[CIT-571] Add download crate and supporting struct in json_sm (#454)

* Add download crate and supporting struct in json_sm
* Add tests for downloader

Signed-off-by: Lukasz Woznicki <lukasz.woznicki@softwareag.com>

9 files changed, 463 insertions, 1 deletions

diff --git a/Cargo.lock b/Cargo.lock
index dc039c84..f74aa20b 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -198,6 +198,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a"
 
 [[package]]
+name = "backoff"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9fe17f59a06fe8b87a6fc8bf53bb70b3aba76d7685f432487a68cd5552853625"
+dependencies = [
+ "futures-core",
+ "getrandom 0.2.3",
+ "instant",
+ "pin-project",
+ "rand 0.8.4",
+ "tokio",
+]
+
+[[package]]
 name = "backtrace"
 version = "0.3.61"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -456,6 +470,22 @@ dependencies = [
 ]
 
 [[package]]
+name = "core-foundation"
+version = "0.9.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0a89e2ae426ea83155dccf10c0fa6b1463ef6d5fcb44cee0b224a408fa640a62"
+dependencies = [
+ "core-foundation-sys",
+ "libc",
+]
+
+[[package]]
+name = "core-foundation-sys"
+version = "0.8.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ea221b5284a47e40033bf9b66f35f984ec0ea2931eb03505246cd27a963f981b"
+
+[[package]]
 name = "cpufeatures"
 version = "0.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -668,6 +698,25 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "4bb454f0228b18c7f4c3b0ebbee346ed9c52e7443b0999cd543ff3571205701d"
 
 [[package]]
+name = "download"
+version = "0.3.1"
+dependencies = [
+ "anyhow",
+ "backoff",
+ "json_sm",
+ "log",
+ "mockito",
+ "regex",
+ "reqwest",
+ "tedge_utils",
+ "tempfile",
+ "thiserror",
+ "tokio",
+ "tokio-test",
+ "url",
+]
+
+[[package]]
 name = "either"
 version = "1.6.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -750,6 +799,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
 
 [[package]]
+name = "foreign-types"
+version = "0.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1"
+dependencies = [
+ "foreign-types-shared",
+]
+
+[[package]]
+name = "foreign-types-shared"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b"
+
+[[package]]
 name = "form_urlencoded"
 version = "1.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -1078,6 +1142,19 @@ dependencies = [
 ]
 
 [[package]]
+name = "hyper-tls"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905"
+dependencies = [
+ "bytes",
+ "hyper",
+ "native-tls",
+ "tokio",
+ "tokio-native-tls",
+]
+
+[[package]]
 name = "idna"
 version = "0.2.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -1440,6 +1517,24 @@ dependencies = [
 ]
 
 [[package]]
+name = "native-tls"
+version = "0.2.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "48ba9f7719b5a0f42f338907614285fb5fd70e53858141f69898a1fb7203b24d"
+dependencies = [
+ "lazy_static",
+ "libc",
+ "log",
+ "openssl",
+ "openssl-probe",
+ "openssl-sys",
+ "schannel",
+ "security-framework",
+ "security-framework-sys",
+ "tempfile",
+]
+
+[[package]]
 name = "nix"
 version = "0.21.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -1557,6 +1652,39 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5"
 
 [[package]]
+name = "openssl"
+version = "0.10.36"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8d9facdb76fec0b73c406f125d44d86fdad818d66fef0531eec9233ca425ff4a"
+dependencies = [
+ "bitflags",
+ "cfg-if 1.0.0",
+ "foreign-types",
+ "libc",
+ "once_cell",
+ "openssl-sys",
+]
+
+[[package]]
+name = "openssl-probe"
+version = "0.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "28988d872ab76095a6e6ac88d99b54fd267702734fd7ffe610ca27f533ddb95a"
+
+[[package]]
+name = "openssl-sys"
+version = "0.9.67"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "69df2d8dfc6ce3aaf44b40dec6f487d5a886516cf6879c49e98e0710f310a058"
+dependencies = [
+ "autocfg",
+ "cc",
+ "libc",
+ "pkg-config",
+ "vcpkg",
+]
+
+[[package]]
 name = "output_vt100"
 version = "0.1.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -1640,6 +1768,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"
 
 [[package]]
+name = "pkg-config"
+version = "0.3.20"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7c9b1041b4387893b91ee6746cddfc28516aff326a3519fb2adf820932c5e6cb"
+
+[[package]]
 name = "plotters"
 version = "0.3.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -2070,11 +2204,13 @@ dependencies = [
  "http-body",
  "hyper",
  "hyper-rustls",
+ "hyper-tls",
  "ipnet",
  "js-sys",
  "lazy_static",
  "log",
  "mime",
+ "native-tls",
  "percent-encoding",
  "pin-project-lite",
  "rustls",
@@ -2082,6 +2218,7 @@ dependencies = [
  "serde_json",
  "serde_urlencoded",
  "tokio",
+ "tokio-native-tls",
  "tokio-rustls",
  "url",
  "wasm-bindgen",
@@ -2251,6 +2388,16 @@ dependencies = [
 ]
 
 [[package]]
+name = "schannel"
+version = "0.1.19"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8f05ba609c234e60bee0d547fe94a4c7e9da733d1c962cf6e59efa4cd9c8bc75"
+dependencies = [
+ "lazy_static",
+ "winapi",
+]
+
+[[package]]
 name = "scoped-tls"
 version = "1.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -2273,6 +2420,29 @@ dependencies = [
 ]
 
 [[package]]
+name = "security-framework"
+version = "2.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "23a2ac85147a3a11d77ecf1bc7166ec0b92febfa4461c37944e180f319ece467"
+dependencies = [
+ "bitflags",
+ "core-foundation",
+ "core-foundation-sys",
+ "libc",
+ "security-framework-sys",
+]
+
+[[package]]
+name = "security-framework-sys"
+version = "2.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7e4effb91b4b8b6fb7732e670b6cee160278ff8e6bf485c7805d9e319d76e284"
+dependencies = [
+ "core-foundation-sys",
+ "libc",
+]
+
+[[package]]
 name = "segments"
 version = "0.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -2792,6 +2962,16 @@ dependencies = [
 ]
 
 [[package]]
+name = "tokio-native-tls"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f7d995660bd2b7f8c1568414c1126076c13fbb725c40112dc0120b78eb9b717b"
+dependencies = [
+ "native-tls",
+ "tokio",
+]
+
+[[package]]
 name = "tokio-rustls"
 version = "0.22.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -3083,6 +3263,12 @@ dependencies = [
 ]
 
 [[package]]
+name = "vcpkg"
+version = "0.2.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426"
+
+[[package]]
 name = "vec_map"
 version = "0.8.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
diff --git a/Cargo.toml b/Cargo.toml
index 3ab12af1..264c9117 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -13,6 +13,7 @@ members = [
     "mapper/cumulocity/c8y_smartrest",
     "mapper/tedge_mapper",
     "mapper/thin_edge_json",
+    "sm/download",
     "sm/json_sm",
     "sm/plugin_sm",
     "sm/tedge_agent",
diff --git a/sm/download/Cargo.toml b/sm/download/Cargo.toml
new file mode 100644
index 00000000..b24514b2
--- /dev/null
+++ b/sm/download/Cargo.toml
@@ -0,0 +1,25 @@
+[package]
+name = "download"
+version = "0.3.1"
+edition = "2018"
+authors = ["thin-edge.io team <info@thin-edge.io>"]
+license = "Apache-2.0"
+description = "download_manager"
+
+[dependencies]
+backoff = { version = "0.3", features = ["tokio"] }
+json_sm = { path = "../json_sm" }
+log = "0.4"
+reqwest = "0.11"
+tedge_utils = { path = "../../common/tedge_utils" }
+tempfile = "3.2"
+thiserror = "1.0"
+tokio = "1.9"
+url = "2.2"
+
+[dev-dependencies]
+anyhow = "1.0"
+mockito = "0.30"
+regex = "1.5"
+tempfile = "3.2"
+tokio-test = "0.4"
diff --git a/sm/download/examples/simple_curl.rs b/sm/download/examples/simple_curl.rs
new file mode 100644
index 00000000..821ac0e3
--- /dev/null
+++ b/sm/download/examples/simple_curl.rs
@@ -0,0 +1,23 @@
+use anyhow::Result;
+use download::Downloader;
+use json_sm::DownloadInfo;
+
+/// This example shows how to use the `downlaoder`.
+#[tokio::main]
+async fn main() -> Result<()> {
+    // Create Download metadata.
+    let url_data = DownloadInfo::new(
+        "https://file-examples-com.github.io/uploads/2017/02/file_example_CSV_5000.csv",
+    );
+
+    // Create downloader instance with desired file path and target directory.
+    let downloader = Downloader::new("test_download", &None, "/tmp");
+
+    // Call `download` method to get data from url.
+    let () = downloader.download(&url_data).await?;
+
+    // Call cleanup method to remove downloaded file if no longer necessary.
+    let () = downloader.cleanup().await?;
+
+    Ok(())
+}
diff --git a/sm/download/src/download.rs b/sm/download/src/download.rs
new file mode 100644
index 00000000..e94069af
--- /dev/null
+++ b/sm/download/src/download.rs
@@ -0,0 +1,141 @@
+use crate::error::DownloadError;
+use backoff::{future::retry, ExponentialBackoff};
+use json_sm::DownloadInfo;
+use log::error;
+use std::{
+    path::{Path, PathBuf},
+    time::Duration,
+};
+
+#[derive(Debug)]
+pub struct Downloader {
+    target_filename: PathBuf,
+    download_target: PathBuf,
+}
+
+impl Downloader {
+    pub fn new(name: &str, version: &Option<String>, target_dir_path: impl AsRef<Path>) -> Self {
+        let mut filename = name.to_string();
+        if let Some(version) = version {
+            filename.push('_');
+            filename.push_str(version.as_str());
+        }
+
+        let mut download_target = PathBuf::new().join(&target_dir_path).join(&filename);
+        download_target.set_extension("tmp");
+        let target_filename = PathBuf::new().join(target_dir_path).join(filename);
+
+        Self {
+            target_filename,
+            download_target,
+        }
+    }
+
+    pub async fn download(&self, url: &DownloadInfo) -> Result<(), DownloadError> {
+        // Default retry is an exponential retry with a limit of 15 minutes total.
+        // Let's set some more reasonable retry policy so we don't block the downloads for too long.
+        let backoff = ExponentialBackoff {
+            initial_interval: Duration::from_secs(30),
+            max_elapsed_time: Some(Duration::from_secs(300)),
+            ..Default::default()
+        };
+
+        let response = retry(backoff, || async {
+            let client = reqwest::Client::new();
+            match &url.auth {
+                Some(json_sm::Auth::Bearer(token)) => {
+                    match client
+                        .get(url.url())
+                        .bearer_auth(token)
+                        .send()
+                        .await
+                        .unwrap()
+                        .error_for_status()
+                    {
+                        Ok(response) => Ok(response),
+                        Err(err) => {
+                            error!("Request returned an error: {:?}", &err);
+                            Err(err.into())
+                        }
+                    }
+                }
+                None => match client.get(url.url()).send().await?.error_for_status() {
+                    Ok(response) => Ok(response),
+                    Err(err) => {
+                        error!("Request returned an error: {:?}", &err);
+                        Err(err.into())
+                    }
+                },
+            }
+        })
+        .await?;
+
+        let content = response.bytes().await?;
+
+        // Cleanup after `disc full` will happen inside atomic write function.
+        tedge_utils::fs::atomically_write_file_async(
+            self.download_target.as_path(),
+            self.target_filename.as_path(),
+            content.as_ref(),
+        )
+        .await?;
+
+        Ok(())
+    }
+
+    pub fn filename(&self) -> &Path {
+        self.target_filename.as_path()
+    }
+
+    pub async fn cleanup(&self) -> Result<(), DownloadError> {
+        let _res = tokio::fs::remove_file(&self.target_filename).await;
+        Ok(())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::Downloader;
+    use json_sm::DownloadInfo;
+    use mockito::mock;
+    use std::path::{Path, PathBuf};
+    use tempfile::TempDir;
+
+    #[test]
+    fn construct_downloader_filename() {
+        let name = "test_download";
+        let version = Some("test1".to_string());
+        let target_dir_path = PathBuf::from("/tmp");
+
+        let downloader = Downloader::new(name, &version, &target_dir_path);
+
+        let expected_path = Path::new("/tmp/test_download_test1");
+        assert_eq!(downloader.filename(), expected_path);
+    }
+
+    #[tokio::test]
+    async fn downloader_download_content_no_auth() -> anyhow::Result<()> {
+        let _mock1 = mock("GET", "/some_file.txt")
+            .with_status(200)
+            .with_body(b"hello")
+            .create();
+
+        let name = "test_download";
+        let version = Some("test1".to_string());
+        let target_dir_path = TempDir::new()?;
+
+        let mut target_url = mockito::server_url();
+        target_url.push_str("/some_file.txt");
+
+        let url = DownloadInfo::new(&target_url);
+
+        let downloader = Downloader::new(&name, &version, target_dir_path.path());
+        let () = downloader.download(&url).await?;
+
+        let log_content = std::fs::read(downloader.filename())?;
+
+        assert_eq!("hello".as_bytes(), log_content);
+
+        Ok(())
+    }
+}
diff --git a/sm/download/src/error.rs b/sm/download/src/error.rs
new file mode 100644
index 00000000..6d1eea19
--- /dev/null
+++ b/sm/download/src/error.rs
@@ -0,0 +1,33 @@
+#[derive(Debug, thiserror::Error)]
+pub enum DownloadError {
+    #[error(transparent)]
+    FromBackoff(#[from] backoff::Error<reqwest::Error>),
+
+    #[error(transparent)]
+    FromElapsed(#[from] tokio::time::error::Elapsed),
+
+    #[error("I/O error: {reason:?}")]
+    FromIo { reason: String },
+
+    #[error("JSON parse error: {reason:?}")]
+    FromReqwest { reason: String },
+
+    #[error(transparent)]
+    FromUrlParse(#[from] url::ParseError),
+}
+
+impl From<reqwest::Error> for DownloadError {
+    fn from(err: reqwest::Error) -> Self {
+        DownloadError::FromReqwest {
+            reason: format!("{}", err),
+        }
+    }
+}
+
+impl From<std::io::Error> for DownloadError {
+    fn from(err: std::io::Error) -> Self {
+        DownloadError::FromIo {
+            reason: format!("{}", err),
+        }
+    }
+}
diff --git a/sm/download/src/lib.rs b/sm/download/src/lib.rs
new file mode 100644
index 00000000..205dd8b8
--- /dev/null
+++ b/sm/download/src/lib.rs
@@ -0,0 +1,5 @@
+mod download;
+mod error;
+
+pub use crate::download::Downloader;
+pub use crate::error::DownloadError;
diff --git a/sm/json_sm/src/lib.rs b/sm/json_sm/src/lib.rs
index 80ac6a32..a39136f2 100644
--- a/sm/json_sm/src/lib.rs
+++ b/sm/json_sm/src/lib.rs
@@ -4,7 +4,7 @@ mod software;
 
 pub use error::*;
 pub use messages::{
-    software_filter_topic, Jsonify, SoftwareListRequest, SoftwareListResponse,
+    software_filter_topic, Auth, DownloadInfo, Jsonify, SoftwareListRequest, SoftwareListResponse,
     SoftwareOperationStatus, SoftwareRequestResponse, SoftwareUpdateRequest,
     SoftwareUpdateResponse,
 };
diff --git a/sm/json_sm/src/messages.rs b/sm/json_sm/src/messages.rs
index cac82da0..33bb7ddd 100644
--- a/sm/json_sm/src/messages.rs
+++ b/sm/json_sm/src/messages.rs
@@ -298,6 +298,54 @@ pub enum SoftwareModuleAction {
     Remove,
 }
 
+#[derive(Debug, Clone, Deserialize, PartialEq, Serialize)]
+#[serde(rename_all = "camelCase")]
+#[serde(deny_unknown_fields)]
+pub struct DownloadInfo {
+    pub url: String,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub auth: Option<Auth>,
+}
+
+impl From<&str> for DownloadInfo {
+    fn from(url: &str) -> Self {
+        Self::new(url)
+    }
+}
+
+impl DownloadInfo {
+    pub fn new(url: &str) -> Self {
+        Self {
+            url: url.into(),
+            auth: None,
+        }
+    }
+
+    pub fn with_auth(self, auth: Auth) -> Self {
+        Self {
+            auth: Some(auth),
+            ..self
+        }
+    }
+
+    pub fn url(&self) -> &str {
+        self.url.as_str()
+    }
+}
+
+#[derive(Debug, Clone, Deserialize, PartialEq, Serialize)]
+#[serde(rename_all = "camelCase")]
+#[serde(deny_unknown_fields)]
+pub enum Auth {
+    Bearer(String),
+}
+
+impl Auth {
+    pub fn new_bearer(token: &str) -> Self {
+        Self::Bearer(token.into())
+    }
+}
+
 /// Software module payload definition.
 #[derive(Debug, Clone, Deserialize, PartialEq, Serialize)]
 #[serde(rename_all = "camelCase")]