diff options
-rw-r--r-- | Cargo.lock | 186 | ||||
-rw-r--r-- | Cargo.toml | 1 | ||||
-rw-r--r-- | sm/download/Cargo.toml | 25 | ||||
-rw-r--r-- | sm/download/examples/simple_curl.rs | 23 | ||||
-rw-r--r-- | sm/download/src/download.rs | 141 | ||||
-rw-r--r-- | sm/download/src/error.rs | 33 | ||||
-rw-r--r-- | sm/download/src/lib.rs | 5 | ||||
-rw-r--r-- | sm/json_sm/src/lib.rs | 2 | ||||
-rw-r--r-- | sm/json_sm/src/messages.rs | 48 |
9 files changed, 463 insertions, 1 deletions
@@ -198,6 +198,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" [[package]] +name = "backoff" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fe17f59a06fe8b87a6fc8bf53bb70b3aba76d7685f432487a68cd5552853625" +dependencies = [ + "futures-core", + "getrandom 0.2.3", + "instant", + "pin-project", + "rand 0.8.4", + "tokio", +] + +[[package]] name = "backtrace" version = "0.3.61" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -456,6 +470,22 @@ dependencies = [ ] [[package]] +name = "core-foundation" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a89e2ae426ea83155dccf10c0fa6b1463ef6d5fcb44cee0b224a408fa640a62" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea221b5284a47e40033bf9b66f35f984ec0ea2931eb03505246cd27a963f981b" + +[[package]] name = "cpufeatures" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -668,6 +698,25 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4bb454f0228b18c7f4c3b0ebbee346ed9c52e7443b0999cd543ff3571205701d" [[package]] +name = "download" +version = "0.3.1" +dependencies = [ + "anyhow", + "backoff", + "json_sm", + "log", + "mockito", + "regex", + "reqwest", + "tedge_utils", + "tempfile", + "thiserror", + "tokio", + "tokio-test", + "url", +] + +[[package]] name = "either" version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -750,6 +799,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" [[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + +[[package]] name = "form_urlencoded" version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -1078,6 +1142,19 @@ dependencies = [ ] [[package]] +name = "hyper-tls" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" +dependencies = [ + "bytes", + "hyper", + "native-tls", + "tokio", + "tokio-native-tls", +] + +[[package]] name = "idna" version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -1440,6 +1517,24 @@ dependencies = [ ] [[package]] +name = "native-tls" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48ba9f7719b5a0f42f338907614285fb5fd70e53858141f69898a1fb7203b24d" +dependencies = [ + "lazy_static", + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + +[[package]] name = "nix" version = "0.21.1" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -1557,6 +1652,39 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" [[package]] +name = "openssl" +version = "0.10.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d9facdb76fec0b73c406f125d44d86fdad818d66fef0531eec9233ca425ff4a" +dependencies = [ + "bitflags", + "cfg-if 1.0.0", + "foreign-types", + "libc", + "once_cell", + "openssl-sys", +] + +[[package]] +name = "openssl-probe" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28988d872ab76095a6e6ac88d99b54fd267702734fd7ffe610ca27f533ddb95a" + +[[package]] +name = "openssl-sys" +version = "0.9.67" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69df2d8dfc6ce3aaf44b40dec6f487d5a886516cf6879c49e98e0710f310a058" +dependencies = [ + "autocfg", + "cc", + "libc", + "pkg-config", + "vcpkg", +] + +[[package]] name = "output_vt100" version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -1640,6 +1768,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] +name = "pkg-config" +version = "0.3.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c9b1041b4387893b91ee6746cddfc28516aff326a3519fb2adf820932c5e6cb" + +[[package]] name = "plotters" version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -2070,11 +2204,13 @@ dependencies = [ "http-body", "hyper", "hyper-rustls", + "hyper-tls", "ipnet", "js-sys", "lazy_static", "log", "mime", + "native-tls", "percent-encoding", "pin-project-lite", "rustls", @@ -2082,6 +2218,7 @@ dependencies = [ "serde_json", "serde_urlencoded", "tokio", + "tokio-native-tls", "tokio-rustls", "url", "wasm-bindgen", @@ -2251,6 +2388,16 @@ dependencies = [ ] [[package]] +name = "schannel" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f05ba609c234e60bee0d547fe94a4c7e9da733d1c962cf6e59efa4cd9c8bc75" +dependencies = [ + "lazy_static", + "winapi", +] + +[[package]] name = "scoped-tls" version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -2273,6 +2420,29 @@ dependencies = [ ] [[package]] +name = "security-framework" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23a2ac85147a3a11d77ecf1bc7166ec0b92febfa4461c37944e180f319ece467" +dependencies = [ + "bitflags", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e4effb91b4b8b6fb7732e670b6cee160278ff8e6bf485c7805d9e319d76e284" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] name = "segments" version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -2792,6 +2962,16 @@ dependencies = [ ] [[package]] +name = "tokio-native-tls" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7d995660bd2b7f8c1568414c1126076c13fbb725c40112dc0120b78eb9b717b" +dependencies = [ + "native-tls", + "tokio", +] + +[[package]] name = "tokio-rustls" version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -3083,6 +3263,12 @@ dependencies = [ ] [[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] name = "vec_map" version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -13,6 +13,7 @@ members = [ "mapper/cumulocity/c8y_smartrest", "mapper/tedge_mapper", "mapper/thin_edge_json", + "sm/download", "sm/json_sm", "sm/plugin_sm", "sm/tedge_agent", diff --git a/sm/download/Cargo.toml b/sm/download/Cargo.toml new file mode 100644 index 00000000..b24514b2 --- /dev/null +++ b/sm/download/Cargo.toml @@ -0,0 +1,25 @@ +[package] +name = "download" +version = "0.3.1" +edition = "2018" +authors = ["thin-edge.io team <info@thin-edge.io>"] +license = "Apache-2.0" +description = "download_manager" + +[dependencies] +backoff = { version = "0.3", features = ["tokio"] } +json_sm = { path = "../json_sm" } +log = "0.4" +reqwest = "0.11" +tedge_utils = { path = "../../common/tedge_utils" } +tempfile = "3.2" +thiserror = "1.0" +tokio = "1.9" +url = "2.2" + +[dev-dependencies] +anyhow = "1.0" +mockito = "0.30" +regex = "1.5" +tempfile = "3.2" +tokio-test = "0.4" diff --git a/sm/download/examples/simple_curl.rs b/sm/download/examples/simple_curl.rs new file mode 100644 index 00000000..821ac0e3 --- /dev/null +++ b/sm/download/examples/simple_curl.rs @@ -0,0 +1,23 @@ +use anyhow::Result; +use download::Downloader; +use json_sm::DownloadInfo; + +/// This example shows how to use the `downlaoder`. +#[tokio::main] +async fn main() -> Result<()> { + // Create Download metadata. + let url_data = DownloadInfo::new( + "https://file-examples-com.github.io/uploads/2017/02/file_example_CSV_5000.csv", + ); + + // Create downloader instance with desired file path and target directory. + let downloader = Downloader::new("test_download", &None, "/tmp"); + + // Call `download` method to get data from url. + let () = downloader.download(&url_data).await?; + + // Call cleanup method to remove downloaded file if no longer necessary. + let () = downloader.cleanup().await?; + + Ok(()) +} diff --git a/sm/download/src/download.rs b/sm/download/src/download.rs new file mode 100644 index 00000000..e94069af --- /dev/null +++ b/sm/download/src/download.rs @@ -0,0 +1,141 @@ +use crate::error::DownloadError; +use backoff::{future::retry, ExponentialBackoff}; +use json_sm::DownloadInfo; +use log::error; +use std::{ + path::{Path, PathBuf}, + time::Duration, +}; + +#[derive(Debug)] +pub struct Downloader { + target_filename: PathBuf, + download_target: PathBuf, +} + +impl Downloader { + pub fn new(name: &str, version: &Option<String>, target_dir_path: impl AsRef<Path>) -> Self { + let mut filename = name.to_string(); + if let Some(version) = version { + filename.push('_'); + filename.push_str(version.as_str()); + } + + let mut download_target = PathBuf::new().join(&target_dir_path).join(&filename); + download_target.set_extension("tmp"); + let target_filename = PathBuf::new().join(target_dir_path).join(filename); + + Self { + target_filename, + download_target, + } + } + + pub async fn download(&self, url: &DownloadInfo) -> Result<(), DownloadError> { + // Default retry is an exponential retry with a limit of 15 minutes total. + // Let's set some more reasonable retry policy so we don't block the downloads for too long. + let backoff = ExponentialBackoff { + initial_interval: Duration::from_secs(30), + max_elapsed_time: Some(Duration::from_secs(300)), + ..Default::default() + }; + + let response = retry(backoff, || async { + let client = reqwest::Client::new(); + match &url.auth { + Some(json_sm::Auth::Bearer(token)) => { + match client + .get(url.url()) + .bearer_auth(token) + .send() + .await + .unwrap() + .error_for_status() + { + Ok(response) => Ok(response), + Err(err) => { + error!("Request returned an error: {:?}", &err); + Err(err.into()) + } + } + } + None => match client.get(url.url()).send().await?.error_for_status() { + Ok(response) => Ok(response), + Err(err) => { + error!("Request returned an error: {:?}", &err); + Err(err.into()) + } + }, + } + }) + .await?; + + let content = response.bytes().await?; + + // Cleanup after `disc full` will happen inside atomic write function. + tedge_utils::fs::atomically_write_file_async( + self.download_target.as_path(), + self.target_filename.as_path(), + content.as_ref(), + ) + .await?; + + Ok(()) + } + + pub fn filename(&self) -> &Path { + self.target_filename.as_path() + } + + pub async fn cleanup(&self) -> Result<(), DownloadError> { + let _res = tokio::fs::remove_file(&self.target_filename).await; + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::Downloader; + use json_sm::DownloadInfo; + use mockito::mock; + use std::path::{Path, PathBuf}; + use tempfile::TempDir; + + #[test] + fn construct_downloader_filename() { + let name = "test_download"; + let version = Some("test1".to_string()); + let target_dir_path = PathBuf::from("/tmp"); + + let downloader = Downloader::new(name, &version, &target_dir_path); + + let expected_path = Path::new("/tmp/test_download_test1"); + assert_eq!(downloader.filename(), expected_path); + } + + #[tokio::test] + async fn downloader_download_content_no_auth() -> anyhow::Result<()> { + let _mock1 = mock("GET", "/some_file.txt") + .with_status(200) + .with_body(b"hello") + .create(); + + let name = "test_download"; + let version = Some("test1".to_string()); + let target_dir_path = TempDir::new()?; + + let mut target_url = mockito::server_url(); + target_url.push_str("/some_file.txt"); + + let url = DownloadInfo::new(&target_url); + + let downloader = Downloader::new(&name, &version, target_dir_path.path()); + let () = downloader.download(&url).await?; + + let log_content = std::fs::read(downloader.filename())?; + + assert_eq!("hello".as_bytes(), log_content); + + Ok(()) + } +} diff --git a/sm/download/src/error.rs b/sm/download/src/error.rs new file mode 100644 index 00000000..6d1eea19 --- /dev/null +++ b/sm/download/src/error.rs @@ -0,0 +1,33 @@ +#[derive(Debug, thiserror::Error)] +pub enum DownloadError { + #[error(transparent)] + FromBackoff(#[from] backoff::Error<reqwest::Error>), + + #[error(transparent)] + FromElapsed(#[from] tokio::time::error::Elapsed), + + #[error("I/O error: {reason:?}")] + FromIo { reason: String }, + + #[error("JSON parse error: {reason:?}")] + FromReqwest { reason: String }, + + #[error(transparent)] + FromUrlParse(#[from] url::ParseError), +} + +impl From<reqwest::Error> for DownloadError { + fn from(err: reqwest::Error) -> Self { + DownloadError::FromReqwest { + reason: format!("{}", err), + } + } +} + +impl From<std::io::Error> for DownloadError { + fn from(err: std::io::Error) -> Self { + DownloadError::FromIo { + reason: format!("{}", err), + } + } +} diff --git a/sm/download/src/lib.rs b/sm/download/src/lib.rs new file mode 100644 index 00000000..205dd8b8 --- /dev/null +++ b/sm/download/src/lib.rs @@ -0,0 +1,5 @@ +mod download; +mod error; + +pub use crate::download::Downloader; +pub use crate::error::DownloadError; diff --git a/sm/json_sm/src/lib.rs b/sm/json_sm/src/lib.rs index 80ac6a32..a39136f2 100644 --- a/sm/json_sm/src/lib.rs +++ b/sm/json_sm/src/lib.rs @@ -4,7 +4,7 @@ mod software; pub use error::*; pub use messages::{ - software_filter_topic, Jsonify, SoftwareListRequest, SoftwareListResponse, + software_filter_topic, Auth, DownloadInfo, Jsonify, SoftwareListRequest, SoftwareListResponse, SoftwareOperationStatus, SoftwareRequestResponse, SoftwareUpdateRequest, SoftwareUpdateResponse, }; diff --git a/sm/json_sm/src/messages.rs b/sm/json_sm/src/messages.rs index cac82da0..33bb7ddd 100644 --- a/sm/json_sm/src/messages.rs +++ b/sm/json_sm/src/messages.rs @@ -298,6 +298,54 @@ pub enum SoftwareModuleAction { Remove, } +#[derive(Debug, Clone, Deserialize, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +#[serde(deny_unknown_fields)] +pub struct DownloadInfo { + pub url: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub auth: Option<Auth>, +} + +impl From<&str> for DownloadInfo { + fn from(url: &str) -> Self { + Self::new(url) + } +} + +impl DownloadInfo { + pub fn new(url: &str) -> Self { + Self { + url: url.into(), + auth: None, + } + } + + pub fn with_auth(self, auth: Auth) -> Self { + Self { + auth: Some(auth), + ..self + } + } + + pub fn url(&self) -> &str { + self.url.as_str() + } +} + +#[derive(Debug, Clone, Deserialize, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +#[serde(deny_unknown_fields)] +pub enum Auth { + Bearer(String), +} + +impl Auth { + pub fn new_bearer(token: &str) -> Self { + Self::Bearer(token.into()) + } +} + /// Software module payload definition. #[derive(Debug, Clone, Deserialize, PartialEq, Serialize)] #[serde(rename_all = "camelCase")] |