Merge branch 'main' into feature/1107/enhance-config-management

18 files changed, 138 insertions, 60 deletions

diff --git a/crates/core/agent_interface/Cargo.toml b/crates/core/agent_interface/Cargo.toml
index 0f5d09da..74d43dab 100644
--- a/crates/core/agent_interface/Cargo.toml
+++ b/crates/core/agent_interface/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "agent_interface"
-version = "0.6.3"
+version = "0.6.4"
 authors = ["thin-edge.io team <info@thin-edge.io>"]
 edition = "2021"
 rust-version = "1.58.1"
diff --git a/crates/core/c8y_api/Cargo.toml b/crates/core/c8y_api/Cargo.toml
index 0a19be66..b97a60dc 100644
--- a/crates/core/c8y_api/Cargo.toml
+++ b/crates/core/c8y_api/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "c8y_api"
-version = "0.6.3"
+version = "0.6.4"
 authors = ["thin-edge.io team <info@thin-edge.io>"]
 edition = "2021"
 rust-version = "1.58.1"
diff --git a/crates/core/c8y_api/src/http_proxy.rs b/crates/core/c8y_api/src/http_proxy.rs
index 9d160bdb..78d8280b 100644
--- a/crates/core/c8y_api/src/http_proxy.rs
+++ b/crates/core/c8y_api/src/http_proxy.rs
@@ -11,8 +11,8 @@ use reqwest::Url;
 use std::path::Path;
 use std::{collections::HashMap, time::Duration};
 use tedge_config::{
-    C8yUrlSetting, ConfigSettingAccessor, ConfigSettingAccessorStringExt, DeviceIdSetting,
-    MqttBindAddressSetting, MqttPortSetting, TEdgeConfig,
+    C8yRootCertPathSetting, C8yUrlSetting, ConfigSettingAccessor, ConfigSettingAccessorStringExt,
+    DeviceIdSetting, MqttBindAddressSetting, MqttPortSetting, TEdgeConfig,
 };
 use time::OffsetDateTime;
 
@@ -212,7 +212,17 @@ impl JwtAuthHttpProxy {
     ) -> Result<JwtAuthHttpProxy, SMCumulocityMapperError> {
         let c8y_host = tedge_config.query_string(C8yUrlSetting)?;
         let device_id = tedge_config.query_string(DeviceIdSetting)?;
-        let http_con = reqwest::ClientBuilder::new().build()?;
+        let root_cert = tedge_config.query(C8yRootCertPathSetting)?;
+
+        let client_builder = reqwest::Client::builder();
+        let http_con = match std::fs::metadata(&root_cert)?.is_file() {
+            true => {
+                let cert = std::fs::read(root_cert)?;
+                let cert_pem = reqwest::Certificate::from_pem(&cert)?;
+                client_builder.add_root_certificate(cert_pem).build()?
+            }
+            false => client_builder.build()?,
+        };
 
         let mqtt_port = tedge_config.query(MqttPortSetting)?.into();
         let mqtt_host = tedge_config.query(MqttBindAddressSetting)?.to_string();
diff --git a/crates/core/c8y_smartrest/Cargo.toml b/crates/core/c8y_smartrest/Cargo.toml
index 8b21b359..2803c8cb 100644
--- a/crates/core/c8y_smartrest/Cargo.toml
+++ b/crates/core/c8y_smartrest/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "c8y_smartrest"
-version = "0.6.3"
+version = "0.6.4"
 authors = ["thin-edge.io team <info@thin-edge.io>"]
 edition = "2021"
 rust-version = "1.58.1"
@@ -20,6 +20,7 @@ tokio = { version = "1.8", features = ["rt", "sync", "time"] }
 toml = "0.5"
 
 [dev-dependencies]
+anyhow = "1.0"
 assert_matches = "1.5"
 assert-json-diff = "2.0"
 serde_json = "1.0"
diff --git a/crates/core/c8y_translator/Cargo.toml b/crates/core/c8y_translator/Cargo.toml
index f6fd28bf..2f7c2d2f 100644
--- a/crates/core/c8y_translator/Cargo.toml
+++ b/crates/core/c8y_translator/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "c8y_translator"
-version = "0.6.3"
+version = "0.6.4"
 authors = ["thin-edge.io team <info@thin-edge.io>"]
 edition = "2021"
 rust-version = "1.58.1"
@@ -17,6 +17,7 @@ anyhow = "1.0"
 assert_matches = "1.5"
 assert-json-diff = "2.0"
 criterion = "0.3"
+pretty_assertions = "1.0"
 proptest = "1.0"
 serde_json = "1.0"
 test-case = "2.0"
diff --git a/crates/core/plugin_sm/Cargo.toml b/crates/core/plugin_sm/Cargo.toml
index ef844a24..f8393219 100644
--- a/crates/core/plugin_sm/Cargo.toml
+++ b/crates/core/plugin_sm/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "plugin_sm"
-version = "0.6.3"
+version = "0.6.4"
 authors = ["thin-edge.io team <info@thin-edge.io>"]
 edition = "2021"
 rust-version = "1.58.1"
@@ -22,7 +22,8 @@ url = "2.2"
 
 [dev-dependencies]
 anyhow = "1.0"
-serial_test = "0.5.1"
+assert_matches = "1.5"
+serial_test = "0.6"
 tempfile = "3.2"
 test-case = "2.0"
 
diff --git a/crates/core/tedge/Cargo.toml b/crates/core/tedge/Cargo.toml
index bc01d456..177e61d0 100644
--- a/crates/core/tedge/Cargo.toml
+++ b/crates/core/tedge/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "tedge"
-version = "0.6.3"
+version = "0.6.4"
 edition = "2021"
 rust-version = "1.58.1"
 authors = ["thin-edge.io team <info@thin-edge.io>"]
@@ -17,6 +17,7 @@ anyhow = "1.0"
 base64 = "0.13"
 certificate = { path = "../../common/certificate" }
 clap = { version = "3", features = ["cargo", "derive"] }
+futures = "0.3"
 hyper = { version = "0.14", default-features = false }
 reqwest = { version = "0.11", default-features = false, features = ["blocking", "json", "rustls-tls", "stream"] }
 rpassword = "5.0"
diff --git a/crates/core/tedge/src/cli/certificate/error.rs b/crates/core/tedge/src/cli/certificate/error.rs
index 92950823..8fffc417 100644
--- a/crates/core/tedge/src/cli/certificate/error.rs
+++ b/crates/core/tedge/src/cli/certificate/error.rs
@@ -1,6 +1,7 @@
 use reqwest::StatusCode;
 use std::error::Error;
 use tedge_config::FilePath;
+use tedge_config::{ConfigSettingError, TEdgeConfigError};
 use tedge_users::UserSwitchError;
 use tedge_utils::paths::PathsError;
 
@@ -76,6 +77,12 @@ pub enum CertError {
 
     #[error("HTTP Connection Problem: {msg} \nHint: {hint}")]
     CertificateValidationFailure { hint: String, msg: String },
+
+    #[error(transparent)]
+    TedgeConfigError(#[from] TEdgeConfigError),
+
+    #[error(transparent)]
+    TedgeConfigSettingError(#[from] ConfigSettingError),
 }
 
 impl CertError {
diff --git a/crates/core/tedge/src/cli/certificate/upload.rs b/crates/core/tedge/src/cli/certificate/upload.rs
index 603f6477..0fdb7313 100644
--- a/crates/core/tedge/src/cli/certificate/upload.rs
+++ b/crates/core/tedge/src/cli/certificate/upload.rs
@@ -44,8 +44,17 @@ impl UploadCertCmd {
             Err(_) => rpassword::read_password_from_tty(Some("Enter password: "))?,
         };
 
-        // Use a builder instead of `Client::new`, `new` could panic, builder adds option to allow invalid certs.
-        let client = reqwest::blocking::Client::builder().build()?;
+        let config = get_tedge_config()?;
+        let root_cert = config.query(C8yRootCertPathSetting)?;
+        let client_builder = reqwest::blocking::Client::builder();
+        let client = match std::fs::metadata(&root_cert)?.is_file() {
+            true => {
+                let cert = std::fs::read(root_cert)?;
+                let cert_pem = reqwest::Certificate::from_pem(&cert)?;
+                client_builder.add_root_certificate(cert_pem).build()?
+            }
+            false => client_builder.build()?,
+        };
 
         // To post certificate c8y requires one of the following endpoints:
         // https://<tenant_id>.cumulocity.url.io/tenant/tenants/<tenant_id>/trusted-certificates
diff --git a/crates/core/tedge/src/cli/config/config_key.rs b/crates/core/tedge/src/cli/config/config_key.rs
index 96227269..161f0e4c 100644
--- a/crates/core/tedge/src/cli/config/config_key.rs
+++ b/crates/core/tedge/src/cli/config/config_key.rs
@@ -45,6 +45,7 @@ impl ConfigKey {
             config_key!(DeviceCertPathSetting),
             config_key!(C8yUrlSetting),
             config_key!(C8yRootCertPathSetting),
+            config_key!(C8ySmartRestTemplates),
             config_key!(AzureUrlSetting),
             config_key!(AzureRootCertPathSetting),
             config_key!(AzureMapperTimestamp),
diff --git a/crates/core/tedge/src/cli/connect/bridge_config.rs b/crates/core/tedge/src/cli/connect/bridge_config.rs
index bb4a7930..f843c7fe 100644
--- a/crates/core/tedge/src/cli/connect/bridge_config.rs
+++ b/crates/core/tedge/src/cli/connect/bridge_config.rs
@@ -39,7 +39,6 @@ impl BridgeConfig {
         }
         writeln!(writer, "address {}", self.address)?;
 
-        // XXX: This has to go away
         if std::fs::metadata(&self.bridge_root_cert_path)?.is_dir() {
             writeln!(writer, "bridge_capath {}", self.bridge_root_cert_path)?;
         } else {
diff --git a/crates/core/tedge/src/cli/connect/bridge_config_c8y.rs b/crates/core/tedge/src/cli/connect/bridge_config_c8y.rs
index 90fdab7e..31330598 100644
--- a/crates/core/tedge/src/cli/connect/bridge_config_c8y.rs
+++ b/crates/core/tedge/src/cli/connect/bridge_config_c8y.rs
@@ -1,5 +1,5 @@
 use crate::cli::connect::BridgeConfig;
-use tedge_config::{ConnectUrl, FilePath};
+use tedge_config::{ConnectUrl, FilePath, TemplatesSet};
 
 #[derive(Debug, PartialEq)]
 pub struct BridgeConfigC8yParams {
@@ -10,6 +10,7 @@ pub struct BridgeConfigC8yParams {
     pub bridge_root_cert_path: FilePath,
     pub bridge_certfile: FilePath,
     pub bridge_keyfile: FilePath,
+    pub smartrest_templates: TemplatesSet,
 }
 
 impl From<BridgeConfigC8yParams> for BridgeConfig {
@@ -22,9 +23,57 @@ impl From<BridgeConfigC8yParams> for BridgeConfig {
             remote_clientid,
             bridge_certfile,
             bridge_keyfile,
+            smartrest_templates,
         } = params;
         let address = format!("{}:{}", connect_url.as_str(), mqtt_tls_port);
 
+        let mut topics: Vec<String> = vec![
+            // Registration
+            r#"s/dcr in 2 c8y/ """#.into(),
+            r#"s/ucr out 2 c8y/ """#.into(),
+            // Templates
+            r#"s/dt in 2 c8y/ """#.into(),
+            r#"s/ut/# out 2 c8y/ """#.into(),
+            // Static templates
+            r#"s/us/# out 2 c8y/ """#.into(),
+            r#"t/us/# out 2 c8y/ """#.into(),
+            r#"q/us/# out 2 c8y/ """#.into(),
+            r#"c/us/# out 2 c8y/ """#.into(),
+            r#"s/ds in 2 c8y/ """#.into(),
+            // Debug
+            r#"s/e in 0 c8y/ """#.into(),
+            // SmartRest2
+            r#"s/uc/# out 2 c8y/ """#.into(),
+            r#"t/uc/# out 2 c8y/ """#.into(),
+            r#"q/uc/# out 2 c8y/ """#.into(),
+            r#"c/uc/# out 2 c8y/ """#.into(),
+            r#"s/dc/# in 2 c8y/ """#.into(),
+            // c8y JSON
+            r#"inventory/managedObjects/update/# out 2 c8y/ """#.into(),
+            r#"measurement/measurements/create out 2 c8y/ """#.into(),
+            r#"event/events/create out 2 c8y/ """#.into(),
+            r#"error in 2 c8y/ """#.into(),
+            // c8y JWT token retrieval
+            r#"s/uat/# out 2 c8y/ """#.into(),
+            r#"s/dat/# in 2 c8y/ """#.into(),
+        ];
+
+        let templates_set = smartrest_templates
+            .0
+            .iter()
+            .flat_map(|s| {
+                // Smartrest templates should be deserialized as:
+                // c8y/s/uc/template-1 (in from localhost), s/uc/template-1
+                // c8y/s/dc/template-1 (out to localhost), s/dc/template-1
+                [
+                    format!(r#"s/uc/{s} out 2 c8y/ """#),
+                    format!(r#"s/dc/{s} in 2 c8y/ """#),
+                ]
+                .into_iter()
+            })
+            .collect::<Vec<String>>();
+        topics.extend(templates_set);
+
         Self {
             cloud_name: "c8y".into(),
             config_file,
@@ -45,36 +94,7 @@ impl From<BridgeConfigC8yParams> for BridgeConfig {
             notifications_local_only: true,
             notification_topic: "tedge/health/mosquitto-c8y-bridge".into(),
             bridge_attempt_unsubscribe: false,
-            topics: vec![
-                // Registration
-                r#"s/dcr in 2 c8y/ """#.into(),
-                r#"s/ucr out 2 c8y/ """#.into(),
-                // Templates
-                r#"s/dt in 2 c8y/ """#.into(),
-                r#"s/ut/# out 2 c8y/ """#.into(),
-                // Static templates
-                r#"s/us/# out 2 c8y/ """#.into(),
-                r#"t/us/# out 2 c8y/ """#.into(),
-                r#"q/us/# out 2 c8y/ """#.into(),
-                r#"c/us/# out 2 c8y/ """#.into(),
-                r#"s/ds in 2 c8y/ """#.into(),
-                // Debug
-                r#"s/e in 0 c8y/ """#.into(),
-                // SmartRest2
-                r#"s/uc/# out 2 c8y/ """#.into(),
-                r#"t/uc/# out 2 c8y/ """#.into(),
-                r#"q/uc/# out 2 c8y/ """#.into(),
-                r#"c/uc/# out 2 c8y/ """#.into(),
-                r#"s/dc/# in 2 c8y/ """#.into(),
-                // c8y JSON
-                r#"inventory/managedObjects/update/# out 2 c8y/ """#.into(),
-                r#"measurement/measurements/create out 2 c8y/ """#.into(),
-                r#"event/events/create out 2 c8y/ """#.into(),
-                r#"error in 2 c8y/ """#.into(),
-                // c8y JWT token retrieval
-                r#"s/uat/# out 2 c8y/ """#.into(),
-                r#"s/dat/# in 2 c8y/ """#.into(),
-            ],
+            topics,
         }
     }
 }
@@ -90,6 +110,7 @@ fn test_bridge_config_from_c8y_params() -> anyhow::Result<()> {
         bridge_root_cert_path: "./test_root.pem".into(),
         bridge_certfile: "./test-certificate.pem".into(),
         bridge_keyfile: "./test-private-key.pem".into(),
+        smartrest_templates: TemplatesSet::try_from(vec!["abc", "def"])?,
     };
 
     let bridge = BridgeConfig::from(params);
@@ -136,6 +157,13 @@ fn test_bridge_config_from_c8y_params() -> anyhow::Result<()> {
             // c8y JWT token retrieval
             r#"s/uat/# out 2 c8y/ """#.into(),
             r#"s/dat/# in 2 c8y/ """#.into(),
+            // Smartrest templates should be deserialized as:
+            // s/uc/template-1 (in from localhost), s/uc/template-1
+            // s/dc/template-1 (out to localhost), s/dc/template-1
+            r#"s/uc/abc out 2 c8y/ """#.into(),
+            r#"s/dc/abc in 2 c8y/ """#.into(),
+            r#"s/uc/def out 2 c8y/ """#.into(),
+            r#"s/dc/def in 2 c8y/ """#.into(),
         ],
         try_private: false,
         start_type: "automatic".into(),
diff --git a/crates/core/tedge/src/cli/connect/c8y_direct_connection.rs b/crates/core/tedge/src/cli/connect/c8y_direct_connection.rs
index d080eda2..bba72e6a 100644
--- a/crates/core/tedge/src/cli/connect/c8y_direct_connection.rs
+++ b/crates/core/tedge/src/cli/connect/c8y_direct_connection.rs
@@ -9,6 +9,7 @@ use rustls_0_19::ClientConfig;
 
 use std::fs;
 use std::io::{Error, ErrorKind};
+use std::path::PathBuf;
 use std::{fs::File, io::BufReader};
 use tedge_config::FilePath;
 
@@ -111,22 +112,33 @@ fn publish_device_create_message(
 
 fn load_root_certs(
     root_store: &mut rustls_0_19::RootCertStore,
-    cert_dir: FilePath,
+    cert_path: FilePath,
 ) -> Result<(), ConnectError> {
-    for file_entry in fs::read_dir(cert_dir)? {
-        let file = file_entry?;
-        let f = File::open(file.path())?;
-        let mut rd = BufReader::new(f);
-        let _ = root_store.add_pem_file(&mut rd).map(|_| ()).map_err(|()| {
-            Error::new(
-                ErrorKind::InvalidData,
-                "could not load PEM file".to_string(),
-            )
-        });
+    if fs::metadata(&cert_path)?.is_dir() {
+        for file_entry in fs::read_dir(cert_path)? {
+            add_root_cert(root_store, file_entry?.path())?;
+        }
+    } else {
+        add_root_cert(root_store, cert_path.into())?;
     }
     Ok(())
 }
 
+fn add_root_cert(
+    root_store: &mut rustls_0_19::RootCertStore,
+    cert_path: PathBuf,
+) -> Result<(), ConnectError> {
+    let f = File::open(cert_path)?;
+    let mut rd = BufReader::new(f);
+    let _ = root_store.add_pem_file(&mut rd).map(|_| ()).map_err(|()| {
+        Error::new(
+            ErrorKind::InvalidData,
+            "could not load PEM file".to_string(),
+        )
+    });
+    Ok(())
+}
+
 fn read_pvt_key(
     user_manager: UserManager,
     key_file: tedge_config::FilePath,
diff --git a/crates/core/tedge/src/cli/connect/command.rs b/crates/core/tedge/src/cli/connect/command.rs
index 30682ddd..7da926d0 100644
--- a/crates/core/tedge/src/cli/connect/command.rs
+++ b/crates/core/tedge/src/cli/connect/command.rs
@@ -203,6 +203,7 @@ impl ConnectCommand {
                     remote_clientid: config.query(DeviceIdSetting)?,
                     bridge_certfile: config.query(DeviceCertPathSetting)?,
                     bridge_keyfile: config.query(DeviceKeyPathSetting)?,
+                    smartrest_templates: config.query(C8ySmartRestTemplates)?,
                 };
 
                 Ok(BridgeConfig::from(params))
diff --git a/crates/core/tedge_agent/Cargo.toml b/crates/core/tedge_agent/Cargo.toml
index 6fd0bc81..0e9a1a11 100644
--- a/crates/core/tedge_agent/Cargo.toml
+++ b/crates/core/tedge_agent/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "tedge_agent"
-version = "0.6.3"
+version = "0.6.4"
 authors = ["thin-edge.io team <info@thin-edge.io>"]
 edition = "2021"
 rust-version = "1.58.1"
@@ -28,6 +28,7 @@ async-trait = "0.1"
 clap = { version = "3.0", features = ["cargo", "derive"] }
 flockfile = { path = "../../common/flockfile" }
 futures = "0.3"
+mockall = "0.10"
 mqtt_channel = { path = "../../common/mqtt_channel" }
 plugin_sm = { path = "../plugin_sm" }
 serde = { version = "1.0", features = ["derive"] }
@@ -47,5 +48,8 @@ assert-json-diff = "2.0"
 once_cell = "1.8"
 mqtt_tests = { path = "../../tests/mqtt_tests" }
 predicates = "2.1"
+tedge_users = { path = "../../common/tedge_users"}
 tedge_utils = { path = "../../common/tedge_utils"}
 tempfile = "3.2"
+tokio-test = "0.4"
+serial_test = "0.6"
diff --git a/crates/core/tedge_mapper/Cargo.toml b/crates/core/tedge_mapper/Cargo.toml
index 34e8e8da..b7561fb3 100644
--- a/crates/core/tedge_mapper/Cargo.toml
+++ b/crates/core/tedge_mapper/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "tedge_mapper"
-version = "0.6.3"
+version = "0.6.4"
 authors = ["thin-edge.io team <info@thin-edge.io>"]
 edition = "2021"
 rust-version = "1.58.1"
@@ -46,6 +46,7 @@ serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
 clap = { version = "3.0", features = ["cargo", "derive"] }
 tedge_config = { path = "../../common/tedge_config" }
+tedge_users = { path = "../../common/tedge_users" }
 tedge_utils = { path = "../../common/tedge_utils", features = ["logging"] }
 thin_edge_json = { path = "../thin_edge_json" }
 thiserror = "1.0"
@@ -61,10 +62,11 @@ serde = "1.0"
 mockito = "0.31"
 mqtt_tests = { path = "../../tests/mqtt_tests" }
 serde_json = "1.0"
-serial_test = "0.5"
+serial_test = "0.6"
 tempfile = "3.2"
 test-case = "2.0"
 time = { version = "0.3", features = ["macros"] }
+tokio-test = "0.4"
 
 [features]
 integration-test = []
diff --git a/crates/core/tedge_watchdog/Cargo.toml b/crates/core/tedge_watchdog/Cargo.toml
index 7ca4e2f4..73d3e659 100644
--- a/crates/core/tedge_watchdog/Cargo.toml
+++ b/crates/core/tedge_watchdog/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "tedge_watchdog"
-version = "0.6.3"
+version = "0.6.4"
 authors = ["thin-edge.io team <info@thin-edge.io>"]
 edition = "2021"
 rust-version = "1.58.1"
diff --git a/crates/core/thin_edge_json/Cargo.toml b/crates/core/thin_edge_json/Cargo.toml
index 44be4ea7..d2831614 100644
--- a/crates/core/thin_edge_json/Cargo.toml
+++ b/crates/core/thin_edge_json/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "thin_edge_json"
-version = "0.6.3"
+version = "0.6.4"
 authors = ["thin-edge.io team <info@thin-edge.io>"]
 edition = "2021"
 rust-version = "1.58.1"
@@ -20,6 +20,7 @@ anyhow = "1.0"
 clock = { path = "../../common/clock" }
 criterion = "0.3"
 mockall = "0.10"
+proptest = "1.0"
 stats_alloc = "0.1"
 walkdir = "2"
 assert_matches = "1.5"