Merge branch 'main' into feature/1030/configuration-management

81 files changed, 2417 insertions, 512 deletions

diff --git a/.github/workflows/ci_pipeline.yml b/.github/workflows/ci_pipeline.yml
new file mode 100644
index 00000000..417ee964
--- /dev/null
+++ b/.github/workflows/ci_pipeline.yml
@@ -0,0 +1,1384 @@
+name: ci_pipeline
+
+# All in one ci pipeline
+#  Note that the jobs system-test_offsite_{abcd} , install-and-use-rpi_m32sd10{abcd}
+#  are there four times and are almost identical. They only differ in a character
+#  a,b,c,d that we use to select a runner.
+#
+# TODO: Reduce complexity by moving functionality to bash scripts
+# TODO: Think about automated generation of this workflow, there is
+#   duplicated code, especially in the later sections
+# TODO: Smoke testing on Azure is disabled, we should check if it works out
+#   with 4 runners in parallel or enable only one
+
+on:
+  push:
+    branches: [main]
+  workflow_dispatch:
+    branches: [main]
+
+env:
+  CARGO_TERM_COLOR: always
+
+jobs:
+  cargo-fmt:
+    name: Run cargo fmt
+    runs-on: Ubuntu-20.04
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Cargo fmt --version
+        uses: actions-rs/cargo@v1
+        # https://github.com/marketplace/actions/rust-cargo
+        with:
+          command: fmt
+          args: --version
+
+      - name: Cargo fmt
+        uses: actions-rs/cargo@v1
+        # https://github.com/marketplace/actions/rust-cargo
+        with:
+          command: fmt
+          args: -- --check
+
+  python-black:
+    name: Python static checks
+    runs-on: Ubuntu-20.04
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Run black
+        uses: rickstaa/action-black@v1
+        with:
+          black_args: ". --check"
+
+  cargo-clippy:
+    name: Run cargo clippy
+    runs-on: Ubuntu-20.04
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: enable toolchain via github action
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: 1.58.1
+          components: rustfmt, clippy
+          override: true
+
+      - name: Enable cache
+        # https://github.com/marketplace/actions/rust-cache
+        uses: Swatinem/rust-cache@v1
+
+      - name: Cargo clippy --version
+        uses: actions-rs/cargo@v1
+        # https://github.com/marketplace/actions/rust-cargo
+        with:
+          toolchain: 1.58.1
+          command: clippy
+          args: --version
+
+      - name: Cargo clippy
+        uses: actions-rs/cargo@v1
+        # https://github.com/marketplace/actions/rust-cargo
+        with:
+          toolchain: 1.58.1
+          command: clippy
+
+  cargo-test:
+    name: Run cargo test
+    runs-on: Ubuntu-20.04
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: enable toolchain via github action
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: 1.58.1
+          override: true
+
+      - name: Enable cache
+        # https://github.com/marketplace/actions/rust-cache
+        uses: Swatinem/rust-cache@v1
+
+      - name: Cargo version
+        uses: actions-rs/cargo@v1
+        # https://github.com/marketplace/actions/rust-cargo
+        with:
+          command: version
+
+      - name: Cargo build dummy plugin
+        uses: actions-rs/cargo@v1
+        # https://github.com/marketplace/actions/rust-cargo
+        with:
+          command: build
+          args: -p tedge_dummy_plugin
+
+      - name: Cargo test
+        uses: actions-rs/cargo@v1
+        # https://github.com/marketplace/actions/rust-cargo
+        with:
+          command: test
+          args: --no-fail-fast
+
+
+###############################################################################
+
+  build_amd64:
+    name: Build tedge and mapper Debian packages for amd64
+    runs-on: Ubuntu-20.04
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: enable toolchain via github action
+        # https://github.com/actions-rs/toolchain
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: 1.58.1
+          override: true
+
+      - name: Enable cache
+        # https://github.com/marketplace/actions/rust-cache
+        uses: Swatinem/rust-cache@v1
+
+      - name: Install cargo-deb
+        uses: actions-rs/cargo@v1
+        # https://github.com/marketplace/actions/rust-cargo
+        with:
+          command: install
+          args: cargo-deb --version 1.34.2
+
+      - name: Build tedge debian package
+        uses: actions-rs/cargo@v1
+        # https://github.com/marketplace/actions/rust-cargo
+        with:
+          command: deb
+          args: -p tedge
+
+      - name: Build tedge_mapper debian package
+        uses: actions-rs/cargo@v1
+        # https://github.com/marketplace/actions/rust-cargo
+        with:
+          command: deb
+          args: -p tedge_mapper
+
+      - name: Build tedge_apt_plugin debian package
+        uses: actions-rs/cargo@v1
+        # https://github.com/marketplace/actions/rust-cargo
+        with:
+          command: deb
+          args: -p tedge_apt_plugin
+
+      - name: Build tedge_apama_plugin debian package
+        uses: actions-rs/cargo@v1
+        # https://github.com/marketplace/actions/rust-cargo
+        with:
+          command: deb
+          args: -p tedge_apama_plugin
+
+      - name: Build tedge_agent debian package
+        uses: actions-rs/cargo@v1
+        # https://github.com/marketplace/actions/rust-cargo
+        with:
+          command: deb
+          args: -p tedge_agent
+
+      - name: Build tedge_logfile_request_plugin debian package
+        uses: actions-rs/cargo@v1
+        # https://github.com/marketplace/actions/rust-cargo
+        with:
+          command: deb
+          args: -p tedge_logfile_request_plugin
+
+      - name: build sawtooth-publisher for amd64
+        uses: actions-rs/cargo@v1
+        # https://github.com/marketplace/actions/rust-cargo
+        with:
+          command: build
+          args: --release -p sawtooth_publisher
+
+      - name: Upload artifacts as zip
+        # https://github.com/marketplace/actions/upload-a-build-artifact
+        uses: actions/upload-artifact@v2
+        with:
+          name: debian-packages-amd64
+          path: target/debian/*.deb
+
+      - name: upload amd64 sawtooth-publisher as zip
+        # https://github.com/marketplace/actions/upload-a-build-artifact
+        uses: actions/upload-artifact@v2
+        with:
+          name: sawtooth_publisher_amd64
+          path: target/release/sawtooth_publisher
+
+      - name: Build tedge_dummy_plugin
+        uses: actions-rs/cargo@v1
+        # https://github.com/marketplace/actions/rust-cargo
+        with:
+          command: build
+          args: --release -p tedge_dummy_plugin
+
+      - name: upload dummy-plugin
+        # https://github.com/marketplace/actions/upload-a-build-artifact
+        uses: actions/upload-artifact@v2
+        with:
+          name: tedge_dummy_plugin_amd64
+          path: target/release/tedge_dummy_plugin
+
+  build_matrix_arm:
+    name: Build tedge and mapper Debian for armv7
+    runs-on: Ubuntu-20.04
+    strategy:
+      matrix:
+        # Add only arm targets here as we use a custom strip binary!
+        target:
+          [
+            aarch64-unknown-linux-gnu,
+            aarch64-unknown-linux-musl,
+            arm-unknown-linux-gnueabihf,
+            armv7-unknown-linux-gnueabihf,
+            armv7-unknown-linux-musleabihf,
+          ]
+
+    steps:
+      - name: checkout
+        uses: actions/checkout@v2
+
+      - name: enable toolchain via github action
+        # https://github.com/actions-rs/toolchain
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: 1.58.1
+          target: ${{ matrix.target }}
+          override: true
+
+      - name: Enable cache
+        # https://github.com/marketplace/actions/rust-cache
+        uses: Swatinem/rust-cache@v1
+
+      - name: install cargo-deb
+        uses: actions-rs/cargo@v1
+        # https://github.com/marketplace/actions/rust-cargo
+        with:
+          command: install
+          args: cargo-deb --version 1.34.2
+
+      - name: install cargo-strip
+        uses: actions-rs/cargo@v1
+        # https://github.com/marketplace/actions/rust-cargo
+        with:
+          command: install
+          args: cargo-strip
+
+      - name: build cross release for target
+        uses: actions-rs/cargo@v1
+        # https://github.com/marketplace/actions/rust-cargo
+        with:
+          use-cross: true
+          command: build
+          args: --release --target=${{ matrix.target }}
+
+      - name: apt update
+        run: sudo apt update
+
+      # armv7 uses `arm-linux-gnueabihf-strip`; aarch64 uses `aarch64-linux-gnu-strip`
+      # It appears `aarch64-linux-gnu-strip` seems to work explicitly on other arm bins but not other way around.
+      - name: Install binutils to add `strip` for striping arm binaries
+        run: sudo apt-get --assume-yes install binutils-arm-linux-gnueabihf binutils-aarch64-linux-gnu
+
+      - name: Strip tedge
+        run: arm-linux-gnueabihf-strip target/${{ matrix.target }}/release/tedge || aarch64-linux-gnu-strip target/${{ matrix.target }}/release/tedge
+
+      - name: Strip tedge_mapper
+        run: arm-linux-gnueabihf-strip target/${{ matrix.target }}/release/tedge_mapper || aarch64-linux-gnu-strip target/${{ matrix.target }}/release/tedge_mapper
+
+      - name: Strip tedge_agent
+        run: arm-linux-gnueabihf-strip target/${{ matrix.target }}/release/tedge_agent || aarch64-linux-gnu-strip target/${{ matrix.target }}/release/tedge_agent
+
+      - name: Strip tedge_apt_plugin
+        run: arm-linux-gnueabihf-strip target/${{ matrix.target }}/release/tedge_apt_plugin || aarch64-linux-gnu-strip target/${{ matrix.target }}/release/tedge_apt_plugin
+
+      - name: Strip tedge_apama_plugin
+        run: arm-linux-gnueabihf-strip target/${{ matrix.target }}/release/tedge_apama_plugin || aarch64-linux-gnu-strip target/${{ matrix.target }}/release/tedge_apama_plugin
+
+      - name: Strip tedge_logfile_request_plugin
+        run: arm-linux-gnueabihf-strip target/${{ matrix.target }}/release/tedge_logfile_request_plugin || aarch64-linux-gnu-strip target/${{ matrix.target }}/release/tedge_logfile_request_plugin
+
+      -