From ad487ffce9c5f2016b9c1caf7f3305d5e4cbe5f7 Mon Sep 17 00:00:00 2001 From: Michael Abel <75477722+abelikt@users.noreply.github.com> Date: Tue, 15 Feb 2022 17:40:26 +0100 Subject: [#835] Dynamic Cumulocity id management in system tests (#839) * Add find_device_id.py from ci branch * Add delete_current_device_c8y.py from ci branch * Use URL from GitHub secrets * Disconnect old bridge before configuring the new one * Add shebang to avoid that we run this with dash * Delete the current device, reconenct and retrieve the new ID * Regenerate bridge in integration test workflow * Use ID from within the script * Remove unused, forgotten variables * Use dynamic C8YDEVICEID in smoke_test * Use dynamic C8YDEVICEID in plugin_test * Use dynamic C8YDEVICEID in software management tests * Use dynamic C8YDEVICEID to run all tests ... ... and avoid an additional bridge configuration * Also configure bridge in offsite workflow ... ... and fix a job name * Move invocation of find device ... ... where we have an enabled environment. And specifically call the interpreter in the environment * Use separate environment in the smoke test and ... ... switch to bash to have source * Also use c8y-api environment for bridge setup * Set executable bits * Also add the c8y API to the requirements * Work on review comments * Replace check for zero with regex * Use name keyword to get the right device --- .github/workflows/integration-test-workflow.yml | 13 ++++- .github/workflows/system-test-offsite.yml | 11 ++++ .github/workflows/system-test-workflow.yml | 3 - ci/ci_run_all_plugin_tests.sh | 4 +- ci/ci_run_all_sm_tests.sh | 6 +- ci/ci_run_all_tests.sh | 7 +-- ci/ci_smoke_test_c8y.sh | 32 ++++------- ci/configure_bridge.sh | 40 +++++++++++++- ci/delete_current_device_c8y.py | 73 +++++++++++++++++++++++++ ci/find_device_id.py | 66 ++++++++++++++++++++++ tests/requirements.txt | 2 + 11 files changed, 224 insertions(+), 33 deletions(-) create mode 100755 ci/delete_current_device_c8y.py create mode 100755 ci/find_device_id.py diff --git a/.github/workflows/integration-test-workflow.yml b/.github/workflows/integration-test-workflow.yml index 9a22c00c..d920932c 100644 --- a/.github/workflows/integration-test-workflow.yml +++ b/.github/workflows/integration-test-workflow.yml @@ -201,6 +201,18 @@ jobs: - name: chmod dummy_plugin run: chmod +x /home/pi/tedge_dummy_plugin/tedge_dummy_plugin + - name: Configure Bridge + run: ./ci/configure_bridge.sh + env: + C8YPASS: ${{ secrets.SECRET_C8YPASS }} + C8YUSERNAME: ${{ secrets.SECRET_C8YUSERNAME }} + C8YTENANT: ${{secrets.SECRET_C8YTENANT}} + C8YDEVICE: ${{ secrets.SECRET_C8YDEVICE }} + TEBASEDIR: /home/pi/actions-runner/_work/thin-edge.io/thin-edge.io/ + EXAMPLEDIR: /home/pi/examples + C8YURL: https://thin-edge-io.eu-latest.cumulocity.com + IOTHUBNAME: ${{ secrets.IOTHUBNAME }} + - name: Run smoke test for Cumulocity run: ./ci/ci_smoke_test_c8y.sh env: @@ -208,7 +220,6 @@ jobs: C8YUSERNAME: ${{ secrets.SECRET_C8YUSERNAME }} C8YTENANT: ${{secrets.SECRET_C8YTENANT}} C8YDEVICE: ${{ secrets.SECRET_C8YDEVICE }} - C8YDEVICEID: ${{ secrets.SECRET_C8YDEVICEID }} TEBASEDIR: /home/pi/actions-runner/_work/thin-edge.io/thin-edge.io/ EXAMPLEDIR: /home/pi/examples C8YURL: https://thin-edge-io.eu-latest.cumulocity.com diff --git a/.github/workflows/system-test-offsite.yml b/.github/workflows/system-test-offsite.yml index a60ed81f..97d4c2ac 100644 --- a/.github/workflows/system-test-offsite.yml +++ b/.github/workflows/system-test-offsite.yml @@ -85,6 +85,17 @@ jobs: - name: chmod dummy_plugin run: chmod +x /home/pi/tedge_dummy_plugin/tedge_dummy_plugin + - name: Configure Bridge + run: ./ci/configure_bridge.sh + env: + C8YPASS: ${{ secrets.SECRET_C8YPASS }} + C8YUSERNAME: ${{ secrets.SECRET_C8YUSERNAME }} + C8YTENANT: ${{secrets.SECRET_C8YTENANT}} + C8YDEVICE: ${{ secrets.SECRET_C8YDEVICE_OFFSITE_A }} + TEBASEDIR: /home/pi/actions-runner/_work/thin-edge.io/thin-edge.io/ + EXAMPLEDIR: /home/pi/examples + C8YURL: https://thin-edge-io.eu-latest.cumulocity.com + - name: Run Smoke Test run: ./ci/ci_smoke_test_c8y.sh env: diff --git a/.github/workflows/system-test-workflow.yml b/.github/workflows/system-test-workflow.yml index c8874c32..7284c6d7 100644 --- a/.github/workflows/system-test-workflow.yml +++ b/.github/workflows/system-test-workflow.yml @@ -28,12 +28,9 @@ jobs: env: C8YPASS: ${{ secrets.SECRET_C8YPASS }} C8YUSERNAME: ${{ secrets.SECRET_C8YUSERNAME }} - C8YTENNANT: ${{secrets.SECRET_C8YTENANT}} C8YDEVICE: ${{ secrets.SECRET_C8YDEVICE }} C8YTENANT: ${{secrets.SECRET_C8YTENANT}} C8YURL: https://thin-edge-io.eu-latest.cumulocity.com - C8YDEVICEID: ${{ secrets.SECRET_C8YDEVICEID }} - TIMEZONE: 01:00 TEBASEDIR: /home/pi/actions-runner/_work/thin-edge.io/thin-edge.io/ EXAMPLEDIR: /home/pi diff --git a/ci/ci_run_all_plugin_tests.sh b/ci/ci_run_all_plugin_tests.sh index 1f9e0962..0775a7d0 100755 --- a/ci/ci_run_all_plugin_tests.sh +++ b/ci/ci_run_all_plugin_tests.sh @@ -8,8 +8,6 @@ # C8YUSERNAME : Cumolocity username # C8YTENANT : Cumolocity tenant # C8YDEVICE : The device name -# C8YDEVICEID : The device ID in Cumolocity -# TIMEZONE : Your timezone (temporary) # TEBASEDIR : Base directory for the Thin-Edge repo # EXAMPLEDIR : The direcory of the sawtooth example # C8YURL : e.g. https://thin-edge-io.eu-latest.cumulocity.com @@ -26,6 +24,8 @@ dpkg -s mosquitto-clients python3 -m venv ~/env-pysys source ~/env-pysys/bin/activate pip3 install -r tests/requirements.txt +export C8YDEVICEID=$(python3 ./ci/find_device_id.py --tenant $C8YTENANT --user $C8YUSERNAME --device $C8YDEVICE --url $C8YURL) + cd tests/PySys/ sudo tedge config set software.plugin.default apt diff --git a/ci/ci_run_all_sm_tests.sh b/ci/ci_run_all_sm_tests.sh index ab6bee1f..c3d698bf 100755 --- a/ci/ci_run_all_sm_tests.sh +++ b/ci/ci_run_all_sm_tests.sh @@ -8,14 +8,13 @@ # C8YUSERNAME : Cumolocity username # C8YTENANT : Cumolocity tenant # C8YDEVICE : The device name -# C8YDEVICEID : The device ID in Cumolocity -# TIMEZONE : Your timezone (temporary) # TEBASEDIR : Base directory for the Thin-Edge repo # EXAMPLEDIR : The direcory of the sawtooth example # C8YURL : e.g. https://thin-edge-io.eu-latest.cumulocity.com set -e + cd $TEBASEDIR # Check if clients are installed @@ -46,6 +45,9 @@ sudo cp tests/PySys/software_management_end_to_end/dummy_plugin_configuration/li python3 -m venv ~/env-pysys source ~/env-pysys/bin/activate pip3 install -r tests/requirements.txt + +export C8YDEVICEID=$(python3 ./ci/find_device_id.py --tenant $C8YTENANT --user $C8YUSERNAME --device $C8YDEVICE --url $C8YURL) + cd tests/PySys/ # Run all software management tests, including the ones for the diff --git a/ci/ci_run_all_tests.sh b/ci/ci_run_all_tests.sh index 836cf571..d26dbcb1 100755 --- a/ci/ci_run_all_tests.sh +++ b/ci/ci_run_all_tests.sh @@ -8,8 +8,6 @@ # C8YUSERNAME : Cumolocity username # C8YTENANT : Cumolocity tenant # C8YDEVICE : The device name -# C8YDEVICEID : The device ID in Cumolocity -# TIMEZONE : Your timezone (temporary) # TEBASEDIR : Base directory for the Thin-Edge repo # EXAMPLEDIR : The direcory of the sawtooth example # C8YURL : e.g. https://thin-edge-io.eu-latest.cumulocity.com @@ -32,13 +30,14 @@ cd $TEBASEDIR # Check if clients are installed dpkg -s mosquitto-clients -./ci/configure_bridge.sh - # Run all PySys tests python3 -m venv ~/env-pysys source ~/env-pysys/bin/activate pip3 install -r tests/requirements.txt + +export C8YDEVICEID=$(python3 ./ci/find_device_id.py --tenant $C8YTENANT --user $C8YUSERNAME --device $C8YDEVICE --url $C8YURL) + cd tests/PySys/ set +e diff --git a/ci/ci_smoke_test_c8y.sh b/ci/ci_smoke_test_c8y.sh index b7ddd123..28182f59 100755 --- a/ci/ci_smoke_test_c8y.sh +++ b/ci/ci_smoke_test_c8y.sh @@ -1,4 +1,4 @@ -#!/usr/bin/sh +#!/usr/bin/bash # Smoke test for Cumulocity # - Rebuild bridge @@ -16,7 +16,6 @@ # C8YUSERNAME # C8YTENANT # C8YPASS -# C8YDEVICEID # EXAMPLEDIR # a simple function to append lines to files if not already there @@ -37,13 +36,6 @@ else echo "Your device: HIDDEN" fi -if [ -z $C8YDEVICEID ]; then - echo "Error: Please supply your Cumulocity device ID name as environment variable C8YDEVICEID" - exit 1 -else - echo "Your device: HIDDEN" -fi - if [ -z $C8YUSERNAME ]; then echo "Error: Please supply your user name as environment variable C8YUSERNAME" exit 1 @@ -82,19 +74,19 @@ fi # Adding sbin seems to be necessary for non Raspberry P OS systems as Debian or Ubuntu PATH=$PATH:/usr/sbin -echo "Disconnect old bridge" +python3 -m venv ~/env-c8y-api +source ~/env-c8y-api/bin/activate +pip3 install c8y-api +export C8YDEVICEID=$(python3 ./ci/find_device_id.py --tenant $C8YTENANT --user $C8YUSERNAME --device $C8YDEVICE --url $C8YURL) -# Disconnect - may fail if not there -sudo tedge disconnect c8y - -# From now on exit if a command exits with a non-zero status. -# Commands above are allowed to fail -set -e - -./ci/configure_bridge.sh +# after calling the script, the ID should be a numeric value +if [[ $C8YDEVICEID =~ ^[0-9]+$ ]]; then + echo "Your device ID: $C8YDEVICEID" +else + echo "Error: Please supply your Cumulocity device ID name as environment variable C8YDEVICEID" + exit 1 +fi -# wait for certificate to to reflect in the c8y cloud -sleep 5 echo "Connect again" sudo tedge connect c8y diff --git a/ci/configure_bridge.sh b/ci/configure_bridge.sh index 5c1f9c77..a66bda5b 100755 --- a/ci/configure_bridge.sh +++ b/ci/configure_bridge.sh @@ -1,15 +1,25 @@ +#!/bin/bash +echo "Disconnect old bridge" + +# Disconnect - may fail if not there +sudo tedge disconnect c8y + +# From now on exit if a command exits with a non-zero status. +# Commands above are allowed to fail set -e echo "Configuring Bridge" +URL=$(echo $C8YURL | cut -c 9- - ) + sudo tedge cert remove sudo tedge cert create --device-id=$C8YDEVICE sudo tedge cert show -sudo tedge config set c8y.url thin-edge-io.eu-latest.cumulocity.com +sudo tedge config set c8y.url $URL sudo tedge config set c8y.root.cert.path /etc/ssl/certs @@ -24,3 +34,31 @@ sudo tedge config list sudo -E tedge cert upload c8y --user $C8YUSERNAME cat /etc/mosquitto/mosquitto.conf + +python3 -m venv ~/env-c8y-api +source ~/env-c8y-api/bin/activate +pip3 install c8y-api + +# Delete the device (ignore error) +set +e +python3 ./ci/delete_current_device_c8y.py --tenant $C8YTENANT --user $C8YUSERNAME --device $C8YDEVICE --url $C8YURL +set -e + +# Give Cumolocity time to process the cert deletion +sleep 2 + +# Connect and disconnect so that we can retrive a new device ID +sudo tedge connect c8y +sudo tedge disconnect c8y + +# Give Cumolocity time to process the cert deletion +sleep 2 + +# Retrieve the Cumulocity device ID + +export C8YDEVICEID=$(python3 ./ci/find_device_id.py --tenant $C8YTENANT --user $C8YUSERNAME --device $C8YDEVICE --url $C8YURL) + +echo "The current device ID is (read from home directory): " $C8YDEVICEID + +deactivate + diff --git a/ci/delete_current_device_c8y.py b/ci/delete_current_device_c8y.py new file mode 100755 index 00000000..f13a50ad --- /dev/null +++ b/ci/delete_current_device_c8y.py @@ -0,0 +1,73 @@ +#!/usr/bin/python3 + +""" +Delete device from Cumulocity with a given name. + +For example: + + python3 ci/delete_current_device.py --tenant t493319102 --user octocat + --device devraspberrypi --url thin-edge-io.eu-latest.cumulocity.com + +""" + +import argparse +import os +import sys +from c8y_api import CumulocityApi + + +def delete_object(c8y, obj): + """Delete object from inventory""" + try: + c8y.inventory.get(obj).delete() + except KeyError: + print(f"Object with {obj} not there") + print(f"Deleted object with ID {obj}") + + +def delete_device(c8y, name, verbose): + """Delete the current device""" + devices = c8y.device_inventory.get_all(name=name) + if len(devices) == 1: + dev = devices[0] + print(f"Device has ID {dev.id}") + delete_object(c8y, dev.id) + return True + return False + + +def main(): + """Main entry point""" + + parser = argparse.ArgumentParser() + parser.add_argument("--tenant", required=True, help="C8y Tenant") + parser.add_argument("--user", required=True, help="C8y User") + parser.add_argument("--device", required=True, help="Device to find") + parser.add_argument("--url", required=True, help="URL of C8y") + parser.add_argument("--verbose", "-v", action="count", default=0) + + args = parser.parse_args() + + tenant = args.tenant + user = args.user + device_name = args.device + url = args.url + verbose = args.verbose + + try: + password = os.environ["C8YPASS"] + except KeyError: + print("Please export your password into $C8YPASS") + sys.exit(1) + + c8y = CumulocityApi(url, tenant, user, password) + + if delete_device(c8y, device_name, verbose): + print("Deleted device from C8y") + else: + print("Failed to delete device from C8y") + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/ci/find_device_id.py b/ci/find_device_id.py new file mode 100755 index 00000000..816274e4 --- /dev/null +++ b/ci/find_device_id.py @@ -0,0 +1,66 @@ +#!/usr/bin/python3 + +"""Find the device ID for a given device in C8y and print it on the +command line. + +For example: + + python3 ci/find_device_id.py --tenant t493319102 --user octocat + --device devraspberrypi --url thin-edge-io.eu-latest.cumulocity.com + +""" + +import argparse +import os +import sys +from c8y_api import CumulocityApi + + +def get_device_id(c8y, name): + """retrive the current device ID""" + + devices = c8y.device_inventory.get_all(name=name) + if len(devices) == 1: + return devices[0].id + return None + + +def main(): + """Main entry point""" + + parser = argparse.ArgumentParser() + parser.add_argument("--tenant", required=True, help="C8y Tenant") + parser.add_argument("--user", required=True, help="C8y User") + parser.add_argument("--device", required=True, help="Device to find") + parser.add_argument("--url", required=True, help="URL of C8y") + parser.add_argument("--verbose", "-v", action="count", default=0) + + args = parser.parse_args() + + tenant = args.tenant + user = args.user + device_name = args.device + url = args.url + verbose = args.verbose + + try: + password = os.environ["C8YPASS"] + except KeyError: + print("Please export your password into $C8YPASS") + sys.exit(1) + + c8y = CumulocityApi(url, tenant, user, password) + + device_id = get_device_id(c8y, device_name) + if device_id: + if verbose: + print("The current device ID is:") + print(device_id) + + else: + print("Failed to find device in C8y") + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/tests/requirements.txt b/tests/requirements.txt index f32fdafc..457f65f5 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -5,3 +5,5 @@ retry == 0.9.2 junitparser == 2.2.0 junit2html == 30.0.6 psutil == 5.9.0 +c8y-api == 1.1.1 + -- cgit v1.2.3