[#835] Dynamic Cumulocity id management in system tests (#839)

* Add find_device_id.py from ci branch

* Add delete_current_device_c8y.py from ci branch

* Use URL from GitHub secrets

* Disconnect old bridge before configuring the new one

* Add shebang to avoid that we run this with dash

* Delete the current device, reconenct and retrieve the new ID

* Regenerate bridge in integration test workflow

* Use ID from within the script

* Remove unused, forgotten variables

* Use dynamic C8YDEVICEID in smoke_test

* Use dynamic C8YDEVICEID in plugin_test

* Use dynamic C8YDEVICEID in software management tests

* Use dynamic C8YDEVICEID to run all tests ...

... and avoid an additional bridge configuration

* Also configure bridge in offsite workflow ...

... and fix a job name

* Move invocation of find device ...

... where we have an enabled environment. And specifically call the
interpreter in the environment

* Use separate environment in the smoke test and ...

... switch to bash to have source

* Also use c8y-api environment for bridge setup

* Set executable bits

* Also add the c8y API to the requirements

* Work on review comments

* Replace check for zero with regex

* Use name keyword to get the right device

11 files changed, 224 insertions, 33 deletions

diff --git a/.github/workflows/integration-test-workflow.yml b/.github/workflows/integration-test-workflow.yml
index 9a22c00c..d920932c 100644
--- a/.github/workflows/integration-test-workflow.yml
+++ b/.github/workflows/integration-test-workflow.yml
@@ -201,6 +201,18 @@ jobs:
       - name: chmod dummy_plugin
         run: chmod +x /home/pi/tedge_dummy_plugin/tedge_dummy_plugin
 
+      - name: Configure Bridge
+        run: ./ci/configure_bridge.sh
+        env:
+          C8YPASS: ${{ secrets.SECRET_C8YPASS }}
+          C8YUSERNAME: ${{ secrets.SECRET_C8YUSERNAME }}
+          C8YTENANT: ${{secrets.SECRET_C8YTENANT}}
+          C8YDEVICE: ${{ secrets.SECRET_C8YDEVICE }}
+          TEBASEDIR: /home/pi/actions-runner/_work/thin-edge.io/thin-edge.io/
+          EXAMPLEDIR: /home/pi/examples
+          C8YURL: https://thin-edge-io.eu-latest.cumulocity.com
+          IOTHUBNAME: ${{ secrets.IOTHUBNAME }}
+
       - name: Run smoke test for Cumulocity
         run: ./ci/ci_smoke_test_c8y.sh
         env:
@@ -208,7 +220,6 @@ jobs:
           C8YUSERNAME: ${{ secrets.SECRET_C8YUSERNAME }}
           C8YTENANT: ${{secrets.SECRET_C8YTENANT}}
           C8YDEVICE: ${{ secrets.SECRET_C8YDEVICE }}
-          C8YDEVICEID: ${{ secrets.SECRET_C8YDEVICEID }}
           TEBASEDIR: /home/pi/actions-runner/_work/thin-edge.io/thin-edge.io/
           EXAMPLEDIR: /home/pi/examples
           C8YURL: https://thin-edge-io.eu-latest.cumulocity.com
diff --git a/.github/workflows/system-test-offsite.yml b/.github/workflows/system-test-offsite.yml
index a60ed81f..97d4c2ac 100644
--- a/.github/workflows/system-test-offsite.yml
+++ b/.github/workflows/system-test-offsite.yml
@@ -85,6 +85,17 @@ jobs:
       - name: chmod dummy_plugin
         run: chmod +x /home/pi/tedge_dummy_plugin/tedge_dummy_plugin
 
+      - name: Configure Bridge
+        run: ./ci/configure_bridge.sh
+        env:
+          C8YPASS: ${{ secrets.SECRET_C8YPASS }}
+          C8YUSERNAME: ${{ secrets.SECRET_C8YUSERNAME }}
+          C8YTENANT: ${{secrets.SECRET_C8YTENANT}}
+          C8YDEVICE: ${{ secrets.SECRET_C8YDEVICE_OFFSITE_A }}
+          TEBASEDIR: /home/pi/actions-runner/_work/thin-edge.io/thin-edge.io/
+          EXAMPLEDIR: /home/pi/examples
+          C8YURL: https://thin-edge-io.eu-latest.cumulocity.com
+
       - name: Run Smoke Test
         run: ./ci/ci_smoke_test_c8y.sh
         env:
diff --git a/.github/workflows/system-test-workflow.yml b/.github/workflows/system-test-workflow.yml
index c8874c32..7284c6d7 100644
--- a/.github/workflows/system-test-workflow.yml
+++ b/.github/workflows/system-test-workflow.yml
@@ -28,12 +28,9 @@ jobs:
       env:
             C8YPASS: ${{ secrets.SECRET_C8YPASS }}
             C8YUSERNAME: ${{ secrets.SECRET_C8YUSERNAME }}
-            C8YTENNANT: ${{secrets.SECRET_C8YTENANT}}
             C8YDEVICE: ${{ secrets.SECRET_C8YDEVICE }}
             C8YTENANT: ${{secrets.SECRET_C8YTENANT}}
             C8YURL: https://thin-edge-io.eu-latest.cumulocity.com
-            C8YDEVICEID: ${{ secrets.SECRET_C8YDEVICEID }}
-            TIMEZONE: 01:00
             TEBASEDIR: /home/pi/actions-runner/_work/thin-edge.io/thin-edge.io/
             EXAMPLEDIR: /home/pi
 
diff --git a/ci/ci_run_all_plugin_tests.sh b/ci/ci_run_all_plugin_tests.sh
index 1f9e0962..0775a7d0 100755
--- a/ci/ci_run_all_plugin_tests.sh
+++ b/ci/ci_run_all_plugin_tests.sh
@@ -8,8 +8,6 @@
 # C8YUSERNAME : Cumolocity username
 # C8YTENANT : Cumolocity tenant
 # C8YDEVICE : The device name
-# C8YDEVICEID : The device ID in Cumolocity
-# TIMEZONE : Your timezone (temporary)
 # TEBASEDIR : Base directory for the Thin-Edge repo
 # EXAMPLEDIR : The direcory of the sawtooth example
 # C8YURL : e.g. https://thin-edge-io.eu-latest.cumulocity.com
@@ -26,6 +24,8 @@ dpkg -s mosquitto-clients
 python3 -m venv ~/env-pysys
 source ~/env-pysys/bin/activate
 pip3 install -r tests/requirements.txt
+export C8YDEVICEID=$(python3 ./ci/find_device_id.py --tenant $C8YTENANT --user $C8YUSERNAME --device $C8YDEVICE --url $C8YURL)
+
 cd tests/PySys/
 
 sudo tedge config set software.plugin.default apt
diff --git a/ci/ci_run_all_sm_tests.sh b/ci/ci_run_all_sm_tests.sh
index ab6bee1f..c3d698bf 100755
--- a/ci/ci_run_all_sm_tests.sh
+++ b/ci/ci_run_all_sm_tests.sh
@@ -8,14 +8,13 @@
 # C8YUSERNAME : Cumolocity username
 # C8YTENANT : Cumolocity tenant
 # C8YDEVICE : The device name
-# C8YDEVICEID : The device ID in Cumolocity
-# TIMEZONE : Your timezone (temporary)
 # TEBASEDIR : Base directory for the Thin-Edge repo
 # EXAMPLEDIR : The direcory of the sawtooth example
 # C8YURL : e.g. https://thin-edge-io.eu-latest.cumulocity.com
 
 set -e
 
+
 cd $TEBASEDIR
 
 # Check if clients are installed
@@ -46,6 +45,9 @@ sudo cp tests/PySys/software_management_end_to_end/dummy_plugin_configuration/li
 python3 -m venv ~/env-pysys
 source ~/env-pysys/bin/activate
 pip3 install -r tests/requirements.txt
+
+export C8YDEVICEID=$(python3 ./ci/find_device_id.py --tenant $C8YTENANT --user $C8YUSERNAME --device $C8YDEVICE --url $C8YURL)
+
 cd tests/PySys/
 
 # Run all software management tests, including the ones for the
diff --git a/ci/ci_run_all_tests.sh b/ci/ci_run_all_tests.sh
index 836cf571..d26dbcb1 100755
--- a/ci/ci_run_all_tests.sh
+++ b/ci/ci_run_all_tests.sh
@@ -8,8 +8,6 @@
 # C8YUSERNAME : Cumolocity username
 # C8YTENANT : Cumolocity tenant
 # C8YDEVICE : The device name
-# C8YDEVICEID : The device ID in Cumolocity
-# TIMEZONE : Your timezone (temporary)
 # TEBASEDIR : Base directory for the Thin-Edge repo
 # EXAMPLEDIR : The direcory of the sawtooth example
 # C8YURL : e.g. https://thin-edge-io.eu-latest.cumulocity.com
@@ -32,13 +30,14 @@ cd $TEBASEDIR
 # Check if clients are installed
 dpkg -s mosquitto-clients
 
-./ci/configure_bridge.sh
-
 # Run all PySys tests
 
 python3 -m venv ~/env-pysys
 source ~/env-pysys/bin/activate
 pip3 install -r tests/requirements.txt
+
+export C8YDEVICEID=$(python3 ./ci/find_device_id.py --tenant $C8YTENANT --user $C8YUSERNAME --device $C8YDEVICE --url $C8YURL)
+
 cd tests/PySys/
 
 set +e
diff --git a/ci/ci_smoke_test_c8y.sh b/ci/ci_smoke_test_c8y.sh
index b7ddd123..28182f59 100755
--- a/ci/ci_smoke_test_c8y.sh
+++ b/ci/ci_smoke_test_c8y.sh
@@ -1,4 +1,4 @@
-#!/usr/bin/sh
+#!/usr/bin/bash
 
 # Smoke test for Cumulocity
 # - Rebuild bridge
@@ -16,7 +16,6 @@
 #    C8YUSERNAME
 #    C8YTENANT
 #    C8YPASS
-#    C8YDEVICEID
 #    EXAMPLEDIR
 
 # a simple function to append lines to files if not already there
@@ -37,13 +36,6 @@ else
     echo "Your device: HIDDEN"
 fi
 
-if [ -z $C8YDEVICEID ]; then
-    echo "Error: Please supply your Cumulocity device ID  name as environment variable C8YDEVICEID"
-    exit 1
-else
-    echo "Your device: HIDDEN"
-fi
-
 if [ -z $C8YUSERNAME ]; then
     echo "Error: Please supply your user name  as environment variable C8YUSERNAME"
     exit 1
@@ -82,19 +74,19 @@ fi
 # Adding sbin seems to be necessary for non Raspberry P OS systems as Debian or Ubuntu
 PATH=$PATH:/usr/sbin
 
-echo "Disconnect old bridge"
+python3 -m venv ~/env-c8y-api
+source ~/env-c8y-api/bin/activate
+pip3 install c8y-api
+export C8YDEVICEID=$(python3 ./ci/find_device_id.py --tenant $C8YTENANT --user $C8YUSERNAME --device $C8YDEVICE --url $C8YURL)
 
-# Disconnect - may fail if not there
-sudo tedge disconnect c8y
-
-# From now on exit if a command exits with a non-zero status.
-# Commands above are allowed to fail
-set -e
-
-./ci/configure_bridge.sh
+# after calling the script, the ID should be a numeric value
+if [[ $C8YDEVICEID =~ ^[0-9]+$ ]]; then
+    echo "Your device ID: $C8YDEVICEID"
+else
+    echo "Error: Please supply your Cumulocity device ID  name as environment variable C8YDEVICEID"
+    exit 1
+fi
 
-# wait for certificate to to reflect in the c8y cloud
-sleep 5
 
 echo "Connect again"
 sudo tedge connect c8y
diff --git a/ci/configure_bridge.sh b/ci/configure_bridge.sh
index 5c1f9c77..a66bda5b 100755
--- a/ci/configure_bridge.sh
+++ b/ci/configure_bridge.sh
@@ -1,15 +1,25 @@
+#!/bin/bash
 
+echo "Disconnect old bridge"
+
+# Disconnect - may fail if not there
+sudo tedge disconnect c8y
+
+# From now on exit if a command exits with a non-zero status.
+# Commands above are allowed to fail
 set -e
 
 echo "Configuring Bridge"
 
+URL=$(echo $C8YURL | cut -c 9- - )
+
 sudo tedge cert remove
 
 sudo tedge cert create --device-id=$C8YDEVICE
 
 sudo tedge cert show
 
-sudo tedge config set c8y.url thin-edge-io.eu-latest.cumulocity.com
+sudo tedge config set c8y.url $URL
 
 sudo tedge config set c8y.root.cert.path /etc/ssl/certs
 
@@ -24,3 +34,31 @@ sudo tedge config list
 sudo -E tedge cert upload c8y --user $C8YUSERNAME
 
 cat /etc/mosquitto/mosquitto.conf
+
+python3 -m venv ~/env-c8y-api
+source ~/env-c8y-api/bin/activate
+pip3 install c8y-api
+
+# Delete the device (ignore error)
+set +e
+python3 ./ci/delete_current_device_c8y.py --tenant $C8YTENANT --user $C8YUSERNAME --device $C8YDEVICE --url $C8YURL
+set -e
+
+# Give Cumolocity time to process the cert deletion
+sleep 2
+
+# Connect and disconnect so that we can retrive a new device ID
+sudo tedge connect c8y
+sudo tedge disconnect c8y
+
+# Give Cumolocity time to process the cert deletion
+sleep 2
+
+# Retrieve the Cumulocity device ID
+
+export C8YDEVICEID=$(python3 ./ci/find_device_id.py --tenant $C8YTENANT --user $C8YUSERNAME --device $C8YDEVICE --url $C8YURL)
+
+echo "The current device ID is (read from home directory): " $C8YDEVICEID
+
+deactivate
+
diff --git a/ci/delete_current_device_c8y.py b/ci/delete_current_device_c8y.py
new file mode 100755
index 00000000..f13a50ad
--- /dev/null
+++ b/ci/delete_current_device_c8y.py
@@ -0,0 +1,73 @@
+#!/usr/bin/python3
+
+"""
+Delete device from Cumulocity with a given name.
+
+For example:
+
+    python3 ci/delete_current_device.py --tenant t493319102 --user octocat
+      --device devraspberrypi --url thin-edge-io.eu-latest.cumulocity.com
+
+"""
+
+import argparse
+import os
+import sys
+from c8y_api import CumulocityApi
+
+
+def delete_object(c8y, obj):
+    """Delete object from inventory"""
+    try:
+        c8y.inventory.get(obj).delete()
+    except KeyError:
+        print(f"Object with {obj} not there")
+    print(f"Deleted object with ID {obj}")
+
+
+def delete_device(c8y, name, verbose):
+    """Delete the current device"""
+    devices = c8y.device_inventory.get_all(name=name)
+    if len(devices) == 1:
+        dev = devices[0]
+        print(f"Device has ID {dev.id}")
+        delete_object(c8y, dev.id)
+        return True
+    return False
+
+
+def main():
+    """Main entry point"""
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--tenant", required=True, help="C8y Tenant")
+    parser.add_argument("--user", required=True, help="C8y User")
+    parser.add_argument("--device", required=True, help="Device to find")
+    parser.add_argument("--url", required=True, help="URL of C8y")
+    parser.add_argument("--verbose", "-v", action="count", default=0)
+
+    args = parser.parse_args()
+
+    tenant = args.tenant
+    user = args.user
+    device_name = args.device
+    url = args.url
+    verbose = args.verbose
+
+    try:
+        password = os.environ["C8YPASS"]
+    except KeyError:
+        print("Please export your password into $C8YPASS")
+        sys.exit(1)
+
+    c8y = CumulocityApi(url, tenant, user, password)
+
+    if delete_device(c8y, device_name, verbose):
+        print("Deleted device from C8y")
+    else:
+        print("Failed to delete device from C8y")
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()
diff --git a/ci/find_device_id.py b/ci/find_device_id.py
new file mode 100755
index 00000000..816274e4
--- /dev/null
+++ b/ci/find_device_id.py
@@ -0,0 +1,66 @@
+#!/usr/bin/python3
+
+"""Find the device ID for a given device in C8y and print it on the
+command line.
+
+For example:
+
+    python3 ci/find_device_id.py --tenant t493319102 --user octocat
+      --device devraspberrypi --url thin-edge-io.eu-latest.cumulocity.com
+
+"""
+
+import argparse
+import os
+import sys
+from c8y_api import CumulocityApi
+
+
+def get_device_id(c8y, name):
+    """retrive the current device ID"""
+
+    devices = c8y.device_inventory.get_all(name=name)
+    if len(devices) == 1:
+        return devices[0].id
+    return None
+
+
+def main():
+    """Main entry point"""
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--tenant", required=True, help="C8y Tenant")
+    parser.add_argument("--user", required=True, help="C8y User")
+    parser.add_argument("--device", required=True, help="Device to find")
+    parser.add_argument("--url", required=True, help="URL of C8y")
+    parser.add_argument("--verbose", "-v", action="count", default=0)
+
+    args = parser.parse_args()
+
+    tenant = args.tenant
+    user = args.user
+    device_name = args.device
+    url = args.url
+    verbose = args.verbose
+
+    try:
+        password = os.environ["C8YPASS"]
+    except KeyError:
+        print("Please export your password into $C8YPASS")
+        sys.exit(1)
+
+    c8y = CumulocityApi(url, tenant, user, password)
+
+    device_id = get_device_id(c8y, device_name)
+    if device_id:
+        if verbose:
+            print("The current device ID is:")
+        print(device_id)
+
+    else:
+        print("Failed to find device in C8y")
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()
diff --git a/tests/requirements.txt b/tests/requirements.txt
index f32fdafc..457f65f5 100644
--- a/tests/requirements.txt
+++ b/tests/requirements.txt
@@ -5,3 +5,5 @@ retry == 0.9.2
 junitparser == 2.2.0
 junit2html == 30.0.6
 psutil == 5.9.0
+c8y-api == 1.1.1
+