diff options
author | Michael Abel <75477722+abelikt@users.noreply.github.com> | 2021-09-09 13:55:06 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2021-09-09 13:55:06 +0200 |
commit | cea1f56a29171c622b28cd333045aac7e0a3da9c (patch) | |
tree | 9c2658f9b8d1fd3b66930f3a87c23834472e6098 /tests/PySys | |
parent | 02c214e24e5b6c75325c311d2881c30f327c5878 (diff) |
[CIT-526] End to end test extension for software management (#414)
* checkout tests from ci branch from folder ...
... tests/PySys/software-management-end-to-end
Signed-off-by: Michael Abel <info@abel-ikt.de>
* Split checking and getting statuses
* Automatically set wait time based on arch
* rename symbol
* Improved to wait until a status
* Run black
* Increase timeout
* Enable them when flag is present
* Equalize name
* Make the fake plugins run again
* Refactor test case
* Update docu
* Fix testcase
* Use environment variable to inject a sw id database
* Fix typo
* Rename myPlatform container
* Rename symbols
* Improve comment and disable wait time
* Rename symbol
* Update test descriptions
* Unify case of property names
* Add sample file and docu about the dummy plugin
* Fix typo
Diffstat (limited to 'tests/PySys')
20 files changed, 829 insertions, 96 deletions
diff --git a/tests/PySys/pysysproject.xml b/tests/PySys/pysysproject.xml index 944c0b1b..e0bf006e 100644 --- a/tests/PySys/pysysproject.xml +++ b/tests/PySys/pysysproject.xml @@ -3,6 +3,7 @@ <requires-pysys>1.6.1</requires-pysys> <requires-python>3.7.3</requires-python> <property name="appHome" value="${env.PYSYS_APP_HOME}" default="${testRootDir}/.." pathMustExist="true"/> + <property name="c8yswrepo" value="${env.C8YSWREPO}" default="" /> <property name="tebasedir" value="${env.TEBASEDIR}" /> <property name="exampledir" value="${env.EXAMPLEDIR}" /> <property name="username" value="${env.C8YUSERNAME}" /> diff --git a/tests/PySys/software-management-end-to-end/dummy-plugin-configuration/Info.md b/tests/PySys/software-management-end-to-end/dummy-plugin-configuration/Info.md new file mode 100644 index 00000000..d3d6678e --- /dev/null +++ b/tests/PySys/software-management-end-to-end/dummy-plugin-configuration/Info.md @@ -0,0 +1,42 @@ + +# Configuration of the dummy plugin + +Build the plugin: + + $ cargo build --bin tedge_dummy_plugin + +On every invocation, the dummy plugin will respond with the +contents of file list-valid.<return_code> . Where the return code +defines the intended return code of the invocation. + +Upon manual execution, this path here will be used: + + .tedge_dummy_plugin/list-valid.0 + +When executed via the software management agent, this path will be used: + + /tmp/.tedge_dummy_plugin/list-valid.0 + +## Emulation of fruits + +The current trick is just to use a static configuration to emulate a funcional +plugin that lets us install and deinstall fruits. +Right now, the apt plugin only cares about the returned text on the list command. +So we just insert a static list and base tests on that. + +Note that even when a package is deinstalled it will still appear in the list +as the response is static. + +**Usage:** + +Copy the dummy plugin to this path: + + /etc/tedge/sm-plugins/fruits + +Copy file list-valid.0 to the following place: + + /tmp/.tedge_dummy_plugin/list-valid.0 + +Manual invocation e.g.: + + $ /etc/tedge/sm-plugins/fruits list diff --git a/tests/PySys/software-management-end-to-end/dummy-plugin-configuration/list-valid.0 b/tests/PySys/software-management-end-to-end/dummy-plugin-configuration/list-valid.0 new file mode 100644 index 00000000..35f67c7e --- /dev/null +++ b/tests/PySys/software-management-end-to-end/dummy-plugin-configuration/list-valid.0 @@ -0,0 +1,27 @@ + +{"name":"apple","version":"2.20.11-0ubuntu27.18"} +{"name":"banana","version":"2.0.6"} +{"name":"cherry","version":"3.1.23-1ubuntu1"} +{"name":"Dates","version":"1:1.16.1-4ubuntu6"} +{"name":"Elderberry","version":"20180224.1"} +{"name":"Figs","version":"11ubuntu5.3"} +{"name":"grapefruit","version":"3.5.47"} +{"name":"HOneydew-melon","version":"5.0-6ubuntu1.1"} +{"name":"Imbir","version":"1.0.8-3ubuntu0.1"} +{"name":"Jackfruit-aarch64-linux-gnu","version":"2.34-6ubuntu1.1"} +{"name":"Kiwi","version":"2.34-6ubuntu1.1"} +{"name":"Lime","version":"5.4.1-2"} +{"name":"mango","version":"12.8ubuntu1.1"} +{"name":"nectarine","version":"20210119~20.04.1"} +{"name":"olive","version":"0.31-7-gd99b2d76-0ubuntu1"} +{"name":"papaya","version":"21.2-3-g899bfaa9-0ubuntu2~20.04.1"} +{"name":"Queen-ann-cherry","version":"0.45ubuntu1"} +{"name":"Raspberries","version":"0.45ubuntu1"} +{"name":"Strawberries","version":"5.9.2.g-1ubuntu5"} +{"name":"tomato","version":"5.9.2.g-1ubuntu5"} +{"name":"ugni","version":"7.68.0-1ubuntu2.6"} +{"name":"vanilla","version":"1.7.8-1"} +{"name":"watermelon","version":"0.5.10.2-6"} +{"name":"Xango","version":"1.12.16-2ubuntu2.1"} +{"name":"Yam-x11","version":"1.12.16-2ubuntu2.1"} + diff --git a/tests/PySys/software-management-end-to-end/environment_sm_management.py b/tests/PySys/software-management-end-to-end/environment_sm_management.py index 8def5405..99e8d93e 100644 --- a/tests/PySys/software-management-end-to-end/environment_sm_management.py +++ b/tests/PySys/software-management-end-to-end/environment_sm_management.py @@ -12,25 +12,52 @@ Better run them in a VM or a container. To run the tests: - pysys.py run 'sm-apt*' -XmyPlatform='specialcontainer' + pysys.py run 'sm-apt*' -XmyPlatform='smcontainer' To run the tests with another tenant url: - pysys.py run 'sm-apt*' -XmyPlatform='specialcontainer' -Xtenant_url='thin-edge-io.eu-latest.cumulocity.com' + pysys.py run 'sm-apt*' -XmyPlatform='smcontainer' -Xtenant_url='thin-edge-io.eu-latest.cumulocity.com' +TODO: Avoid hardcoded ids +TODO: Get software package ids from c8y +TODO: Add management for package creation and removal for c8y + -> Mabe as separate python module to access c8y + +To override the hardcoded software id database you can use C8YSWREPO (format: JSON): + + export C8YSWREPO='{ + "asciijump": "5475278", + "robotfindskitten": "5473003", + "squirrel3": "5474871", + "rolldice": "5445239", + "moon-buggy": "5439204", + "apple": "5495053", + "banana": "5494888", + "cherry": "5495382", + "watermelon": "5494510" }' + +To remove + + unset C8YSWREPO + """ import base64 import time - import json +import platform import requests +import subprocess +import sys import pysys from pysys.basetest import BaseTest +sys.path.append("./environments") +from environment_c8y import EnvironmentC8y + def is_timezone_aware(stamp): """determine if object is timezone aware or naive @@ -39,27 +66,60 @@ def is_timezone_aware(stamp): return stamp.tzinfo is not None and stamp.tzinfo.utcoffset(stamp) is not None -class SoftwareManagement(BaseTest): +class SoftwareManagement(EnvironmentC8y): """Base class for software management tests""" # Static class member that can be overriden by a command line argument # E.g.: - # pysys.py run 'sm-apt*' -XmyPlatform='specialcontainer' + # pysys.py run 'sm-apt*' -XmyPlatform='smcontainer' myPlatform = None + # Static class member that can be overriden by a command line argument + # E.g.: + # pysys.py run 'sm-fake*' -Xfakeplugin='fakeplugin' + # Use it only when you have set up the dummy_plugin to install fruits + + fakeplugin = None + tenant_url = "thin-edge-io.eu-latest.cumulocity.com" def setup(self): """Setup Environment""" - if self.myPlatform != "specialcontainer": + if self.myPlatform != "smcontainer": self.skipTest("Testing the apt plugin is not supported on this platform") + # Database with package IDs taken from the thin-edge.io + # TODO make this somehow not hard-coded + self.pkg_id_db = { + # apt + "asciijump": "5475278", + "robotfindskitten": "5473003", + "squirrel3": "5474871", + "rolldice": "5445239", + "moon-buggy": "5439204", + # fake plugin + "apple": "5495053", + "banana": "5494888", + "cherry": "5495382", + "watermelon": "5494510", + } + + if self.project.c8yswrepo: + self.pkg_id_db = json.loads(self.project.c8yswrepo) + self.log.info("Using sw id database: %s"%self.pkg_id_db) + + super().setup() + self.addCleanupFunction(self.mysmcleanup) + tenant = self.project.tenant user = self.project.username password = self.project.c8ypass + # TODO are we doing something wrong while requesting? + self.timeout_req = 80 # seconds, got timeout with 60s + # Place to save the id of the operation that we started. # This is suitable for one operation and not for multiple ones running # at the same time. @@ -105,11 +165,13 @@ class SoftwareManagement(BaseTest): payload = { "deviceId": self.project.deviceid, - "description": "Apply software changes, triggered from PySys test", + "description": f"Apply software changes, triggered from PySys: {json_content}", "c8y_SoftwareUpdate": json_content, } - req = requests.post(url, json=payload, headers=self.header) + req = requests.post( + url, json=payload, headers=self.header, timeout=self.timeout_req + ) jresponse = json.loads(req.text) @@ -130,20 +192,15 @@ class SoftwareManagement(BaseTest): """Check if the current status is a fail""" if self.operation_id: return self.check_status_of_operation("FAILED") - return self.check_last_status("FAILED") + return self.check_status_of_last_operation("FAILED") def is_status_success(self): """Check if the current status is a success""" if self.operation_id: return self.check_status_of_operation("SUCCESSFUL") - return self.check_last_status("SUCCESSFUL") + return self.check_status_of_last_operation("SUCCESSFUL") - def check_last_status(self, status): - """Check if the last operation is successfull. - Warning: an observation so far is, that installation failures - seem to be at the beginning of the list independent of if we - revert it or not. - """ + def get_status_of_last_operation(self): params = { "deviceId": self.project.deviceid, @@ -158,7 +215,9 @@ class SoftwareManagement(BaseTest): } url = f"https://{self.tenant_url}/devicecontrol/operations" - req = requests.get(url, params=params, headers=self.header) + req = requests.get( + url, params=params, headers=self.header, timeout=self.timeout_req + ) req.raise_for_status() @@ -191,13 +250,30 @@ class SoftwareManagement(BaseTest): "State of current operation: %s", json.dumps(operation, indent=4) ) - return operation.get("status") == status + if not operation.get("status"): + raise SystemError("No valid field status in response") - def check_status_of_operation(self, status): - """Check if the last operation is successfull""" + return operation.get("status") + + def check_status_of_last_operation(self, status): + """Check if the last operation is successfull. + Warning: an observation so far is, that installation failures + seem to be at the beginning of the list independent of if we + revert it or not. + """ + + current_status = self.get_status_of_last_operation() + + return current_status == status + + def get_status_of_operation(self): + """Get the last operation""" + + if not self.operation_id: + raise SystemError("No valid operation ID available") url = f"https://{self.tenant_url}/devicecontrol/operations/{self.operation_id}" - req = requests.get(url, headers=self.header) + req = requests.get(url, headers=self.header, timeout=self.timeout_req) req.raise_for_status() @@ -208,7 +284,16 @@ class SoftwareManagement(BaseTest): "State of operation %s : %s", self.operation_id, operation["status"] ) - return operation.get("status") == status + if not operation.get("status"): + raise SystemError("No valid field status in response") + + return operation.get("status") + + def check_status_of_operation(self, status): + """Check if the last operation is successfull""" + current_status = self.get_status_of_operation() + self.log.info("Expected status: %s, got status %s" % (status, current_status)) + return current_status == status def wait_until_succcess(self): """Wait until c8y reports a success""" @@ -225,36 +310,50 @@ class SoftwareManagement(BaseTest): def wait_until_status(self, status, status2=False): """Wait until c8y reports status or status2.""" - wait_time = 300 + poll_period = 2 # seconds + + # Heuristic about how long to wait for a operation + if platform.machine() == "x86_64": + wait_time = int(40 / poll_period) + else: + wait_time = int(90 / poll_period) # 90s on the Rpi + timeout = 0 # wait for some time to let c8y process a request until we can poll for it - time.sleep(1) + time.sleep(poll_period) while True: if self.operation_id: - stat = self.check_status_of_operation( - status - ) or self.check_status_of_operation(status2) + + current_status = self.get_status_of_operation() + if current_status == status or current_status == status2: + # Invalidate the old operation + self.operation_id = None + break + else: - stat = self.check_last_status(status) or self.check_last_status(status2) - if stat: - # Invalidate the old operation - self.operation_id = None - break + current_status = self.get_status_of_last_operation() + if current_status == status or current_status == status2: + # Invalidate the old operation + self.operation_id = None + break + + time.sleep(poll_period) - time.sleep(1) timeout += 1 if timeout > wait_time: - raise SystemError("Timeout while waiting for a failure") + raise SystemError( + "Timeout while waiting for status %s or %s" % (status, status2) + ) def check_is_installed(self, package_name, version=None): """Check if a package is installed""" url = f"https://{self.tenant_url}/inventory/managedObjects/{self.project.deviceid}" - req = requests.get(url, headers=self.header) + req = requests.get(url, headers=self.header, timeout=self.timeout_req) req.raise_for_status() @@ -278,3 +377,28 @@ class SoftwareManagement(BaseTest): ret = True break return ret + + def get_pkg_version(self, pkg): + """ "Use apt-cache madison to derive a package version from + the apt cache even when it is not installed. + Not very bulletproof yet!!! + """ + output = subprocess.check_output(["/usr/bin/apt-cache", "madison", pkg]) + + # Lets assume it is the package in the first line of the output + return output.split()[2].decode("ascii") # E.g. "1.16-1+b3" + + def get_pkgid(self, pkg): + + pkgid = self.pkg_id_db.get(pkg) + + if pkgid: + return pkgid + else: + raise SystemError("Package ID not in database") + + def mysmcleanup(self): + # Slow down a bit to avoid restarting services too fast. + # Enable again, to experiment with C8y timeouts + # time.sleep(1) + pass diff --git a/tests/PySys/software-management-end-to-end/sm-apt-install-fail/pysystest.xml b/tests/PySys/software-management-end-to-end/sm-apt-install-fail/pysystest.xml index d4441289..cdb91bab 100644 --- a/tests/PySys/software-management-end-to-end/sm-apt-install-fail/pysystest.xml +++ b/tests/PySys/software-management-end-to-end/sm-apt-install-fail/pysystest.xml @@ -2,7 +2,7 @@ <pysystest type="auto"> <description> - <title>Validate end to end behaviour for failing installation</title> + <title>Validate end to end behaviour for a failing installation</title> <purpose><![CDATA[ ]]> </purpose> diff --git a/tests/PySys/software-management-end-to-end/sm-apt-install-fail/run.py b/tests/PySys/software-management-end-to-end/sm-apt-install-fail/run.py index abf8f6a3..eb6c456d 100644 --- a/tests/PySys/software-management-end-to-end/sm-apt-install-fail/run.py +++ b/tests/PySys/software-management-end-to-end/sm-apt-install-fail/run.py @@ -3,7 +3,7 @@ from pysys.basetest import BaseTest import time """ -Validate end to end behaviour for failing installation +Validate end to end behaviour for a failing installation When we install a package that cannot be installed with the apt package manager Then we receive a failure from the apt plugin diff --git a/tests/PySys/software-management-end-to-end/sm-apt-install-remove-default/pysystest.xml b/tests/PySys/software-management-end-to-end/sm-apt-install-remove-default/pysystest.xml new file mode 100644 index 00000000..72fb0bbe --- /dev/null +++ b/tests/PySys/software-management-end-to-end/sm-apt-install-remove-default/pysystest.xml @@ -0,0 +1,25 @@ +<?xml version="1.0" encoding="utf-8"?> +<pysystest type="auto"> + + <description> + <title>Validate end to end behaviour for the default plugin for installation and removal of packages</title> + <purpose><![CDATA[ +]]> + </purpose> + </description> + <classification> + <groups inherit="true"> + <group></group> + </groups> + <modes inherit="true"> + </modes> + </classification> + <data> + <class name="PySysTest" module="run"/> + </data> + <traceability> + <requirements> + <requirement id=""/> + </requirements> + </traceability> +</pysystest> diff --git a/tests/PySys/software-management-end-to-end/sm-apt-install-remove-default/run.py b/tests/PySys/software-management-end-to-end/sm-apt-install-remove-default/run.py new file mode 100644 index 00000000..1d07a5c0 --- /dev/null +++ b/tests/PySys/software-management-end-to-end/sm-apt-install-remove-default/run.py @@ -0,0 +1,48 @@ +from pysys.basetest import BaseTest + +import time + +""" +Validate end to end behaviour for the apt plugin for installation and removal of packages + +For the default plugin with a space as version + +When we install a package +Then it is installed +When we deinstall it again +Then it is not installed +""" + +import json +import requests +import time +import sys + +sys.path.append("software-management-end-to-end") +from environment_sm_management import SoftwareManagement + + +class PySysTest(SoftwareManagement): + def setup(self): + super().setup() + self.assertThat("False == value", value=self.check_is_installed("rolldice")) + + def execute(self): + + self.trigger_action( + "rolldice", self.get_pkgid("rolldice"), " ", "notanurl", "install" + ) + + self.wait_until_succcess() + + self.assertThat("True == value", value=self.check_is_installed("rolldice")) + + self.trigger_action( + "rolldice", self.get_pkgid("rolldice"), " ", "notanurl", "delete" + ) + + self.wait_until_succcess() + + def validate(self): + + self.assertThat("False == value", value=self.check_is_installed("rolldice")) diff --git a/tests/PySys/software-management-end-to-end/sm-apt-install-remove-multiple-mixed/pysystest.xml b/tests/PySys/software-management-end-to-end/sm-apt-install-remove-multiple-mixed/pysystest.xml new file mode 100644 index 00000000..eef13a82 --- /dev/null +++ b/tests/PySys/software-management-end-to-end/sm-apt-install-remove-multiple-mixed/pysystest.xml @@ -0,0 +1,25 @@ +<?xml version="1.0" encoding="utf-8"?> +<pysystest type="auto"> + + <description> + <title>Validate end to end behaviour for the apt plugin for multiple packages with mixed versions</title> + <purpose><![CDATA[ +]]> + </purpose> + </description> + <classification> + <groups inherit="true"> + <group></group> + </groups> + <modes inherit="true"> + </modes> + </classification> + <data> + <class name="PySysTest" module="run"/> + </data> + <traceability> + <requirements> + <requirement id=""/> + </requirements> + </traceability> +</pysystest> diff --git a/tests/PySys/software-management-end-to-end/sm-apt-install-remove-multiple-mixed/run.py b/tests/PySys/software-management-end-to-end/sm-apt-install-remove-multiple-mixed/run.py new file mode 100644 index 00000000..ad52da7b --- /dev/null +++ b/tests/PySys/software-management-end-to-end/sm-apt-install-remove-multiple-mixed/run.py @@ -0,0 +1,115 @@ +from pysys.basetest import BaseTest + +import time + +""" +Validate end to end behaviour for the apt plugin for multiple packages with mixed versions + +When we install a bunch of packages with versions, without and even one twice +Then they are installed +When we deinstall them again +Then they are not installed +""" + +import time +import subprocess +import sys + +sys.path.append("software-management-end-to-end") +from environment_sm_management import SoftwareManagement + + +class PySysTest(SoftwareManagement): + + def get_packages_with_action(self, act): + "create an action that we can use later" + + action = [ + { + "action": act, + "id": self.get_pkgid("asciijump"), + "name": "asciijump", + "url": " ", + "version": "::apt", # apt manager + }, + { + "action": act, + "id": self.get_pkgid("robotfindskitten"), + "name": "robotfindskitten", + "url": " ", + "version": " ", # default manager + }, + { + "action": act, + "id": self.get_pkgid("squirrel3"), + "name": "squirrel3", + "url": " ", + "version": self.get_pkg_version("squirrel3") + + "::apt", # version and manager + }, + { + "action": act, + "id": self.get_pkgid("rolldice"), + "name": "rolldice", + "url": " ", + "version": self.get_pkg_version("rolldice"), # only version + }, + { + "action": act, + "id": self.get_pkgid("moon-buggy"), + "name": "moon-buggy", + "url": " ", + "version": self.get_pkg_version("moon-buggy"), # nothing special + }, + { + "action": act, + "id": self.get_pkgid("asciijump"), + "name": "asciijump", + "url": " ", + "version": "::apt", # again same as above + }, + ] + return action + + def setup(self): + super().setup() + + self.assertThat("False == value", value=self.check_is_installed("asciijump")) + self.assertThat( + "False == value", value=self.check_is_installed("robotfindskitten") + ) + self.assertThat("False == value", value=self.check_is_installed("rolldice")) + self.assertThat("False == value", value=self.check_is_installed("squirrel3")) + self.assertThat("False == value", value=self.check_is_installed("moon-buggy")) + + def execute(self): + + action = self.get_packages_with_action("install") + + self.trigger_action_json(action) + + self.wait_until_succcess() + + self.assertThat("True == value", value=self.check_is_installed("asciijump")) + self.assertThat( + "True == value", value=self.check_is_installed("robotfindskitten") + ) + self.assertThat("True == value", value=self.check_is_installed("rolldice")) + self.assertThat("True == value", value=self.check_is_installed("squirrel3")) + self.assertThat("True == value", value=self.check_is_installed("moon-buggy")) + + action = self.get_packages_with_action("delete") + + self.trigger_action_json(action) + + self.wait_until_succcess() + + def validate(self): + + self.assertThat("False == value", value=self.check_is_installed("asciijump")) + self.assertThat( + "False == value", value=self.check_is_installed("robotfindskitten") + ) + self.assertThat("False == value", value=self.check_is_installed("rolldice")) + self.assertThat("False == value", value=self.check_is_installed("squirrel3")) + self.assertThat("False == value", value=self.check_is_installed("moon-buggy")) diff --git a/tests/PySys/software-management-end-to-end/sm-apt-install-remove-multiple/run.py b/tests/PySys/software-management-end-to-end/sm-apt-install-remove-multiple/run.py index 83ee9402..1c172aa3 100644 --- a/tests/PySys/software-management-end-to-end/sm-apt-install-remove-multiple/run.py +++ b/tests/PySys/software-management-end-to-end/sm-apt-install-remove-multiple/run.py @@ -21,47 +21,50 @@ from environment_sm_management import SoftwareManagement class PySysTest(SoftwareManagement): - def setup(self): - super().setup() - - self.assertThat("False == value", value=self |