diff options
author | Austin S. Hemmelgarn <austin@netdata.cloud> | 2022-02-16 08:53:22 -0500 |
---|---|---|
committer | GitHub <noreply@github.com> | 2022-02-16 08:53:22 -0500 |
commit | 46d73e34c0dbab3777b898fc5418f9d0a0e82489 (patch) | |
tree | 9fc8537a192a915b454d00c111b63ceab4cd0595 | |
parent | 598ce2180a87f9a3dd82d64d93b8a952db65a153 (diff) |
Consolidate build matrix across CI jobs. (#11807)
* Restructure CI build matrix handling for better consolidation.
* Update core packaging code to use new distro.yml file.
* Update repository config packaging code to use new distro.yml file.
-rw-r--r-- | .github/data/build-matrix.json | 98 | ||||
-rw-r--r-- | .github/data/distros.yml | 145 | ||||
-rw-r--r-- | .github/workflows/build.yml | 30 | ||||
-rw-r--r-- | .github/workflows/packaging.yml | 81 | ||||
-rw-r--r-- | .github/workflows/repoconfig-packages.yml | 58 |
5 files changed, 255 insertions, 157 deletions
diff --git a/.github/data/build-matrix.json b/.github/data/build-matrix.json deleted file mode 100644 index 6ceb9fa541..0000000000 --- a/.github/data/build-matrix.json +++ /dev/null @@ -1,98 +0,0 @@ -{ - "include": [ - { - "distro": "alpine:edge", - "artifact_key": "alpine-edge", - "pre": "apk add -U bash", - "rmjsonc": "apk del json-c-dev" - }, - { - "distro": "alpine:3.14", - "artifact_key": "alpine-3.14", - "pre": "apk add -U bash", - "rmjsonc": "apk del json-c-dev" - }, - { - "distro": "alpine:3.13", - "artifact_key": "alpine-3.13", - "pre": "apk add -U bash", - "rmjsonc": "apk del json-c-dev" - }, - { - "distro": "alpine:3.12", - "artifact_key": "alpine-3.12", - "pre": "apk add -U bash", - "rmjsonc": "apk del json-c-dev" - }, - { - "distro": "archlinux:latest", - "artifact_key": "archlinux", - "pre": "pacman --noconfirm -Syu && pacman --noconfirm -Sy grep libffi" - }, - { - "distro": "centos:7", - "artifact_key": "centos7" - }, - { - "distro": "rockylinux/rockylinux:8", - "artifact_key": "rockylinux8", - "rmjsonc": "dnf remove -y json-c-devel" - }, - { - "distro": "debian:11", - "artifact_key": "debian11", - "pre": "apt-get update", - "rmjsonc": "apt-get remove -y libjson-c-dev" - }, - { - "distro": "debian:10", - "artifact_key": "debian10", - "pre": "apt-get update", - "rmjsonc": "apt-get remove -y libjson-c-dev" - }, - { - "distro": "debian:9", - "artifact_key": "debian9", - "pre": "apt-get update", - "rmjsonc": "apt-get remove -y libjson-c-dev" - }, - { - "distro": "fedora:35", - "artifact_key": "fedora35", - "rmjsonc": "dnf remove -y json-c-devel" - }, - { - "distro": "fedora:34", - "artifact_key": "fedora34", - "rmjsonc": "dnf remove -y json-c-devel" - }, - { - "distro": "opensuse/leap:15.3", - "artifact_key": "opensuse15.3", - "rmjsonc": "zypper rm -y libjson-c-devel" - }, - { - "distro": "oraclelinux:8", - "artifact_key": "oraclelinux8", - "rmjsonc": "dnf remove -y json-c-devel" - }, - { - "distro": "ubuntu:21.10", - "artifact_key": "ubuntu21.10", - "pre": "rm -f /etc/apt/apt.conf.d/docker && apt-get update", - "rmjsonc": "apt-get remove -y libjson-c-dev" - }, - { - "distro": "ubuntu:20.04", - "artifact_key": "ubuntu20.04", - "pre": "apt-get update", - "rmjsonc": "apt-get remove -y libjson-c-dev" - }, - { - "distro": "ubuntu:18.04", - "artifact_key": "ubuntu18.04", - "pre": "apt-get update", - "rmjsonc": "apt-get remove -y libjson-c-dev" - } - ] -} diff --git a/.github/data/distros.yml b/.github/data/distros.yml new file mode 100644 index 0000000000..e3d76dc13a --- /dev/null +++ b/.github/data/distros.yml @@ -0,0 +1,145 @@ +# This defines the full set of distros we run CI on. +--- +platform_map: # map packaging architectures to docker platforms + i386: linux/i386 + amd64: linux/amd64 + armhf: linux/arm/v7 + arm64: linux/arm64/v8 +include: + - &alpine + distro: alpine + version: edge + env_prep: | + apk add -U bash + jsonc_removal: | + apk del json-c-dev + - <<: *alpine + version: "3.15" + - <<: *alpine + version: "3.14" + - <<: *alpine + version: "3.13" + - <<: *alpine + version: "3.12" + + - distro: archlinux + version: latest + env_prep: | + pacman --noconfirm -Syu && pacman --noconfirm -Sy grep libffi + + - distro: rockylinux + version: "8" + base_image: rockylinux/rockylinux + jsonc_removal: | + dnf remove -y json-c-devel + packages: + type: rpm + repo_distro: el/8 + arches: + - amd64 + - arm64 + - distro: centos + version: "7" + packages: + type: rpm + repo_distro: el/7 + arches: + - amd64 + + - &debian + distro: debian + version: "11" + env_prep: | + apt-get update + jsonc_removal: | + apt-get purge -y libjson-c-dev + packages: &debian_packages + type: deb + repo_distro: debian/bullseye + arches: + - i386 + - amd64 + - armhf + - arm64 + - <<: *debian + version: "10" + packages: + <<: *debian_packages + repo_distro: debian/buster + - <<: *debian + version: "9" + packages: + <<: *debian_packages + repo_distro: debian/stretch + + - &fedora + distro: fedora + version: "35" + jsonc_removal: | + dnf remove -y json-c-devel + packages: &fedora_packages + type: rpm + repo_distro: fedora/35 + arches: + - amd64 + - armhf + - arm64 + - <<: *fedora + version: "34" + packages: + <<: *fedora_packages + repo_distro: fedora/34 + + - &opensuse + distro: opensuse + version: "15.3" + base_image: opensuse/leap + jsonc_removal: | + zypper rm -y libjson-c-devel + packages: + type: rpm + repo_distro: opensuse/15.3 + arches: + - amd64 + - arm64 + + - distro: oraclelinux + version: "8" + jsonc_removal: | + dnf remove -y json-c-devel + packages: + type: rpm + repo_distro: ol/8 + arches: + - amd64 + - arm64 + + - &ubuntu + distro: ubuntu + version: "21.10" + env_prep: | + rm -f /etc/apt/apt.conf.d/docker && apt-get update + jsonc_removal: | + apt-get remove -y libjson-c-dev + packages: &ubuntu_packages + type: deb + repo_distro: ubuntu/impish + arches: + - amd64 + - armhf + - arm64 + - <<: *ubuntu + version: "20.04" + packages: + <<: *ubuntu_packages + repo_distro: ubuntu/focal + - <<: *ubuntu + version: "18.04" + packages: + <<: *ubuntu_packages + repo_distro: ubuntu/bionic + arches: + - i386 + - amd64 + - armhf + - arm64 diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9bbc8f4a0f..a5034d1e3c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -129,13 +129,29 @@ jobs: uses: actions/checkout@v2 - name: Prepare tools run: | - sudo apt-get update && sudo apt-get install -y jq + sudo apt-get update && sudo apt-get install -y python3-ruamel.yaml - name: Read build matrix id: set-matrix + shell: python3 {0} run: | - TASKS="$(jq -c . .github/data/build-matrix.json)" - echo "Generated Matrix: $TASKS" - echo "::set-output name=matrix::$TASKS" + from ruamel.yaml import YAML + import json + yaml = YAML(typ='safe') + with open('.github/data/distros.yml') as f: + data = yaml.load(f) + del data['platform_map'] + for i, v in enumerate(data['include']): + data['include'][i]['artifact_key'] = data['include'][i]['distro'] + str(data['include'][i]['version']).replace('.', '') + if 'packages' in data['include'][i]: + del data['include'][i]['packages'] + if 'base_image' in data['include'][i]: + data['include'][i]['distro'] = data['include'][i]['base_image'] + del data['include'][i]['base_image'] + data['include'][i]['distro'] = ':'.join([data['include'][i]['distro'], str(data['include'][i]['version'])]) + del data['include'][i]['version'] + matrix = json.dumps(data, sort_keys=True) + print('Generated Matrix: ' + matrix) + print('::set-output name=matrix::' + matrix) prepare-test-images: # Prepare the test environments for our build checks. This also checks dependency handling code for each tested environment. name: Prepare Test Environments @@ -160,8 +176,8 @@ jobs: file: .github/dockerfiles/Dockerfile.build_test build-args: | BASE=${{ matrix.distro }} - PRE=${{ matrix.pre }} - RMJSONC=${{ matrix.rmjsonc }} + PRE=${{ matrix.env_prep }} + RMJSONC=${{ matrix.jsonc_removal }} outputs: type=oci,dest=/tmp/image.tar tags: test:${{ matrix.artifact_key }} - name: Upload image artifact @@ -224,7 +240,7 @@ jobs: docker run --security-opt seccomp=unconfined -w /netdata sha256:${{ steps.load.outputs.image }} \ /bin/sh -c './netdata-installer.sh --dont-wait --dont-start-it --require-cloud' - name: netdata-installer on ${{ matrix.distro }}, require cloud, no JSON-C - if: matrix.rmjsonc != '' + if: matrix.jsonc_removal != '' run: | docker run --security-opt seccomp=unconfined -w /netdata sha256:${{ steps.load.outputs.image }} \ /bin/sh -c '/rmjsonc.sh && ./netdata-installer.sh --dont-wait --dont-start-it --require-cloud' diff --git a/.github/workflows/packaging.yml b/.github/workflows/packaging.yml index 59916c7cf7..d24f4e2b22 100644 --- a/.github/workflows/packaging.yml +++ b/.github/workflows/packaging.yml @@ -21,49 +21,56 @@ concurrency: group: packages-${{ github.ref }}-${{ github.event_name }} cancel-in-progress: true jobs: + matrix: + name: Prepare Build Matrix + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Prepare tools + run: | + sudo apt-get update && sudo apt-get install -y python3-ruamel.yaml + - name: Read build matrix + id: set-matrix + shell: python3 {0} + run: | + from ruamel.yaml import YAML + import json + yaml = YAML(typ='safe') + entries = list() + + with open('.github/data/distros.yml') as f: + data = yaml.load(f) + + for i, v in enumerate(data['include']): + if 'packages' in data['include'][i]: + for arch in data['include'][i]['packages']['arches']: + entries.append({ + 'distro': data['include'][i]['distro'], + 'version': data['include'][i]['version'], + 'pkgclouddistro': data['include'][i]['packages']['repo_distro'], + 'format': data['include'][i]['packages']['type'], + 'base_image': data['include'][i]['base_image'] if 'base_image' in data['include'][i] else data['include'][i]['distro'], + 'platform': data['platform_map'][arch], + 'arch': arch + }) + + entries.sort(key=lambda k: (k['arch'], k['distro'], k['version'])) + matrix = json.dumps({'include': entries}, sort_keys=True) + print('Generated Matrix: ' + matrix) + print('::set-output name=matrix::' + matrix) + build: name: Build runs-on: ubuntu-latest env: DOCKER_CLI_EXPERIMENTAL: enabled + needs: + - matrix strategy: - matrix: - include: - - {distro: debian, version: "9", pkgclouddistro: debian/stretch, format: deb, base_image: debian, platform: linux/amd64, arch: amd64} - - {distro: debian, version: "9", pkgclouddistro: debian/stretch, format: deb, base_image: debian, platform: linux/i386, arch: i386} - - {distro: debian, version: "9", pkgclouddistro: debian/stretch, format: deb, base_image: debian, platform: linux/arm/v7, arch: armhf} - - {distro: debian, version: "9", pkgclouddistro: debian/stretch, format: deb, base_image: debian, platform: linux/arm64/v8, arch: arm64} - - {distro: debian, version: "10", pkgclouddistro: debian/buster, format: deb, base_image: debian, platform: linux/amd64, arch: amd64} - - {distro: debian, version: "10", pkgclouddistro: debian/buster, format: deb, base_image: debian, platform: linux/i386, arch: i386} - - {distro: debian, version: "10", pkgclouddistro: debian/buster, format: deb, base_image: debian, platform: linux/arm/v7, arch: armhf} - - {distro: debian, version: "10", pkgclouddistro: debian/buster, format: deb, base_image: debian, platform: linux/arm64/v8, arch: arm64} - - {distro: debian, version: "11", pkgclouddistro: debian/bullseye, format: deb, base_image: debian, platform: linux/amd64, arch: amd64, alias: bullseye} - - {distro: debian, version: "11", pkgclouddistro: debian/bullseye, format: deb, base_image: debian, platform: linux/i386, arch: i386, alias: bullseye} - - {distro: debian, version: "11", pkgclouddistro: debian/bullseye, format: deb, base_image: debian, platform: linux/arm/v7, arch: armhf, alias: bullseye} - - {distro: debian, version: "11", pkgclouddistro: debian/bullseye, format: deb, base_image: debian, platform: linux/arm64/v8, arch: arm64, alias: bullseye} - - {distro: ubuntu, version: "18.04", pkgclouddistro: ubuntu/bionic, format: deb, base_image: ubuntu, platform: linux/amd64, arch: amd64} - - {distro: ubuntu, version: "18.04", pkgclouddistro: ubuntu/bionic, format: deb, base_image: ubuntu, platform: linux/i386, arch: i386} - - {distro: ubuntu, version: "18.04", pkgclouddistro: ubuntu/bionic, format: deb, base_image: ubuntu, platform: linux/arm/v7, arch: armhf} - - {distro: ubuntu, version: "18.04", pkgclouddistro: ubuntu/bionic, format: deb, base_image: ubuntu, platform: linux/arm64/v8, arch: arm64} - - {distro: ubuntu, version: "20.04", pkgclouddistro: ubuntu/focal, format: deb, base_image: ubuntu, platform: linux/amd64, arch: amd64} - - {distro: ubuntu, version: "20.04", pkgclouddistro: ubuntu/focal, format: deb, base_image: ubuntu, platform: linux/arm/v7, arch: armhf} - - {distro: ubuntu, version: "20.04", pkgclouddistro: ubuntu/focal, format: deb, base_image: ubuntu, platform: linux/arm64/v8, arch: arm64} - - {distro: ubuntu, version: "21.10", pkgclouddistro: ubuntu/impish, format: deb, base_image: ubuntu, platform: linux/amd64, arch: amd64} - - {distro: ubuntu, version: "21.10", pkgclouddistro: ubuntu/impish, format: deb, base_image: ubuntu, platform: linux/arm/v7, arch: armhf} - - {distro: ubuntu, version: "21.10", pkgclouddistro: ubuntu/impish, format: deb, base_image: ubuntu, platform: linux/arm64/v8, arch: arm64} - - {distro: centos, version: "7", pkgclouddistro: el/7, format: rpm, base_image: centos, platform: linux/amd64, arch: amd64} - - {distro: rockylinux, version: "8", pkgclouddistro: el/8, format: rpm, base_image: rockylinux/rockylinux, platform: linux/amd64, arch: amd64} - - {distro: rockylinux, version: "8", pkgclouddistro: el/8, format: rpm, base_image: rockylinux/rockylinux, platform: linux/arm64/v8, arch: arm64} - - {distro: fedora, version: "34", pkgclouddistro: fedora/34, format: rpm, base_image: fedora, platform: linux/amd64, arch: amd64} - - {distro: fedora, version: "34", pkgclouddistro: fedora/34, format: rpm, base_image: fedora, platform: linux/arm/v7, arch: armhf} - - {distro: fedora, version: "34", pkgclouddistro: fedora/34, format: rpm, base_image: fedora, platform: linux/arm64/v8, arch: arm64} - - {distro: fedora, version: "35", pkgclouddistro: fedora/35, format: rpm, base_image: fedora, platform: linux/amd64, arch: amd64} - - {distro: fedora, version: "35", pkgclouddistro: fedora/35, format: rpm, base_image: fedora, platform: linux/arm/v7, arch: armhf} - - {distro: fedora, version: "35", pkgclouddistro: fedora/35, format: rpm, base_image: fedora, platform: linux/arm64/v8, arch: arm64} - - {distro: opensuse, version: "15.3", pkgclouddistro: opensuse/15.3, format: rpm, base_image: opensuse/leap, platform: linux/amd64, arch: amd64} - - {distro: opensuse, version: "15.3", pkgclouddistro: opensuse/15.3, format: rpm, base_image: opensuse/leap, platform: linux/arm64/v8, arch: arm64} - - {distro: oraclelinux, version: "8", pkgclouddistro: ol/8, format: rpm, base_image: oraclelinux, platform: linux/amd64, arch: amd64} - - {distro: oraclelinux, version: "8", pkgclouddistro: ol/8, format: rpm, base_image: oraclelinux, platform: linux/arm64/v8, arch: arm64} + matrix: ${{ fromJson(needs.matrix.outputs.matrix) }} # We intentiaonally disable the fail-fast behavior so that a # build failure for one version doesn't prevent us from publishing # successfully built and tested packages for another version. diff --git a/.github/workflows/repoconfig-packages.yml b/.github/workflows/repoconfig-packages.yml index 9a7395ed31..de2c65e61a 100644 --- a/.github/workflows/repoconfig-packages.yml +++ b/.github/workflows/repoconfig-packages.yml @@ -16,28 +16,56 @@ on: env: DISABLE_TELEMETRY: 1 jobs: + matrix: + name: Prepare Build Matrix + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Prepare tools + run: | + sudo apt-get update && sudo apt-get install -y python3-ruamel.yaml + - name: Read build matrix + id: set-matrix + shell: python3 {0} + run: | + from ruamel.yaml import YAML + import json + yaml = YAML(typ='safe') + entries = list() + + with open('.github/data/distros.yml') as f: + data = yaml.load(f) + + for i, v in enumerate(data['include']): + if 'packages' in data['include'][i]: + entries.append({ + 'distro': data['include'][i]['distro'], + 'version': data['include'][i]['version'], + 'pkgclouddistro': data['include'][i]['packages']['repo_distro'], + 'format': data['include'][i]['packages']['type'], + 'base_image': data['include'][i]['base_image'] if 'base_image' in data['include'][i] else data['include'][i]['distro'], + 'platform': data['platform_map']['amd64'], + 'arch': 'amd64' + }) + + entries.sort(key=lambda k: (k['arch'], k['distro'], k['version'])) + matrix = json.dumps({'include': entries}, sort_keys=True) + print('Generated Matrix: ' + matrix) + print('::set-output name=matrix::' + matrix) + build: name: Build runs-on: ubuntu-latest env: DISABLE_TELEMETRY: 1 DOCKER_CLI_EXPERIMENTAL: enabled + needs: + - matrix strategy: - # This needs to be kept in sync with the matrix in packaging.yml, but should only include the AMD64 lines. - matrix: - include: - - {distro: debian, version: "9", pkgclouddistro: debian/stretch, format: deb, base_image: debian, platform: linux/amd64, arch: amd64} - - {distro: debian, version: "10", pkgclouddistro: debian/buster, format: deb, base_image: debian, platform: linux/amd64, arch: amd64} - - {distro: debian, version: "11", pkgclouddistro: debian/bullseye, format: deb, base_image: debian, platform: linux/amd64, arch: amd64} - - {distro: ubuntu, version: "18.04", pkgclouddistro: ubuntu/bionic, format: deb, base_image: ubuntu, platform: linux/amd64, arch: amd64} - - {distro: ubuntu, version: "20.04", pkgclouddistro: ubuntu/focal, format: deb, base_image: ubuntu, platform: linux/amd64, arch: amd64} - - {distro: ubuntu, version: "21.10", pkgclouddistro: ubuntu/impish, format: deb, base_image: ubuntu, platform: linux/amd64, arch: amd64} - - {distro: centos, version: "7", pkgclouddistro: el/7, format: rpm, base_image: centos, platform: linux/amd64, arch: amd64} - - {distro: rockylinux, version: "8", pkgclouddistro: el/8, format: rpm, base_image: rockylinux/rockylinux, platform: linux/amd64, arch: amd64} - - {distro: fedora, version: "34", pkgclouddistro: fedora/34, format: rpm, base_image: fedora, platform: linux/amd64, arch: amd64} - - {distro: fedora, version: "35", pkgclouddistro: fedora/35, format: rpm, base_image: fedora, platform: linux/amd64, arch: amd64} - - {distro: opensuse, version: "15.3", pkgclouddistro: opensuse/15.3, format: rpm, base_image: opensuse/leap, platform: linux/amd64, arch: amd64} - - {distro: oraclelinux, version: "8", pkgclouddistro: ol/8, format: rpm, base_image: oraclelinux, platform: linux/amd64, arch: amd64} + matrix: ${{ fromJson(needs.matrix.outputs.matrix) }} # We intentiaonally disable the fail-fast behavior so that a # build failure for one version doesn't prevent us from publishing # successfully built and tested packages for another version. |