diff options
author | Austin S. Hemmelgarn <austin@netdata.cloud> | 2022-06-28 09:20:19 -0400 |
---|---|---|
committer | GitHub <noreply@github.com> | 2022-06-28 09:20:19 -0400 |
commit | e86cec2631c961b434031e2e09597701a9ec53f8 (patch) | |
tree | 4639617c8e63b01ae7d79e4aef43e79c4c42d2ab /.github | |
parent | 2b5999c39337e1fc0f6c25a22e48f2d23fc6e9f8 (diff) |
Upload packages to new self-hosted repository infrastructure. (#13240)
* Match packaging arches to distro architecture names.
This will simplify handling of uploads to our new self-hosted packaging
infrastructure.
* Add package uploads for RPM packages on new infrastructure.
* Add package upload support for DEB packages.
* Saner tmpdir handling.
* Upload fixes.
* Differentiate different DEB distros in package names.
* Workflow fixes.
Diffstat (limited to '.github')
-rw-r--r-- | .github/data/distros.yml | 35 | ||||
-rwxr-xr-x | .github/scripts/package-upload.sh | 43 | ||||
-rw-r--r-- | .github/workflows/packaging.yml | 29 | ||||
-rw-r--r-- | .github/workflows/repoconfig-packages.yml | 5 |
4 files changed, 92 insertions, 20 deletions
diff --git a/.github/data/distros.yml b/.github/data/distros.yml index 9139d8a6aa..5166da2890 100644 --- a/.github/data/distros.yml +++ b/.github/data/distros.yml @@ -1,10 +1,21 @@ # This defines the full set of distros we run CI on. --- platform_map: # map packaging architectures to docker platforms - i386: linux/i386 + aarch64: linux/arm64/v8 amd64: linux/amd64 - armhf: linux/arm/v7 arm64: linux/arm64/v8 + armhf: linux/arm/v7 + armhfp: linux/arm/v7 + i386: linux/i386 + x86_64: linux/amd64 +arch_order: # sort order for per-architecture jobs in CI + - amd64 + - x86_64 + - i386 + - armhf + - armhfp + - arm64 + - aarch64 include: - &alpine distro: alpine @@ -37,8 +48,8 @@ include: type: rpm repo_distro: el/9 arches: - - amd64 - - arm64 + - x86_64 + - aarch64 - <<: *alma version: "8" packages: @@ -51,7 +62,7 @@ include: type: rpm repo_distro: el/7 arches: - - amd64 + - x86_64 - &debian distro: debian @@ -88,9 +99,9 @@ include: type: rpm repo_distro: fedora/36 arches: - - amd64 - - armhf - - arm64 + - x86_64 + - armhfp + - aarch64 - <<: *fedora version: "35" packages: @@ -107,8 +118,8 @@ include: type: rpm repo_distro: opensuse/15.4 arches: - - amd64 - - arm64 + - x86_64 + - aarch64 - <<: *opensuse version: "15.3" packages: @@ -123,8 +134,8 @@ include: type: rpm repo_distro: ol/8 arches: - - amd64 - - arm64 + - x86_64 + - aarch64 - &ubuntu distro: ubuntu diff --git a/.github/scripts/package-upload.sh b/.github/scripts/package-upload.sh new file mode 100755 index 0000000000..e07e78f0e7 --- /dev/null +++ b/.github/scripts/package-upload.sh @@ -0,0 +1,43 @@ +#!/bin/sh + +set -e + +host="packages.netdata.cloud" +user="netdatabot" + +distro="${1}" +arch="${2}" +format="${3}" +repo="${4}" + +staging="${TMPDIR:-/tmp}/package-staging" +prefix="/var/www/html/repos/${repo}/" + +packages="$(find artifacts -name "*.${format}")" + +mkdir -p "${staging}" + +case "${format}" in + deb) + src="${staging}/$(echo "${distro}" | cut -f 1 -d '/')/pool/" + mkdir -p "${src}" + + for pkg in ${packages}; do + cp "${pkg}" "${src}" + done + ;; + rpm) + src="${staging}/${distro}/${arch}/" + mkdir -p "${src}" + + for pkg in ${packages}; do + cp "${pkg}" "${src}" + done + ;; + *) + echo "Unrecognized package format ${format}." + exit 1 + ;; +esac + +rsync -vrptO "${staging}/" "${user}@${host}:${prefix}" diff --git a/.github/workflows/packaging.yml b/.github/workflows/packaging.yml index d8f6a337c9..f6cb5534f0 100644 --- a/.github/workflows/packaging.yml +++ b/.github/workflows/packaging.yml @@ -65,14 +65,14 @@ jobs: entries.append({ 'distro': data['include'][i]['distro'], 'version': data['include'][i]['version'], - 'pkgclouddistro': data['include'][i]['packages']['repo_distro'], + 'repo_distro': data['include'][i]['packages']['repo_distro'], 'format': data['include'][i]['packages']['type'], 'base_image': data['include'][i]['base_image'] if 'base_image' in data['include'][i] else data['include'][i]['distro'], 'platform': data['platform_map'][arch], 'arch': arch }) - entries.sort(key=lambda k: (k['arch'], k['distro'], k['version'])) + entries.sort(key=lambda k: (data['arch_order'].index(k['arch']), k['distro'], k['version'])) matrix = json.dumps({'include': entries}, sort_keys=True) print('Generated Matrix: ' + matrix) print('::set-output name=matrix::' + matrix) @@ -223,10 +223,27 @@ jobs: run: | printf "Packages to upload:\n%s" "$(ls artifacts/*.${{ matrix.format }})" for pkgfile in artifacts/*.${{ matrix.format }} ; do - .github/scripts/package_cloud_wrapper.sh yank ${{ needs.version-check.outputs.repo }}/${{ matrix.pkgclouddistro }} \ + .github/scripts/package_cloud_wrapper.sh yank ${{ needs.version-check.outputs.repo }}/${{ matrix.repo_distro }} \ "$(basename "${pkgfile}")" || true - .github/scripts/package_cloud_wrapper.sh push ${{ needs.version-check.outputs.repo }}/${{ matrix.pkgclouddistro }} "${pkgfile}" + .github/scripts/package_cloud_wrapper.sh push ${{ needs.version-check.outputs.repo }}/${{ matrix.repo_distro }} "${pkgfile}" done + - name: SSH setup + id: ssh-setup + if: github.event_name == 'workflow_dispatch' + uses: shimataro/ssh-key-action@v2 + with: + key: ${{ secrets.NETDATABOT_PACKAGES_SSH_KEY }} + name: id_ecdsa + known_hosts: ${{ secrets.PACKAGES_KNOWN_HOSTS }} + - name: Upload to packages.netdata.cloud + id: package-upload + if: github.event_name == 'workflow_dispatch' + run: | + .github/scripts/package-upload.sh \ + ${{ matrix.repo_distro }} \ + ${{ matrix.arch }} \ + ${{ matrix.format }} \ + ${{ needs.version-check.outputs.repo }} - name: Failure Notification uses: rtCamp/action-slack-notify@v2 env: @@ -235,7 +252,7 @@ jobs: SLACK_TITLE: 'Package Build failed:' SLACK_USERNAME: 'GitHub Actions' SLACK_MESSAGE: |- - ${{ github.repository }}: ${{ matrix.pkgclouddistro }} ${{ matrix.version }} package build for ${{ matrix.arch }} failed. + ${{ github.repository }}: ${{ matrix.repo_distro }} ${{ matrix.version }} package build for ${{ matrix.arch }} failed. Checkout: ${{ steps.checkout.outcome }} Setup QEMU: ${{ steps.qemu.outcome }} Setup Docker: ${{ steps.docker-config.outcome }} @@ -243,6 +260,8 @@ jobs: Build: ${{ steps.build.outcome }} Test: ${{ steps.test.outcome }} Publish: ${{ steps.upload.outcome }} + Import SSH Key: ${{ steps.ssh-setup.outcome }} + Publish: ${{ steps.package-upload.outcome }} SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }} if: >- ${{ diff --git a/.github/workflows/repoconfig-packages.yml b/.github/workflows/repoconfig-packages.yml index b0600cc0b4..4d0e75c48b 100644 --- a/.github/workflows/repoconfig-packages.yml +++ b/.github/workflows/repoconfig-packages.yml @@ -50,11 +50,10 @@ jobs: 'pkgclouddistro': data['include'][i]['packages']['repo_distro'], 'format': data['include'][i]['packages']['type'], 'base_image': data['include'][i]['base_image'] if 'base_image' in data['include'][i] else data['include'][i]['distro'], - 'platform': data['platform_map']['amd64'], - 'arch': 'amd64' + 'platform': data['platform_map']['amd64'] }) - entries.sort(key=lambda k: (k['arch'], k['distro'], k['version'])) + entries.sort(key=lambda k: (k['distro'], k['version'])) matrix = json.dumps({'include': entries}, sort_keys=True) print('Generated Matrix: ' + matrix) print('::set-output name=matrix::' + matrix) |