Fix handling of binary package uploads. (#10860)

3 files changed, 29 insertions, 66 deletions

diff --git a/.github/scripts/package_cloud_wrapper.sh b/.github/scripts/package_cloud_wrapper.sh
index 0876b2a363..7640ef484d 100755
--- a/.github/scripts/package_cloud_wrapper.sh
+++ b/.github/scripts/package_cloud_wrapper.sh
@@ -29,7 +29,7 @@ fi
 # Install dependency if not there
 if ! command -v package_cloud > /dev/null 2>&1; then
 	echo "No package cloud gem found, installing"
-	gem install -V package_cloud || (echo "Package cloud installation failed. you might want to check if required dependencies are there (ruby gcc gcc-c++ ruby-devel)" && exit 1)
+	sudo gem install -V package_cloud || (echo "Package cloud installation failed. you might want to check if required dependencies are there (ruby gcc gcc-c++ ruby-devel)" && exit 1)
 else
 	echo "Found package_cloud gem, continuing"
 fi
diff --git a/.github/scripts/parse_packagecloud_dist_id.py b/.github/scripts/parse_packagecloud_dist_id.py
deleted file mode 100755
index 55ddf4bec5..0000000000
--- a/.github/scripts/parse_packagecloud_dist_id.py
+++ /dev/null
@@ -1,39 +0,0 @@
-#!/usr/bin/env python3
-'''
-Parse the PackageCloud distributions JSON data to get a dist ID for uploads.
-
-This takes the JSON distributions data from Packagecloud on stdin and
-the package format, distribution name and version as arguments, and
-prints either an error message or the parsed distribution ID based on
-the arguments.
-'''
-
-import json
-import sys
-
-fmt = sys.argv[1]      # The package format ('deb' or 'rpm')
-distro = sys.argv[2]   # The distro name
-version = sys.argv[3]  # The distro version
-print(fmt)
-print(distro)
-print(version)
-
-data = json.load(sys.stdin)
-versions = []
-
-for entry in data[fmt]:
-    if entry['display_name'] == distro:
-        versions = entry['versions']
-        break
-
-if not versions:
-    print('Could not find version information for the requested distribution.')
-    sys.exit(-1)
-
-for entry in versions:
-    if entry['version_number'] == version:
-        print(entry['id'])
-        sys.exit(0)
-
-print('Unable to find id for requested version.')
-sys.exit(-1)
diff --git a/.github/workflows/packaging.yml b/.github/workflows/packaging.yml
index 1a72f0353d..7340c27bb6 100644
--- a/.github/workflows/packaging.yml
+++ b/.github/workflows/packaging.yml
@@ -24,23 +24,23 @@ jobs:
     strategy:
       matrix:
         include:
-          - {distro: debian, version: "9", pkgclouddistro: Debian, pkgcloudversion: "9.0", format: deb, base_image: debian, platform: linux/amd64, arch: amd64}
-          - {distro: debian, version: "9", pkgclouddistro: Debian, pkgcloudversion: "9.0", format: deb, base_image: debian, platform: linux/i386, arch: i386}
-          - {distro: debian, version: "10", pkgclouddistro: Debian, pkgcloudversion: "10.0", format: deb, base_image: debian, platform: linux/amd64, arch: amd64}
-          - {distro: debian, version: "10", pkgclouddistro: Debian, pkgcloudversion: "10.0", format: deb, base_image: debian, platform: linux/i386, arch: i386}
-          - {distro: ubuntu, version: "16.04", pkgclouddistro: Ubuntu, pkgcloudversion: "16.04", format: deb, base_image: ubuntu, platform: linux/amd64, arch: amd64}
-          - {distro: ubuntu, version: "16.04", pkgclouddistro: Ubuntu, pkgcloudversion: "16.04", format: deb, base_image: ubuntu, platform: linux/i386, arch: i386}
-          - {distro: ubuntu, version: "18.04", pkgclouddistro: Ubuntu, pkgcloudversion: "18.04", format: deb, base_image: ubuntu, platform: linux/amd64, arch: amd64}
-          - {distro: ubuntu, version: "18.04", pkgclouddistro: Ubuntu, pkgcloudversion: "18.04", format: deb, base_image: ubuntu, platform: linux/i386, arch: i386}
-          - {distro: ubuntu, version: "20.04", pkgclouddistro: Ubuntu, pkgcloudversion: "20.04", format: deb, base_image: ubuntu, platform: linux/amd64, arch: amd64}
-          - {distro: ubuntu, version: "20.10", pkgclouddistro: Ubuntu, pkgcloudversion: "20.10", format: deb, base_image: ubuntu, platform: linux/amd64, arch: amd64}
-          - {distro: ubuntu, version: "21.04", pkgclouddistro: Ubuntu, pkgcloudversion: "20.10", format: deb, base_image: ubuntu, platform: linux/amd64, arch: amd64}
-          - {distro: centos, version: "7", pkgclouddistro: Enterprise Linux, pkgcloudversion: "7.0", format: rpm, base_image: centos, platform: linux/amd64, arch: amd64}
-          - {distro: centos, version: "8", pkgclouddistro: Enterprise Linux, pkgcloudversion: "8.0", format: rpm, base_image: centos, platform: linux/amd64, arch: amd64}
-          - {distro: fedora, version: "32", pkgclouddistro: Fedora, pkgcloudversion: "32.0", format: rpm, base_image: fedora, platform: linux/amd64, arch: amd64}
-          - {distro: fedora, version: "33", pkgclouddistro: Fedora, pkgcloudversion: "33.0", format: rpm, base_image: fedora, platform: linux/amd64, arch: amd64}
-          - {distro: fedora, version: "34", pkgclouddistro: Fedora, pkgcloudversion: "33.0", format: rpm, base_image: fedora, platform: linux/amd64, arch: amd64}
-          - {distro: opensuse, version: "15.2", pkgclouddistro: openSUSE, pkgcloudversion: "15.2", format: rpm, base_image: opensuse/leap, platform: linux/amd64, arch: amd64}
+          - {distro: debian, version: "9", pkgclouddistro: debian/stretch, format: deb, base_image: debian, platform: linux/amd64, arch: amd64}
+          - {distro: debian, version: "9", pkgclouddistro: debian/stretch, format: deb, base_image: debian, platform: linux/i386, arch: i386}
+          - {distro: debian, version: "10", pkgclouddistro: debian/buster, format: deb, base_image: debian, platform: linux/amd64, arch: amd64}
+          - {distro: debian, version: "10", pkgclouddistro: debian/buster, format: deb, base_image: debian, platform: linux/i386, arch: i386}
+          - {distro: ubuntu, version: "16.04", pkgclouddistro: ubuntu/xenial, format: deb, base_image: ubuntu, platform: linux/amd64, arch: amd64}
+          - {distro: ubuntu, version: "16.04", pkgclouddistro: ubuntu/xenial, format: deb, base_image: ubuntu, platform: linux/i386, arch: i386}
+          - {distro: ubuntu, version: "18.04", pkgclouddistro: ubuntu/bionic, format: deb, base_image: ubuntu, platform: linux/amd64, arch: amd64}
+          - {distro: ubuntu, version: "18.04", pkgclouddistro: ubuntu/bionic, format: deb, base_image: ubuntu, platform: linux/i386, arch: i386}
+          - {distro: ubuntu, version: "20.04", pkgclouddistro: ubuntu/focal, format: deb, base_image: ubuntu, platform: linux/amd64, arch: amd64}
+          - {distro: ubuntu, version: "20.10", pkgclouddistro: ubuntu/groovy, format: deb, base_image: ubuntu, platform: linux/amd64, arch: amd64}
+          - {distro: ubuntu, version: "21.04", pkgclouddistro: ubuntu/hirsute, format: deb, base_image: ubuntu, platform: linux/amd64, arch: amd64}
+          - {distro: centos, version: "7", pkgclouddistro: el/7, format: rpm, base_image: centos, platform: linux/amd64, arch: amd64}
+          - {distro: centos, version: "8", pkgclouddistro: el/8, format: rpm, base_image: centos, platform: linux/amd64, arch: amd64}
+          - {distro: fedora, version: "32", pkgclouddistro: fedora/32, format: rpm, base_image: fedora, platform: linux/amd64, arch: amd64}
+          - {distro: fedora, version: "33", pkgclouddistro: fedora/33, format: rpm, base_image: fedora, platform: linux/amd64, arch: amd64}
+          - {distro: fedora, version: "34", pkgclouddistro: fedora/34, format: rpm, base_image: fedora, platform: linux/amd64, arch: amd64}
+          - {distro: opensuse, version: "15.2", pkgclouddistro: opensuse/15.2, format: rpm, base_image: opensuse/leap, platform: linux/amd64, arch: amd64}
       # We intentiaonally disable the fail-fast behavior so that a
       # build failure for one version doesn't prevent us from publishing
       # successfully built and tested packages for another version.
@@ -51,12 +51,14 @@ jobs:
         uses: actions/checkout@v2
         with:
           fetch-depth: 0 # We need full history for versioning
+          submodules: true
       - name: Checkout Tag # Otherwise check out the tag that triggered this.
-        if: github.event_name == 'wrokflow_dispatch'
+        if: github.event_name == 'workflow_dispatch'
         uses: actions/checkout@v2
         with:
-          refs: ${{ github.event.ref }}
+          ref: ${{ github.event.ref }}
           fetch-depth: 0 # We need full history for versioning
+          submodules: true
       - name: Check Base Branch
         run: |
           if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
@@ -74,7 +76,7 @@ jobs:
                 ;;
               *)
                 echo "repo=${{ secrets.PACKAGE_CLOUD_REPO }}-devel" >> $GITHUB_ENV
-                echo "pkg_version=$(git describe | sed -e 's/^v//')" >> $GITHUB_ENV
+                echo "pkg_version=0.${GITHUB_SHA}" >> $GITHUB_ENV
                 echo "pkg_retention_days=30" >> $GITHUB_ENV
                 ;;
             esac
@@ -114,13 +116,13 @@ jobs:
       - name: Upload
         if: github.event_name == 'workflow_dispatch'
         shell: bash
+        env:
+          PKG_CLOUD_TOKEN: ${{ secrets.PACKAGE_CLOUD_API_KEY }}
         run: |
-          # This figures out the distribution ID for the upload.
-          DIST_ID="$(curl https://${{ secrets.PACKAGE_CLOUD_API_KEY }}:@packagecloud.io/api/v1/distributions.json | ${GITHUB_WORKSPACE}/.github/scripts/parse_packagecloud_dist_id.py ${{ matrix.format }} ${{ matrix.pkgclouddistro }} ${{ matrix.pkgcloudversion }})"
-          for pkgfile in artifacts/*.${FORMAT} ; do
-            curl -F "package[distro_version_id]=${BUILD_ID}" \
-                 -F "package[package_file]=@${pkgfile}" \
-                 https://${{ secrets.PACKAGE_CLOUD_API_KEY }}:@packagecloud.io/api/v1/repos/${{ env.repo }}/packages.json || exit 1
+          echo "Packages to upload:\n$(ls artifacts/*.${{ matrix.format }})"
+          for pkgfile in artifacts/*.${{ matrix.format }} ; do
+            .github/scripts/package_cloud_wrapper.sh push ${{ env.repo }}/${{ matrix.pkgclouddistro }} ${pkgfile}
+          done
       - name: Clean
         if: github.event_name == 'workflow_dispatch'
         shell: bash