CI: Use new deploy.py script and make deploy.sh less smart (just upload)

2 files changed, 26 insertions, 75 deletions

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index cd7dd0a220..dc99adc6c4 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -118,8 +118,8 @@ jobs:
     runs-on: ${{ matrix.os }}
     name: ${{ matrix.name }}
     outputs:
-      artifact-macos-macosintel: ${{ steps.generate_artifact_metadata.outputs.artifact-macos-macosintel }}
-      artifact-windows-windows64: ${{ steps.generate_artifact_metadata.outputs.artifact-windows-windows64 }}
+      artifact-macos-macosintel: ${{ steps.prepare_deploy.outputs.artifact-macos-macosintel }}
+      artifact-windows-windows64: ${{ steps.prepare_deploy.outputs.artifact-windows-windows64 }}
     steps:
 
     # sccache's handling of the /fp:fast MSVC compiler option is broken, so use our fork with the fix.
@@ -329,6 +329,21 @@ jobs:
       run: signtool sign /f $Env:WINDOWS_CODESIGN_CERTIFICATE_PATH /p $Env:WINDOWS_CODESIGN_CERTIFICATE_PASSWORD *.msi
       working-directory: build
 
+    - name: "Prepare for deployment"
+      # Copy the desired directory structure to the deploy/ directory. This
+      # also generates metadata for file artifact and write it to the job
+      # output using the artifacts_slug value.
+      id: prepare_deploy
+      if: github.event_name == 'push'
+      shell: bash
+      run: >
+        python3 tools/deploy.py prepare-deployment
+        --slug '${{ matrix.artifacts_slug }}'
+        --output-dir 'deploy/'
+        --dest-path 'builds/{branch}/${{ runner.os }}/{filename}'
+        --dest-url 'https://downloads.mixxx.org'
+        ${{ matrix.artifacts_path }}
+
     - name: "[Windows] Install rsync and openssh"
       env:
         SSH_PASSWORD: ${{ secrets.DOWNLOADS_HOSTGATOR_DOT_MIXXX_DOT_ORG_KEY_PASSWORD }}
@@ -338,30 +353,16 @@ jobs:
         pacman -S --noconfirm coreutils bash rsync openssh
         Add-Content -Path "$Env:GITHUB_ENV" -Value "PATH=$Env:PATH"
 
-    - name: "Generate Artifact Metadata"
-      # Generate metadata for file artifact and write it to the job output
-      # using the artifacts_slug value.  This also sets the DEPLOY_DIR
-      # environment variable that is used in the deploy.sh script in the next
-      # step.
-      id: generate_artifact_metadata
-      if: github.event_name == 'push'
-      run: python3 tools/generate_download_metadata.py artifact ${{ matrix.artifacts_path }} "${{ matrix.artifacts_slug }}"
-      env:
-        DEPLOY_BASEURL: "https://downloads.mixxx.org"
-        DESTDIR: builds/{git_branch}/${{ runner.os }}
-
     - name: "[macOS/Windows] Upload build to downloads.mixxx.org"
       # skip deploying Ubuntu builds to downloads.mixxx.org because these are deployed to the PPA
       if: runner.os != 'Linux' && github.event_name == 'push' && env.SSH_PASSWORD != null
-      run: bash tools/deploy.sh ${{ matrix.artifacts_path }}
+      run: bash tools/deploy.sh deploy/
       env:
         DESTDIR: public_html/downloads/
-        DEPLOY_ONLY: 0
         SSH_HOST: downloads-hostgator.mixxx.org
         SSH_KEY: packaging/certificates/downloads-hostgator.mixxx.org.key
         SSH_PASSWORD: ${{ secrets.DOWNLOADS_HOSTGATOR_DOT_MIXXX_DOT_ORG_KEY_PASSWORD }}
         SSH_USER: mixxx
-        UPLOAD_ID: ${{ github.run_id }}
 
     # Workaround for https://github.com/actions/cache/issues/531
     - name: Use system tar & zstd from Chocolatey for caching
@@ -390,24 +391,24 @@ jobs:
       # Retrieve the metadata from the matrix job's outputs, merge them into a
       # single JSON document and then deploy to the server.
       if: github.event_name == 'push' && env.SSH_PASSWORD != null
-      run: python3 tools/generate_download_metadata.py manifest
+      run: >
+        python3 tools/deploy.py generate-manifest
+        --output-dir 'deploy/'
+        --dest-path 'builds/{branch}/manifest.json'
+        --dest-url 'https://downloads.mixxx.org'
       env:
         JOB_DATA: ${{ toJSON(needs.build) }}
-        MANIFEST_URL: "https://downloads.mixxx.org/builds/{git_branch}/manifest.json"
-        DESTDIR: "builds/{git_branch}"
         SSH_PASSWORD: ${{ secrets.DOWNLOADS_HOSTGATOR_DOT_MIXXX_DOT_ORG_KEY_PASSWORD }}
 
     - name: "Deploy Manifest"
       if: github.event_name == 'push' && env.SSH_PASSWORD != null && env.MANIFEST_DIRTY != null && env.MANIFEST_DIRTY != '0'
-      run: bash tools/deploy.sh manifest.json
+      run: bash tools/deploy.sh deploy/
       env:
         DESTDIR: public_html/downloads/
-        DEPLOY_ONLY: 1
         SSH_HOST: downloads-hostgator.mixxx.org
         SSH_KEY: packaging/certificates/downloads-hostgator.mixxx.org.key
         SSH_PASSWORD: ${{ secrets.DOWNLOADS_HOSTGATOR_DOT_MIXXX_DOT_ORG_KEY_PASSWORD }}
         SSH_USER: mixxx
-        UPLOAD_ID: ${{ github.run_id }}
 
     - name: "Trigger Netlify build"
       if: env.NETLIFY_BUILD_HOOK != null && env.MANIFEST_DIRTY != null && env.MANIFEST_DIRTY != '0'
diff --git a/tools/deploy.sh b/tools/deploy.sh
index ec2b417be8..5d792d6cb9 100755
--- a/tools/deploy.sh
+++ b/tools/deploy.sh
@@ -8,16 +8,11 @@ set -eu -o pipefail
 [ -z "${SSH_KEY}" ] && echo "Please set the SSH_KEY env var." >&2 && exit 1
 [ -z "${SSH_PASSWORD}" ] && echo "Please set the SSH_PASSWORD env var." >&2 && exit 1
 [ -z "${SSH_USER}" ] && echo "Please set the SSH_USER env var." >&2 && exit 1
-[ -z "${UPLOAD_ID}" ] && echo "Please set the UPLOAD_ID env var." >&2 && exit 1
 [ -z "${DESTDIR}" ] && echo "Please set the DESTDIR env var." >&2 && exit 1
-[ -z "${DEPLOY_DIR}" ] && echo "Please set DEPLOY_DIR env var." >&2 && exit 1
 
 SSH="ssh -i ${SSH_KEY} -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null"
-GIT_BRANCH="$(git rev-parse --abbrev-ref HEAD)"
-DEST_PATH="${DESTDIR}/${DEPLOY_DIR}"
-TMP_PATH="${DESTDIR}/.tmp/${UPLOAD_ID}"
 
-echo "Deploying to $TMP_PATH, then to $DEST_PATH."
+echo "Deploying contents of $1 to $DESTDIR."
 
 # Remove permissions for group and other users so that ssh-keygen does not
 # complain about the key not being protected.
@@ -26,49 +21,4 @@ chmod go-rwx "${SSH_KEY}"
 # Unlock the key by removing its password. This is easier than messing with ssh-agent.
 ssh-keygen -p -P "${SSH_PASSWORD}" -N "" -f "${SSH_KEY}"
 
-# realpath does not exist on macOS
-command -v realpath >/dev/null 2>&1 || realpath() {
-    [[ "$1" = /* ]] && echo "$1" || echo "${PWD}/${1#./}"
-}
-
-# sha256sum doesn't exist on Windows (Git Bash) or macOS
-command -v sha256sum >/dev/null 2>&1 || sha256sum() {
-    openssl dgst -sha256 "$@" | sed 's/^SHA256(\(.*\))= \(\w\+\)$/\2  \1/'
-}
-
-for FILEPATH in "$@"
-do
-    # Always upload to a temporary path.
-    # This prevents users from downloading an incomplete file from the server which has not yet finished deploying.
-    echo "Deploying artifact: ${FILEPATH}"
-    FILENAME="$(basename "${FILEPATH}")"
-
-    if [ "${DEPLOY_ONLY}" -eq 1 ]
-    then
-        rsync -e "${SSH}" -r --delete-after "${FILEPATH}" "${SSH_USER}@${SSH_HOST}:${DEST_PATH}"
-        continue
-    fi
-
-    FILENAME_HASH="${FILENAME}.sha256sum"
-    FILEPATH_HASH="${FILEPATH}.sha256sum"
-
-    # There should be no path components in the shasum file, so we need to cd to it first.
-    pushd "$(dirname "$(realpath "${FILEPATH}")")"
-    sha256sum "${FILENAME}" > "${FILENAME_HASH}"
-    popd
-
-    FILEEXT="${FILENAME##*.}"
-    SYMLINK_NAME="Mixxx-${GIT_BRANCH}-latest.${FILEEXT}"
-
-    rsync -e "${SSH}" --rsync-path="mkdir -p ${TMP_PATH} && rsync" -r --delete-after "${FILEPATH}" "${FILEPATH_HASH}" "${SSH_USER}@${SSH_HOST}:${TMP_PATH}"
-
-    # Move from the temporary path to the final destination.
-    ${SSH} "${SSH_USER}@${SSH_HOST}" << EOF
-    trap 'rm -rf "${TMP_PATH}"' EXIT
-    mkdir -p "${DEST_PATH}" &&
-    mv "${TMP_PATH}/${FILENAME}" "${TMP_PATH}/${FILENAME_HASH}" "${DEST_PATH}" &&
-    cd "${DEST_PATH}" &&
-    ln -sf "${FILENAME_HASH}" "${SYMLINK_NAME}.sha256sum" &&
-    ln -sf "${FILENAME}" "${SYMLINK_NAME}"
-EOF
-done
+rsync --rsh="${SSH}" --verbose --recursive --checksum --times --delay-updates "$@" "${SSH_USER}@${SSH_HOST}:${DESTDIR}/"