diff options
author | Uwe Klotz <uklotz@mixxx.org> | 2021-03-30 22:22:17 +0200 |
---|---|---|
committer | Uwe Klotz <uklotz@mixxx.org> | 2021-03-30 22:22:17 +0200 |
commit | 29b5abe09744861d786121f70f1e76acd6b50289 (patch) | |
tree | c872605026c5e6d0fd3d8c277265fba619c53051 | |
parent | 778f8830d806530b40543f5ba650d8e3dfe0dc74 (diff) | |
parent | 6da0449448b58958c4ce8fba1e34621591bc9a67 (diff) |
Merge branch '2.3' of git@github.com:mixxxdj/mixxx.git
# Conflicts:
# .github/workflows/build.yml
# src/controllers/midi/hss1394controller.cpp
-rw-r--r-- | .github/workflows/build.yml | 71 | ||||
-rw-r--r-- | CMakeLists.txt | 7 | ||||
-rw-r--r-- | res/controllers/Pioneer-DDJ-200-scripts.js | 30 | ||||
-rw-r--r-- | src/controllers/midi/hss1394controller.cpp | 24 | ||||
-rwxr-xr-x | tools/deploy.sh | 11 | ||||
-rw-r--r-- | tools/generate_download_metadata.py | 211 |
6 files changed, 314 insertions, 40 deletions
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9193e7b154..4f7fa89aa1 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -16,12 +16,11 @@ jobs: -DWARNINGS_FATAL=ON -DBULK=ON -DFFMPEG=ON - -DKEYFINDER=ON -DLOCALECOMPARE=ON -DMAD=ON -DMODPLUG=ON -DWAVPACK=ON - -INSTALL_USER_UDEV_RULES=OFF + -DINSTALL_USER_UDEV_RULES=OFF cmake_generator: Unix Makefiles cmake_build_type: RelWithDebInfo ctest_args: @@ -40,6 +39,8 @@ jobs: -DCOREAUDIO=ON -DHSS1394=ON -DMACOS_BUNDLE=ON + -DMODPLUG=OFF + -DWAVPACK=OFF cmake_generator: Unix Makefiles cmake_build_type: RelWithDebInfo # TODO: Fix this broken test on macOS @@ -51,6 +52,7 @@ jobs: buildenv_script: tools/macos_buildenv.sh artifacts_name: macOS DMG artifacts_path: build/*.dmg + artifacts_slug: macos-macosintel qt_qpa_platform: offscreen - name: Windows 2019 (MSVC) os: windows-2019 @@ -59,10 +61,11 @@ jobs: -DBULK=OFF -DFFMPEG=OFF -DHSS1394=ON - -DKEYFINDER=OFF -DLOCALECOMPARE=ON -DMAD=ON -DMEDIAFOUNDATION=ON + -DMODPLUG=ON + -DWAVPACK=ON cc: cl cxx: cl cmake_generator: Ninja @@ -76,6 +79,7 @@ jobs: buildenv_script: tools/windows_buildenv.bat artifacts_name: Windows Installer artifacts_path: build/*.msi + artifacts_slug: windows-windows64 qt_qpa_platform: windows env: @@ -87,6 +91,9 @@ jobs: runs-on: ${{ matrix.os }} name: ${{ matrix.name }} + outputs: + artifact-macos-macosintel: ${{ steps.generate_artifact_metadata.outputs.artifact-macos-macosintel }} + artifact-windows-windows64: ${{ steps.generate_artifact_metadata.outputs.artifact-windows-windows64 }} steps: # sccache's handling of the /fp:fast MSVC compiler option is broken, so use our fork with the fix. @@ -118,7 +125,7 @@ jobs: with: # This should always match the mininum required version in # our CMakeLists.txt - cmake-version: '3.13.x' + cmake-version: '3.16.x' - name: "[Windows] Set up MSVC Developer Command Prompt" if: runner.os == 'Windows' @@ -206,6 +213,7 @@ jobs: -DBATTERY=ON -DBROADCAST=ON -DHID=ON + -DKEYFINDER=ON -DLILV=ON -DOPUS=ON -DQTKEYCHAIN=ON @@ -304,13 +312,25 @@ jobs: pacman -S --noconfirm coreutils bash rsync openssh Add-Content -Path "$Env:GITHUB_ENV" -Value "PATH=$Env:PATH" + - name: "Generate Artifact Metadata" + # Generate metadata for file artifact and write it to the job output + # using the artifacts_slug value. This also sets the DEPLOY_DIR + # environment variable that is used in the deploy.sh script in the next + # step. + id: generate_artifact_metadata + if: github.event_name == 'push' + run: python3 tools/generate_download_metadata.py artifact ${{ matrix.artifacts_path }} "${{ matrix.artifacts_slug }}" + env: + DEPLOY_BASEURL: "https://downloads.mixxx.org" + DESTDIR: builds/{git_branch}/${{ runner.os }} + - name: "[macOS/Windows] Upload build to downloads.mixxx.org" # skip deploying Ubuntu builds to downloads.mixxx.org because these are deployed to the PPA if: runner.os != 'Linux' && github.event_name == 'push' && env.SSH_PASSWORD != null run: bash tools/deploy.sh ${{ matrix.artifacts_path }} env: - DESTDIR: public_html/downloads/builds - OS: ${{ runner.os }} + DESTDIR: public_html/downloads/ + DEPLOY_ONLY: 0 SSH_HOST: downloads-hostgator.mixxx.org SSH_KEY: packaging/certificates/downloads-hostgator.mixxx.org.key SSH_PASSWORD: ${{ secrets.DOWNLOADS_HOSTGATOR_DOT_MIXXX_DOT_ORG_KEY_PASSWORD }} @@ -329,3 +349,42 @@ jobs: with: name: ${{ matrix.artifacts_name }} path: ${{ matrix.artifacts_path }} + + update_manifest: + name: "Update manifest file on download server" + runs-on: ubuntu-latest + needs: build + steps: + - name: "Check out repository" + uses: actions/checkout@v2 + with: + fetch-depth: 0 + + - name: "Collect Artifacts Metadata & Write Manifest" + # Retrieve the metadata from the matrix job's outputs, merge them into a + # single JSON document and then deploy to the server. + if: github.event_name == 'push' && env.SSH_PASSWORD != null + run: python3 tools/generate_download_metadata.py manifest + env: + JOB_DATA: ${{ toJSON(needs.build) }} + MANIFEST_URL: "https://downloads.mixxx.org/builds/{git_branch}/manifest.json" + DESTDIR: "builds/{git_branch}" + SSH_PASSWORD: ${{ secrets.DOWNLOADS_HOSTGATOR_DOT_MIXXX_DOT_ORG_KEY_PASSWORD }} + + - name: "Deploy Manifest" + if: github.event_name == 'push' && env.SSH_PASSWORD != null && env.MANIFEST_DIRTY != null && env.MANIFEST_DIRTY != '0' + run: bash tools/deploy.sh manifest.json + env: + DESTDIR: public_html/downloads/ + DEPLOY_ONLY: 1 + SSH_HOST: downloads-hostgator.mixxx.org + SSH_KEY: packaging/certificates/downloads-hostgator.mixxx.org.key + SSH_PASSWORD: ${{ secrets.DOWNLOADS_HOSTGATOR_DOT_MIXXX_DOT_ORG_KEY_PASSWORD }} + SSH_USER: mixxx + UPLOAD_ID: ${{ github.run_id }} + + - name: "Trigger Netlify build" + if: env.NETLIFY_BUILD_HOOK != null && env.MANIFEST_DIRTY != null && env.MANIFEST_DIRTY != '0' + run: curl -X POST -d '{}' ${{ env.NETLIFY_BUILD_HOOK }} + env: + NETLIFY_BUILD_HOOK: ${{ secrets.NETLIFY_BUILD_HOOK }} diff --git a/CMakeLists.txt b/CMakeLists.txt index 09b7b813f2..aee17e375f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.13.0) +cmake_minimum_required(VERSION 3.16.0) project(mixxx VERSION 2.3.0) set(CMAKE_PROJECT_HOMEPAGE_URL "https://www.mixxx.org") set(CMAKE_PROJECT_DESCRIPTION "Mixxx is Free DJ software that gives you everything you need to perform live mixes.") @@ -2748,11 +2748,6 @@ set(CPACK_DEBIAN_PACKAGE_DESCRIPTION_MERGED "${CPACK_DEBIAN_PACKAGE_DESCRIPTION} string(PREPEND CPACK_DEBIAN_PACKAGE_DESCRIPTION_MERGED "${CPACK_PACKAGE_DESCRIPTION_SUMMARY}" "\n") string(REPLACE "\n\n" "\n.\n" CPACK_DEBIAN_PACKAGE_DESCRIPTION_MERGED "${CPACK_DEBIAN_PACKAGE_DESCRIPTION_MERGED}") string(REPLACE "\n" "\n " CPACK_DEBIAN_PACKAGE_DESCRIPTION_MERGED "${CPACK_DEBIAN_PACKAGE_DESCRIPTION_MERGED}") -if("3.16.0" VERSION_GREATER CMAKE_VERSION) - # This hack is no longer required with cpack version 3.16.3 - set(CPACK_DEBIAN_PACKAGE_DESCRIPTION "${CPACK_DEBIAN_PACKAGE_DESCRIPTION_MERGED}") - set(CPACK_PACKAGE_DESCRIPTION "${CPACK_DEBIAN_PACKAGE_DESCRIPTION_MERGED}") -endif() # The upstream version must not contain hyphen string(REPLACE "-" "~" CPACK_DEBIAN_UPSTREAM_VERSION "${MIXXX_VERSION}") diff --git a/res/controllers/Pioneer-DDJ-200-scripts.js b/res/controllers/Pioneer-DDJ-200-scripts.js index e6c0f68ff6..25fc5d9700 100644 --- a/res/controllers/Pioneer-DDJ-200-scripts.js +++ b/res/controllers/Pioneer-DDJ-200-scripts.js @@ -281,22 +281,26 @@ DDJ200.cueGotoandstop = function(channel, control, value, status, group) { }; DDJ200.hotcueNActivate = function(channel, control, value, status, group) { - var vDeckNo = DDJ200.vDeckNo[script.deckFromGroup(group)]; - var vgroup = "[Channel" + vDeckNo + "]"; - var hotcue = "hotcue_" + (control + 1); - engine.setValue(vgroup, hotcue + "_activate", true); - midi.sendShortMsg(status, control, - 0x7F * engine.getValue(vgroup, hotcue + "_enabled")); - var deckNo = script.deckFromGroup(group); - midi.sendShortMsg(0x90 + deckNo - 1, 0x0B, 0x7F * - engine.getValue(vgroup, "play")); // set play LED + if (value) { // only if button pressed, not releases, i.e. value === 0 + var vDeckNo = DDJ200.vDeckNo[script.deckFromGroup(group)]; + var vgroup = "[Channel" + vDeckNo + "]"; + var hotcue = "hotcue_" + (control + 1); + engine.setValue(vgroup, hotcue + "_activate", true); + midi.sendShortMsg(status, control, + 0x7F * engine.getValue(vgroup, hotcue + "_enabled")); + var deckNo = script.deckFromGroup(group); + midi.sendShortMsg(0x90 + deckNo - 1, 0x0B, 0x7F * + engine.getValue(vgroup, "play")); // set play LED + } }; DDJ200.hotcueNClear = function(channel, control, value, status, group) { - var vDeckNo = DDJ200.vDeckNo[script.deckFromGroup(group)]; - var vgroup = "[Channel" + vDeckNo + "]"; - engine.setValue(vgroup, "hotcue_" + (control + 1) + "_clear", true); - midi.sendShortMsg(status-1, control, 0x00); // set hotcue LEDs + if (value) { // only if button pressed, not releases, i.e. value === 0 + var vDeckNo = DDJ200.vDeckNo[script.deckFromGroup(group)]; + var vgroup = "[Channel" + vDeckNo + "]"; + engine.setValue(vgroup, "hotcue_" + (control + 1) + "_clear", true); + midi.sendShortMsg(status-1, control, 0x00); // set hotcue LEDs + } }; DDJ200.pfl = function(channel, control, value, status, group) { diff --git a/src/controllers/midi/hss1394controller.cpp b/src/controllers/midi/hss1394controller.cpp index d73e206212..f2d96f455d 100644 --- a/src/controllers/midi/hss1394controller.cpp +++ b/src/controllers/midi/hss1394controller.cpp @@ -22,8 +22,6 @@ void DeviceChannelListener::Process(const hss1394::uint8 *pBuffer, hss1394::uint // If multiple three-byte messages arrive right next to each other, handle them all while (i < uBufferSize) { unsigned char status = pBuffer[i]; - unsigned char opcode = status & 0xF0; - unsigned char channel = status & 0x0F; unsigned char note; unsigned char velocity; switch (status & 0xF0) { @@ -176,7 +174,7 @@ int Hss1394Controller::close() { void Hss1394Controller::sendShortMsg(unsigned char status, unsigned char byte1, unsigned char byte2) { - unsigned char data[3] = { status, byte1, byte2 }; + const unsigned char data[3] = {status, byte1, byte2}; int bytesSent = m_pChannel->SendChannelBytes(data, 3); controllerDebug(MidiUtils::formatMidiMessage(getName(), @@ -184,19 +182,19 @@ void Hss1394Controller::sendShortMsg(unsigned char status, unsigned char byte1, MidiUtils::channelFromStatus(status), MidiUtils::opCodeFromStatus(status))); - //if (bytesSent != 3) { - // qDebug()<<"ERROR: Sent" << bytesSent << "of 3 bytes:" << message; - // //m_pChannel->Flush(); - //} + if (bytesSent != 3) { + qWarning() << "Sent" << bytesSent << "of 3 bytes:" << status << byte1 << byte2; + //m_pChannel->Flush(); + } } void Hss1394Controller::sendBytes(const QByteArray& data) { - int bytesSent = m_pChannel->SendChannelBytes( - (unsigned char*)data.constData(), data.size()); + const int bytesSent = m_pChannel->SendChannelBytes( + reinterpret_cast<const unsigned char*>(data.constData()), data.size()); controllerDebug(MidiUtils::formatSysexMessage(getName(), data)); - //if (bytesSent != length) { - // qDebug()<<"ERROR: Sent" << bytesSent << "of" << length << "bytes (SysEx)"; - // //m_pChannel->Flush(); - //} + if (bytesSent != data.size()) { + qWarning() << "Sent" << bytesSent << "of" << data.size() << "bytes (SysEx)"; + //m_pChannel->Flush(); + } } diff --git a/tools/deploy.sh b/tools/deploy.sh index 328f160ac3..ec2b417be8 100755 --- a/tools/deploy.sh +++ b/tools/deploy.sh @@ -9,12 +9,12 @@ set -eu -o pipefail [ -z "${SSH_PASSWORD}" ] && echo "Please set the SSH_PASSWORD env var." >&2 && exit 1 [ -z "${SSH_USER}" ] && echo "Please set the SSH_USER env var." >&2 && exit 1 [ -z "${UPLOAD_ID}" ] && echo "Please set the UPLOAD_ID env var." >&2 && exit 1 -[ -z "${OS}" ] && echo "Please set the OS env var." >&2 && exit 1 [ -z "${DESTDIR}" ] && echo "Please set the DESTDIR env var." >&2 && exit 1 +[ -z "${DEPLOY_DIR}" ] && echo "Please set DEPLOY_DIR env var." >&2 && exit 1 SSH="ssh -i ${SSH_KEY} -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" GIT_BRANCH="$(git rev-parse --abbrev-ref HEAD)" -DEST_PATH="${DESTDIR}/${GIT_BRANCH}/${OS}" +DEST_PATH="${DESTDIR}/${DEPLOY_DIR}" TMP_PATH="${DESTDIR}/.tmp/${UPLOAD_ID}" echo "Deploying to $TMP_PATH, then to $DEST_PATH." @@ -42,6 +42,13 @@ do # This prevents users from downloading an incomplete file from the server which has not yet finished deploying. echo "Deploying artifact: ${FILEPATH}" FILENAME="$(basename "${FILEPATH}")" + + if [ "${DEPLOY_ONLY}" -eq 1 ] + then + rsync -e "${SSH}" -r --delete-after "${FILEPATH}" "${SSH_USER}@${SSH_HOST}:${DEST_PATH}" + continue + fi + FILENAME_HASH="${FILENAME}.sha256sum" FILEPATH_HASH="${FILEPATH}.sha256sum" diff --git a/tools/generate_download_metadata.py b/tools/generate_download_metadata.py new file mode 100644 index 0000000000..f3dbc954bf --- /dev/null +++ b/tools/generate_download_metadata.py @@ -0,0 +1,211 @@ +#!/usr/bin/env python3 +import argparse +import datetime +import functools +import glob +import hashlib +import json +import os +import posixpath +import subprocess +import urllib.parse +import urllib.request + + +def url_fetch(url, headers=None, **kwargs): + """Make a web request to the given URL and return the response object.""" + request_headers = { + # Override the User-Agent because our download server seems to block + # requests with the default UA value and responds "403 Forbidden". + "User-Agent": ( + "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:86.0) Gecko/20100101 " + "Firefox/86.0" + ), + } + if headers: + request_headers.update(headers) + req = urllib.request.Request(url, headers=request_headers, **kwargs) + return urllib.request.urlopen(req, timeout=10) + + +def url_exists(url): + """Make a HEAD request to the URL and check if the response is "200 OK".""" + try: + resp = url_fetch(url, method="HEAD") + except IOError: + return False + return resp.status == 200 + + +def url_download_json(url): + """Returns the JSON object from the given URL or return None.""" + try: + resp = url_fetch(url) + manifest_data = resp.read().decode() + except IOError: + return None + + return json.loads(manifest_data) + + +def sha256(file_path): + """Returns the sha256 hexdigest for a file.""" + with open(file_path, mode="rb") as fp: + read_chunk = functools.partial(fp.read, 1024) + m = hashlib.sha256() + for data in iter(read_chunk, b""): + m.update(data) + return m.hexdigest() + + +def find_git_branch(path="."): + """Return the checked out git branch for the given path.""" + return subprocess.check_output( + ("git", "rev-parse", "--abbrev-ref", "HEAD"), + cwd=path, + encoding="utf-8", + ).strip() + + +def generate_file_metadata(file_path, destdir): + """ + Generate the file metadata for file_Path. + + The destdir argument is used for for generating the download URL. + """ + file_stat = os.stat(file_path) + file_sha256 = sha256(file_path) + file_name = os.path.basename(file_path) + + commit_id = os.environ["GITHUB_SHA"] + github_run_id = os.environ["GITHUB_RUN_ID"] + github_server_url = os.environ["GITHUB_SERVER_URL"] + github_repository = os.environ["GITHUB_REPOSITORY"] + baseurl = os.environ["DEPLOY_BASEURL"] + + return { + "commit_id": commit_id, + "commit_url": ( + f"{github_server_url}/{github_repository}/commit/{commit_id}" + ), + "build_log_url": ( + f"{github_server_url}/{github_repository}/actions/" + f"runs/{github_run_id}" + ), + "file_url": f"{baseurl}/{destdir}/{file_name}", + "file_size": file_stat.st_size, + "file_date": datetime.datetime.fromtimestamp( + file_stat.st_ctime + ).isoformat(), + "sha256": file_sha256, + "sha256_url": f"{baseurl}/{destdir}/{file_name}.sha256sum", + } + + +def collect_manifest_data(job_data): + """Parse the job metadata dict and return the manifest data.""" + job_result = job_data["result"] + print(f"Build job result: {job_result}") + assert job_result == "success" + + manifest_data = {} + for output_name, output_data in job_data["outputs"].items(): + # Filter out unrelated job outputs that don't start with "artifact-". + prefix, _, slug = output_name.partition("-") + if prefix != "artifact" or not slug: + print(f"Ignoring output '{output_name}'...") + continue + artifact_data = json.loads(output_data) + + url = artifact_data["file_url"] + + # Make sure that the file actually exists on the download server + resp = url_fetch(url, method="HEAD") + if not resp.status == 200: + raise LookupError(f"Unable to find URL '{url}' on remote server") + + manifest_data[slug] = artifact_data + + return manifest_data + + +def main(argv=None): + parser = argparse.ArgumentParser() + subparsers = parser.add_subparsers() + artifact_parser = subparsers.add_parser( + "artifact", help="Generate artifact metadata from file" + ) + artifact_parser.add_argument("file") + artifact_parser.add_argument("slug") + artifact_parser.set_defaults(cmd="artifact") + + manifest_parser = subparsers.add_parser( + "manifest", + help="Collect artifact metadata and generate manifest.json file", + ) + manifest_parser.set_defaults(cmd="manifest") + + args = parser.parse_args(argv) + + git_branch = find_git_branch() + destdir = os.environ["DESTDIR"].format( + git_branch=git_branch, + ) + + if args.cmd == "artifact": + # Check that we have exactly one file artifact + artifact_paths = glob.glob(args.file) + assert len(artifact_paths) == 1 + file_path = artifact_paths[0] + + # Generate metadata and print it + metadata = generate_file_metadata(file_path, destdir) + print(json.dumps(metadata, indent=2, sort_keys=True)) + + if os.getenv("CI") == "true": + # Set GitHub Actions job output + print( + "::set-output name=artifact-{}::{}".format( + args.slug, json.dumps(metadata) + ) + ) + + # Set the DEPLOY_DIR variable for the next build step + url = urllib.parse.urlparse(metadata["file_url"]) + deploy_dir = posixpath.dirname(url.path) + with open(os.environ["GITHUB_ENV"], mode="a") as fp: + fp.write(f"DEPLOY_DIR={deploy_dir}\n") + elif args.cmd == "manifest": + # Parse the JOB_DATA JSON data, generate the manifest data and print it + job_data = json.loads(os.environ["JOB_DATA"]) + manifest_data = collect_manifest_data(job_data) + print(json.dumps(manifest_data, indent=2, sort_keys=True)) + + # Write the manifest.json for subsequent deployment to the server + with open("manifest.json", mode="w") as fp: + json.dump(manifest_data, fp, indent=2, sort_keys=True) + + if os.getenv("CI") == "true": + # Check if generated manifest.json file differs from the one that + # is currently deployed. + manifest_url = os.environ["MANIFEST_URL"].format( + git_branch=git_branch + ) + try: + remote_manifest_data = url_fetch(manifest_url) + except IOError: + remote_manifest_data = None + + if manifest_data == remote_manifest_data: + return + + # The manifest data is different, so we set the DEPLOY_DIR and + # MANIFEST_DIRTY env vars. + deploy_dir = os.environ["DESTDIR"].format(git_branch=git_branch) + with open(os.environ["GITHUB_ENV"], mode="a") as fp: + fp.write(f"DEPLOY_DIR={deploy_dir}\n") + fp.write("MANIFEST_DIRTY=1\n") + + +if __name__ == "__main__": + main() |