diff options
| -rw-r--r-- | .travis.yml | 38 | ||||
| -rwxr-xr-x | .travis/package_management/build_package_in_container.sh | 8 | ||||
| -rwxr-xr-x | .travis/package_management/common.py | 46 | ||||
| -rwxr-xr-x | .travis/package_management/configure_deb_lxc_environment.py | 65 | ||||
| -rwxr-xr-x | .travis/package_management/configure_rpm_lxc_environment.py | 159 | ||||
| -rwxr-xr-x | .travis/package_management/create_lxc_for_build.sh | 8 | ||||
| -rwxr-xr-x | .travis/package_management/deb/configure_lxc_environment.py | 71 | ||||
| -rwxr-xr-x | .travis/package_management/rpm/configure_lxc_environment.py | 89 | ||||
| -rwxr-xr-x | .travis/package_management/rpm/trigger_lxc_build.py | 62 | ||||
| -rwxr-xr-x | .travis/package_management/trigger_deb_lxc_build.py (renamed from .travis/package_management/deb/trigger_lxc_build.py) | 25 | ||||
| -rwxr-xr-x | .travis/package_management/trigger_rpm_lxc_build.py | 55 | ||||
| -rwxr-xr-x | .travis/package_management/yank_stale_rpm.sh | 2 | ||||
| -rw-r--r-- | netdata.spec.in | 365 |
13 files changed, 661 insertions, 332 deletions
diff --git a/.travis.yml b/.travis.yml index ccd2f78d6d..3d3d6ffcc4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,6 +23,13 @@ install: - source tests/installer/slack.sh - export NOTIF_CHANNEL="automation-beta" - if [ "${TRAVIS_REPO_SLUG}" = "netdata/netdata" ]; then export NOTIF_CHANNEL="automation"; fi; +- export BUILD_VERSION="$(cat packaging/version | cut -d'-' -f1)" +- if [[ "${TRAVIS_COMMIT_MESSAGE}" = *"[Build latest]"* ]]; then export BUILD_VERSION="$(cat packaging/version | cut -d'-' -f1,2 | sed -e 's/-/./g').latest"; fi; +- export DEPLOY_REPO="netdata" # Default production packaging repository +- if [[ "${TRAVIS_COMMIT_MESSAGE}" = *"[Build latest]"* ]]; then export DEPLOY_REPO="netdata-edge"; fi; +#- export PACKAGING_USER="$(echo ${TRAVIS_REPO_SLUG} | cut -d'/' -f1)" +#Override to help push packages towards netdata/netdata-edge +- export PACKAGING_USER="netdata" @@ -80,23 +87,23 @@ stages: # Build RPM packages under special conditions # Enterprise linux (Covers CentOS, Redhat, Amazon linux) - name: "Package Enterprise Linux 7" - if: type != cron AND branch = master AND commit_message =~ /(\[Package arm64 RPM Enterprise Linux\]|\[Package arm64 RPM\]|\[Package i386 RPM Enterprise Linux\]|\[Package i386 RPM\]|\[Package AMD64 RPM Enterprise Linux\]|\[Package AMD64 RPM\])/ + if: type != cron AND branch = binary-packages-specfile-cleanup AND commit_message =~ /(\[Package arm64 RPM Enterprise Linux\]|\[Package arm64 RPM\]|\[Package i386 RPM Enterprise Linux\]|\[Package i386 RPM\]|\[Package AMD64 RPM Enterprise Linux\]|\[Package AMD64 RPM\])/ - name: "Package Enterprise linux 6" - if: type != cron AND branch = master AND commit_message =~ /(\[Package i386 RPM Enterprise Linux\]|\[Package i386 RPM\]|\[Package AMD64 RPM Enterprise Linux\]|\[Package AMD64 RPM\])/ + if: type != cron AND branch = binary-packages-specfile-cleanup AND commit_message =~ /(\[Package i386 RPM Enterprise Linux\]|\[Package i386 RPM\]|\[Package AMD64 RPM Enterprise Linux\]|\[Package AMD64 RPM\])/ # Fedora - name: "Package Fedora 30" - if: type != cron AND branch = master AND commit_message =~ /(\[Package arm64 RPM Fedora\]|\[Package arm64 RPM\]|\[Package AMD64 RPM Fedora\]|\[Package AMD64 RPM\])/ + if: type != cron AND branch = binary-packages-specfile-cleanup AND commit_message =~ /(\[Package arm64 RPM Fedora\]|\[Package arm64 RPM\]|\[Package AMD64 RPM Fedora\]|\[Package AMD64 RPM\])/ - name: "Package Fedora 29" - if: type != cron AND branch = master AND commit_message =~ /(\[Package arm64 RPM Fedora\]|\[Package arm64 RPM\]|\[Package AMD64 RPM Fedora\]|\[Package AMD64 RPM\])/ + if: type != cron AND branch = binary-packages-specfile-cleanup AND commit_message =~ /(\[Package arm64 RPM Fedora\]|\[Package arm64 RPM\]|\[Package AMD64 RPM Fedora\]|\[Package AMD64 RPM\])/ - name: "Package Fedora 28" - if: type != cron AND branch = master AND commit_message =~ /(\[Package arm64 RPM Fedora\]|\[Package arm64 RPM\]|\[Package AMD64 RPM Fedora\]|\[Package AMD64 RPM\])/ + if: type != cron AND branch = binary-packages-specfile-cleanup AND commit_message =~ /(\[Package arm64 RPM Fedora\]|\[Package arm64 RPM\]|\[Package AMD64 RPM Fedora\]|\[Package AMD64 RPM\])/ # OpenSuSE - name: "Package OpenSuSE 15.1" - if: type != cron AND branch = master AND commit_message =~ /(\[Package arm64 RPM openSuSE\]|\[Package arm64 RPM\]|\[Package AMD64 RPM openSuSE\]|\[Package AMD64 RPM\])/ + if: type != cron AND branch = binary-packages-specfile-cleanup AND commit_message =~ /(\[Package arm64 RPM openSuSE\]|\[Package arm64 RPM\]|\[Package AMD64 RPM openSuSE\]|\[Package AMD64 RPM\])/ - name: "Package OpenSuSE 15.0" - if: type != cron AND branch = master AND commit_message =~ /(\[Package arm64 RPM openSuSE\]|\[Package arm64 RPM\]|\[Package AMD64 RPM openSuSE\]|\[Package AMD64 RPM\])/ + if: type != cron AND branch = binary-packages-specfile-cleanup AND commit_message =~ /(\[Package arm64 RPM openSuSE\]|\[Package arm64 RPM\]|\[Package AMD64 RPM openSuSE\]|\[Package AMD64 RPM\])/ @@ -109,20 +116,20 @@ stages: before_script: - post_message "TRAVIS_MESSAGE" "Starting package preparation and publishing for ${BUILD_STRING}.${BUILD_ARCH}" "${NOTIF_CHANNEL}" - export PACKAGES_DIRECTORY="$(mktemp -d -t netdata-packaging-contents-dir-XXXXXX)" && echo "Created packaging directory ${PACKAGES_DIRECTORY}" - - export BUILD_VERSION=$(cat packaging/version | cut -d'-' -f1) # We always build on the latest release script: - echo "Creating LXC environment for the build" && sudo -E .travis/package_management/create_lxc_for_build.sh - echo "Building package in container" && sudo -E .travis/package_management/build_package_in_container.sh - sudo chmod -R 755 "/var/lib/lxc" - echo "Preparing RPM packaging contents for upload" && sudo -E .travis/package_management/prepare_packages.sh after_failure: post_message "TRAVIS_MESSAGE" "Failed to build RPM for ${BUILD_STRING}.${BUILD_ARCH}" + before_deploy: - .travis/package_management/yank_stale_rpm.sh "${PACKAGES_DIRECTORY}" "${BUILD_STRING}" || echo "No stale RPM found" deploy: # Beta packages deployment - provider: packagecloud - repository: "netdata" - username: "$(echo ${TRAVIS_REPO_SLUG} | cut -d'/' -f1)" + repository: "${DEPLOY_REPO}" + username: "${PACKAGING_USER}" token: "${PKG_CLOUD_TOKEN}" dist: "${BUILD_STRING}" local_dir: "${PACKAGES_DIRECTORY}" @@ -130,7 +137,7 @@ stages: on: # Only deploy on ${USER}/netdata, master branch, when packages directory is created repo: ${TRAVIS_REPO_SLUG} - branch: master + branch: binary-packages-specfile-cleanup condition: -d "${PACKAGES_DIRECTORY}" # Production release packages deployment - provider: packagecloud @@ -143,14 +150,14 @@ stages: on: # Only deploy on ${USER}/netdata, master branch, when packages directory is created repo: "${TRAVIS_REPO_SLUG}" - branch: master + branch: binary-packages-specfile-cleanup condition: -d "${PACKAGES_DIRECTORY}" after_deploy: - if [ -n "${BUILDER_NAME}" ]; then rm -rf /home/${BUILDER_NAME}/* && echo "Cleared /home/${BUILDER_NAME} directory" || echo "Failed to clean /home/${BUILDER_NAME} directory"; fi; - if [ -d "${PACKAGES_DIRECTORY}" ]; then rm -rf "${PACKAGES_DIRECTORY}"; fi; - + # TODO: This section is stale, will be aligned with the RPM implementation when we get to DEB packaging - stage: &_DEB_TEMPLATE name: "Build & Publish DEB package" before_install: @@ -159,7 +166,6 @@ stages: before_script: - post_message "TRAVIS_MESSAGE" "Starting package preparation and publishing for ${BUILD_STRING}.${BUILD_ARCH}" "${NOTIF_CHANNEL}" - export PACKAGES_DIRECTORY="$(mktemp -d -t netdata-packaging-contents-dir-XXXXXX)" && echo "Created packaging directory ${PACKAGES_DIRECTORY}" - - export BUILD_VERSION=$(cat packaging/version | cut -d'-' -f1) # We always build on the latest release script: - echo "Creating LXC environment for the build" && sudo -E .travis/package_management/create_lxc_for_build.sh - echo "Building package in container" && sudo -E .travis/package_management/build_package_in_container.sh @@ -171,8 +177,8 @@ stages: deploy: # Beta packages deployment - provider: packagecloud - repository: "netdata" - username: "$(echo ${TRAVIS_REPO_SLUG} | cut -d'/' -f1)" + repository: "${DEPLOY_REPO}" + username: "${PACKAGING_USER}" token: "${PKG_CLOUD_TOKEN}" dist: "${BUILD_STRING}" local_dir: "${PACKAGES_DIRECTORY}" diff --git a/.travis/package_management/build_package_in_container.sh b/.travis/package_management/build_package_in_container.sh index 2719e7b6b3..95a68e7a83 100755 --- a/.travis/package_management/build_package_in_container.sh +++ b/.travis/package_management/build_package_in_container.sh @@ -54,24 +54,24 @@ case "${BUILD_ARCH}" in echo "Building for amd64.." export CONTAINER_NAME="${BUILDER_NAME}-${BUILD_DISTRO}${BUILD_RELEASE}-amd64" export LXC_CONTAINER_ROOT="/var/lib/lxc/${CONTAINER_NAME}/rootfs" - .travis/package_management/"${PACKAGE_TYPE}"/trigger_lxc_build.py "${CONTAINER_NAME}" + .travis/package_management/trigger_${PACKAGE_TYPE}_lxc_build.py "${CONTAINER_NAME}" echo "Building for arm64.." export CONTAINER_NAME="${BUILDER_NAME}-${BUILD_DISTRO}${BUILD_RELEASE}-arm64" export LXC_CONTAINER_ROOT="/var/lib/lxc/${CONTAINER_NAME}/rootfs" - .travis/package_management/"${PACKAGE_TYPE}"/trigger_lxc_build.py "${CONTAINER_NAME}" + .travis/package_management/trigger_${PACKAGE_TYPE}_lxc_build.py "${CONTAINER_NAME}" echo "Building for i386.." export CONTAINER_NAME="${BUILDER_NAME}-${BUILD_DISTRO}${BUILD_RELEASE}-i386" export LXC_CONTAINER_ROOT="/var/lib/lxc/${CONTAINER_NAME}/rootfs" - .travis/package_management/"${PACKAGE_TYPE}"/trigger_lxc_build.py "${CONTAINER_NAME}" + .travis/package_management/trigger_${PACKAGE_TYPE}_lxc_build.py "${CONTAINER_NAME}" ;; "amd64"|"arm64"|"i386") echo "Building for ${BUILD_ARCH}.." export CONTAINER_NAME="${BUILDER_NAME}-${BUILD_DISTRO}${BUILD_RELEASE}-${BUILD_ARCH}" export LXC_CONTAINER_ROOT="/var/lib/lxc/${CONTAINER_NAME}/rootfs" - .travis/package_management/"${PACKAGE_TYPE}"/trigger_lxc_build.py "${CONTAINER_NAME}" + .travis/package_management/trigger_${PACKAGE_TYPE}_lxc_build.py "${CONTAINER_NAME}" ;; *) echo "Unknown build architecture '${BUILD_ARCH}', nothing to do for build" diff --git a/.travis/package_management/common.py b/.travis/package_management/common.py new file mode 100755 index 0000000000..6cf59293d9 --- /dev/null +++ b/.travis/package_management/common.py @@ -0,0 +1,46 @@ +# +# +# Python library with commonly used functions within the package management scope +# +# Author : Pavlos Emm. Katsoulakis <paul@netdata.cloud> + +import lxc +import subprocess + +def replace_tag(tag_name, spec, new_tag_content): + print("Fixing tag %s in %s" % (tag_name, spec)) + + ifp = open(spec, "r") + config = ifp.readlines() + ifp.close() + + source_line = -1 + for line in config: + if str(line).count(tag_name + ":") > 0: + source_line = config.index(line) + print("Found line: %s in item %d" % (line, source_line)) + break + + if source_line >= 0: + print("Replacing line %s with %s in spec file" %(config[source_line], new_tag_content)) + config[source_line] = "%s: %s\n" % (tag_name, new_tag_content) + config_str = ''.join(config) + ofp = open(spec, 'w') + ofp.write(config_str) + ofp.close() + +def run_command(container, command): + print("Running command: %s" % command) + command_result = container.attach_wait(lxc.attach_run_command, command) + + if command_result != 0: + raise Exception("Command failed with exit code %d" % command_result) + +def run_command_in_host(cmd): + print("Issue command in host: %s" % str(cmd)) + + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + o, e = proc.communicate() + print('Output: ' + o.decode('ascii')) + print('Error: ' + e.decode('ascii')) + print('code: ' + str(proc.returncode)) diff --git a/.travis/package_management/configure_deb_lxc_environment.py b/.travis/package_management/configure_deb_lxc_environment.py new file mode 100755 index 0000000000..58999ad38b --- /dev/null +++ b/.travis/package_management/configure_deb_lxc_environment.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python3 +# +# Prepare the build environment within the container +# The script attaches to the running container and does the following: +# 1) Create the container +# 2) Start the container up +# 3) Create the builder user +# 4) Prepare the environment for DEB build +# +# Copyright: SPDX-License-Identifier: GPL-3.0-or-later +# +# Author : Pavlos Emm. Katsoulakis <paul@netdata.cloud> + +import common +import os +import sys +import lxc + +if len(sys.argv) != 2: + print('You need to provide a container name to get things started') + sys.exit(1) +container_name=sys.argv[1] + +# Setup the container object +print("Defining container %s" % container_name) +container = lxc.Container(container_name) +if not container.defined: + raise Exception("Container %s not defined!" % container_name) + +# Start the container +if not container.start(): + raise Exception("Failed to start the container") + +if not container.running or not container.state == "RUNNING": + raise Exception('Container %s is not running, configuration process aborted ' % container_name) + +# Wait for connectivity +print("Waiting for container connectivity to start configuration sequence") +if not container.get_ips(timeout=30): + raise Exception("Timeout while waiting for container") + +# Run the required activities now +# 1. Create the builder user +print("1. Adding user %s" % os.environ['BUILDER_NAME']) +common.run_command(container, ["useradd", "-m", os.environ['BUILDER_NAME']]) + +# Fetch package dependencies for the build +print("2. Installing package dependencies within LXC container") +common.run_command(container, ["apt-get", "update", "-y"]) +common.run_command(container, ["apt-get", "install", "-y", "sudo"]) +common.run_command(container, ["apt-get", "install", "-y", "wget"]) +common.run_command(container, ["apt-get", "install", "-y", "bash"]) +common.run_command(container, ["wget", "-T", "15", "-O", "~/.install-required-packages.sh", "https://raw.githubusercontent.com/netdata/netdata-demo-site/master/install-required-packages.sh"]) +common.run_command(container, ["bash", "~/.install-required-packages.sh", "netdata", "--dont-wait", "--non-interactive"]) + +# Download the source +dest_archive="/home/%s/netdata-%s.tar.gz" % (os.environ['BUILDER_NAME'],os.environ['BUILD_VERSION']) +release_url="https://github.com/netdata/netdata/releases/download/%s/netdata-%s.tar.gz" % (os.environ['BUILD_VERSION'], os.environ['BUILD_VERSION']) +print("3. Fetch netdata source (%s -> %s)" % (release_url, dest_archive)) +common.run_command(container, ["sudo", "-u", os.environ['BUILDER_NAME'], "wget", "-T", "15", "--output-document=" + dest_archive, release_url]) + +print("4. Extracting directory contents to /home " + os.environ['BUILDER_NAME']) +common.run_command(container, ["sudo", "-u", os.environ['BUILDER_NAME'], "tar", "xf", dest_archive, "-C", "/home/" + os.environ['BUILDER_NAME']]) + +print("Done!") diff --git a/.travis/package_management/configure_rpm_lxc_environment.py b/.travis/package_management/configure_rpm_lxc_environment.py new file mode 100755 index 0000000000..644e027b7b --- /dev/null +++ b/.travis/package_management/configure_rpm_lxc_environment.py @@ -0,0 +1,159 @@ +#!/usr/bin/env python3 +# +# +# Prepare the build environment within the container +# The script attaches to the running container and does the following: +# 1) Create the container +# 2) Start the container up +# 3) Create the builder user +# 4) Prepare the environment for RPM build +# +# Copyright: SPDX-License-Identifier: GPL-3.0-or-later +# +# Author : Pavlos Emm. Katsoulakis <paul@netdata.cloud> + +import common +import os +import sys +import lxc + +if len(sys.argv) != 2: + print('You need to provide a container name to get things started') + sys.exit(1) +container_name=sys.argv[1] + +# Setup the container object +print("Defining container %s" % container_name) +container = lxc.Container(container_name) +if not container.defined: + raise Exception("Container %s not defined!" % container_name) + +# Start the container +if not container.start(): + raise Exception("Failed to start the container") + +if not container.running or not container.state == "RUNNING": + raise Exception('Container %s is not running, configuration process aborted ' % container_name) + +# Wait for connectivity +print("Waiting for container connectivity to start configuration sequence") +if not container.get_ips(timeout=30): + raise Exception("Timeout while waiting for container") + +# Run the required activities now +# Create the builder user +print("1. Adding user %s" % os.environ['BUILDER_NAME']) +common.run_command(container, ["useradd", "-m", os.environ['BUILDER_NAME']]) + +# Fetch package dependencies for the build +print("2. Installing package dependencies within LXC container") +if str(os.environ["REPO_TOOL"]).count("zypper") == 1: + common.run_command(container, [os.environ["REPO_TOOL"], "clean", "-a"]) + common.run_command(container, [os.environ["REPO_TOOL"], "--no-gpg-checks", "update", "-y"]) + common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "json-glib-devel"]) + +elif str(os.environ["REPO_TOOL"]).count("yum") == 1: + common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "json-c-devel"]) + common.run_command(container, [os.environ["REPO_TOOL"], "clean", "all"]) + common.run_command(container, [os.environ["REPO_TOOL"], "update", "-y"]) + common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "epel-release"]) +else: + common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "json-c-devel"]) + common.run_command(container, [os.environ["REPO_TOOL"], "update", "-y"]) + +common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "sudo"]) +common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "wget"]) +common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "bash"]) +common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "freeipmi-devel"]) +common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "cups-devel"]) + +# Exceptional cases, not available everywhere +# + +# Not on Centos-7 +if os.environ["BUILD_STRING"].count("el/7") <= 0: + common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "libnetfilter_acct-devel"]) + +# Not on Centos-6 +if os.environ["BUILD_STRING"].count("el/6") <= 0: + common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "autoconf-archive"]) + +common.run_command(container, ["wget", "-T", "15", "-O", "/home/%s/.install-required-packages.sh" % (os.environ['BUILDER_NAME']), "https://raw.githubusercontent.com/netdata/netdata-demo-site/master/install-required-packages.sh"]) +common.run_command(container, ["bash", "/home/%s/.install-required-packages.sh" % (os.environ['BUILDER_NAME']), "netdata", "--dont-wait", "--non-interactive"]) + +print("3. Setting up macros") +common.run_command(container, ["sudo", "-u", os.environ['BUILDER_NAME'], "/bin/echo", "'%_topdir %(echo /home/" + os.environ['BUILDER_NAME'] + ")/rpmbuild' > /home/" + os.environ['BUILDER_NAME'] + "/.rpmmacros"]) + +print("4. Create rpmbuild directory") +common.run_command(container, ["sudo", "-u", os.environ['BUILDER_NAME'], "mkdir", "-p", "/home/" + os.environ['BUILDER_NAME'] + "/rpmbuild/BUILD"]) +common.run_command(container, ["sudo", "-u", os.environ['BUILDER_NAME'], "mkdir", "-p", "/home/" + os.environ['BUILDER_NAME'] + "/rpmbuild/RPMS"]) +common.run_command(container, ["sudo", "-u", os.environ['BUILDER_NAME'], "mkdir", "-p", "/home/" + os.environ['BUILDER_NAME'] + "/rpmbuild/SOURCES"]) +common.run_command(container, ["sudo", "-u", os.environ['BUILDER_NAME'], "mkdir", "-p", "/home/" + os.environ['BUILDER_NAME'] + "/rpmbuild/SPECS"]) +common.run_command(container, ["sudo", "-u", os.environ['BUILDER_NAME'], "mkdir", "-p", "/home/" + os.environ['BUILDER_NAME'] + "/rpmbuild/SRPMS"]) +common.run_command(container, ["sudo", "-u", os.environ['BUILDER_NAME'], "ls", "-ltrR", "/home/" + os.environ['BUILDER_NAME'] + "/rpmbuild"]) + +# Download the source +rpm_friendly_version="" +dest_archive="" +download_url="" +spec_file="/home/%s/rpmbuild/SPECS/netdata.spec" % os.environ['BUILDER_NAME'] + +# TODO: Checksum validations +if str(os.environ['BUILD_VERSION']).count(".latest") == 1: + version_list=str(os.environ['BUILD_VERSION']).replace('v', '').split('.') + rpm_friendly_version='.'.join(version_list[0:3]) + "." + version_list[3] + + print("Building latest nightly version of netdata..(%s)" % os.environ['BUILD_VERSION']) + dest_archive="/home/%s/rpmbuild/SOURCES/netdata-%s.tar.gz" % (os.environ['BUILDER_NAME'], rpm_friendly_version) + + print("5. Preparing local latest implementation tarball for version %s" % rpm_friendly_version) + tar_file = os.environ['LXC_CONTAINER_ROOT'] + dest_archive + + print("5.1 Tagging the code with latest version: %s" % rpm_friendly_version) + common.run_command_in_host(['git', 'tag', '-a', rpm_friendly_version, '-m', 'Tagging while packaging on %s' % os.environ["CONTAINER_NAME"]]) + + print("5.2 Run autoreconf -ivf") + common.run_command_in_host(['autoreconf', '-ivf']) + + print("5.3 Run configure") + common.run_command_in_host(['./configure', '--with-math', '--with-zlib', '--with-user=netdata']) + + print("5.4 Run make dist") + common.run_command_in_host(['make', 'dist']) + + print("5.5 Copy generated tarbal to desired path") + if os.path.exists('netdata-%s.tar.gz' % rpm_friendly_version): + common.run_command_in_host(['sudo', 'cp', 'netdata-%s.tar.gz' % rpm_friendly_version, tar_file]) + + print("5.6 Fixing permissions on tarball") + common.run_command_in_host(['sudo', 'chmod', '777', tar_file]) + else: + print("I could not find (%s) on the disk, stopping the build. Kindly check the logs and try again" % 'netdata-%s.tar.gz' % rpm_friendly_version) + sys.exit(1) + + # Extract the spec file in place + print("6. Extract spec file from the source") + common.run_command_in_host(['sudo', 'cp', 'netdata.spec', os.environ['LXC_CONTAINER_ROOT'] + spec_file]) + common.run_command_in_host(['sudo', 'chmod', '777', os.environ['LXC_CONTAINER_ROOT'] + spec_file]) + + print("7. Temporary hack: Change Source0 to %s on spec file %s" % (dest_archive, spec_file)) + common.replace_tag("Source0", os.environ['LXC_CONTAINER_ROOT'] + spec_file, tar_file) +else: + rpm_friendly_version = os.environ['BUILD_VERSION'] + + print("Building latest stable version of netdata.. (%s)" % os.environ['BUILD_VERSION']) + dest_archive="/home/%s/rpmbuild/SOURCES/netdata-%s.tar.gz" % (os.environ['BUILDER_NAME'],os.environ['BUILD_VERSION']) + download_url="https://github.com/netdata/netdata/releases/download/%s/netdata-%s.tar.gz" % (os.environ['BUILD_VERSION'], os.environ['BUILD_VERSION']) + + print("5. Fetch netdata source into the repo structure(%s -> %s)" % (download_url, dest_archive)) + tar_file="%s/netdata-%s.tar.gz" % (os.path.dirname(dest_archive), rpm_friendly_version) + common.run_command(container, ["sudo", "-u", os.environ['BUILDER_NAME'], "wget", "-T", "15", "--output-document=" + dest_archive, download_url]) + + print("6.Extract spec file from the source") + common.run_command(container, ["sudo", "-u", os.environ['BUILDER_NAME'], "tar", "--to-command=cat > %s" % spec_file, "-xvf", dest_archive, "netdata-%s/netdata.spec" % os.environ['BUILD_VERSION']]) + + print("7. Temporary hack: Adjust version string on the spec file (%s) to %s and Source0 to %s" % (os.environ['LXC_CONTAINER_ROOT'] + spec_file, rpm_friendly_version, download_url)) + common.replace_tag("Version", os.environ['LXC_CONTAINER_ROOT'] + spec_file, rpm_friendly_version) + common.replace_tag("Source0", os.environ['LXC_CONTAINER_ROOT'] + spec_file, tar_file) + +print('Done!') diff --git a/.travis/package_management/create_lxc_for_build.sh b/.travis/package_management/create_lxc_for_build.sh index 83ef9d1fc1..ae855a7420 100755 --- a/.travis/package_management/create_lxc_for_build.sh +++ b/.travis/package_management/create_lxc_for_build.sh @@ -62,7 +62,7 @@ case "${BUILD_ARCH}" in lxc-create -n "${CONTAINER_NAME}" -t "download" -- --dist "${BUILD_DISTRO}" --release "${BUILD_RELEASE}" --arch "i386" --no-validate echo "Container(s) ready. Configuring container(s).." - .travis/package_management/"${PACKAGE_TYPE}"/configure_lxc_environment.py "${CONTAINER_NAME}" + .travis/package_management/configure_${PACKAGE_TYPE}_lxc_environment.py "${CONTAINER_NAME}" # amd64 echo "Creating LXC Container for amd64.." @@ -71,7 +71,7 @@ case "${BUILD_ARCH}" in lxc-create -n "${CONTAINER_NAME}" -t "download" -- --dist "${BUILD_DISTRO}" --release "${BUILD_RELEASE}" --arch "amd64" --no-validate echo "Container(s) ready. Configuring container(s).." - .travis/package_management/"${PACKAGE_TYPE}"/configure_lxc_environment.py "${CONTAINER_NAME}" + .travis/package_management/configure_${PACKAGE_TYPE}_lxc_environment.py "${CONTAINER_NAME}" # arm64 echo "Creating LXC Container for arm64.." @@ -80,7 +80,7 @@ case "${BUILD_ARCH}" in lxc-create -n "${CONTAINER_NAME}" -t "download" -- --dist "${BUILD_DISTRO}" --release "${BUILD_RELEASE}" --arch "arm64" --no-validate echo "Container(s) ready. Configuring container(s).." - .travis/package_management/"${PACKAGE_TYPE}"/configure_lxc_environment.py "${CONTAINER_NAME}" + .travis/package_management/configure_${PACKAGE_TYPE}_lxc_environment.py "${CONTAINER_NAME}" ;; "i386"|"amd64"|"arm64") # AMD64 or i386 @@ -90,7 +90,7 @@ case "${BUILD_ARCH}" in lxc-create -n "${CONTAINER_NAME}" -t "download" -- --dist "${BUILD_DISTRO}" --release "${BUILD_RELEASE}" --arch "${BUILD_ARCH}" --no-validate echo "Container(s) ready. Configuring container(s).." - .travis/package_management/"${PACKAGE_TYPE}"/configure_lxc_environment.py "${CONTAINER_NAME}" + .travis/package_management/configure_${PACKAGE_TYPE}_lxc_environment.py "${CONTAINER_NAME}" ;; *) echo "Unknown BUILD_ARCH value '${BUILD_ARCH}' given, process failed" diff --git a/.travis/package_management/deb/configure_lxc_environment.py b/.travis/package_management/deb/configure_lxc_environment.py deleted file mode 100755 index 496cdf5aa5..0000000000 --- a/.travis/package_management/deb/configure_lxc_environment.py +++ /dev/null @@ -1,71 +0,0 @@ -#!/usr/bin/env python3 -# -# Prepare the build environment within the container -# The script attaches to the running container and does the following: -# 1) Create the container -# 2) Start the container up -# 3) Create the builder user -# 4) Prepare the environment for DEB build -# -# Copyright: SPDX-License-Identifier: GPL-3.0-or-later -# -# Author : Pavlos Emm. Katsoulakis <paul@netdata.cloud> - -import os -import sys -import lxc - -def run_command(command): - print ("Running command: %s" % command) - command_result = container.attach_wait(lxc.attach_run_command, command) - - if command_result != 0: - raise Exception("Command failed with exit code %d" % command_result) - -if len(sys.argv) != 2: - print ('You need to provide a container name to get things started') - sys.exit(1) -container_name=sys.argv[1] - -# Setup the container object -print ("Defining container %s" % container_name) -container = lxc.Container(container_name) -if not container.defined: - raise Exception("Container %s not defined!" % container_name) - -# Start the container -if not container.start(): - raise Exception("Failed to start the container") - -if not container.running or not container.state == "RUNNING": - raise Exception('Container %s is not running, configuration process aborted ' % container_name) - -# Wait for connectivity -print ("Waiting for container connectivity to start configuration sequence") -if not container.get_ips(timeout=30): - raise Exception("Timeout while waiting for container") - -# Run the required activities now -# 1. Create the builder user -print ("1. Adding user %s" % os.environ['BUILDER_NAME']) -run_command(["useradd", "-m", os.environ['BUILDER_NAME']]) - -# Fetch package dependencies for the build -print ("2. Installing package dependencies within LXC container") -run_command(["apt-get", "update", "-y"]) -run_command(["apt-get", "install", "-y", "sudo"]) -run_command(["apt-get", "install", "-y", "wget"]) -run_command(["apt-get", "install", "-y", "bash"]) -run_command(["wget", "-T", "15", "-O", "~/.install-required-packages.sh", "https://raw.githubusercontent.com/netdata/netdata-demo-site/master/install-required-packages.sh"]) -run_command(["bash", "~/.install-required-packages.sh", "netdata", "--dont-wait", "--non-interactive"]) - -# Download the source -dest_archive="/home/%s/netdata-%s.tar.gz" % (os.environ['BUILDER_NAME'],os.environ['BUILD_VERSION']) -release_url="https://github.com/netdata/netdata/releases/download/%s/netdata-%s.tar.gz" % (os.environ['BUILD_VERSION'], os.environ['BUILD_VERSION']) -print ("3. Fetch netdata source (%s -> %s)" % (release_url, dest_archive)) -run_command(["sudo", "-u", os.environ['BUILDER_NAME'], "wget", "-T", "15", "--output-document=" + dest_archive, release_url]) - -print ("4. Extracting directory contents to /home " + os.environ['BUILDER_NAME']) -run_command(["sudo", "-u", os.environ['BUILDER_NAME'], "tar", "xf", dest_archive, "-C", "/home/" + os.environ['BUILDER_NAME']]) - -print ('Done!') diff --git a/.travis/package_management/rpm/configure_lxc_environment.py b/.travis/package_management/rpm/configure_lxc_environment.py deleted file mode 100755 index bed75d7489..0000000000 --- a/.travis/package_management/rpm/configure_lxc_environment.py +++ /dev/null @@ -1,89 +0,0 @@ -#!/usr/bin/env python3 -# -# Prepare the build environment within the container -# The script attaches to the running container and does the following: -# 1) Create the container -# 2) Start the container up -# 3) Create the builder user -# 4) Prepare the environment for RPM build -# -# Copyright: SPDX-License-Identifier: GPL-3.0-or-later -# -# Author : Pavlos Emm. Katsoulakis <paul@netdata.cloud> - -import os -import sys -import lxc - -def run_command(command): - print ("Running command: %s" % command) - command_result = container.attach_wait(lxc.attach_run_command, command) - - if command_result != 0: - raise Exception("Command failed with exit code %d" % command_result) - -if len(sys.argv) != 2: - print ('You need to provide a container name to get things started') - sys.exit(1) -container_name=sys.argv[1] - -# Setup the container object -print ("Defining container %s" % container_name) -container = lxc.Container(container_name) -if not container.defined: - raise Exception("Container %s not defined!" % container_name) - -# Start the container -if not container.start(): - raise Exception("Failed to start the container") - -if not container.running or not container.state == "RUNNING": - raise Exception('Container %s is not running, configuration process aborted ' % container_name) - -# Wait for connectivity -print ("Waiting for container connectivity to start configuration sequence") -if not container.get_ips(timeout=30): - raise Exception("Timeout while waiting for container") - -# Run the required activities now -# Create the builder user -print ("1. Adding user %s" % os.environ['BUILDER_NAME']) -run_command(["useradd", "-m", os.environ['BUILDER_NAME']]) - -# Fetch package dependencies for the build -print ("2. Installing package dependencies within LXC container") -if str(os.environ["REPO_TOOL"]).count("zypper") == 1: - run_command([os.environ["REPO_TOOL"], "clean", "-a"]) - run_command([os.environ["REPO_TOOL"], "--no-gpg-checks", "update", "-y"]) -else: - run_command([os.environ["REPO_TOOL"], "update", "-y"]) - -run_command([os.environ["REPO_TOOL"], "install", "-y", "sudo"]) -run_command([os.environ["REPO_TOOL"], "install", "-y", "wget"]) -run_command([os.environ["REPO_TOOL"], "install", "-y", "bash"]) -run_command(["wget", "-T", "15", "-O", "~/.install-required-packages.sh", "https://raw.githubusercontent.com/netdata/netdata-demo-site/master/install-required-packages.sh"]) -run_command(["bash", "~/.install-required-packages.sh", "netdata", "--dont-wait", "--non-interactive"]) - -print ("3. Setting up macros") -run_command(["sudo", "-u", os.environ['BUILDER_NAME'], "/bin/echo", "'%_topdir %(echo /home/" + os.environ['BUILDER_NAME'] + ")/rpmbuild' > /home/" + os.environ['BUILDER_NAME'] + "/.rpmmacros"]) - -print ("4. Create rpmbuild directory") -run_command(["sudo", "-u", os.environ['BUILDER_NAME'], "mkdir", "-p", "/home/" + os.environ['BUILDER_NAME'] + "/rpmbuild/BUILD"]) -run_command(["sudo", "-u", os.environ['BUILDER_NAME'], "mkdir", "-p", "/home/" + os.environ['BUILDER_NAME'] + "/rpmbuild/RPMS"]) -run_command(["sudo", "-u", os.environ['BUILDER_NAME'], "mkdir", "-p", "/home/" + os.environ['BUILDER_NAME'] + "/rpmbuild/SOURCES"]) -run_command(["sudo", "-u", os.environ['BUILDER_NAME'], "mkdir", "-p", "/home/" + os.environ['BUILDER_NAME'] + "/rpmbuild/SPECS"]) -run_command(["sudo", "-u", os.environ['BUILDER_NAME'], "mkdir", "-p", "/home/" + os.environ['BUILDER_NAME'] + "/rpmbuild/SRPMS"]) -run_command(["sudo", "-u", os.environ['BUILDER_NAME'], "ls", "-ltrR", "/home/" + os.environ['BUILDER_NAME'] + "/rpmbuild"]) - -# Download the source -dest_archive="/home/%s/rpmbuild/SOURCES/netdata-%s.tar.gz" % (os.environ['BUILDER_NAME'],os.environ['BUILD_VERSION']) -release_url="https://github.com/netdata/netdata/releases/download/%s/netdata-%s.tar.gz" % (os.environ['BUILD_VERSION'], os.environ['BUILD_VERSION']) -print ("5. Fetch netdata source into the repo structure(%s -> %s)" % (release_url, dest_archive)) -run_command(["sudo", "-u", os.environ['BUILDER_NAME'], "wget", "-T", "15", "--output-document=" + dest_archive, release_url]) - -# Extract the spec file in place -print ("6. Extract spec file from the source") -spec_file="/home/%s/rpmbuild/SPECS/netdata.spec" % os.environ['BUILDER_NAME'] -run_command(["sudo", "-u", os.environ['BUILDER_NAME'], "tar", "--to-command=cat > %s" % spec_file, "-xvf", dest_archive, "netdata-%s/netdata.spec" % os.environ['BUILD_VERSION']]) - -print ('Done!') diff --git a/.travis/package_management/rpm/trigger_lxc_build.py b/.travis/package_management/rpm/trigger_lxc_build.py deleted file mode 100755 index 108e54fe9b..0000000000 --- a/.travis/package_management/rpm/trigger_lxc_build.py +++ /dev/null @@ -1,62 +0,0 @@ -#!/usr/bin/env python3 -# -# This script is responsible for running the RPM build on the running container -# -# Copyright: SPDX-License-Identifier: GPL-3.0-or-later -# -# Author : Pavlos Emm. Katsoulakis <paul@netdata.cloud> - -import os -import sys -import lxc - -def run_command(command): - print ("Running command: %s" % command) - command_result = container.attach_wait(lxc.attach_run_command, command) - - if command_result != 0: - raise Exception("Command failed with exit code %d" % command_result) - -print (sys.argv) -if len(sys.argv) != 2: - print ('You need to provide a container name to get things started') - sys.exit(1) -container_name=sys.argv[1] - -# Load the container, break if its not there -print ("Starting up container %s" % container_name) -container = lxc.Container(container_name) -if not container.defined: - raise Exception("Container %s does not exist!" % container_name) - - |