path: root/Vagrantfile

# -*- mode: ruby -*-
# vi: set ft=ruby :

# Automated creation of testing environments / binaries on misc. platforms

$cpus = Integer(ENV.fetch('VMCPUS', '4'))  # create VMs with that many cpus
$xdistn = Integer(ENV.fetch('XDISTN', '4'))  # dispatch tests to that many pytest workers
$wmem = $xdistn * 256  # give the VM additional memory for workers [MB]

def packages_prepare_wheezy
  return <<-EOF
      # debian 7 wheezy does not have lz4, but it is available from wheezy-backports:
      echo "deb http://http.debian.net/debian wheezy-backports main" > /etc/apt/sources.list.d/wheezy-backports.list
  EOF
end

def packages_debianoid(user)
  return <<-EOF
    apt-get update
    # install all the (security and other) updates
    apt-get dist-upgrade -y
    # for building borgbackup and dependencies:
    apt-get install -y libssl-dev libacl1-dev liblz4-dev libfuse-dev fuse pkg-config
    usermod -a -G fuse #{user}
    chgrp fuse /dev/fuse
    chmod 666 /dev/fuse
    apt-get install -y fakeroot build-essential git
    apt-get install -y python3-dev python3-setuptools
    # for building python:
    apt-get install -y zlib1g-dev libbz2-dev libncurses5-dev libreadline-dev liblzma-dev libsqlite3-dev
    # this way it works on older dists (like ubuntu 12.04) also:
    # for python 3.2 on ubuntu 12.04 we need pip<8 and virtualenv<14 as
    # newer versions are not compatible with py 3.2 any more.
    easy_install3 -i https://pypi.python.org/simple/ 'pip<8.0'
    pip3 install 'virtualenv<14.0'
  EOF
end

def packages_redhatted
  return <<-EOF
    yum install -y epel-release
    yum update -y
    # for building borgbackup and dependencies:
    yum install -y openssl-devel openssl libacl-devel libacl lz4-devel fuse-devel fuse pkgconfig
    usermod -a -G fuse vagrant
    chgrp fuse /dev/fuse
    chmod 666 /dev/fuse
    yum install -y fakeroot gcc git patch
    # needed to compile msgpack-python (otherwise it will use slow fallback code):
    yum install -y gcc-c++
    # for building python:
    yum install -y zlib-devel bzip2-devel ncurses-devel readline-devel xz xz-devel sqlite-devel
    #yum install -y python-pip
    #pip install virtualenv
  EOF
end

def packages_darwin
  return <<-EOF
    # install all the (security and other) updates
    sudo softwareupdate --ignore iTunesX
    sudo softwareupdate --ignore iTunes
    sudo softwareupdate --install --all
    # get osxfuse 3.x release code from github:
    curl -s -L https://github.com/osxfuse/osxfuse/releases/download/osxfuse-3.7.1/osxfuse-3.7.1.dmg >osxfuse.dmg
    MOUNTDIR=$(echo `hdiutil mount osxfuse.dmg | tail -1 | awk '{$1="" ; print $0}'` | xargs -0 echo) \
    && sudo installer -pkg "${MOUNTDIR}/Extras/FUSE for macOS 3.7.1.pkg" -target /
    sudo chown -R vagrant /usr/local  # brew must be able to create stuff here
    ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)"
    brew update
    brew install openssl
    brew install lz4
    brew install xz  # required for python lzma module
    brew install fakeroot
    brew install git
    brew install pkg-config
  EOF
end

def packages_freebsd
  return <<-EOF
    # VM has no hostname set
    hostname freebsd
    # install all the (security and other) updates, base system
    freebsd-update --not-running-from-cron fetch install
    # for building borgbackup and dependencies:
    pkg install -y openssl liblz4 fusefs-libs pkgconf
    pkg install -y git bash
    # for building python:
    pkg install -y sqlite3
    # make bash default / work:
    chsh -s bash vagrant
    mount -t fdescfs fdesc /dev/fd
    echo 'fdesc	/dev/fd		fdescfs		rw	0	0' >> /etc/fstab
    # make FUSE work
    echo 'fuse_load="YES"' >> /boot/loader.conf
    echo 'vfs.usermount=1' >> /etc/sysctl.conf
    kldload fuse
    sysctl vfs.usermount=1
    pw groupmod operator -M vagrant
    # /dev/fuse has group operator
    chmod 666 /dev/fuse
    # install all the (security and other) updates, packages
    pkg update
    yes | pkg upgrade
  EOF
end

def packages_openbsd
  return <<-EOF
    echo 'installpath = http://ftp.hostserver.de/pub/OpenBSD/6.0/packages/amd64/' > /etc/pkg.conf
    echo 'export PKG_PATH=http://ftp.hostserver.de/pub/OpenBSD/6.0/packages/amd64/' >> ~/.profile
    . ~/.profile
    pkg_add bash
    chsh -s /usr/local/bin/bash vagrant
    pkg_add openssl
    pkg_add lz4
    pkg_add git  # no fakeroot
    pkg_add py3-setuptools
    ln -sf /usr/local/bin/python3.4 /usr/local/bin/python3
    ln -sf /usr/local/bin/python3.4 /usr/local/bin/python
    easy_install-3.4 pip
    pip3 install virtualenv
  EOF
end

def packages_netbsd
  return <<-EOF
    hostname netbsd  # the box we use has an invalid hostname
    PKG_PATH="ftp://ftp.NetBSD.org/pub/pkgsrc/packages/NetBSD/amd64/7.0.1/All"
    export PKG_PATH
    pkg_add mozilla-rootcerts lz4 git
    chsh -s bash vagrant
    mkdir -p /usr/local/opt/lz4/include
    mkdir -p /usr/local/opt/lz4/lib
    ln -s /usr/pkg/include/lz4*.h /usr/local/opt/lz4/include/
    ln -s /usr/pkg/lib/liblz4* /usr/local/opt/lz4/lib/
    touch /etc/openssl/openssl.cnf  # avoids a flood of "can't open ..."
    mozilla-rootcerts install
    pkg_add pkg-config  # avoids some "pkg-config missing" error msg, even without fuse pkg
    # pkg_add fuse  # llfuse supports netbsd, but is still buggy.
    # https://bitbucket.org/nikratio/python-llfuse/issues/70/perfuse_open-setsockopt-no-buffer-space
    pkg_add python34 py34-setuptools
    ln -s /usr/pkg/bin/python3.4 /usr/pkg/bin/python
    ln -s /usr/pkg/bin/python3.4 /usr/pkg/bin/python3
    easy_install-3.4 pip
    pip install virtualenv
  EOF
end

def packages_openindiana
  return <<-EOF
    #pkg update  # XXX needs separate provisioning step + reboot
    pkg install python-34 clang-40 lz4 git
    python3 -m ensurepip
    pip3 install -U setuptools pip wheel virtualenv
  EOF
end

# Install required cygwin packages and configure environment
#
# Microsoft/EdgeOnWindows10 image has MLS-OpenSSH installed by default,
# which is based on cygwin x86_64 but should not be used together with cygwin.
# In order to have have cygwin compatible bash 'ImagePath' is replaced with
# cygrunsrv of newly installed cygwin
#
# supported cygwin versions:
#   x86_64
#   x86
def packages_cygwin(version)
  setup_exe = "setup-#{version}.exe"

  return <<-EOF
    mkdir -p /cygdrive/c/cygwin
    powershell -Command '$client = new-object System.Net.WebClient; $client.DownloadFile("https://www.cygwin.com/#{setup_exe}","C:\\cygwin\\#{setup_exe}")'
    echo '
    REM --- Change to use different CygWin platform and final install path
    set CYGSETUP=#{setup_exe}
    REM --- Install build version of CygWin in a subfolder
    set OURPATH=%cd%
    set CYGBUILD="C:\\cygwin\\CygWin"
    set CYGMIRROR=http://mirrors.kernel.org/sourceware/cygwin/
    set BASEPKGS=openssh,rsync
    set BUILDPKGS=python3,python3-setuptools,python-devel,binutils,gcc-g++,libopenssl,openssl-devel,git,make,liblz4-devel,liblz4_1,curl
    %CYGSETUP% -q -B -o -n -R %CYGBUILD% -L -D -s %CYGMIRROR% -P %BASEPKGS%,%BUILDPKGS%
    cd /d C:\\cygwin\\CygWin\\bin
    regtool set /HKLM/SYSTEM/CurrentControlSet/Services/OpenSSHd/ImagePath "C:\\cygwin\\CygWin\\bin\\cygrunsrv.exe"
    bash -c "ssh-host-config --no"
    bash -c "chown sshd_server /cygdrive/c/cygwin/CygWin/var/empty"
    ' > /cygdrive/c/cygwin/install.bat

    echo "alias mkdir='mkdir -p'" > ~/.profile
    echo "export CYGWIN_ROOT=/cygdrive/c/cygwin/CygWin" >> ~/.profile
    echo 'export PATH=$CYGWIN_ROOT/bin:$PATH' >> ~/.profile

    echo '' > ~/.bash_profile

    cmd.exe /c 'setx /m PATH "C:\\cygwin\\CygWin\\bin;%PATH%"'
    source ~/.profile
    cd /cygdrive/c/cygwin && cmd.exe /c install.bat

    echo 'db_home: windows' > $CYGWIN_ROOT/etc/nsswitch.conf
  EOF
end

def install_cygwin_venv
  return <<-EOF
      python3 -m ensurepip -U --default-pip
      pip install virtualenv
  EOF
end

def install_pyenv(boxname)
  script = <<-EOF
    curl -s -L https://raw.githubusercontent.com/yyuu/pyenv-installer/master/bin/pyenv-installer | bash
    echo 'export PATH="$HOME/.pyenv/bin:$PATH"' >> ~/.bash_profile
    echo 'eval "$(pyenv init -)"' >> ~/.bash_profile
    echo 'eval "$(pyenv virtualenv-init -)"' >> ~/.bash_profile
    echo 'export PYTHON_CONFIGURE_OPTS="--enable-shared"' >> ~/.bash_profile
  EOF
  return script
end

def fix_pyenv_darwin(boxname)
  return <<-EOF
    echo 'export PYTHON_CONFIGURE_OPTS="--enable-framework"' >> ~/.bash_profile
  EOF
end

def install_pythons(boxname)
  return <<-EOF
    . ~/.bash_profile
    pyenv install 3.4.0  # tests
    pyenv install 3.5.0  # tests
    pyenv install 3.6.0  # tests
    pyenv install 3.5.4  # binary build, use latest 3.5.x release
    pyenv rehash
  EOF
end

def build_sys_venv(boxname)
  return <<-EOF
    . ~/.bash_profile
    cd /vagrant/borg
    virtualenv --python=python3 borg-env
  EOF
end

def build_pyenv_venv(boxname)
  return <<-EOF
    . ~/.bash_profile
    cd /vagrant/borg
    # use the latest 3.5 release
    pyenv global 3.5.4
    pyenv virtualenv 3.5.4 borg-env
    ln -s ~/.pyenv/versions/borg-env .
  EOF
end

def install_borg(fuse)
  script = <<-EOF
    . ~/.bash_profile
    cd /vagrant/borg
    . borg-env/bin/activate
    pip install -U wheel  # upgrade wheel, too old for 3.5
    cd borg
    # clean up (wrong/outdated) stuff we likely got via rsync:
    rm -rf __pycache__
    find src -name '__pycache__' -exec rm -rf {} \\;
    pip install -r requirements.d/development.txt
    python setup.py clean
  EOF
  if fuse
    script += <<-EOF
      # by using [fuse], setup.py can handle different FUSE requirements:
      pip install -e .[fuse]
    EOF
  else
    script += <<-EOF
      pip install -e .
      # do not install llfuse into the virtualenvs built by tox:
      sed -i.bak '/fuse.txt/d' tox.ini
    EOF
  end
  return script
end

def install_pyinstaller()
  return <<-EOF
    . ~/.bash_profile
    cd /vagrant/borg
    . borg-env/bin/activate
    git clone https://github.com/thomaswaldmann/pyinstaller.git
    cd pyinstaller
    git checkout v3.2.1
    python setup.py install
  EOF
end

def build_binary_with_pyinstaller(boxname)
  return <<-EOF
    . ~/.bash_profile
    cd /vagrant/borg
    . borg-env/bin/activate
    cd borg
    pyinstaller --clean --distpath=/vagrant/borg scripts/borg.exe.spec
    echo 'export PATH="/vagrant/borg:$PATH"' >> ~/.bash_profile
  EOF
end

def run_tests(boxname)
  return <<-EOF
    . ~/.bash_profile
    cd /vagrant/borg/borg
    . ../borg-env/bin/activate
    if which pyenv 2> /dev/null; then
      # for testing, use the earliest point releases of the supported python versions:
      pyenv global 3.4.0 3.5.0 3.6.0
      pyenv local 3.4.0 3.5.0 3.6.0
    fi
    # otherwise: just use the system python
    if which fakeroot 2> /dev/null; then
      echo "Running tox WITH fakeroot -u"
      fakeroot -u tox --skip-missing-interpreters
    else
      echo "Running tox WITHOUT fakeroot -u"
      tox --skip-missing-interpreters
    fi
  EOF
end

def fs_init(user)
  return <<-EOF
    chown -R #{user} /vagrant/borg
    touch ~#{user}/.bash_profile ; chown #{user} ~#{user}/.bash_profile
    echo 'export LANG=en_US.UTF-8' >> ~#{user}/.bash_profile
    echo 'export LC_CTYPE=en_US.UTF-8' >> ~#{user}/.bash_profile
    echo 'export XDISTN=#{$xdistn}' >> ~#{user}/.bash_profile
  EOF
end

Vagrant.configure(2) do |config|
  # use rsync to copy content to the folder
  config.vm.synced_folder ".", "/vagrant/borg/borg", :type => "rsync", :rsync__args => ["--verbose", "--archive", "--delete", "-z"], :rsync__chown => false
  # do not let the VM access . on the host machine via the default shared folder!
  config.vm.synced_folder ".", "/vagrant", disabled: true

  config.vm.provider :virtualbox do |v|
    #v.gui = true
    v.cpus = $cpus
  end

  # Linux
  config.vm.define "centos7_64" do |b|
    b.vm.box = "centos/7"
    b.vm.provider :virtualbox do |v|
      v.memory = 1024 + $wmem
    end
    b.vm.provision "fs init", :type => :shell, :inline => fs_init("vagrant")
    b.vm.provision "install system packages", :type => :shell, :inline => packages_redhatted
    b.vm.provision "install pyenv", :type => :shell, :privileged => false, :inline => install_pyenv("centos7_64")
    b.vm.provision "install pythons", :type => :shell, :privileged => false, :inline => install_pythons("centos7_64")
    b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_pyenv_venv("centos7_64")
    b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg(true)
    b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("centos7_64")
  end

  config.vm.define "centos6_32" do |b|
    b.vm.box = "centos6-32"
    b.vm.provider :virtualbox do |v|
      v.memory = 768 + $wmem
    end
    b.vm.provision "fs init", :type => :shell, :inline => fs_init("vagrant")
    b.