1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
|
# -*- mode: ruby -*-
# vi: set ft=ruby :
# Automated creation of testing environments / binaries on misc. platforms
$cpus = Integer(ENV.fetch('VMCPUS', '4')) # create VMs with that many cpus
$xdistn = Integer(ENV.fetch('XDISTN', '4')) # dispatch tests to that many pytest workers
$wmem = $xdistn * 256 # give the VM additional memory for workers [MB]
def packages_debianoid(user)
return <<-EOF
apt update
# install all the (security and other) updates
apt dist-upgrade -y
# for building borgbackup and dependencies:
apt install -y libssl-dev libacl1-dev liblz4-dev libfuse-dev fuse pkg-config
usermod -a -G fuse #{user}
chgrp fuse /dev/fuse
chmod 666 /dev/fuse
apt install -y fakeroot build-essential git curl
apt install -y python3-dev python3-setuptools python-virtualenv python3-virtualenv
# for building python:
apt install -y zlib1g-dev libbz2-dev libncurses5-dev libreadline-dev liblzma-dev libsqlite3-dev libffi-dev
EOF
end
def packages_arch
return <<-EOF
echo "en_US.UTF-8 UTF-8" >> /etc/locale.gen
locale-gen
localectl set-locale LANG=en_US.UTF-8
chown vagrant.vagrant /vagrant
pacman --sync --noconfirm --refresh python-virtualenv python-pip
EOF
end
def packages_freebsd
return <<-EOF
# in case the VM has no hostname set
hostname freebsd
# install all the (security and other) updates, base system
freebsd-update --not-running-from-cron fetch install
# for building borgbackup and dependencies:
pkg install -y openssl-devel liblz4 fusefs-libs pkgconf
pkg install -y git bash # fakeroot causes lots of troubles on freebsd
# for building python:
pkg install -y sqlite3
pkg install -y py27-virtualenv # provides "virtualenv" command
pkg install -y python36 py36-virtualenv py36-pip
# make sure there is a python3 command
ln -s /usr/local/bin/python3.6 /usr/local/bin/python3
# make bash default / work:
chsh -s bash vagrant
mount -t fdescfs fdesc /dev/fd
echo 'fdesc /dev/fd fdescfs rw 0 0' >> /etc/fstab
# make FUSE work
echo 'fuse_load="YES"' >> /boot/loader.conf
echo 'vfs.usermount=1' >> /etc/sysctl.conf
kldload fuse
sysctl vfs.usermount=1
pw groupmod operator -M vagrant
# /dev/fuse has group operator
chmod 666 /dev/fuse
# install all the (security and other) updates, packages
pkg update
yes | pkg upgrade
EOF
end
def packages_darwin
return <<-EOF
# install all the (security and other) updates
sudo softwareupdate --ignore iTunesX
sudo softwareupdate --ignore iTunes
sudo softwareupdate --ignore "Install macOS High Sierra"
sudo softwareupdate --install --all
# get osxfuse 3.x release code from github:
curl -s -L https://github.com/osxfuse/osxfuse/releases/download/osxfuse-3.8.2/osxfuse-3.8.2.dmg >osxfuse.dmg
MOUNTDIR=$(echo `hdiutil mount osxfuse.dmg | tail -1 | awk '{$1="" ; print $0}'` | xargs -0 echo) \
&& sudo installer -pkg "${MOUNTDIR}/Extras/FUSE for macOS 3.8.2.pkg" -target /
sudo chown -R vagrant /usr/local # brew must be able to create stuff here
ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)"
brew update
brew install openssl
brew install lz4
brew install xz # required for python lzma module
brew install fakeroot
brew install git
brew install pkg-config
EOF
end
def install_pyenv(boxname)
return <<-EOF
curl -s -L https://raw.githubusercontent.com/yyuu/pyenv-installer/master/bin/pyenv-installer | bash
echo 'export PATH="$HOME/.pyenv/bin:$PATH"' >> ~/.bash_profile
echo 'eval "$(pyenv init -)"' >> ~/.bash_profile
echo 'eval "$(pyenv virtualenv-init -)"' >> ~/.bash_profile
echo 'export PYTHON_CONFIGURE_OPTS="--enable-shared"' >> ~/.bash_profile
EOF
end
def fix_pyenv_darwin(boxname)
return <<-EOF
echo 'export PYTHON_CONFIGURE_OPTS="--enable-framework"' >> ~/.bash_profile
EOF
end
def install_pythons(boxname)
return <<-EOF
. ~/.bash_profile
pyenv install 3.7.0 # tests
pyenv install 3.6.0 # tests
pyenv install 3.5.0 # tests
pyenv install 3.6.7 # binary build, use latest 3.6.x release
pyenv rehash
EOF
end
def build_sys_venv(boxname)
return <<-EOF
. ~/.bash_profile
cd /vagrant/borg
virtualenv --python=python3 borg-env
EOF
end
def build_pyenv_venv(boxname)
return <<-EOF
. ~/.bash_profile
cd /vagrant/borg
# use the latest 3.6 release
pyenv global 3.6.7
pyenv virtualenv 3.6.7 borg-env
ln -s ~/.pyenv/versions/borg-env .
EOF
end
def install_borg(fuse)
script = <<-EOF
. ~/.bash_profile
cd /vagrant/borg
. borg-env/bin/activate
pip install -U wheel # upgrade wheel, too old for 3.5
cd borg
pip install -r requirements.d/development.txt
python setup.py clean
EOF
if fuse
script += <<-EOF
# by using [fuse], setup.py can handle different FUSE requirements:
pip install -e .[fuse]
EOF
else
script += <<-EOF
pip install -e .
# do not install llfuse into the virtualenvs built by tox:
sed -i.bak '/fuse.txt/d' tox.ini
EOF
end
return script
end
def install_pyinstaller()
return <<-EOF
. ~/.bash_profile
cd /vagrant/borg
. borg-env/bin/activate
git clone https://github.com/thomaswaldmann/pyinstaller.git
cd pyinstaller
git checkout v3.3.1
python setup.py install
EOF
end
def build_binary_with_pyinstaller(boxname)
return <<-EOF
. ~/.bash_profile
cd /vagrant/borg
. borg-env/bin/activate
cd borg
pyinstaller --clean --distpath=/vagrant/borg scripts/borg.exe.spec
echo 'export PATH="/vagrant/borg:$PATH"' >> ~/.bash_profile
EOF
end
def run_tests(boxname)
return <<-EOF
. ~/.bash_profile
cd /vagrant/borg/borg
. ../borg-env/bin/activate
if which pyenv 2> /dev/null; then
# for testing, use the earliest point releases of the supported python versions:
pyenv global 3.5.0 3.6.0 3.7.0
pyenv local 3.5.0 3.6.0 3.7.0
fi
# otherwise: just use the system python
if which fakeroot 2> /dev/null; then
echo "Running tox WITH fakeroot -u"
fakeroot -u tox --skip-missing-interpreters
else
echo "Running tox WITHOUT fakeroot -u"
tox --skip-missing-interpreters
fi
EOF
end
def fs_init(user)
return <<-EOF
# clean up (wrong/outdated) stuff we likely got via rsync:
rm -rf /vagrant/borg/borg/.tox 2> /dev/null
rm -rf /vagrant/borg/borg/borgbackup.egg-info 2> /dev/null
rm -rf /vagrant/borg/borg/__pycache__ 2> /dev/null
find /vagrant/borg/borg/src -name '__pycache__' -exec rm -rf {} \\; 2> /dev/null
chown -R #{user} /vagrant/borg
touch ~#{user}/.bash_profile ; chown #{user} ~#{user}/.bash_profile
echo 'export LANG=en_US.UTF-8' >> ~#{user}/.bash_profile
echo 'export LC_CTYPE=en_US.UTF-8' >> ~#{user}/.bash_profile
echo 'export XDISTN=#{$xdistn}' >> ~#{user}/.bash_profile
EOF
end
Vagrant.configure(2) do |config|
# use rsync to copy content to the folder
config.vm.synced_folder ".", "/vagrant/borg/borg", :type => "rsync", :rsync__args => ["--verbose", "--archive", "--delete", "-z"], :rsync__chown => false
# do not let the VM access . on the host machine via the default shared folder!
config.vm.synced_folder ".", "/vagrant", disabled: true
config.vm.provider :virtualbox do |v|
#v.gui = true
v.cpus = $cpus
end
config.vm.define "bionic64" do |b|
b.vm.box = "ubuntu/bionic64"
b.vm.provider :virtualbox do |v|
v.memory = 1024 + $wmem
end
b.vm.provision "fs init", :type => :shell, :inline => fs_init("vagrant")
b.vm.provision "packages debianoid", :type => :shell, :inline => packages_debianoid("vagrant")
b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_sys_venv("bionic64")
b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg(true)
b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("bionic64")
end
config.vm.define "stretch64" do |b|
b.vm.box = "debian/stretch64"
b.vm.provider :virtualbox do |v|
v.memory = 1024 + $wmem
end
b.vm.provision "fs init", :type => :shell, :inline => fs_init("vagrant")
b.vm.provision "packages debianoid", :type => :shell, :inline => packages_debianoid("vagrant")
b.vm.provision "install pyenv", :type => :shell, :privileged => false, :inline => install_pyenv("stretch64")
b.vm.provision "install pythons", :type => :shell, :privileged => false, :inline => install_pythons("stretch64")
b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_pyenv_venv("stretch64")
b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg(true)
b.vm.provision "install pyinstaller", :type => :shell, :privileged => false, :inline => install_pyinstaller()
b.vm.provision "build binary with pyinstaller", :type => :shell, :privileged => false, :inline => build_binary_with_pyinstaller("stretch64")
b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("stretch64")
end
config.vm.define "arch64" do |b|
b.vm.box = "terrywang/archlinux"
b.vm.provider :virtualbox do |v|
v.memory = 1024 + $wmem
end
b.vm.provision "fs init", :type => :shell, :inline => fs_init("vagrant")
b.vm.provision "packages arch", :type => :shell, :privileged => true, :inline => packages_arch
b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_sys_venv("arch64")
b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg(true)
b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("arch64")
end
config.vm.define "freebsd64" do |b|
b.vm.box = "freebsd12-amd64"
b.vm.provider :virtualbox do |v|
v.memory = 1024 + $wmem
end
b.ssh.shell = "sh"
b.vm.provision "fs init", :type => :shell, :inline => fs_init("vagrant")
b.vm.provision "packages freebsd", :type => :shell, :inline => packages_freebsd
b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_sys_venv("freebsd64")
b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg(true)
b.vm.provision "install pyinstaller", :type => :shell, :privileged => false, :inline => install_pyinstaller()
b.vm.provision "build binary with pyinstaller", :type => :shell, :privileged => false, :inline => build_binary_with_pyinstaller("freebsd64")
b.vm.provision "run tests", :type => :shell,
|