1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
|
#!/usr/bin/env python3
"""
Converts the LibreOffice `download.lst` file into a Nix expression.
Requires an environment variable named `downloadList` identifying the path
of the input file, and writes the result to stdout.
todo - Ideally we would move as much as possible into derivation dependencies.
"""
import collections, itertools, json, re, subprocess, sys, os
def main():
packages = list(get_packages())
for x in packages:
print(x, file=sys.stderr)
print('[')
for x in packages:
md5 = x['md5']
upstream_sha256 = x['sha256']
if upstream_sha256:
hash = upstream_sha256
hashtype = 'sha256'
else:
hash = md5
hashtype = 'md5'
tarball = x['tarball']
url = construct_url(x)
print('url: {}'.format(url), file=sys.stderr)
path = download(url, tarball, hash, hashtype)
print('path: {}'.format(path), file=sys.stderr)
sha256 = get_sha256(path)
print('sha256: {}'.format(sha256), file=sys.stderr)
print(' {')
print(' name = "{}";'.format(tarball))
print(' url = "{}";'.format(url))
print(' sha256 = "{}";'.format(sha256))
print(' md5 = "{}";'.format(md5))
print(' md5name = "{}-{}";'.format(md5 or upstream_sha256,tarball))
print(' }')
print(']')
def construct_url(x):
if x['brief']:
return 'http://dev-www.libreoffice.org/src/{}{}'.format(
x.get('subdir', ''), x['tarball'])
else:
return 'http://dev-www.libreoffice.org/src/{}{}-{}'.format(
x.get('subdir', ''), x['md5'], x['tarball'])
def download(url, name, hash, hashtype):
cmd = ['nix-prefetch-url', url, hash, '--print-path',
'--type', hashtype, '--name', name]
proc = subprocess.run(cmd, stdout=subprocess.PIPE, check=True,
universal_newlines=True)
return proc.stdout.split('\n')[1].strip()
def get_sha256(path):
cmd = ['sha256sum', path]
proc = subprocess.run(cmd, stdout=subprocess.PIPE, check=True,
universal_newlines=True)
return proc.stdout.split(' ')[0].strip()
def get_packages():
"""
All of the package data: What's parsed from download.lst,
plus our additions.
"""
return apply_additions(get_packages_from_download_list(),
get_additions())
def get_additions():
"""
A mapping from package name (the all-caps identifiers used in
`download.lst`) to a dict of additional attributes to set on the package.
"""
with open('./libreoffice-srcs-additions.json') as f:
return json.load(f)
def apply_additions(xs, additions):
for x in xs:
yield dict_merge([x,
additions.get(x['name'], {})])
def get_packages_from_download_list():
"""
The result of parsing `download.lst`: A list of dicts containing keys
'name', 'tarball', 'md5', 'brief'.
"""
def lines():
for x in sub_symbols(parse_lines(get_lines())):
interpretation = interpret(x)
if interpretation == 'unrecognized':
print_skipped_line(x)
else:
yield dict_merge([x,
interpretation])
def cluster(xs):
"""
Groups lines according to their order within the file, to support
packages that are listed in `download.lst` more than once.
"""
keys = ['tarball', 'md5', 'sha256', 'brief']
a = {k: [x for x in xs if k in x['attrs']] for k in keys}
return zip(*[a[k] for k in keys])
def packages():
for (name, group) in groupby(lines(), lambda x: x['name']):
for xs in cluster(group):
yield {'name': name,
'attrs': dict_merge(x['attrs'] for x in xs),
'index': min(x['index'] for x in xs)}
for x in sorted(packages(), key=lambda x: x['index']):
yield dict_merge([{'name': x['name']},
x['attrs']])
def dict_merge(xs):
"""
>>> dict_merge([{1: 2}, {3: 4}, {3: 5}])
{1: 2, 3: 4}
"""
return dict(collections.ChainMap(*xs))
def groupby(xs, f):
"""
>>> groupby([1, 2, 3, 4], lambda x: x % 2)
[(0, [2, 4]), (1, [1, 3])]
"""
for (k, iter) in itertools.groupby(sorted(xs, key=f), f):
group = list(iter)
yield (f(group[0]), group)
def get_lines():
download_list = os.getenv('downloadList')
with open(download_list) as f:
return f.read().splitlines()
def print_skipped_line(x):
print('Skipped line {}: {}'.format(x['index'],
x['original']),
file=sys.stderr)
def parse_lines(lines):
"""
Input: List of strings (the lines from `download.lst`
Output: Iterator of dicts with keys 'key', 'value', and 'index'
"""
for (index, line) in enumerate(lines):
x = { 'index': index, 'original': line }
result = parse_line(line)
if result == 'nothing':
pass
elif result == 'unrecognized':
print_skipped_line(x)
else:
yield dict_merge([x,
result])
def parse_line(line):
"""
Input: A string
Output: One of 1. A dict with keys 'key', 'value'
2. 'nothing' (if the line contains no information)
2. 'unrecognized' (if parsing failed)
"""
if re.match('\s*(#.*)?$', line):
return 'nothing'
match = re.match('\s*export\s+([^:\s]+)\s*:=\s*(.*)$', line)
if match:
return {
'key': match.group(1),
'value': match.group(2).strip()
}
else:
return 'unrecognized'
def sub_symbols(xs):
"""
Do substitution of variables across all lines.
>>> sub_symbols([{'key': 'a', 'value': 'x'},
... {'key': 'c': 'value': '$(a)yz'}])
[{'key': 'a', 'value': 'x'}, {'key': 'c': 'value': 'xyz'}]
"""
xs = list(xs)
symbols = {x['key']: x for x in xs}
def get_value(k):
x = symbols.get(k)
return x['value'] if x is not None else ''
for x in xs:
yield dict_merge([{'value': sub_str(x['value'], get_value)},
x])
def sub_str(string, func):
"""
Do substitution of variables in a single line.
>>> sub_str("x = $(x)", lambda k: {'x': 'a'}[k])
"x = a"
"""
def func2(m):
x = m.group(1)
result = func(x)
return result if result is not None else x
return re.sub(r'\$\(([^\$\(\)]+)\)', func2, string)
def interpret(x):
"""
Input: Dict with keys 'key' and 'value'
Output: One of 1. Dict with keys 'name' and 'attrs'
2. 'unrecognized' (if interpretation failed)
"""
for f in [interpret_md5, interpret_sha256, interpret_tarball_with_md5, interpret_tarball, interpret_jar]:
result = f(x)
if result is not None:
return result
return 'unrecognized'
def interpret_md5(x):
"""
>>> interpret_md5("ODFGEN_MD5SUM", "32572ea48d9021bbd6fa317ddb697abc")
{'name': 'ODFGEN', 'attrs': {'md5': '32572ea48d9021bbd6fa317ddb697abc'}}
"""
match = re.match('^(.*)_MD5SUM$', x['key'])
if match:
return {'name': match.group(1),
'attrs': {'md5': x['value'], 'sha256': ''}}
def interpret_sha256(x):
match = re.match('^(.*)_SHA256SUM$', x['key'])
if match:
return {'name': match.group(1),
'attrs': {'sha256': x['value'], 'md5': ''}}
def interpret_tarball(x):
"""
>>> interpret_tarball("FREEHAND_TARBALL", "libfreehand-0.1.1.tar.bz2")
{'name': 'FREEHAND',
'attrs': {'tarball': 'libfreehand-0.1.1.tar.bz2', 'brief': True}}
"""
match = re.match('^(.*)_TARBALL$', x['key'])
if match:
return {'name': match.group(1),
'attrs': {'tarball': x['value'], 'brief': True}}
def interpret_jar(x):
match = re.match('^(.*)_JAR$', x['key'])
if match:
return {'name': match.group(1),
'attrs': {'tarball': x['value'], 'brief': True}}
def interpret_tarball_with_md5(x):
"""
>>> interpret_tarball_with_md5("CLUCENE_TARBALL",\
"48d647fbd8ef8889e5a7f422c1bfda94-clucene-core-2.3.3.4.tar.gz")
{'name': 'CLUCENE',
'attrs': {'tarball': 'clucene-core-2.3.3.4.tar.gz',
'md5': '48d647fbd8ef8889e5a7f422c1bfda94', 'brief': False}}
"""
match = {'key': re.match('^(.*)_(TARBALL|JAR)$', x['key']),
'value': re.match('(?P<md5>[0-9a-fA-F]{32})-(?P<tarball>.+)$',
x['value'])}
if match['key'] and match['value']:
return {'name': match['key'].group(1),
'attrs': {'tarball': match['value'].group('tarball'),
'md5': match['value'].group('md5'),
'sha256': '',
'brief'
|