1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
|
#!/usr/libexec/flua
--[[
SPDX-License-Identifier: BSD-2-Clause-FreeBSD
Copyright (c) 2022 Stefan Esser <se@FreeBSD.org>
Generate a list of existing and required CONFLICTS_INSTALL lines
for all ports (limited to ports for which official packages are
provided).
This script depends on the ports-mgmt/pkg-provides port for the list
of files installed by all pre-built packages for the architecture
the script is run on.
The script generates a list of ports by running "pkg provides ." and
a mapping from package base name to origin via "pkg rquery '%n %o'".
The existing CONFLICTS and CONFLICTS_INSTALL definitions are fetched
by "make -C $origin -V CONFLICTS -V CONFLICTS_INSTALL". This list is
only representative for the options configured for each port (i.e.
if non-default options have been selected and registered, these may
lead to a non-default list of conflicts).
The script detects files used by more than one port, than lists by
origin the existing definition and the list of package base names
that have been detected to cause install conflicts followed by the
list of duplicate files separated by a hash character "#".
This script uses the "hidden" LUA interpreter in the FreeBSD base
systems and does not need any port except "pkg-provides" to be run.
The run-time on my system checking the ~32000 packages available
for -CURRENT on amd64 is less than 250 seconds.
Example output:
# Port: games/sol
# Files: bin/sol
# < aisleriot gnome-games
# > aisleriot
portedit merge -ie 'CONFLICTS_INSTALL=aisleriot # bin/sol' /usr/ports/games/sol
The output is per port (for all flavors of the port, if applicable),
gives examples of conflicting files (mostly to understand whether
different versions of a port could co-exist), the current CONFLICTS
and CONFLICTS_INSTALL entries merged, and a suggested new entry.
This information is followed by a portedit command line that should
do the right thing for simple cases, but the result should always
be checked before the resulting Makefile is committed.
--]]
require "lfs"
-------------------------------------------------------------------
local file_pattern = "."
local database = "/var/db/pkg/provides/provides.db"
local max_age = 1 * 24 * 3600 -- maximum age of database file in seconds
-------------------------------------------------------------------
local function table_sorted_keys(t)
local result = {}
for k, _ in pairs(t) do
result[#result + 1] = k
end
table.sort(result)
return result
end
-------------------------------------------------------------------
local function table_sort_uniq(t)
local result = {}
if t then
local last
table.sort(t)
for _, entry in ipairs(t) do
if entry ~= last then
last = entry
result[#result + 1] = entry
end
end
end
return result
end
-------------------------------------------------------------------
local function fnmatch(name, pattern)
local function fnsubst(s)
s = string.gsub(s, "%%", "%%%%")
s = string.gsub(s, "%+", "%%+")
s = string.gsub(s, "%-", "%%-")
s = string.gsub(s, "%.", "%%.")
s = string.gsub(s, "%?", ".")
s = string.gsub(s, "%*", ".*")
return s
end
local rexpr = ""
local left, middle, right
while true do
left, middle, right = string.match(pattern, "([^[]*)(%[[^]]+%])(.*)")
if not left then
break
end
rexpr = rexpr .. fnsubst(left) .. middle
pattern = right
end
rexpr = "^" .. rexpr .. fnsubst(pattern) .. "$"
return string.find(name, rexpr)
end
-------------------------------------------------------------------
local function fetch_pkgs_origins()
local pkgs = {}
local pipe = io.popen("pkg rquery '%n %o'")
for line in pipe:lines() do
local pkgbase, origin = string.match(line, "(%S+) (%S+)")
pkgs[origin] = pkgbase
end
pipe:close()
pipe = io.popen("pkg rquery '%n %o %At %Av'")
for line in pipe:lines() do
local pkgbase, origin, tag, value = string.match(line, "(%S+) (%S+) (%S+) (%S+)")
if tag == "flavor" then
pkgs[origin] = nil
pkgs[origin .. "@" .. value] = pkgbase
end
end
pipe:close()
return pkgs
end
-------------------------------------------------------------------
local BAD_FILE_PATTERN = {
"^[^/]+$",
"^lib/python[%d%.]+/site%-packages/examples/[^/]+$",
"^lib/python[%d%.]+/site%-packages/samples/[^/]+$",
"^lib/python[%d%.]+/site%-packages/test/[^/]+$",
"^lib/python[%d%.]+/site%-packages/test_app/[^/]+$",
"^lib/python[%d%.]+/site%-packages/tests/[^/]+$",
"^lib/python[%d%.]+/site%-packages/tests/unit/[^/]+$",
}
local BAD_FILE_PKGS = {}
local function check_bad_file(pkgbase, file)
for _, pattern in ipairs(BAD_FILE_PATTERN) do
if string.match(file, pattern) then
BAD_FILE_PKGS[pkgbase] = BAD_FILE_PKGS[pkgbase] or {}
table.insert(BAD_FILE_PKGS[pkgbase], file)
break
end
end
end
-------------------------------------------------------------------
local function read_files(pattern)
local files_table = {}
local now = os.time()
local modification_time = lfs.attributes(database, "modification")
if not modification_time then
print("# Aborting: package file database " .. database .. " does not exist.")
print("# Install the 'pkg-provides' package and add it as a module to 'pkg.conf'.")
print("# Then fetch the database with 'pkg update' or 'pkg provides -u'.")
os.exit(1)
end
if now - modification_time > max_age then
print("# Aborting: package file database " .. database)
print("# is older than " .. max_age .. " seconds.")
print("# Use 'pkg provides -u' to update the database.")
os.exit(2)
end
local pipe = io.popen("locate -d " .. database .. " " .. pattern)
if pipe then
for line in pipe:lines() do
local pkgbase, file = string.match(line, "([^*]+)%*([^*]+)")
if file:sub(1, 11) == "/usr/local/" then
file = file:sub(12)
end
check_bad_file(pkgbase, file)
local t = files_table[file] or {}
t[#t + 1] = pkgbase
files_table[file] = t
end
pipe:close()
end
return files_table
end
-------------------------------------------------------------------
local DUPLICATE_FILE = {}
local function fetch_pkg_pairs(pattern)
local pkg_pairs = {}
for file, pkgbases in pairs(read_files(pattern)) do
if #pkgbases >= 2 then
DUPLICATE_FILE[file] = true
for i = 1, #pkgbases -1 do
local pkg_i = pkgbases[i]
for j = i + 1, #pkgbases do
local pkg_j = pkgbases[j]
if pkg_i ~= pkg_j then
local p1 = pkg_pairs[pkg_i] or {}
local p2 = p1[pkg_j] or {}
p2[#p2 + 1] = file
p1[pkg_j] = p2
pkg_pairs[pkg_i] = p1
end
end
end
end
end
return pkg_pairs
end
-------------------------------------------------------------------
local function conflicts_delta(old, new)
local old_seen = {}
local changed
for i = 1, #new do
local matched
for j = 1, #old do
if new[i] == old[j] or fnmatch(new[i], old[j]) then
new[i] = old[j]
old_seen[j] = true
matched = true
break
end
end
changed = changed or not matched
end
if not changed then
for j = 1, #old do
if not old_seen[j] then
changed = true
break
end
end
end
if changed then
return table_sort_uniq(new)
end
end
-------------------------------------------------------------------
local function fetch_port_conflicts(origin)
local dir, flavor = origin:match("([^@]+)@?(.*)")
if flavor ~= "" then
flavor = " FLAVOR=" .. flavor
end
local seen = {}
local portdir = "/usr/ports/" .. dir
local pipe = io.popen("make -C " .. portdir .. flavor .. " -V CONFLICTS -V CONFLICTS_INSTALL 2>/dev/null")
for line in pipe:lines() do
for word in line:gmatch("(%S+)%s?") do
seen[word] = true
end
end
pipe:close()
return table_sorted_keys(seen)
end
-------------------------------------------------------------------
local function conflicting_pkgs(conflicting)
local pkgs = {}
for origin, pkgbase in pairs(fetch_pkgs_origins()) do
if conflicting[pkgbase] then
pkgs[origin] = pkgbase
end
end
return pkgs
end
-------------------------------------------------------------------
local function collect_conflicts(pkg_pairs)
local pkgs = {}
for pkg_i, p1 in pairs(pkg_pairs) do
for pkg_j, _ in pairs(p1) do
pkgs[pkg_i] = pkgs[pkg_i] or {}
table.insert(pkgs[pkg_i], pkg_j)
pkgs[pkg_j] = pkgs[pkg_j] or {}
table.insert(pkgs[pkg_j], pkg_i)
end
end
return pkgs
end
-------------------------------------------------------------------
local function split_origins(origin_list)
local port_list = {}
local flavors = {}
local last_port
for _, origin in ipairs(origin_list) do
local port, flavor = string.match(origin, "([^@]+)@?(.*)")
if port ~= last_port then
port_list[#port_list + 1] = port
if flavor ~= "" then
flavors[port] = {flavor}
end
else
table.insert(flavors[port], flavor)
end
last_port = port
end
return port_list, flavors
end
-------------------------------------------------------------------
local PKG_PAIR_FILES = fetch_pkg_pairs(file_pattern)
local CONFLICT_PKGS = collect_conflicts(PKG_PAIR_FILES)
local PKGBASE = conflicting_pkgs(CONFLICT_PKGS)
local ORIGIN_LIST = table_sorted_keys(PKGBASE)
local PORT_LIST, FLAVORS = split_origins(ORIGIN_LIST)
local function conflicting_files(pkg_i, pkgs)
local files = {}
local all_files = {}
local f
local p1 = PKG_PAIR_FILES[pkg_i]
if p1 then
for _, pkg_j in ipairs(pkgs) do
f = p1[pkg_j]
if f then
table.sort(f)
files[#files + 1] = f[1]
table.move(f, 1, #f, #all_files + 1, all_files)
end
end
end
for _, pkg_j in ipairs(pkgs) do
p1 = PKG_PAIR_FILES[pkg_j]
f = p1 and p1[pkg_i]
if f then
table.sort(f)
files[#files + 1] = f[1]
table.move(f, 1, #f, #all_files + 1, all_files)
end
end
return table_sort_uniq(files), table_sort_uniq(all_files)
end
---------------------------------------------------------------------
local version_pattern = {
"^lib/python%d%.%d/",
"^share/py3%d%d%?-",
"^share/%a+/py3%d%d%?-",
"^lib/lua/%d%.%d/",
"^share/lua/%d%.%d/",
"^lib/perl5/[%d%.]+/",
"^lib/perl5/site_perl/mach/[%d%.]+/",
"^lib/ruby/gems/%d%.%d/",
"^lib/ruby/site_ruby/%d%.%d/",
}
local function generalize_patterns(pkgs, files)
local function match_any(str, pattern_array)
for _, pattern in ipairs(pattern_array) do
if string.match(str, pattern) then
return true
end
end
return false
end
local function unversioned_files()
for i = 1, #files do
if not match_any(files[i], version_pattern) then
return true
end
end
return false
end
local function pkg_wildcards(from, ...)
local to_list = {...}
local result = {}
for i = 1, #pkgs do
local orig_pkg = pkgs[i]
for _, to in ipairs(to_list) do
result[string.gsub(orig_pkg, from, to)] = true
end
end
pkgs = table_sorted_keys(result)
end
local pkg_pfx_php = "php[0-9][0-9]-"
local pkg_sfx_php = "-php[0-9][0-9]"
local pkg_pfx_python2
local pkg_sfx_python2
local pkg_pfx_python3
local pkg_sfx_python3
local pkg_pfx_lua
local pkg_pfx_ruby
pkgs = table_sort_uniq(pkgs)
if unversioned_files() then
pkg_pfx_python2 = "py3[0-9]-" -- e.g. py39-
pkg_sfx_python2 = "-py3[0-9]"
pkg_pfx_python3 = "py3[0-9][0-9]-" -- e.g. py311-
pkg_sfx_python3 = "-py3[0-9][0-9]"
pkg_pfx_lua = "lua[0-9][0-9]-"
pkg_pfx_ruby = "ruby[0-9][0-9]-"
else
pkg_pfx_python2 = "${PYTHON_PKGNAMEPREFIX}"
pkg_sfx_python2 = "${PYTHON_PKGNAMESUFFIX}"
pkg_pfx_python3 = nil
pkg_sfx_python3 = nil
pkg_pfx_lua = "${LUA_PKGNAMEPREFIX}"
pkg_pfx_ruby = "${RUBY_PKGNAMEPREFIX}"
end
pkg_wildcards("^php%d%d%-", pkg_pfx_php)
pkg_wildcards("-php%d%d$", pkg_sfx_php)
pkg_wildcards("^phpunit%d%-", "phpunit[0-9]-")
pkg_wildcards("^py3%d%-", pkg_pfx_python2, pkg_pfx_python3)
pkg_wildcards("-py3%d%$", pkg_sfx_python2, pkg_sfx_python3)
pkg_wildcards("^py3%d%d%-", pkg_pfx_python2, pkg_pfx_python3)
pkg_wildcards("-py3%d%d%$", pkg_sfx_python2, pkg_sfx_python3)
pkg_wildcards("^lua%d%d%-", pkg_pfx_lua)
pkg_wildcards("-emacs_[%a_]*", "-emacs_*")
pkg_wildcards("^ghostscript%d%-", "ghostscript[0-9]-")
pkg_wildcards("^bacula%d%-", "bacula[0-9]-")
pkg_wildcards("^bacula%d%d%-", "bacula[0-9][0-9]-")
pkg_wildcards("^bareos%d%-", "bareos[0-9]-")
pkg_wildcards("^bareos%d%d%-", "bareos[0-9][0-9]-")
pkg_wildcards("^moosefs%d%-", "moosefs[0-9]-")
pkg_wildcards("^ruby%d+-", pkg_pfx_ruby)
return table_sort_uniq(pkgs)
end
-------------------------------------------------------------------
for _, port in ipairs(PORT_LIST) do
local function merge_table(t1, t2)
table.move(t2, 1, #t2, #t1 + 1, t1)
end
local port_conflicts = {}
local files = {}
local msg_files = {}
local conflict_pkgs = {}
local function merge_data(origin)
local pkgbase = PKGBASE[origin]
if not BAD_FILE_PKGS[pkgbase] then
local pkg_confl_file1, pkg_confl_files = conflicting_files(pkgbase, CONFLICT_PKGS[pkgbase])
merge_table(msg_files, pkg_confl_file1) -- 1 file per flavor
merge_table(files, pkg_confl_files) -- all conflicting files
merge_table(conflict_pkgs, CONFLICT_PKGS[pkgbase])
merge_table(port_conflicts, fetch_port_conflicts(origin))
end
end
local flavors = FLAVORS[port]
if flavors then
for _, flavor in ipairs(flavors) do
merge_data(port .. "@" .. flavor)
end
else
merge_data(port)
end
files = table_sort_uniq(files)
msg_files = table_sort_uniq(msg_files)
conflict_pkgs = generalize_patterns(conflict_pkgs, files)
if #port_conflicts then
port_conflicts = table_sort_uniq(port_conflicts)
conflict_pkgs = conflicts_delta(port_conflicts, conflict_pkgs)
end
if conflict_pkgs then
local conflicts_string_cur = table.concat(port_conflicts, " ")
local conflicts_string_new = table.concat(conflict_pkgs, " ")
local file_list = table.concat(msg_files, " ")
print("# Port: " .. port)
print("# Files: " .. file_list)
if conflicts_string_cur ~= "" then
print("# < " .. conflicts_string_cur)
end
print("# > " .. conflicts_string_new)
print("portedit merge -ie 'CONFLICTS_INSTALL=" .. conflicts_string_new ..
" # " .. file_list .. "' /usr/ports/" .. port)
print()
end
end
-------------------------------------------------------------------
local BAD_FILES_ORIGINS = {}
for _, origin in ipairs(ORIGIN_LIST) do
local pkgbase = PKGBASE[origin]
local files = BAD_FILE_PKGS[pkgbase]
if files then
for _, file in ipairs(files) do
if DUPLICATE_FILE[file] then
local port = string.match(origin, "([^@]+)@?")
BAD_FILES_ORIGINS[port] = BAD_FILES_ORIGINS[origin] or {}
table.insert(BAD_FILES_ORIGINS[port], file)
end
end
end
end
-------------------------------------------------------------------
local bad_origins = table_sorted_keys(BAD_FILES_ORIGINS)
if #bad_origins > 0 then
print ("# Ports with badly named files:")
print ()
for _, port in ipairs(bad_origins) do
print ("# " .. port)
local files = BAD_FILES_ORIGINS[port]
table.sort(files)
for _, file in ipairs(files) do
print ("#", file)
end
print()
end
end
|