diff options
Diffstat (limited to 'Tools/scripts/port_conflicts_check.lua')
| -rwxr-xr-x | Tools/scripts/port_conflicts_check.lua | 507 |
1 files changed, 507 insertions, 0 deletions
diff --git a/Tools/scripts/port_conflicts_check.lua b/Tools/scripts/port_conflicts_check.lua new file mode 100755 index 000000000000..c5265af19a5a --- /dev/null +++ b/Tools/scripts/port_conflicts_check.lua @@ -0,0 +1,507 @@ +#!/usr/libexec/flua + +--[[ +SPDX-License-Identifier: BSD-2-Clause-FreeBSD + +Copyright (c) 2022 Stefan Esser <se@FreeBSD.org> + +Generate a list of existing and required CONFLICTS_INSTALL lines +for all ports (limited to ports for which official packages are +provided). + +This script depends on the ports-mgmt/pkg-provides port for the list +of files installed by all pre-built packages for the architecture +the script is run on. + +The script generates a list of ports by running "pkg provides ." and +a mapping from package base name to origin via "pkg rquery '%n %o'". + +The existing CONFLICTS and CONFLICTS_INSTALL definitions are fetched +by "make -C $origin -V CONFLICTS -V CONFLICTS_INSTALL". This list is +only representative for the options configured for each port (i.e. +if non-default options have been selected and registered, these may +lead to a non-default list of conflicts). + +The script detects files used by more than one port, than lists by +origin the existing definition and the list of package base names +that have been detected to cause install conflicts followed by the +list of duplicate files separated by a hash character "#". + +This script uses the "hidden" LUA interpreter in the FreeBSD base +systems and does not need any port except "pkg-provides" to be run. + +The run-time on my system checking the ~32000 packages available +for -CURRENT on amd64 is less than 250 seconds. + +Example output: + +# Port: games/sol +# Files: bin/sol +# < aisleriot gnome-games +# > aisleriot +portedit merge -ie 'CONFLICTS_INSTALL=aisleriot # bin/sol' /usr/ports/games/sol + +The output is per port (for all flavors of the port, if applicable), +gives examples of conflicting files (mostly to understand whether +different versions of a port could co-exist), the current CONFLICTS +and CONFLICTS_INSTALL entries merged, and a suggested new entry. +This information is followed by a portedit command line that should +do the right thing for simple cases, but the result should always +be checked before the resulting Makefile is committed. +--]] + +require "lfs" + +------------------------------------------------------------------- +local file_pattern = "." +local database = "/var/db/pkg/provides/provides.db" +local max_age = 1 * 24 * 3600 -- maximum age of database file in seconds + +------------------------------------------------------------------- +local function table_sorted_keys(t) + local result = {} + for k, _ in pairs(t) do + result[#result + 1] = k + end + table.sort(result) + return result +end + +------------------------------------------------------------------- +local function table_sort_uniq(t) + local result = {} + if t then + local last + table.sort(t) + for _, entry in ipairs(t) do + if entry ~= last then + last = entry + result[#result + 1] = entry + end + end + end + return result +end + +------------------------------------------------------------------- +local function fnmatch(name, pattern) + local function fnsubst(s) + s = string.gsub(s, "%%", "%%%%") + s = string.gsub(s, "%+", "%%+") + s = string.gsub(s, "%-", "%%-") + s = string.gsub(s, "%.", "%%.") + s = string.gsub(s, "%?", ".") + s = string.gsub(s, "%*", ".*") + return s + end + local rexpr = "" + local left, middle, right + while true do + left, middle, right = string.match(pattern, "([^[]*)(%[[^]]+%])(.*)") + if not left then + break + end + rexpr = rexpr .. fnsubst(left) .. middle + pattern = right + end + rexpr = "^" .. rexpr .. fnsubst(pattern) .. "$" + return string.find(name, rexpr) +end + +------------------------------------------------------------------- +local function fetch_pkgs_origins() + local pkgs = {} + local pipe = io.popen("pkg rquery '%n %o'") + for line in pipe:lines() do + local pkgbase, origin = string.match(line, "(%S+) (%S+)") + pkgs[origin] = pkgbase + end + pipe:close() + pipe = io.popen("pkg rquery '%n %o %At %Av'") + for line in pipe:lines() do + local pkgbase, origin, tag, value = string.match(line, "(%S+) (%S+) (%S+) (%S+)") + if tag == "flavor" then + pkgs[origin] = nil + pkgs[origin .. "@" .. value] = pkgbase + end + end + pipe:close() + return pkgs +end + +------------------------------------------------------------------- +local BAD_FILE_PATTERN = { + "^[^/]+$", + "^lib/python[%d%.]+/site%-packages/examples/[^/]+$", + "^lib/python[%d%.]+/site%-packages/samples/[^/]+$", + "^lib/python[%d%.]+/site%-packages/test/[^/]+$", + "^lib/python[%d%.]+/site%-packages/test_app/[^/]+$", + "^lib/python[%d%.]+/site%-packages/tests/[^/]+$", + "^lib/python[%d%.]+/site%-packages/tests/unit/[^/]+$", +} + +local BAD_FILE_PKGS = {} + +local function check_bad_file(pkgbase, file) + for _, pattern in ipairs(BAD_FILE_PATTERN) do + if string.match(file, pattern) then + BAD_FILE_PKGS[pkgbase] = BAD_FILE_PKGS[pkgbase] or {} + table.insert(BAD_FILE_PKGS[pkgbase], file) + break + end + end +end + +------------------------------------------------------------------- +local function read_files(pattern) + local files_table = {} + local now = os.time() + local modification_time = lfs.attributes(database, "modification") + if not modification_time then + print("# Aborting: package file database " .. database .. " does not exist.") + print("# Install the 'pkg-provides' package and add it as a module to 'pkg.conf'.") + print("# Then fetch the database with 'pkg update' or 'pkg provides -u'.") + os.exit(1) + end + if now - modification_time > max_age then + print("# Aborting: package file database " .. database) + print("# is older than " .. max_age .. " seconds.") + print("# Use 'pkg provides -u' to update the database.") + os.exit(2) + end + local pipe = io.popen("locate -d " .. database .. " " .. pattern) + if pipe then + for line in pipe:lines() do + local pkgbase, file = string.match(line, "([^*]+)%*([^*]+)") + if file:sub(1, 11) == "/usr/local/" then + file = file:sub(12) + end + check_bad_file(pkgbase, file) + local t = files_table[file] or {} + t[#t + 1] = pkgbase + files_table[file] = t + end + pipe:close() + end + return files_table +end + +------------------------------------------------------------------- +local DUPLICATE_FILE = {} + +local function fetch_pkg_pairs(pattern) + local pkg_pairs = {} + for file, pkgbases in pairs(read_files(pattern)) do + if #pkgbases >= 2 then + DUPLICATE_FILE[file] = true + for i = 1, #pkgbases -1 do + local pkg_i = pkgbases[i] + for j = i + 1, #pkgbases do + local pkg_j = pkgbases[j] + if pkg_i ~= pkg_j then + local p1 = pkg_pairs[pkg_i] or {} + local p2 = p1[pkg_j] or {} + p2[#p2 + 1] = file + p1[pkg_j] = p2 + pkg_pairs[pkg_i] = p1 + end + end + end + end + end + return pkg_pairs +end + +------------------------------------------------------------------- +local function conflicts_delta(old, new) + local old_seen = {} + local changed + for i = 1, #new do + local matched + for j = 1, #old do + if new[i] == old[j] or fnmatch(new[i], old[j]) then + new[i] = old[j] + old_seen[j] = true + matched = true + break + end + end + changed = changed or not matched + end + if not changed then + for j = 1, #old do + if not old_seen[j] then + changed = true + break + end + end + end + if changed then + return table_sort_uniq(new) + end +end + +------------------------------------------------------------------- +local function fetch_port_conflicts(origin) + local dir, flavor = origin:match("([^@]+)@?(.*)") + if flavor ~= "" then + flavor = " FLAVOR=" .. flavor + end + local seen = {} + local portdir = "/usr/ports/" .. dir + local pipe = io.popen("make -C " .. portdir .. flavor .. " -V CONFLICTS -V CONFLICTS_INSTALL 2>/dev/null") + for line in pipe:lines() do + for word in line:gmatch("(%S+)%s?") do + seen[word] = true + end + end + pipe:close() + return table_sorted_keys(seen) +end + +------------------------------------------------------------------- +local function conflicting_pkgs(conflicting) + local pkgs = {} + for origin, pkgbase in pairs(fetch_pkgs_origins()) do + if conflicting[pkgbase] then + pkgs[origin] = pkgbase + end + end + return pkgs +end + +------------------------------------------------------------------- +local function collect_conflicts(pkg_pairs) + local pkgs = {} + for pkg_i, p1 in pairs(pkg_pairs) do + for pkg_j, _ in pairs(p1) do + pkgs[pkg_i] = pkgs[pkg_i] or {} + table.insert(pkgs[pkg_i], pkg_j) + pkgs[pkg_j] = pkgs[pkg_j] or {} + table.insert(pkgs[pkg_j], pkg_i) + end + end + return pkgs +end + +------------------------------------------------------------------- +local function split_origins(origin_list) + local port_list = {} + local flavors = {} + local last_port + for _, origin in ipairs(origin_list) do + local port, flavor = string.match(origin, "([^@]+)@?(.*)") + if port ~= last_port then + port_list[#port_list + 1] = port + if flavor ~= "" then + flavors[port] = {flavor} + end + else + table.insert(flavors[port], flavor) + end + last_port = port + end + return port_list, flavors +end + +------------------------------------------------------------------- +local PKG_PAIR_FILES = fetch_pkg_pairs(file_pattern) +local CONFLICT_PKGS = collect_conflicts(PKG_PAIR_FILES) +local PKGBASE = conflicting_pkgs(CONFLICT_PKGS) +local ORIGIN_LIST = table_sorted_keys(PKGBASE) +local PORT_LIST, FLAVORS = split_origins(ORIGIN_LIST) + +local function conflicting_files(pkg_i, pkgs) + local files = {} + local all_files = {} + local f + local p1 = PKG_PAIR_FILES[pkg_i] + if p1 then + for _, pkg_j in ipairs(pkgs) do + f = p1[pkg_j] + if f then + table.sort(f) + files[#files + 1] = f[1] + table.move(f, 1, #f, #all_files + 1, all_files) + end + end + end + for _, pkg_j in ipairs(pkgs) do + p1 = PKG_PAIR_FILES[pkg_j] + f = p1 and p1[pkg_i] + if f then + table.sort(f) + files[#files + 1] = f[1] + table.move(f, 1, #f, #all_files + 1, all_files) + end + end + return table_sort_uniq(files), table_sort_uniq(all_files) +end + +--------------------------------------------------------------------- +local version_pattern = { + "^lib/python%d%.%d/", + "^share/py3%d%d%?-", + "^share/%a+/py3%d%d%?-", + "^lib/lua/%d%.%d/", + "^share/lua/%d%.%d/", + "^lib/perl5/[%d%.]+/", + "^lib/perl5/site_perl/mach/[%d%.]+/", + "^lib/ruby/gems/%d%.%d/", + "^lib/ruby/site_ruby/%d%.%d/", +} + +local function generalize_patterns(pkgs, files) + local function match_any(str, pattern_array) + for _, pattern in ipairs(pattern_array) do + if string.match(str, pattern) then + return true + end + end + return false + end + local function unversioned_files() + for i = 1, #files do + if not match_any(files[i], version_pattern) then + return true + end + end + return false + end + local function pkg_wildcards(from, ...) + local to_list = {...} + local result = {} + for i = 1, #pkgs do + local orig_pkg = pkgs[i] + for _, to in ipairs(to_list) do + result[string.gsub(orig_pkg, from, to)] = true + end + end + pkgs = table_sorted_keys(result) + end + local pkg_pfx_php = "php[0-9][0-9]-" + local pkg_sfx_php = "-php[0-9][0-9]" + local pkg_pfx_python2 + local pkg_sfx_python2 + local pkg_pfx_python3 + local pkg_sfx_python3 + local pkg_pfx_lua + local pkg_pfx_ruby + pkgs = table_sort_uniq(pkgs) + if unversioned_files() then + pkg_pfx_python2 = "py3[0-9]-" -- e.g. py39- + pkg_sfx_python2 = "-py3[0-9]" + pkg_pfx_python3 = "py3[0-9][0-9]-" -- e.g. py311- + pkg_sfx_python3 = "-py3[0-9][0-9]" + pkg_pfx_lua = "lua[0-9][0-9]-" + pkg_pfx_ruby = "ruby[0-9][0-9]-" + else + pkg_pfx_python2 = "${PYTHON_PKGNAMEPREFIX}" + pkg_sfx_python2 = "${PYTHON_PKGNAMESUFFIX}" + pkg_pfx_python3 = nil + pkg_sfx_python3 = nil + pkg_pfx_lua = "${LUA_PKGNAMEPREFIX}" + pkg_pfx_ruby = "${RUBY_PKGNAMEPREFIX}" + end + pkg_wildcards("^php%d%d%-", pkg_pfx_php) + pkg_wildcards("-php%d%d$", pkg_sfx_php) + pkg_wildcards("^phpunit%d%-", "phpunit[0-9]-") + pkg_wildcards("^py3%d%-", pkg_pfx_python2, pkg_pfx_python3) + pkg_wildcards("-py3%d%$", pkg_sfx_python2, pkg_sfx_python3) + pkg_wildcards("^py3%d%d%-", pkg_pfx_python2, pkg_pfx_python3) + pkg_wildcards("-py3%d%d%$", pkg_sfx_python2, pkg_sfx_python3) + pkg_wildcards("^lua%d%d%-", pkg_pfx_lua) + pkg_wildcards("-emacs_[%a_]*", "-emacs_*") + pkg_wildcards("^ghostscript%d%-", "ghostscript[0-9]-") + pkg_wildcards("^bacula%d%-", "bacula[0-9]-") + pkg_wildcards("^bacula%d%d%-", "bacula[0-9][0-9]-") + pkg_wildcards("^bareos%d%-", "bareos[0-9]-") + pkg_wildcards("^bareos%d%d%-", "bareos[0-9][0-9]-") + pkg_wildcards("^moosefs%d%-", "moosefs[0-9]-") + pkg_wildcards("^ruby%d+-", pkg_pfx_ruby) + return table_sort_uniq(pkgs) +end + +------------------------------------------------------------------- +for _, port in ipairs(PORT_LIST) do + local function merge_table(t1, t2) + table.move(t2, 1, #t2, #t1 + 1, t1) + end + local port_conflicts = {} + local files = {} + local msg_files = {} + local conflict_pkgs = {} + local function merge_data(origin) + local pkgbase = PKGBASE[origin] + if not BAD_FILE_PKGS[pkgbase] then + local pkg_confl_file1, pkg_confl_files = conflicting_files(pkgbase, CONFLICT_PKGS[pkgbase]) + merge_table(msg_files, pkg_confl_file1) -- 1 file per flavor + merge_table(files, pkg_confl_files) -- all conflicting files + merge_table(conflict_pkgs, CONFLICT_PKGS[pkgbase]) + merge_table(port_conflicts, fetch_port_conflicts(origin)) + end + end + local flavors = FLAVORS[port] + if flavors then + for _, flavor in ipairs(flavors) do + merge_data(port .. "@" .. flavor) + end + else + merge_data(port) + end + files = table_sort_uniq(files) + msg_files = table_sort_uniq(msg_files) + conflict_pkgs = generalize_patterns(conflict_pkgs, files) + if #port_conflicts then + port_conflicts = table_sort_uniq(port_conflicts) + conflict_pkgs = conflicts_delta(port_conflicts, conflict_pkgs) + end + if conflict_pkgs then + local conflicts_string_cur = table.concat(port_conflicts, " ") + local conflicts_string_new = table.concat(conflict_pkgs, " ") + local file_list = table.concat(msg_files, " ") + print("# Port: " .. port) + print("# Files: " .. file_list) + if conflicts_string_cur ~= "" then + print("# < " .. conflicts_string_cur) + end + print("# > " .. conflicts_string_new) + print("portedit merge -ie 'CONFLICTS_INSTALL=" .. conflicts_string_new .. + " # " .. file_list .. "' /usr/ports/" .. port) + print() + end +end + +------------------------------------------------------------------- +local BAD_FILES_ORIGINS = {} + +for _, origin in ipairs(ORIGIN_LIST) do + local pkgbase = PKGBASE[origin] + local files = BAD_FILE_PKGS[pkgbase] + if files then + for _, file in ipairs(files) do + if DUPLICATE_FILE[file] then + local port = string.match(origin, "([^@]+)@?") + BAD_FILES_ORIGINS[port] = BAD_FILES_ORIGINS[origin] or {} + table.insert(BAD_FILES_ORIGINS[port], file) + end + end + end +end + +------------------------------------------------------------------- +local bad_origins = table_sorted_keys(BAD_FILES_ORIGINS) + +if #bad_origins > 0 then + print ("# Ports with badly named files:") + print () + for _, port in ipairs(bad_origins) do + print ("# " .. port) + local files = BAD_FILES_ORIGINS[port] + table.sort(files) + for _, file in ipairs(files) do + print ("#", file) + end + print() + end +end |