This is the mail archive of the
cygwin-apps-cvs
mailing list for the cygwin-apps project.
[calm - Cygwin server-side packaging maintenance script] branch master, updated. 20160705-95-g63037d2
- From: jturney at sourceware dot org
- To: cygwin-apps-cvs at sourceware dot org
- Date: 4 Oct 2017 14:12:45 -0000
- Subject: [calm - Cygwin server-side packaging maintenance script] branch master, updated. 20160705-95-g63037d2
https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=63037d2cfeb1caf0929d30fd170927dcb1d0a84e
commit 63037d2cfeb1caf0929d30fd170927dcb1d0a84e
Author: Jon Turney <jon.turney@dronecode.org.uk>
Date: Tue Oct 3 21:48:58 2017 +0100
Fix a problem with version sorting
Non-alphanumeric separators (i.e. '.') were not ignored as per rpmvercmp.
This led to incorrect sorting for e.g. 15.8b-1 vs 15.8.0.1-2
https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=1edd7bfc022bc1d1e16517ee3af0d6c3765e1e73
commit 1edd7bfc022bc1d1e16517ee3af0d6c3765e1e73
Author: Jon Turney <jon.turney@dronecode.org.uk>
Date: Mon Oct 2 23:10:38 2017 +0100
Remove unused SetupVersion._warn_ambiguous_compare
https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=484dafa7e4ff07fb4fb423697a96b46532ebaaf3
commit 484dafa7e4ff07fb4fb423697a96b46532ebaaf3
Author: Jon Turney <jon.turney@dronecode.org.uk>
Date: Thu Jun 8 16:02:12 2017 +0100
Add a tool for de-duplicating a source package
https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=7cb33c04ecb1ddd0aee793864a95d89d5221fa49
commit 7cb33c04ecb1ddd0aee793864a95d89d5221fa49
Author: Jon Turney <jon.turney@dronecode.org.uk>
Date: Sat Apr 8 19:52:27 2017 +0100
Generate package listing page for source-only packages
Note that we may not have a good sdesc for these packages
https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=d51e7d75d48e1940840a000b50473175e0748e35
commit d51e7d75d48e1940840a000b50473175e0748e35
Author: Jon Turney <jon.turney@dronecode.org.uk>
Date: Fri Jan 20 15:43:50 2017 +0000
Make source packages a thing
Look for source packages in src/
Write Source: lines in setup.ini to reference them
Future work: could do with an indication that a package is a source package,
rather than a heuristic based on package name?
Diff:
---
calm/calm.py | 30 ++++++-----
calm/dedupsrc.py | 156 +++++++++++++++++++++++++++++++++++++++++++++++++++++
calm/package.py | 21 +++++---
calm/pkg2html.py | 27 ++++++---
calm/version.py | 38 ++-----------
setup.py | 1 +
test/test_calm.py | 18 +++++-
7 files changed, 225 insertions(+), 66 deletions(-)
diff --git a/calm/calm.py b/calm/calm.py
index cfffb2f..72325f7 100755
--- a/calm/calm.py
+++ b/calm/calm.py
@@ -112,7 +112,7 @@ def process_relarea(args):
if args.stale:
stale_to_vault = remove_stale_packages(args, packages)
if stale_to_vault:
- for arch in common_constants.ARCHES + ['noarch']:
+ for arch in common_constants.ARCHES + ['noarch', 'src']:
logging.info("vaulting %d old package(s) for arch %s, which are no longer accessible by installer" % (len(stale_to_vault[arch]), arch))
uploads.move_to_vault(args, stale_to_vault[arch])
else:
@@ -143,7 +143,7 @@ def process_uploads(args, state):
# for each arch and noarch
scan_result = {}
skip_maintainer = False
- for arch in common_constants.ARCHES + ['noarch']:
+ for arch in common_constants.ARCHES + ['noarch', 'src']:
logging.debug("reading uploaded arch %s packages from maintainer %s" % (arch, name))
# read uploads
@@ -176,7 +176,7 @@ def process_uploads(args, state):
logging.debug("merging %s package set with uploads from maintainer %s" % (arch, name))
# merge package sets
- merged_packages[arch] = package.merge(state.packages[arch], scan_result[arch].packages, scan_result['noarch'].packages)
+ merged_packages[arch] = package.merge(state.packages[arch], scan_result[arch].packages, scan_result['noarch'].packages, scan_result['src'].packages)
if not merged_packages[arch]:
logging.error("error while merging uploaded %s packages for %s" % (arch, name))
valid = False
@@ -211,7 +211,7 @@ def process_uploads(args, state):
# check for conflicting movelists
conflicts = False
- for arch in common_constants.ARCHES + ['noarch']:
+ for arch in common_constants.ARCHES + ['noarch', 'src']:
conflicts = conflicts or report_movelist_conflicts(scan_result[arch].to_relarea, scan_result[arch].to_vault, "manually")
if args.stale:
conflicts = conflicts or report_movelist_conflicts(scan_result[arch].to_relarea, stale_to_vault[arch], "automatically")
@@ -223,7 +223,7 @@ def process_uploads(args, state):
continue
# for each arch and noarch
- for arch in common_constants.ARCHES + ['noarch']:
+ for arch in common_constants.ARCHES + ['noarch', 'src']:
logging.debug("moving %s packages for maintainer %s" % (arch, name))
# process the move lists
@@ -237,7 +237,7 @@ def process_uploads(args, state):
# for each arch
if args.stale:
- for arch in common_constants.ARCHES + ['noarch']:
+ for arch in common_constants.ARCHES + ['noarch', 'src']:
if stale_to_vault[arch]:
logging.info("vaulting %d old package(s) for arch %s, which are no longer accessible by installer" % (len(stale_to_vault[arch]), arch))
uploads.move_to_vault(args, stale_to_vault[arch])
@@ -246,7 +246,7 @@ def process_uploads(args, state):
for arch in common_constants.ARCHES:
# use merged package list
state.packages[arch] = merged_packages[arch]
- logging.debug("added %d + %d packages from maintainer %s" % (len(scan_result[arch].packages), len(scan_result['noarch'].packages), name))
+ logging.debug("added %d (%s) + %d (noarch) + %d (src) packages from maintainer %s" % (len(scan_result[arch].packages), arch, len(scan_result['noarch'].packages), len(scan_result['src'].packages), name))
# record updated reminder times for maintainers
maintainers.Maintainer.update_reminder_times(mlist)
@@ -280,6 +280,7 @@ def process(args, state):
def remove_stale_packages(args, packages):
to_vault = {}
to_vault['noarch'] = defaultdict(list)
+ to_vault['src'] = defaultdict(list)
for arch in common_constants.ARCHES:
logging.debug("checking for stale packages for arch %s" % (arch))
@@ -308,16 +309,17 @@ def remove_stale_packages(args, packages):
if error:
return None
- # since noarch packages are included in the package set for both arch, we
- # will build (hopefully) identical move lists for those packages for each
- # arch.
+ # since noarch and src packages are included in the package set for both
+ # arch, we will build (hopefully) identical move lists for those packages
+ # for each arch.
#
# de-duplicate these package moves, as rather awkward workaround for that
for path in list(to_vault[common_constants.ARCHES[0]]):
- if path.startswith('noarch'):
- to_vault['noarch'][path] = to_vault[common_constants.ARCHES[0]][path]
- for arch in common_constants.ARCHES:
- del to_vault[arch][path]
+ for prefix in ['noarch', 'src']:
+ if path.startswith(prefix):
+ to_vault[prefix][path] = to_vault[common_constants.ARCHES[0]][path]
+ for arch in common_constants.ARCHES:
+ del to_vault[arch][path]
return to_vault
diff --git a/calm/dedupsrc.py b/calm/dedupsrc.py
new file mode 100755
index 0000000..f531a2e
--- /dev/null
+++ b/calm/dedupsrc.py
@@ -0,0 +1,156 @@
+#!/usr/bin/env python3
+#
+# Copyright (c) 2017 Jon Turney
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#
+
+#
+# Move a given source archive to src/ (assuming it is indentical in x86/ and
+# x86_64/) and adjust hints appropriately.
+#
+
+import argparse
+import copy
+import os
+import re
+import sys
+
+from . import common_constants
+from . import hint
+
+#
+#
+#
+
+
+def hint_file_write(fn, hints):
+ with open(fn, 'w') as f:
+ for k, v in hints.items():
+ print("%s: %s" % (k, v), file=f)
+
+#
+#
+#
+
+
+def dedup(archive, relarea):
+ # split path and filename
+ (path, filename) = os.path.split(archive)
+
+ # parse tarfile name
+ match = re.match(r'^(.+?)-(\d.*)-src\.tar\.(bz2|gz|lzma|xz)$', filename)
+
+ if not match:
+ print('tarfile name %s does not meet expectations' % (filename))
+ sys.exit(1)
+
+ p = match.group(1)
+ vr = match.group(2)
+ ext = match.group(3)
+
+ # compute filenames
+ to_filename = p + '-src-' + vr + '.tar.' + ext
+ hint_filename = p + '-' + vr + '.hint'
+ to_hint_filename = p + '-src-' + vr + '.hint'
+
+ # read hints for both arches
+ hints = {}
+ for arch in ['x86', 'x86_64']:
+ hint_pathname = os.path.join(relarea, arch, path, hint_filename)
+
+ if not os.path.exists(hint_pathname):
+ print('%s not found' % (hint_pathname))
+ return 1
+
+ hints[arch] = hint.hint_file_parse(hint_pathname, hint.pvr)
+
+ if hints['x86'] != hints['x86_64']:
+ print('hints for %s-%s differ between arches' % (p, vr))
+ return 1
+
+ # ensure target directory exists
+ try:
+ os.makedirs(os.path.join(relarea, 'src', path, p + '-src'))
+ except FileExistsError:
+ pass
+
+ # move the src files to src/
+ for arch in ['x86', 'x86_64']:
+ print('%s -> %s' % (os.path.join(relarea, arch, path, filename), os.path.join(relarea, 'src', path, p + '-src', to_filename)))
+ os.rename(os.path.join(relarea, arch, path, filename), os.path.join(relarea, 'src', path, p + '-src', to_filename))
+
+ # write .hint file for new -src package
+ src_hints = copy.copy(hints['x86'])
+
+ if 'source' not in src_hints['sdesc']:
+ sdesc = re.sub(r'"(.*)"', r'\1', src_hints['sdesc'])
+ sdesc += ' (source code)'
+ src_hints['sdesc'] = '"' + sdesc + '"'
+
+ if 'requires' in src_hints:
+ del src_hints['requires']
+
+ if 'external-source' in src_hints:
+ del src_hints['external-source']
+
+ to_hint_pathname = os.path.join(relarea, 'src', path, p + '-src', to_hint_filename)
+ print('writing %s' % (to_hint_pathname))
+ hint_file_write(to_hint_pathname, src_hints)
+
+ # adjust external-source in .hint for all subpackages
+ for arch in ['x86', 'x86_64']:
+ for (dirpath, subdirs, files) in os.walk(os.path.join(relarea, arch, path)):
+ subpkg = os.path.basename(dirpath)
+ filename = subpkg + '-' + vr + '.hint'
+ if filename in files:
+ hint_pathname = os.path.join(dirpath, filename)
+ hints = hint.hint_file_parse(hint_pathname, hint.pvr)
+ if ('skip' in hints):
+ # p was source only, so no package remains
+ print('removing %s' % (hint_pathname))
+ os.remove(hint_pathname)
+ elif ('external-source' not in hints) or (hints['external-source'] == p):
+ hints['external-source'] = p + '-src'
+ print('writing %s' % (hint_pathname))
+ hint_file_write(hint_pathname, hints)
+
+ return 0
+
+#
+#
+#
+
+
+def main():
+ relarea_default = common_constants.FTP
+
+ parser = argparse.ArgumentParser(description='Source package deduplicator')
+ parser.add_argument('archive', metavar='ARCHIVE', nargs=1, help="source archive to deduplicate")
+ parser.add_argument('--releasearea', action='store', metavar='DIR', help="release directory (default: " + relarea_default + ")", default=relarea_default, dest='rel_area')
+ (args) = parser.parse_args()
+
+ return dedup(args.archive[0], args.rel_area)
+
+#
+#
+#
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/calm/package.py b/calm/package.py
index 07ef02b..23954dd 100755
--- a/calm/package.py
+++ b/calm/package.py
@@ -82,8 +82,8 @@ class Tar(object):
def read_packages(rel_area, arch):
packages = defaultdict(Package)
- # both noarch/ and <arch>/ directories are considered
- for root in ['noarch', arch]:
+ # <arch>/ noarch/ and src/ directories are considered
+ for root in ['noarch', 'src', arch]:
releasedir = os.path.join(rel_area, root)
logging.debug('reading packages from %s' % releasedir)
@@ -423,7 +423,7 @@ def validate_packages(args, packages):
for (t, tar) in packages[p].tars.items():
# categorize each tarfile as either 'source' or 'install'
- if re.search(r'-src\.tar', t):
+ if re.search(r'-src.*\.tar', t):
category = 'source'
else:
category = 'install'
@@ -736,7 +736,7 @@ def write_setup_ini(args, packages, arch):
# for each package
for p in sorted(packages.keys(), key=sort_key):
# do nothing if 'skip'
- if packages[p].skip:
+ if packages[p].skip and not p.endswith('-src'):
continue
# write package data
@@ -784,11 +784,16 @@ def write_setup_ini(args, packages, arch):
# if that doesn't exist, follow external-source
elif 'external-source' in packages[p].version_hints[version]:
s = packages[p].version_hints[version]['external-source']
- if 'source' in packages[s].vermap[version]:
- t = packages[s].vermap[version]['source']
- tar_line('source', packages[s], t, f)
+ # external-source points to a real source package (-src)
+ if s.endswith('-src'):
+ print("Source: %s" % (s), file=f)
+ # external-source points to a source file in another package
else:
- logging.warning("package '%s' version '%s' has no source in external-source '%s'" % (p, version, s))
+ if 'source' in packages[s].vermap[version]:
+ t = packages[s].vermap[version]['source']
+ tar_line('source', packages[s], t, f)
+ else:
+ logging.warning("package '%s' version '%s' has no source in external-source '%s'" % (p, version, s))
# helper function to output details for a particular tar file
diff --git a/calm/pkg2html.py b/calm/pkg2html.py
index 9b23a94..83308b6 100755
--- a/calm/pkg2html.py
+++ b/calm/pkg2html.py
@@ -55,6 +55,22 @@ from . import package
#
+# get sdesc for a package
+#
+# some source-only packages don't have an sdesc, since they consist of just
+# 'skip':', in which case we try to make a reasonable one
+#
+
+def desc(packages, p, bv):
+ if 'sdesc' in packages[p].version_hints[bv]:
+ header = packages[p].version_hints[bv]['sdesc']
+ else:
+ header = p
+
+ return header.replace('"', '')
+
+
+#
#
#
@@ -88,10 +104,6 @@ def update_package_listings(args, packages, arch):
for p in packages:
- # do nothing for packages marked 'skip'
- if packages[p].skip:
- continue
-
dir = os.path.join(base, p)
if not args.dryrun:
try:
@@ -140,7 +152,7 @@ def update_package_listings(args, packages, arch):
if not args.dryrun:
with open(listing, 'w') as f:
bv = packages[p].best_version
- header = p + ": " + packages[p].version_hints[bv]['sdesc'].replace('"', '')
+ header = p + ": " + desc(packages, p, bv)
if fver.endswith('-src'):
header = header + " (source code)"
@@ -210,12 +222,9 @@ def update_package_listings(args, packages, arch):
<table class="pkglist">''') % (arch, arch), file=index)
for p in sorted(packages.keys(), key=package.sort_key):
- # don't write anything if 'skip'
- if packages[p].skip:
- continue
bv = packages[p].best_version
- header = packages[p].version_hints[bv]['sdesc'].replace('"', '')
+ header = desc(packages, p, bv)
print('<tr><td><a href="' + arch + '/' + p + '">' + p + '</a></td><td>' + html.escape(header, quote=False) + '</td></tr>', file=index)
diff --git a/calm/version.py b/calm/version.py
index 027372f..f00ddbb 100644
--- a/calm/version.py
+++ b/calm/version.py
@@ -22,7 +22,6 @@
#
import itertools
-import logging
import re
@@ -45,10 +44,14 @@ class SetupVersion:
# split version into [V, R], on the last '-', if any
split = list(itertools.chain(version_string.rsplit('-', 1), ['']))[:2]
- # then split each part into numeric and non-numeric sequences.
+ # then split each part into numeric and alphabetic sequences
+ # non-alphanumeric separators are discarded
# numeric sequences have leading zeroes discarded
for j, i in enumerate(['V', 'R']):
- setattr(self, '_' + i, [re.sub(r'^0+(\d)', r'\1', m.group(1), 1) for m in re.finditer(r'(\d+|\D+)', split[j])])
+ sequences = re.finditer(r'(\d+|[a-zA-Z]+|[^a-zA-Z\d]+)', split[j])
+ sequences = [m for m in sequences if not re.match(r'[^a-zA-Z\d]+', m.group(1))]
+ sequences = [re.sub(r'^0+(\d)', r'\1', m.group(1), 1) for m in sequences]
+ setattr(self, '_' + i, sequences)
def __str__(self):
return '%s (V=%s R=%s)' % (self._version_string, str(self._V), str(self._R))
@@ -63,8 +66,6 @@ class SetupVersion:
return self.__cmp__(other) == -1
def __cmp__(self, other):
- # warn about ill-specified comparisons
- # SetupVersion._warn_ambiguous_compare(self, other)
# compare V
c = SetupVersion._compare(self._V, other._V)
@@ -101,30 +102,3 @@ class SetupVersion:
# if equal length, all components have matched, so equal
# otherwise, the version with a suffix remaining is greater
return cmp(len(a), len(b))
-
- # warn if the comparison of these versions is historically under-specified
- @staticmethod
- def _warn_ambiguous_compare(a, b):
- def classify(s):
- if len(s) == 0:
- return 'e'
- elif s[0].isdigit():
- return 'n'
- elif s[0] in '.-_':
- return s[0]
- elif s[0].isalpha():
- return 'a'
- return 'o'
-
- def is_ambiguous(a, b):
- ambiguous = False
-
- for i in range(0, min(len(a), len(b))):
- if classify(a[i]) != classify(b[i]):
- ambiguous = True
- break
-
- return ambiguous
-
- if is_ambiguous(a._V, b._V) or is_ambiguous(a._R, b._R):
- logging.warning("ordering of versions '%s' and '%s' may not be what you expect" % (a._version_string, b._version_string))
diff --git a/setup.py b/setup.py
index 2941fa4..cdc8121 100644
--- a/setup.py
+++ b/setup.py
@@ -14,6 +14,7 @@ setup(
'calm = calm.calm:main',
'mksetupini = calm.mksetupini:main',
'calm-mkgitoliteconf = calm.mkgitoliteconf:main',
+ 'dedup-source = calm.dedupsrc:main',
],
},
url='https://cygwin.com/git/?p=cygwin-apps/calm.git',
diff --git a/test/test_calm.py b/test/test_calm.py
index 30ada6f..d1cf3af 100755
--- a/test/test_calm.py
+++ b/test/test_calm.py
@@ -153,13 +153,25 @@ class CalmTest(unittest.TestCase):
["1.3.30c-2", "1.3.30c-10", -1],
["2.24.51-1", "2.25-1", -1],
["2.1.5+20120813+gitdcbe778-1", "2.1.5-3", 1],
- ["3.4.1-1", "3.4b1-1", -1],
+ ["3.4.1-1", "3.4b1-1", 1],
["041206-1", "200090325-1", -1],
["0.6.2+git20130413-2", "0.6.2-1", 1],
["2.6.0+bzr6602-1", "2.6.0-2", 1],
- ["2.6.0-2", "2.6b2-1", -1],
- ["2.6.0+bzr6602-1", "2.6b2-1", -1],
+ ["2.6.0-2", "2.6b2-1", 1],
+ ["2.6.0+bzr6602-1", "2.6b2-1", 1],
["0.6.7+20150214+git3a710f9-1", "0.6.7-1", 1],
+ ["15.8b-1", "15.8.0.1-2", -1],
+ ["1.2rc1-1","1.2.0-2", -1],
+ # examples from https://fedoraproject.org/wiki/Archive:Tools/RPM/VersionComparison
+ ["1.0010", "1.9", 1],
+ ["1.05", "1.5", 0],
+ ["1.0", "1", 1],
+ ["2.50", "2.5", 1],
+ ["fc4", "fc.4", 0],
+ ["FC5", "fc4", -1],
+ ["2a", "2.0", -1],
+ ["1.0", "1.fc4", 1],
+ ["3.0.0_fc", "3.0.0.fc", 0],
]
for d in test_data: