From 96c09e9321f3a9a7f5aafb0edb70e98f19b7bf7e Mon Sep 17 00:00:00 2001 From: Jerry Date: Wed, 3 Apr 2019 18:10:00 +0800 Subject: [PATCH] repo.py full functionality --- config.py | 19 +++++++--- repo.py | 101 ++++++++++++++++++++++++++++++++++++++++++++++-------- utils.py | 43 +++++++++++++++++++++-- 3 files changed, 143 insertions(+), 20 deletions(-) diff --git a/config.py b/config.py index 5979d94..54e249d 100644 --- a/config.py +++ b/config.py @@ -1,11 +1,22 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -# config -REPO_NAME='jerryxiao' -PKG_COMPRESSION='xz' +#### config for all ARCHS = ['aarch64', 'any', 'armv7h', 'x86_64'] +REPO_NAME='jerryxiao' +PKG_COMPRESSION='xz' BUILD_ARCHS = ['aarch64', 'any', 'x86_64'] -REPO_CMD = 'repo-add --verify --remove' \ No newline at end of file +#### config for repo.py +REPO_CMD = 'repo-add --verify --remove' +RECENT_VERSIONS_KEPT = 3 +PREFERRED_ANY_BUILD_ARCH = 'x86_64' + +#### config for package.py +# Archlinux-Jerry Build Bot +GPG_KEY = 'BEE4F1D5A661CA1FEA65C38093962CE07A0D5B7D' +GPG_SIGN_CMD = (f'gpg --default-key {GPG_KEY} --no-armor' + '--pinentry-mode loopback --passphrase \'\'' + '--detach-sign --yes -- ') + diff --git a/repo.py b/repo.py index 6225cd0..a213b3d 100755 --- a/repo.py +++ b/repo.py @@ -7,9 +7,10 @@ # buildbot -- buildbot (git) # buildbot/repo -- repo root # /updates/ -- new packages goes in here - # /updates/archive -- archive dir, old packages goes in here + # /recycled/ -- litter bin + # /archive/ -- archive dir, old packages goes in here # /www/ -- http server root - # /www/archive => /updates/archive -- archive dir for users + # /www/archive => /archive -- archive dir for users # /www/aarch64 -- packages for "aarch64" # /www/any -- packages for "any" # /www/armv7h -- packages for "armv7h" (No build bot) @@ -18,8 +19,10 @@ import os from pathlib import Path +from shutil import copyfile as __copy_file import logging -from utils import bash, Pkg, get_pkg_details_from_name, print_exc_plus +from utils import bash, Pkg, get_pkg_details_from_name, \ + print_exc_plus from time import time import argparse @@ -40,16 +43,21 @@ def symlink(dst, src, exist_ok=True): try: dst.symlink_to(src) except FileExistsError: - if not exist_ok: + if (not dst.is_symlink()) or (not exist_ok): raise -def checkenv(): - (Path(abspath).parent / 'recycled').mkdir(mode=0o755, exist_ok=True) - dirs = [Path('updates/archive')] + [Path('www/') / arch for arch in ARCHS] +def copyfile(src, dst): + src = str(src) + dst = str(dst) + __copy_file(src, dst, follow_symlinks=False) + +def prepare_env(): + dirs = [Path('updates/'), Path('archive/'), Path('recycled/')] + \ + [Path('www/') / arch for arch in ARCHS] for mydir in dirs: mydir.mkdir(mode=0o755, exist_ok=True, parents=True) - symlink(Path('www/archive'), '../updates/archive') -checkenv() + symlink(Path('www/archive'), '../archive') +prepare_env() def repo_add(fpaths): @@ -62,11 +70,68 @@ def repo_add(fpaths): def throw_away(fpath): assert issubclass(type(fpath), os.PathLike) - newPath = Path(abspath).parent / 'recycled' / f"{fpath.name}_{time()}" + newPath = Path('recycled') / f"{fpath.name}_{time()}" assert not newPath.exists() logger.warning('Throwing away %s', fpath) fpath.rename(newPath) +def archive_pkg(fpath): + assert issubclass(type(fpath), os.PathLike) + if fpath.is_symlink(): + logger.warning('Not archiving symlink %s', fpath) + throw_away(fpath) + return + newPath = Path('archive') / fpath.name + assert not newPath.exists() + logger.warning('Archiving %s', fpath) + fpath.rename(newPath) + +def filter_old_pkg(fpaths, keep_new=1, archive=False, recycle=False): + ''' + Accepts a list of paths (must be in the same dir) + return a tuple of list of paths + ([new1, new2], [old1, old2]) + packages are arranged from new to old, one by one. + new: pkga-v8, pkga-v7, pkgb-v5, pkgb-v4 + old: pkga-v6, pkga-v5, pkgb-v3, pkgb-v2 + (assume keep_new=2) + ''' + if not fpaths: + return (list(), list()) + assert type(fpaths) is list + for fpath in fpaths: + assert issubclass(type(fpath), os.PathLike) and \ + fpath.name.endswith(PKG_SUFFIX) + assert not (archive and recycle) + assert not [None for fpath in fpaths if fpath.parent != fpaths[0].parent] + + new_pkgs = list() + old_pkgs = list() + pkgs_vers = dict() + for fpath in fpaths: + pkg = get_pkg_details_from_name(fpath.name) + pkgs_vers.setdefault(pkg.pkgname, list()).append(pkg) + for pkgname in pkgs_vers: + family = pkgs_vers[pkgname] + # new packages first + family = sorted(family, reverse=True) + if len(family) > keep_new: + new_pkgs += family[:keep_new] + old_pkgs += family[keep_new:] + else: + new_pkgs += family + for pkg in old_pkgs: + fullpath = fpaths[0].parent / pkg.fname + if archive: + archive_pkg(fullpath) + elif recycle: + throw_away(fullpath) + return (new_pkgs, old_pkgs) + +def _clean_archive(keep_new=3): + basedir = Path('archive') + dir_list = [fpath for fpath in basedir.iterdir()] + filter_old_pkg(dir_list, keep_new=keep_new, recycle=True) def _regenerate(target_archs=ARCHS, just_symlink=False): if just_symlink: @@ -106,6 +171,7 @@ def _regenerate(target_archs=ARCHS, just_symlink=False): logger.error(f'{arch} dir does not exist!') continue pkgfiles = [f for f in basedir.iterdir()] + filter_old_pkg(pkgfiles, keep_new=1, recycle=True) for pkgfile in pkgfiles: if pkgfile.name in repo_files: repo_files_count.append(pkgfile.name) @@ -148,7 +214,9 @@ def _update(): update_path = Path('updates') assert update_path.exists() pkgs_to_add = dict() - for pkg_to_add in update_path.iterdir(): + dir_list = [fpath for fpath in update_path.iterdir()] + filter_old_pkg(dir_list, keep_new=1, archive=True) + for pkg_to_add in dir_list: if pkg_to_add.is_dir(): continue else: @@ -158,10 +226,12 @@ def _update(): arch = get_pkg_details_from_name(pkg_to_add.name).arch pkg_nlocation = pkg_to_add.parent / '..' / 'www' / arch / pkg_to_add.name sig_nlocation = Path(f'{str(pkg_nlocation)}.sig') - logger.info(f'Moving {pkg_to_add} to {pkg_nlocation}, {sigfile} to {sig_nlocation}') + logger.info(f'Copying {pkg_to_add} to {pkg_nlocation}, {sigfile} to {sig_nlocation}') assert not (pkg_nlocation.exists() or sig_nlocation.exists()) - pkg_to_add.rename(pkg_nlocation) - sigfile.rename(sig_nlocation) + copyfile(pkg_to_add, pkg_nlocation) + copyfile(sigfile, sig_nlocation) + archive_pkg(pkg_to_add) + archive_pkg(sigfile) if arch == 'any': for arch in ARCHS: pkg_nlocation = pkg_to_add.parent / '..' / 'www' / arch / pkg_to_add.name @@ -189,6 +259,7 @@ if __name__ == '__main__': parser.add_argument('-a', '--arch', nargs='?', default='all', help='arch to regenerate, split by comma, defaults to all') parser.add_argument('-u', '--update', action='store_true', help='get updates from updates dir, push them to the repo') parser.add_argument('-r', '--regenerate', action='store_true', help='regenerate the whole package database') + parser.add_argument('-c', '--clean', action='store_true', help='clean archive, keep 3 recent versions') args = parser.parse_args() arch = args.arch arch = arch.split(',') if arch != 'all' else ARCHS @@ -197,6 +268,8 @@ if __name__ == '__main__': _update() elif args.regenerate: _regenerate(target_archs=arch) + elif args.clean: + _clean_archive(keep_new=3) else: parser.error("Please choose an action") except Exception as err: diff --git a/utils.py b/utils.py index 16c2ff0..6b1d0eb 100644 --- a/utils.py +++ b/utils.py @@ -31,6 +31,7 @@ def long_bash(cmdline, hours=2): return bash(cmdline, keepalive=True, KEEPALIVE_TIMEOUT=60, RUN_CMD_TIMEOUT=hours*60*60) def run_cmd(cmd, keepalive=False, KEEPALIVE_TIMEOUT=30, RUN_CMD_TIMEOUT=60): + logger.debug('run_cmd: %s', cmd) RUN_CMD_LOOP_TIME = KEEPALIVE_TIMEOUT - 1 if KEEPALIVE_TIMEOUT >= 10 else 5 stopped = False last_read = [int(time()), ""] @@ -94,12 +95,50 @@ def run_cmd(cmd, keepalive=False, KEEPALIVE_TIMEOUT=30, RUN_CMD_TIMEOUT=60): raise subprocess.CalledProcessError(code, cmd, outstr) return outstr + +# pyalpm is an alternative +# due to lack of documentation i'll consider this later. + +def vercmp(ver1, ver2): + ''' + compare ver1 and ver2, return 1, -1, 0 + see https://www.archlinux.org/pacman/vercmp.8.html + ''' + res = run_cmd(['vercmp', str(ver1), str(ver2)]) + res = res.strip() + if res in ('-1', '0', '1'): + return int(res) + class Pkg: - def __init__(self, pkgname, pkgver, pkgrel, arch): + def __init__(self, pkgname, pkgver, pkgrel, arch, fname): self.pkgname = pkgname self.pkgver = pkgver self.pkgrel = pkgrel self.arch = arch + self.fname = fname + self.ver = f'{self.pkgver}-{self.pkgrel}' + def __eq__(self, ver2): + if vercmp(self.ver, ver2.ver) == 0: + return True + else: + return False + def __ge__(self, ver2): + return self > ver2 or self == ver2 + def __gt__(self, ver2): + if vercmp(self.ver, ver2.ver) == 1: + return True + else: + return False + def __le__(self, ver2): + return self < ver2 or self == ver2 + def __lt__(self, ver2): + if vercmp(self.ver, ver2.ver) == -1: + return True + else: + return False + def __repr__(self): + return f'Pkg({self.pkgname}, {self.ver}, {self.arch})' + def get_pkg_details_from_name(name): assert type(name) is str @@ -107,7 +146,7 @@ def get_pkg_details_from_name(name): m = re.match(r'(.+)-([^-]+)-([^-]+)-([^-]+)\.pkg\.tar\.\w+', name) assert m and m.groups() and len(m.groups()) == 4 (pkgname, pkgver, pkgrel, arch) = m.groups() - return Pkg(pkgname, pkgver, pkgrel, arch) + return Pkg(pkgname, pkgver, pkgrel, arch, name) def print_exc_plus(): """