From 1509183847d062cf0e1aabf82f6b8d461f8afd0e Mon Sep 17 00:00:00 2001 From: Jerry Date: Tue, 2 Apr 2019 22:03:28 +0800 Subject: [PATCH] buildbot: repo.py --- .gitignore | 6 +++ README.md | 25 ++++++++++ config.py | 8 ++++ master.py | 2 + package.py | 13 ++++++ repo.py | 131 +++++++++++++++++++++++++++++++++++++++++++++++++++++ utils.py | 108 +++++++++++++++++++++++++++++++++++++++++++ worker.py | 24 ++++++++++ 8 files changed, 317 insertions(+) create mode 100644 .gitignore create mode 100644 README.md create mode 100644 config.py create mode 100755 master.py create mode 100644 package.py create mode 100755 repo.py create mode 100644 utils.py create mode 100755 worker.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ca8f745 --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ +.vscode/* +__pycache__/ +*.py[cod] +buildbot.log* +buildbot.sql +test \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..b16fd38 --- /dev/null +++ b/README.md @@ -0,0 +1,25 @@ +# Buildbot + +## Typical autobuild.yaml format +### Note +Anything starting with bash will be considered as bash commands. +`e.g. - bash ls -al` +All of the four blocks: updates, prebuild, build, postbuild can be ommited, and their first value will be used. +### Example +``` +updates: + - repo (repo only, it means the package will only be built when a new commit is pushed to repo.) + - git * (* means optional) + - ?? (tbd) +prebuild: + - standard (do nothing) + - git-cherrypick push* + - ?? +build: + - standard (makepkg -s, note that missing aur dependencies will not be resolved.) + - ?? +postbuild: + - standard (sign and upload) + - do_nothing (leave it alone) + - ?? +``` \ No newline at end of file diff --git a/config.py b/config.py new file mode 100644 index 0000000..b730c22 --- /dev/null +++ b/config.py @@ -0,0 +1,8 @@ +# config +REPO_NAME='jerryxiao' +PKG_COMPRESSION='xz' + +ARCHS = ['aarch64', 'any', 'armv7h', 'x86_64'] +BUILD_ARCHS = ['aarch64', 'any', 'x86_64'] + +REPO_CMD = 'repo-add --verify --remove' \ No newline at end of file diff --git a/master.py b/master.py new file mode 100755 index 0000000..56fafa5 --- /dev/null +++ b/master.py @@ -0,0 +1,2 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- diff --git a/package.py b/package.py new file mode 100644 index 0000000..f6ed4d4 --- /dev/null +++ b/package.py @@ -0,0 +1,13 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +import logging +from utils import bash + +logger = logging.getLogger(name='package') + + +# makepkg -o +# makepkg -e +# makepkg --nosign +# makepkg --packagelist +# gpg --default-key {GPG_KEY} --no-armor --pinentry-mode loopback --passphrase '' --detach-sign --yes -- aaa diff --git a/repo.py b/repo.py new file mode 100755 index 0000000..af16982 --- /dev/null +++ b/repo.py @@ -0,0 +1,131 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# repo.py: Automatic management tool for an arch repo. +# This file is part of Buildbot by JerryXiao + +# Directory structure of the repo: +# buildbot -- buildbot (git) +# buildbot/repo -- repo root + # /updates/ -- new packages goes in here + # /updates/archive -- archive dir, old packages goes in here + # /www/ -- http server root + # /www/archive => /updates/archive -- archive dir for users + # /www/aarch64 -- packages for "aarch64" + # /www/any -- packages for "any" + # /www/armv7h -- packages for "armv7h" (No build bot) + # /www/x86_64 -- packages for "x86_64" + # /www/robots.txt => /r_r_n/r.txt -- robots.txt + +import os +from pathlib import Path +import logging +from utils import bash, Pkg, get_pkg_details_from_name +from time import time + +from config import REPO_NAME, PKG_COMPRESSION, ARCHS, REPO_CMD + +abspath = os.path.abspath(__file__) +repocwd = Path(abspath).parent / 'repo' +repocwd.mkdir(mode=0o755, exist_ok=True) +os.chdir(repocwd) + +logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') +logger = logging.getLogger(__name__) + + +def symlink(dst, src, exist_ok=True): + assert issubclass(type(dst), os.PathLike) and type(src) is str + try: + dst.symlink_to(src) + except FileExistsError: + if not exist_ok: + raise + +def checkenv(): + (Path(abspath).parent / 'recycled').mkdir(mode=0o755, exist_ok=True) + dirs = [Path('updates/archive')] + [Path('www/') / arch for arch in ARCHS] + for mydir in dirs: + mydir.mkdir(mode=0o755, exist_ok=True, parents=True) + try: + symlink(Path('www/archive'), 'updates/archive') + except FileExistsError: + pass +checkenv() + + +def repo_add(fpath): + assert issubclass(type(fpath), os.PathLike) and fpath.name.endswith(f'.pkg.tar.{PKG_COMPRESSION}') + dbpath = fpath.parent / f'{REPO_NAME}.db.tar.gz' + return bash(f'{REPO_CMD} {dbpath} {fpath}') + +def throw_away(fpath): + assert issubclass(type(fpath), os.PathLike) + newPath = Path(abspath).parent / 'recycled' / f"{fpath.name}_{time()}" + assert not newPath.exists() + fpath.rename(newPath) + +def _check_repo(): + rn = REPO_NAME + repo_files = (f"{rn}.db {rn}.db.tar.gz {rn}.db.tar.gz.old " + f"{rn}.files {rn}.files.tar.gz {rn}.files.tar.gz.old") + repo_files = repo_files.split(' ') + repo_files_essential = [fname for fname in repo_files if not fname.endswith('.old')] + assert repo_files_essential + # make symlink for arch=any pkgs + basedir = Path('www') / 'any' + if basedir.exists(): + for pkgfile in basedir.iterdir(): + if pkgfile.name.endswith(f'.pkg.tar.{PKG_COMPRESSION}') and \ + get_pkg_details_from_name(pkgfile.name).arch == 'any': + sigfile = Path(f"{str(pkgfile)}.sig") + if sigfile.exists(): + logger.info(f'Creating symlink for {pkgfile}, {sigfile}') + for arch in ARCHS: + if arch == 'any': + continue + symlink(pkgfile.parent / '..' / arch / pkgfile.name, f'../any/{pkgfile.name}') + symlink(sigfile.parent / '..' / arch / sigfile.name, f'../any/{sigfile.name}') + else: + logger.error(f'{arch} dir does not exist!') + # run repo_add + for arch in ARCHS: + basedir = Path('www') / arch + repo_files_count = list() + if not basedir.exists(): + logger.error(f'{arch} dir does not exist!') + continue + pkgfiles = [f for f in basedir.iterdir()] + for pkgfile in pkgfiles: + if pkgfile.name in repo_files: + repo_files_count.append(pkgfile.name) + continue + if pkgfile.name.endswith(f'.pkg.tar.{PKG_COMPRESSION}.sig'): + if not Path(str(pkgfile)[:-4]).exists() and pkgfile.exists(): + logger.warning(f"{pkgfile} has no package!") + throw_away(pkgfile) + continue + elif pkgfile.name.endswith(f'.pkg.tar.{PKG_COMPRESSION}'): + sigfile = Path(f"{str(pkgfile)}.sig") + if not sigfile.exists(): + logger.warning(f"{pkgfile} has no signature!") + throw_away(pkgfile) + continue + realarch = get_pkg_details_from_name(pkgfile.name).arch + if realarch != 'any' and realarch != arch: + newpath = pkgfile.parent / '..' / realarch / pkgfile.name + assert not newpath.exists() + pkgfile.rename(newpath) + newSigpath = pkgfile.parent / '..' / realarch / f"{pkgfile.name}.sig" + assert not newSigpath.exists() + sigfile.rename(newSigpath) + logger.info(f'Moving {pkgfile} to {newpath}, {sigfile} to {newSigpath}') + logger.debug("repo-add: %s", repo_add(newpath)) + else: + logger.debug("repo-add: %s", repo_add(pkgfile)) + else: + logger.warning(f"{pkgfile} is garbage!") + throw_away(pkgfile) + for rfile in repo_files_essential: + if rfile not in repo_files_count: + logger.error(f'{rfile} does not exist in {arch}!') + diff --git a/utils.py b/utils.py new file mode 100644 index 0000000..0e80282 --- /dev/null +++ b/utils.py @@ -0,0 +1,108 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +import subprocess +import logging +from time import time +import re +from threading import Thread, Lock + +from config import PKG_COMPRESSION + +logger = logging.getLogger(name='utils') + +def background(func): + def wrapped(*args, **kwargs): + tr = Thread(target=func, args=args, kwargs=kwargs) + tr.daemon = True + tr.start() + return tr + return wrapped + +def bash(cmdline, **kwargs): + assert type(cmdline) is str + logger.info(f'bash: {cmdline}') + return(run_cmd(['/bin/bash', '-x', '-e', '-c', cmdline], **kwargs)) + +def long_bash(cmdline, hours=2): + assert type(hours) is int and hours >= 1 + logger.info(f'longbash{hours}: {cmdline}') + return bash(cmdline, keepalive=True, KEEPALIVE_TIMEOUT=60, RUN_CMD_TIMEOUT=hours*60*60) + +def run_cmd(cmd, keepalive=False, KEEPALIVE_TIMEOUT=30, RUN_CMD_TIMEOUT=60): + RUN_CMD_LOOP_TIME = KEEPALIVE_TIMEOUT - 1 if KEEPALIVE_TIMEOUT >= 10 else 5 + stopped = False + last_read = [int(time()), ""] + output = list() + stdout_lock = Lock() + @background + def check_stdout(stdout): + nonlocal stopped, last_read, output + stdout_lock.acquire() + last_read_time = int(time()) + while stopped is False: + line = stdout.readline(4096) + last_read_time = int(time()) + logger.debug(line) + output.append(line) + last_read[0] = last_read_time + last_read[1] = line + stdout_lock.release() + p = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, encoding='utf-8') + check_stdout(p.stdout) + process_start = int(time()) + while True: + try: + p.wait(timeout=RUN_CMD_LOOP_TIME) + except subprocess.TimeoutExpired: + time_passed = int(time()) - last_read[0] + if time_passed >= KEEPALIVE_TIMEOUT*2: + logger.info('Timeout expired. No action.') + output.append('+ Buildbot: Timeout expired. No action.\n') + elif time_passed >= KEEPALIVE_TIMEOUT: + if keepalive: + logger.info('Timeout expired, writing nl') + output.append('+ Buildbot: Timeout expired, writing nl\n') + p.stdin.write('\n') + p.stdin.flush() + else: + logger.info('Timeout expired, not writing nl') + output.append('+ Buildbot: Timeout expired, not writing nl\n') + if int(time()) - process_start >= RUN_CMD_TIMEOUT: + stopped = True + logger.error('Process timeout expired, terminating.') + output.append('+ Buildbot: Process timeout expired, terminating.\n') + p.terminate() + try: + p.wait(timeout=10) + except subprocess.TimeoutExpired: + logger.error('Cannot terminate, killing.') + output.append('+ Buildbot: Cannot terminate, killing.\n') + p.kill() + break + else: + stopped = True + break + code = p.returncode + + stdout_lock.acquire(10) + outstr = ''.join(output) + + if code != 0: + raise subprocess.CalledProcessError(code, cmd, outstr) + return outstr + +class Pkg: + def __init__(self, pkgname, pkgver, pkgrel, arch): + self.pkgname = pkgname + self.pkgver = pkgver + self.pkgrel = pkgrel + self.arch = arch + +def get_pkg_details_from_name(name): + if name.endswith(f'pkg.tar.{PKG_COMPRESSION}'): + arch = re.match(r'(.+)-([^-]+)-([^-]+)-([^-]+)\.pkg\.tar\.\w+', name) + assert arch and arch.groups() and len(arch.groups()) == 4 + (pkgname, pkgver, pkgrel, arch) = arch.groups() + return Pkg(pkgname, pkgver, pkgrel, arch) + diff --git a/worker.py b/worker.py new file mode 100755 index 0000000..a919215 --- /dev/null +++ b/worker.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +import os +import sys +import logging +from utils import bash +from yaml import load, dump +from pathlib import Path + +logger = logging.getLogger(__name__) + +abspath=os.path.abspath(__file__) +abspath=os.path.dirname(abspath) +os.chdir(abspath) + +# include all autobuild.yaml files + +REPO_NAME = Path('repo') +BUTOBUILD_FNAME = 'autobuild.yaml' +for mydir in REPO_NAME.iterdir(): + if mydir.is_dir() and (mydir / BUTOBUILD_FNAME).exists(): + # parsing yaml + logger.info('Bulidbot: found %s in %s', BUTOBUILD_FNAME, mydir / BUTOBUILD_FNAME) +