From 97311e905114d5262623acbaa4ed97379f68e8dc Mon Sep 17 00:00:00 2001 From: Jerry Date: Sun, 7 Apr 2019 17:14:50 +0800 Subject: [PATCH] buildbot.py: ready to test --- README.md | 2 +- buildbot.py | 124 ++++++++++++++++++++++++++++----------- config.py | 7 ++- repod.py | 2 +- utils.py | 164 +++++++++++++++++++++++++++++++++------------------- 5 files changed, 205 insertions(+), 94 deletions(-) diff --git a/README.md b/README.md index 15234d5..3783ade 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ cleanbuild: true / false timeout: 30 (30 mins, int only) -extra: +extra: (wip) - update: - /bin/true - prebuild: diff --git a/buildbot.py b/buildbot.py index 0760e78..73cf24b 100755 --- a/buildbot.py +++ b/buildbot.py @@ -10,12 +10,14 @@ import os from pathlib import Path from subprocess import CalledProcessError -from utils import print_exc_plus, background - from config import ARCHS, BUILD_ARCHS, BUILD_ARCH_MAPPING, \ MASTER_BIND_ADDRESS, MASTER_BIND_PASSWD, \ - PKGBUILD_DIR, MAKEPKG_PKGLIST_CMD, MAKEPKG_UPD_CMD -from utils import bash, get_pkg_details_from_name, vercmp + PKGBUILD_DIR, MAKEPKG_PKGLIST_CMD, MAKEPKG_UPD_CMD, \ + MAKEPKG_MAKE_CMD, MAKEPKG_MAKE_CMD_CLEAN + +from utils import print_exc_plus, background, \ + bash, get_pkg_details_from_name, vercmp, \ + nspawn_shell, mon_nspawn_shell, get_arch_from_pkgbuild import json @@ -31,53 +33,100 @@ os.chdir(abspath) REPO_ROOT = Path(PKGBUILD_DIR) class Job: - def __init__(self, arch, pkgdir, packagelist, version): - buildarch = BUILD_ARCH_MAPPING.get(arch, None) + def __init__(self, buildarch, pkgconfig, version, multiarch=False): assert buildarch in BUILD_ARCHS - self.arch = arch - self.buildarch = buildarch - self.pkgdir = pkgdir - self.packagelist = packagelist + self.arch = buildarch + self.pkgconfig = pkgconfig self.version = version + self.multiarch = multiarch self.added = time() - self.claimed = 0 class jobsManager: def __init__(self): - self.__buildjobs = dict() - for arch in BUILD_ARCHS: - self.__buildjobs.setdefault(arch, list()) + self.__buildjobs = list() self.__uploadjobs = list() self.__curr_job = None self.pkgconfigs = load_all_yaml() - def _new_buildjob(self, job, buildarch): + def _new_buildjob(self, job): assert type(job) is Job - self.__buildjobs.get(buildarch).append(job) - def claim_job(self, buildarch): - assert buildarch in BUILD_ARCHS + job_to_remove = list() + for previous_job in self.__buildjobs: + if job.pkgconfig.dirname == previous_job.pkgconfig.dirname and \ + job.arch == previous_job.arch: + job_to_remove.append(previous_job) + for oldjob in job_to_remove: + self.__buildjobs.remove(oldjob) + logger.info('removed an old job for %s %s, %s => %s', + job.pkgconfig.dirname, job.arch, + oldjob.version, job.version) + logger.info('new job for %s %s %s', + job.pkgconfig.dirname, job.arch, job.version) + self.__buildjobs.append(job) + def __get_job(self): if self.__curr_job: return None - jobs = self.__buildjobs.get(buildarch, list()) + jobs = self.__buildjobs if jobs: self.__curr_job = jobs.pop(0) return self.__curr_job def __finish_job(self, pkgdir): - assert pkgdir == self.__curr_job.pkgdir + assert pkgdir == self.__curr_job.pkgconfig.dirname # do upload self.__curr_job = None return True + def __makepkg(self, job): + mkcmd = MAKEPKG_MAKE_CMD_CLEAN if job.pkgconfig.cleanbuild \ + else MAKEPKG_MAKE_CMD + cwd = REPO_ROOT / job.pkgconfig.dirname + logger.info('makepkg in %s %s', job.pkgconfig.dirname, job.arch) + return mon_nspawn_shell(arch=job.arch, cwd=cwd, cmdline=mkcmd, + logfile = cwd / 'buildbot.log.update', + short_return = True) + def __clean(self, job): + cwd = REPO_ROOT / job.pkgconfig.dirname + logger.info('cleaning build dir for %s %s', + job.pkgconfig.dirname, job.arch) + nspawn_shell(job.arch, 'rm -rf src pkg', cwd=cwd) + def __sign(self, job): + ''' + wip + ''' + cwd = REPO_ROOT / job.pkgconfig.dirname + print(nspawn_shell(job.arch, 'ls -l', cwd=cwd)) + #nspawn_shell(job.arch, 'rm -rf src pkg', cwd=cwd) + def __upload(self, job): + ''' + wip + ''' + cwd = REPO_ROOT / job.pkgconfig.dirname + print(nspawn_shell(job.arch, 'ls -l', cwd=cwd)) + #nspawn_shell(job.arch, 'rm -rf src pkg', cwd=cwd) def tick(self): ''' check for updates, create new jobs and run them ''' - if self.__curr_job is None: + if not self.__buildjobs: + # This part check for updates updates = updmgr.check_update() for update in updates: - (pkg, packagelist, ver) = update - - + (pkgconfig, ver, buildarchs) = update + march = True if len(buildarchs) >= 2 else False + for arch in buildarchs: + newjob = Job(arch, pkgconfig, ver, multiarch=march) + self._new_buildjob(newjob) + else: + # This part does the job + for job in self.__buildjobs: + cwd = REPO_ROOT / job.pkgconfig.dirname + if job.multiarch: + # wip + pass + else: + self.__makepkg(job) + self.__sign(job) + self.__upload(job) jobsmgr = jobsManager() class updateManager: @@ -109,28 +158,36 @@ class updateManager: f.write(pkgvers) else: logger.error('pkgver.json - Not writable') - def __get_package_list(self, dirname): + def __get_package_list(self, dirname, arch): pkgdir = REPO_ROOT / dirname assert pkgdir.exists() - pkglist = bash(MAKEPKG_PKGLIST_CMD, cwd=pkgdir) + pkglist = nspawn_shell(arch, MAKEPKG_PKGLIST_CMD, cwd=pkgdir) pkglist = pkglist.split('\n') return pkglist - def __get_new_ver(self, dirname): - pkgfiles = self.__get_package_list(dirname) - ver = get_pkg_details_from_name(pkgfiles[0]) - return (ver, pkgfiles) + def __get_new_ver(self, dirname, arch): + pkgfiles = self.__get_package_list(dirname, arch) + ver = get_pkg_details_from_name(pkgfiles[0]).ver + return ver def check_update(self): updates = list() for pkg in jobsmgr.pkgconfigs: pkgdir = REPO_ROOT / pkg.dirname logger.info(f'checking update: {pkg.dirname}') - bash(MAKEPKG_UPD_CMD, cwd=pkgdir, RUN_CMD_TIMEOUT=60*60) + pkgbuild = pkgdir / 'PKGBUILD' + archs = get_arch_from_pkgbuild(pkgbuild) + buildarchs = [BUILD_ARCH_MAPPING.get(arch, None) for arch in archs] + buildarchs = [arch for arch in buildarchs if arch is not None] + # hopefully we only need to check one arch for update + arch = 'x86_64' if 'x86_64' in buildarchs else buildarchs[0] # prefer x86 + mon_nspawn_shell(arch, MAKEPKG_UPD_CMD, cwd=pkgdir, minutes=60, + logfile = pkgdir / 'buildbot.log.update', + short_return = True) if pkg.type in ('git', 'manual'): - (ver, pkgfiles) = self.__get_new_ver(pkg.dirname) + ver = self.__get_new_ver(pkg.dirname, arch) oldver = self.__pkgvers.get(pkg.dirname, None) if oldver is None or vercmp(ver, oldver) == 1: self.__pkgvers[pkg.dirname] = ver - updates.append((pkg, pkgfiles, ver)) + updates.append((pkg, ver, buildarchs)) else: logger.warning(f'package: {pkg.dirname} downgrade attempted') else: @@ -174,6 +231,7 @@ if __name__ == '__main__': if type(myrecv) is list and len(myrecv) == 3: (funcname, args, kwargs) = myrecv funcname = str(funcname) + logger.info('running: %s %s %s', funcname, args, kwargs) conn.send(run(funcname, args=args, kwargs=kwargs)) except Exception: print_exc_plus() diff --git a/config.py b/config.py index 52d0c38..f1587df 100644 --- a/config.py +++ b/config.py @@ -37,7 +37,7 @@ GPG_SIGN_CMD = (f'gpg --default-key {GPG_KEY} --no-armor' '--pinentry-mode loopback --passphrase \'\'' '--detach-sign --yes --') -#### config for master.py +#### config for buildbot.py MASTER_BIND_ADDRESS = ('localhost', 7011) MASTER_BIND_PASSWD = b'mypassword' @@ -49,3 +49,8 @@ MAKEPKG_MAKE_CMD = 'makepkg --syncdeps --noextract' MAKEPKG_MAKE_CMD_CLEAN = 'makepkg --syncdeps --noextract --clean --cleanbuild' MAKEPKG_PKGLIST_CMD = f'{MAKEPKG} --packagelist' + +CONTAINER_BUILDBOT_ROOT = '~/shared/buildbot' +# single quote may cause problem here +SHELL_ARCH_X64 = 'sudo machinectl --quiet shell build@archlinux /bin/bash -c \'{command}\'' +SHELL_ARCH_ARM64 = 'sudo machinectl --quiet shell root@alarm /bin/bash -c $\'su -l alarm -c \\\'{command}\\\'\'' diff --git a/repod.py b/repod.py index 7c775f0..2fb68c3 100644 --- a/repod.py +++ b/repod.py @@ -119,7 +119,7 @@ def add_files(filename, overwrite=False): def run(funcname, args=list(), kwargs=dict()): if funcname in ('clean', 'regenerate', 'remove', 'update', 'push_files', 'add_files'): - logger.info('running: %s %s %s',funcname, args, kwargs) + logger.info('running: %s %s %s', funcname, args, kwargs) ret = eval(funcname)(*args, **kwargs) logger.info('done: %s %s',funcname, ret) return ret diff --git a/utils.py b/utils.py index 4f0e58b..8b92c27 100644 --- a/utils.py +++ b/utils.py @@ -5,10 +5,13 @@ import logging from time import time import re from threading import Thread, Lock +from pathlib import Path +import os import sys import traceback -from config import PKG_COMPRESSION +from config import PKG_COMPRESSION, SHELL_ARCH_ARM64, SHELL_ARCH_X64, \ + CONTAINER_BUILDBOT_ROOT, ARCHS logger = logging.getLogger(name='utils') @@ -22,74 +25,107 @@ def background(func): def bash(cmdline, **kwargs): assert type(cmdline) is str - logger.info(f'bash: {cmdline}') + logger.info(f'bash: {cmdline}, kwargs: {kwargs}') return(run_cmd(['/bin/bash', '-x', '-e', '-c', cmdline], **kwargs)) -def long_bash(cmdline, cwd=None, hours=2): - assert type(hours) is int and hours >= 1 - logger.info(f'longbash{hours}: {cmdline}') - return bash(cmdline, cwd=cwd, keepalive=True, KEEPALIVE_TIMEOUT=60, RUN_CMD_TIMEOUT=hours*60*60) +def mon_bash(cmdline, cwd=None, minutes=30, **kwargs): + assert type(minutes) is int and minutes >= 1 + return bash(cmdline, cwd=cwd, keepalive=True, KEEPALIVE_TIMEOUT=60, + RUN_CMD_TIMEOUT=minutes*60, **kwargs) -def run_cmd(cmd, cwd=None, keepalive=False, KEEPALIVE_TIMEOUT=30, RUN_CMD_TIMEOUT=60): +def nspawn_shell(arch, cmdline, cwd=None, **kwargs): + root = Path(CONTAINER_BUILDBOT_ROOT) + if cwd: + cwd = root / cwd + else: + cwd = root + if arch in ('aarch64', 'arm64'): + return bash(SHELL_ARCH_ARM64.format(command=f'cd \"{cwd}\" || exit 1; {cmdline}')) + elif arch in ('x64', 'x86', 'x86_64'): + return bash(SHELL_ARCH_X64.format(command=f'cd \"{cwd}\" || exit 1; {cmdline}')) + raise TypeError('nspawn_shell: wrong arch') + +def mon_nspawn_shell(arch, cmdline, cwd, minutes=30, **kwargs): + assert type(minutes) is int and minutes >= 1 + return nspawn_shell(arch, cmdline, cwd=cwd, keepalive=True, KEEPALIVE_TIMEOUT=60, + RUN_CMD_TIMEOUT=minutes*60, **kwargs) + +def run_cmd(cmd, cwd=None, keepalive=False, KEEPALIVE_TIMEOUT=30, RUN_CMD_TIMEOUT=60, + logfile=None, short_return=False): logger.debug('run_cmd: %s', cmd) RUN_CMD_LOOP_TIME = KEEPALIVE_TIMEOUT - 1 if KEEPALIVE_TIMEOUT >= 10 else 5 stopped = False last_read = [int(time()), ""] - output = list() + class Output(list): + def append(self, mystring): + if not self.__short_return: + super().append(mystring) + if self.__file and type(mystring) is str: + self.__file.write(mystring) + def __enter__(self, logfile=None, short_return=False): + self.__short_return = short_return + if logfile: + assert issubclass(type(logfile), os.PathLike) + self.__file = open(logfile, 'w') + else: + self.__file = None + def __exit__(self, type, value, traceback): + if self.__file: + self.__file.close() stdout_lock = Lock() - @background - def check_stdout(stdout): - nonlocal stopped, last_read, output - stdout_lock.acquire() - last_read_time = int(time()) - while stopped is False: - line = stdout.readline(4096) + with Output(logfile=logfile, short_return=short_return) as output: + @background + def check_stdout(stdout): + nonlocal stopped, last_read, output + stdout_lock.acquire() last_read_time = int(time()) - logger.debug(line) - output.append(line) - last_read[0] = last_read_time - last_read[1] = line - stdout_lock.release() - p = subprocess.Popen(cmd, cwd=cwd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, encoding='utf-8') - check_stdout(p.stdout) - process_start = int(time()) - while True: - try: - p.wait(timeout=RUN_CMD_LOOP_TIME) - except subprocess.TimeoutExpired: - time_passed = int(time()) - last_read[0] - if time_passed >= KEEPALIVE_TIMEOUT*2: - logger.info('Timeout expired. No action.') - output.append('+ Buildbot: Timeout expired. No action.\n') - elif time_passed >= KEEPALIVE_TIMEOUT: - if keepalive: - logger.info('Timeout expired, writing nl') - output.append('+ Buildbot: Timeout expired, writing nl\n') - p.stdin.write('\n') - p.stdin.flush() - else: - logger.info('Timeout expired, not writing nl') - output.append('+ Buildbot: Timeout expired, not writing nl\n') - if int(time()) - process_start >= RUN_CMD_TIMEOUT: + while stopped is False: + line = stdout.readline(4096) + last_read_time = int(time()) + logger.debug(line) + output.append(line) + last_read[0] = last_read_time + last_read[1] = line + stdout_lock.release() + p = subprocess.Popen(cmd, cwd=cwd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, encoding='utf-8') + check_stdout(p.stdout) + process_start = int(time()) + while True: + try: + p.wait(timeout=RUN_CMD_LOOP_TIME) + except subprocess.TimeoutExpired: + time_passed = int(time()) - last_read[0] + if time_passed >= KEEPALIVE_TIMEOUT*2: + logger.info('Timeout expired. No action.') + output.append('+ Buildbot: Timeout expired. No action.\n') + elif time_passed >= KEEPALIVE_TIMEOUT: + if keepalive: + logger.info('Timeout expired, writing nl') + output.append('+ Buildbot: Timeout expired, writing nl\n') + p.stdin.write('\n') + p.stdin.flush() + else: + logger.info('Timeout expired, not writing nl') + output.append('+ Buildbot: Timeout expired, not writing nl\n') + if int(time()) - process_start >= RUN_CMD_TIMEOUT: + stopped = True + logger.error('Process timeout expired, terminating.') + output.append('+ Buildbot: Process timeout expired, terminating.\n') + p.terminate() + try: + p.wait(timeout=10) + except subprocess.TimeoutExpired: + logger.error('Cannot terminate, killing.') + output.append('+ Buildbot: Cannot terminate, killing.\n') + p.kill() + break + else: stopped = True - logger.error('Process timeout expired, terminating.') - output.append('+ Buildbot: Process timeout expired, terminating.\n') - p.terminate() - try: - p.wait(timeout=10) - except subprocess.TimeoutExpired: - logger.error('Cannot terminate, killing.') - output.append('+ Buildbot: Cannot terminate, killing.\n') - p.kill() break - else: - stopped = True - break - code = p.returncode - - stdout_lock.acquire(10) - outstr = ''.join(output) + code = p.returncode + stdout_lock.acquire(10) + outstr = ''.join(output) if code != 0: raise subprocess.CalledProcessError(code, cmd, outstr) @@ -148,6 +184,18 @@ def get_pkg_details_from_name(name): (pkgname, pkgver, pkgrel, arch) = m.groups() return Pkg(pkgname, pkgver, pkgrel, arch, name) +def get_arch_from_pkgbuild(fpath): + assert issubclass(type(fpath), os.PathLike) + with open(fpath, 'r') as f: + for line in f.readline(): + if line.startswith('arch='): + matches = re.findall('[\'\"]([^\'\"]+)[\'\"]', line) + if not matches: + raise TypeError('Unexpected PKGBUILD format') + assert not [None for match in matches if match not in ARCHS] + return matches + raise TypeError('Unexpected PKGBUILD') + def print_exc_plus(): """ Print the usual traceback information, followed by a listing of all the