From d66d4e1f7b027f93577bfdd2db9ab879af35e617 Mon Sep 17 00:00:00 2001 From: Jerry Date: Sun, 8 Sep 2019 17:07:47 +0800 Subject: [PATCH] buildbot: add extra --- buildbot.py | 37 +++++++++++++++++++++++++++++++++---- client.py | 6 +++--- config.py | 5 +++++ extra.py | 51 ++++++++++++++++++++++++++++++++++++++++++++++++++- 4 files changed, 91 insertions(+), 8 deletions(-) diff --git a/buildbot.py b/buildbot.py index 0a09e04..b3ae0ea 100755 --- a/buildbot.py +++ b/buildbot.py @@ -19,7 +19,8 @@ from config import ARCHS, BUILD_ARCHS, BUILD_ARCH_MAPPING, \ MAKEPKG_MAKE_CMD, MAKEPKG_MAKE_CMD_CLEAN, \ GPG_SIGN_CMD, GPG_VERIFY_CMD, UPDATE_INTERVAL, \ MAKEPKG_MAKE_CMD_MARCH, UPLOAD_CMD, \ - GIT_PULL, GIT_RESET_SUBDIR, CONSOLE_LOGFILE + GIT_PULL, GIT_RESET_SUBDIR, CONSOLE_LOGFILE, \ + MAIN_LOGFILE, PKG_UPDATE_LOGFILE, MAKEPKG_LOGFILE from utils import print_exc_plus, background, \ bash, get_pkg_details_from_name, vercmp, \ @@ -32,12 +33,16 @@ import json from yamlparse import load_all as load_all_yaml +from extra import gen_pkglist as extra_gen_pkglist, \ + readpkglog as extra_readpkglog, \ + readmainlog as extra_readmainlog + abspath=os.path.abspath(__file__) abspath=os.path.dirname(abspath) os.chdir(abspath) logger = logging.getLogger('buildbot') -configure_logger(logger, logfile='buildbot.log', rotate_size=1024*1024*10, enable_notify=True, consolelog=CONSOLE_LOGFILE) +configure_logger(logger, logfile=MAIN_LOGFILE, rotate_size=1024*1024*10, enable_notify=True, consolelog=CONSOLE_LOGFILE) # refuse to run in systemd-nspawn if 'systemd-nspawn' in bash('systemd-detect-virt || true'): @@ -214,7 +219,7 @@ class jobsManager: # actually makepkg try: ret = mon_nspawn_shell(arch=job.arch, cwd=cwd, cmdline=mkcmd, - logfile = cwd / 'buildbot.log.makepkg', + logfile = cwd / MAKEPKG_LOGFILE, short_return = True, seconds=job.pkgconfig.timeout*60) except Exception: @@ -399,6 +404,12 @@ class updateManager: self.__pkgerrs = dict() self.__pkgvers = dict() self.__load() + @property + def pkgvers(self): + return self.__pkgvers + @property + def pkgerrs(self): + return self.__pkgerrs def __load(self): if Path(self.__filename).exists(): with open(self.__filename,"r") as f: @@ -465,7 +476,7 @@ class updateManager: if type(scr) is str: mon_nspawn_shell(arch, scr, cwd=pkgdir, seconds=60*60) mon_nspawn_shell(arch, MAKEPKG_UPD_CMD, cwd=pkgdir, seconds=60*60, - logfile = pkgdir / 'buildbot.log.update', + logfile = pkgdir / PKG_UPDATE_LOGFILE, short_return = True) if pkg.type in ('git', 'manual'): ver = self.__get_new_ver(pkg.dirname, arch) @@ -532,6 +543,24 @@ def force_upload(pkgdirname, overwrite=False): def getup(): return jobsmgr.getup() +def extras(action, pkgname=None): + if action.startswith("pkg"): + p = extra_gen_pkglist(jobsmgr.pkgconfigs, updmgr.pkgvers, updmgr.pkgerrs) + if action == "pkgdetail": + return p[1].get(pkgname, None) + elif action == "pkgdetails": + return p[1] + elif action == "pkglist": + return p[0] + elif action == "mainlog": + return extra_readmainlog(debug=False) + elif action == "debuglog": + return extra_readmainlog(debug=True) + elif action == "readpkglog": + pkgname = str(pkgname) + return extra_readpkglog(pkgname) + return False + def run(funcname, args=list(), kwargs=dict()): if funcname in ('info', 'rebuild_package', 'clean', 'clean_all', 'force_upload', 'getup'): diff --git a/client.py b/client.py index 5ffe130..9ef418a 100755 --- a/client.py +++ b/client.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -# repod.py: Automatic management tool for an arch repo. +# client.py: Automatic management tool for an arch repo. # This file is part of Buildbot by JerryXiao import logging @@ -10,7 +10,7 @@ from time import sleep from config import REPOD_BIND_ADDRESS, REPOD_BIND_PASSWD, \ MASTER_BIND_ADDRESS, MASTER_BIND_PASSWD, \ - CONSOLE_LOGFILE + CONSOLE_LOGFILE, MAIN_LOGFILE from utils import print_exc_plus @@ -47,7 +47,7 @@ if __name__ == '__main__': configure_logger(logger) def print_log(debug=False): if debug: - os.system(f'tail -n 43 -f \"buildbot.log\"') + os.system(f'tail -n 43 -f \"{MAIN_LOGFILE}\"') else: os.system(f'tail -n 43 -f \"{CONSOLE_LOGFILE}\"') try: diff --git a/config.py b/config.py index 4adbcf2..ee25f44 100644 --- a/config.py +++ b/config.py @@ -63,4 +63,9 @@ UPLOAD_CMD = 'rsync -avPh \"{src}\" repoupload:/srv/repo/buildbot/repo/updates/' GIT_PULL = 'git pull' GIT_RESET_SUBDIR = 'git checkout HEAD -- .' + +# logfiles +MAIN_LOGFILE = 'buildbot.log' CONSOLE_LOGFILE = 'buildbot.log.console' +PKG_UPDATE_LOGFILE = 'buildbot.log.update' +MAKEPKG_LOGFILE = 'buildbot.log.makepkg' diff --git a/extra.py b/extra.py index 2198ff2..7456809 100644 --- a/extra.py +++ b/extra.py @@ -1,13 +1,62 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- +# extra.py: Automatic management tool for an arch repo. +# This file is part of Buildbot by JerryXiao + import os import logging -from pathlib import Path +from pathlib import Path from utils import print_exc_plus +from config import PKGBUILD_DIR, MAIN_LOGFILE, CONSOLE_LOGFILE, \ + PKG_UPDATE_LOGFILE, MAKEPKG_LOGFILE + logger = logging.getLogger(f'buildbot.{__name__}') abspath=os.path.abspath(__file__) abspath=os.path.dirname(abspath) os.chdir(abspath) + +REPO_ROOT = Path(PKGBUILD_DIR) + +# generate package list +def gen_pkglist(pkgconfigs, pkgvers, pkgerrs): + # pkgall contains details + # namelist is a list of pkgnames + pkgall = dict() + for pc in pkgconfigs: + ps = ('type', 'cleanbuild', 'timeout') + hps = ('prebuild', 'postbuild', 'update', 'failure') + dps = {p:getattr(pc, p, None) for p in ps} + dhps = {p:'\n'.join(str(getattr(pc, p, None))) for p in hps} + # additional package details + ves = {'version': pkgvers.get(pc.dirname, None), 'errors': pkgerrs.get(pc.dirname, None)} + pkgall[pc.dirname] = {**dps, **dhps, **ves} + namelist = [k for k in pkgall] + return (namelist, pkgall) + +def __simpleread(fpath, limit=4096-100): + with open(fpath, 'r') as f: + c = f.read() + if len(c) > limit: + c = c[-limit:] + return c +# read logs +def readpkglog(pkgdirname, update=False): + cwd = REPO_ROOT / pkgdirname + logfile = PKG_UPDATE_LOGFILE if update else MAKEPKG_LOGFILE + if cwd.exists() and (cwd / logfile).exists(): + logger.debug(f'formatting {"update" if update else "build"} logs in {pkgdirname}') + return __simpleread(cwd / logfile) + else: + logger.debug(f'not found: {"update" if update else "build"} log in dir {pkgdirname}') + return f"{cwd / logfile} cannot be found" +def readmainlog(debug=False): + logfile = MAIN_LOGFILE if debug else CONSOLE_LOGFILE + if (PKGBUILD_DIR / logfile).exists(): + logger.debug(f'formatting buildbot{" debug" if debug else ""} logs') + return __simpleread(PKGBUILD_DIR / logfile) + else: + logger.debug(f'not found: buildbot{" debug" if debug else ""} log') + return f"{PKGBUILD_DIR / logfile} cannot be found"