buildbot: add extra

This commit is contained in:
JerryXiao 2019-09-08 17:07:47 +08:00
parent a61cbc8f7a
commit d66d4e1f7b
Signed by: Jerry
GPG key ID: 9D9CE43650FF2BAA
4 changed files with 91 additions and 8 deletions

View file

@ -19,7 +19,8 @@ from config import ARCHS, BUILD_ARCHS, BUILD_ARCH_MAPPING, \
MAKEPKG_MAKE_CMD, MAKEPKG_MAKE_CMD_CLEAN, \
GPG_SIGN_CMD, GPG_VERIFY_CMD, UPDATE_INTERVAL, \
MAKEPKG_MAKE_CMD_MARCH, UPLOAD_CMD, \
GIT_PULL, GIT_RESET_SUBDIR, CONSOLE_LOGFILE
GIT_PULL, GIT_RESET_SUBDIR, CONSOLE_LOGFILE, \
MAIN_LOGFILE, PKG_UPDATE_LOGFILE, MAKEPKG_LOGFILE
from utils import print_exc_plus, background, \
bash, get_pkg_details_from_name, vercmp, \
@ -32,12 +33,16 @@ import json
from yamlparse import load_all as load_all_yaml
from extra import gen_pkglist as extra_gen_pkglist, \
readpkglog as extra_readpkglog, \
readmainlog as extra_readmainlog
abspath=os.path.abspath(__file__)
abspath=os.path.dirname(abspath)
os.chdir(abspath)
logger = logging.getLogger('buildbot')
configure_logger(logger, logfile='buildbot.log', rotate_size=1024*1024*10, enable_notify=True, consolelog=CONSOLE_LOGFILE)
configure_logger(logger, logfile=MAIN_LOGFILE, rotate_size=1024*1024*10, enable_notify=True, consolelog=CONSOLE_LOGFILE)
# refuse to run in systemd-nspawn
if 'systemd-nspawn' in bash('systemd-detect-virt || true'):
@ -214,7 +219,7 @@ class jobsManager:
# actually makepkg
try:
ret = mon_nspawn_shell(arch=job.arch, cwd=cwd, cmdline=mkcmd,
logfile = cwd / 'buildbot.log.makepkg',
logfile = cwd / MAKEPKG_LOGFILE,
short_return = True,
seconds=job.pkgconfig.timeout*60)
except Exception:
@ -399,6 +404,12 @@ class updateManager:
self.__pkgerrs = dict()
self.__pkgvers = dict()
self.__load()
@property
def pkgvers(self):
return self.__pkgvers
@property
def pkgerrs(self):
return self.__pkgerrs
def __load(self):
if Path(self.__filename).exists():
with open(self.__filename,"r") as f:
@ -465,7 +476,7 @@ class updateManager:
if type(scr) is str:
mon_nspawn_shell(arch, scr, cwd=pkgdir, seconds=60*60)
mon_nspawn_shell(arch, MAKEPKG_UPD_CMD, cwd=pkgdir, seconds=60*60,
logfile = pkgdir / 'buildbot.log.update',
logfile = pkgdir / PKG_UPDATE_LOGFILE,
short_return = True)
if pkg.type in ('git', 'manual'):
ver = self.__get_new_ver(pkg.dirname, arch)
@ -532,6 +543,24 @@ def force_upload(pkgdirname, overwrite=False):
def getup():
return jobsmgr.getup()
def extras(action, pkgname=None):
if action.startswith("pkg"):
p = extra_gen_pkglist(jobsmgr.pkgconfigs, updmgr.pkgvers, updmgr.pkgerrs)
if action == "pkgdetail":
return p[1].get(pkgname, None)
elif action == "pkgdetails":
return p[1]
elif action == "pkglist":
return p[0]
elif action == "mainlog":
return extra_readmainlog(debug=False)
elif action == "debuglog":
return extra_readmainlog(debug=True)
elif action == "readpkglog":
pkgname = str(pkgname)
return extra_readpkglog(pkgname)
return False
def run(funcname, args=list(), kwargs=dict()):
if funcname in ('info', 'rebuild_package', 'clean', 'clean_all',
'force_upload', 'getup'):

View file

@ -1,6 +1,6 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# repod.py: Automatic management tool for an arch repo.
# client.py: Automatic management tool for an arch repo.
# This file is part of Buildbot by JerryXiao
import logging
@ -10,7 +10,7 @@ from time import sleep
from config import REPOD_BIND_ADDRESS, REPOD_BIND_PASSWD, \
MASTER_BIND_ADDRESS, MASTER_BIND_PASSWD, \
CONSOLE_LOGFILE
CONSOLE_LOGFILE, MAIN_LOGFILE
from utils import print_exc_plus
@ -47,7 +47,7 @@ if __name__ == '__main__':
configure_logger(logger)
def print_log(debug=False):
if debug:
os.system(f'tail -n 43 -f \"buildbot.log\"')
os.system(f'tail -n 43 -f \"{MAIN_LOGFILE}\"')
else:
os.system(f'tail -n 43 -f \"{CONSOLE_LOGFILE}\"')
try:

View file

@ -63,4 +63,9 @@ UPLOAD_CMD = 'rsync -avPh \"{src}\" repoupload:/srv/repo/buildbot/repo/updates/'
GIT_PULL = 'git pull'
GIT_RESET_SUBDIR = 'git checkout HEAD -- .'
# logfiles
MAIN_LOGFILE = 'buildbot.log'
CONSOLE_LOGFILE = 'buildbot.log.console'
PKG_UPDATE_LOGFILE = 'buildbot.log.update'
MAKEPKG_LOGFILE = 'buildbot.log.makepkg'

View file

@ -1,13 +1,62 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# extra.py: Automatic management tool for an arch repo.
# This file is part of Buildbot by JerryXiao
import os
import logging
from pathlib import Path
from pathlib import Path
from utils import print_exc_plus
from config import PKGBUILD_DIR, MAIN_LOGFILE, CONSOLE_LOGFILE, \
PKG_UPDATE_LOGFILE, MAKEPKG_LOGFILE
logger = logging.getLogger(f'buildbot.{__name__}')
abspath=os.path.abspath(__file__)
abspath=os.path.dirname(abspath)
os.chdir(abspath)
REPO_ROOT = Path(PKGBUILD_DIR)
# generate package list
def gen_pkglist(pkgconfigs, pkgvers, pkgerrs):
# pkgall contains details
# namelist is a list of pkgnames
pkgall = dict()
for pc in pkgconfigs:
ps = ('type', 'cleanbuild', 'timeout')
hps = ('prebuild', 'postbuild', 'update', 'failure')
dps = {p:getattr(pc, p, None) for p in ps}
dhps = {p:'\n'.join(str(getattr(pc, p, None))) for p in hps}
# additional package details
ves = {'version': pkgvers.get(pc.dirname, None), 'errors': pkgerrs.get(pc.dirname, None)}
pkgall[pc.dirname] = {**dps, **dhps, **ves}
namelist = [k for k in pkgall]
return (namelist, pkgall)
def __simpleread(fpath, limit=4096-100):
with open(fpath, 'r') as f:
c = f.read()
if len(c) > limit:
c = c[-limit:]
return c
# read logs
def readpkglog(pkgdirname, update=False):
cwd = REPO_ROOT / pkgdirname
logfile = PKG_UPDATE_LOGFILE if update else MAKEPKG_LOGFILE
if cwd.exists() and (cwd / logfile).exists():
logger.debug(f'formatting {"update" if update else "build"} logs in {pkgdirname}')
return __simpleread(cwd / logfile)
else:
logger.debug(f'not found: {"update" if update else "build"} log in dir {pkgdirname}')
return f"{cwd / logfile} cannot be found"
def readmainlog(debug=False):
logfile = MAIN_LOGFILE if debug else CONSOLE_LOGFILE
if (PKGBUILD_DIR / logfile).exists():
logger.debug(f'formatting buildbot{" debug" if debug else ""} logs')
return __simpleread(PKGBUILD_DIR / logfile)
else:
logger.debug(f'not found: buildbot{" debug" if debug else ""} log')
return f"{PKGBUILD_DIR / logfile} cannot be found"