buildbot: lots of improvements

This commit is contained in:
JerryXiao 2019-09-05 22:42:08 +08:00
parent fe3b51fadf
commit 07dcaa487a
Signed by: Jerry
GPG key ID: 9D9CE43650FF2BAA
5 changed files with 174 additions and 56 deletions

View file

@ -10,7 +10,7 @@ cleanbuild:
true / false true / false
timeout: timeout:
30 (30 mins, int only) 30 (30 mins, int only)
extra: (wip) extra:
- update: - update:
- /bin/true - /bin/true
- prebuild: - prebuild:

View file

@ -114,7 +114,10 @@ class jobsManager:
if (REPO_ROOT / pkgdirname).exists() and clean: if (REPO_ROOT / pkgdirname).exists() and clean:
self.reset_dir(pkgdirname) self.reset_dir(pkgdirname)
updates = updmgr.check_update(rebuild_package=pkgdirname) updates = updmgr.check_update(rebuild_package=pkgdirname)
if updates and len(updates) == 1: if not (REPO_ROOT / pkgdirname).exists():
ret = f'rebuild failed: no such dir {pkgdirname}'
logger.warning(ret)
elif updates and len(updates) == 1:
(pkgconfig, ver, buildarchs) = updates[0] (pkgconfig, ver, buildarchs) = updates[0]
march = True if len(buildarchs) >= 2 else False march = True if len(buildarchs) >= 2 else False
for arch in buildarchs: for arch in buildarchs:
@ -123,7 +126,7 @@ class jobsManager:
ret = f'rebuild job added for {pkgdirname} {" ".join(buildarchs)}' ret = f'rebuild job added for {pkgdirname} {" ".join(buildarchs)}'
logger.info(ret) logger.info(ret)
else: else:
ret = f'rebuild failed: no such dir {pkgdirname}' ret = 'rebuild failed: cannot check update.'
logger.warning(ret) logger.warning(ret)
return ret return ret
def _new_buildjob(self, job): def _new_buildjob(self, job):
@ -143,7 +146,7 @@ class jobsManager:
self.__buildjobs.append(job) self.__buildjobs.append(job)
def __get_job(self): def __get_job(self):
if self.__curr_job: if self.__curr_job:
logger.error(f'Job {self.__curr_job} failed. Correct the error and rebuild') logger.error(f'Job {self.__curr_job} failed and is not cleaned.')
self.__finish_job(self.__curr_job, force=True) self.__finish_job(self.__curr_job, force=True)
return self.__get_job() return self.__get_job()
jobs = self.__buildjobs jobs = self.__buildjobs
@ -155,6 +158,12 @@ class jobsManager:
assert pkgdir == self.__curr_job.pkgconfig.dirname assert pkgdir == self.__curr_job.pkgconfig.dirname
self.__curr_job = None self.__curr_job = None
return True return True
def clean_failed_job(self):
if self.__curr_job:
logger.error(f'Job {self.__curr_job} failed. Correct the error and rebuild')
self.__finish_job(self.__curr_job, force=True)
else:
raise RuntimeError('Unexpected behavior')
def __makepkg(self, job): def __makepkg(self, job):
cwd = REPO_ROOT / job.pkgconfig.dirname cwd = REPO_ROOT / job.pkgconfig.dirname
if job.multiarch: if job.multiarch:
@ -164,10 +173,38 @@ class jobsManager:
mkcmd = MAKEPKG_MAKE_CMD_CLEAN if job.pkgconfig.cleanbuild \ mkcmd = MAKEPKG_MAKE_CMD_CLEAN if job.pkgconfig.cleanbuild \
else MAKEPKG_MAKE_CMD else MAKEPKG_MAKE_CMD
logger.info('makepkg in %s %s', job.pkgconfig.dirname, job.arch) logger.info('makepkg in %s %s', job.pkgconfig.dirname, job.arch)
return mon_nspawn_shell(arch=job.arch, cwd=cwd, cmdline=mkcmd, # run pre-makepkg-scripts
logger.info('running pre-build scripts')
for scr in getattr(job.pkgconfig, 'prebuild', list()):
if type(scr) is str:
try:
mon_nspawn_shell(arch=job.arch, cwd=cwd, cmdline=scr, seconds=60*60)
except Exception:
print_exc_plus()
# actually makepkg
try:
ret = mon_nspawn_shell(arch=job.arch, cwd=cwd, cmdline=mkcmd,
logfile = cwd / 'buildbot.log.makepkg', logfile = cwd / 'buildbot.log.makepkg',
short_return = True, short_return = True,
seconds=job.pkgconfig.timeout*60) seconds=job.pkgconfig.timeout*60)
except Exception:
logger.error(f'Job {job} failed. Running build-failure scripts')
for scr in getattr(job.pkgconfig, 'failure', list()):
if type(scr) is str:
try:
mon_nspawn_shell(arch=job.arch, cwd=cwd, cmdline=scr, seconds=60*60)
except Exception:
print_exc_plus()
raise
# run post-makepkg-scripts
logger.info('running post-build scripts')
for scr in getattr(job.pkgconfig, 'postbuild', list()):
if type(scr) is str:
try:
mon_nspawn_shell(arch=job.arch, cwd=cwd, cmdline=scr, seconds=60*60)
except Exception:
print_exc_plus()
return ret
def __clean(self, job, remove_pkg=False, rm_src=True): def __clean(self, job, remove_pkg=False, rm_src=True):
cwd = REPO_ROOT / job.pkgconfig.dirname cwd = REPO_ROOT / job.pkgconfig.dirname
logger.info('cleaning build dir for %s, %sremoving pkg', logger.info('cleaning build dir for %s, %sremoving pkg',
@ -199,6 +236,9 @@ class jobsManager:
f_to_upload.append(sigpath) f_to_upload.append(sigpath)
f_to_upload.append(fpath) f_to_upload.append(fpath)
for f in f_to_upload: for f in f_to_upload:
max_tries = 5
for tries in range(max_tries):
try:
size = f.stat().st_size / 1000 / 1000 size = f.stat().st_size / 1000 / 1000
if f.name.endswith(PKG_SUFFIX): if f.name.endswith(PKG_SUFFIX):
for _ in range(10): for _ in range(10):
@ -220,6 +260,17 @@ class jobsManager:
else: else:
logger.error(f'Update failed for {f.name}, reason: {res}') logger.error(f'Update failed for {f.name}, reason: {res}')
suc = False suc = False
except Exception:
time_to_sleep = (tries + 1) * 60
logger.error(f'We are getting problem uploading {f}, wait {time_to_sleep} secs')
print_exc_plus()
if tries + 1 < max_tries:
sleep(time_to_sleep)
else:
break
else:
logger.error(f'Upload {f} failed, abort.')
raise RuntimeError('Unable to upload some files')
return suc return suc
def tick(self): def tick(self):
''' '''
@ -233,8 +284,8 @@ class jobsManager:
if not self.idle: if not self.idle:
logger.info('Buildbot is idling for package updates.') logger.info('Buildbot is idling for package updates.')
self.idle = True self.idle = True
sleep(60) return 60
return else:
self.last_updatecheck = time() self.last_updatecheck = time()
self.idle = False self.idle = False
# git pull repo # git pull repo
@ -250,6 +301,7 @@ class jobsManager:
for arch in buildarchs: for arch in buildarchs:
newjob = Job(arch, pkgconfig, ver, multiarch=march) newjob = Job(arch, pkgconfig, ver, multiarch=march)
self._new_buildjob(newjob) self._new_buildjob(newjob)
return 0
else: else:
# This part does the job # This part does the job
self.idle = False self.idle = False
@ -272,35 +324,41 @@ class jobsManager:
else: else:
self.__clean(job, rm_src=False, remove_pkg=True) self.__clean(job, rm_src=False, remove_pkg=True)
self.__finish_job(job.pkgconfig.dirname) self.__finish_job(job.pkgconfig.dirname)
return 0
jobsmgr = jobsManager() jobsmgr = jobsManager()
class updateManager: class updateManager:
def __init__(self, filename='pkgver.json'): def __init__(self, filename='pkgver.json'):
self.__filename = filename self.__filename = filename
self.__pkgerrs = dict()
self.__pkgvers = dict() self.__pkgvers = dict()
self.__load() self.__load()
def __load(self): def __load(self):
if Path(self.__filename).exists(): if Path(self.__filename).exists():
with open(self.__filename,"r") as f: with open(self.__filename,"r") as f:
try: try:
pkgvers = json.loads(f.read()) pkgdata = json.loads(f.read())
except json.JSONDecodeError: except json.JSONDecodeError:
logger.error('pkgver.json - Bad json') logger.error('pkgver.json - Bad json')
print_exc_plus print_exc_plus
exit(1) exit(1)
else: else:
logger.warning(f'No {self.__filename} found') logger.warning(f'No {self.__filename} found')
pkgvers = dict() pkgdata = dict()
assert type(pkgvers) is dict assert type(pkgdata) is dict
for pkgname in pkgvers: for pkgname in pkgdata:
assert type(pkgname) is str assert type(pkgname) is str
self.__pkgvers = pkgvers assert len(pkgdata[pkgname]) == 2
self.__pkgvers = {pkgname:pkgdata[pkgname][0] for pkgname in pkgdata}
self.__pkgerrs = {pkgname:pkgdata[pkgname][1] for pkgname in pkgdata}
def _save(self): def _save(self):
pkgvers = json.dumps(self.__pkgvers, indent=4) pkgdata = {pkgname:[self.__pkgvers[pkgname], self.__pkgerrs[pkgname]] for pkgname in self.__pkgvers}
pkgvers += '\n' pkgdatastr = json.dumps(pkgdata, indent=4)
pkgdatastr += '\n'
with open(self.__filename,"w") as f: with open(self.__filename,"w") as f:
if f.writable: if f.writable:
f.write(pkgvers) f.write(pkgdatastr)
else: else:
logger.error('pkgver.json - Not writable') logger.error('pkgver.json - Not writable')
def __get_package_list(self, dirname, arch): def __get_package_list(self, dirname, arch):
@ -332,12 +390,21 @@ class updateManager:
continue continue
# hopefully we only need to check one arch for update # hopefully we only need to check one arch for update
arch = 'x86_64' if 'x86_64' in buildarchs else buildarchs[0] # prefer x86 arch = 'x86_64' if 'x86_64' in buildarchs else buildarchs[0] # prefer x86
# run pre_update_scripts
logger.info('running pre-update scripts')
for scr in getattr(pkg, 'update', list()):
if type(scr) is str:
mon_nspawn_shell(arch, scr, cwd=pkgdir, seconds=60*60)
mon_nspawn_shell(arch, MAKEPKG_UPD_CMD, cwd=pkgdir, seconds=60*60, mon_nspawn_shell(arch, MAKEPKG_UPD_CMD, cwd=pkgdir, seconds=60*60,
logfile = pkgdir / 'buildbot.log.update', logfile = pkgdir / 'buildbot.log.update',
short_return = True) short_return = True)
if pkg.type in ('git', 'manual'): if pkg.type in ('git', 'manual'):
ver = self.__get_new_ver(pkg.dirname, arch) ver = self.__get_new_ver(pkg.dirname, arch)
oldver = self.__pkgvers.get(pkg.dirname, None) oldver = self.__pkgvers.get(pkg.dirname, None)
errs = self.__pkgerrs.get(pkg.dirname, 0)
if errs >= 2:
logger.warning(f'package: {pkg.dirname} too many failures checking update')
continue
has_update = False has_update = False
if rebuild_package: if rebuild_package:
has_update = True has_update = True
@ -353,10 +420,12 @@ class updateManager:
has_update = True has_update = True
if has_update: if has_update:
self.__pkgvers[pkg.dirname] = ver self.__pkgvers[pkg.dirname] = ver
self.__pkgerrs[pkg.dirname] = 0
updates.append((pkg, ver, buildarchs)) updates.append((pkg, ver, buildarchs))
else: else:
logger.warning(f'unknown package type: {pkg.type}') logger.warning(f'unknown package type: {pkg.type}')
except Exception: except Exception:
self.__pkgerrs[pkg.dirname] = self.__pkgerrs.get(pkg.dirname, 0) + 1
print_exc_plus() print_exc_plus()
self._save() self._save()
return updates return updates
@ -414,11 +483,23 @@ if __name__ == '__main__':
logger.info('Listener started.') logger.info('Listener started.')
while True: while True:
try: try:
jobsmgr.tick() ret = 1
ret = jobsmgr.tick()
except Exception: except Exception:
try:
jobsmgr.clean_failed_job()
except Exception:
print_exc_plus()
print_exc_plus() print_exc_plus()
except KeyboardInterrupt: except KeyboardInterrupt:
logger.info('KeyboardInterrupt') logger.info('KeyboardInterrupt')
print_exc_plus() print_exc_plus()
break break
if ret is None:
sleep(1)
elif ret == 0:
pass
elif type(ret) in (int, float):
sleep(ret)
else:
sleep(1) sleep(1)

13
extra.py Normal file
View file

@ -0,0 +1,13 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import os
import logging
from pathlib import Path
from utils import print_exc_plus
logger = logging.getLogger(f'buildbot.{__name__}')
abspath=os.path.abspath(__file__)
abspath=os.path.dirname(abspath)
os.chdir(abspath)

16
notify.py Normal file
View file

@ -0,0 +1,16 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# notify.py: Automatic management tool for an arch repo.
# This file is part of Buildbot by JerryXiao
import logging
from utils import background, print_exc_plus, configure_logger
logger = logging.getLogger(f'buildbot.{__name__}')
# wip
# does nothing
@background
def send(content):
pass

View file

@ -261,11 +261,19 @@ def format_exc_plus():
def configure_logger(logger, format='%(asctime)s - %(name)-18s - %(levelname)s - %(message)s', def configure_logger(logger, format='%(asctime)s - %(name)-18s - %(levelname)s - %(message)s',
level=logging.INFO, logfile=None, flevel=logging.DEBUG, rotate_size=None): level=logging.INFO, logfile=None, flevel=logging.DEBUG, rotate_size=None):
try:
from notify import send
except ModuleNotFoundError:
def send(*args):
pass
class ExceptionFormatter(logging.Formatter): class ExceptionFormatter(logging.Formatter):
def format(self, record): def format(self, record):
if record.levelno == 49: if record.levelno == 49:
record.msg = 'Exception caught.\nPrinting stack traceback\n' + record.msg record.msg = 'Exception caught.\nPrinting stack traceback\n' + record.msg
return super().format(record) fmtr = super().format(record)
send(fmtr)
return fmtr
logger.setLevel(logging.DEBUG) logger.setLevel(logging.DEBUG)
formatter = ExceptionFormatter(fmt=format) formatter = ExceptionFormatter(fmt=format)