mirror of
https://github.com/archlinux-jerry/buildbot
synced 2024-11-22 04:50:41 +08:00
buildbot: lots of improvements
This commit is contained in:
parent
fe3b51fadf
commit
07dcaa487a
GPG key ID:
9D9CE43650FF2BAA
5 changed files with 174 additions and 56 deletions
|
|
@ -10,7 +10,7 @@ cleanbuild:
|
|||
true / false
|
||||
timeout:
|
||||
30 (30 mins, int only)
|
||||
extra: (wip)
|
||||
extra:
|
||||
- update:
|
||||
- /bin/true
|
||||
- prebuild:
|
||||
|
|
|
|||
189
buildbot.py
189
buildbot.py
|
|
@ -114,7 +114,10 @@ class jobsManager:
|
|||
if (REPO_ROOT / pkgdirname).exists() and clean:
|
||||
self.reset_dir(pkgdirname)
|
||||
updates = updmgr.check_update(rebuild_package=pkgdirname)
|
||||
if updates and len(updates) == 1:
|
||||
if not (REPO_ROOT / pkgdirname).exists():
|
||||
ret = f'rebuild failed: no such dir {pkgdirname}'
|
||||
logger.warning(ret)
|
||||
elif updates and len(updates) == 1:
|
||||
(pkgconfig, ver, buildarchs) = updates[0]
|
||||
march = True if len(buildarchs) >= 2 else False
|
||||
for arch in buildarchs:
|
||||
|
|
@ -123,7 +126,7 @@ class jobsManager:
|
|||
ret = f'rebuild job added for {pkgdirname} {" ".join(buildarchs)}'
|
||||
logger.info(ret)
|
||||
else:
|
||||
ret = f'rebuild failed: no such dir {pkgdirname}'
|
||||
ret = 'rebuild failed: cannot check update.'
|
||||
logger.warning(ret)
|
||||
return ret
|
||||
def _new_buildjob(self, job):
|
||||
|
|
@ -143,7 +146,7 @@ class jobsManager:
|
|||
self.__buildjobs.append(job)
|
||||
def __get_job(self):
|
||||
if self.__curr_job:
|
||||
logger.error(f'Job {self.__curr_job} failed. Correct the error and rebuild')
|
||||
logger.error(f'Job {self.__curr_job} failed and is not cleaned.')
|
||||
self.__finish_job(self.__curr_job, force=True)
|
||||
return self.__get_job()
|
||||
jobs = self.__buildjobs
|
||||
|
|
@ -155,6 +158,12 @@ class jobsManager:
|
|||
assert pkgdir == self.__curr_job.pkgconfig.dirname
|
||||
self.__curr_job = None
|
||||
return True
|
||||
def clean_failed_job(self):
|
||||
if self.__curr_job:
|
||||
logger.error(f'Job {self.__curr_job} failed. Correct the error and rebuild')
|
||||
self.__finish_job(self.__curr_job, force=True)
|
||||
else:
|
||||
raise RuntimeError('Unexpected behavior')
|
||||
def __makepkg(self, job):
|
||||
cwd = REPO_ROOT / job.pkgconfig.dirname
|
||||
if job.multiarch:
|
||||
|
|
@ -164,10 +173,38 @@ class jobsManager:
|
|||
mkcmd = MAKEPKG_MAKE_CMD_CLEAN if job.pkgconfig.cleanbuild \
|
||||
else MAKEPKG_MAKE_CMD
|
||||
logger.info('makepkg in %s %s', job.pkgconfig.dirname, job.arch)
|
||||
return mon_nspawn_shell(arch=job.arch, cwd=cwd, cmdline=mkcmd,
|
||||
logfile = cwd / 'buildbot.log.makepkg',
|
||||
short_return = True,
|
||||
seconds=job.pkgconfig.timeout*60)
|
||||
# run pre-makepkg-scripts
|
||||
logger.info('running pre-build scripts')
|
||||
for scr in getattr(job.pkgconfig, 'prebuild', list()):
|
||||
if type(scr) is str:
|
||||
try:
|
||||
mon_nspawn_shell(arch=job.arch, cwd=cwd, cmdline=scr, seconds=60*60)
|
||||
except Exception:
|
||||
print_exc_plus()
|
||||
# actually makepkg
|
||||
try:
|
||||
ret = mon_nspawn_shell(arch=job.arch, cwd=cwd, cmdline=mkcmd,
|
||||
logfile = cwd / 'buildbot.log.makepkg',
|
||||
short_return = True,
|
||||
seconds=job.pkgconfig.timeout*60)
|
||||
except Exception:
|
||||
logger.error(f'Job {job} failed. Running build-failure scripts')
|
||||
for scr in getattr(job.pkgconfig, 'failure', list()):
|
||||
if type(scr) is str:
|
||||
try:
|
||||
mon_nspawn_shell(arch=job.arch, cwd=cwd, cmdline=scr, seconds=60*60)
|
||||
except Exception:
|
||||
print_exc_plus()
|
||||
raise
|
||||
# run post-makepkg-scripts
|
||||
logger.info('running post-build scripts')
|
||||
for scr in getattr(job.pkgconfig, 'postbuild', list()):
|
||||
if type(scr) is str:
|
||||
try:
|
||||
mon_nspawn_shell(arch=job.arch, cwd=cwd, cmdline=scr, seconds=60*60)
|
||||
except Exception:
|
||||
print_exc_plus()
|
||||
return ret
|
||||
def __clean(self, job, remove_pkg=False, rm_src=True):
|
||||
cwd = REPO_ROOT / job.pkgconfig.dirname
|
||||
logger.info('cleaning build dir for %s, %sremoving pkg',
|
||||
|
|
@ -199,27 +236,41 @@ class jobsManager:
|
|||
f_to_upload.append(sigpath)
|
||||
f_to_upload.append(fpath)
|
||||
for f in f_to_upload:
|
||||
size = f.stat().st_size / 1000 / 1000
|
||||
if f.name.endswith(PKG_SUFFIX):
|
||||
for _ in range(10):
|
||||
timeout = rrun('push_start', args=(f.name, size))
|
||||
if timeout > 0:
|
||||
break
|
||||
max_tries = 5
|
||||
for tries in range(max_tries):
|
||||
try:
|
||||
size = f.stat().st_size / 1000 / 1000
|
||||
if f.name.endswith(PKG_SUFFIX):
|
||||
for _ in range(10):
|
||||
timeout = rrun('push_start', args=(f.name, size))
|
||||
if timeout > 0:
|
||||
break
|
||||
else:
|
||||
logger.warning('Remote is busy (-1), wait 1 min x10')
|
||||
sleep(60)
|
||||
else:
|
||||
logger.warning('Remote is busy (-1), wait 1 min x10')
|
||||
sleep(60)
|
||||
else:
|
||||
timeout = 60
|
||||
logger.info(f'Uploading {f}, timeout in {timeout}s')
|
||||
mon_bash(UPLOAD_CMD.format(src=f), seconds=int(timeout))
|
||||
if f.name.endswith(PKG_SUFFIX):
|
||||
logger.info(f'Requesting repo update for {f.name}')
|
||||
res = rrun('push_done', args=(f.name,), kwargs={'overwrite': False,})
|
||||
if res is None:
|
||||
logger.info(f'Update success for {f.name}')
|
||||
timeout = 60
|
||||
logger.info(f'Uploading {f}, timeout in {timeout}s')
|
||||
mon_bash(UPLOAD_CMD.format(src=f), seconds=int(timeout))
|
||||
if f.name.endswith(PKG_SUFFIX):
|
||||
logger.info(f'Requesting repo update for {f.name}')
|
||||
res = rrun('push_done', args=(f.name,), kwargs={'overwrite': False,})
|
||||
if res is None:
|
||||
logger.info(f'Update success for {f.name}')
|
||||
else:
|
||||
logger.error(f'Update failed for {f.name}, reason: {res}')
|
||||
suc = False
|
||||
except Exception:
|
||||
time_to_sleep = (tries + 1) * 60
|
||||
logger.error(f'We are getting problem uploading {f}, wait {time_to_sleep} secs')
|
||||
print_exc_plus()
|
||||
if tries + 1 < max_tries:
|
||||
sleep(time_to_sleep)
|
||||
else:
|
||||
logger.error(f'Update failed for {f.name}, reason: {res}')
|
||||
suc = False
|
||||
break
|
||||
else:
|
||||
logger.error(f'Upload {f} failed, abort.')
|
||||
raise RuntimeError('Unable to upload some files')
|
||||
return suc
|
||||
def tick(self):
|
||||
'''
|
||||
|
|
@ -232,24 +283,25 @@ class jobsManager:
|
|||
if time() - self.last_updatecheck <= UPDATE_INTERVAL * 60:
|
||||
if not self.idle:
|
||||
logger.info('Buildbot is idling for package updates.')
|
||||
self.idle = True
|
||||
sleep(60)
|
||||
return
|
||||
self.last_updatecheck = time()
|
||||
self.idle = False
|
||||
# git pull repo
|
||||
try:
|
||||
bash(GIT_PULL, cwd=REPO_ROOT)
|
||||
except Exception:
|
||||
print_exc_plus()
|
||||
self.pkgconfigs = load_all_yaml()
|
||||
updates = updmgr.check_update()
|
||||
for update in updates:
|
||||
(pkgconfig, ver, buildarchs) = update
|
||||
march = True if len(buildarchs) >= 2 else False
|
||||
for arch in buildarchs:
|
||||
newjob = Job(arch, pkgconfig, ver, multiarch=march)
|
||||
self._new_buildjob(newjob)
|
||||
self.idle = True
|
||||
return 60
|
||||
else:
|
||||
self.last_updatecheck = time()
|
||||
self.idle = False
|
||||
# git pull repo
|
||||
try:
|
||||
bash(GIT_PULL, cwd=REPO_ROOT)
|
||||
except Exception:
|
||||
print_exc_plus()
|
||||
self.pkgconfigs = load_all_yaml()
|
||||
updates = updmgr.check_update()
|
||||
for update in updates:
|
||||
(pkgconfig, ver, buildarchs) = update
|
||||
march = True if len(buildarchs) >= 2 else False
|
||||
for arch in buildarchs:
|
||||
newjob = Job(arch, pkgconfig, ver, multiarch=march)
|
||||
self._new_buildjob(newjob)
|
||||
return 0
|
||||
else:
|
||||
# This part does the job
|
||||
self.idle = False
|
||||
|
|
@ -272,35 +324,41 @@ class jobsManager:
|
|||
else:
|
||||
self.__clean(job, rm_src=False, remove_pkg=True)
|
||||
self.__finish_job(job.pkgconfig.dirname)
|
||||
return 0
|
||||
|
||||
jobsmgr = jobsManager()
|
||||
|
||||
class updateManager:
|
||||
def __init__(self, filename='pkgver.json'):
|
||||
self.__filename = filename
|
||||
self.__pkgerrs = dict()
|
||||
self.__pkgvers = dict()
|
||||
self.__load()
|
||||
def __load(self):
|
||||
if Path(self.__filename).exists():
|
||||
with open(self.__filename,"r") as f:
|
||||
try:
|
||||
pkgvers = json.loads(f.read())
|
||||
pkgdata = json.loads(f.read())
|
||||
except json.JSONDecodeError:
|
||||
logger.error('pkgver.json - Bad json')
|
||||
print_exc_plus
|
||||
exit(1)
|
||||
else:
|
||||
logger.warning(f'No {self.__filename} found')
|
||||
pkgvers = dict()
|
||||
assert type(pkgvers) is dict
|
||||
for pkgname in pkgvers:
|
||||
pkgdata = dict()
|
||||
assert type(pkgdata) is dict
|
||||
for pkgname in pkgdata:
|
||||
assert type(pkgname) is str
|
||||
self.__pkgvers = pkgvers
|
||||
assert len(pkgdata[pkgname]) == 2
|
||||
self.__pkgvers = {pkgname:pkgdata[pkgname][0] for pkgname in pkgdata}
|
||||
self.__pkgerrs = {pkgname:pkgdata[pkgname][1] for pkgname in pkgdata}
|
||||
def _save(self):
|
||||
pkgvers = json.dumps(self.__pkgvers, indent=4)
|
||||
pkgvers += '\n'
|
||||
pkgdata = {pkgname:[self.__pkgvers[pkgname], self.__pkgerrs[pkgname]] for pkgname in self.__pkgvers}
|
||||
pkgdatastr = json.dumps(pkgdata, indent=4)
|
||||
pkgdatastr += '\n'
|
||||
with open(self.__filename,"w") as f:
|
||||
if f.writable:
|
||||
f.write(pkgvers)
|
||||
f.write(pkgdatastr)
|
||||
else:
|
||||
logger.error('pkgver.json - Not writable')
|
||||
def __get_package_list(self, dirname, arch):
|
||||
|
|
@ -332,12 +390,21 @@ class updateManager:
|
|||
continue
|
||||
# hopefully we only need to check one arch for update
|
||||
arch = 'x86_64' if 'x86_64' in buildarchs else buildarchs[0] # prefer x86
|
||||
# run pre_update_scripts
|
||||
logger.info('running pre-update scripts')
|
||||
for scr in getattr(pkg, 'update', list()):
|
||||
if type(scr) is str:
|
||||
mon_nspawn_shell(arch, scr, cwd=pkgdir, seconds=60*60)
|
||||
mon_nspawn_shell(arch, MAKEPKG_UPD_CMD, cwd=pkgdir, seconds=60*60,
|
||||
logfile = pkgdir / 'buildbot.log.update',
|
||||
short_return = True)
|
||||
if pkg.type in ('git', 'manual'):
|
||||
ver = self.__get_new_ver(pkg.dirname, arch)
|
||||
oldver = self.__pkgvers.get(pkg.dirname, None)
|
||||
errs = self.__pkgerrs.get(pkg.dirname, 0)
|
||||
if errs >= 2:
|
||||
logger.warning(f'package: {pkg.dirname} too many failures checking update')
|
||||
continue
|
||||
has_update = False
|
||||
if rebuild_package:
|
||||
has_update = True
|
||||
|
|
@ -353,10 +420,12 @@ class updateManager:
|
|||
has_update = True
|
||||
if has_update:
|
||||
self.__pkgvers[pkg.dirname] = ver
|
||||
self.__pkgerrs[pkg.dirname] = 0
|
||||
updates.append((pkg, ver, buildarchs))
|
||||
else:
|
||||
logger.warning(f'unknown package type: {pkg.type}')
|
||||
except Exception:
|
||||
self.__pkgerrs[pkg.dirname] = self.__pkgerrs.get(pkg.dirname, 0) + 1
|
||||
print_exc_plus()
|
||||
self._save()
|
||||
return updates
|
||||
|
|
@ -414,11 +483,23 @@ if __name__ == '__main__':
|
|||
logger.info('Listener started.')
|
||||
while True:
|
||||
try:
|
||||
jobsmgr.tick()
|
||||
ret = 1
|
||||
ret = jobsmgr.tick()
|
||||
except Exception:
|
||||
try:
|
||||
jobsmgr.clean_failed_job()
|
||||
except Exception:
|
||||
print_exc_plus()
|
||||
print_exc_plus()
|
||||
except KeyboardInterrupt:
|
||||
logger.info('KeyboardInterrupt')
|
||||
print_exc_plus()
|
||||
break
|
||||
sleep(1)
|
||||
if ret is None:
|
||||
sleep(1)
|
||||
elif ret == 0:
|
||||
pass
|
||||
elif type(ret) in (int, float):
|
||||
sleep(ret)
|
||||
else:
|
||||
sleep(1)
|
||||
|
|
|
|||
13
extra.py
Normal file
13
extra.py
Normal file
|
|
@ -0,0 +1,13 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
import os
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
from utils import print_exc_plus
|
||||
|
||||
logger = logging.getLogger(f'buildbot.{__name__}')
|
||||
|
||||
abspath=os.path.abspath(__file__)
|
||||
abspath=os.path.dirname(abspath)
|
||||
os.chdir(abspath)
|
||||
16
notify.py
Normal file
16
notify.py
Normal file
|
|
@ -0,0 +1,16 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
# notify.py: Automatic management tool for an arch repo.
|
||||
# This file is part of Buildbot by JerryXiao
|
||||
|
||||
|
||||
import logging
|
||||
from utils import background, print_exc_plus, configure_logger
|
||||
|
||||
logger = logging.getLogger(f'buildbot.{__name__}')
|
||||
|
||||
# wip
|
||||
# does nothing
|
||||
@background
|
||||
def send(content):
|
||||
pass
|
||||
10
utils.py
10
utils.py
|
|
@ -261,11 +261,19 @@ def format_exc_plus():
|
|||
|
||||
def configure_logger(logger, format='%(asctime)s - %(name)-18s - %(levelname)s - %(message)s',
|
||||
level=logging.INFO, logfile=None, flevel=logging.DEBUG, rotate_size=None):
|
||||
try:
|
||||
from notify import send
|
||||
except ModuleNotFoundError:
|
||||
def send(*args):
|
||||
pass
|
||||
|
||||
class ExceptionFormatter(logging.Formatter):
|
||||
def format(self, record):
|
||||
if record.levelno == 49:
|
||||
record.msg = 'Exception caught.\nPrinting stack traceback\n' + record.msg
|
||||
return super().format(record)
|
||||
fmtr = super().format(record)
|
||||
send(fmtr)
|
||||
return fmtr
|
||||
|
||||
logger.setLevel(logging.DEBUG)
|
||||
formatter = ExceptionFormatter(fmt=format)
|
||||
|
|
|
|||
Loading…
Reference in a new issue