425 lines
16 KiB
Python
Executable File
425 lines
16 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
# -*- coding: utf-8 -*-
|
|
# buildbot.py: Automatic management tool for an arch repo.
|
|
# This file is part of Buildbot by JerryXiao
|
|
|
|
import logging
|
|
from multiprocessing.connection import Listener
|
|
from time import time, sleep
|
|
import os
|
|
from pathlib import Path
|
|
from shutil import rmtree
|
|
from subprocess import CalledProcessError
|
|
|
|
from shared_vars import PKG_SUFFIX, PKG_SIG_SUFFIX
|
|
|
|
from config import ARCHS, BUILD_ARCHS, BUILD_ARCH_MAPPING, \
|
|
MASTER_BIND_ADDRESS, MASTER_BIND_PASSWD, \
|
|
PKGBUILD_DIR, MAKEPKG_PKGLIST_CMD, MAKEPKG_UPD_CMD, \
|
|
MAKEPKG_MAKE_CMD, MAKEPKG_MAKE_CMD_CLEAN, \
|
|
GPG_SIGN_CMD, GPG_VERIFY_CMD, UPDATE_INTERVAL, \
|
|
MAKEPKG_MAKE_CMD_MARCH, UPLOAD_CMD, \
|
|
GIT_PULL, GIT_RESET_SUBDIR
|
|
|
|
from utils import print_exc_plus, background, \
|
|
bash, get_pkg_details_from_name, vercmp, \
|
|
nspawn_shell, mon_nspawn_shell, get_arch_from_pkgbuild, \
|
|
configure_logger, mon_bash
|
|
|
|
from client import run as rrun
|
|
|
|
import json
|
|
|
|
from yamlparse import load_all as load_all_yaml
|
|
|
|
abspath=os.path.abspath(__file__)
|
|
abspath=os.path.dirname(abspath)
|
|
os.chdir(abspath)
|
|
|
|
logger = logging.getLogger('buildbot')
|
|
configure_logger(logger, logfile='buildbot.log', rotate_size=1024*1024*10)
|
|
|
|
# refuse to run in systemd-nspawn
|
|
if 'systemd-nspawn' in bash('systemd-detect-virt || true'):
|
|
logger.error('Refused to run in systemd-nspawn.')
|
|
raise AssertionError('Refused to run in systemd-nspawn.')
|
|
|
|
REPO_ROOT = Path(PKGBUILD_DIR)
|
|
|
|
class Job:
|
|
def __init__(self, buildarch, pkgconfig, version, multiarch=False):
|
|
assert buildarch in BUILD_ARCHS
|
|
self.arch = buildarch
|
|
self.pkgconfig = pkgconfig
|
|
self.version = version
|
|
self.multiarch = multiarch
|
|
self.added = time()
|
|
def __repr__(self):
|
|
ret = "Job("
|
|
for myproperty in (
|
|
'arch', 'pkgconfig', 'version', 'multiarch', 'added'
|
|
):
|
|
ret += f'{myproperty}={getattr(self, myproperty, None)},'
|
|
ret += ')'
|
|
return ret
|
|
class jobsManager:
|
|
def __init__(self):
|
|
self.__buildjobs = list()
|
|
self.__uploadjobs = list()
|
|
self.__curr_job = None
|
|
self.pkgconfigs = None
|
|
self.last_updatecheck = 0.0
|
|
self.idle = False
|
|
@property
|
|
def jobs(self):
|
|
return \
|
|
{
|
|
'build_jobs': self.__buildjobs,
|
|
'upload_jobs': self.__uploadjobs,
|
|
'current_job': self.__curr_job
|
|
}
|
|
def __repr__(self):
|
|
ret = "jobsManager("
|
|
for myproperty in (
|
|
'jobs', 'pkgconfigs',
|
|
'last_updatecheck', 'idle'
|
|
):
|
|
ret += f'{myproperty}={getattr(self, myproperty, None)},'
|
|
ret += ')'
|
|
return ret
|
|
def reset_dir(self, pkgdirname=None, all=False):
|
|
if all:
|
|
logger.info('resetting %s', str(REPO_ROOT))
|
|
bash(GIT_RESET_SUBDIR, cwd=REPO_ROOT)
|
|
else:
|
|
if not pkgdirname:
|
|
return False
|
|
cwd = REPO_ROOT / pkgdirname
|
|
if cwd.exists():
|
|
logger.info('resetting %s', str(cwd))
|
|
bash(GIT_RESET_SUBDIR, cwd=cwd)
|
|
for fpath in [f for f in cwd.iterdir()]:
|
|
if fpath.is_dir() and \
|
|
fpath.name in ('pkg', 'src'):
|
|
rmtree(fpath)
|
|
elif fpath.is_file() and \
|
|
(fpath.name.endswith(PKG_SUFFIX) or \
|
|
fpath.name.endswith(PKG_SIG_SUFFIX)):
|
|
fpath.unlink()
|
|
else:
|
|
return False
|
|
return True
|
|
def rebuild_package(self, pkgdirname, clean=False):
|
|
self.pkgconfigs = load_all_yaml()
|
|
if (REPO_ROOT / pkgdirname).exists() and clean:
|
|
self.reset_dir(pkgdirname)
|
|
updates = updmgr.check_update(rebuild_package=pkgdirname)
|
|
if updates and len(updates) == 1:
|
|
(pkgconfig, ver, buildarchs) = updates[0]
|
|
march = True if len(buildarchs) >= 2 else False
|
|
for arch in buildarchs:
|
|
newjob = Job(arch, pkgconfig, ver, multiarch=march)
|
|
self._new_buildjob(newjob)
|
|
ret = f'rebuild job added for {pkgdirname} {" ".join(buildarchs)}'
|
|
logger.info(ret)
|
|
else:
|
|
ret = f'rebuild failed: no such dir {pkgdirname}'
|
|
logger.warning(ret)
|
|
return ret
|
|
def _new_buildjob(self, job):
|
|
assert type(job) is Job
|
|
job_to_remove = list()
|
|
for previous_job in self.__buildjobs:
|
|
if job.pkgconfig.dirname == previous_job.pkgconfig.dirname and \
|
|
job.arch == previous_job.arch:
|
|
job_to_remove.append(previous_job)
|
|
for oldjob in job_to_remove:
|
|
self.__buildjobs.remove(oldjob)
|
|
logger.info('removed an old job for %s %s, %s => %s',
|
|
job.pkgconfig.dirname, job.arch,
|
|
oldjob.version, job.version)
|
|
logger.info('new job for %s %s %s',
|
|
job.pkgconfig.dirname, job.arch, job.version)
|
|
self.__buildjobs.append(job)
|
|
def __get_job(self):
|
|
if self.__curr_job:
|
|
logger.error(f'Job {self.__curr_job} failed. Correct the error and rebuild')
|
|
self.__finish_job(self.__curr_job, force=True)
|
|
return self.__get_job()
|
|
jobs = self.__buildjobs
|
|
if jobs:
|
|
self.__curr_job = jobs.pop(0)
|
|
return self.__curr_job
|
|
def __finish_job(self, pkgdir, force=False):
|
|
if not force:
|
|
assert pkgdir == self.__curr_job.pkgconfig.dirname
|
|
self.__curr_job = None
|
|
return True
|
|
def __makepkg(self, job):
|
|
cwd = REPO_ROOT / job.pkgconfig.dirname
|
|
if job.multiarch:
|
|
# assume a clean env, no source avail
|
|
mkcmd = MAKEPKG_MAKE_CMD_MARCH
|
|
else:
|
|
mkcmd = MAKEPKG_MAKE_CMD_CLEAN if job.pkgconfig.cleanbuild \
|
|
else MAKEPKG_MAKE_CMD
|
|
logger.info('makepkg in %s %s', job.pkgconfig.dirname, job.arch)
|
|
return mon_nspawn_shell(arch=job.arch, cwd=cwd, cmdline=mkcmd,
|
|
logfile = cwd / 'buildbot.log.makepkg',
|
|
short_return = True,
|
|
seconds=job.pkgconfig.timeout*60)
|
|
def __clean(self, job, remove_pkg=False, rm_src=True):
|
|
cwd = REPO_ROOT / job.pkgconfig.dirname
|
|
logger.info('cleaning build dir for %s, %sremoving pkg',
|
|
job.pkgconfig.dirname, '' if remove_pkg else 'not ')
|
|
for fpath in [f for f in cwd.iterdir()]:
|
|
if rm_src and fpath.is_dir() and \
|
|
fpath.name in ('pkg', 'src'):
|
|
rmtree(fpath)
|
|
elif remove_pkg and fpath.is_file() and \
|
|
((not job.multiarch) or job.arch in fpath.name) and \
|
|
(fpath.name.endswith(PKG_SUFFIX) or \
|
|
fpath.name.endswith(PKG_SIG_SUFFIX)):
|
|
fpath.unlink()
|
|
def __sign(self, job):
|
|
logger.info('signing in %s %s', job.pkgconfig.dirname, job.arch)
|
|
cwd = REPO_ROOT / job.pkgconfig.dirname
|
|
for fpath in cwd.iterdir():
|
|
if fpath.name.endswith(PKG_SUFFIX):
|
|
bash(f'{GPG_SIGN_CMD} {fpath.name}', cwd=cwd)
|
|
def __upload(self, job):
|
|
suc = True
|
|
cwd = REPO_ROOT / job.pkgconfig.dirname
|
|
f_to_upload = list()
|
|
for fpath in cwd.iterdir():
|
|
if fpath.name.endswith(PKG_SUFFIX) and \
|
|
get_pkg_details_from_name(fpath.name).ver == job.version:
|
|
sigpath = fpath.parent / f'{fpath.name}.sig'
|
|
assert sigpath.exists()
|
|
f_to_upload.append(sigpath)
|
|
f_to_upload.append(fpath)
|
|
for f in f_to_upload:
|
|
size = f.stat().st_size / 1000 / 1000
|
|
if f.name.endswith(PKG_SUFFIX):
|
|
for _ in range(10):
|
|
timeout = rrun('push_start', args=(f.name, size))
|
|
if timeout > 0:
|
|
break
|
|
else:
|
|
logger.warning('Remote is busy (-1), wait 1 min x10')
|
|
sleep(60)
|
|
else:
|
|
timeout = 60
|
|
logger.info(f'Uploading {f}, timeout in {timeout}s')
|
|
mon_bash(UPLOAD_CMD.format(src=f), seconds=int(timeout))
|
|
if f.name.endswith(PKG_SUFFIX):
|
|
logger.info(f'Requesting repo update for {f.name}')
|
|
res = rrun('push_done', args=(f.name,), kwargs={'overwrite': False,})
|
|
if res is None:
|
|
logger.info(f'Update success for {f.name}')
|
|
else:
|
|
logger.error(f'Update failed for {f.name}, reason: {res}')
|
|
suc = False
|
|
return suc
|
|
def tick(self):
|
|
'''
|
|
check for updates,
|
|
create new jobs
|
|
and run them
|
|
'''
|
|
if not self.__buildjobs:
|
|
# This part check for updates
|
|
if time() - self.last_updatecheck <= UPDATE_INTERVAL * 60:
|
|
if not self.idle:
|
|
logger.info('Buildbot is idling for package updates.')
|
|
self.idle = True
|
|
sleep(60)
|
|
return
|
|
self.last_updatecheck = time()
|
|
self.idle = False
|
|
# git pull repo
|
|
try:
|
|
bash(GIT_PULL, cwd=REPO_ROOT)
|
|
except Exception:
|
|
print_exc_plus()
|
|
self.pkgconfigs = load_all_yaml()
|
|
updates = updmgr.check_update()
|
|
for update in updates:
|
|
(pkgconfig, ver, buildarchs) = update
|
|
march = True if len(buildarchs) >= 2 else False
|
|
for arch in buildarchs:
|
|
newjob = Job(arch, pkgconfig, ver, multiarch=march)
|
|
self._new_buildjob(newjob)
|
|
else:
|
|
# This part does the job
|
|
self.idle = False
|
|
job = self.__get_job()
|
|
if not job:
|
|
logging.error('No job got')
|
|
return
|
|
if job.multiarch:
|
|
self.__clean(job, remove_pkg=True)
|
|
self.__makepkg(job)
|
|
self.__sign(job)
|
|
if self.__upload(job):
|
|
self.__clean(job, remove_pkg=True)
|
|
else:
|
|
self.__makepkg(job)
|
|
self.__sign(job)
|
|
if self.__upload(job):
|
|
if job.pkgconfig.cleanbuild:
|
|
self.__clean(job, remove_pkg=True)
|
|
else:
|
|
self.__clean(job, rm_src=False, remove_pkg=True)
|
|
self.__finish_job(job.pkgconfig.dirname)
|
|
jobsmgr = jobsManager()
|
|
|
|
class updateManager:
|
|
def __init__(self, filename='pkgver.json'):
|
|
self.__filename = filename
|
|
self.__pkgvers = dict()
|
|
self.__load()
|
|
def __load(self):
|
|
if Path(self.__filename).exists():
|
|
with open(self.__filename,"r") as f:
|
|
try:
|
|
pkgvers = json.loads(f.read())
|
|
except json.JSONDecodeError:
|
|
logger.error('pkgver.json - Bad json')
|
|
print_exc_plus
|
|
exit(1)
|
|
else:
|
|
logger.warning(f'No {self.__filename} found')
|
|
pkgvers = dict()
|
|
assert type(pkgvers) is dict
|
|
for pkgname in pkgvers:
|
|
assert type(pkgname) is str
|
|
self.__pkgvers = pkgvers
|
|
def _save(self):
|
|
pkgvers = json.dumps(self.__pkgvers, indent=4)
|
|
pkgvers += '\n'
|
|
with open(self.__filename,"w") as f:
|
|
if f.writable:
|
|
f.write(pkgvers)
|
|
else:
|
|
logger.error('pkgver.json - Not writable')
|
|
def __get_package_list(self, dirname, arch):
|
|
pkgdir = REPO_ROOT / dirname
|
|
assert pkgdir.exists()
|
|
pkglist = nspawn_shell(arch, MAKEPKG_PKGLIST_CMD, cwd=pkgdir)
|
|
pkglist = pkglist.split('\n')
|
|
pkglist = [line for line in pkglist if not line.startswith('+')]
|
|
return pkglist
|
|
def __get_new_ver(self, dirname, arch):
|
|
pkgfiles = self.__get_package_list(dirname, arch)
|
|
ver = get_pkg_details_from_name(pkgfiles[0]).ver
|
|
return ver
|
|
def check_update(self, rebuild_package=None):
|
|
updates = list()
|
|
for pkg in jobsmgr.pkgconfigs:
|
|
try:
|
|
if rebuild_package and \
|
|
rebuild_package != pkg.dirname:
|
|
continue
|
|
pkgdir = REPO_ROOT / pkg.dirname
|
|
logger.info(f'checking update: {pkg.dirname}')
|
|
pkgbuild = pkgdir / 'PKGBUILD'
|
|
archs = get_arch_from_pkgbuild(pkgbuild)
|
|
buildarchs = [BUILD_ARCH_MAPPING.get(arch, None) for arch in archs]
|
|
buildarchs = [arch for arch in buildarchs if arch is not None]
|
|
if not buildarchs:
|
|
logger.warning(f'No build arch for {pkg.dirname}, refuse to build.')
|
|
continue
|
|
# hopefully we only need to check one arch for update
|
|
arch = 'x86_64' if 'x86_64' in buildarchs else buildarchs[0] # prefer x86
|
|
mon_nspawn_shell(arch, MAKEPKG_UPD_CMD, cwd=pkgdir, seconds=60*60,
|
|
logfile = pkgdir / 'buildbot.log.update',
|
|
short_return = True)
|
|
if pkg.type in ('git', 'manual'):
|
|
ver = self.__get_new_ver(pkg.dirname, arch)
|
|
oldver = self.__pkgvers.get(pkg.dirname, None)
|
|
has_update = False
|
|
if rebuild_package:
|
|
has_update = True
|
|
if oldver:
|
|
res = vercmp(ver, oldver)
|
|
if res == 1:
|
|
has_update = True
|
|
elif res == -1:
|
|
logger.warning(f'package: {pkg.dirname} downgrade attempted')
|
|
elif res == 0:
|
|
logger.info(f'package: {pkg.dirname} is up to date')
|
|
else:
|
|
has_update = True
|
|
if has_update:
|
|
self.__pkgvers[pkg.dirname] = ver
|
|
updates.append((pkg, ver, buildarchs))
|
|
else:
|
|
logger.warning(f'unknown package type: {pkg.type}')
|
|
except Exception:
|
|
print_exc_plus()
|
|
self._save()
|
|
return updates
|
|
|
|
updmgr = updateManager()
|
|
|
|
|
|
|
|
|
|
def info():
|
|
ret = str(jobsmgr)
|
|
ret += '\nhuman-readable:\n'
|
|
ret += "".join([f"{k} = {jobsmgr.jobs[k]}\n" for k in jobsmgr.jobs])
|
|
ret += f"idle: {jobsmgr.idle}"
|
|
return ret
|
|
|
|
def rebuild_package(pkgdirname, clean=False):
|
|
return jobsmgr.rebuild_package(pkgdirname, clean=clean)
|
|
|
|
def clean(pkgdirname):
|
|
return jobsmgr.reset_dir(pkgdirname=pkgdirname)
|
|
|
|
def clean_all():
|
|
return jobsmgr.reset_dir(all=True)
|
|
|
|
def run(funcname, args=list(), kwargs=dict()):
|
|
if funcname in ('info', 'rebuild_package', 'clean', 'clean_all'):
|
|
logger.debug('running: %s %s %s',funcname, args, kwargs)
|
|
ret = eval(funcname)(*args, **kwargs)
|
|
logger.info('done: %s %s %s',funcname, args, kwargs)
|
|
return ret
|
|
else:
|
|
logger.error('unexpected: %s %s %s',funcname, args, kwargs)
|
|
return False
|
|
|
|
@background
|
|
def __main():
|
|
while True:
|
|
try:
|
|
with Listener(MASTER_BIND_ADDRESS, authkey=MASTER_BIND_PASSWD) as listener:
|
|
with listener.accept() as conn:
|
|
logger.info('connection accepted from %s', listener.last_accepted)
|
|
myrecv = conn.recv()
|
|
if type(myrecv) is list and len(myrecv) == 3:
|
|
(funcname, args, kwargs) = myrecv
|
|
funcname = str(funcname)
|
|
logger.debug('running: %s %s %s', funcname, args, kwargs)
|
|
conn.send(run(funcname, args=args, kwargs=kwargs))
|
|
except Exception:
|
|
print_exc_plus()
|
|
|
|
if __name__ == '__main__':
|
|
logger.info('Buildbot started.')
|
|
__main() # start the Listener thread
|
|
logger.info('Listener started.')
|
|
while True:
|
|
try:
|
|
jobsmgr.tick()
|
|
except Exception:
|
|
print_exc_plus()
|
|
except KeyboardInterrupt:
|
|
logger.info('KeyboardInterrupt')
|
|
print_exc_plus()
|
|
break
|
|
sleep(1)
|