Archlinux-Jerry
/
buildbot
mirror of https://github.com/archlinux-jerry/buildbot
1
0
Fork 0
Code Issues Releases Wiki Activity

buildbot: add git and client

This commit is contained in:
JerryXiao 2019-04-10 20:36:43 +08:00
parent 84f8afd8a0
commit 7203913efc
Signed by: Jerry
GPG Key ID: 9D9CE43650FF2BAA
5 changed files with 123 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

93
buildbot.py
View File

@ -18,7 +18,8 @@ from config import ARCHS, BUILD_ARCHS, BUILD_ARCH_MAPPING, \
PKGBUILD_DIR, MAKEPKG_PKGLIST_CMD, MAKEPKG_UPD_CMD, \
MAKEPKG_MAKE_CMD, MAKEPKG_MAKE_CMD_CLEAN, \
GPG_SIGN_CMD, GPG_VERIFY_CMD, UPDATE_INTERVAL, \
MAKEPKG_MAKE_CMD_MARCH, UPLOAD_CMD
MAKEPKG_MAKE_CMD_MARCH, UPLOAD_CMD, \
GIT_PULL, GIT_RESET_SUBDIR
from utils import print_exc_plus, background, \
bash, get_pkg_details_from_name, vercmp, \
@ -48,7 +49,14 @@ class Job:
self.version = version
self.multiarch = multiarch
self.added = time()
def __repr__(self):
ret = "Job("
for myproperty in (
'arch', 'pkgconfig', 'version', 'multiarch', 'added'
):
ret += f'{myproperty}={getattr(self, myproperty, None)},'
ret += ')'
return ret
class jobsManager:
def __init__(self):
self.__buildjobs = list()
@ -57,6 +65,52 @@ class jobsManager:
self.pkgconfigs = None
self.last_updatecheck = 0.0
self.idle = False
def __repr__(self):
ret = "jobsManager("
for myproperty in (
'__buildjobs', '__uploadjobs', '__curr_job',
'pkgconfigs', 'last_updatecheck', 'idle'
):
ret += f'{myproperty}={getattr(self, myproperty, None)},'
ret += ')'
return ret
def reset_dir(self, pkgdirname=None, all=False):
if all:
logger.info('git checkout all: %s', bash(GIT_RESET_SUBDIR, cwd=REPO_ROOT))
else:
if not pkgdirname:
return False
cwd = REPO_ROOT / pkgdirname
if cwd.exists():
logger.info('git checkout: %s', bash(GIT_RESET_SUBDIR, cwd=cwd))
for fpath in [f for f in cwd.iterdir()]:
if fpath.is_dir() and \
fpath.name in ('pkg', 'src'):
rmtree(fpath)
elif fpath.is_file() and \
(fpath.name.endswith(PKG_SUFFIX) or \
fpath.name.endswith(PKG_SIG_SUFFIX)):
fpath.unlink()
else:
return False
return True
def rebuild_package(self, pkgdirname, clean=False):
self.pkgconfigs = load_all_yaml()
updates = updmgr.check_update(rebuild_package=pkgdirname)
if updates and len(updates) == 1:
if clean:
self.reset_dir(pkgdirname)
(pkgconfig, ver, buildarchs) = updates[0]
march = True if len(buildarchs) >= 2 else False
for arch in buildarchs:
newjob = Job(arch, pkgconfig, ver, multiarch=march)
self._new_buildjob(newjob)
ret = f'rebuild job added for {pkgdirname} {" ".join(buildarchs)}'
logger.info(ret)
else:
ret = f'rebuild failed: no such dir {pkgdirname}'
logger.warning(ret)
return ret
def _new_buildjob(self, job):
assert type(job) is Job
job_to_remove = list()
@ -74,7 +128,9 @@ class jobsManager:
self.__buildjobs.append(job)
def __get_job(self):
if self.__curr_job:
return None
logger.error(f'Job {self.__curr_job} failed')
self.__finish_job(self.__curr_job)
return self.__get_job()
jobs = self.__buildjobs
if jobs:
self.__curr_job = jobs.pop(0)
@ -168,6 +224,11 @@ class jobsManager:
return
self.last_updatecheck = time()
self.idle = False
# git pull repo
try:
bash(GIT_PULL, cwd=REPO_ROOT)
except Exception:
print_exc_plus()
self.pkgconfigs = load_all_yaml()
updates = updmgr.check_update()
for update in updates:
@ -179,6 +240,9 @@ class jobsManager:
else:
# This part does the job
job = self.__get_job()
if not job:
logging.error('No job got')
return
if job.multiarch:
self.__clean(job, remove_pkg=True)
self.__makepkg(job)
@ -236,15 +300,21 @@ class updateManager:
pkgfiles = self.__get_package_list(dirname, arch)
ver = get_pkg_details_from_name(pkgfiles[0]).ver
return ver
def check_update(self):
def check_update(self, rebuild_package=None):
updates = list()
for pkg in jobsmgr.pkgconfigs:
if rebuild_package and \
rebuild_package != pkg.dirname:
continue
pkgdir = REPO_ROOT / pkg.dirname
logger.info(f'checking update: {pkg.dirname}')
pkgbuild = pkgdir / 'PKGBUILD'
archs = get_arch_from_pkgbuild(pkgbuild)
buildarchs = [BUILD_ARCH_MAPPING.get(arch, None) for arch in archs]
buildarchs = [arch for arch in buildarchs if arch is not None]
if not buildarchs:
logger.warning(f'No build arch for {pkg.dirname}, refuse to build.')
continue
# hopefully we only need to check one arch for update
arch = 'x86_64' if 'x86_64' in buildarchs else buildarchs[0] # prefer x86
mon_nspawn_shell(arch, MAKEPKG_UPD_CMD, cwd=pkgdir, seconds=60*60,
@ -277,11 +347,20 @@ updmgr = updateManager()
def info(*args, **kwargs):
return (args, kwargs)
def info():
return (str(jobsmgr))
def rebuild_package(pkgdirname, clean=False):
return jobsmgr.rebuild_package(pkgdirname, clean=clean)
def clean(pkgdirname):
return jobsmgr.reset_dir(pkgdirname=pkgdirname)
def clean_all():
return jobsmgr.reset_dir(all=True)
def run(funcname, args=list(), kwargs=dict()):
if funcname in ('info',):
if funcname in ('info', 'rebuild_package', 'clean', 'clean_all'):
logger.info('running: %s %s %s',funcname, args, kwargs)
ret = eval(funcname)(*args, **kwargs)
logger.info('done: %s %s',funcname, ret)

41
client.py Normal file → Executable file
View File

@ -7,22 +7,25 @@ import logging
from multiprocessing.connection import Client
from time import sleep
from config import REPOD_BIND_ADDRESS, REPOD_BIND_PASSWD
from config import REPOD_BIND_ADDRESS, REPOD_BIND_PASSWD, \
MASTER_BIND_ADDRESS, MASTER_BIND_PASSWD
from utils import print_exc_plus
logger = logging.getLogger(f'buildbot.{__name__}')
def run(funcname, args=list(), kwargs=dict(), retries=0):
def run(funcname, args=list(), kwargs=dict(), retries=0, server=(REPOD_BIND_ADDRESS, REPOD_BIND_PASSWD)):
try:
logger.info('client: %s %s %s',funcname, args, kwargs)
with Client(REPOD_BIND_ADDRESS, authkey=REPOD_BIND_PASSWD) as conn:
(addr, authkey) = server
with Client(addr, authkey=authkey) as conn:
conn.send([funcname, args, kwargs])
return conn.recv()
except ConnectionRefusedError:
if retries <= 10:
logger.info("Server refused, retry after 60s")
sleep(60)
return ping(funcname, args=args, kwargs=kwargs, retries=retries+1)
return run(funcname, args=args, kwargs=kwargs, retries=retries+1)
else:
logger.error("Server refused")
return False
@ -34,7 +37,29 @@ def run(funcname, args=list(), kwargs=dict(), retries=0):
if __name__ == '__main__':
import argparse
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
logger.info('result: %s', run('push_files', args=('aaa', 1)))
logger.info('result: %s', run('add_files', args=('aaa',)))
#logger.info('result: %s', run('update'))
from utils import configure_logger
configure_logger(logger)
try:
parser = argparse.ArgumentParser(description='Client for buildbot')
parser.add_argument('--update', action='store_true', help='update pushed files to the repo')
parser.add_argument('--cleanall', action='store_true', help='checkout pkgbuilds')
parser.add_argument('--clean', nargs='?', default=None, help='checkout pkgbuilds in one package')
parser.add_argument('--rebuild', nargs='?', default=None, help='rebuild a package with its dirname')
args = parser.parse_args()
if args.update:
server=(REPOD_BIND_ADDRESS, REPOD_BIND_PASSWD)
logger.info(run('update', kwargs={'overwrite': False}, server=server))
elif args.cleanall:
server=(MASTER_BIND_ADDRESS, MASTER_BIND_PASSWD)
logger.info(run('clean_all', server=server))
elif args.clean:
server=(MASTER_BIND_ADDRESS, MASTER_BIND_PASSWD)
logger.info(run('clean', args=(args.clean,), server=server))
elif args.rebuild:
server=(MASTER_BIND_ADDRESS, MASTER_BIND_PASSWD)
logger.info(run('rebuild', args=(args.rebuild,), kwargs={'clean': True}, server=server))
else:
parser.error("Please choose an action")
except Exception:
print_exc_plus()
parser.exit(status=1)

3
config.py
View File

@ -59,3 +59,6 @@ SHELL_ARM64_ADDITIONAL = 'set -e; set -x'
SHELL_TRAP = 'trap \'echo ++ exit $?\' ERR EXIT'
UPLOAD_CMD = 'rsync -avPh \"{src}\" repoupload:/srv/repo/buildbot/repo/updates/'
GIT_PULL = 'git pull'
GIT_RESET_SUBDIR = 'git checkout HEAD -- .'

1
utils.py
View File

@ -211,7 +211,6 @@ def get_arch_from_pkgbuild(fpath):
if not matches:
raise TypeError('Unexpected PKGBUILD format')
matches = [arch for arch in matches if arch in ARCHS]
assert matches
return matches
raise TypeError('Unexpected PKGBUILD')

2
yamlparse.py
View File

@ -65,7 +65,7 @@ class pkgConfig:
'dirname', 'type', 'cleanbuild', 'timeout'
'prebuild', 'postbuild', 'update', 'failure'
):
ret += f'{myproperty}={getattr(self, myproperty)},'
ret += f'{myproperty}={getattr(self, myproperty, None)},'
ret += ')'
return ret