buildbot/buildbot.py

243 lines
8.8 KiB
Python
Raw Normal View History

2019-04-06 20:59:27 +08:00
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# buildbot.py: Automatic management tool for an arch repo.
# This file is part of Buildbot by JerryXiao
import logging
from multiprocessing.connection import Listener
from time import time, sleep
import os
from pathlib import Path
from subprocess import CalledProcessError
from config import ARCHS, BUILD_ARCHS, BUILD_ARCH_MAPPING, \
MASTER_BIND_ADDRESS, MASTER_BIND_PASSWD, \
2019-04-07 17:14:50 +08:00
PKGBUILD_DIR, MAKEPKG_PKGLIST_CMD, MAKEPKG_UPD_CMD, \
MAKEPKG_MAKE_CMD, MAKEPKG_MAKE_CMD_CLEAN
from utils import print_exc_plus, background, \
bash, get_pkg_details_from_name, vercmp, \
nspawn_shell, mon_nspawn_shell, get_arch_from_pkgbuild
2019-04-06 20:59:27 +08:00
import json
from yamlparse import load_all as load_all_yaml
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
logger = logging.getLogger(__name__)
abspath=os.path.abspath(__file__)
abspath=os.path.dirname(abspath)
os.chdir(abspath)
REPO_ROOT = Path(PKGBUILD_DIR)
class Job:
2019-04-07 17:14:50 +08:00
def __init__(self, buildarch, pkgconfig, version, multiarch=False):
2019-04-06 20:59:27 +08:00
assert buildarch in BUILD_ARCHS
2019-04-07 17:14:50 +08:00
self.arch = buildarch
self.pkgconfig = pkgconfig
2019-04-06 20:59:27 +08:00
self.version = version
2019-04-07 17:14:50 +08:00
self.multiarch = multiarch
2019-04-06 20:59:27 +08:00
self.added = time()
class jobsManager:
def __init__(self):
2019-04-07 17:14:50 +08:00
self.__buildjobs = list()
2019-04-06 20:59:27 +08:00
self.__uploadjobs = list()
self.__curr_job = None
self.pkgconfigs = load_all_yaml()
2019-04-07 17:14:50 +08:00
def _new_buildjob(self, job):
2019-04-06 20:59:27 +08:00
assert type(job) is Job
2019-04-07 17:14:50 +08:00
job_to_remove = list()
for previous_job in self.__buildjobs:
if job.pkgconfig.dirname == previous_job.pkgconfig.dirname and \
job.arch == previous_job.arch:
job_to_remove.append(previous_job)
for oldjob in job_to_remove:
self.__buildjobs.remove(oldjob)
logger.info('removed an old job for %s %s, %s => %s',
job.pkgconfig.dirname, job.arch,
oldjob.version, job.version)
logger.info('new job for %s %s %s',
job.pkgconfig.dirname, job.arch, job.version)
self.__buildjobs.append(job)
def __get_job(self):
2019-04-06 20:59:27 +08:00
if self.__curr_job:
return None
2019-04-07 17:14:50 +08:00
jobs = self.__buildjobs
2019-04-06 20:59:27 +08:00
if jobs:
self.__curr_job = jobs.pop(0)
return self.__curr_job
def __finish_job(self, pkgdir):
2019-04-07 17:14:50 +08:00
assert pkgdir == self.__curr_job.pkgconfig.dirname
2019-04-06 20:59:27 +08:00
# do upload
self.__curr_job = None
return True
2019-04-07 17:14:50 +08:00
def __makepkg(self, job):
mkcmd = MAKEPKG_MAKE_CMD_CLEAN if job.pkgconfig.cleanbuild \
else MAKEPKG_MAKE_CMD
cwd = REPO_ROOT / job.pkgconfig.dirname
logger.info('makepkg in %s %s', job.pkgconfig.dirname, job.arch)
return mon_nspawn_shell(arch=job.arch, cwd=cwd, cmdline=mkcmd,
logfile = cwd / 'buildbot.log.update',
short_return = True)
def __clean(self, job):
cwd = REPO_ROOT / job.pkgconfig.dirname
logger.info('cleaning build dir for %s %s',
job.pkgconfig.dirname, job.arch)
nspawn_shell(job.arch, 'rm -rf src pkg', cwd=cwd)
def __sign(self, job):
'''
wip
'''
cwd = REPO_ROOT / job.pkgconfig.dirname
print(nspawn_shell(job.arch, 'ls -l', cwd=cwd))
#nspawn_shell(job.arch, 'rm -rf src pkg', cwd=cwd)
def __upload(self, job):
'''
wip
'''
cwd = REPO_ROOT / job.pkgconfig.dirname
print(nspawn_shell(job.arch, 'ls -l', cwd=cwd))
#nspawn_shell(job.arch, 'rm -rf src pkg', cwd=cwd)
2019-04-06 20:59:27 +08:00
def tick(self):
'''
check for updates,
create new jobs
and run them
'''
2019-04-07 17:14:50 +08:00
if not self.__buildjobs:
# This part check for updates
2019-04-06 20:59:27 +08:00
updates = updmgr.check_update()
for update in updates:
2019-04-07 17:14:50 +08:00
(pkgconfig, ver, buildarchs) = update
march = True if len(buildarchs) >= 2 else False
for arch in buildarchs:
newjob = Job(arch, pkgconfig, ver, multiarch=march)
self._new_buildjob(newjob)
else:
# This part does the job
for job in self.__buildjobs:
cwd = REPO_ROOT / job.pkgconfig.dirname
if job.multiarch:
# wip
pass
else:
self.__makepkg(job)
self.__sign(job)
self.__upload(job)
2019-04-06 20:59:27 +08:00
jobsmgr = jobsManager()
class updateManager:
def __init__(self, filename='pkgver.json'):
self.__filename = filename
self.__pkgvers = dict()
self.__load()
def __load(self):
if Path(self.__filename).exists():
with open(self.__filename,"r") as f:
try:
pkgvers = json.loads(f.read())
except json.JSONDecodeError:
logger.error('pkgver.json - Bad json')
print_exc_plus
exit(1)
else:
logger.warning(f'No {self.__filename} found')
pkgvers = dict()
assert type(pkgvers) is dict
for pkgname in pkgvers:
assert type(pkgname) is str
self.__pkgvers = pkgvers
def _save(self):
pkgvers = json.dumps(self.__pkgvers, indent=4)
pkgvers += '\n'
with open(self.__filename,"w") as f:
if f.writable:
f.write(pkgvers)
else:
logger.error('pkgver.json - Not writable')
2019-04-07 17:14:50 +08:00
def __get_package_list(self, dirname, arch):
2019-04-06 20:59:27 +08:00
pkgdir = REPO_ROOT / dirname
assert pkgdir.exists()
2019-04-07 17:14:50 +08:00
pkglist = nspawn_shell(arch, MAKEPKG_PKGLIST_CMD, cwd=pkgdir)
2019-04-06 20:59:27 +08:00
pkglist = pkglist.split('\n')
return pkglist
2019-04-07 17:14:50 +08:00
def __get_new_ver(self, dirname, arch):
pkgfiles = self.__get_package_list(dirname, arch)
ver = get_pkg_details_from_name(pkgfiles[0]).ver
return ver
2019-04-06 20:59:27 +08:00
def check_update(self):
updates = list()
for pkg in jobsmgr.pkgconfigs:
pkgdir = REPO_ROOT / pkg.dirname
logger.info(f'checking update: {pkg.dirname}')
2019-04-07 17:14:50 +08:00
pkgbuild = pkgdir / 'PKGBUILD'
archs = get_arch_from_pkgbuild(pkgbuild)
buildarchs = [BUILD_ARCH_MAPPING.get(arch, None) for arch in archs]
buildarchs = [arch for arch in buildarchs if arch is not None]
# hopefully we only need to check one arch for update
arch = 'x86_64' if 'x86_64' in buildarchs else buildarchs[0] # prefer x86
mon_nspawn_shell(arch, MAKEPKG_UPD_CMD, cwd=pkgdir, minutes=60,
logfile = pkgdir / 'buildbot.log.update',
short_return = True)
2019-04-06 20:59:27 +08:00
if pkg.type in ('git', 'manual'):
2019-04-07 17:14:50 +08:00
ver = self.__get_new_ver(pkg.dirname, arch)
2019-04-06 20:59:27 +08:00
oldver = self.__pkgvers.get(pkg.dirname, None)
if oldver is None or vercmp(ver, oldver) == 1:
self.__pkgvers[pkg.dirname] = ver
2019-04-07 17:14:50 +08:00
updates.append((pkg, ver, buildarchs))
2019-04-06 20:59:27 +08:00
else:
logger.warning(f'package: {pkg.dirname} downgrade attempted')
else:
logger.warning(f'unknown package type: {pkg.type}')
self._save()
return updates
updmgr = updateManager()
@background
def __main():
2019-04-07 17:16:16 +08:00
while True:
jobsmgr.tick()
2019-04-06 20:59:27 +08:00
def run(funcname, args=list(), kwargs=dict()):
if funcname in ('clean', 'regenerate', 'remove',
'update', 'push_files', 'add_files'):
logger.info('running: %s %s %s',funcname, args, kwargs)
ret = eval(funcname)(*args, **kwargs)
logger.info('done: %s %s',funcname, ret)
return ret
else:
logger.error('unexpected: %s %s %s',funcname, args, kwargs)
return False
if __name__ == '__main__':
__main() # start the main worker thread
while True:
try:
with Listener(MASTER_BIND_ADDRESS, authkey=MASTER_BIND_PASSWD) as listener:
with listener.accept() as conn:
logger.info('connection accepted from %s', listener.last_accepted)
myrecv = conn.recv()
if type(myrecv) is list and len(myrecv) == 3:
(funcname, args, kwargs) = myrecv
funcname = str(funcname)
2019-04-07 17:14:50 +08:00
logger.info('running: %s %s %s', funcname, args, kwargs)
2019-04-06 20:59:27 +08:00
conn.send(run(funcname, args=args, kwargs=kwargs))
except Exception:
print_exc_plus()
except KeyboardInterrupt:
logger.info('KeyboardInterrupt')
print_exc_plus()
break