Archlinux-Jerry
/
buildbot
mirror of https://github.com/archlinux-jerry/buildbot
1
0
Fork 0
Code Issues Releases Wiki Activity

buildbot.py: ready to test

This commit is contained in:
JerryXiao 2019-04-07 17:14:50 +08:00
parent 715d101d01
commit 97311e9051
Signed by: Jerry
GPG Key ID: 9D9CE43650FF2BAA
5 changed files with 205 additions and 94 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
README.md
View File

@ -10,7 +10,7 @@ cleanbuild:
true / false
timeout:
30 (30 mins, int only)
extra:
extra: (wip)
- update:
- /bin/true
- prebuild:

124
buildbot.py
View File

@ -10,12 +10,14 @@ import os
from pathlib import Path
from subprocess import CalledProcessError
from utils import print_exc_plus, background
from config import ARCHS, BUILD_ARCHS, BUILD_ARCH_MAPPING, \
MASTER_BIND_ADDRESS, MASTER_BIND_PASSWD, \
PKGBUILD_DIR, MAKEPKG_PKGLIST_CMD, MAKEPKG_UPD_CMD
from utils import bash, get_pkg_details_from_name, vercmp
PKGBUILD_DIR, MAKEPKG_PKGLIST_CMD, MAKEPKG_UPD_CMD, \
MAKEPKG_MAKE_CMD, MAKEPKG_MAKE_CMD_CLEAN
from utils import print_exc_plus, background, \
bash, get_pkg_details_from_name, vercmp, \
nspawn_shell, mon_nspawn_shell, get_arch_from_pkgbuild
import json
@ -31,53 +33,100 @@ os.chdir(abspath)
REPO_ROOT = Path(PKGBUILD_DIR)
class Job:
def __init__(self, arch, pkgdir, packagelist, version):
buildarch = BUILD_ARCH_MAPPING.get(arch, None)
def __init__(self, buildarch, pkgconfig, version, multiarch=False):
assert buildarch in BUILD_ARCHS
self.arch = arch
self.buildarch = buildarch
self.pkgdir = pkgdir
self.packagelist = packagelist
self.arch = buildarch
self.pkgconfig = pkgconfig
self.version = version
self.multiarch = multiarch
self.added = time()
self.claimed = 0
class jobsManager:
def __init__(self):
self.__buildjobs = dict()
for arch in BUILD_ARCHS:
self.__buildjobs.setdefault(arch, list())
self.__buildjobs = list()
self.__uploadjobs = list()
self.__curr_job = None
self.pkgconfigs = load_all_yaml()
def _new_buildjob(self, job, buildarch):
def _new_buildjob(self, job):
assert type(job) is Job
self.__buildjobs.get(buildarch).append(job)
def claim_job(self, buildarch):
assert buildarch in BUILD_ARCHS
job_to_remove = list()
for previous_job in self.__buildjobs:
if job.pkgconfig.dirname == previous_job.pkgconfig.dirname and \
job.arch == previous_job.arch:
job_to_remove.append(previous_job)
for oldjob in job_to_remove:
self.__buildjobs.remove(oldjob)
logger.info('removed an old job for %s %s, %s => %s',
job.pkgconfig.dirname, job.arch,
oldjob.version, job.version)
logger.info('new job for %s %s %s',
job.pkgconfig.dirname, job.arch, job.version)
self.__buildjobs.append(job)
def __get_job(self):
if self.__curr_job:
return None
jobs = self.__buildjobs.get(buildarch, list())
jobs = self.__buildjobs
if jobs:
self.__curr_job = jobs.pop(0)
return self.__curr_job
def __finish_job(self, pkgdir):
assert pkgdir == self.__curr_job.pkgdir
assert pkgdir == self.__curr_job.pkgconfig.dirname
# do upload
self.__curr_job = None
return True
def __makepkg(self, job):
mkcmd = MAKEPKG_MAKE_CMD_CLEAN if job.pkgconfig.cleanbuild \
else MAKEPKG_MAKE_CMD
cwd = REPO_ROOT / job.pkgconfig.dirname
logger.info('makepkg in %s %s', job.pkgconfig.dirname, job.arch)
return mon_nspawn_shell(arch=job.arch, cwd=cwd, cmdline=mkcmd,
logfile = cwd / 'buildbot.log.update',
short_return = True)
def __clean(self, job):
cwd = REPO_ROOT / job.pkgconfig.dirname
logger.info('cleaning build dir for %s %s',
job.pkgconfig.dirname, job.arch)
nspawn_shell(job.arch, 'rm -rf src pkg', cwd=cwd)
def __sign(self, job):
'''
wip
'''
cwd = REPO_ROOT / job.pkgconfig.dirname
print(nspawn_shell(job.arch, 'ls -l', cwd=cwd))
#nspawn_shell(job.arch, 'rm -rf src pkg', cwd=cwd)
def __upload(self, job):
'''
wip
'''
cwd = REPO_ROOT / job.pkgconfig.dirname
print(nspawn_shell(job.arch, 'ls -l', cwd=cwd))
#nspawn_shell(job.arch, 'rm -rf src pkg', cwd=cwd)
def tick(self):
'''
check for updates,
create new jobs
and run them
'''
if self.__curr_job is None:
if not self.__buildjobs:
# This part check for updates
updates = updmgr.check_update()
for update in updates:
(pkg, packagelist, ver) = update
(pkgconfig, ver, buildarchs) = update
march = True if len(buildarchs) >= 2 else False
for arch in buildarchs:
newjob = Job(arch, pkgconfig, ver, multiarch=march)
self._new_buildjob(newjob)
else:
# This part does the job
for job in self.__buildjobs:
cwd = REPO_ROOT / job.pkgconfig.dirname
if job.multiarch:
# wip
pass
else:
self.__makepkg(job)
self.__sign(job)
self.__upload(job)
jobsmgr = jobsManager()
class updateManager:
@ -109,28 +158,36 @@ class updateManager:
f.write(pkgvers)
else:
logger.error('pkgver.json - Not writable')
def __get_package_list(self, dirname):
def __get_package_list(self, dirname, arch):
pkgdir = REPO_ROOT / dirname
assert pkgdir.exists()
pkglist = bash(MAKEPKG_PKGLIST_CMD, cwd=pkgdir)
pkglist = nspawn_shell(arch, MAKEPKG_PKGLIST_CMD, cwd=pkgdir)
pkglist = pkglist.split('\n')
return pkglist
def __get_new_ver(self, dirname):
pkgfiles = self.__get_package_list(dirname)
ver = get_pkg_details_from_name(pkgfiles[0])
return (ver, pkgfiles)
def __get_new_ver(self, dirname, arch):
pkgfiles = self.__get_package_list(dirname, arch)
ver = get_pkg_details_from_name(pkgfiles[0]).ver
return ver
def check_update(self):
updates = list()
for pkg in jobsmgr.pkgconfigs:
pkgdir = REPO_ROOT / pkg.dirname
logger.info(f'checking update: {pkg.dirname}')
bash(MAKEPKG_UPD_CMD, cwd=pkgdir, RUN_CMD_TIMEOUT=60*60)
pkgbuild = pkgdir / 'PKGBUILD'
archs = get_arch_from_pkgbuild(pkgbuild)
buildarchs = [BUILD_ARCH_MAPPING.get(arch, None) for arch in archs]
buildarchs = [arch for arch in buildarchs if arch is not None]
# hopefully we only need to check one arch for update
arch = 'x86_64' if 'x86_64' in buildarchs else buildarchs[0] # prefer x86
mon_nspawn_shell(arch, MAKEPKG_UPD_CMD, cwd=pkgdir, minutes=60,
logfile = pkgdir / 'buildbot.log.update',
short_return = True)
if pkg.type in ('git', 'manual'):
(ver, pkgfiles) = self.__get_new_ver(pkg.dirname)
ver = self.__get_new_ver(pkg.dirname, arch)
oldver = self.__pkgvers.get(pkg.dirname, None)
if oldver is None or vercmp(ver, oldver) == 1:
self.__pkgvers[pkg.dirname] = ver
updates.append((pkg, pkgfiles, ver))
updates.append((pkg, ver, buildarchs))
else:
logger.warning(f'package: {pkg.dirname} downgrade attempted')
else:
@ -174,6 +231,7 @@ if __name__ == '__main__':
if type(myrecv) is list and len(myrecv) == 3:
(funcname, args, kwargs) = myrecv
funcname = str(funcname)
logger.info('running: %s %s %s', funcname, args, kwargs)
conn.send(run(funcname, args=args, kwargs=kwargs))
except Exception:
print_exc_plus()

7
config.py
View File

@ -37,7 +37,7 @@ GPG_SIGN_CMD = (f'gpg --default-key {GPG_KEY} --no-armor'
'--pinentry-mode loopback --passphrase \'\''
'--detach-sign --yes --')
#### config for master.py
#### config for buildbot.py
MASTER_BIND_ADDRESS = ('localhost', 7011)
MASTER_BIND_PASSWD = b'mypassword'
@ -49,3 +49,8 @@ MAKEPKG_MAKE_CMD = 'makepkg --syncdeps --noextract'
MAKEPKG_MAKE_CMD_CLEAN = 'makepkg --syncdeps --noextract --clean --cleanbuild'
MAKEPKG_PKGLIST_CMD = f'{MAKEPKG} --packagelist'
CONTAINER_BUILDBOT_ROOT = '~/shared/buildbot'
# single quote may cause problem here
SHELL_ARCH_X64 = 'sudo machinectl --quiet shell build@archlinux /bin/bash -c \'{command}\''
SHELL_ARCH_ARM64 = 'sudo machinectl --quiet shell root@alarm /bin/bash -c $\'su -l alarm -c \\\'{command}\\\'\''

2
repod.py
View File

@ -119,7 +119,7 @@ def add_files(filename, overwrite=False):
def run(funcname, args=list(), kwargs=dict()):
if funcname in ('clean', 'regenerate', 'remove',
'update', 'push_files', 'add_files'):
logger.info('running: %s %s %s',funcname, args, kwargs)
logger.info('running: %s %s %s', funcname, args, kwargs)
ret = eval(funcname)(*args, **kwargs)
logger.info('done: %s %s',funcname, ret)
return ret

164
utils.py
View File

@ -5,10 +5,13 @@ import logging
from time import time
import re
from threading import Thread, Lock
from pathlib import Path
import os
import sys
import traceback
from config import PKG_COMPRESSION
from config import PKG_COMPRESSION, SHELL_ARCH_ARM64, SHELL_ARCH_X64, \
CONTAINER_BUILDBOT_ROOT, ARCHS
logger = logging.getLogger(name='utils')
@ -22,74 +25,107 @@ def background(func):
def bash(cmdline, **kwargs):
assert type(cmdline) is str
logger.info(f'bash: {cmdline}')
logger.info(f'bash: {cmdline}, kwargs: {kwargs}')
return(run_cmd(['/bin/bash', '-x', '-e', '-c', cmdline], **kwargs))
def long_bash(cmdline, cwd=None, hours=2):
assert type(hours) is int and hours >= 1
logger.info(f'longbash{hours}: {cmdline}')
return bash(cmdline, cwd=cwd, keepalive=True, KEEPALIVE_TIMEOUT=60, RUN_CMD_TIMEOUT=hours*60*60)
def mon_bash(cmdline, cwd=None, minutes=30, **kwargs):
assert type(minutes) is int and minutes >= 1
return bash(cmdline, cwd=cwd, keepalive=True, KEEPALIVE_TIMEOUT=60,
RUN_CMD_TIMEOUT=minutes*60, **kwargs)
def run_cmd(cmd, cwd=None, keepalive=False, KEEPALIVE_TIMEOUT=30, RUN_CMD_TIMEOUT=60):
def nspawn_shell(arch, cmdline, cwd=None, **kwargs):
root = Path(CONTAINER_BUILDBOT_ROOT)
if cwd:
cwd = root / cwd
else:
cwd = root
if arch in ('aarch64', 'arm64'):
return bash(SHELL_ARCH_ARM64.format(command=f'cd \"{cwd}\" || exit 1; {cmdline}'))
elif arch in ('x64', 'x86', 'x86_64'):
return bash(SHELL_ARCH_X64.format(command=f'cd \"{cwd}\" || exit 1; {cmdline}'))
raise TypeError('nspawn_shell: wrong arch')
def mon_nspawn_shell(arch, cmdline, cwd, minutes=30, **kwargs):
assert type(minutes) is int and minutes >= 1
return nspawn_shell(arch, cmdline, cwd=cwd, keepalive=True, KEEPALIVE_TIMEOUT=60,
RUN_CMD_TIMEOUT=minutes*60, **kwargs)
def run_cmd(cmd, cwd=None, keepalive=False, KEEPALIVE_TIMEOUT=30, RUN_CMD_TIMEOUT=60,
logfile=None, short_return=False):
logger.debug('run_cmd: %s', cmd)
RUN_CMD_LOOP_TIME = KEEPALIVE_TIMEOUT - 1 if KEEPALIVE_TIMEOUT >= 10 else 5
stopped = False
last_read = [int(time()), ""]
output = list()
class Output(list):
def append(self, mystring):
if not self.__short_return:
super().append(mystring)
if self.__file and type(mystring) is str:
self.__file.write(mystring)
def __enter__(self, logfile=None, short_return=False):
self.__short_return = short_return
if logfile:
assert issubclass(type(logfile), os.PathLike)
self.__file = open(logfile, 'w')
else:
self.__file = None
def __exit__(self, type, value, traceback):
if self.__file:
self.__file.close()
stdout_lock = Lock()
@background
def check_stdout(stdout):
nonlocal stopped, last_read, output
stdout_lock.acquire()
last_read_time = int(time())
while stopped is False:
line = stdout.readline(4096)
with Output(logfile=logfile, short_return=short_return) as output:
@background
def check_stdout(stdout):
nonlocal stopped, last_read, output
stdout_lock.acquire()
last_read_time = int(time())
logger.debug(line)
output.append(line)
last_read[0] = last_read_time
last_read[1] = line
stdout_lock.release()
p = subprocess.Popen(cmd, cwd=cwd, stdin=subprocess.PIPE, stdout=subprocess.PIPE,
stderr=subprocess.STDOUT, encoding='utf-8')
check_stdout(p.stdout)
process_start = int(time())
while True:
try:
p.wait(timeout=RUN_CMD_LOOP_TIME)
except subprocess.TimeoutExpired:
time_passed = int(time()) - last_read[0]
if time_passed >= KEEPALIVE_TIMEOUT*2:
logger.info('Timeout expired. No action.')
output.append('+ Buildbot: Timeout expired. No action.\n')
elif time_passed >= KEEPALIVE_TIMEOUT:
if keepalive:
logger.info('Timeout expired, writing nl')
output.append('+ Buildbot: Timeout expired, writing nl\n')
p.stdin.write('\n')
p.stdin.flush()
else:
logger.info('Timeout expired, not writing nl')
output.append('+ Buildbot: Timeout expired, not writing nl\n')
if int(time()) - process_start >= RUN_CMD_TIMEOUT:
while stopped is False:
line = stdout.readline(4096)
last_read_time = int(time())
logger.debug(line)
output.append(line)
last_read[0] = last_read_time
last_read[1] = line
stdout_lock.release()
p = subprocess.Popen(cmd, cwd=cwd, stdin=subprocess.PIPE, stdout=subprocess.PIPE,
stderr=subprocess.STDOUT, encoding='utf-8')
check_stdout(p.stdout)
process_start = int(time())
while True:
try:
p.wait(timeout=RUN_CMD_LOOP_TIME)
except subprocess.TimeoutExpired:
time_passed = int(time()) - last_read[0]
if time_passed >= KEEPALIVE_TIMEOUT*2:
logger.info('Timeout expired. No action.')
output.append('+ Buildbot: Timeout expired. No action.\n')
elif time_passed >= KEEPALIVE_TIMEOUT:
if keepalive:
logger.info('Timeout expired, writing nl')
output.append('+ Buildbot: Timeout expired, writing nl\n')
p.stdin.write('\n')
p.stdin.flush()
else:
logger.info('Timeout expired, not writing nl')
output.append('+ Buildbot: Timeout expired, not writing nl\n')
if int(time()) - process_start >= RUN_CMD_TIMEOUT:
stopped = True
logger.error('Process timeout expired, terminating.')
output.append('+ Buildbot: Process timeout expired, terminating.\n')
p.terminate()
try:
p.wait(timeout=10)
except subprocess.TimeoutExpired:
logger.error('Cannot terminate, killing.')
output.append('+ Buildbot: Cannot terminate, killing.\n')
p.kill()
break
else:
stopped = True
logger.error('Process timeout expired, terminating.')
output.append('+ Buildbot: Process timeout expired, terminating.\n')
p.terminate()
try:
p.wait(timeout=10)
except subprocess.TimeoutExpired:
logger.error('Cannot terminate, killing.')
output.append('+ Buildbot: Cannot terminate, killing.\n')
p.kill()
break
else:
stopped = True
break
code = p.returncode
stdout_lock.acquire(10)
outstr = ''.join(output)
code = p.returncode
stdout_lock.acquire(10)
outstr = ''.join(output)
if code != 0:
raise subprocess.CalledProcessError(code, cmd, outstr)
@ -148,6 +184,18 @@ def get_pkg_details_from_name(name):
(pkgname, pkgver, pkgrel, arch) = m.groups()
return Pkg(pkgname, pkgver, pkgrel, arch, name)
def get_arch_from_pkgbuild(fpath):
assert issubclass(type(fpath), os.PathLike)
with open(fpath, 'r') as f:
for line in f.readline():
if line.startswith('arch='):
matches = re.findall('[\'\"]([^\'\"]+)[\'\"]', line)
if not matches:
raise TypeError('Unexpected PKGBUILD format')
assert not [None for match in matches if match not in ARCHS]
return matches
raise TypeError('Unexpected PKGBUILD')
def print_exc_plus():
"""
Print the usual traceback information, followed by a listing of all the