buildbot.py: ready to test

README.md

@@ -10,7 +10,7 @@ cleanbuild:
     true / false
 timeout:
     30 (30 mins, int only)
-extra:
+extra: (wip)
     - update:
         - /bin/true
     - prebuild:

buildbot.py

@@ -10,12 +10,14 @@ import os
 from pathlib import Path
 from subprocess import CalledProcessError
 
-from utils import print_exc_plus, background
-
 from config import ARCHS, BUILD_ARCHS, BUILD_ARCH_MAPPING, \
                    MASTER_BIND_ADDRESS, MASTER_BIND_PASSWD, \
-                   PKGBUILD_DIR, MAKEPKG_PKGLIST_CMD, MAKEPKG_UPD_CMD
+                   PKGBUILD_DIR, MAKEPKG_PKGLIST_CMD, MAKEPKG_UPD_CMD, \
-from utils import bash, get_pkg_details_from_name, vercmp
+                   MAKEPKG_MAKE_CMD, MAKEPKG_MAKE_CMD_CLEAN
+
+from utils import print_exc_plus, background, \
+                  bash, get_pkg_details_from_name, vercmp, \
+                  nspawn_shell, mon_nspawn_shell, get_arch_from_pkgbuild
 
 import json
 
@@ -31,53 +33,100 @@ os.chdir(abspath)
 REPO_ROOT = Path(PKGBUILD_DIR)
 
 class Job:
-    def __init__(self, arch, pkgdir, packagelist, version):
+    def __init__(self, buildarch, pkgconfig, version, multiarch=False):
-        buildarch = BUILD_ARCH_MAPPING.get(arch, None)
         assert buildarch in BUILD_ARCHS
-        self.arch = arch
+        self.arch = buildarch
-        self.buildarch = buildarch
+        self.pkgconfig = pkgconfig
-        self.pkgdir = pkgdir
-        self.packagelist = packagelist
         self.version = version
+        self.multiarch = multiarch
         self.added = time()
-        self.claimed = 0
 
 class jobsManager:
     def __init__(self):
-        self.__buildjobs = dict()
+        self.__buildjobs = list()
-        for arch in BUILD_ARCHS:
-            self.__buildjobs.setdefault(arch, list())
         self.__uploadjobs = list()
         self.__curr_job = None
         self.pkgconfigs = load_all_yaml()
-    def _new_buildjob(self, job, buildarch):
+    def _new_buildjob(self, job):
         assert type(job) is Job
-        self.__buildjobs.get(buildarch).append(job)
+        job_to_remove = list()
-    def claim_job(self, buildarch):
+        for previous_job in self.__buildjobs:
-        assert buildarch in BUILD_ARCHS
+            if job.pkgconfig.dirname == previous_job.pkgconfig.dirname and \
+               job.arch == previous_job.arch:
+                job_to_remove.append(previous_job)
+        for oldjob in job_to_remove:
+            self.__buildjobs.remove(oldjob)
+            logger.info('removed an old job for %s %s, %s => %s',
+                        job.pkgconfig.dirname, job.arch,
+                        oldjob.version, job.version)
+        logger.info('new job for %s %s %s',
+                     job.pkgconfig.dirname, job.arch, job.version)
+        self.__buildjobs.append(job)
+    def __get_job(self):
         if self.__curr_job:
             return None
-        jobs = self.__buildjobs.get(buildarch, list())
+        jobs = self.__buildjobs
         if jobs:
             self.__curr_job = jobs.pop(0)
             return self.__curr_job
     def __finish_job(self, pkgdir):
-        assert pkgdir == self.__curr_job.pkgdir
+        assert pkgdir == self.__curr_job.pkgconfig.dirname
         # do upload
         self.__curr_job = None
         return True
+    def __makepkg(self, job):
+        mkcmd = MAKEPKG_MAKE_CMD_CLEAN if job.pkgconfig.cleanbuild \
+                                       else MAKEPKG_MAKE_CMD
+        cwd = REPO_ROOT / job.pkgconfig.dirname
+        logger.info('makepkg in %s %s', job.pkgconfig.dirname, job.arch)
+        return mon_nspawn_shell(arch=job.arch, cwd=cwd, cmdline=mkcmd,
+                                logfile = cwd / 'buildbot.log.update',
+                                short_return = True)
+    def __clean(self, job):
+        cwd = REPO_ROOT / job.pkgconfig.dirname
+        logger.info('cleaning build dir for %s %s',
+                    job.pkgconfig.dirname, job.arch)
+        nspawn_shell(job.arch, 'rm -rf src pkg', cwd=cwd)
+    def __sign(self, job):
+        '''
+            wip
+        '''
+        cwd = REPO_ROOT / job.pkgconfig.dirname
+        print(nspawn_shell(job.arch, 'ls -l', cwd=cwd))
+        #nspawn_shell(job.arch, 'rm -rf src pkg', cwd=cwd)
+    def __upload(self, job):
+        '''
+            wip
+        '''
+        cwd = REPO_ROOT / job.pkgconfig.dirname
+        print(nspawn_shell(job.arch, 'ls -l', cwd=cwd))
+        #nspawn_shell(job.arch, 'rm -rf src pkg', cwd=cwd)
     def tick(self):
         '''
             check for updates,
             create new jobs
             and run them
         '''
-        if self.__curr_job is None:
+        if not self.__buildjobs:
+            # This part check for updates
             updates = updmgr.check_update()
             for update in updates:
-                (pkg, packagelist, ver) = update
+                (pkgconfig, ver, buildarchs) = update
-
+                march = True if len(buildarchs) >= 2 else False
-
+                for arch in buildarchs:
+                    newjob = Job(arch, pkgconfig, ver, multiarch=march)
+                    self._new_buildjob(newjob)
+        else:
+            # This part does the job
+            for job in self.__buildjobs:
+                cwd = REPO_ROOT / job.pkgconfig.dirname
+                if job.multiarch:
+                    # wip
+                    pass
+                else:
+                    self.__makepkg(job)
+                    self.__sign(job)
+                    self.__upload(job)
 jobsmgr = jobsManager()
 
 class updateManager:
@@ -109,28 +158,36 @@ class updateManager:
                 f.write(pkgvers)
             else:
                 logger.error('pkgver.json - Not writable')
-    def __get_package_list(self, dirname):
+    def __get_package_list(self, dirname, arch):
         pkgdir = REPO_ROOT / dirname
         assert pkgdir.exists()
-        pkglist = bash(MAKEPKG_PKGLIST_CMD, cwd=pkgdir)
+        pkglist = nspawn_shell(arch, MAKEPKG_PKGLIST_CMD, cwd=pkgdir)
         pkglist = pkglist.split('\n')
         return pkglist
-    def __get_new_ver(self, dirname):
+    def __get_new_ver(self, dirname, arch):
-        pkgfiles = self.__get_package_list(dirname)
+        pkgfiles = self.__get_package_list(dirname, arch)
-        ver = get_pkg_details_from_name(pkgfiles[0])
+        ver = get_pkg_details_from_name(pkgfiles[0]).ver
-        return (ver, pkgfiles)
+        return ver
     def check_update(self):
         updates = list()
         for pkg in jobsmgr.pkgconfigs:
             pkgdir = REPO_ROOT / pkg.dirname
             logger.info(f'checking update: {pkg.dirname}')
-            bash(MAKEPKG_UPD_CMD, cwd=pkgdir, RUN_CMD_TIMEOUT=60*60)
+            pkgbuild = pkgdir / 'PKGBUILD'
+            archs = get_arch_from_pkgbuild(pkgbuild)
+            buildarchs = [BUILD_ARCH_MAPPING.get(arch, None) for arch in archs]
+            buildarchs = [arch for arch in buildarchs if arch is not None]
+            # hopefully we only need to check one arch for update
+            arch = 'x86_64' if 'x86_64' in buildarchs else buildarchs[0] # prefer x86
+            mon_nspawn_shell(arch, MAKEPKG_UPD_CMD, cwd=pkgdir, minutes=60,
+                             logfile = pkgdir / 'buildbot.log.update',
+                             short_return = True)
             if pkg.type in ('git', 'manual'):
-                (ver, pkgfiles) = self.__get_new_ver(pkg.dirname)
+                ver = self.__get_new_ver(pkg.dirname, arch)
                 oldver = self.__pkgvers.get(pkg.dirname, None)
                 if oldver is None or vercmp(ver, oldver) == 1:
                     self.__pkgvers[pkg.dirname] = ver
-                    updates.append((pkg, pkgfiles, ver))
+                    updates.append((pkg, ver, buildarchs))
                 else:
                     logger.warning(f'package: {pkg.dirname} downgrade attempted')
             else:
@@ -174,6 +231,7 @@ if __name__ == '__main__':
                     if type(myrecv) is list and len(myrecv) == 3:
                         (funcname, args, kwargs) = myrecv
                         funcname = str(funcname)
+                        logger.info('running: %s %s %s', funcname, args, kwargs)
                         conn.send(run(funcname, args=args, kwargs=kwargs))
         except Exception:
             print_exc_plus()

config.py

@@ -37,7 +37,7 @@ GPG_SIGN_CMD = (f'gpg --default-key {GPG_KEY} --no-armor'
                  '--pinentry-mode loopback --passphrase \'\''
                  '--detach-sign --yes --')
 
-#### config for master.py
+#### config for buildbot.py
 
 MASTER_BIND_ADDRESS = ('localhost', 7011)
 MASTER_BIND_PASSWD = b'mypassword'
@@ -49,3 +49,8 @@ MAKEPKG_MAKE_CMD = 'makepkg --syncdeps --noextract'
 MAKEPKG_MAKE_CMD_CLEAN = 'makepkg --syncdeps --noextract --clean --cleanbuild'
 
 MAKEPKG_PKGLIST_CMD = f'{MAKEPKG} --packagelist'
+
+CONTAINER_BUILDBOT_ROOT = '~/shared/buildbot'
+# single quote may cause problem here
+SHELL_ARCH_X64 = 'sudo machinectl --quiet shell build@archlinux /bin/bash -c \'{command}\''
+SHELL_ARCH_ARM64 = 'sudo machinectl --quiet shell root@alarm /bin/bash -c $\'su -l alarm -c \\\'{command}\\\'\''

repod.py

@@ -119,7 +119,7 @@ def add_files(filename, overwrite=False):
 def run(funcname, args=list(), kwargs=dict()):
     if funcname in ('clean', 'regenerate', 'remove',
                     'update', 'push_files', 'add_files'):
-        logger.info('running: %s %s %s',funcname, args, kwargs)
+        logger.info('running: %s %s %s', funcname, args, kwargs)
         ret = eval(funcname)(*args, **kwargs)
         logger.info('done: %s %s',funcname, ret)
         return ret

utils.py

@@ -5,10 +5,13 @@ import logging
 from time import time
 import re
 from threading import Thread, Lock
+from pathlib import Path
+import os
 import sys
 import traceback
 
-from config import PKG_COMPRESSION
+from config import PKG_COMPRESSION, SHELL_ARCH_ARM64, SHELL_ARCH_X64, \
+                   CONTAINER_BUILDBOT_ROOT, ARCHS
 
 logger = logging.getLogger(name='utils')
 
@@ -22,74 +25,107 @@ def background(func):
 
 def bash(cmdline, **kwargs):
     assert type(cmdline) is str
-    logger.info(f'bash: {cmdline}')
+    logger.info(f'bash: {cmdline}, kwargs: {kwargs}')
     return(run_cmd(['/bin/bash', '-x', '-e', '-c', cmdline], **kwargs))
 
-def long_bash(cmdline, cwd=None, hours=2):
+def mon_bash(cmdline, cwd=None, minutes=30, **kwargs):
-    assert type(hours) is int and hours >= 1
+    assert type(minutes) is int and minutes >= 1
-    logger.info(f'longbash{hours}: {cmdline}')
+    return bash(cmdline, cwd=cwd, keepalive=True, KEEPALIVE_TIMEOUT=60,
-    return bash(cmdline, cwd=cwd, keepalive=True, KEEPALIVE_TIMEOUT=60, RUN_CMD_TIMEOUT=hours*60*60)
+                RUN_CMD_TIMEOUT=minutes*60, **kwargs)
 
-def run_cmd(cmd, cwd=None, keepalive=False, KEEPALIVE_TIMEOUT=30, RUN_CMD_TIMEOUT=60):
+def nspawn_shell(arch, cmdline, cwd=None, **kwargs):
+    root = Path(CONTAINER_BUILDBOT_ROOT)
+    if cwd:
+        cwd = root / cwd
+    else:
+        cwd = root
+    if arch in ('aarch64', 'arm64'):
+        return bash(SHELL_ARCH_ARM64.format(command=f'cd \"{cwd}\" || exit 1; {cmdline}'))
+    elif arch in ('x64', 'x86', 'x86_64'):
+        return bash(SHELL_ARCH_X64.format(command=f'cd \"{cwd}\" || exit 1; {cmdline}'))
+    raise TypeError('nspawn_shell: wrong arch')
+
+def mon_nspawn_shell(arch, cmdline, cwd, minutes=30, **kwargs):
+    assert type(minutes) is int and minutes >= 1
+    return nspawn_shell(arch, cmdline, cwd=cwd, keepalive=True, KEEPALIVE_TIMEOUT=60,
+                        RUN_CMD_TIMEOUT=minutes*60, **kwargs)
+
+def run_cmd(cmd, cwd=None, keepalive=False, KEEPALIVE_TIMEOUT=30, RUN_CMD_TIMEOUT=60,
+            logfile=None, short_return=False):
     logger.debug('run_cmd: %s', cmd)
     RUN_CMD_LOOP_TIME = KEEPALIVE_TIMEOUT - 1 if KEEPALIVE_TIMEOUT >= 10 else 5
     stopped = False
     last_read = [int(time()), ""]
-    output = list()
+    class Output(list):
+        def append(self, mystring):
+            if not self.__short_return:
+                super().append(mystring)
+            if self.__file and type(mystring) is str:
+                self.__file.write(mystring)
+        def __enter__(self, logfile=None, short_return=False):
+            self.__short_return = short_return
+            if logfile:
+                assert issubclass(type(logfile), os.PathLike)
+                self.__file = open(logfile, 'w')
+            else:
+                self.__file = None
+        def __exit__(self, type, value, traceback):
+            if self.__file:
+                self.__file.close()
     stdout_lock = Lock()
-    @background
+    with Output(logfile=logfile, short_return=short_return) as output:
-    def check_stdout(stdout):
+        @background
-        nonlocal stopped, last_read, output
+        def check_stdout(stdout):
-        stdout_lock.acquire()
+            nonlocal stopped, last_read, output
-        last_read_time = int(time())
+            stdout_lock.acquire()
-        while stopped is False:
-            line = stdout.readline(4096)
             last_read_time = int(time())
-            logger.debug(line)
+            while stopped is False:
-            output.append(line)
+                line = stdout.readline(4096)
-            last_read[0] = last_read_time
+                last_read_time = int(time())
-            last_read[1] = line
+                logger.debug(line)
-        stdout_lock.release()
+                output.append(line)
-    p = subprocess.Popen(cmd, cwd=cwd, stdin=subprocess.PIPE, stdout=subprocess.PIPE,
+                last_read[0] = last_read_time
-                         stderr=subprocess.STDOUT, encoding='utf-8')
+                last_read[1] = line
-    check_stdout(p.stdout)
+            stdout_lock.release()
-    process_start = int(time())
+        p = subprocess.Popen(cmd, cwd=cwd, stdin=subprocess.PIPE, stdout=subprocess.PIPE,
-    while True:
+                            stderr=subprocess.STDOUT, encoding='utf-8')
-        try:
+        check_stdout(p.stdout)
-            p.wait(timeout=RUN_CMD_LOOP_TIME)
+        process_start = int(time())
-        except subprocess.TimeoutExpired:
+        while True:
-            time_passed = int(time()) - last_read[0]
+            try:
-            if time_passed >= KEEPALIVE_TIMEOUT*2:
+                p.wait(timeout=RUN_CMD_LOOP_TIME)
-                logger.info('Timeout expired. No action.')
+            except subprocess.TimeoutExpired:
-                output.append('+ Buildbot: Timeout expired. No action.\n')
+                time_passed = int(time()) - last_read[0]
-            elif time_passed >= KEEPALIVE_TIMEOUT:
+                if time_passed >= KEEPALIVE_TIMEOUT*2:
-                if keepalive:
+                    logger.info('Timeout expired. No action.')
-                    logger.info('Timeout expired, writing nl')
+                    output.append('+ Buildbot: Timeout expired. No action.\n')
-                    output.append('+ Buildbot: Timeout expired, writing nl\n')
+                elif time_passed >= KEEPALIVE_TIMEOUT:
-                    p.stdin.write('\n')
+                    if keepalive:
-                    p.stdin.flush()
+                        logger.info('Timeout expired, writing nl')
-                else:
+                        output.append('+ Buildbot: Timeout expired, writing nl\n')
-                    logger.info('Timeout expired, not writing nl')
+                        p.stdin.write('\n')
-                    output.append('+ Buildbot: Timeout expired, not writing nl\n')
+                        p.stdin.flush()
-            if int(time()) - process_start >= RUN_CMD_TIMEOUT:
+                    else:
+                        logger.info('Timeout expired, not writing nl')
+                        output.append('+ Buildbot: Timeout expired, not writing nl\n')
+                if int(time()) - process_start >= RUN_CMD_TIMEOUT:
+                    stopped = True
+                    logger.error('Process timeout expired, terminating.')
+                    output.append('+ Buildbot: Process timeout expired, terminating.\n')
+                    p.terminate()
+                    try:
+                        p.wait(timeout=10)
+                    except subprocess.TimeoutExpired:
+                        logger.error('Cannot terminate, killing.')
+                        output.append('+ Buildbot: Cannot terminate, killing.\n')
+                        p.kill()
+                    break
+            else:
                 stopped = True
-                logger.error('Process timeout expired, terminating.')
-                output.append('+ Buildbot: Process timeout expired, terminating.\n')
-                p.terminate()
-                try:
-                    p.wait(timeout=10)
-                except subprocess.TimeoutExpired:
-                    logger.error('Cannot terminate, killing.')
-                    output.append('+ Buildbot: Cannot terminate, killing.\n')
-                    p.kill()
                 break
-        else:
+        code = p.returncode
-            stopped = True
+        stdout_lock.acquire(10)
-            break
+        outstr = ''.join(output)
-    code = p.returncode
-
-    stdout_lock.acquire(10)
-    outstr = ''.join(output)
 
     if code != 0:
         raise subprocess.CalledProcessError(code, cmd, outstr)
@@ -148,6 +184,18 @@ def get_pkg_details_from_name(name):
         (pkgname, pkgver, pkgrel, arch) = m.groups()
         return Pkg(pkgname, pkgver, pkgrel, arch, name)
 
+def get_arch_from_pkgbuild(fpath):
+    assert issubclass(type(fpath), os.PathLike)
+    with open(fpath, 'r') as f:
+        for line in f.readline():
+            if line.startswith('arch='):
+                matches = re.findall('[\'\"]([^\'\"]+)[\'\"]', line)
+                if not matches:
+                    raise TypeError('Unexpected PKGBUILD format')
+                assert not [None for match in matches if match not in ARCHS]
+                return matches
+    raise TypeError('Unexpected PKGBUILD')
+
 def print_exc_plus():
     """
     Print the usual traceback information, followed by a listing of all the