completed

buildbot.py

@@ -17,12 +17,13 @@ from config import ARCHS, BUILD_ARCHS, BUILD_ARCH_MAPPING, \
                    MASTER_BIND_ADDRESS, MASTER_BIND_PASSWD, \
                    PKGBUILD_DIR, MAKEPKG_PKGLIST_CMD, MAKEPKG_UPD_CMD, \
                    MAKEPKG_MAKE_CMD, MAKEPKG_MAKE_CMD_CLEAN, \
-                   GPG_SIGN_CMD, GPG_VERIFY_CMD, UPDATE_INTERVAL
+                   GPG_SIGN_CMD, GPG_VERIFY_CMD, UPDATE_INTERVAL, \
+                   MAKEPKG_MAKE_CMD_MARCH, UPLOAD_CMD
 
 from utils import print_exc_plus, background, \
                   bash, get_pkg_details_from_name, vercmp, \
                   nspawn_shell, mon_nspawn_shell, get_arch_from_pkgbuild, \
-                  configure_logger
+                  configure_logger, mon_bash
 
 from client import run as rrun
 
@@ -84,22 +85,28 @@ class jobsManager:
         self.__curr_job = None
         return True
     def __makepkg(self, job):
+        cwd = REPO_ROOT / job.pkgconfig.dirname
+        if job.multiarch:
+            # assume a clean env, no source avail
+            mkcmd = MAKEPKG_MAKE_CMD_MARCH
+        else:
             mkcmd = MAKEPKG_MAKE_CMD_CLEAN if job.pkgconfig.cleanbuild \
                                         else MAKEPKG_MAKE_CMD
-        cwd = REPO_ROOT / job.pkgconfig.dirname
         logger.info('makepkg in %s %s', job.pkgconfig.dirname, job.arch)
         return mon_nspawn_shell(arch=job.arch, cwd=cwd, cmdline=mkcmd,
                                 logfile = cwd / 'buildbot.log.makepkg',
-                                short_return = True)
+                                short_return = True,
-    def __clean(self, job, remove_pkg=False):
+                                seconds=job.pkgconfig.timeout*60)
+    def __clean(self, job, remove_pkg=False, rm_src=True):
         cwd = REPO_ROOT / job.pkgconfig.dirname
         logger.info('cleaning build dir for %s, %sremoving pkg',
                     job.pkgconfig.dirname, '' if remove_pkg else 'not ')
         for fpath in [f for f in cwd.iterdir()]:
-            fpath = Path()
+            if rm_src and fpath.is_dir() and \
-            if fpath.is_dir() and fpath.name in ('pkg', 'src'):
+                          fpath.name in ('pkg', 'src'):
                 rmtree(fpath)
             elif remove_pkg and fpath.is_file() and \
+                 ((not job.multiarch) or job.arch in fpath.name) and \
                  (fpath.name.endswith(PKG_SUFFIX) or \
                   fpath.name.endswith(PKG_SIG_SUFFIX)):
                 fpath.unlink()
@@ -113,9 +120,27 @@ class jobsManager:
             wip
         '''
         cwd = REPO_ROOT / job.pkgconfig.dirname
-        print(bash('ls -l', cwd=cwd))
+        f_to_upload = list()
-        #nspawn_shell(job.arch, 'rm -rf src pkg', cwd=cwd)
+        for fpath in cwd.iterdir():
-        #rrun()
+            if fpath.name.endswith(PKG_SUFFIX) and \
+               get_pkg_details_from_name(fpath.name).ver == job.version:
+                sigpath = fpath.parent / f'{fpath.name}.sig'
+                assert sigpath.exists()
+                f_to_upload.append(sigpath)
+                f_to_upload.append(fpath)
+        for f in f_to_upload:
+            size = f.stat().st_size / 1000 / 1000
+            timeout = rrun('push_start', args=(f.name, size))
+            logger.info(f'Uploading {f}, timeout in {timeout}s')
+            assert timeout > 0
+            mon_bash(f'{UPLOAD_CMD} \"{f}\"', seconds=timeout)
+            if f.name.endswith(PKG_SUFFIX):
+                logger.info(f'Requesting repo update for {f.name}')
+                res = rrun('push_done', kwargs={'overwrite': False,})
+                if res is None:
+                    logger.info(f'Update success for {f.name}')
+                else:
+                    logger.error(f'Update failed for {f.name}, reason: {res}')
     def tick(self):
         '''
             check for updates,
@@ -142,16 +167,19 @@ class jobsManager:
         else:
             # This part does the job
             job = self.__get_job()
-            cwd = REPO_ROOT / job.pkgconfig.dirname
             if job.multiarch:
-                # wip
+                self.__clean(job, remove_pkg=True)
-                pass
+                self.__sign(job)
+                self.__upload(job)
+                self.__clean(job, remove_pkg=True)
             else:
                 self.__makepkg(job)
                 self.__sign(job)
                 self.__upload(job)
                 if job.pkgconfig.cleanbuild:
                     self.__clean(job, remove_pkg=True)
+                else:
+                    self.__clean(job, rm_src=False, remove_pkg=True)
             self.__finish_job(job.pkgconfig.dirname)
 jobsmgr = jobsManager()
 
@@ -206,7 +234,7 @@ class updateManager:
             buildarchs = [arch for arch in buildarchs if arch is not None]
             # hopefully we only need to check one arch for update
             arch = 'x86_64' if 'x86_64' in buildarchs else buildarchs[0] # prefer x86
-            mon_nspawn_shell(arch, MAKEPKG_UPD_CMD, cwd=pkgdir, minutes=60,
+            mon_nspawn_shell(arch, MAKEPKG_UPD_CMD, cwd=pkgdir, seconds=60*60,
                              logfile = pkgdir / 'buildbot.log.update',
                              short_return = True)
             if pkg.type in ('git', 'manual'):
@@ -234,24 +262,13 @@ class updateManager:
 updmgr = updateManager()
 
 
-@background
-def __main():
-    while True:
-        try:
-            jobsmgr.tick()
-        except:
-            print_exc_plus()
-        sleep(1)
-
-
-
-
 
 
+def info(*args, **kwargs):
+    return (args, kwargs)
 
 def run(funcname, args=list(), kwargs=dict()):
-    if funcname in ('clean', 'regenerate', 'remove',
+    if funcname in ('info',):
-                    'update', 'push_files', 'add_files'):
         logger.info('running: %s %s %s',funcname, args, kwargs)
         ret = eval(funcname)(*args, **kwargs)
         logger.info('done: %s %s',funcname, ret)
@@ -260,9 +277,8 @@ def run(funcname, args=list(), kwargs=dict()):
         logger.error('unexpected: %s %s %s',funcname, args, kwargs)
         return False
 
-
+@background
-if __name__ == '__main__':
+def __main():
-    __main() # start the main worker thread
     while True:
         try:
             with Listener(MASTER_BIND_ADDRESS, authkey=MASTER_BIND_PASSWD) as listener:
@@ -276,7 +292,18 @@ if __name__ == '__main__':
                         conn.send(run(funcname, args=args, kwargs=kwargs))
         except Exception:
             print_exc_plus()
+
+if __name__ == '__main__':
+    logger.info('Buildbot started.')
+    __main() # start the Listener thread
+    logger.info('Listener started.')
+    while True:
+        try:
+            jobsmgr.tick()
+        except Exception:
+            print_exc_plus()
         except KeyboardInterrupt:
             logger.info('KeyboardInterrupt')
             print_exc_plus()
             break
+        sleep(1)

config.py

@@ -48,10 +48,14 @@ MAKEPKG = 'makepkg --nosign --needed --noconfirm --noprogressbar --nocolor'
 MAKEPKG_UPD_CMD = 'makepkg --syncdeps --nobuild'
 MAKEPKG_MAKE_CMD = 'makepkg --syncdeps --noextract'
 MAKEPKG_MAKE_CMD_CLEAN = 'makepkg --syncdeps --noextract --clean --cleanbuild'
+MAKEPKG_MAKE_CMD_MARCH = 'makepkg --syncdeps --clean --cleanbuild'
 
 MAKEPKG_PKGLIST_CMD = f'{MAKEPKG} --packagelist'
 
 CONTAINER_BUILDBOT_ROOT = 'shared/buildbot'
-# single quote may cause problem here
+SHELL_ARCH_X64 = ['/usr/bin/sudo', 'machinectl', '--quiet', 'shell', 'build@archlinux', '/bin/bash', '-x', '-e', '-c']
-SHELL_ARCH_X64 = 'sudo machinectl --quiet shell build@archlinux /bin/bash -c \'{command}\''
+SHELL_ARCH_ARM64 = ['/usr/bin/sudo', 'machinectl', '--quiet', 'shell', 'root@alarm', '/bin/su', '-l', 'alarm', '-c']
-SHELL_ARCH_ARM64 = 'sudo machinectl --quiet shell root@alarm /bin/bash -c $\'su -l alarm -c \\\'{command}\\\'\''
+SHELL_ARM64_ADDITIONAL = 'set -e; set -x'
+SHELL_TRAP = 'trap \'echo ++ exit $?\' ERR EXIT'
+
+UPLOAD_CMD = 'rsync -avPh {src} repoupload:/srv/repo/buildbot/repo/updates/'

repod.py

@@ -39,17 +39,20 @@ class pushFm:
     def start(self, fname, size):
         '''
             size is in MB
+            returns -1 when busy
         '''
         if self.is_busy():
-            return f'busy with {self.fname}'
+            return -1
         self.fname = fname
         self.start_time = time()
         self.size = size
         if size <= 7.5:
+            timeout = 120
             self.end_time = self.start_time + 120
         else:
-            self.end_time = self.start_time + size / (REPO_PUSH_BANDWIDTH / 8) * 2
+            timeout = size / (REPO_PUSH_BANDWIDTH / 8) * 2
-        return None
+        self.end_time = self.start_time + timeout
+        return timeout
     def tick(self):
         '''
             return None means success
@@ -107,11 +110,11 @@ class pushFm:
 
 pfm = pushFm()
 
-def push_files(filename, size):
+def push_start(filename, size):
     pfm.tick()
     return pfm.start(filename, size)
 
-def add_files(filename, overwrite=False):
+def push_done(filename, overwrite=False):
     return pfm.done(filename, overwrite=overwrite)
 
 
@@ -120,7 +123,7 @@ def add_files(filename, overwrite=False):
 
 def run(funcname, args=list(), kwargs=dict()):
     if funcname in ('clean', 'regenerate', 'remove',
-                    'update', 'push_files', 'add_files'):
+                    'update', 'push_start', 'push_done'):
         logger.info('running: %s %s %s', funcname, args, kwargs)
         ret = eval(funcname)(*args, **kwargs)
         logger.info('done: %s %s',funcname, ret)

utils.py

@@ -11,6 +11,7 @@ import sys
 import traceback
 
 from config import PKG_COMPRESSION, SHELL_ARCH_ARM64, SHELL_ARCH_X64, \
+                   SHELL_ARM64_ADDITIONAL, SHELL_TRAP, \
                    CONTAINER_BUILDBOT_ROOT, ARCHS
 
 logger = logging.getLogger(f'buildbot.{__name__}')
@@ -28,10 +29,10 @@ def bash(cmdline, **kwargs):
     logger.info(f'bash: {cmdline}, kwargs: {kwargs}')
     return(run_cmd(['/bin/bash', '-x', '-e', '-c', cmdline], **kwargs))
 
-def mon_bash(cmdline, cwd=None, minutes=30, **kwargs):
+def mon_bash(cmdline, seconds=60*30, **kwargs):
-    assert type(minutes) is int and minutes >= 1
+    assert type(seconds) is int and seconds >= 1
-    return bash(cmdline, cwd=cwd, keepalive=True, KEEPALIVE_TIMEOUT=60,
+    return bash(cmdline, keepalive=True, KEEPALIVE_TIMEOUT=60,
-                RUN_CMD_TIMEOUT=minutes*60, **kwargs)
+                RUN_CMD_TIMEOUT=seconds, **kwargs)
 
 def nspawn_shell(arch, cmdline, cwd=None, **kwargs):
     root = Path(CONTAINER_BUILDBOT_ROOT)
@@ -39,22 +40,23 @@ def nspawn_shell(arch, cmdline, cwd=None, **kwargs):
         cwd = root / cwd
     else:
         cwd = root
+    logger.info(f'bash_{arch}: {cmdline}, cwd: {cwd}, kwargs: {kwargs}')
     if arch in ('aarch64', 'arm64'):
-        ret = bash(SHELL_ARCH_ARM64.format(
+        command=f'{SHELL_ARM64_ADDITIONAL}; {SHELL_TRAP}; cd \'{cwd}\'; {cmdline}'
-            command=f'cd \"{cwd}\" || echo \"++ exit 1\" && exit 1; {cmdline}; echo \"++ exit $?\"'), **kwargs)
+        ret = run_cmd(SHELL_ARCH_ARM64 + [command,], **kwargs)
     elif arch in ('x64', 'x86', 'x86_64'):
-        ret = bash(SHELL_ARCH_X64.format(
+        command=f'{SHELL_TRAP}; cd \'{cwd}\'; {cmdline}'
-            command=f'cd \"{cwd}\" || echo \"++ exit 1\" && exit 1; {cmdline}; echo \"++ exit $?\"'), **kwargs)
+        ret = run_cmd(SHELL_ARCH_X64 + [command,], **kwargs)
     else:
         raise TypeError('nspawn_shell: wrong arch')
     if not ret.endswith('++ exit 0\n'):
         raise subprocess.CalledProcessError(1, cmdline, ret)
     return ret
 
-def mon_nspawn_shell(arch, cmdline, cwd, minutes=30, **kwargs):
+def mon_nspawn_shell(arch, cmdline, cwd, seconds=60*30, **kwargs):
-    assert type(minutes) is int and minutes >= 1
+    assert type(seconds) is int and seconds >= 1
     return nspawn_shell(arch, cmdline, cwd=cwd, keepalive=True, KEEPALIVE_TIMEOUT=60,
-                        RUN_CMD_TIMEOUT=minutes*60, **kwargs)
+                        RUN_CMD_TIMEOUT=seconds, **kwargs)
 
 def run_cmd(cmd, cwd=None, keepalive=False, KEEPALIVE_TIMEOUT=30, RUN_CMD_TIMEOUT=60,
             logfile=None, short_return=False):
@@ -96,7 +98,6 @@ def run_cmd(cmd, cwd=None, keepalive=False, KEEPALIVE_TIMEOUT=30, RUN_CMD_TIMEOU
                     continue
                 line.replace('\x0f', '\n')
                 last_read_time = int(time())
-                logger.debug(line)
                 output.append(line)
                 last_read[0] = last_read_time
                 last_read[1] = line
@@ -253,7 +254,7 @@ def format_exc_plus():
             ret += '\n'
     return ret
 
-def configure_logger(logger, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
+def configure_logger(logger, format='%(asctime)s - %(name)-18s - %(levelname)s - %(message)s',
                      level=logging.INFO, logfile=None, flevel=logging.INFO, rotate_size=None):
     class ExceptionFormatter(logging.Formatter):
         def format(self, record):

yamlparse.py

@@ -73,10 +73,10 @@ def load_all():
     pkgconfigs = list()
     for mydir in REPO_ROOT.iterdir():
         try:
-            if mydir.is_dir() and (mydir / AUTOBUILD_FNAME).exists():
+            if mydir.is_dir():
+                if (mydir / AUTOBUILD_FNAME).exists():
                     # parsing yaml
-                logger.info('Bulidbot: found %s in %s', AUTOBUILD_FNAME,
+                    logger.info('Bulidbot: found %s in %s', AUTOBUILD_FNAME, mydir)
-                                                        mydir / AUTOBUILD_FNAME)
                     with open(mydir / AUTOBUILD_FNAME, 'r') as f:
                         content = f.read()
                         content = load(content, Loader=Loader)
@@ -85,6 +85,8 @@ def load_all():
                                 ('type', 'cleanbuild', 'timeout', 'extra')]
                         args = [mydir.name] + args
                         pkgconfigs.append(pkgConfig(*args))
+                else:
+                    logger.warning('Bulidbot: NO %s in %s', AUTOBUILD_FNAME, mydir)
         except Exception:
             logger.error(f'Error while parsing {AUTOBUILD_FNAME} for {mydir.name}')
             print_exc_plus()