added repod.py

client.py

@@ -0,0 +1,40 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# repod.py: Automatic management tool for an arch repo.
+# This file is part of Buildbot by JerryXiao
+
+import logging
+from multiprocessing.connection import Client
+from time import sleep
+
+from config import REPOD_BIND_ADDRESS, REPOD_BIND_PASSWD
+from utils import print_exc_plus
+
+logger = logging.getLogger(__name__)
+
+def ping(funcname, args=list(), kwargs=dict(), retries=0):
+    try:
+        logger.info('client: %s %s %s',funcname, args, kwargs)
+        with Client(REPOD_BIND_ADDRESS, authkey=REPOD_BIND_PASSWD) as conn:
+            conn.send([funcname, args, kwargs])
+            return conn.recv()
+    except ConnectionRefusedError:
+        if retries <= 10:
+            logger.info("Server refused, retry after 60s")
+            sleep(60)
+            return ping(funcname, args=args, kwargs=kwargs, retries=retries+1)
+        else:
+            logger.error("Server refused")
+            return False
+    except EOFError:
+        logger.error('Internal server error')
+        return False
+    except Exception:
+        print_exc_plus()
+
+if __name__ == '__main__':
+    import argparse
+    logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
+    logger.info('result: %s', ping('push_files', args=('aaa', 1)))
+    logger.info('result: %s', ping('add_files', args=('aaa',)))
+    #logger.info('result: %s', ping('update'))

config.py

@@ -8,16 +8,33 @@ REPO_NAME='jerryxiao'
 PKG_COMPRESSION='xz'
 BUILD_ARCHS = ['aarch64', 'any', 'x86_64']
 
+
 #### config for repo.py
+
 REPO_CMD = 'repo-add --verify --remove'
 REPO_REMOVE_CMD = 'repo-remove --verify'
 RECENT_VERSIONS_KEPT = 3
 PREFERRED_ANY_BUILD_ARCH = 'x86_64'
 
+
+#### config for repod.py
+
+REPOD_BIND_ADDRESS = ('localhost', 7010)
+REPOD_BIND_PASSWD = b'mypassword'
+
+REPO_PUSH_BANDWIDTH = 1 # 1Mbps
+GPG_VERIFY_CMD = 'gpg --verify'
+
+
 #### config for package.py
+
 # Archlinux-Jerry Build Bot <buildbot@mail.jerryxiao.cc>
 GPG_KEY = 'BEE4F1D5A661CA1FEA65C38093962CE07A0D5B7D'
 GPG_SIGN_CMD = (f'gpg --default-key {GPG_KEY} --no-armor'
                  '--pinentry-mode loopback --passphrase \'\''
                  '--detach-sign --yes --')
 
+#### config for master.py
+
+MASTER_BIND_ADDRESS = ('localhost', 7011)
+MASTER_BIND_PASSWD = b'mypassword'

repo.py

@@ -24,7 +24,6 @@ import logging
 from utils import bash, Pkg, get_pkg_details_from_name, \
                   print_exc_plus
 from time import time
-import argparse
 
 from config import REPO_NAME, PKG_COMPRESSION, ARCHS, REPO_CMD, \
                    REPO_REMOVE_CMD
@@ -158,6 +157,7 @@ def _clean_archive(keep_new=3):
     dir_list = [fpath for fpath in basedir.iterdir() if fpath.name.endswith(PKG_SUFFIX)]
     filter_old_pkg(dir_list, keep_new=keep_new, recycle=True)
     logger.info('finished clean')
+    return True
 
 def _regenerate(target_archs=ARCHS, just_symlink=False):
     if just_symlink:
@@ -187,7 +187,7 @@ def _regenerate(target_archs=ARCHS, just_symlink=False):
     else:
         logger.error(f'{arch} dir does not exist!')
     if just_symlink:
-        return
+        return True
     # run repo_add
     for arch in target_archs:
         basedir = Path('www') / arch
@@ -236,6 +236,7 @@ def _regenerate(target_archs=ARCHS, just_symlink=False):
             if rfile not in repo_files_count:
                 logger.error(f'{rfile} does not exist in {arch}!')
     logger.info('finished regenerate')
+    return True
 
 def _update():
     logger.info('starting update')
@@ -281,6 +282,7 @@ def _update():
             logger.warning(f"{other} is garbage!")
             throw_away(other)
     logger.info('finished update')
+    return True
 
 def _remove(pkgnames, target_archs=[a for a in ARCHS if a != 'any']):
     assert type(pkgnames) is list and pkgnames
@@ -302,8 +304,10 @@ def _remove(pkgnames, target_archs=[a for a in ARCHS if a != 'any']):
         else:
             logger.warning(f'Nothing to remove in {arch}')
     logger.info('finished remove')
+    return True
 
 if __name__ == '__main__':
+    import argparse
     try:
         parser = argparse.ArgumentParser(description='Automatic management tool for an arch repo.')
         parser.add_argument('-a', '--arch', nargs='?', default=False, help='arch to regenerate, split by comma, defaults to all')
@@ -336,4 +340,4 @@ if __name__ == '__main__':
             parser.error("Please choose an action")
     except Exception as err:
         print_exc_plus()
-
+        parser.exit(status=1)

repod.py

@@ -0,0 +1,146 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# repod.py: Automatic management tool for an arch repo.
+# This file is part of Buildbot by JerryXiao
+
+import logging
+from threading import Thread
+from multiprocessing.connection import Listener
+from time import time, sleep
+from pathlib import Path
+from subprocess import CalledProcessError
+
+from utils import print_exc_plus
+
+from config import REPOD_BIND_ADDRESS, REPOD_BIND_PASSWD, REPO_PUSH_BANDWIDTH, \
+                   GPG_VERIFY_CMD
+
+
+
+
+from repo import _clean_archive as clean, \
+                 _regenerate as regenerate, \
+                 _remove as remove, \
+                 _update as update
+
+from utils import bash
+
+
+logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
+logger = logging.getLogger(__name__)
+
+class pushFm:
+    def __init__(self):
+        self.fname = None
+        self.size = None
+        self.start_time = None
+        self.end_time = None
+    def start(self, fname, size):
+        '''
+            size is in MB
+        '''
+        if self.is_busy():
+            return f'busy with {self.fname}'
+        self.fname = fname
+        self.start_time = time()
+        self.size = size
+        if size <= 7.5:
+            self.end_time = self.start_time + 120
+        else:
+            self.end_time = self.start_time + size / (REPO_PUSH_BANDWIDTH / 8) * 2
+        return None
+    def tick(self):
+        '''
+            return None means success
+            else returns an error string
+        '''
+        if self.is_busy():
+            if time() > self.end_time:
+                ret = f'file {self.fname} is supposed to finish at {self.end_time}'
+                self.__init__()
+                logger.error(f'pfm: {ret}')
+                return ret
+            else:
+                return None
+        else:
+            return None
+    def done(self, fname):
+        '''
+            return None means success
+            else returns an error string
+        '''
+        if fname == self.fname:
+            self.__init__()
+            update_path = Path('updates')
+            pkg_found = False
+            sig_found = False
+            for fpath in update_path.iterdir():
+                if fpath.is_dir:
+                    continue
+                if fpath.name == self.fname:
+                    pkg_found = fpath
+                elif fpath.name == f'{self.fname}.sig':
+                    sig_found = fpath
+            if pkg_found and sig_found:
+                try:
+                    bash(f'{GPG_VERIFY_CMD} {str(sig_found)} {str(pkg_found)}')
+                except CalledProcessError:
+                    print_exc_plus()
+                    return 'GPG verify error'
+                else:
+                    return None
+            else:
+                return f'file missing: pkg {str(pkg_found)} sig {str(sig_found)}'
+            return "unexpected error"
+        else:
+            return "Wrong file"
+    def is_busy(self):
+        return not (self.fname is None)
+
+pfm = pushFm()
+
+def run(funcname, args=list(), kwargs=dict()):
+    if funcname in ('clean', 'regenerate', 'remove',
+                    'update', 'push_files', 'add_files'):
+        logger.info('running: %s %s %s',funcname, args, kwargs)
+        ret = eval(funcname)(*args, **kwargs)
+        logger.info('done: %s %s',funcname, ret)
+        return ret
+    else:
+        logger.error('unexpected: %s %s %s',funcname, args, kwargs)
+        return False
+
+def push_files(filename, size):
+    pfm.tick()
+    return pfm.start(filename, size)
+
+def add_files(filename):
+    res = pfm.done(filename)
+    if res:
+        return res
+    else:
+        try:
+            if update():
+                return None
+        except Exception:
+            print_exc_plus()
+            return 'update error'
+
+
+if __name__ == '__main__':
+    while True:
+        try:
+            with Listener(REPOD_BIND_ADDRESS, authkey=REPOD_BIND_PASSWD) as listener:
+                with listener.accept() as conn:
+                    logger.info('connection accepted from %s', listener.last_accepted)
+                    myrecv = conn.recv()
+                    if type(myrecv) is list and len(myrecv) == 3:
+                        (funcname, args, kwargs) = myrecv
+                        funcname = str(funcname)
+                        conn.send(run(funcname, args=args, kwargs=kwargs))
+        except Exception:
+            print_exc_plus()
+        except KeyboardInterrupt:
+            logger.info('KeyboardInterrupt')
+            print_exc_plus()
+            break