diff --git a/client.py b/client.py new file mode 100644 index 0000000..2e53b52 --- /dev/null +++ b/client.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# repod.py: Automatic management tool for an arch repo. +# This file is part of Buildbot by JerryXiao + +import logging +from multiprocessing.connection import Client +from time import sleep + +from config import REPOD_BIND_ADDRESS, REPOD_BIND_PASSWD +from utils import print_exc_plus + +logger = logging.getLogger(__name__) + +def ping(funcname, args=list(), kwargs=dict(), retries=0): + try: + logger.info('client: %s %s %s',funcname, args, kwargs) + with Client(REPOD_BIND_ADDRESS, authkey=REPOD_BIND_PASSWD) as conn: + conn.send([funcname, args, kwargs]) + return conn.recv() + except ConnectionRefusedError: + if retries <= 10: + logger.info("Server refused, retry after 60s") + sleep(60) + return ping(funcname, args=args, kwargs=kwargs, retries=retries+1) + else: + logger.error("Server refused") + return False + except EOFError: + logger.error('Internal server error') + return False + except Exception: + print_exc_plus() + +if __name__ == '__main__': + import argparse + logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') + logger.info('result: %s', ping('push_files', args=('aaa', 1))) + logger.info('result: %s', ping('add_files', args=('aaa',))) + #logger.info('result: %s', ping('update')) diff --git a/config.py b/config.py index 9277ea4..644f59f 100644 --- a/config.py +++ b/config.py @@ -8,16 +8,33 @@ REPO_NAME='jerryxiao' PKG_COMPRESSION='xz' BUILD_ARCHS = ['aarch64', 'any', 'x86_64'] + #### config for repo.py + REPO_CMD = 'repo-add --verify --remove' REPO_REMOVE_CMD = 'repo-remove --verify' RECENT_VERSIONS_KEPT = 3 PREFERRED_ANY_BUILD_ARCH = 'x86_64' + +#### config for repod.py + +REPOD_BIND_ADDRESS = ('localhost', 7010) +REPOD_BIND_PASSWD = b'mypassword' + +REPO_PUSH_BANDWIDTH = 1 # 1Mbps +GPG_VERIFY_CMD = 'gpg --verify' + + #### config for package.py + # Archlinux-Jerry Build Bot GPG_KEY = 'BEE4F1D5A661CA1FEA65C38093962CE07A0D5B7D' GPG_SIGN_CMD = (f'gpg --default-key {GPG_KEY} --no-armor' '--pinentry-mode loopback --passphrase \'\'' - '--detach-sign --yes -- ') + '--detach-sign --yes --') +#### config for master.py + +MASTER_BIND_ADDRESS = ('localhost', 7011) +MASTER_BIND_PASSWD = b'mypassword' diff --git a/repo.py b/repo.py index afb092d..acb2ed7 100755 --- a/repo.py +++ b/repo.py @@ -24,7 +24,6 @@ import logging from utils import bash, Pkg, get_pkg_details_from_name, \ print_exc_plus from time import time -import argparse from config import REPO_NAME, PKG_COMPRESSION, ARCHS, REPO_CMD, \ REPO_REMOVE_CMD @@ -158,6 +157,7 @@ def _clean_archive(keep_new=3): dir_list = [fpath for fpath in basedir.iterdir() if fpath.name.endswith(PKG_SUFFIX)] filter_old_pkg(dir_list, keep_new=keep_new, recycle=True) logger.info('finished clean') + return True def _regenerate(target_archs=ARCHS, just_symlink=False): if just_symlink: @@ -187,7 +187,7 @@ def _regenerate(target_archs=ARCHS, just_symlink=False): else: logger.error(f'{arch} dir does not exist!') if just_symlink: - return + return True # run repo_add for arch in target_archs: basedir = Path('www') / arch @@ -236,6 +236,7 @@ def _regenerate(target_archs=ARCHS, just_symlink=False): if rfile not in repo_files_count: logger.error(f'{rfile} does not exist in {arch}!') logger.info('finished regenerate') + return True def _update(): logger.info('starting update') @@ -281,6 +282,7 @@ def _update(): logger.warning(f"{other} is garbage!") throw_away(other) logger.info('finished update') + return True def _remove(pkgnames, target_archs=[a for a in ARCHS if a != 'any']): assert type(pkgnames) is list and pkgnames @@ -302,8 +304,10 @@ def _remove(pkgnames, target_archs=[a for a in ARCHS if a != 'any']): else: logger.warning(f'Nothing to remove in {arch}') logger.info('finished remove') + return True if __name__ == '__main__': + import argparse try: parser = argparse.ArgumentParser(description='Automatic management tool for an arch repo.') parser.add_argument('-a', '--arch', nargs='?', default=False, help='arch to regenerate, split by comma, defaults to all') @@ -336,4 +340,4 @@ if __name__ == '__main__': parser.error("Please choose an action") except Exception as err: print_exc_plus() - + parser.exit(status=1) diff --git a/repod.py b/repod.py new file mode 100644 index 0000000..d9f1078 --- /dev/null +++ b/repod.py @@ -0,0 +1,146 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# repod.py: Automatic management tool for an arch repo. +# This file is part of Buildbot by JerryXiao + +import logging +from threading import Thread +from multiprocessing.connection import Listener +from time import time, sleep +from pathlib import Path +from subprocess import CalledProcessError + +from utils import print_exc_plus + +from config import REPOD_BIND_ADDRESS, REPOD_BIND_PASSWD, REPO_PUSH_BANDWIDTH, \ + GPG_VERIFY_CMD + + + + +from repo import _clean_archive as clean, \ + _regenerate as regenerate, \ + _remove as remove, \ + _update as update + +from utils import bash + + +logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') +logger = logging.getLogger(__name__) + +class pushFm: + def __init__(self): + self.fname = None + self.size = None + self.start_time = None + self.end_time = None + def start(self, fname, size): + ''' + size is in MB + ''' + if self.is_busy(): + return f'busy with {self.fname}' + self.fname = fname + self.start_time = time() + self.size = size + if size <= 7.5: + self.end_time = self.start_time + 120 + else: + self.end_time = self.start_time + size / (REPO_PUSH_BANDWIDTH / 8) * 2 + return None + def tick(self): + ''' + return None means success + else returns an error string + ''' + if self.is_busy(): + if time() > self.end_time: + ret = f'file {self.fname} is supposed to finish at {self.end_time}' + self.__init__() + logger.error(f'pfm: {ret}') + return ret + else: + return None + else: + return None + def done(self, fname): + ''' + return None means success + else returns an error string + ''' + if fname == self.fname: + self.__init__() + update_path = Path('updates') + pkg_found = False + sig_found = False + for fpath in update_path.iterdir(): + if fpath.is_dir: + continue + if fpath.name == self.fname: + pkg_found = fpath + elif fpath.name == f'{self.fname}.sig': + sig_found = fpath + if pkg_found and sig_found: + try: + bash(f'{GPG_VERIFY_CMD} {str(sig_found)} {str(pkg_found)}') + except CalledProcessError: + print_exc_plus() + return 'GPG verify error' + else: + return None + else: + return f'file missing: pkg {str(pkg_found)} sig {str(sig_found)}' + return "unexpected error" + else: + return "Wrong file" + def is_busy(self): + return not (self.fname is None) + +pfm = pushFm() + +def run(funcname, args=list(), kwargs=dict()): + if funcname in ('clean', 'regenerate', 'remove', + 'update', 'push_files', 'add_files'): + logger.info('running: %s %s %s',funcname, args, kwargs) + ret = eval(funcname)(*args, **kwargs) + logger.info('done: %s %s',funcname, ret) + return ret + else: + logger.error('unexpected: %s %s %s',funcname, args, kwargs) + return False + +def push_files(filename, size): + pfm.tick() + return pfm.start(filename, size) + +def add_files(filename): + res = pfm.done(filename) + if res: + return res + else: + try: + if update(): + return None + except Exception: + print_exc_plus() + return 'update error' + + +if __name__ == '__main__': + while True: + try: + with Listener(REPOD_BIND_ADDRESS, authkey=REPOD_BIND_PASSWD) as listener: + with listener.accept() as conn: + logger.info('connection accepted from %s', listener.last_accepted) + myrecv = conn.recv() + if type(myrecv) is list and len(myrecv) == 3: + (funcname, args, kwargs) = myrecv + funcname = str(funcname) + conn.send(run(funcname, args=args, kwargs=kwargs)) + except Exception: + print_exc_plus() + except KeyboardInterrupt: + logger.info('KeyboardInterrupt') + print_exc_plus() + break