buildbot/repod.py

202 lines
6.9 KiB
Python
Raw Permalink Normal View History

2019-04-05 00:21:12 +08:00
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# repod.py: Automatic management tool for an arch repo.
# This file is part of Buildbot by JerryXiao
import logging
from multiprocessing.connection import Listener
from time import time, sleep
from pathlib import Path
from subprocess import CalledProcessError
2019-04-09 15:43:17 +08:00
import os
2019-04-05 00:21:12 +08:00
from config import REPOD_BIND_ADDRESS, REPOD_BIND_PASSWD, REPO_PUSH_BANDWIDTH, \
GPG_VERIFY_CMD
2019-09-06 14:59:45 +08:00
from shared_vars import PKG_SUFFIX, PKG_SIG_SUFFIX
2019-04-05 00:21:12 +08:00
from repo import _clean_archive as clean, \
_regenerate as regenerate, \
_remove as remove, \
_update as update
2019-04-09 15:43:17 +08:00
from utils import bash, configure_logger, print_exc_plus
2019-04-05 00:21:12 +08:00
2019-04-09 15:43:17 +08:00
abspath=os.path.abspath(__file__)
abspath=os.path.dirname(abspath)
os.chdir(abspath)
2019-04-05 00:21:12 +08:00
2019-04-09 21:15:03 +08:00
logger = logging.getLogger('buildbot')
2019-09-06 11:45:12 +08:00
configure_logger(logger, logfile='repod.log', rotate_size=1024*1024*10, enable_notify=True)
2019-04-05 00:21:12 +08:00
class pushFm:
def __init__(self):
2019-09-06 14:59:45 +08:00
self.fnames = list()
2019-04-05 00:21:12 +08:00
self.size = None
2019-09-06 14:59:45 +08:00
self.sizes = None
2019-04-05 00:21:12 +08:00
self.start_time = None
self.end_time = None
2019-09-06 14:59:45 +08:00
def start(self, fnames, sizes):
2019-04-05 00:21:12 +08:00
'''
2019-09-06 14:59:45 +08:00
sizes is list in MB
2019-04-09 20:48:37 +08:00
returns -1 when busy
2019-04-05 00:21:12 +08:00
'''
if self.is_busy():
2019-04-09 20:48:37 +08:00
return -1
2019-09-06 14:59:45 +08:00
self.fnames = fnames
2019-04-05 00:21:12 +08:00
self.start_time = time()
2019-09-06 14:59:45 +08:00
self.sizes = sizes
size = 0
for s in sizes:
size += s
2019-04-05 00:21:12 +08:00
self.size = size
2019-09-06 14:59:45 +08:00
def get_timeout(size):
if size <= 7.5:
timeout = 120
else:
timeout = size / (REPO_PUSH_BANDWIDTH / 8) * 2
return timeout
timeouts = [get_timeout(s) for s in sizes]
self.end_time = self.start_time + get_timeout(self.size)
return timeouts
2019-04-05 00:21:12 +08:00
def tick(self):
'''
return None means success
else returns an error string
'''
if self.is_busy():
if time() > self.end_time:
2019-09-06 14:59:45 +08:00
ret = f'files {self.fnames} are supposed to finish at {self.end_time}'
2019-04-05 00:21:12 +08:00
self.__init__()
2019-09-06 14:59:45 +08:00
logger.error(f'tick: {ret}')
2019-04-05 00:21:12 +08:00
return ret
else:
return None
else:
return None
2019-09-06 14:59:45 +08:00
def fail(self, tfname):
update_path = Path('updates')
2019-09-06 14:59:45 +08:00
if tfname in self.fnames:
for fname in self.fnames:
pkg = update_path / fname
sig = update_path / f'{fname}.sig'
for f in (pkg, sig):
if f.exists():
try:
f.unlink()
except Exception:
logger.warning(f'unable to remove {f.name}')
self.__init__()
return None
else:
return "Wrong file"
2019-09-07 11:56:52 +08:00
def add_time(self, tfname, atime):
if tfname in self.fnames:
assert type(atime) in (int, float)
self.end_time += atime
return None
else:
return "Wrong file"
2019-09-06 14:59:45 +08:00
def done(self, fnames, overwrite=False):
2019-04-05 00:21:12 +08:00
'''
return None means success
else returns an error string
'''
2019-09-06 14:59:45 +08:00
if [f for f in fnames if not (f.endswith(PKG_SUFFIX) or f.endswith(PKG_SIG_SUFFIX))]:
2019-09-06 15:48:25 +08:00
return "files to upload are garbage"
2019-09-06 14:59:45 +08:00
filter_sig = lambda fnames:[fname for fname in fnames if not fname.endswith(PKG_SIG_SUFFIX)]
if sorted(filter_sig(fnames)) == sorted(filter_sig(self.fnames)):
2019-04-05 00:45:50 +08:00
try:
2019-04-09 21:26:34 +08:00
update_path = Path('updates')
2019-09-06 14:59:45 +08:00
for pkgfname in filter_sig(fnames):
pkg_found = False
sig_found = False
for fpath in update_path.iterdir():
if fpath.is_dir():
continue
if fpath.name == pkgfname:
pkg_found = fpath
elif fpath.name == f'{pkgfname}.sig':
sig_found = fpath
if pkg_found and sig_found:
2019-04-05 00:45:50 +08:00
try:
2019-09-06 14:59:45 +08:00
bash(f'{GPG_VERIFY_CMD} {sig_found} {pkg_found}')
except CalledProcessError:
ret = f'{pkg_found} GPG verify error'
logger.error(ret)
2019-04-05 00:45:50 +08:00
print_exc_plus()
2019-09-06 14:59:45 +08:00
return ret
else:
2019-09-06 15:48:25 +08:00
# gpg verify success
continue
2019-09-06 14:59:45 +08:00
else:
return f'file missing: pkg {pkg_found} sig {sig_found}'
2019-09-06 15:48:25 +08:00
raise RuntimeError("unexpected error")
2019-04-05 00:21:12 +08:00
else:
2019-09-06 15:48:25 +08:00
# gpg verified for all
try:
if update(overwrite=overwrite):
return None
else:
raise RuntimeError('update return false')
except Exception:
print_exc_plus()
return f'{pkg_found} update error'
raise RuntimeError("unexpected error")
2019-04-05 00:45:50 +08:00
finally:
self.__init__()
2019-04-05 00:21:12 +08:00
else:
return "Wrong file"
def is_busy(self):
2019-09-06 14:59:45 +08:00
return bool(self.fnames)
2019-04-05 00:21:12 +08:00
pfm = pushFm()
2019-09-06 14:59:45 +08:00
def push_start(filenames, sizes):
2019-04-06 20:59:27 +08:00
pfm.tick()
2019-09-06 14:59:45 +08:00
return pfm.start(filenames, sizes)
2019-04-06 20:59:27 +08:00
2019-09-06 14:59:45 +08:00
def push_done(filenames, overwrite=False):
return pfm.done(filenames, overwrite=overwrite)
2019-04-06 20:59:27 +08:00
def push_fail(filename):
return pfm.fail(filename)
2019-04-06 20:59:27 +08:00
2019-09-07 11:56:52 +08:00
def push_add_time(filename, atime):
return pfm.add_time(filename, atime)
2019-04-06 20:59:27 +08:00
# server part
2019-04-05 00:21:12 +08:00
def run(funcname, args=list(), kwargs=dict()):
if funcname in ('clean', 'regenerate', 'remove',
'update', 'push_start', 'push_done',
2019-09-07 11:56:52 +08:00
'push_fail', 'push_add_time'):
2019-04-07 17:14:50 +08:00
logger.info('running: %s %s %s', funcname, args, kwargs)
2019-04-05 00:21:12 +08:00
ret = eval(funcname)(*args, **kwargs)
logger.info('done: %s %s',funcname, ret)
return ret
else:
logger.error('unexpected: %s %s %s',funcname, args, kwargs)
return False
if __name__ == '__main__':
2019-09-06 11:28:25 +08:00
logger.info('Buildbot.repod started.')
2019-04-05 00:21:12 +08:00
while True:
try:
with Listener(REPOD_BIND_ADDRESS, authkey=REPOD_BIND_PASSWD) as listener:
with listener.accept() as conn:
2019-09-06 15:16:24 +08:00
logger.debug('connection accepted from %s', listener.last_accepted)
2019-04-05 00:21:12 +08:00
myrecv = conn.recv()
if type(myrecv) is list and len(myrecv) == 3:
(funcname, args, kwargs) = myrecv
funcname = str(funcname)
conn.send(run(funcname, args=args, kwargs=kwargs))
except Exception:
print_exc_plus()
except KeyboardInterrupt:
logger.info('KeyboardInterrupt')
print_exc_plus()
break