buildbot/repod.py

170 lines
5.4 KiB
Python
Raw Normal View History

2019-04-05 00:21:12 +08:00
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# repod.py: Automatic management tool for an arch repo.
# This file is part of Buildbot by JerryXiao
import logging
from multiprocessing.connection import Listener
from time import time, sleep
from pathlib import Path
from subprocess import CalledProcessError
2019-04-09 15:43:17 +08:00
import os
2019-04-05 00:21:12 +08:00
from config import REPOD_BIND_ADDRESS, REPOD_BIND_PASSWD, REPO_PUSH_BANDWIDTH, \
GPG_VERIFY_CMD
from repo import _clean_archive as clean, \
_regenerate as regenerate, \
_remove as remove, \
_update as update
2019-04-09 15:43:17 +08:00
from utils import bash, configure_logger, print_exc_plus
2019-04-05 00:21:12 +08:00
2019-04-09 15:43:17 +08:00
abspath=os.path.abspath(__file__)
abspath=os.path.dirname(abspath)
os.chdir(abspath)
2019-04-05 00:21:12 +08:00
2019-04-09 21:15:03 +08:00
logger = logging.getLogger('buildbot')
2019-04-09 15:43:17 +08:00
configure_logger(logger, logfile='repod.log', rotate_size=1024*1024*10)
2019-04-05 00:21:12 +08:00
class pushFm:
def __init__(self):
self.fname = None
self.size = None
self.start_time = None
self.end_time = None
def start(self, fname, size):
'''
size is in MB
2019-04-09 20:48:37 +08:00
returns -1 when busy
2019-04-05 00:21:12 +08:00
'''
if self.is_busy():
2019-04-09 20:48:37 +08:00
return -1
2019-04-05 00:21:12 +08:00
self.fname = fname
self.start_time = time()
self.size = size
if size <= 7.5:
2019-04-09 20:48:37 +08:00
timeout = 120
2019-04-05 00:21:12 +08:00
self.end_time = self.start_time + 120
else:
2019-04-09 20:48:37 +08:00
timeout = size / (REPO_PUSH_BANDWIDTH / 8) * 2
self.end_time = self.start_time + timeout
return timeout
2019-04-05 00:21:12 +08:00
def tick(self):
'''
return None means success
else returns an error string
'''
if self.is_busy():
if time() > self.end_time:
ret = f'file {self.fname} is supposed to finish at {self.end_time}'
self.__init__()
logger.error(f'pfm: {ret}')
return ret
else:
return None
else:
return None
def fail(self, fname):
update_path = Path('updates')
if fname == self.fname:
pkg = update_path / self.fname
sig = update_path / f'{self.fname}.sig'
for f in (pkg, sig):
if f.exists():
try:
f.unlink()
except Exception:
logger.warning(f'unable to remove {f.name}')
self.__init__()
return None
else:
return "Wrong file"
2019-04-05 00:45:50 +08:00
def done(self, fname, overwrite=False):
2019-04-05 00:21:12 +08:00
'''
return None means success
else returns an error string
'''
if fname == self.fname:
2019-04-05 00:45:50 +08:00
try:
2019-04-09 21:26:34 +08:00
update_path = Path('updates')
2019-04-05 00:45:50 +08:00
pkg_found = False
sig_found = False
for fpath in update_path.iterdir():
2019-04-09 21:26:34 +08:00
if fpath.is_dir():
2019-04-05 00:45:50 +08:00
continue
if fpath.name == self.fname:
pkg_found = fpath
elif fpath.name == f'{self.fname}.sig':
sig_found = fpath
if pkg_found and sig_found:
try:
bash(f'{GPG_VERIFY_CMD} {sig_found} {pkg_found}')
except CalledProcessError:
print_exc_plus()
return 'GPG verify error'
else:
try:
if update(overwrite=overwrite):
return None
except Exception:
print_exc_plus()
return 'update error'
2019-04-05 00:21:12 +08:00
else:
2019-04-05 00:45:50 +08:00
return f'file missing: pkg {pkg_found} sig {sig_found}'
return "unexpected error"
finally:
self.__init__()
2019-04-05 00:21:12 +08:00
else:
return "Wrong file"
def is_busy(self):
return not (self.fname is None)
pfm = pushFm()
2019-04-09 20:48:37 +08:00
def push_start(filename, size):
2019-04-06 20:59:27 +08:00
pfm.tick()
return pfm.start(filename, size)
2019-04-09 20:48:37 +08:00
def push_done(filename, overwrite=False):
2019-04-06 20:59:27 +08:00
return pfm.done(filename, overwrite=overwrite)
def push_fail(filename):
return pfm.fail(filename)
2019-04-06 20:59:27 +08:00
# server part
2019-04-05 00:21:12 +08:00
def run(funcname, args=list(), kwargs=dict()):
if funcname in ('clean', 'regenerate', 'remove',
'update', 'push_start', 'push_done',
'push_fail'):
2019-04-07 17:14:50 +08:00
logger.info('running: %s %s %s', funcname, args, kwargs)
2019-04-05 00:21:12 +08:00
ret = eval(funcname)(*args, **kwargs)
logger.info('done: %s %s',funcname, ret)
return ret
else:
logger.error('unexpected: %s %s %s',funcname, args, kwargs)
return False
if __name__ == '__main__':
2019-09-06 11:28:25 +08:00
logger.info('Buildbot.repod started.')
2019-04-05 00:21:12 +08:00
while True:
try:
with Listener(REPOD_BIND_ADDRESS, authkey=REPOD_BIND_PASSWD) as listener:
with listener.accept() as conn:
logger.info('connection accepted from %s', listener.last_accepted)
myrecv = conn.recv()
if type(myrecv) is list and len(myrecv) == 3:
(funcname, args, kwargs) = myrecv
funcname = str(funcname)
conn.send(run(funcname, args=args, kwargs=kwargs))
except Exception:
print_exc_plus()
except KeyboardInterrupt:
logger.info('KeyboardInterrupt')
print_exc_plus()
break