Archlinux-Jerry
/
buildbot
mirror of https://github.com/archlinux-jerry/buildbot
1
0
Fork 0
Code Issues Releases Wiki Activity

buildbot.py: fix bugs and rewrite client

This commit is contained in:
JerryXiao 2019-09-06 11:23:07 +08:00
parent 61bbd523f4
commit e68c227ec5
Signed by: Jerry
GPG Key ID: 9D9CE43650FF2BAA
5 changed files with 122 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
buildbot.py
View File

@ -109,7 +109,11 @@ class jobsManager:
else:
return False
return True
def force_upload_package(self, pkgdirname, overwrite=True):
pass
def rebuild_package(self, pkgdirname, clean=False):
if not self.idle:
logger.debug('rebuild requested and not idle.')
self.pkgconfigs = load_all_yaml()
if (REPO_ROOT / pkgdirname).exists() and clean:
self.reset_dir(pkgdirname)
@ -263,6 +267,8 @@ class jobsManager:
except Exception:
time_to_sleep = (tries + 1) * 60
logger.error(f'We are getting problem uploading {f}, wait {time_to_sleep} secs')
if not rrun('push_fail', args=(f.name,)):
logger.error('unable to run push_fail')
print_exc_plus()
if tries + 1 < max_tries:
sleep(time_to_sleep)
@ -272,6 +278,13 @@ class jobsManager:
logger.error(f'Upload {f} failed, abort.')
raise RuntimeError('Unable to upload some files')
return suc
def getup(self):
'''
check for updates now !!!
'''
logger.info('Check for updates now.')
self.last_updatecheck = 0.0
return "buildbot wakes up"
def tick(self):
'''
check for updates,
@ -450,8 +463,16 @@ def clean(pkgdirname):
def clean_all():
return jobsmgr.reset_dir(all=True)
def force_upload(pkgdirname):
#jobsmgr.force_upload_package(pkgdirname)
return "not implemented"
def getup():
return jobsmgr.getup()
def run(funcname, args=list(), kwargs=dict()):
if funcname in ('info', 'rebuild_package', 'clean', 'clean_all'):
if funcname in ('info', 'rebuild_package', 'clean', 'clean_all',
'force_upload', 'getup'):
logger.debug('running: %s %s %s',funcname, args, kwargs)
ret = eval(funcname)(*args, **kwargs)
logger.info('done: %s %s %s',funcname, args, kwargs)
@ -466,7 +487,7 @@ def __main():
try:
with Listener(MASTER_BIND_ADDRESS, authkey=MASTER_BIND_PASSWD) as listener:
with listener.accept() as conn:
logger.info('connection accepted from %s', listener.last_accepted)
logger.debug('connection accepted from %s', listener.last_accepted)
myrecv = conn.recv()
if type(myrecv) is list and len(myrecv) == 3:
(funcname, args, kwargs) = myrecv

72
client.py
View File

@ -71,30 +71,70 @@ if __name__ == '__main__':
lines = list()
sleep(1)
try:
parser = argparse.ArgumentParser(description='Client for buildbot')
parser.add_argument('--info', action='store_true', help='show buildbot info')
parser.add_argument('--update', action='store_true', help='update pushed files to the repo')
parser.add_argument('--cleanall', action='store_true', help='checkout pkgbuilds')
parser.add_argument('--clean', nargs='?', default=None, help='checkout pkgbuilds in one package')
parser.add_argument('--rebuild', nargs='?', default=None, help='rebuild a package with its dirname')
parser.add_argument('--log', action='store_true' , help='print log')
actions = {
'info': 'show buildbot info',
'update': '[--overwrite] update pushed files to the repo',
'clean': '[dir / all] checkout pkgbuilds in packages',
'rebuild': '[dir1 dir2 --clean] rebuild packages',
'log': '[--debug] print log',
'upload': '[dir1 dir2] force upload packages',
'getup': 'check for updates now'
}
parser = argparse.ArgumentParser(description='Client for buildbot',
formatter_class=argparse.RawTextHelpFormatter)
__action_help = "\n".join([f"{a}:\t{actions[a]}" for a in actions])
parser.add_argument('action', nargs='*', help=f'Choose which action to invoke:\n\n{__action_help}')
parser.add_argument('--overwrite', nargs='?', default='False', help='overwrite existing files')
parser.add_argument('--debug', nargs='?', default='False', help='print debug logs')
parser.add_argument('--clean', nargs='?', default='True', help='clean build packages')
args = parser.parse_args()
if args.info:
action = args.action
for switch in ('overwrite', 'debug', 'clean'):
s = getattr(args, switch)
if str(s).lower() in ('false', 'no', 'n', '0'):
setattr(args, switch, False)
else:
setattr(args, switch, True)
if not (action and len(action) >= 1 and action[0] in actions):
parser.print_help()
parser.exit(status=1)
if action[0] == 'info':
server=(MASTER_BIND_ADDRESS, MASTER_BIND_PASSWD)
logger.info(run('info', server=server))
elif args.update:
elif action[0] == 'getup':
server=(MASTER_BIND_ADDRESS, MASTER_BIND_PASSWD)
logger.info(run('getup', server=server))
elif action[0] == 'update':
server=(REPOD_BIND_ADDRESS, REPOD_BIND_PASSWD)
logger.info(run('update', kwargs={'overwrite': False}, server=server))
elif args.cleanall:
elif action[0] == 'clean':
if len(action) <= 1:
print('Error: Need package name')
parser.print_help()
parser.exit(status=1)
server=(MASTER_BIND_ADDRESS, MASTER_BIND_PASSWD)
logger.info(run('clean_all', server=server))
elif args.clean:
if 'all' in action[1:]:
logger.info(run('clean_all', server=server))
else:
for p in action[1:]:
logger.info(run('clean', args=(p,), server=server))
elif action[0] == 'rebuild':
if len(action) <= 1:
print('Error: Need package name')
parser.print_help()
parser.exit(status=1)
server=(MASTER_BIND_ADDRESS, MASTER_BIND_PASSWD)
logger.info(run('clean', args=(args.clean,), server=server))
elif args.rebuild:
for p in action[1:]:
logger.info(run('rebuild_package', args=(p,), kwargs={'clean': args.clean}, server=server))
elif action[0] == 'upload':
if len(action) <= 1:
print('Error: Need package name')
parser.print_help()
parser.exit(status=1)
server=(MASTER_BIND_ADDRESS, MASTER_BIND_PASSWD)
logger.info(run('rebuild_package', args=(args.rebuild,), kwargs={'clean': True}, server=server))
elif args.log:
for p in action[1:]:
logger.info(run('force_upload', args=(p,), server=server))
elif action[0] == 'log':
logger.info('printing logs')
print_log()
else:

21
repod.py
View File

@ -68,6 +68,21 @@ class pushFm:
return None
else:
return None
def fail(self, fname):
update_path = Path('updates')
if fname == self.fname:
pkg = update_path / self.fname
sig = update_path / f'{self.fname}.sig'
for f in (pkg, sig):
if f.exists():
try:
f.unlink()
except Exception:
logger.warning(f'unable to remove {f.name}')
self.__init__()
return None
else:
return "Wrong file"
def done(self, fname, overwrite=False):
'''
return None means success
@ -117,13 +132,15 @@ def push_start(filename, size):
def push_done(filename, overwrite=False):
return pfm.done(filename, overwrite=overwrite)
def push_fail(filename):
return pfm.fail(filename)
# server part
def run(funcname, args=list(), kwargs=dict()):
if funcname in ('clean', 'regenerate', 'remove',
'update', 'push_start', 'push_done'):
'update', 'push_start', 'push_done',
'push_fail'):
logger.info('running: %s %s %s', funcname, args, kwargs)
ret = eval(funcname)(*args, **kwargs)
logger.info('done: %s %s',funcname, ret)

33
utils.py
View File

@ -260,23 +260,36 @@ def format_exc_plus():
return ret
def configure_logger(logger, format='%(asctime)s - %(name)-18s - %(levelname)s - %(message)s',
level=logging.INFO, logfile=None, flevel=logging.DEBUG, rotate_size=None):
try:
from notify import send
except ModuleNotFoundError:
def send(*args):
pass
level=logging.INFO, logfile=None, flevel=logging.DEBUG, rotate_size=None,
enable_notify=False):
def __send(*args):
pass
if enable_notify:
try:
from notify import send
except ModuleNotFoundError:
send = __send
else:
send = __send
class ExceptionFormatter(logging.Formatter):
def __init__(self, *args, notify=False, **kwargs):
self.notify = notify
super().__init__(*args, **kwargs)
def format(self, record):
if record.levelno == 49:
record.msg = 'Exception caught.\nPrinting stack traceback\n' + record.msg
fmtr = super().format(record)
send(fmtr)
if self.notify:
send(fmtr)
return fmtr
logger.setLevel(logging.DEBUG)
formatter = ExceptionFormatter(fmt=format)
fnotify = cnotify = False
fnotify = True if enable_notify and logfile else False
cnotify = True if enable_notify and (not logfile) else False
fformatter = ExceptionFormatter(fmt=format, notify=fnotify)
cformatter = ExceptionFormatter(fmt=format, notify=cnotify)
logging.addLevelName(49, 'Exception')
# create file handler
if logfile:
@ -287,10 +300,10 @@ def configure_logger(logger, format='%(asctime)s - %(name)-18s - %(levelname)s -
else:
fh = logging.FileHandler(logfile)
fh.setLevel(flevel)
fh.setFormatter(formatter)
fh.setFormatter(fformatter)
logger.addHandler(fh)
# create console handler
ch = logging.StreamHandler()
ch.setLevel(level)
ch.setFormatter(formatter)
ch.setFormatter(cformatter)
logger.addHandler(ch)

2
yamlparse.py
View File

@ -73,7 +73,7 @@ def load_all():
pkgconfigs = list()
for mydir in REPO_ROOT.iterdir():
try:
if mydir.is_dir():
if mydir.is_dir() and (not mydir.name.startswith('.')):
if (mydir / AUTOBUILD_FNAME).exists():
# parsing yaml
logger.info('Bulidbot: found %s in %s', AUTOBUILD_FNAME, mydir)