buildbot/utils.py

305 lines
11 KiB
Python

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import subprocess
import logging, logging.handlers
from time import time, sleep
import re
from threading import Thread, Lock
from pathlib import Path
import os
import sys
import traceback
from config import PKG_COMPRESSION, SHELL_ARCH_ARM64, SHELL_ARCH_X64, \
SHELL_ARM64_ADDITIONAL, SHELL_TRAP, \
CONTAINER_BUILDBOT_ROOT, ARCHS
logger = logging.getLogger(f'buildbot.{__name__}')
def background(func):
def wrapped(*args, **kwargs):
tr = Thread(target=func, args=args, kwargs=kwargs)
tr.daemon = True
tr.start()
return tr
return wrapped
def bash(cmdline, **kwargs):
assert type(cmdline) is str
logger.debug(f'bash: {cmdline}, kwargs: {kwargs}')
return(run_cmd(['/bin/bash', '-x', '-e', '-c', cmdline], **kwargs))
def mon_bash(cmdline, seconds=60*30, **kwargs):
assert type(seconds) is int and seconds >= 1
return bash(cmdline, keepalive=True, KEEPALIVE_TIMEOUT=60,
RUN_CMD_TIMEOUT=seconds, **kwargs)
def nspawn_shell(arch, cmdline, cwd=None, **kwargs):
root = Path(CONTAINER_BUILDBOT_ROOT)
if cwd:
cwd = root / cwd
else:
cwd = root
logger.debug(f'bash_{arch}: {cmdline}, cwd: {cwd}, kwargs: {kwargs}')
if arch in ('aarch64', 'arm64'):
command=f'{SHELL_ARM64_ADDITIONAL}; {SHELL_TRAP}; cd \'{cwd}\'; {cmdline}'
ret = run_cmd(SHELL_ARCH_ARM64 + [command,], **kwargs)
elif arch in ('x64', 'x86', 'x86_64'):
command=f'{SHELL_TRAP}; cd \'{cwd}\'; {cmdline}'
ret = run_cmd(SHELL_ARCH_X64 + [command,], **kwargs)
else:
raise TypeError('nspawn_shell: wrong arch')
if not ret.endswith('++ exit 0\n'):
raise subprocess.CalledProcessError(1, cmdline, ret)
return ret
def mon_nspawn_shell(arch, cmdline, cwd, seconds=60*30, **kwargs):
assert type(seconds) is int and seconds >= 1
return nspawn_shell(arch, cmdline, cwd=cwd, keepalive=True, KEEPALIVE_TIMEOUT=60,
RUN_CMD_TIMEOUT=seconds, **kwargs)
def run_cmd(cmd, cwd=None, keepalive=False, KEEPALIVE_TIMEOUT=30, RUN_CMD_TIMEOUT=60,
logfile=None, short_return=False):
logger.debug('run_cmd: %s', cmd)
RUN_CMD_LOOP_TIME = KEEPALIVE_TIMEOUT - 1 if KEEPALIVE_TIMEOUT >= 10 else 5
stopped = False
last_read = [int(time()), ""]
class Output(list):
def __init__(self, logfile=None, short_return=False):
super().__init__()
self.__short_return = short_return
if logfile:
assert issubclass(type(logfile), os.PathLike)
self.__file = open(logfile, 'w')
else:
self.__file = None
def append(self, mystring):
if self.__short_return and super().__len__() >= 20:
super().pop(0)
super().append(mystring)
if self.__file and type(mystring) is str:
self.__file.write(mystring)
self.__file.flush()
def __enter__(self):
return self
def __exit__(self, type, value, traceback):
if self.__file:
self.__file.close()
stdout_lock = Lock()
with Output(logfile=logfile, short_return=short_return) as output:
@background
def check_stdout(stdout):
nonlocal stopped, last_read, output
stdout_lock.acquire()
last_read_time = int(time())
while stopped is False:
line = stdout.readline(4096)
if line is '':
sleep(0.05)
continue
line.replace('\x0f', '\n')
last_read_time = int(time())
output.append(line)
last_read[0] = last_read_time
last_read[1] = line
stdout_lock.release()
p = subprocess.Popen(cmd, cwd=cwd, stdin=subprocess.PIPE, stdout=subprocess.PIPE,
stderr=subprocess.STDOUT, encoding='utf-8')
check_stdout(p.stdout)
process_start = int(time())
while True:
try:
p.wait(timeout=RUN_CMD_LOOP_TIME)
except subprocess.TimeoutExpired:
time_passed = int(time()) - last_read[0]
if time_passed >= KEEPALIVE_TIMEOUT*2:
logger.info('Timeout expired. No action.')
output.append('+ Buildbot: Timeout expired. No action.\n')
elif time_passed >= KEEPALIVE_TIMEOUT:
if keepalive:
logger.info('Timeout expired, writing nl')
output.append('+ Buildbot: Timeout expired, writing nl\n')
p.stdin.write('\n')
p.stdin.flush()
else:
logger.info('Timeout expired, not writing nl')
output.append('+ Buildbot: Timeout expired, not writing nl\n')
if int(time()) - process_start >= RUN_CMD_TIMEOUT:
stopped = True
logger.error('Process timeout expired, terminating.')
output.append('+ Buildbot: Process timeout expired, terminating.\n')
p.terminate()
try:
p.wait(timeout=10)
except subprocess.TimeoutExpired:
logger.error('Cannot terminate, killing.')
output.append('+ Buildbot: Cannot terminate, killing.\n')
p.kill()
break
else:
# sometimes the process ended too quickly and stdout is not captured
sleep(0.1)
stopped = True
break
code = p.returncode
stdout_lock.acquire(10)
outstr = ''.join(output)
if code != 0:
raise subprocess.CalledProcessError(code, cmd, outstr)
if logfile:
logger.debug('run_cmd: logfile written to %s', str(logfile))
else:
logger.debug('run_cmd: %s, ret = %s', cmd, outstr)
return outstr
# pyalpm is an alternative
# due to lack of documentation i'll consider this later.
def vercmp(ver1, ver2):
'''
compare ver1 and ver2, return 1, -1, 0
see https://www.archlinux.org/pacman/vercmp.8.html
'''
res = run_cmd(['vercmp', str(ver1), str(ver2)])
res = res.strip()
if res in ('-1', '0', '1'):
return int(res)
class Pkg:
def __init__(self, pkgname, pkgver, pkgrel, arch, fname):
self.pkgname = pkgname
self.pkgver = pkgver
self.pkgrel = pkgrel
self.arch = arch
self.fname = fname
self.ver = f'{self.pkgver}-{self.pkgrel}'
def __eq__(self, ver2):
if vercmp(self.ver, ver2.ver) == 0:
return True
else:
return False
def __ge__(self, ver2):
return self > ver2 or self == ver2
def __gt__(self, ver2):
if vercmp(self.ver, ver2.ver) == 1:
return True
else:
return False
def __le__(self, ver2):
return self < ver2 or self == ver2
def __lt__(self, ver2):
if vercmp(self.ver, ver2.ver) == -1:
return True
else:
return False
def __repr__(self):
return f'Pkg({self.pkgname}, {self.ver}, {self.arch})'
def get_pkg_details_from_name(name):
assert type(name) is str
if name.endswith(f'pkg.tar.{PKG_COMPRESSION}'):
m = re.match(r'(.+)-([^-]+)-([^-]+)-([^-]+)\.pkg\.tar\.\w+', name)
assert m and m.groups() and len(m.groups()) == 4
(pkgname, pkgver, pkgrel, arch) = m.groups()
return Pkg(pkgname, pkgver, pkgrel, arch, name)
def get_arch_from_pkgbuild(fpath):
assert issubclass(type(fpath), os.PathLike)
with open(fpath, 'r') as f:
for line in f.read().split('\n'):
if line.startswith('arch='):
matches = re.findall(r'[()\s\'\"]([\w]+)[()\s\'\"]', line)
if not matches:
raise TypeError('Unexpected PKGBUILD format')
matches = [arch for arch in matches if arch in ARCHS]
return matches
raise TypeError('Unexpected PKGBUILD')
def print_exc_plus():
logger.critical("Exception caught.\nPrinting stack traceback\n" + format_exc_plus())
def format_exc_plus():
"""
Print the usual traceback information, followed by a listing of all the
local variables in each frame.
from Python Cookbook by David Ascher, Alex Martelli
"""
ret = str()
tb = sys.exc_info()[2]
while True:
if not tb.tb_next:
break
tb = tb.tb_next
stack = []
f = tb.tb_frame
while f:
stack.append(f)
f = f.f_back
stack.reverse()
ret += traceback.format_exc()
ret += "\nLocals by frame, innermost last\n"
for frame in stack:
ret += "Frame %s in %s at line %s\n" % (frame.f_code.co_name,
frame.f_code.co_filename,
frame.f_lineno)
for key, value in frame.f_locals.items( ):
ret += "\t%20s = " % key
# We have to be VERY careful not to cause a new error in our error
# printer! Calling str( ) on an unknown object could cause an
# error we don't want, so we must use try/except to catch it --
# we can't stop it from happening, but we can and should
# stop it from propagating if it does happen!
try:
ret += str(value)
except:
ret += "<ERROR WHILE PRINTING VALUE>"
ret += '\n'
return ret
def configure_logger(logger, format='%(asctime)s - %(name)-18s - %(levelname)s - %(message)s',
level=logging.INFO, logfile=None, flevel=logging.DEBUG, rotate_size=None,
enable_notify=False, consolelog=None):
class NotifyHandler(logging.NullHandler):
def handle(self, record):
send(self.formatter.format(record))
logger.setLevel(logging.DEBUG)
formatter = logging.Formatter(fmt=format)
# create file handler
if logfile:
assert type(logfile) is str
if rotate_size and type(rotate_size) is int and rotate_size >= 1000:
fh = logging.handlers.RotatingFileHandler(logfile, mode='a',
maxBytes=rotate_size, backupCount=8)
else:
fh = logging.FileHandler(logfile)
fh.setLevel(flevel)
fh.setFormatter(formatter)
logger.addHandler(fh)
# create console handler
ch = logging.StreamHandler()
ch.setLevel(level)
ch.setFormatter(formatter)
logger.addHandler(ch)
# for client.printlog
if consolelog:
assert type(consolelog) is str
cfh = logging.FileHandler(consolelog)
cfh.setLevel(level)
cfh.setFormatter(formatter)
logger.addHandler(cfh)
# notify
if enable_notify:
try:
from notify import send
except ModuleNotFoundError:
print('Failed to import notify.send')
else:
nh = NotifyHandler()
nh.setLevel(level)
nh.setFormatter(formatter)
logger.addHandler(nh)