buildbot.py: fix bugs and rewrite client

buildbot.py

@@ -109,7 +109,11 @@ class jobsManager:
             else:
                 return False
         return True
+    def force_upload_package(self, pkgdirname, overwrite=True):
+        pass
     def rebuild_package(self, pkgdirname, clean=False):
+        if not self.idle:
+            logger.debug('rebuild requested and not idle.')
         self.pkgconfigs = load_all_yaml()
         if (REPO_ROOT / pkgdirname).exists() and clean:
             self.reset_dir(pkgdirname)
@@ -263,6 +267,8 @@ class jobsManager:
                 except Exception:
                     time_to_sleep = (tries + 1) * 60
                     logger.error(f'We are getting problem uploading {f}, wait {time_to_sleep} secs')
+                    if not rrun('push_fail', args=(f.name,)):
+                        logger.error('unable to run push_fail')
                     print_exc_plus()
                     if tries + 1 < max_tries:
                         sleep(time_to_sleep)
@@ -272,6 +278,13 @@ class jobsManager:
                 logger.error(f'Upload {f} failed, abort.')
                 raise RuntimeError('Unable to upload some files')
         return suc
+    def getup(self):
+        '''
+            check for updates now !!!
+        '''
+        logger.info('Check for updates now.')
+        self.last_updatecheck = 0.0
+        return "buildbot wakes up"
     def tick(self):
         '''
             check for updates,
@@ -450,8 +463,16 @@ def clean(pkgdirname):
 def clean_all():
     return jobsmgr.reset_dir(all=True)
 
+def force_upload(pkgdirname):
+    #jobsmgr.force_upload_package(pkgdirname)
+    return "not implemented"
+
+def getup():
+    return jobsmgr.getup()
+
 def run(funcname, args=list(), kwargs=dict()):
-    if funcname in ('info', 'rebuild_package', 'clean', 'clean_all'):
+    if funcname in ('info', 'rebuild_package', 'clean', 'clean_all',
+                    'force_upload', 'getup'):
         logger.debug('running: %s %s %s',funcname, args, kwargs)
         ret = eval(funcname)(*args, **kwargs)
         logger.info('done: %s %s %s',funcname, args, kwargs)
@@ -466,7 +487,7 @@ def __main():
         try:
             with Listener(MASTER_BIND_ADDRESS, authkey=MASTER_BIND_PASSWD) as listener:
                 with listener.accept() as conn:
-                    logger.info('connection accepted from %s', listener.last_accepted)
+                    logger.debug('connection accepted from %s', listener.last_accepted)
                     myrecv = conn.recv()
                     if type(myrecv) is list and len(myrecv) == 3:
                         (funcname, args, kwargs) = myrecv

client.py

@@ -71,30 +71,70 @@ if __name__ == '__main__':
                 lines = list()
                 sleep(1)
     try:
-        parser = argparse.ArgumentParser(description='Client for buildbot')
+        actions = {
-        parser.add_argument('--info', action='store_true', help='show buildbot info')
+                    'info':     'show buildbot info',
-        parser.add_argument('--update', action='store_true', help='update pushed files to the repo')
+                    'update':   '[--overwrite] update pushed files to the repo',
-        parser.add_argument('--cleanall', action='store_true', help='checkout pkgbuilds')
+                    'clean':    '[dir / all]   checkout pkgbuilds in packages',
-        parser.add_argument('--clean', nargs='?', default=None, help='checkout pkgbuilds in one package')
+                    'rebuild':  '[dir1 dir2 --clean]   rebuild packages',
-        parser.add_argument('--rebuild', nargs='?', default=None, help='rebuild a package with its dirname')
+                    'log':      '[--debug]     print log',
-        parser.add_argument('--log', action='store_true' , help='print log')
+                    'upload':   '[dir1 dir2]   force upload packages',
+                    'getup':    'check for updates now'
+                  }
+        parser = argparse.ArgumentParser(description='Client for buildbot',
+                                        formatter_class=argparse.RawTextHelpFormatter)
+        __action_help = "\n".join([f"{a}:\t{actions[a]}" for a in actions])
+        parser.add_argument('action', nargs='*', help=f'Choose which action to invoke:\n\n{__action_help}')
+        parser.add_argument('--overwrite', nargs='?', default='False', help='overwrite existing files')
+        parser.add_argument('--debug', nargs='?', default='False', help='print debug logs')
+        parser.add_argument('--clean', nargs='?', default='True', help='clean build packages')
         args = parser.parse_args()
-        if args.info:
+        action = args.action
+        for switch in ('overwrite', 'debug', 'clean'):
+            s = getattr(args, switch)
+            if str(s).lower() in ('false', 'no', 'n', '0'):
+                setattr(args, switch, False)
+            else:
+                setattr(args, switch, True)
+        if not (action and len(action) >= 1 and action[0] in actions):
+            parser.print_help()
+            parser.exit(status=1)
+        if action[0] == 'info':
             server=(MASTER_BIND_ADDRESS, MASTER_BIND_PASSWD)
             logger.info(run('info', server=server))
-        elif args.update:
+        elif action[0] == 'getup':
+            server=(MASTER_BIND_ADDRESS, MASTER_BIND_PASSWD)
+            logger.info(run('getup', server=server))
+        elif action[0] == 'update':
             server=(REPOD_BIND_ADDRESS, REPOD_BIND_PASSWD)
             logger.info(run('update', kwargs={'overwrite': False}, server=server))
-        elif args.cleanall:
+        elif action[0] == 'clean':
+            if len(action) <= 1:
+                print('Error: Need package name')
+                parser.print_help()
+                parser.exit(status=1)
             server=(MASTER_BIND_ADDRESS, MASTER_BIND_PASSWD)
+            if 'all' in action[1:]:
                 logger.info(run('clean_all', server=server))
-        elif args.clean:
+            else:
+                for p in action[1:]:
+                    logger.info(run('clean', args=(p,), server=server))
+        elif action[0] == 'rebuild':
+            if len(action) <= 1:
+                print('Error: Need package name')
+                parser.print_help()
+                parser.exit(status=1)
             server=(MASTER_BIND_ADDRESS, MASTER_BIND_PASSWD)
-            logger.info(run('clean', args=(args.clean,), server=server))
+            for p in action[1:]:
-        elif args.rebuild:
+                logger.info(run('rebuild_package', args=(p,), kwargs={'clean': args.clean}, server=server))
+        elif action[0] == 'upload':
+            if len(action) <= 1:
+                print('Error: Need package name')
+                parser.print_help()
+                parser.exit(status=1)
             server=(MASTER_BIND_ADDRESS, MASTER_BIND_PASSWD)
-            logger.info(run('rebuild_package', args=(args.rebuild,), kwargs={'clean': True}, server=server))
+            for p in action[1:]:
-        elif args.log:
+                logger.info(run('force_upload', args=(p,), server=server))
+        elif action[0] == 'log':
             logger.info('printing logs')
             print_log()
         else:

repod.py

@@ -68,6 +68,21 @@ class pushFm:
                 return None
         else:
             return None
+    def fail(self, fname):
+        update_path = Path('updates')
+        if fname == self.fname:
+            pkg = update_path / self.fname
+            sig = update_path / f'{self.fname}.sig'
+            for f in (pkg, sig):
+                if f.exists():
+                    try:
+                        f.unlink()
+                    except Exception:
+                        logger.warning(f'unable to remove {f.name}')
+            self.__init__()
+            return None
+        else:
+            return "Wrong file"
     def done(self, fname, overwrite=False):
         '''
             return None means success
@@ -117,13 +132,15 @@ def push_start(filename, size):
 def push_done(filename, overwrite=False):
     return pfm.done(filename, overwrite=overwrite)
 
-
+def push_fail(filename):
+    return pfm.fail(filename)
 
 # server part
 
 def run(funcname, args=list(), kwargs=dict()):
     if funcname in ('clean', 'regenerate', 'remove',
-                    'update', 'push_start', 'push_done'):
+                    'update', 'push_start', 'push_done',
+                    'push_fail'):
         logger.info('running: %s %s %s', funcname, args, kwargs)
         ret = eval(funcname)(*args, **kwargs)
         logger.info('done: %s %s',funcname, ret)

utils.py

@@ -260,23 +260,36 @@ def format_exc_plus():
     return ret
 
 def configure_logger(logger, format='%(asctime)s - %(name)-18s - %(levelname)s - %(message)s',
-                     level=logging.INFO, logfile=None, flevel=logging.DEBUG, rotate_size=None):
+                     level=logging.INFO, logfile=None, flevel=logging.DEBUG, rotate_size=None,
+                     enable_notify=False):
+    def __send(*args):
+        pass
+    if enable_notify:
         try:
             from notify import send
         except ModuleNotFoundError:
-        def send(*args):
+            send = __send
-            pass
+    else:
+        send = __send
 
     class ExceptionFormatter(logging.Formatter):
+        def __init__(self, *args, notify=False, **kwargs):
+            self.notify = notify
+            super().__init__(*args, **kwargs)
         def format(self, record):
             if record.levelno == 49:
                 record.msg = 'Exception caught.\nPrinting stack traceback\n' + record.msg
             fmtr = super().format(record)
+            if self.notify:
                 send(fmtr)
             return fmtr
 
     logger.setLevel(logging.DEBUG)
-    formatter = ExceptionFormatter(fmt=format)
+    fnotify = cnotify = False
+    fnotify = True if enable_notify and logfile else False
+    cnotify = True if enable_notify and (not logfile) else False
+    fformatter = ExceptionFormatter(fmt=format, notify=fnotify)
+    cformatter = ExceptionFormatter(fmt=format, notify=cnotify)
     logging.addLevelName(49, 'Exception')
     # create file handler
     if logfile:
@@ -287,10 +300,10 @@ def configure_logger(logger, format='%(asctime)s - %(name)-18s - %(levelname)s -
         else:
             fh = logging.FileHandler(logfile)
         fh.setLevel(flevel)
-        fh.setFormatter(formatter)
+        fh.setFormatter(fformatter)
         logger.addHandler(fh)
     # create console handler
     ch = logging.StreamHandler()
     ch.setLevel(level)
-    ch.setFormatter(formatter)
+    ch.setFormatter(cformatter)
     logger.addHandler(ch)

yamlparse.py

@@ -73,7 +73,7 @@ def load_all():
     pkgconfigs = list()
     for mydir in REPO_ROOT.iterdir():
         try:
-            if mydir.is_dir():
+            if mydir.is_dir() and (not mydir.name.startswith('.')):
                 if (mydir / AUTOBUILD_FNAME).exists():
                     # parsing yaml
                     logger.info('Bulidbot: found %s in %s', AUTOBUILD_FNAME, mydir)