From 7ff38e71fcd1a63ce4e4c6080e3c1f52c696e946 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Mon, 25 Apr 2022 20:15:59 +0300 Subject: only notify with last 1000 lines of log Sponsored-by: author --- ick2/actions.py | 318 +++++++++++++++++++++++++++----------------------------- 1 file changed, 153 insertions(+), 165 deletions(-) diff --git a/ick2/actions.py b/ick2/actions.py index c901d6b..4281e24 100644 --- a/ick2/actions.py +++ b/ick2/actions.py @@ -33,9 +33,9 @@ class UnknownStepError(Exception): class ActionFactory: _classes = { - 'host': ick2.HostEnvironment, - 'chroot': ick2.ChrootEnvironment, - 'container': ick2.ContainerEnvironment, + "host": ick2.HostEnvironment, + "chroot": ick2.ChrootEnvironment, + "container": ick2.ContainerEnvironment, } def __init__(self, build_id, systree, workspace_area, reporter): @@ -75,7 +75,7 @@ class ActionFactory: return area def create_environment(self, spec, project_name): - env = spec.get('where', 'host') + env = spec.get("where", "host") assert env in self.get_allowed_environments() env_class = self._classes[env] area = self.get_workspace_area() @@ -95,46 +95,45 @@ class ActionFactory: def _create_action_object(self, env, spec): rules = [ - ('shell', ShellAction), - ('python', PythonAction), - ('debootstrap', DebootstrapAction), + ("shell", ShellAction), + ("python", PythonAction), + ("debootstrap", DebootstrapAction), ] for key, klass in rules: if key in spec: return klass(env) - if 'archive' in spec: + if "archive" in spec: rules2 = { - 'workspace': ArchiveWorkspaceAction, - 'systree': ArchiveSystreeAction, + "workspace": ArchiveWorkspaceAction, + "systree": ArchiveSystreeAction, } - kind = spec['archive'] + kind = spec["archive"] klass = rules2.get(kind) if klass: return klass(env) - if 'action' in spec: + if "action" in spec: rules2 = { - 'populate_systree': PopulateSystreeAction, - 'populate_workspace': PopulateWorkspaceAction, - 'create_workspace': CreateWorkspaceAction, - 'git': GitAction, - 'git_mirror': GitMirrorAction, - 'rsync': RsyncAction, - 'dput': DputAction, - 'notify': NotifyAction, + "populate_systree": PopulateSystreeAction, + "populate_workspace": PopulateWorkspaceAction, + "create_workspace": CreateWorkspaceAction, + "git": GitAction, + "git_mirror": GitMirrorAction, + "rsync": RsyncAction, + "dput": DputAction, + "notify": NotifyAction, } - kind = spec['action'] + kind = spec["action"] klass = rules2.get(kind) if klass: return klass(env) - raise UnknownStepError('Unknown action %r' % spec) + raise UnknownStepError("Unknown action %r" % spec) class Action: # pragma: no cover - def __init__(self, env): self._env = env self._cc = None @@ -181,20 +180,20 @@ class Action: # pragma: no cover def encode64(self, params): assert isinstance(params, dict) as_text = json.dumps(params) - as_bytes = as_text.encode('UTF-8') + as_bytes = as_text.encode("UTF-8") as_base64 = base64.b64encode(as_bytes) - return as_base64.decode('UTF-8') + return as_base64.decode("UTF-8") def decode64(self, encoded): - as_base64 = encoded.encode('UTF-8') + as_base64 = encoded.encode("UTF-8") as_bytes = base64.b64decode(as_base64) - as_text = as_bytes.decode('UTF-8') + as_text = as_bytes.decode("UTF-8") return json.loads(as_text) def get_authz_headers(self): token = self.get_token() return { - 'Authorization': 'Bearer {}'.format(token), + "Authorization": "Bearer {}".format(token), } def execute(self, params, step): @@ -202,79 +201,75 @@ class Action: # pragma: no cover class ShellAction(Action): - def encode_parameters(self, params): # pragma: no cover encoded = self.encode64(params) return 'params() { echo -n "%s" | base64 -d; }\n' % encoded def execute(self, params, step): prefix = self.encode_parameters(params) - snippet = step['shell'] - argv = ['bash', '-exuc', prefix + snippet] + snippet = step["shell"] + argv = ["bash", "-exuc", prefix + snippet] exit_code = self._env.runcmd(argv) - self._env.report(exit_code, 'action finished\n') + self._env.report(exit_code, "action finished\n") return exit_code class PythonAction(Action): - def encode_parameters(self, params): # pragma: no cover encoded = self.encode64(params) prefix = ( - 'import base64, json, subprocess\n' - 'params = json.loads(base64.b64decode(\n' + "import base64, json, subprocess\n" + "params = json.loads(base64.b64decode(\n" ' "{}").decode("utf8"))\n' - 'def RUN(*args, **kwargs):\n' + "def RUN(*args, **kwargs):\n" ' print("Executing:", args, kwargs)\n' ' if "check" not in kwargs:\n' ' kwargs["check"] = True\n' - ' return subprocess.run(args, **kwargs)\n' - 'def OUT(*args, **kwargs):\n' - ' x = RUN(*args, stdout=subprocess.PIPE, **kwargs)\n' + " return subprocess.run(args, **kwargs)\n" + "def OUT(*args, **kwargs):\n" + " x = RUN(*args, stdout=subprocess.PIPE, **kwargs)\n" ' return x.stdout.decode("UTF-8")\n' - 'def ERR(*args, **kwargs):\n' - ' x = RUN(*args, stderr=subprocess.PIPE, check=False, **kwargs)\n' + "def ERR(*args, **kwargs):\n" + " x = RUN(*args, stderr=subprocess.PIPE, check=False, **kwargs)\n" ' return x.stderr.decode("UTF-8")\n' - 'def OUTERR(*args, **kwargs):\n' - ' x = RUN(*args, stdout=subprocess.PIPE, \n' - ' sterr=subproces.STDOUT, check=False, **kwargs)\n' + "def OUTERR(*args, **kwargs):\n" + " x = RUN(*args, stdout=subprocess.PIPE, \n" + " sterr=subproces.STDOUT, check=False, **kwargs)\n" ' return s.stdout.decode("UTF-8"), x.stderr.decode("UTF-8")\n' - ).format(encoded) return prefix def execute(self, params, step): prefix = self.encode_parameters(params) - snippet = step['python'] - argv = ['python3', '-c', prefix + '\n' + snippet] + snippet = step["python"] + argv = ["python3", "-c", prefix + "\n" + snippet] exit_code = self._env.runcmd(argv) - self._env.report(exit_code, 'action finished\n') + self._env.report(exit_code, "action finished\n") return exit_code class DebootstrapAction(Action): - default_mirror = 'http://deb.debian.org/debian' + default_mirror = "http://deb.debian.org/debian" def encode_parameters(self, params): # pragma: no cover pass def execute(self, params, step): - suite = step.get('debootstrap') - if suite is None or suite == 'auto': - suite = params['debian_codename'] - mirror = step.get('mirror', self.default_mirror) + suite = step.get("debootstrap") + if suite is None or suite == "auto": + suite = params["debian_codename"] + mirror = step.get("mirror", self.default_mirror) env = self.get_env() workspace = env.get_workspace_directory() - argv = ['sudo', 'debootstrap', suite, '.', mirror] + argv = ["sudo", "debootstrap", suite, ".", mirror] exit_code = self._env.host_runcmd(argv, cwd=workspace) - self._env.report(exit_code, 'action finished\n') + self._env.report(exit_code, "action finished\n") return exit_code class CreateWorkspaceAction(Action): - def encode_parameters(self, params): # pragma: no cover pass @@ -282,12 +277,11 @@ class CreateWorkspaceAction(Action): env = self.get_env() dirname = env.get_workspace_directory() make_directory_empty(env, dirname) - self._env.report(0, 'Created or emptied workspace %s\n' % dirname) + self._env.report(0, "Created or emptied workspace %s\n" % dirname) return 0 class ArchiveBaseAction(Action): # pragma: no cover - def get_dirname(self, env): raise NotImplementedError() @@ -298,42 +292,42 @@ class ArchiveBaseAction(Action): # pragma: no cover env = self.get_env() dirname = self.get_dirname(env) - name_from = step.get('name_from', 'artifact_name') + name_from = step.get("name_from", "artifact_name") blob_name = params.get(name_from) if not blob_name: - env.report(1, 'No artifact_name parameter\n') + env.report(1, "No artifact_name parameter\n") return 1 - env.report(None, 'Creating new artifact named {}\n'.format(blob_name)) - env.report(None, 'Artifact will be created from {}\n'.format(dirname)) + env.report(None, "Creating new artifact named {}\n".format(blob_name)) + env.report(None, "Artifact will be created from {}\n".format(dirname)) - globs = step.get('globs') + globs = step.get("globs") if globs is None: - names = ['.'] + names = ["."] else: names = self.match_globs(dirname, globs) url = self.get_blob_upload_url(blob_name) headers = self.get_authz_headers() - self._env.report(None, 'Creating tarball\n') + self._env.report(None, "Creating tarball\n") fd, tarball = tempfile.mkstemp() os.close(fd) - tar = ['sudo', 'tar', '-zvcf', tarball, '-C', dirname] + names + tar = ["sudo", "tar", "-zvcf", tarball, "-C", dirname] + names exit_code = self._env.host_runcmd(tar) if exit_code != 0: - self._env.report(exit_code, 'Tarball generation failed\n') + self._env.report(exit_code, "Tarball generation failed\n") os.remove(tarball) return exit_code - self._env.report(None, 'Tarball generation finished OK\n') - - self._env.report(None, 'Uploading tarball to artifact store\n') - curl = ['curl', '-sk', '-T', tarball] + [ - '-H{}:{}'.format(name, value) - for name, value in headers.items() - ] + [url] + self._env.report(None, "Tarball generation finished OK\n") + + self._env.report(None, "Uploading tarball to artifact store\n") + curl = ( + ["curl", "-sk", "-T", tarball] + + ["-H{}:{}".format(name, value) for name, value in headers.items()] + + [url] + ) exit_code = self._env.host_runcmd(curl) - self._env.report( - exit_code, 'curl upload finished (exit code %s)\n' % exit_code) + self._env.report(exit_code, "curl upload finished (exit code %s)\n" % exit_code) os.remove(tarball) return exit_code @@ -341,20 +335,18 @@ class ArchiveBaseAction(Action): # pragma: no cover def match_globs(self, workspace, globs): names = [] for pat in globs: - abspat = os.path.join(workspace, './' + pat) + abspat = os.path.join(workspace, "./" + pat) for name in glob.glob(abspat): names.append(os.path.normpath(name)) return names class ArchiveSystreeAction(ArchiveBaseAction): # pragma: no cover - def get_dirname(self, env): return env.get_systree_directory() class ArchiveWorkspaceAction(ArchiveBaseAction): # pragma: no cover - def get_dirname(self, env): return env.get_workspace_directory() @@ -371,24 +363,26 @@ class PopulateActionBase(Action): # pragma: no cover env = self.get_env() name = step.get(self.step_field) - if not name or name == 'auto': - name_name = step.get('name_from', self.param_name) + if not name or name == "auto": + name_name = step.get("name_from", self.param_name) name = params.get(name_name) if not name: - msg = '{} in action is {}, but no {} parameter\n'.format( - self.step_field, name, name_name) + msg = "{} in action is {}, but no {} parameter\n".format( + self.step_field, name, name_name + ) env.report(1, msg) return 1 - env.report(None, 'Using {} for artifact name\n'.format(name)) + env.report(None, "Using {} for artifact name\n".format(name)) dirname = self.get_unpack_directory(env) make_directory_empty(env, dirname) exit_code = self.download_and_unpack_artifact(name, dirname) new_code = self.mangle_exit_code(exit_code) env.report( - new_code, '{} finished (exit_code {} -> {})\n'.format( - str(self), exit_code, new_code)) + new_code, + "{} finished (exit_code {} -> {})\n".format(str(self), exit_code, new_code), + ) return new_code def get_unpack_directory(self, env): @@ -400,12 +394,13 @@ class PopulateActionBase(Action): # pragma: no cover def download_and_unpack_artifact(self, name, dirname): url = self.get_blob_upload_url(name) headers = self.get_authz_headers() - curl = ['curl', '-sk'] + [ - '-H{}:{}'.format(name, value) - for name, value in headers.items() - ] + [url] + curl = ( + ["curl", "-sk"] + + ["-H{}:{}".format(name, value) for name, value in headers.items()] + + [url] + ) - untar = ['sudo', 'tar', '-zxf', '-', '-C', dirname] + untar = ["sudo", "tar", "-zxf", "-", "-C", dirname] env = self.get_env() return env.host_runcmd(curl, untar) @@ -413,11 +408,11 @@ class PopulateActionBase(Action): # pragma: no cover class PopulateSystreeAction(PopulateActionBase): # pragma: no cover - step_field = 'systree_name' - param_name = 'systree_name' + step_field = "systree_name" + param_name = "systree_name" def __str__(self): - return 'populate-systree' + return "populate-systree" def get_unpack_directory(self, env): return env.get_systree_directory() @@ -428,11 +423,11 @@ class PopulateSystreeAction(PopulateActionBase): # pragma: no cover class PopulateWorkspaceAction(PopulateActionBase): # pragma: no cover - step_field = 'workspace_name' - param_name = 'workspace_name' + step_field = "workspace_name" + param_name = "workspace_name" def __str__(self): - return 'populate-workspace' + return "populate-workspace" def get_unpack_directory(self, env): return env.get_workspace_directory() @@ -443,7 +438,6 @@ class PopulateWorkspaceAction(PopulateActionBase): # pragma: no cover class GitAction(Action): # pragma: no cover - def encode_parameters(self, params): pass @@ -451,44 +445,43 @@ class GitAction(Action): # pragma: no cover env = self.get_env() workspace = env.get_workspace_directory() - git_dir = params.get('git_dir') + git_dir = params.get("git_dir") if git_dir is None: - env.report(1, 'git_dir not provided\n') - if git_dir.startswith('/') or '..' in git_dir: - env.report(1, 'git_dir not acceptable\n') + env.report(1, "git_dir not provided\n") + if git_dir.startswith("/") or ".." in git_dir: + env.report(1, "git_dir not acceptable\n") - git_url = params.get('git_url') + git_url = params.get("git_url") if git_url is None: - env.report(1, 'git_url not provided\n') + env.report(1, "git_url not provided\n") pathname = os.path.join(workspace, git_dir) if os.path.exists(pathname): - argv = ['git', 'remote', '-v', 'update', '--prune'] + argv = ["git", "remote", "-v", "update", "--prune"] cwd = pathname else: - argv = ['git', 'clone', '-v', git_url, git_dir] + argv = ["git", "clone", "-v", git_url, git_dir] cwd = workspace exit_code = env.host_runcmd(argv, cwd=cwd) - env.report(exit_code, 'git finished (exit code %d)\n' % exit_code) + env.report(exit_code, "git finished (exit code %d)\n" % exit_code) return exit_code class GitMirrorAction(Action): # pragma: no cover - def encode_parameters(self, params): pass def execute(self, params, step): env = self.get_env() workspace = env.get_workspace_directory() - mirrors = os.path.join(workspace, '.mirrors') + mirrors = os.path.join(workspace, ".mirrors") - if step.get('where') != 'host': + if step.get("where") != "host": env.report(1, '"where" must be "host"\n') return 1 - sources = params.get('sources') + sources = params.get("sources") if sources is None: env.report(1, '"sources" parameter not provided\n') return 1 @@ -496,17 +489,15 @@ class GitMirrorAction(Action): # pragma: no cover try: exit_code = self.git_mirror(env, sources, mirrors) except Exception as e: - env.report(1, 'Caught exception: {}\n'.format(e)) + env.report(1, "Caught exception: {}\n".format(e)) return 1 - env.report( - exit_code, - 'git mirror action finished (exit code %d)\n' % exit_code) + env.report(exit_code, "git mirror action finished (exit code %d)\n" % exit_code) return exit_code def git_mirror(self, env, sources, mirrors): if not os.path.exists(mirrors): - env.report(None, 'mkdir {}\n'.format(mirrors)) + env.report(None, "mkdir {}\n".format(mirrors)) os.mkdir(mirrors) checked = self.check_sources(sources) @@ -519,8 +510,8 @@ class GitMirrorAction(Action): # pragma: no cover def check_sources(self, sources): checked = [] for source in sources: - name = source.get('name') - repo = source.get('repo') + name = source.get("name") + repo = source.get("repo") if name is None: raise Exception('source lacks "name" field: {}'.format(source)) if repo is None: @@ -529,27 +520,26 @@ class GitMirrorAction(Action): # pragma: no cover return checked def mirror(self, env, mirrors, name, url): - env.report(None, 'git_mirror: mirrors: {}\n'.format(mirrors)) - env.report(None, 'git_mirror: name: {}\n'.format(name)) - env.report(None, 'git_mirror: url: {}\n'.format(url)) + env.report(None, "git_mirror: mirrors: {}\n".format(mirrors)) + env.report(None, "git_mirror: name: {}\n".format(name)) + env.report(None, "git_mirror: url: {}\n".format(url)) dirname = os.path.join(mirrors, name) - env.report(None, 'git_mirror: dirname: {}\n'.format(dirname)) + env.report(None, "git_mirror: dirname: {}\n".format(dirname)) if os.path.exists(dirname): - argv = ['git', 'remote', 'update', '--prune'] + argv = ["git", "remote", "update", "--prune"] cwd = dirname else: - argv = ['git', 'clone', '--mirror', url, name] + argv = ["git", "clone", "--mirror", url, name] cwd = mirrors os.mkdir(dirname) - env.report(None, 'Running: {} in {}\n'.format(argv, cwd)) + env.report(None, "Running: {} in {}\n".format(argv, cwd)) return env.host_runcmd(argv, cwd=cwd) class RsyncAction(Action): # pragma: no cover - def encode_parameters(self, params): pass @@ -557,40 +547,41 @@ class RsyncAction(Action): # pragma: no cover env = self.get_env() workspace = env.get_workspace_directory() - rsync_src = params.get('rsync_src') + rsync_src = params.get("rsync_src") if rsync_src is None: - env.report(1, 'rsync_src not provided\n') + env.report(1, "rsync_src not provided\n") if not self._is_relative(rsync_src): - env.report(1, 'rsync_src not acceptable\n') + env.report(1, "rsync_src not acceptable\n") - rsync_target = params.get('rsync_target') + rsync_target = params.get("rsync_target") if rsync_target is None: - env.report(1, 'git_url not provided\n') + env.report(1, "git_url not provided\n") if not self._remote(rsync_target): - env.report(1, 'rsync_target not acceptable\n') + env.report(1, "rsync_target not acceptable\n") argv = [ - 'rsync', '-av', '--delete-after', - './{}/.'.format(rsync_src), - '{}/.'.format(rsync_target), + "rsync", + "-av", + "--delete-after", + "./{}/.".format(rsync_src), + "{}/.".format(rsync_target), ] exit_code = env.host_runcmd(argv, cwd=workspace) - env.report(exit_code, 'rsync finished (exit code %d)\n' % exit_code) + env.report(exit_code, "rsync finished (exit code %d)\n" % exit_code) return exit_code def _is_relative(self, src): - if src.startswith('/'): + if src.startswith("/"): return False - if '../' in src: + if "../" in src: return False return True def _remote(self, target): - return ':' in target + return ":" in target class DputAction(Action): # pragma: no cover - def encode_parameters(self, params): pass @@ -600,12 +591,12 @@ class DputAction(Action): # pragma: no cover apt_server = self._cc.get_apt_server() config = self.get_dput_config(apt_server) - logging.debug('dput config:\n%s', config) + logging.debug("dput config:\n%s", config) filename = self.create_dput_config_file(config) - argv = ['sh', '-c', 'dput -c {} ick *.changes'.format(filename)] + argv = ["sh", "-c", "dput -c {} ick *.changes".format(filename)] exit_code = env.host_runcmd(argv, cwd=workspace) - env.report(exit_code, 'dput finished (exit code %d)\n' % exit_code) + env.report(exit_code, "dput finished (exit code %d)\n" % exit_code) os.remove(config) return exit_code @@ -617,7 +608,7 @@ class DputAction(Action): # pragma: no cover return filename def get_dput_config(self, apt_server): - template = '''\ + template = """\ [ick] login = incoming fqdn = {apt_server} @@ -626,12 +617,11 @@ incoming = /srv/apt/incoming allow_unsigned_uploads = 1 check_version = 0 run_dinstall = 0 -''' +""" return template.format(apt_server=apt_server) class NotifyAction(Action): # pragma: no cover - def encode_parameters(self, params): pass @@ -641,45 +631,43 @@ class NotifyAction(Action): # pragma: no cover assert cc is not None build_id = self.get_build_id() - env.report(None, 'Notifying about build ending\n') + env.report(None, "Notifying about build ending\n") - build_path = '/builds/{}'.format(build_id) + build_path = "/builds/{}".format(build_id) build = cc.show(build_path) - params = build.get('parameters', {}) - if 'notify' not in params: - env.report( - 0, - 'NOT notifying about build ending: no "notify" parameter.\n') + params = build.get("parameters", {}) + if "notify" not in params: + env.report(0, 'NOT notifying about build ending: no "notify" parameter.\n') return - recipients = params['notify'] + recipients = params["notify"] log = cc.get_log(build_id) - log = log.decode('utf-8') + log = log.decode("utf-8") + log = "\n".join(log.splitlines()[-1000]) notify = { - 'recipients': recipients, - 'build': self.mangle_build(build), - 'log': log, + "recipients": recipients, + "build": self.mangle_build(build), + "log": log, } cc.notify(notify) - env.report(0, 'Notified about build {} ending\n'.format(build_id)) + env.report(0, "Notified about build {} ending\n".format(build_id)) def mangle_build(self, build): b = copy.deepcopy(build) - exit_code = build.get('exit_code') + exit_code = build.get("exit_code") if exit_code is None: - b['status'] = 'BUILDING' + b["status"] = "BUILDING" elif exit_code == 0: - b['status'] = 'SUCCESS' + b["status"] = "SUCCESS" else: - b['status'] = 'FAILED' + b["status"] = "FAILED" return b def make_directory_empty(env, dirname): - return env.runcmd( - ['sudo', 'find', dirname, '-mindepth', '1', '-delete']) + return env.runcmd(["sudo", "find", dirname, "-mindepth", "1", "-delete"]) -- cgit v1.2.1