diff options
| -rwxr-xr-x | check | 106 | ||||
| -rw-r--r-- | ick2/__init__.py | 4 | ||||
| -rw-r--r-- | ick2/actions.py | 318 | ||||
| -rw-r--r-- | ick2/apibase.py | 4 | ||||
| -rw-r--r-- | ick2/buildsm.py | 63 | ||||
| -rw-r--r-- | ick2/notificationapi.py | 21 | ||||
| -rw-r--r-- | ick2/persistent.py | 61 | ||||
| -rw-r--r-- | ick2/persistent_tests.py | 30 | ||||
| -rw-r--r-- | ick2/projectapi_tests.py | 5 | ||||
| -rw-r--r-- | ick2/workapi_tests.py | 5 | ||||
| -rw-r--r-- | pipelines/systrees.ick | 7 |
11 files changed, 315 insertions, 309 deletions
@@ -17,78 +17,70 @@ set -eu -title() -{ - printf "\n" - echo "$@" - n=77 - for i in $(seq $n) - do - printf "%s" - - done - printf '\n' +title() { + printf "\n" + echo "$@" + n=77 + for i in $(seq $n); do + printf "%s" - + done + printf '\n' } - title Remote yarns? local=yes yarns=no -if [ "$#" -gt 0 ] -then - case "$1" in - https://*) - local=no - yarns=yes - remote_url="$1" - shift 1 - ;; - *) - echo "Don't understand args: $@" 1>&2 - exit 1 - ;; - esac +if [ "$#" -gt 0 ]; then + case "$1" in + https://*) + local=no + yarns=yes + remote_url="$1" + shift 1 + ;; + *) + echo "Don't understand args: $*" 1>&2 + exit 1 + ;; + esac fi +if [ "$local" = yes ]; then + title Unit tests + python3 -m CoverageTestRunner --ignore-missing-from=without-tests ick2 -if [ "$local" = yes ] -then - title Unit tests - python3 -m CoverageTestRunner --ignore-missing-from=without-tests ick2 - - if [ -e .git ] - then - sources="$(git ls-files | grep -Fvxf copyright-exceptions)" + if [ -e .git ]; then + sources="$(git ls-files | grep -Fvxf copyright-exceptions)" - # title Copyright statements - # copyright-statement-lint $sources + # title Copyright statements + # copyright-statement-lint $sources - title Copyright licences - ./is-agpl3+ $sources - fi + title Copyright licences + ./is-agpl3+ $sources + fi - python_sources="ick_controller.py worker_manager ick2 icktool" + python_sources="ick_controller.py worker_manager ick2 icktool" - title pycodestyle - pycodestyle ick2 $python_sources + # title pycodestyle + # pycodestyle ick2 $python_sources - # if command -v pylint3 > /dev/null - # then - # title pylint3 - # pylint3 --rcfile pylint.conf $python_sources - # fi + # if command -v pylint3 > /dev/null + # then + # title pylint3 + # pylint3 --rcfile pylint.conf $python_sources + # fi fi -if [ "$yarns" = yes ] -then - title Yarns - yarn yarns/*.yarn \ - --shell python2 \ - --shell-arg '' \ - --shell-library yarns/lib.py \ - --cd-datadir \ - --env "CONTROLLER=$remote_url" \ - --env "SECRETS=$HOME/.config/qvarn/createtoken.conf" \ - "$@" +if [ "$yarns" = yes ]; then + title Yarns + yarn yarns/*.yarn \ + --shell python2 \ + --shell-arg '' \ + --shell-library yarns/lib.py \ + --cd-datadir \ + --env "CONTROLLER=$remote_url" \ + --env "SECRETS=$HOME/.config/qvarn/createtoken.conf" \ + "$@" fi title OK diff --git a/ick2/__init__.py b/ick2/__init__.py index 0016f13..f489396 100644 --- a/ick2/__init__.py +++ b/ick2/__init__.py @@ -1,4 +1,4 @@ -# Copyright (C) 2017-2019 Lars Wirzenius +# Copyright (C) 2017-2018 Lars Wirzenius # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as published by # the Free Software Foundation, either version 3 of the License, or @@ -16,7 +16,7 @@ from .version import __version__, __version_info__ from .logging import setup_logging, log from .persistent import ( - MemoryPersistentState, + FilePersistentState, NotFound, Resource, resource_from_dict, diff --git a/ick2/actions.py b/ick2/actions.py index c901d6b..2881524 100644 --- a/ick2/actions.py +++ b/ick2/actions.py @@ -33,9 +33,9 @@ class UnknownStepError(Exception): class ActionFactory: _classes = { - 'host': ick2.HostEnvironment, - 'chroot': ick2.ChrootEnvironment, - 'container': ick2.ContainerEnvironment, + "host": ick2.HostEnvironment, + "chroot": ick2.ChrootEnvironment, + "container": ick2.ContainerEnvironment, } def __init__(self, build_id, systree, workspace_area, reporter): @@ -75,7 +75,7 @@ class ActionFactory: return area def create_environment(self, spec, project_name): - env = spec.get('where', 'host') + env = spec.get("where", "host") assert env in self.get_allowed_environments() env_class = self._classes[env] area = self.get_workspace_area() @@ -95,46 +95,45 @@ class ActionFactory: def _create_action_object(self, env, spec): rules = [ - ('shell', ShellAction), - ('python', PythonAction), - ('debootstrap', DebootstrapAction), + ("shell", ShellAction), + ("python", PythonAction), + ("debootstrap", DebootstrapAction), ] for key, klass in rules: if key in spec: return klass(env) - if 'archive' in spec: + if "archive" in spec: rules2 = { - 'workspace': ArchiveWorkspaceAction, - 'systree': ArchiveSystreeAction, + "workspace": ArchiveWorkspaceAction, + "systree": ArchiveSystreeAction, } - kind = spec['archive'] + kind = spec["archive"] klass = rules2.get(kind) if klass: return klass(env) - if 'action' in spec: + if "action" in spec: rules2 = { - 'populate_systree': PopulateSystreeAction, - 'populate_workspace': PopulateWorkspaceAction, - 'create_workspace': CreateWorkspaceAction, - 'git': GitAction, - 'git_mirror': GitMirrorAction, - 'rsync': RsyncAction, - 'dput': DputAction, - 'notify': NotifyAction, + "populate_systree": PopulateSystreeAction, + "populate_workspace": PopulateWorkspaceAction, + "create_workspace": CreateWorkspaceAction, + "git": GitAction, + "git_mirror": GitMirrorAction, + "rsync": RsyncAction, + "dput": DputAction, + "notify": NotifyAction, } - kind = spec['action'] + kind = spec["action"] klass = rules2.get(kind) if klass: return klass(env) - raise UnknownStepError('Unknown action %r' % spec) + raise UnknownStepError("Unknown action %r" % spec) class Action: # pragma: no cover - def __init__(self, env): self._env = env self._cc = None @@ -181,20 +180,20 @@ class Action: # pragma: no cover def encode64(self, params): assert isinstance(params, dict) as_text = json.dumps(params) - as_bytes = as_text.encode('UTF-8') + as_bytes = as_text.encode("UTF-8") as_base64 = base64.b64encode(as_bytes) - return as_base64.decode('UTF-8') + return as_base64.decode("UTF-8") def decode64(self, encoded): - as_base64 = encoded.encode('UTF-8') + as_base64 = encoded.encode("UTF-8") as_bytes = base64.b64decode(as_base64) - as_text = as_bytes.decode('UTF-8') + as_text = as_bytes.decode("UTF-8") return json.loads(as_text) def get_authz_headers(self): token = self.get_token() return { - 'Authorization': 'Bearer {}'.format(token), + "Authorization": "Bearer {}".format(token), } def execute(self, params, step): @@ -202,79 +201,75 @@ class Action: # pragma: no cover class ShellAction(Action): - def encode_parameters(self, params): # pragma: no cover encoded = self.encode64(params) return 'params() { echo -n "%s" | base64 -d; }\n' % encoded def execute(self, params, step): prefix = self.encode_parameters(params) - snippet = step['shell'] - argv = ['bash', '-exuc', prefix + snippet] + snippet = step["shell"] + argv = ["bash", "-exuc", prefix + snippet] exit_code = self._env.runcmd(argv) - self._env.report(exit_code, 'action finished\n') + self._env.report(exit_code, "action finished\n") return exit_code class PythonAction(Action): - def encode_parameters(self, params): # pragma: no cover encoded = self.encode64(params) prefix = ( - 'import base64, json, subprocess\n' - 'params = json.loads(base64.b64decode(\n' + "import base64, json, subprocess\n" + "params = json.loads(base64.b64decode(\n" ' "{}").decode("utf8"))\n' - 'def RUN(*args, **kwargs):\n' + "def RUN(*args, **kwargs):\n" ' print("Executing:", args, kwargs)\n' ' if "check" not in kwargs:\n' ' kwargs["check"] = True\n' - ' return subprocess.run(args, **kwargs)\n' - 'def OUT(*args, **kwargs):\n' - ' x = RUN(*args, stdout=subprocess.PIPE, **kwargs)\n' + " return subprocess.run(args, **kwargs)\n" + "def OUT(*args, **kwargs):\n" + " x = RUN(*args, stdout=subprocess.PIPE, **kwargs)\n" ' return x.stdout.decode("UTF-8")\n' - 'def ERR(*args, **kwargs):\n' - ' x = RUN(*args, stderr=subprocess.PIPE, check=False, **kwargs)\n' + "def ERR(*args, **kwargs):\n" + " x = RUN(*args, stderr=subprocess.PIPE, check=False, **kwargs)\n" ' return x.stderr.decode("UTF-8")\n' - 'def OUTERR(*args, **kwargs):\n' - ' x = RUN(*args, stdout=subprocess.PIPE, \n' - ' sterr=subproces.STDOUT, check=False, **kwargs)\n' + "def OUTERR(*args, **kwargs):\n" + " x = RUN(*args, stdout=subprocess.PIPE, \n" + " sterr=subproces.STDOUT, check=False, **kwargs)\n" ' return s.stdout.decode("UTF-8"), x.stderr.decode("UTF-8")\n' - ).format(encoded) return prefix def execute(self, params, step): prefix = self.encode_parameters(params) - snippet = step['python'] - argv = ['python3', '-c', prefix + '\n' + snippet] + snippet = step["python"] + argv = ["python3", "-c", prefix + "\n" + snippet] exit_code = self._env.runcmd(argv) - self._env.report(exit_code, 'action finished\n') + self._env.report(exit_code, "action finished\n") return exit_code class DebootstrapAction(Action): - default_mirror = 'http://deb.debian.org/debian' + default_mirror = "http://deb.debian.org/debian" def encode_parameters(self, params): # pragma: no cover pass def execute(self, params, step): - suite = step.get('debootstrap') - if suite is None or suite == 'auto': - suite = params['debian_codename'] - mirror = step.get('mirror', self.default_mirror) + suite = step.get("debootstrap") + if suite is None or suite == "auto": + suite = params["debian_codename"] + mirror = step.get("mirror", self.default_mirror) env = self.get_env() workspace = env.get_workspace_directory() - argv = ['sudo', 'debootstrap', suite, '.', mirror] + argv = ["sudo", "debootstrap", suite, ".", mirror] exit_code = self._env.host_runcmd(argv, cwd=workspace) - self._env.report(exit_code, 'action finished\n') + self._env.report(exit_code, "action finished\n") return exit_code class CreateWorkspaceAction(Action): - def encode_parameters(self, params): # pragma: no cover pass @@ -282,12 +277,11 @@ class CreateWorkspaceAction(Action): env = self.get_env() dirname = env.get_workspace_directory() make_directory_empty(env, dirname) - self._env.report(0, 'Created or emptied workspace %s\n' % dirname) + self._env.report(0, "Created or emptied workspace %s\n" % dirname) return 0 class ArchiveBaseAction(Action): # pragma: no cover - def get_dirname(self, env): raise NotImplementedError() @@ -298,42 +292,42 @@ class ArchiveBaseAction(Action): # pragma: no cover env = self.get_env() dirname = self.get_dirname(env) - name_from = step.get('name_from', 'artifact_name') + name_from = step.get("name_from", "artifact_name") blob_name = params.get(name_from) if not blob_name: - env.report(1, 'No artifact_name parameter\n') + env.report(1, "No artifact_name parameter\n") return 1 - env.report(None, 'Creating new artifact named {}\n'.format(blob_name)) - env.report(None, 'Artifact will be created from {}\n'.format(dirname)) + env.report(None, "Creating new artifact named {}\n".format(blob_name)) + env.report(None, "Artifact will be created from {}\n".format(dirname)) - globs = step.get('globs') + globs = step.get("globs") if globs is None: - names = ['.'] + names = ["."] else: names = self.match_globs(dirname, globs) url = self.get_blob_upload_url(blob_name) headers = self.get_authz_headers() - self._env.report(None, 'Creating tarball\n') + self._env.report(None, "Creating tarball\n") fd, tarball = tempfile.mkstemp() os.close(fd) - tar = ['sudo', 'tar', '-zvcf', tarball, '-C', dirname] + names + tar = ["sudo", "tar", "-zvcf", tarball, "-C", dirname] + names exit_code = self._env.host_runcmd(tar) if exit_code != 0: - self._env.report(exit_code, 'Tarball generation failed\n') + self._env.report(exit_code, "Tarball generation failed\n") os.remove(tarball) return exit_code - self._env.report(None, 'Tarball generation finished OK\n') - - self._env.report(None, 'Uploading tarball to artifact store\n') - curl = ['curl', '-sk', '-T', tarball] + [ - '-H{}:{}'.format(name, value) - for name, value in headers.items() - ] + [url] + self._env.report(None, "Tarball generation finished OK\n") + + self._env.report(None, "Uploading tarball to artifact store\n") + curl = ( + ["curl", "-sk", "-T", tarball] + + ["-H{}:{}".format(name, value) for name, value in headers.items()] + + [url] + ) exit_code = self._env.host_runcmd(curl) - self._env.report( - exit_code, 'curl upload finished (exit code %s)\n' % exit_code) + self._env.report(exit_code, "curl upload finished (exit code %s)\n" % exit_code) os.remove(tarball) return exit_code @@ -341,20 +335,18 @@ class ArchiveBaseAction(Action): # pragma: no cover def match_globs(self, workspace, globs): names = [] for pat in globs: - abspat = os.path.join(workspace, './' + pat) + abspat = os.path.join(workspace, "./" + pat) for name in glob.glob(abspat): names.append(os.path.normpath(name)) return names class ArchiveSystreeAction(ArchiveBaseAction): # pragma: no cover - def get_dirname(self, env): return env.get_systree_directory() class ArchiveWorkspaceAction(ArchiveBaseAction): # pragma: no cover - def get_dirname(self, env): return env.get_workspace_directory() @@ -371,24 +363,26 @@ class PopulateActionBase(Action): # pragma: no cover env = self.get_env() name = step.get(self.step_field) - if not name or name == 'auto': - name_name = step.get('name_from', self.param_name) + if not name or name == "auto": + name_name = step.get("name_from", self.param_name) name = params.get(name_name) if not name: - msg = '{} in action is {}, but no {} parameter\n'.format( - self.step_field, name, name_name) + msg = "{} in action is {}, but no {} parameter\n".format( + self.step_field, name, name_name + ) env.report(1, msg) return 1 - env.report(None, 'Using {} for artifact name\n'.format(name)) + env.report(None, "Using {} for artifact name\n".format(name)) dirname = self.get_unpack_directory(env) make_directory_empty(env, dirname) exit_code = self.download_and_unpack_artifact(name, dirname) new_code = self.mangle_exit_code(exit_code) env.report( - new_code, '{} finished (exit_code {} -> {})\n'.format( - str(self), exit_code, new_code)) + new_code, + "{} finished (exit_code {} -> {})\n".format(str(self), exit_code, new_code), + ) return new_code def get_unpack_directory(self, env): @@ -400,12 +394,13 @@ class PopulateActionBase(Action): # pragma: no cover def download_and_unpack_artifact(self, name, dirname): url = self.get_blob_upload_url(name) headers = self.get_authz_headers() - curl = ['curl', '-sk'] + [ - '-H{}:{}'.format(name, value) - for name, value in headers.items() - ] + [url] + curl = ( + ["curl", "-sk"] + + ["-H{}:{}".format(name, value) for name, value in headers.items()] + + [url] + ) - untar = ['sudo', 'tar', '-zxf', '-', '-C', dirname] + untar = ["sudo", "tar", "-zxf", "-", "-C", dirname] env = self.get_env() return env.host_runcmd(curl, untar) @@ -413,11 +408,11 @@ class PopulateActionBase(Action): # pragma: no cover class PopulateSystreeAction(PopulateActionBase): # pragma: no cover - step_field = 'systree_name' - param_name = 'systree_name' + step_field = "systree_name" + param_name = "systree_name" def __str__(self): - return 'populate-systree' + return "populate-systree" def get_unpack_directory(self, env): return env.get_systree_directory() @@ -428,11 +423,11 @@ class PopulateSystreeAction(PopulateActionBase): # pragma: no cover class PopulateWorkspaceAction(PopulateActionBase): # pragma: no cover - step_field = 'workspace_name' - param_name = 'workspace_name' + step_field = "workspace_name" + param_name = "workspace_name" def __str__(self): - return 'populate-workspace' + return "populate-workspace" def get_unpack_directory(self, env): return env.get_workspace_directory() @@ -443,7 +438,6 @@ class PopulateWorkspaceAction(PopulateActionBase): # pragma: no cover class GitAction(Action): # pragma: no cover - def encode_parameters(self, params): pass @@ -451,44 +445,43 @@ class GitAction(Action): # pragma: no cover env = self.get_env() workspace = env.get_workspace_directory() - git_dir = params.get('git_dir') + git_dir = params.get("git_dir") if git_dir is None: - env.report(1, 'git_dir not provided\n') - if git_dir.startswith('/') or '..' in git_dir: - env.report(1, 'git_dir not acceptable\n') + env.report(1, "git_dir not provided\n") + if git_dir.startswith("/") or ".." in git_dir: + env.report(1, "git_dir not acceptable\n") - git_url = params.get('git_url') + git_url = params.get("git_url") if git_url is None: - env.report(1, 'git_url not provided\n') + env.report(1, "git_url not provided\n") pathname = os.path.join(workspace, git_dir) if os.path.exists(pathname): - argv = ['git', 'remote', '-v', 'update', '--prune'] + argv = ["git", "remote", "-v", "update", "--prune"] cwd = pathname else: - argv = ['git', 'clone', '-v', git_url, git_dir] + argv = ["git", "clone", "-v", git_url, git_dir] cwd = workspace exit_code = env.host_runcmd(argv, cwd=cwd) - env.report(exit_code, 'git finished (exit code %d)\n' % exit_code) + env.report(exit_code, "git finished (exit code %d)\n" % exit_code) return exit_code class GitMirrorAction(Action): # pragma: no cover - def encode_parameters(self, params): pass def execute(self, params, step): env = self.get_env() workspace = env.get_workspace_directory() - mirrors = os.path.join(workspace, '.mirrors') + mirrors = os.path.join(workspace, ".mirrors") - if step.get('where') != 'host': + if step.get("where") != "host": env.report(1, '"where" must be "host"\n') return 1 - sources = params.get('sources') + sources = params.get("sources") if sources is None: env.report(1, '"sources" parameter not provided\n') return 1 @@ -496,17 +489,15 @@ class GitMirrorAction(Action): # pragma: no cover try: exit_code = self.git_mirror(env, sources, mirrors) except Exception as e: - env.report(1, 'Caught exception: {}\n'.format(e)) + env.report(1, "Caught exception: {}\n".format(e)) return 1 - env.report( - exit_code, - 'git mirror action finished (exit code %d)\n' % exit_code) + env.report(exit_code, "git mirror action finished (exit code %d)\n" % exit_code) return exit_code def git_mirror(self, env, sources, mirrors): if not os.path.exists(mirrors): - env.report(None, 'mkdir {}\n'.format(mirrors)) + env.report(None, "mkdir {}\n".format(mirrors)) os.mkdir(mirrors) checked = self.check_sources(sources) @@ -519,8 +510,8 @@ class GitMirrorAction(Action): # pragma: no cover def check_sources(self, sources): checked = [] for source in sources: - name = source.get('name') - repo = source.get('repo') + name = source.get("name") + repo = source.get("repo") if name is None: raise Exception('source lacks "name" field: {}'.format(source)) if repo is None: @@ -529,27 +520,26 @@ class GitMirrorAction(Action): # pragma: no cover return checked def mirror(self, env, mirrors, name, url): - env.report(None, 'git_mirror: mirrors: {}\n'.format(mirrors)) - env.report(None, 'git_mirror: name: {}\n'.format(name)) - env.report(None, 'git_mirror: url: {}\n'.format(url)) + env.report(None, "git_mirror: mirrors: {}\n".format(mirrors)) + env.report(None, "git_mirror: name: {}\n".format(name)) + env.report(None, "git_mirror: url: {}\n".format(url)) dirname = os.path.join(mirrors, name) - env.report(None, 'git_mirror: dirname: {}\n'.format(dirname)) + env.report(None, "git_mirror: dirname: {}\n".format(dirname)) if os.path.exists(dirname): - argv = ['git', 'remote', 'update', '--prune'] + argv = ["git", "remote", "update", "--prune"] cwd = dirname else: - argv = ['git', 'clone', '--mirror', url, name] + argv = ["git", "clone", "--mirror", url, name] cwd = mirrors os.mkdir(dirname) - env.report(None, 'Running: {} in {}\n'.format(argv, cwd)) + env.report(None, "Running: {} in {}\n".format(argv, cwd)) return env.host_runcmd(argv, cwd=cwd) class RsyncAction(Action): # pragma: no cover - def encode_parameters(self, params): pass @@ -557,40 +547,41 @@ class RsyncAction(Action): # pragma: no cover env = self.get_env() workspace = env.get_workspace_directory() - rsync_src = params.get('rsync_src') + rsync_src = params.get("rsync_src") if rsync_src is None: - env.report(1, 'rsync_src not provided\n') + env.report(1, "rsync_src not provided\n") if not self._is_relative(rsync_src): - env.report(1, 'rsync_src not acceptable\n') + env.report(1, "rsync_src not acceptable\n") - rsync_target = params.get('rsync_target') + rsync_target = params.get("rsync_target") if rsync_target is None: - env.report(1, 'git_url not provided\n') + env.report(1, "git_url not provided\n") if not self._remote(rsync_target): - env.report(1, 'rsync_target not acceptable\n') + env.report(1, "rsync_target not acceptable\n") argv = [ - 'rsync', '-av', '--delete-after', - './{}/.'.format(rsync_src), - '{}/.'.format(rsync_target), + "rsync", + "-av", + "--delete-after", + "./{}/.".format(rsync_src), + "{}/.".format(rsync_target), ] exit_code = env.host_runcmd(argv, cwd=workspace) - env.report(exit_code, 'rsync finished (exit code %d)\n' % exit_code) + env.report(exit_code, "rsync finished (exit code %d)\n" % exit_code) return exit_code def _is_relative(self, src): - if src.startswith('/'): + if src.startswith("/"): return False - if '../' in src: + if "../" in src: return False return True def _remote(self, target): - return ':' in target + return ":" in target class DputAction(Action): # pragma: no cover - def encode_parameters(self, params): pass @@ -600,12 +591,12 @@ class DputAction(Action): # pragma: no cover apt_server = self._cc.get_apt_server() config = self.get_dput_config(apt_server) - logging.debug('dput config:\n%s', config) + logging.debug("dput config:\n%s", config) filename = self.create_dput_config_file(config) - argv = ['sh', '-c', 'dput -c {} ick *.changes'.format(filename)] + argv = ["sh", "-c", "dput -c {} ick *.changes".format(filename)] exit_code = env.host_runcmd(argv, cwd=workspace) - env.report(exit_code, 'dput finished (exit code %d)\n' % exit_code) + env.report(exit_code, "dput finished (exit code %d)\n" % exit_code) os.remove(config) return exit_code @@ -617,7 +608,7 @@ class DputAction(Action): # pragma: no cover return filename def get_dput_config(self, apt_server): - template = '''\ + template = """\ [ick] login = incoming fqdn = {apt_server} @@ -626,12 +617,11 @@ incoming = /srv/apt/incoming allow_unsigned_uploads = 1 check_version = 0 run_dinstall = 0 -''' +""" return template.format(apt_server=apt_server) class NotifyAction(Action): # pragma: no cover - def encode_parameters(self, params): pass @@ -641,45 +631,43 @@ class NotifyAction(Action): # pragma: no cover assert cc is not None build_id = self.get_build_id() - env.report(None, 'Notifying about build ending\n') + env.report(None, "Notifying about build ending\n") - build_path = '/builds/{}'.format(build_id) + build_path = "/builds/{}".format(build_id) build = cc.show(build_path) - params = build.get('parameters', {}) - if 'notify' not in params: - env.report( - 0, - 'NOT notifying about build ending: no "notify" parameter.\n') + params = build.get("parameters", {}) + if "notify" not in params: + env.report(0, 'NOT notifying about build ending: no "notify" parameter.\n') return - recipients = params['notify'] + recipients = params["notify"] log = cc.get_log(build_id) - log = log.decode('utf-8') + log = log.decode("utf-8") + log = "\n".join(log.splitlines()[-1000:]) notify = { - 'recipients': recipients, - 'build': self.mangle_build(build), - 'log': log, + "recipients": recipients, + "build": self.mangle_build(build), + "log": log, } cc.notify(notify) - env.report(0, 'Notified about build {} ending\n'.format(build_id)) + env.report(0, "Notified about build {} ending\n".format(build_id)) def mangle_build(self, build): b = copy.deepcopy(build) - exit_code = build.get('exit_code') + exit_code = build.get("exit_code") if exit_code is None: - b['status'] = 'BUILDING' + b["status"] = "BUILDING" elif exit_code == 0: - b['status'] = 'SUCCESS' + b["status"] = "SUCCESS" else: - b['status'] = 'FAILED' + b["status"] = "FAILED" return b def make_directory_empty(env, dirname): - return env.runcmd( - ['sudo', 'find', dirname, '-mindepth', '1', '-delete']) + return env.runcmd(["sudo", "find", dirname, "-mindepth", "1", "-delete"]) diff --git a/ick2/apibase.py b/ick2/apibase.py index f330eae..c08f7cf 100644 --- a/ick2/apibase.py +++ b/ick2/apibase.py @@ -1,4 +1,4 @@ -# Copyright (C) 2017-2019 Lars Wirzenius +# Copyright (C) 2017-2018 Lars Wirzenius # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as published by # the Free Software Foundation, either version 3 of the License, or @@ -19,7 +19,7 @@ import ick2 class APIbase: def __init__(self, state): - assert state is None or isinstance(state, ick2.MemoryPersistentState) + assert state is None or isinstance(state, ick2.FilePersistentState) self._trans = ick2.TransactionalState(state) def get_routes(self, path): diff --git a/ick2/buildsm.py b/ick2/buildsm.py index 9177e09..8d71fde 100644 --- a/ick2/buildsm.py +++ b/ick2/buildsm.py @@ -17,15 +17,14 @@ import ick2 -BUILD_TRIGGERED = 'triggered' -BUILD_BUILDING = 'building' -BUILD_NOTIFYING = 'notifying' -BUILD_DONE = 'done' -BUILD_FAILED = 'failed' +BUILD_TRIGGERED = "triggered" +BUILD_BUILDING = "building" +BUILD_NOTIFYING = "notifying" +BUILD_DONE = "done" +BUILD_FAILED = "failed" class StateMachine: - def __init__(self, get_state, set_state): self.transitions = {} self.get_state = get_state @@ -48,7 +47,6 @@ class StateMachine: class BuildStateMachine: - def __init__(self, build): self.build = build self.sm = self.init_sm() @@ -80,10 +78,10 @@ class BuildStateMachine: return sm def get_state(self): - return self.build.resource['status'] + return self.build.resource["status"] def set_state(self, state): - self.build.resource['status'] = state + self.build.resource["status"] = state def handle_event(self, event): old_state = self.get_state() @@ -101,7 +99,7 @@ class BuildStateMachine: graph = self.build.get_graph() action_ids = graph.find_actions(ick2.ACTION_READY) if not action_ids: # pragma: no cover - self.build.resource['exit_code'] = 0 + self.build.resource["exit_code"] = 0 return BUILD_DONE, None action_id = action_ids[0] @@ -113,7 +111,7 @@ class BuildStateMachine: graph = self.build.get_graph() action_ids = graph.find_actions(ick2.ACTION_READY) if not action_ids: # pragma: no cover - self.build.resource['exit_code'] = 0 + self.build.resource["exit_code"] = 0 return BUILD_DONE, None action_id = action_ids[0] @@ -122,34 +120,32 @@ class BuildStateMachine: return BUILD_NOTIFYING, (action_id, action) def mark_action_done(self, event): - self.build.resource['exit_code'] = event.exit_code + self.build.resource["exit_code"] = event.exit_code graph = self.build.get_graph() graph.set_action_status(event.action_id, ick2.ACTION_DONE) graph.unblock() if graph.has_more_to_do(): return BUILD_BUILDING, None - - self.add_notification_action() - return BUILD_NOTIFYING, None + return BUILD_DONE, None def add_notification_action(self): action = { - 'action': 'notify', + "action": "notify", } graph = self.build.get_graph() graph.append_action(action, ick2.ACTION_READY, depends=[]) def mark_notification_done(self, event): if event.exit_code not in (0, None): # pragma: no cover - self.build.resource['exit_code'] = event.exit_code + self.build.resource["exit_code"] = event.exit_code graph = self.build.get_graph() graph.set_action_status(event.action_id, ick2.ACTION_DONE) graph.unblock() if graph.has_more_to_do(): # pragma: no cover return BUILD_NOTIFYING, None - if self.build.resource.get('exit_code') in (0, None): - self.build.resource['exit_code'] = 0 + if self.build.resource.get("exit_code") in (0, None): + self.build.resource["exit_code"] = 0 return BUILD_DONE, None return BUILD_FAILED, None @@ -157,9 +153,9 @@ class BuildStateMachine: def mark_build_failed(self, event): graph = self.build.get_graph() graph.set_action_status(event.action_id, ick2.BUILD_FAILED) - self.build.resource['exit_code'] = event.exit_code + self.build.resource["exit_code"] = event.exit_code self.add_notification_action() - return BUILD_NOTIFYING, None + return BUILD_FAILED, None # Thing should be something we can create a BuildEvent from. @@ -171,8 +167,8 @@ def create_build_event(thing): return NeedWorkEvent() if isinstance(thing, dict): - exit_code = thing.get('exit_code') - action_id = thing.get('action_id') + exit_code = thing.get("exit_code") + action_id = thing.get("action_id") if exit_code is None: return PartialActionOutputEvent() if exit_code == 0: @@ -181,31 +177,26 @@ def create_build_event(thing): class BuildEvent: # pragma: no cover - - event_type = 'BuildEvent' + event_type = "BuildEvent" def __str__(self): return self.event_type class BuildStartsEvent(BuildEvent): - - event_type = 'build-starts' + event_type = "build-starts" class NeedWorkEvent(BuildEvent): - - event_type = 'need-work' + event_type = "need-work" class PartialActionOutputEvent(BuildEvent): - - event_type = 'partial-output' + event_type = "partial-output" class ActionFinishedEvent(BuildEvent): - - event_type = 'action-finished' + event_type = "action-finished" def __init__(self, action_id): self.action_id = action_id @@ -213,8 +204,7 @@ class ActionFinishedEvent(BuildEvent): class ActionFailedEvent(BuildEvent): - - event_type = 'action-failed' + event_type = "action-failed" def __init__(self, action_id, exit_code): self.action_id = action_id @@ -222,6 +212,5 @@ class ActionFailedEvent(BuildEvent): class UnexpectedEvent(Exception): # pragma: no cover - def __init__(self, event, state): - super().__init__('Did not expect %s in %s' % (event, state)) + super().__init__("Did not expect %s in %s" % (event, state)) diff --git a/ick2/notificationapi.py b/ick2/notificationapi.py index ab261bd..877b929 100644 --- a/ick2/notificationapi.py +++ b/ick2/notificationapi.py @@ -17,29 +17,32 @@ import ick2 -class NotificationAPI: +N = 1000 # Max number of line to include in log + +class NotificationAPI: def __init__(self, config): self._config = config def find_missing_route(self, missing_path): return [ { - 'method': 'POST', - 'path': '/notify', - 'callback': self.notify, + "method": "POST", + "path": "/notify", + "callback": self.notify, }, ] def notify(self, content_type, body, **kwargs): - ick2.log.log('info', msg_text='Notification requested', kwargs=kwargs) + ick2.log.log("info", msg_text="Notification requested", kwargs=kwargs) - recipients = body.get('recipients', []) - build = body.get('build', {}) - log = body.get('log', '') + recipients = body.get("recipients", []) + build = body.get("build", {}) + log = body.get("log", "") + log = "".join(f"{line}\n" for line in log.splitlines()[-N:]) sendmail = ick2.Sendmail() sendmail.set_config(self._config) sendmail.send(recipients, build, log) - return ick2.OK('') + return ick2.OK("") diff --git a/ick2/persistent.py b/ick2/persistent.py index bb0aeb6..1d79e3d 100644 --- a/ick2/persistent.py +++ b/ick2/persistent.py @@ -49,33 +49,64 @@ class PersistentStateInterface: # pragma: no cover raise NotImplementedError() -class MemoryPersistentState(PersistentStateInterface): +class FilePersistentState(PersistentStateInterface): def __init__(self): - self._res = {} + self._dir = None - def get_resource_ids(self, kind): - if kind not in self._res: - return [] - return list(self._res[kind].keys()) + def get_directory(self): + return self._dir + + def set_directory(self, dirname): + self._dir = dirname + + def _safe(self, name): + return urllib.parse.quote(name, safe='') + + def _unsafe(self, safe): + return urllib.parse.unquote(safe) + + def _unsafe_list(self, safe_names): + return [self._unsafe(safe) for safe in safe_names] + + def _dirname(self, kind): + return os.path.join(self._dir, self._safe(kind)) + + def _filename(self, kind, rid): + dirname = self._dirname(kind) + return os.path.join(dirname, self._safe(rid)) def has_resource(self, kind, rid): - return kind in self._res and rid in self._res[kind] + filename = self._filename(kind, rid) + return os.path.exists(filename) + + def get_resource_ids(self, kind): + dirname = self._dirname(kind) + if os.path.exists(dirname): + return self._unsafe_list(os.listdir(dirname)) + return [] def get_resource(self, kind, rid): - if kind not in self._res or rid not in self._res[kind]: + filename = self._filename(kind, rid) + if not os.path.exists(filename): raise ick2.NotFound(kind=kind, rid=rid) - return self._res[kind][rid] + with open(filename, 'r') as f: + as_dict = yaml.load(f, Loader=yaml.CSafeLoader) + return resource_from_dict(as_dict) def write_resource(self, kind, rid, resource): - if kind not in self._res: - self._res[kind] = {} - self._res[kind][rid] = resource + dirname = self._dirname(kind) + if not os.path.exists(dirname): + os.makedirs(dirname) + + filename = self._filename(kind, rid) + with open(filename, 'w') as f: + yaml.dump( + resource.as_dict(), stream=f, Dumper=yaml.CSafeDumper) def remove_resource(self, kind, rid): - if kind not in self._res or rid not in self._res[kind]: - raise ick2.NotFound(kind=kind, rid=rid) - del self._res[kind][rid] + filename = self._filename(kind, rid) + os.remove(filename) class NotFound(Exception): diff --git a/ick2/persistent_tests.py b/ick2/persistent_tests.py index 61cd43f..de279a1 100644 --- a/ick2/persistent_tests.py +++ b/ick2/persistent_tests.py @@ -22,7 +22,18 @@ import unittest import ick2 -class PersistentStateTestsMixIn: +class FilePersistentStateTests(unittest.TestCase): + + def setUp(self): + self.tempdir = tempfile.mkdtemp() + self.state = ick2.FilePersistentState() + self.state.set_directory(self.tempdir) + + def tearDown(self): + shutil.rmtree(self.tempdir) + + def test_returns_dirname(self): + self.assertEqual(self.state.get_directory(), self.tempdir) def test_has_no_resources_initially(self): self.assertEqual(self.state.get_resource_ids('silly'), []) @@ -49,20 +60,3 @@ class PersistentStateTestsMixIn: self.state.remove_resource('silly', '#1') self.assertFalse(self.state.has_resource('silly', '#1')) self.assertEqual(self.state.get_resource_ids('silly'), []) - - def test_raises_error_removing_nonexistent_resource_kind(self): - with self.assertRaises(ick2.NotFound): - self.state.remove_resource('silly', '#1') - - def test_raises_error_removing_nonexistent_resource(self): - as_dict = {'foo': 'bar'} - r = ick2.resource_from_dict(as_dict) - self.state.write_resource('silly', '#1', r) - with self.assertRaises(ick2.NotFound): - self.state.remove_resource('silly', '#2') - - -class MemoryPersistentStateTests(unittest.TestCase, PersistentStateTestsMixIn): - - def setUp(self): - self.state = ick2.MemoryPersistentState() diff --git a/ick2/projectapi_tests.py b/ick2/projectapi_tests.py index 5be2a58..b6ec9e9 100644 --- a/ick2/projectapi_tests.py +++ b/ick2/projectapi_tests.py @@ -1,4 +1,4 @@ -# Copyright (C) 2017-2019 Lars Wirzenius +# Copyright (C) 2017-2018 Lars Wirzenius # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as published by # the Free Software Foundation, either version 3 of the License, or @@ -27,7 +27,8 @@ class ProjectAPITests(unittest.TestCase): def setUp(self): self.tempdir = tempfile.mkdtemp() self.statedir = os.path.join(self.tempdir, 'state/dir') - self.state = ick2.MemoryPersistentState() + self.state = ick2.FilePersistentState() + self.state.set_directory(self.statedir) def tearDown(self): shutil.rmtree(self.tempdir) diff --git a/ick2/workapi_tests.py b/ick2/workapi_tests.py index 8652110..c368b4b 100644 --- a/ick2/workapi_tests.py +++ b/ick2/workapi_tests.py @@ -1,4 +1,4 @@ -# Copyright (C) 2017-2019 Lars Wirzenius +# Copyright (C) 2017-2018 Lars Wirzenius # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as published by # the Free Software Foundation, either version 3 of the License, or @@ -28,7 +28,8 @@ class WorkAPITests(unittest.TestCase): def setUp(self): self.tempdir = tempfile.mkdtemp() self.statedir = os.path.join(self.tempdir, 'state/dir') - self.state = ick2.MemoryPersistentState() + self.state = ick2.FilePersistentState() + self.state.set_directory(self.statedir) self.claims = None def tearDown(self): diff --git a/pipelines/systrees.ick b/pipelines/systrees.ick index b6da798..bd52b1c 100644 --- a/pipelines/systrees.ick +++ b/pipelines/systrees.ick @@ -18,6 +18,13 @@ pipelines: - packages - artifact_name actions: + - shell: | + lsb_release -a + dpkg -l debootstrap + sudo chown root:root . + ls -la + where: host + - debootstrap: auto mirror: http://deb.debian.org/debian where: host |