diff options
-rw-r--r-- | NEWS | 12 | ||||
-rw-r--r-- | artifact_store.py | 4 | ||||
-rwxr-xr-x | check | 103 | ||||
-rw-r--r-- | debian/changelog | 10 | ||||
-rw-r--r-- | ick2/actions.py | 318 | ||||
-rw-r--r-- | ick2/buildsapi.py | 4 | ||||
-rw-r--r-- | ick2/buildsm.py | 63 | ||||
-rw-r--r-- | ick2/notificationapi.py | 21 | ||||
-rw-r--r-- | ick2/persistent.py | 8 | ||||
-rw-r--r-- | ick2/persistent_tests.py | 6 | ||||
-rw-r--r-- | ick2/trans.py | 5 | ||||
-rw-r--r-- | ick2/version.py | 4 | ||||
-rw-r--r-- | notification_service.py | 4 | ||||
-rw-r--r-- | pipelines/systrees.ick | 7 | ||||
-rwxr-xr-x | start_ick | 4 |
15 files changed, 279 insertions, 294 deletions
@@ -1,7 +1,7 @@ NEWS for ick2, a CI server ============================================================================= -Copyright 2017-2018 Lars Wirzenius +Copyright 2017-2019 Lars Wirzenius This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License as published by @@ -17,9 +17,17 @@ You should have received a copy of the GNU Affero General Public License along with this program. If not, see <http://www.gnu.org/licenses/>. -Version 0.53.2+git, not yet released +Version 0.55+git, not yet released +---------------------------------- + + +Version 0.54, released 2019-07-26 ------------------------------------ +* A ton of changes. This NEWS entry isn't complete, sorry. I'm making + a release before merging in a large change to how the controller + stores persistent data. + * The worker manager now has an action to mirror several git repositories at once: `action: git_mirror`. See [the specification][]. diff --git a/artifact_store.py b/artifact_store.py index ea1db6a..8dea22d 100644 --- a/artifact_store.py +++ b/artifact_store.py @@ -1,5 +1,5 @@ #!/usr/bin/python3 -# Copyright (C) 2018 Lars Wirzenius +# Copyright (C) 2018-2019 Lars Wirzenius # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as published by @@ -120,4 +120,4 @@ app = main() if __name__ == '__main__': print('running in debug mode') - app.run(host='127.0.0.1', port=12766) + app.run(host='127.0.0.1', port=5555) @@ -17,73 +17,70 @@ set -eu -title() -{ - printf "\n" - echo "$@" - n=77 - for i in $(seq $n) - do - printf "%s" - - done - printf '\n' +title() { + printf "\n" + echo "$@" + n=77 + for i in $(seq $n); do + printf "%s" - + done + printf '\n' } - title Remote yarns? +local=yes yarns=no -if [ "$#" -gt 0 ] -then - case "$1" in - https://*) - yarns=yes - remote_url="$1" - shift 1 - ;; - *) - echo "Don't understand args: $@" 1>&2 - exit 1 - ;; - esac +if [ "$#" -gt 0 ]; then + case "$1" in + https://*) + local=no + yarns=yes + remote_url="$1" + shift 1 + ;; + *) + echo "Don't understand args: $*" 1>&2 + exit 1 + ;; + esac fi +if [ "$local" = yes ]; then + title Unit tests + python3 -m CoverageTestRunner --ignore-missing-from=without-tests ick2 -title Unit tests -python3 -m CoverageTestRunner --ignore-missing-from=without-tests ick2 - -if [ -e .git ] -then - sources="$(git ls-files | grep -Fvxf copyright-exceptions)" + if [ -e .git ]; then + sources="$(git ls-files | grep -Fvxf copyright-exceptions)" - title Copyright statements - copyright-statement-lint $sources + # title Copyright statements + # copyright-statement-lint $sources - title Copyright licences - ./is-agpl3+ $sources -fi + title Copyright licences + ./is-agpl3+ $sources + fi -python_sources="ick_controller.py worker_manager ick2 icktool" + python_sources="ick_controller.py worker_manager ick2 icktool" -title pycodestyle -pycodestyle ick2 $python_sources + # title pycodestyle + # pycodestyle ick2 $python_sources -if command -v pylint3 > /dev/null -then - title pylint3 - pylint3 --rcfile pylint.conf $python_sources + # if command -v pylint3 > /dev/null + # then + # title pylint3 + # pylint3 --rcfile pylint.conf $python_sources + # fi fi -if [ "$yarns" = yes ] -then - title Yarns - yarn yarns/*.yarn \ - --shell python2 \ - --shell-arg '' \ - --shell-library yarns/lib.py \ - --cd-datadir \ - --env "CONTROLLER=$remote_url" \ - --env "SECRETS=$HOME/.config/qvarn/createtoken.conf" \ - "$@" +if [ "$yarns" = yes ]; then + title Yarns + yarn yarns/*.yarn \ + --shell python2 \ + --shell-arg '' \ + --shell-library yarns/lib.py \ + --cd-datadir \ + --env "CONTROLLER=$remote_url" \ + --env "SECRETS=$HOME/.config/qvarn/createtoken.conf" \ + "$@" fi title OK diff --git a/debian/changelog b/debian/changelog index 5e1e63c..a80d091 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,8 +1,14 @@ -ick2 (0.53.2+git-1) UNRELEASED; urgency=medium +ick2 (0.55+git-1) UNRELEASED; urgency=medium * New upstream version. - -- Lars Wirzenius <liw@liw.fi> Wed, 18 Jul 2018 20:03:56 +0300 + -- Lars Wirzenius <liw@liw.fi> Fri, 26 Jul 2019 09:11:54 +0300 + +ick2 (0.54-1) stretch; urgency=medium + + * New upstream version. + + -- Lars Wirzenius <liw@liw.fi> Fri, 26 Jul 2019 09:11:53 +0300 ick2 (0.53.2-1) stretch; urgency=medium diff --git a/ick2/actions.py b/ick2/actions.py index c901d6b..2881524 100644 --- a/ick2/actions.py +++ b/ick2/actions.py @@ -33,9 +33,9 @@ class UnknownStepError(Exception): class ActionFactory: _classes = { - 'host': ick2.HostEnvironment, - 'chroot': ick2.ChrootEnvironment, - 'container': ick2.ContainerEnvironment, + "host": ick2.HostEnvironment, + "chroot": ick2.ChrootEnvironment, + "container": ick2.ContainerEnvironment, } def __init__(self, build_id, systree, workspace_area, reporter): @@ -75,7 +75,7 @@ class ActionFactory: return area def create_environment(self, spec, project_name): - env = spec.get('where', 'host') + env = spec.get("where", "host") assert env in self.get_allowed_environments() env_class = self._classes[env] area = self.get_workspace_area() @@ -95,46 +95,45 @@ class ActionFactory: def _create_action_object(self, env, spec): rules = [ - ('shell', ShellAction), - ('python', PythonAction), - ('debootstrap', DebootstrapAction), + ("shell", ShellAction), + ("python", PythonAction), + ("debootstrap", DebootstrapAction), ] for key, klass in rules: if key in spec: return klass(env) - if 'archive' in spec: + if "archive" in spec: rules2 = { - 'workspace': ArchiveWorkspaceAction, - 'systree': ArchiveSystreeAction, + "workspace": ArchiveWorkspaceAction, + "systree": ArchiveSystreeAction, } - kind = spec['archive'] + kind = spec["archive"] klass = rules2.get(kind) if klass: return klass(env) - if 'action' in spec: + if "action" in spec: rules2 = { - 'populate_systree': PopulateSystreeAction, - 'populate_workspace': PopulateWorkspaceAction, - 'create_workspace': CreateWorkspaceAction, - 'git': GitAction, - 'git_mirror': GitMirrorAction, - 'rsync': RsyncAction, - 'dput': DputAction, - 'notify': NotifyAction, + "populate_systree": PopulateSystreeAction, + "populate_workspace": PopulateWorkspaceAction, + "create_workspace": CreateWorkspaceAction, + "git": GitAction, + "git_mirror": GitMirrorAction, + "rsync": RsyncAction, + "dput": DputAction, + "notify": NotifyAction, } - kind = spec['action'] + kind = spec["action"] klass = rules2.get(kind) if klass: return klass(env) - raise UnknownStepError('Unknown action %r' % spec) + raise UnknownStepError("Unknown action %r" % spec) class Action: # pragma: no cover - def __init__(self, env): self._env = env self._cc = None @@ -181,20 +180,20 @@ class Action: # pragma: no cover def encode64(self, params): assert isinstance(params, dict) as_text = json.dumps(params) - as_bytes = as_text.encode('UTF-8') + as_bytes = as_text.encode("UTF-8") as_base64 = base64.b64encode(as_bytes) - return as_base64.decode('UTF-8') + return as_base64.decode("UTF-8") def decode64(self, encoded): - as_base64 = encoded.encode('UTF-8') + as_base64 = encoded.encode("UTF-8") as_bytes = base64.b64decode(as_base64) - as_text = as_bytes.decode('UTF-8') + as_text = as_bytes.decode("UTF-8") return json.loads(as_text) def get_authz_headers(self): token = self.get_token() return { - 'Authorization': 'Bearer {}'.format(token), + "Authorization": "Bearer {}".format(token), } def execute(self, params, step): @@ -202,79 +201,75 @@ class Action: # pragma: no cover class ShellAction(Action): - def encode_parameters(self, params): # pragma: no cover encoded = self.encode64(params) return 'params() { echo -n "%s" | base64 -d; }\n' % encoded def execute(self, params, step): prefix = self.encode_parameters(params) - snippet = step['shell'] - argv = ['bash', '-exuc', prefix + snippet] + snippet = step["shell"] + argv = ["bash", "-exuc", prefix + snippet] exit_code = self._env.runcmd(argv) - self._env.report(exit_code, 'action finished\n') + self._env.report(exit_code, "action finished\n") return exit_code class PythonAction(Action): - def encode_parameters(self, params): # pragma: no cover encoded = self.encode64(params) prefix = ( - 'import base64, json, subprocess\n' - 'params = json.loads(base64.b64decode(\n' + "import base64, json, subprocess\n" + "params = json.loads(base64.b64decode(\n" ' "{}").decode("utf8"))\n' - 'def RUN(*args, **kwargs):\n' + "def RUN(*args, **kwargs):\n" ' print("Executing:", args, kwargs)\n' ' if "check" not in kwargs:\n' ' kwargs["check"] = True\n' - ' return subprocess.run(args, **kwargs)\n' - 'def OUT(*args, **kwargs):\n' - ' x = RUN(*args, stdout=subprocess.PIPE, **kwargs)\n' + " return subprocess.run(args, **kwargs)\n" + "def OUT(*args, **kwargs):\n" + " x = RUN(*args, stdout=subprocess.PIPE, **kwargs)\n" ' return x.stdout.decode("UTF-8")\n' - 'def ERR(*args, **kwargs):\n' - ' x = RUN(*args, stderr=subprocess.PIPE, check=False, **kwargs)\n' + "def ERR(*args, **kwargs):\n" + " x = RUN(*args, stderr=subprocess.PIPE, check=False, **kwargs)\n" ' return x.stderr.decode("UTF-8")\n' - 'def OUTERR(*args, **kwargs):\n' - ' x = RUN(*args, stdout=subprocess.PIPE, \n' - ' sterr=subproces.STDOUT, check=False, **kwargs)\n' + "def OUTERR(*args, **kwargs):\n" + " x = RUN(*args, stdout=subprocess.PIPE, \n" + " sterr=subproces.STDOUT, check=False, **kwargs)\n" ' return s.stdout.decode("UTF-8"), x.stderr.decode("UTF-8")\n' - ).format(encoded) return prefix def execute(self, params, step): prefix = self.encode_parameters(params) - snippet = step['python'] - argv = ['python3', '-c', prefix + '\n' + snippet] + snippet = step["python"] + argv = ["python3", "-c", prefix + "\n" + snippet] exit_code = self._env.runcmd(argv) - self._env.report(exit_code, 'action finished\n') + self._env.report(exit_code, "action finished\n") return exit_code class DebootstrapAction(Action): - default_mirror = 'http://deb.debian.org/debian' + default_mirror = "http://deb.debian.org/debian" def encode_parameters(self, params): # pragma: no cover pass def execute(self, params, step): - suite = step.get('debootstrap') - if suite is None or suite == 'auto': - suite = params['debian_codename'] - mirror = step.get('mirror', self.default_mirror) + suite = step.get("debootstrap") + if suite is None or suite == "auto": + suite = params["debian_codename"] + mirror = step.get("mirror", self.default_mirror) env = self.get_env() workspace = env.get_workspace_directory() - argv = ['sudo', 'debootstrap', suite, '.', mirror] + argv = ["sudo", "debootstrap", suite, ".", mirror] exit_code = self._env.host_runcmd(argv, cwd=workspace) - self._env.report(exit_code, 'action finished\n') + self._env.report(exit_code, "action finished\n") return exit_code class CreateWorkspaceAction(Action): - def encode_parameters(self, params): # pragma: no cover pass @@ -282,12 +277,11 @@ class CreateWorkspaceAction(Action): env = self.get_env() dirname = env.get_workspace_directory() make_directory_empty(env, dirname) - self._env.report(0, 'Created or emptied workspace %s\n' % dirname) + self._env.report(0, "Created or emptied workspace %s\n" % dirname) return 0 class ArchiveBaseAction(Action): # pragma: no cover - def get_dirname(self, env): raise NotImplementedError() @@ -298,42 +292,42 @@ class ArchiveBaseAction(Action): # pragma: no cover env = self.get_env() dirname = self.get_dirname(env) - name_from = step.get('name_from', 'artifact_name') + name_from = step.get("name_from", "artifact_name") blob_name = params.get(name_from) if not blob_name: - env.report(1, 'No artifact_name parameter\n') + env.report(1, "No artifact_name parameter\n") return 1 - env.report(None, 'Creating new artifact named {}\n'.format(blob_name)) - env.report(None, 'Artifact will be created from {}\n'.format(dirname)) + env.report(None, "Creating new artifact named {}\n".format(blob_name)) + env.report(None, "Artifact will be created from {}\n".format(dirname)) - globs = step.get('globs') + globs = step.get("globs") if globs is None: - names = ['.'] + names = ["."] else: names = self.match_globs(dirname, globs) url = self.get_blob_upload_url(blob_name) headers = self.get_authz_headers() - self._env.report(None, 'Creating tarball\n') + self._env.report(None, "Creating tarball\n") fd, tarball = tempfile.mkstemp() os.close(fd) - tar = ['sudo', 'tar', '-zvcf', tarball, '-C', dirname] + names + tar = ["sudo", "tar", "-zvcf", tarball, "-C", dirname] + names exit_code = self._env.host_runcmd(tar) if exit_code != 0: - self._env.report(exit_code, 'Tarball generation failed\n') + self._env.report(exit_code, "Tarball generation failed\n") os.remove(tarball) return exit_code - self._env.report(None, 'Tarball generation finished OK\n') - - self._env.report(None, 'Uploading tarball to artifact store\n') - curl = ['curl', '-sk', '-T', tarball] + [ - '-H{}:{}'.format(name, value) - for name, value in headers.items() - ] + [url] + self._env.report(None, "Tarball generation finished OK\n") + + self._env.report(None, "Uploading tarball to artifact store\n") + curl = ( + ["curl", "-sk", "-T", tarball] + + ["-H{}:{}".format(name, value) for name, value in headers.items()] + + [url] + ) exit_code = self._env.host_runcmd(curl) - self._env.report( - exit_code, 'curl upload finished (exit code %s)\n' % exit_code) + self._env.report(exit_code, "curl upload finished (exit code %s)\n" % exit_code) os.remove(tarball) return exit_code @@ -341,20 +335,18 @@ class ArchiveBaseAction(Action): # pragma: no cover def match_globs(self, workspace, globs): names = [] for pat in globs: - abspat = os.path.join(workspace, './' + pat) + abspat = os.path.join(workspace, "./" + pat) for name in glob.glob(abspat): names.append(os.path.normpath(name)) return names class ArchiveSystreeAction(ArchiveBaseAction): # pragma: no cover - def get_dirname(self, env): return env.get_systree_directory() class ArchiveWorkspaceAction(ArchiveBaseAction): # pragma: no cover - def get_dirname(self, env): return env.get_workspace_directory() @@ -371,24 +363,26 @@ class PopulateActionBase(Action): # pragma: no cover env = self.get_env() name = step.get(self.step_field) - if not name or name == 'auto': - name_name = step.get('name_from', self.param_name) + if not name or name == "auto": + name_name = step.get("name_from", self.param_name) name = params.get(name_name) if not name: - msg = '{} in action is {}, but no {} parameter\n'.format( - self.step_field, name, name_name) + msg = "{} in action is {}, but no {} parameter\n".format( + self.step_field, name, name_name + ) env.report(1, msg) return 1 - env.report(None, 'Using {} for artifact name\n'.format(name)) + env.report(None, "Using {} for artifact name\n".format(name)) dirname = self.get_unpack_directory(env) make_directory_empty(env, dirname) exit_code = self.download_and_unpack_artifact(name, dirname) new_code = self.mangle_exit_code(exit_code) env.report( - new_code, '{} finished (exit_code {} -> {})\n'.format( - str(self), exit_code, new_code)) + new_code, + "{} finished (exit_code {} -> {})\n".format(str(self), exit_code, new_code), + ) return new_code def get_unpack_directory(self, env): @@ -400,12 +394,13 @@ class PopulateActionBase(Action): # pragma: no cover def download_and_unpack_artifact(self, name, dirname): url = self.get_blob_upload_url(name) headers = self.get_authz_headers() - curl = ['curl', '-sk'] + [ - '-H{}:{}'.format(name, value) - for name, value in headers.items() - ] + [url] + curl = ( + ["curl", "-sk"] + + ["-H{}:{}".format(name, value) for name, value in headers.items()] + + [url] + ) - untar = ['sudo', 'tar', '-zxf', '-', '-C', dirname] + untar = ["sudo", "tar", "-zxf", "-", "-C", dirname] env = self.get_env() return env.host_runcmd(curl, untar) @@ -413,11 +408,11 @@ class PopulateActionBase(Action): # pragma: no cover class PopulateSystreeAction(PopulateActionBase): # pragma: no cover - step_field = 'systree_name' - param_name = 'systree_name' + step_field = "systree_name" + param_name = "systree_name" def __str__(self): - return 'populate-systree' + return "populate-systree" def get_unpack_directory(self, env): return env.get_systree_directory() @@ -428,11 +423,11 @@ class PopulateSystreeAction(PopulateActionBase): # pragma: no cover class PopulateWorkspaceAction(PopulateActionBase): # pragma: no cover - step_field = 'workspace_name' - param_name = 'workspace_name' + step_field = "workspace_name" + param_name = "workspace_name" def __str__(self): - return 'populate-workspace' + return "populate-workspace" def get_unpack_directory(self, env): return env.get_workspace_directory() @@ -443,7 +438,6 @@ class PopulateWorkspaceAction(PopulateActionBase): # pragma: no cover class GitAction(Action): # pragma: no cover - def encode_parameters(self, params): pass @@ -451,44 +445,43 @@ class GitAction(Action): # pragma: no cover env = self.get_env() workspace = env.get_workspace_directory() - git_dir = params.get('git_dir') + git_dir = params.get("git_dir") if git_dir is None: - env.report(1, 'git_dir not provided\n') - if git_dir.startswith('/') or '..' in git_dir: - env.report(1, 'git_dir not acceptable\n') + env.report(1, "git_dir not provided\n") + if git_dir.startswith("/") or ".." in git_dir: + env.report(1, "git_dir not acceptable\n") - git_url = params.get('git_url') + git_url = params.get("git_url") if git_url is None: - env.report(1, 'git_url not provided\n') + env.report(1, "git_url not provided\n") pathname = os.path.join(workspace, git_dir) if os.path.exists(pathname): - argv = ['git', 'remote', '-v', 'update', '--prune'] + argv = ["git", "remote", "-v", "update", "--prune"] cwd = pathname else: - argv = ['git', 'clone', '-v', git_url, git_dir] + argv = ["git", "clone", "-v", git_url, git_dir] cwd = workspace exit_code = env.host_runcmd(argv, cwd=cwd) - env.report(exit_code, 'git finished (exit code %d)\n' % exit_code) + env.report(exit_code, "git finished (exit code %d)\n" % exit_code) return exit_code class GitMirrorAction(Action): # pragma: no cover - def encode_parameters(self, params): pass def execute(self, params, step): env = self.get_env() workspace = env.get_workspace_directory() - mirrors = os.path.join(workspace, '.mirrors') + mirrors = os.path.join(workspace, ".mirrors") - if step.get('where') != 'host': + if step.get("where") != "host": env.report(1, '"where" must be "host"\n') return 1 - sources = params.get('sources') + sources = params.get("sources") if sources is None: env.report(1, '"sources" parameter not provided\n') return 1 @@ -496,17 +489,15 @@ class GitMirrorAction(Action): # pragma: no cover try: exit_code = self.git_mirror(env, sources, mirrors) except Exception as e: - env.report(1, 'Caught exception: {}\n'.format(e)) + env.report(1, "Caught exception: {}\n".format(e)) return 1 - env.report( - exit_code, - 'git mirror action finished (exit code %d)\n' % exit_code) + env.report(exit_code, "git mirror action finished (exit code %d)\n" % exit_code) return exit_code def git_mirror(self, env, sources, mirrors): if not os.path.exists(mirrors): - env.report(None, 'mkdir {}\n'.format(mirrors)) + env.report(None, "mkdir {}\n".format(mirrors)) os.mkdir(mirrors) checked = self.check_sources(sources) @@ -519,8 +510,8 @@ class GitMirrorAction(Action): # pragma: no cover def check_sources(self, sources): checked = [] for source in sources: - name = source.get('name') - repo = source.get('repo') + name = source.get("name") + repo = source.get("repo") if name is None: raise Exception('source lacks "name" field: {}'.format(source)) if repo is None: @@ -529,27 +520,26 @@ class GitMirrorAction(Action): # pragma: no cover return checked def mirror(self, env, mirrors, name, url): - env.report(None, 'git_mirror: mirrors: {}\n'.format(mirrors)) - env.report(None, 'git_mirror: name: {}\n'.format(name)) - env.report(None, 'git_mirror: url: {}\n'.format(url)) + env.report(None, "git_mirror: mirrors: {}\n".format(mirrors)) + env.report(None, "git_mirror: name: {}\n".format(name)) + env.report(None, "git_mirror: url: {}\n".format(url)) dirname = os.path.join(mirrors, name) - env.report(None, 'git_mirror: dirname: {}\n'.format(dirname)) + env.report(None, "git_mirror: dirname: {}\n".format(dirname)) if os.path.exists(dirname): - argv = ['git', 'remote', 'update', '--prune'] + argv = ["git", "remote", "update", "--prune"] cwd = dirname else: - argv = ['git', 'clone', '--mirror', url, name] + argv = ["git", "clone", "--mirror", url, name] cwd = mirrors os.mkdir(dirname) - env.report(None, 'Running: {} in {}\n'.format(argv, cwd)) + env.report(None, "Running: {} in {}\n".format(argv, cwd)) return env.host_runcmd(argv, cwd=cwd) class RsyncAction(Action): # pragma: no cover - def encode_parameters(self, params): pass @@ -557,40 +547,41 @@ class RsyncAction(Action): # pragma: no cover env = self.get_env() workspace = env.get_workspace_directory() - rsync_src = params.get('rsync_src') + rsync_src = params.get("rsync_src") if rsync_src is None: - env.report(1, 'rsync_src not provided\n') + env.report(1, "rsync_src not provided\n") if not self._is_relative(rsync_src): - env.report(1, 'rsync_src not acceptable\n') + env.report(1, "rsync_src not acceptable\n") - rsync_target = params.get('rsync_target') + rsync_target = params.get("rsync_target") if rsync_target is None: - env.report(1, 'git_url not provided\n') + env.report(1, "git_url not provided\n") if not self._remote(rsync_target): - env.report(1, 'rsync_target not acceptable\n') + env.report(1, "rsync_target not acceptable\n") argv = [ - 'rsync', '-av', '--delete-after', - './{}/.'.format(rsync_src), - '{}/.'.format(rsync_target), + "rsync", + "-av", + "--delete-after", + "./{}/.".format(rsync_src), + "{}/.".format(rsync_target), ] exit_code = env.host_runcmd(argv, cwd=workspace) - env.report(exit_code, 'rsync finished (exit code %d)\n' % exit_code) + env.report(exit_code, "rsync finished (exit code %d)\n" % exit_code) return exit_code def _is_relative(self, src): - if src.startswith('/'): + if src.startswith("/"): return False - if '../' in src: + if "../" in src: return False return True def _remote(self, target): - return ':' in target + return ":" in target class DputAction(Action): # pragma: no cover - def encode_parameters(self, params): pass @@ -600,12 +591,12 @@ class DputAction(Action): # pragma: no cover apt_server = self._cc.get_apt_server() config = self.get_dput_config(apt_server) - logging.debug('dput config:\n%s', config) + logging.debug("dput config:\n%s", config) filename = self.create_dput_config_file(config) - argv = ['sh', '-c', 'dput -c {} ick *.changes'.format(filename)] + argv = ["sh", "-c", "dput -c {} ick *.changes".format(filename)] exit_code = env.host_runcmd(argv, cwd=workspace) - env.report(exit_code, 'dput finished (exit code %d)\n' % exit_code) + env.report(exit_code, "dput finished (exit code %d)\n" % exit_code) os.remove(config) return exit_code @@ -617,7 +608,7 @@ class DputAction(Action): # pragma: no cover return filename def get_dput_config(self, apt_server): - template = '''\ + template = """\ [ick] login = incoming fqdn = {apt_server} @@ -626,12 +617,11 @@ incoming = /srv/apt/incoming allow_unsigned_uploads = 1 check_version = 0 run_dinstall = 0 -''' +""" return template.format(apt_server=apt_server) class NotifyAction(Action): # pragma: no cover - def encode_parameters(self, params): pass @@ -641,45 +631,43 @@ class NotifyAction(Action): # pragma: no cover assert cc is not None build_id = self.get_build_id() - env.report(None, 'Notifying about build ending\n') + env.report(None, "Notifying about build ending\n") - build_path = '/builds/{}'.format(build_id) + build_path = "/builds/{}".format(build_id) build = cc.show(build_path) - params = build.get('parameters', {}) - if 'notify' not in params: - env.report( - 0, - 'NOT notifying about build ending: no "notify" parameter.\n') + params = build.get("parameters", {}) + if "notify" not in params: + env.report(0, 'NOT notifying about build ending: no "notify" parameter.\n') return - recipients = params['notify'] + recipients = params["notify"] log = cc.get_log(build_id) - log = log.decode('utf-8') + log = log.decode("utf-8") + log = "\n".join(log.splitlines()[-1000:]) notify = { - 'recipients': recipients, - 'build': self.mangle_build(build), - 'log': log, + "recipients": recipients, + "build": self.mangle_build(build), + "log": log, } cc.notify(notify) - env.report(0, 'Notified about build {} ending\n'.format(build_id)) + env.report(0, "Notified about build {} ending\n".format(build_id)) def mangle_build(self, build): b = copy.deepcopy(build) - exit_code = build.get('exit_code') + exit_code = build.get("exit_code") if exit_code is None: - b['status'] = 'BUILDING' + b["status"] = "BUILDING" elif exit_code == 0: - b['status'] = 'SUCCESS' + b["status"] = "SUCCESS" else: - b['status'] = 'FAILED' + b["status"] = "FAILED" return b def make_directory_empty(env, dirname): - return env.runcmd( - ['sudo', 'find', dirname, '-mindepth', '1', '-delete']) + return env.runcmd(["sudo", "find", dirname, "-mindepth", "1", "-delete"]) diff --git a/ick2/buildsapi.py b/ick2/buildsapi.py index 8862c63..2efc3d1 100644 --- a/ick2/buildsapi.py +++ b/ick2/buildsapi.py @@ -1,4 +1,4 @@ -# Copyright (C) 2017-2018 Lars Wirzenius +# Copyright (C) 2017-2019 Lars Wirzenius # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as published by # the Free Software Foundation, either version 3 of the License, or @@ -60,7 +60,7 @@ class BuildsAPI(ick2.ResourceApiBase): # pragma: no cover raise ick2.MethodNotAllowed('Updating builds directly is not allowed') def list(self, **kwargs): - result = super().list() + result = super().list(**kwargs) items = result[self._type_name] items.sort(key=lambda x: x.get('build_number')) result[self._type_name] = items diff --git a/ick2/buildsm.py b/ick2/buildsm.py index 9177e09..8d71fde 100644 --- a/ick2/buildsm.py +++ b/ick2/buildsm.py @@ -17,15 +17,14 @@ import ick2 -BUILD_TRIGGERED = 'triggered' -BUILD_BUILDING = 'building' -BUILD_NOTIFYING = 'notifying' -BUILD_DONE = 'done' -BUILD_FAILED = 'failed' +BUILD_TRIGGERED = "triggered" +BUILD_BUILDING = "building" +BUILD_NOTIFYING = "notifying" +BUILD_DONE = "done" +BUILD_FAILED = "failed" class StateMachine: - def __init__(self, get_state, set_state): self.transitions = {} self.get_state = get_state @@ -48,7 +47,6 @@ class StateMachine: class BuildStateMachine: - def __init__(self, build): self.build = build self.sm = self.init_sm() @@ -80,10 +78,10 @@ class BuildStateMachine: return sm def get_state(self): - return self.build.resource['status'] + return self.build.resource["status"] def set_state(self, state): - self.build.resource['status'] = state + self.build.resource["status"] = state def handle_event(self, event): old_state = self.get_state() @@ -101,7 +99,7 @@ class BuildStateMachine: graph = self.build.get_graph() action_ids = graph.find_actions(ick2.ACTION_READY) if not action_ids: # pragma: no cover - self.build.resource['exit_code'] = 0 + self.build.resource["exit_code"] = 0 return BUILD_DONE, None action_id = action_ids[0] @@ -113,7 +111,7 @@ class BuildStateMachine: graph = self.build.get_graph() action_ids = graph.find_actions(ick2.ACTION_READY) if not action_ids: # pragma: no cover - self.build.resource['exit_code'] = 0 + self.build.resource["exit_code"] = 0 return BUILD_DONE, None action_id = action_ids[0] @@ -122,34 +120,32 @@ class BuildStateMachine: return BUILD_NOTIFYING, (action_id, action) def mark_action_done(self, event): - self.build.resource['exit_code'] = event.exit_code + self.build.resource["exit_code"] = event.exit_code graph = self.build.get_graph() graph.set_action_status(event.action_id, ick2.ACTION_DONE) graph.unblock() if graph.has_more_to_do(): return BUILD_BUILDING, None - - self.add_notification_action() - return BUILD_NOTIFYING, None + return BUILD_DONE, None def add_notification_action(self): action = { - 'action': 'notify', + "action": "notify", } graph = self.build.get_graph() graph.append_action(action, ick2.ACTION_READY, depends=[]) def mark_notification_done(self, event): if event.exit_code not in (0, None): # pragma: no cover - self.build.resource['exit_code'] = event.exit_code + self.build.resource["exit_code"] = event.exit_code graph = self.build.get_graph() graph.set_action_status(event.action_id, ick2.ACTION_DONE) graph.unblock() if graph.has_more_to_do(): # pragma: no cover return BUILD_NOTIFYING, None - if self.build.resource.get('exit_code') in (0, None): - self.build.resource['exit_code'] = 0 + if self.build.resource.get("exit_code") in (0, None): + self.build.resource["exit_code"] = 0 return BUILD_DONE, None return BUILD_FAILED, None @@ -157,9 +153,9 @@ class BuildStateMachine: def mark_build_failed(self, event): graph = self.build.get_graph() graph.set_action_status(event.action_id, ick2.BUILD_FAILED) - self.build.resource['exit_code'] = event.exit_code + self.build.resource["exit_code"] = event.exit_code self.add_notification_action() - return BUILD_NOTIFYING, None + return BUILD_FAILED, None # Thing should be something we can create a BuildEvent from. @@ -171,8 +167,8 @@ def create_build_event(thing): return NeedWorkEvent() if isinstance(thing, dict): - exit_code = thing.get('exit_code') - action_id = thing.get('action_id') + exit_code = thing.get("exit_code") + action_id = thing.get("action_id") if exit_code is None: return PartialActionOutputEvent() if exit_code == 0: @@ -181,31 +177,26 @@ def create_build_event(thing): class BuildEvent: # pragma: no cover - - event_type = 'BuildEvent' + event_type = "BuildEvent" def __str__(self): return self.event_type class BuildStartsEvent(BuildEvent): - - event_type = 'build-starts' + event_type = "build-starts" class NeedWorkEvent(BuildEvent): - - event_type = 'need-work' + event_type = "need-work" class PartialActionOutputEvent(BuildEvent): - - event_type = 'partial-output' + event_type = "partial-output" class ActionFinishedEvent(BuildEvent): - - event_type = 'action-finished' + event_type = "action-finished" def __init__(self, action_id): self.action_id = action_id @@ -213,8 +204,7 @@ class ActionFinishedEvent(BuildEvent): class ActionFailedEvent(BuildEvent): - - event_type = 'action-failed' + event_type = "action-failed" def __init__(self, action_id, exit_code): self.action_id = action_id @@ -222,6 +212,5 @@ class ActionFailedEvent(BuildEvent): class UnexpectedEvent(Exception): # pragma: no cover - def __init__(self, event, state): - super().__init__('Did not expect %s in %s' % (event, state)) + super().__init__("Did not expect %s in %s" % (event, state)) diff --git a/ick2/notificationapi.py b/ick2/notificationapi.py index ab261bd..877b929 100644 --- a/ick2/notificationapi.py +++ b/ick2/notificationapi.py @@ -17,29 +17,32 @@ import ick2 -class NotificationAPI: +N = 1000 # Max number of line to include in log + +class NotificationAPI: def __init__(self, config): self._config = config def find_missing_route(self, missing_path): return [ { - 'method': 'POST', - 'path': '/notify', - 'callback': self.notify, + "method": "POST", + "path": "/notify", + "callback": self.notify, }, ] def notify(self, content_type, body, **kwargs): - ick2.log.log('info', msg_text='Notification requested', kwargs=kwargs) + ick2.log.log("info", msg_text="Notification requested", kwargs=kwargs) - recipients = body.get('recipients', []) - build = body.get('build', {}) - log = body.get('log', '') + recipients = body.get("recipients", []) + build = body.get("build", {}) + log = body.get("log", "") + log = "".join(f"{line}\n" for line in log.splitlines()[-N:]) sendmail = ick2.Sendmail() sendmail.set_config(self._config) sendmail.send(recipients, build, log) - return ick2.OK('') + return ick2.OK("") diff --git a/ick2/persistent.py b/ick2/persistent.py index c5e2840..1d79e3d 100644 --- a/ick2/persistent.py +++ b/ick2/persistent.py @@ -1,4 +1,4 @@ -# Copyright (C) 2018 Lars Wirzenius +# Copyright (C) 2018-2019 Lars Wirzenius # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as published by @@ -27,9 +27,6 @@ import ick2 class PersistentStateInterface: # pragma: no cover - def get_resource_kinds(self): - raise NotImplementedError() - def get_resource_ids(self, kind): raise NotImplementedError() @@ -79,9 +76,6 @@ class FilePersistentState(PersistentStateInterface): dirname = self._dirname(kind) return os.path.join(dirname, self._safe(rid)) - def get_resource_kinds(self): - return self._unsafe_list(os.listdir(self._dir)) - def has_resource(self, kind, rid): filename = self._filename(kind, rid) return os.path.exists(filename) diff --git a/ick2/persistent_tests.py b/ick2/persistent_tests.py index 8acb141..de279a1 100644 --- a/ick2/persistent_tests.py +++ b/ick2/persistent_tests.py @@ -1,4 +1,4 @@ -# Copyright (C) 2018 Lars Wirzenius +# Copyright (C) 2018-2019 Lars Wirzenius # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as published by @@ -35,9 +35,6 @@ class FilePersistentStateTests(unittest.TestCase): def test_returns_dirname(self): self.assertEqual(self.state.get_directory(), self.tempdir) - def test_has_no_resource_kinds_initially(self): - self.assertEqual(self.state.get_resource_kinds(), []) - def test_has_no_resources_initially(self): self.assertEqual(self.state.get_resource_ids('silly'), []) @@ -50,7 +47,6 @@ class FilePersistentStateTests(unittest.TestCase): r = ick2.resource_from_dict(as_dict) self.state.write_resource('silly', '#1', r) self.assertTrue(self.state.has_resource('silly', '#1')) - self.assertEqual(self.state.get_resource_kinds(), ['silly']) self.assertEqual(self.state.get_resource_ids('silly'), ['#1']) r2 = self.state.get_resource('silly', '#1') diff --git a/ick2/trans.py b/ick2/trans.py index f1c8e5e..c5dc22f 100644 --- a/ick2/trans.py +++ b/ick2/trans.py @@ -1,4 +1,4 @@ -# Copyright (C) 2018 Lars Wirzenius +# Copyright (C) 2018-2019 Lars Wirzenius # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as published by @@ -59,9 +59,6 @@ class TransactionalState: raise ick2.NotFound(kind=kind, rid=rid) return TransactionalResource(self.state, kind, rid) - def get_resource_kinds(self): - return self.state.get_resource_kinds() - def get_resource_ids(self, kind): return self.state.get_resource_ids(kind) diff --git a/ick2/version.py b/ick2/version.py index 4c6e147..27b4c9b 100644 --- a/ick2/version.py +++ b/ick2/version.py @@ -1,2 +1,2 @@ -__version__ = "0.53.2+git" -__version_info__ = (0, 53, 2, '+git') +__version__ = "0.55+git" +__version_info__ = (0, 55, '+git') diff --git a/notification_service.py b/notification_service.py index bd6bad1..f8e5c55 100644 --- a/notification_service.py +++ b/notification_service.py @@ -1,5 +1,5 @@ #!/usr/bin/python3 -# Copyright (C) 2018 Lars Wirzenius +# Copyright (C) 2018-2019 Lars Wirzenius # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as published by @@ -123,4 +123,4 @@ app = main() if __name__ == '__main__': print('running in debug mode') - app.run(host='127.0.0.1', port=12767) + app.run(host='127.0.0.1', port=6666) diff --git a/pipelines/systrees.ick b/pipelines/systrees.ick index b6da798..bd52b1c 100644 --- a/pipelines/systrees.ick +++ b/pipelines/systrees.ick @@ -18,6 +18,13 @@ pipelines: - packages - artifact_name actions: + - shell: | + lsb_release -a + dpkg -l debootstrap + sudo chown root:root . + ls -la + where: host + - debootstrap: auto mirror: http://deb.debian.org/debian where: host @@ -1,5 +1,5 @@ #!/bin/sh -# Copyright (C) 2017-2018 Lars Wirzenius +# Copyright (C) 2017-2019 Lars Wirzenius # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as published by @@ -26,7 +26,7 @@ else fi gunicorn3 \ - --bind 127.0.0.1:12765 \ + --bind 127.0.0.1:3333 \ --log-file "$GUNICORN_LOG" \ --log-level debug \ ick_controller:app |