From 9d4e01e2df83a108c2c44cc4471666b5cecf4c36 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Mon, 23 Apr 2018 13:27:22 +0300 Subject: Change: all actions must have a "where" --- NEWS | 4 ++ ick2/__init__.py | 2 +- ick2/pipelineapi.py | 18 ++++++++ ick2/workapi.py | 1 + ick2/workapi_tests.py | 11 +++-- ick2/workerapi.py | 24 ---------- yarns/150-pipelines.yarn | 39 ++++++++-------- yarns/400-build.yarn | 110 +++++++++++++++++++++++++++------------------- yarns/500-build-fail.yarn | 19 ++++---- 9 files changed, 126 insertions(+), 102 deletions(-) diff --git a/NEWS b/NEWS index 85c2db8..0e55170 100644 --- a/NEWS +++ b/NEWS @@ -23,6 +23,10 @@ Version 0.46+git, not yet released * Worker manager now sets `LC_ALL` and `DEBIAN_FRONTEND` variables inside the container. +* Each action in a pipeline MUST now have a `where` field. Previously + you could leave it out and it would act as if `where: host` was + specified. + Version 0.46, released 2018-04-22 ---------------------------------- diff --git a/ick2/__init__.py b/ick2/__init__.py index 8b784c9..8ceb989 100644 --- a/ick2/__init__.py +++ b/ick2/__init__.py @@ -42,7 +42,7 @@ from .apibase import APIbase, ResourceApiBase from .buildsapi import BuildsAPI from .logapi import LogAPI from .versionapi import VersionAPI -from .pipelineapi import PipelineAPI +from .pipelineapi import (PipelineAPI, NoWhere) from .projectapi import ProjectAPI from .workapi import WorkAPI from .workerapi import WorkerAPI diff --git a/ick2/pipelineapi.py b/ick2/pipelineapi.py index 22db2b8..da774d4 100644 --- a/ick2/pipelineapi.py +++ b/ick2/pipelineapi.py @@ -23,3 +23,21 @@ class PipelineAPI(ick2.ResourceApiBase): def get_resource_name(self, resource): return resource.get('pipeline') + + def create(self, body, **kwargs): + resource = self.mangle_new_resource(body) + + actions = resource.get('actions', []) + for action in actions: + where = action.get('where') + if where is None: + raise NoWhere(action) + + return super().create(body, **kwargs) + + +class NoWhere(Exception): + + def __init__(self, action): + super().__init__( + 'Every action MUST specify a "where": {}'.format(action)) diff --git a/ick2/workapi.py b/ick2/workapi.py index fa9c2bc..1f8c827 100644 --- a/ick2/workapi.py +++ b/ick2/workapi.py @@ -106,6 +106,7 @@ class WorkAPI(ick2.APIbase): parameters = project.get('parameters', {}) create_workspace = { 'action': 'create_workspace', + 'where': 'host', } actions = [create_workspace] + self._get_actions(project) build = { diff --git a/ick2/workapi_tests.py b/ick2/workapi_tests.py index 92d34e5..d2d9df6 100644 --- a/ick2/workapi_tests.py +++ b/ick2/workapi_tests.py @@ -37,8 +37,8 @@ class WorkAPITests(unittest.TestCase): pipeline = { 'pipeline': 'build', 'actions': [ - {'shell': 'step-1'}, - {'shell': 'step-2'}, + {'shell': 'step-1', 'where': 'host'}, + {'shell': 'step-2', 'where': 'host'}, ], } @@ -88,6 +88,7 @@ class WorkAPITests(unittest.TestCase): }, 'step': { 'action': 'create_workspace', + 'where': 'host', }, 'log': '/logs/foo/1', } @@ -113,6 +114,7 @@ class WorkAPITests(unittest.TestCase): }, 'step': { 'action': 'create_workspace', + 'where': 'host', }, 'log': '/logs/foo/1', } @@ -139,7 +141,7 @@ class WorkAPITests(unittest.TestCase): work.update_work(done) # We should get the next step now. - expected['step'] = {'shell': 'step-1'} + expected['step'] = {'shell': 'step-1', 'where': 'host'} self.assertEqual(work.get_work('asterix'), expected) # Finish the step. @@ -147,7 +149,7 @@ class WorkAPITests(unittest.TestCase): work.update_work(done) # We should get the next step now. - expected['step'] = {'shell': 'step-2'} + expected['step'] = {'shell': 'step-2', 'where': 'host'} self.assertEqual(work.get_work('asterix'), expected) # Finish the step. @@ -179,6 +181,7 @@ class WorkAPITests(unittest.TestCase): }, 'step': { 'action': 'create_workspace', + 'where': 'host', }, 'log': '/logs/foo/1', } diff --git a/ick2/workerapi.py b/ick2/workerapi.py index 2e53758..7f2f0b7 100644 --- a/ick2/workerapi.py +++ b/ick2/workerapi.py @@ -21,30 +21,6 @@ class WorkerAPI(ick2.ResourceApiBase): # pragma: no cover def __init__(self, state): super().__init__('workers', state) - def get_routes(self, path): - return [ - { - 'method': 'POST', - 'path': path, - 'callback': self.POST(self.create), - }, - { - 'method': 'GET', - 'path': path, - 'callback': self.GET(self.list), - }, - { - 'method': 'GET', - 'path': '{}/'.format(path), - 'callback': self.GET(self.show), - }, - { - 'method': 'DELETE', - 'path': '{}/'.format(path), - 'callback': self.DELETE(self.delete), - }, - ] - def get_resource_name(self, resource): return resource['worker'] diff --git a/yarns/150-pipelines.yarn b/yarns/150-pipelines.yarn index dd5cde2..a63ce02 100644 --- a/yarns/150-pipelines.yarn +++ b/yarns/150-pipelines.yarn @@ -31,13 +31,16 @@ They're described like resources like this: }, "actions": [ { - "shell": "git clone git://git.liw.fi/ick2-website src" } + "shell": "git clone git://git.liw.fi/ick2-website src", + "where": "host" }, { - "shell": "cd src && ikiwiki --setup ikiwiki.setup" + "shell": "cd src && ikiwiki --setup ikiwiki.setup", + "where": "host" }, { - "shell": "cd html && rsync -a --delete . server::/srv/http/ick2/." + "shell": "cd html && rsync -a --delete . server::/srv/http/ick2/.", + "where": "host" } ] } @@ -75,9 +78,9 @@ running them. We start by starting an instance of the controller. ... { ... "pipeline": "build_website", ... "actions": [ - ... { "shell": "git clone git://repo src" }, - ... { "shell": "mkdir html" }, - ... { "shell": "ikiwiki src html" } + ... { "where": "host", "shell": "git clone git://repo src" }, + ... { "where": "host", "shell": "mkdir html" }, + ... { "where": "host", "shell": "ikiwiki src html" } ... ] ... } THEN result has status code 201 @@ -85,9 +88,9 @@ running them. We start by starting an instance of the controller. ... { ... "pipeline": "build_website", ... "actions": [ - ... { "shell": "git clone git://repo src" }, - ... { "shell": "mkdir html" }, - ... { "shell": "ikiwiki src html" } + ... { "where": "host", "shell": "git clone git://repo src" }, + ... { "where": "host", "shell": "mkdir html" }, + ... { "where": "host", "shell": "ikiwiki src html" } ... ] ... } @@ -107,9 +110,9 @@ Creating a new pipeline with the same name is forbidden. ... { ... "pipeline": "build_website", ... "actions": [ - ... { "shell": "git clone git://repo src" }, - ... { "shell": "mkdir html" }, - ... { "shell": "ikiwiki src html" } + ... { "where": "host", "shell": "git clone git://repo src" }, + ... { "where": "host", "shell": "mkdir html" }, + ... { "where": "host", "shell": "ikiwiki src html" } ... ] ... } ... ] @@ -123,9 +126,9 @@ Creating a new pipeline with the same name is forbidden. ... { ... "pipeline": "build_website", ... "actions": [ - ... { "shell": "git clone git://repo src" }, - ... { "shell": "mkdir html" }, - ... { "shell": "ikiwiki src html" } + ... { "where": "host", "shell": "git clone git://repo src" }, + ... { "where": "host", "shell": "mkdir html" }, + ... { "where": "host", "shell": "ikiwiki src html" } ... ] ... } @@ -134,7 +137,7 @@ Creating a new pipeline with the same name is forbidden. ... { ... "pipeline": "build_website", ... "actions": [ - ... { "shell": "build-it" } + ... { "where": "host", "shell": "build-it" } ... ] ... } THEN result has status code 200 @@ -142,7 +145,7 @@ Creating a new pipeline with the same name is forbidden. ... { ... "pipeline": "build_website", ... "actions": [ - ... { "shell": "build-it" } + ... { "where": "host", "shell": "build-it" } ... ] ... } @@ -152,7 +155,7 @@ Creating a new pipeline with the same name is forbidden. ... { ... "pipeline": "build_website", ... "actions": [ - ... { "shell": "build-it" } + ... { "where": "host", "shell": "build-it" } ... ] ... } diff --git a/yarns/400-build.yarn b/yarns/400-build.yarn index 796bd20..13257a0 100644 --- a/yarns/400-build.yarn +++ b/yarns/400-build.yarn @@ -47,8 +47,8 @@ Add up a project with some named pipelines. ... { ... "pipeline": "construct", ... "actions": [ - ... { "shell": "day 1" }, - ... { "shell": "day 2" } + ... { "where": "host", "shell": "day 1" }, + ... { "where": "host", "shell": "day 2" } ... ] ... } THEN result has status code 201 @@ -126,7 +126,8 @@ the worker to construct a new workspace for the build. ... "foo": "bar" ... }, ... "step": { - ... "action": "create_workspace" + ... "action": "create_workspace", + ... "where": "host" ... } ... } @@ -143,7 +144,8 @@ the worker to construct a new workspace for the build. ... "foo": "bar" ... }, ... "step": { - ... "action": "create_workspace" + ... "action": "create_workspace", + ... "where": "host" ... } ... } @@ -171,7 +173,8 @@ User can now see pipeline is running and which worker is building it. ... "foo": "bar" ... }, ... "step": { - ... "action": "create_workspace" + ... "action": "create_workspace", + ... "where": "host" ... } ... } ... } @@ -188,9 +191,9 @@ User can now see pipeline is running and which worker is building it. ... "worker": "obelix", ... "project": "rome", ... "actions": [ - ... { "action": "create_workspace" }, - ... { "shell": "day 1" }, - ... { "shell": "day 2" } + ... { "where": "host", "action": "create_workspace" }, + ... { "where": "host", "shell": "day 1" }, + ... { "where": "host", "shell": "day 2" } ... ], ... "current_action": 0, ... "parameters": { @@ -236,7 +239,8 @@ Worker requests more work, and gets the first actual build step. ... "foo": "bar" ... }, ... "step": { - ... "shell": "day 1" + ... "shell": "day 1", + ... "where": "host" ... } ... } @@ -271,7 +275,8 @@ didnt't finish. ... "foo": "bar" ... }, ... "step": { - ... "shell": "day 1" + ... "shell": "day 1", + ... "where": "host" ... } ... } @@ -315,9 +320,9 @@ The build status now shows the next step as the active one. ... "worker": "obelix", ... "project": "rome", ... "actions": [ - ... { "action": "create_workspace" }, - ... { "shell": "day 1" }, - ... { "shell": "day 2" } + ... { "where": "host", "action": "create_workspace" }, + ... { "where": "host", "shell": "day 1" }, + ... { "where": "host", "shell": "day 2" } ... ], ... "current_action": 2, ... "parameters": { @@ -344,7 +349,8 @@ Now there's another step to do. ... "foo": "bar" ... }, ... "step": { - ... "shell": "day 2" + ... "shell": "day 2", + ... "where": "host" ... } ... } @@ -364,7 +370,8 @@ User sees changed status. ... "foo": "bar" ... }, ... "step": { - ... "shell": "day 2" + ... "shell": "day 2", + ... "where": "host" ... }, ... "log": "/logs/rome/1" ... } @@ -411,9 +418,9 @@ no current action. ... "worker": "obelix", ... "project": "rome", ... "actions": [ - ... { "action": "create_workspace" }, - ... { "shell": "day 1" }, - ... { "shell": "day 2" } + ... { "where": "host", "action": "create_workspace" }, + ... { "where": "host", "shell": "day 1" }, + ... { "where": "host", "shell": "day 2" } ... ], ... "current_action": null, ... "parameters": { @@ -434,9 +441,9 @@ no current action. ... "worker": "obelix", ... "project": "rome", ... "actions": [ - ... { "action": "create_workspace" }, - ... { "shell": "day 1" }, - ... { "shell": "day 2" } + ... { "where": "host", "action": "create_workspace" }, + ... { "where": "host", "shell": "day 1" }, + ... { "where": "host", "shell": "day 2" } ... ], ... "current_action": null, ... "parameters": { @@ -469,7 +476,8 @@ Start build again. This should become build number 2. ... "foo": "bar" ... }, ... "step": { - ... "action": "create_workspace" + ... "action": "create_workspace", + ... "where": "host" ... } ... } @@ -485,9 +493,9 @@ Start build again. This should become build number 2. ... "worker": "obelix", ... "project": "rome", ... "actions": [ - ... { "action": "create_workspace" }, - ... { "shell": "day 1" }, - ... { "shell": "day 2" } + ... { "where": "host", "action": "create_workspace" }, + ... { "where": "host", "shell": "day 1" }, + ... { "where": "host", "shell": "day 2" } ... ], ... "current_action": null, ... "parameters": { @@ -502,9 +510,9 @@ Start build again. This should become build number 2. ... "worker": "obelix", ... "project": "rome", ... "actions": [ - ... { "action": "create_workspace" }, - ... { "shell": "day 1" }, - ... { "shell": "day 2" } + ... { "where": "host", "action": "create_workspace" }, + ... { "where": "host", "shell": "day 1" }, + ... { "where": "host", "shell": "day 2" } ... ], ... "current_action": 0, ... "parameters": { @@ -540,7 +548,8 @@ Start build again. This should become build number 2. ... "foo": "bar" ... }, ... "step": { - ... "shell": "day 1" + ... "shell": "day 1", + ... "where": "host" ... } ... } @@ -583,9 +592,9 @@ Start build again. This should become build number 2. ... "worker": "obelix", ... "project": "rome", ... "actions": [ - ... { "action": "create_workspace" }, - ... { "shell": "day 1" }, - ... { "shell": "day 2" } + ... { "where": "host", "action": "create_workspace" }, + ... { "where": "host", "shell": "day 1" }, + ... { "where": "host", "shell": "day 2" } ... ], ... "current_action": null, ... "parameters": { @@ -600,9 +609,9 @@ Start build again. This should become build number 2. ... "worker": "obelix", ... "project": "rome", ... "actions": [ - ... { "action": "create_workspace" }, - ... { "shell": "day 1" }, - ... { "shell": "day 2" } + ... { "where": "host", "action": "create_workspace" }, + ... { "where": "host", "shell": "day 1" }, + ... { "where": "host", "shell": "day 2" } ... ], ... "current_action": null, ... "parameters": { @@ -646,7 +655,7 @@ Add a couple of projects. ... { ... "pipeline": "do_something", ... "actions": [ - ... { "shell": "something" } + ... { "where": "host", "shell": "something" } ... ] ... } THEN result has status code 201 @@ -684,7 +693,8 @@ Build the first project. WHEN worker-manager makes request GET /work/obelix THEN result is step ... { - ... "action": "create_workspace" + ... "action": "create_workspace", + ... "where": "host" ... } WHEN worker-manager makes request POST /work with a valid token and body @@ -703,7 +713,8 @@ Build the first project. WHEN worker-manager makes request GET /work/obelix THEN result is step ... { - ... "shell": "something" + ... "shell": "something", + ... "where": "host" ... } WHEN worker-manager makes request POST /work with a valid token and body @@ -731,7 +742,8 @@ Build second project. WHEN worker-manager makes request GET /work/obelix THEN result is step ... { - ... "action": "create_workspace" + ... "action": "create_workspace", + ... "where": "host" ... } WHEN worker-manager makes request POST /work with a valid token and body @@ -749,7 +761,8 @@ Build second project. WHEN worker-manager makes request GET /work/obelix THEN result is step ... { - ... "shell": "something" + ... "shell": "something", + ... "where": "host" ... } WHEN worker-manager makes request POST /work with a valid token and body @@ -802,7 +815,7 @@ Add a couple of projects. ... { ... "pipeline": "do_something", ... "actions": [ - ... { "shell": "something" } + ... { "where": "host", "shell": "something" } ... ] ... } THEN result has status code 201 @@ -855,7 +868,8 @@ Trigger both projects. WHEN asterix makes request GET /work/asterix THEN result is step ... { - ... "action": "create_workspace" + ... "action": "create_workspace", + ... "where": "host" ... } WHEN user requests list of builds @@ -868,7 +882,8 @@ Trigger both projects. WHEN obelix makes request GET /work/obelix THEN result is step ... { - ... "action": "create_workspace" + ... "action": "create_workspace", + ... "where": "host" ... } WHEN user requests list of builds @@ -890,13 +905,15 @@ Trigger both projects. WHEN asterix makes request GET /work/asterix THEN result is step ... { - ... "shell": "something" + ... "shell": "something", + ... "where": "host" ... } WHEN obelix makes request GET /work/obelix THEN result is step ... { - ... "action": "create_workspace" + ... "action": "create_workspace", + ... "where": "host" ... } WHEN obelix makes request POST /work with a valid token and body @@ -915,7 +932,8 @@ Trigger both projects. WHEN obelix makes request GET /work/obelix THEN result is step ... { - ... "shell": "something" + ... "shell": "something", + ... "where": "host" ... } WHEN asterix makes request POST /work with a valid token and body diff --git a/yarns/500-build-fail.yarn b/yarns/500-build-fail.yarn index 0a12178..3b29499 100644 --- a/yarns/500-build-fail.yarn +++ b/yarns/500-build-fail.yarn @@ -48,8 +48,8 @@ Add up a project and its pipelines. ... { ... "pipeline": "construct", ... "actions": [ - ... { "shell": "day 1" }, - ... { "shell": "day 2" } + ... { "shell": "day 1", "where": "host" }, + ... { "shell": "day 2", "where": "host" } ... ] ... } THEN result has status code 201 @@ -91,7 +91,8 @@ Worker wants work and gets the first step to run. ... "project": "rome", ... "parameters": {}, ... "step": { - ... "action": "create_workspace" + ... "action": "create_workspace", + ... "where": "host" ... } ... } @@ -147,9 +148,9 @@ Also, there's a build with a log. ... "worker": "obelix", ... "project": "rome", ... "actions": [ - ... { "action": "create_workspace" }, - ... { "shell": "day 1" }, - ... { "shell": "day 2" } + ... { "where": "host", "action": "create_workspace" }, + ... { "where": "host", "shell": "day 1" }, + ... { "where": "host", "shell": "day 2" } ... ], ... "current_action": null, ... "parameters": {}, @@ -168,9 +169,9 @@ Also, there's a build with a log. ... "worker": "obelix", ... "project": "rome", ... "actions": [ - ... { "action": "create_workspace" }, - ... { "shell": "day 1" }, - ... { "shell": "day 2" } + ... { "where": "host", "action": "create_workspace" }, + ... { "where": "host", "shell": "day 1" }, + ... { "where": "host", "shell": "day 2" } ... ], ... "current_action": null, ... "parameters": {}, -- cgit v1.2.1