From efba85dcfa234b40160864e6eeddd472fb9853a9 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Sat, 13 Jan 2018 15:06:15 +0200 Subject: Add: prepend a create_workspace action to pipelines This replaces the old fresh_workspace attribute of work resources. It's better this way, because it moves policy from worker-manager to controller, where it belongs. --- ick2/workapi.py | 8 ++-- ick2/workapi_tests.py | 18 +++++---- worker_manager | 6 +-- yarns/400-build.yarn | 95 ++++++++++++++++++++++++++++++++++++++++------- yarns/500-build-fail.yarn | 5 ++- 5 files changed, 103 insertions(+), 29 deletions(-) diff --git a/ick2/workapi.py b/ick2/workapi.py index d1b3431..8237ff4 100644 --- a/ick2/workapi.py +++ b/ick2/workapi.py @@ -66,7 +66,6 @@ class WorkAPI(ick2.APIbase): 'project': project['project'], 'pipeline': pipeline['name'], 'parameters': project.get('parameters', {}), - 'fresh_workspace': True, 'step': actions[current_action], 'log': '/logs/{}'.format(build_id), } @@ -102,6 +101,10 @@ class WorkAPI(ick2.APIbase): def _start_build(self, project, pipeline, worker, build_id): ick2.log.log('info', msg_text='Starting new build', build_id=build_id) parameters = project.get('parameters', {}) + create_workspace = { + 'action': 'create_workspace', + } + actions = [create_workspace] + list(pipeline['actions']) build = { 'build_id': build_id, 'log': '/logs/{}'.format(build_id), @@ -110,7 +113,7 @@ class WorkAPI(ick2.APIbase): 'pipeline': pipeline['name'], 'parameters': parameters, 'status': 'building', - 'actions': pipeline['actions'], + 'actions': actions, 'current_action': 0, } self._builds.add(build_id, build) @@ -155,7 +158,6 @@ class WorkAPI(ick2.APIbase): build['current_action'] = index self._update_build(build) doing['step'] = actions[index] - doing['fresh_workspace'] = False worker_state = { 'worker': update['worker'], diff --git a/ick2/workapi_tests.py b/ick2/workapi_tests.py index d0e8e8d..48c9bbb 100644 --- a/ick2/workapi_tests.py +++ b/ick2/workapi_tests.py @@ -86,9 +86,8 @@ class WorkAPITests(unittest.TestCase): 'parameters': { 'foo': 'bar', }, - 'fresh_workspace': True, 'step': { - 'shell': 'step-1', + 'action': 'create_workspace', }, 'log': '/logs/1', } @@ -112,9 +111,8 @@ class WorkAPITests(unittest.TestCase): 'parameters': { 'foo': 'bar', }, - 'fresh_workspace': True, 'step': { - 'shell': 'step-1', + 'action': 'create_workspace', }, 'log': '/logs/1', } @@ -141,9 +139,16 @@ class WorkAPITests(unittest.TestCase): done['exit_code'] = 0 work.update_work(done) + # We should get the next step now. + expected['step'] = {'shell': 'step-1'} + self.assertEqual(work.get_work('asterix'), expected) + + # Finish the step. + done['exit_code'] = 0 + work.update_work(done) + # We should get the next step now. expected['step'] = {'shell': 'step-2'} - expected['fresh_workspace'] = False self.assertEqual(work.get_work('asterix'), expected) # Finish the step. @@ -173,9 +178,8 @@ class WorkAPITests(unittest.TestCase): 'parameters': { 'foo': 'bar', }, - 'fresh_workspace': True, 'step': { - 'shell': 'step-1', + 'action': 'create_workspace', }, 'log': '/logs/1', } diff --git a/worker_manager b/worker_manager index dcb6fda..bdd982c 100755 --- a/worker_manager +++ b/worker_manager @@ -269,10 +269,6 @@ class Worker: step = work['step'] logging.info('Running step: %r', step) exit_code = 0 - if work.get('fresh_workspace'): - logging.info('Make an empty workspace') - cleaner = WorkspaceCleaner(None, self._workspace, post) - exit_code = cleaner.do(work) if exit_code == 0: klass = self.worker_factory(step) if klass is None: @@ -306,6 +302,8 @@ class Worker: return DebootstrapWorker elif 'archive' in step: return WorkspaceArchiver + elif step.get('action') == 'create_workspace': + return WorkspaceCleaner logging.warning('Cannot find worker for %s', step) return None diff --git a/yarns/400-build.yarn b/yarns/400-build.yarn index 047c2b8..b18a0e6 100644 --- a/yarns/400-build.yarn +++ b/yarns/400-build.yarn @@ -108,6 +108,9 @@ Worker wants work and gets the first step to run. If the worker asks again, it gets the same answer. **FIXME: should the name of the worker be in the path or can we get it in the access token?** +Note that the controller has inserted a special additional step to get +the worker to construct a new workspace for the build. + WHEN worker-manager makes request GET /work/obelix THEN result has status code 200 AND body matches @@ -120,9 +123,8 @@ be in the path or can we get it in the access token?** ... "parameters": { ... "foo": "bar" ... }, - ... "fresh_workspace": true, ... "step": { - ... "shell": "day 1" + ... "action": "create_workspace" ... } ... } @@ -138,9 +140,8 @@ be in the path or can we get it in the access token?** ... "parameters": { ... "foo": "bar" ... }, - ... "fresh_workspace": true, ... "step": { - ... "shell": "day 1" + ... "action": "create_workspace" ... } ... } @@ -167,9 +168,8 @@ User can now see pipeline is running and which worker is building it. ... "parameters": { ... "foo": "bar" ... }, - ... "fresh_workspace": true, ... "step": { - ... "shell": "day 1" + ... "action": "create_workspace" ... } ... } ... } @@ -186,6 +186,7 @@ User can now see pipeline is running and which worker is building it. ... "project": "rome", ... "pipeline": "construct", ... "actions": [ + ... { "action": "create_workspace" }, ... { "shell": "day 1" }, ... { "shell": "day 2" } ... ], @@ -204,7 +205,42 @@ User can now see pipeline is running and which worker is building it. AND result has header Content-Type: text/plain AND body text is "" -Worker reports some build output. Note the null exit code. +Worker reports workspace creation is done. Note the zero exit code. + + WHEN worker-manager makes request POST /work with a valid token and body + ... { + ... "build_id": 1, + ... "worker": "obelix", + ... "project": "rome", + ... "pipeline": "construct", + ... "exit_code": 0, + ... "stdout": "", + ... "stderr": "", + ... "timestamp": "2017-10-27T17:08:49" + ... } + THEN result has status code 201 + +Worker requests more work, and gets the first actual build step. + + WHEN worker-manager makes request GET /work/obelix + THEN result has status code 200 + AND body matches + ... { + ... "build_id": 1, + ... "log": "/logs/1", + ... "worker": "obelix", + ... "project": "rome", + ... "pipeline": "construct", + ... "parameters": { + ... "foo": "bar" + ... }, + ... "step": { + ... "shell": "day 1" + ... } + ... } + +Worker reports some build output. Note the null exit code. The step +hasn't finished yet. WHEN worker-manager makes request POST /work with a valid token and body ... { @@ -234,7 +270,6 @@ didnt't finish. ... "parameters": { ... "foo": "bar" ... }, - ... "fresh_workspace": true, ... "step": { ... "shell": "day 1" ... } @@ -281,10 +316,11 @@ The build status now shows the next step as the active one. ... "project": "rome", ... "pipeline": "construct", ... "actions": [ + ... { "action": "create_workspace" }, ... { "shell": "day 1" }, ... { "shell": "day 2" } ... ], - ... "current_action": 1, + ... "current_action": 2, ... "parameters": { ... "foo": "bar" ... }, @@ -308,7 +344,6 @@ Now there's another step to do. ... "parameters": { ... "foo": "bar" ... }, - ... "fresh_workspace": false, ... "step": { ... "shell": "day 2" ... } @@ -329,7 +364,6 @@ User sees changed status. ... "parameters": { ... "foo": "bar" ... }, - ... "fresh_workspace": false, ... "step": { ... "shell": "day 2" ... }, @@ -379,6 +413,7 @@ no current action. ... "project": "rome", ... "pipeline": "construct", ... "actions": [ + ... { "action": "create_workspace" }, ... { "shell": "day 1" }, ... { "shell": "day 2" } ... ], @@ -402,6 +437,7 @@ no current action. ... "project": "rome", ... "pipeline": "construct", ... "actions": [ + ... { "action": "create_workspace" }, ... { "shell": "day 1" }, ... { "shell": "day 2" } ... ], @@ -436,9 +472,8 @@ Start build again. This should become build number 2. ... "parameters": { ... "foo": "bar" ... }, - ... "fresh_workspace": true, ... "step": { - ... "shell": "day 1" + ... "action": "create_workspace" ... } ... } @@ -454,6 +489,7 @@ Start build again. This should become build number 2. ... "project": "rome", ... "pipeline": "construct", ... "actions": [ + ... { "action": "create_workspace" }, ... { "shell": "day 1" }, ... { "shell": "day 2" } ... ], @@ -470,6 +506,7 @@ Start build again. This should become build number 2. ... "project": "rome", ... "pipeline": "construct", ... "actions": [ + ... { "action": "create_workspace" }, ... { "shell": "day 1" }, ... { "shell": "day 2" } ... ], @@ -482,6 +519,36 @@ Start build again. This should become build number 2. ... ] ... } + WHEN worker-manager makes request POST /work with a valid token and body + ... { + ... "build_id": 2, + ... "worker": "obelix", + ... "project": "rome", + ... "pipeline": "construct", + ... "exit_code": 0, + ... "stdout": "", + ... "stderr": "", + ... "timestamp": "2017-10-27T17:08:49" + ... } + THEN result has status code 201 + + WHEN worker-manager makes request GET /work/obelix + THEN result has status code 200 + AND body matches + ... { + ... "build_id": 2, + ... "log": "/logs/2", + ... "worker": "obelix", + ... "project": "rome", + ... "pipeline": "construct", + ... "parameters": { + ... "foo": "bar" + ... }, + ... "step": { + ... "shell": "day 1" + ... } + ... } + WHEN worker-manager makes request POST /work with a valid token and body ... { ... "build_id": 2, @@ -523,6 +590,7 @@ Start build again. This should become build number 2. ... "project": "rome", ... "pipeline": "construct", ... "actions": [ + ... { "action": "create_workspace" }, ... { "shell": "day 1" }, ... { "shell": "day 2" } ... ], @@ -539,6 +607,7 @@ Start build again. This should become build number 2. ... "project": "rome", ... "pipeline": "construct", ... "actions": [ + ... { "action": "create_workspace" }, ... { "shell": "day 1" }, ... { "shell": "day 2" } ... ], diff --git a/yarns/500-build-fail.yarn b/yarns/500-build-fail.yarn index 1c2a37c..433c3e0 100644 --- a/yarns/500-build-fail.yarn +++ b/yarns/500-build-fail.yarn @@ -88,9 +88,8 @@ Worker wants work and gets the first step to run. ... "project": "rome", ... "pipeline": "construct", ... "parameters": {}, - ... "fresh_workspace": true, ... "step": { - ... "shell": "day 1" + ... "action": "create_workspace" ... } ... } @@ -147,6 +146,7 @@ Also, there's a build with a log. ... "project": "rome", ... "pipeline": "construct", ... "actions": [ + ... { "action": "create_workspace" }, ... { "shell": "day 1" }, ... { "shell": "day 2" } ... ], @@ -168,6 +168,7 @@ Also, there's a build with a log. ... "project": "rome", ... "pipeline": "construct", ... "actions": [ + ... { "action": "create_workspace" }, ... { "shell": "day 1" }, ... { "shell": "day 2" } ... ], -- cgit v1.2.1