diff options
author | Lars Wirzenius <liw@liw.fi> | 2017-11-29 20:40:28 +0100 |
---|---|---|
committer | Lars Wirzenius <liw@liw.fi> | 2017-11-29 20:40:28 +0100 |
commit | 5d311716861dbfede64d884621ca787b96e574f9 (patch) | |
tree | 87c12f13bb47936499ababfe661e21dd82c76d18 | |
parent | e323455e21d8d9141ada6f32ba1f609291c077c7 (diff) | |
parent | 6532045949fe17538e277710f332b3c0727fe4b9 (diff) | |
download | ick2-5d311716861dbfede64d884621ca787b96e574f9.tar.gz |
Merge branch 'liw/workspace'
-rw-r--r-- | debian/ick2.dirs | 1 | ||||
-rw-r--r-- | debian/ick2.postinst | 7 | ||||
-rw-r--r-- | ick2/apibase.py | 3 | ||||
-rw-r--r-- | ick2/workapi.py | 4 | ||||
-rw-r--r-- | ick2/workapi_tests.py | 4 | ||||
-rwxr-xr-x | icktool | 58 | ||||
-rwxr-xr-x | worker_manager | 39 | ||||
-rw-r--r-- | worker_manager.yaml | 1 | ||||
-rw-r--r-- | yarns/400-build.yarn | 7 | ||||
-rw-r--r-- | yarns/500-build-fail.yarn | 1 |
10 files changed, 101 insertions, 24 deletions
diff --git a/debian/ick2.dirs b/debian/ick2.dirs index f3a2c00..a398f42 100644 --- a/debian/ick2.dirs +++ b/debian/ick2.dirs @@ -1 +1,2 @@ var/lib/ick +var/lib/ick/workspace diff --git a/debian/ick2.postinst b/debian/ick2.postinst index 028139b..f66dbda 100644 --- a/debian/ick2.postinst +++ b/debian/ick2.postinst @@ -37,10 +37,13 @@ create_user() create_user _ick /var/lib/ick /var/log/ick create_user _ickwm /var/lib/ick/wm /var/log/ickwm +# Create controller config dir. +install -d -m 0755 -o root -g root /etc/ick + # Create controller state dir install -d -m 0755 -o _ick -g _ick /var/lib/ick/state -# Create controller config dir. -install -d -m 0755 -o root -g root /etc/ick +# Create worker-manager workspace +install -d -m 0755 -o _ickwm -g _ickwm /var/lib/ick/workspace #DEBHELPER# diff --git a/ick2/apibase.py b/ick2/apibase.py index 1e4fd14..088a2b6 100644 --- a/ick2/apibase.py +++ b/ick2/apibase.py @@ -61,7 +61,8 @@ class APIbase: body = callback(**kwargs) except ick2.NotFound as e: ick2.log.log( - 'warning', msg_text='GET Not found', kwargs=kwargs) + 'error', msg_text='GET Not found', kwargs=kwargs, + exc_info=True) return ick2.not_found(e) if isinstance(body, dict): return ick2.OK(body) diff --git a/ick2/workapi.py b/ick2/workapi.py index f140108..f958132 100644 --- a/ick2/workapi.py +++ b/ick2/workapi.py @@ -62,6 +62,7 @@ class WorkAPI(ick2.APIbase): 'project': project['project'], 'pipeline': pipeline['name'], 'parameters': project.get('parameters', {}), + 'fresh_workspace': True, 'step': pipeline['actions'][index], 'step_index': index, 'log': '/logs/{}'.format(build_id), @@ -154,7 +155,8 @@ class WorkAPI(ick2.APIbase): self._finish_build(update) else: doing['step_index'] = index - doing['step'] = actions[index] + doing['step'] = dict(actions[index]) + doing['fresh_workspace'] = False worker_state = { 'worker': update['worker'], diff --git a/ick2/workapi_tests.py b/ick2/workapi_tests.py index c855895..d7ebfa9 100644 --- a/ick2/workapi_tests.py +++ b/ick2/workapi_tests.py @@ -86,6 +86,7 @@ class WorkAPITests(unittest.TestCase): 'parameters': { 'foo': 'bar', }, + 'fresh_workspace': True, 'step': { 'shell': 'step-1', }, @@ -112,6 +113,7 @@ class WorkAPITests(unittest.TestCase): 'parameters': { 'foo': 'bar', }, + 'fresh_workspace': True, 'step': { 'shell': 'step-1', }, @@ -144,6 +146,7 @@ class WorkAPITests(unittest.TestCase): # We should get the next step now. expected['step'] = {'shell': 'step-2'} expected['step_index'] = 1 + expected['fresh_workspace'] = False self.assertEqual(work.get_work('asterix'), expected) # Finish the step. @@ -173,6 +176,7 @@ class WorkAPITests(unittest.TestCase): 'parameters': { 'foo': 'bar', }, + 'fresh_workspace': True, 'step': { 'shell': 'step-1', }, @@ -40,6 +40,22 @@ def scopes(base): return [x.format(base) for x in patterns] +def scopes_for_types(types): + result = [] + for type_name in types: + result.extend(scopes(type_name)) + return result + + +types = [ + 'projects', + 'pipelines', + 'workers', + 'work', + 'builds', + 'logs', +] + class Icktool(cliapp.Application): _default_scopes = [ @@ -47,7 +63,7 @@ class Icktool(cliapp.Application): 'uapi_work_post', 'uapi_projects_id_pipelines_id_get', 'uapi_projects_id_pipelines_id_put', - ] + scopes('projects') + scopes('workers') + scopes('work') + scopes('builds') + scopes('logs') + ] + scopes_for_types(types) def add_settings(self): self.settings.string( @@ -110,12 +126,14 @@ class Icktool(cliapp.Application): pipelines = project['pipelines'] for pipeline in pipelines: build = self._get_latest_build( - project['project'], pipeline['name'], builds) + project['project'], pipeline, builds) + status = self._get_pipeline_status( + project['project'], pipeline) row = { 'project': project['project'], - 'pipeline': pipeline['name'], + 'pipeline': pipeline, 'build_id': build['build_id'], - 'status': pipeline['status'], + 'status': status['status'], 'log': build['log'], } rows.append(row) @@ -160,15 +178,39 @@ class Icktool(cliapp.Application): name = args[0] rc.delete(name) + def cmd_list_pipelines(self, args): + rc = self._new_rc('/pipelines', 'name') + self._prettyson(rc.list()) + + def cmd_create_pipeline(self, args): + rc = self._new_rc('/pipelines', 'name') + obj = self._read_object() + rc.create(obj) + + def cmd_update_pipeline(self, args): + rc = self._new_rc('/pipelines', 'name') + obj = self._read_object() + rc.update(obj) + def cmd_show_pipeline(self, args): - project = args[0] - pipeline = args[1] + rc = self._new_rc('/pipelines', 'pipeline') + name = args[0] + self._prettyson(rc.show(name)) + + def cmd_delete_pipeline(self, args): + rc = self._new_rc('/pipelines', 'pipeline') + name = args[0] + rc.delete(name) + + def cmd_show_pipeline_status(self, args): + self._prettyson(self._get_pipeline_status(args[0], args[1])) + + def _get_pipeline_status(self, project, pipeline): path = '/projects/{}/pipelines/{}'.format(project, pipeline) api = self._new_api() code, text = api.get(path) self._report(code, 200, text) - obj = json.loads(text) - self._prettyson(obj) + return json.loads(text) def _report(self, code, expected, text): if code != expected: diff --git a/worker_manager b/worker_manager index f13b431..a480558 100755 --- a/worker_manager +++ b/worker_manager @@ -83,6 +83,13 @@ class WorkerManager(cliapp.Application): default=5, ) + self.settings.string( + ['workspace'], + 'use DIR as the workspace where commands are run', + metavar='DIR', + default='/var/lib/ick/workspace', + ) + def process_args(self, args): self.settings.require('name') name = self.settings['name'] @@ -94,7 +101,7 @@ class WorkerManager(cliapp.Application): if work and self.settings['pretend']: self.report_pretend_work(url, name, work) elif work: - self.do_work(url, name, work) + self.do_work(url, name, work, self.settings['workspace']) else: self.show_msg('Nothing to do') self.sleep_a_little() @@ -180,7 +187,7 @@ class WorkerManager(cliapp.Application): } self.post_snippet(snippet_url, snippet) - def do_work(self, url, name, work): + def do_work(self, url, name, work, workspace): self.show_msg('Doing work') snippet_url = '{}/work'.format(url) snippet = { @@ -200,16 +207,24 @@ class WorkerManager(cliapp.Application): s[stream_name] = data self.post_snippet(snippet_url, s) - shell_cmd = work['step']['shell'] - exit_code, _, _ = cliapp.runcmd_unchecked( - ['bash', '-xeuc', shell_cmd], - stdout_callback=lambda data: post('stdout', data), - stderr_callback=lambda data: post('stderr', data), - ) - - end_snippet = dict(snippet) - end_snippet['exit_code'] = exit_code - self.post_snippet(snippet_url, end_snippet) + def run_shell(shell_cmd): + exit_code, _, _ = cliapp.runcmd_unchecked( + ['bash', '-xeuc', shell_cmd], + stdout_callback=lambda data: post('stdout', data), + stderr_callback=lambda data: post('stderr', data), + cwd=workspace, + ) + end_snippet = dict(snippet) + end_snippet['exit_code'] = exit_code + self.post_snippet(snippet_url, end_snippet) + return exit_code + + exit_code = 0 + if work.get('fresh_workspace'): + exit_code = run_shell('find . -delete') + if exit_code == 0: + shell_cmd = work['step']['shell'] + run_shell(shell_cmd) def now(self): return time.strftime('%Y-%m-%dT%H:%M:%S') diff --git a/worker_manager.yaml b/worker_manager.yaml index 1279336..d7876ab 100644 --- a/worker_manager.yaml +++ b/worker_manager.yaml @@ -21,3 +21,4 @@ config: log-keep: 10 token-key: /etc/ick/token_key token-key-pub: /etc/ick/token_key.pub + workspace: /var/lib/ick/workspace diff --git a/yarns/400-build.yarn b/yarns/400-build.yarn index ec3e8e5..4591459 100644 --- a/yarns/400-build.yarn +++ b/yarns/400-build.yarn @@ -120,6 +120,7 @@ be in the path or can we get it in the access token?** ... "parameters": { ... "foo": "bar" ... }, + ... "fresh_workspace": true, ... "step": { ... "shell": "day 1" ... }, @@ -138,6 +139,7 @@ be in the path or can we get it in the access token?** ... "parameters": { ... "foo": "bar" ... }, + ... "fresh_workspace": true, ... "step": { ... "shell": "day 1" ... }, @@ -167,6 +169,7 @@ User can now see pipeline is running and which worker is building it. ... "parameters": { ... "foo": "bar" ... }, + ... "fresh_workspace": true, ... "step_index": 0, ... "step": { ... "shell": "day 1" @@ -225,6 +228,7 @@ Still the same job, since the first build step didnt't finish. ... "parameters": { ... "foo": "bar" ... }, + ... "fresh_workspace": true, ... "step": { ... "shell": "day 1" ... }, @@ -272,6 +276,7 @@ Now there's another step to do. ... "parameters": { ... "foo": "bar" ... }, + ... "fresh_workspace": false, ... "step": { ... "shell": "day 2" ... }, @@ -293,6 +298,7 @@ User sees changed status. ... "parameters": { ... "foo": "bar" ... }, + ... "fresh_workspace": false, ... "step_index": 1, ... "step": { ... "shell": "day 2" @@ -383,6 +389,7 @@ Start build again. This should become build number 2. ... "parameters": { ... "foo": "bar" ... }, + ... "fresh_workspace": true, ... "step": { ... "shell": "day 1" ... }, diff --git a/yarns/500-build-fail.yarn b/yarns/500-build-fail.yarn index f0afa32..bd6f22b 100644 --- a/yarns/500-build-fail.yarn +++ b/yarns/500-build-fail.yarn @@ -88,6 +88,7 @@ Worker wants work and gets the first step to run. ... "project": "rome", ... "pipeline": "construct", ... "parameters": {}, + ... "fresh_workspace": true, ... "step": { ... "shell": "day 1" ... }, |