diff options
author | Lars Wirzenius <liw@liw.fi> | 2017-11-18 22:09:07 +0100 |
---|---|---|
committer | Lars Wirzenius <liw@liw.fi> | 2017-11-18 22:09:07 +0100 |
commit | 83aea836a74da1cd006e87c753900c49d54ee50a (patch) | |
tree | eb2f132c877546b4c398d516856b564d6193c455 | |
parent | 1c9c77331874c8932f221c454346568b23c25024 (diff) | |
parent | 72c05cb232bc2eec173dba9b4c199703a180b651 (diff) | |
download | ick2-83aea836a74da1cd006e87c753900c49d54ee50a.tar.gz |
Merge: controllerapi code cleanups
-rw-r--r-- | ick2/__init__.py | 23 | ||||
-rw-r--r-- | ick2/apibase.py | 157 | ||||
-rw-r--r-- | ick2/buildsapi.py | 38 | ||||
-rw-r--r-- | ick2/controllerapi.py | 569 | ||||
-rw-r--r-- | ick2/controllerapi_tests.py | 353 | ||||
-rw-r--r-- | ick2/exceptions.py | 30 | ||||
-rw-r--r-- | ick2/logapi.py | 36 | ||||
-rw-r--r-- | ick2/projectapi.py | 73 | ||||
-rw-r--r-- | ick2/projectapi_tests.py | 184 | ||||
-rw-r--r-- | ick2/responses.py | 60 | ||||
-rw-r--r-- | ick2/versionapi.py | 49 | ||||
-rw-r--r-- | ick2/versionapi_tests.py | 36 | ||||
-rw-r--r-- | ick2/workapi.py | 232 | ||||
-rw-r--r-- | ick2/workapi_tests.py | 192 | ||||
-rw-r--r-- | ick2/workerapi.py | 25 | ||||
-rw-r--r-- | without-tests | 8 |
16 files changed, 1143 insertions, 922 deletions
diff --git a/ick2/__init__.py b/ick2/__init__.py index 02a8e76..abbde96 100644 --- a/ick2/__init__.py +++ b/ick2/__init__.py @@ -16,10 +16,25 @@ from .version import __version__, __version_info__ from .logging import setup_logging, log from .state import ControllerState, NotFound, WrongPipelineStatus +from .exceptions import ( + BadUpdate, + IckException, + MethodNotAllowed, +) +from .responses import ( + OK, + bad_request, + created, + not_found, + text_plain, +) +from .apibase import APIbase, ResourceApiBase +from .buildsapi import BuildsAPI +from .logapi import LogAPI +from .versionapi import VersionAPI +from .projectapi import ProjectAPI +from .workapi import WorkAPI +from .workerapi import WorkerAPI from .controllerapi import ( ControllerAPI, - ProjectAPI, - VersionAPI, - WorkAPI, - WorkerAPI, ) diff --git a/ick2/apibase.py b/ick2/apibase.py new file mode 100644 index 0000000..c7abc45 --- /dev/null +++ b/ick2/apibase.py @@ -0,0 +1,157 @@ +# Copyright (C) 2017 Lars Wirzenius +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + + +import ick2 + + +class APIbase: + + def __init__(self, state): + self._state = state + + def get_routes(self, path): + return [ + { + 'method': 'POST', + 'path': path, + 'callback': self.POST(self.create), + }, + { + 'method': 'GET', + 'path': path, + 'callback': self.GET(self.list), + }, + { + 'method': 'GET', + 'path': '{}/<name>'.format(path), + 'callback': self.GET(self.show), + }, + { + 'method': 'PUT', + 'path': '{}/<name>'.format(path), + 'callback': self.PUT(self.update), + }, + { + 'method': 'DELETE', + 'path': '{}/<name>'.format(path), + 'callback': self.DELETE(self.delete), + }, + ] + + def GET(self, callback): + def wrapper(content_type, body, **kwargs): + ick2.log.log( + 'trace', msg_text='GET called', kwargs=kwargs, + content_type=content_type, body=body) + try: + if 'raw_uri_path' in kwargs: + del kwargs['raw_uri_path'] + body = callback(**kwargs) + except ick2.NotFound as e: + return ick2.not_found(e) + if isinstance(body, dict): + return ick2.OK(body) + elif isinstance(body, str): + return ick2.text_plain(body) + raise Exception('this must not happen') + return wrapper + + def POST(self, callback): + def wrapper(content_type, body, **kwargs): + ick2.log.log( + 'trace', msg_text='POST called', kwargs=kwargs, + content_type=content_type, body=body) + body = callback(body) + ick2.log.log('trace', msg_text='returned body', body=repr(body)) + return ick2.created(body) + return wrapper + + def PUT(self, callback): + def wrapper(content_type, body, **kwargs): + ick2.log.log( + 'trace', msg_text='PUT called', kwargs=kwargs, + content_type=content_type, body=body) + if 'raw_uri_path' in kwargs: + del kwargs['raw_uri_path'] + try: + body = callback(body, **kwargs) + except ick2.NotFound as e: + return ick2.not_found(e) + except ick2.WrongPipelineStatus as e: + ick2.log.log( + 'error', + msg_text='Wrong state for pipeline', + exception=str(e)) + return ick2.bad_request(e) + ick2.log.log('trace', msg_text='returned body', body=repr(body)) + return ick2.OK(body) + return wrapper + + def DELETE(self, callback): + def wrapper(content_type, body, **kwargs): + ick2.log.log( + 'trace', msg_text='DELETE called', kwargs=kwargs, + content_type=content_type, body=body) + try: + if 'raw_uri_path' in kwargs: + del kwargs['raw_uri_path'] + body = callback(**kwargs) + except ick2.NotFound as e: + return ick2.not_found(e) + return ick2.OK(body) + return wrapper + + def create(self, body): + raise NotImplementedError() + + def update(self, body, name): + raise NotImplementedError() + + def delete(self, name): + raise NotImplementedError() + + def list(self): + raise NotImplementedError() + + def show(self, name): + raise NotImplementedError() + + +class ResourceApiBase(APIbase): + + def __init__(self, type_name, state): + super().__init__(state) + self._type_name = type_name + + def list(self): + return { + self._type_name: self._state.get_resources(self._type_name), + } + + def show(self, name): + return self._state.get_resource(self._type_name, name) + + def create(self, body): + return self._state.add_resource( + self._type_name, self.get_resource_name(body), body) + + def get_resource_name(self, resource): # pragma: no cover + raise NotImplementedError() + + def update(self, body, name): + return self._state.update_resource(self._type_name, name, body) + + def delete(self, name): + self._state.remove_resource(self._type_name, name) diff --git a/ick2/buildsapi.py b/ick2/buildsapi.py new file mode 100644 index 0000000..2d5b5f9 --- /dev/null +++ b/ick2/buildsapi.py @@ -0,0 +1,38 @@ +# Copyright (C) 2017 Lars Wirzenius +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + + +import ick2 + + +class BuildsAPI(ick2.ResourceApiBase): # pragma: no cover + + def __init__(self, state): + super().__init__('builds', state) + + def get_resource_name(self, resource): + return resource['build'] + + def create(self, body): # pragma: no cover + raise ick2.MethodNotAllowed('Creating builds directly is not allowed') + + def update(self, body, name): # pragma: no cover + raise ick2.MethodNotAllowed('Updating builds directly is not allowed') + + def list(self): + result = super().list() + items = result[self._type_name] + items.sort(key=lambda x: x.get('build_id')) + result[self._type_name] = items + return result diff --git a/ick2/controllerapi.py b/ick2/controllerapi.py index ef20daa..4b67d10 100644 --- a/ick2/controllerapi.py +++ b/ick2/controllerapi.py @@ -13,9 +13,6 @@ # along with this program. If not, see <http://www.gnu.org/licenses/>. -import apifw - - import ick2 @@ -32,12 +29,12 @@ class ControllerAPI: def find_missing_route(self, missing_path): # pragma: no cover apis = { - '/version': VersionAPI, - '/builds': BuildsAPI, - '/logs': LogAPI, - '/projects': ProjectAPI, - '/work': WorkAPI, - '/workers': WorkerAPI, + '/version': ick2.VersionAPI, + '/builds': ick2.BuildsAPI, + '/logs': ick2.LogAPI, + '/projects': ick2.ProjectAPI, + '/work': ick2.WorkAPI, + '/workers': ick2.WorkerAPI, } routes = [] @@ -47,557 +44,3 @@ class ControllerAPI: routes.extend(api.get_routes(path)) ick2.log.log('info', msg_texg='Found routes', routes=routes) return routes - - -class APIbase: # pragma: no cover - - def __init__(self, state): - self._state = state - - def get_routes(self, path): - return [ - { - 'method': 'POST', - 'path': path, - 'callback': self.POST(self.create), - }, - { - 'method': 'GET', - 'path': path, - 'callback': self.GET(self.list), - }, - { - 'method': 'GET', - 'path': '{}/<name>'.format(path), - 'callback': self.GET(self.show), - }, - { - 'method': 'PUT', - 'path': '{}/<name>'.format(path), - 'callback': self.PUT(self.update), - }, - { - 'method': 'DELETE', - 'path': '{}/<name>'.format(path), - 'callback': self.DELETE(self.delete), - }, - ] - - def GET(self, callback): - def wrapper(content_type, body, **kwargs): - ick2.log.log( - 'trace', msg_text='GET called', kwargs=kwargs, - content_type=content_type, body=body) - try: - if 'raw_uri_path' in kwargs: - del kwargs['raw_uri_path'] - body = callback(**kwargs) - except ick2.NotFound as e: - return not_found(e) - if isinstance(body, dict): - return OK(body) - elif isinstance(body, str): - return text_plain(body) - raise Exception('this must not happen') - return wrapper - - def POST(self, callback): - def wrapper(content_type, body, **kwargs): - ick2.log.log( - 'trace', msg_text='POST called', kwargs=kwargs, - content_type=content_type, body=body) - body = callback(body) - ick2.log.log('trace', msg_text='returned body', body=repr(body)) - return created(body) - return wrapper - - def PUT(self, callback): - def wrapper(content_type, body, **kwargs): - ick2.log.log( - 'trace', msg_text='PUT called', kwargs=kwargs, - content_type=content_type, body=body) - if 'raw_uri_path' in kwargs: - del kwargs['raw_uri_path'] - try: - body = callback(body, **kwargs) - except ick2.NotFound as e: - return not_found(e) - except ick2.WrongPipelineStatus as e: - ick2.log.log( - 'error', - msg_text='Wrong state for pipeline', - exception=str(e)) - return bad_request(e) - ick2.log.log('trace', msg_text='returned body', body=repr(body)) - return OK(body) - return wrapper - - def DELETE(self, callback): - def wrapper(content_type, body, **kwargs): - ick2.log.log( - 'trace', msg_text='DELETE called', kwargs=kwargs, - content_type=content_type, body=body) - try: - if 'raw_uri_path' in kwargs: - del kwargs['raw_uri_path'] - body = callback(**kwargs) - except ick2.NotFound as e: - return not_found(e) - return OK(body) - return wrapper - - def create(self, body): - raise NotImplementedError() - - def update(self, body, name): - raise NotImplementedError() - - def delete(self, name): - raise NotImplementedError() - - def list(self): - raise NotImplementedError() - - def show(self, name): - raise NotImplementedError() - - -class VersionAPI(APIbase): - - def __init__(self, state): - super().__init__(state) - - def get_routes(self, path): # pragma: no cover - return [ - { - 'method': 'GET', - 'path': path, - 'callback': self.GET(self.get_version), - } - ] - - def get_version(self): - return {'version': ick2.__version__} - - def create(self, *args): # pragma: no cover - pass - - def update(self, *args): # pragma: no cover - pass - - def delete(self, *args): # pragma: no cover - pass - - def list(self): # pragma: no cover - pass - - def show(self, *args): # pragma: no cover - pass - - -class ResourceApiBase(APIbase): - - def __init__(self, type_name, state): - super().__init__(state) - self._type_name = type_name - - def list(self): - return { - self._type_name: self._state.get_resources(self._type_name), - } - - def show(self, name): - return self._state.get_resource(self._type_name, name) - - def create(self, body): - return self._state.add_resource( - self._type_name, self.get_resource_name(body), body) - - def get_resource_name(self, resource): # pragma: no cover - raise NotImplementedError - - def update(self, body, name): - return self._state.update_resource(self._type_name, name, body) - - def delete(self, name): - self._state.remove_resource(self._type_name, name) - - -class WorkerAPI(ResourceApiBase): # pragma: no cover - - def __init__(self, state): - super().__init__('workers', state) - - def get_resource_name(self, resource): - return resource['worker'] - - -class BuildsAPI(ResourceApiBase): # pragma: no cover - - def __init__(self, state): - super().__init__('builds', state) - - def get_resource_name(self, resource): - return resource['build'] - - def create(self, body): # pragma: no cover - raise MethodNotAllowed('Creating builds directly is not allowed') - - def update(self, body, name): # pragma: no cover - raise MethodNotAllowed('Updating builds directly is not allowed') - - def list(self): - result = super().list() - items = result[self._type_name] - items.sort(key=lambda x: x.get('build_id')) - result[self._type_name] = items - return result - - -class LogAPI(ResourceApiBase): # pragma: no cover - - def __init__(self, state): - super().__init__('log', state) - - def get_resource_name(self, resource): - return resource['log'] - - def create(self, body): # pragma: no cover - raise MethodNotAllowed('Creating builds directly is not allowed') - - def update(self, body, name): # pragma: no cover - raise MethodNotAllowed('Updating builds directly is not allowed') - - def show(self, name): - log = self._state.get_resource('log', str(name)) - ick2.log.log('info', msg_text='Returning log', log=log) - return log['log'] - - -class ProjectAPI(ResourceApiBase): - - def __init__(self, state): - super().__init__('projects', state) - - def get_resource_name(self, resource): - return resource['project'] - - def get_routes(self, path): # pragma: no cover - return super().get_routes(path) + self.get_pipeline_routes(path) - - def get_pipeline_routes(self, path): # pragma: no cover - pipeline_path = '{}/<project>/pipelines/<pipeline>'.format(path) - builds_path = '{}/<project>/builds'.format(path) - return [ - { - 'method': 'GET', - 'path': pipeline_path, - 'callback': self.GET(self.get_pipeline), - }, - { - 'method': 'PUT', - 'path': pipeline_path, - 'callback': self.PUT(self.set_pipeline_callback), - }, - { - 'method': 'GET', - 'path': builds_path, - 'callback': self.GET(self.get_builds), - }, - ] - - def get_pipeline(self, project, pipeline): - p = self._state.get_resource(self._type_name, project) - for pl in p['pipelines']: - if pl['name'] == pipeline: - return { - 'status': pl.get('status', 'idle'), - } - raise ick2.NotFound() - - def set_pipeline_callback( - self, body, project, pipeline): # pragma: no cover - return self.set_pipeline(body['status'], project, pipeline) - - def set_pipeline(self, state, project, pipeline): - allowed_changes = { - 'idle': 'triggered', - 'triggered': 'building', - 'building': 'idle', - } - p = self._state.get_resource(self._type_name, project) - for pl in p['pipelines']: - if pl['name'] == pipeline: - old_state = pl.get('status', 'idle') - if allowed_changes[old_state] != state: - raise ick2.WrongPipelineStatus(state) - pl['status'] = state - self._state.update_resource(self._type_name, project, p) - return {'status': state} - raise ick2.NotFound() - - def get_builds(self, project): - p = self._state.get_resource(self._type_name, project) - return { - 'project': project, - 'builds': p.get('builds', []), - } - - -class WorkAPI(APIbase): - - def __init__(self, state): - super().__init__(state) - self._type_name = 'work' - - def get_routes(self, path): # pragma: no cover - return [ - { - 'method': 'GET', - 'path': '{}/<worker>'.format(path), - 'callback': self.GET(self.get_work), - }, - { - 'method': 'POST', - 'path': path, - 'callback': self.POST(self.update_work), - }, - ] - - def get_work(self, worker): - worker_state = self._get_worker(worker) - if not worker_state.get('doing'): - project, pipeline = self._pick_triggered_pipeline() - if project is None: - doing = {} - else: - pipeline['status'] = 'building' - self._update_project(project) - - build_id = self._start_build(project, pipeline, worker) - self._start_log(build_id) - - index = 0 - doing = { - 'build_id': build_id, - 'worker': worker, - 'project': project['project'], - 'pipeline': pipeline['name'], - 'step': pipeline['actions'][index], - 'step_index': index, - 'log': '/logs/{}'.format(build_id), - } - - worker_state = { - 'worker': worker, - 'doing': doing, - } - self._update_worker(worker_state) - - return worker_state['doing'] - - def _get_worker(self, worker): # pragma: no cover - try: - return self._state.get_resource('workers', worker) - except ick2.NotFound: - return { - 'worker': worker, - } - - def _update_worker(self, worker_state): - self._state.update_resource( - 'workers', worker_state['worker'], worker_state) - - def _pick_triggered_pipeline(self): - projects = self._get_projects() - for project in projects: - for pipeline in project['pipelines']: - if pipeline.get('status') == 'triggered': - return project, pipeline - return None, None - - def _get_projects(self): - return self._state.get_resources('projects') - - def _update_project(self, project): - self._state.update_resource('projects', project['project'], project) - - def update_work(self, update): - if 'worker' not in update: # pragma: no cover - raise BadUpdate('no worker specified') - - worker_state = self._get_worker(update['worker']) - doing = worker_state.get('doing', {}) - self._check_work_update(doing, update) - - project, pipeline = self._get_pipeline( - update['project'], update['pipeline']) - self._append_to_build_log(update) - - ick2.log.log( - 'trace', - msg_text='xxx update_work', - update=update, - project=project, - pipeline=pipeline, - doing=doing) - - exit_code = update.get('exit_code') - if exit_code == 0: - index = doing['step_index'] + 1 - actions = pipeline['actions'] - if index >= len(actions): - pipeline['status'] = 'idle' - doing = {} - self._finish_build(update) - else: - doing['step_index'] = index - doing['step'] = actions[index] - self._update_project(project) - - worker_state = { - 'worker': update['worker'], - 'doing': doing, - } - self._update_worker(worker_state) - elif exit_code is not None: - assert isinstance(exit_code, int) - assert exit_code != 0 - pipeline['status'] = 'idle' - self._update_project(project) - self._finish_build(update) - - worker_state = { - 'worker': update['worker'], - 'doing': {}, - } - self._update_worker(worker_state) - - def _check_work_update(self, doing, update): # pragma: no cover - must_match = ['worker', 'project', 'pipeline', 'build_id'] - for name in must_match: - if name not in update: - raise BadUpdate('{} not specified'.format(name)) - if doing.get(name) != update[name]: - raise BadUpdate( - '{} differs from current work: {} vs {}'.format( - name, doing.get(name), update[name])) - - def _get_pipeline(self, project, pipeline): # pragma: no cover - projects = self._get_projects() - for p in projects: - for pl in p['pipelines']: - if pl.get('name') == pipeline: - return p, pl - raise ick2.NotFound() - - def _start_build(self, project, pipeline, worker): - ick2.log.log('info', msg_text='Starting new build') - build_id = project.get('build_id', 0) - build_id += 1 - project['build_id'] = build_id - self._update_project(project) - build = { - 'build_id': build_id, - 'log': '/logs/{}'.format(build_id), - 'worker': worker, - 'project': project['project'], - 'pipeline': pipeline['name'], - 'status': 'building', - } - self._state.add_resource('builds', str(build_id), build) - return build_id - - def _start_log(self, build_id): - ick2.log.log('info', msg_text='Starting new log', build_id=build_id) - log = { - 'build_id': build_id, - 'log': '', - } - self._state.add_resource('log', str(build_id), log) - return build_id - - def _append_to_build_log(self, update): - build_id = update['build_id'] - log = self._state.get_resource('log', str(build_id)) - for kind in ['stdout', 'stderr']: - text = update.get(kind, '') - if text is not None: - log['log'] += text - self._state.update_resource('log', str(build_id), log) - - def _finish_build(self, update): - build = self._state.get_resource('builds', str(update['build_id'])) - build['status'] = update['exit_code'] - self._state.update_resource('builds', str(update['build_id']), build) - - def create(self, *args, **kwargs): # pragma: no cover - pass - - def update(self, *args, **kwargs): # pragma: no cover - pass - - def list(self, *args, **kwargs): # pragma: no cover - pass - - def show(self, *args, **kwargs): # pragma: no cover - pass - - def delete(self, *args, **kwargs): # pragma: no cover - pass - - -class BadUpdate(Exception): # pragma: no cover - - def __init__(self, how): - super().__init__('Work update is BAD: {}'.format(how)) - - -class MethodNotAllowed(Exception): # pragma: no cover - - def __init__(self, wat): - super().__init__(wat) - - -def response(status_code, body, headers): # pragma: no cover - obj = { - 'status': status_code, - 'body': body, - 'headers': headers, - } - return apifw.Response(obj) - - -def OK(body): # pragma: no cover - headers = { - 'Content-Type': 'application/json', - } - return response(apifw.HTTP_OK, body, headers) - - -def text_plain(body): # pragma: no cover - headers = { - 'Content-Type': 'text/plain', - } - return response(apifw.HTTP_OK, body, headers) - - -def not_found(error): # pragma: no cover - headers = { - 'Content-Type': 'text/plain', - } - return response(apifw.HTTP_NOT_FOUND, str(error), headers) - - -def bad_request(error): # pragma: no cover - headers = { - 'Content-Type': 'text/plain', - } - return response(apifw.HTTP_BAD_REQUEST, str(error), headers) - - -def created(body): # pragma: no cover - headers = { - 'Content-Type': 'application/json', - } - return response(apifw.HTTP_CREATED, body, headers) diff --git a/ick2/controllerapi_tests.py b/ick2/controllerapi_tests.py index ad638b6..bb24e4a 100644 --- a/ick2/controllerapi_tests.py +++ b/ick2/controllerapi_tests.py @@ -46,356 +46,3 @@ class ControllerAPITests(unittest.TestCase): statedir = api.get_state_directory() self.assertEqual(statedir, self.statedir) self.assertTrue(os.path.exists(statedir)) - - -class VersionAPITests(unittest.TestCase): - - def create_api(self): - api = ick2.VersionAPI(None) - return api - - def test_returns_version_correcly(self): - api = ick2.VersionAPI(None) - response = api.get_version() - self.assertEqual( - response, - { - 'version': ick2.__version__, - } - ) - - -class ProjectAPITests(unittest.TestCase): - - def setUp(self): - self.tempdir = tempfile.mkdtemp() - self.statedir = os.path.join(self.tempdir, 'state/dir') - self.state = ick2.ControllerState() - self.state.set_state_directory(self.statedir) - - def tearDown(self): - shutil.rmtree(self.tempdir) - - def create_api(self): - return ick2.ProjectAPI(self.state) - - def test_has_not_projects_initially(self): - api = self.create_api() - self.assertEqual(api.list(), {'projects': []}) - - def test_creates_project(self): - project = { - 'project': 'foo', - 'pipelines': [ - { - 'name': 'build', - 'actions': [ - { - 'shell': 'step-1', - }, - ], - }, - ], - } - api = self.create_api() - self.assertEqual(api.create(project), project) - self.assertEqual(api.list(), {'projects': [project]}) - self.assertEqual(api.get_pipeline('foo', 'build'), {'status': 'idle'}) - self.assertEqual( - api.get_builds('foo'), - {'project': 'foo', 'builds': []} - ) - - def test_raises_error_when_getting_missing_pipeline(self): - project = { - 'project': 'foo', - 'pipelines': [ - { - 'name': 'build', - 'actions': [ - { - 'shell': 'step-1', - }, - ], - }, - ], - } - api = self.create_api() - api.create(project) - with self.assertRaises(ick2.NotFound): - api.get_pipeline('foo', 'does-not-exist') - - def test_loads_projects_from_state_directory(self): - project = { - 'project': 'foo', - 'shell_steps': ['build'], - } - api = self.create_api() - api.create(project) - - api2 = self.create_api() - self.assertEqual(api2.list(), {'projects': [project]}) - - def test_gets_named_project(self): - project = { - 'project': 'foo', - 'shell_steps': ['build'], - } - api = self.create_api() - api.create(project) - self.assertEqual(api.show('foo'), project) - - def test_updates_named_project(self): - project_v1 = { - 'project': 'foo', - 'shell_steps': ['build'], - } - project_v2 = dict(project_v1) - project_v2['shell_steps'] = ['build it using magic'] - api = self.create_api() - api.create(project_v1) - updated = api.update(project_v2, 'foo') - self.assertEqual(updated, project_v2) - self.assertEqual(api.show('foo'), project_v2) - - def test_deletes_named_project(self): - project = { - 'project': 'foo', - 'shell_steps': ['build'], - } - api = self.create_api() - api.create(project) - api.delete('foo') - self.assertEqual(api.list(), {'projects': []}) - with self.assertRaises(ick2.NotFound): - api.show('foo') - - def test_raises_error_deleting_missing_project(self): - api = self.create_api() - with self.assertRaises(ick2.NotFound): - api.delete('foo') - - def test_updates_pipeline_status(self): - project = { - 'project': 'foo', - 'pipelines': [ - { - 'name': 'build', - 'actions': [ - { - 'shell': 'step-1', - }, - ], - }, - ], - } - api = self.create_api() - api.create(project) - self.assertEqual(api.get_pipeline('foo', 'build'), {'status': 'idle'}) - - with self.assertRaises(ick2.WrongPipelineStatus): - api.set_pipeline('building', 'foo', 'build') - - api.set_pipeline('triggered', 'foo', 'build') - self.assertEqual( - api.get_pipeline('foo', 'build'), - {'status': 'triggered'} - ) - - with self.assertRaises(ick2.WrongPipelineStatus): - api.set_pipeline('idle', 'foo', 'build') - - api.set_pipeline('building', 'foo', 'build') - self.assertEqual( - api.get_pipeline('foo', 'build'), - {'status': 'building'} - ) - - with self.assertRaises(ick2.WrongPipelineStatus): - api.set_pipeline('triggered', 'foo', 'build') - - api.set_pipeline('idle', 'foo', 'build') - self.assertEqual( - api.get_pipeline('foo', 'build'), - {'status': 'idle'} - ) - - def test_raises_error_updating_status_of_missing_pipeline(self): - project = { - 'project': 'foo', - 'pipelines': [], - } - api = self.create_api() - api.create(project) - with self.assertRaises(ick2.NotFound): - api.set_pipeline('idle', 'foo', 'build') - - -class WorkAPITests(unittest.TestCase): - - def setUp(self): - self.tempdir = tempfile.mkdtemp() - self.statedir = os.path.join(self.tempdir, 'state/dir') - self.state = ick2.ControllerState() - self.state.set_state_directory(self.statedir) - - def tearDown(self): - shutil.rmtree(self.tempdir) - - def create_project_api(self): - project = { - 'project': 'foo', - 'pipelines': [ - { - 'name': 'build', - 'actions': [ - { - 'shell': 'step-1', - }, - { - 'shell': 'step-2', - }, - ], - }, - ], - } - api = ick2.ProjectAPI(self.state) - api.create(project) - return api - - def create_worker_api(self): - worker = { - 'worker': 'asterix', - } - api = ick2.WorkerAPI(self.state) - api.create(worker) - return api - - def create_work_api(self): - return ick2.WorkAPI(self.state) - - def test_worker_gets_no_work_when_no_pipeline_is_triggered(self): - self.create_project_api() - self.create_worker_api() - work = self.create_work_api() - self.assertEqual(work.get_work('asterix'), {}) - - def test_worker_gets_work_when_a_pipeline_is_triggered(self): - projects = self.create_project_api() - projects.set_pipeline('triggered', 'foo', 'build') - self.create_worker_api() - work = self.create_work_api() - expected = { - 'build_id': 1, - 'worker': 'asterix', - 'project': 'foo', - 'pipeline': 'build', - 'step': { - 'shell': 'step-1', - }, - 'step_index': 0, - 'log': '/logs/1', - } - self.assertEqual(work.get_work('asterix'), expected) - - # Check we get the same thing twice. - self.assertEqual(work.get_work('asterix'), expected) - - def test_worker_manager_posts_work_updates(self): - projects = self.create_project_api() - projects.set_pipeline('triggered', 'foo', 'build') - self.create_worker_api() - work = self.create_work_api() - - # Ask for some work. - expected = { - 'build_id': 1, - 'worker': 'asterix', - 'project': 'foo', - 'pipeline': 'build', - 'step': { - 'shell': 'step-1', - }, - 'step_index': 0, - 'log': '/logs/1', - } - self.assertEqual(work.get_work('asterix'), expected) - - # Post a partial update. - done = { - 'build_id': 1, - 'worker': 'asterix', - 'project': 'foo', - 'pipeline': 'build', - 'exit_code': None, - 'stdout': 'out', - 'stderr': 'err', - 'timestamp': '2000-01-01T00:00:00', - } - work.update_work(done) - - # Ask for work again. We didn't finish the previous step, so - # should get same thing. - self.assertEqual(work.get_work('asterix'), expected) - - # Finish the step. - done['exit_code'] = 0 - work.update_work(done) - - # We should get the next step now. - expected['step'] = {'shell': 'step-2'} - expected['step_index'] = 1 - self.assertEqual(work.get_work('asterix'), expected) - - # Finish the step. - done['exit_code'] = 0 - work.update_work(done) - - # We now get nothing further to do. - self.assertEqual(work.get_work('asterix'), {}) - - # An pipeline status has changed. - self.assertEqual( - projects.get_pipeline('foo', 'build'), - {'status': 'idle'}) - - def test_worker_manager_posts_failure(self): - projects = self.create_project_api() - projects.set_pipeline('triggered', 'foo', 'build') - self.create_worker_api() - work = self.create_work_api() - - # Ask for some work. - expected = { - 'build_id': 1, - 'worker': 'asterix', - 'project': 'foo', - 'pipeline': 'build', - 'step': { - 'shell': 'step-1', - }, - 'step_index': 0, - 'log': '/logs/1', - } - self.assertEqual(work.get_work('asterix'), expected) - - # Post a partial update. - done = { - 'build_id': 1, - 'worker': 'asterix', - 'project': 'foo', - 'pipeline': 'build', - 'exit_code': 1, - 'stdout': 'out', - 'stderr': 'err', - 'timestamp': '2000-01-01T00:00:00', - } - work.update_work(done) - - # Ask for work again. - self.assertEqual(work.get_work('asterix'), {}) - - # An pipeline status has changed. - self.assertEqual( - projects.get_pipeline('foo', 'build'), - {'status': 'idle'}) diff --git a/ick2/exceptions.py b/ick2/exceptions.py new file mode 100644 index 0000000..c99c3e5 --- /dev/null +++ b/ick2/exceptions.py @@ -0,0 +1,30 @@ +# Copyright (C) 2017 Lars Wirzenius +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + + +class IckException(Exception): + + pass + + +class BadUpdate(IckException): + + def __init__(self, how): + super().__init__('Work update is BAD: {}'.format(how)) + + +class MethodNotAllowed(IckException): + + def __init__(self, wat): + super().__init__(wat) diff --git a/ick2/logapi.py b/ick2/logapi.py new file mode 100644 index 0000000..2a6f2f4 --- /dev/null +++ b/ick2/logapi.py @@ -0,0 +1,36 @@ +# Copyright (C) 2017 Lars Wirzenius +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + + +import ick2 + + +class LogAPI(ick2.ResourceApiBase): # pragma: no cover + + def __init__(self, state): + super().__init__('log', state) + + def get_resource_name(self, resource): + return resource['log'] + + def create(self, body): # pragma: no cover + raise ick2.MethodNotAllowed('Creating builds directly is not allowed') + + def update(self, body, name): # pragma: no cover + raise ick2.MethodNotAllowed('Updating builds directly is not allowed') + + def show(self, name): + log = self._state.get_resource('log', str(name)) + ick2.log.log('info', msg_text='Returning log', log=log) + return log['log'] diff --git a/ick2/projectapi.py b/ick2/projectapi.py new file mode 100644 index 0000000..71fe5ee --- /dev/null +++ b/ick2/projectapi.py @@ -0,0 +1,73 @@ +# Copyright (C) 2017 Lars Wirzenius +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + + +import ick2 + + +class ProjectAPI(ick2.ResourceApiBase): + + def __init__(self, state): + super().__init__('projects', state) + + def get_resource_name(self, resource): + return resource['project'] + + def get_routes(self, path): # pragma: no cover + return super().get_routes(path) + self.get_pipeline_routes(path) + + def get_pipeline_routes(self, path): # pragma: no cover + pipeline_path = '{}/<project>/pipelines/<pipeline>'.format(path) + return [ + { + 'method': 'GET', + 'path': pipeline_path, + 'callback': self.GET(self.get_pipeline), + }, + { + 'method': 'PUT', + 'path': pipeline_path, + 'callback': self.PUT(self.set_pipeline_callback), + }, + ] + + def get_pipeline(self, project, pipeline): + p = self._state.get_resource(self._type_name, project) + for pl in p['pipelines']: + if pl['name'] == pipeline: + return { + 'status': pl.get('status', 'idle'), + } + raise ick2.NotFound() + + def set_pipeline_callback( + self, body, project, pipeline): # pragma: no cover + return self.set_pipeline(body['status'], project, pipeline) + + def set_pipeline(self, state, project, pipeline): + allowed_changes = { + 'idle': 'triggered', + 'triggered': 'building', + 'building': 'idle', + } + p = self._state.get_resource(self._type_name, project) + for pl in p['pipelines']: + if pl['name'] == pipeline: + old_state = pl.get('status', 'idle') + if allowed_changes[old_state] != state: + raise ick2.WrongPipelineStatus(state) + pl['status'] = state + self._state.update_resource(self._type_name, project, p) + return {'status': state} + raise ick2.NotFound() diff --git a/ick2/projectapi_tests.py b/ick2/projectapi_tests.py new file mode 100644 index 0000000..975018c --- /dev/null +++ b/ick2/projectapi_tests.py @@ -0,0 +1,184 @@ +# Copyright (C) 2017 Lars Wirzenius +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + + +import os +import shutil +import tempfile +import unittest + + +import ick2 + + +class ProjectAPITests(unittest.TestCase): + + def setUp(self): + self.tempdir = tempfile.mkdtemp() + self.statedir = os.path.join(self.tempdir, 'state/dir') + self.state = ick2.ControllerState() + self.state.set_state_directory(self.statedir) + + def tearDown(self): + shutil.rmtree(self.tempdir) + + def create_api(self): + return ick2.ProjectAPI(self.state) + + def test_has_not_projects_initially(self): + api = self.create_api() + self.assertEqual(api.list(), {'projects': []}) + + def test_creates_project(self): + project = { + 'project': 'foo', + 'pipelines': [ + { + 'name': 'build', + 'actions': [ + { + 'shell': 'step-1', + }, + ], + }, + ], + } + api = self.create_api() + self.assertEqual(api.create(project), project) + self.assertEqual(api.list(), {'projects': [project]}) + self.assertEqual(api.get_pipeline('foo', 'build'), {'status': 'idle'}) + + def test_raises_error_when_getting_missing_pipeline(self): + project = { + 'project': 'foo', + 'pipelines': [ + { + 'name': 'build', + 'actions': [ + { + 'shell': 'step-1', + }, + ], + }, + ], + } + api = self.create_api() + api.create(project) + with self.assertRaises(ick2.NotFound): + api.get_pipeline('foo', 'does-not-exist') + + def test_loads_projects_from_state_directory(self): + project = { + 'project': 'foo', + 'shell_steps': ['build'], + } + api = self.create_api() + api.create(project) + + api2 = self.create_api() + self.assertEqual(api2.list(), {'projects': [project]}) + + def test_gets_named_project(self): + project = { + 'project': 'foo', + 'shell_steps': ['build'], + } + api = self.create_api() + api.create(project) + self.assertEqual(api.show('foo'), project) + + def test_updates_named_project(self): + project_v1 = { + 'project': 'foo', + 'shell_steps': ['build'], + } + project_v2 = dict(project_v1) + project_v2['shell_steps'] = ['build it using magic'] + api = self.create_api() + api.create(project_v1) + updated = api.update(project_v2, 'foo') + self.assertEqual(updated, project_v2) + self.assertEqual(api.show('foo'), project_v2) + + def test_deletes_named_project(self): + project = { + 'project': 'foo', + 'shell_steps': ['build'], + } + api = self.create_api() + api.create(project) + api.delete('foo') + self.assertEqual(api.list(), {'projects': []}) + with self.assertRaises(ick2.NotFound): + api.show('foo') + + def test_raises_error_deleting_missing_project(self): + api = self.create_api() + with self.assertRaises(ick2.NotFound): + api.delete('foo') + + def test_updates_pipeline_status(self): + project = { + 'project': 'foo', + 'pipelines': [ + { + 'name': 'build', + 'actions': [ + { + 'shell': 'step-1', + }, + ], + }, + ], + } + api = self.create_api() + api.create(project) + self.assertEqual(api.get_pipeline('foo', 'build'), {'status': 'idle'}) + + with self.assertRaises(ick2.WrongPipelineStatus): + api.set_pipeline('building', 'foo', 'build') + + api.set_pipeline('triggered', 'foo', 'build') + self.assertEqual( + api.get_pipeline('foo', 'build'), + {'status': 'triggered'} + ) + + with self.assertRaises(ick2.WrongPipelineStatus): + api.set_pipeline('idle', 'foo', 'build') + + api.set_pipeline('building', 'foo', 'build') + self.assertEqual( + api.get_pipeline('foo', 'build'), + {'status': 'building'} + ) + + with self.assertRaises(ick2.WrongPipelineStatus): + api.set_pipeline('triggered', 'foo', 'build') + + api.set_pipeline('idle', 'foo', 'build') + self.assertEqual( + api.get_pipeline('foo', 'build'), + {'status': 'idle'} + ) + + def test_raises_error_updating_status_of_missing_pipeline(self): + project = { + 'project': 'foo', + 'pipelines': [], + } + api = self.create_api() + api.create(project) + with self.assertRaises(ick2.NotFound): + api.set_pipeline('idle', 'foo', 'build') diff --git a/ick2/responses.py b/ick2/responses.py new file mode 100644 index 0000000..ef68783 --- /dev/null +++ b/ick2/responses.py @@ -0,0 +1,60 @@ +# Copyright (C) 2017 Lars Wirzenius +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + + +import apifw + + +def response(status_code, body, headers): # pragma: no cover + obj = { + 'status': status_code, + 'body': body, + 'headers': headers, + } + return apifw.Response(obj) + + +def OK(body): # pragma: no cover + headers = { + 'Content-Type': 'application/json', + } + return response(apifw.HTTP_OK, body, headers) + + +def text_plain(body): # pragma: no cover + headers = { + 'Content-Type': 'text/plain', + } + return response(apifw.HTTP_OK, body, headers) + + +def not_found(error): # pragma: no cover + headers = { + 'Content-Type': 'text/plain', + } + return response(apifw.HTTP_NOT_FOUND, str(error), headers) + + +def bad_request(error): # pragma: no cover + headers = { + 'Content-Type': 'text/plain', + } + return response(apifw.HTTP_BAD_REQUEST, str(error), headers) + + +def created(body): # pragma: no cover + headers = { + 'Content-Type': 'application/json', + } + return response(apifw.HTTP_CREATED, body, headers) diff --git a/ick2/versionapi.py b/ick2/versionapi.py new file mode 100644 index 0000000..f17c333 --- /dev/null +++ b/ick2/versionapi.py @@ -0,0 +1,49 @@ +# Copyright (C) 2017 Lars Wirzenius +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + + +import ick2 + + +class VersionAPI(ick2.APIbase): + + def __init__(self, state): + super().__init__(state) + + def get_routes(self, path): # pragma: no cover + return [ + { + 'method': 'GET', + 'path': path, + 'callback': self.GET(self.get_version), + } + ] + + def get_version(self): + return {'version': ick2.__version__} + + def create(self, *args): # pragma: no cover + pass + + def update(self, *args): # pragma: no cover + pass + + def delete(self, *args): # pragma: no cover + pass + + def list(self): # pragma: no cover + pass + + def show(self, *args): # pragma: no cover + pass diff --git a/ick2/versionapi_tests.py b/ick2/versionapi_tests.py new file mode 100644 index 0000000..500d7e6 --- /dev/null +++ b/ick2/versionapi_tests.py @@ -0,0 +1,36 @@ +# Copyright (C) 2017 Lars Wirzenius +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + + +import unittest + + +import ick2 + + +class VersionAPITests(unittest.TestCase): + + def create_api(self): + api = ick2.VersionAPI(None) + return api + + def test_returns_version_correcly(self): + api = ick2.VersionAPI(None) + response = api.get_version() + self.assertEqual( + response, + { + 'version': ick2.__version__, + } + ) diff --git a/ick2/workapi.py b/ick2/workapi.py new file mode 100644 index 0000000..ccfbf71 --- /dev/null +++ b/ick2/workapi.py @@ -0,0 +1,232 @@ +# Copyright (C) 2017 Lars Wirzenius +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + + +import ick2 + + +class WorkAPI(ick2.APIbase): + + def __init__(self, state): + super().__init__(state) + self._workers = Workers(state) + self._projects = Projects(state) + self._type_name = 'work' + + def get_routes(self, path): # pragma: no cover + return [ + { + 'method': 'GET', + 'path': '{}/<worker>'.format(path), + 'callback': self.GET(self.get_work), + }, + { + 'method': 'POST', + 'path': path, + 'callback': self.POST(self.update_work), + }, + ] + + def get_work(self, worker): + worker_state = self._workers.get_worker(worker) + if not worker_state.get('doing'): + project, pipeline = self._pick_triggered_pipeline() + if project is None: + doing = {} + else: + pipeline['status'] = 'building' + self._projects.update_project(project) + + build_id = project.get('build_id', 0) + 1 + project['build_id'] = build_id + self._projects.update_project(project) + + self._start_build(project, pipeline, worker, build_id) + self._start_log(build_id) + + index = 0 + doing = { + 'build_id': build_id, + 'worker': worker, + 'project': project['project'], + 'pipeline': pipeline['name'], + 'step': pipeline['actions'][index], + 'step_index': index, + 'log': '/logs/{}'.format(build_id), + } + + worker_state = { + 'worker': worker, + 'doing': doing, + } + self._workers.update_worker(worker_state) + + return worker_state['doing'] + + def _pick_triggered_pipeline(self): + projects = self._projects.get_projects() + for project in projects: + for pipeline in project['pipelines']: + if pipeline.get('status') == 'triggered': + return project, pipeline + return None, None + + def _start_build(self, project, pipeline, worker, build_id): + ick2.log.log('info', msg_text='Starting new build', build_id=build_id) + build = { + 'build_id': build_id, + 'log': '/logs/{}'.format(build_id), + 'worker': worker, + 'project': project['project'], + 'pipeline': pipeline['name'], + 'status': 'building', + } + self._state.add_resource('builds', str(build_id), build) + return build_id + + def _start_log(self, build_id): + ick2.log.log('info', msg_text='Starting new log', build_id=build_id) + log = { + 'build_id': build_id, + 'log': '', + } + self._state.add_resource('log', str(build_id), log) + + def update_work(self, update): + if 'worker' not in update: # pragma: no cover + raise ick2.BadUpdate('no worker specified') + + worker_state = self._workers.get_worker(update['worker']) + doing = worker_state.get('doing', {}) + self._check_work_update(doing, update) + + project, pipeline = self._get_pipeline( + update['project'], update['pipeline']) + self._append_to_build_log(update) + + ick2.log.log( + 'trace', + msg_text='xxx update_work', + update=update, + project=project, + pipeline=pipeline, + doing=doing) + + exit_code = update.get('exit_code') + if exit_code == 0: + index = doing['step_index'] + 1 + actions = pipeline['actions'] + if index >= len(actions): + pipeline['status'] = 'idle' + doing = {} + self._finish_build(update) + else: + doing['step_index'] = index + doing['step'] = actions[index] + self._projects.update_project(project) + + worker_state = { + 'worker': update['worker'], + 'doing': doing, + } + self._workers.update_worker(worker_state) + elif exit_code is not None: + assert isinstance(exit_code, int) + assert exit_code != 0 + pipeline['status'] = 'idle' + self._projects.update_project(project) + self._finish_build(update) + + worker_state = { + 'worker': update['worker'], + 'doing': {}, + } + self._workers.update_worker(worker_state) + + def _check_work_update(self, doing, update): # pragma: no cover + must_match = ['worker', 'project', 'pipeline', 'build_id'] + for name in must_match: + if name not in update: + raise ick2.BadUpdate('{} not specified'.format(name)) + if doing.get(name) != update[name]: + raise ick2.BadUpdate( + '{} differs from current work: {} vs {}'.format( + name, doing.get(name), update[name])) + + def _get_pipeline(self, project, pipeline): # pragma: no cover + projects = self._projects.get_projects() + for p in projects: + for pl in p['pipelines']: + if pl.get('name') == pipeline: + return p, pl + raise ick2.NotFound() + + def _append_to_build_log(self, update): + build_id = update['build_id'] + log = self._state.get_resource('log', str(build_id)) + for kind in ['stdout', 'stderr']: + text = update.get(kind, '') + if text is not None: + log['log'] += text + self._state.update_resource('log', str(build_id), log) + + def _finish_build(self, update): + build = self._state.get_resource('builds', str(update['build_id'])) + build['status'] = update['exit_code'] + self._state.update_resource('builds', str(update['build_id']), build) + + def create(self, *args, **kwargs): # pragma: no cover + pass + + def update(self, *args, **kwargs): # pragma: no cover + pass + + def list(self, *args, **kwargs): # pragma: no cover + pass + + def show(self, *args, **kwargs): # pragma: no cover + pass + + def delete(self, *args, **kwargs): # pragma: no cover + pass + + +class Workers: # pragma: no cover + + def __init__(self, state): + self._state = state + + def get_worker(self, name): + try: + return self._state.get_resource('workers', name) + except ick2.NotFound: + return { + 'worker': name, + } + + def update_worker(self, worker): + self._state.update_resource( + 'workers', worker['worker'], worker) + + +class Projects: # pragma: no cover + + def __init__(self, state): + self._state = state + + def get_projects(self): + return self._state.get_resources('projects') + + def update_project(self, project): + self._state.update_resource('projects', project['project'], project) diff --git a/ick2/workapi_tests.py b/ick2/workapi_tests.py new file mode 100644 index 0000000..03b25e7 --- /dev/null +++ b/ick2/workapi_tests.py @@ -0,0 +1,192 @@ +# Copyright (C) 2017 Lars Wirzenius +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + + +import os +import shutil +import tempfile +import unittest + + +import ick2 + + +class WorkAPITests(unittest.TestCase): + + def setUp(self): + self.tempdir = tempfile.mkdtemp() + self.statedir = os.path.join(self.tempdir, 'state/dir') + self.state = ick2.ControllerState() + self.state.set_state_directory(self.statedir) + + def tearDown(self): + shutil.rmtree(self.tempdir) + + def create_project_api(self): + project = { + 'project': 'foo', + 'pipelines': [ + { + 'name': 'build', + 'actions': [ + { + 'shell': 'step-1', + }, + { + 'shell': 'step-2', + }, + ], + }, + ], + } + api = ick2.ProjectAPI(self.state) + api.create(project) + return api + + def create_worker_api(self): + worker = { + 'worker': 'asterix', + } + api = ick2.WorkerAPI(self.state) + api.create(worker) + return api + + def create_work_api(self): + return ick2.WorkAPI(self.state) + + def test_worker_gets_no_work_when_no_pipeline_is_triggered(self): + self.create_project_api() + self.create_worker_api() + work = self.create_work_api() + self.assertEqual(work.get_work('asterix'), {}) + + def test_worker_gets_work_when_a_pipeline_is_triggered(self): + projects = self.create_project_api() + projects.set_pipeline('triggered', 'foo', 'build') + self.create_worker_api() + work = self.create_work_api() + expected = { + 'build_id': 1, + 'worker': 'asterix', + 'project': 'foo', + 'pipeline': 'build', + 'step': { + 'shell': 'step-1', + }, + 'step_index': 0, + 'log': '/logs/1', + } + self.assertEqual(work.get_work('asterix'), expected) + + # Check we get the same thing twice. + self.assertEqual(work.get_work('asterix'), expected) + + def test_worker_manager_posts_work_updates(self): + projects = self.create_project_api() + projects.set_pipeline('triggered', 'foo', 'build') + self.create_worker_api() + work = self.create_work_api() + + # Ask for some work. + expected = { + 'build_id': 1, + 'worker': 'asterix', + 'project': 'foo', + 'pipeline': 'build', + 'step': { + 'shell': 'step-1', + }, + 'step_index': 0, + 'log': '/logs/1', + } + self.assertEqual(work.get_work('asterix'), expected) + + # Post a partial update. + done = { + 'build_id': 1, + 'worker': 'asterix', + 'project': 'foo', + 'pipeline': 'build', + 'exit_code': None, + 'stdout': 'out', + 'stderr': 'err', + 'timestamp': '2000-01-01T00:00:00', + } + work.update_work(done) + + # Ask for work again. We didn't finish the previous step, so + # should get same thing. + self.assertEqual(work.get_work('asterix'), expected) + + # Finish the step. + done['exit_code'] = 0 + work.update_work(done) + + # We should get the next step now. + expected['step'] = {'shell': 'step-2'} + expected['step_index'] = 1 + self.assertEqual(work.get_work('asterix'), expected) + + # Finish the step. + done['exit_code'] = 0 + work.update_work(done) + + # We now get nothing further to do. + self.assertEqual(work.get_work('asterix'), {}) + + # An pipeline status has changed. + self.assertEqual( + projects.get_pipeline('foo', 'build'), + {'status': 'idle'}) + + def test_worker_manager_posts_failure(self): + projects = self.create_project_api() + projects.set_pipeline('triggered', 'foo', 'build') + self.create_worker_api() + work = self.create_work_api() + + # Ask for some work. + expected = { + 'build_id': 1, + 'worker': 'asterix', + 'project': 'foo', + 'pipeline': 'build', + 'step': { + 'shell': 'step-1', + }, + 'step_index': 0, + 'log': '/logs/1', + } + self.assertEqual(work.get_work('asterix'), expected) + + # Post a partial update. + done = { + 'build_id': 1, + 'worker': 'asterix', + 'project': 'foo', + 'pipeline': 'build', + 'exit_code': 1, + 'stdout': 'out', + 'stderr': 'err', + 'timestamp': '2000-01-01T00:00:00', + } + work.update_work(done) + + # Ask for work again. + self.assertEqual(work.get_work('asterix'), {}) + + # An pipeline status has changed. + self.assertEqual( + projects.get_pipeline('foo', 'build'), + {'status': 'idle'}) diff --git a/ick2/workerapi.py b/ick2/workerapi.py new file mode 100644 index 0000000..5a7e835 --- /dev/null +++ b/ick2/workerapi.py @@ -0,0 +1,25 @@ +# Copyright (C) 2017 Lars Wirzenius +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + + +import ick2 + + +class WorkerAPI(ick2.ResourceApiBase): # pragma: no cover + + def __init__(self, state): + super().__init__('workers', state) + + def get_resource_name(self, resource): + return resource['worker'] diff --git a/without-tests b/without-tests index 5af6235..b587c70 100644 --- a/without-tests +++ b/without-tests @@ -1,5 +1,9 @@ ick2/__init__.py +ick2/apibase.py +ick2/buildsapi.py +ick2/exceptions.py +ick2/logapi.py ick2/logging.py +ick2/responses.py ick2/version.py - - +ick2/workerapi.py |