diff options
author | Lars Wirzenius <liw@liw.fi> | 2017-10-16 19:52:32 +0300 |
---|---|---|
committer | Lars Wirzenius <liw@liw.fi> | 2017-10-16 19:52:32 +0300 |
commit | 5340eaeb454b454b12caeb28be29a6d282c9f4f2 (patch) | |
tree | 77e9eb34eefe4166b0d428a0aff2cc9082cb8b37 | |
parent | ffa4ed102eb7d3548d5700fa362b51c4a3c54319 (diff) | |
parent | 1eaab3cb5aca52d99a80fc397ea78d9aaf2937be (diff) | |
download | ick2-5340eaeb454b454b12caeb28be29a6d282c9f4f2.tar.gz |
Merge branch 'liw/workers'
-rw-r--r-- | NEWS | 2 | ||||
-rw-r--r-- | ick2/__init__.py | 6 | ||||
-rw-r--r-- | ick2/controllerapi.py | 160 | ||||
-rw-r--r-- | ick2/controllerapi_tests.py | 57 | ||||
-rw-r--r-- | ick2/state.py | 48 | ||||
-rw-r--r-- | ick2/state_tests.py | 57 | ||||
-rw-r--r-- | ick_controller.py | 3 | ||||
-rw-r--r-- | yarns/300-workers.yarn | 159 | ||||
-rw-r--r-- | yarns/900-local.yarn | 5 | ||||
-rw-r--r-- | yarns/900-remote.yarn | 3 |
10 files changed, 394 insertions, 106 deletions
@@ -20,6 +20,8 @@ along with this program. If not, see <http://www.gnu.org/licenses/>. Version 0.12+git, not yet released ---------------------------------- +* Added the `/workers` endpoint, for managing information about + workers. Version 0.12, released 2017-10-15 ---------------------------------- diff --git a/ick2/__init__.py b/ick2/__init__.py index e24092e..605c6fa 100644 --- a/ick2/__init__.py +++ b/ick2/__init__.py @@ -16,4 +16,8 @@ from .version import __version__, __version_info__ from .logging import setup_logging, log from .state import ControllerState, NotFound -from .controllerapi import ControllerAPI +from .controllerapi import ( + ControllerAPI, + ProjectAPI, + VersionAPI, +) diff --git a/ick2/controllerapi.py b/ick2/controllerapi.py index cb553d7..6d25a7d 100644 --- a/ick2/controllerapi.py +++ b/ick2/controllerapi.py @@ -30,62 +30,80 @@ class ControllerAPI: def set_state_directory(self, dirname): self._state.set_state_directory(dirname) - def load_projects(self): - return self._state.load_projects() + def find_missing_route(self, missing_path): # pragma: no cover + apis = { + '/version': VersionAPI, + '/projects': ProjectAPI, + '/workers': WorkerAPI, + } + + routes = [] + for path in apis: + self._state.load_resources(path[1:]) + api = apis[path](self._state) + routes.extend(api.get_routes(path)) + ick2.log.log('info', msg_texg='Found routes', routes=routes) + return routes + - def find_missing_route(self, path): # pragma: no cover +class APIbase: # pragma: no cover + + def __init__(self, state): + self._state = state + + def get_routes(self, path): return [ { - 'method': 'GET', - 'path': '/version', - 'callback': self.GET(self.get_version), - }, - { 'method': 'POST', - 'path': '/projects', - 'callback': self.POST(self.post_projects), + 'path': path, + 'callback': self.POST(self.create), }, { 'method': 'GET', - 'path': '/projects', - 'callback': self.GET(self.get_projects), + 'path': path, + 'callback': self.GET(self.list), }, { 'method': 'GET', - 'path': '/projects/<project>', - 'callback': self.GET(self.get_project), + 'path': '{}/<name>'.format(path), + 'callback': self.GET(self.show), }, { 'method': 'PUT', - 'path': '/projects/<project>', - 'callback': self.PUT(self.put_projects), + 'path': '{}/<name>'.format(path), + 'callback': self.PUT(self.update), }, { 'method': 'DELETE', - 'path': '/projects/<project>', - 'callback': self.DELETE(self.delete_projects), + 'path': '{}/<name>'.format(path), + 'callback': self.DELETE(self.delete), }, ] - def GET(self, callback): # pragma: no cover + def GET(self, callback): def wrapper(content_type, body, **kwargs): + ick2.log.log( + 'xxx', msg_text='GET called', kwargs=kwargs, + content_type=content_type, body=body) try: if 'raw_uri_path' in kwargs: del kwargs['raw_uri_path'] - body = callback(**kwargs) + body = callback(**kwargs) + ick2.log.log( + 'xxx', msg_text='GET callback returned', body=body) except ick2.NotFound as e: return not_found(e) return OK(body) return wrapper - def POST(self, callback): # pragma: no cover + def POST(self, callback): def wrapper(content_type, body, **kwargs): body = callback(body) ick2.log.log('trace', msg_text='returned body', body=repr(body)) return created(body) return wrapper - def PUT(self, callback): # pragma: no cover + def PUT(self, callback): def wrapper(content_type, body, **kwargs): if 'raw_uri_path' in kwargs: del kwargs['raw_uri_path'] @@ -94,7 +112,7 @@ class ControllerAPI: return OK(body) return wrapper - def DELETE(self, callback): # pragma: no cover + def DELETE(self, callback): def wrapper(content_type, body, **kwargs): try: if 'raw_uri_path' in kwargs: @@ -105,27 +123,99 @@ class ControllerAPI: return OK(body) return wrapper + def create(self, body): + raise NotImplementedError() + + def update(self, body, name): + raise NotImplementedError() + + def delete(self, name): + raise NotImplementedError() + + def list(self): + raise NotImplementedError() + + def show(self, name): + raise NotImplementedError() + + +class VersionAPI(APIbase): + + def __init__(self, state): + super().__init__(state) + + def get_routes(self, path): # pragma: no cover + return [ + { + 'method': 'GET', + 'path': path, + 'callback': self.GET(self.get_version), + } + ] + def get_version(self): return {'version': ick2.__version__} - def get_projects(self): + def create(self, *args): # pragma: no cover + pass + + def update(self, *args): # pragma: no cover + pass + + def delete(self, *args): # pragma: no cover + pass + + def list(self): # pragma: no cover + pass + + def show(self, *args): # pragma: no cover + pass + + +class SubAPI(APIbase): + + def __init__(self, type_name, state): + super().__init__(state) + self._type_name = type_name + + def list(self): return { - 'projects': self._state.get_projects(), + self._type_name: self._state.get_resources(self._type_name), } - def get_project(self, project=None): - assert project is not None - return self._state.get_project(project) + def show(self, name): + return self._state.get_resource(self._type_name, name) + + def create(self, body): + return self._state.add_resource( + self._type_name, self.get_resource_name(body), body) + + def get_resource_name(self, resource): # pragma: no cover + raise NotImplementedError + + def update(self, body, name): + return self._state.update_resource(self._type_name, name, body) + + def delete(self, name): + self._state.remove_resource(self._type_name, name) + + +class ProjectAPI(SubAPI): + + def __init__(self, state): + super().__init__('projects', state) + + def get_resource_name(self, resource): + return resource['project'] + - def post_projects(self, project): - return self._state.add_project(project) +class WorkerAPI(SubAPI): # pragma: no cover - def put_projects(self, body, project=None): - assert project is not None - return self._state.update_project(project, body) + def __init__(self, state): + super().__init__('workers', state) - def delete_projects(self, project): - self._state.remove_project(project) + def get_resource_name(self, resource): + return resource['worker'] def response(status_code, body, headers): # pragma: no cover diff --git a/ick2/controllerapi_tests.py b/ick2/controllerapi_tests.py index e37dff4..42d1003 100644 --- a/ick2/controllerapi_tests.py +++ b/ick2/controllerapi_tests.py @@ -47,8 +47,15 @@ class ControllerAPITests(unittest.TestCase): self.assertEqual(statedir, self.statedir) self.assertTrue(os.path.exists(statedir)) + +class VersionAPITests(unittest.TestCase): + + def create_api(self): + api = ick2.VersionAPI(None) + return api + def test_returns_version_correcly(self): - api = self.create_api() + api = ick2.VersionAPI(None) response = api.get_version() self.assertEqual( response, @@ -57,9 +64,24 @@ class ControllerAPITests(unittest.TestCase): } ) + +class ProjectAPITests(unittest.TestCase): + + def setUp(self): + self.tempdir = tempfile.mkdtemp() + self.statedir = os.path.join(self.tempdir, 'state/dir') + self.state = ick2.ControllerState() + self.state.set_state_directory(self.statedir) + + def tearDown(self): + shutil.rmtree(self.tempdir) + + def create_api(self): + return ick2.ProjectAPI(self.state) + def test_has_not_projects_initially(self): api = self.create_api() - self.assertEqual(api.get_projects(), {'projects': []}) + self.assertEqual(api.list(), {'projects': []}) def test_creates_project(self): project = { @@ -67,8 +89,8 @@ class ControllerAPITests(unittest.TestCase): 'shell_steps': ['build'], } api = self.create_api() - self.assertEqual(api.post_projects(project), project) - self.assertEqual(api.get_projects(), {'projects': [project]}) + self.assertEqual(api.create(project), project) + self.assertEqual(api.list(), {'projects': [project]}) def test_loads_projects_from_state_directory(self): project = { @@ -76,11 +98,10 @@ class ControllerAPITests(unittest.TestCase): 'shell_steps': ['build'], } api = self.create_api() - api.post_projects(project) + api.create(project) api2 = self.create_api() - api2.load_projects() - self.assertEqual(api2.get_projects(), {'projects': [project]}) + self.assertEqual(api2.list(), {'projects': [project]}) def test_gets_named_project(self): project = { @@ -88,8 +109,8 @@ class ControllerAPITests(unittest.TestCase): 'shell_steps': ['build'], } api = self.create_api() - api.post_projects(project) - self.assertEqual(api.get_project('foo'), project) + api.create(project) + self.assertEqual(api.show('foo'), project) def test_updates_named_project(self): project_v1 = { @@ -99,10 +120,10 @@ class ControllerAPITests(unittest.TestCase): project_v2 = dict(project_v1) project_v2['shell_steps'] = ['build it using magic'] api = self.create_api() - api.post_projects(project_v1) - updated = api.put_projects(project_v2, project='foo') + api.create(project_v1) + updated = api.update(project_v2, 'foo') self.assertEqual(updated, project_v2) - self.assertEqual(api.get_project('foo'), project_v2) + self.assertEqual(api.show('foo'), project_v2) def test_deletes_named_project(self): project = { @@ -110,13 +131,13 @@ class ControllerAPITests(unittest.TestCase): 'shell_steps': ['build'], } api = self.create_api() - api.post_projects(project) - api.delete_projects('foo') - self.assertEqual(api.get_projects(), {'projects': []}) + api.create(project) + api.delete('foo') + self.assertEqual(api.list(), {'projects': []}) with self.assertRaises(ick2.NotFound): - api.get_project('foo') + api.show('foo') - def test_raises_errro_deleting_missing_project(self): + def test_raises_error_deleting_missing_project(self): api = self.create_api() with self.assertRaises(ick2.NotFound): - api.delete_projects('foo') + api.delete('foo') diff --git a/ick2/state.py b/ick2/state.py index ab5d391..67cd057 100644 --- a/ick2/state.py +++ b/ick2/state.py @@ -34,52 +34,52 @@ class ControllerState: if not os.path.exists(self._statedir): os.makedirs(self._statedir) - def get_project_directory(self): - return os.path.join(self.get_state_directory(), 'projects') + def get_resource_directory(self, type_name): + return os.path.join(self.get_state_directory(), type_name) - def get_project_filename(self, project_name): + def get_resource_filename(self, type_name, project_name): return os.path.join( - self.get_project_directory(), project_name + '.yaml') + self.get_resource_directory(type_name), project_name + '.yaml') - def load_projects(self): + def load_resources(self, type_name): assert self._statedir is not None - projects = [] - dirname = self.get_project_directory() + resources = [] + dirname = self.get_resource_directory(type_name) for filename in glob.glob(dirname + '/*.yaml'): - obj = self.load_project(filename) - projects.append(obj) - return projects + obj = self.load_resource(filename) + resources.append(obj) + return resources - def load_project(self, filename): + def load_resource(self, filename): with open(filename, 'r') as f: return yaml.safe_load(f) - def get_projects(self): - return self.load_projects() + def get_resources(self, type_name): + return self.load_resources(type_name) - def get_project(self, project): - filename = self.get_project_filename(project) + def get_resource(self, type_name, resource_name): + filename = self.get_resource_filename(type_name, resource_name) if os.path.exists(filename): - return self.load_project(filename) + return self.load_resource(filename) raise NotFound() - def add_project(self, project): - filename = self.get_project_filename(project['project']) + def add_resource(self, type_name, resource_name, resource): + filename = self.get_resource_filename(type_name, resource_name) dirname = os.path.dirname(filename) if not os.path.exists(dirname): os.makedirs(dirname) with open(filename, 'w') as f: - yaml.safe_dump(project, stream=f) - return project + yaml.safe_dump(resource, stream=f) + return resource - def update_project(self, project, body): - filename = self.get_project_filename(project) + def update_resource(self, type_name, resource_name, body): + filename = self.get_resource_filename(type_name, resource_name) with open(filename, 'w') as f: yaml.safe_dump(body, stream=f) return body - def remove_project(self, project): - filename = self.get_project_filename(project) + def remove_resource(self, type_name, resource_name): + filename = self.get_resource_filename(type_name, resource_name) if not os.path.exists(filename): raise NotFound() os.remove(filename) diff --git a/ick2/state_tests.py b/ick2/state_tests.py index 15a0a24..146ed6e 100644 --- a/ick2/state_tests.py +++ b/ick2/state_tests.py @@ -23,7 +23,7 @@ import unittest import ick2 -class ControllerStateTests(unittest.TestCase): +class StateTestsBase: def setUp(self): self.tempdir = tempfile.mkdtemp() @@ -37,6 +37,9 @@ class ControllerStateTests(unittest.TestCase): state.set_state_directory(self.statedir) return state + +class ControllerStateTests(StateTestsBase, unittest.TestCase): + def test_has_no_state_directory_initially(self): state = ick2.ControllerState() self.assertTrue(state.get_state_directory() is None) @@ -47,42 +50,46 @@ class ControllerStateTests(unittest.TestCase): self.assertEqual(statedir, self.statedir) self.assertTrue(os.path.exists(statedir)) - def test_has_not_projects_initially(self): + +class ResourceStateTests(StateTestsBase, unittest.TestCase): + + def test_has_not_resources_initially(self): state = self.create_state() - self.assertEqual(state.get_projects(), []) + self.assertEqual(state.get_resources('project'), []) - def test_creates_project(self): + def test_creates_resource(self): project = { 'project': 'foo', 'shell_steps': ['build'], } state = self.create_state() - state.add_project(project) - self.assertEqual(state.get_projects(), [project]) - self.assertTrue(os.path.exists(state.get_project_filename('foo'))) + state.add_resource('projects', project['project'], project) + self.assertEqual(state.get_resources('projects'), [project]) + self.assertTrue( + os.path.exists(state.get_resource_filename('projects', 'foo'))) - def test_loads_projects_from_state_directory(self): + def test_loads_resource_from_state_directory(self): project = { 'project': 'foo', 'shell_steps': ['build'], } state = self.create_state() - state.add_project(project) + state.add_resource('projects', project['project'], project) state2 = self.create_state() - state2.load_projects() - self.assertEqual(state2.get_projects(), [project]) + state2.load_resources('projects') + self.assertEqual(state2.get_resources('projects'), [project]) - def test_gets_named_project(self): + def test_gets_named_resource(self): project = { 'project': 'foo', 'shell_steps': ['build'], } state = self.create_state() - state.add_project(project) - self.assertEqual(state.get_project('foo'), project) + state.add_resource('projects', project['project'], project) + self.assertEqual(state.get_resource('projects', 'foo'), project) - def test_updates_named_project(self): + def test_updates_named_resource(self): project_v1 = { 'project': 'foo', 'shell_steps': ['build'], @@ -90,24 +97,24 @@ class ControllerStateTests(unittest.TestCase): project_v2 = dict(project_v1) project_v2['shell_steps'] = ['build it using magic'] state = self.create_state() - state.add_project(project_v1) - updated = state.update_project('foo', project_v2) + state.add_resource('projects', 'foo', project_v1) + updated = state.update_resource('projects', 'foo', project_v2) self.assertEqual(updated, project_v2) - self.assertEqual(state.get_project('foo'), project_v2) + self.assertEqual(state.get_resource('projects', 'foo'), project_v2) - def test_deletes_named_project(self): + def test_deletes_named_resource(self): project = { 'project': 'foo', 'shell_steps': ['build'], } state = self.create_state() - state.add_project(project) - state.remove_project('foo') - self.assertEqual(state.get_projects(), []) + state.add_resource('projects', project['project'], project) + state.remove_resource('projects', 'foo') + self.assertEqual(state.get_resources('projects'), []) with self.assertRaises(ick2.NotFound): - state.get_project('foo') + state.get_resource('projects', 'foo') - def test_raises_error_deleting_missing_project(self): + def test_raises_error_deleting_missing_resource(self): state = self.create_state() with self.assertRaises(ick2.NotFound): - state.remove_project('foo') + state.remove_resource('projects', 'foo') diff --git a/ick_controller.py b/ick_controller.py index a7a1c51..58b05e8 100644 --- a/ick_controller.py +++ b/ick_controller.py @@ -86,9 +86,6 @@ def main(): ick2.log.log( 'info', msg_text='called ControllerAPI.set_state_directory') - api.load_projects() - ick2.log.log('info', msg_text='called ControllerAPI.load_projects') - application = apifw.create_bottle_application( api, counter, dict_logger, config) ick2.log.log('info', msg_text='called apifw.create_bottle_application') diff --git a/yarns/300-workers.yarn b/yarns/300-workers.yarn new file mode 100644 index 0000000..6386ae9 --- /dev/null +++ b/yarns/300-workers.yarn @@ -0,0 +1,159 @@ +<!-- + +Copyright 2017 Lars Wirzenius + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see <http://www.gnu.org/licenses/>. + +--> + +# Controller worker management + +The Ick2 controller manages information about workers. Workers are +things that actually do the builds. A worker is described by a resource +like this: + + EXAMPLE project resource + { + "worker": "obelix", + "protocol": "ssh", + "address": "obelix.ick.example.com", + "user": "ick", + "keywords": { + "debian_release": "debian9", + "debian_codename": "stretch", + "debian_arch": "amd64" + } + } + +In other words, there are several things that define a worker: + +* The name, in the `worker` field. This is used for referreing to the + project in the API. +* Information for how to access the worker from the outside. The + worker manager needs this, in case it's running on a different host + from the worker. +* Some keywords to describe the worker. The controller will use this + to pick the appropriate worker to give a build to. + +## Managing workers + +Note that this only tests managing information about workers via the +controller API. It doesn't actually talk to the worker itself. + + SCENARIO managing workers + GIVEN an RSA key pair for token signing + AND an access token for scopes + ... uapi_workers_get + ... uapi_workers_post + ... uapi_workers_id_get + ... uapi_workers_id_put + ... uapi_workers_id_delete + AND controller config uses statedir at the state directory + AND a running ick controller + + WHEN user makes request GET /workers + THEN result has status code 200 + AND body matches { "workers": [] } + + WHEN user makes request POST /workers + ... { + ... "worker": "obelix", + ... "protocol": "ssh", + ... "address": "obelix.ick.example", + ... "user": "ick", + ... "keywords": { + ... "debian_codename": "stretch" + ... } + ... } + THEN result has status code 201 + AND body matches + ... { + ... "worker": "obelix", + ... "protocol": "ssh", + ... "address": "obelix.ick.example", + ... "user": "ick", + ... "keywords": { + ... "debian_codename": "stretch" + ... } + ... } + AND controller state directory contains worker obelix + + WHEN user makes request GET /workers + THEN result has status code 200 + AND body matches + ... { + ... "workers": [ + ... { + ... "worker": "obelix", + ... "protocol": "ssh", + ... "address": "obelix.ick.example", + ... "user": "ick", + ... "keywords": { + ... "debian_codename": "stretch" + ... } + ... } + ... ] + ... } + + WHEN user stops ick controller + GIVEN a running ick controller + WHEN user makes request GET /workers/obelix + THEN result has status code 200 + AND body matches + ... { + ... "worker": "obelix", + ... "protocol": "ssh", + ... "address": "obelix.ick.example", + ... "user": "ick", + ... "keywords": { + ... "debian_codename": "stretch" + ... } + ... } + + WHEN user makes request PUT /workers/obelix + ... { + ... "worker": "obelix", + ... "protocol": "local", + ... "keywords": { + ... "debian_codename": "unstable" + ... } + ... } + THEN result has status code 200 + AND body matches + ... { + ... "worker": "obelix", + ... "protocol": "local", + ... "keywords": { + ... "debian_codename": "unstable" + ... } + ... } + AND controller state directory contains worker obelix + + WHEN user makes request GET /workers/obelix + THEN result has status code 200 + AND body matches + ... { + ... "worker": "obelix", + ... "protocol": "local", + ... "keywords": { + ... "debian_codename": "unstable" + ... } + ... } + + WHEN user makes request DELETE /workers/obelix + THEN result has status code 200 + WHEN user makes request GET /workers/obelix + THEN result has status code 404 + + FINALLY stop ick controller diff --git a/yarns/900-local.yarn b/yarns/900-local.yarn index 2657662..2bc4e0e 100644 --- a/yarns/900-local.yarn +++ b/yarns/900-local.yarn @@ -94,3 +94,8 @@ along with this program. If not, see <http://www.gnu.org/licenses/>. name = get_next_match() filename = os.path.join(vars['statedir'], 'projects', name + '.yaml') assertTrue(os.path.exists(filename)) + + IMPLEMENTS THEN controller state directory contains worker (\S+) + name = get_next_match() + filename = os.path.join(vars['statedir'], 'workers', name + '.yaml') + assertTrue(os.path.exists(filename)) diff --git a/yarns/900-remote.yarn b/yarns/900-remote.yarn index 8c5c8b7..0a28c81 100644 --- a/yarns/900-remote.yarn +++ b/yarns/900-remote.yarn @@ -57,3 +57,6 @@ along with this program. If not, see <http://www.gnu.org/licenses/>. IMPLEMENTS THEN controller state directory contains project (\S+) pass + + IMPLEMENTS THEN controller state directory contains worker (\S+) + pass |