From 5cf3781ad4beddfab5aa7181c0990a5462ffe06f Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Wed, 7 Feb 2018 18:54:13 +0200 Subject: Change: make all .service units be conditional on config If the config file is missing, the unit won't start. --- NEWS | 5 +++++ blob-service.service | 28 ---------------------------- debian/ick2.install | 5 +---- ick-blob-service.service | 29 +++++++++++++++++++++++++++++ ick-controller.service | 29 +++++++++++++++++++++++++++++ ick-worker.service | 31 +++++++++++++++++++++++++++++++ ick2-worker-manager.service | 30 ------------------------------ ick2.service | 28 ---------------------------- ick2/controllerapi.py | 19 ++++++++++++++++--- ick2/versionapi.py | 14 ++++++++++++-- ick2/versionapi_tests.py | 9 ++++----- ick_controller.py | 8 ++++++-- start_blob_service | 4 ++-- start_ick | 4 ++-- worker_manager | 44 ++++++++++++++++++++++++++++++++------------ worker_manager.yaml | 24 ------------------------ yarns/100-projects.yarn | 1 + yarns/150-pipelines.yarn | 1 + yarns/200-version.yarn | 4 +++- yarns/300-workers.yarn | 3 ++- yarns/400-build.yarn | 2 ++ yarns/500-build-fail.yarn | 1 + yarns/600-unauthz.yarn | 1 + yarns/900-local.yarn | 13 +++++++++++++ 24 files changed, 193 insertions(+), 144 deletions(-) delete mode 100644 blob-service.service create mode 100644 ick-blob-service.service create mode 100644 ick-controller.service create mode 100644 ick-worker.service delete mode 100644 ick2-worker-manager.service delete mode 100644 ick2.service delete mode 100644 worker_manager.yaml diff --git a/NEWS b/NEWS index a4356ce..7d8c2d7 100644 --- a/NEWS +++ b/NEWS @@ -42,6 +42,11 @@ Version 0.23+git, not yet released * The systemd unit for the worker manager now restarts the worker manager automatically if it crashes, after 10 seconds. +* The service files are now called ick-controller, ick-worker, and + ick-blob-service. They don't start unless the corresponding + configuration file in /etc exist. This makes the ick Debian package + safe to install, if one only wants icktool. + Version 0.23, released 2018-01-22 ---------------------------------- diff --git a/blob-service.service b/blob-service.service deleted file mode 100644 index 8f2f046..0000000 --- a/blob-service.service +++ /dev/null @@ -1,28 +0,0 @@ -# Copyright 2017 Lars Wirzenius -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - -[Unit] -Description=ick2 blob service -After=network.target - -[Service] -Type=simple -User=_ickbs -Group=_ickbs -WorkingDirectory=/var/lib/ick/blobs -ExecStart=/usr/bin/start_blob_service -KillSignal=QUIT - -[Install] -WantedBy=multi-user.target diff --git a/debian/ick2.install b/debian/ick2.install index bda5847..0e320db 100644 --- a/debian/ick2.install +++ b/debian/ick2.install @@ -1,4 +1 @@ -ick2.service lib/systemd/system -ick2-worker-manager.service lib/systemd/system -blob-service.service lib/systemd/system -worker_manager.yaml etc/ick +ick-*.service lib/systemd/system diff --git a/ick-blob-service.service b/ick-blob-service.service new file mode 100644 index 0000000..2fbb2f2 --- /dev/null +++ b/ick-blob-service.service @@ -0,0 +1,29 @@ +# Copyright 2017-2018 Lars Wirzenius +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +[Unit] +Description=ick blob service +After=network.target +ConditionPathExists=/etc/ick/blob-service.yaml + +[Service] +Type=simple +User=_ickbs +Group=_ickbs +WorkingDirectory=/var/lib/ick/blobs +ExecStart=/usr/bin/start_blob_service +KillSignal=QUIT + +[Install] +WantedBy=multi-user.target diff --git a/ick-controller.service b/ick-controller.service new file mode 100644 index 0000000..7a58da9 --- /dev/null +++ b/ick-controller.service @@ -0,0 +1,29 @@ +# Copyright 2017-2018 Lars Wirzenius +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +[Unit] +Description=ick controller +After=network.target +ConditionPathExists=/etc/ick/controller.yaml + +[Service] +Type=simple +User=_ick +Group=_ick +WorkingDirectory=/var/lib/ick +ExecStart=/usr/bin/start_ick +KillSignal=QUIT + +[Install] +WantedBy=multi-user.target diff --git a/ick-worker.service b/ick-worker.service new file mode 100644 index 0000000..14c0995 --- /dev/null +++ b/ick-worker.service @@ -0,0 +1,31 @@ +# Copyright 2017-2018 Lars Wirzenius +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +[Unit] +Description=ick worker +After=network.target +ConditionPathExists=/etc/ick/worker_manager.yaml + +[Service] +Type=simple +Restart=on-failure +RestartSec=10s +User=_ickwm +Group=_ickwm +WorkingDirectory=/var/lib/ick/workspace +ExecStart=/usr/bin/worker_manager --config /etc/ick/worker_manager.yaml +KillSignal=QUIT + +[Install] +WantedBy=multi-user.target diff --git a/ick2-worker-manager.service b/ick2-worker-manager.service deleted file mode 100644 index 18c08f3..0000000 --- a/ick2-worker-manager.service +++ /dev/null @@ -1,30 +0,0 @@ -# Copyright 2017-2018 Lars Wirzenius -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - -[Unit] -Description=ick2 worker manager -After=network.target - -[Service] -Type=simple -Restart=on-failure -RestartSec=10s -User=_ickwm -Group=_ickwm -WorkingDirectory=/var/lib/ick/workspace -ExecStart=/usr/bin/worker_manager --config /etc/ick/worker_manager.yaml -KillSignal=QUIT - -[Install] -WantedBy=multi-user.target diff --git a/ick2.service b/ick2.service deleted file mode 100644 index 57fd209..0000000 --- a/ick2.service +++ /dev/null @@ -1,28 +0,0 @@ -# Copyright 2017 Lars Wirzenius -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - -[Unit] -Description=ick2 controller -After=network.target - -[Service] -Type=simple -User=_ick -Group=_ick -WorkingDirectory=/var/lib/ick -ExecStart=/usr/bin/start_ick -KillSignal=QUIT - -[Install] -WantedBy=multi-user.target diff --git a/ick2/controllerapi.py b/ick2/controllerapi.py index fe28c29..074fa2e 100644 --- a/ick2/controllerapi.py +++ b/ick2/controllerapi.py @@ -1,4 +1,4 @@ -# Copyright (C) 2017 Lars Wirzenius +# Copyright (C) 2017-2018 Lars Wirzenius # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as published by # the Free Software Foundation, either version 3 of the License, or @@ -20,6 +20,16 @@ class ControllerAPI: def __init__(self, state): self._state = state + self._apis = {} + + def set_blob_service_url(self, url): # pragma: no cover + self.find_missing_route('/version') + api = self._apis.get('/version') + if api: + api.set_blob_service_url(url) + ick2.log.log( + 'info', msg_text='Set blob service url', url=url, + version=api.get_version()) def find_missing_route(self, missing_path): # pragma: no cover apis = { @@ -32,10 +42,13 @@ class ControllerAPI: '/workers': ick2.WorkerAPI, } - routes = [] for path in apis: + if path not in self._apis: + self._apis[path] = apis[path](self._state) + + routes = [] + for path, api in self._apis.items(): self._state.load_resources(path[1:]) - api = apis[path](self._state) routes.extend(api.get_routes(path)) ick2.log.log('info', msg_texg='Found routes', routes=routes) return routes diff --git a/ick2/versionapi.py b/ick2/versionapi.py index fddfdcc..40d1dda 100644 --- a/ick2/versionapi.py +++ b/ick2/versionapi.py @@ -1,4 +1,4 @@ -# Copyright (C) 2017 Lars Wirzenius +# Copyright (C) 2017-2018 Lars Wirzenius # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as published by # the Free Software Foundation, either version 3 of the License, or @@ -18,6 +18,13 @@ import ick2 class VersionAPI(ick2.APIbase): + def __init__(self, state): + super().__init__(state) + self._blob_service_url = None + + def set_blob_service_url(self, url): + self._blob_service_url = url + def get_routes(self, path): # pragma: no cover return [ { @@ -28,7 +35,10 @@ class VersionAPI(ick2.APIbase): ] def get_version(self, **kwargs): - return {'version': ick2.__version__} + return { + 'version': ick2.__version__, + 'blob_service': self._blob_service_url, + } def create(self, body, **kwargs): # pragma: no cover pass diff --git a/ick2/versionapi_tests.py b/ick2/versionapi_tests.py index 500d7e6..143e1cc 100644 --- a/ick2/versionapi_tests.py +++ b/ick2/versionapi_tests.py @@ -1,4 +1,4 @@ -# Copyright (C) 2017 Lars Wirzenius +# Copyright (C) 2017-2018 Lars Wirzenius # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as published by # the Free Software Foundation, either version 3 of the License, or @@ -21,16 +21,15 @@ import ick2 class VersionAPITests(unittest.TestCase): - def create_api(self): - api = ick2.VersionAPI(None) - return api - def test_returns_version_correcly(self): + bloburl = 'https://blobs.example.com' api = ick2.VersionAPI(None) + api.set_blob_service_url(bloburl) response = api.get_version() self.assertEqual( response, { 'version': ick2.__version__, + 'blob_service': bloburl, } ) diff --git a/ick_controller.py b/ick_controller.py index 85c0bca..a176351 100644 --- a/ick_controller.py +++ b/ick_controller.py @@ -1,5 +1,5 @@ #!/usr/bin/python3 -# Copyright (C) 2017 Lars Wirzenius +# Copyright (C) 2017-2018 Lars Wirzenius # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as published by @@ -46,6 +46,7 @@ default_config = { 'token-issuer': None, 'log': [], 'statedir': None, + 'blob-service': None, } @@ -93,7 +94,10 @@ def main(): state.set_state_directory(config['statedir']) api = ick2.ControllerAPI(state) - ick2.log.log('info', msg_text='created ControllerAPI') + api.set_blob_service_url(config['blob-service']) + ick2.log.log( + 'info', msg_text='created ControllerAPI', + blob_service=config['blob-service']) application = apifw.create_bottle_application( api, counter, dict_logger, config) diff --git a/start_blob_service b/start_blob_service index 88bd770..b4a929f 100644 --- a/start_blob_service +++ b/start_blob_service @@ -1,5 +1,5 @@ #!/bin/sh -# Copyright (C) 2017 Lars Wirzenius +# Copyright (C) 2017-2018 Lars Wirzenius # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as published by @@ -16,7 +16,7 @@ set -eux -export BLOB_SERVICE_CONFIG=/etc/ick/blob_service.conf +export BLOB_SERVICE_CONFIG=/etc/ick/blob-service.yaml gunicorn3 \ --bind 127.0.0.1:12766 \ --log-file /var/log/ickbs/gunicorn3.log \ diff --git a/start_ick b/start_ick index add1f61..96f745c 100644 --- a/start_ick +++ b/start_ick @@ -1,5 +1,5 @@ #!/bin/sh -# Copyright (C) 2017 Lars Wirzenius +# Copyright (C) 2017-2018 Lars Wirzenius # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as published by @@ -16,7 +16,7 @@ set -eux -export ICK_CONTROLLER_CONFIG=/etc/ick/controller.conf +export ICK_CONTROLLER_CONFIG=/etc/ick/controller.yaml gunicorn3 \ --bind 127.0.0.1:12765 \ --log-file /var/log/ick/gunicorn3.log \ diff --git a/worker_manager b/worker_manager index efb84cd..ad51619 100755 --- a/worker_manager +++ b/worker_manager @@ -120,6 +120,7 @@ class ControllerAPI: def __init__(self, name, url, token_generator): self._name = name self._url = url + self._blob_url = None self._token_generator = token_generator self._httpapi = HttpApi() @@ -130,7 +131,9 @@ class ControllerAPI: body = { 'worker': self._name, } - self._httpapi.post(url, headers, body) + code = self._httpapi.post(url, headers, body) + if code not in [200, 201, 409]: + raise cliapp.AppException('Failed to register worker') def get_work(self): url = self.url('/work/{}'.format(self._name)) @@ -149,26 +152,42 @@ class ControllerAPI: raise cliapp.AppException( 'Error posting data to controller: {}'.format(code)) + def url(self, path): + return '{}{}'.format(self._url, path) + + def get_auth_headers(self): + token = self._token_generator.get_token() + return { + 'Authorization': 'Bearer {}'.format(token), + } + def upload_blob(self, blob_id, blob): logging.info('Upload blob %s', blob_id) - url = self.url('/blobs/{}'.format(blob_id)) + url = self.bloburl(blob_id) headers = self.get_auth_headers() code = self._httpapi.put(url, headers, blob) def download_blob(self, blob_id): logging.info('Download blob %s', blob_id) - url = self.url('/blobs/{}'.format(blob_id)) + url = self.bloburl(blob_id) headers = self.get_auth_headers() return self._httpapi.get_blob(url, headers) - def url(self, path): - return '{}{}'.format(self._url, path) + def bloburl(self, blob_id): + if self._blob_url is None: + self._blob_url = self.get_blob_service_url() + if self._blob_url is not None: + return '{}/blobs/{}'.format(self._blob_url, blob_id) + logging.error('Do not have blob service URL') + return None - def get_auth_headers(self): - token = self._token_generator.get_token() - return { - 'Authorization': 'Bearer {}'.format(token), - } + def get_blob_service_url(self): + url = self.url('/version') + headers = self.get_auth_headers() + version = self._httpapi.get(url, headers) + logging.info('Version: %r', version) + if version: + return version.get('blob_service') class HttpApi: @@ -208,6 +227,7 @@ class TokenGenerator: iss = 'localhost' aud = 'localhost' scopes = ' '.join([ + 'uapi_version_get', 'uapi_work_id_get', 'uapi_work_post', 'uapi_workers_post', @@ -541,7 +561,7 @@ class WorkspaceArchiver(WorkerBase): logging.info('Uploading %s', blob_id) self.report(b'Uploading workspace\n') - url = self._api.url('/blobs/{}'.format(blob_id)) + url = self._api.bloburl(blob_id) headers = self._api.get_auth_headers() upload = ['curl', '-sk', '-T', filename] for name, value in headers.items(): @@ -591,7 +611,7 @@ class SystreePopulator(WorkerBase): return self.execute_argv(['sudo', 'find', '-delete'], cwd=dirname) def download_and_unpack_systree(self, systree_name, dirname): - url = self._api.url('/blobs/{}'.format(systree_name)) + url = self._api.bloburl(systree_name) headers = self._api.get_auth_headers() download = ['curl', '-k'] for name, value in headers.items(): diff --git a/worker_manager.yaml b/worker_manager.yaml deleted file mode 100644 index d7876ab..0000000 --- a/worker_manager.yaml +++ /dev/null @@ -1,24 +0,0 @@ -# Copyright 2017 Lars Wirzenius -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - -config: - controller: https://127.0.0.1 - name: worker1 - log: /var/log/ickwm/worker_manager.log - log-level: debug - log-max: 10M - log-keep: 10 - token-key: /etc/ick/token_key - token-key-pub: /etc/ick/token_key.pub - workspace: /var/lib/ick/workspace diff --git a/yarns/100-projects.yarn b/yarns/100-projects.yarn index 63572e7..7c18952 100644 --- a/yarns/100-projects.yarn +++ b/yarns/100-projects.yarn @@ -55,6 +55,7 @@ building them. We start by starting an instance of the controller. ... uapi_projects_id_put ... uapi_projects_id_delete AND controller config uses statedir at the state directory + AND controller config uses https://blobs.example.com as blob service AND a running ick controller WHEN user makes request GET /projects diff --git a/yarns/150-pipelines.yarn b/yarns/150-pipelines.yarn index 7169dd5..feda6d0 100644 --- a/yarns/150-pipelines.yarn +++ b/yarns/150-pipelines.yarn @@ -63,6 +63,7 @@ running them. We start by starting an instance of the controller. ... uapi_pipelines_id_put ... uapi_pipelines_id_delete AND controller config uses statedir at the state directory + AND controller config uses https://blobs.example.com as blob service AND a running ick controller WHEN user makes request GET /pipelines diff --git a/yarns/200-version.yarn b/yarns/200-version.yarn index fe92b34..2f87ac7 100644 --- a/yarns/200-version.yarn +++ b/yarns/200-version.yarn @@ -1,6 +1,6 @@