Change: make all .service units be conditional on config

If the config file is missing, the unit won't start.

21 files changed, 112 insertions, 63 deletions

diff --git a/NEWS b/NEWS
index a4356ce..7d8c2d7 100644
--- a/NEWS
+++ b/NEWS
@@ -42,6 +42,11 @@ Version 0.23+git, not yet released
 * The systemd unit for the worker manager now restarts the worker
   manager automatically if it crashes, after 10 seconds.
 
+* The service files are now called ick-controller, ick-worker, and
+  ick-blob-service. They don't start unless the corresponding
+  configuration file in /etc exist. This makes the ick Debian package
+  safe to install, if one only wants icktool.
+
 Version 0.23, released 2018-01-22
 ----------------------------------
 
diff --git a/debian/ick2.install b/debian/ick2.install
index bda5847..0e320db 100644
--- a/debian/ick2.install
+++ b/debian/ick2.install
@@ -1,4 +1 @@
-ick2.service lib/systemd/system
-ick2-worker-manager.service lib/systemd/system
-blob-service.service lib/systemd/system
-worker_manager.yaml etc/ick
+ick-*.service lib/systemd/system
diff --git a/blob-service.service b/ick-blob-service.service
index 8f2f046..2fbb2f2 100644
--- a/blob-service.service
+++ b/ick-blob-service.service
@@ -1,4 +1,4 @@
-# Copyright 2017 Lars Wirzenius
+# Copyright 2017-2018 Lars Wirzenius
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
@@ -13,8 +13,9 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 [Unit]
-Description=ick2 blob service
+Description=ick blob service
 After=network.target
+ConditionPathExists=/etc/ick/blob-service.yaml
 
 [Service]
 Type=simple
diff --git a/ick2.service b/ick-controller.service
index 57fd209..7a58da9 100644
--- a/ick2.service
+++ b/ick-controller.service
@@ -1,4 +1,4 @@
-# Copyright 2017 Lars Wirzenius
+# Copyright 2017-2018 Lars Wirzenius
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
@@ -13,8 +13,9 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 [Unit]
-Description=ick2 controller
+Description=ick controller
 After=network.target
+ConditionPathExists=/etc/ick/controller.yaml
 
 [Service]
 Type=simple
diff --git a/ick2-worker-manager.service b/ick-worker.service
index 18c08f3..14c0995 100644
--- a/ick2-worker-manager.service
+++ b/ick-worker.service
@@ -13,8 +13,9 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 [Unit]
-Description=ick2 worker manager
+Description=ick worker
 After=network.target
+ConditionPathExists=/etc/ick/worker_manager.yaml
 
 [Service]
 Type=simple
diff --git a/ick2/controllerapi.py b/ick2/controllerapi.py
index fe28c29..074fa2e 100644
--- a/ick2/controllerapi.py
+++ b/ick2/controllerapi.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2017  Lars Wirzenius
+# Copyright (C) 2017-2018  Lars Wirzenius
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
@@ -20,6 +20,16 @@ class ControllerAPI:
 
     def __init__(self, state):
         self._state = state
+        self._apis = {}
+
+    def set_blob_service_url(self, url):  # pragma: no cover
+        self.find_missing_route('/version')
+        api = self._apis.get('/version')
+        if api:
+            api.set_blob_service_url(url)
+            ick2.log.log(
+                'info', msg_text='Set blob service url', url=url,
+                version=api.get_version())
 
     def find_missing_route(self, missing_path):  # pragma: no cover
         apis = {
@@ -32,10 +42,13 @@ class ControllerAPI:
             '/workers': ick2.WorkerAPI,
         }
 
-        routes = []
         for path in apis:
+            if path not in self._apis:
+                self._apis[path] = apis[path](self._state)
+
+        routes = []
+        for path, api in self._apis.items():
             self._state.load_resources(path[1:])
-            api = apis[path](self._state)
             routes.extend(api.get_routes(path))
         ick2.log.log('info', msg_texg='Found routes', routes=routes)
         return routes
diff --git a/ick2/versionapi.py b/ick2/versionapi.py
index fddfdcc..40d1dda 100644
--- a/ick2/versionapi.py
+++ b/ick2/versionapi.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2017  Lars Wirzenius
+# Copyright (C) 2017-2018  Lars Wirzenius
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
@@ -18,6 +18,13 @@ import ick2
 
 class VersionAPI(ick2.APIbase):
 
+    def __init__(self, state):
+        super().__init__(state)
+        self._blob_service_url = None
+
+    def set_blob_service_url(self, url):
+        self._blob_service_url = url
+
     def get_routes(self, path):  # pragma: no cover
         return [
             {
@@ -28,7 +35,10 @@ class VersionAPI(ick2.APIbase):
         ]
 
     def get_version(self, **kwargs):
-        return {'version': ick2.__version__}
+        return {
+            'version': ick2.__version__,
+            'blob_service': self._blob_service_url,
+        }
 
     def create(self, body, **kwargs):  # pragma: no cover
         pass
diff --git a/ick2/versionapi_tests.py b/ick2/versionapi_tests.py
index 500d7e6..143e1cc 100644
--- a/ick2/versionapi_tests.py
+++ b/ick2/versionapi_tests.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2017  Lars Wirzenius
+# Copyright (C) 2017-2018  Lars Wirzenius
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
@@ -21,16 +21,15 @@ import ick2
 
 class VersionAPITests(unittest.TestCase):
 
-    def create_api(self):
-        api = ick2.VersionAPI(None)
-        return api
-
     def test_returns_version_correcly(self):
+        bloburl = 'https://blobs.example.com'
         api = ick2.VersionAPI(None)
+        api.set_blob_service_url(bloburl)
         response = api.get_version()
         self.assertEqual(
             response,
             {
                 'version': ick2.__version__,
+                'blob_service': bloburl,
             }
         )
diff --git a/ick_controller.py b/ick_controller.py
index 85c0bca..a176351 100644
--- a/ick_controller.py
+++ b/ick_controller.py
@@ -1,5 +1,5 @@
 #!/usr/bin/python3
-# Copyright (C) 2017  Lars Wirzenius
+# Copyright (C) 2017-2018  Lars Wirzenius
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -46,6 +46,7 @@ default_config = {
     'token-issuer': None,
     'log': [],
     'statedir': None,
+    'blob-service': None,
 }
 
 
@@ -93,7 +94,10 @@ def main():
         state.set_state_directory(config['statedir'])
 
         api = ick2.ControllerAPI(state)
-        ick2.log.log('info', msg_text='created ControllerAPI')
+        api.set_blob_service_url(config['blob-service'])
+        ick2.log.log(
+            'info', msg_text='created ControllerAPI',
+            blob_service=config['blob-service'])
 
         application = apifw.create_bottle_application(
             api, counter, dict_logger, config)
diff --git a/start_blob_service b/start_blob_service
index 88bd770..b4a929f 100644
--- a/start_blob_service
+++ b/start_blob_service
@@ -1,5 +1,5 @@
 #!/bin/sh
-# Copyright (C) 2017  Lars Wirzenius
+# Copyright (C) 2017-2018  Lars Wirzenius
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -16,7 +16,7 @@
 
 set -eux
 
-export BLOB_SERVICE_CONFIG=/etc/ick/blob_service.conf
+export BLOB_SERVICE_CONFIG=/etc/ick/blob-service.yaml
 gunicorn3 \
     --bind 127.0.0.1:12766 \
     --log-file /var/log/ickbs/gunicorn3.log \
diff --git a/start_ick b/start_ick
index add1f61..96f745c 100644
--- a/start_ick
+++ b/start_ick
@@ -1,5 +1,5 @@
 #!/bin/sh
-# Copyright (C) 2017  Lars Wirzenius
+# Copyright (C) 2017-2018  Lars Wirzenius
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -16,7 +16,7 @@
 
 set -eux
 
-export ICK_CONTROLLER_CONFIG=/etc/ick/controller.conf
+export ICK_CONTROLLER_CONFIG=/etc/ick/controller.yaml
 gunicorn3 \
     --bind 127.0.0.1:12765 \
     --log-file /var/log/ick/gunicorn3.log \
diff --git a/worker_manager b/worker_manager
index efb84cd..ad51619 100755
--- a/worker_manager
+++ b/worker_manager
@@ -120,6 +120,7 @@ class ControllerAPI:
     def __init__(self, name, url, token_generator):
         self._name = name
         self._url = url
+        self._blob_url = None
         self._token_generator = token_generator
         self._httpapi = HttpApi()
 
@@ -130,7 +131,9 @@ class ControllerAPI:
         body = {
             'worker': self._name,
         }
-        self._httpapi.post(url, headers, body)
+        code = self._httpapi.post(url, headers, body)
+        if code not in [200, 201, 409]:
+            raise cliapp.AppException('Failed to register worker')
 
     def get_work(self):
         url = self.url('/work/{}'.format(self._name))
@@ -149,26 +152,42 @@ class ControllerAPI:
             raise cliapp.AppException(
                 'Error posting data to controller: {}'.format(code))
 
+    def url(self, path):
+        return '{}{}'.format(self._url, path)
+
+    def get_auth_headers(self):
+        token = self._token_generator.get_token()
+        return {
+            'Authorization': 'Bearer {}'.format(token),
+        }
+
     def upload_blob(self, blob_id, blob):
         logging.info('Upload blob %s', blob_id)
-        url = self.url('/blobs/{}'.format(blob_id))
+        url = self.bloburl(blob_id)
         headers = self.get_auth_headers()
         code = self._httpapi.put(url, headers, blob)
 
     def download_blob(self, blob_id):
         logging.info('Download blob %s', blob_id)
-        url = self.url('/blobs/{}'.format(blob_id))
+        url = self.bloburl(blob_id)
         headers = self.get_auth_headers()
         return self._httpapi.get_blob(url, headers)
 
-    def url(self, path):
-        return '{}{}'.format(self._url, path)
+    def bloburl(self, blob_id):
+        if self._blob_url is None:
+            self._blob_url = self.get_blob_service_url()
+        if self._blob_url is not None:
+            return '{}/blobs/{}'.format(self._blob_url, blob_id)
+        logging.error('Do not have blob service URL')
+        return None
 
-    def get_auth_headers(self):
-        token = self._token_generator.get_token()
-        return {
-            'Authorization': 'Bearer {}'.format(token),
-        }
+    def get_blob_service_url(self):
+        url = self.url('/version')
+        headers = self.get_auth_headers()
+        version = self._httpapi.get(url, headers)
+        logging.info('Version: %r', version)
+        if version:
+            return version.get('blob_service')
 
 
 class HttpApi:
@@ -208,6 +227,7 @@ class TokenGenerator:
     iss = 'localhost'
     aud = 'localhost'
     scopes = ' '.join([
+        'uapi_version_get',
         'uapi_work_id_get',
         'uapi_work_post',
         'uapi_workers_post',
@@ -541,7 +561,7 @@ class WorkspaceArchiver(WorkerBase):
         logging.info('Uploading %s', blob_id)
         self.report(b'Uploading workspace\n')
 
-        url = self._api.url('/blobs/{}'.format(blob_id))
+        url = self._api.bloburl(blob_id)
         headers = self._api.get_auth_headers()
         upload = ['curl', '-sk', '-T', filename]
         for name, value in headers.items():
@@ -591,7 +611,7 @@ class SystreePopulator(WorkerBase):
         return self.execute_argv(['sudo', 'find', '-delete'], cwd=dirname)
 
     def download_and_unpack_systree(self, systree_name, dirname):
-        url = self._api.url('/blobs/{}'.format(systree_name))
+        url = self._api.bloburl(systree_name)
         headers = self._api.get_auth_headers()
         download = ['curl', '-k']
         for name, value in headers.items():
diff --git a/worker_manager.yaml b/worker_manager.yaml
deleted file mode 100644
index d7876ab..0000000
--- a/worker_manager.yaml
+++ /dev/null
@@ -1,24 +0,0 @@
-# Copyright 2017 Lars Wirzenius
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-config:
-  controller: https://127.0.0.1
-  name: worker1
-  log: /var/log/ickwm/worker_manager.log
-  log-level: debug
-  log-max: 10M
-  log-keep: 10
-  token-key: /etc/ick/token_key
-  token-key-pub: /etc/ick/token_key.pub
-  workspace: /var/lib/ick/workspace
diff --git a/yarns/100-projects.yarn b/yarns/100-projects.yarn
index 63572e7..7c18952 100644
--- a/yarns/100-projects.yarn
+++ b/yarns/100-projects.yarn
@@ -55,6 +55,7 @@ building them. We start by starting an instance of the controller.
     ... uapi_projects_id_put
     ... uapi_projects_id_delete
     AND controller config uses statedir at the state directory
+    AND controller config uses https://blobs.example.com as blob service
     AND a running ick controller
 
     WHEN user makes request GET /projects
diff --git a/yarns/150-pipelines.yarn b/yarns/150-pipelines.yarn
index 7169dd5..feda6d0 100644
--- a/yarns/150-pipelines.yarn
+++ b/yarns/150-pipelines.yarn
@@ -63,6 +63,7 @@ running them. We start by starting an instance of the controller.
     ... uapi_pipelines_id_put
     ... uapi_pipelines_id_delete
     AND controller config uses statedir at the state directory
+    AND controller config uses https://blobs.example.com as blob service
     AND a running ick controller
 
     WHEN user makes request GET /pipelines
diff --git a/yarns/200-version.yarn b/yarns/200-version.yarn
index fe92b34..2f87ac7 100644
--- a/yarns/200-version.yarn
+++ b/yarns/200-version.yarn
@@ -1,6 +1,6 @@
 <!--
 
-Copyright 2017 Lars Wirzenius
+Copyright 2017-2018 Lars Wirzenius
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU Affero General Public License as published by
@@ -26,11 +26,13 @@ The Ick controller reports is version upon request.
     AND an access token for user with scopes
     ... uapi_version_get
     AND controller config uses statedir at the state directory
+    AND controller config uses https://blobs.example.com as blob service
     AND a running ick controller
 
     WHEN user makes request GET /version
     THEN result has status code 200
     AND version in body matches version from setup.py
+    AND blob service URL is https://blobs.example.com
 
     FINALLY stop ick controller
 
diff --git a/yarns/300-workers.yarn b/yarns/300-workers.yarn
index c94f47c..2cc7ea2 100644
--- a/yarns/300-workers.yarn
+++ b/yarns/300-workers.yarn
@@ -1,6 +1,6 @@
 <!--
 
-Copyright 2017 Lars Wirzenius
+Copyright 2017-2018 Lars Wirzenius
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU Affero General Public License as published by
@@ -60,6 +60,7 @@ controller API. It doesn't actually talk to the worker itself.
     ... uapi_workers_id_put
     ... uapi_workers_id_delete
     AND controller config uses statedir at the state directory
+    AND controller config uses https://blobs.example.com as blob service
     AND a running ick controller
 
     WHEN user makes request GET /workers
diff --git a/yarns/400-build.yarn b/yarns/400-build.yarn
index e0af107..5de9768 100644
--- a/yarns/400-build.yarn
+++ b/yarns/400-build.yarn
@@ -27,6 +27,7 @@ Set up the controller.
 
     GIVEN an RSA key pair for token signing
     AND controller config uses statedir at the state directory
+    AND controller config uses https://blobs.example.com as blob service
     AND an access token for user with scopes
     ... uapi_pipelines_post
     ... uapi_projects_post
@@ -639,6 +640,7 @@ Set up the controller.
 
     GIVEN an RSA key pair for token signing
     AND controller config uses statedir at the state directory
+    AND controller config uses https://blobs.example.com as blob service
     AND an access token for user with scopes
     ... uapi_pipelines_post
     ... uapi_projects_post
diff --git a/yarns/500-build-fail.yarn b/yarns/500-build-fail.yarn
index cde859f..dcc551d 100644
--- a/yarns/500-build-fail.yarn
+++ b/yarns/500-build-fail.yarn
@@ -28,6 +28,7 @@ Set up the controller.
 
     GIVEN an RSA key pair for token signing
     AND controller config uses statedir at the state directory
+    AND controller config uses https://blobs.example.com as blob service
     AND an access token for user with scopes
     ... uapi_pipelines_post
     ... uapi_projects_post
diff --git a/yarns/600-unauthz.yarn b/yarns/600-unauthz.yarn
index e8d9664..0a602af 100644
--- a/yarns/600-unauthz.yarn
+++ b/yarns/600-unauthz.yarn
@@ -28,6 +28,7 @@ Set up the controller.
 
     GIVEN an RSA key pair for token signing
     AND controller config uses statedir at the state directory
+    AND controller config uses https://blobs.example.com as blob service
     AND an access token for user with scopes
     ... uapi_projects_post
     ... uapi_projects_id_pipelines_id_put
diff --git a/yarns/900-local.yarn b/yarns/900-local.yarn
index 2739542..5efad69 100644
--- a/yarns/900-local.yarn
+++ b/yarns/900-local.yarn
@@ -46,6 +46,9 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
     IMPLEMENTS GIVEN controller config uses (\S+) at the state directory
     vars['statedir'] = get_next_match()
 
+    IMPLEMENTS GIVEN controller config uses (\S+) as blob service
+    vars['blob_service'] = get_next_match()
+
 ## Start and stop the controller
 
     IMPLEMENTS GIVEN a running ick controller
@@ -64,6 +67,7 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
             },
         ],
         'statedir': vars['statedir'],
+        'blob-service': vars['blob_service'],
     }
     env = dict(os.environ)
     env['ICK_CONTROLLER_CONFIG'] = 'ick_controller.yaml'
@@ -103,6 +107,15 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
     filename = os.path.join(vars['statedir'], 'workers', basename + '.yaml')
     assertTrue(os.path.exists(filename))
 
+## Check version result
+
+    IMPLEMENTS THEN blob service URL is (\S+)
+    expected = get_next_match()
+    body = vars['body']
+    obj = json.loads(body)
+    actual = obj['blob_service']
+    assertEqual(actual, expected)
+
 ## Start and stop blob service
 
     IMPLEMENTS GIVEN blob service config uses (\S+) at the blob directory