Add: production controller application

Rewrite the controller to be a proper application. This is not quite
production ready, actually: it only runs in debug mode. Later on I'll
add support for uwsgi and systemd units for managing it, but for now
this is enough.

7 files changed, 269 insertions, 103 deletions

diff --git a/controller b/controller
index 6a23afe..66522c9 100755
--- a/controller
+++ b/controller
@@ -1,108 +1,7 @@
 #!/usr/bin/env python2
 
 
-import os
-import random
-import sys
-
-import bottle
-import yaml
-
 import ick2lib
 
 
-@bottle.route('/')
-def root():
-    log('GET /')
-    return 'This is the root'
-
-@bottle.route('/version')
-def version():
-    log('GET /version')
-    return { 'version': '1.0' }
-
-@bottle.route('/projects')
-def projects():
-    return {'projects': [project.name]}
-
-@bottle.route('/worker/bar')
-def worker():
-    log('GET /worker/bar')
-    shell = project.get_current_build_step()
-    if shell is not None:
-        return {
-            'project': project.name,
-            'shell': shell,
-        }
-
-@bottle.route('/projects/foo/logs/current')
-def current():
-    log('GET current log')
-    return project.get_current_log()
-
-@bottle.route('/projects/foo/logs/previous')
-def previous():
-    log('GET previous log')
-    return project.get_previous_log()
-
-@bottle.route('/projects/foo/+trigger')
-def current():
-    log('GET +trigger')
-    project.trigger_build()
-
-@bottle.post('/worker/bar/snippet')
-def snippet():
-    log('POST snippet')
-    obj = bottle.request.json
-    log('body: {}'.format(obj))
-    project.append_to_current_log(obj['stdout'] + obj['stderr'])
-    if obj['exit-code'] is not None:
-        log('exit-code is {}, not None, rotating logs'.format(obj['exit-code']))
-        project.finish_current_log()
-        project.finish_current_build_step()
-
-
-# Command line args.
-
-pid_file = sys.argv[1]
-port_file = sys.argv[2]
-projects_file = sys.argv[3]
-
-
-
-log_file = open('log', 'a')
-def log(msg):
-    log_file.write('{} {}\n'.format(os.getpid(), msg))
-    log_file.flush()
-
-
-# Write pid to named file.
-
-with open(pid_file, 'w') as f:
-    f.write('{}\n'.format(os.getpid()))
-
-
-# Pick a random port and write it to named file.
-port = random.randint(1025, 32767)
-with open(port_file, 'w') as f:
-    f.write('{}\n'.format(port))
-
-
-# Load projects.
-if os.path.exists(projects_file):
-    with open(projects_file) as f:
-        projects_config = yaml.safe_load(f)
-        projects = {}
-        for name, config in projects_config['projects'].items():
-            p = ick2lib.Project(name)
-            projects[name] = p
-            for shell in config['shell_steps']:
-                p.add_build_step(shell)
-
-    project = projects['foo']
-else:
-    project = None
-
-
-log('starting daemon on port {}'.format(port))
-bottle.run(port=port, quiet=True)
+ick2lib.ApiApp().run()
diff --git a/ick2lib/__init__.py b/ick2lib/__init__.py
index 1426276..a195273 100644
--- a/ick2lib/__init__.py
+++ b/ick2lib/__init__.py
@@ -18,3 +18,5 @@
 
 from .version import __version__, __version_info__
 from .project import Project
+from .apiservice import ApiService
+from .app import ApiApp
diff --git a/ick2lib/apiservice.py b/ick2lib/apiservice.py
new file mode 100644
index 0000000..363f3c0
--- /dev/null
+++ b/ick2lib/apiservice.py
@@ -0,0 +1,159 @@
+# Copyright 2017  Lars Wirzenius
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# =*= License: GPL-3+ =*=
+
+
+import logging
+import time
+
+
+import bottle
+
+
+import ick2lib
+
+
+class ApiService(object):
+
+    def __init__(self):
+        self._projects = {}
+        self._app = self._create_bottle_app()
+
+    def _create_bottle_app(self):
+        app = bottle.Bottle()
+
+        routes = [
+            {
+                'method': 'GET',
+                'path': '/',
+                'callback': lambda: None,
+            },
+            {
+                'method': 'GET',
+                'path': '/version',
+                'callback': lambda: { 'version': '1.0' },
+            },
+            {
+                'method': 'GET',
+                'path': '/projects',
+                'callback': self._projects_cb,
+            },
+            {
+                'method': 'GET',
+                'path': '/projects/<project>/logs/current',
+                'callback': self._current_log_cb,
+            },
+            {
+                'method': 'GET',
+                'path': '/projects/<project>/logs/previous',
+                'callback': self._previous_log_cb,
+            },
+            {
+                'method': 'GET',
+                'path': '/projects/<project>/+trigger',
+                'callback': self._trigger_cb,
+            },
+            {
+                'method': 'GET',
+                'path': '/worker/<worker>',
+                'callback': self._work_cb,
+            },
+            {
+                'method': 'POST',
+                'path': '/worker/<worker>/snippet',
+                'callback': self._snippet_cb,
+            },
+        ]
+
+        for route in routes:
+            app.route(**route)
+
+        app.install(LoggingPlugin())
+        return app
+
+    def _projects_cb(self):
+        return {
+            'projects': list(sorted(self._projects.keys())),
+        }
+
+    def _current_log_cb(self, project):
+        return self._projects[project].get_current_log()
+
+    def _previous_log_cb(self, project):
+        return self._projects[project].get_previous_log()
+
+    def _trigger_cb(self, project):
+        return self._projects[project].trigger_build()
+
+    def _work_cb(self, worker):
+        for p in self._projects.values():
+            step = p.get_current_build_step()
+            if step is not None:
+                return {
+                    'project': p.name,
+                    'shell': step,
+                }
+        return None
+
+    def _snippet_cb(self, worker):
+        obj = bottle.request.json
+        p = self._projects[obj['project']]
+        logging.debug('stdout: %r', obj['stdout'])
+        logging.debug('stderr: %r', obj['stderr'])
+        p.append_to_current_log(obj['stdout'] + obj['stderr'])
+        logging.debug('log is now: %r', p.get_current_log())
+        if obj['exit-code'] is not None:
+            p.finish_current_log()
+            p.finish_current_build_step()
+
+    def set_projects(self, projects):
+        self._projects = projects
+
+    def run_debug(self, port):
+        self._app.run(port=port, quiet=True)
+
+
+
+class LoggingPlugin(object):
+
+    def apply(self, callback, route):
+
+        def wrapper(*args, **kwargs):
+            # Do the thing and catch any exceptions.
+            try:
+                self.log_request()
+                data = callback(*args, **kwargs)
+                self.add_response_headers()
+                self.log_response()
+                return data
+            except SystemExit:
+                raise
+            except BaseException as e:
+                logging.error(str(e), exc_info=True)
+                raise bottle.HTTPError(status=500, body=str(e))
+
+        return wrapper
+
+    def add_response_headers(self):
+        rfc822 = time.strftime('%a, %d %b %Y %H:%M:%S %z')
+        bottle.response.set_header('Date', rfc822)
+
+    def log_request(self):
+        r = bottle.request
+        logging.info('Request: %s %s', r.method, r.path)
+
+    def log_response(self):
+        logging.info('Response: %s', bottle.response.status_code)
diff --git a/ick2lib/app.py b/ick2lib/app.py
new file mode 100644
index 0000000..6abf03f
--- /dev/null
+++ b/ick2lib/app.py
@@ -0,0 +1,98 @@
+# Copyright 2017  Lars Wirzenius
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# =*= License: GPL-3+ =*=
+
+
+import logging
+import os
+import random
+
+
+import cliapp
+import yaml
+
+
+import ick2lib
+
+
+class ApiApp(cliapp.Application):
+
+    def add_settings(self):
+        self.settings.boolean(
+            ['debug'],
+            'Turn on debug mode (incl. for yarn tests)'
+        )
+
+        self.settings.string(
+            ['pid-file'],
+            'Store application PID in FILE',
+            metavar='FILE'
+        )
+
+        self.settings.string(
+            ['port-file'],
+            'Store randomly chosen listening port (in debug mode) in FILE',
+            metavar='FILE'
+        )
+
+        self.settings.string(
+            ['projects'],
+            'Read project list from YAML format FILE',
+            metavar='FILE'
+        )
+
+    def write_pid_file(self):
+        write_file(self.settings['pid-file'], str(os.getpid()))
+
+    def pick_random_port(self):
+        return random.randint(1025, 32767)
+
+    def write_port_file(self, port):
+        write_file(self.settings['port-file'], str(port))
+
+    def load_projects(self, filename):
+        with open(filename) as f:
+            projects_config = yaml.safe_load(f)
+
+        projects = {}
+        for name, config in projects_config['projects'].items():
+            p = ick2lib.Project(name)
+            projects[name] = p
+            for shell in config['shell_steps']:
+                p.add_build_step(shell)
+
+        return projects
+
+    def process_args(self, args):
+        assert self.settings['debug']
+
+        projects = {}
+        if os.path.exists(self.settings['projects']):
+            projects = self.load_projects(self.settings['projects'])
+
+        self.write_pid_file()
+        port = self.pick_random_port()
+        self.write_port_file(port)
+
+        apiapp = ick2lib.ApiService()
+        apiapp.set_projects(projects)
+        apiapp.run_debug(port)
+
+
+def write_file(filename, content):
+    logging.info('Writing file %s: %r', filename, content)
+    with open(filename, 'w') as f:
+        f.write('{}\n'.format(content))
diff --git a/without-tests b/without-tests
index 7afaae3..bb41680 100644
--- a/without-tests
+++ b/without-tests
@@ -1,3 +1,5 @@
 ick2lib/__init__.py
+ick2lib/apiservice.py
+ick2lib/app.py
 ick2lib/version.py
 
diff --git a/yarns/200-build.yarn b/yarns/200-build.yarn
index 77daebc..40cb1c9 100644
--- a/yarns/200-build.yarn
+++ b/yarns/200-build.yarn
@@ -41,6 +41,7 @@ pretend output we sent ends up in the current build log.
 
     WHEN worker manager calls POST /worker/bar/snippet,
     ... with JSON body '{
+    ...     "project": "foo",
     ...     "stdout": "ikiwiki build output",
     ...     "stderr": "",
     ...     "exit-code": null
@@ -62,6 +63,7 @@ that we get a new thing to run.
 
     WHEN worker manager calls POST /worker/bar/snippet,
     ... with JSON body '{
+    ...     "project": "foo",
     ...     "stdout": "|more output",
     ...     "stderr": "",
     ...     "exit-code": 0
@@ -86,6 +88,7 @@ previous log will contain the previously current log.
 
     WHEN worker manager calls POST /worker/bar/snippet,
     ... with JSON body '{
+    ...     "project": "foo",
     ...     "stdout": "rsync output",
     ...     "stderr": "",
     ...     "exit-code": 0
diff --git a/yarns/900.yarn b/yarns/900.yarn
index 23ae291..82ab42a 100644
--- a/yarns/900.yarn
+++ b/yarns/900.yarn
@@ -4,7 +4,10 @@
 
     IMPLEMENTS GIVEN a running controller instance
     controller = os.path.join(srcdir, 'controller')
-    cliapp.runcmd(['/usr/sbin/daemonize', '-c.', controller, 'pid', 'port', 'ick.ick'])
+    cliapp.runcmd(
+        ['/usr/sbin/daemonize', '-c.',
+         controller, '--debug', '--pid-file=pid',
+         '--port-file=port', '--projects=ick.ick', '--log=log'])
     vars['pid'] = cat('pid').strip()
     vars['port'] = cat('port').strip()