Merge: deployment, worker-manager, icktool fixes

With this, it should be possible to deploy a working Ick2 with a
worker. Mostly-working, at least. There's still bugs.

5 files changed, 120 insertions, 28 deletions

diff --git a/NEWS b/NEWS
index 17f7325..22846e6 100644
--- a/NEWS
+++ b/NEWS
@@ -20,6 +20,12 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 Version 0.13+git, not yet released
 ----------------------------------
 
+* The contoller can now schedule work for a worker, to run a build.
+
+* `icktool` can now show and set the status of a pipeline, show
+  builds, and show logs.
+
+* `worker-manager` works with the new version of the controller.
 
 Version 0.13, released 2017-10-16
 ----------------------------------
diff --git a/debian/control b/debian/control
index 50a8bfb..1e20006 100644
--- a/debian/control
+++ b/debian/control
@@ -34,5 +34,6 @@ Depends: ${python3:Depends}, ${misc:Depends},
   python3-requests,
   gunicorn3,
   python3-yaml,
-Description: a work-in-progress CI server
+  adduser
+Description: work-in-progress CI server
  This should be written.
diff --git a/debian/ick2.dirs b/debian/ick2.dirs
index 7bcc774..f3a2c00 100644
--- a/debian/ick2.dirs
+++ b/debian/ick2.dirs
@@ -1 +1 @@
-var/lib/ick2
+var/lib/ick
diff --git a/icktool b/icktool
index 999bd50..3c8dec9 100755
--- a/icktool
+++ b/icktool
@@ -20,7 +20,6 @@ import logging
 import sys
 
 import cliapp
-import urllib3
 import requests
 
 import ick2
@@ -41,7 +40,8 @@ class Icktool(cliapp.Application):
 
     _default_scopes = [
         'uapi_version_get',
-    ] + scopes('projects') + scopes('workers')
+        'uapi_work_post',
+    ] + scopes('projects') + scopes('work') + scopes('builds') + scopes('logs')
 
     def add_settings(self):
         self.settings.string(
@@ -120,6 +120,35 @@ class Icktool(cliapp.Application):
         name = args[0]
         rc.delete(name)
 
+    def cmd_show_pipeline(self, args):
+        project = args[0]
+        pipeline = args[1]
+        path = '/projects/{}/pipelines/{}'.format(project, pipeline)
+        api = self._new_api()
+        code, text = api.get(path)
+        self._report(code, 200, text)
+        obj = json.loads(text)
+        self._prettyson(obj)
+
+    def _report(self, code, expected, text):
+        if code != expected:
+            sys.stderr.write('HTTP status {}\n'.format(code))
+            sys.stderr.write(text)
+            if not text.endswith('\n'):
+                sys.stderr.write('\n')
+            sys.exit(1)
+
+    def cmd_set_pipeline(self, args):
+        project = args[0]
+        pipeline = args[1]
+        state = args[2]
+        path = '/projects/{}/pipelines/{}'.format(project, pipeline)
+        api = self._new_api()
+        code, text = api.put(path, {'status': state})
+        self._report(code, 200, text)
+        obj = json.loads(text)
+        self._prettyson(obj)
+
     def cmd_list_workers(self, args):
         rc = self._new_rc('/workers', 'worker')
         self._prettyson(rc.list())
@@ -144,6 +173,22 @@ class Icktool(cliapp.Application):
         name = args[0]
         rc.delete(name)
 
+    def cmd_list_builds(self, args):
+        rc = self._new_rc('/builds', 'build')
+        self._prettyson(rc.list())
+
+    def cmd_list_logs(self, args):
+        rc = self._new_rc('/logs', 'log')
+        self._prettyson(rc.list())
+
+    def cmd_show_log(self, args):
+        log = args[0]
+        path = '/logs/{}'.format(log)
+        api = self._new_api()
+        code, text = api.get(path)
+        self._report(code, 200, text)
+        self.output.write(text)
+
     def _new_token(self):
         scopes = self.settings['scope']
         cmd = self.settings['token-private-key-cmd']
diff --git a/worker_manager b/worker_manager
index bddc7a6..f5492cb 100755
--- a/worker_manager
+++ b/worker_manager
@@ -16,15 +16,21 @@
 
 
 import json
-import os
+import logging
+import sys
 import time
 
 import cliapp
 import requests
+import urllib3
 
 import ick2
 
 
+urllib3.disable_warnings()
+logging.captureWarnings(False)
+
+
 class WorkerManager(cliapp.Application):
 
     def add_settings(self):
@@ -45,6 +51,12 @@ class WorkerManager(cliapp.Application):
             metavar='URL',
         )
 
+        self.settings.string(
+            ['token'],
+            'use TOKEN for all controller HTTP requests',
+            metavar='TOKEN',
+        )
+
         self.settings.integer(
             ['sleep'],
             'sleep for SECS seconds if there is no work currently',
@@ -56,7 +68,8 @@ class WorkerManager(cliapp.Application):
         self.settings.require('name')
         name = self.settings['name']
         url = self.settings['controller']
-        print('Worker manager {} starts, controller is {}'.format(name, url))
+        self.show_msg(
+            'Worker manager {} starts, controller is {}'.format(name, url))
         while True:
             work = self.get_work(url, name)
             if work and self.settings['pretend']:
@@ -64,73 +77,100 @@ class WorkerManager(cliapp.Application):
             elif work:
                 self.do_work(url, name, work)
             else:
+                self.show_msg('Nothing to do')
                 self.sleep_a_little()
 
     def sleep_a_little(self):
         secs = self.settings['sleep']
-        print('Sleeping for {} seconds'.format(secs))
+        self.show_msg('Sleeping for {} seconds'.format(secs))
         time.sleep(secs)
 
     def get_work(self, url, name):
-        get_work_url = '{}/worker/{}'.format(url, name)
-        print('Getting work from {}'.format(get_work_url))
-        r = requests.get(get_work_url)
+        get_work_url = '{}/work/{}'.format(url, name)
+        self.show_msg('Getting work from {}'.format(get_work_url))
+        headers = self.get_auth_headers()
+        r = requests.get(get_work_url, headers=headers, verify=False)
         if r.status_code != 200 or not r.text:
             return None
-        print('Response: {!r}'.format(r.text))
-        return r.json()
+        work = r.json()
+        self.show_json('Response:', work)
+        return work
+
+    def get_auth_headers(self):
+        token = self.settings['token']
+        return {
+            'Authorization': 'Bearer {}'.format(token),
+        }
 
     def report_pretend_work(self, url, name, work):
-        print('Pretending to work: {!r}'.format(work))
-        snippet_url = '{}/worker/{}/snippet'.format(url, name)
+        self.show_msg('Pretending to work: {!r}'.format(work))
+        snippet_url = '{}/work/{}'.format(url, name)
         snippet = {
+            'build_id': work['build_id'],
+            'worker': name,
             'project': work['project'],
-            'stdout': 'pretending: {}'.format(work['shell']),
+            'stdout': '',
             'stderr': '',
-            'exit-code': 0,
+            'exit_code': None,
+            'timestamp': self.now(),
         }
         self.post_snippet(snippet_url, snippet)
 
     def do_work(self, url, name, work):
-        print('Doing work: {!r}'.format(work))
-        snippet_url = '{}/worker/{}/snippet'.format(url, name)
+        self.show_msg('Doing work')
+        snippet_url = '{}/work'.format(url)
         snippet = {
+            'build_id': work['build_id'],
+            'worker': name,
             'project': work['project'],
+            'pipeline': work['pipeline'],
             'stdout': '',
             'stderr': '',
-            'exit-code': None,
+            'exit_code': None,
+            'timestamp': self.now(),
         }
 
         def post(stream_name, data):
-            print('{}: {!r}'.format(stream_name, data))
+            data = data.decode('UTF-8')
             s = dict(snippet)
             s[stream_name] = data
             self.post_snippet(snippet_url, s)
 
-        env = dict(os.environ)
-        env['ICK_URL'] = work['git']
-
+        shell_cmd = work['step']['shell']
         exit_code, _, _ = cliapp.runcmd_unchecked(
-            ['bash', '-xeuc', work['shell']],
+            ['bash', '-xeuc', shell_cmd],
             stdout_callback=lambda data: post('stdout', data),
             stderr_callback=lambda data: post('stderr', data),
-            env=env,
         )
 
         end_snippet = dict(snippet)
-        end_snippet['exit-code'] = exit_code
+        end_snippet['exit_code'] = exit_code
         self.post_snippet(snippet_url, end_snippet)
 
+    def now(self):
+        return time.strftime('%Y-%m-%dT%H:%M:%S')
+
     def post_snippet(self, url, snippet):
         headers = {
             'Content-Type': 'application/json',
         }
-        print('POST {} {!r}'.format(url, snippet))
-        r = requests.post(url, headers=headers, data=json.dumps(snippet))
+        headers.update(self.get_auth_headers())
+        self.show_json('POST {}'.format(url), snippet)
+        r = requests.post(
+            url, headers=headers, data=json.dumps(snippet), verify=False)
         if not r.ok:
             raise cliapp.AppException(
                 'Error posting data to controller: {} {!r}'.format(
                     r.status_code, r.text))
 
+    def show_msg(self, msg):
+        sys.stdout.write('------\n{}\n'.format(msg))
+        logging.info(msg)
+
+    def show_json(self, prelude, obj):
+        msg = '{}: {}'.format(prelude, json.dumps(obj, indent=4))
+        sys.stdout.write('------\n{}\n'.format(msg))
+        logging.info(msg)
+
 
 WorkerManager(version=ick2.__version__).run()