Merge branch 'liw/workspace'

10 files changed, 101 insertions, 24 deletions

diff --git a/debian/ick2.dirs b/debian/ick2.dirs
index f3a2c00..a398f42 100644
--- a/debian/ick2.dirs
+++ b/debian/ick2.dirs
@@ -1 +1,2 @@
 var/lib/ick
+var/lib/ick/workspace
diff --git a/debian/ick2.postinst b/debian/ick2.postinst
index 028139b..f66dbda 100644
--- a/debian/ick2.postinst
+++ b/debian/ick2.postinst
@@ -37,10 +37,13 @@ create_user()
 create_user _ick /var/lib/ick /var/log/ick
 create_user _ickwm /var/lib/ick/wm /var/log/ickwm
 
+# Create controller config dir.
+install -d -m 0755 -o root -g root /etc/ick
+
 # Create controller state dir
 install -d  -m 0755 -o _ick -g _ick /var/lib/ick/state
 
-# Create controller config dir.
-install -d -m 0755 -o root -g root /etc/ick
+# Create worker-manager workspace
+install -d  -m 0755 -o _ickwm -g _ickwm /var/lib/ick/workspace
 
 #DEBHELPER#
diff --git a/ick2/apibase.py b/ick2/apibase.py
index 1e4fd14..088a2b6 100644
--- a/ick2/apibase.py
+++ b/ick2/apibase.py
@@ -61,7 +61,8 @@ class APIbase:
                 body = callback(**kwargs)
             except ick2.NotFound as e:
                 ick2.log.log(
-                    'warning', msg_text='GET Not found', kwargs=kwargs)
+                    'error', msg_text='GET Not found', kwargs=kwargs,
+                    exc_info=True)
                 return ick2.not_found(e)
             if isinstance(body, dict):
                 return ick2.OK(body)
diff --git a/ick2/workapi.py b/ick2/workapi.py
index f140108..f958132 100644
--- a/ick2/workapi.py
+++ b/ick2/workapi.py
@@ -62,6 +62,7 @@ class WorkAPI(ick2.APIbase):
                     'project': project['project'],
                     'pipeline': pipeline['name'],
                     'parameters': project.get('parameters', {}),
+                    'fresh_workspace': True,
                     'step': pipeline['actions'][index],
                     'step_index': index,
                     'log': '/logs/{}'.format(build_id),
@@ -154,7 +155,8 @@ class WorkAPI(ick2.APIbase):
                 self._finish_build(update)
             else:
                 doing['step_index'] = index
-                doing['step'] = actions[index]
+                doing['step'] = dict(actions[index])
+                doing['fresh_workspace'] = False
 
             worker_state = {
                 'worker': update['worker'],
diff --git a/ick2/workapi_tests.py b/ick2/workapi_tests.py
index c855895..d7ebfa9 100644
--- a/ick2/workapi_tests.py
+++ b/ick2/workapi_tests.py
@@ -86,6 +86,7 @@ class WorkAPITests(unittest.TestCase):
             'parameters': {
                 'foo': 'bar',
             },
+            'fresh_workspace': True,
             'step': {
                 'shell': 'step-1',
             },
@@ -112,6 +113,7 @@ class WorkAPITests(unittest.TestCase):
             'parameters': {
                 'foo': 'bar',
             },
+            'fresh_workspace': True,
             'step': {
                 'shell': 'step-1',
             },
@@ -144,6 +146,7 @@ class WorkAPITests(unittest.TestCase):
         # We should get the next step now.
         expected['step'] = {'shell': 'step-2'}
         expected['step_index'] = 1
+        expected['fresh_workspace'] = False
         self.assertEqual(work.get_work('asterix'), expected)
 
         # Finish the step.
@@ -173,6 +176,7 @@ class WorkAPITests(unittest.TestCase):
             'parameters': {
                 'foo': 'bar',
             },
+            'fresh_workspace': True,
             'step': {
                 'shell': 'step-1',
             },
diff --git a/icktool b/icktool
index 8b89a84..0090b2b 100755
--- a/icktool
+++ b/icktool
@@ -40,6 +40,22 @@ def scopes(base):
     return [x.format(base) for x in patterns]
 
 
+def scopes_for_types(types):
+    result = []
+    for type_name in types:
+        result.extend(scopes(type_name))
+    return result
+
+
+types = [
+    'projects',
+    'pipelines',
+    'workers',
+    'work',
+    'builds',
+    'logs',
+]
+
 class Icktool(cliapp.Application):
 
     _default_scopes = [
@@ -47,7 +63,7 @@ class Icktool(cliapp.Application):
         'uapi_work_post',
         'uapi_projects_id_pipelines_id_get',
         'uapi_projects_id_pipelines_id_put',
-    ] + scopes('projects') + scopes('workers') + scopes('work') + scopes('builds') + scopes('logs')
+    ] + scopes_for_types(types)
 
     def add_settings(self):
         self.settings.string(
@@ -110,12 +126,14 @@ class Icktool(cliapp.Application):
             pipelines = project['pipelines']
             for pipeline in pipelines:
                 build = self._get_latest_build(
-                    project['project'], pipeline['name'], builds)
+                    project['project'], pipeline, builds)
+                status = self._get_pipeline_status(
+                    project['project'], pipeline)
                 row = {
                     'project': project['project'],
-                    'pipeline': pipeline['name'],
+                    'pipeline': pipeline,
                     'build_id': build['build_id'],
-                    'status': pipeline['status'],
+                    'status': status['status'],
                     'log': build['log'],
                 }
                 rows.append(row)
@@ -160,15 +178,39 @@ class Icktool(cliapp.Application):
         name = args[0]
         rc.delete(name)
 
+    def cmd_list_pipelines(self, args):
+        rc = self._new_rc('/pipelines', 'name')
+        self._prettyson(rc.list())
+
+    def cmd_create_pipeline(self, args):
+        rc = self._new_rc('/pipelines', 'name')
+        obj = self._read_object()
+        rc.create(obj)
+
+    def cmd_update_pipeline(self, args):
+        rc = self._new_rc('/pipelines', 'name')
+        obj = self._read_object()
+        rc.update(obj)
+
     def cmd_show_pipeline(self, args):
-        project = args[0]
-        pipeline = args[1]
+        rc = self._new_rc('/pipelines', 'pipeline')
+        name = args[0]
+        self._prettyson(rc.show(name))
+
+    def cmd_delete_pipeline(self, args):
+        rc = self._new_rc('/pipelines', 'pipeline')
+        name = args[0]
+        rc.delete(name)
+
+    def cmd_show_pipeline_status(self, args):
+        self._prettyson(self._get_pipeline_status(args[0], args[1]))
+
+    def _get_pipeline_status(self, project, pipeline):
         path = '/projects/{}/pipelines/{}'.format(project, pipeline)
         api = self._new_api()
         code, text = api.get(path)
         self._report(code, 200, text)
-        obj = json.loads(text)
-        self._prettyson(obj)
+        return json.loads(text)
 
     def _report(self, code, expected, text):
         if code != expected:
diff --git a/worker_manager b/worker_manager
index f13b431..a480558 100755
--- a/worker_manager
+++ b/worker_manager
@@ -83,6 +83,13 @@ class WorkerManager(cliapp.Application):
             default=5,
         )
 
+        self.settings.string(
+            ['workspace'],
+            'use DIR as the workspace where commands are run',
+            metavar='DIR',
+            default='/var/lib/ick/workspace',
+        )
+
     def process_args(self, args):
         self.settings.require('name')
         name = self.settings['name']
@@ -94,7 +101,7 @@ class WorkerManager(cliapp.Application):
             if work and self.settings['pretend']:
                 self.report_pretend_work(url, name, work)
             elif work:
-                self.do_work(url, name, work)
+                self.do_work(url, name, work, self.settings['workspace'])
             else:
                 self.show_msg('Nothing to do')
                 self.sleep_a_little()
@@ -180,7 +187,7 @@ class WorkerManager(cliapp.Application):
         }
         self.post_snippet(snippet_url, snippet)
 
-    def do_work(self, url, name, work):
+    def do_work(self, url, name, work, workspace):
         self.show_msg('Doing work')
         snippet_url = '{}/work'.format(url)
         snippet = {
@@ -200,16 +207,24 @@ class WorkerManager(cliapp.Application):
             s[stream_name] = data
             self.post_snippet(snippet_url, s)
 
-        shell_cmd = work['step']['shell']
-        exit_code, _, _ = cliapp.runcmd_unchecked(
-            ['bash', '-xeuc', shell_cmd],
-            stdout_callback=lambda data: post('stdout', data),
-            stderr_callback=lambda data: post('stderr', data),
-        )
-
-        end_snippet = dict(snippet)
-        end_snippet['exit_code'] = exit_code
-        self.post_snippet(snippet_url, end_snippet)
+        def run_shell(shell_cmd):
+            exit_code, _, _ = cliapp.runcmd_unchecked(
+                ['bash', '-xeuc', shell_cmd],
+                stdout_callback=lambda data: post('stdout', data),
+                stderr_callback=lambda data: post('stderr', data),
+                cwd=workspace,
+            )
+            end_snippet = dict(snippet)
+            end_snippet['exit_code'] = exit_code
+            self.post_snippet(snippet_url, end_snippet)
+            return exit_code
+
+        exit_code = 0
+        if work.get('fresh_workspace'):
+            exit_code = run_shell('find . -delete')
+        if exit_code == 0:
+            shell_cmd = work['step']['shell']
+            run_shell(shell_cmd)
 
     def now(self):
         return time.strftime('%Y-%m-%dT%H:%M:%S')
diff --git a/worker_manager.yaml b/worker_manager.yaml
index 1279336..d7876ab 100644
--- a/worker_manager.yaml
+++ b/worker_manager.yaml
@@ -21,3 +21,4 @@ config:
   log-keep: 10
   token-key: /etc/ick/token_key
   token-key-pub: /etc/ick/token_key.pub
+  workspace: /var/lib/ick/workspace
diff --git a/yarns/400-build.yarn b/yarns/400-build.yarn
index ec3e8e5..4591459 100644
--- a/yarns/400-build.yarn
+++ b/yarns/400-build.yarn
@@ -120,6 +120,7 @@ be in the path or can we get it in the access token?**
     ...     "parameters": {
     ...         "foo": "bar"
     ...     },
+    ...     "fresh_workspace": true,
     ...     "step": {
     ...         "shell": "day 1"
     ...     },
@@ -138,6 +139,7 @@ be in the path or can we get it in the access token?**
     ...     "parameters": {
     ...         "foo": "bar"
     ...     },
+    ...     "fresh_workspace": true,
     ...     "step": {
     ...         "shell": "day 1"
     ...     },
@@ -167,6 +169,7 @@ User can now see pipeline is running and which worker is building it.
     ...         "parameters": {
     ...             "foo": "bar"
     ...         },
+    ...         "fresh_workspace": true,
     ...         "step_index": 0,
     ...         "step": {
     ...             "shell": "day 1"
@@ -225,6 +228,7 @@ Still the same job, since the first build step didnt't finish.
     ...     "parameters": {
     ...         "foo": "bar"
     ...     },
+    ...     "fresh_workspace": true,
     ...     "step": {
     ...         "shell": "day 1"
     ...     },
@@ -272,6 +276,7 @@ Now there's another step to do.
     ...     "parameters": {
     ...         "foo": "bar"
     ...     },
+    ...     "fresh_workspace": false,
     ...     "step": {
     ...         "shell": "day 2"
     ...     },
@@ -293,6 +298,7 @@ User sees changed status.
     ...         "parameters": {
     ...             "foo": "bar"
     ...         },
+    ...         "fresh_workspace": false,
     ...         "step_index": 1,
     ...         "step": {
     ...             "shell": "day 2"
@@ -383,6 +389,7 @@ Start build again. This should become build number 2.
     ...     "parameters": {
     ...         "foo": "bar"
     ...     },
+    ...     "fresh_workspace": true,
     ...     "step": {
     ...         "shell": "day 1"
     ...     },
diff --git a/yarns/500-build-fail.yarn b/yarns/500-build-fail.yarn
index f0afa32..bd6f22b 100644
--- a/yarns/500-build-fail.yarn
+++ b/yarns/500-build-fail.yarn
@@ -88,6 +88,7 @@ Worker wants work and gets the first step to run.
     ...     "project": "rome",
     ...     "pipeline": "construct",
     ...     "parameters": {},
+    ...     "fresh_workspace": true,
     ...     "step": {
     ...         "shell": "day 1"
     ...     },