Add: classes for pipeline actions

3 files changed, 533 insertions, 0 deletions

diff --git a/ick2/__init__.py b/ick2/__init__.py
index 2a03a25..8673046 100644
--- a/ick2/__init__.py
+++ b/ick2/__init__.py
@@ -67,3 +67,13 @@ from .actionenvs import (
     ContainerEnvironment,
 )
 from .workspace import WorkspaceArea, Workspace
+from .actions import (
+    ActionFactory,
+    UnknownStepError,
+    ShellAction,
+    PythonAction,
+    DebootstrapAction,
+    CreateWorkspaceAction,
+    ArchiveWorkspaceAction,
+    PopulateSystreeAction,
+)
diff --git a/ick2/actions.py b/ick2/actions.py
new file mode 100644
index 0000000..fef6889
--- /dev/null
+++ b/ick2/actions.py
@@ -0,0 +1,293 @@
+# Copyright (C) 2018  Lars Wirzenius
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+import base64
+import json
+import logging
+import os
+import tempfile
+
+
+import cliapp
+
+
+import ick2
+
+
+class UnknownStepError(Exception):
+
+    pass
+
+
+class ActionFactory:
+
+    _classes = {
+        'host': ick2.HostEnvironment,
+        'chroot': ick2.ChrootEnvironment,
+        'container': ick2.ContainerEnvironment,
+    }
+
+    def __init__(self, systree, workspace_area, reporter):
+        self._systree = systree
+        self._workspace_area = workspace_area
+        self._reporter = reporter
+        self._token = None
+        self._blob_url = None
+
+    def set_token(self, token):
+        self._token = token
+
+    def get_token(self):
+        return self._token
+
+    def set_blob_url_func(self, func):
+        self._blob_url = func
+
+    def get_blob_url_func(self):
+        return self._blob_url
+
+    def get_allowed_environments(self):
+        return list(self._classes.keys())
+
+    def create_environment(self, spec, project_name):
+        env = spec.get('where', 'host')
+        assert env in self.get_allowed_environments()
+        env_class = self._classes[env]
+        area = ick2.WorkspaceArea()
+        area.set_root(self._workspace_area)
+        ws = area.create_workspace(project_name)
+        return env_class(self._systree, ws.get_directory(), self._reporter)
+
+    def create_action(self, spec, project_name):
+        env = self.create_environment(spec, project_name)
+        action = self._create_action_object(env, spec)
+        action.set_token(self.get_token())
+        action.set_blob_url_func(self.get_blob_url_func())
+        return action
+
+    def _create_action_object(self, env, spec):
+        if 'shell' in spec:
+            return ShellAction(env)
+        if 'python' in spec:
+            return PythonAction(env)
+        if 'debootstrap' in spec:
+            return DebootstrapAction(env)
+        if 'archive' in spec:
+            return ArchiveWorkspaceAction(env)
+        if spec.get('action') == 'populate_systree':
+            return PopulateSystreeAction(env)
+        if spec.get('action') == 'create_workspace':
+            return CreateWorkspaceAction(env)
+        raise UnknownStepError('Unknown action %r' % spec)
+
+
+class Action:  # pragma: no cover
+
+    def __init__(self, env):
+        self._env = env
+        self._token = None
+        self._blob_url = None
+
+    def set_token(self, token):
+        self._token = token
+
+    def get_token(self):
+        return self._token
+
+    def set_blob_url_func(self, func):
+        self._blob_url = func
+
+    def get_blob_upload_url(self, blob_name):
+        return self._blob_url(blob_name)
+
+    def get_env(self):
+        return self._env
+
+    def get_workspace_area(self):
+        env = self.get_env()
+        area = ick2.WorkspaceArea()
+        area.set_root(env.get_workspace_directory())
+        return area
+
+    def encode_parameters(self, parsms):
+        raise NotImplementedError()
+
+    def encode64(self, params):
+        assert isinstance(params, dict)
+        as_text = json.dumps(params)
+        as_bytes = as_text.encode('UTF-8')
+        as_base64 = base64.b64encode(as_bytes)
+        return as_base64.decode('UTF-8')
+
+    def decode64(self, encoded):
+        as_base64 = encoded.encode('UTF-8')
+        as_bytes = base64.b64decode(as_base64)
+        as_text = as_bytes.decode('UTF-8')
+        return json.loads(as_text)
+
+    def get_authz_headers(self):
+        token = self.get_token()
+        return {
+            'Authorization': 'Bearer {}'.format(token),
+        }
+
+    def execute(self, params, work):
+        raise NotImplementedError()
+
+
+class ShellAction(Action):
+
+    def encode_parameters(self, params):  # pragma: no cover
+        encoded = self.encode64(params)
+        return 'params() { echo -n "%s" | base64 -d; }\n' % encoded
+
+    def execute(self, params, step):
+        prefix = self.encode_parameters(params)
+        snippet = step['shell']
+        argv = ['bash', '-exuc', prefix + snippet]
+        exit_code = self._env.runcmd(argv)
+        self._env.report(exit_code, 'action finished\n')
+        return exit_code
+
+
+class PythonAction(Action):
+
+    def encode_parameters(self, params):  # pragma: no cover
+        encoded = self.encode64(params)
+        prefix = (
+            'import base64, json\n'
+            'params = json.loads(base64.b64decode(\n'
+            '  "{}").decode("utf8"))\n'
+        ).format(encoded)
+        return prefix
+
+    def execute(self, params, step):
+        prefix = self.encode_parameters(params)
+        snippet = step['python']
+        argv = ['python3', '-c', prefix + '\n' + snippet]
+        exit_code = self._env.runcmd(argv)
+        self._env.report(exit_code, 'action finished\n')
+        return exit_code
+
+
+class DebootstrapAction(Action):
+
+    default_mirror = 'http://deb.debian.org/debian'
+
+    def encode_parameters(self, params):  # pragma: no cover
+        pass
+
+    def execute(self, params, step):
+        suite = step.get('debootstrap')
+        if suite is None or suite == 'auto':
+            suite = params['debian_codename']
+        mirror = step.get('mirror', self.default_mirror)
+
+        env = self.get_env()
+        workspace = env.get_workspace_directory()
+        argv = ['sudo', 'debootstrap', suite, '.', mirror]
+        exit_code = self._env.host_runcmd(argv, cwd=workspace)
+        self._env.report(exit_code, 'action finished\n')
+        return exit_code
+
+
+class CreateWorkspaceAction(Action):
+
+    def encode_parameters(self, params):  # pragma: no cover
+        pass
+
+    def execute(self, params, step):
+        logging.debug('CreateWorkspaceAction: params=%r', params)
+        logging.debug('CreateWorkspaceAction: step=%r', step)
+
+        env = self.get_env()
+        workspace = env.get_workspace_directory()
+        self._env.report(0, 'Created workspace %s\n' % workspace)
+        return 0
+
+
+class ArchiveWorkspaceAction(Action):  # pragma: no cover
+
+    def encode_parameters(self, params):
+        pass
+
+    def execute(self, params, step):
+        blob_name = params.get('systree_name', 'noname')
+
+        env = self.get_env()
+        dirname = env.get_workspace_directory()
+
+        url = self.get_blob_upload_url(blob_name)
+        headers = self.get_authz_headers()
+
+        logging.debug('ArchiveWorkspaceAction: url=%r', url)
+        logging.debug('ArchiveWorkspaceAction: headers=%r', headers)
+
+        assert url is not None
+        assert headers is not None
+
+        fd, tarball = tempfile.mkstemp()
+        os.close(fd)
+        tar = ['sudo', 'tar', '-zcf', tarball, '-C', dirname, '.']
+        exit_code = self._env.host_runcmd(tar)
+        if exit_code != 0:
+            self._env.report(exit_code, 'tarball generation finished\n')
+            os.remove(tarball)
+            return exit_code
+        self._env.report(None, 'tarball generation finished\n')
+
+        curl = ['curl', '-sk', '-T', tarball] + [
+            '-H{}:{}'.format(name, value)
+            for name, value in headers.items()
+        ] + [url]
+        exit_code = self._env.host_runcmd(curl)
+        self._env.report(
+            exit_code, 'curl upload finished (exit code %s)\n' % exit_code)
+        os.remove(tarball)
+        return exit_code
+
+
+class PopulateSystreeAction(Action):  # pragma: no cover
+
+    def encode_parameters(self, params):
+        pass
+
+    def execute(self, params, step):
+        systree_name = step.get('systree_name')
+        if not systree_name:
+            return 1
+
+        env = self.get_env()
+        systree_dir = env.get_systree_directory()
+        self.make_directory_empty(systree_dir)
+        return self.download_and_unpack_systree(systree_name, systree_dir)
+
+    def make_directory_empty(self, dirname):
+        return cliapp.runcmd(
+            ['sudo', 'find', dirname, '-mindepth', '1', '-delete'])
+
+    def download_and_unpack_systree(self, systree_name, dirname):
+        url = self.get_blob_upload_url(systree_name)
+        headers = self.get_authz_headers()
+        curl = ['curl', '-sk'] + [
+            '-H{}:{}'.format(name, value)
+            for name, value in headers.items()
+        ] + [url]
+
+        untar = ['sudo', 'tar', '-zxf', '-', '-C', dirname]
+
+        exit_code = self._env.host_runcmd(curl, untar)
+        self._env.report(exit_code, 'action finished\n')
+        return exit_code
diff --git a/ick2/actions_tests.py b/ick2/actions_tests.py
new file mode 100644
index 0000000..8220fe2
--- /dev/null
+++ b/ick2/actions_tests.py
@@ -0,0 +1,230 @@
+# Copyright (C) 2018  Lars Wirzenius
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+import os
+import shutil
+import tempfile
+import unittest
+
+
+import ick2
+
+
+class ActionFactoryTests(unittest.TestCase):
+
+    def setUp(self):
+        self.tempdir = tempfile.mkdtemp()
+        self.workspace_area = self.tempdir
+        self.project = 'magic'
+        self.af = ick2.ActionFactory('systree', self.workspace_area, None)
+
+    def tearDown(self):
+        shutil.rmtree(self.tempdir)
+
+    def test_sets_token(self):
+        token = 'this is a token'
+        self.af.set_token(token)
+        self.assertEqual(self.af.get_token(), token)
+
+    def test_lists_allows_environments(self):
+        allowed = self.af.get_allowed_environments()
+        self.assertTrue(isinstance(allowed, list))
+        self.assertNotEqual(allowed, [])
+
+    def test_creates_host_environment_by_default(self):
+        spec = {
+            'shell': 'echo hello world',
+        }
+        env = self.af.create_environment(spec, self.project)
+        self.assertTrue(isinstance(env, ick2.HostEnvironment))
+
+        self.assertEqual(
+            env.get_workspace_directory(),
+            os.path.join(self.workspace_area, self.project))
+
+    def test_creates_host_environment_when_asked_explicitly(self):
+        spec = {
+            'where': 'host',
+        }
+        env = self.af.create_environment(spec, self.project)
+        self.assertTrue(isinstance(env, ick2.HostEnvironment))
+
+    def test_creates_chroot_environment_by_default(self):
+        spec = {
+            'where': 'chroot',
+        }
+        env = self.af.create_environment(spec, self.project)
+        self.assertTrue(isinstance(env, ick2.ChrootEnvironment))
+
+    def test_creates_container_environment_by_default(self):
+        spec = {
+            'where': 'container',
+        }
+        env = self.af.create_environment(spec, self.project)
+        self.assertTrue(isinstance(env, ick2.ContainerEnvironment))
+
+    def test_creates_shell_action_on_host(self):
+        def url(blob_name):
+            return blob_name
+
+        spec = {
+            'shell': 'echo hello, world',
+        }
+        token = 'secret token'
+        self.af.set_token(token)
+        self.af.set_blob_url_func(url)
+        action = self.af.create_action(spec, self.project)
+        self.assertTrue(isinstance(action, ick2.ShellAction))
+        self.assertTrue(isinstance(action.get_env(), ick2.HostEnvironment))
+        self.assertEqual(action.get_token(), token)
+        self.assertEqual(action.get_blob_upload_url('foo'), 'foo')
+
+    def test_creates_python_action_on_host(self):
+        spec = {
+            'python': 'pass',
+        }
+        action = self.af.create_action(spec, self.project)
+        self.assertTrue(isinstance(action, ick2.PythonAction))
+        self.assertTrue(isinstance(action.get_env(), ick2.HostEnvironment))
+
+    def test_creates_debootstrap_action_on_host(self):
+        spec = {
+            'debootstrap': 'auto',
+        }
+        action = self.af.create_action(spec, self.project)
+        self.assertTrue(isinstance(action, ick2.DebootstrapAction))
+        self.assertTrue(isinstance(action.get_env(), ick2.HostEnvironment))
+
+    def test_creates_create_workspace_action_on_host(self):
+        spec = {
+            'action': 'create_workspace',
+        }
+        action = self.af.create_action(spec, self.project)
+        self.assertTrue(isinstance(action, ick2.CreateWorkspaceAction))
+        self.assertTrue(isinstance(action.get_env(), ick2.HostEnvironment))
+
+    def test_creates_archive_workspace_action_on_host(self):
+        spec = {
+            'archive': 'workspace',
+        }
+        action = self.af.create_action(spec, self.project)
+        self.assertTrue(isinstance(action, ick2.ArchiveWorkspaceAction))
+        self.assertTrue(isinstance(action.get_env(), ick2.HostEnvironment))
+
+    def test_creates_populate_sysgtre_action_on_host(self):
+        spec = {
+            'action': 'populate_systree',
+        }
+        action = self.af.create_action(spec, self.project)
+        self.assertTrue(isinstance(action, ick2.PopulateSystreeAction))
+        self.assertTrue(isinstance(action.get_env(), ick2.HostEnvironment))
+
+    def test_raises_exception_for_unknown_step(self):
+        with self.assertRaises(ick2.UnknownStepError):
+            self.af.create_action({}, self.project)
+
+
+class ShellActionTests(unittest.TestCase):
+
+    def test_encodes_parameters(self):
+        params = {
+            'foo': 'bar',
+        }
+        action = ick2.ShellAction(None)
+        encoded = action.encode64(params)
+        decoded = action.decode64(encoded)
+        self.assertEqual(params, decoded)
+
+    def test_runs_argv(self):
+        params = {}
+        step = {
+            'shell': 'echo hello, world',
+        }
+        env = DummyEnvironment()
+        env.exit_code = 42
+        action = ick2.ShellAction(env)
+        exit_code = action.execute(params, step)
+        self.assertEqual(exit_code, env.exit_code)
+
+
+class PythonActionTests(unittest.TestCase):
+
+    def test_runs_argv(self):
+        params = {}
+        step = {
+            'python': 'pass',
+        }
+        env = DummyEnvironment()
+        env.exit_code = 42
+        action = ick2.PythonAction(env)
+        exit_code = action.execute(params, step)
+        self.assertEqual(exit_code, env.exit_code)
+
+
+class DebootstrapActionTests(unittest.TestCase):
+
+    def test_runs_argv(self):
+        params = {
+            'debian_codename': 'stretch',
+        }
+        step = {
+            'debootstrap': 'auto',
+            'mirror': 'http://deb.debian.org/debian',
+        }
+        env = DummyEnvironment()
+        env.exit_code = 42
+        action = ick2.DebootstrapAction(env)
+        exit_code = action.execute(params, step)
+        self.assertEqual(exit_code, env.exit_code)
+
+
+class CreateWorkspaceActionTests(unittest.TestCase):
+
+    def setUp(self):
+        self.tempdir = tempfile.mkdtemp()
+
+    def tearDown(self):
+        shutil.rmtree(self.tempdir)
+
+    def test_runs_argv(self):
+        params = {}
+        step = {
+            'action': 'create_workspace',
+        }
+        env = DummyEnvironment(tempdir=self.tempdir)
+        action = ick2.CreateWorkspaceAction(env)
+        exit_code = action.execute(params, step)
+        self.assertEqual(exit_code, 0)
+
+
+class DummyEnvironment:
+
+    def __init__(self, tempdir=None):
+        self.exit_code = None
+        self.argv = None
+        self.tempdir = tempdir or tempfile.mkdtemp()
+
+    def report(self, exit_code, msg):
+        pass
+
+    def host_runcmd(self, argv, **kwargs):
+        self.argv = argv
+        return self.exit_code
+
+    def runcmd(self, argv, **kwargs):
+        return self.host_runcmd(argv)
+
+    def get_workspace_directory(self):
+        return self.tempdir