diff options
author | Lars Wirzenius <liw@liw.fi> | 2018-03-17 19:35:30 +0200 |
---|---|---|
committer | Lars Wirzenius <liw@liw.fi> | 2018-03-30 11:52:19 +0300 |
commit | c4841e5b3af2ebaa448a1a0b43a65941cf5c6c3e (patch) | |
tree | 5044362b227e3e92254847c40c00f02671f13106 | |
parent | 114a276e06aeb19cd9150175efa53d7267b6a6ff (diff) | |
download | ick2-c4841e5b3af2ebaa448a1a0b43a65941cf5c6c3e.tar.gz |
Add: ActionEnvironment class for executing in various contexts
-rw-r--r-- | ick2/__init__.py | 9 | ||||
-rw-r--r-- | ick2/actionenvs.py | 167 | ||||
-rw-r--r-- | ick2/actionenvs_tests.py | 138 |
3 files changed, 313 insertions, 1 deletions
diff --git a/ick2/__init__.py b/ick2/__init__.py index 95f3914..2a03a25 100644 --- a/ick2/__init__.py +++ b/ick2/__init__.py @@ -52,8 +52,15 @@ from .controllerapi import ( from .blobapi import BlobAPI from .blob_store import BlobStore -from .client import HttpAPI, HttpError, ControllerClient, BlobClient, Reporter +from .client import ( + HttpAPI, + HttpError, + ControllerClient, + BlobClient, + Reporter, +) from .actionenvs import ( + Runner, ActionEnvironment, HostEnvironment, ChrootEnvironment, diff --git a/ick2/actionenvs.py b/ick2/actionenvs.py new file mode 100644 index 0000000..7a2b57a --- /dev/null +++ b/ick2/actionenvs.py @@ -0,0 +1,167 @@ +# Copyright (C) 2018 Lars Wirzenius +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + + +import logging +import os +import subprocess + + +import cliapp + + +class Runner: + + def __init__(self, reporter, maxbuf=128*1024): + self._reporter = reporter + self._buffers = { + 'stdout': b'', + 'stderr': b'', + } + self._maxbuf = maxbuf + self._timeout = 1.0 + + def runcmd(self, *argvs, **kwargs): + for argv in argvs: + logging.debug('Runner.runcmd: argv: %r', argv) + for key in kwargs: + logging.debug('Runner.runcmd: kwargs: %r=%r', key, kwargs[key]) + assert all(argv is not None for argv in argvs) + exit_code, _, _ = cliapp.runcmd_unchecked( + *argvs, + stdout_callback=self.capture_stdout, + stderr=subprocess.STDOUT, + output_timeout=self._timeout, + timeout_callback=self.flush, + **kwargs + ) + self.flush() + logging.debug('Runner.runcmd: finished, exit code: %d', exit_code) + return exit_code + + def capture_stdout(self, data): + return self.capture('stdout', data) + + def capture(self, stream_name, data): + self._buffers[stream_name] += data + if len(self._buffers[stream_name]) >= self._maxbuf: + self.flush() + + return b'' + + def flush(self): + stdout = self._buffers['stdout'] + stderr = self._buffers['stderr'] + self._reporter.report(None, stdout, stderr) + self._buffers['stdout'] = b'' + self._buffers['stderr'] = b'' + + +class Mounter: # pragma: no cover + + def __init__(self, mounts, runner): + self._mounts = mounts + self._runner = runner + + def __enter__(self): + self.mount() + return self + + def __exit__(self, *args): + self.unmount() + + def mount(self): + for dirname, mp in self._mounts: + if not os.path.exists(mp): + os.mkdir(mp) + self._runner.runcmd(['sudo', 'mount', '--bind', dirname, mp]) + + def unmount(self): + for dirname, mp in reversed(self._mounts): + try: + self._runner.runcmd(['sudo', 'umount', mp]) + except BaseException as e: + logging.error( + 'Ignoring error while unmounting %s: %s', mp, str(e)) + + +class ActionEnvironment: # pragma: no cover + + def __init__(self, systree, workspace, reporter): + super().__init__() + self._systree = systree + self._workspace = workspace + self._reporter = reporter + + def get_systree_directory(self): + return self._systree + + def get_workspace_directory(self): + return self._workspace + + def get_mounts(self): + return [] + + def report(self, exit_code, msg): + self._reporter.report(exit_code, msg, None) + + def runcmd(self, argv): + raise NotImplementedError() + + def host_runcmd(self, *argvs, cwd=None): + env = self.get_env_vars() + runner = Runner(self._reporter) + mounts = self.get_mounts() + with Mounter(mounts, runner): + return runner.runcmd(*argvs, cwd=cwd, env=env) + + def get_env_vars(self): + env = dict(os.environ) + env.update({ + 'LC_ALL': 'C', + 'DEBIAN_FRONTEND': 'noninteractive', + }) + return env + + +class HostEnvironment(ActionEnvironment): + + def runcmd(self, argv): + return self.host_runcmd(argv, cwd=self._workspace) + + +class ChrootEnvironment(ActionEnvironment): + + def get_mounts(self): # pragma: no cover + return [ + ('/proc', os.path.join(self._workspace, 'proc')), + ('/sys', os.path.join(self._workspace, 'sys')), + ] + + def runcmd(self, argv): + prefix = ['sudo', 'chroot', self._workspace] + return self.host_runcmd(prefix + argv) + + +class ContainerEnvironment(ActionEnvironment): + + def runcmd(self, argv): + bind = '{}:/workspace'.format(self._workspace) + prefix = [ + 'sudo', 'systemd-nspawn', + '-D', self._systree, + '--bind', bind, + '--chdir', '/workspace', + ] + return self.host_runcmd(prefix + argv) diff --git a/ick2/actionenvs_tests.py b/ick2/actionenvs_tests.py new file mode 100644 index 0000000..4643527 --- /dev/null +++ b/ick2/actionenvs_tests.py @@ -0,0 +1,138 @@ +# Copyright (C) 2018 Lars Wirzenius +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + + +import unittest + + +import ick2 + + +class FakeReporter: + + def __init__(self): + self.stdout = b'' + self.stderr = b'' + + def report(self, exit_code, stdout, stderr): + self.stdout += stdout + self.stderr += stderr + + +class RunnerTests(unittest.TestCase): + + def test_runs_echo(self): + reporter = FakeReporter() + r = ick2.Runner(reporter) + exit_code = r.runcmd(['echo', 'hello', 'world']) + self.assertEqual(exit_code, 0) + self.assertEqual(reporter.stdout, b'hello world\n') + + def test_captures_lots_of_output(self): + reporter = FakeReporter() + r = ick2.Runner(reporter, maxbuf=1) + exit_code = r.runcmd(['echo', 'hello', 'world']) + self.assertEqual(exit_code, 0) + self.assertEqual(reporter.stdout, b'hello world\n') + + def test_captures_stderr(self): + reporter = FakeReporter() + r = ick2.Runner(reporter) + exit_code = r.runcmd(['sh', '-c', 'echo eek 1>&2 ; exit 2']) + self.assertEqual(exit_code, 2) + self.assertEqual(reporter.stdout, b'eek\n') + + +class EnvironmentTestsBase(unittest.TestCase): + + def setUp(self): + self.workspace = '/workspace' + self.fake = FakeRuncmd() + self.env = self.create_env() + self.env.host_runcmd = self.fake + + def create_env(self): + return DummyEnvironment(None, self.workspace, None) + + def expected_argv(self, argv): + return argv + + def test_runs_argv(self): + argv = ['echo', 'hello', 'world'] + self.fake.exit_code = 42 + exit_code = self.env.runcmd(argv) + self.assertEqual(self.fake.argv, self.expected_argv(argv)) + self.assertEqual(self.fake.exit_code, exit_code) + + +class DummyEnvironment(ick2.ActionEnvironment): + + def runcmd(self, argv): + return self.host_runcmd(argv) + + +class HostEnvironmentTests(EnvironmentTestsBase): + + def create_env(self): + return ick2.HostEnvironment(None, self.workspace, None) + + def expected_argv(self, argv): + return argv + + def test_pipeline(self): + reporter = FakeReporter() + env = ick2.HostEnvironment(None, None, reporter) + + echo = ['echo', 'hello', 'world'] + cat = ['cat'] + exit_code = env.host_runcmd(echo, cat) + self.assertEqual(exit_code, 0) + self.assertEqual(reporter.stdout, b'hello world\n') + self.assertEqual(reporter.stderr, b'') + + +class ChrootEnvironmentTests(EnvironmentTestsBase): + + def create_env(self): + return ick2.ChrootEnvironment(None, self.workspace, None) + + def expected_argv(self, argv): + return ['sudo', 'chroot', self.workspace] + argv + + +class ContainerEnvironmentTests(EnvironmentTestsBase): + + def create_env(self): + self.systree = 'systree' + self.workspace = 'workspace' + return ick2.ContainerEnvironment(self.systree, self.workspace, None) + + def expected_argv(self, argv): + prefix = [ + 'sudo', 'systemd-nspawn', '-D', self.systree, + '--bind', '{}:/workspace'.format(self.workspace), + '--chdir', '/workspace', + ] + return prefix + argv + + +class FakeRuncmd: + + def __init__(self): + self.argv = None + self.exit_code = None + + def __call__(self, argv, cwd=None): + self.argv = argv + return self.exit_code |