From 746ddbc6e17d9ab8238cc4a8e3a348ccb44878b5 Mon Sep 17 00:00:00 2001
From: Lars Wirzenius <liw@liw.fi>
Date: Tue, 8 Sep 2020 12:23:48 +0300
Subject: test: add a subplot to verify the roles work

---
 subplot/daemon.py    |  84 +++++++++++++++++++++
 subplot/qemumgr.py   | 210 +++++++++++++++++++++++++++++++++++++++++++++++++++
 subplot/runcmd.py    |  77 +++++++++++++++++++
 subplot/subplot.py   | 120 +++++++++++++++++++++++++++++
 subplot/subplot.yaml |  15 ++++
 5 files changed, 506 insertions(+)
 create mode 100644 subplot/daemon.py
 create mode 100755 subplot/qemumgr.py
 create mode 100644 subplot/runcmd.py
 create mode 100644 subplot/subplot.py
 create mode 100644 subplot/subplot.yaml

(limited to 'subplot')

diff --git a/subplot/daemon.py b/subplot/daemon.py
new file mode 100644
index 0000000..e223505
--- /dev/null
+++ b/subplot/daemon.py
@@ -0,0 +1,84 @@
+#############################################################################
+# Start and stop daemons, or background processes.
+
+
+import logging
+import os
+import signal
+import time
+
+
+# Start a process in the background.
+def start_daemon(ctx, name, argv):
+    runcmd = globals()["runcmd"]
+    exit_code_is = globals()["exit_code_is"]
+
+    logging.debug(f"Starting daemon {name}")
+    logging.debug(f"  ctx={ctx.as_dict()}")
+    logging.debug(f"  name={name}")
+    logging.debug(f"  argv={argv}")
+
+    if "daemon" not in ctx.as_dict():
+        ctx["daemon"] = {}
+    assert name not in ctx["daemon"]
+    this = ctx["daemon"][name] = {
+        "pid-file": f"{name}.pid",
+        "stderr": f"{name}.stderr",
+        "stdout": f"{name}.stdout",
+    }
+    runcmd(
+        ctx,
+        [
+            "/usr/sbin/daemonize",
+            "-c",
+            os.getcwd(),
+            "-p",
+            this["pid-file"],
+            "-e",
+            this["stderr"],
+            "-o",
+            this["stdout"],
+        ]
+        + argv,
+    )
+
+    # Wait for a bit for daemon to start and maybe find a problem and die.
+    time.sleep(3)
+    if ctx["exit"] != 0:
+        logging.error(f"obnam-server stderr: {ctx['stderr']}")
+
+    exit_code_is(ctx, 0)
+    this["pid"] = int(open(this["pid-file"]).read().strip())
+    assert process_exists(this["pid"])
+
+    logging.debug(f"Started daemon {name}")
+    logging.debug(f"  ctx={ctx.as_dict()}")
+
+
+# Stop a daemon.
+def stop_daemon(ctx, name):
+    logging.debug(f"Stopping daemon {name}")
+    logging.debug(f"  ctx={ctx.as_dict()}")
+    logging.debug(f"  ctx['daemon']={ctx.as_dict()['daemon']}")
+
+    this = ctx["daemon"][name]
+    terminate_process(this["pid"], signal.SIGKILL)
+
+
+# Does a process exist?
+def process_exists(pid):
+    try:
+        os.kill(pid, 0)
+    except ProcessLookupError:
+        return False
+    return True
+
+
+# Terminate process.
+def terminate_process(pid, signalno):
+    logging.debug(f"Terminating process {pid} with signal {signalno}")
+    try:
+        os.kill(pid, signalno)
+    except ProcessLookupError:
+        logging.debug("Process did not actually exist (anymore?)")
+        pass
diff --git a/subplot/qemumgr.py b/subplot/qemumgr.py
new file mode 100755
index 0000000..8951122
--- /dev/null
+++ b/subplot/qemumgr.py
@@ -0,0 +1,210 @@
+#!/usr/bin/env python3
+
+
+import logging
+import os
+import shlex
+import shutil
+import signal
+import subprocess
+
+# import tempfile
+import time
+
+
+CLOUD_INIT_META_TEMPLATE = """\
+# Amazon EC2 style metadata
+local-hostname: {hostname}
+"""
+
+CLOUD_INIT_USER_TEMPLATE = """\
+#cloud-config
+ssh_authorized_keys:
+- {pubkey}
+"""
+
+
+class QemuSystem:
+    def __init__(self, name, base_image, disk_size, pubkey):
+        self._name = name
+        self._dirname = "."
+        self._image = self._copy_image(base_image, disk_size)
+        self._pubkey = pubkey
+        self._memory = 512
+        self._vcpus = 1
+        self._username = None
+        self._port = None
+        self._pid = None
+
+    def _join(self, *names):
+        return os.path.join(self._dirname, *names)
+
+    def set_disk_size(self, size):
+        self._disk_size = str(size * 1024 * 1024)
+
+    def set_memory(self, memory):
+        self._memory = memory
+
+    def set_vcpus(self, vcpus):
+        self._vcpus = vcpus
+
+    def set_username(self, username):
+        self._username = username
+
+    def _copy_image(self, base_image, size):
+        image = self._join("qemu.img")
+        logging.debug(f"QemuSystem: actual disk image: {image}")
+        shutil.copy(base_image, image)
+        if size is not None:
+            subprocess.check_call(["qemu-img", "resize", "-q", image, str(size)])
+        return image
+
+    def _cloud_init_iso(self):
+        iso = self._join("cloud-init.iso")
+
+        config = self._join("cloud-init")
+        os.mkdir(config)
+
+        meta = os.path.join(config, "meta-data")
+        write_file(meta, CLOUD_INIT_META_TEMPLATE.format(hostname=self._name))
+
+        user = os.path.join(config, "user-data")
+        write_file(user, CLOUD_INIT_USER_TEMPLATE.format(pubkey=self._pubkey))
+
+        if os.path.exists(iso):
+            os.remove(iso)
+        subprocess.check_call(
+            [
+                "genisoimage",
+                "-quiet",
+                "-joliet",
+                "-rock",
+                "-output",
+                iso,
+                "-volid",
+                "cidata",
+                config,
+            ]
+        )
+
+        return iso
+
+    def start(self):
+        iso = self._cloud_init_iso()
+
+        self._port = 2222  # FIXME
+        pid_file = self._join("qemu.pid")
+        out_file = self._join("qemu.out")
+        err_file = self._join("qemu.err")
+
+        MiB = 1024 ** 2
+        memory = int(self._memory / MiB)
+
+        argv = [
+            "/usr/sbin/daemonize",
+            "-c",
+            self._dirname,
+            "-p",
+            pid_file,
+            "-e",
+            err_file,
+            "-o",
+            out_file,
+            "/usr/bin/qemu-system-x86_64",
+            "-name",
+            self._name,
+            "-m",
+            str(memory),
+            "-cpu",
+            "host",
+            "-smp",
+            f"cpus={self._vcpus}",
+            "-drive",
+            f"file={self._image},format=qcow2,cache=none,if=virtio",
+            "-drive",
+            f"file={iso},if=ide,media=cdrom",
+            "-device",
+            "virtio-net,netdev=user.0",
+            "-netdev",
+            f"user,id=user.0,hostfwd=tcp::{self._port}-:22",
+            "-nographic",
+            "-enable-kvm",
+        ]
+
+        subprocess.check_call(argv, stdout=None, stderr=None)
+        self._pid = int(wait_for(lambda: got_pid(pid_file), 1))
+
+        logging.debug(f"started qemu-system")
+        logging.debug(f"  argv: {argv}")
+        logging.debug(f"  pid: {self._pid}")
+
+    def stop(self):
+        if self._pid is not None:
+            logging.debug(f"killing qemu-system process {self._pid}")
+            os.kill(self._pid, signal.SIGTERM)
+
+    def wait_for_ssh(self):
+        def ssh_ok():
+            output, exit = self.ssh(["true"])
+            logging.debug(f"tried ssh, exit={exit}")
+            if exit == 0:
+                return True
+            return None
+
+        return wait_for(ssh_ok, 60, sleep=5)
+
+    def ssh(self, argv):
+        assert self._username is not None
+
+        srcdir = globals()["srcdir"]
+
+        ssh_opts = [
+            "-i",
+            os.path.join(srcdir, "ssh", "id"),
+            "-p",
+            str(self._port),
+            "-l",
+            self._username,
+            "-ouserknownhostsfile=/dev/null",
+            "-ostricthostkeychecking=accept-new",
+            "-opasswordauthentication=no",
+        ]
+
+        real_argv = (
+            ["ssh"] + ssh_opts + ["localhost", "--"] + [shlex.quote(x) for x in argv]
+        )
+
+        logging.debug(f"running on VM: {real_argv}")
+        p = subprocess.Popen(
+            real_argv, stdout=subprocess.PIPE, stderr=subprocess.STDOUT
+        )
+        output, _ = p.communicate()
+        logging.debug(f"exit code: {p.returncode}")
+        logging.debug(f"output: {output}")
+        return output, p.returncode
+
+
+def read_file(filename):
+    with open(filename, "r") as f:
+        return f.read()
+
+
+def write_file(filename, content):
+    with open(filename, "w") as f:
+        f.write(content)
+
+
+def wait_for(func, timeout, sleep=0.1):
+    start = time.time()
+    while time.time() < start + timeout:
+        val = func()
+        if val is not None:
+            return val
+        time.sleep(sleep)
+
+
+def got_pid(pid_file):
+    if os.path.exists(pid_file):
+        pid = read_file(pid_file)
+        if pid:
+            return pid.strip()
diff --git a/subplot/runcmd.py b/subplot/runcmd.py
new file mode 100644
index 0000000..7193c15
--- /dev/null
+++ b/subplot/runcmd.py
@@ -0,0 +1,77 @@
+# Some step implementations for running commands and capturing the result.
+
+import subprocess
+
+
+# Run a command, capture its stdout, stderr, and exit code in context.
+def runcmd(ctx, argv, **kwargs):
+    p = subprocess.Popen(argv, stdout=subprocess.PIPE, stderr=subprocess.PIPE, **kwargs)
+    stdout, stderr = p.communicate("")
+    ctx["argv"] = argv
+    ctx["stdout"] = stdout.decode("utf-8")
+    ctx["stderr"] = stderr.decode("utf-8")
+    ctx["exit"] = p.returncode
+
+
+# Check that latest exit code captured by runcmd was a specific one.
+def exit_code_is(ctx, wanted):
+    if ctx.get("exit") != wanted:
+        print("context:", ctx.as_dict())
+    assert_eq(ctx.get("exit"), wanted)
+
+
+# Check that latest exit code captured by runcmd was not a specific one.
+def exit_code_is_not(ctx, unwanted):
+    if ctx.get("exit") == unwanted:
+        print("context:", ctx.as_dict())
+    assert_ne(ctx.get("exit"), unwanted)
+
+
+# Check that latest exit code captured by runcmd was zero.
+def exit_code_zero(ctx):
+    exit_code_is(ctx, 0)
+
+
+# Check that latest exit code captured by runcmd was not zero.
+def exit_code_nonzero(ctx):
+    exit_code_is_not(ctx, 0)
+
+
+# Check that stdout of latest runcmd contains a specific string.
+def stdout_contains(ctx, pattern=None):
+    stdout = ctx.get("stdout", "")
+    if pattern not in stdout:
+        print("pattern:", repr(pattern))
+        print("stdout:", repr(stdout))
+        print("ctx:", ctx.as_dict())
+    assert_eq(pattern in stdout, True)
+
+
+# Check that stdout of latest runcmd does not contain a specific string.
+def stdout_does_not_contain(ctx, pattern=None):
+    stdout = ctx.get("stdout", "")
+    if pattern in stdout:
+        print("pattern:", repr(pattern))
+        print("stdout:", repr(stdout))
+        print("ctx:", ctx.as_dict())
+    assert_eq(pattern not in stdout, True)
+
+
+# Check that stderr of latest runcmd does contains a specific string.
+def stderr_contains(ctx, pattern=None):
+    stderr = ctx.get("stderr", "")
+    if pattern not in stderr:
+        print("pattern:", repr(pattern))
+        print("stderr:", repr(stderr))
+        print("ctx:", ctx.as_dict())
+    assert_eq(pattern in stderr, True)
+
+
+# Check that stderr of latest runcmd does not contain a specific string.
+def stderr_does_not_contain(ctx, pattern=None):
+    stderr = ctx.get("stderr", "")
+    if pattern not in stderr:
+        print("pattern:", repr(pattern))
+        print("stderr:", repr(stderr))
+        print("ctx:", ctx.as_dict())
+    assert_eq(pattern not in stderr, True)
diff --git a/subplot/subplot.py b/subplot/subplot.py
new file mode 100644
index 0000000..4434f24
--- /dev/null
+++ b/subplot/subplot.py
@@ -0,0 +1,120 @@
+import logging
+import os
+import yaml
+
+
+def fixme(ctx, **kwargs):
+    assert 0
+
+
+def create_vm(ctx):
+    QemuSystem = globals()["QemuSystem"]
+    srcdir = globals()["srcdir"]
+
+    name = "debian-ansible-test"
+    base_image = "/home/liw/tmp/debian-10-openstack-amd64.qcow2"
+    GiB = 1024 ** 3
+    disk_size = 10 * GiB
+    pubkey = open(os.path.join(srcdir, "ssh", "id.pub")).read().strip()
+    memory = 1 * GiB
+    cpus = 2
+    username = "debian"
+
+    logging.info("starting a VM using qemu-system")
+    logging.info(f"  name    : {name}")
+    logging.info(f"  image   : {base_image}")
+    logging.info(f"  disk    : {disk_size}")
+    logging.info(f"  pubkey  : {pubkey}")
+    logging.info(f"  memory  : {memory}")
+    logging.info(f"  cpus    : {cpus}")
+    logging.info(f"  username: {username}")
+
+    qemu = QemuSystem(name, base_image, disk_size, pubkey)
+    qemu.set_memory(memory)
+    qemu.set_vcpus(cpus)
+    qemu.set_username(username)
+    qemu.start()
+
+    logging.debug("waiting for SSH to be ready")
+    if qemu.wait_for_ssh():
+        logging.debug("SSH is ready")
+    else:
+        logging.error("SSH did not get ready")
+        assert 0
+    logging.info("a qemu-system VM is up and running and accessible over SSH")
+    ctx["qemu"] = qemu
+
+
+def destroy_vm(ctx):
+    logging.debug(f"destroying qemu running")
+    qemu = ctx["qemu"]
+    qemu.stop()
+
+
+def run_true_on_host(ctx):
+    qemu = ctx["qemu"]
+    qemu.ssh(["/bin/true"])
+
+
+def use_role_in_playbook(ctx, role=None):
+    empty_playbook = {
+        "hosts": "test-host",
+        "remote_user": "debian",  # FIXME: don't hardcode this
+        "become": True,
+        "roles": [],
+    }
+    playbook = ctx.get("playbook", dict(empty_playbook))
+    playbook["roles"].append(role)
+    ctx["playbook"] = playbook
+
+
+def set_vars_file(ctx, filename=None):
+    get_file = globals()["get_file"]
+    data = get_file(filename)
+    with open("vars.yml", "wb") as f:
+        f.write(data)
+
+
+def run_playbook(ctx):
+    runcmd = globals()["runcmd"]
+    exit_code_zero = globals()["exit_code_zero"]
+    assert_ne = globals()["assert_ne"]
+    srcdir = globals()["srcdir"]
+
+    with open("hosts", "w") as f:
+        f.write("test-host\n")
+
+    if not os.path.exists("vars.yml"):
+        with open("vars.yml", "w") as f:
+            yaml.safe_dump([], stream=f)
+
+    playbook = [ctx["playbook"]]
+    assert_ne(playbook, None)
+    with open("playbook.yml", "w") as f:
+        yaml.safe_dump(playbook, stream=f)
+
+    ssh_opts = [
+        "-ouserknownhostsfile=/dev/null",
+        "-ostricthostkeychecking=accept-new",
+        "-i",
+        os.path.join(srcdir, "ssh", "id"),
+        "-p2222",
+    ]
+
+    env = dict(os.environ)
+    env["ANSIBLE_SSH_ARGS"] = " ".join(ssh_opts)
+    env["ANSIBLE_LOG"] = "ansible.log"
+    env["ANSIBLE_ROLES_PATH"] = os.path.join(srcdir, "roles")
+
+    argv = [
+        "ansible-playbook",
+        "-i",
+        "hosts",
+        f"-e@vars.yml",
+        "-eansible_ssh_host=localhost",
+        "-eansible_ssh_port=2222",
+        "playbook.yml",
+    ]
+
+    runcmd(ctx, argv, env=env)
+    exit_code_zero(ctx)
diff --git a/subplot/subplot.yaml b/subplot/subplot.yaml
new file mode 100644
index 0000000..01c85f5
--- /dev/null
+++ b/subplot/subplot.yaml
@@ -0,0 +1,15 @@
+- given: a host running Debian
+  function: create_vm
+  cleanup: destroy_vm
+
+- then: I can run /bin/true on the host
+  function: run_true_on_host
+
+- when: I use role {role}
+  function: use_role_in_playbook
+
+- when: I use variables from {filename}
+  function: set_vars_file
+
+- when: I run the playbook
+  function: run_playbook
-- 
cgit v1.2.1