Merge pull request 'simpler-boot-script' (#14) from simpler-boot-script into main

Reviewed-on: https://codeberg.org/ambient/ambient-build-vm/pulls/14

12 files changed, 194 insertions, 180 deletions

diff --git a/README.md b/README.md
index 575f283..65edc77 100644
--- a/README.md
+++ b/README.md
@@ -9,7 +9,7 @@ A software developer would use this program to create an image for a
 build virtual machine like this:
 
 ```sh
-$ ambient-build-vm --image=my.qcow2
+$ ./ambient-build-vm --image=my.qcow2
 ```
 
 This will take several minutes to run, but will produce the file
@@ -28,11 +28,20 @@ See <https://ambient.liw.fi/> for more information about Ambient.
 ## Requirements
 
 * MUST build a Debian VM with desired software installed.
+* MUST run the `run-ci` script provided in a tar archive on
+  `/dev/vdb`.
 
 ## Architecture
 
 `ambient-build-vm` runs [vmdb2](https://vmdb2.liw.fi/) to build a
-Debian image. Support for other systems will be added later.
+Debian image. Support for other operating systems will be added later.
+
+The built image itself is set up to run the `ambient-boot` script at
+boot up, and then power off. The boot script output is redirected to
+`/dev/ttyS1`, so that it can be captured. The script unpacks a tar
+archive from `/dev/vdb` into a temporary directory, and runs
+`./run-ci` in that directory. The `run-ci` executable does whatever is
+needed to be done to run CI on a project.
 
 ## Building
 
@@ -41,12 +50,21 @@ from the source tree. It doesn't need to be built.
 
 ## Testing
 
-Build image. Use it with `ambient-run`, and if the build succeeds, be
-happy.
+To also test the built image, to ensure it works as expected for
+Ambient CI, the `build-and-test.sh` script can be used:
+
+~~~sh
+$ ./build-and-test my.qcow2 /tmp
+~~~
+
+This builds the image, by running `ambient-build-vm`, and then runs it
+under QEMU to make sure it works. The running should add about 10 or
+20 seconds to the run time. You need `kvm` installed and configured.
 
 ## Deployment
 
-Not supported at this time.
+There is Debian packaging. You can build the `.deb` package and
+install that.
 
 ## Contributing
 
diff --git a/ambient-boot b/ambient-boot
new file mode 100755
index 0000000..c00e50c
--- /dev/null
+++ b/ambient-boot
@@ -0,0 +1,43 @@
+#!/bin/sh
+#
+# Boot strap running Ambient CI in a VM built for Ambient. Read a tar
+# archive from the specified device, and extract it into a temporary
+# directory. Then run the executable "run-ci" that's in that
+# directory. The executable can then do the rest of the work to
+# execute a CI run.
+
+set -eu
+
+die() {
+	echo "ERROR: $*" 1>&2
+	exit 1
+}
+
+log() {
+	echo "INFO: $*"
+}
+
+if [ "$#" != 1 ]; then
+	die "usage: $0 /dev/vdx"
+fi
+
+dev="$1"
+
+case "$dev" in
+/*) true ;;
+*) die "Argument MUST be an absolute path name" ;;
+esac
+
+tmp="$(mktemp -d)"
+
+cd "$tmp"
+log "Extracting tar archive from $dev"
+tar -xvf "$dev"
+echo
+log "Running run-ci from $dev"
+echo ================================ BEGIN ================================
+if ./run-ci; then
+	echo "EXIT CODE: 0"
+else
+	echo "EXIT CODE: $?"
+fi
diff --git a/ambient-build-vm b/ambient-build-vm
index a2eebc6..3eeec39 100755
--- a/ambient-build-vm
+++ b/ambient-build-vm
@@ -40,7 +40,7 @@ def parse_args():
 
     p.add_argument(
         "--playbook",
-        action="store",
+        action="append",
         help="extra Ansible playbook to use with vmdb to build image (default is none)",
     )
 
@@ -50,6 +50,14 @@ def parse_args():
         help="filename for resulting image (required)",
         required=True,
     )
+
+    p.add_argument(
+        "--debian",
+        action="store",
+        help="install Debian version (codename) instead of the default (%default)",
+        default="bookworm",
+    )
+
     return p.parse_args()
 
 
@@ -70,19 +78,22 @@ def write_yaml(filename, obj):
 def prepare_build_files(tmp, args):
     vmdb_filename = os.path.join(tmp, "image.vmdb")
     playbook_filename = os.path.join(tmp, "playbook.yml")
-    extra_playbook_filename = os.path.join(tmp, "extra.yml")
 
     vmdb = load_yaml(join(args, BASE_VMDB))
 
-    if args.playbook is not None:
+    for step in vmdb["steps"]:
+        if "debootstrap" in step:
+            step["debootstrap"] = args.debian
+
+    for playbook in args.playbook:
         vmdb["steps"].append(
             {
                 # note: the value MUST be the root file system tag in the VMDB file
                 "ansible": "/",
-                "playbook": args.playbook,
+                "playbook": playbook,
             }
         )
-        shutil.copyfile(args.playbook, extra_playbook_filename)
+        shutil.copyfile(playbook, os.path.join(tmp, playbook))
 
     write_yaml(vmdb_filename, vmdb)
     shutil.copyfile(join(args, BASE_PLAYBOOK), playbook_filename)
diff --git a/ambient-build.service b/ambient-build.service
deleted file mode 100644
index b767e6d..0000000
--- a/ambient-build.service
+++ /dev/null
@@ -1,12 +0,0 @@
-[Unit]
-Description=Run build job for Ambient CI
-Before=serial-getty@ttyS0.service
-
-[Install]
-WantedBy=serial-getty@ttyS0.service
-
-[Service]
-Type=oneshot
-ExitType=cgroup
-ExecStart=/bin/bash -c '/bin/ambient-run-script -t /dev/ttyS1 -s /dev/vdb -a /dev/vdc -c /dev/vdd -d /dev/vde >/dev/ttyS0 2>&1 || true'
-ExecStartPost=/sbin/poweroff
diff --git a/ambient-run-script b/ambient-run-script
deleted file mode 100755
index 40d51c6..0000000
--- a/ambient-run-script
+++ /dev/null
@@ -1,146 +0,0 @@
-#!/usr/bin/python3
-
-import argparse
-import os
-import subprocess
-import sys
-import tarfile
-
-
-class RunScript:
-    def __init__(self, args, log):
-        self.args = args
-        self.act = not args.dry_run
-        self.root = args.root
-        self.log = log
-
-    def msg(self, txt):
-        self.log.write(f"{txt}\n")
-        self.log.flush()
-
-    def marker(self, symbol):
-        line = "=" * 22
-        self.msg(f"\n{line} {symbol} {line}\n")
-
-    def join(self, path):
-        return os.path.join(self.root, f"./{path}")
-
-    def mkdir(self, path):
-        path = self.join(path)
-        self.msg(f"mkdir {path}")
-        if not os.path.isdir(path):
-            os.mkdir(path)
-
-    def untar(self, tar_filename, target_directory):
-        self.msg(f"untar {tar_filename} to {target_directory}")
-        tar = tarfile.open(name=tar_filename, mode="r:")
-        tar.extractall(path=self.join(target_directory))
-
-    def tar(self, filename, directory):
-        self.msg(f"tar up {directory} as {filename}")
-        tar = tarfile.open(name=filename, mode="w:")
-        tar.add(directory, arcname=".")
-        tar.close()
-
-    def run(self):
-        self.marker("BEGIN")
-        self.create_workspace()
-        self.extract_sources()
-        if self.args.cache:
-            self.extract_cache()
-        if self.args.dependencies:
-            self.extract_deps()
-        self.build()
-        if self.args.cache:
-            self.save_cache()
-
-    def create_workspace(self):
-        self.msg("create /workspace")
-        if self.act:
-            self.mkdir("/workspace")
-            for subdir in ("src", "cache", "deps"):
-                self.mkdir(f"/workspace/{subdir}")
-
-    def extract_sources(self):
-        self.msg(f"extract {self.args.src} to /workspace/src")
-        if self.act:
-            self.untar(self.args.src, "/workspace/src")
-
-    def extract_cache(self):
-        self.msg(f"extract {self.args.cache} to /workspace/cache")
-        if self.act:
-            self.untar(self.args.cache, "/workspace/cache")
-
-    def extract_deps(self):
-        self.msg(f"extract {self.args.dependencies} to /workspace/deps")
-        if self.act:
-            self.untar(self.args.dependencies, "/workspace/deps")
-
-    def build(self):
-        self.msg(f"build in /workspace/src")
-        if self.act:
-            src = self.join("/workspace/src")
-            script = ".ambient-script"
-            self.msg(f"run {script} in {src}")
-            os.chmod(os.path.join(src, script), 0o755)
-            with open(self.args.tty, "wb") as stdout:
-                p = subprocess.run(
-                    [f"./{script}", os.path.abspath(self.args.artifact)],
-                    stdout=stdout,
-                    stderr=subprocess.STDOUT,
-                    cwd=src,
-                )
-            self.msg(f"\nEXIT CODE: {p.returncode}\n")
-
-    def save_cache(self):
-        self.msg(f"save /workspace/cache to {self.args.cache}")
-        if self.act:
-            cache = self.join("/workspace/cache")
-            self.tar(self.args.cache, cache)
-
-
-def parse_args():
-    p = argparse.ArgumentParser(prog="ambient-run-script")
-    p.add_argument("--dump-config")
-    p.add_argument("--dry-run")
-    p.add_argument("-t", "--tty", required=True)
-    p.add_argument("-s", "--source", required=True, dest="src")
-    p.add_argument("-a", "--artifact")
-    p.add_argument("-c", "--cache")
-    p.add_argument("-d", "--dependencies")
-    p.add_argument("--root", default="/")
-    return p.parse_args()
-
-
-def dump_config(args, log):
-    log.write("ambient-run-script:\n")
-    log.write(f"- dump: {args.dump_config}\n")
-    log.write(f"- tty: {args.tty}\n")
-    log.write(f"- src: {args.src}\n")
-    log.write(f"- artifact: {args.artifact}\n")
-    log.write(f"- cache: {args.cache}\n")
-    log.write(f"- dependencies: {args.dependencies}\n")
-    log.write(f"- root: {args.root}\n")
-    log.write(f"- dry_run: {args.dry_run}\n")
-    log.flush()
-
-
-def run(args, log):
-    RunScript(args, log).run()
-
-
-def main():
-    args = parse_args()
-    print(args)
-    if args.dump_config is not None:
-        with open(args.dump_config, "w") as log:
-            dump_config(args, log)
-    elif args.dry_run:
-        with open(args.dry_run, "w") as log:
-            run(args, log)
-    else:
-        with open(args.tty, "w") as log:
-            run(args, log)
-
-
-main()
diff --git a/ambient.service b/ambient.service
new file mode 100644
index 0000000..a16bd91
--- /dev/null
+++ b/ambient.service
@@ -0,0 +1,8 @@
+[Unit]
+Description=Run build job for Ambient CI
+After=dev-ttyS0.device
+
+[Service]
+Type=oneshot
+ExecStart=/bin/bash -c '/bin/ambient-boot /dev/vdb >/dev/ttyS1 2>&1 || true'
+ExecStartPost=/sbin/poweroff
diff --git a/base.vmdb b/base.vmdb
index d8cedeb..0545fcb 100644
--- a/base.vmdb
+++ b/base.vmdb
@@ -29,7 +29,7 @@ steps:
 
   - unpack-rootfs: /
 
-  - debootstrap: bullseye
+  - debootstrap: bookworm
     mirror: http://deb.debian.org/debian
     variant: minbase
     target: /
@@ -46,6 +46,7 @@ steps:
     tag: /
     packages:
       - zstd
+      - nano
 
   - cache-rootfs: /
     unless: rootfs_unpacked
diff --git a/build-and-test.sh b/build-and-test.sh
new file mode 100755
index 0000000..dd6cbfb
--- /dev/null
+++ b/build-and-test.sh
@@ -0,0 +1,47 @@
+#!/bin/bash
+
+set -euo pipefail
+
+image="$1"
+cache="$2"
+shift 2
+
+echo Building image.
+rm -f "$cache/ambient.tar.gz"
+./ambient-build-vm --data . --cache "$cache" --image "$image" "$@"
+
+tmp="$(mktemp -d)"
+trap 'rm -rf "$tmp"' EXIT
+
+mkdir "$tmp/files"
+cat <<EOF >"$tmp/files/run-ci"
+echo xyzzy
+EOF
+chmod +x "$tmp/files/run-ci"
+
+tar -cf "$tmp/run-ci.tar" -C "$tmp/files" .
+cp "$image" "$tmp/img.qcow2"
+
+QVMF_FD="/usr/share/ovmf/OVMF.fd"
+cp "$QVMF_FD" "$tmp/vars"
+
+echo Testing image.
+kvm \
+	-m 1024 \
+	-smp cpus=1 \
+	-display none \
+	-serial "file:/$tmp/log0" \
+	-serial "file:/$tmp/log1" \
+	-drive if=pflash,format=raw,unit=0,file="$QVMF_FD",readonly=on \
+	-drive if=pflash,format=raw,unit=1,file="$tmp/vars" \
+	-drive format=qcow2,if=virtio,file="$tmp/img.qcow2" \
+	-drive format=raw,if=virtio,file="$tmp/run-ci.tar",readonly=on \
+	-nodefaults
+
+if ! grep -q '^xyzzy' "$tmp/log1" >/dev/null; then
+	echo "FAILED to perform a simple CI run" 1>&2
+	cat -v "$tmp/log1"
+	exit 1
+fi
+
+echo "OK. $image is ready to use."
diff --git a/debian/rules b/debian/rules
index 26678c8..8e0f536 100755
--- a/debian/rules
+++ b/debian/rules
@@ -21,7 +21,6 @@ override_dh_auto_install:
 	install README.md debian/$(NAME)/usr/share/doc//$(NAME)
 
 	install -d debian/$(NAME)/usr/share/$(NAME)
-	install ambient-build.service ambient-run-script base.vmdb playbook.yml \
-		debian/$(NAME)/usr/share/$(NAME)
+	install ambient.service base.vmdb playbook.yml debian/$(NAME)/usr/share/$(NAME)
 
 	find debian
diff --git a/ikiwiki.yml b/ikiwiki.yml
index ded7bd7..e41143b 100644
--- a/ikiwiki.yml
+++ b/ikiwiki.yml
@@ -51,5 +51,19 @@
           - perlmagick
           - plantuml
 
+    - name: "clone ikiwiki plugin for Pandoc"
+      git:
+        repo: https://github.com/dubiousjim/pandoc-iki
+        dest: /root/pandoc-iki
+
+    - name: "create ikiwiki plugin directory"
+      file:
+        state: directory
+        path: /root/.ikiwiki/IkiWiki/Plugin
+
+    - name: "install Pandoc plugin for ikiwiki"
+      shell: |
+        cp /root/pandoc-iki/pandoc.pm /root/.ikiwiki/IkiWiki/Plugin/pandoc.pm
+
   vars:
     ansible_python_interpreter: /usr/bin/python3
diff --git a/playbook.yml b/playbook.yml
index c9e9c25..69f42b8 100644
--- a/playbook.yml
+++ b/playbook.yml
@@ -40,21 +40,21 @@
       shell: |
         sed -i '/^root:[^:]*:/s//root::/' /etc/passwd
 
-    - name: "create script to simulate build"
+    - name: "install script to boot strap build"
       copy:
-        src: ./ambient-run-script
-        dest: /bin/ambient-run-script
+        src: ./ambient-boot
+        dest: /bin/ambient-boot
         mode: 0755
 
-    - name: "install systemd service unit to run build"
+    - name: "install systemd service unit to boot strap build"
       copy:
-        src: ambient-build.service
-        dest: /etc/systemd/system/ambient-build.service
+        src: ambient.service
+        dest: /etc/systemd/system/ambient.service
 
-    - name: "make sure ambient-build.service is run at boot"
+    - name: "make sure ambient.service is run at boot"
       shell: |
-        ln -nsf /etc/systemd/system/ambient-build.service \
-                /etc/systemd/system/multi-user.target.wants/ambient-build.service
+        ln -nsf /etc/systemd/system/ambient.service \
+                /etc/systemd/system/multi-user.target.wants/ambient.service
 
   vars:
     ansible_python_interpreter: /usr/bin/python3
diff --git a/try-boot.sh b/try-boot.sh
new file mode 100755
index 0000000..eef7b8a
--- /dev/null
+++ b/try-boot.sh
@@ -0,0 +1,31 @@
+#!/bin/bash
+
+set -euo pipefail
+
+img="$1"
+
+QVMF_FD="/usr/share/ovmf/OVMF.fd"
+
+tmp="$(mktemp -d)"
+trap 'rm -rf "$tmp"' EXIT
+
+mkdir "$tmp/run-ci"
+cat <<EOF >"$tmp/run-ci/run-ci"
+echo xyzzy
+EOF
+chmod +x "$tmp/run-ci/run-ci"
+tar -C "$tmp/run-ci" -cf "$tmp/run-ci.tar" .
+
+rm -f log0 log1
+cp "$QVMF_FD" "$tmp/vars"
+
+kvm \
+	-m 1024 \
+	-smp cpus=1 \
+	-display none \
+	-serial file:log0 \
+	-serial file:log1 \
+	-drive if=pflash,format=raw,unit=0,file="$tmp/vars" \
+	-drive format=qcow2,if=virtio,file="$img" \
+	-drive format=raw,if=virtio,file="$tmp/run-ci.tar",readonly=on \
+	-nodefaults