Merge branch 'liw/subplot' into 'main'

feat! rewrite ambient-run-script in Python, support cache, deps

See merge request larswirzenius/ambient-ci!18

9 files changed, 346 insertions, 35 deletions

diff --git a/.gitignore b/.gitignore
index 977e1e7..0821999 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,5 +1,2 @@
-runner.qcow2
-temp.img
-OVMF_VARS.fd
-expect.out
-expect.err
+test.log
+test.py
diff --git a/ambient-build.service b/ambient-build.service
index b7d078e..b62a970 100644
--- a/ambient-build.service
+++ b/ambient-build.service
@@ -6,5 +6,5 @@ WantedBy=multi-user.target
 
 [Service]
 Type=oneshot
-ExecStart=/bin/ambient-run-script /dev/ttyS0 /dev/vdb /dev/vdc
+ExecStart=/bin/ambient-run-script -t /dev/ttyS0 -s /dev/vdb -a /dev/vdc
 ExecStart=/sbin/poweroff
diff --git a/ambient-run-script b/ambient-run-script
index fde2ddc..0f6bfce 100644..100755
--- a/ambient-run-script
+++ b/ambient-run-script
@@ -1,39 +1,142 @@
-#!/bin/bash
+#!/usr/bin/python3
 
-set -xeuo pipefail
+import argparse
+import os
+import subprocess
+import sys
+import tarfile
 
-script=".ambient-script"
 
-run() {
-	local tty="$1"
-	local inputdev="$2"
-	local outputdev="$3"
+class RunScript:
+    def __init__(self, args, log):
+        self.args = args
+        self.act = not args.dry_run
+        self.root = args.root
+        self.log = log
 
-	echo "============================================================"
-	echo "hello from ambient-run-script"
+    def msg(self, txt):
+        self.log.write(f"{txt}\n")
+        self.log.flush()
+        sys.stdout.write(f"{txt}\n")
 
-	ls -l "$tty"
-	ls -l "$inputdev"
-	ls -l "$outputdev"
+    def join(self, path):
+        return os.path.join(self.root, f"./{path}")
 
-	echo "creating /workspace"
-	install -d /workspace
+    def mkdir(self, path):
+        path = self.join(path)
+        self.msg(f"mkdir {path}")
+        if not os.path.isdir(path):
+            os.mkdir(path)
 
-	echo "extracting source from $inputdev"
-	cd /workspace
-	tar -xvvf "$inputdev"
+    def untar(self, tar_filename, target_directory):
+        self.msg(f"untar {tar_filename} to {target_directory}")
+        tar = tarfile.open(name=tar_filename, mode="r:")
+        tar.extractall(path=self.join(target_directory))
 
-	if [ ! -x $script ]; then
-		echo "making $script executable, because it wasn't"
-		chmod +x "$script"
-	fi
+    def tar(self, filename, directory):
+        self.msg(f"tar up {directory} as {filename}")
+        tar = tarfile.open(name=filename, mode="w:")
+        tar.add(directory, arcname=".")
+        tar.close()
 
-	echo "running $script"
-	if ./$script "$outputdev" 2>&1; then
-		echo "build OK"
-	else
-		echo "build FAILED"
-	fi
-}
+    def run(self):
+        self.create_workspace()
+        self.extract_sources()
+        if self.args.cache:
+            self.extract_cache()
+        if self.args.dependencies:
+            self.extract_deps()
+        self.build()
+        if self.args.cache:
+            self.save_cache()
 
-run "$@" </dev/null >"$1"
+    def create_workspace(self):
+        self.msg("create /workspace")
+        if self.act:
+            self.mkdir("/workspace")
+            for subdir in ("src", "cache", "deps"):
+                self.mkdir(f"/workspace/{subdir}")
+
+    def extract_sources(self):
+        self.msg(f"extract {self.args.src} to /workspace/src")
+        if self.act:
+            self.untar(self.args.src, "/workspace/src")
+
+    def extract_cache(self):
+        self.msg(f"extract {self.args.cache} to /workspace/cache")
+        if self.act:
+            self.untar(self.args.cache, "/workspace/cache")
+
+    def extract_deps(self):
+        self.msg(f"extract {self.args.dependencies} to /workspace/deps")
+        if self.act:
+            self.untar(self.args.dependencies, "/workspace/deps")
+
+    def build(self):
+        self.msg(f"build in /workspace/src")
+        if self.act:
+            src = self.join("/workspace/src")
+            script = ".ambient-script"
+            self.msg(f"run {script} in {src}")
+            os.chmod(os.path.join(src, script), 0o755)
+            with open(self.args.tty, "wb") as stdout:
+                p = subprocess.run(
+                    [f"./{script}", os.path.abspath(self.args.artifact)],
+                    stdout=subprocess.PIPE,
+                    stderr=subprocess.STDOUT,
+                    cwd=src,
+                )
+                self.msg(f"stdout: {p.stdout.decode()}")
+
+    def save_cache(self):
+        self.msg(f"save /workspace/cache to {self.args.cache}")
+        if self.act:
+            cache = self.join("/workspace/cache")
+            self.tar(self.args.cache, cache)
+
+
+def parse_args():
+    p = argparse.ArgumentParser(prog="ambient-run-script")
+    p.add_argument("--dump-config")
+    p.add_argument("--dry-run")
+    p.add_argument("-t", "--tty", required=True)
+    p.add_argument("-s", "--source", required=True, dest="src")
+    p.add_argument("-a", "--artifact")
+    p.add_argument("-c", "--cache")
+    p.add_argument("-d", "--dependencies")
+    p.add_argument("--root", default="/")
+    return p.parse_args()
+
+
+def dump_config(args, log):
+    log.write("ambient-run-script:\n")
+    log.write(f"- dump: {args.dump_config}\n")
+    log.write(f"- tty: {args.tty}\n")
+    log.write(f"- src: {args.src}\n")
+    log.write(f"- artifact: {args.artifact}\n")
+    log.write(f"- cache: {args.cache}\n")
+    log.write(f"- dependencies: {args.dependencies}\n")
+    log.write(f"- root: {args.root}\n")
+    log.write(f"- dry_run: {args.dry_run}\n")
+    log.flush()
+
+
+def run(args, log):
+    RunScript(args, log).run()
+
+
+def main():
+    args = parse_args()
+    print(args)
+    if args.dump_config is not None:
+        with open(args.dump_config, "w") as log:
+            dump_config(args, log)
+    elif args.dry_run:
+        with open(args.dry_run, "w") as log:
+            run(args, log)
+    else:
+        with open(args.tty, "w") as log:
+            run(args, log)
+
+
+main()
diff --git a/ambient.md b/ambient.md
index 8d20a59..de78993 100644
--- a/ambient.md
+++ b/ambient.md
@@ -247,3 +247,156 @@ We run the VM with a pre-determined amount of disk, number of CPUs,
 amount of memory. We fail the build if it exceeds a pre-determined
 time limit. We fail the build if the amount of output via the serial
 console exceeds a pre-determined limit.
+
+# Acceptance criteria
+
+[Subplot]: https://subplot.tech
+
+These acceptance criteria are written for the [Subplot][] tool to
+process. They are verified using scenarios expressed in a
+given/when/then language. For details, please see the Subplot
+documentation.
+
+## `ambient-run-script`
+
+This section concerns itself with `ambient-run-script`, which is part
+of the VM in which the build is run.
+
+### Accepts the various devices
+
+_Requirement: `ambient-run-script` accepts the various input and output
+devices correctly._
+
+We verify this by running the script with an option that only exists
+for this purpose to dump its configuration as text.
+
+~~~scenario
+given an installed ambient-run-script
+given file tars.txt
+when I run ./ambient-run-script --dump-config=dump.txt -t /dev/ttyS0 -s input.tar -a output.tar -c cache.tar -d deps.tar
+then files dump.txt and tars.txt match
+~~~
+
+~~~{#tars.txt .file .text}
+ambient-run-script:
+- dump: dump.txt
+- tty: /dev/ttyS0
+- src: input.tar
+- artifact: output.tar
+- cache: cache.tar
+- dependencies: deps.tar
+- root: /
+- dry_run: None
+~~~
+
+
+### Lists steps in happy path
+
+This scenario verifies two requirements, for the sake of simplicity of
+test implementation.
+
+* _Requirement: `ambient-run-script` must perform the same steps every
+  time, unless something goes wrong._
+
+  We verify this by having `ambient-run-script` list the steps is would
+  do, without actually doing them, using the `--dry-run` option.
+
+~~~scenario
+given an installed ambient-run-script
+given file expected-steps.txt
+when I run ./ambient-run-script --dry-run=steps.txt -t /dev/ttyS0 -s input.tar -a output.tar -c cache.tar -d deps.tar
+then files steps.txt and expected-steps.txt match
+~~~
+
+~~~{#expected-steps.txt .file .text}
+create /workspace
+extract input.tar to /workspace/src
+extract cache.tar to /workspace/cache
+extract deps.tar to /workspace/deps
+build in /workspace/src
+save /workspace/cache to cache.tar
+~~~
+
+### Performs expected steps in happy path
+
+_Requirement: `ambient-run-script` must prepare to run a build in a VM._
+
+`ambient-run-script` in inside the VM, so we verify this requirement
+by having it run a build that we prepare so it's safe to run in our
+test environment, without a VM. We make use of a special option so
+that paths used by the program are relative to the current working
+directory, instead of absolute. We also verity that the output device
+gets output written, and that the cache device gets updated.
+
+This is a bit of a long scenario, so it's divided into chunks. First
+we set things up.
+
+~~~scenario
+given an installed ambient-run-script
+
+given file project/.ambient-script from simple-ambient-script
+given tar archive input.tar with contents of project
+
+given file cached/data from cache.data
+given tar archive cache.tar with contents of cached
+
+given file deps/deps.data from deps.data
+given tar archive deps.tar with contents of deps
+~~~
+
+We now have the source files, cached data, and dependencies and we can
+run `ambient-run-script` with them.
+
+~~~scenario
+when I run ./ambient-run-script --root=. -t log -s input.tar -a output.tar -c cache.tar -d deps.tar
+~~~
+
+The workspace must now exists and have the expected contents.
+
+~~~scenario
+then file workspace/src/.ambient-script exists
+then file workspace/cache/data exists
+then file workspace/cache/cached contains "hello, cache"
+then file workspace/deps/deps.data exists
+then file log contains "hello, ambient script"
+~~~
+
+The artifact tar archive must contain the expected contents.
+
+~~~scenario
+when I create directory untar-output
+when I run tar -C untar-output -xvf output.tar
+then file untar-output/greeting contains "hello, there"
+~~~
+
+The cache tar archive must contain the expected contents.
+
+~~~scenario
+when I create directory untar-cache
+when I run tar -C untar-cache -xvf cache.tar
+then file untar-cache/cached contains "hello, cache"
+then file untar-cache/data exists
+~~~
+
+That's all, folks.
+
+~~~{#simple-ambient-script .file .sh}
+#!/bin/bash
+
+set -xeuo pipefail
+
+echo hello, ambient script
+
+echo hello, cache > ../cache/cached
+
+echo hello, there > greeting
+tar -cf "$1" greeting
+~~~
+
+~~~{#cache.data .file}
+This is cached data.
+~~~
+
+~~~{#deps.data .file}
+This is dependency data.
+~~~
diff --git a/ambient.py b/ambient.py
new file mode 100644
index 0000000..5d4ffcf
--- /dev/null
+++ b/ambient.py
@@ -0,0 +1,21 @@
+def install_ambient_run_script(ctx):
+    import os, shutil
+
+    srcdir = globals()["srcdir"]
+    script = os.path.join(srcdir, "ambient-run-script")
+
+    shutil.copy(script, ".")
+
+
+def create_tar(ctx, filename=None, directory=None):
+    import tarfile
+
+    tar = tarfile.open(name=filename, mode="w:")
+    tar.add(directory, arcname=".")
+    tar.close()
+
+
+def create_empty_tar(ctx, filename=None):
+    import tarfile
+
+    tarfile.open(name=filename, mode="w:").close()
diff --git a/ambient.subplot b/ambient.subplot
new file mode 100644
index 0000000..50e223a
--- /dev/null
+++ b/ambient.subplot
@@ -0,0 +1,14 @@
+title: Ambient CI acceptance criteria
+authors:
+  - Lars Wirzenius
+markdowns:
+  - ambient.md
+bindings:
+  - ambient.yaml
+  - lib/files.yaml
+  - lib/runcmd.yaml
+impls:
+  python:
+    - ambient.py
+    - lib/files.py
+    - lib/runcmd.py
diff --git a/ambient.yaml b/ambient.yaml
new file mode 100644
index 0000000..1874639
--- /dev/null
+++ b/ambient.yaml
@@ -0,0 +1,14 @@
+- given: an installed ambient-run-script
+  impl:
+    python:
+      function: install_ambient_run_script
+
+- given: tar archive {filename} with contents of {directory}
+  impl:
+    python:
+      function: create_tar
+
+- given: empty tar archive {filename}
+  impl:
+    python:
+      function: create_empty_tar
diff --git a/check b/check
new file mode 100755
index 0000000..fd55d66
--- /dev/null
+++ b/check
@@ -0,0 +1,7 @@
+#!/bin/bash
+
+set -euo pipefail
+
+subplot codegen ambient.subplot -o test.py
+rm -f test.log
+python3 test.py --log test.log "$@"
diff --git a/test-project/.ambient-script b/test-project/.ambient-script
index 797ac74..5819e76 100755
--- a/test-project/.ambient-script
+++ b/test-project/.ambient-script
@@ -8,5 +8,7 @@ cat /proc/cmdline
 
 output="$1"
 
+echo this is an output file >output.txt
+
 echo "produce output tar"
 tar -cf "$output" .