Add yarn tests vmdebootstrap

10 files changed, 377 insertions, 0 deletions

diff --git a/yarns/000-meta.yarn b/yarns/000-meta.yarn
new file mode 100644
index 0000000..13de949
--- /dev/null
+++ b/yarns/000-meta.yarn
@@ -0,0 +1,4 @@
+---
+title: vmdebootstrap integration test suite
+date: PRE-ALPHA
+...
diff --git a/yarns/100-intro.yarn b/yarns/100-intro.yarn
new file mode 100644
index 0000000..c0646cd
--- /dev/null
+++ b/yarns/100-intro.yarn
@@ -0,0 +1,13 @@
+# Introduction
+
+This document is the integration test suite for vmdebootstrap. It is
+both human-readable and machine-executable. It is written for the
+[yarn][] tool. This document is aimed at those developing
+vmdebootstrap.
+
+* See the vmdebootstrap documentation about what the software is for
+  and how it is used.
+* See the [yarn documentation][] for how to use yarn.
+
+[yarn]: http://liw.fi/cmdtest/
+[yarn documentation]: http://liw.fi/cmdtest/README.yarn/
diff --git a/yarns/200-fast-tests.yarn b/yarns/200-fast-tests.yarn
new file mode 100644
index 0000000..6e30fa2
--- /dev/null
+++ b/yarns/200-fast-tests.yarn
@@ -0,0 +1,8 @@
+# Fast option check tests
+
+* A scenario, or set of scenarios, that verify that certain
+  combinations of vmdebootstrap options work or fail as expected, but
+  without actually building images. The goal is to test command line
+  handling quickly. For example, options that conflict should cause
+  error messages.
+    * FIXME: Check with Neil for suggestions for options.
diff --git a/yarns/300-slow-build-tests.yarn b/yarns/300-slow-build-tests.yarn
new file mode 100644
index 0000000..41d11a3
--- /dev/null
+++ b/yarns/300-slow-build-tests.yarn
@@ -0,0 +1,20 @@
+# Slow image building tests
+
+In this chapter, we have test scenarios that actually build an image
+and test the output. The images are not booted, but that may be added
+later. Instead, all the tests on the images are static.
+
+These tests are slow, since building images is slow.
+
+
+## Build a very basic Debian 8 image
+
+    SCENARIO build a basic Debian 8 image
+    ASSUMING build tests are requested
+    GIVEN user wants to build an image FOO.img that is 2GiB in size
+    WHEN the user runs vmdebootstrap --sparse --extlinux
+    THEN the image has the correct size
+    AND the image has one partition
+    AND partition 1 has the boot flag set
+    AND partition 1 has an ext4 filesystem
+    AND partition 1 has file /etc/debian_version matching ^8\..*$
diff --git a/yarns/800-future.yarn b/yarns/800-future.yarn
new file mode 100644
index 0000000..1368b8f
--- /dev/null
+++ b/yarns/800-future.yarn
@@ -0,0 +1,9 @@
+# Thoughts for future tests
+
+* Build image, run it under Qemu or some other way. Then do some basic
+  testing of the running system. For example, boot it under
+  qemu-system-i386, with a serial consoler, and verify that it's
+  possible to log in via the serial consoler, and ping the host.
+  More detailed testing of built systems may or may not be
+  appropriate: a lot of more intricate testing may be beyond the scope
+  of testing of vmdebootstrap.
diff --git a/yarns/900-implements.yarn b/yarns/900-implements.yarn
new file mode 100644
index 0000000..e9e4ec7
--- /dev/null
+++ b/yarns/900-implements.yarn
@@ -0,0 +1,89 @@
+# Scenario step implementations
+
+This chapter has implementations of scenario steps used elsewhere in
+this document.
+
+
+## Should a scenario be run?
+
+We provide a way to control which classes of scenarios get run. This
+is done by passing in an environment variable `TESTS` to vmdebootstrap
+(`--env` option), with a comma-separated list of classes:
+
+* `fast`---run fast tests
+* `build`---run scenarios that do builds
+
+If `TESTS` is not set, everything gets run.
+
+Scenarios can use the ASSUMING statements defined here to let the user
+to allow them to run or not to run.
+
+    IMPLEMENTS ASSUMING build tests are requested
+    test_requested build
+
+
+## Building an image
+
+To keep individual steps shorter, we provide some steps to set common
+parts, such as the name of the image being built.
+
+    IMPLEMENTS GIVEN user wants to build an image (\S+) that is (\S+) in size
+    remember_setting IMAGE "$MATCH_1"
+    remember_setting IMAGE_SIZE "$(size_in_bytes "$MATCH_2")"
+
+Actually build an image. This looks like it can invoke any command,
+but it's actually restricted to vmdebootstrap in the source tree.
+
+    IMPLEMENTS WHEN the user runs vmdebootstrap (.*)
+    PYTHONPATH="$SRCDIR" "$SRCDIR/bin/vmdebootstrap" \
+        --image "$IMAGE" \
+        --size "$IMAGE_SIZE" \
+        $MATCH_1
+
+
+## Static tests on disk images
+
+The steps in this section do static tests of disk image. These all
+operate on the image specified in the step "GIVEN user wants to
+build...".
+
+Test the size of an image. This tests the length, not disk usage, of
+the image.
+
+    IMPLEMENTS THEN the image has the correct size
+    actual="$(stat -c %s "$IMAGE")"
+    [ "$actual" = "$IMAGE_SIZE" ]
+
+Check the partition table on the image.
+
+    IMPLEMENTS THEN the image has one partition
+    parted --script "$IMAGE" print |
+        sed '1,/^Number/d' |
+        grep -c . |
+        grep -Fx 1
+
+Check partition boot flag.
+
+    IMPLEMENTS THEN partition (\d+) has the boot flag set
+    parted --script "$IMAGE" print |
+        awk -v "PART=$MATCH_1" '/^ [0-9]+ / && $1 == PART && $7 == "boot"' |
+        grep .
+
+Check filesystem on a partition. This checks the actual filesystem,
+not a type declared in the partition table.
+
+    IMPLEMENTS THEN partition (\d+) has an? (\S+) filesystem
+    device="$(kpartx_image_partition "$IMAGE" "$MATCH_1")"
+    trap "unkpartx_image \"$IMAGE\"" EXIT
+    blkid "$device" | grep "TYPE=\"$MATCH_2\""
+
+Check that the partition contains a file with some content matching a
+regular expression.
+
+    IMPLEMENTS THEN partition (\d+) has file (\S+) matching (.+)
+    device="$(kpartx_image_partition "$IMAGE" "$MATCH_1")"
+    trap "unkpartx_image \"$IMAGE\"" EXIT
+    mp="$(mktemp -d)"
+    mount -r "$device" "$mp"
+    trap "umount \"$mp\"; unkpartx_image \"$IMAGE\"" EXIT
+    grep -P -e "$MATCH_3" "$mp/$MATCH_2"
diff --git a/yarns/Makefile b/yarns/Makefile
new file mode 100644
index 0000000..41367b4
--- /dev/null
+++ b/yarns/Makefile
@@ -0,0 +1,37 @@
+# Copyright 2015  Lars Wirzenius
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# =*= License: GPL-3+ =*=
+
+
+yarns = $(shell ls [0-9][0-9][0-9]-*.yarn)
+
+all: vmdebootstrap-yarns.pdf vmdebootstrap-yarns.html
+
+vmdebootstrap-yarns.pdf: $(yarns) Makefile
+	pandoc --chapters \
+		--toc \
+		--number-sections \
+		-V documentclass:report \
+		-o $@ $(yarns)
+
+vmdebootstrap-yarns.html: $(yarns) Makefile yarns.css
+	pandoc --chapters \
+		--toc \
+		--number-sections \
+		--standalone \
+		--self-contained \
+		-H yarns.css \
+		-o $@ $(yarns)
diff --git a/yarns/run-tests b/yarns/run-tests
new file mode 100755
index 0000000..4ef337b
--- /dev/null
+++ b/yarns/run-tests
@@ -0,0 +1,22 @@
+#!/bin/sh
+# Copyright 2015  Lars Wirzenius
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# =*= License: GPL-3+ =*=
+
+set -eu
+
+cd "$(dirname "$0")"
+yarn -s shell.lib *.yarn "$@"
diff --git a/yarns/shell.lib b/yarns/shell.lib
new file mode 100644
index 0000000..08835ab
--- /dev/null
+++ b/yarns/shell.lib
@@ -0,0 +1,96 @@
+# A shell library for yarn scenario step implementations.
+
+# Copyright 2015  Lars Wirzenius
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# =*= License: GPL-3+ =*=
+
+
+# Is running of scenarios of a given type requested? If nothing is
+# requested ($TESTS is empty), then everything is.
+
+test_requested()
+{
+    if [ "${TESTS:-}" = "" ]
+    then
+        return 0
+    elif echo "${TESTS}" | tr , '\n' | grep -Fx "$1"
+    then
+        return 0
+    else
+        return 1
+    fi
+}
+
+
+# Convert a size with a unit, such a kB, to plain bytes.
+
+size_in_bytes()
+{
+    local amount="$(echo "$1" | sed 's/[^0-9]*$//')"
+    local unit="$(echo "$1" | sed 's/^[0-9]*//' | tr A-Z a-z)"
+    local factor=1
+    case "$unit" in
+        k|kb) factor=1000 ;;
+        m|mb) factor=1000000 ;;
+        g|gb) factor=1000000000 ;;
+        t|tb) factor=1000000000000 ;;
+        ki|kib) factor=1024 ;;
+        mi|mib) factor=1048576 ;;
+        gi|gib) factor=1073741824 ;;
+        ti|tib) factor=1099511627776 ;;
+    esac
+    echo "$amount * $factor" | bc -lq
+}
+
+
+# Make a partition in a disk image accessible as a block device.
+# Return it's device file.
+
+kpartx_image_partition()
+{
+    kpartx -sav "$1" | awk -v "n=$2" 'NR == n { print "/dev/mapper/" $3 }'
+}
+
+
+# Undo kpartx_image_partition.
+
+unkpartx_image()
+{
+    kpartx -d "$1"
+}
+
+
+# Rembember and retrieve settings between scenario steps. This is
+# implemented by a shell script snippet that gets sourced at the end
+# of this shell library.
+
+remember_setting()
+{
+    echo "$1=$2" >> "$DATADIR/settings.sh"
+}
+
+if [ -e "$DATADIR/settings.sh" ]
+then
+    . "$DATADIR/settings.sh"
+fi
+
+
+# Make sure the current working directory is $DATADIR, not $SRCDIR.
+# This makes it a little simpler to write scenario step
+# implementations by not having to be specifying $DATADIR explicitly
+# everywhere.
+
+cd "$DATADIR"
diff --git a/yarns/yarns.css b/yarns/yarns.css
new file mode 100644
index 0000000..78cd374
--- /dev/null
+++ b/yarns/yarns.css
@@ -0,0 +1,79 @@
+<style>
+html {
+    background: white;
+    font-family: serif;
+    margin-left: 3em;
+    margin-right: 2em;
+    margin-top: 2em;
+}
+
+form#searchform {
+    font-family: monospace;
+    text-align: right;
+}
+
+div.actions {
+    font-family: monospace;
+    text-align: right;
+}
+
+div.actions ul, div.actions li {
+    display: inline;
+}
+
+div.pageheader {
+    font-family: monospace;
+    margin-bottom: 2em;
+}
+
+div.pageheader span.title {
+    display: block;
+    font-size: 200%;
+    font-weight: bold;
+    font-family: sans-serif;
+    margin-top: 0.5em;
+}
+
+div#pagebody {
+}
+
+div.pagefooter {
+    font-family: monospace;
+    margin-top: 3em;
+}
+
+div#pagebody {
+}
+
+div#TOC ul {
+    list-style: none;
+}
+
+h1, h2, h3, h4, h5, h6 {
+    font-family: sans-serif;
+    font-weight: bold;
+    margin-top: 2em;
+}
+
+h1 {
+    font-size: 150%;
+}
+
+h2 {
+    font-size: 120%;
+}
+
+h3 {
+    font-size: 100%;
+}
+
+ul li, ol li {
+    margin-top: 0.5em;
+    margin-bottom: 0.5em;
+}
+
+pre {
+    margin-left: 4em;
+}
+
+</style>