Merge branch 'doc' into 'master'

Fix manual to have steps in alphabetical order

Closes #6

See merge request larswirzenius/vmdb2!14

38 files changed, 560 insertions, 338 deletions

diff --git a/check b/check
index aad834c..5210727 100755
--- a/check
+++ b/check
@@ -1,6 +1,7 @@
-#!/bin/sh
+#!/bin/bash
 
 set -eu
+set -o pipefail
 
 echo Running unit tests ============================================
 python3 -m CoverageTestRunner --ignore-missing-from=without-tests yarns vmdb
@@ -21,3 +22,32 @@ yarn \
     --env "PYTHONPATH=$(pwd)/yarns" \
     --cd-datadir \
     yarns/*.yarn "$@"
+echo
+
+echo Formatting docs ========================================
+./format.sh
+
+steps() {
+    sed -n '/<h2 id="step-.*> Step: /s///p' vmdb2.html
+}
+
+code() {
+    steps | grep '<code>' || true
+}
+
+panic() {
+    echo "ERROR: $@" 1>&2
+    exit 1
+}
+
+n="$(code | wc -l)"
+if [ "$n" != 0 ]
+then
+    code
+    panic "Documentation has steps that use code in title"
+fi
+
+if ! diff -u <(steps) <(steps | sort)
+then
+    panic "Steps are not in sorted order"
+fi
diff --git a/vmdb/plugins/ansible.mdwn b/vmdb/plugins/ansible.mdwn
index 8929c57..fa014ad 100644
--- a/vmdb/plugins/ansible.mdwn
+++ b/vmdb/plugins/ansible.mdwn
@@ -1,4 +1,4 @@
-Step: `ansible`
+Step: ansible
 -----------------------------------------------------------------------------
 
 Run Ansible using a provided playbook, to configure the image. vmdb2
diff --git a/vmdb/plugins/apt.mdwn b/vmdb/plugins/apt.mdwn
index dd83f18..a05922a 100644
--- a/vmdb/plugins/apt.mdwn
+++ b/vmdb/plugins/apt.mdwn
@@ -1,4 +1,4 @@
-Step: `apt`
+Step: apt
 -----------------------------------------------------------------------------
 
 Install packages using apt, which needs to already have been
diff --git a/vmdb/plugins/cache_rootfs.mdwn b/vmdb/plugins/cache_rootfs.mdwn
new file mode 100644
index 0000000..c4270a7
--- /dev/null
+++ b/vmdb/plugins/cache_rootfs.mdwn
@@ -0,0 +1,13 @@
+Step: cache-rootfs
+-----------------------------------------------------------------------------
+
+Create a tarball of the root filesystem in the image.
+
+Step keys:
+
+* `cache-rootfs` &mdash; REQUIRED; tag of root filesystem on image.
+
+Example (in the .vmdb file):
+
+    - cache-rootfs: root
+      unless: rootfs_unpacked
diff --git a/vmdb/plugins/rootfs_cache_plugin.py b/vmdb/plugins/cache_rootfs_plugin.py
index 2db907f..c114945 100644
--- a/vmdb/plugins/rootfs_cache_plugin.py
+++ b/vmdb/plugins/cache_rootfs_plugin.py
@@ -24,7 +24,7 @@ import cliapp
 import vmdb
 
 
-class RootFSCachePlugin(cliapp.Plugin):
+class CacheRootFSPlugin(cliapp.Plugin):
 
     def enable(self):
         self.app.settings.string(
@@ -33,7 +33,6 @@ class RootFSCachePlugin(cliapp.Plugin):
             metavar='FILE')
 
         self.app.step_runners.add(MakeCacheStepRunner())
-        self.app.step_runners.add(UnpackCacheStepRunner())
 
 
 class MakeCacheStepRunner(vmdb.StepRunnerInterface):
@@ -74,25 +73,3 @@ class MakeCacheStepRunner(vmdb.StepRunnerInterface):
         if dirname == rootdir:
             return '.'
         return dirname[len(rootdir) + 1:]
-
-
-class UnpackCacheStepRunner(vmdb.StepRunnerInterface):
-
-    def get_required_keys(self):
-        return ['unpack-rootfs']
-
-    def run(self, step, settings, state):
-        fs_tag = step['unpack-rootfs']
-        rootdir = state.tags.get_builder_mount_point(fs_tag)
-        tar_path = settings['rootfs-tarball']
-        if not tar_path:
-            raise Exception('--rootfs-tarball MUST be set')
-        if os.path.exists(tar_path):
-            vmdb.runcmd(
-                ['tar', '-C', rootdir, '-xf', tar_path, '--numeric-owner'])
-            self.copy_resolv_conf(rootdir)
-            state.rootfs_unpacked = True
-
-    def copy_resolv_conf(self, rootdir):
-        filename = os.path.join(rootdir, 'etc', 'resolv.conf')
-        vmdb.runcmd(['cp', '/etc/resolv.conf', filename])
diff --git a/vmdb/plugins/chroot.mdwn b/vmdb/plugins/chroot.mdwn
index 87fd5ec..9db7bd0 100644
--- a/vmdb/plugins/chroot.mdwn
+++ b/vmdb/plugins/chroot.mdwn
@@ -1,4 +1,4 @@
-Step: `chroot`
+Step: chroot
 -----------------------------------------------------------------------------
 
 Run a shell snippet in a chroot inside the image.
@@ -14,22 +14,3 @@ Example (in the .vmdb file):
     - chroot: root
       shell: |
           echo I am in chroot
-
-
-Step: `shell`
------------------------------------------------------------------------------
-
-Run a shell snippet on the host. This is not run in a chroot, and can
-access the host system.
-
-Step keys:
-
-* `root-fs` &mdash; REQUIRED; value is the tag for the root filesystem.
-
-* `shell` &mdash; REQUIRED; the shell snippet to run
-
-Example (in the .vmdb file):
-
-    - root-fs: root
-      shell: |
-          echo I am in NOT in chroot.
diff --git a/vmdb/plugins/chroot_plugin.py b/vmdb/plugins/chroot_plugin.py
index 1304e31..4eb5847 100644
--- a/vmdb/plugins/chroot_plugin.py
+++ b/vmdb/plugins/chroot_plugin.py
@@ -28,7 +28,6 @@ class ChrootPlugin(cliapp.Plugin):
 
     def enable(self):
         self.app.step_runners.add(ChrootStepRunner())
-        self.app.step_runners.add(ShellStepRunner())
 
 
 class ChrootStepRunner(vmdb.StepRunnerInterface):
@@ -42,17 +41,3 @@ class ChrootStepRunner(vmdb.StepRunnerInterface):
 
         mount_point = state.tags.get_builder_mount_point(fs_tag)
         vmdb.runcmd_chroot(mount_point, ['sh', '-ec', shell])
-
-
-class ShellStepRunner(vmdb.StepRunnerInterface):
-
-    def get_required_keys(self):
-        return ['shell', 'root-fs']
-
-    def run(self, step, settings, state):
-        shell = step['shell']
-        fs_tag = step['root-fs']
-
-        env = dict(os.environ)
-        env['ROOT'] = state.tags.get_builder_mount_point(fs_tag)
-        vmdb.runcmd(['sh', '-ec', shell], env=env)
diff --git a/vmdb/plugins/copy_file.mdwn b/vmdb/plugins/copy_file.mdwn
new file mode 100644
index 0000000..0796190
--- /dev/null
+++ b/vmdb/plugins/copy_file.mdwn
@@ -0,0 +1,20 @@
+Step: copy-file
+-----------------------------------------------------------------------------
+
+Copy a file from outside into the target filesystem.
+
+Step keys:
+
+* `copy-file` &mdash; REQUIRED; the full (starting from the new
+  filesystem root) path name of the file to create. Any missing
+  directories will be created (owner root, group root, mode 0511).
+* `src` &mdash; REQUIRED; filename on the host filesystem, outside the
+  chroot, relative to the current working directory of the vmdb2
+  process.
+* `perm` &mdash; OPTIONAL; the numeric (octal) representation of the
+  file's permissions. Defaults to 0644.
+* `uid` &mdash; OPTIONAL; the numeric user ID of the file's
+  owner. Defaults to 0 (root).
+* `gid` &mdash; OPTIONAL; the numeric user ID of the file's
+  group. Defaults to 0 (root).
+
diff --git a/vmdb/plugins/files_plugin.py b/vmdb/plugins/copy_file_plugin.py
index d7a1097..7683d7e 100644
--- a/vmdb/plugins/files_plugin.py
+++ b/vmdb/plugins/copy_file_plugin.py
@@ -20,38 +20,12 @@ import vmdb
 import os
 import logging
 
-class CreateFilePlugin(cliapp.Plugin):
+class CopyFilePlugin(cliapp.Plugin):
 
     def enable(self):
-        self.app.step_runners.add(CreateDirStepRunner())
-        self.app.step_runners.add(CreateFileStepRunner())
         self.app.step_runners.add(CopyFileStepRunner())
 
 
-class CreateFileStepRunner(vmdb.StepRunnerInterface):
-
-    def get_required_keys(self):
-        return ['create-file', 'contents']
-
-    def run(self, step, settings, state):
-        root = state.tags.get_builder_from_target_mount_point('/')
-        newfile = step['create-file']
-        contents = step['contents']
-        perm = step.get('perm', 0o644)
-        uid = step.get('uid', 0)
-        gid = step.get('gid', 0)
-
-        filename = '/'.join([root,newfile])
-
-        logging.info('Creating file %s, uid %d, gid %d, perms %o' % (filename, uid, gid, perm))
-        fd = open(filename, 'w')
-        fd.write(contents)
-        fd.close
-
-        os.chown(filename, uid, gid)
-        os.chmod(filename, perm)
-
-
 class CopyFileStepRunner(vmdb.StepRunnerInterface):
 
     def get_required_keys(self):
@@ -80,22 +54,3 @@ class CopyFileStepRunner(vmdb.StepRunnerInterface):
 
         os.chown(filename, uid, gid)
         os.chmod(filename, perm)
-
-
-class CreateDirStepRunner(vmdb.StepRunnerInterface):
-
-    def get_required_keys(self):
-        return ['create-dir']
-
-    def run(self, step, settings, state):
-        root = state.tags.get_builder_from_target_mount_point('/')
-        newdir = step['create-dir']
-        path = '/'.join([root, newdir])
-        perm = step.get('perm', 0o755)
-        uid = step.get('uid', 0)
-        gid = step.get('gid', 0)
-
-        logging.info('Creating directory %s, uid %d, gid %d, perms %o' % (path, uid, gid, perm))
-
-        os.makedirs(path, perm)
-        os.chown(path, uid, gid)
diff --git a/vmdb/plugins/create_dir.mdwn b/vmdb/plugins/create_dir.mdwn
new file mode 100644
index 0000000..4fce053
--- /dev/null
+++ b/vmdb/plugins/create_dir.mdwn
@@ -0,0 +1,17 @@
+Step: create-dir
+-----------------------------------------------------------------------------
+
+Create a directory in the target filesystem
+
+Step keys:
+
+* `create-dir` &mdash; REQUIRED; the full (starting from the new
+  filesystem root) path name of the directory to create. It will work
+  as a `mkdir -p` — Any intermediate directories that do not yet exist
+  will be created.
+* `perm` &mdash; OPTIONAL; the numeric (octal) representation of the
+  directory's permissions. Defaults to 0755.
+* `uid` &mdash; OPTIONAL; the numeric user ID of the directory's
+  owner. Defaults to 0 (root).
+* `gid` &mdash; OPTIONAL; the numeric user ID of the directory's
+  group. Defaults to 0 (root).
diff --git a/vmdb/plugins/create_dir_plugin.py b/vmdb/plugins/create_dir_plugin.py
new file mode 100644
index 0000000..c48b662
--- /dev/null
+++ b/vmdb/plugins/create_dir_plugin.py
@@ -0,0 +1,45 @@
+# Copyright 2019 Gunnar Wolf
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# =*= License: GPL-3+ =*=
+
+import cliapp
+import vmdb
+import os
+import logging
+
+class CreateDirPlugin(cliapp.Plugin):
+
+    def enable(self):
+        self.app.step_runners.add(CreateDirStepRunner())
+
+
+class CreateDirStepRunner(vmdb.StepRunnerInterface):
+
+    def get_required_keys(self):
+        return ['create-dir']
+
+    def run(self, step, settings, state):
+        root = state.tags.get_builder_from_target_mount_point('/')
+        newdir = step['create-dir']
+        path = '/'.join([root, newdir])
+        perm = step.get('perm', 0o755)
+        uid = step.get('uid', 0)
+        gid = step.get('gid', 0)
+
+        logging.info('Creating directory %s, uid %d, gid %d, perms %o' % (path, uid, gid, perm))
+
+        os.makedirs(path, perm)
+        os.chown(path, uid, gid)
diff --git a/vmdb/plugins/create_file.mdwn b/vmdb/plugins/create_file.mdwn
new file mode 100644
index 0000000..c154b66
--- /dev/null
+++ b/vmdb/plugins/create_file.mdwn
@@ -0,0 +1,19 @@
+Step: create-file
+-----------------------------------------------------------------------------
+
+Create an empty file in the target filesystem.
+
+Step keys:
+
+* `create-file` &mdash; REQUIRED; the full (starting from the new
+  filesystem root) path name of the file to create. It will *not*
+  create any directories; if they need to be created, please use
+  `create-dir` first.
+* `contents` &mdash; REQUIRED; the contents to be written to the
+  generated file.
+* `perm` &mdash; OPTIONAL; the numeric (octal) representation of the
+  file's permissions. Defaults to 0644.
+* `uid` &mdash; OPTIONAL; the numeric user ID of the file's
+  owner. Defaults to 0 (root).
+* `gid` &mdash; OPTIONAL; the numeric user ID of the file's
+  group. Defaults to 0 (root).
diff --git a/vmdb/plugins/create_file_plugin.py b/vmdb/plugins/create_file_plugin.py
new file mode 100644
index 0000000..d6f9d7a
--- /dev/null
+++ b/vmdb/plugins/create_file_plugin.py
@@ -0,0 +1,51 @@
+# Copyright 2019 Gunnar Wolf
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# =*= License: GPL-3+ =*=
+
+import cliapp
+import vmdb
+import os
+import logging
+
+
+class CreateFilePlugin(cliapp.Plugin):
+
+    def enable(self):
+        self.app.step_runners.add(CreateFileStepRunner())
+
+
+class CreateFileStepRunner(vmdb.StepRunnerInterface):
+
+    def get_required_keys(self):
+        return ['create-file', 'contents']
+
+    def run(self, step, settings, state):
+        root = state.tags.get_builder_from_target_mount_point('/')
+        newfile = step['create-file']
+        contents = step['contents']
+        perm = step.get('perm', 0o644)
+        uid = step.get('uid', 0)
+        gid = step.get('gid', 0)
+
+        filename = '/'.join([root,newfile])
+
+        logging.info('Creating file %s, uid %d, gid %d, perms %o' % (filename, uid, gid, perm))
+        fd = open(filename, 'w')
+        fd.write(contents)
+        fd.close
+
+        os.chown(filename, uid, gid)
+        os.chmod(filename, perm)
diff --git a/vmdb/plugins/debootstrap.mdwn b/vmdb/plugins/debootstrap.mdwn
index 18f083a..c1773d3 100644
--- a/vmdb/plugins/debootstrap.mdwn
+++ b/vmdb/plugins/debootstrap.mdwn
@@ -1,4 +1,4 @@
-Step: `debootstrap`
+Step: debootstrap
 -----------------------------------------------------------------------------
 
 Create a directory tree with a basic Debian installation. This does
diff --git a/vmdb/plugins/files.mdwn b/vmdb/plugins/files.mdwn
deleted file mode 100644
index bd6fece..0000000
--- a/vmdb/plugins/files.mdwn
+++ /dev/null
@@ -1,59 +0,0 @@
-Step: create-dir
------------------------------------------------------------------------------
-
-Create a directory in the target filesystem
-
-Step keys:
-
-* `create-dir` &mdash; REQUIRED; the full (starting from the new
-  filesystem root) path name of the directory to create. It will work
-  as a `mkdir -p` — Any intermediate directories that do not yet exist
-  will be created.
-* `perm` &mdash; OPTIONAL; the numeric (octal) representation of the
-  directory's permissions. Defaults to 0755.
-* `uid` &mdash; OPTIONAL; the numeric user ID of the directory's
-  owner. Defaults to 0 (root).
-* `gid` &mdash; OPTIONAL; the numeric user ID of the directory's
-  group. Defaults to 0 (root).
-
-Step: create-file
------------------------------------------------------------------------------
-
-Create an empty file in the target filesystem.
-
-Step keys:
-
-* `create-file` &mdash; REQUIRED; the full (starting from the new
-  filesystem root) path name of the file to create. It will *not*
-  create any directories; if they need to be created, please use
-  `create-dir` first.
-* `contents` &mdash; REQUIRED; the contents to be written to the
-  generated file.
-* `perm` &mdash; OPTIONAL; the numeric (octal) representation of the
-  file's permissions. Defaults to 0644.
-* `uid` &mdash; OPTIONAL; the numeric user ID of the file's
-  owner. Defaults to 0 (root).
-* `gid` &mdash; OPTIONAL; the numeric user ID of the file's
-  group. Defaults to 0 (root).
-
-
-Step: copy-file
------------------------------------------------------------------------------
-
-Copy a file from outside into the target filesystem.
-
-Step keys:
-
-* `copy-file` &mdash; REQUIRED; the full (starting from the new
-  filesystem root) path name of the file to create. Any missing
-  directories will be created (owner root, group root, mode 0511).
-* `src` &mdash; REQUIRED; filename on the host filesystem, outside the
-  chroot, relative to the current working directory of the vmdb2
-  process.
-* `perm` &mdash; OPTIONAL; the numeric (octal) representation of the
-  file's permissions. Defaults to 0644.
-* `uid` &mdash; OPTIONAL; the numeric user ID of the file's
-  owner. Defaults to 0 (root).
-* `gid` &mdash; OPTIONAL; the numeric user ID of the file's
-  group. Defaults to 0 (root).
-
diff --git a/vmdb/plugins/fstab.mdwn b/vmdb/plugins/fstab.mdwn
index a2c8eef..46f905a 100644
--- a/vmdb/plugins/fstab.mdwn
+++ b/vmdb/plugins/fstab.mdwn
@@ -1,4 +1,4 @@
-Step: `fstab`
+Step: fstab
 -----------------------------------------------------------------------------
 
 Create `/etc/fstab` inside the the image.
diff --git a/vmdb/plugins/grub.mdwn b/vmdb/plugins/grub.mdwn
index 1d69460..706e662 100644
--- a/vmdb/plugins/grub.mdwn
+++ b/vmdb/plugins/grub.mdwn
@@ -1,4 +1,4 @@
-Step: `grub`
+Step: grub
 -----------------------------------------------------------------------------
 
 Install the GRUB bootloader to the image. Works on a PC, for
diff --git a/vmdb/plugins/kpartx.mdwn b/vmdb/plugins/kpartx.mdwn
new file mode 100644
index 0000000..9bb81a7
--- /dev/null
+++ b/vmdb/plugins/kpartx.mdwn
@@ -0,0 +1,13 @@
+Step: kpartx
+-----------------------------------------------------------------------------
+
+Create loop devices for partitions in an image file. Not needed when
+installing to a real block device, instead of an image file.
+
+Step keys:
+
+* `kpartx` &mdash; REQUIRED; filename of block device with partitions.
+
+Example (in the .vmdb file):
+
+    - kpartx: "{{ output }}"
diff --git a/vmdb/plugins/kpartx_plugin.py b/vmdb/plugins/kpartx_plugin.py
new file mode 100644
index 0000000..dca61bb
--- /dev/null
+++ b/vmdb/plugins/kpartx_plugin.py
@@ -0,0 +1,57 @@
+# Copyright 2017  Lars Wirzenius
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# =*= License: GPL-3+ =*=
+
+
+
+import os
+import stat
+
+import cliapp
+
+import vmdb
+
+
+class KpartxPlugin(cliapp.Plugin):
+
+    def enable(self):
+        self.app.step_runners.add(KpartxStepRunner())
+
+
+class KpartxStepRunner(vmdb.StepRunnerInterface):
+
+    def get_required_keys(self):
+        return ['kpartx']
+
+    def run(self, step, settings, state):
+        device = step['kpartx']
+        tags = state.tags.get_tags()
+        devs = self.kpartx(device)
+        for tag, dev in zip(tags, devs):
+            vmdb.progress('remembering {} as {}'.format(dev, tag))
+            state.tags.set_dev(tag, dev)
+
+    def kpartx(self, device):
+        output = vmdb.runcmd(['kpartx', '-asv', device]).decode('UTF-8')
+        for line in output.splitlines():
+            words = line.split()
+            if words[0] == 'add':
+                name = words[2]
+                yield '/dev/mapper/{}'.format(name)
+
+    def teardown(self, step, settings, state):
+        device = step['kpartx']
+        vmdb.runcmd(['kpartx', '-dsv', device])
diff --git a/vmdb/plugins/luks.mdwn b/vmdb/plugins/luks.mdwn
index 4b80f63..663d7b7 100644
--- a/vmdb/plugins/luks.mdwn
+++ b/vmdb/plugins/luks.mdwn
@@ -1,4 +1,4 @@
-Step: `luks`
+Step: luks
 -----------------------------------------------------------------------------
 
 Set up disk encryption using LUKS with the `cryptsetup` utility. The
diff --git a/vmdb/plugins/lvcreate.mdwn b/vmdb/plugins/lvcreate.mdwn
new file mode 100644
index 0000000..fcb350d
--- /dev/null
+++ b/vmdb/plugins/lvcreate.mdwn
@@ -0,0 +1,18 @@
+Step: lvcreate
+-----------------------------------------------------------------------------
+
+Create an LVM2 logical volume (LV) in an existing volume group.
+
+Step keys:
+
+* `lvcreate` &mdash; REQUIRED; value is the tag for the volume group.
+
+* `name` &mdash; REQUIRED; tag for the new LV block device.
+
+* `size` &mdash; REQUIRED; size of the new LV.
+
+Example (in the .vmdb file):
+
+    - lvcreate: rootvg
+      name: rootfs
+      size: 1G
diff --git a/vmdb/plugins/lvcreate_plugin.py b/vmdb/plugins/lvcreate_plugin.py
new file mode 100644
index 0000000..c8a83b8
--- /dev/null
+++ b/vmdb/plugins/lvcreate_plugin.py
@@ -0,0 +1,48 @@
+# Copyright 2018  Lars Wirzenius
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# =*= License: GPL-3+ =*=
+
+
+
+import os
+
+import cliapp
+
+import vmdb
+
+
+class LvcreatePlugin(cliapp.Plugin):
+
+    def enable(self):
+        self.app.step_runners.add(LvcreateStepRunner())
+
+
+class LvcreateStepRunner(vmdb.StepRunnerInterface):
+
+    def get_required_keys(self):
+        return ['lvcreate']
+
+    def run(self, step, settings, state):
+        vgname = step['lvcreate']
+        lvname = step['name']
+        size = step['size']
+
+        vmdb.runcmd(['lvcreate', '--name', lvname, '--size', size, vgname])
+
+        lvdev = '/dev/{}/{}'.format(vgname, lvname)
+        assert os.path.exists(lvdev)
+        state.tags.append(lvname)
+        state.tags.set_dev(lvname, lvdev)
diff --git a/vmdb/plugins/mkfs.mdwn b/vmdb/plugins/mkfs.mdwn
index fb433b8..fed8609 100644
--- a/vmdb/plugins/mkfs.mdwn
+++ b/vmdb/plugins/mkfs.mdwn
@@ -1,4 +1,4 @@
-Step: `mkfs`
+Step: mkfs
 -----------------------------------------------------------------------------
 
 Create a filesystem.
diff --git a/vmdb/plugins/mkimg.mdwn b/vmdb/plugins/mkimg.mdwn
index e2998ac..2eb866a 100644
--- a/vmdb/plugins/mkimg.mdwn
+++ b/vmdb/plugins/mkimg.mdwn
@@ -1,4 +1,4 @@
-Step: `mkimg`
+Step: mkimg
 -----------------------------------------------------------------------------
 
 Create a new image file of a desired size.
diff --git a/vmdb/plugins/mklabel.mdwn b/vmdb/plugins/mklabel.mdwn
new file mode 100644
index 0000000..e954fc3
--- /dev/null
+++ b/vmdb/plugins/mklabel.mdwn
@@ -0,0 +1,16 @@
+Step: mklabel
+-----------------------------------------------------------------------------
+
+Create a partition table on a block device.
+
+Step keys:
+
+* `mklabel` &mdash; REQUIRED; type of partition table, MUST be one of
+  `msdos` and `gpt`.
+
+* `device` &mdash; REQUIRED; tag for the block device.
+
+Example (in the .vmdb file):
+
+    - mklabel: msdos
+      device: "{{ output }}"
diff --git a/vmdb/plugins/mklabel_plugin.py b/vmdb/plugins/mklabel_plugin.py
new file mode 100644
index 0000000..39e8c8d
--- /dev/null
+++ b/vmdb/plugins/mklabel_plugin.py
@@ -0,0 +1,42 @@
+# Copyright 2017  Lars Wirzenius
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# =*= License: GPL-3+ =*=
+
+
+
+import os
+import stat
+
+import cliapp
+
+import vmdb
+
+
+class MklabelPlugin(cliapp.Plugin):
+
+    def enable(self):
+        self.app.step_runners.add(MklabelStepRunner())
+
+
+class MklabelStepRunner(vmdb.StepRunnerInterface):
+
+    def get_required_keys(self):
+        return ['mklabel', 'device']
+
+    def run(self, step, settings, state):
+        label_type = step['mklabel']
+        device = step['device']
+        vmdb.runcmd(['parted', '-s', device, 'mklabel', label_type])
diff --git a/vmdb/plugins/mkpart.mdwn b/vmdb/plugins/mkpart.mdwn
new file mode 100644
index 0000000..bfbea5d
--- /dev/null
+++ b/vmdb/plugins/mkpart.mdwn
@@ -0,0 +1,26 @@
+Step: mkpart
+-----------------------------------------------------------------------------
+
+Create a partition.
+
+Step keys:
+
+* `mkpart` &mdash; REQUIRED; type of partition to create: use
+  `primary` (but any value acceped by `parted` is OK).
+
+* `device` &mdash; REQUIRED; filename of block device where to create
+  partition.
+
+* `start` &mdash; REQUIRED; where does the partition start?
+
+* `end` &mdash; REQUIRED; where does the partition end?
+
+* `tag` &mdash; REQUIRED; tag for the new partition.
+
+Example (in the .vmdb file):
+
+    - mkpart: primary
+      device: "{{ output }}"
+      start: 0%
+      end: 100%
+      tag: root
diff --git a/vmdb/plugins/partition_plugin.py b/vmdb/plugins/mkpart_plugin.py
index 60d4225..45c4148 100644
--- a/vmdb/plugins/partition_plugin.py
+++ b/vmdb/plugins/mkpart_plugin.py
@@ -25,23 +25,10 @@ import cliapp
 import vmdb
 
 
-class PartitionPlugin(cliapp.Plugin):
+class MkpartPlugin(cliapp.Plugin):
 
     def enable(self):
-        self.app.step_runners.add(MklabelStepRunner())
         self.app.step_runners.add(MkpartStepRunner())
-        self.app.step_runners.add(KpartxStepRunner())
-
-
-class MklabelStepRunner(vmdb.StepRunnerInterface):
-
-    def get_required_keys(self):
-        return ['mklabel', 'device']
-
-    def run(self, step, settings, state):
-        label_type = step['mklabel']
-        device = step['device']
-        vmdb.runcmd(['parted', '-s', device, 'mklabel', label_type])
 
 
 class MkpartStepRunner(vmdb.StepRunnerInterface):
@@ -93,29 +80,3 @@ class MkpartStepRunner(vmdb.StepRunnerInterface):
             for line in new
             if line not in old
         ]
-
-
-class KpartxStepRunner(vmdb.StepRunnerInterface):
-
-    def get_required_keys(self):
-        return ['kpartx']
-
-    def run(self, step, settings, state):
-        device = step['kpartx']
-        tags = state.tags.get_tags()
-        devs = self.kpartx(device)
-        for tag, dev in zip(tags, devs):
-            vmdb.progress('remembering {} as {}'.format(dev, tag))
-            state.tags.set_dev(tag, dev)
-
-    def kpartx(self, device):
-        output = vmdb.runcmd(['kpartx', '-asv', device]).decode('UTF-8')
-        for line in output.splitlines():
-            words = line.split()
-            if words[0] == 'add':
-                name = words[2]
-                yield '/dev/mapper/{}'.format(name)
-
-    def teardown(self, step, settings, state):
-        device = step['kpartx']
-        vmdb.runcmd(['kpartx', '-dsv', device])
diff --git a/vmdb/plugins/mount.mdwn b/vmdb/plugins/mount.mdwn
index cf447f1..9c830ef 100644
--- a/vmdb/plugins/mount.mdwn
+++ b/vmdb/plugins/mount.mdwn
@@ -1,4 +1,4 @@
-Step: `mount`
+Step: mount
 -----------------------------------------------------------------------------
 
 Mount a filesystem.
diff --git a/vmdb/plugins/partition.mdwn b/vmdb/plugins/partition.mdwn
deleted file mode 100644
index b058056..0000000
--- a/vmdb/plugins/partition.mdwn
+++ /dev/null
@@ -1,57 +0,0 @@
-Step: `mklabel`
------------------------------------------------------------------------------
-
-Create a partition table on a block device.
-
-Step keys:
-
-* `mklabel` &mdash; REQUIRED; type of partition table, MUST be one of
-  `msdos` and `gpt`.
-
-* `device` &mdash; REQUIRED; tag for the block device.
-
-Example (in the .vmdb file):
-
-    - mklabel: msdos
-      device: "{{ output }}"
-
-Step: `mkpart`
------------------------------------------------------------------------------
-
-Create a partition.
-
-Step keys:
-
-* `mkpart` &mdash; REQUIRED; type of partition to create: use
-  `primary` (but any value acceped by `parted` is OK).
-
-* `device` &mdash; REQUIRED; filename of block device where to create
-  partition.
-
-* `start` &mdash; REQUIRED; where does the partition start?
-
-* `end` &mdash; REQUIRED; where does the partition end?
-
-* `tag` &mdash; REQUIRED; tag for the new partition.
-
-Example (in the .vmdb file):
-
-    - mkpart: primary
-      device: "{{ output }}"
-      start: 0%
-      end: 100%
-      tag: root
-
-Step: `kpartx`
------------------------------------------------------------------------------
-
-Create loop devices for partitions in an image file. Not needed when
-installing to a real block device, instead of an image file.
-
-Step keys:
-
-* `kpartx` &mdash; REQUIRED; filename of block device with partitions.
-
-Example (in the .vmdb file):
-
-    - kpartx: "{{ output }}"
diff --git a/vmdb/plugins/qemudebootstrap.mdwn b/vmdb/plugins/qemudebootstrap.mdwn
index 9b62ab0..bb9adc4 100644
--- a/vmdb/plugins/qemudebootstrap.mdwn
+++ b/vmdb/plugins/qemudebootstrap.mdwn
@@ -1,4 +1,4 @@
-tep: `qemu-debootstrap`
+Step: qemu-debootstrap
 -----------------------------------------------------------------------------
 
 Install packages using apt, which needs to already have been
diff --git a/vmdb/plugins/shell.mdwn b/vmdb/plugins/shell.mdwn
new file mode 100644
index 0000000..02ed0b9
--- /dev/null
+++ b/vmdb/plugins/shell.mdwn
@@ -0,0 +1,17 @@
+Step: shell
+-----------------------------------------------------------------------------
+
+Run a shell snippet on the host. This is not run in a chroot, and can
+access the host system.
+
+Step keys:
+
+* `root-fs` &mdash; REQUIRED; value is the tag for the root filesystem.
+
+* `shell` &mdash; REQUIRED; the shell snippet to run
+
+Example (in the .vmdb file):
+
+    - root-fs: root
+      shell: |
+          echo I am in NOT in chroot.
diff --git a/vmdb/plugins/shell_plugin.mdwn b/vmdb/plugins/shell_plugin.mdwn
new file mode 100644
index 0000000..ecbbbfa
--- /dev/null
+++ b/vmdb/plugins/shell_plugin.mdwn
@@ -0,0 +1,44 @@
+# Copyright 2017  Lars Wirzenius
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# =*= License: GPL-3+ =*=
+
+
+
+import os
+
+import cliapp
+
+import vmdb
+
+
+class ShellPlugin(cliapp.Plugin):
+
+    def enable(self):
+        self.app.step_runners.add(ShellStepRunner())
+
+
+class ShellStepRunner(vmdb.StepRunnerInterface):
+
+    def get_required_keys(self):
+        return ['shell', 'root-fs']
+
+    def run(self, step, settings, state):
+        shell = step['shell']
+        fs_tag = step['root-fs']
+
+        env = dict(os.environ)
+        env['ROOT'] = state.tags.get_builder_mount_point(fs_tag)
+        vmdb.runcmd(['sh', '-ec', shell], env=env)
diff --git a/vmdb/plugins/rootfs_cache.mdwn b/vmdb/plugins/unpack_rootfs.mdwn
index 3f4ee0b..2883937 100644
--- a/vmdb/plugins/rootfs_cache.mdwn
+++ b/vmdb/plugins/unpack_rootfs.mdwn
@@ -1,19 +1,4 @@
-Step: `cache-rootfs`
------------------------------------------------------------------------------
-
-Create a tarball of the root filesystem in the image.
-
-Step keys:
-
-* `cache-rootfs` &mdash; REQUIRED; tag of root filesystem on image.
-
-Example (in the .vmdb file):
-
-    - cache-rootfs: root
-      unless: rootfs_unpacked
-
-
-Step: `unpack-rootfs`
+Step: unpack-rootfs
 -----------------------------------------------------------------------------
 
 Unpack a tarball of the root filesystem to the image, and set the
diff --git a/vmdb/plugins/unpack_rootfs_plugin.py b/vmdb/plugins/unpack_rootfs_plugin.py
new file mode 100644
index 0000000..8e6aada
--- /dev/null
+++ b/vmdb/plugins/unpack_rootfs_plugin.py
@@ -0,0 +1,52 @@
+# Copyright 2017  Lars Wirzenius
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# =*= License: GPL-3+ =*=
+
+
+
+import os
+
+import cliapp
+
+import vmdb
+
+
+class UnpackRootFSPlugin(cliapp.Plugin):
+
+    def enable(self):
+        self.app.step_runners.add(UnpackCacheStepRunner())
+
+
+class UnpackCacheStepRunner(vmdb.StepRunnerInterface):
+
+    def get_required_keys(self):
+        return ['unpack-rootfs']
+
+    def run(self, step, settings, state):
+        fs_tag = step['unpack-rootfs']
+        rootdir = state.tags.get_builder_mount_point(fs_tag)
+        tar_path = settings['rootfs-tarball']
+        if not tar_path:
+            raise Exception('--rootfs-tarball MUST be set')
+        if os.path.exists(tar_path):
+            vmdb.runcmd(
+                ['tar', '-C', rootdir, '-xf', tar_path, '--numeric-owner'])
+            self.copy_resolv_conf(rootdir)
+            state.rootfs_unpacked = True
+
+    def copy_resolv_conf(self, rootdir):
+        filename = os.path.join(rootdir, 'etc', 'resolv.conf')
+        vmdb.runcmd(['cp', '/etc/resolv.conf', filename])
diff --git a/vmdb/plugins/lvm2.mdwn b/vmdb/plugins/vgcreate.mdwn
index 76f5892..02207e5 100644
--- a/vmdb/plugins/lvm2.mdwn
+++ b/vmdb/plugins/vgcreate.mdwn
@@ -1,4 +1,4 @@
-Step: `vgcreate`
+Step: vgcreate
 -----------------------------------------------------------------------------
 
 Create an LVM2 volume group (VG), and also initialise the physical
@@ -20,23 +20,3 @@ Example (in the .vmdb file):
       physical:
       - my_partition
       - other_partition
-
-
-Step: `lvcreate`
------------------------------------------------------------------------------
-
-Create an LVM2 logical volume (LV) in an existing volume group.
-
-Step keys:
-
-* `lvcreate` &mdash; REQUIRED; value is the tag for the volume group.
-
-* `name` &mdash; REQUIRED; tag for the new LV block device.
-
-* `size` &mdash; REQUIRED; size of the new LV.
-
-Example (in the .vmdb file):
-
-    - lvcreate: rootvg
-      name: rootfs
-      size: 1G
diff --git a/vmdb/plugins/lvm2_plugin.py b/vmdb/plugins/vgcreate_plugin.py
index 03a98c0..2816265 100644
--- a/vmdb/plugins/lvm2_plugin.py
+++ b/vmdb/plugins/vgcreate_plugin.py
@@ -24,11 +24,10 @@ import cliapp
 import vmdb
 
 
-class Lvm2Plugin(cliapp.Plugin):
+class VgcreatePlugin(cliapp.Plugin):
 
     def enable(self):
         self.app.step_runners.add(VgcreateStepRunner())
-        self.app.step_runners.add(LvcreateStepRunner())
 
 
 class VgcreateStepRunner(vmdb.StepRunnerInterface):
@@ -56,21 +55,3 @@ class VgcreateStepRunner(vmdb.StepRunnerInterface):
             state.tags.get_dev(tag)
             for tag in step['physical']
         ]
-
-
-class LvcreateStepRunner(vmdb.StepRunnerInterface):
-
-    def get_required_keys(self):
-        return ['lvcreate']
-
-    def run(self, step, settings, state):
-        vgname = step['lvcreate']
-        lvname = step['name']
-        size = step['size']
-
-        vmdb.runcmd(['lvcreate', '--name', lvname, '--size', size, vgname])
-
-        lvdev = '/dev/{}/{}'.format(vgname, lvname)
-        assert os.path.exists(lvdev)
-        state.tags.append(lvname)
-        state.tags.set_dev(lvname, lvdev)
diff --git a/without-tests b/without-tests
index d15cfe3..4c84d9c 100644
--- a/without-tests
+++ b/without-tests
@@ -1,26 +1,31 @@
-yarns/lib.py
 vmdb/__init__.py
 vmdb/app.py
-vmdb/runcmd.py
-vmdb/state.py
-vmdb/version.py
 vmdb/plugins/__init__.py
 vmdb/plugins/ansible_plugin.py
 vmdb/plugins/apt_plugin.py
+vmdb/plugins/cache_rootfs_plugin.py
 vmdb/plugins/chroot_plugin.py
+vmdb/plugins/copy_file_plugin.py
+vmdb/plugins/create_dir_plugin.py
+vmdb/plugins/create_file_plugin.py
 vmdb/plugins/debootstrap_plugin.py
 vmdb/plugins/echo_plugin.py
 vmdb/plugins/error_plugin.py
 vmdb/plugins/fstab_plugin.py
 vmdb/plugins/grub_plugin.py
-vmdb/plugins/kernel_plugin.py
-vmdb/plugins/lvm2_plugin.py
+vmdb/plugins/kpartx_plugin.py
 vmdb/plugins/luks_plugin.py
+vmdb/plugins/lvcreate_plugin.py
 vmdb/plugins/mkfs_plugin.py
 vmdb/plugins/mkimg_plugin.py
+vmdb/plugins/mklabel_plugin.py
+vmdb/plugins/mkpart_plugin.py
 vmdb/plugins/mount_plugin.py
 vmdb/plugins/qemudebootstrap_plugin.py
-vmdb/plugins/partition_plugin.py
-vmdb/plugins/rootfs_cache_plugin.py
+vmdb/plugins/unpack_rootfs_plugin.py
+vmdb/plugins/vgcreate_plugin.py
 vmdb/plugins/virtuals_plugin.py
-vmdb/plugins/files_plugin.py
+vmdb/runcmd.py
+vmdb/state.py
+vmdb/version.py
+yarns/lib.py