Add a rootfs caching plugin

Also, add a generic "unless" field for steps.

7 files changed, 114 insertions, 0 deletions

diff --git a/NEWS b/NEWS
index ba73b4e..879bc83 100644
--- a/NEWS
+++ b/NEWS
@@ -4,6 +4,18 @@ NEWS for vmdb2, the Debian disk image builder
 Version 0.1+git, not yet released
 ---------------------------------
 
+* Add plugin to provide steps to cache the rootfs, and to unpack the
+  rootfs from the cache, instead of running debootstrap or installing
+  packages. This speeds up iteration time from about 9 minutes to 40
+  seconds on my laptop. See small.vmdb for an example.
+
+* Add a generic "unless this condition is true" functionality to
+  steps. If a step has a field "unless: foo" it is skipped the
+  variable foo exists and is true. The variables are set by steps, and
+  currently only the rootfs unpacking step sets a variable (the
+  `rootfs_unpacked` variable). This allows debootstrap to be skipped
+  if the rootfs has already been created by unpacking a cached
+  tarball.
 
 Version 0.1, released 2017-05-13
 -----------------------------------------------------------------------------
diff --git a/small.vmdb b/small.vmdb
index cbc1971..177c1b9 100644
--- a/small.vmdb
+++ b/small.vmdb
@@ -26,12 +26,19 @@ steps:
   - mount: root-part
     fs-tag: root-fs
 
+  - unpack-rootfs: root-fs
+
   - debootstrap: stretch
     mirror: http://http.debian.net/debian
     target: root-fs
+    unless: rootfs_unpacked
 
   - apt: linux-image-amd64
     fs-tag: root-fs
+    unless: rootfs_unpacked
+
+  - cache-rootfs: root-fs
+    unless: rootfs_unpacked
 
   - grub: uefi
     root-fs: root-fs
diff --git a/vmdb/plugins/apt_plugin.py b/vmdb/plugins/apt_plugin.py
index b567e75..9da8ef4 100644
--- a/vmdb/plugins/apt_plugin.py
+++ b/vmdb/plugins/apt_plugin.py
@@ -37,6 +37,9 @@ class AptStepRunner(vmdb.StepRunnerInterface):
         return ['apt', 'fs-tag']
 
     def run(self, step, settings, state):
+        if getattr(state, 'rootfs_unpacked', False):
+            return
+
         package = step['apt']
         fstag = step['fs-tag']
         mount_point = state.mounts[fstag]
diff --git a/vmdb/plugins/debootstrap_plugin.py b/vmdb/plugins/debootstrap_plugin.py
index 91b2852..824c42c 100644
--- a/vmdb/plugins/debootstrap_plugin.py
+++ b/vmdb/plugins/debootstrap_plugin.py
@@ -37,6 +37,9 @@ class DebootstrapStepRunner(vmdb.StepRunnerInterface):
         return ['debootstrap', 'target', 'mirror']
 
     def run(self, step, settings, state):
+        if getattr(state, 'rootfs_unpacked', False):
+            return
+
         suite = step['debootstrap']
         tag = step['target']
         target = state.mounts[tag]
diff --git a/vmdb/plugins/rootfs_cache_plugin.py b/vmdb/plugins/rootfs_cache_plugin.py
new file mode 100644
index 0000000..d37b199
--- /dev/null
+++ b/vmdb/plugins/rootfs_cache_plugin.py
@@ -0,0 +1,70 @@
+# Copyright 2017  Lars Wirzenius
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# =*= License: GPL-3+ =*=
+
+
+
+import logging
+import os
+import sys
+
+import cliapp
+
+import vmdb
+
+
+class RootFSCachePlugin(cliapp.Plugin):
+
+    def enable(self):
+        self.app.settings.string(
+            ['rootfs-tarball'],
+            'store rootfs cache tar archives in FILE',
+            metavar='FILE')
+
+        self.app.step_runners.add(MakeCacheStepRunner())
+        self.app.step_runners.add(UnpackCacheStepRunner())
+        
+
+class MakeCacheStepRunner(vmdb.StepRunnerInterface):
+
+    def get_required_keys(self):
+        return ['cache-rootfs']
+
+    def run(self, step, settings, state):
+        fs_tag = step['cache-rootfs']
+        rootdir = state.mounts[fs_tag]
+        tar_path = settings['rootfs-tarball']
+        if not os.path.exists(tar_path):
+            vmdb.progress(
+                'Caching contents of {} to {}'.format(rootdir, tar_path))
+            cliapp.runcmd(['tar', '-C', rootdir, '-caf', tar_path, '.'])
+        
+
+class UnpackCacheStepRunner(vmdb.StepRunnerInterface):
+
+    def get_required_keys(self):
+        return ['unpack-rootfs']
+
+    def run(self, step, settings, state):
+        fs_tag = step['unpack-rootfs']
+        rootdir = state.mounts[fs_tag]
+        tar_path = settings['rootfs-tarball']
+        if os.path.exists(tar_path):
+            vmdb.progress(
+                'Unpacking rootfs from {} to {}'.format(tar_path, rootdir))
+            cliapp.runcmd(
+                ['tar', '-C', rootdir, '-xf', tar_path, '--numeric-owner'])
+            state.rootfs_unpacked = True
diff --git a/vmdb/step_list.py b/vmdb/step_list.py
index a9ee1c3..b5a7c7c 100644
--- a/vmdb/step_list.py
+++ b/vmdb/step_list.py
@@ -32,6 +32,24 @@ class StepRunnerInterface(object):  # pragma: no cover
         # need to have a nop teardown.
         pass
 
+    def skip(self, step_spec, settings, state):
+        # Return true if step should be skipped and not run. Does not
+        # apply to teardowns.
+
+        # Skipping is indicated by the step having a field 'unless',
+        # which is either the name of a variable (field in state), or
+        # a list of such names. If all variables have a value that
+        # evaluates as truth, the step is skipped.
+
+        value = step_spec.get('unless', None)
+        if value is None:
+            return False
+
+        if isinstance(value, list):
+            return all(getattr(state, field, False) for field in value)
+        else:
+            return getattr(state, value, False)
+
 
 class StepRunnerList(object):
 
diff --git a/without-tests b/without-tests
index 895420f..c950117 100644
--- a/without-tests
+++ b/without-tests
@@ -15,3 +15,4 @@ vmdb/plugins/mkfs_plugin.py
 vmdb/plugins/mkimg_plugin.py
 vmdb/plugins/mount_plugin.py
 vmdb/plugins/partition_plugin.py
+vmdb/plugins/rootfs_cache_plugin.py