use a constants handler

Add support for using squashfs as an alternative to creating
a tarball of a directory tree.

7 files changed, 169 insertions, 87 deletions

diff --git a/README b/README
index 2b58ccf..b6fa46c 100644
--- a/README
+++ b/README
@@ -68,12 +68,40 @@ $ sudo PYTHONPATH=. ./bin/vmdebootstrap
 This has changed slightly with version 1.0 with the need for
 PYTHONPATH to reference the module approach for support handlers.
 
+vmdebootstrap modules
+---------------------
+
+The single vmdebootstrap script has been refactored to be the top
+level settings parser and validator and the point where the other
+modules (handlers) get to be called in a collaborative sequence.
+
+The new modules are an attempt to work with a DRY process as well
+as keeping the source code itself maintainable. Handler functions
+need to check settings at the start so that calls to the handlers
+can be retained in a simple flow. Where a function needs code from
+multiple handlers, that function needs to be in the vmdebootstrap
+script but these should, ideally, be single calls into dedicated
+calls from the relevant handlers which can return True|False or
+raise cliapp.AppException to affect subsequent flow. Handlers must
+NOT hook into other handlers, except Base or constants, only the
+vmdebootstrap script has the full set, so use function arguments to
+pass variables populated by different handlers. Wherever possible,
+large sections of new functionality need to be added as new handlers.
+
 pylint
 ------
 
-vmdebootstrap uses pylint and contains comments to disable certain
-pylint checks in certain areas. pylint compatibility will make it
-easier to accept patches, just follow the existing pattern
+When using pylint, the following option is advised:
+
+ $ pylint --ignore-imports=y vmdebootstrap
+
+(Despite the name of the option, this only ignores imports when
+computing similarities and various handlers will end up needing
+similar imports, it makes no sense to complain about that.)
+
+Apart from that, vmdebootstrap uses pylint and contains comments to
+disable certain pylint checks in certain areas. pylint compatibility
+will make it easier to accept patches, just follow the existing pattern
 of pylint usage. pylint is far from perfect but can be helpful.
 
 Testing UEFI support
diff --git a/bin/vmdebootstrap b/bin/vmdebootstrap
index 876ec5f..b13d976 100755
--- a/bin/vmdebootstrap
+++ b/bin/vmdebootstrap
@@ -34,8 +34,8 @@ from vmdebootstrap.grub import GrubHandler
 from vmdebootstrap.extlinux import ExtLinux
 from vmdebootstrap.codenames import Codenames
 from vmdebootstrap.filesystem import Filesystem
-from vmdebootstrap.uefi import Uefi, arch_table
-
+from vmdebootstrap.uefi import Uefi
+from vmdebootstrap.constants import arch_table
 
 __version__ = '1.0'
 
@@ -168,6 +168,11 @@ class VmDebootstrap(cliapp.Application):  # pylint: disable=too-many-public-meth
             'the build is complete.')
         self.settings.boolean(
             ['squash'], 'use squashfs on the final image.')
+        self.settings.string(
+            ['squash-file'], 'filename for the squashfs '
+            '- cannot be used with --image',
+            metavar='FILE',
+            default='rootfs.squash')
         self.settings.boolean(
             ['configure-apt'], 'Create an apt source based on '
             'the distribution and mirror selected.')
@@ -193,9 +198,11 @@ class VmDebootstrap(cliapp.Application):  # pylint: disable=too-many-public-meth
             handler.define_settings(self.settings)
 
         distro = self.handlers[Codenames.name]
-        if not self.settings['image'] and not self.settings['tarball']:
+        if not self.settings['image'] and not (
+                self.settings['tarball'] or self.settings['squash']):
             raise cliapp.AppException(
-                'You must give disk image filename, or tarball filename')
+                'You must give disk image filename or use either a '
+                'tarball filename or use squash')
         if self.settings['image'] and not self.settings['size']:
             raise cliapp.AppException(
                 'If disk image is specified, you must give image size.')
@@ -247,7 +254,7 @@ class VmDebootstrap(cliapp.Application):  # pylint: disable=too-many-public-meth
         roottype = filesystem.devices['roottype']
         bootdev = filesystem.devices['bootdev']
         if self.settings['swap'] > 0:
-            self.message("Creating swap space")
+            base.message("Creating swap space")
             runcmd(['mkswap', filesystem.devices['swapdev']])
         filesystem.mkfs(rootdev, fstype=roottype)
         rootdir = self.mount(rootdev)
@@ -268,12 +275,13 @@ class VmDebootstrap(cliapp.Application):  # pylint: disable=too-many-public-meth
         grub = self.handlers[GrubHandler.name]
         extlinux = self.handlers[ExtLinux.name]
         base = self.handlers[Base.name]
+        uefi = self.handlers[Uefi.name]
         filesystem = self.handlers[Filesystem.name]
         if self.settings['use-uefi']:
             grub.install_grub_uefi(rootdir)
-            uefi.configure_efi()
+            uefi.configure_efi(rootdir)
             grub.install_extra_grub_uefi(rootdir)
-            uefi.configure_extra_efi()
+            uefi.configure_extra_efi(rootdir)
         elif self.settings['grub']:
             if not grub.install_grub2(rootdev, rootdir):
                 extlinux.install_extlinux(rootdev, rootdir)
@@ -281,7 +289,7 @@ class VmDebootstrap(cliapp.Application):  # pylint: disable=too-many-public-meth
             extlinux.install_extlinux(rootdev, rootdir)
         base.append_serial_console(rootdir)
         self.optimize_image(rootdir)
-        filesystem.squash()
+        filesystem.squash_image()
 
     def start_ops(self):
         base = self.handlers[Base.name]
@@ -293,6 +301,7 @@ class VmDebootstrap(cliapp.Application):  # pylint: disable=too-many-public-meth
                 rootdev = filesystem.devices['rootdev']
             else:
                 rootdir = self.mkdtemp()
+                rootdev = filesystem.devices['rootdev']
             logging.debug("rootdir=%s rootdev=%s", rootdir, rootdev)
             self.debootstrap(rootdir)
             filesystem.set_hostname()
@@ -316,38 +325,23 @@ class VmDebootstrap(cliapp.Application):  # pylint: disable=too-many-public-meth
 
             if self.settings['tarball']:
                 base.create_tarball(rootdir)
+            else:
+                filesystem.squash_filesystem()
             filesystem.chown()
 
         except BaseException as e:
-            self.message('EEEK! Something bad happened...')
+            base.message('EEEK! Something bad happened...')
+            rootdir = filesystem.devices['rootdir']
             if rootdir:
                 db_log = os.path.join(rootdir, 'debootstrap', 'debootstrap.log')
                 if os.path.exists(db_log):
                     shutil.copy(db_log, os.getcwd())
-            self.message(e)
+            base.message(e)
             self.cleanup_system()
             raise
         else:
             self.cleanup_system()
 
-    def message(self, msg):
-        logging.info(msg)
-        if self.settings['verbose']:
-            print msg
-
-    def runcmd(self, argv, stdin='', ignore_fail=False, env=None, **kwargs):
-        logging.debug('runcmd: %s %s %s', argv, env, kwargs)
-        p = subprocess.Popen(argv, stdin=subprocess.PIPE,
-                             stdout=subprocess.PIPE, stderr=subprocess.PIPE,
-                             env=env, **kwargs)
-        out, err = p.communicate(stdin)
-        if p.returncode != 0:
-            msg = 'command failed: %s\n%s\n%s' % (argv, out, err)
-            logging.error(msg)
-            if not ignore_fail:
-                raise cliapp.AppException(msg)
-        return out
-
     def mkdtemp(self):
         dirname = tempfile.mkdtemp()
         self.remove_dirs.append(dirname)
@@ -355,11 +349,12 @@ class VmDebootstrap(cliapp.Application):  # pylint: disable=too-many-public-meth
         return dirname
 
     def mount(self, device, path=None):
+        base = self.handlers[Base.name]
         if not path:
             mount_point = self.mkdtemp()
         else:
             mount_point = path
-        self.message('Mounting %s on %s' % (device, mount_point))
+        base.message('Mounting %s on %s' % (device, mount_point))
         runcmd(['mount', device, mount_point])
         self.mount_points.append(mount_point)
         logging.debug('mounted %s on %s', device, mount_point)
@@ -372,7 +367,8 @@ class VmDebootstrap(cliapp.Application):  # pylint: disable=too-many-public-meth
         starts at that offset to allow customisation scripts to
         put bootloader images into the space, e.g. u-boot.
         """
-        self.message('Creating partitions')
+        base = self.handlers[Base.name]
+        base.message('Creating partitions')
         runcmd(['parted', '-s', self.settings['image'],
                 'mklabel', self.settings['part-type']])
         partoffset = 0
@@ -399,13 +395,13 @@ class VmDebootstrap(cliapp.Application):  # pylint: disable=too-many-public-meth
                     partoffset, self.settings['bootoffset'])
             else:
                 partoffset = self.settings['bootoffset'] / (1024 * 1024)
-                self.message("Using bootoffset: %smib %s bytes" % (partoffset, self.settings['bootoffset']))
+                base.message("Using bootoffset: %smib %s bytes" % (partoffset, self.settings['bootoffset']))
         if self.settings['bootsize'] and self.settings['bootsize'] is not '0%':
             if self.settings['grub'] and not partoffset:
                 partoffset = 1
             bootsize = self.settings['bootsize'] / (1024 * 1024)
             bootsize += partoffset
-            self.message("Using bootsize %smib: %s bytes" % (bootsize, self.settings['bootsize']))
+            base.message("Using bootsize %smib: %s bytes" % (bootsize, self.settings['bootsize']))
             logging.debug("Starting boot partition at %sMb", bootsize)
             runcmd(['parted', '-s', self.settings['image'],
                     'mkpart', 'primary', 'fat16', str(partoffset), str(bootsize)])
@@ -463,6 +459,7 @@ class VmDebootstrap(cliapp.Application):  # pylint: disable=too-many-public-meth
         return include
 
     def _debootstrap_second_stage(self, rootdir):
+        base = self.handlers[Base.name]
         # set a noninteractive debconf environment for secondstage
         env = {
             "DEBIAN_FRONTEND": "noninteractive",
@@ -472,17 +469,18 @@ class VmDebootstrap(cliapp.Application):  # pylint: disable=too-many-public-meth
         # add the mapping to the complete environment.
         env.update(os.environ)
         # First copy the binfmt handler over
-        self.message('Setting up binfmt handler')
+        base.message('Setting up binfmt handler')
         shutil.copy(self.settings['foreign'], '%s/usr/bin/' % rootdir)
         # Next, run the package install scripts etc.
-        self.message('Running debootstrap second stage')
+        base.message('Running debootstrap second stage')
         runcmd(['chroot', rootdir,
                 '/debootstrap/debootstrap', '--second-stage'],
                env=env)
 
     def debootstrap(self, rootdir):
+        base = self.handlers[Base.name]
         msg = "(%s)" % self.settings['variant'] if self.settings['variant'] else ''
-        self.message(
+        base.message(
             'Debootstrapping %s [%s] %s' % (
                 self.settings['distribution'], self.settings['arch'], msg))
         include = self._bootstrap_packages()
@@ -505,9 +503,10 @@ class VmDebootstrap(cliapp.Application):  # pylint: disable=too-many-public-meth
             self._debootstrap_second_stage(rootdir)
 
     def install_debs(self, rootdir):
+        base = self.handlers[Base.name]
         if not self.settings['custom-package']:
             return
-        self.message('Installing custom packages')
+        base.message('Installing custom packages')
         tmp = os.path.join(rootdir, 'tmp', 'install_debs')
         os.mkdir(tmp)
         for deb in self.settings['custom-package']:
@@ -533,7 +532,8 @@ class VmDebootstrap(cliapp.Application):  # pylint: disable=too-many-public-meth
             runcmd(['rm', '-f', zeros])
 
     def setup_networking(self, rootdir):
-        self.message('Setting up networking')
+        base = self.handlers[Base.name]
+        base.message('Setting up networking')
         distro = self.handlers[Codenames.name]
         # unconditionally write for wheezy (which became oldstable on 04/25/2015)
         if distro.was_oldstable(datetime.date(2015, 4, 26)):
@@ -556,9 +556,10 @@ class VmDebootstrap(cliapp.Application):  # pylint: disable=too-many-public-meth
                 eth.write('iface eth0 inet dhcp\n')
 
     def cleanup_system(self):
+        base = self.handlers[Base.name]
         # Clean up after any errors.
 
-        self.message('Cleaning up')
+        base.message('Cleaning up')
 
         # Umount in the reverse mount order
         if self.settings['image']:
diff --git a/setup.py b/setup.py
index 1c59145..fb25e27 100644
--- a/setup.py
+++ b/setup.py
@@ -2,9 +2,9 @@
 # -*- coding: utf-8 -*-
 #
 #  setup.py
-#  
+#
 #  Copyright 2015 Neil Williams <codehelp@debian.org>
-#  
+#
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
diff --git a/vmdebootstrap/constants.py b/vmdebootstrap/constants.py
new file mode 100644
index 0000000..9f39415
--- /dev/null
+++ b/vmdebootstrap/constants.py
@@ -0,0 +1,52 @@
+"""
+  Constants which can be used by any handler
+"""
+# -*- coding: utf-8 -*-
+#
+#  constants.py
+#
+#  Copyright 2015 Neil Williams <codehelp@debian.org>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+arch_table = {  # pylint: disable=invalid-name
+    'amd64': {
+        'removable': '/EFI/boot/bootx64.efi',  # destination location
+        'install': '/EFI/debian/grubx64.efi',  # package location
+        'package': 'grub-efi-amd64',  # bootstrap package
+        'bin_package': 'grub-efi-amd64-bin',  # binary only
+        'extra': 'i386',  # architecture to add binary package
+        'exclusive': False,  # only EFI supported for this arch.
+        'target': 'x86_64-efi',  # grub target name
+    },
+    'i386': {
+        'removable': '/EFI/boot/bootia32.efi',
+        'install': '/EFI/debian/grubia32.efi',
+        'package': 'grub-efi-ia32',
+        'bin_package': 'grub-efi-ia32-bin',
+        'extra': None,
+        'exclusive': False,
+        'target': 'i386-efi',
+    },
+    'arm64': {
+        'removable': '/EFI/boot/bootaa64.efi',
+        'install': '/EFI/debian/grubaa64.efi',
+        'package': 'grub-efi-arm64',
+        'bin_package': 'grub-efi-arm64-bin',
+        'extra': None,
+        'exclusive': True,
+        'target': 'arm64-efi',
+    }
+}
diff --git a/vmdebootstrap/filesystem.py b/vmdebootstrap/filesystem.py
index 07ec9ca..ccc95a4 100644
--- a/vmdebootstrap/filesystem.py
+++ b/vmdebootstrap/filesystem.py
@@ -160,7 +160,37 @@ class Filesystem(Base):
             elif self.settings['swap'] > 0:
                 fstab.write("/dev/sda2 swap swap defaults 0 0\n")
 
-    def squash(self):
+    def squash_filesystem(self):
+        """
+        Run squashfs on the temporary directory
+        """
+        if not self.settings['squash']:
+            return
+        if self.settings['image']:
+            return
+        if not os.path.exists('/usr/bin/mksquashfs'):
+            logging.warning("Squash selected but mksquashfs not found!")
+            return
+        logging.debug(
+            "%s usage: %s", self.settings['image'],
+            runcmd(['du', self.settings['image']]))
+        self.message("Running mksquashfs")
+        output = self.settings['squash-file']
+        if os.path.exists(output):
+            os.unlink(output)
+        msg = runcmd(
+            ['mksquashfs', self.devices['rootdir'],
+             output, '-no-progress', '-comp', 'xz'], ignore_fail=False)
+        logging.debug(msg)
+        check_size = os.path.getsize(output)
+        if check_size < (1024 * 1024):
+            logging.warning(
+                "%s appears to be too small! %s bytes",
+                output, check_size)
+        else:
+            logging.debug("squashed size: %s", check_size)
+
+    def squash_image(self):
         """
         Run squashfs on the image.
         """
diff --git a/vmdebootstrap/grub.py b/vmdebootstrap/grub.py
index 22d7658..e511462 100644
--- a/vmdebootstrap/grub.py
+++ b/vmdebootstrap/grub.py
@@ -20,6 +20,9 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+# pylint: disable=missing-docstring,duplicate-code
+
+
 import os
 import cliapp
 import logging
@@ -29,9 +32,7 @@ from vmdebootstrap.base import (
     mount_wrapper,
     umount_wrapper
 )
-from vmdebootstrap.uefi import Uefi, arch_table
-
-# pylint: disable=missing-docstring
+from vmdebootstrap.uefi import arch_table
 
 
 def grub_serial_console(rootdir):
@@ -53,7 +54,6 @@ class GrubHandler(Base):
 
     def __init__(self):
         super(GrubHandler, self).__init__()
-        self.uefi = Uefi()
 
     def install_grub2(self, rootdev, rootdir):
         self.message("Configuring grub2")
diff --git a/vmdebootstrap/uefi.py b/vmdebootstrap/uefi.py
index 1acae14..215e97e 100644
--- a/vmdebootstrap/uefi.py
+++ b/vmdebootstrap/uefi.py
@@ -21,6 +21,9 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 
+# pylint: disable=missing-docstring,duplicate-code
+
+
 import os
 import cliapp
 import logging
@@ -30,39 +33,7 @@ from vmdebootstrap.base import (
     mount_wrapper,
     umount_wrapper,
 )
-
-# pylint: disable=missing-docstring
-
-
-arch_table = {  # pylint: disable=invalid-name
-    'amd64': {
-        'removable': '/EFI/boot/bootx64.efi',  # destination location
-        'install': '/EFI/debian/grubx64.efi',  # package location
-        'package': 'grub-efi-amd64',  # bootstrap package
-        'bin_package': 'grub-efi-amd64-bin',  # binary only
-        'extra': 'i386',  # architecture to add binary package
-        'exclusive': False,  # only EFI supported for this arch.
-        'target': 'x86_64-efi',  # grub target name
-    },
-    'i386': {
-        'removable': '/EFI/boot/bootia32.efi',
-        'install': '/EFI/debian/grubia32.efi',
-        'package': 'grub-efi-ia32',
-        'bin_package': 'grub-efi-ia32-bin',
-        'extra': None,
-        'exclusive': False,
-        'target': 'i386-efi',
-    },
-    'arm64': {
-        'removable': '/EFI/boot/bootaa64.efi',
-        'install': '/EFI/debian/grubaa64.efi',
-        'package': 'grub-efi-arm64',
-        'bin_package': 'grub-efi-arm64-bin',
-        'extra': None,
-        'exclusive': True,
-        'target': 'arm64-efi',
-    }
-}
+from vmdebootstrap.constants import arch_table
 
 
 class Uefi(Base):
@@ -105,28 +76,28 @@ class Uefi(Base):
             os.unlink(efi_output)
         os.rename(efi_input, efi_output)
 
-    def configure_efi(self):
+    def configure_efi(self, rootdir):
         """
         Copy the bootloader file from the package into the location
         so needs to be after grub and kernel already installed.
         """
         self.message('Configuring EFI')
-        mount_wrapper()
+        mount_wrapper(rootdir)
         efi_removable = str(arch_table[self.settings['arch']]['removable'])
         efi_install = str(arch_table[self.settings['arch']]['install'])
         self.message('Installing UEFI support binary')
         self.copy_efi_binary(efi_removable, efi_install)
-        umount_wrapper()
+        umount_wrapper(rootdir)
 
-    def configure_extra_efi(self):
+    def configure_extra_efi(self, rootdir):
         extra = str(arch_table[self.settings['arch']]['extra'])
         if extra:
-            mount_wrapper()
+            mount_wrapper(rootdir)
             efi_removable = str(arch_table[extra]['removable'])
             efi_install = str(arch_table[extra]['install'])
             self.message('Copying UEFI support binary for %s' % extra)
             self.copy_efi_binary(efi_removable, efi_install)
-            umount_wrapper()
+            umount_wrapper(rootdir)
 
     def partition_esp(self):
         espsize = self.settings['esp-size'] / (1024 * 1024)