From 087b65ebe2e81ed51f1796a999d06485bb84a712 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Sun, 9 Jan 2022 13:35:40 +0200 Subject: feat: replace options with a YAML specification file Files can be kept in git. Command line invocation are less convenient. Also: discard all the drives, allow extra LVs to be created, and refactor the code to be easier to follow. Sponsored-by: author --- v-i | 438 ++++++++++++++++++++++++++++++++++++++++++++------------------------ 1 file changed, 285 insertions(+), 153 deletions(-) (limited to 'v-i') diff --git a/v-i b/v-i index c1957fa..2f613a7 100755 --- a/v-i +++ b/v-i @@ -82,8 +82,8 @@ def is_luks(path): return p.returncode == 0 -def clean_up_disks(device): - log("clean up disks from old installs") +def clean_up_disks(drives): + log(f"clean system LVM2 and drives from everything: {drives}") mounts = mount_points() pvs = physical_volumes() @@ -115,118 +115,223 @@ def clean_up_disks(device): log(f"open LUKS volume {mapping} (just in case it is one)") run(["cryptsetup", "close", mapping], check=False, capture_output=True) - log(f"blkdiscard {device}") - run(["blkdiscard", "--force", device], check=True, capture_output=True) + for drive in drives: + log(f"blkdiscard {drive}") + run( + ["blkdiscard", "--force", drive], + check=True, + capture_output=True, + ) -def vmdb_spec(cryptsetup_password, playbook, playbook2, extra_vars): - device = "{{ image }}" - spec = { - "steps": [ - { - "mklabel": "gpt", - "device": device, - }, - { - "mkpart": "primary", - "device": device, - "start": "0%", - "end": "500M", - "tag": "efi", - }, - { - "mkpart": "primary", - "device": device, - "start": "500M", - "end": "1G", - "tag": "boot", - }, - ] +def mklabel(device): + return { + # Create a GPT partition table. It works well for all modern + # systems. + "mklabel": "gpt", + "device": device, + } + + +def mkpart(device, tag, start, end): + return { + "mkpart": "primary", + "device": device, + "start": start, + "end": end, + "tag": tag, + } + + +def mkfs(tag, fstype): + return { + "mkfs": fstype, + "partition": tag, + } + + +def cryptsetup(tag, password, name): + return { + "cryptsetup": "cryptsetup0", + "password": password, + "name": name, } + +def vgcreate(tag, drives): + return { + "vgcreate": tag, + "physical": drives, + } + + +def lvcreate(vg, name, size): + return { + "lvcreate": vg, + "name": name, + "size": size, + } + + +def mount(tag, dirname=None, mount_on=None): + d = {"mount": tag} + if dirname is not None: + d["dirname"] = dirname + if mount_on is not None: + d["mount-on"] = mount_on + return d + + +def unpack_rootfs(tag): + return { + "unpack-rootfs": tag, + } + + +def cache_rootfs(tag): + return { + "cache-rootfs": tag, + "unless": "rootfs_unpacked", + } + + +def debootstrap(tag): + return { + "debootstrap": "bullseye", + "mirror": "http://deb.debian.org/debian", + "target": tag, + "unless": "rootfs_unpacked", + } + + +def apt(tag, packages, unless=True): + d = { + "apt": "install", + "packages": [ + "console-setup", + "dosfstools", + "linux-image-amd64", + "locales-all", + "lvm2", + "psmisc", + "python3", + "ssh", + "strace", + ], + "tag": tag, + } + if unless: + d["unless"] = "rootfs_unpacked" + return d + + +def virtual_filesystems(tag): + return { + "virtual-filesystems": "root", + } + + +def fstab(tag): + return { + "fstab": tag, + } + + +def grub(device, root, efi): + return { + "grub": "uefi", + "tag": root, + "efi": efi, + "quiet": True, + "image-dev": device, + } + + +def vmdb_spec(system, ansible_vars): + device = "{{ image }}" + steps = [ + mklabel(device), + # Create a partition for UEFI to boot from. + mkpart(device, "efi", "0%", "500M"), + # Create a separate /boot partition, in case we want to have LUKS. + mkpart(device, "boot", "500M", "1G"), + # Format the EFI partition. This MUST be vfat. + mkfs("efi", "vfat"), + # Format /boot. This is conventionally ext2. This file system + # gets very little I/O, but it MUST be supported by GRUB, so + # ext2 seems like a nice, safe choice. + mkfs("boot", "ext2"), + ] + # Set up pv0 for lvm2, either encrypted or cleartext. - if cryptsetup_password: - spec["steps"].extend( + if system.luks: + steps.extend( [ - { - "mkpart": "primary", - "device": device, - "start": "1G", - "end": "100%", - "tag": "cryptsetup0", - }, - { - "cryptsetup": "cryptsetup0", - "password": cryptsetup_password, - "name": "pv0", - }, + mkpart(device, "cryptsetup0", "1G", "100%"), + cryptsetup("cryptsetup0", system.luks, "pv0"), ] ) + for (i, drive) in enumerate(system.extra_drives): + steps.append(cryptsetup(f"cryptsetuo{i+1}", system.luks, f"pv{i+1}")) else: - spec["steps"].extend( + steps.append(mkpart(device, "pv0", "1G", "100%")) + for (i, drive) in enumerate(system.extra_drives): + steps.extend( + [ + mklabel(drive), + mkpart(drive, f"pv{i+1}", "0%", "100%"), + ] + ) + + # Create file systems and install Debian. + steps.extend( + [ + # Create an LVM2 volume group using pv0, which we create + # earlier. At this point, if LUKS is used, pv0 is the unlocked, + # open, cleartext block device. + vgcreate( + "vg0", + [f"pv{i}" for i in range(len([system.drive] + system.extra_drives))], + ), + # Create a 20 gigabyte LV for the root file system. That's big + # enough for a desktop system. + lvcreate("vg0", "root", "20G"), + # format the root file system. This gets a fair bit of use, so + # ext4 seems like a safe choice. If you wanted another file + # system, sorry. + mkfs("root", "ext4"), + # Mount the root file system. + mount("root"), + # Mount /boot on top of the root file system. + mount("boot", dirname="/boot", mount_on="root"), + # Mount /boot/efi. + mount("efi", dirname="/boot/efi", mount_on="boot"), + ] + ) + + # Add any additional LVs. + for lv in system.extra_lvs: + steps.extend( [ - { - "mkpart": "primary", - "device": device, - "start": "1G", - "end": "100%", - "tag": "pv0", - }, + lvcreate("vg0", lv["name"], lv["size"]), + mkfs(lv["name"], "ext4"), ] ) - # Create file systems and install Debian. - spec["steps"].extend( + steps.extend( [ - { - "mkfs": "vfat", - "partition": "efi", - }, - { - "mkfs": "ext2", - "partition": "boot", - }, - { - "vgcreate": "vg0", - "physical": ["pv0"], - }, - { - "lvcreate": "vg0", - "name": "root", - "size": "10G", - }, - { - "mkfs": "ext4", - "partition": "root", - }, - { - "mount": "root", - }, - { - "mount": "boot", - "dirname": "/boot", - "mount-on": "root", - }, - { - "mount": "efi", - "dirname": "/boot/efi", - "mount-on": "boot", - }, - { - "unpack-rootfs": "root", - }, - { - "debootstrap": "bullseye", - "mirror": "http://deb.debian.org/debian", - "target": "root", - "unless": "rootfs_unpacked", - }, - { - "apt": "install", - "packages": [ + # If we have a cached version of the installed system, unpack + # it now. Otherwise do nothing. Note that if you make any + # changes to the steps marked "unless: rootfs_unpacked", you + # have to remember to manually remove the cache file. v-i or + # vmdb2 won't do that automatically for you. + unpack_rootfs("root"), + debootstrap("root"), + apt( + "root", + [ "console-setup", "dosfstools", - # "ifupdown", "linux-image-amd64", "locales-all", "lvm2", @@ -235,84 +340,112 @@ def vmdb_spec(cryptsetup_password, playbook, playbook2, extra_vars): "ssh", "strace", ], - "tag": "root", - "unless": "rootfs_unpacked", - }, - { - "cache-rootfs": "root", - "unless": "rootfs_unpacked", - }, - { - # This MUST be after the debootstrap step. - "virtual-filesystems": "root", - }, - { - "fstab": "root", - }, - { - # These MUST come after the fstab step so that they add the - # crypttab in the initramfs. - "apt": "install", - "packages": [ + ), + # If we didn't unpack an existing cache archive, make one now. + # Otherwise, skip this step. + cache_rootfs("root"), + # This MUST be after the debootstrap step. + virtual_filesystems("root"), + # Create /etc/fstab (and, if LUKS is used, /etc/crypttab). + fstab("root"), + # These MUST come after the fstab step so that they add the + # crypttab in the initramfs. We install them regardless of + # whether LUKS is used: they're harmless if LUKS isn't used. + apt( + "root", + [ "cryptsetup", "cryptsetup-initramfs", ], - "tag": "root", - }, - { - # This also MUST come outside the rootfs caching, as it install - # things outside the file systems. - "grub": "uefi", - "tag": "root", - "efi": "efi", - "quiet": True, - "image-dev": device, - }, + unless=False, + ), + # This also MUST come outside the rootfs caching, as it install + # things outside the file systems, and those won't be in the + # cache. + grub(device, "root", "efi"), ] ) # If playbooks have been specified, add ansible steps. - for p in (playbook, playbook2): + for p in ["std.yml"] + system.extra_playbooks: if p: - spec["steps"].append( - {"ansible": "root", "playbook": p, "extra_vars": extra_vars} - ) + steps.append({"ansible": "root", "playbook": p, "extra_vars": ansible_vars}) + + return {"steps": steps} + + +class SystemSpec: + def __init__(self, filename): + REQUIRED = "required" + self._obj = { + "hostname": REQUIRED, + "drive": REQUIRED, + "extra_drives": [], + "extra_lvs": [], + "extra_playbooks": [], + "ansible_vars": {}, + "luks": None, + } + with open(filename) as f: + obj = yaml.safe_load(f) + + # Check for unknown keys. + for key in obj: + if key not in self._obj: + sys.exit(f"spec has unknown key: {key}") + + # Check for missing required keys. + for key in self._obj: + if self._obj[key] == REQUIRED and key not in obj: + sys.exit(f"spec lacks required key {key}") + + # Check for types of values. + for key in self._obj: + if key in obj: + e = type(self._obj[key]) + a = type(obj[key]) + if a != e: + sys.exit(f"spec key {key} has unexpected type {a}, wanted {e}") - return spec + self._obj.update(obj) + + for key in self._obj: + setattr(self, key, self._obj[key]) + del self._obj + + def __repr__(self): + r = {key: getattr(self, key) for key in dir(self) if not key.startswith("_")} + return repr(r) def main(): p = argparse.ArgumentParser() p.add_argument("--verbose", action="store_true") + p.add_argument("--very-verbose", action="store_true") p.add_argument("--log", default="install.log") p.add_argument("--cache", default="cache.tar.gz") - p.add_argument("--playbook", default="std.yml") - p.add_argument("--playbook2") - p.add_argument("--vars") - p.add_argument("--luks") - p.add_argument("device") + p.add_argument("spec") args = p.parse_args() - if args.verbose: - global verbose - verbose = args.verbose + global verbose + verbose = args.verbose - extra_vars = {} - if args.vars: - with open(args.vars) as f: - extra_vars = yaml.safe_load(f) + system = SystemSpec(args.spec) + log(f"spec: {system!r}") - clean_up_disks(args.device) + clean_up_disks([system.drive] + system.extra_drives) - spec = vmdb_spec(args.luks, args.playbook, args.playbook2, extra_vars) + ansible_vars = dict(system.ansible_vars) + ansible_vars["hostname"] = system.hostname + vmdb = vmdb_spec(system, ansible_vars) tmp = tempfile.mkdtemp() specfile = os.path.join(tmp, "spec.yaml") - if args.verbose: - yaml.dump(spec, stream=sys.stdout, indent=4) + if args.very_verbose: + yaml.dump(vmdb, stream=sys.stdout, indent=4) with open(specfile, "w") as f: - yaml.dump(spec, stream=f, indent=4) + yaml.dump(vmdb, stream=f, indent=4) - log(f"run vmdb2 to install on {args.device}") + log(f"run vmdb2 to install on {system.drive}") env = dict(os.environ) env["ANSIBLE_STDOUT_CALLBACK"] = "yaml" env["ANSIBLE_NOCOWS"] = "1" @@ -322,12 +455,11 @@ def main(): "vmdb2", f"--rootfs-tarball={args.cache}", f"--log={args.log}", - f"--image={args.device}", + f"--image={system.drive}", specfile, ] if verbose: argv.append("--verbose") - run(argv, check=True, capture_output=True) log("cleanup") -- cgit v1.2.1