From f097c61b1c1435d4849a33930cb75332dc7158dc Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Fri, 25 Dec 2015 13:06:49 +0100 Subject: Rewrite obbench, adding yarns and Debian packaging --- yarns/000.yarn | 6 ++++ yarns/100-intro.yarn | 26 +++++++++++++++++ yarns/200-config.yarn | 74 +++++++++++++++++++++++++++++++++++++++++++++++ yarns/300-running.yarn | 70 ++++++++++++++++++++++++++++++++++++++++++++ yarns/900-implements.yarn | 58 +++++++++++++++++++++++++++++++++++++ yarns/Makefile | 30 +++++++++++++++++++ 6 files changed, 264 insertions(+) create mode 100644 yarns/000.yarn create mode 100644 yarns/100-intro.yarn create mode 100644 yarns/200-config.yarn create mode 100644 yarns/300-running.yarn create mode 100644 yarns/900-implements.yarn create mode 100644 yarns/Makefile (limited to 'yarns') diff --git a/yarns/000.yarn b/yarns/000.yarn new file mode 100644 index 0000000..fd1b150 --- /dev/null +++ b/yarns/000.yarn @@ -0,0 +1,6 @@ +--- +title: Obnam benchmarking tool +author: Lars Wirzenius +date: in-development version from git +... + diff --git a/yarns/100-intro.yarn b/yarns/100-intro.yarn new file mode 100644 index 0000000..b59b142 --- /dev/null +++ b/yarns/100-intro.yarn @@ -0,0 +1,26 @@ +# Introduction + +This is the manual for `obbench`, a tool for benchmarking the +[Obnam][] backup program. The obbench program is at least the fifth +generation of the tool: the author seems to have a big difficulty +trying to figure out how benchmarking should be done. + +[Obnam]: http://obnam.org/ + +The obbench approach is that its operation is defined in a benchmark +specification file, rather than code. The code provides some primitive +operations, which can be combined in various ways in the specification +file. The goal is that new benchmarks can be specified easily, without +changing obbench itself. + +Additionally, obbench measures various aspects of running Obnam and +the resulting backup repository, and generates a static website to +report the results. + +## Installation + +Obbench can be run from its git source tree (`./obbench`), without any +build steps required. Additionally, the author will provide Debian +packages on [code.liw.fi][]. + +[code.liw.fi]: http://liw.fi/code/ diff --git a/yarns/200-config.yarn b/yarns/200-config.yarn new file mode 100644 index 0000000..5202b2a --- /dev/null +++ b/yarns/200-config.yarn @@ -0,0 +1,74 @@ +# Configuration + +Obbench configuration is split into two: a benchmark specification +file in YAML, and a user- and site-specific configuration. The former +specifies the benchmarks to run, and the latter where state and +results should be put on the local machine. + + +## Benchmark specification YAML file + +The benchmark specification file uses [YAML][] as the syntax. The top +level object is an "associative array", corresponding to a Perl +hashmap, a Python dict, or more generally a set of key/value pairs. + +[YAML]: https://en.wikipedia.org/wiki/YAML + +An example: + + EXAMPLE benchmark specification file + git: /home/liw/obnam/obnam + obnam_config: + repository-format: green-albatross + benchmarks: + - name: many_files + description: backup many small files + steps: + - live: genbackupdata . --create=10 --file-size=1 + obnam: backup + reference: 31 + - obnam: restore + reference: 32 + - obnam: backup + +The top level keys are: + +key |type |description +--------------|-------------|------ +`git` |URL |location of Obnam git repository +`obnam_config`|dict |Obnam configuration variables +`benchmarks` |list of dicts|actual benchmark specifications + +There are no default values, so for example the git URL to the Obnam +repository needs to specified every time. + +The Obnam configuration variables are as specified by Obnam itself. +Any configuration variables may be used. Note that obbench itself +provides the following settings for Obnam: `quiet`, `repository`, +`root`, and `log`. The benchmark specification should probably not +override those. + +The benchmark have the following key/value pairs. + +key |type |description +-------------|-------------|----------- +`name` |symbol |name of the benchmark +`description`|text |description of the benchmark +`steps` |list of dicts|list of steps in the benchmark + +The steps can use the following keys: + +key |type |description +-------|----------|----------- +`live` |shell |modify live data for step +`obnam`|subcommand|Obnam subcommand to run + +Every step optionally modifies the live data to be backed up. This is +done by running an arbitrary shell command. Additionally, an Obnam +subcommand can be run on the live data. Currently, the supported +subcommands are `backup` and `restore`. Arbitrary Obnam subcommands +are not supported. + +## Local configuration + +FIXME. This needs to be implemented and documented. diff --git a/yarns/300-running.yarn b/yarns/300-running.yarn new file mode 100644 index 0000000..c196280 --- /dev/null +++ b/yarns/300-running.yarn @@ -0,0 +1,70 @@ +# Running obbench + +To run obbench we need to set up a benchmark specification. The +examples below use the [yarn][] syntax, as they also form an automated +test suite for obbench. + +[yarn]: http://liw.fi/cmdtest/ + + SCENARIO running obbench + +For this example, we use a very simple benchmark specification. Note +that due to yarn syntax limitations, we use the JSON variant of YAML. + + GIVEN a benchmark specification file simple.yaml containing + ... { + ... git: "/home/liw/code/obnam/obnam", + ... description: "Sample benchmark description.\n\n**There you go.**", + ... benchmarks: [ + ... { + ... name: silly, + ... description: "It's a silly benchmark", + ... steps: [ + ... { + ... live: "genbackupdata --create=1k .", + ... obnam: backup + ... }, + ... { + ... obnam: restore, + ... } + ... ] + ... }, + ... { + ... name: moresilly, + ... description: "It's **another** silly benchmark!", + ... steps: [ + ... { + ... live: "genbackupdata --create=1k .", + ... obnam: backup + ... }, + ... { + ... obnam: backup + ... }, + ... ] + ... } + ... ] + ... } + +We also create a local configuration, also using YAML's JSON syntax: + + GIVEN an obbench configuration file local.yaml containing + ... { + ... config: { + ... state: "statedir" + ... } + ... } + +We then run obbench for the first time, for several commits. We run it +twice for the tip of master, so that we know obbench handles running +it twice for the same commit (e.g., because of environmental changes). + + WHEN I run obbench --config local.yaml simple.yaml + ... master~1 master master + +We now have some results. + + THEN directory statedir exists + AND directory statedir/git/.git exists + AND files matching statedir/results/*.yaml exist + AND file statedir/html/index.html exists + AND file statedir/html/obbench.css exists diff --git a/yarns/900-implements.yarn b/yarns/900-implements.yarn new file mode 100644 index 0000000..a0004cf --- /dev/null +++ b/yarns/900-implements.yarn @@ -0,0 +1,58 @@ +# Scenario step implementations + +This chapter contains implementations of the scenario steps, so that +this manual may be used as an automated test suite for obbench. See +[yarn][] documentation for understanding this. + +We use Python to implement the steps. This requires at least +version 0.19 of yarn. + +## Create benchmark specification file + + IMPLEMENTS GIVEN a benchmark specification file (\S+) containing (.*) + import os + filename = os.environ['MATCH_1'] + config_text = os.environ['MATCH_2'] + with open(filename, 'w') as f: + f.write(config_text) + +## Create a local configuration file + + IMPLEMENTS GIVEN an obbench configuration file (\S+) containing (.*) + import os + filename = os.environ['MATCH_1'] + config_text = os.environ['MATCH_2'] + with open(filename, 'w') as f: + f.write(config_text) + +## Run obbench, with arguments + + IMPLEMENTS WHEN I run obbench (.*) + import os + import cliapp + arg_string = os.environ['MATCH_1'] + args = arg_string.split() + srcdir = os.environ['SRCDIR'] + obbench = os.path.join(srcdir, 'obbench') + cliapp.runcmd([obbench] + args, stdout=None, stderr=None) + +### Check directory existence + + IMPLEMENTS THEN directory (\S+) exists + import os + dirname = os.environ['MATCH_1'] + assert os.path.isdir(dirname), "dir {} doesn't exist".format(dirname) + +### Check file existence + + IMPLEMENTS THEN file (\S+) exists + import os + filename = os.environ['MATCH_1'] + assert os.path.isfile(filename), "file {} doesn't exist".format(filename) + +### Check glob matching + + IMPLEMENTS THEN files matching (\S+) exist + import glob, os + pattern = os.environ['MATCH_1'] + assert glob.glob(pattern) != [], "glob {} doesn't match".format(pattern) diff --git a/yarns/Makefile b/yarns/Makefile new file mode 100644 index 0000000..4a8cfe1 --- /dev/null +++ b/yarns/Makefile @@ -0,0 +1,30 @@ +# Copyright 2013-2015 Lars Wirzenius +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# =*= License: GPL-3+ =*= + + +yarns = $(shell ls *.yarn) +css = ../obbenchlib/obbench.css + +all: yarns.pdf yarns.html + +yarns.pdf: $(yarns) Makefile + pandoc --chapters --toc -o yarns.pdf $(yarns) + +yarns.html: $(yarns) Makefile $(css) + pandoc -H $(css) --smart --toc --chapters --number-sections \ + -V geometry:lettersize \ + --standalone --self-contained -o yarns.html $(yarns) -- cgit v1.2.1