From 1f8646b2d291d6666ae86dcd496b492f326c41ad Mon Sep 17 00:00:00 2001
From: Lars Wirzenius <liw@liw.fi>
Date: Fri, 3 Dec 2021 18:32:50 +0200
Subject: doc: start documenting obnam-benchmark and its requirements

Sponsored-by: author
---
 obnam-benchmark.md     | 154 +++++++++++++++++++++++++++++++++++++++++++++++++
 subplot/benchmark.yaml |  22 +++++++
 2 files changed, 176 insertions(+)
 create mode 100644 obnam-benchmark.md
 create mode 100644 subplot/benchmark.yaml

diff --git a/obnam-benchmark.md b/obnam-benchmark.md
new file mode 100644
index 0000000..1913206
--- /dev/null
+++ b/obnam-benchmark.md
@@ -0,0 +1,154 @@
+---
+title: "`obnam-benchmark`---tool to run benchmarks"
+author: "The Obnam project"
+documentclass: report
+bindings:
+- subplot/benchmark.yaml
+- lib/files.yaml
+- lib/runcmd.yaml
+impls:
+  rust: []
+...
+
+
+# Introduction
+
+Obnam is a backup program. It aims to have good performance. To
+achieve that, we run benchmarks, and we automate that. To allow
+flexibility in what benchmark scenarios we have, we have a tool that
+reads benchmark specifications from a file and runs them. This
+document is the description of that tool, and its acceptance criteria.
+
+# Overview
+
+The `obnam-benchmark` tool runs a set of specified benchmarks for one
+version of Obnam. It works as follows:
+
+1. Read the specification file.
+2. Build the specified Obnam version.
+3. Start the Obnam server with an empty chunk directory.
+4. Run each specified benchmark, and take measurements.
+5. Stop the Obnam server. Clean up.
+6. Store measurements in a form that can be processed later.
+
+A benchmark specification file is in YAML format. It has a list of
+individual benchmarks. Each benchmark has a name, and a description of
+how to prepare the test data for each backup generation. Example:
+
+```{.yaml .numberLines}
+- benchmark: maildir
+  backups:
+  - change:
+    - create:
+        files: 100000
+        file_size: 0
+    - rename:
+        files: 1000
+  - change: []
+- benchmark: video-footage
+  backups:
+  - change:
+    - create:
+        files: 1000
+        filee_size: 1G
+  - change: []
+```
+
+The example above specifies two benchmarks: "maildir", and
+"video-footage". The names are not interpreted by `obnam-benchmark`,
+they are for human consumption only. The `backups` field specifies a
+list of changes to the test data; the benchmark tool runs a backup for
+each item in the list. Each change can create, copy, delete, or rename
+files, compared to the previous backup. The created files will have
+the specified size, and the content is randomly generated,
+non-repetitive, binary junk. The files will be stored in a directory
+tree avoiding very large numbers of files per directory.
+
+By specifying what kind of data is generated, and how it changes from
+backup to backup, and how many backups are made, the specification
+file can describe synthetic benchmarks for different use cases.
+
+# Acceptance criteria
+
+## Smoke test
+
+_Requirement: The benchmark tool must be able to handle a simplistic
+specification file and report results that are very fast._
+
+This requirement verifies that we can run `obnam-benchmark` in the
+simplest possible case: to run a benchmark with one backup, using test
+data consisting only of an empty directory. If this works, then we
+know that the executable program exists, can be invoked, at least some
+of its command line and specification code works. If this scenario
+doesn't work, then we can't expect anything else to work either.
+
+```scenario
+given the benchmark system is set up
+given file smoke.yaml
+when I run obnam-benchmark smoke.yaml
+then file smoke.json exists
+then benchmark "smoke" in smoke.json lasted less than 1000 milliseconds
+```
+
+```{#smoke.yaml .yaml .file .numberLines}
+- benchmark: smoke
+  backups:
+  - change: []
+```
+
+
+## Parses a specification file
+
+_Requirement: The benchmark tool can parse a specification file._
+
+We verify this by having the tool output a YAML specification file as
+JSON. Given the formats are different, this will check that the tool
+actually parses the file. We use an input file that contains aspects
+of benchmark specification that are not just YAML, with the assumption
+that the tool uses a YAML parser that works well, so that we only need
+to care about things that are specific to the benchmark tool.
+
+Specifically, we verify that the tool parses file sizes such as "1K"
+correctly.
+
+```scenario
+given an installed obnam-benchmark
+given file spec.yaml
+when I run obnam-benchmark dump-spec spec.yaml
+then stdout, as JSON, matches spec.jsona
+```
+
+```{#spec.yaml .yaml .file .numberLines}
+- benchmark: foo
+  backups:
+  - change:
+    - create:
+        files: 1j
+        size: 1K
+    - create:
+        files: 2
+        size: 2M
+    - create:
+        files: 3
+        size: 3G
+```
+
+```{#spec.json .yaml .file .numberLines}
+{
+  "benchmark": "foo",
+  "backups": [
+    [
+      {
+        "change": { "files": 1, "size": 1024 }
+      },
+      {
+        "change": { "files": 2, "size": 2048 }
+      },
+      {
+        "change": { "files": 3, "size": 3072 }
+      }
+    ]
+  ]
+}
+```
+
diff --git a/subplot/benchmark.yaml b/subplot/benchmark.yaml
new file mode 100644
index 0000000..4be4765
--- /dev/null
+++ b/subplot/benchmark.yaml
@@ -0,0 +1,22 @@
+- given: an installed obnam-benchmark
+  impl:
+    rust:
+      function: install_obnam_benchmark
+
+- given: the benchmark system is set up
+  impl:
+    rust:
+      function: set_up_benchmark
+      cleanup: stop_benchmark
+
+- then: benchmark "{name}" in {filename} lasted less than {ms:int} milliseconds
+  impl:
+    rust:
+      function: benchmark_duration_less_than
+
+- then: stdout, as JSON, matches {filename}
+  impl:
+    rust:
+      function: stdout_matches_json
+  type:
+    filename: file
-- 
cgit v1.2.1