summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorLars Wirzenius <liw@liw.fi>2021-12-03 17:34:46 +0000
committerLars Wirzenius <liw@liw.fi>2021-12-03 17:34:46 +0000
commit6c61d025c8cdace3c985ab8733a4e115191e35a8 (patch)
tree18d8d19ef5d57d63df713785a480bf34e0666b9e
parente94abe763dd4a1bf4e1b84269c054a4fb56329b0 (diff)
parent6fd336eda6d012136b79ee4f1c37463d43822b96 (diff)
downloadobnam2-6c61d025c8cdace3c985ab8733a4e115191e35a8.tar.gz
Merge branch 'bench-manyfiles' into 'main'
perf: add benchmark script for the "many empty files" scenario Closes #158 See merge request obnam/obnam!195
-rwxr-xr-xbench-manyfiles.sh58
-rw-r--r--manyfiles.py39
2 files changed, 97 insertions, 0 deletions
diff --git a/bench-manyfiles.sh b/bench-manyfiles.sh
new file mode 100755
index 0000000..39e6c2e
--- /dev/null
+++ b/bench-manyfiles.sh
@@ -0,0 +1,58 @@
+#!/bin/bash
+#
+# Run a simple benchmark of an incremental backup of many empty files,
+# when the live data hasn't changed.
+#
+# Edit the if-statement towards the end to get a flamegraph to see
+# where time is actually spent.
+#
+# This is very simplistic and could do with a lot of improvement. But
+# it's a start.
+
+set -euo pipefail
+
+N=100000
+
+TMP="$(mktemp -d)"
+trap 'rm -rf "$TMP"' EXIT
+
+chunks="$TMP/chunks"
+live="$TMP/live"
+
+mkdir "$chunks"
+mkdir "$live"
+python3 manyfiles.py "$live" "$N"
+
+cat <<EOF >"$TMP/server.yaml"
+address: localhost:8888
+chunks: $chunks
+tls_key: test.key
+tls_cert: test.pem
+EOF
+
+cat <<EOF >"$TMP/client.yaml"
+server_url: https://localhost:8888
+verify_tls_cert: false
+roots:
+ - $live
+log: $TMP/client.log
+EOF
+
+cargo build -q --release --all-targets
+
+OBNAM_SERVER_LOG=error cargo run -q --release --bin obnam-server -- "$TMP/server.yaml" >/dev/null &
+pid="$!"
+
+cargo run -q --release --bin obnam -- --config "$TMP/client.yaml" init --insecure-passphrase=hunter2
+
+# Initial backup.
+cargo run -q --release --bin obnam -- --config "$TMP/client.yaml" backup >/dev/null
+
+# Incremental backup.
+if true; then
+ /usr/bin/time --format=%e cargo run -q --release --bin obnam -- --config "$TMP/client.yaml" backup >/dev/null
+else
+ cargo flamegraph --bin obnam -o obnam.svg -- --config "$TMP/client.yaml" backup >/dev/null
+fi
+
+kill "$pid"
diff --git a/manyfiles.py b/manyfiles.py
new file mode 100644
index 0000000..a45d3cd
--- /dev/null
+++ b/manyfiles.py
@@ -0,0 +1,39 @@
+#!/usr/bin/python3
+#
+# Create the desired number of empty files in a directory. A thousand files per
+# subdirectory.
+
+import os
+import sys
+
+
+def subdir(dirname, dirno):
+ pathname = os.path.join(dirname, str(dirno))
+ os.mkdir(pathname)
+ return pathname
+
+
+def create(filename):
+ open(filename, "w").close()
+
+
+DIRFILES = 1000
+
+dirname = sys.argv[1]
+n = int(sys.argv[2])
+
+dirno = 0
+subdirpath = subdir(dirname, dirno)
+fileno = 0
+thisdir = 0
+
+while fileno < n:
+ filename = os.path.join(subdirpath, str(thisdir))
+ create(filename)
+
+ fileno += 1
+ thisdir += 1
+ if thisdir >= DIRFILES:
+ dirno += 1
+ subdirpath = subdir(dirname, dirno)
+ thisdir = 0