feat: add programs to benchmark server chunk storage

8 files changed, 181 insertions, 0 deletions

diff --git a/benchmark.sh b/benchmark.sh
new file mode 100755
index 0000000..cf7491a
--- /dev/null
+++ b/benchmark.sh
@@ -0,0 +1,25 @@
+#!/bin/bash
+
+set -euo pipefail
+
+chunkdir="$1"
+bin="$2"
+
+cleanup()
+{
+    echo "emptying $chunkdir" 1>&2
+    find "$chunkdir" -mindepth 1 -delete
+}
+
+cleanup
+
+echo "running benchmarks for various sizes"
+for n in 1 10 100 1000 10000 100000 1000000
+do
+    echo "size $n" 1>&2
+    for prog in benchmark-null benchmark-index benchmark-store benchmark-indexedstore
+    do
+	/usr/bin/time --format "$prog $n %e" "$bin/$prog" "$chunkdir" "$n" 2>&1
+	cleanup
+    done
+done | awk '{ printf "%-30s %10s %10s\n", $1, $2, $3 }'
diff --git a/src/benchmark.rs b/src/benchmark.rs
new file mode 100644
index 0000000..b313868
--- /dev/null
+++ b/src/benchmark.rs
@@ -0,0 +1,32 @@
+use crate::chunk::DataChunk;
+use crate::chunkid::ChunkId;
+use crate::chunkmeta::ChunkMeta;
+
+// Generate a desired number of empty data chunks with id and metadata.
+pub struct ChunkGenerator {
+    goal: u32,
+    next: u32,
+}
+
+impl ChunkGenerator {
+    pub fn new(goal: u32) -> Self {
+        Self { goal, next: 0 }
+    }
+}
+
+impl Iterator for ChunkGenerator {
+    type Item = (ChunkId, String, ChunkMeta, DataChunk);
+
+    fn next(&mut self) -> Option<Self::Item> {
+        if self.next >= self.goal {
+            None
+        } else {
+            let id = ChunkId::new();
+            let checksum = id.sha256();
+            let meta = ChunkMeta::new(&checksum);
+            let chunk = DataChunk::new(vec![]);
+            self.next += 1;
+            Some((id, checksum, meta, chunk))
+        }
+    }
+}
diff --git a/src/bin/benchmark-index.rs b/src/bin/benchmark-index.rs
new file mode 100644
index 0000000..5008660
--- /dev/null
+++ b/src/bin/benchmark-index.rs
@@ -0,0 +1,33 @@
+use obnam::benchmark::ChunkGenerator;
+use obnam::index::Index;
+use std::path::PathBuf;
+use structopt::StructOpt;
+
+#[derive(Debug, StructOpt)]
+#[structopt(
+    name = "benchmark-index",
+    about = "Benhcmark the store index in memory"
+)]
+struct Opt {
+    // We don't use this, but we accept it for command line
+    // compatibility with other benchmark programs.
+    #[structopt(parse(from_os_str))]
+    chunks: PathBuf,
+
+    #[structopt()]
+    num: u32,
+}
+
+fn main() -> anyhow::Result<()> {
+    pretty_env_logger::init();
+
+    let opt = Opt::from_args();
+    let gen = ChunkGenerator::new(opt.num);
+
+    let mut index = Index::default();
+    for (id, checksum, _, _) in gen {
+        index.insert(id, "sha25", &checksum);
+    }
+
+    Ok(())
+}
diff --git a/src/bin/benchmark-indexedstore.rs b/src/bin/benchmark-indexedstore.rs
new file mode 100644
index 0000000..a4191ac
--- /dev/null
+++ b/src/bin/benchmark-indexedstore.rs
@@ -0,0 +1,28 @@
+use obnam::benchmark::ChunkGenerator;
+use obnam::indexedstore::IndexedStore;
+use std::path::PathBuf;
+use structopt::StructOpt;
+
+#[derive(Debug, StructOpt)]
+#[structopt(name = "benchmark-store", about = "Benhcmark the store without HTTP")]
+struct Opt {
+    #[structopt(parse(from_os_str))]
+    chunks: PathBuf,
+
+    #[structopt()]
+    num: u32,
+}
+
+fn main() -> anyhow::Result<()> {
+    pretty_env_logger::init();
+
+    let opt = Opt::from_args();
+    let gen = ChunkGenerator::new(opt.num);
+
+    let mut store = IndexedStore::new(&opt.chunks);
+    for (_, _, meta, chunk) in gen {
+        store.save(&meta, &chunk)?;
+    }
+
+    Ok(())
+}
diff --git a/src/bin/benchmark-null.rs b/src/bin/benchmark-null.rs
new file mode 100644
index 0000000..6df8ca1
--- /dev/null
+++ b/src/bin/benchmark-null.rs
@@ -0,0 +1,29 @@
+use obnam::benchmark::ChunkGenerator;
+use std::path::PathBuf;
+use structopt::StructOpt;
+
+#[derive(Debug, StructOpt)]
+#[structopt(
+    name = "benchmark-index",
+    about = "Benhcmark the store index in memory"
+)]
+struct Opt {
+    // We don't use this, but we accept it for command line
+    // compatibility with other benchmark programs.
+    #[structopt(parse(from_os_str))]
+    chunks: PathBuf,
+
+    #[structopt()]
+    num: u32,
+}
+
+fn main() -> anyhow::Result<()> {
+    pretty_env_logger::init();
+
+    let opt = Opt::from_args();
+    let gen = ChunkGenerator::new(opt.num);
+
+    for (_, _, _, _) in gen {}
+
+    Ok(())
+}
diff --git a/src/bin/benchmark-store.rs b/src/bin/benchmark-store.rs
new file mode 100644
index 0000000..f7c82b1
--- /dev/null
+++ b/src/bin/benchmark-store.rs
@@ -0,0 +1,28 @@
+use obnam::benchmark::ChunkGenerator;
+use obnam::store::Store;
+use std::path::PathBuf;
+use structopt::StructOpt;
+
+#[derive(Debug, StructOpt)]
+#[structopt(name = "benchmark-store", about = "Benhcmark the store without HTTP")]
+struct Opt {
+    #[structopt(parse(from_os_str))]
+    chunks: PathBuf,
+
+    #[structopt()]
+    num: u32,
+}
+
+fn main() -> anyhow::Result<()> {
+    pretty_env_logger::init();
+
+    let opt = Opt::from_args();
+    let gen = ChunkGenerator::new(opt.num);
+
+    let store = Store::new(&opt.chunks);
+    for (id, _, meta, chunk) in gen {
+        store.save(&id, &meta, &chunk)?;
+    }
+
+    Ok(())
+}
diff --git a/src/chunkid.rs b/src/chunkid.rs
index 781e497..2a38775 100644
--- a/src/chunkid.rs
+++ b/src/chunkid.rs
@@ -1,3 +1,4 @@
+use crate::checksummer::sha256;
 use rusqlite::types::ToSqlOutput;
 use rusqlite::ToSql;
 use serde::{Deserialize, Serialize};
@@ -34,6 +35,10 @@ impl ChunkId {
             id: Uuid::new_v4().to_string(),
         }
     }
+
+    pub fn sha256(&self) -> String {
+        sha256(self.id.as_bytes())
+    }
 }
 
 impl ToSql for ChunkId {
diff --git a/src/lib.rs b/src/lib.rs
index bdc790f..995cdeb 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -1,3 +1,4 @@
+pub mod benchmark;
 pub mod checksummer;
 pub mod chunk;
 pub mod chunker;