From af2c79d1963c49402d5b47e916ffb6b0210c79d1 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Wed, 25 Nov 2020 08:11:57 +0200 Subject: feat: add programs to benchmark server chunk storage --- benchmark.sh | 25 +++++++++++++++++++++++++ src/benchmark.rs | 32 ++++++++++++++++++++++++++++++++ src/bin/benchmark-index.rs | 33 +++++++++++++++++++++++++++++++++ src/bin/benchmark-indexedstore.rs | 28 ++++++++++++++++++++++++++++ src/bin/benchmark-null.rs | 29 +++++++++++++++++++++++++++++ src/bin/benchmark-store.rs | 28 ++++++++++++++++++++++++++++ src/chunkid.rs | 5 +++++ src/lib.rs | 1 + 8 files changed, 181 insertions(+) create mode 100755 benchmark.sh create mode 100644 src/benchmark.rs create mode 100644 src/bin/benchmark-index.rs create mode 100644 src/bin/benchmark-indexedstore.rs create mode 100644 src/bin/benchmark-null.rs create mode 100644 src/bin/benchmark-store.rs diff --git a/benchmark.sh b/benchmark.sh new file mode 100755 index 0000000..cf7491a --- /dev/null +++ b/benchmark.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +set -euo pipefail + +chunkdir="$1" +bin="$2" + +cleanup() +{ + echo "emptying $chunkdir" 1>&2 + find "$chunkdir" -mindepth 1 -delete +} + +cleanup + +echo "running benchmarks for various sizes" +for n in 1 10 100 1000 10000 100000 1000000 +do + echo "size $n" 1>&2 + for prog in benchmark-null benchmark-index benchmark-store benchmark-indexedstore + do + /usr/bin/time --format "$prog $n %e" "$bin/$prog" "$chunkdir" "$n" 2>&1 + cleanup + done +done | awk '{ printf "%-30s %10s %10s\n", $1, $2, $3 }' diff --git a/src/benchmark.rs b/src/benchmark.rs new file mode 100644 index 0000000..b313868 --- /dev/null +++ b/src/benchmark.rs @@ -0,0 +1,32 @@ +use crate::chunk::DataChunk; +use crate::chunkid::ChunkId; +use crate::chunkmeta::ChunkMeta; + +// Generate a desired number of empty data chunks with id and metadata. +pub struct ChunkGenerator { + goal: u32, + next: u32, +} + +impl ChunkGenerator { + pub fn new(goal: u32) -> Self { + Self { goal, next: 0 } + } +} + +impl Iterator for ChunkGenerator { + type Item = (ChunkId, String, ChunkMeta, DataChunk); + + fn next(&mut self) -> Option { + if self.next >= self.goal { + None + } else { + let id = ChunkId::new(); + let checksum = id.sha256(); + let meta = ChunkMeta::new(&checksum); + let chunk = DataChunk::new(vec![]); + self.next += 1; + Some((id, checksum, meta, chunk)) + } + } +} diff --git a/src/bin/benchmark-index.rs b/src/bin/benchmark-index.rs new file mode 100644 index 0000000..5008660 --- /dev/null +++ b/src/bin/benchmark-index.rs @@ -0,0 +1,33 @@ +use obnam::benchmark::ChunkGenerator; +use obnam::index::Index; +use std::path::PathBuf; +use structopt::StructOpt; + +#[derive(Debug, StructOpt)] +#[structopt( + name = "benchmark-index", + about = "Benhcmark the store index in memory" +)] +struct Opt { + // We don't use this, but we accept it for command line + // compatibility with other benchmark programs. + #[structopt(parse(from_os_str))] + chunks: PathBuf, + + #[structopt()] + num: u32, +} + +fn main() -> anyhow::Result<()> { + pretty_env_logger::init(); + + let opt = Opt::from_args(); + let gen = ChunkGenerator::new(opt.num); + + let mut index = Index::default(); + for (id, checksum, _, _) in gen { + index.insert(id, "sha25", &checksum); + } + + Ok(()) +} diff --git a/src/bin/benchmark-indexedstore.rs b/src/bin/benchmark-indexedstore.rs new file mode 100644 index 0000000..a4191ac --- /dev/null +++ b/src/bin/benchmark-indexedstore.rs @@ -0,0 +1,28 @@ +use obnam::benchmark::ChunkGenerator; +use obnam::indexedstore::IndexedStore; +use std::path::PathBuf; +use structopt::StructOpt; + +#[derive(Debug, StructOpt)] +#[structopt(name = "benchmark-store", about = "Benhcmark the store without HTTP")] +struct Opt { + #[structopt(parse(from_os_str))] + chunks: PathBuf, + + #[structopt()] + num: u32, +} + +fn main() -> anyhow::Result<()> { + pretty_env_logger::init(); + + let opt = Opt::from_args(); + let gen = ChunkGenerator::new(opt.num); + + let mut store = IndexedStore::new(&opt.chunks); + for (_, _, meta, chunk) in gen { + store.save(&meta, &chunk)?; + } + + Ok(()) +} diff --git a/src/bin/benchmark-null.rs b/src/bin/benchmark-null.rs new file mode 100644 index 0000000..6df8ca1 --- /dev/null +++ b/src/bin/benchmark-null.rs @@ -0,0 +1,29 @@ +use obnam::benchmark::ChunkGenerator; +use std::path::PathBuf; +use structopt::StructOpt; + +#[derive(Debug, StructOpt)] +#[structopt( + name = "benchmark-index", + about = "Benhcmark the store index in memory" +)] +struct Opt { + // We don't use this, but we accept it for command line + // compatibility with other benchmark programs. + #[structopt(parse(from_os_str))] + chunks: PathBuf, + + #[structopt()] + num: u32, +} + +fn main() -> anyhow::Result<()> { + pretty_env_logger::init(); + + let opt = Opt::from_args(); + let gen = ChunkGenerator::new(opt.num); + + for (_, _, _, _) in gen {} + + Ok(()) +} diff --git a/src/bin/benchmark-store.rs b/src/bin/benchmark-store.rs new file mode 100644 index 0000000..f7c82b1 --- /dev/null +++ b/src/bin/benchmark-store.rs @@ -0,0 +1,28 @@ +use obnam::benchmark::ChunkGenerator; +use obnam::store::Store; +use std::path::PathBuf; +use structopt::StructOpt; + +#[derive(Debug, StructOpt)] +#[structopt(name = "benchmark-store", about = "Benhcmark the store without HTTP")] +struct Opt { + #[structopt(parse(from_os_str))] + chunks: PathBuf, + + #[structopt()] + num: u32, +} + +fn main() -> anyhow::Result<()> { + pretty_env_logger::init(); + + let opt = Opt::from_args(); + let gen = ChunkGenerator::new(opt.num); + + let store = Store::new(&opt.chunks); + for (id, _, meta, chunk) in gen { + store.save(&id, &meta, &chunk)?; + } + + Ok(()) +} diff --git a/src/chunkid.rs b/src/chunkid.rs index 781e497..2a38775 100644 --- a/src/chunkid.rs +++ b/src/chunkid.rs @@ -1,3 +1,4 @@ +use crate::checksummer::sha256; use rusqlite::types::ToSqlOutput; use rusqlite::ToSql; use serde::{Deserialize, Serialize}; @@ -34,6 +35,10 @@ impl ChunkId { id: Uuid::new_v4().to_string(), } } + + pub fn sha256(&self) -> String { + sha256(self.id.as_bytes()) + } } impl ToSql for ChunkId { diff --git a/src/lib.rs b/src/lib.rs index bdc790f..995cdeb 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,3 +1,4 @@ +pub mod benchmark; pub mod checksummer; pub mod chunk; pub mod chunker; -- cgit v1.2.1