From 57d11bd2f4823387a0b1a421428d554c6608cf5e Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Fri, 5 Mar 2021 11:07:41 +0200 Subject: fix: generate deterministic chunk ids for benchmarks This allows us to generate the same sequence many times. --- src/benchmark.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/benchmark.rs b/src/benchmark.rs index b313868..b484aa1 100644 --- a/src/benchmark.rs +++ b/src/benchmark.rs @@ -21,7 +21,7 @@ impl Iterator for ChunkGenerator { if self.next >= self.goal { None } else { - let id = ChunkId::new(); + let id = ChunkId::from_str(&format!("{}", self.next)); let checksum = id.sha256(); let meta = ChunkMeta::new(&checksum); let chunk = DataChunk::new(vec![]); -- cgit v1.2.1 From 5e4bbf7131660a41fd9460225c763b9a6ea4bdce Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Fri, 5 Mar 2021 11:07:41 +0200 Subject: perf: allow benchmark-index to do lookups by checksum --- src/bin/benchmark-index.rs | 82 +++++++++++++++++++++++++++++++++++++--------- 1 file changed, 67 insertions(+), 15 deletions(-) diff --git a/src/bin/benchmark-index.rs b/src/bin/benchmark-index.rs index d49a6c3..6a1dc13 100644 --- a/src/bin/benchmark-index.rs +++ b/src/bin/benchmark-index.rs @@ -1,35 +1,87 @@ use obnam::benchmark::ChunkGenerator; -use obnam::chunkmeta::ChunkMeta; use obnam::index::Index; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use structopt::StructOpt; #[derive(Debug, StructOpt)] #[structopt( name = "benchmark-index", - about = "Benhcmark the store index in memory" + about = "Benhcmark the chunk store index without HTTP" )] -struct Opt { - // We don't use this, but we accept it for command line - // compatibility with other benchmark programs. - #[structopt(parse(from_os_str))] - chunks: PathBuf, - - #[structopt()] - num: u32, +enum Opt { + Create { + #[structopt(parse(from_os_str))] + chunks: PathBuf, + + #[structopt()] + num: u32, + }, + + Lookup { + #[structopt(parse(from_os_str))] + chunks: PathBuf, + + #[structopt()] + warmup_count: u32, + + #[structopt()] + hot_count: u32, + }, } fn main() -> anyhow::Result<()> { pretty_env_logger::init(); let opt = Opt::from_args(); - let gen = ChunkGenerator::new(opt.num); - let mut index = Index::new(".")?; - for (id, checksum, _, _) in gen { - let meta = ChunkMeta::new(&checksum); + match opt { + Opt::Create { chunks, num } => create(&chunks, num)?, + Opt::Lookup { + chunks, + warmup_count, + hot_count, + } => { + let mut index = Index::new(chunks)?; + warmup(&mut index, warmup_count)?; + hot(&mut index, hot_count)?; + } + } + + Ok(()) +} + +fn create(chunks: &Path, num: u32) -> anyhow::Result<()> { + let mut index = Index::new(chunks)?; + let gen = ChunkGenerator::new(num); + + for (id, _, meta, _) in gen { index.insert_meta(id, meta)?; } Ok(()) } + +fn warmup(index: &mut Index, num: u32) -> anyhow::Result<()> { + println!("warming up cache"); + lookup(index, num) +} + +fn hot(index: &mut Index, num: u32) -> anyhow::Result<()> { + println!("using hot cache"); + lookup(index, num) +} + +fn lookup(index: &mut Index, num: u32) -> anyhow::Result<()> { + let mut done = 0; + + loop { + let gen = ChunkGenerator::new(num); + for (_, _, meta, _) in gen { + index.find_by_sha256(&meta.sha256())?; + done += 1; + if done >= num { + return Ok(()); + } + } + } +} -- cgit v1.2.1 From ef3960bd60a571a7c4a5b50cfefa8d7ba29a521b Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Fri, 5 Mar 2021 11:07:41 +0200 Subject: perf: allow benchmark-indexedstore do lookups by checksum --- src/bin/benchmark-indexedstore.rs | 77 ++++++++++++++++++++++++++++++++++----- 1 file changed, 68 insertions(+), 9 deletions(-) diff --git a/src/bin/benchmark-indexedstore.rs b/src/bin/benchmark-indexedstore.rs index 3ee4c38..9783c3c 100644 --- a/src/bin/benchmark-indexedstore.rs +++ b/src/bin/benchmark-indexedstore.rs @@ -1,28 +1,87 @@ use obnam::benchmark::ChunkGenerator; use obnam::indexedstore::IndexedStore; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use structopt::StructOpt; #[derive(Debug, StructOpt)] -#[structopt(name = "benchmark-store", about = "Benhcmark the store without HTTP")] -struct Opt { - #[structopt(parse(from_os_str))] - chunks: PathBuf, +#[structopt( + name = "benchmark-indexedstore", + about = "Benhcmark the store without HTTP" +)] +enum Opt { + Create { + #[structopt(parse(from_os_str))] + chunks: PathBuf, - #[structopt()] - num: u32, + #[structopt()] + num: u32, + }, + + Lookup { + #[structopt(parse(from_os_str))] + chunks: PathBuf, + + #[structopt()] + warmup_count: u32, + + #[structopt()] + hot_count: u32, + }, } fn main() -> anyhow::Result<()> { pretty_env_logger::init(); let opt = Opt::from_args(); - let gen = ChunkGenerator::new(opt.num); - let mut store = IndexedStore::new(&opt.chunks)?; + match opt { + Opt::Create { chunks, num } => create(&chunks, num)?, + Opt::Lookup { + chunks, + warmup_count, + hot_count, + } => { + let mut index = IndexedStore::new(&chunks)?; + warmup(&mut index, warmup_count)?; + hot(&mut index, hot_count)?; + } + } + + Ok(()) +} + +fn create(chunks: &Path, num: u32) -> anyhow::Result<()> { + let mut store = IndexedStore::new(chunks)?; + let gen = ChunkGenerator::new(num); + for (_, _, meta, chunk) in gen { store.save(&meta, &chunk)?; } Ok(()) } + +fn warmup(index: &mut IndexedStore, num: u32) -> anyhow::Result<()> { + println!("warming up cache"); + lookup(index, num) +} + +fn hot(index: &mut IndexedStore, num: u32) -> anyhow::Result<()> { + println!("using hot cache"); + lookup(index, num) +} + +fn lookup(index: &mut IndexedStore, num: u32) -> anyhow::Result<()> { + let mut done = 0; + + loop { + let gen = ChunkGenerator::new(num); + for (_, _, meta, _) in gen { + index.find_by_sha256(&meta.sha256())?; + done += 1; + if done >= num { + return Ok(()); + } + } + } +} -- cgit v1.2.1