summaryrefslogtreecommitdiff
path: root/src/bin
diff options
context:
space:
mode:
Diffstat (limited to 'src/bin')
-rw-r--r--src/bin/benchmark-index.rs35
-rw-r--r--src/bin/benchmark-indexedstore.rs28
-rw-r--r--src/bin/benchmark-null.rs29
-rw-r--r--src/bin/benchmark-store.rs28
-rw-r--r--src/bin/obnam-server.rs148
-rw-r--r--src/bin/obnam.rs170
6 files changed, 145 insertions, 293 deletions
diff --git a/src/bin/benchmark-index.rs b/src/bin/benchmark-index.rs
deleted file mode 100644
index d49a6c3..0000000
--- a/src/bin/benchmark-index.rs
+++ /dev/null
@@ -1,35 +0,0 @@
-use obnam::benchmark::ChunkGenerator;
-use obnam::chunkmeta::ChunkMeta;
-use obnam::index::Index;
-use std::path::PathBuf;
-use structopt::StructOpt;
-
-#[derive(Debug, StructOpt)]
-#[structopt(
- name = "benchmark-index",
- about = "Benhcmark the store index in memory"
-)]
-struct Opt {
- // We don't use this, but we accept it for command line
- // compatibility with other benchmark programs.
- #[structopt(parse(from_os_str))]
- chunks: PathBuf,
-
- #[structopt()]
- num: u32,
-}
-
-fn main() -> anyhow::Result<()> {
- pretty_env_logger::init();
-
- let opt = Opt::from_args();
- let gen = ChunkGenerator::new(opt.num);
-
- let mut index = Index::new(".")?;
- for (id, checksum, _, _) in gen {
- let meta = ChunkMeta::new(&checksum);
- index.insert_meta(id, meta)?;
- }
-
- Ok(())
-}
diff --git a/src/bin/benchmark-indexedstore.rs b/src/bin/benchmark-indexedstore.rs
deleted file mode 100644
index 3ee4c38..0000000
--- a/src/bin/benchmark-indexedstore.rs
+++ /dev/null
@@ -1,28 +0,0 @@
-use obnam::benchmark::ChunkGenerator;
-use obnam::indexedstore::IndexedStore;
-use std::path::PathBuf;
-use structopt::StructOpt;
-
-#[derive(Debug, StructOpt)]
-#[structopt(name = "benchmark-store", about = "Benhcmark the store without HTTP")]
-struct Opt {
- #[structopt(parse(from_os_str))]
- chunks: PathBuf,
-
- #[structopt()]
- num: u32,
-}
-
-fn main() -> anyhow::Result<()> {
- pretty_env_logger::init();
-
- let opt = Opt::from_args();
- let gen = ChunkGenerator::new(opt.num);
-
- let mut store = IndexedStore::new(&opt.chunks)?;
- for (_, _, meta, chunk) in gen {
- store.save(&meta, &chunk)?;
- }
-
- Ok(())
-}
diff --git a/src/bin/benchmark-null.rs b/src/bin/benchmark-null.rs
deleted file mode 100644
index 6df8ca1..0000000
--- a/src/bin/benchmark-null.rs
+++ /dev/null
@@ -1,29 +0,0 @@
-use obnam::benchmark::ChunkGenerator;
-use std::path::PathBuf;
-use structopt::StructOpt;
-
-#[derive(Debug, StructOpt)]
-#[structopt(
- name = "benchmark-index",
- about = "Benhcmark the store index in memory"
-)]
-struct Opt {
- // We don't use this, but we accept it for command line
- // compatibility with other benchmark programs.
- #[structopt(parse(from_os_str))]
- chunks: PathBuf,
-
- #[structopt()]
- num: u32,
-}
-
-fn main() -> anyhow::Result<()> {
- pretty_env_logger::init();
-
- let opt = Opt::from_args();
- let gen = ChunkGenerator::new(opt.num);
-
- for (_, _, _, _) in gen {}
-
- Ok(())
-}
diff --git a/src/bin/benchmark-store.rs b/src/bin/benchmark-store.rs
deleted file mode 100644
index f7c82b1..0000000
--- a/src/bin/benchmark-store.rs
+++ /dev/null
@@ -1,28 +0,0 @@
-use obnam::benchmark::ChunkGenerator;
-use obnam::store::Store;
-use std::path::PathBuf;
-use structopt::StructOpt;
-
-#[derive(Debug, StructOpt)]
-#[structopt(name = "benchmark-store", about = "Benhcmark the store without HTTP")]
-struct Opt {
- #[structopt(parse(from_os_str))]
- chunks: PathBuf,
-
- #[structopt()]
- num: u32,
-}
-
-fn main() -> anyhow::Result<()> {
- pretty_env_logger::init();
-
- let opt = Opt::from_args();
- let gen = ChunkGenerator::new(opt.num);
-
- let store = Store::new(&opt.chunks);
- for (id, _, meta, chunk) in gen {
- store.save(&id, &meta, &chunk)?;
- }
-
- Ok(())
-}
diff --git a/src/bin/obnam-server.rs b/src/bin/obnam-server.rs
index 19f2e99..9b5a557 100644
--- a/src/bin/obnam-server.rs
+++ b/src/bin/obnam-server.rs
@@ -1,42 +1,43 @@
-use bytes::Bytes;
+use anyhow::Context;
+use clap::Parser;
use log::{debug, error, info};
-use obnam::chunk::DataChunk;
use obnam::chunkid::ChunkId;
use obnam::chunkmeta::ChunkMeta;
-use obnam::indexedstore::IndexedStore;
-use serde::{Deserialize, Serialize};
+use obnam::chunkstore::ChunkStore;
+use obnam::label::Label;
+use obnam::server::{ServerConfig, ServerConfigError};
+use serde::Serialize;
use std::collections::HashMap;
use std::default::Default;
use std::net::{SocketAddr, ToSocketAddrs};
use std::path::{Path, PathBuf};
use std::sync::Arc;
-use structopt::StructOpt;
use tokio::sync::Mutex;
use warp::http::StatusCode;
+use warp::hyper::body::Bytes;
use warp::Filter;
-#[derive(Debug, StructOpt)]
-#[structopt(name = "obnam2-server", about = "Backup server")]
+#[derive(Debug, Parser)]
+#[clap(name = "obnam2-server", about = "Backup server")]
struct Opt {
- #[structopt(parse(from_os_str))]
config: PathBuf,
}
#[tokio::main]
async fn main() -> anyhow::Result<()> {
- pretty_env_logger::init();
+ pretty_env_logger::init_custom_env("OBNAM_SERVER_LOG");
- let opt = Opt::from_args();
- let config = Config::read_config(&opt.config).unwrap();
+ let opt = Opt::parse();
+ let config = load_config(&opt.config)?;
let addresses: Vec<SocketAddr> = config.address.to_socket_addrs()?.collect();
if addresses.is_empty() {
error!("specified address is empty set: {:?}", addresses);
eprintln!("ERROR: server address is empty: {:?}", addresses);
- return Err(ConfigError::BadServerAddress.into());
+ return Err(ServerConfigError::BadServerAddress.into());
}
- let store = IndexedStore::new(&config.chunks)?;
+ let store = ChunkStore::local(&config.chunks)?;
let store = Arc::new(Mutex::new(store));
let store = warp::any().map(move || Arc::clone(&store));
@@ -45,32 +46,32 @@ async fn main() -> anyhow::Result<()> {
debug!("Configuration: {:#?}", config);
let create = warp::post()
+ .and(warp::path("v1"))
.and(warp::path("chunks"))
+ .and(warp::path::end())
.and(store.clone())
.and(warp::header("chunk-meta"))
.and(warp::filters::body::bytes())
.and_then(create_chunk);
let fetch = warp::get()
+ .and(warp::path("v1"))
.and(warp::path("chunks"))
.and(warp::path::param())
+ .and(warp::path::end())
.and(store.clone())
.and_then(fetch_chunk);
let search = warp::get()
+ .and(warp::path("v1"))
.and(warp::path("chunks"))
+ .and(warp::path::end())
.and(warp::query::<HashMap<String, String>>())
.and(store.clone())
.and_then(search_chunks);
- let delete = warp::delete()
- .and(warp::path("chunks"))
- .and(warp::path::param())
- .and(store.clone())
- .and_then(delete_chunk);
-
let log = warp::log("obnam");
- let webroot = create.or(fetch).or(search).or(delete).with(log);
+ let webroot = create.or(fetch).or(search).with(log);
debug!("starting warp");
warp::serve(webroot)
@@ -82,57 +83,22 @@ async fn main() -> anyhow::Result<()> {
Ok(())
}
-#[derive(Debug, Deserialize, Clone)]
-pub struct Config {
- pub chunks: PathBuf,
- pub address: String,
- pub tls_key: PathBuf,
- pub tls_cert: PathBuf,
-}
-
-#[derive(Debug, thiserror::Error)]
-enum ConfigError {
- #[error("Directory for chunks {0} does not exist")]
- ChunksDirNotFound(PathBuf),
-
- #[error("TLS certificate {0} does not exist")]
- TlsCertNotFound(PathBuf),
-
- #[error("TLS key {0} does not exist")]
- TlsKeyNotFound(PathBuf),
-
- #[error("server address can't be resolved")]
- BadServerAddress,
-}
-
-impl Config {
- pub fn read_config(filename: &Path) -> anyhow::Result<Config> {
- let config = std::fs::read_to_string(filename)?;
- let config: Config = serde_yaml::from_str(&config)?;
- config.check()?;
- Ok(config)
- }
-
- pub fn check(&self) -> anyhow::Result<()> {
- if !self.chunks.exists() {
- return Err(ConfigError::ChunksDirNotFound(self.chunks.clone()).into());
- }
- if !self.tls_cert.exists() {
- return Err(ConfigError::TlsCertNotFound(self.tls_cert.clone()).into());
- }
- if !self.tls_key.exists() {
- return Err(ConfigError::TlsKeyNotFound(self.tls_key.clone()).into());
- }
- Ok(())
- }
+fn load_config(filename: &Path) -> Result<ServerConfig, anyhow::Error> {
+ let config = ServerConfig::read_config(filename).with_context(|| {
+ format!(
+ "Couldn't read default configuration file {}",
+ filename.display()
+ )
+ })?;
+ Ok(config)
}
pub async fn create_chunk(
- store: Arc<Mutex<IndexedStore>>,
+ store: Arc<Mutex<ChunkStore>>,
meta: String,
data: Bytes,
) -> Result<impl warp::Reply, warp::Rejection> {
- let mut store = store.lock().await;
+ let store = store.lock().await;
let meta: ChunkMeta = match meta.parse() {
Ok(s) => s,
@@ -142,9 +108,7 @@ pub async fn create_chunk(
}
};
- let chunk = DataChunk::new(data.to_vec());
-
- let id = match store.save(&meta, &chunk) {
+ let id = match store.put(data.to_vec(), &meta).await {
Ok(id) => id,
Err(e) => {
error!("couldn't save: {}", e);
@@ -152,17 +116,17 @@ pub async fn create_chunk(
}
};
- info!("created chunk {}: {:?}", id, meta);
+ info!("created chunk {}", id);
Ok(ChunkResult::Created(id))
}
pub async fn fetch_chunk(
id: String,
- store: Arc<Mutex<IndexedStore>>,
+ store: Arc<Mutex<ChunkStore>>,
) -> Result<impl warp::Reply, warp::Rejection> {
let store = store.lock().await;
let id: ChunkId = id.parse().unwrap();
- match store.load(&id) {
+ match store.get(&id).await {
Ok((data, meta)) => {
info!("found chunk {}: {:?}", id, meta);
Ok(ChunkResult::Fetched(meta, data))
@@ -176,20 +140,23 @@ pub async fn fetch_chunk(
pub async fn search_chunks(
query: HashMap<String, String>,
- store: Arc<Mutex<IndexedStore>>,
+ store: Arc<Mutex<ChunkStore>>,
) -> Result<impl warp::Reply, warp::Rejection> {
let store = store.lock().await;
let mut query = query.iter();
let found = if let Some((key, value)) = query.next() {
- if query.next() != None {
+ if query.next().is_some() {
error!("search has more than one key to search for");
return Ok(ChunkResult::BadRequest);
}
- if key == "generation" && value == "true" {
- store.find_generations().expect("SQL lookup failed")
- } else if key == "sha256" {
- store.find_by_sha256(value).expect("SQL lookup failed")
+ if key == "label" {
+ let label = Label::deserialize(value).unwrap();
+ let label = ChunkMeta::new(&label);
+ store
+ .find_by_label(&label)
+ .await
+ .expect("SQL lookup failed")
} else {
error!("unknown search key {:?}", key);
return Ok(ChunkResult::BadRequest);
@@ -201,7 +168,7 @@ pub async fn search_chunks(
let mut hits = SearchHits::default();
for chunk_id in found {
- let meta = match store.load_meta(&chunk_id) {
+ let (_, meta) = match store.get(&chunk_id).await {
Ok(meta) => {
info!("search found chunk {}", chunk_id);
meta
@@ -240,30 +207,10 @@ impl SearchHits {
}
}
-pub async fn delete_chunk(
- id: String,
- store: Arc<Mutex<IndexedStore>>,
-) -> Result<impl warp::Reply, warp::Rejection> {
- let mut store = store.lock().await;
- let id: ChunkId = id.parse().unwrap();
-
- match store.remove(&id) {
- Ok(_) => {
- info!("chunk deleted: {}", id);
- Ok(ChunkResult::Deleted)
- }
- Err(e) => {
- error!("could not delete chunk {}: {:?}", id, e);
- Ok(ChunkResult::NotFound)
- }
- }
-}
-
enum ChunkResult {
Created(ChunkId),
- Fetched(ChunkMeta, DataChunk),
+ Fetched(ChunkMeta, Vec<u8>),
Found(SearchHits),
- Deleted,
NotFound,
BadRequest,
InternalServerError,
@@ -292,13 +239,12 @@ impl warp::Reply for ChunkResult {
);
into_response(
StatusCode::OK,
- chunk.data(),
+ &chunk,
"application/octet-stream",
Some(headers),
)
}
ChunkResult::Found(hits) => json_response(StatusCode::OK, hits.to_json(), None),
- ChunkResult::Deleted => status_response(StatusCode::OK),
ChunkResult::BadRequest => status_response(StatusCode::BAD_REQUEST),
ChunkResult::NotFound => status_response(StatusCode::NOT_FOUND),
ChunkResult::InternalServerError => status_response(StatusCode::INTERNAL_SERVER_ERROR),
diff --git a/src/bin/obnam.rs b/src/bin/obnam.rs
index e9f30ca..240960b 100644
--- a/src/bin/obnam.rs
+++ b/src/bin/obnam.rs
@@ -1,100 +1,126 @@
+use clap::Parser;
+use directories_next::ProjectDirs;
use log::{debug, error, info, LevelFilter};
use log4rs::append::file::FileAppender;
-use log4rs::config::{Appender, Config, Logger, Root};
-use obnam::client::ClientConfig;
-use obnam::cmd::{backup, get_chunk, list, list_files, restore, show_generation};
+use log4rs::config::{Appender, Logger, Root};
+use obnam::cmd::backup::Backup;
+use obnam::cmd::chunk::{DecryptChunk, EncryptChunk};
+use obnam::cmd::chunkify::Chunkify;
+use obnam::cmd::gen_info::GenInfo;
+use obnam::cmd::get_chunk::GetChunk;
+use obnam::cmd::init::Init;
+use obnam::cmd::inspect::Inspect;
+use obnam::cmd::list::List;
+use obnam::cmd::list_backup_versions::ListSchemaVersions;
+use obnam::cmd::list_files::ListFiles;
+use obnam::cmd::resolve::Resolve;
+use obnam::cmd::restore::Restore;
+use obnam::cmd::show_config::ShowConfig;
+use obnam::cmd::show_gen::ShowGeneration;
+use obnam::config::ClientConfig;
+use obnam::performance::{Clock, Performance};
use std::path::{Path, PathBuf};
-use structopt::StructOpt;
-const BUFFER_SIZE: usize = 1024 * 1024;
+const QUALIFIER: &str = "";
+const ORG: &str = "";
+const APPLICATION: &str = "obnam";
-fn main() -> anyhow::Result<()> {
- let opt = Opt::from_args();
- let config_file = match opt.config {
- None => default_config(),
- Some(ref path) => path.to_path_buf(),
- };
- let config = ClientConfig::read_config(&config_file)?;
- if let Some(ref log) = config.log {
- setup_logging(&log)?;
+fn main() {
+ let mut perf = Performance::default();
+ perf.start(Clock::RunTime);
+ if let Err(err) = main_program(&mut perf) {
+ error!("{}", err);
+ eprintln!("ERROR: {}", err);
+ std::process::exit(1);
}
+ perf.stop(Clock::RunTime);
+ perf.log();
+}
+
+fn main_program(perf: &mut Performance) -> anyhow::Result<()> {
+ let opt = Opt::parse();
+ let config = ClientConfig::read(&config_filename(&opt))?;
+ setup_logging(&config.log)?;
info!("client starts");
debug!("{:?}", opt);
+ debug!("configuration: {:#?}", config);
- let result = match opt.cmd {
- Command::Backup => backup(&config, BUFFER_SIZE),
- Command::List => list(&config),
- Command::ShowGeneration { gen_id } => show_generation(&config, &gen_id),
- Command::ListFiles { gen_id } => list_files(&config, &gen_id),
- Command::Restore { gen_id, to } => restore(&config, &gen_id, &to),
- Command::GetChunk { chunk_id } => get_chunk(&config, &chunk_id),
- };
-
- if let Err(ref e) = result {
- error!("{}", e);
- eprintln!("ERROR: {}", e);
- return result;
- }
+ match opt.cmd {
+ Command::Init(x) => x.run(&config),
+ Command::ListBackupVersions(x) => x.run(&config),
+ Command::Backup(x) => x.run(&config, perf),
+ Command::Inspect(x) => x.run(&config),
+ Command::Chunkify(x) => x.run(&config),
+ Command::List(x) => x.run(&config),
+ Command::ShowGeneration(x) => x.run(&config),
+ Command::ListFiles(x) => x.run(&config),
+ Command::Resolve(x) => x.run(&config),
+ Command::Restore(x) => x.run(&config),
+ Command::GenInfo(x) => x.run(&config),
+ Command::GetChunk(x) => x.run(&config),
+ Command::Config(x) => x.run(&config),
+ Command::EncryptChunk(x) => x.run(&config),
+ Command::DecryptChunk(x) => x.run(&config),
+ }?;
info!("client ends successfully");
Ok(())
}
+fn setup_logging(filename: &Path) -> anyhow::Result<()> {
+ let logfile = FileAppender::builder().build(filename)?;
+
+ let config = log4rs::Config::builder()
+ .appender(Appender::builder().build("obnam", Box::new(logfile)))
+ .logger(Logger::builder().build("obnam", LevelFilter::Debug))
+ .build(Root::builder().appender("obnam").build(LevelFilter::Debug))?;
+
+ log4rs::init_config(config)?;
+
+ Ok(())
+}
+
+fn config_filename(opt: &Opt) -> PathBuf {
+ match opt.config {
+ None => default_config(),
+ Some(ref filename) => filename.to_path_buf(),
+ }
+}
+
fn default_config() -> PathBuf {
- if let Some(path) = dirs::config_dir() {
- path.join("obnam").join("obnam.yaml")
- } else if let Some(path) = dirs::home_dir() {
- path.join(".config").join("obnam").join("obnam.yaml")
+ if let Some(dirs) = ProjectDirs::from(QUALIFIER, ORG, APPLICATION) {
+ dirs.config_dir().join("obnam.yaml")
} else {
- panic!("can't find config dir or home dir");
+ panic!("can't figure out the configuration directory");
}
}
-#[derive(Debug, StructOpt)]
-#[structopt(name = "obnam-backup", about = "Simplistic backup client")]
+#[derive(Debug, Parser)]
+#[clap(name = "obnam-backup", version, about = "Simplistic backup client")]
struct Opt {
- #[structopt(long, short, parse(from_os_str))]
+ #[clap(long, short)]
config: Option<PathBuf>,
- #[structopt(subcommand)]
+ #[clap(subcommand)]
cmd: Command,
}
-#[derive(Debug, StructOpt)]
+#[derive(Debug, Parser)]
enum Command {
- Backup,
- List,
- ListFiles {
- #[structopt(default_value = "latest")]
- gen_id: String,
- },
- Restore {
- #[structopt()]
- gen_id: String,
-
- #[structopt(parse(from_os_str))]
- to: PathBuf,
- },
- ShowGeneration {
- #[structopt(default_value = "latest")]
- gen_id: String,
- },
- GetChunk {
- #[structopt()]
- chunk_id: String,
- },
-}
-
-fn setup_logging(filename: &Path) -> anyhow::Result<()> {
- let logfile = FileAppender::builder().build(filename)?;
-
- let config = Config::builder()
- .appender(Appender::builder().build("obnam", Box::new(logfile)))
- .logger(Logger::builder().build("obnam", LevelFilter::Debug))
- .build(Root::builder().appender("obnam").build(LevelFilter::Debug))?;
-
- log4rs::init_config(config)?;
-
- Ok(())
+ Init(Init),
+ Backup(Backup),
+ Inspect(Inspect),
+ Chunkify(Chunkify),
+ List(List),
+ ListBackupVersions(ListSchemaVersions),
+ ListFiles(ListFiles),
+ Restore(Restore),
+ GenInfo(GenInfo),
+ ShowGeneration(ShowGeneration),
+ Resolve(Resolve),
+ GetChunk(GetChunk),
+ Config(ShowConfig),
+ EncryptChunk(EncryptChunk),
+ DecryptChunk(DecryptChunk),
}
generated by cgit v1.2.1 (git 2.18.0) at 2024-05-04 11:36:28 +0000