From 14ab788c930846aeb2c472b421a9030b0e9e95f3 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Wed, 26 Oct 2022 17:33:52 +0300 Subject: feat! finish chunk store abstraction This builds on Alexander's work to show me how to get past the problem I had. There's additional changes to finish off the changes. Drop chunk deletion from server: it's not a good idea to have it until the server API is authenticated. Sponsored-by: author --- obnam.md | 26 -------------------------- src/bin/obnam-server.rs | 45 +++++++++++---------------------------------- src/chunkstore.rs | 2 +- subplot/server.py | 6 ++++-- 4 files changed, 16 insertions(+), 63 deletions(-) diff --git a/obnam.md b/obnam.md index 0c36a8b..ef6daea 100644 --- a/obnam.md +++ b/obnam.md @@ -1259,21 +1259,6 @@ and content-type is application/json and the JSON body matches {"":{"label":"0abc"}} ~~~ -Finally, we must be able to delete it. After that, we must not be able -to retrieve it, or find it using metadata. - -~~~scenario -when I DELETE /v1/chunks/ -then HTTP status code is 200 - -when I GET /v1/chunks/ -then HTTP status code is 404 - -when I GET /v1/chunks?label=0abc -then HTTP status code is 200 -and content-type is application/json -and the JSON body matches {} -~~~ ## Retrieve a chunk that does not exist @@ -1298,17 +1283,6 @@ and content-type is application/json and the JSON body matches {} ~~~ -## Delete chunk that does not exist - -We must get the right error when deleting a chunk that doesn't exist. - -~~~scenario -given a working Obnam system -when I try to DELETE /v1/chunks/any.random.string -then HTTP status code is 404 -~~~ - - ## Persistent across restarts Chunk storage, and the index of chunk metadata for searches, needs to diff --git a/src/bin/obnam-server.rs b/src/bin/obnam-server.rs index c9e93bc..102e8b6 100644 --- a/src/bin/obnam-server.rs +++ b/src/bin/obnam-server.rs @@ -4,6 +4,7 @@ use log::{debug, error, info}; use obnam::chunkid::ChunkId; use obnam::chunkmeta::ChunkMeta; use obnam::chunkstore::ChunkStore; +use obnam::label::Label; use obnam::server::{ServerConfig, ServerConfigError}; use serde::Serialize; use std::collections::HashMap; @@ -69,16 +70,8 @@ async fn main() -> anyhow::Result<()> { .and(store.clone()) .and_then(search_chunks); - let delete = warp::delete() - .and(warp::path("v1")) - .and(warp::path("chunks")) - .and(warp::path::param()) - .and(warp::path::end()) - .and(store.clone()) - .and_then(delete_chunk); - let log = warp::log("obnam"); - let webroot = create.or(fetch).or(search).or(delete).with(log); + let webroot = create.or(fetch).or(search).with(log); debug!("starting warp"); warp::serve(webroot) @@ -105,7 +98,7 @@ pub async fn create_chunk( meta: String, data: Bytes, ) -> Result { - let mut store = store.lock().await; + let store = store.lock().await; let meta: ChunkMeta = match meta.parse() { Ok(s) => s, @@ -147,7 +140,7 @@ pub async fn fetch_chunk( pub async fn search_chunks( query: HashMap, - store: Arc>, + store: Arc>, ) -> Result { let store = store.lock().await; @@ -158,7 +151,12 @@ pub async fn search_chunks( return Ok(ChunkResult::BadRequest); } if key == "label" { - store.find_by_label(value).expect("SQL lookup failed") + let label = Label::deserialize(value).unwrap(); + let label = ChunkMeta::new(&label); + store + .find_by_label(&label) + .await + .expect("SQL lookup failed") } else { error!("unknown search key {:?}", key); return Ok(ChunkResult::BadRequest); @@ -170,7 +168,7 @@ pub async fn search_chunks( let mut hits = SearchHits::default(); for chunk_id in found { - let meta = match store.load_meta(&chunk_id) { + let (_, meta) = match store.get(&chunk_id).await { Ok(meta) => { info!("search found chunk {}", chunk_id); meta @@ -209,30 +207,10 @@ impl SearchHits { } } -pub async fn delete_chunk( - id: String, - store: Arc>, -) -> Result { - let mut store = store.lock().await; - let id: ChunkId = id.parse().unwrap(); - - match store.remove(&id) { - Ok(_) => { - info!("chunk deleted: {}", id); - Ok(ChunkResult::Deleted) - } - Err(e) => { - error!("could not delete chunk {}: {:?}", id, e); - Ok(ChunkResult::NotFound) - } - } -} - enum ChunkResult { Created(ChunkId), Fetched(ChunkMeta, Vec), Found(SearchHits), - Deleted, NotFound, BadRequest, InternalServerError, @@ -267,7 +245,6 @@ impl warp::Reply for ChunkResult { ) } ChunkResult::Found(hits) => json_response(StatusCode::OK, hits.to_json(), None), - ChunkResult::Deleted => status_response(StatusCode::OK), ChunkResult::BadRequest => status_response(StatusCode::BAD_REQUEST), ChunkResult::NotFound => status_response(StatusCode::NOT_FOUND), ChunkResult::InternalServerError => status_response(StatusCode::INTERNAL_SERVER_ERROR), diff --git a/src/chunkstore.rs b/src/chunkstore.rs index 85d5007..2b93720 100644 --- a/src/chunkstore.rs +++ b/src/chunkstore.rs @@ -124,7 +124,7 @@ impl LocalStore { let b = bytes[1]; let c = bytes[2]; let dir = self.path.join(format!("{}/{}/{}", a, b, c)); - let filename = dir.join(format!("{}", id)); + let filename = dir.join(format!("{}.data", id)); (dir, filename) } } diff --git a/subplot/server.py b/subplot/server.py index 1f4506f..a604733 100644 --- a/subplot/server.py +++ b/subplot/server.py @@ -87,7 +87,9 @@ def delete_chunk_by_id(ctx, chunk_id=None): def make_chunk_file_be_empty(ctx, chunk_id=None): chunk_id = ctx["vars"][chunk_id] chunks = ctx["config"]["chunks"] - for (dirname, _, _) in os.walk(chunks): + logging.debug(f"trying to empty chunk {chunk_id}") + for (dirname, _, filenames) in os.walk(chunks): + logging.debug(f"found directory {dirname}, with {filenames}") filename = os.path.join(dirname, chunk_id + ".data") if os.path.exists(filename): logging.debug(f"emptying chunk file {filename}") @@ -136,7 +138,7 @@ def server_has_n_chunks(ctx, n=None): assert_eq = globals()["assert_eq"] n = int(n) files = find_files(ctx["config"]["chunks"]) - files = [json.load(open(x)) for x in files if x.endswith(".meta")] + files = [x for x in files if x.endswith(".data")] logging.debug(f"server_has_n_file_chunks: n={n}") logging.debug(f"server_has_n_file_chunks: len(files)={len(files)}") logging.debug(f"server_has_n_file_chunks: files={files}") -- cgit v1.2.1