feat! finish chunk store abstraction

This builds on Alexander's work to show me how to get past the problem
I had. There's additional changes to finish off the changes.

Drop chunk deletion from server: it's not a good idea to have it until
the server API is authenticated.

Sponsored-by: author

4 files changed, 16 insertions, 63 deletions

diff --git a/obnam.md b/obnam.md
index 0c36a8b..ef6daea 100644
--- a/obnam.md
+++ b/obnam.md
@@ -1259,21 +1259,6 @@ and content-type is application/json
 and the JSON body matches {"<ID>":{"label":"0abc"}}
 ~~~
 
-Finally, we must be able to delete it. After that, we must not be able
-to retrieve it, or find it using metadata.
-
-~~~scenario
-when I DELETE /v1/chunks/<ID>
-then HTTP status code is 200
-
-when I GET /v1/chunks/<ID>
-then HTTP status code is 404
-
-when I GET /v1/chunks?label=0abc
-then HTTP status code is 200
-and content-type is application/json
-and the JSON body matches {}
-~~~
 
 ## Retrieve a chunk that does not exist
 
@@ -1298,17 +1283,6 @@ and content-type is application/json
 and the JSON body matches {}
 ~~~
 
-## Delete chunk that does not exist
-
-We must get the right error when deleting a chunk that doesn't exist.
-
-~~~scenario
-given a working Obnam system
-when I try to DELETE /v1/chunks/any.random.string
-then HTTP status code is 404
-~~~
-
-
 ## Persistent across restarts
 
 Chunk storage, and the index of chunk metadata for searches, needs to
diff --git a/src/bin/obnam-server.rs b/src/bin/obnam-server.rs
index c9e93bc..102e8b6 100644
--- a/src/bin/obnam-server.rs
+++ b/src/bin/obnam-server.rs
@@ -4,6 +4,7 @@ use log::{debug, error, info};
 use obnam::chunkid::ChunkId;
 use obnam::chunkmeta::ChunkMeta;
 use obnam::chunkstore::ChunkStore;
+use obnam::label::Label;
 use obnam::server::{ServerConfig, ServerConfigError};
 use serde::Serialize;
 use std::collections::HashMap;
@@ -69,16 +70,8 @@ async fn main() -> anyhow::Result<()> {
         .and(store.clone())
         .and_then(search_chunks);
 
-    let delete = warp::delete()
-        .and(warp::path("v1"))
-        .and(warp::path("chunks"))
-        .and(warp::path::param())
-        .and(warp::path::end())
-        .and(store.clone())
-        .and_then(delete_chunk);
-
     let log = warp::log("obnam");
-    let webroot = create.or(fetch).or(search).or(delete).with(log);
+    let webroot = create.or(fetch).or(search).with(log);
 
     debug!("starting warp");
     warp::serve(webroot)
@@ -105,7 +98,7 @@ pub async fn create_chunk(
     meta: String,
     data: Bytes,
 ) -> Result<impl warp::Reply, warp::Rejection> {
-    let mut store = store.lock().await;
+    let store = store.lock().await;
 
     let meta: ChunkMeta = match meta.parse() {
         Ok(s) => s,
@@ -147,7 +140,7 @@ pub async fn fetch_chunk(
 
 pub async fn search_chunks(
     query: HashMap<String, String>,
-    store: Arc<Mutex<IndexedStore>>,
+    store: Arc<Mutex<ChunkStore>>,
 ) -> Result<impl warp::Reply, warp::Rejection> {
     let store = store.lock().await;
 
@@ -158,7 +151,12 @@ pub async fn search_chunks(
             return Ok(ChunkResult::BadRequest);
         }
         if key == "label" {
-            store.find_by_label(value).expect("SQL lookup failed")
+            let label = Label::deserialize(value).unwrap();
+            let label = ChunkMeta::new(&label);
+            store
+                .find_by_label(&label)
+                .await
+                .expect("SQL lookup failed")
         } else {
             error!("unknown search key {:?}", key);
             return Ok(ChunkResult::BadRequest);
@@ -170,7 +168,7 @@ pub async fn search_chunks(
 
     let mut hits = SearchHits::default();
     for chunk_id in found {
-        let meta = match store.load_meta(&chunk_id) {
+        let (_, meta) = match store.get(&chunk_id).await {
             Ok(meta) => {
                 info!("search found chunk {}", chunk_id);
                 meta
@@ -209,30 +207,10 @@ impl SearchHits {
     }
 }
 
-pub async fn delete_chunk(
-    id: String,
-    store: Arc<Mutex<IndexedStore>>,
-) -> Result<impl warp::Reply, warp::Rejection> {
-    let mut store = store.lock().await;
-    let id: ChunkId = id.parse().unwrap();
-
-    match store.remove(&id) {
-        Ok(_) => {
-            info!("chunk deleted: {}", id);
-            Ok(ChunkResult::Deleted)
-        }
-        Err(e) => {
-            error!("could not delete chunk {}: {:?}", id, e);
-            Ok(ChunkResult::NotFound)
-        }
-    }
-}
-
 enum ChunkResult {
     Created(ChunkId),
     Fetched(ChunkMeta, Vec<u8>),
     Found(SearchHits),
-    Deleted,
     NotFound,
     BadRequest,
     InternalServerError,
@@ -267,7 +245,6 @@ impl warp::Reply for ChunkResult {
                 )
             }
             ChunkResult::Found(hits) => json_response(StatusCode::OK, hits.to_json(), None),
-            ChunkResult::Deleted => status_response(StatusCode::OK),
             ChunkResult::BadRequest => status_response(StatusCode::BAD_REQUEST),
             ChunkResult::NotFound => status_response(StatusCode::NOT_FOUND),
             ChunkResult::InternalServerError => status_response(StatusCode::INTERNAL_SERVER_ERROR),
diff --git a/src/chunkstore.rs b/src/chunkstore.rs
index 85d5007..2b93720 100644
--- a/src/chunkstore.rs
+++ b/src/chunkstore.rs
@@ -124,7 +124,7 @@ impl LocalStore {
         let b = bytes[1];
         let c = bytes[2];
         let dir = self.path.join(format!("{}/{}/{}", a, b, c));
-        let filename = dir.join(format!("{}", id));
+        let filename = dir.join(format!("{}.data", id));
         (dir, filename)
     }
 }
diff --git a/subplot/server.py b/subplot/server.py
index 1f4506f..a604733 100644
--- a/subplot/server.py
+++ b/subplot/server.py
@@ -87,7 +87,9 @@ def delete_chunk_by_id(ctx, chunk_id=None):
 def make_chunk_file_be_empty(ctx, chunk_id=None):
     chunk_id = ctx["vars"][chunk_id]
     chunks = ctx["config"]["chunks"]
-    for (dirname, _, _) in os.walk(chunks):
+    logging.debug(f"trying to empty chunk {chunk_id}")
+    for (dirname, _, filenames) in os.walk(chunks):
+        logging.debug(f"found directory {dirname}, with {filenames}")
         filename = os.path.join(dirname, chunk_id + ".data")
         if os.path.exists(filename):
             logging.debug(f"emptying chunk file {filename}")
@@ -136,7 +138,7 @@ def server_has_n_chunks(ctx, n=None):
     assert_eq = globals()["assert_eq"]
     n = int(n)
     files = find_files(ctx["config"]["chunks"])
-    files = [json.load(open(x)) for x in files if x.endswith(".meta")]
+    files = [x for x in files if x.endswith(".data")]
     logging.debug(f"server_has_n_file_chunks: n={n}")
     logging.debug(f"server_has_n_file_chunks: len(files)={len(files)}")
     logging.debug(f"server_has_n_file_chunks: files={files}")