From e839c5f1c93e1fe024a2656d319721a7b23c6461 Mon Sep 17 00:00:00 2001
From: Lars Wirzenius <liw@liw.fi>
Date: Sat, 29 May 2021 11:39:31 +0300
Subject: refactor: count chunks via file system, not via chunk server API

---
 src/bin/obnam-server.rs |  2 --
 src/indexedstore.rs     | 25 +------------------------
 subplot/server.py       | 18 +++++++++++++-----
 subplot/server.yaml     |  4 ++--
 4 files changed, 16 insertions(+), 33 deletions(-)

diff --git a/src/bin/obnam-server.rs b/src/bin/obnam-server.rs
index efee77e..29ea9ff 100644
--- a/src/bin/obnam-server.rs
+++ b/src/bin/obnam-server.rs
@@ -155,8 +155,6 @@ pub async fn search_chunks(
         }
         if key == "generation" && value == "true" {
             store.find_generations().expect("SQL lookup failed")
-        } else if key == "data" && value == "true" {
-            store.find_file_chunks().expect("SQL lookup failed")
         } else if key == "sha256" {
             store.find_by_sha256(value).expect("SQL lookup failed")
         } else {
diff --git a/src/indexedstore.rs b/src/indexedstore.rs
index 982e2d9..b05cfba 100644
--- a/src/indexedstore.rs
+++ b/src/indexedstore.rs
@@ -1,9 +1,8 @@
-use crate::chunk::{DataChunk, GenerationChunk, GenerationChunkError};
+use crate::chunk::{DataChunk, GenerationChunkError};
 use crate::chunkid::ChunkId;
 use crate::chunkmeta::ChunkMeta;
 use crate::index::{Index, IndexError};
 use crate::store::{Store, StoreError};
-use std::collections::HashSet;
 use std::path::Path;
 
 /// A store for chunks and their metadata.
@@ -68,28 +67,6 @@ impl IndexedStore {
         Ok(self.index.find_generations()?)
     }
 
-    pub fn find_file_chunks(&self) -> IndexedResult<Vec<ChunkId>> {
-        let gen_ids = self.find_generations()?;
-
-        let mut sql_chunks: HashSet<ChunkId> = HashSet::new();
-        for id in gen_ids {
-            let gen_chunk = self.store.load(&id)?;
-            let gen = GenerationChunk::from_data_chunk(&gen_chunk)?;
-            for sqlite_chunk_id in gen.chunk_ids() {
-                sql_chunks.insert(sqlite_chunk_id.clone());
-            }
-        }
-
-        let all_chunk_ids = self.index.all_chunks()?;
-        let file_chunks = all_chunk_ids
-            .iter()
-            .filter(|id| !sql_chunks.contains(id))
-            .cloned()
-            .collect();
-
-        Ok(file_chunks)
-    }
-
     pub fn remove(&mut self, id: &ChunkId) -> IndexedResult<()> {
         self.index.remove_meta(id)?;
         self.store.delete(id)?;
diff --git a/subplot/server.py b/subplot/server.py
index df594f7..cfe91ab 100644
--- a/subplot/server.py
+++ b/subplot/server.py
@@ -134,13 +134,21 @@ def json_body_matches(ctx, wanted=None):
         assert_eq(body.get(key, "not.there"), wanted[key])
 
 
-def server_has_n_file_chunks(ctx, n=None):
+def server_has_n_chunks(ctx, n=None):
     assert_eq = globals()["assert_eq"]
     n = int(n)
-    url = f"{ctx['server_url']}/chunks?data=true"
-    _request(ctx, requests.get, url)
-    num_chunks = len(ctx["http.json"])
-    assert_eq(n, num_chunks)
+    files = find_files(ctx["config"]["chunks"])
+    files = [json.load(open(x)) for x in files if x.endswith(".meta")]
+    logging.debug(f"server_has_n_file_chunks: n={n}")
+    logging.debug(f"server_has_n_file_chunks: len(files)={len(files)}")
+    logging.debug(f"server_has_n_file_chunks: files={files}")
+    assert_eq(n, len(files))
+
+
+def find_files(root):
+    for dirname, _, names in os.walk(root):
+        for name in names:
+            yield os.path.join(dirname, name)
 
 
 # Make an HTTP request.
diff --git a/subplot/server.yaml b/subplot/server.yaml
index 60f8a44..5b8a242 100644
--- a/subplot/server.yaml
+++ b/subplot/server.yaml
@@ -44,5 +44,5 @@
 - then: "the body matches file {filename}"
   function: body_matches_file
 
-- then: "server has {n:int} file chunks"
-  function: server_has_n_file_chunks
+- then: "server has {n:int} chunks"
+  function: server_has_n_chunks
-- 
cgit v1.2.1