summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorLars Wirzenius <liw@liw.fi>2021-05-29 11:39:31 +0300
committerLars Wirzenius <liw@liw.fi>2021-05-29 15:02:31 +0300
commite839c5f1c93e1fe024a2656d319721a7b23c6461 (patch)
tree3828b84f7e46d8f9976d1a828e8625e22f79c84a
parent03a8b0f9cba08ad09cb8494579f6e318aee762f4 (diff)
downloadobnam2-e839c5f1c93e1fe024a2656d319721a7b23c6461.tar.gz
refactor: count chunks via file system, not via chunk server API
-rw-r--r--src/bin/obnam-server.rs2
-rw-r--r--src/indexedstore.rs25
-rw-r--r--subplot/server.py18
-rw-r--r--subplot/server.yaml4
4 files changed, 16 insertions, 33 deletions
diff --git a/src/bin/obnam-server.rs b/src/bin/obnam-server.rs
index efee77e..29ea9ff 100644
--- a/src/bin/obnam-server.rs
+++ b/src/bin/obnam-server.rs
@@ -155,8 +155,6 @@ pub async fn search_chunks(
}
if key == "generation" && value == "true" {
store.find_generations().expect("SQL lookup failed")
- } else if key == "data" && value == "true" {
- store.find_file_chunks().expect("SQL lookup failed")
} else if key == "sha256" {
store.find_by_sha256(value).expect("SQL lookup failed")
} else {
diff --git a/src/indexedstore.rs b/src/indexedstore.rs
index 982e2d9..b05cfba 100644
--- a/src/indexedstore.rs
+++ b/src/indexedstore.rs
@@ -1,9 +1,8 @@
-use crate::chunk::{DataChunk, GenerationChunk, GenerationChunkError};
+use crate::chunk::{DataChunk, GenerationChunkError};
use crate::chunkid::ChunkId;
use crate::chunkmeta::ChunkMeta;
use crate::index::{Index, IndexError};
use crate::store::{Store, StoreError};
-use std::collections::HashSet;
use std::path::Path;
/// A store for chunks and their metadata.
@@ -68,28 +67,6 @@ impl IndexedStore {
Ok(self.index.find_generations()?)
}
- pub fn find_file_chunks(&self) -> IndexedResult<Vec<ChunkId>> {
- let gen_ids = self.find_generations()?;
-
- let mut sql_chunks: HashSet<ChunkId> = HashSet::new();
- for id in gen_ids {
- let gen_chunk = self.store.load(&id)?;
- let gen = GenerationChunk::from_data_chunk(&gen_chunk)?;
- for sqlite_chunk_id in gen.chunk_ids() {
- sql_chunks.insert(sqlite_chunk_id.clone());
- }
- }
-
- let all_chunk_ids = self.index.all_chunks()?;
- let file_chunks = all_chunk_ids
- .iter()
- .filter(|id| !sql_chunks.contains(id))
- .cloned()
- .collect();
-
- Ok(file_chunks)
- }
-
pub fn remove(&mut self, id: &ChunkId) -> IndexedResult<()> {
self.index.remove_meta(id)?;
self.store.delete(id)?;
diff --git a/subplot/server.py b/subplot/server.py
index df594f7..cfe91ab 100644
--- a/subplot/server.py
+++ b/subplot/server.py
@@ -134,13 +134,21 @@ def json_body_matches(ctx, wanted=None):
assert_eq(body.get(key, "not.there"), wanted[key])
-def server_has_n_file_chunks(ctx, n=None):
+def server_has_n_chunks(ctx, n=None):
assert_eq = globals()["assert_eq"]
n = int(n)
- url = f"{ctx['server_url']}/chunks?data=true"
- _request(ctx, requests.get, url)
- num_chunks = len(ctx["http.json"])
- assert_eq(n, num_chunks)
+ files = find_files(ctx["config"]["chunks"])
+ files = [json.load(open(x)) for x in files if x.endswith(".meta")]
+ logging.debug(f"server_has_n_file_chunks: n={n}")
+ logging.debug(f"server_has_n_file_chunks: len(files)={len(files)}")
+ logging.debug(f"server_has_n_file_chunks: files={files}")
+ assert_eq(n, len(files))
+
+
+def find_files(root):
+ for dirname, _, names in os.walk(root):
+ for name in names:
+ yield os.path.join(dirname, name)
# Make an HTTP request.
diff --git a/subplot/server.yaml b/subplot/server.yaml
index 60f8a44..5b8a242 100644
--- a/subplot/server.yaml
+++ b/subplot/server.yaml
@@ -44,5 +44,5 @@
- then: "the body matches file {filename}"
function: body_matches_file
-- then: "server has {n:int} file chunks"
- function: server_has_n_file_chunks
+- then: "server has {n:int} chunks"
+ function: server_has_n_chunks