use crate::checksummer::sha256; use crate::chunk::DataChunk; use crate::chunk::GenerationChunk; use crate::chunker::Chunker; use crate::chunkid::ChunkId; use crate::chunkmeta::ChunkMeta; use crate::error::ObnamError; use crate::fsentry::{FilesystemEntry, FilesystemKind}; use crate::generation::{FinishedGeneration, LocalGeneration}; use crate::genlist::GenerationList; use anyhow::Context; use chrono::{DateTime, Local}; use log::{debug, error, info, trace}; use reqwest::blocking::Client; use serde::Deserialize; use std::collections::HashMap; use std::fs::File; use std::io::prelude::*; use std::path::{Path, PathBuf}; #[derive(Debug, Deserialize, Clone)] pub struct ClientConfig { pub server_url: String, pub root: PathBuf, pub log: Option, } impl ClientConfig { pub fn read_config(filename: &Path) -> anyhow::Result { trace!("read_config: filename={:?}", filename); let config = std::fs::read_to_string(filename) .with_context(|| format!("reading configuration file {}", filename.display()))?; let config = serde_yaml::from_str(&config)?; Ok(config) } } #[derive(Debug, thiserror::Error)] pub enum ClientError { #[error("Server successful response to creating chunk lacked chunk id")] NoCreatedChunkId, #[error("Server does not have chunk {0}")] ChunkNotFound(String), #[error("Server does not have generation {0}")] GenerationNotFound(String), } pub struct BackupClient { client: Client, base_url: String, } impl BackupClient { pub fn new(base_url: &str) -> anyhow::Result { let client = Client::builder() .danger_accept_invalid_certs(true) .build()?; Ok(Self { client, base_url: base_url.to_string(), }) } pub fn upload_filesystem_entry( &self, e: &FilesystemEntry, size: usize, ) -> anyhow::Result> { info!("upload entry: {:?}", e); let ids = match e.kind() { FilesystemKind::Regular => self.read_file(e.pathbuf(), size)?, FilesystemKind::Directory => vec![], FilesystemKind::Symlink => vec![], }; Ok(ids) } pub fn upload_generation(&self, filename: &Path, size: usize) -> anyhow::Result { info!("upload SQLite {}", filename.display()); let ids = self.read_file(filename.to_path_buf(), size)?; let gen = GenerationChunk::new(ids); let data = gen.to_data_chunk()?; let meta = ChunkMeta::new_generation(&sha256(data.data()), ¤t_timestamp()); let gen_id = self.upload_gen_chunk(meta.clone(), gen)?; info!("uploaded generation {}, meta {:?}", gen_id, meta); Ok(gen_id) } fn read_file(&self, filename: PathBuf, size: usize) -> anyhow::Result> { info!("upload file {}", filename.display()); let file = std::fs::File::open(filename)?; let chunker = Chunker::new(size, file); let chunk_ids = self.upload_new_file_chunks(chunker)?; Ok(chunk_ids) } fn base_url(&self) -> &str { &self.base_url } fn chunks_url(&self) -> String { format!("{}/chunks", self.base_url()) } pub fn has_chunk(&self, meta: &ChunkMeta) -> anyhow::Result> { trace!("has_chunk: url={:?}", self.base_url()); let req = self .client .get(&self.chunks_url()) .query(&[("sha256", meta.sha256())]) .build()?; let res = self.client.execute(req)?; debug!("has_chunk: status={}", res.status()); let has = if res.status() != 200 { debug!("has_chunk: error from server"); None } else { let text = res.text()?; debug!("has_chunk: text={:?}", text); let hits: HashMap = serde_json::from_str(&text)?; debug!("has_chunk: hits={:?}", hits); let mut iter = hits.iter(); if let Some((chunk_id, _)) = iter.next() { debug!("has_chunk: chunk_id={:?}", chunk_id); Some(chunk_id.into()) } else { None } }; info!("has_chunk result: {:?}", has); Ok(has) } pub fn upload_chunk(&self, meta: ChunkMeta, chunk: DataChunk) -> anyhow::Result { let res = self .client .post(&self.chunks_url()) .header("chunk-meta", meta.to_json()) .body(chunk.data().to_vec()) .send()?; debug!("upload_chunk: res={:?}", res); let res: HashMap = res.json()?; let chunk_id = if let Some(chunk_id) = res.get("chunk_id") { debug!("upload_chunk: id={}", chunk_id); chunk_id.parse().unwrap() } else { return Err(ClientError::NoCreatedChunkId.into()); }; info!("uploaded_chunk {} meta {:?}", chunk_id, meta); Ok(chunk_id) } pub fn upload_gen_chunk( &self, meta: ChunkMeta, gen: GenerationChunk, ) -> anyhow::Result { let res = self .client .post(&self.chunks_url()) .header("chunk-meta", meta.to_json()) .body(serde_json::to_string(&gen)?) .send()?; debug!("upload_chunk: res={:?}", res); let res: HashMap = res.json()?; let chunk_id = if let Some(chunk_id) = res.get("chunk_id") { debug!("upload_chunk: id={}", chunk_id); chunk_id.parse().unwrap() } else { return Err(ClientError::NoCreatedChunkId.into()); }; info!("uploaded_generation chunk {}", chunk_id); Ok(chunk_id) } pub fn upload_new_file_chunks(&self, chunker: Chunker) -> anyhow::Result> { let mut chunk_ids = vec![]; for item in chunker { let (meta, chunk) = item?; if let Some(chunk_id) = self.has_chunk(&meta)? { chunk_ids.push(chunk_id.clone()); info!("reusing existing chunk {}", chunk_id); } else { let chunk_id = self.upload_chunk(meta, chunk)?; chunk_ids.push(chunk_id.clone()); info!("created new chunk {}", chunk_id); } } Ok(chunk_ids) } pub fn list_generations(&self) -> anyhow::Result { let url = format!("{}?generation=true", &self.chunks_url()); trace!("list_generations: url={:?}", url); let req = self.client.get(&url).build()?; let res = self.client.execute(req)?; debug!("list_generations: status={}", res.status()); let body = res.bytes()?; debug!("list_generations: body={:?}", body); let map: HashMap = serde_yaml::from_slice(&body)?; debug!("list_generations: map={:?}", map); let finished = map .iter() .map(|(id, meta)| FinishedGeneration::new(id, meta.ended().map_or("", |s| s))) .collect(); Ok(GenerationList::new(finished)) } pub fn fetch_chunk(&self, chunk_id: &ChunkId) -> anyhow::Result { info!("fetch chunk {}", chunk_id); let url = format!("{}/{}", &self.chunks_url(), chunk_id); let req = self.client.get(&url).build()?; let res = self.client.execute(req)?; if res.status() != 200 { let err = ClientError::ChunkNotFound(chunk_id.to_string()); error!("fetching chunk {} failed: {}", chunk_id, err); return Err(err.into()); } let headers = res.headers(); let meta = headers.get("chunk-meta"); if meta.is_none() { let err = ObnamError::NoChunkMeta(chunk_id.clone()); error!("fetching chunk {} failed: {}", chunk_id, err); return Err(err.into()); } let meta = meta.unwrap().to_str()?; debug!("fetching chunk {}: meta={:?}", chunk_id, meta); let meta: ChunkMeta = serde_json::from_str(meta)?; debug!("fetching chunk {}: meta={:?}", chunk_id, meta); let body = res.bytes()?; let body = body.to_vec(); let actual = sha256(&body); if actual != meta.sha256() { let err = ObnamError::WrongChecksum(chunk_id.clone(), actual, meta.sha256().to_string()); error!("fetching chunk {} failed: {}", chunk_id, err); return Err(err.into()); } let chunk: DataChunk = DataChunk::new(body); Ok(chunk) } fn fetch_generation_chunk(&self, gen_id: &str) -> anyhow::Result { let chunk_id = ChunkId::from_str(gen_id); let chunk = self.fetch_chunk(&chunk_id)?; let gen = GenerationChunk::from_data_chunk(&chunk)?; Ok(gen) } pub fn fetch_generation(&self, gen_id: &str, dbname: &Path) -> anyhow::Result { let gen = self.fetch_generation_chunk(gen_id)?; // Fetch the SQLite file, storing it in the named file. let mut dbfile = File::create(&dbname)?; for id in gen.chunk_ids() { let chunk = self.fetch_chunk(id)?; dbfile.write_all(chunk.data())?; } info!("downloaded generation to {}", dbname.display()); let gen = LocalGeneration::open(dbname)?; Ok(gen) } } fn current_timestamp() -> String { let now: DateTime = Local::now(); format!("{}", now.format("%Y-%m-%d %H:%M:%S.%f %z")) }