Merge changes to allow user to choose node, chunk, chunk group sizes.

14 files changed, 82 insertions, 44 deletions

diff --git a/blackboxtest b/blackboxtest
index 61ab18b9..35108bed 100755
--- a/blackboxtest
+++ b/blackboxtest
@@ -390,7 +390,7 @@ class ReusesChunks(BlackBoxTest):
         self.verify(data, restored)
         fsf = obnamlib.VfsFactory()
         fs = fsf.new(store)
-        s = obnamlib.Store(fs)
+        s = obnamlib.Store(fs, obnamlib.DEFAULT_NODE_SIZE)
         s.open_host(self.hostid)
         self.assertEqual('ReusesChunks', len(s.list_chunks()), 1,
                          'only one chunk when all files are identical')
@@ -401,7 +401,9 @@ class UsesChunkGroups(BlackBoxTest):
     def test(self):
         data = self.tempdir()
         ngroups = 2
-        nbytes = obnamlib.CHUNK_SIZE * obnamlib.CHUNK_GROUP_SIZE * ngroups
+        nbytes = (obnamlib.DEFAULT_CHUNK_SIZE * 
+                  obnamlib.DEFAULT_CHUNK_GROUP_SIZE * 
+                  ngroups)
         self.create_file(data, 'file', 'x' * nbytes)
         store = self.tempdir()
         self.backup(store, [data])
@@ -409,7 +411,7 @@ class UsesChunkGroups(BlackBoxTest):
         self.verify(data, restored)
         fsf = obnamlib.VfsFactory()
         fs = fsf.new(store)
-        s = obnamlib.Store(fs)
+        s = obnamlib.Store(fs, obnamlib.DEFAULT_NODE_SIZE)
         s.open_host(self.hostid)
         self.assertEqual('UsesChunkGroups', len(s.list_chunk_groups()), 
                          ngroups,
diff --git a/dumpobjs b/dumpobjs
index e0517cce..4edcd833 100755
--- a/dumpobjs
+++ b/dumpobjs
@@ -27,7 +27,7 @@ def find_objids(fs):
 
 
 fs = obnamlib.LocalFS(sys.argv[1])
-store = obnamlib.Store(fs)
+store = obnamlib.Store(fs, obnamlib.DEFAULT_NODE_SIZE)
 for objid in find_objids(fs):
     obj = store.get_object(objid)
     print 'id %s (%s):' % (obj.id, obj.__class__.__name__)
diff --git a/obnamlib/__init__.py b/obnamlib/__init__.py
index b1952403..04872c94 100644
--- a/obnamlib/__init__.py
+++ b/obnamlib/__init__.py
@@ -26,8 +26,9 @@ class AppException(Exception):
 class Error(Exception):
     pass
 
-CHUNK_SIZE = 4096
-CHUNK_GROUP_SIZE = 16
+DEFAULT_NODE_SIZE = 64 * 1024
+DEFAULT_CHUNK_SIZE = 4096
+DEFAULT_CHUNK_GROUP_SIZE = 16
 
 from sizeparse import SizeSyntaxError, UnitNameError, ByteSizeParser
 
diff --git a/obnamlib/app.py b/obnamlib/app.py
index 8f7401f0..e3f299c6 100644
--- a/obnamlib/app.py
+++ b/obnamlib/app.py
@@ -42,6 +42,22 @@ class App(object):
                                 'do not write or remove anything, just '
                                 'pretend to do that')
 
+        self.config.new_bytesize(['node-size'],
+                                 'size of B-tree nodes on disk '
+                                 '(default: %default)')
+        self.config['node-size'] = '%s' % obnamlib.DEFAULT_NODE_SIZE
+
+        self.config.new_bytesize(['chunk-size'],
+                                 'size of chunks of file data backed up '
+                                 '(default: %default)')
+        self.config['chunk-size'] = '%s' % obnamlib.DEFAULT_CHUNK_SIZE
+
+        self.config.new_bytesize(['chunk-group-size'],
+                                 'number of chunks per chunk group '
+                                 '(default: %default)')
+        self.config['chunk-group-size'] = \
+            '%s' % obnamlib.DEFAULT_CHUNK_GROUP_SIZE
+
         self.pm = obnamlib.PluginManager()
         self.pm.locations = [self.plugins_dir()]
         self.pm.plugin_arguments = (self,)
diff --git a/obnamlib/cfg.py b/obnamlib/cfg.py
index 89137a1d..ff635b32 100644
--- a/obnamlib/cfg.py
+++ b/obnamlib/cfg.py
@@ -95,6 +95,12 @@ class Configuration(object):
         self.new_string(names, help)
         self.processors[names[0]] = callback
 
+    def new_bytesize(self, names, help):
+        def callback(value):
+            p = obnamlib.ByteSizeParser()
+            return p.parse(value)
+        self.new_processed(names, help, callback)
+
     def new_list(self, names, help):
         self.new_setting('list', names, help, 'append', [])
 
diff --git a/obnamlib/cfg_tests.py b/obnamlib/cfg_tests.py
index 248437fb..551a84ad 100644
--- a/obnamlib/cfg_tests.py
+++ b/obnamlib/cfg_tests.py
@@ -73,3 +73,13 @@ class ConfigurationTests(unittest.TestCase):
         self.cfg.load(['--size=123'])
         self.assertEqual(self.cfg['size'], 123)
 
+    def test_handles_unadorned_bytesize(self):
+        self.cfg.new_bytesize(['size'], 'size help')
+        self.cfg.load(['--size=123'])
+        self.assertEqual(self.cfg['size'], 123)
+
+    def test_handles_bytesize_with_units(self):
+        self.cfg.new_bytesize(['size'], 'size help')
+        self.cfg.load(['--size=123KiB'])
+        self.assertEqual(self.cfg['size'], 123*1024)
+
diff --git a/obnamlib/plugins/backup_plugin.py b/obnamlib/plugins/backup_plugin.py
index 0f7f4cdc..839ef7ae 100644
--- a/obnamlib/plugins/backup_plugin.py
+++ b/obnamlib/plugins/backup_plugin.py
@@ -55,7 +55,7 @@ class BackupPlugin(obnamlib.ObnamPlugin):
         storepath = self.app.config['store']
         logging.debug('store: %s' % storepath)
         storefs = self.app.fsf.new(storepath)
-        self.store = obnamlib.Store(storefs)
+        self.store = obnamlib.Store(storefs, self.app.config['node-size'])
 
         hostname = self.app.config['hostname']
         logging.debug('hostname: %s' % hostname)
@@ -190,13 +190,15 @@ class BackupPlugin(obnamlib.ObnamPlugin):
         cgids = []
         groupsum = self.store.new_checksummer()
         f = self.fs.open(filename, 'r')
+        chunk_size = int(self.app.config['chunk-size'])
+        chunk_group_size = int(self.app.config['chunk-group-size'])
         while True:
-            data = f.read(obnamlib.CHUNK_SIZE)
+            data = f.read(chunk_size)
             if not data:
                 break
             chunkids.append(self.backup_file_chunk(data))
             groupsum.update(data)
-            if len(chunkids) == obnamlib.CHUNK_GROUP_SIZE:
+            if len(chunkids) == chunk_group_size:
                 checksum = groupsum.hexdigest()
                 cgid = self.store.put_chunk_group(chunkids, checksum)
                 cgids.append(cgid)
diff --git a/obnamlib/plugins/forget_plugin.py b/obnamlib/plugins/forget_plugin.py
index f9f074f9..f6a1c634 100644
--- a/obnamlib/plugins/forget_plugin.py
+++ b/obnamlib/plugins/forget_plugin.py
@@ -34,7 +34,7 @@ class ForgetPlugin(obnamlib.ObnamPlugin):
         self.app.config.require('hostname')
 
         fs = self.app.fsf.new(self.app.config['store'])
-        self.store = obnamlib.Store(fs)
+        self.store = obnamlib.Store(fs, self.app.config['node-size'])
         self.store.lock_host(self.app.config['hostname'])
 
         if args:
diff --git a/obnamlib/plugins/fsck_plugin.py b/obnamlib/plugins/fsck_plugin.py
index eef2c521..5ecc3bf1 100644
--- a/obnamlib/plugins/fsck_plugin.py
+++ b/obnamlib/plugins/fsck_plugin.py
@@ -31,7 +31,7 @@ class FsckPlugin(obnamlib.ObnamPlugin):
 
         storefs = self.app.fsf.new(self.app.config['store'])
         storefs.connect()
-        self.store = obnamlib.Store(storefs)
+        self.store = obnamlib.Store(storefs, self.app.config['node-size'])
         
         self.check_root()
 
diff --git a/obnamlib/plugins/restore_plugin.py b/obnamlib/plugins/restore_plugin.py
index 6f00f30e..11b2a22f 100644
--- a/obnamlib/plugins/restore_plugin.py
+++ b/obnamlib/plugins/restore_plugin.py
@@ -81,7 +81,7 @@ class RestorePlugin(obnamlib.ObnamPlugin):
             args = ['/']
     
         storefs = self.app.fsf.new(self.app.config['store'])
-        self.store = obnamlib.Store(storefs)
+        self.store = obnamlib.Store(storefs, self.app.config['node-size'])
         self.store.open_host(self.app.config['hostname'])
         self.fs = self.app.fsf.new(self.app.config['to'])
 
diff --git a/obnamlib/plugins/show_plugin.py b/obnamlib/plugins/show_plugin.py
index 05cc6088..3ed00895 100644
--- a/obnamlib/plugins/show_plugin.py
+++ b/obnamlib/plugins/show_plugin.py
@@ -40,7 +40,7 @@ class ShowPlugin(obnamlib.ObnamPlugin):
         self.app.config.require('store')
         self.app.config.require('hostname')
         fs = self.app.fsf.new(self.app.config['store'])
-        self.store = obnamlib.Store(fs)
+        self.store = obnamlib.Store(fs, self.app.config['node-size'])
         self.store.open_host(self.app.config['hostname'])
 
     def hosts(self, args):
diff --git a/obnamlib/plugins/verify_plugin.py b/obnamlib/plugins/verify_plugin.py
index db272e3f..c24b8bfd 100644
--- a/obnamlib/plugins/verify_plugin.py
+++ b/obnamlib/plugins/verify_plugin.py
@@ -49,7 +49,7 @@ class VerifyPlugin(obnamlib.ObnamPlugin):
     
         fs = self.app.fsf.new(self.app.config['store'])
         fs.connect()
-        self.store = obnamlib.Store(fs)
+        self.store = obnamlib.Store(fs, self.app.config['node-size'])
         self.store.open_host(self.app.config['hostname'])
         self.fs = self.app.fsf.new(self.app.config['root'][0])
         self.fs.connect()
diff --git a/obnamlib/store.py b/obnamlib/store.py
index 5696f291..dc93299e 100644
--- a/obnamlib/store.py
+++ b/obnamlib/store.py
@@ -158,11 +158,9 @@ class HostList(StoreTree):
     '''Store list of hosts.'''
 
     key_bytes = 8 # 64-bit counter as key
-    node_size = 4096 # typical size of disk block
 
-    def __init__(self, fs):
-        StoreTree.__init__(self, fs, 'hostlist', self.key_bytes, 
-                           self.node_size)
+    def __init__(self, fs, node_size):
+        StoreTree.__init__(self, fs, 'hostlist', self.key_bytes, node_size)
         self.minkey = self.key(0)
         self.maxkey = self.key(2**64-1)
 
@@ -234,8 +232,6 @@ class GenerationStore(StoreTree):
 
     '''
 
-    node_size = 64 * 1024
-
     TYPE_MAX = 255
     SUBKEY_MAX = struct.pack('!Q', 2**64-1)
 
@@ -257,10 +253,10 @@ class GenerationStore(StoreTree):
     FILE_NAME = 0
     FILE_METADATA = 1
     
-    def __init__(self, fs, hostname):
+    def __init__(self, fs, hostname, node_size):
         key_bytes = len(self.key('', 0, 0))
         # FIXME: We should handle evil hostnames.
-        StoreTree.__init__(self, fs, hostname, key_bytes, self.node_size)
+        StoreTree.__init__(self, fs, hostname, key_bytes, node_size)
         self.curgen = None
 
     def hash_name(self, filename):
@@ -477,10 +473,10 @@ class ChecksumTree(StoreTree):
 
     '''
 
-    def __init__(self, fs, name, checksum_length):
+    def __init__(self, fs, name, checksum_length, node_size):
         self.sumlen = checksum_length
         key_bytes = len(self.key('', 0))
-        StoreTree.__init__(self, fs, name, key_bytes, 64*1024)
+        StoreTree.__init__(self, fs, name, key_bytes, node_size)
         self.max_id = 2**64 - 1
 
     def key(self, checksum, number):
@@ -526,9 +522,9 @@ class ChunkGroupTree(StoreTree):
     # We store things using the chunk group id as tkey key. The ids of
     # the chunks are stored as the value, as a blob, using struct.
 
-    def __init__(self, fs):
+    def __init__(self, fs, node_size):
         StoreTree.__init__(self, fs, 'chunkgroups', 
-                           len(self.key(0)), 64*1024)
+                           len(self.key(0)), node_size)
         self.max_id = 2**64 - 1
 
     def key(self, cgid):
@@ -621,10 +617,11 @@ class Store(object):
 
     '''
 
-    def __init__(self, fs):
+    def __init__(self, fs, node_size):
         self.fs = fs
+        self.node_size = node_size
         self.got_root_lock = False
-        self.hostlist = HostList(fs)
+        self.hostlist = HostList(fs, node_size)
         self.got_host_lock = False
         self.host_lockfile = None
         self.current_host = None
@@ -633,9 +630,11 @@ class Store(object):
         self.removed_hosts = []
         self.removed_generations = []
         self.genstore = None
-        self.chunksums = ChecksumTree(fs, 'chunksums', len(self.checksum('')))
-        self.groupsums = ChecksumTree(fs, 'groupsums', len(self.checksum('')))
-        self.chunkgroups = ChunkGroupTree(fs)
+        self.chunksums = ChecksumTree(fs, 'chunksums', len(self.checksum('')),
+                                      node_size)
+        self.groupsums = ChecksumTree(fs, 'groupsums', len(self.checksum('')),
+                                      node_size)
+        self.chunkgroups = ChunkGroupTree(fs, node_size)
         self.prev_chunkid = None
 
     def checksum(self, data):
@@ -743,7 +742,7 @@ class Store(object):
         self.current_host = hostname
         self.added_generations = []
         self.removed_generations = []
-        self.genstore = GenerationStore(self.fs, hostname)
+        self.genstore = GenerationStore(self.fs, hostname, self.node_size)
         self.genstore.require_forest()
 
     @require_host_lock
@@ -779,7 +778,7 @@ class Store(object):
         if hostname not in self.list_hosts():
             raise obnamlib.Error('%s is not an existing host' % hostname)
         self.current_host = hostname
-        self.genstore = GenerationStore(self.fs, hostname)
+        self.genstore = GenerationStore(self.fs, hostname, self.node_size)
         self.genstore.init_forest()
         
     @require_open_host
diff --git a/obnamlib/store_tests.py b/obnamlib/store_tests.py
index 7ac5c45c..3c399b3a 100644
--- a/obnamlib/store_tests.py
+++ b/obnamlib/store_tests.py
@@ -30,7 +30,8 @@ class ChecksumTreeTests(unittest.TestCase):
         self.tempdir = tempfile.mkdtemp()
         fs = obnamlib.LocalFS(self.tempdir)
         self.checksum = hashlib.md5('foo').digest()
-        self.tree = obnamlib.store.ChecksumTree(fs, 'x', len(self.checksum))
+        self.tree = obnamlib.store.ChecksumTree(fs, 'x', len(self.checksum), 
+                                                obnamlib.DEFAULT_NODE_SIZE)
 
     def tearDown(self):
         self.tree.commit()
@@ -67,7 +68,8 @@ class ChunkGroupTreeTests(unittest.TestCase):
     def setUp(self):
         self.tempdir = tempfile.mkdtemp()
         fs = obnamlib.LocalFS(self.tempdir)
-        self.tree = obnamlib.store.ChunkGroupTree(fs)
+        self.tree = obnamlib.store.ChunkGroupTree(fs, 
+                                                  obnamlib.DEFAULT_NODE_SIZE)
 
     def tearDown(self):
         shutil.rmtree(self.tempdir)
@@ -114,10 +116,10 @@ class StoreRootNodeTests(unittest.TestCase):
         self.tempdir = tempfile.mkdtemp()
 
         self.fs = obnamlib.LocalFS(self.tempdir)
-        self.store = obnamlib.Store(self.fs)
+        self.store = obnamlib.Store(self.fs, obnamlib.DEFAULT_NODE_SIZE)
         
         self.otherfs = obnamlib.LocalFS(self.tempdir)
-        self.other = obnamlib.Store(self.fs)
+        self.other = obnamlib.Store(self.fs, obnamlib.DEFAULT_NODE_SIZE)
 
     def tearDown(self):
         shutil.rmtree(self.tempdir)
@@ -181,7 +183,7 @@ class StoreRootNodeTests(unittest.TestCase):
         self.store.lock_root()
         self.store.add_host('foo')
         self.store.commit_root()
-        s2 = obnamlib.Store(self.fs)
+        s2 = obnamlib.Store(self.fs, obnamlib.DEFAULT_NODE_SIZE)
         self.assertEqual(s2.list_hosts(), ['foo'])
         
     def test_adding_existing_host_fails(self):
@@ -246,13 +248,13 @@ class StoreHostTests(unittest.TestCase):
         self.tempdir = tempfile.mkdtemp()
 
         self.fs = obnamlib.LocalFS(self.tempdir)
-        self.store = obnamlib.Store(self.fs)
+        self.store = obnamlib.Store(self.fs, obnamlib.DEFAULT_NODE_SIZE)
         self.store.lock_root()
         self.store.add_host('hostname')
         self.store.commit_root()
         
         self.otherfs = obnamlib.LocalFS(self.tempdir)
-        self.other = obnamlib.Store(self.otherfs)
+        self.other = obnamlib.Store(self.otherfs, obnamlib.DEFAULT_NODE_SIZE)
         
         self.dir_meta = obnamlib.Metadata()
         self.dir_meta.st_mode = stat.S_IFDIR | 0777
@@ -500,7 +502,7 @@ class StoreChunkTests(unittest.TestCase):
         self.tempdir = tempfile.mkdtemp()
 
         self.fs = obnamlib.LocalFS(self.tempdir)
-        self.store = obnamlib.Store(self.fs)
+        self.store = obnamlib.Store(self.fs, obnamlib.DEFAULT_NODE_SIZE)
         self.store.lock_root()
         self.store.add_host('hostname')
         self.store.commit_root()
@@ -559,7 +561,7 @@ class StoreChunkGroupTests(unittest.TestCase):
         self.tempdir = tempfile.mkdtemp()
 
         self.fs = obnamlib.LocalFS(self.tempdir)
-        self.store = obnamlib.Store(self.fs)
+        self.store = obnamlib.Store(self.fs, obnamlib.DEFAULT_NODE_SIZE)
         self.store.lock_root()
         self.store.add_host('hostname')
         self.store.commit_root()
@@ -607,7 +609,7 @@ class StoreGetSetChunksAndGroupsTests(unittest.TestCase):
         self.tempdir = tempfile.mkdtemp()
 
         self.fs = obnamlib.LocalFS(self.tempdir)
-        self.store = obnamlib.Store(self.fs)
+        self.store = obnamlib.Store(self.fs, obnamlib.DEFAULT_NODE_SIZE)
         self.store.lock_root()
         self.store.add_host('hostname')
         self.store.commit_root()
@@ -643,7 +645,7 @@ class StoreGenspecTests(unittest.TestCase):
 
         storedir = os.path.join(self.tempdir, 'store')
         fs = obnamlib.VfsFactory().new(storedir)
-        self.store = obnamlib.Store(fs)
+        self.store = obnamlib.Store(fs, obnamlib.DEFAULT_NODE_SIZE)
         self.store.lock_host('hostname')
 
     def tearDown(self):