Add --node-size option and make everywhere use that.

12 files changed, 46 insertions, 39 deletions

diff --git a/blackboxtest b/blackboxtest
index 7f09bbf6..35108bed 100755
--- a/blackboxtest
+++ b/blackboxtest
@@ -390,7 +390,7 @@ class ReusesChunks(BlackBoxTest):
         self.verify(data, restored)
         fsf = obnamlib.VfsFactory()
         fs = fsf.new(store)
-        s = obnamlib.Store(fs)
+        s = obnamlib.Store(fs, obnamlib.DEFAULT_NODE_SIZE)
         s.open_host(self.hostid)
         self.assertEqual('ReusesChunks', len(s.list_chunks()), 1,
                          'only one chunk when all files are identical')
@@ -411,7 +411,7 @@ class UsesChunkGroups(BlackBoxTest):
         self.verify(data, restored)
         fsf = obnamlib.VfsFactory()
         fs = fsf.new(store)
-        s = obnamlib.Store(fs)
+        s = obnamlib.Store(fs, obnamlib.DEFAULT_NODE_SIZE)
         s.open_host(self.hostid)
         self.assertEqual('UsesChunkGroups', len(s.list_chunk_groups()), 
                          ngroups,
diff --git a/dumpobjs b/dumpobjs
index e0517cce..4edcd833 100755
--- a/dumpobjs
+++ b/dumpobjs
@@ -27,7 +27,7 @@ def find_objids(fs):
 
 
 fs = obnamlib.LocalFS(sys.argv[1])
-store = obnamlib.Store(fs)
+store = obnamlib.Store(fs, obnamlib.DEFAULT_NODE_SIZE)
 for objid in find_objids(fs):
     obj = store.get_object(objid)
     print 'id %s (%s):' % (obj.id, obj.__class__.__name__)
diff --git a/obnamlib/__init__.py b/obnamlib/__init__.py
index 1f53237f..04872c94 100644
--- a/obnamlib/__init__.py
+++ b/obnamlib/__init__.py
@@ -26,6 +26,7 @@ class AppException(Exception):
 class Error(Exception):
     pass
 
+DEFAULT_NODE_SIZE = 64 * 1024
 DEFAULT_CHUNK_SIZE = 4096
 DEFAULT_CHUNK_GROUP_SIZE = 16
 
diff --git a/obnamlib/app.py b/obnamlib/app.py
index 6ffe4623..e3f299c6 100644
--- a/obnamlib/app.py
+++ b/obnamlib/app.py
@@ -42,6 +42,11 @@ class App(object):
                                 'do not write or remove anything, just '
                                 'pretend to do that')
 
+        self.config.new_bytesize(['node-size'],
+                                 'size of B-tree nodes on disk '
+                                 '(default: %default)')
+        self.config['node-size'] = '%s' % obnamlib.DEFAULT_NODE_SIZE
+
         self.config.new_bytesize(['chunk-size'],
                                  'size of chunks of file data backed up '
                                  '(default: %default)')
diff --git a/obnamlib/plugins/backup_plugin.py b/obnamlib/plugins/backup_plugin.py
index 84dcb5c6..839ef7ae 100644
--- a/obnamlib/plugins/backup_plugin.py
+++ b/obnamlib/plugins/backup_plugin.py
@@ -55,7 +55,7 @@ class BackupPlugin(obnamlib.ObnamPlugin):
         storepath = self.app.config['store']
         logging.debug('store: %s' % storepath)
         storefs = self.app.fsf.new(storepath)
-        self.store = obnamlib.Store(storefs)
+        self.store = obnamlib.Store(storefs, self.app.config['node-size'])
 
         hostname = self.app.config['hostname']
         logging.debug('hostname: %s' % hostname)
diff --git a/obnamlib/plugins/forget_plugin.py b/obnamlib/plugins/forget_plugin.py
index f9f074f9..f6a1c634 100644
--- a/obnamlib/plugins/forget_plugin.py
+++ b/obnamlib/plugins/forget_plugin.py
@@ -34,7 +34,7 @@ class ForgetPlugin(obnamlib.ObnamPlugin):
         self.app.config.require('hostname')
 
         fs = self.app.fsf.new(self.app.config['store'])
-        self.store = obnamlib.Store(fs)
+        self.store = obnamlib.Store(fs, self.app.config['node-size'])
         self.store.lock_host(self.app.config['hostname'])
 
         if args:
diff --git a/obnamlib/plugins/fsck_plugin.py b/obnamlib/plugins/fsck_plugin.py
index eef2c521..5ecc3bf1 100644
--- a/obnamlib/plugins/fsck_plugin.py
+++ b/obnamlib/plugins/fsck_plugin.py
@@ -31,7 +31,7 @@ class FsckPlugin(obnamlib.ObnamPlugin):
 
         storefs = self.app.fsf.new(self.app.config['store'])
         storefs.connect()
-        self.store = obnamlib.Store(storefs)
+        self.store = obnamlib.Store(storefs, self.app.config['node-size'])
         
         self.check_root()
 
diff --git a/obnamlib/plugins/restore_plugin.py b/obnamlib/plugins/restore_plugin.py
index 6f00f30e..11b2a22f 100644
--- a/obnamlib/plugins/restore_plugin.py
+++ b/obnamlib/plugins/restore_plugin.py
@@ -81,7 +81,7 @@ class RestorePlugin(obnamlib.ObnamPlugin):
             args = ['/']
     
         storefs = self.app.fsf.new(self.app.config['store'])
-        self.store = obnamlib.Store(storefs)
+        self.store = obnamlib.Store(storefs, self.app.config['node-size'])
         self.store.open_host(self.app.config['hostname'])
         self.fs = self.app.fsf.new(self.app.config['to'])
 
diff --git a/obnamlib/plugins/show_plugin.py b/obnamlib/plugins/show_plugin.py
index 05cc6088..3ed00895 100644
--- a/obnamlib/plugins/show_plugin.py
+++ b/obnamlib/plugins/show_plugin.py
@@ -40,7 +40,7 @@ class ShowPlugin(obnamlib.ObnamPlugin):
         self.app.config.require('store')
         self.app.config.require('hostname')
         fs = self.app.fsf.new(self.app.config['store'])
-        self.store = obnamlib.Store(fs)
+        self.store = obnamlib.Store(fs, self.app.config['node-size'])
         self.store.open_host(self.app.config['hostname'])
 
     def hosts(self, args):
diff --git a/obnamlib/plugins/verify_plugin.py b/obnamlib/plugins/verify_plugin.py
index db272e3f..c24b8bfd 100644
--- a/obnamlib/plugins/verify_plugin.py
+++ b/obnamlib/plugins/verify_plugin.py
@@ -49,7 +49,7 @@ class VerifyPlugin(obnamlib.ObnamPlugin):
     
         fs = self.app.fsf.new(self.app.config['store'])
         fs.connect()
-        self.store = obnamlib.Store(fs)
+        self.store = obnamlib.Store(fs, self.app.config['node-size'])
         self.store.open_host(self.app.config['hostname'])
         self.fs = self.app.fsf.new(self.app.config['root'][0])
         self.fs.connect()
diff --git a/obnamlib/store.py b/obnamlib/store.py
index 5696f291..dc93299e 100644
--- a/obnamlib/store.py
+++ b/obnamlib/store.py
@@ -158,11 +158,9 @@ class HostList(StoreTree):
     '''Store list of hosts.'''
 
     key_bytes = 8 # 64-bit counter as key
-    node_size = 4096 # typical size of disk block
 
-    def __init__(self, fs):
-        StoreTree.__init__(self, fs, 'hostlist', self.key_bytes, 
-                           self.node_size)
+    def __init__(self, fs, node_size):
+        StoreTree.__init__(self, fs, 'hostlist', self.key_bytes, node_size)
         self.minkey = self.key(0)
         self.maxkey = self.key(2**64-1)
 
@@ -234,8 +232,6 @@ class GenerationStore(StoreTree):
 
     '''
 
-    node_size = 64 * 1024
-
     TYPE_MAX = 255
     SUBKEY_MAX = struct.pack('!Q', 2**64-1)
 
@@ -257,10 +253,10 @@ class GenerationStore(StoreTree):
     FILE_NAME = 0
     FILE_METADATA = 1
     
-    def __init__(self, fs, hostname):
+    def __init__(self, fs, hostname, node_size):
         key_bytes = len(self.key('', 0, 0))
         # FIXME: We should handle evil hostnames.
-        StoreTree.__init__(self, fs, hostname, key_bytes, self.node_size)
+        StoreTree.__init__(self, fs, hostname, key_bytes, node_size)
         self.curgen = None
 
     def hash_name(self, filename):
@@ -477,10 +473,10 @@ class ChecksumTree(StoreTree):
 
     '''
 
-    def __init__(self, fs, name, checksum_length):
+    def __init__(self, fs, name, checksum_length, node_size):
         self.sumlen = checksum_length
         key_bytes = len(self.key('', 0))
-        StoreTree.__init__(self, fs, name, key_bytes, 64*1024)
+        StoreTree.__init__(self, fs, name, key_bytes, node_size)
         self.max_id = 2**64 - 1
 
     def key(self, checksum, number):
@@ -526,9 +522,9 @@ class ChunkGroupTree(StoreTree):
     # We store things using the chunk group id as tkey key. The ids of
     # the chunks are stored as the value, as a blob, using struct.
 
-    def __init__(self, fs):
+    def __init__(self, fs, node_size):
         StoreTree.__init__(self, fs, 'chunkgroups', 
-                           len(self.key(0)), 64*1024)
+                           len(self.key(0)), node_size)
         self.max_id = 2**64 - 1
 
     def key(self, cgid):
@@ -621,10 +617,11 @@ class Store(object):
 
     '''
 
-    def __init__(self, fs):
+    def __init__(self, fs, node_size):
         self.fs = fs
+        self.node_size = node_size
         self.got_root_lock = False
-        self.hostlist = HostList(fs)
+        self.hostlist = HostList(fs, node_size)
         self.got_host_lock = False
         self.host_lockfile = None
         self.current_host = None
@@ -633,9 +630,11 @@ class Store(object):
         self.removed_hosts = []
         self.removed_generations = []
         self.genstore = None
-        self.chunksums = ChecksumTree(fs, 'chunksums', len(self.checksum('')))
-        self.groupsums = ChecksumTree(fs, 'groupsums', len(self.checksum('')))
-        self.chunkgroups = ChunkGroupTree(fs)
+        self.chunksums = ChecksumTree(fs, 'chunksums', len(self.checksum('')),
+                                      node_size)
+        self.groupsums = ChecksumTree(fs, 'groupsums', len(self.checksum('')),
+                                      node_size)
+        self.chunkgroups = ChunkGroupTree(fs, node_size)
         self.prev_chunkid = None
 
     def checksum(self, data):
@@ -743,7 +742,7 @@ class Store(object):
         self.current_host = hostname
         self.added_generations = []
         self.removed_generations = []
-        self.genstore = GenerationStore(self.fs, hostname)
+        self.genstore = GenerationStore(self.fs, hostname, self.node_size)
         self.genstore.require_forest()
 
     @require_host_lock
@@ -779,7 +778,7 @@ class Store(object):
         if hostname not in self.list_hosts():
             raise obnamlib.Error('%s is not an existing host' % hostname)
         self.current_host = hostname
-        self.genstore = GenerationStore(self.fs, hostname)
+        self.genstore = GenerationStore(self.fs, hostname, self.node_size)
         self.genstore.init_forest()
         
     @require_open_host
diff --git a/obnamlib/store_tests.py b/obnamlib/store_tests.py
index 7ac5c45c..3c399b3a 100644
--- a/obnamlib/store_tests.py
+++ b/obnamlib/store_tests.py
@@ -30,7 +30,8 @@ class ChecksumTreeTests(unittest.TestCase):
         self.tempdir = tempfile.mkdtemp()
         fs = obnamlib.LocalFS(self.tempdir)
         self.checksum = hashlib.md5('foo').digest()
-        self.tree = obnamlib.store.ChecksumTree(fs, 'x', len(self.checksum))
+        self.tree = obnamlib.store.ChecksumTree(fs, 'x', len(self.checksum), 
+                                                obnamlib.DEFAULT_NODE_SIZE)
 
     def tearDown(self):
         self.tree.commit()
@@ -67,7 +68,8 @@ class ChunkGroupTreeTests(unittest.TestCase):
     def setUp(self):
         self.tempdir = tempfile.mkdtemp()
         fs = obnamlib.LocalFS(self.tempdir)
-        self.tree = obnamlib.store.ChunkGroupTree(fs)
+        self.tree = obnamlib.store.ChunkGroupTree(fs, 
+                                                  obnamlib.DEFAULT_NODE_SIZE)
 
     def tearDown(self):
         shutil.rmtree(self.tempdir)
@@ -114,10 +116,10 @@ class StoreRootNodeTests(unittest.TestCase):
         self.tempdir = tempfile.mkdtemp()
 
         self.fs = obnamlib.LocalFS(self.tempdir)
-        self.store = obnamlib.Store(self.fs)
+        self.store = obnamlib.Store(self.fs, obnamlib.DEFAULT_NODE_SIZE)
         
         self.otherfs = obnamlib.LocalFS(self.tempdir)
-        self.other = obnamlib.Store(self.fs)
+        self.other = obnamlib.Store(self.fs, obnamlib.DEFAULT_NODE_SIZE)
 
     def tearDown(self):
         shutil.rmtree(self.tempdir)
@@ -181,7 +183,7 @@ class StoreRootNodeTests(unittest.TestCase):
         self.store.lock_root()
         self.store.add_host('foo')
         self.store.commit_root()
-        s2 = obnamlib.Store(self.fs)
+        s2 = obnamlib.Store(self.fs, obnamlib.DEFAULT_NODE_SIZE)
         self.assertEqual(s2.list_hosts(), ['foo'])
         
     def test_adding_existing_host_fails(self):
@@ -246,13 +248,13 @@ class StoreHostTests(unittest.TestCase):
         self.tempdir = tempfile.mkdtemp()
 
         self.fs = obnamlib.LocalFS(self.tempdir)
-        self.store = obnamlib.Store(self.fs)
+        self.store = obnamlib.Store(self.fs, obnamlib.DEFAULT_NODE_SIZE)
         self.store.lock_root()
         self.store.add_host('hostname')
         self.store.commit_root()
         
         self.otherfs = obnamlib.LocalFS(self.tempdir)
-        self.other = obnamlib.Store(self.otherfs)
+        self.other = obnamlib.Store(self.otherfs, obnamlib.DEFAULT_NODE_SIZE)
         
         self.dir_meta = obnamlib.Metadata()
         self.dir_meta.st_mode = stat.S_IFDIR | 0777
@@ -500,7 +502,7 @@ class StoreChunkTests(unittest.TestCase):
         self.tempdir = tempfile.mkdtemp()
 
         self.fs = obnamlib.LocalFS(self.tempdir)
-        self.store = obnamlib.Store(self.fs)
+        self.store = obnamlib.Store(self.fs, obnamlib.DEFAULT_NODE_SIZE)
         self.store.lock_root()
         self.store.add_host('hostname')
         self.store.commit_root()
@@ -559,7 +561,7 @@ class StoreChunkGroupTests(unittest.TestCase):
         self.tempdir = tempfile.mkdtemp()
 
         self.fs = obnamlib.LocalFS(self.tempdir)
-        self.store = obnamlib.Store(self.fs)
+        self.store = obnamlib.Store(self.fs, obnamlib.DEFAULT_NODE_SIZE)
         self.store.lock_root()
         self.store.add_host('hostname')
         self.store.commit_root()
@@ -607,7 +609,7 @@ class StoreGetSetChunksAndGroupsTests(unittest.TestCase):
         self.tempdir = tempfile.mkdtemp()
 
         self.fs = obnamlib.LocalFS(self.tempdir)
-        self.store = obnamlib.Store(self.fs)
+        self.store = obnamlib.Store(self.fs, obnamlib.DEFAULT_NODE_SIZE)
         self.store.lock_root()
         self.store.add_host('hostname')
         self.store.commit_root()
@@ -643,7 +645,7 @@ class StoreGenspecTests(unittest.TestCase):
 
         storedir = os.path.join(self.tempdir, 'store')
         fs = obnamlib.VfsFactory().new(storedir)
-        self.store = obnamlib.Store(fs)
+        self.store = obnamlib.Store(fs, obnamlib.DEFAULT_NODE_SIZE)
         self.store.lock_host('hostname')
 
     def tearDown(self):