diff options
author | Lars Wirzenius <liw@liw.fi> | 2010-07-04 17:17:06 +1200 |
---|---|---|
committer | Lars Wirzenius <liw@liw.fi> | 2010-07-04 17:17:06 +1200 |
commit | 824df096b82ef55d72742cdfc89d446325dd813b (patch) | |
tree | c3567c70e52b07a51f2b0dc3213e15e72e7452b3 | |
parent | ba786e32fcd4ea3feabdc16b4971538202110955 (diff) | |
parent | c92957fec8ed845e3a72b6104bf1b05ba0c0d2a5 (diff) | |
download | obnam-824df096b82ef55d72742cdfc89d446325dd813b.tar.gz |
Merge changes to allow user to choose node, chunk, chunk group sizes.
-rwxr-xr-x | blackboxtest | 8 | ||||
-rwxr-xr-x | dumpobjs | 2 | ||||
-rw-r--r-- | obnamlib/__init__.py | 5 | ||||
-rw-r--r-- | obnamlib/app.py | 16 | ||||
-rw-r--r-- | obnamlib/cfg.py | 6 | ||||
-rw-r--r-- | obnamlib/cfg_tests.py | 10 | ||||
-rw-r--r-- | obnamlib/plugins/backup_plugin.py | 8 | ||||
-rw-r--r-- | obnamlib/plugins/forget_plugin.py | 2 | ||||
-rw-r--r-- | obnamlib/plugins/fsck_plugin.py | 2 | ||||
-rw-r--r-- | obnamlib/plugins/restore_plugin.py | 2 | ||||
-rw-r--r-- | obnamlib/plugins/show_plugin.py | 2 | ||||
-rw-r--r-- | obnamlib/plugins/verify_plugin.py | 2 | ||||
-rw-r--r-- | obnamlib/store.py | 37 | ||||
-rw-r--r-- | obnamlib/store_tests.py | 24 |
14 files changed, 82 insertions, 44 deletions
diff --git a/blackboxtest b/blackboxtest index 61ab18b9..35108bed 100755 --- a/blackboxtest +++ b/blackboxtest @@ -390,7 +390,7 @@ class ReusesChunks(BlackBoxTest): self.verify(data, restored) fsf = obnamlib.VfsFactory() fs = fsf.new(store) - s = obnamlib.Store(fs) + s = obnamlib.Store(fs, obnamlib.DEFAULT_NODE_SIZE) s.open_host(self.hostid) self.assertEqual('ReusesChunks', len(s.list_chunks()), 1, 'only one chunk when all files are identical') @@ -401,7 +401,9 @@ class UsesChunkGroups(BlackBoxTest): def test(self): data = self.tempdir() ngroups = 2 - nbytes = obnamlib.CHUNK_SIZE * obnamlib.CHUNK_GROUP_SIZE * ngroups + nbytes = (obnamlib.DEFAULT_CHUNK_SIZE * + obnamlib.DEFAULT_CHUNK_GROUP_SIZE * + ngroups) self.create_file(data, 'file', 'x' * nbytes) store = self.tempdir() self.backup(store, [data]) @@ -409,7 +411,7 @@ class UsesChunkGroups(BlackBoxTest): self.verify(data, restored) fsf = obnamlib.VfsFactory() fs = fsf.new(store) - s = obnamlib.Store(fs) + s = obnamlib.Store(fs, obnamlib.DEFAULT_NODE_SIZE) s.open_host(self.hostid) self.assertEqual('UsesChunkGroups', len(s.list_chunk_groups()), ngroups, @@ -27,7 +27,7 @@ def find_objids(fs): fs = obnamlib.LocalFS(sys.argv[1]) -store = obnamlib.Store(fs) +store = obnamlib.Store(fs, obnamlib.DEFAULT_NODE_SIZE) for objid in find_objids(fs): obj = store.get_object(objid) print 'id %s (%s):' % (obj.id, obj.__class__.__name__) diff --git a/obnamlib/__init__.py b/obnamlib/__init__.py index b1952403..04872c94 100644 --- a/obnamlib/__init__.py +++ b/obnamlib/__init__.py @@ -26,8 +26,9 @@ class AppException(Exception): class Error(Exception): pass -CHUNK_SIZE = 4096 -CHUNK_GROUP_SIZE = 16 +DEFAULT_NODE_SIZE = 64 * 1024 +DEFAULT_CHUNK_SIZE = 4096 +DEFAULT_CHUNK_GROUP_SIZE = 16 from sizeparse import SizeSyntaxError, UnitNameError, ByteSizeParser diff --git a/obnamlib/app.py b/obnamlib/app.py index 8f7401f0..e3f299c6 100644 --- a/obnamlib/app.py +++ b/obnamlib/app.py @@ -42,6 +42,22 @@ class App(object): 'do not write or remove anything, just ' 'pretend to do that') + self.config.new_bytesize(['node-size'], + 'size of B-tree nodes on disk ' + '(default: %default)') + self.config['node-size'] = '%s' % obnamlib.DEFAULT_NODE_SIZE + + self.config.new_bytesize(['chunk-size'], + 'size of chunks of file data backed up ' + '(default: %default)') + self.config['chunk-size'] = '%s' % obnamlib.DEFAULT_CHUNK_SIZE + + self.config.new_bytesize(['chunk-group-size'], + 'number of chunks per chunk group ' + '(default: %default)') + self.config['chunk-group-size'] = \ + '%s' % obnamlib.DEFAULT_CHUNK_GROUP_SIZE + self.pm = obnamlib.PluginManager() self.pm.locations = [self.plugins_dir()] self.pm.plugin_arguments = (self,) diff --git a/obnamlib/cfg.py b/obnamlib/cfg.py index 89137a1d..ff635b32 100644 --- a/obnamlib/cfg.py +++ b/obnamlib/cfg.py @@ -95,6 +95,12 @@ class Configuration(object): self.new_string(names, help) self.processors[names[0]] = callback + def new_bytesize(self, names, help): + def callback(value): + p = obnamlib.ByteSizeParser() + return p.parse(value) + self.new_processed(names, help, callback) + def new_list(self, names, help): self.new_setting('list', names, help, 'append', []) diff --git a/obnamlib/cfg_tests.py b/obnamlib/cfg_tests.py index 248437fb..551a84ad 100644 --- a/obnamlib/cfg_tests.py +++ b/obnamlib/cfg_tests.py @@ -73,3 +73,13 @@ class ConfigurationTests(unittest.TestCase): self.cfg.load(['--size=123']) self.assertEqual(self.cfg['size'], 123) + def test_handles_unadorned_bytesize(self): + self.cfg.new_bytesize(['size'], 'size help') + self.cfg.load(['--size=123']) + self.assertEqual(self.cfg['size'], 123) + + def test_handles_bytesize_with_units(self): + self.cfg.new_bytesize(['size'], 'size help') + self.cfg.load(['--size=123KiB']) + self.assertEqual(self.cfg['size'], 123*1024) + diff --git a/obnamlib/plugins/backup_plugin.py b/obnamlib/plugins/backup_plugin.py index 0f7f4cdc..839ef7ae 100644 --- a/obnamlib/plugins/backup_plugin.py +++ b/obnamlib/plugins/backup_plugin.py @@ -55,7 +55,7 @@ class BackupPlugin(obnamlib.ObnamPlugin): storepath = self.app.config['store'] logging.debug('store: %s' % storepath) storefs = self.app.fsf.new(storepath) - self.store = obnamlib.Store(storefs) + self.store = obnamlib.Store(storefs, self.app.config['node-size']) hostname = self.app.config['hostname'] logging.debug('hostname: %s' % hostname) @@ -190,13 +190,15 @@ class BackupPlugin(obnamlib.ObnamPlugin): cgids = [] groupsum = self.store.new_checksummer() f = self.fs.open(filename, 'r') + chunk_size = int(self.app.config['chunk-size']) + chunk_group_size = int(self.app.config['chunk-group-size']) while True: - data = f.read(obnamlib.CHUNK_SIZE) + data = f.read(chunk_size) if not data: break chunkids.append(self.backup_file_chunk(data)) groupsum.update(data) - if len(chunkids) == obnamlib.CHUNK_GROUP_SIZE: + if len(chunkids) == chunk_group_size: checksum = groupsum.hexdigest() cgid = self.store.put_chunk_group(chunkids, checksum) cgids.append(cgid) diff --git a/obnamlib/plugins/forget_plugin.py b/obnamlib/plugins/forget_plugin.py index f9f074f9..f6a1c634 100644 --- a/obnamlib/plugins/forget_plugin.py +++ b/obnamlib/plugins/forget_plugin.py @@ -34,7 +34,7 @@ class ForgetPlugin(obnamlib.ObnamPlugin): self.app.config.require('hostname') fs = self.app.fsf.new(self.app.config['store']) - self.store = obnamlib.Store(fs) + self.store = obnamlib.Store(fs, self.app.config['node-size']) self.store.lock_host(self.app.config['hostname']) if args: diff --git a/obnamlib/plugins/fsck_plugin.py b/obnamlib/plugins/fsck_plugin.py index eef2c521..5ecc3bf1 100644 --- a/obnamlib/plugins/fsck_plugin.py +++ b/obnamlib/plugins/fsck_plugin.py @@ -31,7 +31,7 @@ class FsckPlugin(obnamlib.ObnamPlugin): storefs = self.app.fsf.new(self.app.config['store']) storefs.connect() - self.store = obnamlib.Store(storefs) + self.store = obnamlib.Store(storefs, self.app.config['node-size']) self.check_root() diff --git a/obnamlib/plugins/restore_plugin.py b/obnamlib/plugins/restore_plugin.py index 6f00f30e..11b2a22f 100644 --- a/obnamlib/plugins/restore_plugin.py +++ b/obnamlib/plugins/restore_plugin.py @@ -81,7 +81,7 @@ class RestorePlugin(obnamlib.ObnamPlugin): args = ['/'] storefs = self.app.fsf.new(self.app.config['store']) - self.store = obnamlib.Store(storefs) + self.store = obnamlib.Store(storefs, self.app.config['node-size']) self.store.open_host(self.app.config['hostname']) self.fs = self.app.fsf.new(self.app.config['to']) diff --git a/obnamlib/plugins/show_plugin.py b/obnamlib/plugins/show_plugin.py index 05cc6088..3ed00895 100644 --- a/obnamlib/plugins/show_plugin.py +++ b/obnamlib/plugins/show_plugin.py @@ -40,7 +40,7 @@ class ShowPlugin(obnamlib.ObnamPlugin): self.app.config.require('store') self.app.config.require('hostname') fs = self.app.fsf.new(self.app.config['store']) - self.store = obnamlib.Store(fs) + self.store = obnamlib.Store(fs, self.app.config['node-size']) self.store.open_host(self.app.config['hostname']) def hosts(self, args): diff --git a/obnamlib/plugins/verify_plugin.py b/obnamlib/plugins/verify_plugin.py index db272e3f..c24b8bfd 100644 --- a/obnamlib/plugins/verify_plugin.py +++ b/obnamlib/plugins/verify_plugin.py @@ -49,7 +49,7 @@ class VerifyPlugin(obnamlib.ObnamPlugin): fs = self.app.fsf.new(self.app.config['store']) fs.connect() - self.store = obnamlib.Store(fs) + self.store = obnamlib.Store(fs, self.app.config['node-size']) self.store.open_host(self.app.config['hostname']) self.fs = self.app.fsf.new(self.app.config['root'][0]) self.fs.connect() diff --git a/obnamlib/store.py b/obnamlib/store.py index 5696f291..dc93299e 100644 --- a/obnamlib/store.py +++ b/obnamlib/store.py @@ -158,11 +158,9 @@ class HostList(StoreTree): '''Store list of hosts.''' key_bytes = 8 # 64-bit counter as key - node_size = 4096 # typical size of disk block - def __init__(self, fs): - StoreTree.__init__(self, fs, 'hostlist', self.key_bytes, - self.node_size) + def __init__(self, fs, node_size): + StoreTree.__init__(self, fs, 'hostlist', self.key_bytes, node_size) self.minkey = self.key(0) self.maxkey = self.key(2**64-1) @@ -234,8 +232,6 @@ class GenerationStore(StoreTree): ''' - node_size = 64 * 1024 - TYPE_MAX = 255 SUBKEY_MAX = struct.pack('!Q', 2**64-1) @@ -257,10 +253,10 @@ class GenerationStore(StoreTree): FILE_NAME = 0 FILE_METADATA = 1 - def __init__(self, fs, hostname): + def __init__(self, fs, hostname, node_size): key_bytes = len(self.key('', 0, 0)) # FIXME: We should handle evil hostnames. - StoreTree.__init__(self, fs, hostname, key_bytes, self.node_size) + StoreTree.__init__(self, fs, hostname, key_bytes, node_size) self.curgen = None def hash_name(self, filename): @@ -477,10 +473,10 @@ class ChecksumTree(StoreTree): ''' - def __init__(self, fs, name, checksum_length): + def __init__(self, fs, name, checksum_length, node_size): self.sumlen = checksum_length key_bytes = len(self.key('', 0)) - StoreTree.__init__(self, fs, name, key_bytes, 64*1024) + StoreTree.__init__(self, fs, name, key_bytes, node_size) self.max_id = 2**64 - 1 def key(self, checksum, number): @@ -526,9 +522,9 @@ class ChunkGroupTree(StoreTree): # We store things using the chunk group id as tkey key. The ids of # the chunks are stored as the value, as a blob, using struct. - def __init__(self, fs): + def __init__(self, fs, node_size): StoreTree.__init__(self, fs, 'chunkgroups', - len(self.key(0)), 64*1024) + len(self.key(0)), node_size) self.max_id = 2**64 - 1 def key(self, cgid): @@ -621,10 +617,11 @@ class Store(object): ''' - def __init__(self, fs): + def __init__(self, fs, node_size): self.fs = fs + self.node_size = node_size self.got_root_lock = False - self.hostlist = HostList(fs) + self.hostlist = HostList(fs, node_size) self.got_host_lock = False self.host_lockfile = None self.current_host = None @@ -633,9 +630,11 @@ class Store(object): self.removed_hosts = [] self.removed_generations = [] self.genstore = None - self.chunksums = ChecksumTree(fs, 'chunksums', len(self.checksum(''))) - self.groupsums = ChecksumTree(fs, 'groupsums', len(self.checksum(''))) - self.chunkgroups = ChunkGroupTree(fs) + self.chunksums = ChecksumTree(fs, 'chunksums', len(self.checksum('')), + node_size) + self.groupsums = ChecksumTree(fs, 'groupsums', len(self.checksum('')), + node_size) + self.chunkgroups = ChunkGroupTree(fs, node_size) self.prev_chunkid = None def checksum(self, data): @@ -743,7 +742,7 @@ class Store(object): self.current_host = hostname self.added_generations = [] self.removed_generations = [] - self.genstore = GenerationStore(self.fs, hostname) + self.genstore = GenerationStore(self.fs, hostname, self.node_size) self.genstore.require_forest() @require_host_lock @@ -779,7 +778,7 @@ class Store(object): if hostname not in self.list_hosts(): raise obnamlib.Error('%s is not an existing host' % hostname) self.current_host = hostname - self.genstore = GenerationStore(self.fs, hostname) + self.genstore = GenerationStore(self.fs, hostname, self.node_size) self.genstore.init_forest() @require_open_host diff --git a/obnamlib/store_tests.py b/obnamlib/store_tests.py index 7ac5c45c..3c399b3a 100644 --- a/obnamlib/store_tests.py +++ b/obnamlib/store_tests.py @@ -30,7 +30,8 @@ class ChecksumTreeTests(unittest.TestCase): self.tempdir = tempfile.mkdtemp() fs = obnamlib.LocalFS(self.tempdir) self.checksum = hashlib.md5('foo').digest() - self.tree = obnamlib.store.ChecksumTree(fs, 'x', len(self.checksum)) + self.tree = obnamlib.store.ChecksumTree(fs, 'x', len(self.checksum), + obnamlib.DEFAULT_NODE_SIZE) def tearDown(self): self.tree.commit() @@ -67,7 +68,8 @@ class ChunkGroupTreeTests(unittest.TestCase): def setUp(self): self.tempdir = tempfile.mkdtemp() fs = obnamlib.LocalFS(self.tempdir) - self.tree = obnamlib.store.ChunkGroupTree(fs) + self.tree = obnamlib.store.ChunkGroupTree(fs, + obnamlib.DEFAULT_NODE_SIZE) def tearDown(self): shutil.rmtree(self.tempdir) @@ -114,10 +116,10 @@ class StoreRootNodeTests(unittest.TestCase): self.tempdir = tempfile.mkdtemp() self.fs = obnamlib.LocalFS(self.tempdir) - self.store = obnamlib.Store(self.fs) + self.store = obnamlib.Store(self.fs, obnamlib.DEFAULT_NODE_SIZE) self.otherfs = obnamlib.LocalFS(self.tempdir) - self.other = obnamlib.Store(self.fs) + self.other = obnamlib.Store(self.fs, obnamlib.DEFAULT_NODE_SIZE) def tearDown(self): shutil.rmtree(self.tempdir) @@ -181,7 +183,7 @@ class StoreRootNodeTests(unittest.TestCase): self.store.lock_root() self.store.add_host('foo') self.store.commit_root() - s2 = obnamlib.Store(self.fs) + s2 = obnamlib.Store(self.fs, obnamlib.DEFAULT_NODE_SIZE) self.assertEqual(s2.list_hosts(), ['foo']) def test_adding_existing_host_fails(self): @@ -246,13 +248,13 @@ class StoreHostTests(unittest.TestCase): self.tempdir = tempfile.mkdtemp() self.fs = obnamlib.LocalFS(self.tempdir) - self.store = obnamlib.Store(self.fs) + self.store = obnamlib.Store(self.fs, obnamlib.DEFAULT_NODE_SIZE) self.store.lock_root() self.store.add_host('hostname') self.store.commit_root() self.otherfs = obnamlib.LocalFS(self.tempdir) - self.other = obnamlib.Store(self.otherfs) + self.other = obnamlib.Store(self.otherfs, obnamlib.DEFAULT_NODE_SIZE) self.dir_meta = obnamlib.Metadata() self.dir_meta.st_mode = stat.S_IFDIR | 0777 @@ -500,7 +502,7 @@ class StoreChunkTests(unittest.TestCase): self.tempdir = tempfile.mkdtemp() self.fs = obnamlib.LocalFS(self.tempdir) - self.store = obnamlib.Store(self.fs) + self.store = obnamlib.Store(self.fs, obnamlib.DEFAULT_NODE_SIZE) self.store.lock_root() self.store.add_host('hostname') self.store.commit_root() @@ -559,7 +561,7 @@ class StoreChunkGroupTests(unittest.TestCase): self.tempdir = tempfile.mkdtemp() self.fs = obnamlib.LocalFS(self.tempdir) - self.store = obnamlib.Store(self.fs) + self.store = obnamlib.Store(self.fs, obnamlib.DEFAULT_NODE_SIZE) self.store.lock_root() self.store.add_host('hostname') self.store.commit_root() @@ -607,7 +609,7 @@ class StoreGetSetChunksAndGroupsTests(unittest.TestCase): self.tempdir = tempfile.mkdtemp() self.fs = obnamlib.LocalFS(self.tempdir) - self.store = obnamlib.Store(self.fs) + self.store = obnamlib.Store(self.fs, obnamlib.DEFAULT_NODE_SIZE) self.store.lock_root() self.store.add_host('hostname') self.store.commit_root() @@ -643,7 +645,7 @@ class StoreGenspecTests(unittest.TestCase): storedir = os.path.join(self.tempdir, 'store') fs = obnamlib.VfsFactory().new(storedir) - self.store = obnamlib.Store(fs) + self.store = obnamlib.Store(fs, obnamlib.DEFAULT_NODE_SIZE) self.store.lock_host('hostname') def tearDown(self): |