diff options
author | Lars Wirzenius <liw@liw.fi> | 2014-01-23 20:11:19 +0000 |
---|---|---|
committer | Lars Wirzenius <liw@liw.fi> | 2014-01-23 20:11:19 +0000 |
commit | bc0180fc5746f8f8de0453f6dc495af35c0ca837 (patch) | |
tree | 4e465f601e7596ce5ecfb7ca4b93d6afdd307208 | |
parent | 1b385fcc1bd758ffb801661112d78174b86acbe4 (diff) | |
download | obnam-bc0180fc5746f8f8de0453f6dc495af35c0ca837.tar.gz |
Split of metadata encoding for fmt6 and general stuff
-rwxr-xr-x | metadata-speed | 8 | ||||
-rw-r--r-- | obnamlib/__init__.py | 7 | ||||
-rw-r--r-- | obnamlib/fmt_6/__init__.py | 1 | ||||
-rw-r--r-- | obnamlib/fmt_6/clientmetadatatree.py | 8 | ||||
-rw-r--r-- | obnamlib/fmt_6/clientmetadatatree_tests.py | 9 | ||||
-rw-r--r-- | obnamlib/fmt_6/metadata_codec.py | 136 | ||||
-rw-r--r-- | obnamlib/fmt_6/metadata_codec_tests.py | 85 | ||||
-rw-r--r-- | obnamlib/fmt_6/repo_fmt_6.py | 9 | ||||
-rw-r--r-- | obnamlib/metadata.py (renamed from obnamlib/fmt_6/metadata.py) | 118 | ||||
-rw-r--r-- | obnamlib/metadata_tests.py (renamed from obnamlib/fmt_6/metadata_tests.py) | 66 | ||||
-rw-r--r-- | obnamlib/repo.py | 4 |
11 files changed, 248 insertions, 203 deletions
diff --git a/metadata-speed b/metadata-speed index 23cb1825..51d94405 100755 --- a/metadata-speed +++ b/metadata-speed @@ -37,10 +37,12 @@ def main(): fs = obnamlib.LocalFS('.') fs.connect() metadata = obnamlib.read_metadata(fs, '.') - encoded = obnamlib.encode_metadata(metadata) + encoded = obnamlib.fmt_6.metadata_codec.encode_metadata(metadata) calibrate = measure(n, lambda: None) - encode = measure(n, lambda: obnamlib.encode_metadata(metadata)) - decode = measure(n, lambda: obnamlib.decode_metadata(encoded)) + encode = measure( + n, lambda: obnamlib.fmt_6.metadata_codec.encode_metadata(metadata)) + decode = measure( + n, lambda: obnamlib.fmt_6.metadata_codec.decode_metadata(encoded)) print 'encode: %.1f s' % (n/(encode - calibrate)) print 'decode: %.1f s' % (n/(decode - calibrate)) diff --git a/obnamlib/__init__.py b/obnamlib/__init__.py index 70c3ee57..b70a4174 100644 --- a/obnamlib/__init__.py +++ b/obnamlib/__init__.py @@ -154,14 +154,11 @@ from repo_dummy import RepositoryFormatDummy # Repository format 6 specific modules. # -from fmt_6.metadata import ( +from metadata import ( Metadata, read_metadata, set_metadata, - metadata_fields, - metadata_verify_fields, - encode_metadata, - decode_metadata) + metadata_fields) from fmt_6.repo_fmt_6 import RepositoryFormat6 from fmt_6.repo_tree import RepositoryTree from fmt_6.chunklist import ChunkList diff --git a/obnamlib/fmt_6/__init__.py b/obnamlib/fmt_6/__init__.py index e69de29b..bfad0228 100644 --- a/obnamlib/fmt_6/__init__.py +++ b/obnamlib/fmt_6/__init__.py @@ -0,0 +1 @@ +import metadata_codec diff --git a/obnamlib/fmt_6/clientmetadatatree.py b/obnamlib/fmt_6/clientmetadatatree.py index 1019f71b..375ec8f0 100644 --- a/obnamlib/fmt_6/clientmetadatatree.py +++ b/obnamlib/fmt_6/clientmetadatatree.py @@ -356,9 +356,10 @@ class ClientMetadataTree(obnamlib.RepositoryTree): except KeyError: old_metadata = None else: # pragma: no cover - old = obnamlib.decode_metadata(old_metadata) + old = obnamlib.fmt_6.metadata_codec.decode_metadata(old_metadata) - metadata = obnamlib.decode_metadata(encoded_metadata) + metadata = obnamlib.fmt_6.metadata_codec.decode_metadata( + encoded_metadata) if encoded_metadata != old_metadata: tracing.trace('new or changed metadata') @@ -409,7 +410,8 @@ class ClientMetadataTree(obnamlib.RepositoryTree): except KeyError: pass else: - metadata = obnamlib.decode_metadata(encoded_metadata) + metadata = obnamlib.fmt_6.metadata_codec.decode_metadata( + encoded_metadata) # Remove any children. minkey = self.fskey(file_id, self.DIR_CONTENTS, 0) diff --git a/obnamlib/fmt_6/clientmetadatatree_tests.py b/obnamlib/fmt_6/clientmetadatatree_tests.py index 07bf92ae..4fc67a90 100644 --- a/obnamlib/fmt_6/clientmetadatatree_tests.py +++ b/obnamlib/fmt_6/clientmetadatatree_tests.py @@ -41,7 +41,8 @@ class ClientMetadataTreeTests(unittest.TestCase): self.file_size = 123 self.file_metadata = obnamlib.Metadata(st_mode=stat.S_IFREG | 0666, st_size=self.file_size) - self.file_encoded = obnamlib.encode_metadata(self.file_metadata) + self.file_encoded = obnamlib.fmt_6.metadata_codec.encode_metadata( + self.file_metadata) def tearDown(self): shutil.rmtree(self.tempdir) @@ -155,9 +156,11 @@ class ClientMetadataTreeFileOpsTests(unittest.TestCase): self.client.start_generation() self.clientid = self.client.get_generation_id(self.client.tree) self.file_metadata = obnamlib.Metadata(st_mode=stat.S_IFREG | 0666) - self.file_encoded = obnamlib.encode_metadata(self.file_metadata) + self.file_encoded = obnamlib.fmt_6.metadata_codec.encode_metadata( + self.file_metadata) self.dir_metadata = obnamlib.Metadata(st_mode=stat.S_IFDIR | 0777) - self.dir_encoded = obnamlib.encode_metadata(self.dir_metadata) + self.dir_encoded = obnamlib.fmt_6.metadata_codec.encode_metadata( + self.dir_metadata) def tearDown(self): shutil.rmtree(self.tempdir) diff --git a/obnamlib/fmt_6/metadata_codec.py b/obnamlib/fmt_6/metadata_codec.py new file mode 100644 index 00000000..93c93bdb --- /dev/null +++ b/obnamlib/fmt_6/metadata_codec.py @@ -0,0 +1,136 @@ +# Copyright (C) 2009-2014 Lars Wirzenius +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + + +import struct + +import obnamlib + + +metadata_format = struct.Struct('!Q' + # flags + 'Q' + # st_mode + 'qQ' + # st_mtime_sec and _nsec + 'qQ' + # st_atime_sec and _nsec + 'Q' + # st_nlink + 'Q' + # st_size + 'Q' + # st_uid + 'Q' + # st_gid + 'Q' + # st_dev + 'Q' + # st_ino + 'Q' + # st_blocks + 'Q' + # len of groupname + 'Q' + # len of username + 'Q' + # len of symlink target + 'Q' + # len of md5 + 'Q' + # len of xattr + '') + +def encode_metadata(metadata): + flags = 0 + for i, name in enumerate(obnamlib.metadata_fields): + if getattr(metadata, name) is not None: + flags |= (1 << i) + + try: + packed = metadata_format.pack(flags, + metadata.st_mode or 0, + metadata.st_mtime_sec or 0, + metadata.st_mtime_nsec or 0, + metadata.st_atime_sec or 0, + metadata.st_atime_nsec or 0, + metadata.st_nlink or 0, + metadata.st_size or 0, + metadata.st_uid or 0, + metadata.st_gid or 0, + metadata.st_dev or 0, + metadata.st_ino or 0, + metadata.st_blocks or 0, + len(metadata.groupname or ''), + len(metadata.username or ''), + len(metadata.target or ''), + len(metadata.md5 or ''), + len(metadata.xattr or '')) + except TypeError, e: # pragma: no cover + logging.error('ERROR: Packing error due to %s' % str(e)) + logging.error('ERROR: st_mode=%s' % repr(metadata.st_mode)) + logging.error('ERROR: st_mtime_sec=%s' % repr(metadata.st_mtime_sec)) + logging.error('ERROR: st_mtime_nsec=%s' % repr(metadata.st_mtime_nsec)) + logging.error('ERROR: st_atime_sec=%s' % repr(metadata.st_atime_sec)) + logging.error('ERROR: st_atime_nsec=%s' % repr(metadata.st_atime_nsec)) + logging.error('ERROR: st_nlink=%s' % repr(metadata.st_nlink)) + logging.error('ERROR: st_size=%s' % repr(metadata.st_size)) + logging.error('ERROR: st_uid=%s' % repr(metadata.st_uid)) + logging.error('ERROR: st_gid=%s' % repr(metadata.st_gid)) + logging.error('ERROR: st_dev=%s' % repr(metadata.st_dev)) + logging.error('ERROR: st_ino=%s' % repr(metadata.st_ino)) + logging.error('ERROR: st_blocks=%s' % repr(metadata.st_blocks)) + logging.error('ERROR: groupname=%s' % repr(metadata.groupname)) + logging.error('ERROR: username=%s' % repr(metadata.username)) + logging.error('ERROR: target=%s' % repr(metadata.target)) + logging.error('ERROR: md5=%s' % repr(metadata.md5)) + logging.error('ERROR: xattr=%s' % repr(metadata.xattr)) + raise + return (packed + + (metadata.groupname or '') + + (metadata.username or '') + + (metadata.target or '') + + (metadata.md5 or '') + + (metadata.xattr or '')) + + +def decode_metadata(encoded): + + items = metadata_format.unpack_from(encoded) + flags = items[0] + pos = [1, metadata_format.size] + metadata = obnamlib.Metadata() + + def is_present(field): + i = obnamlib.metadata_fields.index(field) + return (flags & (1 << i)) != 0 + + def decode(field, num_items, inc_offset, getvalue): + if is_present(field): + value = getvalue(pos[0], pos[1]) + setattr(metadata, field, value) + if inc_offset: + pos[1] += len(value) + pos[0] += num_items + + def decode_integer(field): + decode(field, 1, False, lambda i, o: items[i]) + + def decode_string(field): + decode(field, 1, True, lambda i, o: encoded[o:o + items[i]]) + + decode_integer('st_mode') + decode_integer('st_mtime_sec') + decode_integer('st_mtime_nsec') + decode_integer('st_atime_sec') + decode_integer('st_atime_nsec') + decode_integer('st_nlink') + decode_integer('st_size') + decode_integer('st_uid') + decode_integer('st_gid') + decode_integer('st_dev') + decode_integer('st_ino') + decode_integer('st_blocks') + decode_string('groupname') + decode_string('username') + decode_string('target') + decode_string('md5') + decode_string('xattr') + + return metadata diff --git a/obnamlib/fmt_6/metadata_codec_tests.py b/obnamlib/fmt_6/metadata_codec_tests.py new file mode 100644 index 00000000..47f8b312 --- /dev/null +++ b/obnamlib/fmt_6/metadata_codec_tests.py @@ -0,0 +1,85 @@ +# Copyright (C) 2009-2014 Lars Wirzenius +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + + +import unittest + +import obnamlib + + +class MetadataCodingTests(unittest.TestCase): + + def equal(self, meta1, meta2): + for name in dir(meta1): + if name in obnamlib.metadata_fields: + value1 = getattr(meta1, name) + value2 = getattr(meta2, name) + self.assertEqual( + value1, + value2, + 'attribute %s must be equal (%s vs %s)' % + (name, value1, value2)) + + def test_round_trip(self): + metadata = obnamlib.Metadata( + st_mode=1, + st_mtime_sec=2, + st_mtime_nsec=12756, + st_nlink=3, + st_size=4, + st_uid=5, + st_blocks=6, + st_dev=7, + st_gid=8, + st_ino=9, + st_atime_sec=10, + st_atime_nsec=123, + groupname='group', + username='user', + target='target', + md5='checksum') + encoded = obnamlib.fmt_6.metadata_codec.encode_metadata(metadata) + decoded = obnamlib.fmt_6.metadata_codec.decode_metadata(encoded) + self.equal(metadata, decoded) + + def test_round_trip_for_None_values(self): + metadata = obnamlib.Metadata() + encoded = obnamlib.fmt_6.metadata_codec.encode_metadata(metadata) + decoded = obnamlib.fmt_6.metadata_codec.decode_metadata(encoded) + for name in dir(metadata): + if name in obnamlib.metadata_fields: + self.assertEqual(getattr(decoded, name), None, + 'attribute %s must be None' % name) + + def test_round_trip_for_maximum_values(self): + unsigned_max = 2**64 - 1 + signed_max = 2**63 - 1 + metadata = obnamlib.Metadata( + st_mode=unsigned_max, + st_mtime_sec=signed_max, + st_mtime_nsec=unsigned_max, + st_nlink=unsigned_max, + st_size=signed_max, + st_uid=unsigned_max, + st_blocks=signed_max, + st_dev=unsigned_max, + st_gid=unsigned_max, + st_ino=unsigned_max, + st_atime_sec=signed_max, + st_atime_nsec=unsigned_max) + encoded = obnamlib.fmt_6.metadata_codec.encode_metadata(metadata) + decoded = obnamlib.fmt_6.metadata_codec.decode_metadata(encoded) + self.equal(metadata, decoded) + diff --git a/obnamlib/fmt_6/repo_fmt_6.py b/obnamlib/fmt_6/repo_fmt_6.py index df06eacb..185e9ed5 100644 --- a/obnamlib/fmt_6/repo_fmt_6.py +++ b/obnamlib/fmt_6/repo_fmt_6.py @@ -847,7 +847,8 @@ class RepositoryFormat6(obnamlib.RepositoryInterface): client_name, gen_number = self._unpack_gen_id(generation_id) self._require_client_lock(client_name) client = self._open_client(client_name) - encoded_metadata = obnamlib.encode_metadata(obnamlib.Metadata()) + encoded_metadata = obnamlib.fmt_6.metadata_codec.encode_metadata( + obnamlib.Metadata()) client.create(filename, encoded_metadata) def remove_file(self, generation_id, filename): @@ -883,7 +884,8 @@ class RepositoryFormat6(obnamlib.RepositoryInterface): client_name, generation_number = self._unpack_gen_id(generation_id) dirty, metadata = value if dirty: - encoded_metadata = obnamlib.encode_metadata(metadata) + encoded_metadata = \ + obnamlib.fmt_6.metadata_codec.encode_metadata(metadata) client = self._open_client(client_name) # FIXME: Only sets in unfinished generation client.set_metadata(filename, encoded_metadata) @@ -893,7 +895,8 @@ class RepositoryFormat6(obnamlib.RepositoryInterface): client_name, gen_number = self._unpack_gen_id(generation_id) client = self._open_client(client_name) encoded_metadata = client.get_metadata(gen_number, filename) - metadata = obnamlib.decode_metadata(encoded_metadata) + metadata = obnamlib.fmt_6.metadata_codec.decode_metadata( + encoded_metadata) cache_key = self._get_file_key_cache_key(generation_id, filename) self._file_key_cache[cache_key] = (False, metadata) diff --git a/obnamlib/fmt_6/metadata.py b/obnamlib/metadata.py index 61daa655..1dea8dfc 100644 --- a/obnamlib/fmt_6/metadata.py +++ b/obnamlib/metadata.py @@ -265,121 +265,3 @@ def set_metadata(fs, filename, metadata, fs.lutimes(filename, metadata.st_atime_sec, metadata.st_atime_nsec, metadata.st_mtime_sec, metadata.st_mtime_nsec) - - -metadata_format = struct.Struct('!Q' + # flags - 'Q' + # st_mode - 'qQ' + # st_mtime_sec and _nsec - 'qQ' + # st_atime_sec and _nsec - 'Q' + # st_nlink - 'Q' + # st_size - 'Q' + # st_uid - 'Q' + # st_gid - 'Q' + # st_dev - 'Q' + # st_ino - 'Q' + # st_blocks - 'Q' + # len of groupname - 'Q' + # len of username - 'Q' + # len of symlink target - 'Q' + # len of md5 - 'Q' + # len of xattr - '') - -def encode_metadata(metadata): - flags = 0 - for i, name in enumerate(obnamlib.metadata_fields): - if getattr(metadata, name) is not None: - flags |= (1 << i) - - try: - packed = metadata_format.pack(flags, - metadata.st_mode or 0, - metadata.st_mtime_sec or 0, - metadata.st_mtime_nsec or 0, - metadata.st_atime_sec or 0, - metadata.st_atime_nsec or 0, - metadata.st_nlink or 0, - metadata.st_size or 0, - metadata.st_uid or 0, - metadata.st_gid or 0, - metadata.st_dev or 0, - metadata.st_ino or 0, - metadata.st_blocks or 0, - len(metadata.groupname or ''), - len(metadata.username or ''), - len(metadata.target or ''), - len(metadata.md5 or ''), - len(metadata.xattr or '')) - except TypeError, e: # pragma: no cover - logging.error('ERROR: Packing error due to %s' % str(e)) - logging.error('ERROR: st_mode=%s' % repr(metadata.st_mode)) - logging.error('ERROR: st_mtime_sec=%s' % repr(metadata.st_mtime_sec)) - logging.error('ERROR: st_mtime_nsec=%s' % repr(metadata.st_mtime_nsec)) - logging.error('ERROR: st_atime_sec=%s' % repr(metadata.st_atime_sec)) - logging.error('ERROR: st_atime_nsec=%s' % repr(metadata.st_atime_nsec)) - logging.error('ERROR: st_nlink=%s' % repr(metadata.st_nlink)) - logging.error('ERROR: st_size=%s' % repr(metadata.st_size)) - logging.error('ERROR: st_uid=%s' % repr(metadata.st_uid)) - logging.error('ERROR: st_gid=%s' % repr(metadata.st_gid)) - logging.error('ERROR: st_dev=%s' % repr(metadata.st_dev)) - logging.error('ERROR: st_ino=%s' % repr(metadata.st_ino)) - logging.error('ERROR: st_blocks=%s' % repr(metadata.st_blocks)) - logging.error('ERROR: groupname=%s' % repr(metadata.groupname)) - logging.error('ERROR: username=%s' % repr(metadata.username)) - logging.error('ERROR: target=%s' % repr(metadata.target)) - logging.error('ERROR: md5=%s' % repr(metadata.md5)) - logging.error('ERROR: xattr=%s' % repr(metadata.xattr)) - raise - return (packed + - (metadata.groupname or '') + - (metadata.username or '') + - (metadata.target or '') + - (metadata.md5 or '') + - (metadata.xattr or '')) - - -def decode_metadata(encoded): - - items = metadata_format.unpack_from(encoded) - flags = items[0] - pos = [1, metadata_format.size] - metadata = obnamlib.Metadata() - - def is_present(field): - i = obnamlib.metadata_fields.index(field) - return (flags & (1 << i)) != 0 - - def decode(field, num_items, inc_offset, getvalue): - if is_present(field): - value = getvalue(pos[0], pos[1]) - setattr(metadata, field, value) - if inc_offset: - pos[1] += len(value) - pos[0] += num_items - - def decode_integer(field): - decode(field, 1, False, lambda i, o: items[i]) - - def decode_string(field): - decode(field, 1, True, lambda i, o: encoded[o:o + items[i]]) - - decode_integer('st_mode') - decode_integer('st_mtime_sec') - decode_integer('st_mtime_nsec') - decode_integer('st_atime_sec') - decode_integer('st_atime_nsec') - decode_integer('st_nlink') - decode_integer('st_size') - decode_integer('st_uid') - decode_integer('st_gid') - decode_integer('st_dev') - decode_integer('st_ino') - decode_integer('st_blocks') - decode_string('groupname') - decode_string('username') - decode_string('target') - decode_string('md5') - decode_string('xattr') - - return metadata - diff --git a/obnamlib/fmt_6/metadata_tests.py b/obnamlib/metadata_tests.py index 49376dd9..9ee15f41 100644 --- a/obnamlib/fmt_6/metadata_tests.py +++ b/obnamlib/metadata_tests.py @@ -248,69 +248,3 @@ class SetMetadataTests(unittest.TestCase): self.assertEqual(st.st_mode & stat.S_ISUID, stat.S_ISUID) self.assertEqual(st.st_mode & stat.S_ISGID, stat.S_ISGID) - - -class MetadataCodingTests(unittest.TestCase): - - def equal(self, meta1, meta2): - for name in dir(meta1): - if name in obnamlib.metadata_fields: - value1 = getattr(meta1, name) - value2 = getattr(meta2, name) - self.assertEqual( - value1, - value2, - 'attribute %s must be equal (%s vs %s)' % - (name, value1, value2)) - - def test_round_trip(self): - metadata = obnamlib.Metadata( - st_mode=1, - st_mtime_sec=2, - st_mtime_nsec=12756, - st_nlink=3, - st_size=4, - st_uid=5, - st_blocks=6, - st_dev=7, - st_gid=8, - st_ino=9, - st_atime_sec=10, - st_atime_nsec=123, - groupname='group', - username='user', - target='target', - md5='checksum') - encoded = obnamlib.encode_metadata(metadata) - decoded = obnamlib.decode_metadata(encoded) - self.equal(metadata, decoded) - - def test_round_trip_for_None_values(self): - metadata = obnamlib.Metadata() - encoded = obnamlib.encode_metadata(metadata) - decoded = obnamlib.decode_metadata(encoded) - for name in dir(metadata): - if name in obnamlib.metadata_fields: - self.assertEqual(getattr(decoded, name), None, - 'attribute %s must be None' % name) - - def test_round_trip_for_maximum_values(self): - unsigned_max = 2**64 - 1 - signed_max = 2**63 - 1 - metadata = obnamlib.Metadata( - st_mode=unsigned_max, - st_mtime_sec=signed_max, - st_mtime_nsec=unsigned_max, - st_nlink=unsigned_max, - st_size=signed_max, - st_uid=unsigned_max, - st_blocks=signed_max, - st_dev=unsigned_max, - st_gid=unsigned_max, - st_ino=unsigned_max, - st_atime_sec=signed_max, - st_atime_nsec=unsigned_max) - encoded = obnamlib.encode_metadata(metadata) - decoded = obnamlib.decode_metadata(encoded) - self.equal(metadata, decoded) - diff --git a/obnamlib/repo.py b/obnamlib/repo.py index 12b6572b..f816e401 100644 --- a/obnamlib/repo.py +++ b/obnamlib/repo.py @@ -602,12 +602,12 @@ class Repository(object): encoded = self.client.get_metadata(gen, filename) except KeyError: raise obnamlib.Error('%s does not exist' % filename) - return obnamlib.decode_metadata(encoded) + return obnamlib.fmt_6.metadata_codec.decode_metadata(encoded) def create(self, filename, metadata): '''Create a new (empty) file in the new generation.''' self.require_started_generation() - encoded = obnamlib.encode_metadata(metadata) + encoded = obnamlib.fmt_6.metadata_codec.encode_metadata(metadata) self.client.create(filename, encoded) def remove(self, filename): |