Merge speedups to metadata coding.

3 files changed, 124 insertions, 3 deletions

diff --git a/metadata-speed b/metadata-speed
new file mode 100755
index 00000000..91ec5212
--- /dev/null
+++ b/metadata-speed
@@ -0,0 +1,48 @@
+#!/usr/bin/python
+# Copyright 2010  Lars Wirzenius
+# 
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+# 
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+import os
+import random
+import shutil
+import sys
+import time
+
+import obnamlib
+
+
+def measure(n, func):
+    start = time.clock()
+    for i in range(n):
+        func()
+    end = time.clock()
+    return end - start
+
+
+def main():
+    n = int(sys.argv[1])
+    fs = obnamlib.LocalFS('.')
+    fs.connect()
+    metadata = obnamlib.read_metadata(fs, '.')
+    encoded = obnamlib.store.encode_metadata(metadata)
+    calibrate = measure(n, lambda: None)
+    encode = measure(n, lambda: obnamlib.store.encode_metadata(metadata))
+    decode = measure(n, lambda: obnamlib.store.decode_metadata(encoded))
+    print 'encode: %.1f s' % (n/(encode - calibrate))
+    print 'decode: %.1f s' % (n/(decode - calibrate))
+
+if __name__ == '__main__':
+    main()
diff --git a/obnamlib/store.py b/obnamlib/store.py
index 8e6226b5..229bd16b 100644
--- a/obnamlib/store.py
+++ b/obnamlib/store.py
@@ -23,7 +23,6 @@ import btree
 import errno
 import hashlib
 import os
-import pickle
 import random
 import struct
 import time
@@ -84,12 +83,50 @@ def require_started_generation(method):
     return helper
 
 
+numeric_fields = [x for x in obnamlib.metadata_fields if x.startswith('st_')]
+string_fields = [x for x in obnamlib.metadata_fields 
+                 if x not in numeric_fields]
+all_fields = numeric_fields + string_fields
+num_numeric = len(numeric_fields)
+metadata_format = struct.Struct('!Q' + 'Q' * len(obnamlib.metadata_fields))
+
+
 def encode_metadata(metadata):
-    return pickle.dumps(metadata)
+    flags = 0
+    for i, name in enumerate(obnamlib.metadata_fields):
+        if getattr(metadata, name) is not None:
+            flags |= (1 << i)
+    fields = ([flags] +
+              [getattr(metadata, x) or 0 for x in numeric_fields] +
+              [len(getattr(metadata, x) or '') for x in string_fields])
+    string = ''.join(getattr(metadata, x) or '' for x in string_fields)
+    return metadata_format.pack(*fields) + string
     
 
+def flagtonone(flags, values):
+    for i, value in enumerate(values):
+        if flags & (1 << i):
+            yield value
+        else:
+            yield None
+
+
 def decode_metadata(encoded):
-    return pickle.loads(encoded)
+    buf = buffer(encoded)
+    items = metadata_format.unpack_from(buf)
+
+    flags = items[0]
+    values = list(items[1:len(numeric_fields)+1])
+    lengths = items[len(numeric_fields)+1:]
+    
+    offset = metadata_format.size
+    append = values.append
+    for length in lengths:
+        append(encoded[offset:offset + length])
+        offset += length
+
+    args = dict(zip(all_fields, flagtonone(flags, values)))
+    return obnamlib.Metadata(**args)
 
 
 class NodeStoreVfs(btree.NodeStoreDisk):
diff --git a/obnamlib/store_tests.py b/obnamlib/store_tests.py
index 38b8daef..75f56bfc 100644
--- a/obnamlib/store_tests.py
+++ b/obnamlib/store_tests.py
@@ -24,6 +24,42 @@ import unittest
 import obnamlib
 
 
+class MetadataCodingTests(unittest.TestCase):
+
+    def test_round_trip(self):
+        metadata = obnamlib.metadata.Metadata(st_mode=1, 
+                                              st_mtime=2, 
+                                              st_nlink=3,
+                                              st_size=4, 
+                                              st_uid=5, 
+                                              st_blocks=6, 
+                                              st_dev=7,
+                                              st_gid=8, 
+                                              st_ino=9,  
+                                              st_atime=10, 
+                                              groupname='group',
+                                              username='user',
+                                              target='target')
+        encoded = obnamlib.store.encode_metadata(metadata)
+        decoded = obnamlib.store.decode_metadata(encoded)
+        for name in dir(metadata):
+            if name in obnamlib.metadata.metadata_fields:
+                self.assertEqual(getattr(metadata, name), 
+                                 getattr(decoded, name),
+                                 'attribute %s must be equal (%s vs %s)' % 
+                                    (name, getattr(metadata, name),
+                                     getattr(decoded, name)))
+
+    def test_round_trip_for_None_values(self):
+        metadata = obnamlib.metadata.Metadata()
+        encoded = obnamlib.store.encode_metadata(metadata)
+        decoded = obnamlib.store.decode_metadata(encoded)
+        for name in dir(metadata):
+            if name in obnamlib.metadata.metadata_fields:
+                self.assertEqual(getattr(decoded, name), None,
+                                 'attribute %s must be None' % name)
+
+
 class ChecksumTreeTests(unittest.TestCase):
 
     def setUp(self):