Merge changes to implement checkpoints during backups.

12 files changed, 261 insertions, 7 deletions

diff --git a/obnamlib/__init__.py b/obnamlib/__init__.py
index 79bb17ef..b1952403 100644
--- a/obnamlib/__init__.py
+++ b/obnamlib/__init__.py
@@ -29,6 +29,8 @@ class Error(Exception):
 CHUNK_SIZE = 4096
 CHUNK_GROUP_SIZE = 16
 
+from sizeparse import SizeSyntaxError, UnitNameError, ByteSizeParser
+
 from hooks import Hook, HookManager
 from cfg import Configuration
 from interp import Interpreter
diff --git a/obnamlib/cfg.py b/obnamlib/cfg.py
index 2b960223..89137a1d 100644
--- a/obnamlib/cfg.py
+++ b/obnamlib/cfg.py
@@ -73,6 +73,7 @@ class Configuration(object):
         self.settings = {}
         self.parser = optparse.OptionParser()
         self.args = []
+        self.processors = {}
 
     def new_setting(self, kind, names, help, action, value):
         setting = Setting(kind, copy.copy(value))
@@ -89,6 +90,10 @@ class Configuration(object):
         
     def new_string(self, names, help):
         self.new_setting('str', names, help, 'store', '')
+        
+    def new_processed(self, names, help, callback):
+        self.new_string(names, help)
+        self.processors[names[0]] = callback
 
     def new_list(self, names, help):
         self.new_setting('list', names, help, 'append', [])
@@ -114,6 +119,8 @@ class Configuration(object):
                         item = [s.strip() for s in item.split(',')]
                         self.settings[name].value += item
                 else:
+                    if name in self.processors:
+                        value = self.processors[name](value)
                     self.settings[name].value = value
 
     def require(self, name):
diff --git a/obnamlib/cfg_tests.py b/obnamlib/cfg_tests.py
index 94639e17..248437fb 100644
--- a/obnamlib/cfg_tests.py
+++ b/obnamlib/cfg_tests.py
@@ -66,3 +66,10 @@ class ConfigurationTests(unittest.TestCase):
         self.cfg.load(['--foo'])
         self.assertEqual(self.cfg.require('foo'), None)
 
+    def test_calls_callback_for_processed_option(self):
+        def callback(value):
+            return int(value)
+        self.cfg.new_processed(['size'], 'size help', callback)
+        self.cfg.load(['--size=123'])
+        self.assertEqual(self.cfg['size'], 123)
+
diff --git a/obnamlib/plugins/backup_plugin.py b/obnamlib/plugins/backup_plugin.py
index 52f1d895..eeda1199 100644
--- a/obnamlib/plugins/backup_plugin.py
+++ b/obnamlib/plugins/backup_plugin.py
@@ -26,9 +26,20 @@ class BackupPlugin(obnamlib.ObnamPlugin):
     def enable(self):
         self.app.register_command('backup', self.backup)
         self.app.config.new_list(['root'], 'what to backup')
+        self.app.config.new_processed(['checkpoint'],
+                                      'make a checkpoint after a given size, '
+                                      'default unit is MiB (%default)',
+                                      self.parse_checkpoint_size)
+        self.app.config['checkpoint'] = '10 MiB'
+
+    def parse_checkpoint_size(self, value):
+        p = obnamlib.ByteSizeParser()
+        p.set_default_unit('MiB')
+        return p.parse(value)
         
     def backup(self, args):
         logging.debug('backup starts')
+        logging.debug('checkpoints every %s' % self.app.config['checkpoint'])
 
         self.app.config.require('store')
         self.app.config.require('hostname')
@@ -40,8 +51,6 @@ class BackupPlugin(obnamlib.ObnamPlugin):
         logging.debug('store: %s' % storepath)
         storefs = self.app.fsf.new(storepath)
         self.store = obnamlib.Store(storefs)
-        self.done = 0
-        self.total = 0
 
         hostname = self.app.config['hostname']
         logging.debug('hostname: %s' % hostname)
@@ -76,6 +85,14 @@ class BackupPlugin(obnamlib.ObnamPlugin):
                     self.app.hooks.call('error-message', 
                                         'Could not back up %s: %s' %
                                         (pathname, e.strerror))
+                if storefs.written >= self.app.config['checkpoint']:
+                    logging.debug('Making checkpoint')
+                    self.backup_parents('.')
+                    self.store.commit_host(checkpoint=True)
+                    self.store.lock_host(hostname)
+                    self.store.start_generation()
+                    storefs.written = 0
+
             self.backup_parents('.')
 
         if self.fs:
@@ -171,7 +188,6 @@ class BackupPlugin(obnamlib.ObnamPlugin):
             self.store.set_file_chunk_groups(filename, cgids)
         else:
             self.store.set_file_chunks(filename, chunkids)
-            
 
     def backup_file_chunk(self, data):
         '''Back up a chunk of data by putting it into the store.'''
diff --git a/obnamlib/plugins/show_plugin.py b/obnamlib/plugins/show_plugin.py
index 29c1c29f..05cc6088 100644
--- a/obnamlib/plugins/show_plugin.py
+++ b/obnamlib/plugins/show_plugin.py
@@ -52,10 +52,16 @@ class ShowPlugin(obnamlib.ObnamPlugin):
         self.open_store()
         for gen in self.store.list_generations():
             start, end = self.store.get_generation_times(gen)
-            sys.stdout.write('%s\t%s .. %s\n' %
+            is_checkpoint = self.store.get_is_checkpoint(gen)
+            if is_checkpoint:
+                checkpoint = ' (checkpoint)'
+            else:
+                checkpoint = ''
+            sys.stdout.write('%s\t%s .. %s%s\n' %
                              (gen, 
                               self.format_time(start), 
-                              self.format_time(end)))
+                              self.format_time(end),
+                              checkpoint))
 
     def ls(self, args):
         self.open_store()
diff --git a/obnamlib/sizeparse.py b/obnamlib/sizeparse.py
new file mode 100644
index 00000000..0f37a039
--- /dev/null
+++ b/obnamlib/sizeparse.py
@@ -0,0 +1,77 @@
+# Copyright 2010  Lars Wirzenius
+# 
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+# 
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+import re
+
+
+class UnitError(Exception):
+
+    def __str__(self):
+        return self.msg
+
+
+class SizeSyntaxError(UnitError):
+
+    def __init__(self, string):
+        self.msg = '"%s" is not a valid size' % string
+
+
+class UnitNameError(UnitError):
+
+    def __init__(self, string):
+        self.msg = '"%s" is not a valid unit' % string
+
+
+class ByteSizeParser(object):
+
+    '''Parse sizes of data in bytes, kilobytes, kibibytes, etc.'''
+    
+    pat = re.compile(r'^(?P<size>\d+(\.\d+)?)\s*'
+                     r'(?P<unit>[kmg]?i?b?)?$', re.I)
+    
+    units = {
+        'b': 1,
+        'k': 1000,
+        'kb': 1000,
+        'kib': 1024,
+        'm': 1000**2,
+        'mb': 1000**2,
+        'mib': 1024**2,
+        'g': 1000**3,
+        'gb': 1000**3,
+        'gib': 1024**3,
+    }
+    
+    def __init__(self):
+        self.set_default_unit('B')
+        
+    def set_default_unit(self, unit):
+        if unit.lower() not in self.units:
+            raise UnitNameError(unit)
+        self.default_unit = unit
+        
+    def parse(self, string):
+        m = self.pat.match(string)
+        if not m:
+            raise SizeSyntaxError(string)
+        size = float(m.group('size'))
+        unit = m.group('unit')
+        if not unit:
+            unit = self.default_unit
+        elif unit.lower() not in self.units:
+            raise UnitNameError(unit)
+        factor = self.units[unit.lower()]
+        return size * factor
diff --git a/obnamlib/sizeparse_tests.py b/obnamlib/sizeparse_tests.py
new file mode 100644
index 00000000..6b679226
--- /dev/null
+++ b/obnamlib/sizeparse_tests.py
@@ -0,0 +1,92 @@
+# Copyright 2010  Lars Wirzenius
+# 
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+# 
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+import unittest
+
+import obnamlib
+
+
+class ByteSizeParserTests(unittest.TestCase):
+
+    def setUp(self):
+        self.p = obnamlib.ByteSizeParser()
+        
+    def test_parses_zero(self):
+        self.assertEqual(self.p.parse('0'), 0)
+        
+    def test_parses_unadorned_size_as_bytes(self):
+        self.assertEqual(self.p.parse('123'), 123)
+        
+    def test_parses_unadorned_size_using_default_unit(self):
+        self.p.set_default_unit('KiB')
+        self.assertEqual(self.p.parse('123'), 123 * 1024)
+        
+    def test_parses_size_with_byte_unit(self):
+        self.assertEqual(self.p.parse('123 B'), 123)
+        
+    def test_parses_size_with_kilo_unit(self):
+        self.assertEqual(self.p.parse('123 k'), 123 * 1000)
+        
+    def test_parses_size_with_kilobyte_unit(self):
+        self.assertEqual(self.p.parse('123 kB'), 123 * 1000)
+        
+    def test_parses_size_with_kibibyte_unit(self):
+        self.assertEqual(self.p.parse('123 KiB'), 123 * 1024)
+        
+    def test_parses_size_with_mega_unit(self):
+        self.assertEqual(self.p.parse('123 m'), 123 * 1000**2)
+        
+    def test_parses_size_with_megabyte_unit(self):
+        self.assertEqual(self.p.parse('123 MB'), 123 * 1000**2)
+        
+    def test_parses_size_with_mebibyte_unit(self):
+        self.assertEqual(self.p.parse('123 MiB'), 123 * 1024**2)
+        
+    def test_parses_size_with_giga_unit(self):
+        self.assertEqual(self.p.parse('123 g'), 123 * 1000**3)
+        
+    def test_parses_size_with_gigabyte_unit(self):
+        self.assertEqual(self.p.parse('123 GB'), 123 * 1000**3)
+        
+    def test_parses_size_with_gibibyte_unit(self):
+        self.assertEqual(self.p.parse('123 GiB'), 123 * 1024**3)
+
+    def test_raises_error_for_empty_string(self):
+        self.assertRaises(obnamlib.SizeSyntaxError, self.p.parse, '')
+
+    def test_raises_error_for_missing_size(self):
+        self.assertRaises(obnamlib.SizeSyntaxError, self.p.parse, 'KiB')
+
+    def test_raises_error_for_bad_unit(self):
+        self.assertRaises(obnamlib.SizeSyntaxError, self.p.parse, '1 km')
+
+    def test_raises_error_for_bad_unit_thats_similar_to_real_one(self):
+        self.assertRaises(obnamlib.UnitNameError, self.p.parse, '1 ib')
+
+    def test_raises_error_for_bad_default_unit(self):
+        self.assertRaises(obnamlib.UnitNameError, 
+                          self.p.set_default_unit, 'km')
+
+    def test_size_syntax_error_includes_input_string(self):
+        text = 'asdf asdf'
+        e = obnamlib.SizeSyntaxError(text)
+        self.assert_(text in str(e), str(e))
+
+    def test_unit_name_error_includes_input_string(self):
+        text = 'asdf asdf'
+        e = obnamlib.UnitNameError(text)
+        self.assert_(text in str(e), str(e))
+
diff --git a/obnamlib/store.py b/obnamlib/store.py
index e94e9e97..5696f291 100644
--- a/obnamlib/store.py
+++ b/obnamlib/store.py
@@ -252,6 +252,7 @@ class GenerationStore(StoreTree):
     GEN_META_ID = 0
     GEN_META_STARTED = 1
     GEN_META_ENDED = 2
+    GEN_META_IS_CHECKPOINT = 3
 
     FILE_NAME = 0
     FILE_METADATA = 1
@@ -348,6 +349,16 @@ class GenerationStore(StoreTree):
         self._insert_int(self.curgen, self.genkey(self.GEN_META_ID), gen_id)
         self._insert_int(self.curgen, self.genkey(self.GEN_META_STARTED), now)
 
+    def set_current_generation_is_checkpoint(self, is_checkpoint):
+        value = 1 if is_checkpoint else 0
+        key = self.genkey(self.GEN_META_IS_CHECKPOINT)
+        self._insert_int(self.curgen, key, value)
+
+    def get_is_checkpoint(self, genid):
+        tree = self.find_generation(genid)
+        key = self.genkey(self.GEN_META_IS_CHECKPOINT)
+        return self._lookup_int(tree, key)
+
     def remove_generation(self, genid):
         tree = self.find_generation(genid)
         if tree == self.curgen:
@@ -750,8 +761,10 @@ class Store(object):
         self.current_host = None
 
     @require_host_lock
-    def commit_host(self):
+    def commit_host(self, checkpoint=False):
         '''Commit changes to and unlock currently locked host.'''
+        if self.new_generation:
+            self.genstore.set_current_generation_is_checkpoint(checkpoint)
         self.added_generations = []
         for genid in self.removed_generations:
             self._really_remove_generation(genid)
@@ -774,6 +787,11 @@ class Store(object):
         '''List existing generations for currently open host.'''
         return self.genstore.list_generations()
         
+    @require_open_host
+    def get_is_checkpoint(self, genid):
+        '''Is a generation a checkpoint one?'''
+        return self.genstore.get_is_checkpoint(genid)
+        
     @require_host_lock
     def start_generation(self):
         '''Start a new generation.
diff --git a/obnamlib/store_tests.py b/obnamlib/store_tests.py
index 7b04ee07..7ac5c45c 100644
--- a/obnamlib/store_tests.py
+++ b/obnamlib/store_tests.py
@@ -282,6 +282,22 @@ class StoreHostTests(unittest.TestCase):
         self.store.commit_host()
         self.assertFalse(self.store.got_host_lock)
 
+    def test_commit_does_not_mark_as_checkpoint_by_default(self):
+        self.store.lock_host('hostname')
+        self.store.start_generation()
+        genid = self.store.new_generation
+        self.store.commit_host()
+        self.store.open_host('hostname')
+        self.assertFalse(self.store.get_is_checkpoint(genid))
+
+    def test_commit_marks_as_checkpoint_when_requested(self):
+        self.store.lock_host('hostname')
+        self.store.start_generation()
+        genid = self.store.new_generation
+        self.store.commit_host(checkpoint=True)
+        self.store.open_host('hostname')
+        self.assert_(self.store.get_is_checkpoint(genid))
+
     def test_commit_host_without_lock_fails(self):
         self.assertRaises(obnamlib.LockFail, self.store.commit_host)
         
diff --git a/obnamlib/vfs.py b/obnamlib/vfs.py
index b3fb62f1..82ec173e 100644
--- a/obnamlib/vfs.py
+++ b/obnamlib/vfs.py
@@ -45,7 +45,7 @@ class VirtualFileSystem(object):
     """
 
     def __init__(self, baseurl):
-        pass
+        self.written = 0
 
     def connect(self):
         """Connect to filesystem."""
diff --git a/obnamlib/vfs_local.py b/obnamlib/vfs_local.py
index 4ed4cb23..7ba752e0 100644
--- a/obnamlib/vfs_local.py
+++ b/obnamlib/vfs_local.py
@@ -138,6 +138,7 @@ class LocalFS(obnamlib.VirtualFileSystem):
             os.remove(name)
             raise
         os.remove(name)
+        self.written += len(contents)
 
     def overwrite_file(self, pathname, contents, make_backup=True):
         path = self.join(pathname)
@@ -167,6 +168,7 @@ class LocalFS(obnamlib.VirtualFileSystem):
                 os.remove(bak)
             except OSError:
                 pass
+        self.written += len(contents)
 
     def listdir(self, dirname):
         return os.listdir(self.join(dirname))
diff --git a/obnamlib/vfs_local_tests.py b/obnamlib/vfs_local_tests.py
index 7c0d300a..dbbc841e 100644
--- a/obnamlib/vfs_local_tests.py
+++ b/obnamlib/vfs_local_tests.py
@@ -263,6 +263,17 @@ class LocalFSTests(unittest.TestCase):
         self.fs.write_file("foo", "bar")
         self.fs.overwrite_file("foo", "foobar")
         self.assertEqual(self.fs.cat("foo"), "foobar")
+    
+    def test_has_written_nothing_initially(self):
+        self.assertEqual(self.fs.written, 0)
+    
+    def test_write_updates_written(self):
+        self.fs.write_file('foo', 'foo')
+        self.assertEqual(self.fs.written, 3)
+    
+    def test_overwrite_updates_written(self):
+        self.fs.overwrite_file('foo', 'foo')
+        self.assertEqual(self.fs.written, 3)
 
 
 class DepthFirstTests(unittest.TestCase):