Fsck fixes from Antoine Brenner

3 files changed, 80 insertions, 27 deletions

diff --git a/fsck-larch b/fsck-larch
index aef6647..4de0bce 100755
--- a/fsck-larch
+++ b/fsck-larch
@@ -14,10 +14,14 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+# To debug, one can create a tracing logfile by adding arguments like:
+# --trace=fsck --log=fsck.logfile
+
 
 import cliapp
 import logging
 import sys
+
 import tracing
 import ttystatus
 
@@ -34,37 +38,32 @@ class Fsck(cliapp.Application):
     def process_args(self, args):
         for pattern in self.settings['trace']:
             tracing.trace_add_pattern(pattern)
-    
-        self.ts = ttystatus.TerminalStatus(period=0.1)
-        self.ts['check'] = 0
-        self.ts['checks'] = 0
-        self.ts['checkname'] = ''
-        self.ts.add(ttystatus.PercentDone('check', 'checks', decimals=2))
-        self.ts.add(ttystatus.Literal(' '))
-        self.ts.add(ttystatus.RemainingTime('check', 'checks'))
-        self.ts.add(ttystatus.Literal(' remaining; now: '))
-        self.ts.add(ttystatus.String('checkname'))
 
-        self.errors = False
+        at_least_one_error = False
         for dirname in args:
+            self.errors = False
+            forest = larch.open_forest(
+                allow_writes=self.settings['fix'], dirname=dirname)
+            self.ts = ttystatus.TerminalStatus(period=0.1)
+            self.ts['item'] = None
+            self.ts['items'] = 0
+            self.ts['last_id'] = forest.last_id
+            self.ts.format(
+                'Checking %Counter(item)/%Integer(last_id): %String(item)')
             self.ts.notify('fsck-larch for %s' % dirname)
-            forest = larch.open_forest(dirname=dirname)
             fsck = larch.fsck.Fsck(forest, self.warning, self.error, 
                                    self.settings['fix'])
-            all_work = list(fsck.find_work())
-            
-            self.ts['checks'] = len(all_work)
-            self.ts['check'] = 0
-            for work in all_work:
-                self.ts['check'] += 1
-                self.ts['checkname'] = str(work)
-                work.do()
-
-        self.ts.finish()
+            fsck.run_fsck( ts = self.ts )
+            self.ts.finish()
+            if self.errors:
+                at_least_one_error = True
+            else:
+                print 'fsck-larch for %s: No errors found' % dirname
         
         if self.errors:
             sys.exit(1)
 
+
     def error(self, msg):
         self.errors = True
         self.ts.notify(msg)
diff --git a/larch/fsck.py b/larch/fsck.py
index cd6fbc9..68b151a 100755
--- a/larch/fsck.py
+++ b/larch/fsck.py
@@ -46,6 +46,9 @@ class WorkItem(object):
     def do(self):
         pass
 
+    def __iter__(self):
+        return iter([self])
+
     def warning(self, msg):
         self.fsck.warning('warning: %s: %s' % (self.name, msg))
 
@@ -74,7 +77,9 @@ class CheckIndexNode(WorkItem):
     def __init__(self, fsck, node):
         self.fsck = fsck
         self.node = node
-        self.name = 'node %s in %s' % (self.node.id, self.fsck.forest_name)
+        self.name = (
+            'CheckIndexNode: checking index node %s in %s' %
+            (self.node.id, self.fsck.forest_name))
 
     def do(self):
         tracing.trace('node.id=%s' % self.node.id)
@@ -94,7 +99,6 @@ class CheckIndexNode(WorkItem):
         # Increase refcounts for all children, and check that the child
         # nodes exist. If the children are index nodes, create work
         # items to check those. Leaf nodes get no further checking.
-
         drop_keys = []
         for key in self.node:
             child_id = self.node[key]
@@ -112,6 +116,8 @@ class CheckIndexNode(WorkItem):
             self.start_modification(self.node)
             for key in drop_keys:
                 self.node.remove(key)
+                self.warning('index node %s: dropped key %s' %
+                    (self.node.id, key))
             self.put_node(self.node)
 
 
@@ -119,9 +125,10 @@ class CheckForest(WorkItem):
 
     def __init__(self, fsck):
         self.fsck = fsck
-        self.name = 'forest %s' % self.fsck.forest_name
+        self.name = 'CheckForest: forest %s' % self.fsck.forest_name
 
     def do(self):
+        tracing.trace("CheckForest: checking forest %s" % self.name )
         for tree in self.fsck.forest.trees:
             self.fsck.count(tree.root.id)
             root_node = self.get_node(tree.root.id)
@@ -133,10 +140,14 @@ class CheckRefcounts(WorkItem):
 
     def __init__(self, fsck):
         self.fsck = fsck
-        self.name = 'refcounts in %s' % self.fsck.forest_name
+        self.name = 'CheckRefcounts: refcounts in %s' % self.fsck.forest_name
 
     def do(self):
+        tracing.trace(
+            'CheckRefcounts : %s nodes to check' % 
+            len(self.fsck.refcounts) )
         for node_id in self.fsck.refcounts:
+            tracing.trace('CheckRefcounts checking node %s' % node_id)
             refcount = self.fsck.forest.node_store.get_refcount(node_id)
             if refcount != self.fsck.refcounts[node_id]:
                 self.error(
@@ -148,13 +159,16 @@ class CheckRefcounts(WorkItem):
                 if self.fsck.fix:
                     self.fsck.forest.node_store.set_refcount(
                         node_id, self.fsck.refcounts[node_id])
+                    self.warning('node %s: refcount was set to %s' %
+                        (node_id, self.fsck.refcounts[node_id]))
 
 
 class CommitForest(WorkItem):
 
     def __init__(self, fsck):
         self.fsck = fsck
-        self.name = 'committing fixes to %s' % self.fsck.forest_name
+        self.name = ('CommitForest: committing fixes to %s' % 
+                     self.fsck.forest_name)
 
     def do(self):
         tracing.trace('committing changes to %s' % self.fsck.forest_name)
@@ -183,3 +197,40 @@ class Fsck(object):
     def count(self, node_id):
         self.refcounts[node_id] = self.refcounts.get(node_id, 0) + 1
 
+
+    def run_work(self, work_generators, ts=None):
+        """run work_generator.do() recursively as needed
+
+        work_generators : list of generators (eg list( self.find_work() ))
+                          who return objects with .do() methods that
+                          either return None or other generators.
+
+        if a ttystatus.TerminalStatus instance is passed as ts,
+        report fsck progress via ts
+        """
+        while work_generators:
+            work_generator = work_generators.pop(0)
+            for work in work_generator:
+                if ts:
+                    ts.increase('items', 1)
+                    ts['item'] = work
+                generator_or_none = work.do()
+                if generator_or_none:
+                    # Run new work before carrying-on with work_generators
+                    # (required for proper refcount check)
+                    work_generators.insert(0,generator_or_none)
+
+    def run_fsck(self, ts=None):
+        """Runs full fsck
+
+        if a ttystatus.TerminalStatus instance is passed as ts,
+        report fsck progress via ts item/items updates
+        """
+        # Make sure that we pass list( self.find_work() ) and not
+        # [ self.find_work() ] so that when CheckForest.do() returns
+        # work generators, the work generators are actually called
+        # before the CheckRefcounts check.
+        work_generators = list( self.find_work() )
+        self.run_work(work_generators, ts=ts)
+
+
diff --git a/speed-test b/speed-test
index 4fc5a24..59620d0 100755
--- a/speed-test
+++ b/speed-test
@@ -17,6 +17,9 @@
 # Excercise my B-tree implementation, for simple benchmarking purposes.
 # The benchmark gets a location and nb of keys to use as command line
 # arguments --location=LOCATION and --keys=KEYS.
+
+# To debug, one can create a tracing logfile by adding arguments like:
+# --trace=refcount --log=refcount.logfile
 #
 # If the location is the empty string, an in-memory node store is used.
 # Otherwise it must be a non-existent directory name.