From 6b5c82dd16d0a0192e23829b13a2b979c26b44fd Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Fri, 6 Sep 2013 19:30:36 +0100 Subject: fsck fixes from Antoine Brenner --- fsck-larch | 42 ++++++++++++++++++++---------------------- larch/fsck.py | 55 ++++++++++++++++++++++++++++++++++++++++++++++++++----- speed-test | 3 +++ 3 files changed, 73 insertions(+), 27 deletions(-) diff --git a/fsck-larch b/fsck-larch index aef6647..4dca010 100755 --- a/fsck-larch +++ b/fsck-larch @@ -14,10 +14,14 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +# To debug, one can create a tracing logfile by adding arguments like: +# --trace=fsck --log=fsck.logfile + import cliapp import logging import sys + import tracing import ttystatus @@ -34,37 +38,31 @@ class Fsck(cliapp.Application): def process_args(self, args): for pattern in self.settings['trace']: tracing.trace_add_pattern(pattern) - - self.ts = ttystatus.TerminalStatus(period=0.1) - self.ts['check'] = 0 - self.ts['checks'] = 0 - self.ts['checkname'] = '' - self.ts.add(ttystatus.PercentDone('check', 'checks', decimals=2)) - self.ts.add(ttystatus.Literal(' ')) - self.ts.add(ttystatus.RemainingTime('check', 'checks')) - self.ts.add(ttystatus.Literal(' remaining; now: ')) - self.ts.add(ttystatus.String('checkname')) - self.errors = False + at_least_one_error = False for dirname in args: + self.errors = False + forest = larch.open_forest(allow_writes=self.settings['fix'],dirname=dirname) + self.ts = ttystatus.TerminalStatus(period=0.1) + self.ts['item'] = None + self.ts['items'] = 0 + self.ts['last_id'] = forest.last_id + self.ts.format( + 'Checking %Counter(item)/%Integer(last_id): %String(item)') self.ts.notify('fsck-larch for %s' % dirname) - forest = larch.open_forest(dirname=dirname) fsck = larch.fsck.Fsck(forest, self.warning, self.error, self.settings['fix']) - all_work = list(fsck.find_work()) - - self.ts['checks'] = len(all_work) - self.ts['check'] = 0 - for work in all_work: - self.ts['check'] += 1 - self.ts['checkname'] = str(work) - work.do() - - self.ts.finish() + fsck.run_fsck( ts = self.ts ) + self.ts.finish() + if self.errors: + at_least_one_error = True + else: + print 'fsck-larch for %s: No errors found' % dirname if self.errors: sys.exit(1) + def error(self, msg): self.errors = True self.ts.notify(msg) diff --git a/larch/fsck.py b/larch/fsck.py index cd6fbc9..e3c37b1 100755 --- a/larch/fsck.py +++ b/larch/fsck.py @@ -46,6 +46,9 @@ class WorkItem(object): def do(self): pass + def __iter__(self): + return iter([self]) + def warning(self, msg): self.fsck.warning('warning: %s: %s' % (self.name, msg)) @@ -74,7 +77,7 @@ class CheckIndexNode(WorkItem): def __init__(self, fsck, node): self.fsck = fsck self.node = node - self.name = 'node %s in %s' % (self.node.id, self.fsck.forest_name) + self.name = 'CheckIndexNode: checking index node %s in %s' % (self.node.id, self.fsck.forest_name) def do(self): tracing.trace('node.id=%s' % self.node.id) @@ -94,7 +97,6 @@ class CheckIndexNode(WorkItem): # Increase refcounts for all children, and check that the child # nodes exist. If the children are index nodes, create work # items to check those. Leaf nodes get no further checking. - drop_keys = [] for key in self.node: child_id = self.node[key] @@ -112,6 +114,8 @@ class CheckIndexNode(WorkItem): self.start_modification(self.node) for key in drop_keys: self.node.remove(key) + self.warning('index node %s: dropped key %s' % + (self.node.id, key)) self.put_node(self.node) @@ -119,9 +123,10 @@ class CheckForest(WorkItem): def __init__(self, fsck): self.fsck = fsck - self.name = 'forest %s' % self.fsck.forest_name + self.name = 'CheckForest: forest %s' % self.fsck.forest_name def do(self): + tracing.trace("CheckForest: checking forest %s" % self.name ) for tree in self.fsck.forest.trees: self.fsck.count(tree.root.id) root_node = self.get_node(tree.root.id) @@ -133,10 +138,12 @@ class CheckRefcounts(WorkItem): def __init__(self, fsck): self.fsck = fsck - self.name = 'refcounts in %s' % self.fsck.forest_name + self.name = 'CheckRefcounts: refcounts in %s' % self.fsck.forest_name def do(self): + tracing.trace('CheckRefcounts : %s nodes to check' % len(self.fsck.refcounts) ) for node_id in self.fsck.refcounts: + tracing.trace('CheckRefcounts checking node %s' % node_id) refcount = self.fsck.forest.node_store.get_refcount(node_id) if refcount != self.fsck.refcounts[node_id]: self.error( @@ -148,13 +155,15 @@ class CheckRefcounts(WorkItem): if self.fsck.fix: self.fsck.forest.node_store.set_refcount( node_id, self.fsck.refcounts[node_id]) + self.warning('node %s: refcount was set to %s' % + (node_id, self.fsck.refcounts[node_id])) class CommitForest(WorkItem): def __init__(self, fsck): self.fsck = fsck - self.name = 'committing fixes to %s' % self.fsck.forest_name + self.name = 'CommitForest: committing fixes to %s' % self.fsck.forest_name def do(self): tracing.trace('committing changes to %s' % self.fsck.forest_name) @@ -183,3 +192,39 @@ class Fsck(object): def count(self, node_id): self.refcounts[node_id] = self.refcounts.get(node_id, 0) + 1 + + def run_work(self, work_generators, ts=None): + """run work_generator.do() recursively as needed + + work_generators : list of generators (eg list( self.find_work() )) + who return objects with .do() methods that + either return None or other generators. + + if a ttystatus.TerminalStatus instance is passed as ts, + report fsck progress via ts + """ + while work_generators: + work_generator = work_generators.pop(0) + for work in work_generator: + if ts: + ts.increase('items', 1) + ts['item'] = work + generator_or_none = work.do() + if generator_or_none: + # Run new work before carrying-on with work_generators (required for proper refcount check) + work_generators.insert(0,generator_or_none) + + def run_fsck(self, ts=None): + """Runs full fsck + + if a ttystatus.TerminalStatus instance is passed as ts, + report fsck progress via ts item/items updates + """ + # Make sure that we pass list( self.find_work() ) and not + # [ self.find_work() ] so that when CheckForest.do() returns + # work generators, the work generators are actually called + # before the CheckRefcounts check. + work_generators = list( self.find_work() ) + self.run_work(work_generators, ts=ts) + + diff --git a/speed-test b/speed-test index 4fc5a24..59620d0 100755 --- a/speed-test +++ b/speed-test @@ -17,6 +17,9 @@ # Excercise my B-tree implementation, for simple benchmarking purposes. # The benchmark gets a location and nb of keys to use as command line # arguments --location=LOCATION and --keys=KEYS. + +# To debug, one can create a tracing logfile by adding arguments like: +# --trace=refcount --log=refcount.logfile # # If the location is the empty string, an in-memory node store is used. # Otherwise it must be a non-existent directory name. -- cgit v1.2.1 From 5d5cad304495c701234a6a963592f4b83c21dbba Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Fri, 6 Sep 2013 19:35:01 +0100 Subject: Break long lines --- fsck-larch | 3 ++- larch/fsck.py | 14 ++++++++++---- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/fsck-larch b/fsck-larch index 4dca010..4de0bce 100755 --- a/fsck-larch +++ b/fsck-larch @@ -42,7 +42,8 @@ class Fsck(cliapp.Application): at_least_one_error = False for dirname in args: self.errors = False - forest = larch.open_forest(allow_writes=self.settings['fix'],dirname=dirname) + forest = larch.open_forest( + allow_writes=self.settings['fix'], dirname=dirname) self.ts = ttystatus.TerminalStatus(period=0.1) self.ts['item'] = None self.ts['items'] = 0 diff --git a/larch/fsck.py b/larch/fsck.py index e3c37b1..68b151a 100755 --- a/larch/fsck.py +++ b/larch/fsck.py @@ -77,7 +77,9 @@ class CheckIndexNode(WorkItem): def __init__(self, fsck, node): self.fsck = fsck self.node = node - self.name = 'CheckIndexNode: checking index node %s in %s' % (self.node.id, self.fsck.forest_name) + self.name = ( + 'CheckIndexNode: checking index node %s in %s' % + (self.node.id, self.fsck.forest_name)) def do(self): tracing.trace('node.id=%s' % self.node.id) @@ -141,7 +143,9 @@ class CheckRefcounts(WorkItem): self.name = 'CheckRefcounts: refcounts in %s' % self.fsck.forest_name def do(self): - tracing.trace('CheckRefcounts : %s nodes to check' % len(self.fsck.refcounts) ) + tracing.trace( + 'CheckRefcounts : %s nodes to check' % + len(self.fsck.refcounts) ) for node_id in self.fsck.refcounts: tracing.trace('CheckRefcounts checking node %s' % node_id) refcount = self.fsck.forest.node_store.get_refcount(node_id) @@ -163,7 +167,8 @@ class CommitForest(WorkItem): def __init__(self, fsck): self.fsck = fsck - self.name = 'CommitForest: committing fixes to %s' % self.fsck.forest_name + self.name = ('CommitForest: committing fixes to %s' % + self.fsck.forest_name) def do(self): tracing.trace('committing changes to %s' % self.fsck.forest_name) @@ -211,7 +216,8 @@ class Fsck(object): ts['item'] = work generator_or_none = work.do() if generator_or_none: - # Run new work before carrying-on with work_generators (required for proper refcount check) + # Run new work before carrying-on with work_generators + # (required for proper refcount check) work_generators.insert(0,generator_or_none) def run_fsck(self, ts=None): -- cgit v1.2.1