Fix everything to use flag-files to file removal

Also, add a list_files method to the journal so NodeStoreDisk
can easily find a list, without having to detect externally
whether specific files exist or not, and whether they're in the
journal or not.

3 files changed, 99 insertions, 9 deletions

diff --git a/larch/journal.py b/larch/journal.py
index 886b2ed..efc7344 100644
--- a/larch/journal.py
+++ b/larch/journal.py
@@ -105,6 +105,11 @@ class Journal(object):
     def _deleted(self, filename):
         '''Return name for temporary name for file to be deleted.'''
         return os.path.join(self.deletedir, self._relative(filename))
+
+    def _realname(self, journaldir, filename):
+        '''Return real name for a file in a journal temporary directory.'''
+        assert filename.startswith(journaldir)
+        return os.path.join(self.storedir, filename[len(journaldir):])
     
     def exists(self, filename):
         if self.allow_writes:
@@ -128,11 +133,15 @@ class Journal(object):
         self.fs.overwrite_file(self._new(filename), contents)
 
     def cat(self, filename):
-        new = self._new(filename)
-        if self.allow_writes and self.fs.exists(new):
-            return self.fs.cat(new)
-        else:
-            return self.fs.cat(filename)
+        if self.allow_writes:
+            new = self._new(filename)
+            deleted = self._deleted(filename)
+            if self.fs.exists(new):
+                return self.fs.cat(new)
+            elif self.fs.exists(deleted):
+                raise OSError((errno.ENOENT, os.strerror(errno.ENOENT), 
+                               filename))
+        return self.fs.cat(filename)
             
     def remove(self, filename):
         tracing.trace(filename)
@@ -148,6 +157,33 @@ class Journal(object):
         else:
             self.fs.overwrite_file(deleted, '')
 
+    def list_files(self, dirname):
+        '''List all files.
+        
+        Files only, no directories.
+        
+        '''
+
+        assert not dirname.startswith(self.newdir)
+        assert not dirname.startswith(self.deletedir)
+
+        if self.allow_writes:
+            if self.fs.exists(dirname):
+                for x in self.climb(dirname, files_only=True):
+                    if self.exists(x):
+                        yield x
+            new = self._new(dirname)
+            if self.fs.exists(new):
+                for x in self.climb(new, files_only=True):
+                    yield self._realname(self.newdir, x)
+        else:
+            if self.fs.exists(dirname):
+                for x in self.climb(dirname, files_only=True):
+                    in_new = x.startswith(self.newdir)
+                    in_deleted = x.startswith(self.deletedir)
+                    if not in_new and not in_deleted:
+                        yield x
+
     def climb(self, dirname, files_only=False):
         basenames = self.fs.listdir(dirname)
         filenames = []
@@ -178,7 +214,7 @@ class Journal(object):
         all_excludes = [dirname] + exclude
         for pathname in self.climb(dirname):
             if pathname not in all_excludes:
-                r = os.path.join(self.storedir, pathname[len(dirname):])
+                r = self._realname(dirname, pathname)
                 parent = os.path.dirname(r)
                 if self.fs.isdir(pathname):
                     if not self.fs.exists(r):
@@ -198,16 +234,25 @@ class Journal(object):
             self._clear_directory(self.newdir)
 
         if self.fs.exists(self.deletedir):
-            self._vivify(self.deletedir, [])
+            self._clear_directory(self.deletedir)
 
         tracing.trace('%s done' % self.storedir)
 
+    def _really_delete(self, deletedir):
+        tracing.trace(deletedir)
+        for pathname in self.climb(deletedir, files_only=True):
+            if pathname != deletedir:
+                realname = self._realname(deletedir, pathname)
+                if not self.fs.isdir(realname):
+                    self.fs.remove(realname)
+                self.fs.remove(pathname)
+
     def commit(self, skip=[]):
         tracing.trace('%s start' % self.storedir)
         self._require_rw()
 
         if self.fs.exists(self.deletedir):
-            self._clear_directory(self.deletedir)
+            self._really_delete(self.deletedir)
 
         if self.fs.exists(self.newdir):
             skip = [self._new(x) for x in skip]
diff --git a/larch/journal_tests.py b/larch/journal_tests.py
index dd5ccf7..ddb5311 100644
--- a/larch/journal_tests.py
+++ b/larch/journal_tests.py
@@ -92,6 +92,13 @@ class JournalTests(unittest.TestCase):
         self.j.overwrite_file(filename, 'yo')
         self.assertEqual(self.j.cat(filename), 'yo')
 
+    def test_cat_does_not_find_deleted_file(self):
+        filename = self.join('foo/bar')
+        self.j.overwrite_file(filename, 'bar')
+        self.j.commit()
+        self.j.remove(filename)
+        self.assertRaises(OSError, self.j.cat, filename)
+
     def test_rollback_brings_back_old_file(self):
         filename = self.join('foo/bar')
         self.j.overwrite_file(filename, 'bar')
@@ -228,3 +235,41 @@ class ReadOnlyJournalTests(unittest.TestCase):
         self.rw.remove(filename)
         self.assertEqual(self.ro.cat(filename), 'bar')
 
+    def tests_lists_no_files_initially(self):
+        dirname = self.join('foo')
+        self.assertEqual(list(self.ro.list_files(dirname)), [])
+
+    def test_lists_files_correctly_when_no_changes(self):
+        dirname = self.join('foo')
+        filename = self.join('foo/bar')
+        self.rw.overwrite_file(filename, 'bar')
+        self.rw.commit()
+        self.assertEqual(list(self.ro.list_files(dirname)), [filename])
+
+    def test_lists_added_file_correctly(self):
+        dirname = self.join('foo')
+        filename = self.join('foo/bar')
+        self.rw.overwrite_file(filename, 'bar')
+        self.assertEqual(list(self.rw.list_files(dirname)), [filename])
+        self.assertEqual(list(self.ro.list_files(dirname)), [])
+
+    def test_lists_added_file_correctly_when_dir_existed_already(self):
+        dirname = self.join('foo')
+        filename = self.join('foo/bar')
+        filename2 = self.join('foo/foobar')
+        self.rw.overwrite_file(filename, 'bar')
+        self.rw.commit()
+        self.rw.overwrite_file(filename2, 'yoyo')
+        self.assertEqual(sorted(list(self.rw.list_files(dirname))),
+                         sorted([filename, filename2]))
+        self.assertEqual(list(self.ro.list_files(dirname)), [filename])
+
+    def test_lists_removed_file_correctly(self):
+        dirname = self.join('foo')
+        filename = self.join('foo/bar')
+        self.rw.overwrite_file(filename, 'bar')
+        self.rw.commit()
+        self.rw.remove(filename)
+        self.assertEqual(list(self.rw.list_files(dirname)), [])
+        self.assertEqual(list(self.ro.list_files(dirname)), [filename])
+
diff --git a/larch/nodestore_disk.py b/larch/nodestore_disk.py
index 1ba90a7..cece8d4 100644
--- a/larch/nodestore_disk.py
+++ b/larch/nodestore_disk.py
@@ -263,7 +263,7 @@ class NodeStoreDisk(larch.NodeStore):
         nodedir = os.path.join(self.dirname, self.nodedir)
         uploaded = []
         if self.journal.exists(nodedir):
-            for filename in self.journal.climb(nodedir, files_only=True):
+            for filename in self.journal.list_files(nodedir):
                 uploaded.append(int(os.path.basename(filename), 16))
         return queued + uploaded