diff options
author | Lars Wirzenius <liw@liw.fi> | 2012-04-23 08:08:58 +0100 |
---|---|---|
committer | Lars Wirzenius <liw@liw.fi> | 2012-04-23 08:08:58 +0100 |
commit | 69591ee375dcff4d48c5f9a9a4e00eb7c9a31cff (patch) | |
tree | ede6cdf96f0cb92882d8580b4746413686b4bc9e /larch | |
parent | 89d7cd82ca37938be5b60c6bdb042b7cd370395b (diff) | |
download | larch-69591ee375dcff4d48c5f9a9a4e00eb7c9a31cff.tar.gz |
Fix everything to use flag-files to file removal
Also, add a list_files method to the journal so NodeStoreDisk
can easily find a list, without having to detect externally
whether specific files exist or not, and whether they're in the
journal or not.
Diffstat (limited to 'larch')
-rw-r--r-- | larch/journal.py | 61 | ||||
-rw-r--r-- | larch/journal_tests.py | 45 | ||||
-rw-r--r-- | larch/nodestore_disk.py | 2 |
3 files changed, 99 insertions, 9 deletions
diff --git a/larch/journal.py b/larch/journal.py index 886b2ed..efc7344 100644 --- a/larch/journal.py +++ b/larch/journal.py @@ -105,6 +105,11 @@ class Journal(object): def _deleted(self, filename): '''Return name for temporary name for file to be deleted.''' return os.path.join(self.deletedir, self._relative(filename)) + + def _realname(self, journaldir, filename): + '''Return real name for a file in a journal temporary directory.''' + assert filename.startswith(journaldir) + return os.path.join(self.storedir, filename[len(journaldir):]) def exists(self, filename): if self.allow_writes: @@ -128,11 +133,15 @@ class Journal(object): self.fs.overwrite_file(self._new(filename), contents) def cat(self, filename): - new = self._new(filename) - if self.allow_writes and self.fs.exists(new): - return self.fs.cat(new) - else: - return self.fs.cat(filename) + if self.allow_writes: + new = self._new(filename) + deleted = self._deleted(filename) + if self.fs.exists(new): + return self.fs.cat(new) + elif self.fs.exists(deleted): + raise OSError((errno.ENOENT, os.strerror(errno.ENOENT), + filename)) + return self.fs.cat(filename) def remove(self, filename): tracing.trace(filename) @@ -148,6 +157,33 @@ class Journal(object): else: self.fs.overwrite_file(deleted, '') + def list_files(self, dirname): + '''List all files. + + Files only, no directories. + + ''' + + assert not dirname.startswith(self.newdir) + assert not dirname.startswith(self.deletedir) + + if self.allow_writes: + if self.fs.exists(dirname): + for x in self.climb(dirname, files_only=True): + if self.exists(x): + yield x + new = self._new(dirname) + if self.fs.exists(new): + for x in self.climb(new, files_only=True): + yield self._realname(self.newdir, x) + else: + if self.fs.exists(dirname): + for x in self.climb(dirname, files_only=True): + in_new = x.startswith(self.newdir) + in_deleted = x.startswith(self.deletedir) + if not in_new and not in_deleted: + yield x + def climb(self, dirname, files_only=False): basenames = self.fs.listdir(dirname) filenames = [] @@ -178,7 +214,7 @@ class Journal(object): all_excludes = [dirname] + exclude for pathname in self.climb(dirname): if pathname not in all_excludes: - r = os.path.join(self.storedir, pathname[len(dirname):]) + r = self._realname(dirname, pathname) parent = os.path.dirname(r) if self.fs.isdir(pathname): if not self.fs.exists(r): @@ -198,16 +234,25 @@ class Journal(object): self._clear_directory(self.newdir) if self.fs.exists(self.deletedir): - self._vivify(self.deletedir, []) + self._clear_directory(self.deletedir) tracing.trace('%s done' % self.storedir) + def _really_delete(self, deletedir): + tracing.trace(deletedir) + for pathname in self.climb(deletedir, files_only=True): + if pathname != deletedir: + realname = self._realname(deletedir, pathname) + if not self.fs.isdir(realname): + self.fs.remove(realname) + self.fs.remove(pathname) + def commit(self, skip=[]): tracing.trace('%s start' % self.storedir) self._require_rw() if self.fs.exists(self.deletedir): - self._clear_directory(self.deletedir) + self._really_delete(self.deletedir) if self.fs.exists(self.newdir): skip = [self._new(x) for x in skip] diff --git a/larch/journal_tests.py b/larch/journal_tests.py index dd5ccf7..ddb5311 100644 --- a/larch/journal_tests.py +++ b/larch/journal_tests.py @@ -92,6 +92,13 @@ class JournalTests(unittest.TestCase): self.j.overwrite_file(filename, 'yo') self.assertEqual(self.j.cat(filename), 'yo') + def test_cat_does_not_find_deleted_file(self): + filename = self.join('foo/bar') + self.j.overwrite_file(filename, 'bar') + self.j.commit() + self.j.remove(filename) + self.assertRaises(OSError, self.j.cat, filename) + def test_rollback_brings_back_old_file(self): filename = self.join('foo/bar') self.j.overwrite_file(filename, 'bar') @@ -228,3 +235,41 @@ class ReadOnlyJournalTests(unittest.TestCase): self.rw.remove(filename) self.assertEqual(self.ro.cat(filename), 'bar') + def tests_lists_no_files_initially(self): + dirname = self.join('foo') + self.assertEqual(list(self.ro.list_files(dirname)), []) + + def test_lists_files_correctly_when_no_changes(self): + dirname = self.join('foo') + filename = self.join('foo/bar') + self.rw.overwrite_file(filename, 'bar') + self.rw.commit() + self.assertEqual(list(self.ro.list_files(dirname)), [filename]) + + def test_lists_added_file_correctly(self): + dirname = self.join('foo') + filename = self.join('foo/bar') + self.rw.overwrite_file(filename, 'bar') + self.assertEqual(list(self.rw.list_files(dirname)), [filename]) + self.assertEqual(list(self.ro.list_files(dirname)), []) + + def test_lists_added_file_correctly_when_dir_existed_already(self): + dirname = self.join('foo') + filename = self.join('foo/bar') + filename2 = self.join('foo/foobar') + self.rw.overwrite_file(filename, 'bar') + self.rw.commit() + self.rw.overwrite_file(filename2, 'yoyo') + self.assertEqual(sorted(list(self.rw.list_files(dirname))), + sorted([filename, filename2])) + self.assertEqual(list(self.ro.list_files(dirname)), [filename]) + + def test_lists_removed_file_correctly(self): + dirname = self.join('foo') + filename = self.join('foo/bar') + self.rw.overwrite_file(filename, 'bar') + self.rw.commit() + self.rw.remove(filename) + self.assertEqual(list(self.rw.list_files(dirname)), []) + self.assertEqual(list(self.ro.list_files(dirname)), [filename]) + diff --git a/larch/nodestore_disk.py b/larch/nodestore_disk.py index 1ba90a7..cece8d4 100644 --- a/larch/nodestore_disk.py +++ b/larch/nodestore_disk.py @@ -263,7 +263,7 @@ class NodeStoreDisk(larch.NodeStore): nodedir = os.path.join(self.dirname, self.nodedir) uploaded = [] if self.journal.exists(nodedir): - for filename in self.journal.climb(nodedir, files_only=True): + for filename in self.journal.list_files(nodedir): uploaded.append(int(os.path.basename(filename), 16)) return queued + uploaded |