Merge changes to move VFS to plugins and to fix SftpFS so it works.

18 files changed, 816 insertions, 533 deletions

diff --git a/README b/README
index 34ea3213..2d6857b7 100644
--- a/README
+++ b/README
@@ -51,6 +51,7 @@ To build:
 To run automatic tests:
 
     make check
+    python test-sftpfs # Read it first, though. Requires ssh setup.
     
 You need my CoverageTestRunner to run tests, get it from:
 
diff --git a/_obnammodule.c b/_obnammodule.c
index 829bb817..e45acf8d 100644
--- a/_obnammodule.c
+++ b/_obnammodule.c
@@ -35,6 +35,7 @@
 
 
 #define _XOPEN_SOURCE 600
+#include <errno.h>
 #include <fcntl.h>
 #include <stdio.h>
 
@@ -72,6 +73,8 @@ lutimes_wrapper(PyObject *self, PyObject *args)
     tv[1].tv_sec = mtime;
     tv[1].tv_usec = 0;
     ret = lutimes(filename, tv);
+    if (ret == -1)
+        ret = errno;
     return Py_BuildValue("i", ret);
 }
 
diff --git a/blackboxtest b/blackboxtest
index 3621cb42..42abe3f2 100755
--- a/blackboxtest
+++ b/blackboxtest
@@ -388,8 +388,7 @@ class ReusesChunks(BlackBoxTest):
         self.backup(store, [data])
         restored = self.restore(store)
         self.verify(data, restored)
-        fsf = obnamlib.VfsFactory()
-        fs = fsf.new(store)
+        fs = obnamlib.LocalFS(store)
         s = obnamlib.Store(fs, obnamlib.DEFAULT_NODE_SIZE, 
                            obnamlib.DEFAULT_UPLOAD_QUEUE_SIZE)
         s.open_host(self.hostid)
@@ -410,8 +409,7 @@ class UsesChunkGroups(BlackBoxTest):
         self.backup(store, [data])
         restored = self.restore(store)
         self.verify(data, restored)
-        fsf = obnamlib.VfsFactory()
-        fs = fsf.new(store)
+        fs = obnamlib.LocalFS(store)
         s = obnamlib.Store(fs, obnamlib.DEFAULT_NODE_SIZE,
                            obnamlib.DEFAULT_UPLOAD_QUEUE_SIZE)
         s.open_host(self.hostid)
diff --git a/obnamlib/__init__.py b/obnamlib/__init__.py
index 7c0963c1..336ccc33 100644
--- a/obnamlib/__init__.py
+++ b/obnamlib/__init__.py
@@ -37,7 +37,7 @@ from hooks import Hook, HookManager
 from cfg import Configuration
 from interp import Interpreter
 from pluginbase import ObnamPlugin
-from vfs import VirtualFileSystem, VfsFactory
+from vfs import VirtualFileSystem, VfsFactory, VfsTests
 from vfs_local import LocalFS
 from metadata import (read_metadata, set_metadata, Metadata, metadata_fields,
                       metadata_verify_fields)
diff --git a/obnamlib/plugins/__init__.py b/obnamlib/plugins/__init__.py
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/obnamlib/plugins/__init__.py
diff --git a/obnamlib/plugins/backup_plugin.py b/obnamlib/plugins/backup_plugin.py
index b6cde4c4..53159e9e 100644
--- a/obnamlib/plugins/backup_plugin.py
+++ b/obnamlib/plugins/backup_plugin.py
@@ -55,6 +55,7 @@ class BackupPlugin(obnamlib.ObnamPlugin):
         storepath = self.app.config['store']
         logging.debug('store: %s' % storepath)
         storefs = self.app.fsf.new(storepath)
+        storefs.connect()
         self.store = obnamlib.Store(storefs, self.app.config['node-size'],
                                     self.app.config['upload-queue-size'])
 
@@ -93,13 +94,13 @@ class BackupPlugin(obnamlib.ObnamPlugin):
                     self.app.hooks.call('error-message', 
                                         'Could not back up %s: %s' %
                                         (pathname, e.strerror))
-                if storefs.written >= self.app.config['checkpoint']:
+                if storefs.bytes_written >= self.app.config['checkpoint']:
                     logging.debug('Making checkpoint')
                     self.backup_parents('.')
                     self.store.commit_host(checkpoint=True)
                     self.store.lock_host(hostname)
                     self.store.start_generation()
-                    storefs.written = 0
+                    storefs.bytes_written = 0
 
             self.backup_parents('.')
 
diff --git a/obnamlib/plugins/forget_plugin.py b/obnamlib/plugins/forget_plugin.py
index 0ecc8bd0..419c8e19 100644
--- a/obnamlib/plugins/forget_plugin.py
+++ b/obnamlib/plugins/forget_plugin.py
@@ -34,6 +34,7 @@ class ForgetPlugin(obnamlib.ObnamPlugin):
         self.app.config.require('hostname')
 
         fs = self.app.fsf.new(self.app.config['store'])
+        fs.connect()
         self.store = obnamlib.Store(fs, self.app.config['node-size'], 
                                     self.app.config['upload-queue-size'])
         self.store.lock_host(self.app.config['hostname'])
diff --git a/obnamlib/plugins/restore_plugin.py b/obnamlib/plugins/restore_plugin.py
index b51d3977..3a6a9412 100644
--- a/obnamlib/plugins/restore_plugin.py
+++ b/obnamlib/plugins/restore_plugin.py
@@ -81,10 +81,12 @@ class RestorePlugin(obnamlib.ObnamPlugin):
             args = ['/']
     
         storefs = self.app.fsf.new(self.app.config['store'])
+        storefs.connect()
         self.store = obnamlib.Store(storefs, self.app.config['node-size'], 
                                     self.app.config['node-size'])
         self.store.open_host(self.app.config['hostname'])
         self.fs = self.app.fsf.new(self.app.config['to'])
+        self.fs.connect()
 
         self.hardlinks = Hardlinks()
         
diff --git a/obnamlib/plugins/sftp_plugin.py b/obnamlib/plugins/sftp_plugin.py
new file mode 100644
index 00000000..65655d11
--- /dev/null
+++ b/obnamlib/plugins/sftp_plugin.py
@@ -0,0 +1,318 @@
+# Copyright (C) 2009  Lars Wirzenius <liw@liw.fi>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+
+import errno
+import logging
+import os
+import pwd
+import random
+import stat
+import urlparse
+
+# As of 2010-07-10, Debian's paramiko package triggers
+# RandomPool_DeprecationWarning. This will eventually be fixed. Until
+# then, there is no point in spewing the warning to the user, who can't
+# do nothing.
+# http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=586925
+import warnings
+with warnings.catch_warnings():
+    warnings.simplefilter('ignore')
+    import paramiko
+
+import obnamlib
+
+
+DEFAULT_SSH_PORT = 22
+
+
+def ioerror_to_oserror(method):
+    '''Decorator to convert an IOError exception to OSError.
+    
+    Python's os.* raise OSError, mostly, but paramiko's corresponding
+    methods raise IOError. This decorator fixes that.
+    
+    '''
+    
+    def helper(self, filename, *args, **kwargs):
+        try:
+            return method(self, filename, *args, **kwargs)
+        except IOError, e:
+            raise OSError(e.errno, e.strerror, filename)
+    
+    return helper
+
+
+class SftpFS(obnamlib.VirtualFileSystem):
+
+    '''A VFS implementation for SFTP.
+    
+    
+    
+    '''
+
+    def __init__(self, baseurl):
+        obnamlib.VirtualFileSystem.__init__(self, baseurl)
+        self.sftp = None
+        self.reinit(baseurl)
+        
+    def connect(self):
+        self.transport = paramiko.Transport((self.host, self.port))
+        self.transport.connect()
+        self._check_host_key(self.host)
+        self._authenticate(self.user)
+        self.sftp = paramiko.SFTPClient.from_transport(self.transport)
+        self.chdir(self.path)
+
+    def _check_host_key(self, hostname):
+        key = self.transport.get_remote_server_key()
+        known_hosts = os.path.expanduser('~/.ssh/known_hosts')
+        keys = paramiko.util.load_host_keys(known_hosts)
+        if hostname not in keys:
+            raise obnamlib.AppException('Host not in known_hosts: %s' % 
+                                        hostname)
+        elif not keys[hostname].has_key(key.get_name()):
+            raise obnamlib.AppException('No host key for %s' % hostname)
+        elif keys[hostname][key.get_name()] != key:
+            raise obnamlib.AppException('Host key has changed for %s' % 
+                                        hostname)
+    
+    def _authenticate(self, username):
+        agent = paramiko.Agent()
+        agent_keys = agent.get_keys()
+        for key in agent_keys:
+            try:
+                self.transport.auth_publickey(username, key)
+                return
+            except paramiko.SSHException:
+                pass
+        raise obnamlib.AppException('Can\'t authenticate to SSH server '
+                                    'using agent.')
+
+    def close(self):
+        self.sftp.close()
+        self.transport.close()
+        self.sftp = None
+
+    def reinit(self, baseurl):
+        scheme, netloc, path, query, fragment = urlparse.urlsplit(baseurl)
+
+        if scheme != 'sftp':
+            raise obnamlib.Error('SftpFS used with non-sftp URL: %s' % baseurl)
+
+        if '@' in netloc:
+            user, netloc = netloc.split('@', 1)
+        else:
+            user = self._get_username()
+
+        if ':' in netloc:
+            host, port = netloc.split(':', 1)
+            port = int(port)
+        else:
+            host = netloc
+            port = DEFAULT_SSH_PORT
+
+        if path.startswith('/~/'):
+            path = path[3:]
+
+        self.host = host
+        self.port = port
+        self.user = user
+        self.path = path
+        
+        if self.sftp:
+            self.sftp.chdir(path)
+
+    def _get_username(self):
+        return pwd.getpwuid(os.getuid()).pw_name
+
+    def getcwd(self):
+        return self.sftp.getcwd()
+
+    @ioerror_to_oserror
+    def chdir(self, pathname):
+        self.sftp.chdir(pathname)
+
+    @ioerror_to_oserror
+    def listdir(self, pathname):
+        return self.sftp.listdir(pathname)
+
+    def lock(self, lockname):
+        try:
+            self.write_file(lockname, '')
+        except IOError, e:
+            if e.errno == errno.EEXIST:
+                raise obnamlib.AppException('Lock %s already exists' % 
+                                            lockname)
+            else:
+                raise
+
+    def unlock(self, lockname):
+        if self.exists(lockname):
+            self.remove(lockname)
+
+    def exists(self, pathname):
+        try:
+            self.lstat(pathname)
+        except OSError:
+            return False
+        else:
+            return True
+
+    def isdir(self, pathname):
+        try:
+            st = self.lstat(pathname)
+        except OSError:
+            return False
+        else:
+            return stat.S_ISDIR(st.st_mode)
+
+    @ioerror_to_oserror
+    def mkdir(self, pathname):
+        self.sftp.mkdir(pathname)
+        
+    @ioerror_to_oserror
+    def makedirs(self, pathname):
+        parent = os.path.dirname(pathname)
+        if parent and parent != pathname and not self.exists(parent):
+            self.makedirs(parent)
+        self.mkdir(pathname)
+
+    @ioerror_to_oserror
+    def rmdir(self, pathname):
+        self.sftp.rmdir(pathname)
+        
+    @ioerror_to_oserror
+    def remove(self, pathname):
+        self.sftp.remove(pathname)
+
+    @ioerror_to_oserror
+    def rename(self, old, new):
+        if self.exists(new):
+            self.remove(new)
+        self.sftp.rename(old, new)
+    
+    @ioerror_to_oserror
+    def lstat(self, pathname):
+        return self.sftp.lstat(pathname)
+
+    @ioerror_to_oserror
+    def chown(self, pathname, uid, gid):
+        self.sftp.chown(pathname, uid, gid)
+        
+    @ioerror_to_oserror
+    def chmod(self, pathname, mode):
+        self.sftp.chmod(pathname, mode)
+        
+    @ioerror_to_oserror
+    def lutimes(self, pathname, atime, mtime):
+        # FIXME: This does not work for symlinks!
+        # Sftp does not have a way of doing that. This means if the restore
+        # target is over sftp, symlinks and their targets will have wrong
+        # mtimes.
+        if getattr(self, 'lutimes_warned', False):
+            logging.warning('lutimes used over SFTP, this does not work '
+                            'against symlinks (warning appears only first '
+                            'time)')
+            self.lutimes_warned = True
+        self.sftp.utime(pathname, (atime, mtime))
+
+    def link(self, existing_path, new_path):
+        raise obnamlib.AppException('Cannot hardlink on SFTP. Sorry.')
+
+    def readlink(self, symlink):
+        return self.sftp.readlink(symlink)
+
+    @ioerror_to_oserror
+    def symlink(self, source, destination):
+        self.sftp.symlink(source, destination)
+
+    def open(self, pathname, mode):
+        return self.sftp.file(pathname, mode)
+
+    def cat(self, pathname):
+        f = self.open(pathname, 'r')
+        chunks = []
+        while True:
+            # 32 KiB is the chunk size that gives me the fastest speed
+            # for sftp transfers. I don't know why the size matters.
+            chunk = f.read(32 * 1024)
+            if not chunk:
+                break
+            chunks.append(chunk)
+            self.bytes_read += len(chunk)
+        f.close()
+        return ''.join(chunks)
+
+    def write_file(self, pathname, contents):
+        if self.exists(pathname):
+            raise OSError(errno.EEXIST, 'File exists', pathname)
+        self._write_helper(pathname, 'wx', contents)
+
+    def _tempfile(self, dirname):
+        '''Generate a filename that does not exist.
+        
+        This is _not_ as safe as tempfile.mkstemp. Plenty of race
+        conditions. But seems to be as good as SFTP will allow.
+        
+        '''
+        
+        while True:
+            i = random.randint(0, 2**64-1)
+            basename = 'tmp.%x' % i
+            pathname = os.path.join(dirname, basename)
+            if not self.exists(pathname):
+                return pathname
+
+    def overwrite_file(self, pathname, contents, make_backup=True):
+        dirname = os.path.dirname(pathname)
+        tempname = self._tempfile(dirname)
+        self._write_helper(tempname, 'wx', contents)
+
+        # Rename existing to have a .bak suffix. If _that_ file already
+        # exists, remove that.
+        bak = pathname + ".bak"
+        try:
+            self.remove(bak)
+        except OSError:
+            pass
+        if self.exists(pathname):
+            self.rename(pathname, bak)
+        self.rename(tempname, pathname)
+        if not make_backup:
+            try:
+                self.remove(bak)
+            except OSError:
+                pass
+        
+    def _write_helper(self, pathname, mode, contents):
+        dirname = os.path.dirname(pathname)
+        if dirname and not self.exists(dirname):
+            self.makedirs(dirname)
+        f = self.open(pathname, mode)
+        chunk_size = 32 * 1024
+        for pos in range(0, len(contents), chunk_size):
+            chunk = contents[pos:pos + chunk_size]
+            f.write(chunk)
+            self.bytes_written += len(chunk)
+        f.close()
+
+
+class SftpPlugin(obnamlib.ObnamPlugin):
+
+    def enable(self):
+        self.app.fsf.register('sftp', SftpFS)
+
diff --git a/obnamlib/plugins/show_plugin.py b/obnamlib/plugins/show_plugin.py
index d650a828..6c3548f2 100644
--- a/obnamlib/plugins/show_plugin.py
+++ b/obnamlib/plugins/show_plugin.py
@@ -40,6 +40,7 @@ class ShowPlugin(obnamlib.ObnamPlugin):
         self.app.config.require('store')
         self.app.config.require('hostname')
         fs = self.app.fsf.new(self.app.config['store'])
+        fs.connect()
         self.store = obnamlib.Store(fs, self.app.config['node-size'], 
                                     self.app.config['node-size'])
         self.store.open_host(self.app.config['hostname'])
diff --git a/obnamlib/plugins/vfs_local_plugin.py b/obnamlib/plugins/vfs_local_plugin.py
new file mode 100644
index 00000000..ff0fc048
--- /dev/null
+++ b/obnamlib/plugins/vfs_local_plugin.py
@@ -0,0 +1,29 @@
+# Copyright 2010  Lars Wirzenius
+# 
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+# 
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+import logging
+import os
+import re
+import stat
+
+import obnamlib
+
+
+class VfsLocalPlugin(obnamlib.ObnamPlugin):
+
+    def enable(self):
+        self.app.fsf.register('', obnamlib.LocalFS)
+
diff --git a/obnamlib/store_tests.py b/obnamlib/store_tests.py
index 75f56bfc..8178c495 100644
--- a/obnamlib/store_tests.py
+++ b/obnamlib/store_tests.py
@@ -690,7 +690,7 @@ class StoreGenspecTests(unittest.TestCase):
         self.tempdir = tempfile.mkdtemp()
 
         storedir = os.path.join(self.tempdir, 'store')
-        fs = obnamlib.VfsFactory().new(storedir)
+        fs = obnamlib.LocalFS(storedir)
         self.store = obnamlib.Store(fs, obnamlib.DEFAULT_NODE_SIZE,
                                     obnamlib.DEFAULT_UPLOAD_QUEUE_SIZE)
         self.store.lock_host('hostname')
diff --git a/obnamlib/vfs.py b/obnamlib/vfs.py
index 82ec173e..619c97f2 100644
--- a/obnamlib/vfs.py
+++ b/obnamlib/vfs.py
@@ -15,6 +15,7 @@
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
 
+import errno
 import os
 import urlparse
 
@@ -23,7 +24,7 @@ import obnamlib
 
 class VirtualFileSystem(object):
 
-    """A virtual filesystem interface.
+    '''A virtual filesystem interface.
     
     The backup program needs to access both local and remote files.
     To make it easier to support all kinds of files both locally and
@@ -41,17 +42,19 @@ class VirtualFileSystem(object):
     for the relative paths: directory components separated by
     slashes, and an initial slash indicating the root of the
     filesystem (in this case, the base URL).
-
-    """
+    
+    '''
 
     def __init__(self, baseurl):
-        self.written = 0
+        self.baseurl = baseurl
+        self.bytes_read = 0
+        self.bytes_written = 0
 
     def connect(self):
-        """Connect to filesystem."""
+        '''Connect to filesystem.'''
         
     def close(self):
-        """Close connection to filesystem."""
+        '''Close connection to filesystem.'''
 
     def reinit(self, new_baseurl):
         '''Go back to the beginning.
@@ -67,86 +70,87 @@ class VirtualFileSystem(object):
         return os.path.abspath(os.path.join(self.getcwd(), pathname))
 
     def getcwd(self):
-        """Return current working directory as absolute pathname."""
+        '''Return current working directory as absolute pathname.'''
         
     def chdir(self, pathname):
-        """Change current working directory to pathname."""
+        '''Change current working directory to pathname.'''
 
     def listdir(self, pathname):
-        """Return list of basenames of entities at pathname."""
+        '''Return list of basenames of entities at pathname.'''
 
     def lock(self, lockname):
-        """Create a lock file with the given name."""
+        '''Create a lock file with the given name.'''
 
     def unlock(self, lockname):
-        """Remove a lock file."""
+        '''Remove a lock file.'''
 
     def exists(self, pathname):
-        """Does the file or directory exist?"""
+        '''Does the file or directory exist?'''
 
     def isdir(self, pathname):
-        """Is it a directory?"""
+        '''Is it a directory?'''
 
     def mkdir(self, pathname):
-        """Create a directory.
+        '''Create a directory.
         
         Parent directories must already exist.
         
-        """
+        '''
         
     def makedirs(self, pathname):
-        """Create a directory, and missing parents."""
+        '''Create a directory, and missing parents.'''
 
     def rmdir(self, pathname):
         '''Remove an empty directory.'''
 
     def rmtree(self, dirname):
         '''Remove a directory tree, including its contents.'''
-        for dirname, dirnames, basenames in self.depth_first(dirname):
-            for basename in basenames:
-                self.remove(os.path.join(dirname, basename))
-            self.rmdir(dirname)
+        if self.isdir(dirname):
+            for dirname, dirnames, basenames in self.depth_first(dirname):
+                for basename in basenames:
+                    self.remove(os.path.join(dirname, basename))
+                self.rmdir(dirname)
 
     def remove(self, pathname):
-        """Remove a file."""
+        '''Remove a file.'''
 
     def rename(self, old, new):
-        """Rename a file."""
+        '''Rename a file.'''
 
     def lstat(self, pathname):
-        """Like os.lstat."""
+        '''Like os.lstat.'''
 
     def chown(self, pathname, uid, gid):
         '''Like os.chown.'''
 
     def chmod(self, pathname, mode):
-        """Like os.chmod."""
+        '''Like os.chmod.'''
 
     def lutimes(self, pathname, atime, mtime):
-        """Like lutimes(2)."""
+        '''Like lutimes(2).'''
 
     def link(self, existing_path, new_path):
-        """Like os.link."""
+        '''Like os.link.'''
 
     def readlink(self, symlink):
-        """Like os.readlink."""
+        '''Like os.readlink.'''
 
     def symlink(self, source, destination):
-        """Like os.symlink."""
+        '''Like os.symlink.'''
 
     def open(self, pathname, mode):
-        """Open a file, like the builtin open() or file() function.
+        '''Open a file, like the builtin open() or file() function.
 
         The return value is a file object like the ones returned
         by the builtin open() function.
 
-        """
+        '''
 
     def cat(self, pathname):
-        """Return the contents of a file."""
+        '''Return the contents of a file.'''
 
     def write_file(self, pathname, contents):
-        """Write a new file.
+        '''Write a new file.
 
         The file must not yet exist. The file is written atomically,
         so that the given name will only exist when the file is
@@ -154,19 +158,19 @@ class VirtualFileSystem(object):
         
         Any directories in pathname will be created if necessary.
 
-        """
+        '''
 
     def overwrite_file(self, pathname, contents, make_backup=True):
-        """Like write_file, but overwrites existing file.
+        '''Like write_file, but overwrites existing file.
 
         The old file isn't immediately lost, it gets renamed with
         a backup suffix. The backup file is removed if make_backup is
         set to False (default is True).
 
-        """
+        '''
 
     def depth_first(self, top, prune=None):
-        """Walk a directory tree depth-first, except for unwanted subdirs.
+        '''Walk a directory tree depth-first, except for unwanted subdirs.
         
         This is, essentially, 'os.walk(top, topdown=False)', except that
         if the prune argument is set, we call it before descending to 
@@ -178,15 +182,15 @@ class VirtualFileSystem(object):
         and must modify the two lists _in_place_. For example:
         
         def prune(dirname, dirnames, filenames):
-            if ".bzr" in dirnames:
-                dirnames.remove(".bzr")
+            if '.bzr' in dirnames:
+                dirnames.remove('.bzr')
         
         The dirnames and filenames lists contain basenames, relative to
         dirname.
         
         top is relative to VFS root, and so is the returned directory name.
         
-        """
+        '''
 
         names = self.listdir(top)
         dirs = []
@@ -207,12 +211,338 @@ class VirtualFileSystem(object):
         
 class VfsFactory:
 
-    """Create new instances of VirtualFileSystem."""
-    
+    '''Create new instances of VirtualFileSystem.'''
+
+    def __init__(self):
+        self.implementations = {}
+        
+    def register(self, scheme, implementation):
+        if scheme in self.implementations:
+            raise obnamlib.Error('URL scheme %s already registered' % scheme)
+        self.implementations[scheme] = implementation
+
     def new(self, url):
-        """Create a new VFS appropriate for a given URL."""
+        '''Create a new VFS appropriate for a given URL.'''
         scheme, netloc, path, params, query, fragment = urlparse.urlparse(url)
-        if scheme == "sftp":
-            return obnamlib.SftpFS(url)
-        else:
-            return obnamlib.LocalFS(url)
+        if scheme in self.implementations:
+            return self.implementations[scheme](url)
+        raise obnamlib.Error('Unknown VFS type %s' % url)
+            
+            
+class VfsTests(object): # pragma: no cover
+
+    '''Re-useable tests for VirtualFileSystem implementations.
+    
+    The base class can't be usefully instantiated itself.
+    Instead you are supposed to sub-class it and implement the API in
+    a suitable way for yourself.
+    
+    This class implements a number of tests that the API implementation
+    must pass. The implementation's own test class should inherit from
+    this class, and unittest.TestCase.
+    
+    The test sub-class should define a setUp method that sets the following:
+    
+    * self.fs to an instance of the API implementation sub-class
+    * self.basepath to the path to the base of the filesystem
+    
+    basepath must be operable as a pathname using os.path tools. If
+    the VFS implemenation operates remotely and wants to operate on a
+    URL like 'http://domain/path' as the baseurl, then basepath must be
+    just the path portion of the URL.
+    
+    The directory indicated by basepath must exist, but must be empty
+    at start.
+    
+    '''
+
+    def test_abspath_returns_input_for_absolute_path(self):
+        self.assertEqual(self.fs.abspath('/foo/bar'), '/foo/bar')
+
+    def test_abspath_returns_absolute_path_for_relative_input(self):
+        self.assertEqual(self.fs.abspath('foo'),
+                         os.path.join(self.basepath, 'foo'))
+
+    def test_abspath_normalizes_path(self):
+        self.assertEqual(self.fs.abspath('foo/..'), self.basepath)
+
+    def test_reinit_works(self):
+        self.fs.chdir('/')
+        self.fs.reinit(self.fs.baseurl)
+        self.assertEqual(self.fs.getcwd(), self.basepath)
+
+    def test_getcwd_returns_dirname(self):
+        self.assertEqual(self.fs.getcwd(), self.basepath)
+
+    def test_chdir_changes_only_fs_cwd_not_process_cwd(self):
+        process_cwd = os.getcwd()
+        self.fs.chdir('/')
+        self.assertEqual(self.fs.getcwd(), '/')
+        self.assertEqual(os.getcwd(), process_cwd)
+
+    def test_chdir_to_nonexistent_raises_exception(self):
+        self.assertRaises(OSError, self.fs.chdir, '/foobar')
+
+    def test_chdir_to_relative_works(self):
+        pathname = os.path.join(self.basepath, 'foo')
+        os.mkdir(pathname)
+        self.fs.chdir('foo')
+        self.assertEqual(self.fs.getcwd(), pathname)
+
+    def test_chdir_to_dotdot_works(self):
+        pathname = os.path.join(self.basepath, 'foo')
+        os.mkdir(pathname)
+        self.fs.chdir('foo')
+        self.fs.chdir('..')
+        self.assertEqual(self.fs.getcwd(), self.basepath)
+
+    def test_creates_lock_file(self):
+        self.fs.lock('lock')
+        self.assert_(self.fs.exists('lock'))
+
+    def test_second_lock_fails(self):
+        self.fs.lock('lock')
+        self.assertRaises(Exception, self.fs.lock, 'lock')
+
+    def test_lock_raises_oserror_without_eexist(self):
+        def raise_it(relative_path, contents):
+            e = OSError()
+            e.errno = errno.EAGAIN
+            raise e
+        self.fs.write_file = raise_it
+        self.assertRaises(OSError, self.fs.lock, 'foo')
+
+    def test_unlock_removes_lock(self):
+        self.fs.lock('lock')
+        self.fs.unlock('lock')
+        self.assertFalse(self.fs.exists('lock'))
+
+    def test_exists_returns_false_for_nonexistent_file(self):
+        self.assertFalse(self.fs.exists('foo'))
+
+    def test_exists_returns_true_for_existing_file(self):
+        self.fs.write_file('foo', '')
+        self.assert_(self.fs.exists('foo'))
+
+    def test_isdir_returns_false_for_nonexistent_file(self):
+        self.assertFalse(self.fs.isdir('foo'))
+
+    def test_isdir_returns_false_for_nondir(self):
+        self.fs.write_file('foo', '')
+        self.assertFalse(self.fs.isdir('foo'))
+
+    def test_isdir_returns_true_for_existing_dir(self):
+        self.fs.mkdir('foo')
+        self.assert_(self.fs.isdir('foo'))
+
+    def test_listdir_raises_oserror_if_directory_does_not_exist(self):
+        self.assertRaises(OSError, self.fs.listdir, 'foo')
+
+    def test_mkdir_raises_oserror_if_directory_exists(self):
+        self.assertRaises(OSError, self.fs.mkdir, '.')
+
+    def test_mkdir_raises_oserror_if_parent_does_not_exist(self):
+        self.assertRaises(OSError, self.fs.mkdir, 'foo/bar')
+    
+    def test_makedirs_raises_oserror_when_directory_exists(self):
+        self.fs.mkdir('foo')
+        self.assertRaises(OSError, self.fs.makedirs, 'foo')
+    
+    def test_makedirs_creates_directory_when_parent_exists(self):
+        self.fs.makedirs('foo')
+        self.assert_(self.fs.isdir('foo'))
+    
+    def test_makedirs_creates_directory_when_parent_does_not_exist(self):
+        self.fs.makedirs('foo/bar')
+        self.assert_(self.fs.isdir('foo/bar'))
+
+    def test_rmdir_removes_directory(self):
+        self.fs.mkdir('foo')
+        self.fs.rmdir('foo')
+        self.assertFalse(self.fs.exists('foo'))
+
+    def test_rmdir_raises_oserror_if_directory_does_not_exist(self):
+        self.assertRaises(OSError, self.fs.rmdir, 'foo')
+
+    def test_rmdir_raises_oserror_if_directory_is_not_empty(self):
+        self.fs.mkdir('foo')
+        self.fs.write_file('foo/bar', '')
+        self.assertRaises(OSError, self.fs.rmdir, 'foo')
+
+    def test_rmtree_removes_directory_tree(self):
+        self.fs.mkdir('foo')
+        self.fs.write_file('foo/bar', '')
+        self.fs.rmtree('foo')
+        self.assertFalse(self.fs.exists('foo'))
+
+    def test_rmtree_is_silent_when_target_does_not_exist(self):
+        self.assertEqual(self.fs.rmtree('foo'), None)
+
+    def test_remove_removes_file(self):
+        self.fs.write_file('foo', '')
+        self.fs.remove('foo')
+        self.assertFalse(self.fs.exists('foo'))
+
+    def test_remove_raises_oserror_if_file_does_not_exist(self):
+        self.assertRaises(OSError, self.fs.remove, 'foo')
+
+    def test_rename_renames_file(self):
+        self.fs.write_file('foo', 'xxx')
+        self.fs.rename('foo', 'bar')
+        self.assertFalse(self.fs.exists('foo'))
+        self.assertEqual(self.fs.cat('bar'), 'xxx')
+
+    def test_rename_raises_oserror_if_file_does_not_exist(self):
+        self.assertRaises(OSError, self.fs.rename, 'foo', 'bar')
+
+    def test_rename_works_if_target_exists(self):
+        self.fs.write_file('foo', 'foo')
+        self.fs.write_file('bar', 'bar')
+        self.fs.rename('foo', 'bar')
+        self.assertEqual(self.fs.cat('bar'), 'foo')
+
+    def test_lstat_returns_result(self):
+        self.assert_(self.fs.lstat('.'))
+
+    def test_lstat_raises_oserror_for_nonexistent_entry(self):
+        self.assertRaises(OSError, self.fs.lstat, 'notexists')
+
+    def test_chmod_sets_permissions_correctly(self):
+        self.fs.mkdir('foo')
+        self.fs.chmod('foo', 0777)
+        self.assertEqual(self.fs.lstat('foo').st_mode & 0777, 0777)
+
+    def test_chmod_raises_oserror_for_nonexistent_entry(self):
+        self.assertRaises(OSError, self.fs.chmod, 'notexists', 0)
+
+    def test_lutimes_sets_times_correctly(self):
+        self.fs.mkdir('foo')
+        self.fs.lutimes('foo', 1, 2)
+        self.assertEqual(self.fs.lstat('foo').st_atime, 1)
+        self.assertEqual(self.fs.lstat('foo').st_mtime, 2)
+
+    def test_lutimes_raises_oserror_for_nonexistent_entry(self):
+        self.assertRaises(OSError, self.fs.lutimes, 'notexists', 1, 2)
+
+    def test_link_creates_hard_link(self):
+        self.fs.write_file('foo', 'foo')
+        self.fs.link('foo', 'bar')
+        st1 = self.fs.lstat('foo')
+        st2 = self.fs.lstat('bar')
+        self.assertEqual(st1, st2)
+
+    def test_symlink_creates_soft_link(self):
+        self.fs.symlink('foo', 'bar')
+        target = self.fs.readlink('bar')
+        self.assertEqual(target, 'foo')
+
+    def test_symlink_raises_oserror_if_name_exists(self):
+        self.fs.write_file('foo', 'foo')
+        self.assertRaises(OSError, self.fs.symlink, 'bar', 'foo')
+
+    def test_opens_existing_file_ok(self):
+        self.fs.write_file('foo', '')
+        self.assert_(self.fs.open('foo', 'w'))
+
+    def test_open_fails_for_nonexistent_file(self):
+        self.assertRaises(IOError, self.fs.open, 'foo', 'r')
+
+    def test_cat_reads_existing_file_ok(self):
+        self.fs.write_file('foo', 'bar')
+        self.assertEqual(self.fs.cat('foo'), 'bar')
+
+    def test_cat_fails_for_nonexistent_file(self):
+        self.assertRaises(IOError, self.fs.cat, 'foo')
+    
+    def test_has_read_nothing_initially(self):
+        self.assertEqual(self.fs.bytes_read, 0)
+    
+    def test_cat_updates_bytes_read(self):
+        self.fs.write_file('foo', 'bar')
+        self.fs.cat('foo')
+        self.assertEqual(self.fs.bytes_read, 3)
+
+    def test_write_fails_if_file_exists_already(self):
+        self.fs.write_file('foo', 'bar')
+        self.assertRaises(OSError, self.fs.write_file, 'foo', 'foobar')
+
+    def test_write_creates_missing_directories(self):
+        self.fs.write_file('foo/bar', 'yo')
+        self.assertEqual(self.fs.cat('foo/bar'), 'yo')
+
+    def test_write_leaves_existing_file_intact(self):
+        self.fs.write_file('foo', 'bar')
+        try:
+            self.fs.write_file('foo', 'foobar')
+        except OSError:
+            pass
+        self.assertEqual(self.fs.cat('foo'), 'bar')
+
+    def test_overwrite_creates_new_file_ok(self):
+        self.fs.overwrite_file('foo', 'bar')
+        self.assertEqual(self.fs.cat('foo'), 'bar')
+
+    def test_overwrite_renames_existing_file(self):
+        self.fs.write_file('foo', 'bar')
+        self.fs.overwrite_file('foo', 'foobar')
+        self.assert_(self.fs.exists('foo.bak'))
+
+    def test_overwrite_removes_existing_bak_file(self):
+        self.fs.write_file('foo', 'bar')
+        self.fs.write_file('foo.bak', 'baz')
+        self.fs.overwrite_file('foo', 'foobar')
+        self.assertEqual(self.fs.cat('foo.bak'), 'bar')
+
+    def test_overwrite_removes_bak_file(self):
+        self.fs.write_file('foo', 'bar')
+        self.fs.overwrite_file('foo', 'foobar', make_backup=False)
+        self.assertFalse(self.fs.exists('foo.bak'))
+
+    def test_overwrite_is_ok_without_bak(self):
+        self.fs.overwrite_file('foo', 'foobar', make_backup=False)
+        self.assertFalse(self.fs.exists('foo.bak'))
+
+    def test_overwrite_replaces_existing_file(self):
+        self.fs.write_file('foo', 'bar')
+        self.fs.overwrite_file('foo', 'foobar')
+        self.assertEqual(self.fs.cat('foo'), 'foobar')
+    
+    def test_has_written_nothing_initially(self):
+        self.assertEqual(self.fs.bytes_written, 0)
+    
+    def test_write_updates_written(self):
+        self.fs.write_file('foo', 'foo')
+        self.assertEqual(self.fs.bytes_written, 3)
+    
+    def test_overwrite_updates_written(self):
+        self.fs.overwrite_file('foo', 'foo')
+        self.assertEqual(self.fs.bytes_written, 3)
+
+    def set_up_depth_first(self):
+        self.dirs = ['foo', 'foo/bar', 'foobar']
+        self.dirs = [os.path.join(self.basepath, x) for x in self.dirs]
+        for dirname in self.dirs:
+            self.fs.mkdir(dirname)
+        self.dirs.insert(0, self.basepath)
+    
+    def test_depth_first_finds_all_dirs(self):
+        self.set_up_depth_first()
+        dirs = [x[0] for x in self.fs.depth_first(self.basepath)]
+        self.failUnlessEqual(sorted(dirs), sorted(self.dirs))
+
+    def prune(self, dirname, dirnames, filenames):
+        if 'foo' in dirnames:
+            dirnames.remove('foo')
+
+    def test_depth_first_finds_all_airs_except_the_pruned_one(self):
+        self.set_up_depth_first()
+        correct = [x 
+                   for x in self.dirs 
+                   if not x.endswith('/foo') and not '/foo/' in x]
+        dirs = [x[0] 
+                for x in self.fs.depth_first(self.basepath, prune=self.prune)]
+        self.failUnlessEqual(sorted(dirs), sorted(correct))
+
+    def test_depth_first_raises_oserror_if_directory_does_not_exist(self):
+        self.assertRaises(OSError, list, self.fs.depth_first('notexist'))
+
diff --git a/obnamlib/vfs_local.py b/obnamlib/vfs_local.py
index 7ba752e0..e4ce3d92 100644
--- a/obnamlib/vfs_local.py
+++ b/obnamlib/vfs_local.py
@@ -79,7 +79,9 @@ class LocalFS(obnamlib.VirtualFileSystem):
         os.chmod(self.join(pathname), mode)
 
     def lutimes(self, pathname, atime, mtime):
-        obnamlib._obnam.lutimes(self.join(pathname), atime, mtime)
+        ret = obnamlib._obnam.lutimes(self.join(pathname), atime, mtime)
+        if ret != 0:
+            raise OSError(ret, errno.errorcode[ret], pathname)
 
     def link(self, existing, new):
         os.link(self.join(existing), self.join(new))
@@ -116,6 +118,7 @@ class LocalFS(obnamlib.VirtualFileSystem):
             if not chunk:
                 break
             chunks.append(chunk)
+            self.bytes_read += len(chunk)
         f.close()
         data = "".join(chunks)
         return data
@@ -131,6 +134,7 @@ class LocalFS(obnamlib.VirtualFileSystem):
             chunk = contents[pos:pos+self.chunk_size]
             os.write(fd, chunk)
             pos += len(chunk)
+            self.bytes_written += len(chunk)
         os.close(fd)
         try:
             os.link(name, path)
@@ -138,7 +142,6 @@ class LocalFS(obnamlib.VirtualFileSystem):
             os.remove(name)
             raise
         os.remove(name)
-        self.written += len(contents)
 
     def overwrite_file(self, pathname, contents, make_backup=True):
         path = self.join(pathname)
@@ -149,6 +152,7 @@ class LocalFS(obnamlib.VirtualFileSystem):
             chunk = contents[pos:pos+self.chunk_size]
             os.write(fd, chunk)
             pos += len(chunk)
+            self.bytes_written += len(chunk)
         os.close(fd)
 
         # Rename existing to have a .bak suffix. If _that_ file already
@@ -168,7 +172,6 @@ class LocalFS(obnamlib.VirtualFileSystem):
                 os.remove(bak)
             except OSError:
                 pass
-        self.written += len(contents)
 
     def listdir(self, dirname):
         return os.listdir(self.join(dirname))
diff --git a/obnamlib/vfs_local_tests.py b/obnamlib/vfs_local_tests.py
index dbbc841e..7acdfc29 100644
--- a/obnamlib/vfs_local_tests.py
+++ b/obnamlib/vfs_local_tests.py
@@ -24,284 +24,21 @@ import unittest
 import obnamlib
 
 
-class LocalFSTests(unittest.TestCase):
+class LocalFSTests(obnamlib.VfsTests, unittest.TestCase):
 
     def setUp(self):
-        self.dirname = tempfile.mkdtemp()
-        self.fs = obnamlib.LocalFS(self.dirname)
+        self.basepath = tempfile.mkdtemp()
+        self.fs = obnamlib.LocalFS(self.basepath)
 
     def tearDown(self):
         self.fs.close()
-        shutil.rmtree(self.dirname)
+        shutil.rmtree(self.basepath)
 
     def test_joins_relative_path_ok(self):
-        self.assertEqual(self.fs.join("foo"), 
-                         os.path.join(self.dirname, "foo"))
+        self.assertEqual(self.fs.join('foo'), 
+                         os.path.join(self.basepath, 'foo'))
 
     def test_join_treats_absolute_path_as_absolute(self):
-        self.assertEqual(self.fs.join("/foo"), "/foo")
+        self.assertEqual(self.fs.join('/foo'), '/foo')
 
-    def test_abspath_returns_input_for_absolute_path(self):
-        self.assertEqual(self.fs.abspath('/foo/bar'), '/foo/bar')
 
-    def test_abspath_returns_absolute_path_for_relative_input(self):
-        self.assertEqual(self.fs.abspath('foo'),
-                         os.path.join(self.dirname, 'foo'))
-
-    def test_abspath_normalizes_path(self):
-        self.assertEqual(self.fs.abspath('foo/..'), self.dirname)
-
-    def test_reinit_works(self):
-        self.fs.reinit('.')
-        self.assertEqual(self.fs.cwd, os.getcwd())
-
-    def test_getcwd_returns_dirname(self):
-        self.assertEqual(self.fs.getcwd(), self.dirname)
-
-    def test_chdir_changes_only_fs_cwd_not_process_cwd(self):
-        process_cwd = os.getcwd()
-        self.fs.chdir('/')
-        self.assertEqual(self.fs.getcwd(), '/')
-        self.assertEqual(os.getcwd(), process_cwd)
-
-    def test_chdir_to_nonexistent_raises_exception(self):
-        self.assertRaises(OSError, self.fs.chdir, '/foobar')
-
-    def test_chdir_to_relative_works(self):
-        pathname = os.path.join(self.dirname, 'foo')
-        os.mkdir(pathname)
-        self.fs.chdir('foo')
-        self.assertEqual(self.fs.getcwd(), pathname)
-
-    def test_chdir_to_dotdot_works(self):
-        pathname = os.path.join(self.dirname, 'foo')
-        os.mkdir(pathname)
-        self.fs.chdir('foo')
-        self.fs.chdir('..')
-        self.assertEqual(self.fs.getcwd(), self.dirname)
-
-    def test_creates_lock_file(self):
-        self.fs.lock("lock")
-        self.assert_(self.fs.exists("lock"))
-        self.assert_(os.path.exists(os.path.join(self.dirname, "lock")))
-
-    def test_second_lock_fails(self):
-        self.fs.lock("lock")
-        self.assertRaises(Exception, self.fs.lock, "lock")
-
-    def test_lock_raises_oserror_without_eexist(self):
-        def raise_it(relative_path, contents):
-            e = OSError()
-            e.errno = errno.EAGAIN
-            raise e
-        self.fs.write_file = raise_it
-        self.assertRaises(OSError, self.fs.lock, "foo")
-
-    def test_unlock_removes_lock(self):
-        self.fs.lock("lock")
-        self.fs.unlock("lock")
-        self.assertFalse(self.fs.exists("lock"))
-        self.assertFalse(os.path.exists(os.path.join(self.dirname, "lock")))
-
-    def test_exists_returns_false_for_nonexistent_file(self):
-        self.assertFalse(self.fs.exists("foo"))
-
-    def test_exists_returns_true_for_existing_file(self):
-        file(os.path.join(self.dirname, "foo"), "w").close()
-        self.assert_(self.fs.exists("foo"))
-
-    def test_isdir_returns_false_for_nonexistent_file(self):
-        self.assertFalse(self.fs.isdir("foo"))
-
-    def test_isdir_returns_false_for_nondir(self):
-        file(os.path.join(self.dirname, "foo"), "w").close()
-        self.assertFalse(self.fs.isdir("foo"))
-
-    def test_isdir_returns_true_for_existing_dir(self):
-        os.mkdir(os.path.join(self.dirname, "foo"))
-        self.assert_(self.fs.isdir("foo"))
-
-    def test_mkdir_creates_directory(self):
-        self.fs.mkdir("foo")
-        self.assert_(os.path.isdir(os.path.join(self.dirname, "foo")))
-        
-    def test_mkdir_raises_oserror_if_directory_exists(self):
-        self.assertRaises(OSError, self.fs.mkdir, ".")
-
-    def test_mkdir_raises_oserror_if_parent_does_not_exist(self):
-        self.assertRaises(OSError, self.fs.mkdir, "foo/bar")
-    
-    def test_makedirs_creates_directory_when_parent_exists(self):
-        self.fs.makedirs("foo")
-        self.assert_(os.path.isdir(os.path.join(self.dirname, "foo")))
-    
-    def test_makedirs_creates_directory_when_parent_does_not_exist(self):
-        self.fs.makedirs("foo/bar")
-        self.assert_(os.path.isdir(os.path.join(self.dirname, "foo/bar")))
-
-    def test_rmdir_removes_directory(self):
-        self.fs.mkdir('foo')
-        self.fs.rmdir('foo')
-        self.assertFalse(self.fs.exists('foo'))
-
-    def test_rmdir_raises_oserror_if_directory_does_not_exist(self):
-        self.assertRaises(OSError, self.fs.rmdir, 'foo')
-
-    def test_rmdir_raises_oserror_if_directory_is_not_empty(self):
-        self.fs.mkdir('foo')
-        self.fs.write_file('foo/bar', '')
-        self.assertRaises(OSError, self.fs.rmdir, 'foo')
-
-    def test_rmtree_removes_directory_tree(self):
-        self.fs.mkdir('foo')
-        self.fs.write_file('foo/bar', '')
-        self.fs.rmtree('foo')
-        self.assertFalse(self.fs.exists('foo'))
-
-    def test_remove_removes_file(self):
-        self.fs.write_file('foo', '')
-        self.fs.remove('foo')
-        self.assertFalse(self.fs.exists('foo'))
-
-    def test_rename_renames_file(self):
-        self.fs.write_file('foo', 'xxx')
-        self.fs.rename('foo', 'bar')
-        self.assertFalse(self.fs.exists('foo'))
-        self.assertEqual(self.fs.cat('bar'), 'xxx')
-
-    def test_lstat_returns_result(self):
-        self.assert_(self.fs.lstat("."))
-
-    def test_chmod_sets_permissions_correctly(self):
-        self.fs.mkdir("foo")
-        self.fs.chmod("foo", 0777)
-        self.assertEqual(self.fs.lstat("foo").st_mode & 0777, 0777)
-
-    def test_lutimes_sets_times_correctly(self):
-        self.fs.mkdir("foo")
-        self.fs.lutimes("foo", 1, 2)
-        self.assertEqual(self.fs.lstat("foo").st_atime, 1)
-        self.assertEqual(self.fs.lstat("foo").st_mtime, 2)
-
-    def test_link_creates_hard_link(self):
-        f = self.fs.open("foo", "w")
-        f.write("foo")
-        f.close()
-        self.fs.link("foo", "bar")
-        st1 = self.fs.lstat("foo")
-        st2 = self.fs.lstat("bar")
-        self.assertEqual(st1, st2)
-
-    def test_symlink_creates_soft_link(self):
-        self.fs.symlink("foo", "bar")
-        target = os.readlink(os.path.join(self.dirname, "bar"))
-        self.assertEqual(target, "foo")
-
-    def test_readlink_reads_link_target(self):
-        self.fs.symlink("foo", "bar")
-        self.assertEqual(self.fs.readlink("bar"), "foo")
-
-    def test_opens_existing_file_ok(self):
-        file(os.path.join(self.dirname, "foo"), "w").close()
-        self.assert_(self.fs.open("foo", "w"))
-
-    def test_open_fails_for_nonexistent_file(self):
-        self.assertRaises(IOError, self.fs.open, "foo", "r")
-
-    def test_cat_reads_existing_file_ok(self):
-        file(os.path.join(self.dirname, "foo"), "w").write("bar")
-        self.assertEqual(self.fs.cat("foo"), "bar")
-
-    def test_cat_fails_for_nonexistent_file(self):
-        self.assertRaises(IOError, self.fs.cat, "foo")
-
-    def test_write_file_writes_file_ok(self):
-        self.fs.write_file("foo", "bar")
-        self.assertEqual(self.fs.cat("foo"), "bar")
-
-    def test_write_fails_if_file_exists_already(self):
-        file(os.path.join(self.dirname, "foo"), "w").write("bar")
-        self.assertRaises(OSError, self.fs.write_file, "foo", "foobar")
-
-    def test_write_creates_missing_directories(self):
-        self.fs.write_file("foo/bar", "yo")
-        self.assertEqual(self.fs.cat("foo/bar"), "yo")
-
-    def test_write_leaves_existing_file_intact(self):
-        file(os.path.join(self.dirname, "foo"), "w").write("bar")
-        try:
-            self.fs.write_file("foo", "foobar")
-        except OSError:
-            pass
-        self.assertEqual(self.fs.cat("foo"), "bar")
-
-    def test_overwrite_creates_new_file_ok(self):
-        self.fs.overwrite_file("foo", "bar")
-        self.assertEqual(self.fs.cat("foo"), "bar")
-
-    def test_overwrite_renames_existing_file(self):
-        self.fs.write_file("foo", "bar")
-        self.fs.overwrite_file("foo", "foobar")
-        self.assert_(self.fs.exists("foo.bak"))
-
-    def test_overwrite_removes_existing_bak_file(self):
-        self.fs.write_file("foo", "bar")
-        self.fs.write_file("foo.bak", "baz")
-        self.fs.overwrite_file("foo", "foobar")
-        self.assertEqual(self.fs.cat("foo.bak"), "bar")
-
-    def test_overwrite_removes_bak_file(self):
-        self.fs.write_file("foo", "bar")
-        self.fs.overwrite_file("foo", "foobar", make_backup=False)
-        self.assertFalse(self.fs.exists("foo.bak"))
-
-    def test_overwrite_is_ok_without_bak(self):
-        self.fs.overwrite_file("foo", "foobar", make_backup=False)
-        self.assertFalse(self.fs.exists("foo.bak"))
-
-    def test_overwrite_replaces_existing_file(self):
-        self.fs.write_file("foo", "bar")
-        self.fs.overwrite_file("foo", "foobar")
-        self.assertEqual(self.fs.cat("foo"), "foobar")
-    
-    def test_has_written_nothing_initially(self):
-        self.assertEqual(self.fs.written, 0)
-    
-    def test_write_updates_written(self):
-        self.fs.write_file('foo', 'foo')
-        self.assertEqual(self.fs.written, 3)
-    
-    def test_overwrite_updates_written(self):
-        self.fs.overwrite_file('foo', 'foo')
-        self.assertEqual(self.fs.written, 3)
-
-
-class DepthFirstTests(unittest.TestCase):
-
-    def setUp(self):
-        self.root = tempfile.mkdtemp()
-        self.dirs = ["foo", "foo/bar", "foobar"]
-        self.dirs = [os.path.join(self.root, x) for x in self.dirs]
-        for dir in self.dirs:
-            os.mkdir(dir)
-        self.dirs.insert(0, self.root)
-        self.fs = obnamlib.LocalFS("/")
-    
-    def tearDown(self):
-        shutil.rmtree(self.root)
-
-    def testFindsAllDirs(self):
-        dirs = [x[0] for x in self.fs.depth_first(self.root)]
-        self.failUnlessEqual(sorted(dirs), sorted(self.dirs))
-
-    def prune(self, dirname, dirnames, filenames):
-        if "foo" in dirnames:
-            dirnames.remove("foo")
-
-    def testFindsAllDirsExceptThePrunedOne(self):
-        correct = [x 
-                   for x in self.dirs 
-                   if not x.endswith("/foo") and not "/foo/" in x]
-        dirs = [x[0] 
-                for x in self.fs.depth_first(self.root, prune=self.prune)]
-        self.failUnlessEqual(sorted(dirs), sorted(correct))
diff --git a/obnamlib/vfs_sftp.py b/obnamlib/vfs_sftp.py
deleted file mode 100644
index e2ba0073..00000000
--- a/obnamlib/vfs_sftp.py
+++ /dev/null
@@ -1,204 +0,0 @@
-# Copyright (C) 2009  Lars Wirzenius <liw@liw.fi>
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-
-import errno
-import logging
-import os
-import pwd
-import stat
-import urlparse
-
-import paramiko
-
-import obnamlib
-
-
-class SftpFS(obnamlib.VirtualFileSystem):
-
-    """A VFS implementation for SFTP."""
-
-    def __init__(self, baseurl, progress):
-        obnamlib.VirtualFileSystem.__init__(self, baseurl, progress)
-        self.first_lutimes = True
-
-    def connect(self):
-        user = host = port = path = None
-        scheme, netloc, path, query, fragment = urlparse.urlsplit(self.baseurl)
-        assert scheme == "sftp", "wrong scheme in %s" % self.baseurl
-        if "@" in netloc:
-            user, netloc = netloc.split("@", 1)
-        else:
-            user = self.get_username()
-        if ":" in netloc:
-            host, port = netloc.split(":", 1)
-            port = int(port)
-        else:
-            host = netloc
-            port = 22
-        if path.startswith("/~/"):
-            path = path[3:]
-        self.basepath = path
-        self.transport = paramiko.Transport((host, port))
-        self.transport.connect()
-        self.check_host_key(host)
-        self.authenticate(user)
-        self.sftp = paramiko.SFTPClient.from_transport(self.transport)
-
-    def get_username(self):
-        return pwd.getpwuid(os.getuid()).pw_name
-
-    def check_host_key(self, hostname):
-        key = self.transport.get_remote_server_key()
-        known_hosts = os.path.expanduser('~/.ssh/known_hosts')
-        keys = paramiko.util.load_host_keys(known_hosts)
-        if hostname not in keys:
-            raise obnamlib.AppException("Host key for %s not found" % hostname)
-        elif not keys[hostname].has_key(key.get_name()):
-            raise obnamlib.AppException("Unknown host key for %s" % hostname)
-        elif keys[hostname][key.get_name()] != key:
-            print '*** WARNING: Host key has changed!!!'
-            raise obnamlib.AppException("Host key has changed for %s" % hostname)
-    
-    def authenticate(self, username):
-        if self.authenticate_via_agent(username):
-            return
-        raise obnamlib.AppException("Can't authenticate to SSH server.")
-
-    def authenticate_via_agent(self, username):
-        agent = paramiko.Agent()
-        agent_keys = agent.get_keys()
-        for key in agent_keys:
-            try:
-                self.transport.auth_publickey(username, key)
-                return True
-            except paramiko.SSHException:
-                pass
-        return False
-    
-    def close(self):
-        self.transport.close()
-
-    def join(self, relative_path):
-        return os.path.join(self.basepath, relative_path.lstrip("/"))
-
-    def listdir(self, relative_path):
-        return self.sftp.listdir(self.join(relative_path))
-
-    def lock(self, lockname):
-        try:
-            self.write_file(lockname, "")
-        except IOError, e:
-            if e.errno == errno.EEXIST:
-                raise obnamlib.AppException("Lock %s already exists" % lockname)
-            else:
-                raise
-
-    def unlock(self, lockname):
-        if self.exists(lockname):
-            self.remove(lockname)
-
-    def remove(self, relative_path):
-        self.sftp.remove(self.join(relative_path))
-
-    def lstat(self, relative_path):
-        return self.sftp.lstat(self.join(relative_path))
-
-    def chown(self, relative_path, uid, gid):
-        self.sftp.chown(self.join(relative_path), uid, gid)
-
-    def chmod(self, relative_path, mode):
-        self.sftp.chmod(self.join(relative_path), mode)
-
-    def lutimes(self, relative_path, atime, mtime):
-        # FIXME: This does not work for symlinks!
-        # Sftp does not have a way of doing that. This means if the restore
-        # target is over sftp, symlinks and their targets will have wrong
-        # mtimes.
-        if self.first_lutimes:
-            logging.warning("lutimes used over SFTP, this does not work "
-                            "against symlinks (warning appears only first "
-                            "time)")
-            self.first_lutimes = False
-        self.sftp.utime(self.join(relative_path), (atime, mtime))
-
-    def link(self, existing, new):
-        raise obnamlib.AppException("Cannot link on SFTP. Sorry.")
-
-    def readlink(self, relative_path):
-        return self.sftp.readlink(self.join(relative_path))
-
-    def symlink(self, existing, new):
-        self.sftp.symlink(existing, self.join(new))
-
-    def open(self, relative_path, mode):
-        return self.sftp.file(self.join(relative_path), mode)
-
-    def exists(self, relative_path):
-        try:
-            self.lstat(relative_path)
-            return True
-        except IOError:
-            return False
-
-    def isdir(self, relative_path):
-        try:
-            st = self.lstat(relative_path)
-        except IOError:
-            return False
-        return stat.S_ISDIR(st.st_mode)
-
-    def mkdir(self, relative_path):
-        self.sftp.mkdir(self.join(relative_path))
-
-    def makedirs(self, relative_path):
-        if self.isdir(relative_path):
-            return
-        parent = os.path.dirname(relative_path)
-        if parent and parent != relative_path:
-            self.makedirs(parent)
-        self.mkdir(relative_path)
-
-    def cat(self, relative_path):
-        f = self.open(relative_path, "r")
-        chunks = []
-        while True:
-            # 32 KiB is the chunk size that gives me the fastest speed
-            # for sftp transfers. I don't know why the size matters.
-            chunk = f.read(32 * 1024)
-            if not chunk:
-                break
-            chunks.append(chunk)
-            self.progress["bytes-received"] += len(chunk)
-        f.close()
-        return "".join(chunks)
-
-    def write_helper(self, relative_path, mode, contents):
-        self.makedirs(os.path.dirname(relative_path))
-        f = self.open(relative_path, mode)
-        chunk_size = 32 * 1024
-        for pos in range(0, len(contents), chunk_size):
-            chunk = contents[pos:pos + chunk_size]
-            f.write(chunk)
-            self.progress["bytes-sent"] += len(chunk)
-        f.close()
-
-    def write_file(self, relative_path, contents):
-        self.write_helper(relative_path, 'wx', contents)
-
-    def overwrite_file(self, relative_path, contents):
-        self.write_helper(relative_path, 'w', contents)
-
diff --git a/test-sftpfs b/test-sftpfs
new file mode 100644
index 00000000..becf1f4f
--- /dev/null
+++ b/test-sftpfs
@@ -0,0 +1,61 @@
+#!/usr/bin/python
+# Copyright 2010  Lars Wirzenius
+# 
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+# 
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+'''Test SftpFS.
+
+This can't be part of the normal unit tests, since it requires access
+to a (real) ssh server.
+
+To run these tests, you must arrange for localhost to be able to accept
+ssh connections using the ssh agent.
+
+'''
+
+
+import os
+import shutil
+import tempfile
+import unittest
+
+import obnamlib
+import obnamlib.plugins.sftp_plugin
+
+
+class SftpTests(unittest.TestCase, obnamlib.VfsTests):
+
+    def setUp(self):
+        self.basepath = tempfile.mkdtemp()
+        baseurl = 'sftp://localhost%s' % self.basepath
+        self.fs = obnamlib.plugins.sftp_plugin.SftpFS(baseurl)
+        self.fs.connect()
+        
+    def tearDown(self):
+        self.fs.close()
+        shutil.rmtree(self.basepath)
+        
+    def test_sets_path_to_absolute_path(self):
+        self.assert_(self.fs.path.startswith('/'))
+        
+    def test_initial_cwd_is_basepath(self):
+        self.assertEqual(self.fs.getcwd(), self.fs.path)
+
+    def test_link_creates_hard_link(self):
+        pass # sftp does not support hardlinking, so not testing it
+        
+        
+if __name__ == '__main__':
+    unittest.main()
diff --git a/without-tests b/without-tests
index 36388eb8..42660e06 100644
--- a/without-tests
+++ b/without-tests
@@ -3,7 +3,6 @@
 ./obnamlib/app.py
 ./obnamlib/status.py
 ./obnamlib/vfs.py
-./obnamlib/vfs_sftp.py
 ./obnamlib/objs.py
 ./obnamlib/plugins/foo_plugin.py
 ./obnamlib/plugins/backup_plugin.py
@@ -15,6 +14,9 @@
 ./test-plugins/oldhello_plugin.py
 ./test-plugins/aaa_hello_plugin.py
 ./test-plugins/wrongversion_plugin.py
-
 ./obnamlib/plugins/forget_plugin.py
 ./obnamlib/plugins/fsck_plugin.py
+./obnamlib/plugins/vfs_local_plugin.py
+./obnamlib/plugins/sftp_plugin.py
+
+./obnamlib/plugins/__init__.py