diff options
| -rw-r--r-- | Makefile | 10 | ||||
| -rw-r--r-- | _obnammodule.c | 92 | ||||
| -rw-r--r-- | obnamlib/__init__.py | 4 | ||||
| -rw-r--r-- | obnamlib/vfs.py | 182 | ||||
| -rw-r--r-- | obnamlib/vfs_local.py | 150 | ||||
| -rw-r--r-- | obnamlib/vfs_local_tests.py | 219 | ||||
| -rw-r--r-- | obnamlib/vfs_sftp.py | 202 | ||||
| -rw-r--r-- | setup.py | 28 | ||||
| -rw-r--r-- | without-tests | 3 |
9 files changed, 889 insertions, 1 deletions
@@ -1,4 +1,12 @@ -all: +PYTHON = python + +all: _obnam.so + +_obnam.so: _obnammodule.c + if $(PYTHON) setup.py build > setup.log 2>&1; then \ + rm setup.log; else cat setup.log; exit 1; fi + cp build/lib*/*.so . + rm -rf build check: python -m CoverageTestRunner --ignore-missing-from=without-tests diff --git a/_obnammodule.c b/_obnammodule.c new file mode 100644 index 00000000..829bb817 --- /dev/null +++ b/_obnammodule.c @@ -0,0 +1,92 @@ +/* + * _obnammodule.c -- Python extensions for Obna + * + * Copyright (C) 2008, 2009 Lars Wirzenius <liw@liw.fi> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + + +/* + * This is a Python extension module written for Obnam, the backup + * software. + * + * This module provides a way to call the posix_fadvise function from + * Python. Obnam uses this to use set the POSIX_FADV_SEQUENTIAL and + * POSIX_FADV_DONTNEED flags, to make sure the kernel knows that it will + * read files sequentially and that the data does not need to be cached. + * This makes Obnam not trash the disk buffer cache, which is nice. + */ + + +#include <Python.h> + + +#define _XOPEN_SOURCE 600 +#include <fcntl.h> +#include <stdio.h> + + +static PyObject * +fadvise_dontneed(PyObject *self, PyObject *args) +{ + int fd; + /* Can't use off_t for offset and len, since PyArg_ParseTuple + doesn't know it. */ + unsigned long long offset; + unsigned long long len; + int ret; + + if (!PyArg_ParseTuple(args, "iLL", &fd, &offset, &len)) + return NULL; + ret = posix_fadvise(fd, offset, len, POSIX_FADV_DONTNEED); + return Py_BuildValue("i", ret); +} + + +static PyObject * +lutimes_wrapper(PyObject *self, PyObject *args) +{ + int ret; + long atime; + long mtime; + struct timeval tv[2]; + const char *filename; + + if (!PyArg_ParseTuple(args, "sll", &filename, &atime, &mtime)) + return NULL; + tv[0].tv_sec = atime; + tv[0].tv_usec = 0; + tv[1].tv_sec = mtime; + tv[1].tv_usec = 0; + ret = lutimes(filename, tv); + return Py_BuildValue("i", ret); +} + + +static PyMethodDef methods[] = { + {"fadvise_dontneed", fadvise_dontneed, METH_VARARGS, + "Call posix_fadvise(2) with POSIX_FADV_DONTNEED argument."}, + {"lutimes", lutimes_wrapper, METH_VARARGS, + "lutimes(2) wrapper; args are filename, atime, and mtime."}, + {NULL, NULL, 0, NULL} /* Sentinel */ +}; + + +PyMODINIT_FUNC +init_obnam(void) +{ + (void) Py_InitModule("_obnam", methods); +} diff --git a/obnamlib/__init__.py b/obnamlib/__init__.py index 59b177d6..6e7c29f4 100644 --- a/obnamlib/__init__.py +++ b/obnamlib/__init__.py @@ -14,6 +14,10 @@ # along with this program. If not, see <http://www.gnu.org/licenses/>. +import _obnam + from hooks import Hook, HookManager from cfg import Configuration from interp import Interpreter +from vfs import VirtualFileSystem, VfsFactory +from vfs_local import LocalFS diff --git a/obnamlib/vfs.py b/obnamlib/vfs.py new file mode 100644 index 00000000..829778e3 --- /dev/null +++ b/obnamlib/vfs.py @@ -0,0 +1,182 @@ +# Copyright (C) 2008 Lars Wirzenius <liw@liw.fi> +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +import os +import urlparse + +import obnamlib + + +class VirtualFileSystem(object): + + """A virtual filesystem interface. + + The backup program needs to access both local and remote files. + To make it easier to support all kinds of files both locally and + remotely, we use a custom virtual filesystem interface so that + all filesystem access is done the same way. This way, we can + easily support user data and backup stores in any combination of + local and remote filesystems. + + This class defines the interface for such virtual filesystems. + Sub-classes will actually implement the interface. + + When a VFS is instantiated, it is bound to a base URL. When + accessing the virtual filesystem, all paths are then given + relative to the base URL. The Unix syntax for files is used + for the relative paths: directory components separated by + slashes, and an initial slash indicating the root of the + filesystem (in this case, the base URL). + + """ + + def __init__(self, baseurl): + self.baseurl = baseurl + + def connect(self): + """Connect to filesystem.""" + + def close(self): + """Close connection to filesystem.""" + + def listdir(self, relative_path): + """Return list of basenames of entities at relative_path.""" + + def lock(self, lockname): + """Create a lock file with the given name.""" + + def unlock(self, lockname): + """Remove a lock file.""" + + def exists(self, relative_path): + """Does the file or directory exist?""" + + def isdir(self, relative_path): + """Is it a directory?""" + + def mkdir(self, relative_path): + """Create a directory. + + Parent directories must already exist. + + """ + + def makedirs(self, relative_path): + """Create a directory, and missing parents.""" + + def remove(self, relative_path): + """Remove a file.""" + + def lstat(self, relative_path): + """Like os.lstat.""" + + def chmod(self, relative_path, mode): + """Like os.chmod.""" + + def lutimes(self, relative_path, atime, mtime): + """Like lutimes(2).""" + + def link(self, existing_path, new_path): + """Like os.link.""" + + def readlink(self, symlink): + """Like os.readlink.""" + + def symlink(self, source, destination): + """Like os.symlink.""" + + def open(self, relative_path, mode): + """Open a file, like the builtin open() or file() function. + + The return value is a file object like the ones returned + by the builtin open() function. + + """ + + def cat(self, relative_path): + """Return the contents of a file.""" + + def write_file(self, relative_path, contents): + """Write a new file. + + The file must not yet exist. The file is written atomically, + so that the given name will only exist when the file is + completely written. + + Any directories in relative_path will be created if necessary. + + """ + + def overwrite_file(self, relative_path, contents): + """Like write_file, but overwrites existing file. + + The old file isn't immediately lost, it gets renamed with + a backup suffix. + + """ + + def depth_first(self, top, prune=None): + """Walk a directory tree depth-first, except for unwanted subdirs. + + This is, essentially, 'os.walk(top, topdown=False)', except that + if the prune argument is set, we call it before descending to + sub-directories to allow it to remove any directories and files + the caller does not want to know about. + + If set, prune must be a function that gets three arguments (current + directory, list of sub-directory names, list of files in directory), + and must modify the two lists _in_place_. For example: + + def prune(dirname, dirnames, filenames): + if ".bzr" in dirnames: + dirnames.remove(".bzr") + + The dirnames and filenames lists contain basenames, relative to + dirname. + + top is relative to VFS root, and so is the returned directory name. + + """ + + names = self.listdir(top) + dirs = [] + nondirs = [] + for name in names: + if self.isdir(os.path.join(top, name)): + dirs.append(name) + else: + nondirs.append(name) + if prune: + prune(top, dirs, nondirs) + for name in dirs: + path = os.path.join(top, name) + for x in self.depth_first(path, prune=prune): + yield x + yield top, dirs, nondirs + + +class VfsFactory: + + """Create new instances of VirtualFileSystem.""" + + def new(self, url): + """Create a new VFS appropriate for a given URL.""" + scheme, netloc, path, params, query, fragment = urlparse.urlparse(url) + if scheme == "sftp": + return obnamlib.SftpFS(url) + else: + return obnamlib.LocalFS(url) diff --git a/obnamlib/vfs_local.py b/obnamlib/vfs_local.py new file mode 100644 index 00000000..42a00892 --- /dev/null +++ b/obnamlib/vfs_local.py @@ -0,0 +1,150 @@ +# Copyright (C) 2008 Lars Wirzenius <liw@liw.fi> +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +import errno +import logging +import os +import tempfile + +import obnamlib + + +class LocalFS(obnamlib.VirtualFileSystem): + + """A VFS implementation for local filesystems.""" + + chunk_size = 1024 * 1024 + + def lock(self, lockname): + try: + self.write_file(lockname, "") + except OSError, e: + if e.errno == errno.EEXIST: + raise obnamlib.Exception("Lock %s already exists" % lockname) + else: + raise + + def unlock(self, lockname): + if self.exists(lockname): + self.remove(lockname) + + def join(self, relative_path): + return os.path.join(self.baseurl, relative_path.lstrip("/")) + + def remove(self, relative_path): + os.remove(self.join(relative_path)) + + def lstat(self, relative_path): + return os.lstat(self.join(relative_path)) + + def chmod(self, relative_path, mode): + os.chmod(self.join(relative_path), mode) + + def lutimes(self, relative_path, atime, mtime): + obnamlib._obnam.lutimes(self.join(relative_path), atime, mtime) + + def link(self, existing, new): + os.link(self.join(existing), self.join(new)) + + def readlink(self, relative_path): + return os.readlink(self.join(relative_path)) + + def symlink(self, existing, new): + os.symlink(existing, self.join(new)) + + def open(self, relative_path, mode): + return file(self.join(relative_path), mode) + + def exists(self, relative_path): + return os.path.exists(self.join(relative_path)) + + def isdir(self, relative_path): + return os.path.isdir(self.join(relative_path)) + + def mkdir(self, relative_path): + os.mkdir(self.join(relative_path)) + + def makedirs(self, relative_path): + os.makedirs(self.join(relative_path)) + + def cat(self, relative_path): + logging.debug("LocalFS: Reading %s" % relative_path) + f = self.open(relative_path, "r") + chunks = [] + while True: + chunk = f.read(self.chunk_size) + if not chunk: + break + chunks.append(chunk) +# self.progress["bytes-received"] += len(chunk) + f.close() + data = "".join(chunks) + logging.debug("LocalFS: %s had %d bytes" % (relative_path, len(data))) + return data + + def write_file(self, relative_path, contents): + logging.debug("LocalFS: Writing %s (%d)" % + (relative_path, len(contents))) + path = self.join(relative_path) + dirname = os.path.dirname(path) + if not os.path.exists(dirname): + os.makedirs(dirname) + fd, name = tempfile.mkstemp(dir=dirname) + pos = 0 + while pos < len(contents): + chunk = contents[pos:pos+self.chunk_size] + os.write(fd, chunk) +# self.progress["bytes-sent"] += len(chunk) + pos += len(chunk) + os.close(fd) + try: + os.link(name, path) + except OSError: + os.remove(name) + raise + os.remove(name) + logging.debug("LocalFS: write_file updates bytes-sent") + + def overwrite_file(self, relative_path, contents): + logging.debug("LocalFS: Over-writing %s (%d)" % + (relative_path, len(contents))) + path = self.join(relative_path) + dirname = os.path.dirname(path) + fd, name = tempfile.mkstemp(dir=dirname) + pos = 0 + while pos < len(contents): + chunk = contents[pos:pos+self.chunk_size] + os.write(fd, chunk) +# self.progress["bytes-sent"] += len(chunk) + pos += len(chunk) + os.close(fd) + + # Rename existing to have a .bak suffix. If _that_ file already + # exists, remove that. + bak = path + ".bak" + try: + os.remove(bak) + except OSError: + pass + try: + os.link(path, bak) + except OSError: + pass + os.rename(name, path) + + def listdir(self, dirname): + return os.listdir(self.join(dirname)) diff --git a/obnamlib/vfs_local_tests.py b/obnamlib/vfs_local_tests.py new file mode 100644 index 00000000..37d22dfb --- /dev/null +++ b/obnamlib/vfs_local_tests.py @@ -0,0 +1,219 @@ +# Copyright (C) 2008 Lars Wirzenius <liw@liw.fi> +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +import errno +import os +import shutil +import tempfile +import unittest + +import obnamlib + + +class LocalFSTests(unittest.TestCase): + + def setUp(self): + self.dirname = tempfile.mkdtemp() + self.fs = obnamlib.LocalFS(self.dirname) + + def tearDown(self): + self.fs.close() + shutil.rmtree(self.dirname) + + def test_joins_relative_path_ok(self): + self.assertEqual(self.fs.join("foo"), + os.path.join(self.dirname, "foo")) + + def test_join_treats_absolute_path_as_relative(self): + self.assertEqual(self.fs.join("/foo"), + os.path.join(self.dirname, "foo")) + + def test_creates_lock_file(self): + self.fs.lock("lock") + self.assert_(self.fs.exists("lock")) + self.assert_(os.path.exists(os.path.join(self.dirname, "lock"))) + + def test_second_lock_fails(self): + self.fs.lock("lock") + self.assertRaises(Exception, self.fs.lock, "lock") + + def test_lock_raises_oserror_without_eexist(self): + def raise_it(relative_path, contents): + e = OSError() + e.errno = errno.EAGAIN + raise e + self.fs.write_file = raise_it + self.assertRaises(OSError, self.fs.lock, "foo") + + def test_unlock_removes_lock(self): + self.fs.lock("lock") + self.fs.unlock("lock") + self.assertFalse(self.fs.exists("lock")) + self.assertFalse(os.path.exists(os.path.join(self.dirname, "lock"))) + + def test_exists_returns_false_for_nonexistent_file(self): + self.assertFalse(self.fs.exists("foo")) + + def test_exists_returns_true_for_existing_file(self): + file(os.path.join(self.dirname, "foo"), "w").close() + self.assert_(self.fs.exists("foo")) + + def test_isdir_returns_false_for_nonexistent_file(self): + self.assertFalse(self.fs.isdir("foo")) + + def test_isdir_returns_false_for_nondir(self): + file(os.path.join(self.dirname, "foo"), "w").close() + self.assertFalse(self.fs.isdir("foo")) + + def test_isdir_returns_true_for_existing_dir(self): + os.mkdir(os.path.join(self.dirname, "foo")) + self.assert_(self.fs.isdir("foo")) + + def test_mkdir_creates_directory(self): + self.fs.mkdir("foo") + self.assert_(os.path.isdir(os.path.join(self.dirname, "foo"))) + + def test_mkdir_raises_oserror_if_directory_exists(self): + self.assertRaises(OSError, self.fs.mkdir, ".") + + def test_mkdir_raises_oserror_if_parent_does_not_exist(self): + self.assertRaises(OSError, self.fs.mkdir, "foo/bar") + + def test_makedirs_creates_directory_when_parent_exists(self): + self.fs.makedirs("foo") + self.assert_(os.path.isdir(os.path.join(self.dirname, "foo"))) + + def test_makedirs_creates_directory_when_parent_does_not_exist(self): + self.fs.makedirs("foo/bar") + self.assert_(os.path.isdir(os.path.join(self.dirname, "foo/bar"))) + + def test_lstat_returns_result(self): + self.assert_(self.fs.lstat(".")) + + def test_chmod_sets_permissions_correctly(self): + self.fs.mkdir("foo") + self.fs.chmod("foo", 0777) + self.assertEqual(self.fs.lstat("foo").st_mode & 0777, 0777) + + def test_lutimes_sets_times_correctly(self): + self.fs.mkdir("foo") + self.fs.lutimes("foo", 1, 2) + self.assertEqual(self.fs.lstat("foo").st_atime, 1) + self.assertEqual(self.fs.lstat("foo").st_mtime, 2) + + def test_link_creates_hard_link(self): + f = self.fs.open("foo", "w") + f.write("foo") + f.close() + self.fs.link("foo", "bar") + st1 = self.fs.lstat("foo") + st2 = self.fs.lstat("bar") + self.assertEqual(st1, st2) + + def test_symlink_creates_soft_link(self): + self.fs.symlink("foo", "bar") + target = os.readlink(os.path.join(self.dirname, "bar")) + self.assertEqual(target, "foo") + + def test_readlink_reads_link_target(self): + self.fs.symlink("foo", "bar") + self.assertEqual(self.fs.readlink("bar"), "foo") + + def test_opens_existing_file_ok(self): + file(os.path.join(self.dirname, "foo"), "w").close() + self.assert_(self.fs.open("foo", "w")) + + def test_open_fails_for_nonexistent_file(self): + self.assertRaises(IOError, self.fs.open, "foo", "r") + + def test_cat_reads_existing_file_ok(self): + file(os.path.join(self.dirname, "foo"), "w").write("bar") + self.assertEqual(self.fs.cat("foo"), "bar") + + def test_cat_fails_for_nonexistent_file(self): + self.assertRaises(IOError, self.fs.cat, "foo") + + def test_write_file_writes_file_ok(self): + self.fs.write_file("foo", "bar") + self.assertEqual(self.fs.cat("foo"), "bar") + + def test_write_fails_if_file_exists_already(self): + file(os.path.join(self.dirname, "foo"), "w").write("bar") + self.assertRaises(OSError, self.fs.write_file, "foo", "foobar") + + def test_write_creates_missing_directories(self): + self.fs.write_file("foo/bar", "yo") + self.assertEqual(self.fs.cat("foo/bar"), "yo") + + def test_write_leaves_existing_file_intact(self): + file(os.path.join(self.dirname, "foo"), "w").write("bar") + try: + self.fs.write_file("foo", "foobar") + except OSError: + pass + self.assertEqual(self.fs.cat("foo"), "bar") + + def test_overwrite_creates_new_file_ok(self): + self.fs.overwrite_file("foo", "bar") + self.assertEqual(self.fs.cat("foo"), "bar") + + def test_overwrite_renames_existing_file(self): + self.fs.write_file("foo", "bar") + self.fs.overwrite_file("foo", "foobar") + self.assert_(self.fs.exists("foo.bak")) + + def test_overwrite_removes_existing_bak_file(self): + self.fs.write_file("foo", "bar") + self.fs.write_file("foo.bak", "baz") + self.fs.overwrite_file("foo", "foobar") + self.assertEqual(self.fs.cat("foo.bak"), "bar") + + def test_overwrite_replaces_existing_file(self): + self.fs.write_file("foo", "bar") + self.fs.overwrite_file("foo", "foobar") + self.assertEqual(self.fs.cat("foo"), "foobar") + + +class DepthFirstTests(unittest.TestCase): + + def setUp(self): + self.root = tempfile.mkdtemp() + self.dirs = ["foo", "foo/bar", "foobar"] + self.dirs = [os.path.join(self.root, x) for x in self.dirs] + for dir in self.dirs: + os.mkdir(dir) + self.dirs.insert(0, self.root) + self.fs = obnamlib.LocalFS("/") + + def tearDown(self): + shutil.rmtree(self.root) + + def testFindsAllDirs(self): + dirs = [x[0] for x in self.fs.depth_first(self.root)] + self.failUnlessEqual(sorted(dirs), sorted(self.dirs)) + + def prune(self, dirname, dirnames, filenames): + if "foo" in dirnames: + dirnames.remove("foo") + + def testFindsAllDirsExceptThePrunedOne(self): + correct = [x + for x in self.dirs + if not x.endswith("/foo") and not "/foo/" in x] + dirs = [x[0] + for x in self.fs.depth_first(self.root, prune=self.prune)] + self.failUnlessEqual(sorted(dirs), sorted(correct)) diff --git a/obnamlib/vfs_sftp.py b/obnamlib/vfs_sftp.py new file mode 100644 index 00000000..cf862132 --- /dev/null +++ b/obnamlib/vfs_sftp.py @@ -0,0 +1,202 @@ +# Copyright (C) 2009 Lars Wirzenius <liw@liw.fi> +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +import errno +import getpass +import logging +import os +import pwd +import stat +import urlparse + +import paramiko + +import obnamlib + + +class SftpFS(obnamlib.VirtualFileSystem): + + """A VFS implementation for SFTP.""" + + def __init__(self, baseurl, progress): + obnamlib.VirtualFileSystem.__init__(self, baseurl, progress) + self.first_lutimes = True + + def connect(self): + user = host = port = path = None + scheme, netloc, path, query, fragment = urlparse.urlsplit(self.baseurl) + assert scheme == "sftp", "wrong scheme in %s" % self.baseurl + if "@" in netloc: + user, netloc = netloc.split("@", 1) + else: + user = self.get_username() + if ":" in netloc: + host, port = netloc.split(":", 1) + port = int(port) + else: + host = netloc + port = 22 + if path.startswith("/~/"): + path = path[3:] + self.basepath = path + self.transport = paramiko.Transport((host, port)) + self.transport.connect() + self.check_host_key(host) + self.authenticate(user) + self.sftp = paramiko.SFTPClient.from_transport(self.transport) + + def get_username(self): + return pwd.getpwuid(os.getuid()).pw_name + + def check_host_key(self, hostname): + key = self.transport.get_remote_server_key() + known_hosts = os.path.expanduser('~/.ssh/known_hosts') + keys = paramiko.util.load_host_keys(known_hosts) + if hostname not in keys: + raise obnamlib.Exception("Host key for %s not found" % hostname) + elif not keys[hostname].has_key(key.get_name()): + raise obnamlib.Exception("Unknown host key for %s" % hostname) + elif keys[hostname][key.get_name()] != key: + print '*** WARNING: Host key has changed!!!' + raise obnamlib.Exception("Host key has changed for %s" % hostname) + + def authenticate(self, username): + if self.authenticate_via_agent(username): + return + raise obnamlib.Exception("Can't authenticate to SSH server.") + + def authenticate_via_agent(self, username): + agent = paramiko.Agent() + agent_keys = agent.get_keys() + for key in agent_keys: + try: + self.transport.auth_publickey(username, key) + return True + except paramiko.SSHException: + pass + return False + + def close(self): + self.transport.close() + + def join(self, relative_path): + return os.path.join(self.basepath, relative_path.lstrip("/")) + + def listdir(self, relative_path): + return self.sftp.listdir(self.join(relative_path)) + + def lock(self, lockname): + try: + self.write_file(lockname, "") + except IOError, e: + if e.errno == errno.EEXIST: + raise obnamlib.Exception("Lock %s already exists" % lockname) + else: + raise + + def unlock(self, lockname): + if self.exists(lockname): + self.remove(lockname) + + def remove(self, relative_path): + self.sftp.remove(self.join(relative_path)) + + def lstat(self, relative_path): + return self.sftp.lstat(self.join(relative_path)) + + def chmod(self, relative_path, mode): + self.sftp.chmod(self.join(relative_path), mode) + + def lutimes(self, relative_path, atime, mtime): + # FIXME: This does not work for symlinks! + # Sftp does not have a way of doing that. This means if the restore + # target is over sftp, symlinks and their targets will have wrong + # mtimes. + if self.first_lutimes: + logging.warning("lutimes used over SFTP, this does not work " + "against symlinks (warning appears only first " + "time)") + self.first_lutimes = False + self.sftp.utime(self.join(relative_path), (atime, mtime)) + + def link(self, existing, new): + raise obnamlib.Exception("Cannot link on SFTP. Sorry.") + + def readlink(self, relative_path): + return self.sftp.readlink(self.join(relative_path)) + + def symlink(self, existing, new): + self.sftp.symlink(existing, self.join(new)) + + def open(self, relative_path, mode): + return self.sftp.file(self.join(relative_path), mode) + + def exists(self, relative_path): + try: + self.lstat(relative_path) + return True + except IOError: + return False + + def isdir(self, relative_path): + try: + st = self.lstat(relative_path) + except IOError: + return False + return stat.S_ISDIR(st.st_mode) + + def mkdir(self, relative_path): + self.sftp.mkdir(self.join(relative_path)) + + def makedirs(self, relative_path): + if self.isdir(relative_path): + return + parent = os.path.dirname(relative_path) + if parent and parent != relative_path: + self.makedirs(parent) + self.mkdir(relative_path) + + def cat(self, relative_path): + f = self.open(relative_path, "r") + chunks = [] + while True: + # 32 KiB is the chunk size that gives me the fastest speed + # for sftp transfers. I don't know why the size matters. + chunk = f.read(32 * 1024) + if not chunk: + break + chunks.append(chunk) + self.progress["bytes-received"] += len(chunk) + f.close() + return "".join(chunks) + + def write_helper(self, relative_path, mode, contents): + self.makedirs(os.path.dirname(relative_path)) + f = self.open(relative_path, mode) + chunk_size = 32 * 1024 + for pos in range(0, len(contents), chunk_size): + chunk = contents[pos:pos + chunk_size] + f.write(chunk) + self.progress["bytes-sent"] += len(chunk) + f.close() + + def write_file(self, relative_path, contents): + self.write_helper(relative_path, 'wx', contents) + + def overwrite_file(self, relative_path, contents): + self.write_helper(relative_path, 'w', contents) + diff --git a/setup.py b/setup.py new file mode 100644 index 00000000..f2d11dfb --- /dev/null +++ b/setup.py @@ -0,0 +1,28 @@ +#!/usr/bin/python +# Copyright (C) 2008 Lars Wirzenius <liw@liw.fi> +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +from distutils.core import setup, Extension + +setup(name='obnam', + version='0.0.0', + description='Backup software', + author='Lars Wirzenius', + author_email='liw@liw.fi', + url='http://braawi.org/obnam.html', + packages=['obnamlib'], + ext_modules=[Extension('_obnam', sources=['_obnammodule.c'])], + ) diff --git a/without-tests b/without-tests index af7fc51f..3024c259 100644 --- a/without-tests +++ b/without-tests @@ -1,2 +1,5 @@ +./setup.py ./obnamlib/__init__.py ./obnamlib/status.py +./obnamlib/vfs.py +./obnamlib/vfs_sftp.py |