diff options
| author | Lars Wirzenius <liw@liw.fi> | 2015-08-29 13:53:06 +0300 |
|---|---|---|
| committer | Lars Wirzenius <liw@liw.fi> | 2015-08-29 13:53:06 +0300 |
| commit | e6a41b789f8f17a4c932970389c5db557a91107a (patch) | |
| tree | 1be71f80120d1ab364dc43ec26b1336ae11546a7 | |
| parent | 1fa5246916db88e4adb39c94b1c444cb9ba60eb0 (diff) | |
| parent | 8b92b78257a1d0cac28070f8501fce65574dd224 (diff) | |
| download | obnam-e6a41b789f8f17a4c932970389c5db557a91107a.tar.gz | |
Fix bugs and style issues found by pylint
Also, run pylint in "setup.py -n".
49 files changed, 426 insertions, 386 deletions
diff --git a/obnamlib/__init__.py b/obnamlib/__init__.py index 92d71afc..ff594e8d 100644 --- a/obnamlib/__init__.py +++ b/obnamlib/__init__.py @@ -30,7 +30,7 @@ class DummyExtension(object): def __getattr__(self, name): raise Exception('Trying to use _obnam, but that was not found.') try: - import _obnam + import obnamlib._obnam except ImportError: _obnam = DummyExtension() @@ -38,7 +38,7 @@ except ImportError: # Exceptions defined by Obnam itself. They should all be a subclass # of obnamlib.ObnamError. -from structurederror import StructuredError +from .structurederror import StructuredError class ObnamError(StructuredError): @@ -77,51 +77,52 @@ option_group = { } -from sizeparse import SizeSyntaxError, UnitNameError, ByteSizeParser +from .sizeparse import SizeSyntaxError, UnitNameError, ByteSizeParser -from encryption import (generate_symmetric_key, - encrypt_symmetric, - decrypt_symmetric, - get_public_key, - get_public_key_user_ids, - Keyring, - SecretKeyring, - encrypt_with_keyring, - decrypt_with_secret_keys, - SymmetricKeyCache, - EncryptionError) +from .encryption import ( + generate_symmetric_key, + encrypt_symmetric, + decrypt_symmetric, + get_public_key, + get_public_key_user_ids, + Keyring, + SecretKeyring, + encrypt_with_keyring, + decrypt_with_secret_keys, + SymmetricKeyCache, + EncryptionError) -from hooks import ( +from .hooks import ( Hook, MissingFilterError, NoFilterTagError, FilterHook, HookManager) -from pluginbase import ObnamPlugin -from vfs import ( +from .pluginbase import ObnamPlugin +from .vfs import ( VirtualFileSystem, VfsFactory, VfsTests, LockFail, NEW_DIR_MODE, NEW_FILE_MODE) -from vfs_local import LocalFS -from fsck_work_item import WorkItem -from repo_fs import RepositoryFS -from lockmgr import LockManager -from forget_policy import ForgetPolicy -from app import App, ObnamIOError, ObnamSystemError -from humanise import humanise_duration, humanise_size, humanise_speed -from chunkid_token_map import ChunkIdTokenMap -from pathname_excluder import PathnameExcluder -from splitpath import split_pathname - -from obj_serialiser import serialise_object, deserialise_object -from bag import Bag, BagIdNotSetError, make_object_id, parse_object_id -from bag_store import BagStore, serialise_bag, deserialise_bag -from blob_store import BlobStore - -from repo_factory import ( +from .vfs_local import LocalFS +from .fsck_work_item import WorkItem +from .repo_fs import RepositoryFS +from .lockmgr import LockManager +from .forget_policy import ForgetPolicy +from .app import App, ObnamIOError, ObnamSystemError +from .humanise import humanise_duration, humanise_size, humanise_speed +from .chunkid_token_map import ChunkIdTokenMap +from .pathname_excluder import PathnameExcluder +from .splitpath import split_pathname + +from .obj_serialiser import serialise_object, deserialise_object +from .bag import Bag, BagIdNotSetError, make_object_id, parse_object_id +from .bag_store import BagStore, serialise_bag, deserialise_bag +from .blob_store import BlobStore + +from .repo_factory import ( RepositoryFactory, UnknownRepositoryFormat, UnknownRepositoryFormatWanted) -from repo_interface import ( +from .repo_interface import ( RepositoryInterface, RepositoryInterfaceTests, RepositoryClientAlreadyExists, @@ -178,7 +179,7 @@ from .delegator import RepositoryDelegator, GenerationId # Repository format green-albatross specific modules. # -from fmt_ga import ( +from .fmt_ga import ( RepositoryFormatGA, GAClientList, GAClient, @@ -194,17 +195,17 @@ from fmt_ga import ( # Repository format 6 specific modules. # -from metadata import ( +from .metadata import ( Metadata, read_metadata, set_metadata, SetMetadataError, metadata_fields) -from fmt_6.repo_fmt_6 import RepositoryFormat6 -from fmt_6.repo_tree import RepositoryTree -from fmt_6.chunklist import ChunkList -from fmt_6.clientlist import ClientList -from fmt_6.checksumtree import ChecksumTree -from fmt_6.clientmetadatatree import ClientMetadataTree +from .fmt_6.repo_fmt_6 import RepositoryFormat6 +from .fmt_6.repo_tree import RepositoryTree +from .fmt_6.chunklist import ChunkList +from .fmt_6.clientlist import ClientList +from .fmt_6.checksumtree import ChecksumTree +from .fmt_6.clientmetadatatree import ClientMetadataTree __all__ = locals() diff --git a/obnamlib/bag_store.py b/obnamlib/bag_store.py index abe33baf..5cbc3196 100644 --- a/obnamlib/bag_store.py +++ b/obnamlib/bag_store.py @@ -16,7 +16,6 @@ # =*= License: GPL-3+ =*= -import errno import os import random @@ -56,12 +55,12 @@ class BagStore(object): filename = self._make_bag_filename(bag_id) try: st = self._fs.lstat(filename) - except (IOError, OSError) as e: # pragma: no cover + except (IOError, OSError): # pragma: no cover return False return st.st_size > 0 def get_bag_ids(self): - for pathname, st in self._fs.scan_tree(self._dirname): + for pathname, _ in self._fs.scan_tree(self._dirname): if self._is_bag_filename(pathname): yield self._get_bag_id_from_filename(pathname) diff --git a/obnamlib/delegator.py b/obnamlib/delegator.py index 78eccc98..b3cc2bcf 100644 --- a/obnamlib/delegator.py +++ b/obnamlib/delegator.py @@ -23,6 +23,8 @@ class RepositoryDelegator(obnamlib.RepositoryInterface): '''Implement RepositoryInterface by delegating to other objects.''' + format = None + def __init__(self, **kwargs): self._fs = None self._hooks = kwargs['hooks'] @@ -63,6 +65,16 @@ class RepositoryDelegator(obnamlib.RepositoryInterface): self._chunk_indexes.set_fs(self._fs) # + # Repository methods. + # + + def init_repo(self): + raise NotImplementedError() + + def close(self): + raise NotImplementedError() + + # # Client list methods. # diff --git a/obnamlib/encryption_tests.py b/obnamlib/encryption_tests.py index 96bcdae5..c695b731 100644 --- a/obnamlib/encryption_tests.py +++ b/obnamlib/encryption_tests.py @@ -184,12 +184,9 @@ class PublicKeyEncryptionTests(unittest.TestCase): def test_roundtrip_works(self): cleartext = 'hello, world' - passphrase = 'password1' - pubring = os.path.join(self.gpghome, 'pubring.gpg') - secring = os.path.join(self.gpghome, 'secring.gpg') + pubring = os.path.join(self.gpghome, 'pubring.gpg') keyring = obnamlib.Keyring(cat(pubring)) - seckeys = obnamlib.SecretKeyring(cat(secring)) encrypted = obnamlib.encrypt_with_keyring(cleartext, keyring) decrypted = obnamlib.decrypt_with_secret_keys( diff --git a/obnamlib/fmt_6/__init__.py b/obnamlib/fmt_6/__init__.py index 7f18e5bc..a8626888 100644 --- a/obnamlib/fmt_6/__init__.py +++ b/obnamlib/fmt_6/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2014 Lars Wirzenius +# Copyright 2014-2015 Lars Wirzenius # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -16,4 +16,4 @@ # =*= License: GPL-3+ =*= -import metadata_codec +import obnamlib.fmt_6.metadata_codec diff --git a/obnamlib/fmt_6/checksumtree.py b/obnamlib/fmt_6/checksumtree.py index 1a0eb6da..c24dadf0 100644 --- a/obnamlib/fmt_6/checksumtree.py +++ b/obnamlib/fmt_6/checksumtree.py @@ -62,7 +62,7 @@ class ChecksumTree(obnamlib.RepositoryTree): maxkey = self.key(checksum, obnamlib.MAX_ID, obnamlib.MAX_ID) t = self.forest.trees[-1] pairs = t.lookup_range(minkey, maxkey) - return [self.unkey(key)[1] for key, value in pairs] + return [self.unkey(key)[1] for key, _ in pairs] else: return [] diff --git a/obnamlib/fmt_6/clientlist.py b/obnamlib/fmt_6/clientlist.py index 329a38b7..965cfb8e 100644 --- a/obnamlib/fmt_6/clientlist.py +++ b/obnamlib/fmt_6/clientlist.py @@ -87,7 +87,7 @@ class ClientList(obnamlib.RepositoryTree): minkey = self.key(client_name, 0, 0) maxkey = self.key(client_name, obnamlib.MAX_ID, self.SUBKEY_MAX) for k, v in t.lookup_range(minkey, maxkey): - checksum, client_id, subkey = self.unkey(k) + _, client_id, subkey = self.unkey(k) if subkey == self.CLIENT_NAME and v == client_name: return client_id return None @@ -99,7 +99,7 @@ class ClientList(obnamlib.RepositoryTree): return self.find_client_id(t, client_name) def add_client(self, client_name): - logging.info('Adding client %s' % client_name) + logging.info('Adding client %s', client_name) self.start_changes() if self.find_client_id(self.tree, client_name) is None: while True: @@ -111,10 +111,10 @@ class ClientList(obnamlib.RepositoryTree): break key = self.key(client_name, candidate_id, self.CLIENT_NAME) self.tree.insert(key, client_name) - logging.debug('Client %s has id %s' % (client_name, candidate_id)) + logging.debug('Client %s has id %s', client_name, candidate_id) def remove_client(self, client_name): - logging.info('Removing client %s' % client_name) + logging.info('Removing client %s', client_name) self.start_changes() client_id = self.find_client_id(self.tree, client_name) if client_id is not None: @@ -127,12 +127,12 @@ class ClientList(obnamlib.RepositoryTree): client_id = self.find_client_id(t, client_name) if client_id is not None: key = self.key(client_name, client_id, self.KEYID) - for k, v in t.lookup_range(key, key): + for _, v in t.lookup_range(key, key): return v return None def set_client_keyid(self, client_name, keyid): - logging.info('Setting client %s to use key %s' % (client_name, keyid)) + logging.info('Setting client %s to use key %s', client_name, keyid) self.start_changes() client_id = self.find_client_id(self.tree, client_name) key = self.key(client_name, client_id, self.KEYID) diff --git a/obnamlib/fmt_6/clientlist_tests.py b/obnamlib/fmt_6/clientlist_tests.py index 44b0b79e..523334b1 100644 --- a/obnamlib/fmt_6/clientlist_tests.py +++ b/obnamlib/fmt_6/clientlist_tests.py @@ -42,7 +42,7 @@ class ClientListTests(unittest.TestCase): def test_unkey_unpacks_key_correctly(self): key = self.list.key('client name', 12765, 42) - client_hash, client_id, subkey = self.list.unkey(key) + _, client_id, subkey = self.list.unkey(key) self.assertEqual(client_id, 12765) self.assertEqual(subkey, 42) diff --git a/obnamlib/fmt_6/clientmetadatatree.py b/obnamlib/fmt_6/clientmetadatatree.py index 761d5be7..88a91843 100644 --- a/obnamlib/fmt_6/clientmetadatatree.py +++ b/obnamlib/fmt_6/clientmetadatatree.py @@ -15,7 +15,6 @@ import hashlib -import logging import os import random import struct @@ -208,7 +207,7 @@ class ClientMetadataTree(obnamlib.RepositoryTree): def _lookup_int(self, tree, key): try: return struct.unpack('!Q', tree.lookup(key))[0] - except: + except KeyError: return None def _insert_int(self, tree, key, value): @@ -357,11 +356,6 @@ class ClientMetadataTree(obnamlib.RepositoryTree): old_metadata = self.get_metadata(gen_id, filename) except KeyError: old_metadata = None - else: # pragma: no cover - old = obnamlib.fmt_6.metadata_codec.decode_metadata(old_metadata) - - metadata = obnamlib.fmt_6.metadata_codec.decode_metadata( - encoded_metadata) if encoded_metadata != old_metadata: tracing.trace('new or changed metadata') @@ -407,14 +401,6 @@ class ClientMetadataTree(obnamlib.RepositoryTree): file_id = self.get_file_id(self.tree, filename) genid = self.get_generation_id(self.tree) - try: - encoded_metadata = self.get_metadata(genid, filename) - except KeyError: - pass - else: - metadata = obnamlib.fmt_6.metadata_codec.decode_metadata( - encoded_metadata) - # Remove any children. minkey = self.fskey(file_id, self.DIR_CONTENTS, 0) maxkey = self.fskey(file_id, self.DIR_CONTENTS, obnamlib.MAX_ID) @@ -453,7 +439,7 @@ class ClientMetadataTree(obnamlib.RepositoryTree): minkey = self.fskey(dir_id, self.DIR_CONTENTS, 0) maxkey = self.fskey(dir_id, self.DIR_CONTENTS, self.SUBKEY_MAX) basenames = [] - for key, value in tree.lookup_range(minkey, maxkey): + for _, value in tree.lookup_range(minkey, maxkey): basenames.append(value) return basenames @@ -467,7 +453,7 @@ class ClientMetadataTree(obnamlib.RepositoryTree): maxkey = self.fskey(file_id, self.FILE_CHUNKS, self.SUBKEY_MAX) pairs = tree.lookup_range(minkey, maxkey) chunkids = [] - for key, value in pairs: + for _, value in pairs: chunkids.extend(self._decode_chunks(value)) return chunkids @@ -494,7 +480,7 @@ class ClientMetadataTree(obnamlib.RepositoryTree): minkey = self.fskey(file_id, self.FILE_CHUNKS, 0) maxkey = self.fskey(file_id, self.FILE_CHUNKS, self.SUBKEY_MAX) - for key, value in self.tree.lookup_range(minkey, maxkey): + for _, value in self.tree.lookup_range(minkey, maxkey): for chunkid in self._decode_chunks(value): k = self.chunk_key(chunkid, file_id) self.tree.remove_range(k, k) diff --git a/obnamlib/fmt_6/metadata_codec.py b/obnamlib/fmt_6/metadata_codec.py index a0cad418..b5505a6a 100644 --- a/obnamlib/fmt_6/metadata_codec.py +++ b/obnamlib/fmt_6/metadata_codec.py @@ -65,26 +65,24 @@ def encode_metadata(metadata): len(metadata.md5 or ''), len(metadata.xattr or '')) except TypeError, e: # pragma: no cover - logging.error('ERROR: Packing error due to %s' % str(e)) - logging.error('ERROR: st_mode=%s' % repr(metadata.st_mode)) - logging.error('ERROR: st_mtime_sec=%s' % repr(metadata.st_mtime_sec)) - logging.error( - 'ERROR: st_mtime_nsec=%s' % repr(metadata.st_mtime_nsec)) - logging.error('ERROR: st_atime_sec=%s' % repr(metadata.st_atime_sec)) - logging.error( - 'ERROR: st_atime_nsec=%s' % repr(metadata.st_atime_nsec)) - logging.error('ERROR: st_nlink=%s' % repr(metadata.st_nlink)) - logging.error('ERROR: st_size=%s' % repr(metadata.st_size)) - logging.error('ERROR: st_uid=%s' % repr(metadata.st_uid)) - logging.error('ERROR: st_gid=%s' % repr(metadata.st_gid)) - logging.error('ERROR: st_dev=%s' % repr(metadata.st_dev)) - logging.error('ERROR: st_ino=%s' % repr(metadata.st_ino)) - logging.error('ERROR: st_blocks=%s' % repr(metadata.st_blocks)) - logging.error('ERROR: groupname=%s' % repr(metadata.groupname)) - logging.error('ERROR: username=%s' % repr(metadata.username)) - logging.error('ERROR: target=%s' % repr(metadata.target)) - logging.error('ERROR: md5=%s' % repr(metadata.md5)) - logging.error('ERROR: xattr=%s' % repr(metadata.xattr)) + logging.error('ERROR: Packing error due to %s', str(e)) + logging.error('ERROR: st_mode=%s', repr(metadata.st_mode)) + logging.error('ERROR: st_mtime_sec=%s', repr(metadata.st_mtime_sec)) + logging.error('ERROR: st_mtime_nsec=%s', repr(metadata.st_mtime_nsec)) + logging.error('ERROR: st_atime_sec=%s', repr(metadata.st_atime_sec)) + logging.error('ERROR: st_atime_nsec=%s', repr(metadata.st_atime_nsec)) + logging.error('ERROR: st_nlink=%s', repr(metadata.st_nlink)) + logging.error('ERROR: st_size=%s', repr(metadata.st_size)) + logging.error('ERROR: st_uid=%s', repr(metadata.st_uid)) + logging.error('ERROR: st_gid=%s', repr(metadata.st_gid)) + logging.error('ERROR: st_dev=%s', repr(metadata.st_dev)) + logging.error('ERROR: st_ino=%s', repr(metadata.st_ino)) + logging.error('ERROR: st_blocks=%s', repr(metadata.st_blocks)) + logging.error('ERROR: groupname=%s', repr(metadata.groupname)) + logging.error('ERROR: username=%s', repr(metadata.username)) + logging.error('ERROR: target=%s', repr(metadata.target)) + logging.error('ERROR: md5=%s', repr(metadata.md5)) + logging.error('ERROR: xattr=%s', repr(metadata.xattr)) raise return (packed + (metadata.groupname or '') + diff --git a/obnamlib/fmt_6/repo_fmt_6.py b/obnamlib/fmt_6/repo_fmt_6.py index 9424df05..b69389d8 100644 --- a/obnamlib/fmt_6/repo_fmt_6.py +++ b/obnamlib/fmt_6/repo_fmt_6.py @@ -28,11 +28,6 @@ import tracing import obnamlib -class ToplevelIsFileError(obnamlib.ObnamError): - - msg = 'File at repository root: {filename}' - - class _OpenClientInfo(object): def __init__(self, client): @@ -74,7 +69,7 @@ class RepositoryFormat6(obnamlib.RepositoryInterface): self._setup_file_keys() @classmethod - def setup_hooks(self, hooks): + def setup_hooks(cls, hooks): hooks.new('repository-toplevel-init') hooks.new_filter('repository-data') hooks.new('repository-add-client') @@ -285,18 +280,18 @@ class RepositoryFormat6(obnamlib.RepositoryInterface): del self._open_client_infos[client_name] def client_is_locked(self, client_name): - logging.info('Checking if %s is locked' % client_name) + logging.info('Checking if %s is locked', client_name) client_id = self._get_client_id(client_name) client_dir = self._get_client_dir(client_id) return self._lockmgr.is_locked(client_dir) def lock_client(self, client_name): - logging.info('Locking client %s' % client_name) + logging.info('Locking client %s', client_name) self._setup_file_key_cache() self._raw_lock_client(client_name) def unlock_client(self, client_name): - logging.info('Unlocking client %s' % client_name) + logging.info('Unlocking client %s', client_name) self._require_existing_client(client_name) self._require_client_lock(client_name) self._reset_unused_chunks() @@ -353,7 +348,7 @@ class RepositoryFormat6(obnamlib.RepositoryInterface): def find_gens_to_keep(): keep = [] for gen_id in self.get_client_generation_ids(client_name): - a, gen_number = self._unpack_gen_id(gen_id) + _, gen_number = self._unpack_gen_id(gen_id) if gen_number not in remove_gen_nos: keep.append(gen_number) return keep @@ -714,7 +709,7 @@ class RepositoryFormat6(obnamlib.RepositoryInterface): self._fs, self._node_size, self._upload_queue_size, self._lru_size, self) self._chunksums = obnamlib.ChecksumTree( - self._fs, 'chunksums', len(self._checksum('')), self._node_size, + self._fs, 'chunksums', len(self._checksum('')), self._node_size, self._upload_queue_size, self._lru_size, self) def _chunk_index_dirs_to_lock(self): @@ -1045,6 +1040,8 @@ class RepositoryFormat6(obnamlib.RepositoryInterface): class CheckBTree(obnamlib.WorkItem): # pragma: no cover + settings = None + def __init__(self, fs, dirname, skip_setting): self.fs = fs self.dirname = dirname @@ -1056,10 +1053,10 @@ class CheckBTree(obnamlib.WorkItem): # pragma: no cover return if not self.fs.exists(self.dirname): - logging.debug('B-tree %s does not exist, skipping' % self.dirname) + logging.debug('B-tree %s does not exist, skipping', self.dirname) return - logging.debug('Checking B-tree %s' % self.dirname) + logging.debug('Checking B-tree %s', self.dirname) fix = self.settings['fsck-fix'] forest = larch.open_forest( diff --git a/obnamlib/fmt_ga/chunk_store.py b/obnamlib/fmt_ga/chunk_store.py index f7e486d5..df0ca4c8 100644 --- a/obnamlib/fmt_ga/chunk_store.py +++ b/obnamlib/fmt_ga/chunk_store.py @@ -16,10 +16,6 @@ # =*= License: GPL-3+ =*= -import errno -import os -import random - import obnamlib diff --git a/obnamlib/fmt_ga/client.py b/obnamlib/fmt_ga/client.py index 4895a65a..b151ff58 100644 --- a/obnamlib/fmt_ga/client.py +++ b/obnamlib/fmt_ga/client.py @@ -422,9 +422,6 @@ class GAGeneration(object): def set_number(self, new_id): self._id = new_id - def keys(self): - return self._keys.keys() - def get_key(self, key, default=None): return self._keys.get_key(key, default=default) @@ -488,7 +485,7 @@ class GAFileMetadata(object): def file_exists(self, filename): if filename in self._added_files: return True - dir_obj, dir_path, basename = self._get_dir_obj(filename) + dir_obj, _, basename = self._get_dir_obj(filename) return dir_obj and basename in dir_obj.get_file_basenames() def _get_dir_obj(self, filename): @@ -541,7 +538,7 @@ class GAFileMetadata(object): def get_file_key(self, filename, key): if filename in self._added_files: return self._added_files.get_file_key(filename, key) - dir_obj, dir_path, basename = self._get_dir_obj(filename) + dir_obj, _, basename = self._get_dir_obj(filename) if dir_obj: return dir_obj.get_file_key(basename, key) else: @@ -552,7 +549,7 @@ class GAFileMetadata(object): return self._make_metadata( lambda key: self._added_files.get_file_key(filename, key)) else: - dir_obj, dir_path, basename = self._get_dir_obj(filename) + dir_obj, _, basename = self._get_dir_obj(filename) # We've already verifed the file exists, so we don't need # to handle the case where dir_obj is None. assert dir_obj is not None @@ -656,7 +653,7 @@ class GAFileMetadata(object): chunk_ids = self._added_files.get_file_chunk_ids(filename) return chunk_ids - dir_obj, dir_path, basename = self._get_dir_obj(filename) + dir_obj, _, basename = self._get_dir_obj(filename) if dir_obj: chunk_ids = dir_obj.get_file_chunk_ids(basename) return chunk_ids diff --git a/obnamlib/fmt_ga/client_list.py b/obnamlib/fmt_ga/client_list.py index ccc49a94..78dc1d03 100644 --- a/obnamlib/fmt_ga/client_list.py +++ b/obnamlib/fmt_ga/client_list.py @@ -16,7 +16,6 @@ # =*= License: GPL-3+ =*= -import errno import os import random diff --git a/obnamlib/fmt_ga/tree.py b/obnamlib/fmt_ga/tree.py index 54c1e942..a10a7c48 100644 --- a/obnamlib/fmt_ga/tree.py +++ b/obnamlib/fmt_ga/tree.py @@ -18,8 +18,6 @@ import os -import tracing - import obnamlib diff --git a/obnamlib/fmt_ga/tree_tests.py b/obnamlib/fmt_ga/tree_tests.py index 37df70fa..c0bb69a5 100644 --- a/obnamlib/fmt_ga/tree_tests.py +++ b/obnamlib/fmt_ga/tree_tests.py @@ -118,7 +118,6 @@ class GATreeTests(unittest.TestCase): self.assertEqual(tree3.get_directory('/foo/bar'), None) def test_removes_nonexistent_directory(self): - dir_obj = obnamlib.GADirectory() self.tree.remove_directory('/foo/bar') self.assertEqual(self.tree.get_directory('/foo/bar'), None) diff --git a/obnamlib/forget_policy_tests.py b/obnamlib/forget_policy_tests.py index 79f0355e..536a834e 100644 --- a/obnamlib/forget_policy_tests.py +++ b/obnamlib/forget_policy_tests.py @@ -67,7 +67,7 @@ class ForgetPolicyMatchTests(unittest.TestCase): def match2(self, spec, times): rules = self.fp.parse(spec) - return [dt for i, dt in self.fp.match(rules, list(enumerate(times)))] + return [dt for _, dt in self.fp.match(rules, list(enumerate(times)))] def test_hourly_matches(self): h0m0 = datetime.datetime(2000, 1, 1, 0, 0) @@ -122,7 +122,7 @@ class ForgetPolicyMatchTests(unittest.TestCase): genlist = list(enumerate([d1h0m0, d1h0m1, d2h0m0, d2h0m1, d3h0m0, d3h0m1])) rules = self.fp.parse('1h,2d') - self.assertEqual([dt for genid, dt in self.fp.match(rules, genlist)], + self.assertEqual([dt for _, dt in self.fp.match(rules, genlist)], [d2h0m1, d3h0m1]) def test_hourly_and_daily_together_when_only_daily_backups(self): diff --git a/obnamlib/fsck_work_item.py b/obnamlib/fsck_work_item.py index 5b316a1a..350a9d41 100644 --- a/obnamlib/fsck_work_item.py +++ b/obnamlib/fsck_work_item.py @@ -1,4 +1,4 @@ -# Copyright 2013 Lars Wirzenius +# Copyright 2013,2015 Lars Wirzenius # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -28,3 +28,7 @@ class WorkItem(larch.fsck.WorkItem): implementations) can provide work items. ''' + + repo = None + chunkids_seen = None + settings = None diff --git a/obnamlib/hooks.py b/obnamlib/hooks.py index bdb3fcea..4a285a5c 100644 --- a/obnamlib/hooks.py +++ b/obnamlib/hooks.py @@ -26,7 +26,6 @@ application) will register callbacks. ''' -import logging import tracing import obnamlib @@ -99,9 +98,9 @@ class FilterHook(Hook): self.bytag = {} def add_callback(self, callback, priority=Hook.DEFAULT_PRIORITY): - assert(hasattr(callback, "tag")) - assert(hasattr(callback, "filter_read")) - assert(hasattr(callback, "filter_write")) + assert hasattr(callback, "tag") + assert hasattr(callback, "filter_read") + assert hasattr(callback, "filter_write") self.bytag[callback.tag] = callback return Hook.add_callback(self, callback, priority) diff --git a/obnamlib/lockmgr.py b/obnamlib/lockmgr.py index a9c4216b..bf56f7c7 100644 --- a/obnamlib/lockmgr.py +++ b/obnamlib/lockmgr.py @@ -14,7 +14,6 @@ # along with this program. If not, see <http://www.gnu.org/licenses/>. -import errno import os import time diff --git a/obnamlib/metadata.py b/obnamlib/metadata.py index 5efb21c8..e323061a 100644 --- a/obnamlib/metadata.py +++ b/obnamlib/metadata.py @@ -33,7 +33,7 @@ metadata_verify_fields = ( 'xattr', ) metadata_fields = metadata_verify_fields + ( - 'st_blocks', 'st_dev', 'st_gid', 'st_ino', 'st_atime_sec', + 'st_blocks', 'st_dev', 'st_gid', 'st_ino', 'st_atime_sec', 'st_atime_nsec', 'md5', ) @@ -81,6 +81,8 @@ class Metadata(object): ''' def __init__(self, **kwargs): + self.md5 = None # Silence pylint. + self.st_size = None # Silence pylint. for field in metadata_fields: setattr(self, field, None) for field, value in kwargs.iteritems(): @@ -164,7 +166,8 @@ def get_xattrs_as_blob(fs, filename): # pragma: no cover names.remove(name) logging.warning( '%s has extended attribute named %s without value, ' - 'ignoring attribute' % (filename, name)) + 'ignoring attribute', + filename, name) else: raise else: @@ -288,7 +291,7 @@ def set_metadata(fs, filename, metadata, try: uid = -1 # no change to user fs.lchown(filename, uid, metadata.st_gid) - except (OSError), e: + except OSError: sys.exc_clear() # If we are not the owner, and not root, do not restore setuid/setgid, @@ -312,7 +315,7 @@ def set_metadata(fs, filename, metadata, _set_something( filename, 'xattrs', lambda: - set_xattrs_from_blob(fs, filename, metadata.xattr, user_only)) + set_xattrs_from_blob(fs, filename, metadata.xattr, user_only)) _set_something( filename, 'timestamps', diff --git a/obnamlib/obj_serialiser.py b/obnamlib/obj_serialiser.py index 37cc5722..3bcdb470 100644 --- a/obnamlib/obj_serialiser.py +++ b/obnamlib/obj_serialiser.py @@ -43,11 +43,12 @@ def serialise_object(obj): def deserialise_object(serialised): type_byte = serialised[0] + # We skip decoding of the value length, since we can assume all of # the rest of the value is to be decoded. value = serialised[1 + _length_size:] - func = _deserialisers[serialised[0]] + func = _deserialisers[type_byte] return func(value) diff --git a/obnamlib/pluginbase.py b/obnamlib/pluginbase.py index ddb5abb4..42cdaa16 100644 --- a/obnamlib/pluginbase.py +++ b/obnamlib/pluginbase.py @@ -24,5 +24,8 @@ class ObnamPlugin(cliapp.Plugin): def __init__(self, app): self.app = app + def enable(self): + raise NotImplementedError() + def disable(self): pass diff --git a/obnamlib/pluginbase_tests.py b/obnamlib/pluginbase_tests.py index 53f2367e..1281038f 100644 --- a/obnamlib/pluginbase_tests.py +++ b/obnamlib/pluginbase_tests.py @@ -34,5 +34,8 @@ class ObnamPluginTests(unittest.TestCase): def test_has_an_app(self): self.assertEqual(self.plugin.app, self.fakeapp) + def test_enable_is_not_implemented(self): + self.assertRaises(NotImplementedError, self.plugin.enable) + def test_disable_is_implemented(self): self.plugin.disable() diff --git a/obnamlib/plugins/backup_plugin.py b/obnamlib/plugins/backup_plugin.py index 389f066d..d711ab94 100644 --- a/obnamlib/plugins/backup_plugin.py +++ b/obnamlib/plugins/backup_plugin.py @@ -308,7 +308,7 @@ class BackupPlugin(obnamlib.ObnamPlugin): self.remove_checkpoints() self.finish_backup(args) except BaseException, e: - logging.debug('Handling exception %s' % str(e)) + logging.debug('Handling exception %s', str(e)) logging.debug(traceback.format_exc()) self.unlock_when_error() raise @@ -349,7 +349,7 @@ class BackupPlugin(obnamlib.ObnamPlugin): if self.pretend: try: return self.app.get_repository_object() - except Exception as e: + except Exception: self.progress.error( 'Are you using --pretend without an existing ' 'repository? That does not\n' @@ -471,8 +471,7 @@ class BackupPlugin(obnamlib.ObnamPlugin): 'Attempting to unlock shared trees because of error') self.repo.unlock_chunk_indexes() except BaseException, e2: - logging.warning( - 'Error while unlocking due to error: %s' % str(e2)) + logging.warning('Error while unlocking due to error: %s', str(e2)) logging.debug(traceback.format_exc()) else: logging.info('Successfully unlocked') @@ -508,7 +507,7 @@ class BackupPlugin(obnamlib.ObnamPlugin): self.fs.close() def backup_root(self, root, absroots): - logging.info('Backing up root %s' % root) + logging.info('Backing up root %s', root) self.progress.what('connecting to live data %s' % root) self.reopen_fs(root) @@ -525,7 +524,7 @@ class BackupPlugin(obnamlib.ObnamPlugin): self.root_metadata = self.fs.lstat(absroot) for pathname, metadata in self.find_files(absroot): - logging.info('Backing up %s' % pathname) + logging.info('Backing up %s', pathname) if not self.pretend: existed = self.repo.file_exists(self.new_generation, pathname) try: @@ -637,7 +636,7 @@ class BackupPlugin(obnamlib.ObnamPlugin): # details of the old repository class. Should # be cleaned up, someday. pass - except Exception as ee: + except (OSError, IOError, obnamlib.ObnamError) as ee: msg = ( 'Error removing partly backed up file %s: %s' % (pathname, repr(ee))) @@ -729,6 +728,7 @@ class BackupPlugin(obnamlib.ObnamPlugin): exclude = [False] self.app.hooks.call( 'backup-exclude', + progress=self.progress, fs=self.fs, pathname=pathname, stat_result=st, @@ -776,7 +776,7 @@ class BackupPlugin(obnamlib.ObnamPlugin): try: old = self.get_metadata_from_generation(gen, pathname) - except obnamlib.ObnamError as e: + except obnamlib.ObnamError: # File does not exist in the previous generation, so it # does need to be backed up. return True @@ -828,9 +828,9 @@ class BackupPlugin(obnamlib.ObnamPlugin): metadata = obnamlib.read_metadata(self.fs, root) except OSError, e: logging.warning( - 'Failed to get metadata for %s: %s: %s' % - (root, e.errno or 0, e.strerror)) - logging.warning('Using fake metadata instead for %s' % root) + 'Failed to get metadata for %s: %s: %s', + root, e.errno or 0, e.strerror) + logging.warning('Using fake metadata instead for %s', root) metadata = dummy_metadata if not self.pretend: self.add_file_to_generation(root, metadata) @@ -940,10 +940,9 @@ class BackupPlugin(obnamlib.ObnamPlugin): if data == data2: share(chunkid) return chunkid - else: - chunkid = put() - share(chunkid) - return chunkid + chunkid = put() + share(chunkid) + return chunkid elif mode == 'fatalist': existing = find() if existing: diff --git a/obnamlib/plugins/compression_plugin.py b/obnamlib/plugins/compression_plugin.py index b2ade5ba..a9eb2d21 100644 --- a/obnamlib/plugins/compression_plugin.py +++ b/obnamlib/plugins/compression_plugin.py @@ -14,7 +14,6 @@ # along with this program. If not, see <http://www.gnu.org/licenses/>. -import logging import zlib import obnamlib diff --git a/obnamlib/plugins/encryption_plugin.py b/obnamlib/plugins/encryption_plugin.py index 2ff62a69..ec3bccab 100644 --- a/obnamlib/plugins/encryption_plugin.py +++ b/obnamlib/plugins/encryption_plugin.py @@ -162,12 +162,13 @@ class EncryptionPlugin(obnamlib.ObnamPlugin): def remove_from_userkeys(self, repo, toplevel, keyid): userkeys = self.read_keyring(repo, toplevel) if keyid in userkeys: - logging.debug('removing key %s from %s' % (keyid, toplevel)) + logging.debug('removing key %s from %s', keyid, toplevel) userkeys.remove(keyid) self.write_keyring(repo, toplevel, userkeys) else: - logging.debug('unable to remove key %s from %s (not there)' % - (keyid, toplevel)) + logging.debug( + 'unable to remove key %s from %s (not there)', + keyid, toplevel) def rewrite_symmetric_key(self, repo, toplevel): symmetric_key = self.get_symmetric_key(repo, toplevel) @@ -231,7 +232,7 @@ class EncryptionPlugin(obnamlib.ObnamPlugin): if self.quit_if_unencrypted(): return repo = self.app.get_repository_object() - keys, tops = self._find_keys_and_toplevels(repo) + keys, _ = self._find_keys_and_toplevels(repo) for keyid in keys: print 'key: %s' % self._get_key_string(keyid) for toplevel in keys[keyid]: @@ -242,7 +243,7 @@ class EncryptionPlugin(obnamlib.ObnamPlugin): if self.quit_if_unencrypted(): return repo = self.app.get_repository_object() - keys, tops = self._find_keys_and_toplevels(repo) + _, tops = self._find_keys_and_toplevels(repo) for toplevel in tops: print 'toplevel: %s' % toplevel for keyid in tops[toplevel]: @@ -284,7 +285,7 @@ class EncryptionPlugin(obnamlib.ObnamPlugin): repo = self.app.get_repository_object() repo.lock_client_list() for client_name in args: - logging.info('removing client %s' % client_name) + logging.info('removing client %s', client_name) repo.remove_client(client_name) repo.commit_client_list() repo.unlock_client_list() diff --git a/obnamlib/plugins/exclude_caches_plugin.py b/obnamlib/plugins/exclude_caches_plugin.py index e5990a74..8347527c 100644 --- a/obnamlib/plugins/exclude_caches_plugin.py +++ b/obnamlib/plugins/exclude_caches_plugin.py @@ -46,12 +46,12 @@ class ExcludeCachesPlugin(obnamlib.ObnamPlugin): if stat.S_ISDIR(stat_result.st_mode): tag_filename = 'CACHEDIR.TAG' tag_contents = 'Signature: 8a477f597d28d172789f06886806bc55' - tag_path = os.path.join(pathname, 'CACHEDIR.TAG') + tag_path = os.path.join(pathname, tag_filename) if fs.exists(tag_path): # Can't use with, because Paramiko's SFTPFile does not work. f = fs.open(tag_path, 'rb') data = f.read(len(tag_contents)) f.close() if data == tag_contents: - logging.debug('Excluding (cache dir): %s' % pathname) + logging.debug('Excluding (cache dir): %s', pathname) exclude[0] = True diff --git a/obnamlib/plugins/exclude_pathnames_plugin.py b/obnamlib/plugins/exclude_pathnames_plugin.py index 9aad8070..63ab1c22 100644 --- a/obnamlib/plugins/exclude_pathnames_plugin.py +++ b/obnamlib/plugins/exclude_pathnames_plugin.py @@ -14,11 +14,8 @@ # along with this program. If not, see <http://www.gnu.org/licenses/>. -import hashlib import logging -import os -import stat -import time +import re import obnamlib @@ -53,10 +50,16 @@ class ExcludePathnamesPlugin(obnamlib.ObnamPlugin): def config_loaded(self): self.app.hooks.add_callback('backup-exclude', self.exclude) self.pathname_excluder = obnamlib.PathnameExcluder() - self.compile_exclusion_patterns() - self.compile_inclusion_patterns() + self.patterns_have_been_compiled = False + + def exclude(self, pathname=None, stat_result=None, exclude=None, + progress=None, **kwargs): + + if not self.patterns_have_been_compiled: + self.compile_exclusion_patterns(progress) + self.compile_inclusion_patterns(progress) + self.patterns_have_been_compiled = True - def exclude(self, pathname=None, stat_result=None, exclude=None, **kwargs): is_excluded, regexp = self.pathname_excluder.exclude(pathname) if is_excluded: logging.debug('Exclude (pattern): %s', pathname) @@ -64,7 +67,7 @@ class ExcludePathnamesPlugin(obnamlib.ObnamPlugin): elif regexp is not None: logging.debug('Include due to regexp: %s', pathname) - def compile_exclusion_patterns(self): + def compile_exclusion_patterns(self, progress): regexps = self.read_patterns_from_files( self.app.settings['exclude-from']) @@ -77,15 +80,17 @@ class ExcludePathnamesPlugin(obnamlib.ObnamPlugin): regexps.append(log) logging.debug('Compiling exclusion patterns') - self.compile_regexps(regexps, self.pathname_excluder.exclude_regexp) + self.compile_regexps( + regexps, self.pathname_excluder.exclude_regexp, progress) - def compile_inclusion_patterns(self): + def compile_inclusion_patterns(self, progress): logging.debug('Compiling inclusion patterns') self.compile_regexps( self.app.settings['include'], - self.pathname_excluder.allow_regexp) + self.pathname_excluder.allow_regexp, + progress) - def compile_regexps(self, regexps, compiler): + def compile_regexps(self, regexps, compiler, progress): for regexp in regexps: if not regexp: logging.debug('Ignoring empty pattern') @@ -94,9 +99,10 @@ class ExcludePathnamesPlugin(obnamlib.ObnamPlugin): try: compiler(regexp) except re.error as e: - msg = ('error compiling regular expression "%s": %s' % (x, e)) + msg = ('error compiling regular expression "%s": %s' % + (regexp, e)) logging.error(msg) - self.progress.error(msg) + progress.error(msg) def read_patterns_from_files(self, filenames): patterns = [] diff --git a/obnamlib/plugins/force_lock_plugin.py b/obnamlib/plugins/force_lock_plugin.py index e688043d..85995bec 100644 --- a/obnamlib/plugins/force_lock_plugin.py +++ b/obnamlib/plugins/force_lock_plugin.py @@ -38,7 +38,7 @@ class ForceLockPlugin(obnamlib.ObnamPlugin): repourl = self.app.settings['repository'] client_name = self.app.settings['client-name'] logging.info('Forcing lock') - logging.info('Repository: %s' % repourl) + logging.info('Repository: %s', repourl) try: repo = self.app.get_repository_object() @@ -66,8 +66,8 @@ class ForceLockPlugin(obnamlib.ObnamPlugin): repourl = self.app.settings['repository'] client_name = self.app.settings['client-name'] logging.info('Creating lock') - logging.info('Repository: %s' % repourl) - logging.info('Client: %s' % client_name) + logging.info('Repository: %s', repourl) + logging.info('Client: %s', client_name) try: repo = self.app.get_repository_object() diff --git a/obnamlib/plugins/forget_plugin.py b/obnamlib/plugins/forget_plugin.py index c7352c99..1f9cb236 100644 --- a/obnamlib/plugins/forget_plugin.py +++ b/obnamlib/plugins/forget_plugin.py @@ -127,7 +127,7 @@ class ForgetPlugin(obnamlib.ObnamPlugin): rules = fp.parse(self.app.settings['keep']) keeplist = fp.match(rules, genlist) keepids = set(genid for genid, dt in keeplist) - return [genid for genid, dt in genlist if genid not in keepids] + return [genid for genid, _ in genlist if genid not in keepids] def remove(self, genid): if self.app.settings['pretend']: diff --git a/obnamlib/plugins/fsck_plugin.py b/obnamlib/plugins/fsck_plugin.py index d4bb1fa4..b541ae5c 100644 --- a/obnamlib/plugins/fsck_plugin.py +++ b/obnamlib/plugins/fsck_plugin.py @@ -31,7 +31,7 @@ class CheckChunk(WorkItem): self.name = 'chunk %s' % chunkid def do(self): - logging.debug('Checking chunk %s' % self.chunkid) + logging.debug('Checking chunk %s', self.chunkid) if not self.repo.has_chunk(self.chunkid): self.error('chunk %s does not exist' % self.chunkid) else: @@ -55,7 +55,7 @@ class CheckFileChecksum(WorkItem): self.checksummer = checksummer def do(self): - logging.debug('Checking whole-file checksum for %s' % self.filename) + logging.debug('Checking whole-file checksum for %s', self.filename) if self.correct != self.checksummer.digest(): self.error('%s whole-file checksum mismatch' % self.name) @@ -69,8 +69,9 @@ class CheckFile(WorkItem): self.name = 'file %s:%s:%s' % (client_name, genid, filename) def do(self): - logging.debug('Checking client=%s genid=%s filename=%s' % - (self.client_name, self.genid, self.filename)) + logging.debug( + 'Checking client=%s genid=%s filename=%s', + self.client_name, self.genid, self.filename) mode = self.repo.get_file_key( self.genid, self.filename, obnamlib.REPO_FILE_MODE) if stat.S_ISREG(mode) and not self.settings['fsck-ignore-chunks']: @@ -93,8 +94,9 @@ class CheckDirectory(WorkItem): self.name = 'dir %s:%s:%s' % (client_name, genid, dirname) def do(self): - logging.debug('Checking client=%s genid=%s dirname=%s' % - (self.client_name, self.genid, self.dirname)) + logging.debug( + 'Checking client=%s genid=%s dirname=%s', + self.client_name, self.genid, self.dirname) for pathname in self.repo.get_file_children(self.genid, self.dirname): mode = self.repo.get_file_key( self.genid, pathname, obnamlib.REPO_FILE_MODE) @@ -112,8 +114,8 @@ class CheckGeneration(WorkItem): self.name = 'generation %s:%s' % (client_name, genid) def do(self): - logging.debug('Checking client=%s genid=%s' % - (self.client_name, self.genid)) + logging.debug( + 'Checking client=%s genid=%s', self.client_name, self.genid) started = self.repo.get_generation_key( self.genid, obnamlib.REPO_GENERATION_STARTED) @@ -151,8 +153,8 @@ class CheckGenerationIdsAreDifferent(WorkItem): self.genids = list(genids) def do(self): - logging.debug('Checking genid uniqueness for client=%s' % - self.client_name) + logging.debug( + 'Checking genid uniqueness for client=%s', self.client_name) done = set() while self.genids: genid = self.genids.pop() @@ -169,7 +171,7 @@ class CheckClient(WorkItem): self.name = 'client %s' % client_name def do(self): - logging.debug('Checking client=%s' % self.client_name) + logging.debug('Checking client=%s', self.client_name) genids = self.repo.get_client_generation_ids(self.client_name) yield CheckGenerationIdsAreDifferent(self.client_name, genids) if self.settings['fsck-skip-generations']: @@ -299,7 +301,7 @@ class FsckPlugin(obnamlib.ObnamPlugin): def fsck(self, args): '''Verify internal consistency of backup repository.''' self.app.settings.require('repository') - logging.debug('fsck on %s' % self.app.settings['repository']) + logging.debug('fsck on %s', self.app.settings['repository']) rm_unused_chunks = self.app.settings['fsck-rm-unused'] \ or self.app.settings['fsck-fix'] @@ -324,7 +326,7 @@ class FsckPlugin(obnamlib.ObnamPlugin): while self.work_items: work = self.work_items.pop(0) - logging.debug('doing: %s' % str(work)) + logging.debug('doing: %s', str(work)) self.app.ts['item'] = work self.app.ts.increase('this_item', 1) pos = 0 @@ -347,7 +349,7 @@ class FsckPlugin(obnamlib.ObnamPlugin): sys.exit(1) def add_item(self, work, append=False, pos=0): - logging.debug('adding: %s' % str(work)) + logging.debug('adding: %s', str(work)) work.warning = self.warning work.error = self.error work.repo = self.repo diff --git a/obnamlib/plugins/fuse_plugin.py b/obnamlib/plugins/fuse_plugin.py index a52b77d6..8c63bd44 100644 --- a/obnamlib/plugins/fuse_plugin.py +++ b/obnamlib/plugins/fuse_plugin.py @@ -32,7 +32,7 @@ except ImportError: # this plugin file can be imported. If the module isn't there, the # plugin won't work, and it will tell the user it won't work, but # at least Obnam won't crash at startup. - class Bunch: + class Bunch(object): def __init__(self, **kwds): self.__dict__.update(kwds) fuse = Bunch(Fuse=object) @@ -98,9 +98,8 @@ class ObnamFuseFile(object): if flags & self.write_flags: raise IOError(errno.EROFS, 'Read only filesystem') - if path == '/.pid': - self.read = self.read_pid - self.release = self.release_pid + self.reading_pid = path == '/.pid' + if self.reading_pid: return try: @@ -113,6 +112,18 @@ class ObnamFuseFile(object): if not stat.S_ISREG(self.metadata.st_mode): raise IOError(errno.EINVAL, 'Invalid argument') + def read(self, length, offset): + if self.reading_pid: + return self.read_pid(length, offset) + else: + return self.read_data(length, offset) + + def release(self, flags): + if self.reading_pid: + return self.release_pid(flags) + else: + return self.release_data(flags) + def read_pid(self, length, offset): tracing.trace('length=%r', length) tracing.trace('offset=%r', offset) @@ -130,7 +141,7 @@ class ObnamFuseFile(object): tracing.trace('called') return self.fuse_fs.getattr(self.path) - def read(self, length, offset): + def read_data(self, length, offset): tracing.trace('self.path=%r', self.path) tracing.trace('length=%r', length) tracing.trace('offset=%r', offset) @@ -185,7 +196,7 @@ class ObnamFuseFile(object): return ''.join(output) - def release(self, flags): + def release_data(self, flags): tracing.trace('flags=%r', flags) return 0 @@ -239,16 +250,13 @@ class ObnamFuse(fuse.Fuse): path = '/' + genspec try: genstat = self.get_stat_in_generation(path) - start = self.obnam.repo.get_generation_key( - gen, obnamlib.REPO_GENERATION_STARTED) end = self.obnam.repo.get_generation_key( gen, obnamlib.REPO_GENERATION_ENDED) genstat.st_ctime = genstat.st_mtime = end self.rootlist[path] = genstat used_generations.append(gen) except obnamlib.ObnamError as e: - logging.warning('Ignoring error %s' % str(e)) - pass + logging.warning('Ignoring error %s', str(e)) assert used_generations @@ -380,7 +388,7 @@ class ObnamFuse(fuse.Fuse): total_data = sum( self.obnam.repo.get_generation_key( gen, obnamlib.REPO_GENERATION_TOTAL_DATA) - for gen in obnamlib.repo.get_clientgeneration_ids(client_name)) + for gen in self.obnam.repo.get_clientgeneration_ids(client_name)) files = sum( self.obnam.repo.get_generation_key( diff --git a/obnamlib/plugins/list_formats_plugin.py b/obnamlib/plugins/list_formats_plugin.py index ae6d0ea9..f7e0d49c 100644 --- a/obnamlib/plugins/list_formats_plugin.py +++ b/obnamlib/plugins/list_formats_plugin.py @@ -14,12 +14,6 @@ # along with this program. If not, see <http://www.gnu.org/licenses/>. -import hashlib -import logging -import os -import stat -import time - import obnamlib @@ -34,5 +28,5 @@ class ListFormatsPlugin(obnamlib.ObnamPlugin): def list_formats(self, args): factory = obnamlib.RepositoryFactory() formats = factory.get_implementation_classes() - for format in formats: - self.app.output.write('%s\n' % format.format) + for format_class in formats: + self.app.output.write('%s\n' % format_class.format) diff --git a/obnamlib/plugins/one_file_system_plugin.py b/obnamlib/plugins/one_file_system_plugin.py index 71315795..94101b8d 100644 --- a/obnamlib/plugins/one_file_system_plugin.py +++ b/obnamlib/plugins/one_file_system_plugin.py @@ -43,5 +43,5 @@ class OneFileSystemPlugin(obnamlib.ObnamPlugin): exclude = kwargs['exclude'] if st.st_dev != root_metadata.st_dev: - logging.debug('Excluding (one-file-system): %s' % pathname) + logging.debug('Excluding (one-file-system): %s', pathname) exclude[0] = True diff --git a/obnamlib/plugins/restore_plugin.py b/obnamlib/plugins/restore_plugin.py index 71266b62..239eef45 100644 --- a/obnamlib/plugins/restore_plugin.py +++ b/obnamlib/plugins/restore_plugin.py @@ -124,11 +124,11 @@ class RestorePlugin(obnamlib.ObnamPlugin): self.app.settings.require('generation') self.app.settings.require('to') - logging.debug('restoring generation %s' % - self.app.settings['generation']) - logging.debug('restoring to %s' % self.app.settings['to']) + logging.debug( + 'restoring generation %s', self.app.settings['generation']) + logging.debug('restoring to %s', self.app.settings['to']) - logging.debug('restoring what: %s' % repr(args)) + logging.debug('restoring what: %s', repr(args)) if not args: logging.debug('no args given, so restoring everything') args = ['/'] @@ -230,7 +230,7 @@ class RestorePlugin(obnamlib.ObnamPlugin): except obnamlib.SetMetadataError as e: self.app.ts.error(str(e)) self.errors = True - except Exception, e: + except Exception, e: # pylint: disable=broad-except # Reaching this code path means we've hit a bug, so we log # a full traceback. msg = "Failed to restore %s:" % (pathname,) @@ -239,7 +239,7 @@ class RestorePlugin(obnamlib.ObnamPlugin): self.errors = True def restore_dir(self, gen, root, metadata): - logging.debug('restoring dir %s' % root) + logging.debug('restoring dir %s', root) if self.write_ok: if not self.fs.exists('./' + root): self.fs.mkdir('./' + root) @@ -247,13 +247,13 @@ class RestorePlugin(obnamlib.ObnamPlugin): 'after recursing through %s' % repr(root)) def restore_hardlink(self, filename, link, metadata): - logging.debug('restoring hardlink %s to %s' % (filename, link)) + logging.debug('restoring hardlink %s to %s', filename, link) if self.write_ok: self.fs.link('./' + link, './' + filename) self.hardlinks.forget(metadata) def restore_symlink(self, gen, filename, metadata): - logging.debug('restoring symlink %s' % filename) + logging.debug('restoring symlink %s', filename) def restore_first_link(self, gen, filename, metadata): if stat.S_ISREG(metadata.st_mode): @@ -271,7 +271,7 @@ class RestorePlugin(obnamlib.ObnamPlugin): self.app.ts.notify(msg) def restore_regular_file(self, gen, filename, metadata): - logging.debug('restoring regular %s' % filename) + logging.debug('restoring regular %s', filename) if self.write_ok: f = self.fs.open('./' + filename, 'wb') summer = hashlib.md5() @@ -329,71 +329,35 @@ class RestorePlugin(obnamlib.ObnamPlugin): pass def restore_fifo(self, gen, filename, metadata): - logging.debug('restoring fifo %s' % filename) + logging.debug('restoring fifo %s', filename) if self.write_ok: self.fs.mknod('./' + filename, metadata.st_mode) def restore_socket(self, gen, filename, metadata): - logging.debug('restoring socket %s' % filename) + logging.debug('restoring socket %s', filename) if self.write_ok: self.fs.mknod('./' + filename, metadata.st_mode) def restore_device(self, gen, filename, metadata): - logging.debug('restoring device %s' % filename) + logging.debug('restoring device %s', filename) if self.write_ok: self.fs.mknod('./' + filename, metadata.st_mode) def report_stats(self): - size_table = [ - (1024**4, 'TiB'), - (1024**3, 'GiB'), - (1024**2, 'MiB'), - (1024**1, 'KiB'), - (0, 'B') - ] - - for size_base, size_unit in size_table: - if self.downloaded_bytes >= size_base: - if size_base > 0: - size_amount = (float(self.downloaded_bytes) / - float(size_base)) - else: - size_amount = float(self.downloaded_bytes) - break - - speed_table = [ - (1024**3, 'GiB/s'), - (1024**2, 'MiB/s'), - (1024**1, 'KiB/s'), - (0, 'B/s') - ] duration = time.time() - self.started - speed = float(self.downloaded_bytes) / duration - for speed_base, speed_unit in speed_table: - if speed >= speed_base: - if speed_base > 0: - speed_amount = speed / speed_base - else: - speed_amount = speed - break - - duration_string = '' - seconds = duration - if seconds >= 3600: - duration_string += '%dh' % int(seconds/3600) - seconds %= 3600 - if seconds >= 60: - duration_string += '%dm' % int(seconds/60) - seconds %= 60 - if seconds > 0: - duration_string += '%ds' % round(seconds) + size_amount, size_unit = obnamlib.humanise_size( + self.downloaded_bytes) + speed_amount, speed_unit = obnamlib.humanise_speed( + self.downloaded_bytes, duration) + duration_string = obnamlib.humanise_duration(duration) logging.info('Restore performance statistics:') - logging.info('* files restored: %s' % self.file_count) - logging.info('* downloaded data: %s bytes (%s %s)' % - (self.downloaded_bytes, size_amount, size_unit)) - logging.info('* duration: %s s' % duration) - logging.info('* average speed: %s %s' % (speed_amount, speed_unit)) + logging.info('* files restored: %s', self.file_count) + logging.info( + '* downloaded data: %s bytes (%s %s)', + self.downloaded_bytes, size_amount, size_unit) + logging.info('* duration: %s s', duration) + logging.info('* average speed: %s %s', speed_amount, speed_unit) self.app.ts.notify( 'Restored %d files, ' 'downloaded %.1f %s in %s at %.1f %s average speed' % diff --git a/obnamlib/plugins/sftp_plugin.py b/obnamlib/plugins/sftp_plugin.py index 7351d911..5136058d 100644 --- a/obnamlib/plugins/sftp_plugin.py +++ b/obnamlib/plugins/sftp_plugin.py @@ -176,8 +176,8 @@ class SftpFS(obnamlib.VirtualFileSystem): def log_stats(self): obnamlib.VirtualFileSystem.log_stats(self) - logging.info('VFS: baseurl=%s roundtrips=%s' % - (self.baseurl, self._roundtrips)) + logging.info( + 'VFS: baseurl=%s roundtrips=%s', self.baseurl, self._roundtrips) def _to_string(self, str_or_unicode): if type(str_or_unicode) is unicode: @@ -188,7 +188,7 @@ class SftpFS(obnamlib.VirtualFileSystem): def _create_root_if_missing(self): try: self.mkdir(self.path) - except OSError, e: + except OSError: # sftp/paramiko does not give us a useful errno so we hope # for the best pass @@ -230,7 +230,7 @@ class SftpFS(obnamlib.VirtualFileSystem): # prepend the executable to the argument list args.insert(0, executable) - logging.debug('executing openssh: %s' % args) + logging.debug('executing openssh: %s', args) try: proc = subprocess.Popen(args, stdin=subprocess.PIPE, @@ -245,14 +245,13 @@ class SftpFS(obnamlib.VirtualFileSystem): def _connect_paramiko(self): remote = (self.host, self.port or 22) - logging.debug( - 'connect_paramiko: host=%s port=%s' % remote) + logging.debug('connect_paramiko: host=%s port=%s', *remote) self.transport = paramiko.Transport(remote) self.transport.connect() logging.debug('connect_paramiko: connected') try: self._check_host_key(self.host) - except BaseException, e: + except BaseException: self.transport.close() self.transport = None raise @@ -263,7 +262,7 @@ class SftpFS(obnamlib.VirtualFileSystem): logging.debug('connect_paramiko: end') def _check_host_key(self, hostname): - logging.debug('checking ssh host key for %s' % hostname) + logging.debug('checking ssh host key for %s', hostname) offered_key = self.transport.get_remote_server_key() @@ -274,8 +273,8 @@ class SftpFS(obnamlib.VirtualFileSystem): if known_keys is None: if self.settings['ssh-host-keys-check'] == 'yes': raise NoHostKeyError(hostname=hostname) - logging.warning('No known host keys for %s; accepting offered key' - % hostname) + logging.warning( + 'No known host keys for %s; accepting offered key', hostname) return offered_type = offered_key.get_name() @@ -283,14 +282,15 @@ class SftpFS(obnamlib.VirtualFileSystem): if self.settings['ssh-host-keys-check'] == 'yes': raise NoHostKeyOfWantedTypeError( key_type=offered_type, hostname=hostname) - logging.warning('No known host key of type %s for %s; accepting ' - 'offered key' % (offered_type, hostname)) + logging.warning( + 'No known host key of type %s for %s; accepting offered key', + offered_type, hostname) known_key = known_keys[offered_type] if offered_key != known_key: raise WrongHostKeyError(hostname=hostname) - logging.debug('Host key for %s OK' % hostname) + logging.debug('Host key for %s OK', hostname) def _authenticate(self, username): if not username: @@ -334,7 +334,7 @@ class SftpFS(obnamlib.VirtualFileSystem): @ioerror_to_oserror def reinit(self, baseurl, create=False): - scheme, netloc, path, query, fragment = urlparse.urlsplit(baseurl) + scheme, netloc, path, _, _ = urlparse.urlsplit(baseurl) if scheme != 'sftp': raise WrongURLSchemeError(url=baseurl) @@ -374,7 +374,7 @@ class SftpFS(obnamlib.VirtualFileSystem): if self.sftp: if create: self._create_root_if_missing() - logging.debug('chdir to %s' % path) + logging.debug('chdir to %s', path) self.sftp.chdir(self._initial_dir) self.sftp.chdir(path) @@ -536,7 +536,7 @@ class SftpFS(obnamlib.VirtualFileSystem): def lchown(self, pathname, uid, gid): self._delay() if stat.S_ISLNK(self.lstat(pathname).st_mode): - logging.warning('NOT changing ownership of symlink %s' % pathname) + logging.warning('NOT changing ownership of symlink %s', pathname) else: self.sftp.chown(pathname, uid, gid) @@ -653,7 +653,7 @@ class SftpFS(obnamlib.VirtualFileSystem): # on creation, so we set it separately. This leaves a # short window where the file is possible to open. self.chmod_not_symlink(pathname, obnamlib.NEW_FILE_MODE) - except (IOError, OSError) as e: + except (IOError, OSError): if try_number == max_tries - 1: raise else: diff --git a/obnamlib/plugins/show_plugin.py b/obnamlib/plugins/show_plugin.py index fb7606b7..2e942f03 100644 --- a/obnamlib/plugins/show_plugin.py +++ b/obnamlib/plugins/show_plugin.py @@ -147,8 +147,6 @@ class ShowPlugin(obnamlib.ObnamPlugin): for gen_id in self.repo.get_client_generation_ids(client_name): start = self.repo.get_generation_key( gen_id, obnamlib.REPO_GENERATION_STARTED) - end = self.repo.get_generation_key( - gen_id, obnamlib.REPO_GENERATION_ENDED) if most_recent is None or start > most_recent: most_recent = start self.repo.close() @@ -235,7 +233,7 @@ class ShowPlugin(obnamlib.ObnamPlugin): self.traverse(self.show_hdr_ls, self.show_item_ls, args) def show_hdr_ls(self, comment): - self.app.output.write(comment) + self.app.output.write(comment) def show_item_ls(self, gen_id, filename): fields = self.fields(gen_id, filename) diff --git a/obnamlib/plugins/verify_plugin.py b/obnamlib/plugins/verify_plugin.py index 9662aee2..e2a6752e 100644 --- a/obnamlib/plugins/verify_plugin.py +++ b/obnamlib/plugins/verify_plugin.py @@ -54,14 +54,14 @@ class VerifyPlugin(obnamlib.ObnamPlugin): raise WrongNumberOfGenerationsForVerify() logging.debug( - 'verifying generation %s' % self.app.settings['generation']) + 'verifying generation %s', self.app.settings['generation']) if not args: self.app.settings.require('root') args = self.app.settings['root'] if not args: logging.debug('no roots/args given, so verifying everything') args = ['/'] - logging.debug('verifying what: %s' % repr(args)) + logging.debug('verifying what: %s', repr(args)) self.repo = self.app.get_repository_object() client_name = self.app.settings['client-name'] @@ -69,8 +69,8 @@ class VerifyPlugin(obnamlib.ObnamPlugin): self.fs.connect() t = urlparse.urlparse(args[0]) root_url = urlparse.urlunparse((t[0], t[1], '/', t[3], t[4], t[5])) - logging.debug('t: %s' % repr(t)) - logging.debug('root_url: %s' % repr(root_url)) + logging.debug('t: %s', repr(t)) + logging.debug('root_url: %s', repr(root_url)) self.fs.reinit(root_url) self.failed = False @@ -95,10 +95,10 @@ class VerifyPlugin(obnamlib.ObnamPlugin): if num_randomly == 0: self.app.ts['total'] = \ self.repo.get_generation_key( - gen_id, obnamlib.REPO_GENERATION_FILE_COUNT) + gen_id, obnamlib.REPO_GENERATION_FILE_COUNT) self.app.ts['total_bytes'] = \ self.repo.get_generation_key( - gen_id, obnamlib.REPO_GENERATION_TOTAL_DATA) + gen_id, obnamlib.REPO_GENERATION_TOTAL_DATA) for filename in self.walk(gen_id, args): self.app.ts['filename'] = filename try: @@ -115,7 +115,7 @@ class VerifyPlugin(obnamlib.ObnamPlugin): self.log_fail(e) self.app.ts['done'] += 1 else: - logging.debug('verifying %d files randomly' % num_randomly) + logging.debug('verifying %d files randomly', num_randomly) self.app.ts['total'] = num_randomly self.app.ts.notify('finding all files to choose randomly') @@ -127,7 +127,7 @@ class VerifyPlugin(obnamlib.ObnamPlugin): filenames.append(filename) chosen = [] - for i in range(min(num_randomly, len(filenames))): + for _ in range(min(num_randomly, len(filenames))): filename = random.choice(filenames) filenames.remove(filename) chosen.append(filename) @@ -191,7 +191,7 @@ class VerifyPlugin(obnamlib.ObnamPlugin): X(obnamlib.REPO_FILE_XATTR_BLOB, 'xattr') def verify_regular_file(self, gen_id, filename): - logging.debug('verifying regular %s' % filename) + logging.debug('verifying regular %s', filename) f = self.fs.open(filename, 'r') chunkids = self.repo.get_file_chunk_ids(gen_id, filename) @@ -221,7 +221,7 @@ class VerifyPlugin(obnamlib.ObnamPlugin): ''' for arg in args: - scheme, netloc, path, query, fragment = urlparse.urlsplit(arg) + _, _, path, _, _ = urlparse.urlsplit(arg) arg = os.path.normpath(path) for x in self.repo.walk_generation(gen_id, arg): yield x diff --git a/obnamlib/plugins/vfs_local_plugin.py b/obnamlib/plugins/vfs_local_plugin.py index 7a3fe34a..6825bf9f 100644 --- a/obnamlib/plugins/vfs_local_plugin.py +++ b/obnamlib/plugins/vfs_local_plugin.py @@ -14,8 +14,6 @@ # along with this program. If not, see <http://www.gnu.org/licenses/>. -import logging - import obnamlib diff --git a/obnamlib/repo_factory.py b/obnamlib/repo_factory.py index 24949ec1..8b8feaee 100644 --- a/obnamlib/repo_factory.py +++ b/obnamlib/repo_factory.py @@ -117,8 +117,8 @@ class RepositoryFactory(object): try: fs.write_file('metadata/format', '%s\n' % wanted_format.format) except OSError as e: - logging.debug('create_repo: e=%s' % e, exc_info=1) - logging.debug('create_repo: e.errno=%s' % e.errno) + logging.debug('create_repo: e=%s', e, exc_info=1) + logging.debug('create_repo: e.errno=%s', e.errno) # SFTP (paramiko) sets errno to None when file creation # fails when the file already exists. Local filesystems # set it to EEXIST. Life is wonderful. diff --git a/obnamlib/repo_fs.py b/obnamlib/repo_fs.py index e37fae62..c025b464 100644 --- a/obnamlib/repo_fs.py +++ b/obnamlib/repo_fs.py @@ -20,6 +20,8 @@ import os import tracing +import obnamlib + class RepositoryFS(object): @@ -83,3 +85,8 @@ class RepositoryFS(object): data = self.hooks.filter_write('repository-data', data, repo=self.repo, toplevel=toplevel) self.fs.overwrite_file(filename, data) + + +class ToplevelIsFileError(obnamlib.ObnamError): + + msg = 'File at repository root: {filename}' diff --git a/obnamlib/repo_interface.py b/obnamlib/repo_interface.py index f51dbc56..c618217a 100644 --- a/obnamlib/repo_interface.py +++ b/obnamlib/repo_interface.py @@ -38,48 +38,49 @@ import obnamlib # little magic to get a) automatic enumeration b) mapping between # values and names. -_string_keys = [ - "REPO_CLIENT_TEST_KEY", - "REPO_GENERATION_TEST_KEY", - "REPO_FILE_TEST_KEY", - "REPO_FILE_USERNAME", - "REPO_FILE_GROUPNAME", - "REPO_FILE_SYMLINK_TARGET", - "REPO_FILE_XATTR_BLOB", - "REPO_FILE_MD5", -] - -_integer_keys = [ - "REPO_GENERATION_STARTED", - "REPO_GENERATION_ENDED", - "REPO_GENERATION_IS_CHECKPOINT", - "REPO_GENERATION_FILE_COUNT", - "REPO_GENERATION_TOTAL_DATA", - - "REPO_FILE_MODE", - "REPO_FILE_MTIME_SEC", - "REPO_FILE_MTIME_NSEC", - "REPO_FILE_ATIME_SEC", - "REPO_FILE_ATIME_NSEC", - "REPO_FILE_NLINK", - "REPO_FILE_SIZE", - "REPO_FILE_UID", - "REPO_FILE_GID", - "REPO_FILE_BLOCKS", - "REPO_FILE_DEV", - "REPO_FILE_INO", -] - -_repo_key_names = {} -for i, name in enumerate(_string_keys + _integer_keys): - globals()[name] = i - _repo_key_names[i] = name +REPO_CLIENT_TEST_KEY = 0 +REPO_GENERATION_TEST_KEY = 1 +REPO_FILE_TEST_KEY = 2 +REPO_FILE_USERNAME = 3 +REPO_FILE_GROUPNAME = 4 +REPO_FILE_SYMLINK_TARGET = 5 +REPO_FILE_XATTR_BLOB = 6 +REPO_FILE_MD5 = 7 + +_MAX_STRING_KEY = REPO_FILE_MD5 + +REPO_GENERATION_STARTED = 8 +REPO_GENERATION_ENDED = 9 +REPO_GENERATION_IS_CHECKPOINT = 10 +REPO_GENERATION_FILE_COUNT = 11 +REPO_GENERATION_TOTAL_DATA = 12 +REPO_FILE_MODE = 13 +REPO_FILE_MTIME_SEC = 14 +REPO_FILE_MTIME_NSEC = 15 +REPO_FILE_ATIME_SEC = 16 +REPO_FILE_ATIME_NSEC = 17 +REPO_FILE_NLINK = 18 +REPO_FILE_SIZE = 19 +REPO_FILE_UID = 20 +REPO_FILE_GID = 21 +REPO_FILE_BLOCKS = 22 +REPO_FILE_DEV = 23 +REPO_FILE_INO = 24 + + +_repo_key_names = dict( + (globals()[name], name) + for name in globals() + if name.startswith('REPO_')) def _filter_integer_keys(prefix): - return [globals()[name] - for name in _integer_keys - if name.startswith(prefix)] + return [ + globals()[name] + for name in globals() + if name.startswith(prefix) and globals()[name] > _MAX_STRING_KEY + ] + REPO_GENERATION_INTEGER_KEYS = _filter_integer_keys('REPO_GENERATION_') REPO_FILE_INTEGER_KEYS = _filter_integer_keys('REPO_FILE_') @@ -333,7 +334,7 @@ class RepositoryInterface(object): # Operations on the repository itself. @classmethod - def setup_hooks(self, hooks): # pragma: no cover + def setup_hooks(cls, hooks): # pragma: no cover '''Create any hooks for this repository format. Note that this is a class method. @@ -824,7 +825,7 @@ class RepositoryInterface(object): '''Write any pending new chunks to repository.''' raise NotImplementedError() - def remove_unused_chunk(self): + def remove_unused_chunks(self): '''Remove chunks that are no longer used by any client. The caller MUST commit any changes to clients or chunk indexes @@ -970,6 +971,8 @@ class RepositoryInterfaceTests(unittest.TestCase): # pragma: no cover # Tests for repository level things. + repo = None + def test_has_format_attribute(self): self.assertEqual(type(self.repo.format), str) @@ -1452,7 +1455,7 @@ class RepositoryInterfaceTests(unittest.TestCase): # pragma: no cover def test_unlocking_client_removes_created_generation(self): self.setup_client() self.repo.lock_client('fooclient') - new_id = self.repo.create_generation('fooclient') + self.repo.create_generation('fooclient') self.repo.unlock_client('fooclient') self.assertEqual(self.repo.get_client_generation_ids('fooclient'), []) @@ -1612,7 +1615,7 @@ class RepositoryInterfaceTests(unittest.TestCase): # pragma: no cover self.repo.commit_client('fooclient') self.repo.unlock_client('fooclient') self.repo.lock_client('fooclient') - gen_id_2 = self.repo.create_generation('fooclient') + self.repo.create_generation('fooclient') genspec = self.repo.make_generation_spec(gen_id) self.repo.remove_generation(gen_id) self.assertRaises( @@ -1672,7 +1675,7 @@ class RepositoryInterfaceTests(unittest.TestCase): # pragma: no cover self.repo.unlock_client('fooclient') self.repo.lock_client('fooclient') - gen_id_2 = self.repo.create_generation('fooclient') + self.repo.create_generation('fooclient') self.repo.remove_file(gen_id, '/foo/bar') self.repo.unlock_client('fooclient') @@ -1826,19 +1829,19 @@ class RepositoryInterfaceTests(unittest.TestCase): # pragma: no cover gen_id = self.create_generation() self.repo.add_file(gen_id, '/foo/bar') self.repo.set_file_key( - gen_id, '/foo/bar', obnamlib.REPO_FILE_MTIME, 123) + gen_id, '/foo/bar', obnamlib.REPO_FILE_MTIME_SEC, 123) # Remove the file. Key should be removed. self.repo.remove_file(gen_id, '/foo/bar') self.assertRaises( obnamlib.RepositoryFileDoesNotExistInGeneration, self.repo.get_file_key, - gen_id, '/foo/bar', obnamlib.REPO_FILE_MTIME) + gen_id, '/foo/bar', obnamlib.REPO_FILE_MTIME_SEC) # Add the file back. Key should still be removed. self.repo.add_file(gen_id, '/foo/bar') value = self.repo.get_file_key( - gen_id, '/foo/bar', obnamlib.REPO_FILE_MTIME) + gen_id, '/foo/bar', obnamlib.REPO_FILE_MTIME_SEC) self.assertEqual(value, 0) def test_can_add_a_file_then_remove_then_add_it_again(self): diff --git a/obnamlib/structurederror.py b/obnamlib/structurederror.py index 581a762e..01bbbcae 100644 --- a/obnamlib/structurederror.py +++ b/obnamlib/structurederror.py @@ -78,6 +78,7 @@ class StructuredError(Exception): ''' def __init__(self, **kwargs): + Exception.__init__(self) self.kwargs = kwargs @property diff --git a/obnamlib/vfs.py b/obnamlib/vfs.py index 8ea51e3c..c9de5ed1 100644 --- a/obnamlib/vfs.py +++ b/obnamlib/vfs.py @@ -87,12 +87,12 @@ class VirtualFileSystem(object): self.baseurl = baseurl self.bytes_read = 0 self.bytes_written = 0 - logging.debug('VFS: __init__: baseurl=%s' % self.baseurl) + logging.debug('VFS: __init__: baseurl=%s', self.baseurl) def log_stats(self): logging.debug( - 'VFS: baseurl=%s read=%d written=%d' % - (self.baseurl, self.bytes_read, self.bytes_written)) + 'VFS: baseurl=%s read=%d written=%d', + self.baseurl, self.bytes_read, self.bytes_written) def connect(self): '''Connect to filesystem.''' @@ -146,7 +146,7 @@ class VirtualFileSystem(object): def isdir(self, pathname): '''Is it a directory?''' - def mkdir(self, pathname): + def mkdir(self, pathname, mode=NEW_DIR_MODE): '''Create a directory. Parent directories must already exist. @@ -231,7 +231,7 @@ class VirtualFileSystem(object): def symlink(self, source, destination): '''Like os.symlink.''' - def open(self, pathname, mode): + def open(self, pathname, mode, bufsize=None): '''Open a file, like the builtin open() or file() function. The return value is a file object like the ones returned @@ -318,7 +318,7 @@ class VirtualFileSystem(object): yield dirname, dirst -class VfsFactory: +class VfsFactory(object): '''Create new instances of VirtualFileSystem.''' @@ -332,7 +332,7 @@ class VfsFactory: def new(self, url, create=False): '''Create a new VFS appropriate for a given URL.''' - scheme, netloc, path, params, query, fragment = urlparse.urlparse(url) + scheme, _, _, _, _, _ = urlparse.urlparse(url) if scheme in self.implementations: klass, kwargs = self.implementations[scheme] return klass(url, create=create, **kwargs) @@ -368,6 +368,25 @@ class VfsTests(object): # pragma: no cover non_ascii_name = u'm\u00e4kel\u00e4'.encode('utf-8') + fs = None + basepath = None + + # Add some dummy methods to silence pylint. + + def assertTrue(self, *args, **kwargs): + raise NotImplementedError() + + def assertFalse(self, *args, **kwargs): + raise NotImplementedError() + + def assertEqual(self, *args, **kwargs): + raise NotImplementedError() + + def assertRaises(self, *args, **kwargs): + raise NotImplementedError() + + # Actual tests. + def test_abspath_returns_input_for_absolute_path(self): self.assertEqual(self.fs.abspath('/foo/bar'), '/foo/bar') @@ -447,7 +466,7 @@ class VfsTests(object): # pragma: no cover def test_exists_returns_true_for_existing_file(self): self.fs.write_file('foo', '') - self.assert_(self.fs.exists('foo')) + self.assertTrue(self.fs.exists('foo')) def test_isdir_returns_false_for_nonexistent_file(self): self.assertFalse(self.fs.isdir('foo')) @@ -458,7 +477,7 @@ class VfsTests(object): # pragma: no cover def test_isdir_returns_true_for_existing_dir(self): self.fs.mkdir('foo') - self.assert_(self.fs.isdir('foo')) + self.assertTrue(self.fs.isdir('foo')) def test_listdir_returns_plain_strings_only(self): self.fs.write_file(u'M\u00E4kel\u00E4'.encode('utf-8'), 'data') @@ -490,14 +509,14 @@ class VfsTests(object): # pragma: no cover unicodedata.normalize('NFKD', name_unicode), unicodedata.normalize('NFKD', funny_unicode)) - self.assert_(hasattr(st, 'st_mode')) + self.assertTrue(hasattr(st, 'st_mode')) self.assertFalse(hasattr(st, 'st_mtime')) - self.assert_(hasattr(st, 'st_mtime_sec')) - self.assert_(hasattr(st, 'st_mtime_nsec')) + self.assertTrue(hasattr(st, 'st_mtime_sec')) + self.assertTrue(hasattr(st, 'st_mtime_nsec')) def test_listdir2_returns_plain_strings_only(self): self.fs.write_file(u'M\u00E4kel\u00E4'.encode('utf-8'), 'data') - names = [name for name, st in self.fs.listdir2('.')] + names = [name for name, _ in self.fs.listdir2('.')] types = [type(x) for x in names] self.assertEqual(types, [str]) @@ -520,11 +539,11 @@ class VfsTests(object): # pragma: no cover def test_makedirs_creates_directory_when_parent_exists(self): self.fs.makedirs('foo') - self.assert_(self.fs.isdir('foo')) + self.assertTrue(self.fs.isdir('foo')) def test_makedirs_creates_directory_when_parent_does_not_exist(self): self.fs.makedirs('foo/bar') - self.assert_(self.fs.isdir('foo/bar')) + self.assertTrue(self.fs.isdir('foo/bar')) def test_rmdir_removes_directory(self): self.fs.mkdir('foo') @@ -580,7 +599,7 @@ class VfsTests(object): # pragma: no cover def test_lstat_returns_right_filetype_for_directory(self): st = self.fs.lstat('.') - self.assert_(stat.S_ISDIR(st.st_mode)) + self.assertTrue(stat.S_ISDIR(st.st_mode)) def test_lstat_raises_oserror_for_nonexistent_entry(self): self.assertRaises(OSError, self.fs.lstat, 'notexists') @@ -605,10 +624,10 @@ class VfsTests(object): # pragma: no cover # not all filesystems support sub-second timestamps; those that # do not, return 0, so we have to accept either that or the correct # value, but no other values - self.assert_(self.fs.lstat('foo').st_atime_nsec in [0, 2*1000]) + self.assertTrue(self.fs.lstat('foo').st_atime_nsec in [0, 2*1000]) self.assertEqual(self.fs.lstat('foo').st_mtime_sec, 3) - self.assert_(self.fs.lstat('foo').st_mtime_nsec in [0, 4*1000]) + self.assertTrue(self.fs.lstat('foo').st_mtime_nsec in [0, 4*1000]) def test_lutimes_raises_oserror_for_nonexistent_entry(self): self.assertRaises(OSError, self.fs.lutimes, 'notexists', 1, 2, 3, 4) @@ -637,11 +656,11 @@ class VfsTests(object): # pragma: no cover def test_opens_existing_file_ok_for_reading(self): self.fs.write_file('foo', '') - self.assert_(self.fs.open('foo', 'r')) + self.assertTrue(self.fs.open('foo', 'r')) def test_opens_existing_file_ok_for_writing(self): self.fs.write_file('foo', '') - self.assert_(self.fs.open('foo', 'w')) + self.assertTrue(self.fs.open('foo', 'w')) def test_open_fails_for_nonexistent_file(self): self.assertRaises(IOError, self.fs.open, 'foo', 'r') @@ -718,13 +737,13 @@ class VfsTests(object): # pragma: no cover self.fs.listdir2 = raiser result = list(self.fs.scan_tree(self.basepath, log=logerror)) self.assertEqual(len(result), 1) - pathname, st = result[0] + pathname, _ = result[0] self.assertEqual(pathname, self.basepath) def test_scan_tree_returns_the_right_stuff(self): self.set_up_scan_tree() result = list(self.fs.scan_tree(self.basepath)) - pathnames = [pathname for pathname, st in result] + pathnames = [pathname for pathname, _ in result] self.assertEqual(sorted(pathnames), sorted(self.pathnames)) def test_scan_tree_filters_away_unwanted(self): @@ -732,5 +751,5 @@ class VfsTests(object): # pragma: no cover return stat.S_ISDIR(st.st_mode) self.set_up_scan_tree() result = list(self.fs.scan_tree(self.basepath, ok=ok)) - pathnames = [pathname for pathname, st in result] + pathnames = [pathname for pathname, _ in result] self.assertEqual(sorted(pathnames), sorted(self.dirs)) diff --git a/obnamlib/vfs_local.py b/obnamlib/vfs_local.py index bbb084fb..6e023b13 100644 --- a/obnamlib/vfs_local.py +++ b/obnamlib/vfs_local.py @@ -18,7 +18,6 @@ import errno import fcntl import grp -import logging import os import pwd import tempfile @@ -28,6 +27,12 @@ import tracing import obnamlib +# Pylint doesn't see the function defined in _obnam. We silence, for +# this module only, the no-member warning. +# +# pylint: disable=no-member + + # O_NOATIME is Linux specific: EXTRA_OPEN_FLAGS = getattr(os, "O_NOATIME", 0) @@ -184,7 +189,7 @@ class LocalFS(obnamlib.VirtualFileSystem): # Try link(2) for creating target file. try: os.link(tempname, path) - except OSError, e: + except OSError: pass else: os.remove(tempname) @@ -198,7 +203,7 @@ class LocalFS(obnamlib.VirtualFileSystem): obnamlib.NEW_FILE_MODE) os.close(fd) os.rename(tempname, path) - except OSError, e: + except OSError: # Give up. os.remove(tempname) raise @@ -288,7 +293,8 @@ class LocalFS(obnamlib.VirtualFileSystem): def chmod_symlink(self, pathname, mode): # pragma: no cover tracing.trace('chmod_symlink %s %o', pathname, mode) if self.got_lchmod: - os.lchmod(self.join(pathname), mode) + lchmod = getattr(os, 'lchmod') + lchmod(self.join(pathname), mode) else: self.lstat(pathname) @@ -322,7 +328,7 @@ class LocalFS(obnamlib.VirtualFileSystem): os.symlink(existing, self.join(new)) self.maybe_crash() - def open(self, pathname, mode): + def open(self, pathname, mode, bufsize=None): tracing.trace('pathname=%s', pathname) tracing.trace('mode=%s', mode) f = LocalFSFile(self.join(pathname), mode) @@ -348,7 +354,7 @@ class LocalFS(obnamlib.VirtualFileSystem): tracing.trace('mode=%o', mode) os.mknod(self.join(pathname), mode) - def mkdir(self, pathname): + def mkdir(self, pathname, mode=obnamlib.NEW_DIR_MODE): tracing.trace('mkdir %s', pathname) os.mkdir(self.join(pathname), obnamlib.NEW_DIR_MODE) self.maybe_crash() diff --git a/obnamlib/vfs_local_tests.py b/obnamlib/vfs_local_tests.py index befcab14..6ed4aea4 100644 --- a/obnamlib/vfs_local_tests.py +++ b/obnamlib/vfs_local_tests.py @@ -25,7 +25,13 @@ import obnamlib from obnamlib import _obnam -class LocalFSTests(obnamlib.VfsTests, unittest.TestCase): +# Pylint doesn't see the function defined in _obnam. We silence, for +# this module only, the no-member warning. +# +# pylint: disable=no-member + + +class LocalFSTests(unittest.TestCase, obnamlib.VfsTests): def setUp(self): self.basepath = tempfile.mkdtemp() @@ -69,7 +75,7 @@ class XAttrTests(unittest.TestCase): # attribute with the command line tool. try: - exitcode, out, err = cliapp.runcmd_unchecked( + exitcode, _, _ = cliapp.runcmd_unchecked( ['setfattr', '-n', 'user.foo', 'bar', self.other]) except OSError: # Either xattr aren't supported, or setfattr isn't diff --git a/pylint.conf b/pylint.conf new file mode 100644 index 00000000..9ae639b7 --- /dev/null +++ b/pylint.conf @@ -0,0 +1,32 @@ +[MASTER] +persistent=no + +[MESSAGES CONTROL] +disable= + fixme, + too-many-lines, + abstract-class-little-used, + abstract-class-not-used, + interface-not-implemented, + locally-disabled, + attribute-defined-outside-init, + cyclic-import, + invalid-name, + missing-docstring, + no-self-use, + protected-access, + star-args, + too-few-public-methods, + too-many-arguments, + too-many-branches, + too-many-instance-attributes, + too-many-locals, + too-many-public-methods, + too-many-statements, + unused-argument + +[REPORTS] +reports=no + +[SIMILARITIES] +min-similarity-lines=999 @@ -235,6 +235,7 @@ class Check(Command): def run_nitpick_checks(self): self.check_with_pep8() + self.check_with_pylint() if os.path.exists('.git'): sources = self.find_all_source_files() self.check_sources_for_nitpicks(sources) @@ -245,6 +246,11 @@ class Check(Command): def check_with_pep8(self): cliapp.runcmd(['pep8', 'obnamlib'], stdout=None, stderr=None) + def check_with_pylint(self): + cliapp.runcmd( + ['pylint', '--rcfile=pylint.conf', 'obnamlib'], + stdout=None, stderr=None) + def check_sources_for_nitpicks(self, sources): cliapp.runcmd(['./nitpicker'] + sources, stdout=None, stderr=None) |