diff options
| author | Lars Wirzenius <liw@liw.fi> | 2015-07-04 16:32:03 +0300 |
|---|---|---|
| committer | Lars Wirzenius <liw@liw.fi> | 2015-07-04 16:32:03 +0300 |
| commit | 4417aee6f26adabc46552b53481656d9e37281b1 (patch) | |
| tree | 16c671c1f1089f09142a5c17f231ab0ce17d8240 | |
| parent | 572faf19054c59bf6ecf181b448c4392b35548df (diff) | |
| download | obnam-4417aee6f26adabc46552b53481656d9e37281b1.tar.gz | |
Cleanups suggested by pep8
53 files changed, 358 insertions, 324 deletions
diff --git a/obnamlib/__init__.py b/obnamlib/__init__.py index d4b5e451..4d2479c4 100644 --- a/obnamlib/__init__.py +++ b/obnamlib/__init__.py @@ -20,11 +20,12 @@ import cliapp __version__ = '1.11' - # Import _obnam if it is there. We need to be able to do things without # it, especially at build time, while we're generating manual pages. # If _obnam is not there, substitute a dummy that throws an exception # if used. + + class DummyExtension(object): def __getattr__(self, name): raise Exception('Trying to use _obnam, but that was not found.') @@ -39,6 +40,7 @@ except ImportError: from structurederror import StructuredError + class ObnamError(StructuredError): pass diff --git a/obnamlib/app.py b/obnamlib/app.py index d65e7cad..cfd711cb 100644 --- a/obnamlib/app.py +++ b/obnamlib/app.py @@ -285,4 +285,3 @@ class App(cliapp.Application): return time.mktime(t) else: return time.time() - diff --git a/obnamlib/delegator.py b/obnamlib/delegator.py index 9e1ace9a..52eb3223 100644 --- a/obnamlib/delegator.py +++ b/obnamlib/delegator.py @@ -392,8 +392,8 @@ class GenerationId(object): self.client_name == other.client_name and self.gen_number == other.gen_number) - def __str__(self): # pragma: no cover + def __str__(self): # pragma: no cover return '%s:%s' % (self.client_name, self.gen_number) - def __repr__(self): # pragma: no cover + def __repr__(self): # pragma: no cover return 'GenerationId(%s,%s)' % (self.client_name, self.gen_number) diff --git a/obnamlib/encryption.py b/obnamlib/encryption.py index 6218a7d4..7030359d 100644 --- a/obnamlib/encryption.py +++ b/obnamlib/encryption.py @@ -102,7 +102,7 @@ def _gpg_pipe(args, data, passphrase): os.close(keypipe[0]) # Return output data, or deal with errors. - if p.returncode: # pragma: no cover + if p.returncode: # pragma: no cover raise GpgError(returncode=p.returncode, stderr=err) return out @@ -134,7 +134,7 @@ def _gpg(args, stdin='', gpghome=None): out, err = p.communicate(stdin) # Return output data, or deal with errors. - if p.returncode: # pragma: no cover + if p.returncode: # pragma: no cover raise GpgError(returncode=p.returncode, stderr=err) return out @@ -145,7 +145,7 @@ def get_public_key(keyid, gpghome=None): return _gpg(['--export', '--armor', keyid], gpghome=gpghome) -def get_public_key_user_ids(keyid, gpghome=None): # pragma: no cover +def get_public_key_user_ids(keyid, gpghome=None): # pragma: no cover '''Return the ASCII armored export form of a given public key.''' user_ids = [] output = _gpg(['--with-colons', '--list-keys', keyid], gpghome=gpghome) @@ -229,7 +229,7 @@ class Keyring(object): kwargs['gpghome'] = self._gpghome try: result = _gpg(*args, **kwargs) - except BaseException: # pragma: no cover + except BaseException: # pragma: no cover self._cleanup() raise else: @@ -261,16 +261,16 @@ def encrypt_with_keyring(cleartext, keyring): recipients = [] for keyid in keyring.keyids(): recipients += ['-r', keyid] - return keyring.gpg(False, - ['-e', - '--trust-model', 'always', - '--no-encrypt-to', - '--no-default-recipient', - ] + recipients, - stdin=cleartext) + + opts = [ + '-e', + '--trust-model', 'always', + '--no-encrypt-to', + '--no-default-recipient', + ] + return keyring.gpg(False, opts + recipients, stdin=cleartext) def decrypt_with_secret_keys(encrypted, gpghome=None): '''Decrypt data using secret keys GnuPG finds on its own.''' return _gpg(['-d'], stdin=encrypted, gpghome=gpghome) - diff --git a/obnamlib/encryption_tests.py b/obnamlib/encryption_tests.py index 689ce2ea..96bcdae5 100644 --- a/obnamlib/encryption_tests.py +++ b/obnamlib/encryption_tests.py @@ -40,12 +40,12 @@ class SymmetricEncryptionTests(unittest.TestCase): def test_generates_key_of_correct_length(self): numbits = 16 key = obnamlib.generate_symmetric_key(numbits, filename='/dev/zero') - self.assertEqual(len(key) * 8 / 2, numbits) # /2 for hex encoding + self.assertEqual(len(key) * 8 / 2, numbits) # /2 for hex encoding def test_generates_key_with_size_rounded_up(self): numbits = 15 key = obnamlib.generate_symmetric_key(numbits, filename='/dev/zero') - self.assertEqual(len(key)/2, 2) # /2 for hex encoding + self.assertEqual(len(key)/2, 2) # /2 for hex encoding def test_encrypts_into_different_string_than_cleartext(self): cleartext = 'hello world' @@ -196,4 +196,3 @@ class PublicKeyEncryptionTests(unittest.TestCase): encrypted, gpghome=self.gpghome) self.assertEqual(decrypted, cleartext) - diff --git a/obnamlib/fmt_6/checksumtree.py b/obnamlib/fmt_6/checksumtree.py index 90820d1c..0b880331 100644 --- a/obnamlib/fmt_6/checksumtree.py +++ b/obnamlib/fmt_6/checksumtree.py @@ -43,7 +43,7 @@ class ChecksumTree(obnamlib.RepositoryTree): def key_range(self, checksum, chunk_id): return self.key(checksum, chunk_id, 0), \ - self.key(checksum, chunk_id, obnamlib.MAX_ID) + self.key(checksum, chunk_id, obnamlib.MAX_ID) def unkey(self, key): return struct.unpack(self.fmt, key) diff --git a/obnamlib/fmt_6/checksumtree_tests.py b/obnamlib/fmt_6/checksumtree_tests.py index e2760e7d..7298c84f 100644 --- a/obnamlib/fmt_6/checksumtree_tests.py +++ b/obnamlib/fmt_6/checksumtree_tests.py @@ -77,4 +77,3 @@ class ChecksumTreeTests(unittest.TestCase): def test_known_chunk_is_used(self): self.tree.add(self.checksum, 0, 1) self.assertTrue(self.tree.chunk_is_used(self.checksum, 0)) - diff --git a/obnamlib/fmt_6/chunklist.py b/obnamlib/fmt_6/chunklist.py index 4f72506c..0f517af4 100644 --- a/obnamlib/fmt_6/chunklist.py +++ b/obnamlib/fmt_6/chunklist.py @@ -60,4 +60,3 @@ class ChunkList(obnamlib.RepositoryTree): self.start_changes() key = self.key(chunk_id) self.tree.remove_range(key, key) - diff --git a/obnamlib/fmt_6/chunklist_tests.py b/obnamlib/fmt_6/chunklist_tests.py index 1f83ddc8..d2b48db2 100644 --- a/obnamlib/fmt_6/chunklist_tests.py +++ b/obnamlib/fmt_6/chunklist_tests.py @@ -52,4 +52,3 @@ class ChunkListTests(unittest.TestCase): self.list.add(0, 'checksum') self.list.remove(0) self.assertRaises(KeyError, self.list.get_checksum, 0) - diff --git a/obnamlib/fmt_6/clientlist.py b/obnamlib/fmt_6/clientlist.py index d4ba87a1..91791f47 100644 --- a/obnamlib/fmt_6/clientlist.py +++ b/obnamlib/fmt_6/clientlist.py @@ -78,8 +78,8 @@ class ClientList(obnamlib.RepositoryTree): if self.init_forest() and self.forest.trees: t = self.forest.trees[-1] return [v - for k, v in t.lookup_range(self.minkey, self.maxkey) - if self.unkey(k)[2] == self.CLIENT_NAME] + for k, v in t.lookup_range(self.minkey, self.maxkey) + if self.unkey(k)[2] == self.CLIENT_NAME] else: return [] @@ -140,4 +140,3 @@ class ClientList(obnamlib.RepositoryTree): self.tree.remove_range(key, key) else: self.tree.insert(key, keyid) - diff --git a/obnamlib/fmt_6/clientlist_tests.py b/obnamlib/fmt_6/clientlist_tests.py index 9e14f290..77e90d54 100644 --- a/obnamlib/fmt_6/clientlist_tests.py +++ b/obnamlib/fmt_6/clientlist_tests.py @@ -104,4 +104,3 @@ class ClientListTests(unittest.TestCase): self.list.set_client_keyid('foo', 'cafebeef') self.list.set_client_keyid('foo', None) self.assertEqual(self.list.get_client_keyid('foo'), None) - diff --git a/obnamlib/fmt_6/clientmetadatatree.py b/obnamlib/fmt_6/clientmetadatatree.py index 81b31364..95fb7e19 100644 --- a/obnamlib/fmt_6/clientmetadatatree.py +++ b/obnamlib/fmt_6/clientmetadatatree.py @@ -45,7 +45,7 @@ class ClientMetadataTree(obnamlib.RepositoryTree): DIR_CONTENTS = 4 # subkey type for list of directory contents FILE_DATA = 5 # subkey type for file data (instead of chunk) - FILE_METADATA_ENCODED = 0 # subkey value for encoded obnamlib.Metadata(). + FILE_METADATA_ENCODED = 0 # subkey value for encoded obnamlib.Metadata(). # References to chunks in this generation. # Main key is the chunk id, subkey type is always 0, subkey is file id @@ -88,8 +88,10 @@ class ClientMetadataTree(obnamlib.RepositoryTree): def default_file_id(self, filename): '''Return hash of filename suitable for use as main key.''' tracing.trace(repr(filename)) + def shorthash(s): return hashlib.md5(s).digest()[:4] + dirname = os.path.dirname(filename) basename = os.path.basename(filename) return shorthash(dirname) + shorthash(basename) @@ -283,46 +285,46 @@ class ClientMetadataTree(obnamlib.RepositoryTree): def _get_generation_id_or_None(self, tree): try: return self.get_generation_id(tree) - except KeyError: # pragma: no cover + except KeyError: # pragma: no cover return None - def _lookup_time(self, tree, what): # pragma: no cover + def _lookup_time(self, tree, what): # pragma: no cover try: return self._lookup_int(tree, self.genkey(what)) except KeyError: return None - def _lookup_string(self, tree, what): # pragma: no cover + def _lookup_string(self, tree, what): # pragma: no cover try: return tree.lookup(self.genkey(what)) except KeyError: return None - def get_generation_times(self, genid): # pragma: no cover + def get_generation_times(self, genid): # pragma: no cover tree = self.find_generation(genid) return (self._lookup_time(tree, self.GEN_STARTED), self._lookup_time(tree, self.GEN_ENDED)) - def set_generation_started(self, timestamp): # pragma: no cover + def set_generation_started(self, timestamp): # pragma: no cover self._insert_int(self.tree, self.genkey(self.GEN_STARTED), timestamp) - def set_generation_ended(self, timestamp): # pragma: no cover + def set_generation_ended(self, timestamp): # pragma: no cover self._insert_int(self.tree, self.genkey(self.GEN_ENDED), timestamp) - def get_generation_test_data(self): # pragma: no cover + def get_generation_test_data(self): # pragma: no cover return self._lookup_string(self.tree, self.GEN_TEST_DATA) - def set_generation_test_data(self, value): # pragma: no cover + def set_generation_test_data(self, value): # pragma: no cover key = self.genkey(self.GEN_TEST_DATA) self.tree.insert(key, value) - def get_generation_data(self, genid): # pragma: no cover + def get_generation_data(self, genid): # pragma: no cover return self._lookup_count(genid, self.GEN_TOTAL_DATA) - def set_generation_data(self, gen_id, num_bytes): # pragma: no cover + def set_generation_data(self, gen_id, num_bytes): # pragma: no cover self._insert_count(gen_id, self.GEN_TOTAL_DATA, num_bytes) - def _lookup_count(self, genid, count_type): # pragma: no cover + def _lookup_count(self, genid, count_type): # pragma: no cover tree = self.find_generation(genid) key = self.genkey(count_type) try: @@ -330,21 +332,21 @@ class ClientMetadataTree(obnamlib.RepositoryTree): except KeyError: return None - def _insert_count(self, genid, count_type, count): # pragma: no cover + def _insert_count(self, genid, count_type, count): # pragma: no cover tree = self.find_generation(genid) key = self.genkey(count_type) return self._insert_int(tree, key, count) - def get_generation_file_count(self, genid): # pragma: no cover + def get_generation_file_count(self, genid): # pragma: no cover return self._lookup_count(genid, self.GEN_FILE_COUNT) - def set_generation_file_count(self, gen_id, count): # pragma: no cover + def set_generation_file_count(self, gen_id, count): # pragma: no cover self._insert_count(gen_id, self.GEN_FILE_COUNT, count) - def get_generation_total_data(self, genid): # pragma: no cover + def get_generation_total_data(self, genid): # pragma: no cover return self._lookup_count(genid, self.GEN_TOTAL_DATA) - def set_generation_total_data(self, gen_id, count): # pragma: no cover + def set_generation_total_data(self, gen_id, count): # pragma: no cover self._insert_count(gen_id, self.GEN_TOTAL_DATA, count) def create(self, filename, encoded_metadata): @@ -355,7 +357,7 @@ class ClientMetadataTree(obnamlib.RepositoryTree): old_metadata = self.get_metadata(gen_id, filename) except KeyError: old_metadata = None - else: # pragma: no cover + else: # pragma: no cover old = obnamlib.fmt_6.metadata_codec.decode_metadata(old_metadata) metadata = obnamlib.fmt_6.metadata_codec.decode_metadata( @@ -368,7 +370,7 @@ class ClientMetadataTree(obnamlib.RepositoryTree): # Add to parent's contents, unless already there. parent = os.path.dirname(filename) tracing.trace('parent=%s', parent) - if parent != filename: # root dir is its own parent + if parent != filename: # root dir is its own parent basename = os.path.basename(filename) parent_id = self.set_file_id(parent) key = self.fskey(parent_id, self.DIR_CONTENTS, file_id) @@ -376,7 +378,7 @@ class ClientMetadataTree(obnamlib.RepositoryTree): # churn in the tree if nothing changes. try: self.tree.lookup(key) - tracing.trace('was already in parent') # pragma: no cover + tracing.trace('was already in parent') # pragma: no cover except KeyError: self.tree.insert(key, basename) tracing.trace('added to parent') @@ -436,7 +438,7 @@ class ClientMetadataTree(obnamlib.RepositoryTree): # Also remove from parent's contents. parent = os.path.dirname(filename) - if parent != filename: # root dir is its own parent + if parent != filename: # root dir is its own parent parent_id = self.set_file_id(parent) key = self.fskey(parent_id, self.DIR_CONTENTS, file_id) # The range removal will work even if the key does not exist. @@ -536,7 +538,7 @@ class ClientMetadataTree(obnamlib.RepositoryTree): return list(set(self.chunk_unkey(key)[0] for key, value in t.lookup_range(minkey, maxkey))) - def set_file_data(self, filename, contents): # pragma: no cover + def set_file_data(self, filename, contents): # pragma: no cover '''Store contents of file, if small, in B-tree instead of chunk. The length of the contents should be small enough to fit in a @@ -550,7 +552,7 @@ class ClientMetadataTree(obnamlib.RepositoryTree): key = self.fskey(file_id, self.FILE_DATA, 0) self.tree.insert(key, contents) - def get_file_data(self, gen_id, filename): # pragma: no cover + def get_file_data(self, gen_id, filename): # pragma: no cover '''Return contents of file, if set, or None.''' tree = self.find_generation(gen_id) file_id = self.get_file_id(tree, filename) @@ -559,4 +561,3 @@ class ClientMetadataTree(obnamlib.RepositoryTree): return tree.lookup(key) except KeyError: return None - diff --git a/obnamlib/fmt_6/clientmetadatatree_tests.py b/obnamlib/fmt_6/clientmetadatatree_tests.py index 21a0534f..166d2e6c 100644 --- a/obnamlib/fmt_6/clientmetadatatree_tests.py +++ b/obnamlib/fmt_6/clientmetadatatree_tests.py @@ -34,10 +34,13 @@ class ClientMetadataTreeTests(unittest.TestCase): fs = obnamlib.LocalFS(self.tempdir) self.hooks = obnamlib.HookManager() self.hooks.new('repository-toplevel-init') - self.client = obnamlib.ClientMetadataTree(fs, 'clientid', - obnamlib.DEFAULT_NODE_SIZE, - obnamlib.DEFAULT_UPLOAD_QUEUE_SIZE, - obnamlib.DEFAULT_LRU_SIZE, self) + self.client = obnamlib.ClientMetadataTree( + fs, + 'clientid', + obnamlib.DEFAULT_NODE_SIZE, + obnamlib.DEFAULT_UPLOAD_QUEUE_SIZE, + obnamlib.DEFAULT_LRU_SIZE, + self) self.file_size = 123 self.file_metadata = obnamlib.Metadata(st_mode=stat.S_IFREG | 0666, st_size=self.file_size) @@ -321,4 +324,3 @@ class ClientMetadataTreeFileOpsTests(unittest.TestCase): self.client.set_file_chunks('/foo', [0]) self.client.set_file_chunks('/bar', [0]) self.assertEqual(self.client.list_chunks_in_generation(gen_id), [0]) - diff --git a/obnamlib/fmt_6/metadata_codec.py b/obnamlib/fmt_6/metadata_codec.py index 9e17f919..a0cad418 100644 --- a/obnamlib/fmt_6/metadata_codec.py +++ b/obnamlib/fmt_6/metadata_codec.py @@ -38,6 +38,7 @@ metadata_format = struct.Struct('!Q' + # flags 'Q' + # len of xattr '') + def encode_metadata(metadata): flags = 0 for i, name in enumerate(obnamlib.metadata_fields): @@ -63,7 +64,7 @@ def encode_metadata(metadata): len(metadata.target or ''), len(metadata.md5 or ''), len(metadata.xattr or '')) - except TypeError, e: # pragma: no cover + except TypeError, e: # pragma: no cover logging.error('ERROR: Packing error due to %s' % str(e)) logging.error('ERROR: st_mode=%s' % repr(metadata.st_mode)) logging.error('ERROR: st_mtime_sec=%s' % repr(metadata.st_mtime_sec)) @@ -86,11 +87,11 @@ def encode_metadata(metadata): logging.error('ERROR: xattr=%s' % repr(metadata.xattr)) raise return (packed + - (metadata.groupname or '') + - (metadata.username or '') + - (metadata.target or '') + - (metadata.md5 or '') + - (metadata.xattr or '')) + (metadata.groupname or '') + + (metadata.username or '') + + (metadata.target or '') + + (metadata.md5 or '') + + (metadata.xattr or '')) def decode_metadata(encoded): diff --git a/obnamlib/fmt_6/metadata_codec_tests.py b/obnamlib/fmt_6/metadata_codec_tests.py index 47f8b312..0fab39de 100644 --- a/obnamlib/fmt_6/metadata_codec_tests.py +++ b/obnamlib/fmt_6/metadata_codec_tests.py @@ -30,7 +30,7 @@ class MetadataCodingTests(unittest.TestCase): value1, value2, 'attribute %s must be equal (%s vs %s)' % - (name, value1, value2)) + (name, value1, value2)) def test_round_trip(self): metadata = obnamlib.Metadata( @@ -82,4 +82,3 @@ class MetadataCodingTests(unittest.TestCase): encoded = obnamlib.fmt_6.metadata_codec.encode_metadata(metadata) decoded = obnamlib.fmt_6.metadata_codec.decode_metadata(encoded) self.equal(metadata, decoded) - diff --git a/obnamlib/fmt_6/repo_fmt_6.py b/obnamlib/fmt_6/repo_fmt_6.py index 2c66c110..4f6029a0 100644 --- a/obnamlib/fmt_6/repo_fmt_6.py +++ b/obnamlib/fmt_6/repo_fmt_6.py @@ -223,14 +223,14 @@ class RepositoryFormat6(obnamlib.RepositoryInterface): if client_name not in self._open_client_infos: tracing.trace('client_name=%s', client_name) client_id = self._get_client_id(client_name) - if client_id is None: # pragma: no cover + if client_id is None: # pragma: no cover raise obnamlib.RepositoryClientDoesNotExist( client_name=client_name) client_dir = self._get_client_dir(client_id) client = obnamlib.ClientMetadataTree( self._fs, client_dir, self._node_size, - self._upload_queue_size, self._lru_size, self) + self._upload_queue_size, self._lru_size, self) client.init_forest() self._open_client_infos[client_name] = _OpenClientInfo(client) @@ -262,7 +262,7 @@ class RepositoryFormat6(obnamlib.RepositoryInterface): client_name=client_name) client_id = self._get_client_id(client_name) - if client_id is None: # pragma: no cover + if client_id is None: # pragma: no cover raise obnamlib.RepositoryClientDoesNotExist( client_name=client_name) @@ -325,20 +325,22 @@ class RepositoryFormat6(obnamlib.RepositoryInterface): self._flush_file_key_cache() - self._open_client(client_name) # Ensure client is open + self._open_client(client_name) # Ensure client is open open_client_info = self._open_client_infos[client_name] if open_client_info.current_generation_number: open_client_info.client.set_generation_ended(self._current_time()) - if (open_client_info.current_generation_number or - open_client_info.generations_removed): + need_to_commit = ( + open_client_info.current_generation_number or + open_client_info.generations_removed) + if need_to_commit: open_client_info.client.commit() self._raw_unlock_client(client_name) def _remove_chunks_from_removed_generations( - self, client_name, remove_gen_nos): + self, client_name, remove_gen_nos): def find_chunkids_in_gens(gen_nos): chunkids = set() @@ -356,7 +358,7 @@ class RepositoryFormat6(obnamlib.RepositoryInterface): keep.append(gen_number) return keep - def remove_chunks(chunk_ids): # pragma: no cover + def remove_chunks(chunk_ids): # pragma: no cover for chunk_id in chunk_ids: try: checksum = self._chunklist.get_checksum(chunk_id) @@ -378,7 +380,7 @@ class RepositoryFormat6(obnamlib.RepositoryInterface): def get_allowed_client_keys(self): return [] - def get_client_key(self, client_name, key): # pragma: no cover + def get_client_key(self, client_name, key): # pragma: no cover raise obnamlib.RepositoryClientKeyNotAllowed( format=self.format, client_name=client_name, @@ -391,15 +393,14 @@ class RepositoryFormat6(obnamlib.RepositoryInterface): key_name=obnamlib.repo_key_name(key)) def get_client_generation_ids(self, client_name): - self._open_client(client_name) # Ensure client is open + self._open_client(client_name) # Ensure client is open client_info = self._get_open_client_info(client_name) self._refresh_open_client_info_cached_generation_ids( client_name, client_info) return client_info.cached_generation_ids - def _refresh_open_client_info_cached_generation_ids(self, - client_name, - client_info): + def _refresh_open_client_info_cached_generation_ids( + self, client_name, client_info): if client_info.cached_generation_ids is None: client_info.cached_generation_ids = [ self._construct_gen_id(client_name, gen_number) @@ -410,11 +411,11 @@ class RepositoryFormat6(obnamlib.RepositoryInterface): gen_id): ids = client_info.cached_generation_ids assert ids is not None - if gen_id not in ids: # pragma: no cover + if gen_id not in ids: # pragma: no cover ids.append(gen_id) - def _forget_open_client_info_cached_generation(self, - client_info, gen_id): + def _forget_open_client_info_cached_generation( + self, client_info, gen_id): ids = client_info.cached_generation_ids if ids is not None: if gen_id in ids: @@ -425,7 +426,7 @@ class RepositoryFormat6(obnamlib.RepositoryInterface): self._require_existing_client(client_name) self._require_client_lock(client_name) - self._open_client(client_name) # Ensure client is open + self._open_client(client_name) # Ensure client is open open_client_info = self._open_client_infos[client_name] if open_client_info.current_generation_number is not None: raise obnamlib.RepositoryClientGenerationUnfinished( @@ -478,7 +479,7 @@ class RepositoryFormat6(obnamlib.RepositoryInterface): obnamlib.REPO_GENERATION_TOTAL_DATA, ] - def get_generation_key(self, generation_id, key): # pragma: no cover + def get_generation_key(self, generation_id, key): # pragma: no cover client_name, gen_number = self._unpack_gen_id(generation_id) client = self._open_client(client_name) @@ -503,7 +504,7 @@ class RepositoryFormat6(obnamlib.RepositoryInterface): key_name=obnamlib.repo_key_name(key)) def set_generation_key( - self, generation_id, key, value): # pragma: no cover + self, generation_id, key, value): # pragma: no cover # FIXME: This is not working for generations other than the currently # started one. There should at least be an assert about it. @@ -552,7 +553,7 @@ class RepositoryFormat6(obnamlib.RepositoryInterface): self._require_client_lock(client_name) self._require_existing_generation(gen_id) - self._open_client(client_name) # Ensure client is open + self._open_client(client_name) # Ensure client is open open_client_info = self._open_client_infos[client_name] if gen_number == open_client_info.current_generation_number: open_client_info.current_generation_number = None @@ -592,13 +593,13 @@ class RepositoryFormat6(obnamlib.RepositoryInterface): self._idpath_skip) def _construct_in_tree_chunk_id( - self, gen_id, filename): # pragma: no cover + self, gen_id, filename): # pragma: no cover # This constructs a synthetic chunk id for in-tree data for a # file. The file is expected to have in-tree data. return (gen_id, filename) - def _unpack_in_tree_chunk_id(self, chunk_id): # pragma: no cover + def _unpack_in_tree_chunk_id(self, chunk_id): # pragma: no cover # Return gen_id and filename for in-tree chunk. return chunk_id @@ -621,7 +622,7 @@ class RepositoryFormat6(obnamlib.RepositoryInterface): filename = self._chunk_filename(chunk_id) try: self._fs.write_file(filename, data) - except OSError, e: # pragma: no cover + except OSError, e: # pragma: no cover if e.errno == errno.EEXIST: self._prev_chunk_id = self._random_chunk_id() continue @@ -634,7 +635,7 @@ class RepositoryFormat6(obnamlib.RepositoryInterface): return chunk_id def get_chunk_content(self, chunk_id): - if self._is_in_tree_chunk_id(chunk_id): # pragma: no cover + if self._is_in_tree_chunk_id(chunk_id): # pragma: no cover gen_id, filename = self._unpack_in_tree_chunk_id(chunk_id) client_name, gen_number = self._unpack_gen_id(gen_id) client = self._open_client(client_name) @@ -648,10 +649,10 @@ class RepositoryFormat6(obnamlib.RepositoryInterface): raise obnamlib.RepositoryChunkDoesNotExist( chunk_id=str(chunk_id), filename=filename) - raise # pragma: no cover + raise # pragma: no cover def has_chunk(self, chunk_id): - if self._is_in_tree_chunk_id(chunk_id): # pragma: no cover + if self._is_in_tree_chunk_id(chunk_id): # pragma: no cover gen_id, filename = self._unpack_in_tree_chunk_id(chunk_id) client_name, gen_number = gen_id client = self._open_client(client_name) @@ -666,7 +667,7 @@ class RepositoryFormat6(obnamlib.RepositoryInterface): # Note: we ignore in-tree data, on the assumption that if # it gets removed, the whole file gets removed from the # generation anyway. This should probably be fixed some day. - if self._is_in_tree_chunk_id(chunk_id): # pragma: no cover + if self._is_in_tree_chunk_id(chunk_id): # pragma: no cover return filename = self._chunk_filename(chunk_id) @@ -684,7 +685,7 @@ class RepositoryFormat6(obnamlib.RepositoryInterface): # Note: This does not cover for in-tree chunk data. We cannot # realistically iterate over all per-client B-trees to find # such data. - + pat = re.compile(r'^.*/.*/[0-9a-fA-F]+$') if self._fs.exists('chunks'): for pathname, st in self._fs.scan_tree('chunks'): @@ -796,7 +797,7 @@ class RepositoryFormat6(obnamlib.RepositoryInterface): self._require_chunk_indexes_lock() try: checksum = self._chunklist.get_checksum(chunk_id) - except KeyError: # pragma: no cover + except KeyError: # pragma: no cover tracing.trace('chunk does not exist in chunklist tree') # Because commit_chunk_indexes commits _chunklist before # _chunksums, at this point we know the chunk isn't going @@ -813,7 +814,7 @@ class RepositoryFormat6(obnamlib.RepositoryInterface): raise obnamlib.RepositoryChunkContentNotInIndexes() def validate_chunk_content(self, chunk_id): - if self._is_in_tree_chunk_id(chunk_id): # pragma: no cover + if self._is_in_tree_chunk_id(chunk_id): # pragma: no cover gen_id, filename = self._unpack_in_tree_chunk_id(chunk_id) client_name, gen_number = self._unpack_gen_id(gen_id) client = self._open_client(client_name) @@ -830,7 +831,7 @@ class RepositoryFormat6(obnamlib.RepositoryInterface): actual_checksum = self._checksum(content) try: expected_checksum = self._chunklist.get_checksum(chunk_id) - except KeyError: # pragma: no cover + except KeyError: # pragma: no cover # Chunk is not in the checksum tree, so we cannot validate # its checksum. We'll just assume it's OK. return True @@ -861,7 +862,6 @@ class RepositoryFormat6(obnamlib.RepositoryInterface): obnamlib.REPO_FILE_DEV: 'st_dev', obnamlib.REPO_FILE_INO: 'st_ino', obnamlib.REPO_FILE_MD5: 'md5', - } self._setup_file_key_cache() @@ -901,7 +901,7 @@ class RepositoryFormat6(obnamlib.RepositoryInterface): self._require_client_lock(client_name) self._flush_file_key_cache() client = self._open_client(client_name) - client.remove(filename) # FIXME: Only removes from unfinished gen! + client.remove(filename) # FIXME: Only removes from unfinished gen! def get_allowed_file_keys(self): return self._file_keys.keys() @@ -992,7 +992,7 @@ class RepositoryFormat6(obnamlib.RepositoryInterface): client_name, gen_number = self._unpack_gen_id(generation_id) client = self._open_client(client_name) in_tree_data = client.get_file_data(gen_number, filename) - if in_tree_data is not None: # pragma: no cover + if in_tree_data is not None: # pragma: no cover return [self._construct_in_tree_chunk_id(generation_id, filename)] return client.get_file_chunks(gen_number, filename) @@ -1003,7 +1003,7 @@ class RepositoryFormat6(obnamlib.RepositoryInterface): client_name, gen_number = self._unpack_gen_id(generation_id) self._require_client_lock(client_name) client = self._open_client(client_name) - client.set_file_chunks(filename, []) # FIXME: current gen only + client.set_file_chunks(filename, []) # FIXME: current gen only def append_file_chunk_id(self, generation_id, filename, chunk_id): assert not self._is_in_tree_chunk_id(chunk_id) @@ -1011,7 +1011,7 @@ class RepositoryFormat6(obnamlib.RepositoryInterface): client_name, gen_number = self._unpack_gen_id(generation_id) self._require_client_lock(client_name) client = self._open_client(client_name) - client.append_file_chunks(filename, [chunk_id]) # FIXME: curgen only + client.append_file_chunks(filename, [chunk_id]) # FIXME: curgen only def get_file_children(self, generation_id, filename): self._require_existing_file(generation_id, filename) @@ -1022,7 +1022,7 @@ class RepositoryFormat6(obnamlib.RepositoryInterface): # Fsck. - def get_fsck_work_items(self): # pragma: no cover + def get_fsck_work_items(self): # pragma: no cover yield CheckBTree(self._fs, 'clientlist', 'fsck-skip-shared-b-trees') for client_name in self.get_client_names(): client_id = self._get_client_id(client_name) @@ -1032,8 +1032,8 @@ class RepositoryFormat6(obnamlib.RepositoryInterface): yield CheckBTree(self._fs, 'chunksums', 'fsck-skip-shared-b-trees') -class CheckBTree(obnamlib.WorkItem): # pragma: no cover - +class CheckBTree(obnamlib.WorkItem): # pragma: no cover + def __init__(self, fs, dirname, skip_setting): self.fs = fs self.dirname = dirname diff --git a/obnamlib/fmt_6/repo_fmt_6_tests.py b/obnamlib/fmt_6/repo_fmt_6_tests.py index b333280e..4e270dbc 100644 --- a/obnamlib/fmt_6/repo_fmt_6_tests.py +++ b/obnamlib/fmt_6/repo_fmt_6_tests.py @@ -32,4 +32,3 @@ class RepositoryFormat6Tests(obnamlib.RepositoryInterfaceTests): def tearDown(self): shutil.rmtree(self.tempdir) - diff --git a/obnamlib/fmt_6/repo_tree.py b/obnamlib/fmt_6/repo_tree.py index d9067f61..7ca340ce 100644 --- a/obnamlib/fmt_6/repo_tree.py +++ b/obnamlib/fmt_6/repo_tree.py @@ -90,8 +90,10 @@ class RepositoryTree(object): if self.tree is None and create_tree: if self.forest.trees: self.tree = self.forest.new_tree(self.forest.trees[-1]) - tracing.trace('use newest tree %s (of %d)', self.tree.root.id, - len(self.forest.trees)) + tracing.trace( + 'use newest tree %s (of %d)', + self.tree.root.id, + len(self.forest.trees)) else: self.tree = self.forest.new_tree() tracing.trace('new tree root id %s', self.tree.root.id) @@ -106,4 +108,3 @@ class RepositoryTree(object): self.forest.remove_tree(self.forest.trees[0]) self.forest.commit() self.tree = None - diff --git a/obnamlib/fmt_ga/client_list.py b/obnamlib/fmt_ga/client_list.py index 58c65ac9..6de64945 100644 --- a/obnamlib/fmt_ga/client_list.py +++ b/obnamlib/fmt_ga/client_list.py @@ -139,7 +139,7 @@ class GAClientList(object): del clients[old_client_name] self._data['clients'] = clients - if old_client_name in self._added_clients: # pragma: no cover + if old_client_name in self._added_clients: # pragma: no cover self._added_clients.remove(old_client_name) self._added_clients.append(new_client_name) diff --git a/obnamlib/fmt_ga/format.py b/obnamlib/fmt_ga/format.py index 8f63e30a..2c4796dd 100644 --- a/obnamlib/fmt_ga/format.py +++ b/obnamlib/fmt_ga/format.py @@ -54,7 +54,7 @@ class RepositoryFormatGA(obnamlib.RepositoryDelegator): def get_allowed_client_keys(self): return [] - def get_client_key(self, client_name, key): # pragma: no cover + def get_client_key(self, client_name, key): # pragma: no cover raise obnamlib.RepositoryClientKeyNotAllowed( format=self.format, client_name=client_name, @@ -66,7 +66,7 @@ class RepositoryFormatGA(obnamlib.RepositoryDelegator): client_name=client_name, key_name=obnamlib.repo_key_name(key)) - def get_client_extra_data_directory(self, client_name): # pragma: no cover + def get_client_extra_data_directory(self, client_name): # pragma: no cover if client_name not in self.get_client_names(): raise obnamlib.RepositoryClientDoesNotExist( client_name=client_name) diff --git a/obnamlib/fmt_simple/simple.py b/obnamlib/fmt_simple/simple.py index 2e0b3a08..c2d7c99f 100644 --- a/obnamlib/fmt_simple/simple.py +++ b/obnamlib/fmt_simple/simple.py @@ -127,7 +127,7 @@ class SimpleClientList(SimpleToplevel): self._hooks.call('repository-add-client', self, client_name) self._data.save() self._added_clients = [] - + def get_client_names(self): return self._data.get('clients', {}).keys() @@ -144,7 +144,7 @@ class SimpleClientList(SimpleToplevel): self._data['clients'] = clients self._added_clients.append(client_name) - + def remove_client(self, client_name): self._require_client_exists(client_name) @@ -164,7 +164,7 @@ class SimpleClientList(SimpleToplevel): del clients[old_client_name] self._data['clients'] = clients - if old_client_name in self._added_clients: # pragma: no cover + if old_client_name in self._added_clients: # pragma: no cover self._added_clients.remove(old_client_name) self._added_clients.append(new_client_name) @@ -418,7 +418,7 @@ class SimpleChunkStore(object): filename = self._chunk_filename(chunk_id) try: self._fs.write_file(filename, content) - except OSError, e: # pragma: no cover + except OSError, e: # pragma: no cover if e.errno == errno.EEXIST: continue raise @@ -574,7 +574,7 @@ class RepositoryFormatSimple(obnamlib.RepositoryDelegator): def get_allowed_client_keys(self): return [] - def get_client_key(self, client_name, key): # pragma: no cover + def get_client_key(self, client_name, key): # pragma: no cover raise obnamlib.RepositoryClientKeyNotAllowed( format=self.format, client_name=client_name, @@ -586,7 +586,7 @@ class RepositoryFormatSimple(obnamlib.RepositoryDelegator): client_name=client_name, key_name=obnamlib.repo_key_name(key)) - def get_client_extra_data_directory(self, client_name): # pragma: no cover + def get_client_extra_data_directory(self, client_name): # pragma: no cover if client_name not in self.get_client_names(): raise obnamlib.RepositoryClientDoesNotExist( client_name=client_name) diff --git a/obnamlib/forget_policy.py b/obnamlib/forget_policy.py index 3c61939f..f50394b9 100644 --- a/obnamlib/forget_policy.py +++ b/obnamlib/forget_policy.py @@ -35,8 +35,6 @@ class SeparatorError(obnamlib.ObnamError): 'see position {position}: {policy}') - - class ForgetPolicy(object): '''Parse and interpret a policy for what to forget and what to keep. @@ -133,4 +131,3 @@ class ForgetPolicy(object): return [(genid, dt) for genid, dt in genlist if genid in result_ids] - diff --git a/obnamlib/forget_policy_tests.py b/obnamlib/forget_policy_tests.py index 0e450a93..bc3ac76f 100644 --- a/obnamlib/forget_policy_tests.py +++ b/obnamlib/forget_policy_tests.py @@ -38,20 +38,26 @@ class ForgetPolicyParseTests(unittest.TestCase): self.assertRaises(obnamlib.ObnamError, self.fp.parse, '1h 2d') def test_parses_single_rule(self): - self.assertEqual(self.fp.parse('7d'), - { 'hourly': 0, - 'daily': 7, - 'weekly': 0, - 'monthly': 0, - 'yearly': 0 }) + self.assertEqual( + self.fp.parse('7d'), + { + 'hourly': 0, + 'daily': 7, + 'weekly': 0, + 'monthly': 0, + 'yearly': 0 + }) def test_parses_multiple_rules(self): - self.assertEqual(self.fp.parse('1h,2d,3w,4m,255y'), - { 'hourly': 1, - 'daily': 2, - 'weekly': 3, - 'monthly': 4, - 'yearly': 255 }) + self.assertEqual( + self.fp.parse('1h,2d,3w,4m,255y'), + { + 'hourly': 1, + 'daily': 2, + 'weekly': 3, + 'monthly': 4, + 'yearly': 255 + }) class ForgetPolicyMatchTests(unittest.TestCase): @@ -64,25 +70,25 @@ class ForgetPolicyMatchTests(unittest.TestCase): return [dt for i, dt in self.fp.match(rules, list(enumerate(times)))] def test_hourly_matches(self): - h0m0 = datetime.datetime(2000, 1, 1, 0, 0) + h0m0 = datetime.datetime(2000, 1, 1, 0, 0) h0m59 = datetime.datetime(2000, 1, 1, 0, 59) - h1m0 = datetime.datetime(2000, 1, 1, 1, 0) + h1m0 = datetime.datetime(2000, 1, 1, 1, 0) h1m59 = datetime.datetime(2000, 1, 1, 1, 59) self.assertEqual(self.match2('1h', [h0m0, h0m59, h1m0, h1m59]), [h1m59]) def test_two_hourly_matches(self): - h0m0 = datetime.datetime(2000, 1, 1, 0, 0) + h0m0 = datetime.datetime(2000, 1, 1, 0, 0) h0m59 = datetime.datetime(2000, 1, 1, 0, 59) - h1m0 = datetime.datetime(2000, 1, 1, 1, 0) + h1m0 = datetime.datetime(2000, 1, 1, 1, 0) h1m59 = datetime.datetime(2000, 1, 1, 1, 59) self.assertEqual(self.match2('2h', [h0m0, h0m59, h1m0, h1m59]), [h0m59, h1m59]) def test_daily_matches(self): - d1h0 = datetime.datetime(2000, 1, 1, 0, 0) + d1h0 = datetime.datetime(2000, 1, 1, 0, 0) d1h23 = datetime.datetime(2000, 1, 1, 23, 0) - d2h0 = datetime.datetime(2000, 1, 2, 0, 0) + d2h0 = datetime.datetime(2000, 1, 2, 0, 0) d2h23 = datetime.datetime(2000, 1, 2, 23, 0) self.assertEqual(self.match2('1d', [d1h0, d1h23, d2h0, d2h23]), [d2h23]) @@ -91,17 +97,17 @@ class ForgetPolicyMatchTests(unittest.TestCase): # a sensible test case right now. def test_monthly_matches(self): - m1d1 = datetime.datetime(2000, 1, 1, 0, 0) + m1d1 = datetime.datetime(2000, 1, 1, 0, 0) m1d28 = datetime.datetime(2000, 1, 28, 0, 0) - m2d1 = datetime.datetime(2000, 2, 1, 0, 0) + m2d1 = datetime.datetime(2000, 2, 1, 0, 0) m2d28 = datetime.datetime(2000, 2, 28, 0, 0) self.assertEqual(self.match2('1m', [m1d1, m1d28, m2d1, m2d28]), [m2d28]) def test_yearly_matches(self): - y1m1 = datetime.datetime(2000, 1, 1, 0, 0) + y1m1 = datetime.datetime(2000, 1, 1, 0, 0) y1m12 = datetime.datetime(2000, 12, 1, 0, 0) - y2m1 = datetime.datetime(2001, 1, 1, 0, 0) + y2m1 = datetime.datetime(2001, 1, 1, 0, 0) y2m12 = datetime.datetime(2001, 12, 1, 0, 0) self.assertEqual(self.match2('1y', [y1m1, y1m12, y2m1, y2m12]), [y2m12]) @@ -125,4 +131,3 @@ class ForgetPolicyMatchTests(unittest.TestCase): d3 = datetime.datetime(2000, 1, 3, 0, 0) self.assertEqual(self.match2('10h,1d', [d1, d2, d3]), [d1, d2, d3]) - diff --git a/obnamlib/hooks.py b/obnamlib/hooks.py index 6865aeba..0c252259 100644 --- a/obnamlib/hooks.py +++ b/obnamlib/hooks.py @@ -54,8 +54,8 @@ class Hook(object): if callback not in self.callbacks: self.priorities[callback] = priority self.callbacks.append(callback) - self.callbacks.sort(lambda x,y: cmp(self.priorities[x], - self.priorities[y])) + self.callbacks.sort( + lambda x, y: cmp(self.priorities[x], self.priorities[y])) return callback @@ -138,7 +138,7 @@ class FilterHook(Hook): tracing.trace('calling %s' % filt) new_data = filt.filter_write(data, *args, **kwargs) assert new_data is not None, \ - filt.tag + ": Returned None from filter_write()" + filt.tag + ": Returned None from filter_write()" if data != new_data: tracing.trace('filt.tag=%s' % filt.tag) data = filt.tag + "\0" + new_data diff --git a/obnamlib/hooks_tests.py b/obnamlib/hooks_tests.py index 222b7413..94e4d041 100644 --- a/obnamlib/hooks_tests.py +++ b/obnamlib/hooks_tests.py @@ -20,6 +20,7 @@ import obnamlib import base64 + class HookTests(unittest.TestCase): def setUp(self): @@ -63,13 +64,14 @@ class HookTests(unittest.TestCase): self.hook.add_callback(self.callback) self.hook.call_callbacks('bar', kwarg='foobar') self.assertEqual(self.args, ('bar',)) - self.assertEqual(self.kwargs, { 'kwarg': 'foobar' }) + self.assertEqual(self.kwargs, {'kwarg': 'foobar'}) def test_removes_callback(self): cb_id = self.hook.add_callback(self.callback) self.hook.remove_callback(cb_id) self.assertEqual(self.hook.callbacks, []) + class NeverAddsFilter(object): def __init__(self): @@ -87,6 +89,7 @@ class NeverAddsFilter(object): self.wasread = False return data + class Base64Filter(object): def __init__(self): @@ -104,6 +107,7 @@ class Base64Filter(object): self.wasread = False return base64.b64encode(data) + class FilterHookTests(unittest.TestCase): def setUp(self): @@ -146,6 +150,7 @@ class FilterHookTests(unittest.TestCase): def test_call_callbacks_raises(self): self.assertRaises(NotImplementedError, self.hook.call_callbacks, "") + class HookManagerTests(unittest.TestCase): def setUp(self): @@ -161,7 +166,7 @@ class HookManagerTests(unittest.TestCase): self.assertEqual(hooks.hooks, {}) def test_adds_new_hook(self): - self.assert_(self.hooks.hooks.has_key('foo')) + self.assertIn('foo', self.hooks.hooks) def test_adds_new_filter_hook(self): self.hooks.new_filter('bar') @@ -180,7 +185,7 @@ class HookManagerTests(unittest.TestCase): self.hooks.add_callback('foo', self.callback) self.hooks.call('foo', 'bar', kwarg='foobar') self.assertEqual(self.args, ('bar',)) - self.assertEqual(self.kwargs, { 'kwarg': 'foobar' }) + self.assertEqual(self.kwargs, {'kwarg': 'foobar'}) def test_filter_write_returns_value_of_callbacks(self): self.hooks.new_filter('bar') diff --git a/obnamlib/lockmgr.py b/obnamlib/lockmgr.py index 7e3c298a..a9c4216b 100644 --- a/obnamlib/lockmgr.py +++ b/obnamlib/lockmgr.py @@ -30,10 +30,10 @@ class LockManager(object): self.timeout = timeout self._got_locks = [] - def _time(self): # pragma: no cover + def _time(self): # pragma: no cover return time.time() - def _sleep(self): # pragma: no cover + def _sleep(self): # pragma: no cover time.sleep(1) def sort(self, dirnames): diff --git a/obnamlib/metadata.py b/obnamlib/metadata.py index ca26f4f5..70878638 100644 --- a/obnamlib/metadata.py +++ b/obnamlib/metadata.py @@ -95,7 +95,7 @@ class Metadata(object): def isfile(self): return self.st_mode is not None and stat.S_ISREG(self.st_mode) - def __repr__(self): # pragma: no cover + def __repr__(self): # pragma: no cover fields = ', '.join('%s=%s' % (k, getattr(self, k)) for k in metadata_fields) return 'Metadata(%s)' % fields @@ -118,19 +118,24 @@ class Metadata(object): # change during the runtime of the backup. _uid_to_username = {} -def _cached_getpwuid(uid): # pragma: no cover + + +def _cached_getpwuid(uid): # pragma: no cover if uid not in _uid_to_username: _uid_to_username[uid] = pwd.getpwuid(uid) return _uid_to_username[uid] + _gid_to_groupname = {} -def _cached_getgrgid(gid): # pragma: no cover + + +def _cached_getgrgid(gid): # pragma: no cover if gid not in _gid_to_groupname: _gid_to_groupname[gid] = grp.getgrgid(gid) return _gid_to_groupname[gid] -def get_xattrs_as_blob(fs, filename): # pragma: no cover +def get_xattrs_as_blob(fs, filename): # pragma: no cover tracing.trace('filename=%s' % filename) try: @@ -179,11 +184,11 @@ def get_xattrs_as_blob(fs, filename): # pragma: no cover value_blob)) -def set_xattrs_from_blob(fs, filename, blob, user_only): # pragma: no cover +def set_xattrs_from_blob(fs, filename, blob, user_only): # pragma: no cover sizesize = struct.calcsize('!Q') name_blob_size = struct.unpack('!Q', blob[:sizesize])[0] - name_blob = blob[sizesize : sizesize + name_blob_size] - value_blob = blob[sizesize + name_blob_size : ] + name_blob = blob[sizesize:sizesize + name_blob_size] + value_blob = blob[sizesize + name_blob_size:] names = [s for s in name_blob.split('\0')[:-1]] fmt = '!' + 'Q' * len(names) @@ -237,7 +242,7 @@ class SetMetadataError(obnamlib.ObnamError): msg = "{filename}: Couldn't set metadata {metadata}: {errno}: {strerror}" -def _set_something(filename, what, func): # pragma: no cover +def _set_something(filename, what, func): # pragma: no cover try: func() except OSError as e: @@ -249,7 +254,7 @@ def _set_something(filename, what, func): # pragma: no cover strerror=e.strerror) -def set_metadata(fs, filename, metadata, +def set_metadata(fs, filename, metadata, getuid=None, always_set_id_bits=False): '''Set metadata for a filesystem entry. @@ -281,7 +286,7 @@ def set_metadata(fs, filename, metadata, # normal users can set the group if they are in the group, try to # restore the group, ignoring any errors try: - uid = -1 # no change to user + uid = -1 # no change to user fs.lchown(filename, uid, metadata.st_gid) except (OSError), e: sys.exc_clear() @@ -290,7 +295,7 @@ def set_metadata(fs, filename, metadata, # unless explicitly told to do so. mode = metadata.st_mode set_id_bits = always_set_id_bits or (getuid() in (0, metadata.st_uid)) - if not set_id_bits: # pragma: no cover + if not set_id_bits: # pragma: no cover mode = mode & (~stat.S_ISUID) mode = mode & (~stat.S_ISGID) if symlink: @@ -302,7 +307,7 @@ def set_metadata(fs, filename, metadata, filename, 'chmod', lambda: fs.chmod_not_symlink(filename, mode)) - if metadata.xattr: # pragma: no cover + if metadata.xattr: # pragma: no cover user_only = getuid() != 0 _set_something( filename, 'xattrs', @@ -311,7 +316,7 @@ def set_metadata(fs, filename, metadata, _set_something( filename, 'timestamps', - lambda: + lambda: fs.lutimes( filename, metadata.st_atime_sec, metadata.st_atime_nsec, metadata.st_mtime_sec, metadata.st_mtime_nsec)) diff --git a/obnamlib/metadata_tests.py b/obnamlib/metadata_tests.py index cc56d389..cdd676a8 100644 --- a/obnamlib/metadata_tests.py +++ b/obnamlib/metadata_tests.py @@ -134,7 +134,7 @@ class ReadMetadataTests(unittest.TestCase): metadata = obnamlib.read_metadata(self.fakefs, 'foo', getpwuid=self.fakefs.getpwuid, getgrgid=self.fakefs.getgrgid) - fields = ['st_atime_sec','st_atime_nsec', 'st_blocks', 'st_dev', + fields = ['st_atime_sec', 'st_atime_nsec', 'st_blocks', 'st_dev', 'st_gid', 'st_ino', 'st_mode', 'st_mtime_sec', 'st_mtime_nsec', 'st_nlink', 'st_size', 'st_uid', 'groupname', 'username'] diff --git a/obnamlib/obj_serialiser.py b/obnamlib/obj_serialiser.py index f43dc368..37cc5722 100644 --- a/obnamlib/obj_serialiser.py +++ b/obnamlib/obj_serialiser.py @@ -56,9 +56,11 @@ def deserialise_object(serialised): _length_fmt = '!Q' _length_size = struct.calcsize(_length_fmt) + def _serialise_length(num_bytes): return struct.pack(_length_fmt, num_bytes) + def _deserialise_length(serialised): return struct.unpack(_length_fmt, serialised)[0] @@ -67,9 +69,11 @@ def _deserialise_length(serialised): _none_size_encoded = _serialise_length(0) + def _serialise_none(obj): return _NONE + _none_size_encoded + def _deserialise_none(serialised): return None @@ -80,6 +84,7 @@ def _serialise_integer(obj): s = str(obj) return _INT + _serialise_length(len(s)) + s + def _deserialise_integer(serialised): return int(serialised) @@ -89,10 +94,12 @@ def _deserialise_integer(serialised): _bool_fmt = '!c' _bool_size_serialised = _serialise_length(struct.calcsize(_bool_fmt)) + def _serialise_bool(obj): return (_BOOL + _bool_size_serialised + struct.pack(_bool_fmt, chr(int(obj)))) + def _deserialise_bool(obj): return bool(ord(struct.unpack(_bool_fmt, obj)[0])) @@ -102,6 +109,7 @@ def _deserialise_bool(obj): def _serialise_str(obj): return _STR + _serialise_length(len(obj)) + obj + def _deserialise_str(obj): return obj @@ -112,6 +120,7 @@ def _serialise_list(obj): items = ''.join(serialise_object(item) for item in obj) return _LIST + _serialise_length(len(items)) + items + def _deserialise_list(serialised): items = [] pos = 0 @@ -122,11 +131,13 @@ def _deserialise_list(serialised): pos = end return items + def _extract_length(serialised, pos): start = pos + 1 end = start + _length_size return _deserialise_length(serialised[start:end]) + def _next_object(pos, length): return pos + 1 + _length_size + length @@ -139,6 +150,7 @@ def _serialise_dict(obj): for key, value in obj.iteritems()) return _DICT + _serialise_length(len(pairs)) + pairs + def _deserialise_dict(serialised): result = {} pos = 0 @@ -148,6 +160,7 @@ def _deserialise_dict(serialised): result[key] = value return result + def _deserialise_prefix(serialised, pos): length = _extract_length(serialised, pos) end = _next_object(pos, length) diff --git a/obnamlib/pluginbase_tests.py b/obnamlib/pluginbase_tests.py index 2b23769f..282b956b 100644 --- a/obnamlib/pluginbase_tests.py +++ b/obnamlib/pluginbase_tests.py @@ -24,6 +24,7 @@ class FakeApp(object): def __init__(self): self.hooks = self + class ObnamPluginTests(unittest.TestCase): def setUp(self): diff --git a/obnamlib/plugins/backup_plugin.py b/obnamlib/plugins/backup_plugin.py index e5b06df4..a5f1f0d7 100644 --- a/obnamlib/plugins/backup_plugin.py +++ b/obnamlib/plugins/backup_plugin.py @@ -159,7 +159,6 @@ class BackupProgress(object): scanned_amount, scanned_unit = obnamlib.humanise_size( self.scanned_bytes) - self._ts.notify( 'Backed up %d files (of %d found), containing %.1f %s.' % @@ -203,7 +202,7 @@ class CheckpointManager(object): self.last_checkpoint = 0 def time_for_checkpoint(self): - bytes_since = (self.repo.get_fs().bytes_written - + bytes_since = (self.repo.get_fs().bytes_written - self.last_checkpoint) return bytes_since >= self.interval @@ -329,10 +328,10 @@ class BackupPlugin(obnamlib.ObnamPlugin): def start_backup(self): self.configure_progress_reporting() self.progress.what('setting up') - + self.memory_dump_counter = 0 self.chunkid_token_map = obnamlib.ChunkIdTokenMap() - + self.progress.what('connecting to repository') self.repo = self.open_repository() if not self.pretend: @@ -360,7 +359,7 @@ class BackupPlugin(obnamlib.ObnamPlugin): raise else: return self.app.get_repository_object(create=True) - + def prepare_repository_for_client(self): self.progress.what('adding client') self.add_client(self.client_name) @@ -373,7 +372,7 @@ class BackupPlugin(obnamlib.ObnamPlugin): self.progress.what('initialising shared directories') self.repo.lock_chunk_indexes() self.repo.unlock_chunk_indexes() - + def start_generation(self): self.progress.what('starting new generation') self.new_generation = self.repo.create_generation(self.client_name) @@ -386,7 +385,7 @@ class BackupPlugin(obnamlib.ObnamPlugin): self.progress.what(prefix + 'adding chunks to shared B-trees') self.add_chunks_to_shared() - + self.progress.what(prefix + 'updating generation metadata') self.repo.set_generation_key( self.new_generation, @@ -400,11 +399,11 @@ class BackupPlugin(obnamlib.ObnamPlugin): self.new_generation, obnamlib.REPO_GENERATION_IS_CHECKPOINT, False) - + self.progress.what(prefix + 'committing client') self.repo.flush_chunks() self.repo.commit_client(self.client_name) - + self.progress.what(prefix + 'committing shared B-trees') self.repo.commit_chunk_indexes() @@ -481,10 +480,10 @@ class BackupPlugin(obnamlib.ObnamPlugin): if client_name not in self.repo.get_client_names(): tracing.trace('adding new client %s' % client_name) tracing.trace('client list before adding: %s' % - self.repo.get_client_names()) + self.repo.get_client_names()) self.repo.add_client(client_name) tracing.trace('client list after adding: %s' % - self.repo.get_client_names()) + self.repo.get_client_names()) self.repo.commit_client_list() self.repo = self.app.get_repository_object(repofs=self.repo.get_fs()) @@ -826,7 +825,7 @@ class BackupPlugin(obnamlib.ObnamPlugin): for field, key in things: self.repo.set_file_key( - self.new_generation, filename, key, + self.new_generation, filename, key, getattr(metadata, field)) def backup_parents(self, root): @@ -843,7 +842,7 @@ class BackupPlugin(obnamlib.ObnamPlugin): except OSError, e: logging.warning( 'Failed to get metadata for %s: %s: %s' % - (root, e.errno or 0, e.strerror)) + (root, e.errno or 0, e.strerror)) logging.warning('Using fake metadata instead for %s' % root) metadata = dummy_metadata if not self.pretend: @@ -993,7 +992,7 @@ class BackupPlugin(obnamlib.ObnamPlugin): new_pathnames = [os.path.join(root, x) for x in new_basenames] if self.repo.file_exists(self.new_generation, root): old_pathnames = self.repo.get_file_children( - self.new_generation, root) + self.new_generation, root) else: old_pathnames = [] @@ -1007,8 +1006,8 @@ class BackupPlugin(obnamlib.ObnamPlugin): pass else: # remove paths that were excluded recently - if (not old in no_delete_paths) and \ - (not self.can_be_backed_up(old, st)): + if (old not in no_delete_paths) and \ + (not self.can_be_backed_up(old, st)): self.repo.remove_file(self.new_generation, old) # Files that are created after the previous generation will be diff --git a/obnamlib/plugins/compression_plugin.py b/obnamlib/plugins/compression_plugin.py index 82b6c6e2..b2ade5ba 100644 --- a/obnamlib/plugins/compression_plugin.py +++ b/obnamlib/plugins/compression_plugin.py @@ -19,6 +19,7 @@ import zlib import obnamlib + class DeflateCompressionFilter(object): def __init__(self, app): diff --git a/obnamlib/plugins/encryption_plugin.py b/obnamlib/plugins/encryption_plugin.py index bf4414d8..53988199 100644 --- a/obnamlib/plugins/encryption_plugin.py +++ b/obnamlib/plugins/encryption_plugin.py @@ -117,8 +117,7 @@ class EncryptionPlugin(obnamlib.ObnamPlugin): pubkeys.add(self.pubkey) symmetric_key = obnamlib.generate_symmetric_key( - self.symmetric_key_bits, - filename=self.devrandom) + self.symmetric_key_bits, filename=self.devrandom) encrypted = obnamlib.encrypt_with_keyring(symmetric_key, pubkeys) self._write_file(repo, os.path.join(toplevel, 'key'), encrypted) @@ -288,4 +287,3 @@ class EncryptionPlugin(obnamlib.ObnamPlugin): logging.info('removing client %s' % client_name) repo.remove_client(client_name) repo.commit_client_list() - diff --git a/obnamlib/plugins/exclude_pathnames_plugin.py b/obnamlib/plugins/exclude_pathnames_plugin.py index 1bdf19be..9aad8070 100644 --- a/obnamlib/plugins/exclude_pathnames_plugin.py +++ b/obnamlib/plugins/exclude_pathnames_plugin.py @@ -97,7 +97,7 @@ class ExcludePathnamesPlugin(obnamlib.ObnamPlugin): msg = ('error compiling regular expression "%s": %s' % (x, e)) logging.error(msg) self.progress.error(msg) - + def read_patterns_from_files(self, filenames): patterns = [] for filename in filenames: diff --git a/obnamlib/plugins/forget_plugin.py b/obnamlib/plugins/forget_plugin.py index 3769020a..860cd116 100644 --- a/obnamlib/plugins/forget_plugin.py +++ b/obnamlib/plugins/forget_plugin.py @@ -24,8 +24,8 @@ class ForgetPlugin(obnamlib.ObnamPlugin): '''Forget generations.''' def enable(self): - self.app.add_subcommand('forget', self.forget, - arg_synopsis='[GENERATION]...') + self.app.add_subcommand( + 'forget', self.forget, arg_synopsis='[GENERATION]...') self.app.settings.string( ['keep'], 'policy for what generations to keep ' @@ -83,11 +83,11 @@ class ForgetPlugin(obnamlib.ObnamPlugin): genid = self.repo.interpret_generation_spec( client_name, genspec) self.app.ts.notify( - 'Forgetting generation %s' % + 'Forgetting generation %s' % self.repo.make_generation_spec(genid)) self.remove(genid) self.app.dump_memory_profile( - 'after removing %s' % + 'after removing %s' % self.repo.make_generation_spec(genid)) elif self.app.settings['keep']: genlist = [] @@ -112,7 +112,7 @@ class ForgetPlugin(obnamlib.ObnamPlugin): self.app.ts['gen'] = genid self.remove(genid) self.app.dump_memory_profile( - 'after removing %s' % + 'after removing %s' % self.repo.make_generation_spec(genid)) # Commit or unlock everything. @@ -127,7 +127,7 @@ class ForgetPlugin(obnamlib.ObnamPlugin): def remove(self, genid): if self.app.settings['pretend']: self.app.ts.notify( - 'Pretending to remove generation %s' % + 'Pretending to remove generation %s' % self.repo.make_generation_spec(genid)) else: self.repo.remove_generation(genid) diff --git a/obnamlib/plugins/fsck_plugin.py b/obnamlib/plugins/fsck_plugin.py index 59a296c5..7c0cbbf5 100644 --- a/obnamlib/plugins/fsck_plugin.py +++ b/obnamlib/plugins/fsck_plugin.py @@ -70,7 +70,7 @@ class CheckFile(WorkItem): def do(self): logging.debug('Checking client=%s genid=%s filename=%s' % - (self.client_name, self.genid, self.filename)) + (self.client_name, self.genid, self.filename)) mode = self.repo.get_file_key( self.genid, self.filename, obnamlib.REPO_FILE_MODE) if stat.S_ISREG(mode) and not self.settings['fsck-ignore-chunks']: @@ -94,7 +94,7 @@ class CheckDirectory(WorkItem): def do(self): logging.debug('Checking client=%s genid=%s dirname=%s' % - (self.client_name, self.genid, self.dirname)) + (self.client_name, self.genid, self.dirname)) for pathname in self.repo.get_file_children(self.genid, self.dirname): mode = self.repo.get_file_key( self.genid, pathname, obnamlib.REPO_FILE_MODE) @@ -113,7 +113,7 @@ class CheckGeneration(WorkItem): def do(self): logging.debug('Checking client=%s genid=%s' % - (self.client_name, self.genid)) + (self.client_name, self.genid)) started = self.repo.get_generation_key( self.genid, obnamlib.REPO_GENERATION_STARTED) @@ -121,10 +121,10 @@ class CheckGeneration(WorkItem): self.genid, obnamlib.REPO_GENERATION_ENDED) if started is None: self.error('%s:%s: no generation start time' % - (self.client_name, self.genid)) + (self.client_name, self.genid)) if ended is None: self.error('%s:%s: no generation end time' % - (self.client_name, self.genid)) + (self.client_name, self.genid)) n = self.repo.get_generation_key( self.genid, obnamlib.REPO_GENERATION_FILE_COUNT) @@ -152,7 +152,7 @@ class CheckGenerationIdsAreDifferent(WorkItem): def do(self): logging.debug('Checking genid uniqueness for client=%s' % - self.client_name) + self.client_name) done = set() while self.genids: genid = self.genids.pop() @@ -244,7 +244,7 @@ class FsckPlugin(obnamlib.ObnamPlugin): self.app.settings.boolean( ['fsck-ignore-chunks'], 'ignore chunks when checking repository integrity (assume all ' - 'chunks exist and are correct)', + 'chunks exist and are correct)', group=group) self.app.settings.string_list( @@ -296,7 +296,7 @@ class FsckPlugin(obnamlib.ObnamPlugin): logging.debug('fsck on %s' % self.app.settings['repository']) rm_unused_chunks = self.app.settings['fsck-rm-unused'] \ - or self.app.settings['fsck-fix'] + or self.app.settings['fsck-fix'] self.configure_ttystatus() @@ -361,4 +361,3 @@ class FsckPlugin(obnamlib.ObnamPlugin): def warning(self, msg): logging.warning(msg) self.app.ts.notify(msg) - diff --git a/obnamlib/plugins/fuse_plugin.py b/obnamlib/plugins/fuse_plugin.py index 8f84c648..2bf8d0a7 100644 --- a/obnamlib/plugins/fuse_plugin.py +++ b/obnamlib/plugins/fuse_plugin.py @@ -35,7 +35,7 @@ except ImportError: class Bunch: def __init__(self, **kwds): self.__dict__.update(kwds) - fuse = Bunch(Fuse = object) + fuse = Bunch(Fuse=object) class FileNotFoundError(obnamlib.ObnamError): @@ -85,9 +85,9 @@ class ObnamFuseFile(object): # Flags that indicate the caller wants to write to the file. # Since we're read-only, we'll have to fail the request. write_flags = ( - os.O_WRONLY | os.O_RDWR | os.O_CREAT | os.O_EXCL | os.O_TRUNC | + os.O_WRONLY | os.O_RDWR | os.O_CREAT | os.O_EXCL | os.O_TRUNC | os.O_APPEND) - + def __init__(self, path, flags, *mode): tracing.trace('path=%r', path) tracing.trace('flags=%r', flags) @@ -112,7 +112,7 @@ class ObnamFuseFile(object): # if not a regular file return EINVAL if not stat.S_ISREG(self.metadata.st_mode): raise IOError(errno.EINVAL, 'Invalid argument') - + def read_pid(self, length, offset): tracing.trace('length=%r', length) tracing.trace('offset=%r', offset) @@ -176,7 +176,7 @@ class ObnamFuseFile(object): if contents is None: contents = self.fuse_fs.obnam.repo.get_chunk_content( chunkid) - output.append(contents[start : start+n]) + output.append(contents[start:start+n]) output_length += n assert output_length <= length if output_length == length: @@ -295,6 +295,7 @@ class ObnamFuse(fuse.Fuse): def construct_metadata_object(self, repo, gen, filename): allowed = set(repo.get_allowed_file_keys()) + def K(key): if key in allowed: return repo.get_file_key(gen, filename, key) @@ -416,24 +417,24 @@ class ObnamFuse(fuse.Fuse): for gen in self.obnam.repo.get_client_generation_ids(client_name)) stv = fuse.StatVfs() - stv.f_bsize = 65536 - stv.f_frsize = 0 - stv.f_blocks = total_data / 65536 - stv.f_bfree = 0 - stv.f_bavail = 0 - stv.f_files = files - stv.f_ffree = 0 - stv.f_favail = 0 - stv.f_flag = 0 + stv.f_bsize = 65536 + stv.f_frsize = 0 + stv.f_blocks = total_data / 65536 + stv.f_bfree = 0 + stv.f_bavail = 0 + stv.f_files = files + stv.f_ffree = 0 + stv.f_favail = 0 + stv.f_flag = 0 stv.f_namemax = 255 - #raise OSError(errno.ENOSYS, 'Unimplemented') + # raise OSError(errno.ENOSYS, 'Unimplemented') return stv def getxattr(self, path, name, size): tracing.trace('path=%r', path) tracing.trace('name=%r', name) tracing.trace('size=%r', size) - + try: gen_id, repopath = self.get_gen_path(path) except obnamlib.RepositoryClientHasNoGenerations: @@ -454,10 +455,10 @@ class ObnamFuse(fuse.Fuse): blob = metadata.xattr sizesize = struct.calcsize('!Q') name_blob_size = struct.unpack('!Q', blob[:sizesize])[0] - name_blob = blob[sizesize : sizesize + name_blob_size] + name_blob = blob[sizesize:sizesize + name_blob_size] name_list = name_blob.split('\0')[:-1] if name in name_list: - value_blob = blob[sizesize + name_blob_size : ] + value_blob = blob[sizesize + name_blob_size:] idx = name_list.index(name) fmt = '!' + 'Q' * len(name_list) lengths_size = sizesize * len(name_list) @@ -496,7 +497,7 @@ class ObnamFuse(fuse.Fuse): name_blob_size = struct.unpack('!Q', blob[:sizesize])[0] if size == 0: return name_blob_size - name_blob = blob[sizesize : sizesize + name_blob_size] + name_blob = blob[sizesize:sizesize + name_blob_size] return name_blob.split('\0')[:-1] except obnamlib.ObnamError: raise IOError(errno.ENOENT, 'No such file or directory') @@ -585,7 +586,7 @@ class MountPlugin(obnamlib.ObnamPlugin): ls -l my-fuse ls -l my-fuse/latest diff -u my-fuse/latest/home/liw/README ~/README - + You can also restore files by copying them from the my-fuse directory: diff --git a/obnamlib/plugins/restore_plugin.py b/obnamlib/plugins/restore_plugin.py index 677d8200..3f85d74f 100644 --- a/obnamlib/plugins/restore_plugin.py +++ b/obnamlib/plugins/restore_plugin.py @@ -125,7 +125,7 @@ class RestorePlugin(obnamlib.ObnamPlugin): self.app.settings.require('to') logging.debug('restoring generation %s' % - self.app.settings['generation']) + self.app.settings['generation']) logging.debug('restoring to %s' % self.app.settings['to']) logging.debug('restoring what: %s' % repr(args)) @@ -157,7 +157,7 @@ class RestorePlugin(obnamlib.ObnamPlugin): # the permissions will eventually be correct. self.fs.chmod_not_symlink('.', 0700) else: - self.fs = None # this will trigger error if we try to really write + self.fs = None # this will trigger error if we try to really write self.hardlinks = Hardlinks() @@ -200,6 +200,7 @@ class RestorePlugin(obnamlib.ObnamPlugin): def construct_metadata_object(self, gen, filename): allowed = set(self.repo.get_allowed_file_keys()) + def K(key): if key in allowed: return self.repo.get_file_key(gen, filename, key) @@ -384,7 +385,7 @@ class RestorePlugin(obnamlib.ObnamPlugin): if self.downloaded_bytes >= size_base: if size_base > 0: size_amount = (float(self.downloaded_bytes) / - float(size_base)) + float(size_base)) else: size_amount = float(self.downloaded_bytes) break @@ -419,12 +420,12 @@ class RestorePlugin(obnamlib.ObnamPlugin): logging.info('Restore performance statistics:') logging.info('* files restored: %s' % self.file_count) logging.info('* downloaded data: %s bytes (%s %s)' % - (self.downloaded_bytes, size_amount, size_unit)) + (self.downloaded_bytes, size_amount, size_unit)) logging.info('* duration: %s s' % duration) logging.info('* average speed: %s %s' % (speed_amount, speed_unit)) self.app.ts.notify( 'Restored %d files, ' 'downloaded %.1f %s in %s at %.1f %s average speed' % - (self.file_count, - size_amount, size_unit, - duration_string, speed_amount, speed_unit)) + (self.file_count, + size_amount, size_unit, + duration_string, speed_amount, speed_unit)) diff --git a/obnamlib/plugins/sftp_plugin.py b/obnamlib/plugins/sftp_plugin.py index 0047de24..7351d911 100644 --- a/obnamlib/plugins/sftp_plugin.py +++ b/obnamlib/plugins/sftp_plugin.py @@ -177,7 +177,7 @@ class SftpFS(obnamlib.VirtualFileSystem): def log_stats(self): obnamlib.VirtualFileSystem.log_stats(self) logging.info('VFS: baseurl=%s roundtrips=%s' % - (self.baseurl, self._roundtrips)) + (self.baseurl, self._roundtrips)) def _to_string(self, str_or_unicode): if type(str_or_unicode) is unicode: @@ -192,7 +192,7 @@ class SftpFS(obnamlib.VirtualFileSystem): # sftp/paramiko does not give us a useful errno so we hope # for the best pass - self.create_path_if_missing = False # only create once + self.create_path_if_missing = False # only create once def connect(self): try_openssh = not self.settings or not self.settings['pure-paramiko'] @@ -218,14 +218,14 @@ class SftpFS(obnamlib.VirtualFileSystem): args += ['-l', self.user] if self.settings and self.settings['ssh-key']: args += ['-i', self.settings['ssh-key']] - if (self.settings and - self.settings['ssh-host-keys-check'] != "ssh-config"): - value = self.settings['ssh-host-keys-check'] - args += ['-o', 'StrictHostKeyChecking=%s' % (value,)] + if self.settings: + if self.settings['ssh-host-keys-check'] != "ssh-config": + value = self.settings['ssh-host-keys-check'] + args += ['-o', 'StrictHostKeyChecking=%s' % (value,)] if self.settings and self.settings['ssh-known-hosts']: args += ['-o', 'UserKnownHostsFile=%s' % - self.settings['ssh-known-hosts']] + self.settings['ssh-known-hosts']] args += [self.host, 'sftp'] # prepend the executable to the argument list @@ -279,7 +279,7 @@ class SftpFS(obnamlib.VirtualFileSystem): return offered_type = offered_key.get_name() - if not known_keys.has_key(offered_type): + if offered_type not in known_keys: if self.settings['ssh-host-keys-check'] == 'yes': raise NoHostKeyOfWantedTypeError( key_type=offered_type, hostname=hostname) @@ -314,7 +314,7 @@ class SftpFS(obnamlib.VirtualFileSystem): key = paramiko.RSAKey.from_private_key_file(filename) except paramiko.PasswordRequiredException: password = getpass.getpass('RSA key password for %s: ' % - filename) + filename) key = paramiko.RSAKey.from_private_key_file(filename, password) return key @@ -399,11 +399,11 @@ class SftpFS(obnamlib.VirtualFileSystem): if timestamp is None: return None - max_int32 = 2**31 - 1 # max positive 32 signed integer value + max_int32 = 2**31 - 1 # max positive 32 signed integer value if timestamp > max_int32: timestamp -= 2**32 if timestamp > max_int32: - timestamp = max_int32 # it's too large, need to lose info + timestamp = max_int32 # it's too large, need to lose info return timestamp def _fix_stat(self, pathname, st): diff --git a/obnamlib/plugins/show_plugin.py b/obnamlib/plugins/show_plugin.py index 3a2d72b7..fb7606b7 100644 --- a/obnamlib/plugins/show_plugin.py +++ b/obnamlib/plugins/show_plugin.py @@ -161,16 +161,16 @@ class ShowPlugin(obnamlib.ObnamPlugin): elif now - most_recent > critical_age: self.app.output.write( 'CRITICAL: backup is old. last backup was %s.\n' % - (self.format_time(most_recent))) + (self.format_time(most_recent))) sys.exit(2) elif now - most_recent > warn_age: self.app.output.write( 'WARNING: backup is old. last backup was %s.\n' % - self.format_time(most_recent)) + self.format_time(most_recent)) sys.exit(1) self.app.output.write( 'OK: backup is recent. last backup was %s.\n' % - self.format_time(most_recent)) + self.format_time(most_recent)) def genids(self, args): '''List generation ids for client.''' @@ -240,13 +240,13 @@ class ShowPlugin(obnamlib.ObnamPlugin): def show_item_ls(self, gen_id, filename): fields = self.fields(gen_id, filename) widths = [ - 1, # mode - 5, # nlink - -8, # owner - -8, # group - 10, # size - 1, # mtime - -1, # name + 1, # mode + 5, # nlink + -8, # owner + -8, # group + 10, # size + 1, # mtime + -1, # name ] result = [] @@ -279,20 +279,28 @@ class ShowPlugin(obnamlib.ObnamPlugin): mtime_sec = self.repo.get_file_key( gen_id, filename, obnamlib.REPO_FILE_MTIME_SEC) - if stat.S_ISREG(mode): mode_str = "F\t" - elif stat.S_ISDIR(mode): mode_str = "D " - elif stat.S_ISLNK(mode): mode_str = "L\t" - elif stat.S_ISBLK(mode): mode_str = "BlockDev\t" - elif stat.S_ISCHR(mode): mode_str = "CharDev\t" - elif stat.S_ISSOCK(mode): mode_str = "Socket\t" + if stat.S_ISREG(mode): + mode_str = "F\t" + elif stat.S_ISDIR(mode): + mode_str = "D " + elif stat.S_ISLNK(mode): + mode_str = "L\t" + elif stat.S_ISBLK(mode): + mode_str = "BlockDev\t" + elif stat.S_ISCHR(mode): + mode_str = "CharDev\t" + elif stat.S_ISSOCK(mode): + mode_str = "Socket\t" enc_filename = filename.replace("%", "%25") enc_filename = enc_filename.replace(" ", "%20") enc_filename = enc_filename.replace("\t", "%09") - if filename == "/": return + if filename == "/": + return - self.app.output.write("%s%s\t%d\t%#x\n" % + self.app.output.write( + "%s%s\t%d\t%#x\n" % (mode_str, enc_filename, size, mtime_sec)) def show_diff_for_file(self, gen_id, fullname, change_char): @@ -398,12 +406,12 @@ class ShowPlugin(obnamlib.ObnamPlugin): perms = ['?'] + ['-'] * 9 tab = [ (stat.S_IFDIR, 0, 'd'), - (stat.S_IFCHR, 0, 'c'), # character device - (stat.S_IFBLK, 0, 'b'), # block device + (stat.S_IFCHR, 0, 'c'), # character device + (stat.S_IFBLK, 0, 'b'), # block device (stat.S_IFREG, 0, '-'), (stat.S_IFIFO, 0, 'p'), (stat.S_IFLNK, 0, 'l'), - #(stat.S_IFSOCK, 0, 's'), # not stored, listed for completeness + # (stat.S_IFSOCK, 0, 's'), # not stored, listed for completeness (stat.S_IRUSR, 1, 'r'), (stat.S_IWUSR, 2, 'w'), (stat.S_IXUSR, 3, 'x'), @@ -441,12 +449,12 @@ class ShowPlugin(obnamlib.ObnamPlugin): name = filename return (perms, - str(nlink), - username, - groupname, - str(size), - timestamp, - name) + str(nlink), + username, + groupname, + str(size), + timestamp, + name) def align(self, width, field, field_no): if field_no in self.leftists: @@ -456,10 +464,12 @@ class ShowPlugin(obnamlib.ObnamPlugin): def _convert_time(self, s, default_unit='h'): m = re.match('([0-9]+)([smhdw])?$', s) - if m is None: raise ValueError + if m is None: + raise ValueError ticks = int(m.group(1)) unit = m.group(2) - if unit is None: unit = default_unit + if unit is None: + unit = default_unit if unit == 's': pass @@ -474,4 +484,3 @@ class ShowPlugin(obnamlib.ObnamPlugin): else: raise ValueError return ticks - diff --git a/obnamlib/plugins/verify_plugin.py b/obnamlib/plugins/verify_plugin.py index c2c56838..9662aee2 100644 --- a/obnamlib/plugins/verify_plugin.py +++ b/obnamlib/plugins/verify_plugin.py @@ -37,8 +37,8 @@ class Fail(obnamlib.ObnamError): class VerifyPlugin(obnamlib.ObnamPlugin): def enable(self): - self.app.add_subcommand('verify', self.verify, - arg_synopsis='[DIRECTORY]...') + self.app.add_subcommand( + 'verify', self.verify, arg_synopsis='[DIRECTORY]...') self.app.settings.integer( ['verify-randomly'], 'verify N files randomly from the backup ' @@ -53,8 +53,8 @@ class VerifyPlugin(obnamlib.ObnamPlugin): if len(self.app.settings['generation']) != 1: raise WrongNumberOfGenerationsForVerify() - logging.debug('verifying generation %s' % - self.app.settings['generation']) + logging.debug( + 'verifying generation %s' % self.app.settings['generation']) if not args: self.app.settings.require('root') args = self.app.settings['root'] @@ -178,7 +178,7 @@ class VerifyPlugin(obnamlib.ObnamPlugin): if v1 != v2: raise Fail( filename=filename, - reason='metadata change: %s (%s vs %s)' % + reason='metadata change: %s (%s vs %s)' % (field_name, repr(v1), repr(v2))) X(obnamlib.REPO_FILE_MODE, 'st_mode') diff --git a/obnamlib/plugins/vfs_local_plugin.py b/obnamlib/plugins/vfs_local_plugin.py index c0b00540..7a3fe34a 100644 --- a/obnamlib/plugins/vfs_local_plugin.py +++ b/obnamlib/plugins/vfs_local_plugin.py @@ -23,4 +23,3 @@ class VfsLocalPlugin(obnamlib.ObnamPlugin): def enable(self): self.app.fsf.register('', obnamlib.LocalFS) - diff --git a/obnamlib/repo_factory.py b/obnamlib/repo_factory.py index 6da6a05d..e8a40737 100644 --- a/obnamlib/repo_factory.py +++ b/obnamlib/repo_factory.py @@ -56,7 +56,7 @@ class RepositoryFactory(object): def get_implementation_classes(self): return self._implementations - def setup_hooks(self, hooks): # pragma: no cover + def setup_hooks(self, hooks): # pragma: no cover '''Create all repository related hooks. The factory instantiates all supported repository format classes. @@ -78,7 +78,7 @@ class RepositoryFactory(object): try: existing_format = self._read_existing_format(fs) - except EnvironmentError as e: # pragma: no cover + except EnvironmentError as e: # pragma: no cover if e.errno == errno.ENOENT: raise NotARepository(url=fs.baseurl) raise @@ -125,7 +125,7 @@ class RepositoryFactory(object): # set it to EEXIST. Life is wonderful. if e.errno in (errno.EEXIST, None): return self.open_existing_repo(fs, **kwargs) - raise # pragma: no cover + raise # pragma: no cover else: logging.debug('create_repo: metadata/format created ok') repo = self._open_repo(wanted_format, fs, kwargs) diff --git a/obnamlib/repo_factory_tests.py b/obnamlib/repo_factory_tests.py index e4411541..c5900942 100644 --- a/obnamlib/repo_factory_tests.py +++ b/obnamlib/repo_factory_tests.py @@ -66,7 +66,7 @@ class RepositoryFormatTests(unittest.TestCase): fs = obnamlib.LocalFS(self.repodir) factory = obnamlib.RepositoryFactory() self.assertRaises( - obnamlib.UnknownRepositoryFormatWanted, + obnamlib.UnknownRepositoryFormatWanted, factory.create_repo, fs, int) def test_create_repo_is_ok_with_existing_repo(self): diff --git a/obnamlib/repo_fs.py b/obnamlib/repo_fs.py index e7971ccd..e37fae62 100644 --- a/obnamlib/repo_fs.py +++ b/obnamlib/repo_fs.py @@ -49,12 +49,12 @@ class RepositoryFS(object): parts = filename.split(os.sep) if len(parts) >= 1: return parts[0] - else: # pragma: no cover + else: # pragma: no cover raise ToplevelIsFileError(filename=filename) def cat(self, filename, runfilters=True): data = self.fs.cat(filename) - if not runfilters: # pragma: no cover + if not runfilters: # pragma: no cover return data toplevel = self._get_toplevel(filename) return self.hooks.filter_read('repository-data', data, diff --git a/obnamlib/repo_interface.py b/obnamlib/repo_interface.py index 0081aeb3..59f92d95 100644 --- a/obnamlib/repo_interface.py +++ b/obnamlib/repo_interface.py @@ -73,6 +73,7 @@ _integer_keys = [ for i, name in enumerate(_string_keys + _integer_keys): globals()[name] = i + def _filter_integer_keys(prefix): return [globals()[name] for name in _integer_keys @@ -311,7 +312,7 @@ class RepositoryInterface(object): # Operations on the repository itself. @classmethod - def setup_hooks(self, hooks): # pragma: no cover + def setup_hooks(self, hooks): # pragma: no cover '''Create any hooks for this repository format. Note that this is a class method. @@ -692,7 +693,7 @@ class RepositoryInterface(object): ''' raise NotImplementedError() - def walk_generation(self, gen_id, dirname): # pragma: no cover + def walk_generation(self, gen_id, dirname): # pragma: no cover '''Like os.walk, but for a generation. This is a generator. Each return value is a pathname. @@ -858,7 +859,7 @@ class RepositoryInterface(object): raise NotImplementedError() -class RepositoryInterfaceTests(unittest.TestCase): # pragma: no cover +class RepositoryInterfaceTests(unittest.TestCase): # pragma: no cover '''Tests for implementations of RepositoryInterface. diff --git a/obnamlib/sizeparse.py b/obnamlib/sizeparse.py index 5cfd8e0e..fe9affee 100644 --- a/obnamlib/sizeparse.py +++ b/obnamlib/sizeparse.py @@ -74,4 +74,3 @@ class ByteSizeParser(object): raise UnitNameError(unit=unit) factor = self.units[unit.lower()] return int(size * factor) - diff --git a/obnamlib/sizeparse_tests.py b/obnamlib/sizeparse_tests.py index 2ba8a25f..a844e2b6 100644 --- a/obnamlib/sizeparse_tests.py +++ b/obnamlib/sizeparse_tests.py @@ -92,4 +92,3 @@ class ByteSizeParserTests(unittest.TestCase): text = 'asdf asdf' e = obnamlib.UnitNameError(unit=text) self.assert_(text in str(e), str(e)) - diff --git a/obnamlib/structurederror.py b/obnamlib/structurederror.py index ceddbb6c..581a762e 100644 --- a/obnamlib/structurederror.py +++ b/obnamlib/structurederror.py @@ -112,7 +112,7 @@ class StructuredError(Exception): else: dedented = (textwrap.dedent(lines[0]) + textwrap.dedent(''.join(lines[1:]))) - + try: formatted_msg = dedented.format(**self.kwargs) except KeyError as e: diff --git a/obnamlib/structurederror_tests.py b/obnamlib/structurederror_tests.py index c951313b..c2cd7a59 100644 --- a/obnamlib/structurederror_tests.py +++ b/obnamlib/structurederror_tests.py @@ -40,7 +40,6 @@ class EmptyError(obnamlib.StructuredError): msg = '' - class StructuredErrorTests(unittest.TestCase): def test_ids_differ_between_classes(self): diff --git a/obnamlib/vfs.py b/obnamlib/vfs.py index aeb19869..8ea51e3c 100644 --- a/obnamlib/vfs.py +++ b/obnamlib/vfs.py @@ -60,7 +60,6 @@ class LockFail(obnamlib.ObnamError): msg = "Couldn't create lock {lock_name}: {reason}" - class VirtualFileSystem(object): '''A virtual filesystem interface. @@ -93,7 +92,7 @@ class VirtualFileSystem(object): def log_stats(self): logging.debug( 'VFS: baseurl=%s read=%d written=%d' % - (self.baseurl, self.bytes_read, self.bytes_written)) + (self.baseurl, self.bytes_read, self.bytes_written)) def connect(self): '''Connect to filesystem.''' @@ -340,7 +339,7 @@ class VfsFactory: raise UnknownVFSError(url=url) -class VfsTests(object): # pragma: no cover +class VfsTests(object): # pragma: no cover '''Re-useable tests for VirtualFileSystem implementations. @@ -709,10 +708,13 @@ class VfsTests(object): # pragma: no cover def test_scan_tree_returns_nothing_if_listdir_fails(self): self.set_up_scan_tree() + def raiser(dirname): raise OSError(123, 'oops', dirname) + def logerror(msg): pass + self.fs.listdir2 = raiser result = list(self.fs.scan_tree(self.basepath, log=logerror)) self.assertEqual(len(result), 1) @@ -732,4 +734,3 @@ class VfsTests(object): # pragma: no cover result = list(self.fs.scan_tree(self.basepath, ok=ok)) pathnames = [pathname for pathname, st in result] self.assertEqual(sorted(pathnames), sorted(self.dirs)) - diff --git a/obnamlib/vfs_local.py b/obnamlib/vfs_local.py index a13cb1aa..bbb084fb 100644 --- a/obnamlib/vfs_local.py +++ b/obnamlib/vfs_local.py @@ -83,12 +83,13 @@ class LocalFS(obnamlib.VirtualFileSystem): # Do we have lchmod? self.got_lchmod = hasattr(os, 'lchmod') - def maybe_crash(self): # pragma: no cover + def maybe_crash(self): # pragma: no cover if self.crash_limit is not None: self.crash_counter += 1 if self.crash_counter >= self.crash_limit: - raise Exception('Crashing as requested after %d writes' % - self.crash_counter) + raise Exception( + 'Crashing as requested after %d writes' % + self.crash_counter) def reinit(self, baseurl, create=False): # We fake chdir so that it doesn't mess with the caller's @@ -97,7 +98,7 @@ class LocalFS(obnamlib.VirtualFileSystem): tracing.trace('baseurl=%s', baseurl) tracing.trace('create=%s', create) self.cwd = os.path.abspath(baseurl) - if os.path.exists(self.cwd): # pragma: no cover + if os.path.exists(self.cwd): # pragma: no cover if not os.path.isdir(self.cwd): raise RootIsNotADirectory(baseurl=baseurl) if not self.isdir('.'): @@ -105,7 +106,7 @@ class LocalFS(obnamlib.VirtualFileSystem): tracing.trace('creating %s', baseurl) try: os.mkdir(baseurl) - except OSError, e: # pragma: no cover + except OSError, e: # pragma: no cover # The directory might have been created concurrently # by someone else! if e.errno != errno.EEXIST: @@ -134,7 +135,7 @@ class LocalFS(obnamlib.VirtualFileSystem): raise obnamlib.LockFail( lock_name=lockname, reason='already exists') else: - raise # pragma: no cover + raise # pragma: no cover tracing.trace('got lockname=%s', lockname) tracing.trace('time=%f' % time.time()) self.our_locks.add(lockname) @@ -146,7 +147,7 @@ class LocalFS(obnamlib.VirtualFileSystem): tracing.trace('os.makedirs(%s)' % dirname) try: os.makedirs(dirname, mode=obnamlib.NEW_DIR_MODE) - except OSError as e: # pragma: no cover + except OSError as e: # pragma: no cover # This avoids a race condition: another Obnam process # may have created the directory between our check and # creation attempt. If so, we ignore it. As long as @@ -230,23 +231,23 @@ class LocalFS(obnamlib.VirtualFileSystem): if ret != 0: raise OSError(ret, os.strerror(ret), pathname) return obnamlib.Metadata( - st_dev=dev, - st_ino=ino, - st_mode=mode, - st_nlink=nlink, - st_uid=uid, - st_gid=gid, - st_rdev=rdev, - st_size=size, - st_blksize=blksize, - st_blocks=blocks, - st_atime_sec=atime_sec, - st_atime_nsec=atime_nsec, - st_mtime_sec=mtime_sec, - st_mtime_nsec=mtime_nsec, - st_ctime_sec=ctime_sec, - st_ctime_nsec=ctime_nsec - ) + st_dev=dev, + st_ino=ino, + st_mode=mode, + st_nlink=nlink, + st_uid=uid, + st_gid=gid, + st_rdev=rdev, + st_size=size, + st_blksize=blksize, + st_blocks=blocks, + st_atime_sec=atime_sec, + st_atime_nsec=atime_nsec, + st_mtime_sec=mtime_sec, + st_mtime_nsec=mtime_nsec, + st_ctime_sec=ctime_sec, + st_ctime_nsec=ctime_nsec + ) def get_username(self, uid): return pwd.getpwuid(uid)[0] @@ -254,7 +255,7 @@ class LocalFS(obnamlib.VirtualFileSystem): def get_groupname(self, gid): return grp.getgrgid(gid)[0] - def llistxattr(self, filename): # pragma: no cover + def llistxattr(self, filename): # pragma: no cover ret = obnamlib._obnam.llistxattr(self.join(filename)) if ret is None: raise MallocError(function='llistxattr') @@ -262,7 +263,7 @@ class LocalFS(obnamlib.VirtualFileSystem): raise OSError(ret, os.strerror(ret), filename) return [s for s in ret.split('\0') if s] - def lgetxattr(self, filename, attrname): # pragma: no cover + def lgetxattr(self, filename, attrname): # pragma: no cover ret = obnamlib._obnam.lgetxattr(self.join(filename), attrname) if ret is None: raise MallocError(function='llistxattr') @@ -270,13 +271,13 @@ class LocalFS(obnamlib.VirtualFileSystem): raise OSError(ret, os.strerror(ret), filename) return ret - def lsetxattr(self, filename, attrname, attrvalue): # pragma: no cover + def lsetxattr(self, filename, attrname, attrvalue): # pragma: no cover ret = obnamlib._obnam.lsetxattr(self.join(filename), attrname, attrvalue) if ret != 0: raise OSError(ret, os.strerror(ret), filename) - def lchown(self, pathname, uid, gid): # pragma: no cover + def lchown(self, pathname, uid, gid): # pragma: no cover tracing.trace('lchown %s %d %d', pathname, uid, gid) os.lchown(self.join(pathname), uid, gid) @@ -284,7 +285,7 @@ class LocalFS(obnamlib.VirtualFileSystem): # either has lchmod or doesn't, and accordingly either branch of # the if statement is taken, and the other branch shows up as not # being tested by the unit tests. - def chmod_symlink(self, pathname, mode): # pragma: no cover + def chmod_symlink(self, pathname, mode): # pragma: no cover tracing.trace('chmod_symlink %s %o', pathname, mode) if self.got_lchmod: os.lchmod(self.join(pathname), mode) @@ -330,9 +331,9 @@ class LocalFS(obnamlib.VirtualFileSystem): flags = fcntl.fcntl(f.fileno(), fcntl.F_GETFL) flags |= EXTRA_OPEN_FLAGS fcntl.fcntl(f.fileno(), fcntl.F_SETFL, flags) - except IOError, e: # pragma: no cover + except IOError, e: # pragma: no cover tracing.trace('fcntl F_SETFL failed: %s', repr(e)) - return f # ignore any problems setting flags + return f # ignore any problems setting flags tracing.trace('returning ok') return f @@ -377,7 +378,7 @@ class LocalFS(obnamlib.VirtualFileSystem): data = ''.join(chunks) return data - def write_file(self, pathname, contents): # pragma: no cover + def write_file(self, pathname, contents): # pragma: no cover tempname = self._create_tempfile(pathname) f = self.open(tempname, 'wb') f.write(contents) @@ -409,7 +410,7 @@ class LocalFS(obnamlib.VirtualFileSystem): for name in self.listdir(dirname): try: st = self.lstat(os.path.join(dirname, name)) - except OSError, e: # pragma: no cover + except OSError, e: # pragma: no cover st = e ino = -1 else: @@ -420,4 +421,3 @@ class LocalFS(obnamlib.VirtualFileSystem): # when backing up. result.sort() return [(name, st) for ino, name, st in result] - @@ -234,6 +234,7 @@ class Check(Command): runcmd(['./test-sftpfs']) def run_nitpick_checks(self): + self.check_with_pep8(sources) if os.path.exists('.git'): sources = self.find_all_source_files() self.check_sources_for_nitpicks(sources) @@ -241,6 +242,9 @@ class Check(Command): else: print "no .git, no nitpick for you" + def check_with_pep8(self): + cliapp.runcmd(['pep8', 'obnamlib'], stdout=None, stderr=None) + def check_sources_for_nitpicks(self, sources): cliapp.runcmd(['./nitpicker'] + sources, stdout=None, stderr=None) |