From 761c367c9d2a101fc92b225cc234f1da5d28f4bf Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Sun, 11 Jun 2017 15:38:43 +0300 Subject: Add: backupprogress.py unit tests --- obnamlib/backup_progress.py | 82 +++++++++++++++++---------- obnamlib/backup_progress_tests.py | 115 ++++++++++++++++++++++++++++++++++++++ without-tests | 1 - 3 files changed, 169 insertions(+), 29 deletions(-) create mode 100644 obnamlib/backup_progress_tests.py diff --git a/obnamlib/backup_progress.py b/obnamlib/backup_progress.py index 34509575..026dd604 100644 --- a/obnamlib/backup_progress.py +++ b/obnamlib/backup_progress.py @@ -30,12 +30,14 @@ class BackupProgress(object): self.started = None self.errors = False + self._now = time.time + self._ts = ts self._ts['current-file'] = '' self._ts['scanned-bytes'] = 0 self._ts['uploaded-bytes'] = 0 - if self.ttystatus_supports_multiline(): + if self.ttystatus_supports_multiline(): # pragma: no cover self._ts.format( '%ElapsedTime() Backing up: ' 'found %Counter(current-file) files, ' @@ -43,7 +45,7 @@ class BackupProgress(object): 'uploaded: %ByteSize(uploaded-bytes)\n' '%String(what)' ) - else: + else: # pragma: no cover self._ts.format( '%ElapsedTime() ' '%Counter(current-file) ' @@ -51,13 +53,16 @@ class BackupProgress(object): '%ByteSize(scanned-bytes) scanned: ' '%String(what)') + def set_time_func(self, now): + self._now = now + def ttystatus_supports_multiline(self): return hasattr(self._ts, 'start_new_line') - def clear(self): + def clear(self): # pragma: no cover self._ts.clear() - def finish(self): + def finish(self): # pragma: no cover self._ts.finish() def error(self, msg, exc=None): @@ -68,11 +73,11 @@ class BackupProgress(object): def what(self, what_what): if self.started is None: - self.started = time.time() + self.started = self._now() self._ts['what'] = what_what self._ts.flush() - def update_progress(self): + def update_progress(self): # pragma: no cover self._ts['not-shown'] = 'not shown' def update_progress_with_file(self, filename, metadata): @@ -88,68 +93,89 @@ class BackupProgress(object): self.uploaded_bytes += amount self._ts['uploaded-bytes'] = self.uploaded_bytes - def update_progress_with_removed_checkpoint(self, gen): + def update_progress_with_removed_checkpoint(self, gen): # pragma: no cover self._ts['checkpoint'] = gen - def report_stats(self, output, fs, quiet): - duration = time.time() - self.started - duration_string = obnamlib.humanise_duration(duration) + def compute_report(self, fs): + duration = self._now() - self.started + overhead = fs.bytes_written + fs.bytes_read - self.uploaded_bytes + speed = self.uploaded_bytes / float(duration) + + return { + 'duration': duration, + 'file-count': self.file_count, + 'backed-up-count': self.backed_up_count, + 'scanned-bytes': self.scanned_bytes, + 'uploaded-chunk-bytes': self.uploaded_bytes, + 'uploaded-total-bytes': fs.bytes_written, + 'downloaded-total-bytes': fs.bytes_read, + 'overhead-total-bytes': overhead, + 'effective-upload-speed': speed, + } + + def report_stats(self, output, fs, quiet, report=None): # pragma: no cover + if report is None: + report = self.compute_report(fs) + + duration_string = obnamlib.humanise_duration(report['duration']) chunk_amount, chunk_unit = obnamlib.humanise_size( - self.uploaded_bytes) + report['uploaded-total-bytes']) - ul_amount, ul_unit = obnamlib.humanise_size(fs.bytes_written) + ul_amount, ul_unit = obnamlib.humanise_size( + report['uploaded-total-bytes']) - dl_amount, dl_unit = obnamlib.humanise_size(fs.bytes_read) + dl_amount, dl_unit = obnamlib.humanise_size( + report['downloaded-total-bytes']) overhead_bytes = ( - fs.bytes_read + (fs.bytes_written - self.uploaded_bytes)) + report['downloaded-total-bytes'] + + (report['uploaded-total-bytes'] - report['uploaded-total-bytes'])) overhead_bytes = max(0, overhead_bytes) - overhead_amount, overhead_unit = obnamlib.humanise_size( - overhead_bytes) - if fs.bytes_written > 0: - overhead_percent = 100.0 * overhead_bytes / fs.bytes_written + overhead_amount, overhead_unit = obnamlib.humanise_size(overhead_bytes) + if report['uploaded-total-bytes'] > 0: + overhead_percent = 100.0 * overhead_bytes / report['uploaded-total-bytes'] else: overhead_percent = 0.0 speed_amount, speed_unit = obnamlib.humanise_speed( - self.uploaded_bytes, duration) + report['uploaded-total-bytes'], report['duration']) logging.info( 'Backup performance statistics:') logging.info( '* files found: %s', - self.file_count) + report['file-count']) logging.info( '* files backed up: %s', - self.backed_up_count) + report['backed-up-count']) logging.info( '* uploaded chunk data: %s bytes (%s %s)', - self.uploaded_bytes, chunk_amount, chunk_unit) + report['uploaded-total-bytes'], chunk_amount, chunk_unit) logging.info( '* total uploaded data (incl. metadata): %s bytes (%s %s)', - fs.bytes_written, ul_amount, ul_unit) + report['uploaded-total-bytes'], ul_amount, ul_unit) logging.info( '* total downloaded data (incl. metadata): %s bytes (%s %s)', - fs.bytes_read, dl_amount, dl_unit) + report['downloaded-total-bytes'], dl_amount, dl_unit) logging.info( '* transfer overhead: %s bytes (%s %s)', overhead_bytes, overhead_amount, overhead_unit) logging.info( '* duration: %s s (%s)', - duration, duration_string) + report['duration'], duration_string) logging.info( '* average speed: %s %s', speed_amount, speed_unit) scanned_amount, scanned_unit = obnamlib.humanise_size( - self.scanned_bytes) + report['scanned-bytes']) if not quiet: output.write( 'Backed up %d files (of %d found), containing %.1f %s.\n' % - (self.backed_up_count, - self.file_count, + (report['backed-up-count'], + report['file-count'], scanned_amount, scanned_unit)) output.write( diff --git a/obnamlib/backup_progress_tests.py b/obnamlib/backup_progress_tests.py new file mode 100644 index 00000000..c3d2dc01 --- /dev/null +++ b/obnamlib/backup_progress_tests.py @@ -0,0 +1,115 @@ +# Copyright (C) 2017 Lars Wirzenius +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + +import unittest + +import obnamlib + + +class BackupProgressTests(unittest.TestCase): + + def test_initialised_properly(self): + ts = DummyTerminalStatus() + bp = obnamlib.BackupProgress(ts) + self.assertEqual(bp.file_count, 0) + self.assertEqual(bp.backed_up_count, 0) + self.assertEqual(bp.uploaded_bytes, 0) + self.assertEqual(bp.scanned_bytes, 0) + self.assertEqual(bp.started, None) + self.assertFalse(bp.errors) + + def test_setting_what_records_started_time(self): + ts = DummyTerminalStatus() + bp = obnamlib.BackupProgress(ts) + bp.what('foo') + self.assertNotEqual(bp.started, None) + + def test_reporting_error_sets_errors_flag(self): + ts = DummyTerminalStatus() + bp = obnamlib.BackupProgress(ts) + bp.error('foo') + self.assertTrue(bp.errors) + + def test_files_get_counted(self): + ts = DummyTerminalStatus() + bp = obnamlib.BackupProgress(ts) + bp.update_progress_with_file('foo', None) + self.assertEqual(bp.file_count, 1) + + def test_scanned_bytes_get_counted(self): + ts = DummyTerminalStatus() + bp = obnamlib.BackupProgress(ts) + bp.update_progress_with_scanned(12765) + self.assertEqual(bp.scanned_bytes, 12765) + + def test_uploaded_bytes_get_counted(self): + ts = DummyTerminalStatus() + bp = obnamlib.BackupProgress(ts) + bp.update_progress_with_upload(12765) + self.assertEqual(bp.uploaded_bytes, 12765) + + def test_report_is_correct(self): + ts = DummyTerminalStatus() + bp = obnamlib.BackupProgress(ts) + bp.set_time_func(lambda: 0) + + # Pretend we run a little backup of one file that is 1000 + # bytes in length and we need to upload it all. + bp.what('backing up') + bp.update_progress_with_file('foo', None) + bp.update_progress_with_scanned(1000) + bp.backed_up_count += 1 + bp.update_progress_with_upload(1000) + + # Fake VFS for the backup. + fs = DummyFS() + fs.bytes_written = 2000 + fs.bytes_read = 1000 + + # Check that report is OK. + bp.set_time_func(lambda: 10) + r = bp.compute_report(fs) + self.assertEqual(r['duration'], 10) + self.assertEqual(r['file-count'], 1) + self.assertEqual(r['backed-up-count'], 1) + self.assertEqual(r['scanned-bytes'], 1000) + self.assertEqual(r['uploaded-chunk-bytes'], 1000) + self.assertEqual(r['uploaded-total-bytes'], 2000) + self.assertEqual(r['downloaded-total-bytes'], 1000) + self.assertEqual(r['overhead-total-bytes'], 2000 + 1000 - 1000) + self.assertEqual(r['effective-upload-speed'], 1000.0 / 10.0) + + +class DummyTerminalStatus(object): + + def format(self, format): + pass + + def flush(self): + pass + + def error(self, msg): + pass + + def __setitem__(self, key, value): + pass + + +class DummyFS(object): + + def __init__(self): + self.bytes_written = 0 + self.bytes_read = 0 diff --git a/without-tests b/without-tests index fc8b281c..f5979198 100644 --- a/without-tests +++ b/without-tests @@ -1,7 +1,6 @@ setup.py obnamlib/app.py -obnamlib/backup_progress.py obnamlib/defaults.py obnamlib/delegator.py obnamlib/fmt_6/__init__.py -- cgit v1.2.1