From 49f340f5c5168b03894436bfd3331a6bed48275d Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Sun, 2 Jan 2011 12:32:28 +0000 Subject: Add skeleton blackboxtest script. This is based on the one from obnam. --- blackboxtest | 241 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 241 insertions(+) create mode 100755 blackboxtest diff --git a/blackboxtest b/blackboxtest new file mode 100755 index 0000000..b9e5114 --- /dev/null +++ b/blackboxtest @@ -0,0 +1,241 @@ +#!/usr/bin/python +# +# Copyright (C) 2009, 2010 Lars Wirzenius +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +'''Run some black box tests for genbackupdata.''' + + +import logging +import os +import random +import re +import shutil +import stat +import subprocess +import sys +import tempfile +import traceback +import unittest + +import obnamlib + + +class GenbackupdataTestCase(unittest.TestCase): + + '''Base class for genbackupdata test cases. + + We use the unittest framework even though these are black box tests, + not unit tests. unittest makes implementation of these black box + tests convenient, even though that might not be true for all black + box tests. + + This base class provides a fresh environment for each test, and + cleans up afterwards. It provides helpers for doing the usual + backup operations, and for verifyting results. + + ''' + + def setUp(self): + self.tempdir = tempfile.mkdtemp() + self.setUpHook() + + def setUpHook(self): + pass + + def tearDown(self): + self.tearDownHook() + shutil.rmtree(self.tempdir) + + def tearDownHook(self): + pass + + def mkdir(self, dirname): + abs_dirname = os.path.join(self.tempdir, dirname) + os.makedirs(abs_dirname) + return abs_dirname + + def runcmd(self, argv, stderr_ignore=None): + '''Run an external command. + + If the command fails (non-zero exit), raise an exception. + + If stderr_ignore is not None, it must be a string with a + regexp for lines in stderr to ignore. + + ''' + + logging.debug('executing %s' % argv) + + p = subprocess.Popen(argv, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + stdout, stderr = p.communicate() + if stderr_ignore: + lines = [line for line in stderr.splitlines() + if not re.match(stderr_ignore, line)] + stderr = ''.join(lines) + sys.stderr.write(stderr) + if p.returncode != 0: + raise subprocess.CalledProcessError(p.returncode, argv) + return stdout + + def genbackupdata(self, args, stderr_ignore=None): + '''Run genbackupdata, with some default arguments.''' + return self.runcmd(['./genbackupdata'] + + args, stderr_ignore=stderr_ignore) + + def create_file(self, dirname, relative, contents): + '''Create a new file with the desired contents.''' + + pathname = os.path.join(dirname, relative) + logging.debug('creating file %s' % pathname) + f = open(pathname, 'w') + f.write(contents) + f.close() + + def remove_file(self, root, relative): + '''Remove a file.''' + + pathname = os.path.join(root, relative) + logging.debug('removing file %s' % pathname) + os.remove(pathname) + + def create_dir(self, root, pathname): + '''Create a new directory, return name.''' + fullname = os.path.join(root, pathname) + logging.debug('mkdir %s' % fullname) + os.makedirs(fullname) + return fullname + + def get_info(self, root, pathname): + '''Get the information about a given file. + + Return a tuple (relativepath, stat) where relativepath is the + path relative to root, and stat is the result of os.lstat. + + ''' + + root_base = os.path.basename(root) + del_prefix = root[:-len(root_base)] + if pathname == root: + return None + assert pathname.startswith(root + os.sep), (pathname, root) + return pathname[len(root + os.sep):], os.lstat(pathname) + + def find_everything(self, root): + '''Find all filesystem objects inside a directory tree. + + Return list of (pathname, stat) tuples. The pathname will be + relative to the root of the directory tree. The stat tuples + will be the result of os.lstat for each pathname. + + ''' + + result = [] + for dirname, dirnames, filenames in os.walk(root): + result.append(self.get_info(root, dirname)) + for filename in filenames: + pathname = os.path.join(dirname, filename) + result.append(self.get_info(root, pathname)) + return [x for x in result if x] + + def open_store(self): + '''Open the store.''' + + fs = obnamlib.LocalFS(self.store) + s = obnamlib.Store(fs, obnamlib.DEFAULT_NODE_SIZE, + obnamlib.DEFAULT_UPLOAD_QUEUE_SIZE, + obnamlib.DEFAULT_LRU_SIZE) + s.open_client(self.client_name) + return s + + def assert_equal_stat_fields(self, filename, stat1, stat2, fieldname): + field1 = getattr(stat1, fieldname) + field2 = getattr(stat2, fieldname) + self.assertEqual(field1, field2, + '%s stat field %s difference: %s vs %s' % + (filename, fieldname, repr(field1), repr(field2))) + + def assert_same_stat(self, name, stat1, stat2): + '''Are two stat results effectively identical?''' + + class Fake(object): + + def __init__(self, stat_result): + self.st = stat_result + + def __getattr__(self, name): + if name == 'st_mtime': + return int(getattr(self.st, name)) + else: + return getattr(self.st, name) + + self.assert_equal_stat_fields(name, stat1, stat2, 'st_blocks') + self.assert_equal_stat_fields(name, stat1, stat2, 'st_gid') + self.assert_equal_stat_fields(name, stat1, stat2, 'st_mode') + self.assert_equal_stat_fields(name, Fake(stat1), Fake(stat2), 'st_mtime') + self.assert_equal_stat_fields(name, stat1, stat2, 'st_nlink') + self.assert_equal_stat_fields(name, stat1, stat2, 'st_size') + self.assert_equal_stat_fields(name, stat1, stat2, 'st_uid') + + def assert_same_contents(self, relative, root1, root2): + '''Verify that file contents has been restored correctly.''' + + path1 = os.path.join(root1, relative) + path2 = os.path.join(root2, relative) + + self.assertFilesEqual(path1, path2) + + def assertFileExists(self, path): + self.assert_(os.path.exists(path), '%s does not exist' % path) + + def assertIsRegularFile(self, path): + self.assert_(os.path.isfile(path), '%s is not a regular file' % path) + + def assertFilesEqual(self, path1, path2): + '''Verify that file contents are equal.''' + + self.assertFileExists(path1) + self.assertFileExists(path2) + self.assertIsRegularFile(path1) + self.assertIsRegularFile(path2) + + f1 = open(path1, 'r') + f2 = open(path2, 'r') + + data1 = f1.read() + data2 = f2.read() + + f1.close() + f2.close() + + self.assertEqual(data1, data2, + 'contents of %s and %s differ' % (path1, path2)) + + +class GenbackupdataTests(GenbackupdataTestCase): + + def test_returns_success_with_help_option(self): + self.genbackupdata(['--help']) + self.assertTrue(True) + + +if __name__ == '__main__': + logging.basicConfig(filename='blackboxtest.log', + level=logging.DEBUG, + format='%(levelname)s: %(message)s') + unittest.main() -- cgit v1.2.1 From b2975da0a87cb093c57f4f0ae04b6701ba7de36d Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Sun, 2 Jan 2011 12:33:47 +0000 Subject: Add blackboxtest to "make check". --- Makefile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 79cbdcd..4aea314 100644 --- a/Makefile +++ b/Makefile @@ -5,9 +5,11 @@ check: python-coverage -x tests.py python-coverage -r -m -o /usr,/var | \ awk '{ print } /^TOTAL/ && $$2 != $$3 {exit 1}' + ./blackboxtest clean: - rm -rf *.pyc *.pyo build dist MANIFEST + rm -rf *.pyc *.pyo build dist MANIFEST + rm -f blackboxtest.log blackboxtest-genbackupdata.log dist: python setup.py sdist -- cgit v1.2.1 From 42b396f0d124c710b0eef4568e7fe65f6ccab869 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Sun, 2 Jan 2011 12:44:31 +0000 Subject: Ignore blackobxtest.log. --- .bzrignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.bzrignore b/.bzrignore index 6350e98..1cbec07 100644 --- a/.bzrignore +++ b/.bzrignore @@ -1 +1,2 @@ .coverage +blackboxtest.log -- cgit v1.2.1 From 5b39572858adfa97bae6b1f53d0de0f8cf451e4d Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Sun, 2 Jan 2011 12:48:25 +0000 Subject: Add test that right amount of data gets created. --- blackboxtest | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/blackboxtest b/blackboxtest index b9e5114..ee5ef30 100755 --- a/blackboxtest +++ b/blackboxtest @@ -32,8 +32,6 @@ import tempfile import traceback import unittest -import obnamlib - class GenbackupdataTestCase(unittest.TestCase): @@ -64,6 +62,9 @@ class GenbackupdataTestCase(unittest.TestCase): def tearDownHook(self): pass + def path(self, *relatives): + return os.path.join(self.tempdir, *relatives) + def mkdir(self, dirname): abs_dirname = os.path.join(self.tempdir, dirname) os.makedirs(abs_dirname) @@ -153,15 +154,17 @@ class GenbackupdataTestCase(unittest.TestCase): result.append(self.get_info(root, pathname)) return [x for x in result if x] - def open_store(self): - '''Open the store.''' - - fs = obnamlib.LocalFS(self.store) - s = obnamlib.Store(fs, obnamlib.DEFAULT_NODE_SIZE, - obnamlib.DEFAULT_UPLOAD_QUEUE_SIZE, - obnamlib.DEFAULT_LRU_SIZE) - s.open_client(self.client_name) - return s + def apparent_size(self, root): + '''Return sum of length of regular files in directory, recursively.''' + + size = 0 + for dirname, subdirs, filenames in os.walk(self.path(root)): + for filename in filenames: + pathname = self.path(dirname, filename) + st = os.lstat(pathname) + if stat.S_ISREG(st.st_mode): + size += st.st_size + return size def assert_equal_stat_fields(self, filename, stat1, stat2, fieldname): field1 = getattr(stat1, fieldname) @@ -232,6 +235,11 @@ class GenbackupdataTests(GenbackupdataTestCase): def test_returns_success_with_help_option(self): self.genbackupdata(['--help']) self.assertTrue(True) + + def test_creates_requested_amount_of_data(self): + kilobytes = 4096 + self.genbackupdata([self.path('data'), '--create=%dk' % kilobytes]) + self.assertEqual(self.apparent_size('data'), kilobytes * 1024) if __name__ == '__main__': -- cgit v1.2.1 From 310a017ff0ced22f4fd555450f1414b2b56ff8c4 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Sun, 2 Jan 2011 12:48:46 +0000 Subject: Tweak size to not be a multiple of 4096. --- blackboxtest | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/blackboxtest b/blackboxtest index ee5ef30..7d07136 100755 --- a/blackboxtest +++ b/blackboxtest @@ -237,7 +237,7 @@ class GenbackupdataTests(GenbackupdataTestCase): self.assertTrue(True) def test_creates_requested_amount_of_data(self): - kilobytes = 4096 + kilobytes = 12765 self.genbackupdata([self.path('data'), '--create=%dk' % kilobytes]) self.assertEqual(self.apparent_size('data'), kilobytes * 1024) -- cgit v1.2.1 From 7e626a7c7a4387577b39e13d1fdeb2688e0d27ab Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Sun, 2 Jan 2011 12:49:21 +0000 Subject: Tweak size to be a multiple of bytes. --- blackboxtest | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/blackboxtest b/blackboxtest index 7d07136..b1d3ccd 100755 --- a/blackboxtest +++ b/blackboxtest @@ -237,9 +237,9 @@ class GenbackupdataTests(GenbackupdataTestCase): self.assertTrue(True) def test_creates_requested_amount_of_data(self): - kilobytes = 12765 - self.genbackupdata([self.path('data'), '--create=%dk' % kilobytes]) - self.assertEqual(self.apparent_size('data'), kilobytes * 1024) + bytes = 12765 + self.genbackupdata([self.path('data'), '--create=%d' % bytes]) + self.assertEqual(self.apparent_size('data'), bytes) if __name__ == '__main__': -- cgit v1.2.1 From bb4089be305a86216b63c11f508445b129a57716 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Sun, 2 Jan 2011 13:03:46 +0000 Subject: Remove code. It is going to get rewritten. It does not pass even simple black box testing at the moment. --- blackboxtest | 37 ++ createfiles-ext3-with-dirindex-0B.png | Bin 5518 -> 0 bytes createfiles-ext3-with-dirindex-16384B.png | Bin 4634 -> 0 bytes createfiles-ext3-with-dirindex-4096B.png | Bin 6216 -> 0 bytes createfiles-ext3-with-dirindex-8192B.png | Bin 5567 -> 0 bytes createfiles-plot.sh | 51 --- createfiles.output-ext3-with-dirindex | 528 ------------------------ createfiles.py | 162 -------- genbackupdata.py | 660 ------------------------------ tests.py | 565 ------------------------- 10 files changed, 37 insertions(+), 1966 deletions(-) delete mode 100644 createfiles-ext3-with-dirindex-0B.png delete mode 100644 createfiles-ext3-with-dirindex-16384B.png delete mode 100644 createfiles-ext3-with-dirindex-4096B.png delete mode 100644 createfiles-ext3-with-dirindex-8192B.png delete mode 100755 createfiles-plot.sh delete mode 100644 createfiles.output-ext3-with-dirindex delete mode 100755 createfiles.py delete mode 100644 genbackupdata.py delete mode 100755 tests.py diff --git a/blackboxtest b/blackboxtest index b1d3ccd..e54519e 100755 --- a/blackboxtest +++ b/blackboxtest @@ -20,6 +20,7 @@ '''Run some black box tests for genbackupdata.''' +import hashlib import logging import os import random @@ -166,6 +167,33 @@ class GenbackupdataTestCase(unittest.TestCase): size += st.st_size return size + def checksum(self, pathname): + '''Return MD5 checksum for contents of a file.''' + s = hashlib.new('md5') + f = open(pathname, 'rb') + while True: + data = f.read(64*1024) + if not data: + break + s.update(data) + f.close() + return s.hexdigest() + + def checksums(self, root): + '''Return sorted list of (pathname, checksum) pairs for reg. files.''' + result = [] + prefix = self.path(root) + os.sep + for dirname, subdirs, filenames in os.walk(self.path(root)): + for filename in filenames: + pathname = self.path(dirname, filename) + st = os.lstat(pathname) + if stat.S_ISREG(st.st_mode): + assert pathname.startswith(prefix) + relative = pathname[len(prefix):] + result.append((relative, self.checksum(pathname))) + result.sort() + return result + def assert_equal_stat_fields(self, filename, stat1, stat2, fieldname): field1 = getattr(stat1, fieldname) field2 = getattr(stat2, fieldname) @@ -241,6 +269,15 @@ class GenbackupdataTests(GenbackupdataTestCase): self.genbackupdata([self.path('data'), '--create=%d' % bytes]) self.assertEqual(self.apparent_size('data'), bytes) + def test_creates_same_data_every_time(self): + size = '10m' # big enough to allow both ample text and binary data + self.genbackupdata([self.path('data1'), '--create', size]) + self.genbackupdata([self.path('data2'), '--create', size]) + sums1 = self.checksums('data1') + sums2 = self.checksums('data2') + self.assertEqual(len(sums1), len(sums2)) + for n in range(1, len(sums1)): + self.assertEqual(sums1[:n], sums2[:n]) if __name__ == '__main__': logging.basicConfig(filename='blackboxtest.log', diff --git a/createfiles-ext3-with-dirindex-0B.png b/createfiles-ext3-with-dirindex-0B.png deleted file mode 100644 index 102801f..0000000 Binary files a/createfiles-ext3-with-dirindex-0B.png and /dev/null differ diff --git a/createfiles-ext3-with-dirindex-16384B.png b/createfiles-ext3-with-dirindex-16384B.png deleted file mode 100644 index eb9e863..0000000 Binary files a/createfiles-ext3-with-dirindex-16384B.png and /dev/null differ diff --git a/createfiles-ext3-with-dirindex-4096B.png b/createfiles-ext3-with-dirindex-4096B.png deleted file mode 100644 index 6682c3b..0000000 Binary files a/createfiles-ext3-with-dirindex-4096B.png and /dev/null differ diff --git a/createfiles-ext3-with-dirindex-8192B.png b/createfiles-ext3-with-dirindex-8192B.png deleted file mode 100644 index e63387d..0000000 Binary files a/createfiles-ext3-with-dirindex-8192B.png and /dev/null differ diff --git a/createfiles-plot.sh b/createfiles-plot.sh deleted file mode 100755 index c4a7e19..0000000 --- a/createfiles-plot.sh +++ /dev/null @@ -1,51 +0,0 @@ -#!/bin/sh -# -# A small script to generate graphs from createfiles.py output. There will -# be one createfiles-$SIZE.png file for each size of output, graphing the -# various algorithms at various sizes. - -set -e - -input="$1" - -algos=$(awk '$2 == "files/s" { print $3 }' "$input" | sort -u) -sizes=$(awk '/^Measuring/ { print $(NF-1) }' "$input" | sort -un) - -for size in $sizes -do - plot="" - - for x in $algos - do - awk -vx=$x -vsize=$size ' - /^Measuring/ { - if ($(NF-1) == size) { - n = $6 - ok = 1 - } else { - ok = 0 - } - } - ok && $3 == x { - print n, $1 >> x ".dat" - }' "$input" - if [ "$plot" = "" ] - then - plot="plot \"$x.dat\" with lines" - else - plot="$plot, \"$x.dat\" with lines" - fi - done - - gnuplot < "createfiles-$size.png" -set terminal png -set title "Size $size" -$plot -eof - - for x in $algos - do - rm "$x.dat" - done - -done diff --git a/createfiles.output-ext3-with-dirindex b/createfiles.output-ext3-with-dirindex deleted file mode 100644 index 72e9a46..0000000 --- a/createfiles.output-ext3-with-dirindex +++ /dev/null @@ -1,528 +0,0 @@ -Measuring 7 functions for creating 10 files of 0 bytes - - 41487 files/s onedir All files in one directory - 34721 files/s subdirsof100 Files in subdirs, each with up to 100 files - 35727 files/s subdirsof250 Files in subdirs, each with up to 250 files - 34606 files/s subdirsof1000 Files in subdirs, each with up to 1000 files - 15082 files/s subsubdirsof100 Files in two-level tree, up to 100 files each - 15169 files/s subsubdirsof250 Files in two-level tree, up to 250 files each - 15219 files/s subsubdirsof1000 Files in two-level tree, up to 1000 files each - - -Measuring 7 functions for creating 1000 files of 0 bytes - - 38065 files/s onedir All files in one directory - 44621 files/s subdirsof100 Files in subdirs, each with up to 100 files - 41822 files/s subdirsof250 Files in subdirs, each with up to 250 files - 36725 files/s subdirsof1000 Files in subdirs, each with up to 1000 files - 16732 files/s subsubdirsof100 Files in two-level tree, up to 100 files each - 16327 files/s subsubdirsof250 Files in two-level tree, up to 250 files each - 15492 files/s subsubdirsof1000 Files in two-level tree, up to 1000 files each - - -Measuring 7 functions for creating 2000 files of 0 bytes - - 37583 files/s onedir All files in one directory - 43994 files/s subdirsof100 Files in subdirs, each with up to 100 files - 41007 files/s subdirsof250 Files in subdirs, each with up to 250 files - 36047 files/s subdirsof1000 Files in subdirs, each with up to 1000 files - 16692 files/s subsubdirsof100 Files in two-level tree, up to 100 files each - 16200 files/s subsubdirsof250 Files in two-level tree, up to 250 files each - 15404 files/s subsubdirsof1000 Files in two-level tree, up to 1000 files each - - -Measuring 7 functions for creating 4000 files of 0 bytes - - 36298 files/s onedir All files in one directory - 43217 files/s subdirsof100 Files in subdirs, each with up to 100 files - 40305 files/s subdirsof250 Files in subdirs, each with up to 250 files - 35675 files/s subdirsof1000 Files in subdirs, each with up to 1000 files - 16520 files/s subsubdirsof100 Files in two-level tree, up to 100 files each - 16102 files/s subsubdirsof250 Files in two-level tree, up to 250 files each - 15307 files/s subsubdirsof1000 Files in two-level tree, up to 1000 files each - - -Measuring 7 functions for creating 8000 files of 0 bytes - - 35039 files/s onedir All files in one directory - 42752 files/s subdirsof100 Files in subdirs, each with up to 100 files - 39964 files/s subdirsof250 Files in subdirs, each with up to 250 files - 35321 files/s subdirsof1000 Files in subdirs, each with up to 1000 files - 15394 files/s subsubdirsof100 Files in two-level tree, up to 100 files each - 15941 files/s subsubdirsof250 Files in two-level tree, up to 250 files each - 15280 files/s subsubdirsof1000 Files in two-level tree, up to 1000 files each - - -Measuring 7 functions for creating 16000 files of 0 bytes - - 34941 files/s onedir All files in one directory - 42378 files/s subdirsof100 Files in subdirs, each with up to 100 files - 39506 files/s subdirsof250 Files in subdirs, each with up to 250 files - 35237 files/s subdirsof1000 Files in subdirs, each with up to 1000 files - 14456 files/s subsubdirsof100 Files in two-level tree, up to 100 files each - 15912 files/s subsubdirsof250 Files in two-level tree, up to 250 files each - 15183 files/s subsubdirsof1000 Files in two-level tree, up to 1000 files each - - -Measuring 7 functions for creating 32000 files of 0 bytes - - 34194 files/s onedir All files in one directory - 42020 files/s subdirsof100 Files in subdirs, each with up to 100 files - 38924 files/s subdirsof250 Files in subdirs, each with up to 250 files - 35041 files/s subdirsof1000 Files in subdirs, each with up to 1000 files - 16385 files/s subsubdirsof100 Files in two-level tree, up to 100 files each - 15051 files/s subsubdirsof250 Files in two-level tree, up to 250 files each - 15164 files/s subsubdirsof1000 Files in two-level tree, up to 1000 files each - - -Measuring 7 functions for creating 64000 files of 0 bytes - - 29784 files/s onedir All files in one directory - 41785 files/s subdirsof100 Files in subdirs, each with up to 100 files - 38778 files/s subdirsof250 Files in subdirs, each with up to 250 files - 31200 files/s subdirsof1000 Files in subdirs, each with up to 1000 files - 15282 files/s subsubdirsof100 Files in two-level tree, up to 100 files each - 14932 files/s subsubdirsof250 Files in two-level tree, up to 250 files each - 14309 files/s subsubdirsof1000 Files in two-level tree, up to 1000 files each - - -Measuring 7 functions for creating 128000 files of 0 bytes - - 31150 files/s onedir All files in one directory - 30689 files/s subdirsof100 Files in subdirs, each with up to 100 files - 38397 files/s subdirsof250 Files in subdirs, each with up to 250 files - 30766 files/s subdirsof1000 Files in subdirs, each with up to 1000 files - 14921 files/s subsubdirsof100 Files in two-level tree, up to 100 files each - 15011 files/s subsubdirsof250 Files in two-level tree, up to 250 files each - 14034 files/s subsubdirsof1000 Files in two-level tree, up to 1000 files each - - -Measuring 7 functions for creating 256000 files of 0 bytes - - 29211 files/s onedir All files in one directory - 19747 files/s subdirsof100 Files in subdirs, each with up to 100 files - 35337 files/s subdirsof250 Files in subdirs, each with up to 250 files - 28877 files/s subdirsof1000 Files in subdirs, each with up to 1000 files - 14432 files/s subsubdirsof100 Files in two-level tree, up to 100 files each - 14121 files/s subsubdirsof250 Files in two-level tree, up to 250 files each - 14065 files/s subsubdirsof1000 Files in two-level tree, up to 1000 files each - - -Measuring 7 functions for creating 512000 files of 0 bytes - - 24449 files/s onedir All files in one directory - 21297 files/s subdirsof100 Files in subdirs, each with up to 100 files - 24421 files/s subdirsof250 Files in subdirs, each with up to 250 files - 25494 files/s subdirsof1000 Files in subdirs, each with up to 1000 files - 13814 files/s subsubdirsof100 Files in two-level tree, up to 100 files each - 13897 files/s subsubdirsof250 Files in two-level tree, up to 250 files each - 13321 files/s subsubdirsof1000 Files in two-level tree, up to 1000 files each - - -Measuring 7 functions for creating 1024000 files of 0 bytes - - 20239 files/s onedir All files in one directory - 20728 files/s subdirsof100 Files in subdirs, each with up to 100 files - 23730 files/s subdirsof250 Files in subdirs, each with up to 250 files - 26271 files/s subdirsof1000 Files in subdirs, each with up to 1000 files - 13227 files/s subsubdirsof100 Files in two-level tree, up to 100 files each - 13837 files/s subsubdirsof250 Files in two-level tree, up to 250 files each - 12941 files/s subsubdirsof1000 Files in two-level tree, up to 1000 files each - - -Measuring 7 functions for creating 10 files of 4096 bytes - - 23630 files/s onedir All files in one directory - 21654 files/s subdirsof100 Files in subdirs, each with up to 100 files - 21687 files/s subdirsof250 Files in subdirs, each with up to 250 files - 21834 files/s subdirsof1000 Files in subdirs, each with up to 1000 files - 11364 files/s subsubdirsof100 Files in two-level tree, up to 100 files each - 11822 files/s subsubdirsof250 Files in two-level tree, up to 250 files each - 11808 files/s subsubdirsof1000 Files in two-level tree, up to 1000 files each - - -Measuring 7 functions for creating 1000 files of 4096 bytes - - 21001 files/s onedir All files in one directory - 24392 files/s subdirsof100 Files in subdirs, each with up to 100 files - 23646 files/s subdirsof250 Files in subdirs, each with up to 250 files - 21926 files/s subdirsof1000 Files in subdirs, each with up to 1000 files - 12504 files/s subsubdirsof100 Files in two-level tree, up to 100 files each - 12236 files/s subsubdirsof250 Files in two-level tree, up to 250 files each - 11767 files/s subsubdirsof1000 Files in two-level tree, up to 1000 files each - - -Measuring 7 functions for creating 2000 files of 4096 bytes - - 21391 files/s onedir All files in one directory - 24029 files/s subdirsof100 Files in subdirs, each with up to 100 files - 21909 files/s subdirsof250 Files in subdirs, each with up to 250 files - 20476 files/s subdirsof1000 Files in subdirs, each with up to 1000 files - 11577 files/s subsubdirsof100 Files in two-level tree, up to 100 files each - 11167 files/s subsubdirsof250 Files in two-level tree, up to 250 files each - 10509 files/s subsubdirsof1000 Files in two-level tree, up to 1000 files each - - -Measuring 7 functions for creating 4000 files of 4096 bytes - - 21258 files/s onedir All files in one directory - 23132 files/s subdirsof100 Files in subdirs, each with up to 100 files - 21925 files/s subdirsof250 Files in subdirs, each with up to 250 files - 20587 files/s subdirsof1000 Files in subdirs, each with up to 1000 files - 12205 files/s subsubdirsof100 Files in two-level tree, up to 100 files each - 11882 files/s subsubdirsof250 Files in two-level tree, up to 250 files each - 11340 files/s subsubdirsof1000 Files in two-level tree, up to 1000 files each - - -Measuring 7 functions for creating 8000 files of 4096 bytes - - 19734 files/s onedir All files in one directory - 18496 files/s subdirsof100 Files in subdirs, each with up to 100 files - 12523 files/s subdirsof250 Files in subdirs, each with up to 250 files - 20460 files/s subdirsof1000 Files in subdirs, each with up to 1000 files - 12078 files/s subsubdirsof100 Files in two-level tree, up to 100 files each - 11650 files/s subsubdirsof250 Files in two-level tree, up to 250 files each - 11207 files/s subsubdirsof1000 Files in two-level tree, up to 1000 files each - - -Measuring 7 functions for creating 16000 files of 4096 bytes - - 20482 files/s onedir All files in one directory - 20108 files/s subdirsof100 Files in subdirs, each with up to 100 files - 19301 files/s subdirsof250 Files in subdirs, each with up to 250 files - 20069 files/s subdirsof1000 Files in subdirs, each with up to 1000 files - 10945 files/s subsubdirsof100 Files in two-level tree, up to 100 files each - 10651 files/s subsubdirsof250 Files in two-level tree, up to 250 files each - 10582 files/s subsubdirsof1000 Files in two-level tree, up to 1000 files each - - -Measuring 7 functions for creating 32000 files of 4096 bytes - - 9755 files/s onedir All files in one directory - 6502 files/s subdirsof100 Files in subdirs, each with up to 100 files - 18511 files/s subdirsof250 Files in subdirs, each with up to 250 files - 18066 files/s subdirsof1000 Files in subdirs, each with up to 1000 files - 10939 files/s subsubdirsof100 Files in two-level tree, up to 100 files each - 9046 files/s subsubdirsof250 Files in two-level tree, up to 250 files each - 10583 files/s subsubdirsof1000 Files in two-level tree, up to 1000 files each - - -Measuring 7 functions for creating 64000 files of 4096 bytes - - 3890 files/s onedir All files in one directory - 5069 files/s subdirsof100 Files in subdirs, each with up to 100 files - 14446 files/s subdirsof250 Files in subdirs, each with up to 250 files - 18703 files/s subdirsof1000 Files in subdirs, each with up to 1000 files - 10159 files/s subsubdirsof100 Files in two-level tree, up to 100 files each - 9593 files/s subsubdirsof250 Files in two-level tree, up to 250 files each - 9773 files/s subsubdirsof1000 Files in two-level tree, up to 1000 files each - - -Measuring 7 functions for creating 128000 files of 4096 bytes - - 5949 files/s onedir All files in one directory - 6611 files/s subdirsof100 Files in subdirs, each with up to 100 files - 15273 files/s subdirsof250 Files in subdirs, each with up to 250 files - 5965 files/s subdirsof1000 Files in subdirs, each with up to 1000 files - 9935 files/s subsubdirsof100 Files in two-level tree, up to 100 files each - 9203 files/s subsubdirsof250 Files in two-level tree, up to 250 files each - 6478 files/s subsubdirsof1000 Files in two-level tree, up to 1000 files each - - -Measuring 7 functions for creating 256000 files of 4096 bytes - - 4751 files/s onedir All files in one directory - 8139 files/s subdirsof100 Files in subdirs, each with up to 100 files - 6566 files/s subdirsof250 Files in subdirs, each with up to 250 files - 10514 files/s subdirsof1000 Files in subdirs, each with up to 1000 files - 7560 files/s subsubdirsof100 Files in two-level tree, up to 100 files each - 7301 files/s subsubdirsof250 Files in two-level tree, up to 250 files each - 9719 files/s subsubdirsof1000 Files in two-level tree, up to 1000 files each - - -Measuring 7 functions for creating 512000 files of 4096 bytes - - 3535 files/s onedir All files in one directory - 3437 files/s subdirsof100 Files in subdirs, each with up to 100 files - 4330 files/s subdirsof250 Files in subdirs, each with up to 250 files - 6881 files/s subdirsof1000 Files in subdirs, each with up to 1000 files - 4924 files/s subsubdirsof100 Files in two-level tree, up to 100 files each - 4542 files/s subsubdirsof250 Files in two-level tree, up to 250 files each - 7061 files/s subsubdirsof1000 Files in two-level tree, up to 1000 files each - - -Measuring 7 functions for creating 1024000 files of 4096 bytes - - 2710 files/s onedir All files in one directory - 4207 files/s subdirsof100 Files in subdirs, each with up to 100 files - 4564 files/s subdirsof250 Files in subdirs, each with up to 250 files - 5296 files/s subdirsof1000 Files in subdirs, each with up to 1000 files - 4537 files/s subsubdirsof100 Files in two-level tree, up to 100 files each - 4645 files/s subsubdirsof250 Files in two-level tree, up to 250 files each - 5819 files/s subsubdirsof1000 Files in two-level tree, up to 1000 files each - - -Measuring 7 functions for creating 10 files of 8192 bytes - - 19373 files/s onedir All files in one directory - 18316 files/s subdirsof100 Files in subdirs, each with up to 100 files - 18658 files/s subdirsof250 Files in subdirs, each with up to 250 files - 17765 files/s subdirsof1000 Files in subdirs, each with up to 1000 files - 10673 files/s subsubdirsof100 Files in two-level tree, up to 100 files each - 10719 files/s subsubdirsof250 Files in two-level tree, up to 250 files each - 10763 files/s subsubdirsof1000 Files in two-level tree, up to 1000 files each - - -Measuring 7 functions for creating 1000 files of 8192 bytes - - 18659 files/s onedir All files in one directory - 19490 files/s subdirsof100 Files in subdirs, each with up to 100 files - 17477 files/s subdirsof250 Files in subdirs, each with up to 250 files - 17948 files/s subdirsof1000 Files in subdirs, each with up to 1000 files - 11172 files/s subsubdirsof100 Files in two-level tree, up to 100 files each - 10809 files/s subsubdirsof250 Files in two-level tree, up to 250 files each - 10212 files/s subsubdirsof1000 Files in two-level tree, up to 1000 files each - - -Measuring 7 functions for creating 2000 files of 8192 bytes - - 18249 files/s onedir All files in one directory - 19500 files/s subdirsof100 Files in subdirs, each with up to 100 files - 18248 files/s subdirsof250 Files in subdirs, each with up to 250 files - 17029 files/s subdirsof1000 Files in subdirs, each with up to 1000 files - 10853 files/s subsubdirsof100 Files in two-level tree, up to 100 files each - 10562 files/s subsubdirsof250 Files in two-level tree, up to 250 files each - 10424 files/s subsubdirsof1000 Files in two-level tree, up to 1000 files each - - -Measuring 7 functions for creating 4000 files of 8192 bytes - - 17060 files/s onedir All files in one directory - 18711 files/s subdirsof100 Files in subdirs, each with up to 100 files - 17258 files/s subdirsof250 Files in subdirs, each with up to 250 files - 16282 files/s subdirsof1000 Files in subdirs, each with up to 1000 files - 9925 files/s subsubdirsof100 Files in two-level tree, up to 100 files each - 10461 files/s subsubdirsof250 Files in two-level tree, up to 250 files each - 10164 files/s subsubdirsof1000 Files in two-level tree, up to 1000 files each - - -Measuring 7 functions for creating 8000 files of 8192 bytes - - 16615 files/s onedir All files in one directory - 18280 files/s subdirsof100 Files in subdirs, each with up to 100 files - 18477 files/s subdirsof250 Files in subdirs, each with up to 250 files - 17209 files/s subdirsof1000 Files in subdirs, each with up to 1000 files - 10573 files/s subsubdirsof100 Files in two-level tree, up to 100 files each - 10560 files/s subsubdirsof250 Files in two-level tree, up to 250 files each - 10102 files/s subsubdirsof1000 Files in two-level tree, up to 1000 files each - - -Measuring 7 functions for creating 16000 files of 8192 bytes - - 15661 files/s onedir All files in one directory - 19164 files/s subdirsof100 Files in subdirs, each with up to 100 files - 18020 files/s subdirsof250 Files in subdirs, each with up to 250 files - 16883 files/s subdirsof1000 Files in subdirs, each with up to 1000 files - 10678 files/s subsubdirsof100 Files in two-level tree, up to 100 files each - 10305 files/s subsubdirsof250 Files in two-level tree, up to 250 files each - 10060 files/s subsubdirsof1000 Files in two-level tree, up to 1000 files each - - -Measuring 7 functions for creating 32000 files of 8192 bytes - - 16703 files/s onedir All files in one directory - 5068 files/s subdirsof100 Files in subdirs, each with up to 100 files - 17810 files/s subdirsof250 Files in subdirs, each with up to 250 files - 16753 files/s subdirsof1000 Files in subdirs, each with up to 1000 files - 10521 files/s subsubdirsof100 Files in two-level tree, up to 100 files each - 9306 files/s subsubdirsof250 Files in two-level tree, up to 250 files each - 9704 files/s subsubdirsof1000 Files in two-level tree, up to 1000 files each - - -Measuring 7 functions for creating 64000 files of 8192 bytes - - 8254 files/s onedir All files in one directory - 4493 files/s subdirsof100 Files in subdirs, each with up to 100 files - 10132 files/s subdirsof250 Files in subdirs, each with up to 250 files - 9293 files/s subdirsof1000 Files in subdirs, each with up to 1000 files - 9015 files/s subsubdirsof100 Files in two-level tree, up to 100 files each - 10250 files/s subsubdirsof250 Files in two-level tree, up to 250 files each - 8196 files/s subsubdirsof1000 Files in two-level tree, up to 1000 files each - - -Measuring 7 functions for creating 128000 files of 8192 bytes - - 5450 files/s onedir All files in one directory - 5504 files/s subdirsof100 Files in subdirs, each with up to 100 files - 5132 files/s subdirsof250 Files in subdirs, each with up to 250 files - 3632 files/s subdirsof1000 Files in subdirs, each with up to 1000 files - 6036 files/s subsubdirsof100 Files in two-level tree, up to 100 files each - 6438 files/s subsubdirsof250 Files in two-level tree, up to 250 files each - 6124 files/s subsubdirsof1000 Files in two-level tree, up to 1000 files each - - -Measuring 7 functions for creating 256000 files of 8192 bytes - - 3934 files/s onedir All files in one directory - 4512 files/s subdirsof100 Files in subdirs, each with up to 100 files - 4862 files/s subdirsof250 Files in subdirs, each with up to 250 files - 4884 files/s subdirsof1000 Files in subdirs, each with up to 1000 files - 4759 files/s subsubdirsof100 Files in two-level tree, up to 100 files each - 4793 files/s subsubdirsof250 Files in two-level tree, up to 250 files each - 4859 files/s subsubdirsof1000 Files in two-level tree, up to 1000 files each - - -Measuring 7 functions for creating 512000 files of 8192 bytes - - 3896 files/s onedir All files in one directory - 2877 files/s subdirsof100 Files in subdirs, each with up to 100 files - 3482 files/s subdirsof250 Files in subdirs, each with up to 250 files - 3635 files/s subdirsof1000 Files in subdirs, each with up to 1000 files - 3426 files/s subsubdirsof100 Files in two-level tree, up to 100 files each - 3750 files/s subsubdirsof250 Files in two-level tree, up to 250 files each - 3865 files/s subsubdirsof1000 Files in two-level tree, up to 1000 files each - - -Measuring 7 functions for creating 1024000 files of 8192 bytes - - 3826 files/s onedir All files in one directory - 2922 files/s subdirsof100 Files in subdirs, each with up to 100 files - 3340 files/s subdirsof250 Files in subdirs, each with up to 250 files - 3487 files/s subdirsof1000 Files in subdirs, each with up to 1000 files - 3076 files/s subsubdirsof100 Files in two-level tree, up to 100 files each - 3731 files/s subsubdirsof250 Files in two-level tree, up to 250 files each - 3703 files/s subsubdirsof1000 Files in two-level tree, up to 1000 files each - - -Measuring 7 functions for creating 10 files of 16384 bytes - - 15104 files/s onedir All files in one directory - 14559 files/s subdirsof100 Files in subdirs, each with up to 100 files - 14619 files/s subdirsof250 Files in subdirs, each with up to 250 files - 14533 files/s subdirsof1000 Files in subdirs, each with up to 1000 files - 9310 files/s subsubdirsof100 Files in two-level tree, up to 100 files each - 9243 files/s subsubdirsof250 Files in two-level tree, up to 250 files each - 9259 files/s subsubdirsof1000 Files in two-level tree, up to 1000 files each - - -Measuring 7 functions for creating 1000 files of 16384 bytes - - 14222 files/s onedir All files in one directory - 14999 files/s subdirsof100 Files in subdirs, each with up to 100 files - 14608 files/s subdirsof250 Files in subdirs, each with up to 250 files - 13958 files/s subdirsof1000 Files in subdirs, each with up to 1000 files - 9369 files/s subsubdirsof100 Files in two-level tree, up to 100 files each - 9190 files/s subsubdirsof250 Files in two-level tree, up to 250 files each - 8849 files/s subsubdirsof1000 Files in two-level tree, up to 1000 files each - - -Measuring 7 functions for creating 2000 files of 16384 bytes - - 13653 files/s onedir All files in one directory - 14240 files/s subdirsof100 Files in subdirs, each with up to 100 files - 13868 files/s subdirsof250 Files in subdirs, each with up to 250 files - 13063 files/s subdirsof1000 Files in subdirs, each with up to 1000 files - 8971 files/s subsubdirsof100 Files in two-level tree, up to 100 files each - 8574 files/s subsubdirsof250 Files in two-level tree, up to 250 files each - 8931 files/s subsubdirsof1000 Files in two-level tree, up to 1000 files each - - -Measuring 7 functions for creating 4000 files of 16384 bytes - - 13586 files/s onedir All files in one directory - 14100 files/s subdirsof100 Files in subdirs, each with up to 100 files - 13760 files/s subdirsof250 Files in subdirs, each with up to 250 files - 12727 files/s subdirsof1000 Files in subdirs, each with up to 1000 files - 8825 files/s subsubdirsof100 Files in two-level tree, up to 100 files each - 8567 files/s subsubdirsof250 Files in two-level tree, up to 250 files each - 7580 files/s subsubdirsof1000 Files in two-level tree, up to 1000 files each - - -Measuring 7 functions for creating 8000 files of 16384 bytes - - 13429 files/s onedir All files in one directory - 14343 files/s subdirsof100 Files in subdirs, each with up to 100 files - 13183 files/s subdirsof250 Files in subdirs, each with up to 250 files - 12703 files/s subdirsof1000 Files in subdirs, each with up to 1000 files - 9021 files/s subsubdirsof100 Files in two-level tree, up to 100 files each - 9018 files/s subsubdirsof250 Files in two-level tree, up to 250 files each - 8445 files/s subsubdirsof1000 Files in two-level tree, up to 1000 files each - - -Measuring 7 functions for creating 16000 files of 16384 bytes - - 12358 files/s onedir All files in one directory - 13828 files/s subdirsof100 Files in subdirs, each with up to 100 files - 13826 files/s subdirsof250 Files in subdirs, each with up to 250 files - 13549 files/s subdirsof1000 Files in subdirs, each with up to 1000 files - 8919 files/s subsubdirsof100 Files in two-level tree, up to 100 files each - 8951 files/s subsubdirsof250 Files in two-level tree, up to 250 files each - 8736 files/s subsubdirsof1000 Files in two-level tree, up to 1000 files each - - -Measuring 7 functions for creating 32000 files of 16384 bytes - - 7278 files/s onedir All files in one directory - 8634 files/s subdirsof100 Files in subdirs, each with up to 100 files - 9575 files/s subdirsof250 Files in subdirs, each with up to 250 files - 9640 files/s subdirsof1000 Files in subdirs, each with up to 1000 files - 9092 files/s subsubdirsof100 Files in two-level tree, up to 100 files each - 8398 files/s subsubdirsof250 Files in two-level tree, up to 250 files each - 8670 files/s subsubdirsof1000 Files in two-level tree, up to 1000 files each - - -Measuring 7 functions for creating 64000 files of 16384 bytes - - 3188 files/s onedir All files in one directory - 2988 files/s subdirsof100 Files in subdirs, each with up to 100 files - 3205 files/s subdirsof250 Files in subdirs, each with up to 250 files - 3400 files/s subdirsof1000 Files in subdirs, each with up to 1000 files - 3310 files/s subsubdirsof100 Files in two-level tree, up to 100 files each - 3233 files/s subsubdirsof250 Files in two-level tree, up to 250 files each - 3392 files/s subsubdirsof1000 Files in two-level tree, up to 1000 files each - - -Measuring 7 functions for creating 128000 files of 16384 bytes - - 2506 files/s onedir All files in one directory - 2420 files/s subdirsof100 Files in subdirs, each with up to 100 files - 2596 files/s subdirsof250 Files in subdirs, each with up to 250 files - 2671 files/s subdirsof1000 Files in subdirs, each with up to 1000 files - 2489 files/s subsubdirsof100 Files in two-level tree, up to 100 files each - 2191 files/s subsubdirsof250 Files in two-level tree, up to 250 files each - 2496 files/s subsubdirsof1000 Files in two-level tree, up to 1000 files each - - -Measuring 7 functions for creating 256000 files of 16384 bytes - - 2098 files/s onedir All files in one directory - 2010 files/s subdirsof100 Files in subdirs, each with up to 100 files - 2093 files/s subdirsof250 Files in subdirs, each with up to 250 files - 2084 files/s subdirsof1000 Files in subdirs, each with up to 1000 files - 2148 files/s subsubdirsof100 Files in two-level tree, up to 100 files each - 2208 files/s subsubdirsof250 Files in two-level tree, up to 250 files each - 2085 files/s subsubdirsof1000 Files in two-level tree, up to 1000 files each - - -Measuring 7 functions for creating 512000 files of 16384 bytes - - 2031 files/s onedir All files in one directory - 1585 files/s subdirsof100 Files in subdirs, each with up to 100 files - 1930 files/s subdirsof250 Files in subdirs, each with up to 250 files - 1996 files/s subdirsof1000 Files in subdirs, each with up to 1000 files - 1924 files/s subsubdirsof100 Files in two-level tree, up to 100 files each - 1984 files/s subsubdirsof250 Files in two-level tree, up to 250 files each - 2001 files/s subsubdirsof1000 Files in two-level tree, up to 1000 files each - - -Measuring 7 functions for creating 1024000 files of 16384 bytes - - 2002 files/s onedir All files in one directory - 1689 files/s subdirsof100 Files in subdirs, each with up to 100 files - 1927 files/s subdirsof250 Files in subdirs, each with up to 250 files - 1980 files/s subdirsof1000 Files in subdirs, each with up to 1000 files - 1747 files/s subsubdirsof100 Files in two-level tree, up to 100 files each - 1959 files/s subsubdirsof250 Files in two-level tree, up to 250 files each - 1954 files/s subsubdirsof1000 Files in two-level tree, up to 1000 files each - - diff --git a/createfiles.py b/createfiles.py deleted file mode 100755 index 3a46c53..0000000 --- a/createfiles.py +++ /dev/null @@ -1,162 +0,0 @@ -#!/usr/bin/python - -# Create lots of (empty) files in various ways, to see which way is fastest. - - -import errno -import gc -import os -import optparse -import shutil -import sys -import timeit - - -ROOT = "create.tmp" - - -def create_dir(dirname): - full_name = os.path.join(ROOT, dirname) - try: - os.mkdir(full_name) - except os.error, e: - if e.errno != errno.EEXIST: - raise e - -def create_file(filename, size): - full_name = os.path.join(ROOT, filename) - f = file(full_name, "w") - f.write("x" * size) - f.close() - -def cleanup(): - while os.path.exists(ROOT): - try: - shutil.rmtree(ROOT) - except os.error, e: - if e.errno != errno.ENOENT: - raise e - os.mkdir(ROOT) - -def onedir(count, size): - """All files in one directory""" - for i in range(count): - create_file("%d" % i, size) - -def subdirsofN(N, count, size): - prev_subdir = None - for i in range(count): - subdir = i / N - if subdir != prev_subdir: - create_dir("%d" % subdir) - prev_subdir = subdir - create_file("%d/%d" % (subdir, i), size) - -def subdirsof100(count, size): - """Files in subdirs, each with up to 100 files""" - subdirsofN(100, count, size) - -def subdirsof250(count, size): - """Files in subdirs, each with up to 250 files""" - subdirsofN(250, count, size) - -def subdirsof1000(count, size): - """Files in subdirs, each with up to 1000 files""" - subdirsofN(1000, count, size) - -def subsubdirsofN(N, count, size): - prev = None - for i in range(count): - subsubdir = i / N - subdir = subsubdir / N - if (subdir, subsubdir) != prev: - create_dir("%d" % subdir) - create_dir("%d/%d" % (subdir, subsubdir)) - prev_subdir = (subdir, subsubdir) - create_file("%d/%d/%d" % (subdir, subsubdir, i), size) - -def subsubdirsof100(count, size): - """Files in two-level tree, up to 100 files each""" - subsubdirsofN(100, count, size) - -def subsubdirsof250(count, size): - """Files in two-level tree, up to 250 files each""" - subsubdirsofN(250, count, size) - -def subsubdirsof1000(count, size): - """Files in two-level tree, up to 1000 files each""" - subsubdirsofN(1000, count, size) - -funcs = [ - onedir, # disabled, because it exposes a bug in ext3+dir_index - subdirsof100, - subdirsof250, - subdirsof1000, - subsubdirsof100, - subsubdirsof250, - subsubdirsof1000, - ] - -def measure(func, count, size): - cleanup() - gc.collect() - timer = timeit.Timer(stmt='%s(%d, %d)' % (func.func_name, count, size), - setup="from __main__ import %s" % func.func_name) - return timer.timeit(number=1) - -def check(func, count, size): - cleanup() - func(count, size) - actual_count = 0 - for dirname, _, filenames in os.walk(ROOT): - actual_count += len(filenames) - for name in [os.path.join(dirname, x) for x in filenames]: - actual_size = os.path.getsize(name) - assert size == actual_size, \ - "size is %d, should be %d" % (actual_size, size) - assert count == actual_count, \ - "count=%d and actual_count=%d should be identical" % \ - (count, actual_count) - -def parse_args(args): - p = optparse.OptionParser() - - p.add_option("-c", "--check", action="store_true", default=False, - help="Check that all functions actually work") - p.add_option("-s", "--size", action="store", type="int", metavar="SIZE", - default=0, help="Create files of SIZE bytes (default: 0)") - - return p.parse_args(args) - - -def main(): - options, counts = parse_args(sys.argv[1:]) - - if counts: - counts = [int(x) for x in counts] - else: - counts = [10000] - - for count in counts: - print "Measuring %d functions for creating %d files of %d bytes" % \ - (len(funcs), count, options.size) - print - - namelen = max(len(func.func_name) for func in funcs) - - for func in funcs: - if options.check: - check(func, count, options.size) - secs = measure(func, count, options.size) - speed = count/secs - print "%6.0f files/s %-*s %s" % \ - (speed, namelen, func.func_name, func.__doc__) - sys.stdout.flush() - print - print - - cleanup() - os.rmdir(ROOT) - -if __name__ == "__main__": - main() diff --git a/genbackupdata.py b/genbackupdata.py deleted file mode 100644 index 2fbb416..0000000 --- a/genbackupdata.py +++ /dev/null @@ -1,660 +0,0 @@ -# Copyright (C) 2007 Lars Wirzenius -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - -"""Generate backup test data""" - - -import hashlib -import optparse -import os -import random -import sys - - -KiB = 2 ** 10 # A kibibyte -MiB = 2 ** 20 # A mebibyte -GiB = 2 ** 30 # A gibibyte -TiB = 2 ** 40 # A tebibyte - -# Defaults for various settings in the BackupData class. -DEFAULT_SEED = 0 -DEFAULT_BINARY_CHUNK_SIZE = KiB -DEFAULT_TEXT_FILE_SIZE = 10 * KiB -DEFAULT_BINARY_FILE_SIZE = 10 * MiB -DEFAULT_TEXT_DATA_PERCENTAGE = 10.0 -DEFAULT_MAX_FILES_PER_DIRECTORY = 256 -DEFAULT_MODIFY_PERCENTAGE = 10 - -# Random filler text for generating text data. -if True: #pragma: no cover: - LOREM_IPSUM = """ -Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod -tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim -veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea -commodo consequat. Duis aute irure dolor in reprehenderit in voluptate -velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint -occaecat cupidatat non proident, sunt in culpa qui officia deserunt -mollit anim id est laborum. -""" - - -class BackupData: - - """This class represents the directory with backup data""" - - def __init__(self): - self._dirname = None - self._seed = 0 - self._prng = None - self._chunk_size = DEFAULT_BINARY_CHUNK_SIZE - self._text_file_size = DEFAULT_TEXT_FILE_SIZE - self._binary_file_size = DEFAULT_BINARY_FILE_SIZE - self._text_data_percentage = DEFAULT_TEXT_DATA_PERCENTAGE - self._max_files_per_directory = DEFAULT_MAX_FILES_PER_DIRECTORY - self._modify_percentage = DEFAULT_MODIFY_PERCENTAGE - self._preexisting_file_count = 0 - self._preexisting_data_size = 0 - self._filename_counter = 0 - self._current_dir_no = 0 - self._next_filecount = 0 - - # The zlib compression algorithm gives up if it gets a block of - # 32 KiB bytes it can't find in its dictionary. It completely - # ignores such a block, meaning that if it is repeated, then - # it ignores it repeatedly. Most importantly for us, it doesn't - # compress the repeats, either. Thus, to generate lots of - # uncompressible binary data, we can generate a blob and repeat - # that. Thanks to Richard Braakman for the idea. - self._binary_blob_size = 64 * 1024 # Safety factor of 2 - self._binary_blob = None - - def set_directory(self, dirname): - """Set the directory to be operated on - - Note that this must be set exactly once. Setting it twice will cause - an assertion error, and not setting it will cause other errors. - - """ - - assert self._dirname is None - self._dirname = dirname - - def get_directory(self): - """Return the directory being operated on, or None if not set""" - return self._dirname - - def create_directory(self): - """Create the backup data directory, if it doesn't exist already""" - if not os.path.exists(self._dirname): - os.mkdir(self._dirname) - - def get_seed(self): - """Return the initial seed for the pseudo-random number generator""" - return self._seed - - def set_seed(self, seed): - """Set the initial seed for the pseudo-random number generator - - The seed will be used when the generator is first initialized. - It is initialized implicitly as soon as something in this class - needs randomness. Setting the seed after the generator has been - initialized causes an assertion failure. - - """ - - assert self.get_prng() is None - self._seed = seed - - def get_prng(self): - """Return reference to the psuedo-random number generator being used - - Return None, if one hasn't be initialized yet. - - """ - - return self._prng - - def init_prng(self): - """Initialize the psuedo-random number generator (using seed)""" - if self._prng is None: - self._prng = random.Random() - self._prng.seed(self._seed) - - def get_text_file_size(self): - """Return size of newly created text files""" - return self._text_file_size - - def set_text_file_size(self, size): - """Set size of newly created text files""" - self._text_file_size = size - - def get_binary_file_size(self): - """Return size of newly created binary files""" - return self._binary_file_size - - def set_binary_file_size(self, size): - """Set size of newly created binary files""" - self._binary_file_size = size - - def get_text_data_percentage(self): - """Return percentage of text data of new data that gets created""" - return self._text_data_percentage - - def set_text_data_percentage(self, percent): - """Set percentage of text data of new data that gets created""" - self._text_data_percentage = percent - - def get_max_files_per_directory(self): - """Return current setting of maximum number of files per directory""" - return self._max_files_per_directory - - def set_max_files_per_directory(self, count): - """Set maximum number of files per directory""" - self._max_files_per_directory = count - - def get_preexisting_file_count(self): - """Return count of files that existed in directory in the beginning""" - return self._preexisting_file_count - - def set_preexisting_file_count(self, count): - """Set count of files that existed in directory in the beginning - - This is useful only for unit tests. - - """ - self._preexisting_file_count = count - - def get_preexisting_data_size(self): - """Return size of data that existed in directory in the beginning""" - return self._preexisting_data_size - - def set_preexisting_data_size(self, size): - """Set size of data that existed in directory in the beginning - - This is useful only for unit tests. - - """ - self._preexisting_data_size = size - - def get_relative_file_count(self, percent): - """Return PERCENT percent of pre-existing file count""" - return int(0.01 * percent * self.get_preexisting_file_count()) - - def get_relative_data_size(self, percent): - """Return PERCENT percent of pre-existing data""" - return int(0.01 * percent * self.get_preexisting_data_size()) - - def find_preexisting_files(self): - """Find all the files that exists in the directory right now""" - count = 0 - size = 0 - if os.path.exists(self._dirname): - for root, dirs, filenames in os.walk(self._dirname): - count += len(filenames) - for filename in filenames: - size += os.path.getsize(os.path.join(root, filename)) - self.set_preexisting_file_count(count) - self.set_preexisting_data_size(size) - - def _choose_directory(self): - """Choose directory in which to create the next file""" - - while True: - dirname = os.path.join(self._dirname, - "dir%d" % self._current_dir_no) - if not os.path.exists(dirname): - self._next_filecount = 0 - break - if (self._next_filecount > 0 and - self._next_filecount < self._max_files_per_directory): - break - self._current_dir_no += 1 - return dirname - - def next_filename(self): - """Choose the name of the next filename - - The file does not currently exist. This is not, however, a guarantee - that no other process won't create it before we do. Thus, this - is NOT a secure way to create temporary files. But it's good enough - for our intended purpose. - - For simplified unit testing, the names are very easily predictable, - but it is probably a bad idea for external code to rely on this. - - """ - - dirname = self._choose_directory() - while True: - filename = os.path.join(dirname, - "file%d" % self._filename_counter) - if not os.path.exists(filename): - self._next_filecount += 1 - return filename - self._filename_counter += 1 - - def generate_text_data(self, size): - """Generate SIZE characters of text data""" - if size <= len(LOREM_IPSUM): - return LOREM_IPSUM[:size] - else: - full = size / len(LOREM_IPSUM) - rest = size % len(LOREM_IPSUM) - return "".join(([LOREM_IPSUM] * full) + [LOREM_IPSUM[:rest]]) - - def generate_binary_data_well(self, size): - """Generate SIZE bytes of more or less random binary junk""" - - # The following code has had some fine manual fine tuning done - # to it. This has made it a bit ugly, but faster. On a - # "Intel(R) Core(TM)2 Duo CPU L9400 @ 1.86GHz", it produces - # about 25 MB/s. - - chunks = [] - sum = hashlib.sha1() - chunk_size = len(sum.digest()) - - initial_bytes = min(size, 128) - for i in range(initial_bytes / chunk_size): - sum.update(chr(random.getrandbits(8))) - chunk = sum.digest() - chunks.append(chunk) - - size -= len(chunks) * chunk_size - for i in range(size / chunk_size): - sum.update("a") - chunk = sum.digest() - chunks.append(chunk) - - if size % chunk_size > 0: - sum.update(chr(random.getrandbits(8))) - chunk = sum.digest() - chunks.append(chunk[:size % chunk_size]) - - return "".join(chunks) - - - def generate_binary_data(self, size): - """Generate SIZE bytes of binary junk. - - This is different from generate_binary_data_well in that - it makes use of _binary_blob (and generates that if it does - not yet exist). - - """ - - if self._binary_blob is None: - self._binary_blob = self.generate_binary_data_well( - self._binary_blob_size) - if size <= len(self._binary_blob): - return self._binary_blob[:size] - else: - full = size / len(self._binary_blob) - rest = size % len(self._binary_blob) - return "".join(([self._binary_blob] * full) + - [self._binary_blob[:rest]]) - - def create_subdirectories(self, filename): - """Create the sub-directories that are needed to create filename""" - subdir = os.path.dirname(filename) - if not os.path.exists(subdir): - os.makedirs(subdir) - - def create_text_file(self, size): - """Create a new text file of the desired size""" - filename = self.next_filename() - self.create_subdirectories(filename) - f = file(filename, "w") - f.write(self.generate_text_data(size)) - f.close() - - def get_binary_chunk_size(self): - """Return the size of chunks used when writing binary data""" - return self._chunk_size - - def set_binary_chunk_size(self, size): - """Set the size of chunks used when writing binary data""" - self._chunk_size = size - - def create_binary_file(self, size): - """Create a new binary file of the desired size""" - filename = self.next_filename() - self.create_subdirectories(filename) - f = file(filename, "w") - # We write the data in chunks, so as not to keep the entire file - # contents in memory at a time. Since the size may be very large, - # we might otherwise run out of swap. - while size >= self._chunk_size: - f.write(self.generate_binary_data(self._chunk_size)) - size -= self._chunk_size - f.write(self.generate_binary_data(size)) - f.close() - - def _create_files_of_a_kind(self, size, file_size, create_one): - """Create files with create_one""" - while size > 0: - this_size = min(size, file_size) - create_one(this_size) - size -= this_size - - def create_files(self, size): - """Create new files, totalling SIZE bytes in size""" - text_size = int(0.01 * self._text_data_percentage * size) - bin_size = size - text_size - - self._create_files_of_a_kind(text_size, self.get_text_file_size(), - self.create_text_file) - self._create_files_of_a_kind(bin_size, self.get_binary_file_size(), - self.create_binary_file) - - def find_files(self): - """Find all non-directory files in the test data set""" - files = [] - for root, dirs, filenames in os.walk(self._dirname): - for filename in filenames: - files.append(os.path.join(root, filename)) - return files - - def choose_files_randomly(self, count): - """Choose COUNT files randomly""" - files = self.find_files() - if len(files) >= count: - self.init_prng() - files = self._prng.sample(files, count) - return files - - def delete_files(self, count): - """Delete COUNT files""" - if os.path.exists(self._dirname): - for file in self.choose_files_randomly(count): - os.remove(file) - - def rename_files(self, count): - """Rename COUNT files to new names""" - if os.path.exists(self._dirname): - for file in self.choose_files_randomly(count): - new_file = self.next_filename() - self.create_subdirectories(new_file) - os.rename(file, new_file) - - def link_files(self, count): - """Create COUNT new filenames that are hard links to existing files""" - if os.path.exists(self._dirname): - for file in self.choose_files_randomly(count): - new_file = self.next_filename() - self.create_subdirectories(new_file) - os.link(file, new_file) - - def get_modify_percentage(self): - """Return how many percent to grow each file with modify_files()""" - return self._modify_percentage - - def set_modify_percentage(self, percent): - """Set how many percent to grow each file with modify_files()""" - self._modify_percentage = percent - - def append_data(self, filename, data): - """Append data to a file""" - f = file(filename, "a") - f.write(data) - f.close() - - def _modify_files_of_a_kind(self, filenames, size, generate_data): - """Modify files by appending data to them""" - while size > 0: - filename = self._prng.choice(filenames) - this_size = os.path.getsize(filename) - amount = min(int(0.01 * self._modify_percentage * this_size), - size) - self.append_data(filename, generate_data(amount)) - size -= amount - - def modify_files(self, size): - """Modify existing files by appending to them - - SIZE gives the total amount of new data for all files. - Files are chosen at random, and new data is appended to them. - The amount appended to each file is set by - set_modify_percentage. The data is split between text and - binary data according to set_text_data_percentage. - - """ - - if os.path.exists(self._dirname): - files = self.find_files() - - text_size = int(0.01 * self._text_data_percentage * size) - bin_size = size - text_size - - self.init_prng() - self._modify_files_of_a_kind(files, text_size, - self.generate_text_data) - self._modify_files_of_a_kind(files, bin_size, - self.generate_binary_data) - - - -class CommandLineParser: - - """Parse the command line for the genbackupdata utility""" - - def __init__(self, backup_data): - self._bd = backup_data - self._parser = self._create_option_parser() - - def _create_option_parser(self): - """Create the OptionParser we need""" - - p = optparse.OptionParser() - - p.add_option("--seed", - help="Set pseudo-random number generator seed to SEED") - - p.add_option("--max-count", - action="store", - metavar="COUNT", - help="Allow at most COUNT files per directory") - - p.add_option("-p", "--percentage-text-data", - action="store", - metavar="PERCENT", - help="Make PERCENT of new data textual, not binary") - - p.add_option("-t", "--text-file-size", - action="store", - metavar="SIZE", - help="Make new text files be of size SIZE") - - p.add_option("-b", "--binary-file-size", - action="store", - metavar="SIZE", - help="Make new binary files be of size SIZE") - - p.add_option("-c", "--create", - action="store", - metavar="SIZE", - help="Create SIZE amount of new files") - - p.add_option("-d", "--delete", - action="store", - metavar="COUNT", - help="Delete COUNT files") - - p.add_option("-r", "--rename", - action="store", - metavar="COUNT", - help="Rename COUNT files") - - p.add_option("-l", "--link", - action="store", - metavar="COUNT", - help="Create COUNT new hard links") - - p.add_option("-m", "--modify", - action="store", - metavar="SIZE", - help="Grow total data size by SIZE") - - p.add_option("--modify-percentage", - action="store", - metavar="PERCENT", - help="Increase file size by PERCENT") - - return p - - def parse_size(self, size, base_size=None): - """Parse a SIZE argument (absolute, relative, with/without suffix)""" - - suffixes = (("k", KiB), ("m", MiB), ("g", GiB), ("t", TiB)) - - for suffix, factor in suffixes: - if size.lower().endswith(suffix): - return int(float(size[:-len(suffix)]) * factor) - - if size.endswith("%"): - if base_size is None: - return 0 - else: - return int(float(size[:-1]) * 0.01 * base_size) - - return int(size) - - def parse_count(self, count, base_count=None): - """Parse a COUNT argument (absolute, relative, with/without suffix)""" - - suffixes = (("k", 10**3), ("m", 10**6), ("g", 10**9), ("t", 10**12)) - - for suffix, factor in suffixes: - if count.lower().endswith(suffix): - return int(float(count[:-len(suffix)]) * factor) - - if count.endswith("%"): - if base_count is None: - return 0 - else: - return int(float(count[:-1]) * 0.01 * base_count) - - return int(count) - - def parse(self, args): - """Parse command line arguments""" - options, args = self._parser.parse_args(args) - - if len(args) == 1: - self._bd.set_directory(args[0]) - self._bd.find_preexisting_files() - - if options.seed: - self._bd.set_seed(int(options.seed)) - - if options.max_count: - self._bd.set_max_files_per_directory(int(options.max_count)) - - if options.percentage_text_data: - self._bd.set_text_data_percentage( - float(options.percentage_text_data)) - - if options.modify_percentage: - self._bd.set_modify_percentage(float(options.modify_percentage)) - - if options.text_file_size: - self._bd.set_text_file_size( - self.parse_size(options.text_file_size)) - - if options.binary_file_size: - self._bd.set_binary_file_size( - self.parse_size(options.binary_file_size)) - - if options.create: - options.create = self.parse_size(options.create, - self._bd.get_preexisting_data_size()) - - if options.modify: - options.modify = self.parse_size(options.modify, - self._bd.get_preexisting_data_size()) - - if options.delete: - options.delete = self.parse_count(options.delete, - self._bd.get_preexisting_file_count()) - - if options.rename: - options.rename = self.parse_count(options.rename, - self._bd.get_preexisting_file_count()) - - if options.link: - options.link = self.parse_count(options.link, - self._bd.get_preexisting_file_count()) - - return options, args - - -class AppException(Exception): - - def __str__(self): - return self._str - - -class NeedExactlyOneDirectoryName(AppException): - - def __init__(self): - self._str = ("Need exactly one command line argument, " - "giving directory name") - - -class Application: - - """The main program""" - - def __init__(self, args): - self._args = args - self._bd = BackupData() - self._clp = CommandLineParser(self._bd) - self._error = sys.stderr.write - - def set_error_writer(self, writer): - self._error = writer - - def run(self): - """Execute the desired operations""" - try: - options, args = self._clp.parse(self._args) - - if len(args) != 1: - raise NeedExactlyOneDirectoryName() - - if options.delete: - self._bd.delete_files(options.delete) - - if options.rename: - self._bd.rename_files(options.rename) - - if options.link: - self._bd.link_files(options.link) - - if options.modify: - self._bd.modify_files(options.modify) - - if options.create: - self._bd.create_files(options.create) - - except AppException, e: - self._error(str(e) + "\n") - sys.exit(1) - - -if __name__ == "__main__": #pragma: no cover - Application(sys.argv[1:]).run() diff --git a/tests.py b/tests.py deleted file mode 100755 index 4466fc0..0000000 --- a/tests.py +++ /dev/null @@ -1,565 +0,0 @@ -#!/usr/bin/python -# -# Copyright (C) 2007 Lars Wirzenius -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - -"""Unit tests for genbackupdata.py""" - - -import os -import shutil -import unittest -import zlib - -import genbackupdata - - -class BackupDataTests(unittest.TestCase): - - dirname = "tests.dir" - - def remove_dir(self): - """Remove a directory, if it exists""" - if os.path.exists(self.dirname): - shutil.rmtree(self.dirname) - - def create(self, filename, contents): - """Create a new file with the desired contents""" - dirname = os.path.dirname(filename) - if not os.path.exists(dirname): - os.makedirs(dirname) - f = file(filename, "w") - f.write(contents) - f.close() - - def read_file(self, filename): - """Return the entire contents of a file""" - f = file(filename) - data = f.read() - f.close() - return data - - def setUp(self): - self.remove_dir() - self.bd = genbackupdata.BackupData() - self.bd.set_directory(self.dirname) - - def tearDown(self): - del self.bd - self.remove_dir() - - def testSetsDirectoryCorrect(self): - self.failUnlessEqual(self.bd.get_directory(), self.dirname) - - def testDoesNotCreateDirectoryAtOnce(self): - self.failIf(os.path.exists(self.dirname)) - - def testCreatesDirectory(self): - self.bd.create_directory() - self.failUnless(os.path.exists(self.dirname)) - - def testDoesNotFailWhenCreatingDirectoryWhenItExistsAlready(self): - self.bd.create_directory() - self.bd.create_directory() - - def testHasCorrectDefaultSeed(self): - self.failUnlessEqual(self.bd.get_seed(), genbackupdata.DEFAULT_SEED) - - def testCorrectlySetsSeed(self): - self.bd.set_seed(12765) - self.failUnlessEqual(self.bd.get_seed(), 12765) - - def testHasNoPRNGInitially(self): - self.failUnlessEqual(self.bd.get_prng(), None) - - def testCreatesPRNGWhenRequested(self): - self.bd.init_prng() - self.failIfEqual(self.bd.get_prng(), None) - - def testFailsIfSeedGetsSetAfterPRNGHasBeenCreatedTwice(self): - self.bd.init_prng() - self.failUnlessRaises(AssertionError, self.bd.set_seed, 12765) - - def testHasCorrectDefaultBinaryChunkSize(self): - self.failUnlessEqual(self.bd.get_binary_chunk_size(), - genbackupdata.DEFAULT_BINARY_CHUNK_SIZE) - - def testCorrectlySetsBinaryChunkSize(self): - self.bd.set_binary_chunk_size(12765) - self.failUnlessEqual(self.bd.get_binary_chunk_size(), 12765) - - def testHasCorrectDefaultTextFileSize(self): - self.failUnlessEqual(self.bd.get_text_file_size(), - genbackupdata.DEFAULT_TEXT_FILE_SIZE) - - def testCorrectlySetsTextFileSize(self): - self.bd.set_text_file_size(12765) - self.failUnlessEqual(self.bd.get_text_file_size(), 12765) - - def testHasCorrectDefaultBinaryFileSize(self): - self.failUnlessEqual(self.bd.get_binary_file_size(), - genbackupdata.DEFAULT_BINARY_FILE_SIZE) - - def testCorrectlySetsBinaryFileSize(self): - self.bd.set_binary_file_size(12765) - self.failUnlessEqual(self.bd.get_binary_file_size(), 12765) - - def testHasCorrectDefaultTextDataPercentage(self): - self.failUnlessEqual(self.bd.get_text_data_percentage(), - genbackupdata.DEFAULT_TEXT_DATA_PERCENTAGE) - - def testCorrectlySetsTextDataPercentage(self): - self.bd.set_text_data_percentage(42.0) - self.failUnlessEqual(self.bd.get_text_data_percentage(), 42.0) - - def testHasCorrectDefaultMaxFilesPerDirectory(self): - self.failUnlessEqual(self.bd.get_max_files_per_directory(), - genbackupdata.DEFAULT_MAX_FILES_PER_DIRECTORY) - - def testCorrectlySetsMaxFilesPerDirectory(self): - self.bd.set_max_files_per_directory(12765) - self.failUnlessEqual(self.bd.get_max_files_per_directory(), 12765) - - def testSetsPreExistingFileCountToZeroByDefault(self): - self.failUnlessEqual(self.bd.get_preexisting_file_count(), 0) - - def testCanFakePreExistingFileCount(self): - self.bd.set_preexisting_file_count(12765) - self.failUnlessEqual(self.bd.get_preexisting_file_count(), 12765) - - def testSetsPreExistingDataSizeToZeroByDefault(self): - self.failUnlessEqual(self.bd.get_preexisting_data_size(), 0) - - def testCanFakePreExistingDataSize(self): - self.bd.set_preexisting_data_size(12765) - self.failUnlessEqual(self.bd.get_preexisting_data_size(), 12765) - - def testFindsPreExistingFilesAndDAtaCorrectly(self): - file_count = 10 - file_size = self.bd.get_text_file_size() - data = self.bd.generate_text_data(file_size) - for i in range(file_count): - self.create(self.bd.next_filename(), data) - self.bd.find_preexisting_files() - self.failUnlessEqual(self.bd.get_preexisting_file_count(), file_count) - self.failUnlessEqual(self.bd.get_preexisting_data_size(), - file_count * file_size) - - def testComputesRelativeFileCountCorrectly(self): - self.bd.set_preexisting_file_count(12765) - self.failUnlessEqual(self.bd.get_relative_file_count(10), 1276) - - def testComputesRelativeDataSizeCorrectly(self): - self.bd.set_preexisting_data_size(12765) - self.failUnlessEqual(self.bd.get_relative_data_size(10), 1276) - - def testFindsNoPreExistingFilesWhenDirectoryDoesNotExist(self): - self.bd.find_preexisting_files() - self.failUnlessEqual(self.bd.get_preexisting_file_count(), 0) - - def testFindsNoPreExistingDataWhenDirectoryDoesNotExist(self): - self.bd.find_preexisting_files() - self.failUnlessEqual(self.bd.get_preexisting_data_size(), 0) - - def testChoosesFirstFilenameCorrectly(self): - filename = self.bd.next_filename() - self.failUnless(filename.startswith(self.dirname)) - - def testChoosesFirstFilenameCorrectlyTwice(self): - filename1 = self.bd.next_filename() - filename2 = self.bd.next_filename() - self.failUnlessEqual(filename1, filename2) - - def testChoosesFilenameCorrectlyWhenFirstOneExistsAlready(self): - self.bd.create_directory() - filename1 = self.bd.next_filename() - self.create(filename1, "") - filename2 = self.bd.next_filename() - self.failIfEqual(filename1, filename2) - - def testChoosesSameDirectoryUntilMaxFileLimitIsReached(self): - self.bd.set_max_files_per_directory(10) # For speed - self.bd.create_directory() - chosen = self.bd._choose_directory() - for i in range(self.bd.get_max_files_per_directory()): - self.failUnlessEqual(self.bd._choose_directory(), chosen) - self.create(self.bd.next_filename(), "") - - def testChoosesNewDirectoryWhenMaxFileLimitIsReached(self): - self.bd.set_max_files_per_directory(10) # For speed - self.bd.create_directory() - chosen = self.bd._choose_directory() - for i in range(self.bd.get_max_files_per_directory()): - self.create(self.bd.next_filename(), "") - self.failIfEqual(self.bd._choose_directory(), chosen) - - def testGeneratesSmallAmountOfTextDataCorrectly(self): - n = 128 - self.failUnlessEqual(self.bd.generate_text_data(n), - genbackupdata.LOREM_IPSUM[:n]) - - def testGeneratesLargeAmountOfTextDataCorrectly(self): - n = len(genbackupdata.LOREM_IPSUM) - self.failUnlessEqual(self.bd.generate_text_data(n * 2), - genbackupdata.LOREM_IPSUM * 2) - - def testGeneratesRequestedAmountOfBinaryDataWell(self): - n = 37 - self.failUnlessEqual(len(self.bd.generate_binary_data_well(n)), n) - - def testGeneratesRequestedAmountOfBinaryDataQuickly(self): - n = 128 - self.failUnlessEqual(len(self.bd.generate_binary_data(n)), n) - - def testGeneratesRequestedLargeAmountOfBinaryData(self): - n = self.bd._binary_blob_size + 1 - self.failUnlessEqual(len(self.bd.generate_binary_data(n)), n) - - def testGeneratesBinaryDataWhichDoesNotCompressWell(self): - n = self.bd._binary_blob_size * 4 - data = zlib.compress(self.bd.generate_binary_data(n)) - self.failUnless(len(data) > 0.95* n) - - def testCreatesSubdirectoriesCorrectly(self): - filename = os.path.join(self.dirname, "subdir", "filename") - self.bd.create_subdirectories(filename) - self.failUnless(os.path.isdir(os.path.dirname(filename))) - - def testCreatesTextFileCorrectly(self): - size = self.bd.get_text_file_size() - filename = self.bd.next_filename() - self.bd.create_text_file(size) - self.failUnless(os.path.isfile(filename)) - self.failUnlessEqual(self.read_file(filename), - self.bd.generate_text_data(size)) - - def testCreatesBinaryFileCorrectly(self): - size = self.bd.get_binary_chunk_size() + 1 - filename = self.bd.next_filename() - self.bd.create_binary_file(size) - self.failUnless(os.path.isfile(filename)) - self.failUnlessEqual(os.path.getsize(filename), size) - - def failUnlessTextFile(self, filename): - data = self.read_file(filename) - self.failUnlessEqual(data, self.bd.generate_text_data(len(data))) - - def testCreatesTextFilesOnlyCorrectly(self): - self.bd.set_text_data_percentage(100) - count = 10 - size = count * self.bd.get_text_file_size() - self.bd.create_files(size) - self.bd.find_preexisting_files() - self.failUnlessEqual(self.bd.get_preexisting_file_count(), count) - self.failUnlessEqual(self.bd.get_preexisting_data_size(), size) - for root, dirs, filenames in os.walk(self.dirname): - for filename in filenames: - pathname = os.path.join(root, filename) - self.failUnlessTextFile(pathname) - - def failIfTextFile(self, filename): - data = self.read_file(filename) - self.failIfEqual(data, self.bd.generate_text_data(len(data))) - - def testCreatesNoTextFilesCorrectly(self): - self.bd.set_text_data_percentage(0) - count = 1 - size = count * self.bd.get_text_file_size() - self.bd.create_files(size) - self.bd.find_preexisting_files() - self.failUnlessEqual(self.bd.get_preexisting_file_count(), 1) - self.failUnlessEqual(self.bd.get_preexisting_data_size(), size) - for root, dirs, filenames in os.walk(self.dirname): - for filename in filenames: - pathname = os.path.join(root, filename) - self.failIfTextFile(pathname) - - def testDeletesFilesCorrectly(self): - size = 100 - self.bd.set_text_file_size(1) - self.bd.set_binary_file_size(1) - self.bd.create_files(size) - self.bd.find_preexisting_files() - count = self.bd.get_preexisting_file_count() - to_delete = count/3 - remaining = count - to_delete - self.bd.delete_files(to_delete) - self.bd.find_preexisting_files() - self.failUnlessEqual(self.bd.get_preexisting_file_count(), remaining) - - def testRenamesFilesCorrectly(self): - self.bd.create_directory() - filename = self.bd.next_filename() - self.create(filename, "") - new_filename = self.bd.next_filename() - self.bd.rename_files(1) - self.failIf(os.path.exists(filename)) - self.failUnless(os.path.exists(new_filename)) - - def testCreatesLinksCorrectly(self): - self.bd.create_directory() - filename = self.bd.next_filename() - self.create(filename, "") - new_filename = self.bd.next_filename() - self.bd.link_files(1) - self.failUnless(os.path.samefile(filename, new_filename)) - - def testHasCorrectDefaultModifyPercentage(self): - self.failUnlessEqual(self.bd.get_modify_percentage(), - genbackupdata.DEFAULT_MODIFY_PERCENTAGE) - - def testCorrectlySetsModifyPercentage(self): - self.bd.set_modify_percentage(42.0) - self.failUnlessEqual(self.bd.get_modify_percentage(), 42.0) - - def testModifyFiles(self): - size = 100 - self.bd.create_directory() - filename = self.bd.next_filename() - orig_data = "x" * size - self.create(filename, orig_data) - self.bd.modify_files(size) - self.failUnlessEqual(len(self.read_file(filename)), 2 * size) - - -class CommandLineParserTests(unittest.TestCase): - - dirname = "tests.dir" - - def setUp(self): - self.bd = genbackupdata.BackupData() - self.bd.set_directory(self.dirname) - self.clp = genbackupdata.CommandLineParser(self.bd) - - def tearDown(self): - del self.bd - del self.clp - - def testDoesNotTouchDefaultsWithEmptyCommandLine(self): - self.failUnlessEqual(self.bd.get_seed(), genbackupdata.DEFAULT_SEED) - self.failUnlessEqual(self.bd.get_binary_chunk_size(), - genbackupdata.DEFAULT_BINARY_CHUNK_SIZE) - self.failUnlessEqual(self.bd.get_text_file_size(), - genbackupdata.DEFAULT_TEXT_FILE_SIZE) - self.failUnlessEqual(self.bd.get_binary_file_size(), - genbackupdata.DEFAULT_BINARY_FILE_SIZE) - self.failUnlessEqual(self.bd.get_text_data_percentage(), - genbackupdata.DEFAULT_TEXT_DATA_PERCENTAGE) - self.failUnlessEqual(self.bd.get_max_files_per_directory(), - genbackupdata.DEFAULT_MAX_FILES_PER_DIRECTORY) - self.failUnlessEqual(self.bd.get_modify_percentage(), - genbackupdata.DEFAULT_MODIFY_PERCENTAGE) - - def testParsesPlainSizeCorrectly(self): - self.failUnlessEqual(self.clp.parse_size("12765"), 12765) - - def testParsesAbsoluteSizeSuffixesCorrectly(self): - self.failUnlessEqual(self.clp.parse_size("3k"), 3 * genbackupdata.KiB) - self.failUnlessEqual(self.clp.parse_size("3K"), 3 * genbackupdata.KiB) - self.failUnlessEqual(self.clp.parse_size("3m"), 3 * genbackupdata.MiB) - self.failUnlessEqual(self.clp.parse_size("3M"), 3 * genbackupdata.MiB) - self.failUnlessEqual(self.clp.parse_size("3g"), 3 * genbackupdata.GiB) - self.failUnlessEqual(self.clp.parse_size("3G"), 3 * genbackupdata.GiB) - self.failUnlessEqual(self.clp.parse_size("3t"), 3 * genbackupdata.TiB) - self.failUnlessEqual(self.clp.parse_size("3T"), 3 * genbackupdata.TiB) - - def testParsesRelativeSizeSuffixCorrectly(self): - self.failUnlessEqual(self.clp.parse_size("10%", 12765), 1276) - - def testParsesRelativeSizeSuffixCorrectlyWithMissingBaseSize(self): - self.failUnlessEqual(self.clp.parse_size("10%"), 0) - - def testParsesPlainCountCorrectly(self): - self.failUnlessEqual(self.clp.parse_count("12765"), 12765) - - def testParsesAbsoluteCountSuffixesCorrectly(self): - self.failUnlessEqual(self.clp.parse_count("3k"), 3 * 10**3) - self.failUnlessEqual(self.clp.parse_count("3K"), 3 * 10**3) - self.failUnlessEqual(self.clp.parse_count("3m"), 3 * 10**6) - self.failUnlessEqual(self.clp.parse_count("3M"), 3 * 10**6) - self.failUnlessEqual(self.clp.parse_count("3g"), 3 * 10**9) - self.failUnlessEqual(self.clp.parse_count("3G"), 3 * 10**9) - self.failUnlessEqual(self.clp.parse_count("3t"), 3 * 10**12) - self.failUnlessEqual(self.clp.parse_count("3T"), 3 * 10**12) - - def testParsesRelativeCountSuffixCorrectly(self): - self.failUnlessEqual(self.clp.parse_count("10%", 12765), 1276) - - def testParsesRelativeCountSuffixCorrectlyWithMissingBaseSize(self): - self.failUnlessEqual(self.clp.parse_count("10%"), 0) - - def testHandlesOptionForSeed(self): - optons, args = self.clp.parse(["--seed=12765"]) - self.failUnlessEqual(args, []) - self.failUnlessEqual(self.bd.get_seed(), 12765) - - def testHandlesOptionForMaxCount(self): - options, args = self.clp.parse(["--max-count=12765"]) - self.failUnlessEqual(args, []) - self.failUnlessEqual(self.bd.get_max_files_per_directory(), 12765) - - def testHandlesOptionForPercentageTextData(self): - options, args = self.clp.parse(["--percentage-text-data=4.2"]) - self.failUnlessEqual(args, []) - self.failUnlessEqual(self.bd.get_text_data_percentage(), 4.2) - - def testHandlesOptionForTextFileSize(self): - options, args = self.clp.parse(["--text-file-size=12765"]) - self.failUnlessEqual(args, []) - self.failUnlessEqual(self.bd.get_text_file_size(), 12765) - - def testHandlesOptionForTextFileSizeWithSuffix(self): - options, args = self.clp.parse(["--text-file-size=1t"]) - self.failUnlessEqual(args, []) - self.failUnlessEqual(self.bd.get_text_file_size(), genbackupdata.TiB) - - def testHandlesOptionForBinaryFileSizeWithSuffix(self): - options, args = self.clp.parse(["--binary-file-size=1t"]) - self.failUnlessEqual(args, []) - self.failUnlessEqual(self.bd.get_binary_file_size(), - genbackupdata.TiB) - - def testHandlesOptionForCreate(self): - options, args = self.clp.parse(["--create=1t"]) - self.failUnlessEqual(args, []) - self.failUnlessEqual(options.create, genbackupdata.TiB) - - def testHandlesOptionForCreateWithRelativeSize(self): - self.bd.set_preexisting_data_size(12765) - options, args = self.clp.parse(["--create=10%"]) - self.failUnlessEqual(args, []) - self.failUnlessEqual(options.create, 1276) - - def testHandlesOptionForDelete(self): - options, args = self.clp.parse(["--delete=12765"]) - self.failUnlessEqual(args, []) - self.failUnlessEqual(options.delete, 12765) - - def testHandlesOptionForDeleteWithRelativeCount(self): - self.bd.set_preexisting_file_count(12765) - options, args = self.clp.parse(["--delete=10%"]) - self.failUnlessEqual(args, []) - self.failUnlessEqual(options.delete, 1276) - - def testHandlesOptionForRename(self): - options, args = self.clp.parse(["--rename=12765"]) - self.failUnlessEqual(args, []) - self.failUnlessEqual(options.rename, 12765) - - def testHandlesOptionForLink(self): - options, args = self.clp.parse(["--link=12765"]) - self.failUnlessEqual(args, []) - self.failUnlessEqual(options.link, 12765) - - def testHandlesOptionForModify(self): - options, args = self.clp.parse(["--modify=12765"]) - self.failUnlessEqual(args, []) - self.failUnlessEqual(options.modify, 12765) - - def testHandlesOptionForModifyPercentage(self): - options, args = self.clp.parse(["--modify-percentage=4.2"]) - self.failUnlessEqual(args, []) - self.failUnlessEqual(self.bd.get_modify_percentage(), 4.2) - - -class ApplicationTests(unittest.TestCase): - - dirname = "tests.dir" - - def remove_dir(self): - """Remove a directory, if it exists""" - if os.path.exists(self.dirname): - shutil.rmtree(self.dirname) - - def setUp(self): - self.remove_dir() - - def tearDown(self): - self.remove_dir() - - def file_count(self): - count = 0 - for root, dirs, filenames in os.walk(self.dirname): - count += len(filenames) - return count - - def data_size(self): - size = 0 - for root, dirs, filenames in os.walk(self.dirname): - for filename in filenames: - size += os.path.getsize(os.path.join(root, filename)) - return size - - def file_list(self): - files = [] - for root, dirs, filenames in os.walk(self.dirname): - for filename in filenames: - files.append(os.path.join(root, filename)) - return files - - def nop(self, *args): - pass - - def testTerminatesWithoutDirectory(self): - app = genbackupdata.Application([]) - app.set_error_writer(self.nop) - self.failUnlessRaises(SystemExit, app.run) - - def apprun(self, args): - app = genbackupdata.Application(args) - app.run() - - def testCreatesFirstGenerationCorrectly(self): - self.apprun(["-c10k", self.dirname]) - self.failUnlessEqual(self.data_size(), 10 * genbackupdata.KiB) - - def testIncreasesSecondGenerationCorrectly(self): - self.apprun(["-c10k", self.dirname]) - self.apprun(["-c10k", self.dirname]) - self.failUnlessEqual(self.data_size(), 20 * genbackupdata.KiB) - - def testModifiesSecondGenerationCorrectly(self): - self.apprun(["-c10k", self.dirname]) - count = self.file_count() - self.apprun(["-m1k", self.dirname]) - self.failUnlessEqual(self.data_size(), 11 * genbackupdata.KiB) - self.failUnlessEqual(self.file_count(), count) - - def testDeletesFilesForSecondGenerationCorrectly(self): - self.apprun(["-c10k", self.dirname]) - count = self.file_count() - self.apprun(["-d2", self.dirname]) - self.failUnlessEqual(self.file_count(), count - 2) - - def testRenamesFilesForSecondGenerationCorrectly(self): - self.apprun(["-c10k", self.dirname]) - files1 = self.file_list() - self.apprun(["-r2", self.dirname]) - files2 = self.file_list() - self.failIfEqual(sorted(files1), sorted(files2)) - self.failUnlessEqual(len(files1), len(files2)) - - def testLinksFilesForSecondGenerationCorrectly(self): - self.apprun(["-c10k", self.dirname]) - files1 = self.file_list() - self.apprun(["-l2", self.dirname]) - files2 = self.file_list() - self.failUnlessEqual(len(files1) + 2, len(files2)) - - -if __name__ == "__main__": - unittest.main() -- cgit v1.2.1 From 3678e7ec5d1369fb32a9f64f81a32aa03841cef7 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Sun, 2 Jan 2011 13:10:22 +0000 Subject: Disable black box testing for now. --- Makefile | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 4aea314..b79bece 100644 --- a/Makefile +++ b/Makefile @@ -1,11 +1,8 @@ all: check: - python-coverage -e - python-coverage -x tests.py - python-coverage -r -m -o /usr,/var | \ - awk '{ print } /^TOTAL/ && $$2 != $$3 {exit 1}' - ./blackboxtest + python -m CoverageTestRunner --ignore-missing without-tests +# ./blackboxtest clean: rm -rf *.pyc *.pyo build dist MANIFEST -- cgit v1.2.1 From 2075429c300b6a3bdbdcc8967395289b7fd705d5 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Sun, 2 Jan 2011 13:10:34 +0000 Subject: Remove undesired benchmark script. --- binaryjunk.py | 225 ---------------------------------------------------------- 1 file changed, 225 deletions(-) delete mode 100644 binaryjunk.py diff --git a/binaryjunk.py b/binaryjunk.py deleted file mode 100644 index 4df82ca..0000000 --- a/binaryjunk.py +++ /dev/null @@ -1,225 +0,0 @@ -# Generate incompressible random data in various ways, and measure speeds. - - -import random -import gc -import timeit -import hashlib -import zlib - - -def randint(size): - """just call random.randint(0, 255)""" - bytes = [] - for i in range(size): - bytes.append(chr(random.randint(0, 255))) - return "".join(bytes) - - -def getrandbits(size): - """just call random.getrandbits(8)""" - bytes = [] - for i in range(size): - bytes.append(chr(random.getrandbits(8))) - return "".join(bytes) - - -def md5ofgetrandbits(size): - """catenate successive MD5 of random byte stream""" - chunks = [] - sum = hashlib.md5() - while size > 0: - sum.update(chr(random.getrandbits(8))) - chunk = sum.digest() - chunks.append(chunk) - size -= len(chunk) - if size < 0: - chunks[-1] = chunks[-1][:size] - return "".join(chunks) - - -def md5ofgetrandbits2(size): - """catenate successive MD5 of random byte stream""" - chunks = [] - sum = hashlib.md5() - for i in range(size/16): - sum.update(chr(random.getrandbits(8))) - chunk = sum.digest() - chunks.append(chunk) - if len(chunks) * 16 < size: - sum.update(chr(random.getrandbits(8))) - chunk = sum.digest() - chunks.append(chunk[:size % 16]) - return "".join(chunks) - - -def sha1ofgetrandbits(size): - """catenate successive SHA1 of random byte stream""" - chunks = [] - sum = hashlib.sha1() - while size > 0: - sum.update(chr(random.getrandbits(8))) - chunk = sum.digest() - chunks.append(chunk) - size -= len(chunk) - if size < 0: - chunks[-1] = chunks[-1][:size] - return "".join(chunks) - - -def sha1ofgetrandbits2(size): - """catenate successive SHA1 of random byte stream""" - chunks = [] - sum = hashlib.sha1() - chunk_size = len(sum.digest()) - for byte in [chr(random.getrandbits(8)) for i in xrange(size / chunk_size)]: - sum.update(byte) - chunk = sum.digest() - chunks.append(chunk) - if size % chunk_size > 0: - sum.update(chr(random.getrandbits(8))) - chunk = sum.digest() - chunks.append(chunk[:size % chunk_size]) - return "".join(chunks) - - -def sha512ofgetrandbits(size): - """catenate successive SHA512 of random byte stream""" - chunks = [] - sum = hashlib.sha512() - chunk_size = len(sum.digest()) - for byte in [chr(random.getrandbits(8)) for i in xrange(size / chunk_size)]: - sum.update(byte) - chunk = sum.digest() - chunks.append(chunk) - if size % chunk_size > 0: - sum.update(chr(random.getrandbits(8))) - chunk = sum.digest() - chunks.append(chunk[:size % chunk_size]) - return "".join(chunks) - - -def md5ofrandomandstatic(size): - """MD5 first of random byte stream, then constant""" - chunks = [] - sum = hashlib.md5() - - initial_size = 128 - while size > 0 and initial_size > 0: - sum.update(chr(random.getrandbits(8))) - chunk = sum.digest() - chunks.append(chunk) - size -= len(chunk) - - while size > 0: - sum.update("a") - chunk = sum.digest() - chunks.append(chunk) - size -= len(chunk) - - if size < 0: - chunks[-1] = chunks[-1][:size] - - return "".join(chunks) - - -def md5ofrandomandstatic2(size): - """MD5 first of random byte stream, then constant""" - chunks = [] - sum = hashlib.md5() - chunk_size = len(sum.digest()) - - initial_bytes = min(size, chunk_size * 8) - for i in range(initial_bytes / chunk_size): - sum.update(chr(random.getrandbits(8))) - chunks.append(sum.digest()) - - size -= len(chunks) * chunk_size - for i in range(size / chunk_size): - sum.update("a") - chunks.append(sum.digest()) - - if size % chunk_size > 0: - sum.update(chr(random.getrandbits(8))) - chunks.append(sum.digest()[:size % chunk_size]) - - return "".join(chunks) - - -def sha1ofrandomandstatic2(size): - """SHA1 first of random byte stream, then constant""" - chunks = [] - sum = hashlib.sha1() - chunk_size = len(sum.digest()) - - initial_bytes = min(size, 128) - for i in range(initial_bytes / chunk_size): - sum.update(chr(random.getrandbits(8))) - chunk = sum.digest() - chunks.append(chunk) - - size -= len(chunks) * chunk_size - for i in range(size / chunk_size): - sum.update("a") - chunk = sum.digest() - chunks.append(chunk) - - if size % chunk_size > 0: - sum.update(chr(random.getrandbits(8))) - chunk = sum.digest() - chunks.append(chunk[:size % chunk_size]) - - return "".join(chunks) - - -funcs = [ - randint, - getrandbits, - md5ofgetrandbits, - md5ofgetrandbits2, - sha1ofgetrandbits, - sha1ofgetrandbits2, - md5ofrandomandstatic, - md5ofrandomandstatic2, - sha1ofrandomandstatic2, - sha512ofgetrandbits, - ] - - -def measure(func, block, count): - gc.collect() - timer = timeit.Timer(stmt='%s(%d)' % (func.func_name, block), - setup="from __main__ import %s" % func.func_name) - return min(timer.repeat(repeat=count, number=1)) - - -def check(func, block): - data = func(block) - assert len(data) == block, \ - "data is %d bytes, should be %d" % (len(data), block) - assert len(zlib.compress(data)) >= 0.9 * block, \ - "compressed data is %d bytes, should be at least %d" % \ - (len(zlib.compress(data)), 0.9 * block) - - -def main(): - block = 1024**2 - count = 10 - print "Measuring %d functions for generating uncompressible binary junk"%\ - len(funcs) - print "Each function generates %d times %d bytes" % (count, block) - print "This will take a while" - print - - namelen = max(len(func.func_name) for func in funcs) - - for func in funcs: - check(func, block) - secs = measure(func, block, count) - speed = block/secs/(1024**2) - print "%4.1f MB/s %-*s %s" % \ - (speed, namelen, func.func_name, func.__doc__) - - -if __name__ == "__main__": - main() -- cgit v1.2.1 From d3bf8fd9fff9a825e79e14c5005b6ba209b417d2 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Sun, 2 Jan 2011 13:11:51 +0000 Subject: Add setup.py to list of modules without test modules. --- Makefile | 2 +- without-tests | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 without-tests diff --git a/Makefile b/Makefile index b79bece..86c5013 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ all: check: - python -m CoverageTestRunner --ignore-missing without-tests + python -m CoverageTestRunner --ignore-missing-from without-tests # ./blackboxtest clean: diff --git a/without-tests b/without-tests new file mode 100644 index 0000000..2c4830c --- /dev/null +++ b/without-tests @@ -0,0 +1,2 @@ +./setup.py + -- cgit v1.2.1 From aa0aa384c178994f054c76aaafee8782d778feb1 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Sun, 2 Jan 2011 13:12:19 +0000 Subject: Add initial code package. --- genbackupdatalib/__init__.py | 18 ++++++++++++++++++ genbackupdatalib/__init___tests.py | 25 +++++++++++++++++++++++++ 2 files changed, 43 insertions(+) create mode 100644 genbackupdatalib/__init__.py create mode 100644 genbackupdatalib/__init___tests.py diff --git a/genbackupdatalib/__init__.py b/genbackupdatalib/__init__.py new file mode 100644 index 0000000..0b306c2 --- /dev/null +++ b/genbackupdatalib/__init__.py @@ -0,0 +1,18 @@ +# Copyright 2010 Lars Wirzenius +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + +version = '1.3' + diff --git a/genbackupdatalib/__init___tests.py b/genbackupdatalib/__init___tests.py new file mode 100644 index 0000000..d1bf452 --- /dev/null +++ b/genbackupdatalib/__init___tests.py @@ -0,0 +1,25 @@ +# Copyright 2010 Lars Wirzenius +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + +import unittest + +import genbackupdatalib + + +class GenbackupdataTests(unittest.TestCase): + + def test_provides_version_number(self): + self.assertTrue(hasattr(genbackupdatalib, 'version')) -- cgit v1.2.1 From 10c371cae1ee954095f93e7d5ce17d0db80db27c Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Sun, 2 Jan 2011 13:29:37 +0000 Subject: Remove __init__ test module. It make CoverageTestRunner's module importing break. --- genbackupdatalib/__init___tests.py | 25 ------------------------- without-tests | 2 +- 2 files changed, 1 insertion(+), 26 deletions(-) delete mode 100644 genbackupdatalib/__init___tests.py diff --git a/genbackupdatalib/__init___tests.py b/genbackupdatalib/__init___tests.py deleted file mode 100644 index d1bf452..0000000 --- a/genbackupdatalib/__init___tests.py +++ /dev/null @@ -1,25 +0,0 @@ -# Copyright 2010 Lars Wirzenius -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - - -import unittest - -import genbackupdatalib - - -class GenbackupdataTests(unittest.TestCase): - - def test_provides_version_number(self): - self.assertTrue(hasattr(genbackupdatalib, 'version')) diff --git a/without-tests b/without-tests index 2c4830c..f849124 100644 --- a/without-tests +++ b/without-tests @@ -1,2 +1,2 @@ ./setup.py - +./genbackupdatalib/__init__.py -- cgit v1.2.1 From 05e8ae72cfc315d35018df0d819aadf8d3659d43 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Sun, 2 Jan 2011 13:29:50 +0000 Subject: Remove byte code files from Python package. --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 86c5013..a02d0a2 100644 --- a/Makefile +++ b/Makefile @@ -5,7 +5,7 @@ check: # ./blackboxtest clean: - rm -rf *.pyc *.pyo build dist MANIFEST + rm -rf *.py[co] */*.py[co] build dist MANIFEST rm -f blackboxtest.log blackboxtest-genbackupdata.log dist: -- cgit v1.2.1 From a4ac69a5ef3e645e1814f73b075e55e69308487a Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Sun, 2 Jan 2011 13:30:11 +0000 Subject: Add dummy DataGenerator class, with some tests. --- genbackupdatalib/__init__.py | 2 ++ genbackupdatalib/generator.py | 26 ++++++++++++++++++++++++++ genbackupdatalib/generator_tests.py | 35 +++++++++++++++++++++++++++++++++++ 3 files changed, 63 insertions(+) create mode 100644 genbackupdatalib/generator.py create mode 100644 genbackupdatalib/generator_tests.py diff --git a/genbackupdatalib/__init__.py b/genbackupdatalib/__init__.py index 0b306c2..0a5de5c 100644 --- a/genbackupdatalib/__init__.py +++ b/genbackupdatalib/__init__.py @@ -16,3 +16,5 @@ version = '1.3' +from generator import DataGenerator + diff --git a/genbackupdatalib/generator.py b/genbackupdatalib/generator.py new file mode 100644 index 0000000..58ad03d --- /dev/null +++ b/genbackupdatalib/generator.py @@ -0,0 +1,26 @@ +# Copyright 2010 Lars Wirzenius +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + +class DataGenerator(object): + + '''Generate random binary data.''' + + def __init__(self, seed): + pass + + def generate(self, size): + return '' + diff --git a/genbackupdatalib/generator_tests.py b/genbackupdatalib/generator_tests.py new file mode 100644 index 0000000..e962c11 --- /dev/null +++ b/genbackupdatalib/generator_tests.py @@ -0,0 +1,35 @@ +# Copyright 2010 Lars Wirzenius +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + +import unittest + +import genbackupdatalib + + +class DataGeneratorTests(unittest.TestCase): + + def setUp(self): + self.g1 = genbackupdatalib.DataGenerator(0) + self.g2 = genbackupdatalib.DataGenerator(0) + + def test_every_generator_returns_same_sequence(self): + amount = 1024 + self.assertEqual(self.g1.generate(amount), self.g2.generate(amount)) + + def test_returns_different_sequence_for_different_seed(self): + amount = 1024 + g3 = genbackupdatalib.DataGenerator(1) + self.assertNotEqual(self.g1.generate(amount), g3.generate(amount)) -- cgit v1.2.1 From ed86ea775f6711d091500c0f08fba2b282fdfc42 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Sun, 2 Jan 2011 13:55:43 +0000 Subject: Implement DataGenerator. --- genbackupdatalib/generator.py | 40 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 38 insertions(+), 2 deletions(-) diff --git a/genbackupdatalib/generator.py b/genbackupdatalib/generator.py index 58ad03d..8cf349c 100644 --- a/genbackupdatalib/generator.py +++ b/genbackupdatalib/generator.py @@ -14,13 +14,49 @@ # along with this program. If not, see . +import random +import struct + + class DataGenerator(object): '''Generate random binary data.''' + # We generate data by using a blob of suitable size. The output + # sequence repeats the blob, where each repetition is preceded by + # a 64-bit counter. + # + # We need to be relatively prime with obnam's chunk size, which + # defaults to 64 KiB (65536 bytes). This is so that obnam does not + # notice a lot of duplicated data, resulting in unrealistically + # high amounts of compression in the backup store. + # + # Ideally, we would not generate any repeating data, but the random + # number generator is not fast enough for that. We need to generate + # data about as fast as the disk can write it, and the random number + # generator is orders of magnitude slower than that. + + _blob_size = 65521 + _blob_size = 1021 + def __init__(self, seed): - pass + self._random = random.Random(seed) + self._blob = self._generate_blob() + self._counter = 0 + self._buffer = '' + + def _generate_blob(self): + return ''.join(chr(self._random.randint(0, 255)) + for i in range(self._blob_size)) def generate(self, size): - return '' + while size > len(self._buffer): + self._buffer += self._generate_more_data() + data = self._buffer[:size] + self._buffer = self._buffer[size:] + return data + + def _generate_more_data(self): + self._counter += 1 + return struct.pack('!Q', self._counter) + self._blob -- cgit v1.2.1 From bb97f9e099c305128c5129bc396fb2983fc5fa90 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Sun, 2 Jan 2011 14:16:10 +0000 Subject: Add script to measure speed of data generation. --- generate-speed | 70 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100755 generate-speed diff --git a/generate-speed b/generate-speed new file mode 100755 index 0000000..472c8f2 --- /dev/null +++ b/generate-speed @@ -0,0 +1,70 @@ +#!/usr/bin/python +# Copyright 2010 Lars Wirzenius +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + +import cProfile +import sys +import time + +import genbackupdatalib + + +def measure(repeats, func, arg, do_profile, profname): + def helper(): + for i in range(repeats): + func(arg) + + print 'measuring', profname + start_time = time.time() + start = time.clock() + if do_profile: + globaldict = globals().copy() + localdict = locals().copy() + cProfile.runctx('helper()', globaldict, localdict, + '%s.prof' % profname) + else: + helper() + end = time.clock() + end_time = time.time() + return end - start, end_time - start_time + + +def nop(size): + pass + + +def main(): + repeats = int(sys.argv[1]) + size1 = int(sys.argv[2]) + do_profile = sys.argv[3] == 'yes' + looptime = measure(repeats, nop, None, do_profile, 'calibrate') + + g = genbackupdatalib.DataGenerator(0) + result = measure(repeats, g.generate, size1, do_profile, 'generate') + + def speed(result, i): + total_data = repeats * size1 + return total_data / (result[i] - looptime[i]) + def humansize(size): + return '%4.1f MiB/s' % (size / 1024 / 1024) + def report(label, result): + cpu, wall = result + print '%-12s: %5.3f s (%8s)' % \ + (label, cpu, humansize(speed(result, 0))) + report('generate', result) + +if __name__ == '__main__': + main() -- cgit v1.2.1 From 357c3eb2073af7bf7d9889ba148f6f0174259e12 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Sun, 2 Jan 2011 14:25:21 +0000 Subject: Make sure distinct chunks get generated. --- genbackupdatalib/generator_tests.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/genbackupdatalib/generator_tests.py b/genbackupdatalib/generator_tests.py index e962c11..80d12b4 100644 --- a/genbackupdatalib/generator_tests.py +++ b/genbackupdatalib/generator_tests.py @@ -33,3 +33,10 @@ class DataGeneratorTests(unittest.TestCase): amount = 1024 g3 = genbackupdatalib.DataGenerator(1) self.assertNotEqual(self.g1.generate(amount), g3.generate(amount)) + + def test_returns_distinct_64k_chunks(self): + size = 64 * 1024 + chunk1 = self.g1.generate(size) + num_chunks = 100 + for i in range(num_chunks): + self.assertNotEqual(self.g1.generate(size), chunk1) -- cgit v1.2.1 From f1359104817d7454bd078b1e6d507a763a5a2ef5 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Thu, 6 Jan 2011 21:58:02 +0000 Subject: Remove useless main program. --- genbackupdata | 6 ------ 1 file changed, 6 deletions(-) delete mode 100755 genbackupdata diff --git a/genbackupdata b/genbackupdata deleted file mode 100755 index 8467396..0000000 --- a/genbackupdata +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/python -# -# Run genbackupdata.Application as a command line utility. - -import sys, genbackupdata -genbackupdata.Application(sys.argv[1:]).run() -- cgit v1.2.1 From b7532e9b4f753b16808581a58d4f127db6c81ebb Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Thu, 20 Jan 2011 20:33:09 +0000 Subject: Add beginnings of a generator for output file names. --- genbackupdatalib/__init__.py | 2 +- genbackupdatalib/names.py | 25 +++++++++++++++++++++++++ genbackupdatalib/names_tests.py | 37 +++++++++++++++++++++++++++++++++++++ 3 files changed, 63 insertions(+), 1 deletion(-) create mode 100644 genbackupdatalib/names.py create mode 100644 genbackupdatalib/names_tests.py diff --git a/genbackupdatalib/__init__.py b/genbackupdatalib/__init__.py index 0a5de5c..b7771db 100644 --- a/genbackupdatalib/__init__.py +++ b/genbackupdatalib/__init__.py @@ -17,4 +17,4 @@ version = '1.3' from generator import DataGenerator - +from names import NameGenerator diff --git a/genbackupdatalib/names.py b/genbackupdatalib/names.py new file mode 100644 index 0000000..1ce0c20 --- /dev/null +++ b/genbackupdatalib/names.py @@ -0,0 +1,25 @@ +# Copyright 2011 Lars Wirzenius +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + +class NameGenerator(object): + + '''Generate names for new output files.''' + + def __init__(self, dirname): + self.dirname = dirname + + def new(self): + return 'foo' diff --git a/genbackupdatalib/names_tests.py b/genbackupdatalib/names_tests.py new file mode 100644 index 0000000..7590921 --- /dev/null +++ b/genbackupdatalib/names_tests.py @@ -0,0 +1,37 @@ +# Copyright 2011 Lars Wirzenius +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + +import os +import shutil +import tempfile +import unittest + +import genbackupdatalib + + +class NameGeneratorTests(unittest.TestCase): + + def setUp(self): + self.tempdir = tempfile.mkdtemp() + self.names = genbackupdatalib.NameGenerator(self.tempdir) + + def tearDown(self): + shutil.rmtree(self.tempdir) + + def test_generates_name_that_does_not_exist(self): + name = self.names.new() + self.assertFalse(os.path.exists(name)) + -- cgit v1.2.1 From 6865d3ff7d9d0718fdbc6b8b8deb36bca9534dfb Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Thu, 20 Jan 2011 20:43:21 +0000 Subject: Add more functionality and tests to NameGenerator. --- genbackupdatalib/names.py | 15 +++++++++++++-- genbackupdatalib/names_tests.py | 28 ++++++++++++++++++++++++++-- 2 files changed, 39 insertions(+), 4 deletions(-) diff --git a/genbackupdatalib/names.py b/genbackupdatalib/names.py index 1ce0c20..8cca24c 100644 --- a/genbackupdatalib/names.py +++ b/genbackupdatalib/names.py @@ -14,12 +14,23 @@ # along with this program. If not, see . +import os + + class NameGenerator(object): '''Generate names for new output files.''' def __init__(self, dirname): self.dirname = dirname - + self.counter = 0 + + def _next_candidate_name(self): + self.counter += 1 + return os.path.join(self.dirname, 'file%d' % self.counter) + def new(self): - return 'foo' + while True: + name = self._next_candidate_name() + if not os.path.exists(name): + return name diff --git a/genbackupdatalib/names_tests.py b/genbackupdatalib/names_tests.py index 7590921..55cdfd7 100644 --- a/genbackupdatalib/names_tests.py +++ b/genbackupdatalib/names_tests.py @@ -31,7 +31,31 @@ class NameGeneratorTests(unittest.TestCase): def tearDown(self): shutil.rmtree(self.tempdir) - def test_generates_name_that_does_not_exist(self): + def test_generates_name_that_is_inside_target_directory(self): name = self.names.new() - self.assertFalse(os.path.exists(name)) + self.assert_(name.startswith(self.tempdir + os.sep)) + + def test_generates_different_names_every_time(self): + names = set(self.names.new() for i in range(10)) + self.assertEqual(len(names), 10) + + def test_generates_names_that_do_not_exist(self): + for i in range(10): + name = self.names.new() + self.assertFalse(os.path.exists(name)) + + def test_generates_the_same_sequence_with_every_instance(self): + n = 10 + first = [self.names.new() for i in range(n)] + names2 = genbackupdatalib.NameGenerator(self.tempdir) + second = [names2.new() for i in range(n)] + self.assertEqual(first, second) + + def test_does_not_generate_names_of_existing_files(self): + name = self.names.new() + file(name, 'w').close() + names2 = genbackupdatalib.NameGenerator(self.tempdir) + name2 = names2.new() + self.assertNotEqual(name, name2) + self.assertFalse(os.path.exists(name2)) -- cgit v1.2.1 From f3d7ef0e9ec984874dec13c1e5eb366c29bee8d2 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Sun, 23 Jan 2011 16:13:08 +0000 Subject: Create file path tuples for arbitrary file sequence numbers. --- genbackupdatalib/names.py | 41 +++++++++++++++++++++++++++++++++++++++-- genbackupdatalib/names_tests.py | 26 +++++++++++++++++++++++--- 2 files changed, 62 insertions(+), 5 deletions(-) diff --git a/genbackupdatalib/names.py b/genbackupdatalib/names.py index 8cca24c..02a47ef 100644 --- a/genbackupdatalib/names.py +++ b/genbackupdatalib/names.py @@ -19,12 +19,49 @@ import os class NameGenerator(object): - '''Generate names for new output files.''' + '''Generate names for new output files. - def __init__(self, dirname): + If the target directory is empty, the sequence of output files is + always the same for the same parameters. + + A directory structure is also generated. The shape of the tree is + defined by two parameters: 'max' and 'depth'. 'depth' is the number + of levels of subdirectories to create, and 'max' is the maximum + number of files/dirs to allow per output directory. Thus, if max is + 3 and depth is 2, the output files are: 0/0/0, 0/0/1, 0/0/2, + 0/1/0, 0/1/1, etc. + + If depth is zero, all output files go directly to the target + directory, and max is ignored. + + ''' + + def __init__(self, dirname, depth, max): self.dirname = dirname + self.depth = depth + self.max = max self.counter = 0 + def _path_tuple(self, n): + '''Return tuple for dir/file numbers for nth output file. + + The last item in the tuple gives the file number, the precding + items the directory numbers. Thus, a tuple (1, 2, 3) would + mean path '1/2/3', but it is given as a tuple for easier + manipulation. + + ''' + + if self.depth == 0: + return (n,) + else: + items = [] + for i in range(self.depth + 1): # +1 for filenames + items.append(n % self.max) + n /= self.max + items.reverse() + return tuple(items) + def _next_candidate_name(self): self.counter += 1 return os.path.join(self.dirname, 'file%d' % self.counter) diff --git a/genbackupdatalib/names_tests.py b/genbackupdatalib/names_tests.py index 55cdfd7..f31a61b 100644 --- a/genbackupdatalib/names_tests.py +++ b/genbackupdatalib/names_tests.py @@ -26,10 +26,16 @@ class NameGeneratorTests(unittest.TestCase): def setUp(self): self.tempdir = tempfile.mkdtemp() - self.names = genbackupdatalib.NameGenerator(self.tempdir) + self.depth = 2 + self.max = 3 + self.names = self.new() def tearDown(self): shutil.rmtree(self.tempdir) + + def new(self): + return genbackupdatalib.NameGenerator(self.tempdir, self.depth, + self.max) def test_generates_name_that_is_inside_target_directory(self): name = self.names.new() @@ -47,15 +53,29 @@ class NameGeneratorTests(unittest.TestCase): def test_generates_the_same_sequence_with_every_instance(self): n = 10 first = [self.names.new() for i in range(n)] - names2 = genbackupdatalib.NameGenerator(self.tempdir) + names2 = self.new() second = [names2.new() for i in range(n)] self.assertEqual(first, second) def test_does_not_generate_names_of_existing_files(self): name = self.names.new() file(name, 'w').close() - names2 = genbackupdatalib.NameGenerator(self.tempdir) + names2 = self.new() name2 = names2.new() self.assertNotEqual(name, name2) self.assertFalse(os.path.exists(name2)) + def test_converts_file_sequence_number_into_right_path_tuple(self): + self.assertEqual(self.names._path_tuple(0), (0, 0, 0)) + self.assertEqual(self.names._path_tuple(1), (0, 0, 1)) + self.assertEqual(self.names._path_tuple(2), (0, 0, 2)) + self.assertEqual(self.names._path_tuple(3), (0, 1, 0)) + self.assertEqual(self.names._path_tuple(4), (0, 1, 1)) + self.assertEqual(self.names._path_tuple(5), (0, 1, 2)) + self.assertEqual(self.names._path_tuple(6), (0, 2, 0)) + self.assertEqual(self.names._path_tuple(9), (1, 0, 0)) + + def test_returns_1tuple_for_depth_zero(self): + names = genbackupdatalib.NameGenerator(self.tempdir, 0, 1) + self.assertEqual(names._path_tuple(42), (42,)) + -- cgit v1.2.1 From f26fbfcffe542c3919b0132190a2b86372592836 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Sun, 23 Jan 2011 16:15:22 +0000 Subject: Use _path_tuple to generate names. --- genbackupdatalib/names.py | 3 ++- genbackupdatalib/names_tests.py | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/genbackupdatalib/names.py b/genbackupdatalib/names.py index 02a47ef..f6ba4e1 100644 --- a/genbackupdatalib/names.py +++ b/genbackupdatalib/names.py @@ -63,8 +63,9 @@ class NameGenerator(object): return tuple(items) def _next_candidate_name(self): + items = self._path_tuple(self.counter) self.counter += 1 - return os.path.join(self.dirname, 'file%d' % self.counter) + return os.path.join(self.dirname, *[str(i) for i in items]) def new(self): while True: diff --git a/genbackupdatalib/names_tests.py b/genbackupdatalib/names_tests.py index f31a61b..2f3483f 100644 --- a/genbackupdatalib/names_tests.py +++ b/genbackupdatalib/names_tests.py @@ -59,6 +59,7 @@ class NameGeneratorTests(unittest.TestCase): def test_does_not_generate_names_of_existing_files(self): name = self.names.new() + os.makedirs(os.path.dirname(name)) file(name, 'w').close() names2 = self.new() name2 = names2.new() -- cgit v1.2.1 From c4371b9cc34a40155d46e7bef348b0a2c5a1b3b3 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Sun, 23 Jan 2011 16:24:50 +0000 Subject: Fix path tuples when there are more than max subdirs at root. --- genbackupdatalib/names.py | 3 ++- genbackupdatalib/names_tests.py | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/genbackupdatalib/names.py b/genbackupdatalib/names.py index f6ba4e1..287112d 100644 --- a/genbackupdatalib/names.py +++ b/genbackupdatalib/names.py @@ -56,9 +56,10 @@ class NameGenerator(object): return (n,) else: items = [] - for i in range(self.depth + 1): # +1 for filenames + for i in range(self.depth): items.append(n % self.max) n /= self.max + items.append(n) items.reverse() return tuple(items) diff --git a/genbackupdatalib/names_tests.py b/genbackupdatalib/names_tests.py index 2f3483f..60b4d79 100644 --- a/genbackupdatalib/names_tests.py +++ b/genbackupdatalib/names_tests.py @@ -75,6 +75,8 @@ class NameGeneratorTests(unittest.TestCase): self.assertEqual(self.names._path_tuple(5), (0, 1, 2)) self.assertEqual(self.names._path_tuple(6), (0, 2, 0)) self.assertEqual(self.names._path_tuple(9), (1, 0, 0)) + self.assertEqual(self.names._path_tuple(18), (2, 0, 0)) + self.assertEqual(self.names._path_tuple(27), (3, 0, 0)) def test_returns_1tuple_for_depth_zero(self): names = genbackupdatalib.NameGenerator(self.tempdir, 0, 1) -- cgit v1.2.1 From 4dea0df19b82ecc2d6e51bf8528d1acd94666493 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Sun, 23 Jan 2011 17:09:06 +0000 Subject: Start on new main program. --- genbackupdata | 104 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 104 insertions(+) create mode 100755 genbackupdata diff --git a/genbackupdata b/genbackupdata new file mode 100755 index 0000000..6233989 --- /dev/null +++ b/genbackupdata @@ -0,0 +1,104 @@ +#!/usr/bin/python +# Copyright 2011 Lars Wirzenius +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + +import cliapp +import os +import sys +import ttystatus + +import genbackupdatalib + + +class GenbackupdataApp(cliapp.Application): + + def add_settings(self): + self.add_bytesize_setting(['create', 'c'], + 'how much data to create ' + '(default: %default)') + self.add_bytesize_setting(['file-size'], + 'size of one file (default: %default)', + default=16*1024) + self.add_bytesize_setting(['chunk-size'], + 'generate data in chunks of this size ' + '(default: %default)', + default=16*1024) + self.add_integer_setting(['depth'], + 'depth of directory tree (default: %default)', + default=3) + self.add_integer_setting(['max-files'], + 'max files/dirs per dir (default: %default)', + default=128) + self.add_integer_setting(['seed'], + 'seed for random number generator ' + '(default: %default)', + default=0) + + def process_args(self, args): + outputdir = args[0] + bytes = self['create'] + self.gen = genbackupdatalib.DataGenerator(self['seed']) + self.names = genbackupdatalib.NameGenerator(outputdir, self['depth'], + self['max-files']) + + self.setup_ttystatus() + self.status['total'] = bytes + while bytes > 0: + n = min(self['file-size'], bytes) + self.create_file(n) + bytes -= n + self.status.finish() + + def create_file(self, bytes): + '''Generate one output file.''' + + file_size = self['file-size'] + chunk_size = self['chunk-size'] + pathname = self.names.new() + dirname = os.path.dirname(pathname) + if not os.path.exists(dirname): + os.makedirs(dirname) + f = open(pathname, 'wb') + while bytes >= chunk_size: + self.write_bytes(f, chunk_size) + bytes -= chunk_size + if bytes > 0: + self.write(f, bytes) + f.close() + + def write_bytes(self, f, bytes): + chunk = self.gen.generate(bytes) + f.write(chunk) + self.status['written'] += bytes + + def setup_ttystatus(self): + self.status = ttystatus.TerminalStatus(period=0.1) + self.status['written'] = 0 + self.status['total'] = 0 + self.status.add(ttystatus.Literal('Generating: ')) + self.status.add(ttystatus.ByteSize('written')) + self.status.add(ttystatus.Literal(' of ')) + self.status.add(ttystatus.ByteSize('total')) + self.status.add(ttystatus.Literal(' ')) + self.status.add(ttystatus.PercentDone('written', 'total')) + self.status.add(ttystatus.Literal(' (')) + self.status.add(ttystatus.ByteSpeed('written')) + self.status.add(ttystatus.Literal(')')) + + +if __name__ == '__main__': + GenbackupdataApp().run() + -- cgit v1.2.1 From 82ecdda87b711f34d72e88421069cb7afdf9b984 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Tue, 1 Feb 2011 21:29:24 +0000 Subject: Enable blackboxtest in 'make check'. --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index a02d0a2..129f2e4 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ all: check: python -m CoverageTestRunner --ignore-missing-from without-tests -# ./blackboxtest + ./blackboxtest clean: rm -rf *.py[co] */*.py[co] build dist MANIFEST -- cgit v1.2.1 From 10517c005c7fb6c4b43cade43a506aa7d9415459 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Tue, 1 Feb 2011 21:30:06 +0000 Subject: Fix bug that called wrong method name. --- genbackupdata | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/genbackupdata b/genbackupdata index 6233989..dbdb905 100755 --- a/genbackupdata +++ b/genbackupdata @@ -76,7 +76,7 @@ class GenbackupdataApp(cliapp.Application): self.write_bytes(f, chunk_size) bytes -= chunk_size if bytes > 0: - self.write(f, bytes) + self.write_bytes(f, bytes) f.close() def write_bytes(self, f, bytes): @@ -101,4 +101,4 @@ class GenbackupdataApp(cliapp.Application): if __name__ == '__main__': GenbackupdataApp().run() - + -- cgit v1.2.1 From d027e70ab0b2ac89a1789a3eb6a8e2c8465e8c82 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Tue, 1 Feb 2011 21:31:43 +0000 Subject: Add --quiet option. --- genbackupdata | 3 +++ 1 file changed, 3 insertions(+) diff --git a/genbackupdata b/genbackupdata index dbdb905..becd5e9 100755 --- a/genbackupdata +++ b/genbackupdata @@ -46,6 +46,7 @@ class GenbackupdataApp(cliapp.Application): 'seed for random number generator ' '(default: %default)', default=0) + self.add_boolean_setting(['quiet'], 'do not report progress') def process_args(self, args): outputdir = args[0] @@ -86,6 +87,8 @@ class GenbackupdataApp(cliapp.Application): def setup_ttystatus(self): self.status = ttystatus.TerminalStatus(period=0.1) + if self['quiet']: + self.status.disable() self.status['written'] = 0 self.status['total'] = 0 self.status.add(ttystatus.Literal('Generating: ')) -- cgit v1.2.1 From ce7ae3a3816adfa469140a535977dd656849ea7c Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Tue, 1 Feb 2011 21:32:19 +0000 Subject: Make genbackupdata runs during blackboxtests be quiet. --- blackboxtest | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/blackboxtest b/blackboxtest index e54519e..c4d79c0 100755 --- a/blackboxtest +++ b/blackboxtest @@ -97,7 +97,7 @@ class GenbackupdataTestCase(unittest.TestCase): def genbackupdata(self, args, stderr_ignore=None): '''Run genbackupdata, with some default arguments.''' - return self.runcmd(['./genbackupdata'] + + return self.runcmd(['./genbackupdata', '--quiet'] + args, stderr_ignore=stderr_ignore) def create_file(self, dirname, relative, contents): -- cgit v1.2.1