summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorLars Wirzenius <liw@liw.fi>2011-02-01 21:48:06 +0000
committerLars Wirzenius <liw@liw.fi>2011-02-01 21:48:06 +0000
commit5638f50298796476296bf3954540cb1a9835024e (patch)
treefdbcbb387a9cfe338a7a96fa8e88b7aae724c27e
parent4ebabf3aea7f081ac83d8a6e9f186fb33130dd27 (diff)
parentce7ae3a3816adfa469140a535977dd656849ea7c (diff)
downloadgenbackupdata-5638f50298796476296bf3954540cb1a9835024e.tar.gz
Merge rewrite.
-rw-r--r--.bzrignore1
-rw-r--r--Makefile9
-rw-r--r--binaryjunk.py225
-rwxr-xr-xblackboxtest286
-rw-r--r--createfiles-ext3-with-dirindex-0B.pngbin5518 -> 0 bytes
-rw-r--r--createfiles-ext3-with-dirindex-16384B.pngbin4634 -> 0 bytes
-rw-r--r--createfiles-ext3-with-dirindex-4096B.pngbin6216 -> 0 bytes
-rw-r--r--createfiles-ext3-with-dirindex-8192B.pngbin5567 -> 0 bytes
-rwxr-xr-xcreatefiles-plot.sh51
-rw-r--r--createfiles.output-ext3-with-dirindex528
-rwxr-xr-xcreatefiles.py162
-rwxr-xr-xgenbackupdata109
-rw-r--r--genbackupdata.py660
-rw-r--r--genbackupdatalib/__init__.py20
-rw-r--r--genbackupdatalib/generator.py62
-rw-r--r--genbackupdatalib/generator_tests.py42
-rw-r--r--genbackupdatalib/names.py75
-rw-r--r--genbackupdatalib/names_tests.py84
-rwxr-xr-xgenerate-speed70
-rwxr-xr-xtests.py565
-rw-r--r--without-tests2
21 files changed, 751 insertions, 2200 deletions
diff --git a/.bzrignore b/.bzrignore
index 6350e98..1cbec07 100644
--- a/.bzrignore
+++ b/.bzrignore
@@ -1 +1,2 @@
.coverage
+blackboxtest.log
diff --git a/Makefile b/Makefile
index 79cbdcd..129f2e4 100644
--- a/Makefile
+++ b/Makefile
@@ -1,13 +1,12 @@
all:
check:
- python-coverage -e
- python-coverage -x tests.py
- python-coverage -r -m -o /usr,/var | \
- awk '{ print } /^TOTAL/ && $$2 != $$3 {exit 1}'
+ python -m CoverageTestRunner --ignore-missing-from without-tests
+ ./blackboxtest
clean:
- rm -rf *.pyc *.pyo build dist MANIFEST
+ rm -rf *.py[co] */*.py[co] build dist MANIFEST
+ rm -f blackboxtest.log blackboxtest-genbackupdata.log
dist:
python setup.py sdist
diff --git a/binaryjunk.py b/binaryjunk.py
deleted file mode 100644
index 4df82ca..0000000
--- a/binaryjunk.py
+++ /dev/null
@@ -1,225 +0,0 @@
-# Generate incompressible random data in various ways, and measure speeds.
-
-
-import random
-import gc
-import timeit
-import hashlib
-import zlib
-
-
-def randint(size):
- """just call random.randint(0, 255)"""
- bytes = []
- for i in range(size):
- bytes.append(chr(random.randint(0, 255)))
- return "".join(bytes)
-
-
-def getrandbits(size):
- """just call random.getrandbits(8)"""
- bytes = []
- for i in range(size):
- bytes.append(chr(random.getrandbits(8)))
- return "".join(bytes)
-
-
-def md5ofgetrandbits(size):
- """catenate successive MD5 of random byte stream"""
- chunks = []
- sum = hashlib.md5()
- while size > 0:
- sum.update(chr(random.getrandbits(8)))
- chunk = sum.digest()
- chunks.append(chunk)
- size -= len(chunk)
- if size < 0:
- chunks[-1] = chunks[-1][:size]
- return "".join(chunks)
-
-
-def md5ofgetrandbits2(size):
- """catenate successive MD5 of random byte stream"""
- chunks = []
- sum = hashlib.md5()
- for i in range(size/16):
- sum.update(chr(random.getrandbits(8)))
- chunk = sum.digest()
- chunks.append(chunk)
- if len(chunks) * 16 < size:
- sum.update(chr(random.getrandbits(8)))
- chunk = sum.digest()
- chunks.append(chunk[:size % 16])
- return "".join(chunks)
-
-
-def sha1ofgetrandbits(size):
- """catenate successive SHA1 of random byte stream"""
- chunks = []
- sum = hashlib.sha1()
- while size > 0:
- sum.update(chr(random.getrandbits(8)))
- chunk = sum.digest()
- chunks.append(chunk)
- size -= len(chunk)
- if size < 0:
- chunks[-1] = chunks[-1][:size]
- return "".join(chunks)
-
-
-def sha1ofgetrandbits2(size):
- """catenate successive SHA1 of random byte stream"""
- chunks = []
- sum = hashlib.sha1()
- chunk_size = len(sum.digest())
- for byte in [chr(random.getrandbits(8)) for i in xrange(size / chunk_size)]:
- sum.update(byte)
- chunk = sum.digest()
- chunks.append(chunk)
- if size % chunk_size > 0:
- sum.update(chr(random.getrandbits(8)))
- chunk = sum.digest()
- chunks.append(chunk[:size % chunk_size])
- return "".join(chunks)
-
-
-def sha512ofgetrandbits(size):
- """catenate successive SHA512 of random byte stream"""
- chunks = []
- sum = hashlib.sha512()
- chunk_size = len(sum.digest())
- for byte in [chr(random.getrandbits(8)) for i in xrange(size / chunk_size)]:
- sum.update(byte)
- chunk = sum.digest()
- chunks.append(chunk)
- if size % chunk_size > 0:
- sum.update(chr(random.getrandbits(8)))
- chunk = sum.digest()
- chunks.append(chunk[:size % chunk_size])
- return "".join(chunks)
-
-
-def md5ofrandomandstatic(size):
- """MD5 first of random byte stream, then constant"""
- chunks = []
- sum = hashlib.md5()
-
- initial_size = 128
- while size > 0 and initial_size > 0:
- sum.update(chr(random.getrandbits(8)))
- chunk = sum.digest()
- chunks.append(chunk)
- size -= len(chunk)
-
- while size > 0:
- sum.update("a")
- chunk = sum.digest()
- chunks.append(chunk)
- size -= len(chunk)
-
- if size < 0:
- chunks[-1] = chunks[-1][:size]
-
- return "".join(chunks)
-
-
-def md5ofrandomandstatic2(size):
- """MD5 first of random byte stream, then constant"""
- chunks = []
- sum = hashlib.md5()
- chunk_size = len(sum.digest())
-
- initial_bytes = min(size, chunk_size * 8)
- for i in range(initial_bytes / chunk_size):
- sum.update(chr(random.getrandbits(8)))
- chunks.append(sum.digest())
-
- size -= len(chunks) * chunk_size
- for i in range(size / chunk_size):
- sum.update("a")
- chunks.append(sum.digest())
-
- if size % chunk_size > 0:
- sum.update(chr(random.getrandbits(8)))
- chunks.append(sum.digest()[:size % chunk_size])
-
- return "".join(chunks)
-
-
-def sha1ofrandomandstatic2(size):
- """SHA1 first of random byte stream, then constant"""
- chunks = []
- sum = hashlib.sha1()
- chunk_size = len(sum.digest())
-
- initial_bytes = min(size, 128)
- for i in range(initial_bytes / chunk_size):
- sum.update(chr(random.getrandbits(8)))
- chunk = sum.digest()
- chunks.append(chunk)
-
- size -= len(chunks) * chunk_size
- for i in range(size / chunk_size):
- sum.update("a")
- chunk = sum.digest()
- chunks.append(chunk)
-
- if size % chunk_size > 0:
- sum.update(chr(random.getrandbits(8)))
- chunk = sum.digest()
- chunks.append(chunk[:size % chunk_size])
-
- return "".join(chunks)
-
-
-funcs = [
- randint,
- getrandbits,
- md5ofgetrandbits,
- md5ofgetrandbits2,
- sha1ofgetrandbits,
- sha1ofgetrandbits2,
- md5ofrandomandstatic,
- md5ofrandomandstatic2,
- sha1ofrandomandstatic2,
- sha512ofgetrandbits,
- ]
-
-
-def measure(func, block, count):
- gc.collect()
- timer = timeit.Timer(stmt='%s(%d)' % (func.func_name, block),
- setup="from __main__ import %s" % func.func_name)
- return min(timer.repeat(repeat=count, number=1))
-
-
-def check(func, block):
- data = func(block)
- assert len(data) == block, \
- "data is %d bytes, should be %d" % (len(data), block)
- assert len(zlib.compress(data)) >= 0.9 * block, \
- "compressed data is %d bytes, should be at least %d" % \
- (len(zlib.compress(data)), 0.9 * block)
-
-
-def main():
- block = 1024**2
- count = 10
- print "Measuring %d functions for generating uncompressible binary junk"%\
- len(funcs)
- print "Each function generates %d times %d bytes" % (count, block)
- print "This will take a while"
- print
-
- namelen = max(len(func.func_name) for func in funcs)
-
- for func in funcs:
- check(func, block)
- secs = measure(func, block, count)
- speed = block/secs/(1024**2)
- print "%4.1f MB/s %-*s %s" % \
- (speed, namelen, func.func_name, func.__doc__)
-
-
-if __name__ == "__main__":
- main()
diff --git a/blackboxtest b/blackboxtest
new file mode 100755
index 0000000..c4d79c0
--- /dev/null
+++ b/blackboxtest
@@ -0,0 +1,286 @@
+#!/usr/bin/python
+#
+# Copyright (C) 2009, 2010 Lars Wirzenius <liw@liw.fi>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+
+'''Run some black box tests for genbackupdata.'''
+
+
+import hashlib
+import logging
+import os
+import random
+import re
+import shutil
+import stat
+import subprocess
+import sys
+import tempfile
+import traceback
+import unittest
+
+
+class GenbackupdataTestCase(unittest.TestCase):
+
+ '''Base class for genbackupdata test cases.
+
+ We use the unittest framework even though these are black box tests,
+ not unit tests. unittest makes implementation of these black box
+ tests convenient, even though that might not be true for all black
+ box tests.
+
+ This base class provides a fresh environment for each test, and
+ cleans up afterwards. It provides helpers for doing the usual
+ backup operations, and for verifyting results.
+
+ '''
+
+ def setUp(self):
+ self.tempdir = tempfile.mkdtemp()
+ self.setUpHook()
+
+ def setUpHook(self):
+ pass
+
+ def tearDown(self):
+ self.tearDownHook()
+ shutil.rmtree(self.tempdir)
+
+ def tearDownHook(self):
+ pass
+
+ def path(self, *relatives):
+ return os.path.join(self.tempdir, *relatives)
+
+ def mkdir(self, dirname):
+ abs_dirname = os.path.join(self.tempdir, dirname)
+ os.makedirs(abs_dirname)
+ return abs_dirname
+
+ def runcmd(self, argv, stderr_ignore=None):
+ '''Run an external command.
+
+ If the command fails (non-zero exit), raise an exception.
+
+ If stderr_ignore is not None, it must be a string with a
+ regexp for lines in stderr to ignore.
+
+ '''
+
+ logging.debug('executing %s' % argv)
+
+ p = subprocess.Popen(argv, stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE)
+ stdout, stderr = p.communicate()
+ if stderr_ignore:
+ lines = [line for line in stderr.splitlines()
+ if not re.match(stderr_ignore, line)]
+ stderr = ''.join(lines)
+ sys.stderr.write(stderr)
+ if p.returncode != 0:
+ raise subprocess.CalledProcessError(p.returncode, argv)
+ return stdout
+
+ def genbackupdata(self, args, stderr_ignore=None):
+ '''Run genbackupdata, with some default arguments.'''
+ return self.runcmd(['./genbackupdata', '--quiet'] +
+ args, stderr_ignore=stderr_ignore)
+
+ def create_file(self, dirname, relative, contents):
+ '''Create a new file with the desired contents.'''
+
+ pathname = os.path.join(dirname, relative)
+ logging.debug('creating file %s' % pathname)
+ f = open(pathname, 'w')
+ f.write(contents)
+ f.close()
+
+ def remove_file(self, root, relative):
+ '''Remove a file.'''
+
+ pathname = os.path.join(root, relative)
+ logging.debug('removing file %s' % pathname)
+ os.remove(pathname)
+
+ def create_dir(self, root, pathname):
+ '''Create a new directory, return name.'''
+ fullname = os.path.join(root, pathname)
+ logging.debug('mkdir %s' % fullname)
+ os.makedirs(fullname)
+ return fullname
+
+ def get_info(self, root, pathname):
+ '''Get the information about a given file.
+
+ Return a tuple (relativepath, stat) where relativepath is the
+ path relative to root, and stat is the result of os.lstat.
+
+ '''
+
+ root_base = os.path.basename(root)
+ del_prefix = root[:-len(root_base)]
+ if pathname == root:
+ return None
+ assert pathname.startswith(root + os.sep), (pathname, root)
+ return pathname[len(root + os.sep):], os.lstat(pathname)
+
+ def find_everything(self, root):
+ '''Find all filesystem objects inside a directory tree.
+
+ Return list of (pathname, stat) tuples. The pathname will be
+ relative to the root of the directory tree. The stat tuples
+ will be the result of os.lstat for each pathname.
+
+ '''
+
+ result = []
+ for dirname, dirnames, filenames in os.walk(root):
+ result.append(self.get_info(root, dirname))
+ for filename in filenames:
+ pathname = os.path.join(dirname, filename)
+ result.append(self.get_info(root, pathname))
+ return [x for x in result if x]
+
+ def apparent_size(self, root):
+ '''Return sum of length of regular files in directory, recursively.'''
+
+ size = 0
+ for dirname, subdirs, filenames in os.walk(self.path(root)):
+ for filename in filenames:
+ pathname = self.path(dirname, filename)
+ st = os.lstat(pathname)
+ if stat.S_ISREG(st.st_mode):
+ size += st.st_size
+ return size
+
+ def checksum(self, pathname):
+ '''Return MD5 checksum for contents of a file.'''
+ s = hashlib.new('md5')
+ f = open(pathname, 'rb')
+ while True:
+ data = f.read(64*1024)
+ if not data:
+ break
+ s.update(data)
+ f.close()
+ return s.hexdigest()
+
+ def checksums(self, root):
+ '''Return sorted list of (pathname, checksum) pairs for reg. files.'''
+ result = []
+ prefix = self.path(root) + os.sep
+ for dirname, subdirs, filenames in os.walk(self.path(root)):
+ for filename in filenames:
+ pathname = self.path(dirname, filename)
+ st = os.lstat(pathname)
+ if stat.S_ISREG(st.st_mode):
+ assert pathname.startswith(prefix)
+ relative = pathname[len(prefix):]
+ result.append((relative, self.checksum(pathname)))
+ result.sort()
+ return result
+
+ def assert_equal_stat_fields(self, filename, stat1, stat2, fieldname):
+ field1 = getattr(stat1, fieldname)
+ field2 = getattr(stat2, fieldname)
+ self.assertEqual(field1, field2,
+ '%s stat field %s difference: %s vs %s' %
+ (filename, fieldname, repr(field1), repr(field2)))
+
+ def assert_same_stat(self, name, stat1, stat2):
+ '''Are two stat results effectively identical?'''
+
+ class Fake(object):
+
+ def __init__(self, stat_result):
+ self.st = stat_result
+
+ def __getattr__(self, name):
+ if name == 'st_mtime':
+ return int(getattr(self.st, name))
+ else:
+ return getattr(self.st, name)
+
+ self.assert_equal_stat_fields(name, stat1, stat2, 'st_blocks')
+ self.assert_equal_stat_fields(name, stat1, stat2, 'st_gid')
+ self.assert_equal_stat_fields(name, stat1, stat2, 'st_mode')
+ self.assert_equal_stat_fields(name, Fake(stat1), Fake(stat2), 'st_mtime')
+ self.assert_equal_stat_fields(name, stat1, stat2, 'st_nlink')
+ self.assert_equal_stat_fields(name, stat1, stat2, 'st_size')
+ self.assert_equal_stat_fields(name, stat1, stat2, 'st_uid')
+
+ def assert_same_contents(self, relative, root1, root2):
+ '''Verify that file contents has been restored correctly.'''
+
+ path1 = os.path.join(root1, relative)
+ path2 = os.path.join(root2, relative)
+
+ self.assertFilesEqual(path1, path2)
+
+ def assertFileExists(self, path):
+ self.assert_(os.path.exists(path), '%s does not exist' % path)
+
+ def assertIsRegularFile(self, path):
+ self.assert_(os.path.isfile(path), '%s is not a regular file' % path)
+
+ def assertFilesEqual(self, path1, path2):
+ '''Verify that file contents are equal.'''
+
+ self.assertFileExists(path1)
+ self.assertFileExists(path2)
+ self.assertIsRegularFile(path1)
+ self.assertIsRegularFile(path2)
+
+ f1 = open(path1, 'r')
+ f2 = open(path2, 'r')
+
+ data1 = f1.read()
+ data2 = f2.read()
+
+ f1.close()
+ f2.close()
+
+ self.assertEqual(data1, data2,
+ 'contents of %s and %s differ' % (path1, path2))
+
+
+class GenbackupdataTests(GenbackupdataTestCase):
+
+ def test_returns_success_with_help_option(self):
+ self.genbackupdata(['--help'])
+ self.assertTrue(True)
+
+ def test_creates_requested_amount_of_data(self):
+ bytes = 12765
+ self.genbackupdata([self.path('data'), '--create=%d' % bytes])
+ self.assertEqual(self.apparent_size('data'), bytes)
+
+ def test_creates_same_data_every_time(self):
+ size = '10m' # big enough to allow both ample text and binary data
+ self.genbackupdata([self.path('data1'), '--create', size])
+ self.genbackupdata([self.path('data2'), '--create', size])
+ sums1 = self.checksums('data1')
+ sums2 = self.checksums('data2')
+ self.assertEqual(len(sums1), len(sums2))
+ for n in range(1, len(sums1)):
+ self.assertEqual(sums1[:n], sums2[:n])
+
+if __name__ == '__main__':
+ logging.basicConfig(filename='blackboxtest.log',
+ level=logging.DEBUG,
+ format='%(levelname)s: %(message)s')
+ unittest.main()
diff --git a/createfiles-ext3-with-dirindex-0B.png b/createfiles-ext3-with-dirindex-0B.png
deleted file mode 100644
index 102801f..0000000
--- a/createfiles-ext3-with-dirindex-0B.png
+++ /dev/null
Binary files differ
diff --git a/createfiles-ext3-with-dirindex-16384B.png b/createfiles-ext3-with-dirindex-16384B.png
deleted file mode 100644
index eb9e863..0000000
--- a/createfiles-ext3-with-dirindex-16384B.png
+++ /dev/null
Binary files differ
diff --git a/createfiles-ext3-with-dirindex-4096B.png b/createfiles-ext3-with-dirindex-4096B.png
deleted file mode 100644
index 6682c3b..0000000
--- a/createfiles-ext3-with-dirindex-4096B.png
+++ /dev/null
Binary files differ
diff --git a/createfiles-ext3-with-dirindex-8192B.png b/createfiles-ext3-with-dirindex-8192B.png
deleted file mode 100644
index e63387d..0000000
--- a/createfiles-ext3-with-dirindex-8192B.png
+++ /dev/null
Binary files differ
diff --git a/createfiles-plot.sh b/createfiles-plot.sh
deleted file mode 100755
index c4a7e19..0000000
--- a/createfiles-plot.sh
+++ /dev/null
@@ -1,51 +0,0 @@
-#!/bin/sh
-#
-# A small script to generate graphs from createfiles.py output. There will
-# be one createfiles-$SIZE.png file for each size of output, graphing the
-# various algorithms at various sizes.
-
-set -e
-
-input="$1"
-
-algos=$(awk '$2 == "files/s" { print $3 }' "$input" | sort -u)
-sizes=$(awk '/^Measuring/ { print $(NF-1) }' "$input" | sort -un)
-
-for size in $sizes
-do
- plot=""
-
- for x in $algos
- do
- awk -vx=$x -vsize=$size '
- /^Measuring/ {
- if ($(NF-1) == size) {
- n = $6
- ok = 1
- } else {
- ok = 0
- }
- }
- ok && $3 == x {
- print n, $1 >> x ".dat"
- }' "$input"
- if [ "$plot" = "" ]
- then
- plot="plot \"$x.dat\" with lines"
- else
- plot="$plot, \"$x.dat\" with lines"
- fi
- done
-
- gnuplot <<eof > "createfiles-$size.png"
-set terminal png
-set title "Size $size"
-$plot
-eof
-
- for x in $algos
- do
- rm "$x.dat"
- done
-
-done
diff --git a/createfiles.output-ext3-with-dirindex b/createfiles.output-ext3-with-dirindex
deleted file mode 100644
index 72e9a46..0000000
--- a/createfiles.output-ext3-with-dirindex
+++ /dev/null
@@ -1,528 +0,0 @@
-Measuring 7 functions for creating 10 files of 0 bytes
-
- 41487 files/s onedir All files in one directory
- 34721 files/s subdirsof100 Files in subdirs, each with up to 100 files
- 35727 files/s subdirsof250 Files in subdirs, each with up to 250 files
- 34606 files/s subdirsof1000 Files in subdirs, each with up to 1000 files
- 15082 files/s subsubdirsof100 Files in two-level tree, up to 100 files each
- 15169 files/s subsubdirsof250 Files in two-level tree, up to 250 files each
- 15219 files/s subsubdirsof1000 Files in two-level tree, up to 1000 files each
-
-
-Measuring 7 functions for creating 1000 files of 0 bytes
-
- 38065 files/s onedir All files in one directory
- 44621 files/s subdirsof100 Files in subdirs, each with up to 100 files
- 41822 files/s subdirsof250 Files in subdirs, each with up to 250 files
- 36725 files/s subdirsof1000 Files in subdirs, each with up to 1000 files
- 16732 files/s subsubdirsof100 Files in two-level tree, up to 100 files each
- 16327 files/s subsubdirsof250 Files in two-level tree, up to 250 files each
- 15492 files/s subsubdirsof1000 Files in two-level tree, up to 1000 files each
-
-
-Measuring 7 functions for creating 2000 files of 0 bytes
-
- 37583 files/s onedir All files in one directory
- 43994 files/s subdirsof100 Files in subdirs, each with up to 100 files
- 41007 files/s subdirsof250 Files in subdirs, each with up to 250 files
- 36047 files/s subdirsof1000 Files in subdirs, each with up to 1000 files
- 16692 files/s subsubdirsof100 Files in two-level tree, up to 100 files each
- 16200 files/s subsubdirsof250 Files in two-level tree, up to 250 files each
- 15404 files/s subsubdirsof1000 Files in two-level tree, up to 1000 files each
-
-
-Measuring 7 functions for creating 4000 files of 0 bytes
-
- 36298 files/s onedir All files in one directory
- 43217 files/s subdirsof100 Files in subdirs, each with up to 100 files
- 40305 files/s subdirsof250 Files in subdirs, each with up to 250 files
- 35675 files/s subdirsof1000 Files in subdirs, each with up to 1000 files
- 16520 files/s subsubdirsof100 Files in two-level tree, up to 100 files each
- 16102 files/s subsubdirsof250 Files in two-level tree, up to 250 files each
- 15307 files/s subsubdirsof1000 Files in two-level tree, up to 1000 files each
-
-
-Measuring 7 functions for creating 8000 files of 0 bytes
-
- 35039 files/s onedir All files in one directory
- 42752 files/s subdirsof100 Files in subdirs, each with up to 100 files
- 39964 files/s subdirsof250 Files in subdirs, each with up to 250 files
- 35321 files/s subdirsof1000 Files in subdirs, each with up to 1000 files
- 15394 files/s subsubdirsof100 Files in two-level tree, up to 100 files each
- 15941 files/s subsubdirsof250 Files in two-level tree, up to 250 files each
- 15280 files/s subsubdirsof1000 Files in two-level tree, up to 1000 files each
-
-
-Measuring 7 functions for creating 16000 files of 0 bytes
-
- 34941 files/s onedir All files in one directory
- 42378 files/s subdirsof100 Files in subdirs, each with up to 100 files
- 39506 files/s subdirsof250 Files in subdirs, each with up to 250 files
- 35237 files/s subdirsof1000 Files in subdirs, each with up to 1000 files
- 14456 files/s subsubdirsof100 Files in two-level tree, up to 100 files each
- 15912 files/s subsubdirsof250 Files in two-level tree, up to 250 files each
- 15183 files/s subsubdirsof1000 Files in two-level tree, up to 1000 files each
-
-
-Measuring 7 functions for creating 32000 files of 0 bytes
-
- 34194 files/s onedir All files in one directory
- 42020 files/s subdirsof100 Files in subdirs, each with up to 100 files
- 38924 files/s subdirsof250 Files in subdirs, each with up to 250 files
- 35041 files/s subdirsof1000 Files in subdirs, each with up to 1000 files
- 16385 files/s subsubdirsof100 Files in two-level tree, up to 100 files each
- 15051 files/s subsubdirsof250 Files in two-level tree, up to 250 files each
- 15164 files/s subsubdirsof1000 Files in two-level tree, up to 1000 files each
-
-
-Measuring 7 functions for creating 64000 files of 0 bytes
-
- 29784 files/s onedir All files in one directory
- 41785 files/s subdirsof100 Files in subdirs, each with up to 100 files
- 38778 files/s subdirsof250 Files in subdirs, each with up to 250 files
- 31200 files/s subdirsof1000 Files in subdirs, each with up to 1000 files
- 15282 files/s subsubdirsof100 Files in two-level tree, up to 100 files each
- 14932 files/s subsubdirsof250 Files in two-level tree, up to 250 files each
- 14309 files/s subsubdirsof1000 Files in two-level tree, up to 1000 files each
-
-
-Measuring 7 functions for creating 128000 files of 0 bytes
-
- 31150 files/s onedir All files in one directory
- 30689 files/s subdirsof100 Files in subdirs, each with up to 100 files
- 38397 files/s subdirsof250 Files in subdirs, each with up to 250 files
- 30766 files/s subdirsof1000 Files in subdirs, each with up to 1000 files
- 14921 files/s subsubdirsof100 Files in two-level tree, up to 100 files each
- 15011 files/s subsubdirsof250 Files in two-level tree, up to 250 files each
- 14034 files/s subsubdirsof1000 Files in two-level tree, up to 1000 files each
-
-
-Measuring 7 functions for creating 256000 files of 0 bytes
-
- 29211 files/s onedir All files in one directory
- 19747 files/s subdirsof100 Files in subdirs, each with up to 100 files
- 35337 files/s subdirsof250 Files in subdirs, each with up to 250 files
- 28877 files/s subdirsof1000 Files in subdirs, each with up to 1000 files
- 14432 files/s subsubdirsof100 Files in two-level tree, up to 100 files each
- 14121 files/s subsubdirsof250 Files in two-level tree, up to 250 files each
- 14065 files/s subsubdirsof1000 Files in two-level tree, up to 1000 files each
-
-
-Measuring 7 functions for creating 512000 files of 0 bytes
-
- 24449 files/s onedir All files in one directory
- 21297 files/s subdirsof100 Files in subdirs, each with up to 100 files
- 24421 files/s subdirsof250 Files in subdirs, each with up to 250 files
- 25494 files/s subdirsof1000 Files in subdirs, each with up to 1000 files
- 13814 files/s subsubdirsof100 Files in two-level tree, up to 100 files each
- 13897 files/s subsubdirsof250 Files in two-level tree, up to 250 files each
- 13321 files/s subsubdirsof1000 Files in two-level tree, up to 1000 files each
-
-
-Measuring 7 functions for creating 1024000 files of 0 bytes
-
- 20239 files/s onedir All files in one directory
- 20728 files/s subdirsof100 Files in subdirs, each with up to 100 files
- 23730 files/s subdirsof250 Files in subdirs, each with up to 250 files
- 26271 files/s subdirsof1000 Files in subdirs, each with up to 1000 files
- 13227 files/s subsubdirsof100 Files in two-level tree, up to 100 files each
- 13837 files/s subsubdirsof250 Files in two-level tree, up to 250 files each
- 12941 files/s subsubdirsof1000 Files in two-level tree, up to 1000 files each
-
-
-Measuring 7 functions for creating 10 files of 4096 bytes
-
- 23630 files/s onedir All files in one directory
- 21654 files/s subdirsof100 Files in subdirs, each with up to 100 files
- 21687 files/s subdirsof250 Files in subdirs, each with up to 250 files
- 21834 files/s subdirsof1000 Files in subdirs, each with up to 1000 files
- 11364 files/s subsubdirsof100 Files in two-level tree, up to 100 files each
- 11822 files/s subsubdirsof250 Files in two-level tree, up to 250 files each
- 11808 files/s subsubdirsof1000 Files in two-level tree, up to 1000 files each
-
-
-Measuring 7 functions for creating 1000 files of 4096 bytes
-
- 21001 files/s onedir All files in one directory
- 24392 files/s subdirsof100 Files in subdirs, each with up to 100 files
- 23646 files/s subdirsof250 Files in subdirs, each with up to 250 files
- 21926 files/s subdirsof1000 Files in subdirs, each with up to 1000 files
- 12504 files/s subsubdirsof100 Files in two-level tree, up to 100 files each
- 12236 files/s subsubdirsof250 Files in two-level tree, up to 250 files each
- 11767 files/s subsubdirsof1000 Files in two-level tree, up to 1000 files each
-
-
-Measuring 7 functions for creating 2000 files of 4096 bytes
-
- 21391 files/s onedir All files in one directory
- 24029 files/s subdirsof100 Files in subdirs, each with up to 100 files
- 21909 files/s subdirsof250 Files in subdirs, each with up to 250 files
- 20476 files/s subdirsof1000 Files in subdirs, each with up to 1000 files
- 11577 files/s subsubdirsof100 Files in two-level tree, up to 100 files each
- 11167 files/s subsubdirsof250 Files in two-level tree, up to 250 files each
- 10509 files/s subsubdirsof1000 Files in two-level tree, up to 1000 files each
-
-
-Measuring 7 functions for creating 4000 files of 4096 bytes
-
- 21258 files/s onedir All files in one directory
- 23132 files/s subdirsof100 Files in subdirs, each with up to 100 files
- 21925 files/s subdirsof250 Files in subdirs, each with up to 250 files
- 20587 files/s subdirsof1000 Files in subdirs, each with up to 1000 files
- 12205 files/s subsubdirsof100 Files in two-level tree, up to 100 files each
- 11882 files/s subsubdirsof250 Files in two-level tree, up to 250 files each
- 11340 files/s subsubdirsof1000 Files in two-level tree, up to 1000 files each
-
-
-Measuring 7 functions for creating 8000 files of 4096 bytes
-
- 19734 files/s onedir All files in one directory
- 18496 files/s subdirsof100 Files in subdirs, each with up to 100 files
- 12523 files/s subdirsof250 Files in subdirs, each with up to 250 files
- 20460 files/s subdirsof1000 Files in subdirs, each with up to 1000 files
- 12078 files/s subsubdirsof100 Files in two-level tree, up to 100 files each
- 11650 files/s subsubdirsof250 Files in two-level tree, up to 250 files each
- 11207 files/s subsubdirsof1000 Files in two-level tree, up to 1000 files each
-
-
-Measuring 7 functions for creating 16000 files of 4096 bytes
-
- 20482 files/s onedir All files in one directory
- 20108 files/s subdirsof100 Files in subdirs, each with up to 100 files
- 19301 files/s subdirsof250 Files in subdirs, each with up to 250 files
- 20069 files/s subdirsof1000 Files in subdirs, each with up to 1000 files
- 10945 files/s subsubdirsof100 Files in two-level tree, up to 100 files each
- 10651 files/s subsubdirsof250 Files in two-level tree, up to 250 files each
- 10582 files/s subsubdirsof1000 Files in two-level tree, up to 1000 files each
-
-
-Measuring 7 functions for creating 32000 files of 4096 bytes
-
- 9755 files/s onedir All files in one directory
- 6502 files/s subdirsof100 Files in subdirs, each with up to 100 files
- 18511 files/s subdirsof250 Files in subdirs, each with up to 250 files
- 18066 files/s subdirsof1000 Files in subdirs, each with up to 1000 files
- 10939 files/s subsubdirsof100 Files in two-level tree, up to 100 files each
- 9046 files/s subsubdirsof250 Files in two-level tree, up to 250 files each
- 10583 files/s subsubdirsof1000 Files in two-level tree, up to 1000 files each
-
-
-Measuring 7 functions for creating 64000 files of 4096 bytes
-
- 3890 files/s onedir All files in one directory
- 5069 files/s subdirsof100 Files in subdirs, each with up to 100 files
- 14446 files/s subdirsof250 Files in subdirs, each with up to 250 files
- 18703 files/s subdirsof1000 Files in subdirs, each with up to 1000 files
- 10159 files/s subsubdirsof100 Files in two-level tree, up to 100 files each
- 9593 files/s subsubdirsof250 Files in two-level tree, up to 250 files each
- 9773 files/s subsubdirsof1000 Files in two-level tree, up to 1000 files each
-
-
-Measuring 7 functions for creating 128000 files of 4096 bytes
-
- 5949 files/s onedir All files in one directory
- 6611 files/s subdirsof100 Files in subdirs, each with up to 100 files
- 15273 files/s subdirsof250 Files in subdirs, each with up to 250 files
- 5965 files/s subdirsof1000 Files in subdirs, each with up to 1000 files
- 9935 files/s subsubdirsof100 Files in two-level tree, up to 100 files each
- 9203 files/s subsubdirsof250 Files in two-level tree, up to 250 files each
- 6478 files/s subsubdirsof1000 Files in two-level tree, up to 1000 files each
-
-
-Measuring 7 functions for creating 256000 files of 4096 bytes
-
- 4751 files/s onedir All files in one directory
- 8139 files/s subdirsof100 Files in subdirs, each with up to 100 files
- 6566 files/s subdirsof250 Files in subdirs, each with up to 250 files
- 10514 files/s subdirsof1000 Files in subdirs, each with up to 1000 files
- 7560 files/s subsubdirsof100 Files in two-level tree, up to 100 files each
- 7301 files/s subsubdirsof250 Files in two-level tree, up to 250 files each
- 9719 files/s subsubdirsof1000 Files in two-level tree, up to 1000 files each
-
-
-Measuring 7 functions for creating 512000 files of 4096 bytes
-
- 3535 files/s onedir All files in one directory
- 3437 files/s subdirsof100 Files in subdirs, each with up to 100 files
- 4330 files/s subdirsof250 Files in subdirs, each with up to 250 files
- 6881 files/s subdirsof1000 Files in subdirs, each with up to 1000 files
- 4924 files/s subsubdirsof100 Files in two-level tree, up to 100 files each
- 4542 files/s subsubdirsof250 Files in two-level tree, up to 250 files each
- 7061 files/s subsubdirsof1000 Files in two-level tree, up to 1000 files each
-
-
-Measuring 7 functions for creating 1024000 files of 4096 bytes
-
- 2710 files/s onedir All files in one directory
- 4207 files/s subdirsof100 Files in subdirs, each with up to 100 files
- 4564 files/s subdirsof250 Files in subdirs, each with up to 250 files
- 5296 files/s subdirsof1000 Files in subdirs, each with up to 1000 files
- 4537 files/s subsubdirsof100 Files in two-level tree, up to 100 files each
- 4645 files/s subsubdirsof250 Files in two-level tree, up to 250 files each
- 5819 files/s subsubdirsof1000 Files in two-level tree, up to 1000 files each
-
-
-Measuring 7 functions for creating 10 files of 8192 bytes
-
- 19373 files/s onedir All files in one directory
- 18316 files/s subdirsof100 Files in subdirs, each with up to 100 files
- 18658 files/s subdirsof250 Files in subdirs, each with up to 250 files
- 17765 files/s subdirsof1000 Files in subdirs, each with up to 1000 files
- 10673 files/s subsubdirsof100 Files in two-level tree, up to 100 files each
- 10719 files/s subsubdirsof250 Files in two-level tree, up to 250 files each
- 10763 files/s subsubdirsof1000 Files in two-level tree, up to 1000 files each
-
-
-Measuring 7 functions for creating 1000 files of 8192 bytes
-
- 18659 files/s onedir All files in one directory
- 19490 files/s subdirsof100 Files in subdirs, each with up to 100 files
- 17477 files/s subdirsof250 Files in subdirs, each with up to 250 files
- 17948 files/s subdirsof1000 Files in subdirs, each with up to 1000 files
- 11172 files/s subsubdirsof100 Files in two-level tree, up to 100 files each
- 10809 files/s subsubdirsof250 Files in two-level tree, up to 250 files each
- 10212 files/s subsubdirsof1000 Files in two-level tree, up to 1000 files each
-
-
-Measuring 7 functions for creating 2000 files of 8192 bytes
-
- 18249 files/s onedir All files in one directory
- 19500 files/s subdirsof100 Files in subdirs, each with up to 100 files
- 18248 files/s subdirsof250 Files in subdirs, each with up to 250 files
- 17029 files/s subdirsof1000 Files in subdirs, each with up to 1000 files
- 10853 files/s subsubdirsof100 Files in two-level tree, up to 100 files each
- 10562 files/s subsubdirsof250 Files in two-level tree, up to 250 files each
- 10424 files/s subsubdirsof1000 Files in two-level tree, up to 1000 files each
-
-
-Measuring 7 functions for creating 4000 files of 8192 bytes
-
- 17060 files/s onedir All files in one directory
- 18711 files/s subdirsof100 Files in subdirs, each with up to 100 files
- 17258 files/s subdirsof250 Files in subdirs, each with up to 250 files
- 16282 files/s subdirsof1000 Files in subdirs, each with up to 1000 files
- 9925 files/s subsubdirsof100 Files in two-level tree, up to 100 files each
- 10461 files/s subsubdirsof250 Files in two-level tree, up to 250 files each
- 10164 files/s subsubdirsof1000 Files in two-level tree, up to 1000 files each
-
-
-Measuring 7 functions for creating 8000 files of 8192 bytes
-
- 16615 files/s onedir All files in one directory
- 18280 files/s subdirsof100 Files in subdirs, each with up to 100 files
- 18477 files/s subdirsof250 Files in subdirs, each with up to 250 files
- 17209 files/s subdirsof1000 Files in subdirs, each with up to 1000 files
- 10573 files/s subsubdirsof100 Files in two-level tree, up to 100 files each
- 10560 files/s subsubdirsof250 Files in two-level tree, up to 250 files each
- 10102 files/s subsubdirsof1000 Files in two-level tree, up to 1000 files each
-
-
-Measuring 7 functions for creating 16000 files of 8192 bytes
-
- 15661 files/s onedir All files in one directory
- 19164 files/s subdirsof100 Files in subdirs, each with up to 100 files
- 18020 files/s subdirsof250 Files in subdirs, each with up to 250 files
- 16883 files/s subdirsof1000 Files in subdirs, each with up to 1000 files
- 10678 files/s subsubdirsof100 Files in two-level tree, up to 100 files each
- 10305 files/s subsubdirsof250 Files in two-level tree, up to 250 files each
- 10060 files/s subsubdirsof1000 Files in two-level tree, up to 1000 files each
-
-
-Measuring 7 functions for creating 32000 files of 8192 bytes
-
- 16703 files/s onedir All files in one directory
- 5068 files/s subdirsof100 Files in subdirs, each with up to 100 files
- 17810 files/s subdirsof250 Files in subdirs, each with up to 250 files
- 16753 files/s subdirsof1000 Files in subdirs, each with up to 1000 files
- 10521 files/s subsubdirsof100 Files in two-level tree, up to 100 files each
- 9306 files/s subsubdirsof250 Files in two-level tree, up to 250 files each
- 9704 files/s subsubdirsof1000 Files in two-level tree, up to 1000 files each
-
-
-Measuring 7 functions for creating 64000 files of 8192 bytes
-
- 8254 files/s onedir All files in one directory
- 4493 files/s subdirsof100 Files in subdirs, each with up to 100 files
- 10132 files/s subdirsof250 Files in subdirs, each with up to 250 files
- 9293 files/s subdirsof1000 Files in subdirs, each with up to 1000 files
- 9015 files/s subsubdirsof100 Files in two-level tree, up to 100 files each
- 10250 files/s subsubdirsof250 Files in two-level tree, up to 250 files each
- 8196 files/s subsubdirsof1000 Files in two-level tree, up to 1000 files each
-
-
-Measuring 7 functions for creating 128000 files of 8192 bytes
-
- 5450 files/s onedir All files in one directory
- 5504 files/s subdirsof100 Files in subdirs, each with up to 100 files
- 5132 files/s subdirsof250 Files in subdirs, each with up to 250 files
- 3632 files/s subdirsof1000 Files in subdirs, each with up to 1000 files
- 6036 files/s subsubdirsof100 Files in two-level tree, up to 100 files each
- 6438 files/s subsubdirsof250 Files in two-level tree, up to 250 files each
- 6124 files/s subsubdirsof1000 Files in two-level tree, up to 1000 files each
-
-
-Measuring 7 functions for creating 256000 files of 8192 bytes
-
- 3934 files/s onedir All files in one directory
- 4512 files/s subdirsof100 Files in subdirs, each with up to 100 files
- 4862 files/s subdirsof250 Files in subdirs, each with up to 250 files
- 4884 files/s subdirsof1000 Files in subdirs, each with up to 1000 files
- 4759 files/s subsubdirsof100 Files in two-level tree, up to 100 files each
- 4793 files/s subsubdirsof250 Files in two-level tree, up to 250 files each
- 4859 files/s subsubdirsof1000 Files in two-level tree, up to 1000 files each
-
-
-Measuring 7 functions for creating 512000 files of 8192 bytes
-
- 3896 files/s onedir All files in one directory
- 2877 files/s subdirsof100 Files in subdirs, each with up to 100 files
- 3482 files/s subdirsof250 Files in subdirs, each with up to 250 files
- 3635 files/s subdirsof1000 Files in subdirs, each with up to 1000 files
- 3426 files/s subsubdirsof100 Files in two-level tree, up to 100 files each
- 3750 files/s subsubdirsof250 Files in two-level tree, up to 250 files each
- 3865 files/s subsubdirsof1000 Files in two-level tree, up to 1000 files each
-
-
-Measuring 7 functions for creating 1024000 files of 8192 bytes
-
- 3826 files/s onedir All files in one directory
- 2922 files/s subdirsof100 Files in subdirs, each with up to 100 files
- 3340 files/s subdirsof250 Files in subdirs, each with up to 250 files
- 3487 files/s subdirsof1000 Files in subdirs, each with up to 1000 files
- 3076 files/s subsubdirsof100 Files in two-level tree, up to 100 files each
- 3731 files/s subsubdirsof250 Files in two-level tree, up to 250 files each
- 3703 files/s subsubdirsof1000 Files in two-level tree, up to 1000 files each
-
-
-Measuring 7 functions for creating 10 files of 16384 bytes
-
- 15104 files/s onedir All files in one directory
- 14559 files/s subdirsof100 Files in subdirs, each with up to 100 files
- 14619 files/s subdirsof250 Files in subdirs, each with up to 250 files
- 14533 files/s subdirsof1000 Files in subdirs, each with up to 1000 files
- 9310 files/s subsubdirsof100 Files in two-level tree, up to 100 files each
- 9243 files/s subsubdirsof250 Files in two-level tree, up to 250 files each
- 9259 files/s subsubdirsof1000 Files in two-level tree, up to 1000 files each
-
-
-Measuring 7 functions for creating 1000 files of 16384 bytes
-
- 14222 files/s onedir All files in one directory
- 14999 files/s subdirsof100 Files in subdirs, each with up to 100 files
- 14608 files/s subdirsof250 Files in subdirs, each with up to 250 files
- 13958 files/s subdirsof1000 Files in subdirs, each with up to 1000 files
- 9369 files/s subsubdirsof100 Files in two-level tree, up to 100 files each
- 9190 files/s subsubdirsof250 Files in two-level tree, up to 250 files each
- 8849 files/s subsubdirsof1000 Files in two-level tree, up to 1000 files each
-
-
-Measuring 7 functions for creating 2000 files of 16384 bytes
-
- 13653 files/s onedir All files in one directory
- 14240 files/s subdirsof100 Files in subdirs, each with up to 100 files
- 13868 files/s subdirsof250 Files in subdirs, each with up to 250 files
- 13063 files/s subdirsof1000 Files in subdirs, each with up to 1000 files
- 8971 files/s subsubdirsof100 Files in two-level tree, up to 100 files each
- 8574 files/s subsubdirsof250 Files in two-level tree, up to 250 files each
- 8931 files/s subsubdirsof1000 Files in two-level tree, up to 1000 files each
-
-
-Measuring 7 functions for creating 4000 files of 16384 bytes
-
- 13586 files/s onedir All files in one directory
- 14100 files/s subdirsof100 Files in subdirs, each with up to 100 files
- 13760 files/s subdirsof250 Files in subdirs, each with up to 250 files
- 12727 files/s subdirsof1000 Files in subdirs, each with up to 1000 files
- 8825 files/s subsubdirsof100 Files in two-level tree, up to 100 files each
- 8567 files/s subsubdirsof250 Files in two-level tree, up to 250 files each
- 7580 files/s subsubdirsof1000 Files in two-level tree, up to 1000 files each
-
-
-Measuring 7 functions for creating 8000 files of 16384 bytes
-
- 13429 files/s onedir All files in one directory
- 14343 files/s subdirsof100 Files in subdirs, each with up to 100 files
- 13183 files/s subdirsof250 Files in subdirs, each with up to 250 files
- 12703 files/s subdirsof1000 Files in subdirs, each with up to 1000 files
- 9021 files/s subsubdirsof100 Files in two-level tree, up to 100 files each
- 9018 files/s subsubdirsof250 Files in two-level tree, up to 250 files each
- 8445 files/s subsubdirsof1000 Files in two-level tree, up to 1000 files each
-
-
-Measuring 7 functions for creating 16000 files of 16384 bytes
-
- 12358 files/s onedir All files in one directory
- 13828 files/s subdirsof100 Files in subdirs, each with up to 100 files
- 13826 files/s subdirsof250 Files in subdirs, each with up to 250 files
- 13549 files/s subdirsof1000 Files in subdirs, each with up to 1000 files
- 8919 files/s subsubdirsof100 Files in two-level tree, up to 100 files each
- 8951 files/s subsubdirsof250 Files in two-level tree, up to 250 files each
- 8736 files/s subsubdirsof1000 Files in two-level tree, up to 1000 files each
-
-
-Measuring 7 functions for creating 32000 files of 16384 bytes
-
- 7278 files/s onedir All files in one directory
- 8634 files/s subdirsof100 Files in subdirs, each with up to 100 files
- 9575 files/s subdirsof250 Files in subdirs, each with up to 250 files
- 9640 files/s subdirsof1000 Files in subdirs, each with up to 1000 files
- 9092 files/s subsubdirsof100 Files in two-level tree, up to 100 files each
- 8398 files/s subsubdirsof250 Files in two-level tree, up to 250 files each
- 8670 files/s subsubdirsof1000 Files in two-level tree, up to 1000 files each
-
-
-Measuring 7 functions for creating 64000 files of 16384 bytes
-
- 3188 files/s onedir All files in one directory
- 2988 files/s subdirsof100 Files in subdirs, each with up to 100 files
- 3205 files/s subdirsof250 Files in subdirs, each with up to 250 files
- 3400 files/s subdirsof1000 Files in subdirs, each with up to 1000 files
- 3310 files/s subsubdirsof100 Files in two-level tree, up to 100 files each
- 3233 files/s subsubdirsof250 Files in two-level tree, up to 250 files each
- 3392 files/s subsubdirsof1000 Files in two-level tree, up to 1000 files each
-
-
-Measuring 7 functions for creating 128000 files of 16384 bytes
-
- 2506 files/s onedir All files in one directory
- 2420 files/s subdirsof100 Files in subdirs, each with up to 100 files
- 2596 files/s subdirsof250 Files in subdirs, each with up to 250 files
- 2671 files/s subdirsof1000 Files in subdirs, each with up to 1000 files
- 2489 files/s subsubdirsof100 Files in two-level tree, up to 100 files each
- 2191 files/s subsubdirsof250 Files in two-level tree, up to 250 files each
- 2496 files/s subsubdirsof1000 Files in two-level tree, up to 1000 files each
-
-
-Measuring 7 functions for creating 256000 files of 16384 bytes
-
- 2098 files/s onedir All files in one directory
- 2010 files/s subdirsof100 Files in subdirs, each with up to 100 files
- 2093 files/s subdirsof250 Files in subdirs, each with up to 250 files
- 2084 files/s subdirsof1000 Files in subdirs, each with up to 1000 files
- 2148 files/s subsubdirsof100 Files in two-level tree, up to 100 files each
- 2208 files/s subsubdirsof250 Files in two-level tree, up to 250 files each
- 2085 files/s subsubdirsof1000 Files in two-level tree, up to 1000 files each
-
-
-Measuring 7 functions for creating 512000 files of 16384 bytes
-
- 2031 files/s onedir All files in one directory
- 1585 files/s subdirsof100 Files in subdirs, each with up to 100 files
- 1930 files/s subdirsof250 Files in subdirs, each with up to 250 files
- 1996 files/s subdirsof1000 Files in subdirs, each with up to 1000 files
- 1924 files/s subsubdirsof100 Files in two-level tree, up to 100 files each
- 1984 files/s subsubdirsof250 Files in two-level tree, up to 250 files each
- 2001 files/s subsubdirsof1000 Files in two-level tree, up to 1000 files each
-
-
-Measuring 7 functions for creating 1024000 files of 16384 bytes
-
- 2002 files/s onedir All files in one directory
- 1689 files/s subdirsof100 Files in subdirs, each with up to 100 files
- 1927 files/s subdirsof250 Files in subdirs, each with up to 250 files
- 1980 files/s subdirsof1000 Files in subdirs, each with up to 1000 files
- 1747 files/s subsubdirsof100 Files in two-level tree, up to 100 files each
- 1959 files/s subsubdirsof250 Files in two-level tree, up to 250 files each
- 1954 files/s subsubdirsof1000 Files in two-level tree, up to 1000 files each
-
-
diff --git a/createfiles.py b/createfiles.py
deleted file mode 100755
index 3a46c53..0000000
--- a/createfiles.py
+++ /dev/null
@@ -1,162 +0,0 @@
-#!/usr/bin/python
-
-# Create lots of (empty) files in various ways, to see which way is fastest.
-
-
-import errno
-import gc
-import os
-import optparse
-import shutil
-import sys
-import timeit
-
-
-ROOT = "create.tmp"
-
-
-def create_dir(dirname):
- full_name = os.path.join(ROOT, dirname)
- try:
- os.mkdir(full_name)
- except os.error, e:
- if e.errno != errno.EEXIST:
- raise e
-
-def create_file(filename, size):
- full_name = os.path.join(ROOT, filename)
- f = file(full_name, "w")
- f.write("x" * size)
- f.close()
-
-def cleanup():
- while os.path.exists(ROOT):
- try:
- shutil.rmtree(ROOT)
- except os.error, e:
- if e.errno != errno.ENOENT:
- raise e
- os.mkdir(ROOT)
-
-def onedir(count, size):
- """All files in one directory"""
- for i in range(count):
- create_file("%d" % i, size)
-
-def subdirsofN(N, count, size):
- prev_subdir = None
- for i in range(count):
- subdir = i / N
- if subdir != prev_subdir:
- create_dir("%d" % subdir)
- prev_subdir = subdir
- create_file("%d/%d" % (subdir, i), size)
-
-def subdirsof100(count, size):
- """Files in subdirs, each with up to 100 files"""
- subdirsofN(100, count, size)
-
-def subdirsof250(count, size):
- """Files in subdirs, each with up to 250 files"""
- subdirsofN(250, count, size)
-
-def subdirsof1000(count, size):
- """Files in subdirs, each with up to 1000 files"""
- subdirsofN(1000, count, size)
-
-def subsubdirsofN(N, count, size):
- prev = None
- for i in range(count):
- subsubdir = i / N
- subdir = subsubdir / N
- if (subdir, subsubdir) != prev:
- create_dir("%d" % subdir)
- create_dir("%d/%d" % (subdir, subsubdir))
- prev_subdir = (subdir, subsubdir)
- create_file("%d/%d/%d" % (subdir, subsubdir, i), size)
-
-def subsubdirsof100(count, size):
- """Files in two-level tree, up to 100 files each"""
- subsubdirsofN(100, count, size)
-
-def subsubdirsof250(count, size):
- """Files in two-level tree, up to 250 files each"""
- subsubdirsofN(250, count, size)
-
-def subsubdirsof1000(count, size):
- """Files in two-level tree, up to 1000 files each"""
- subsubdirsofN(1000, count, size)
-
-funcs = [
- onedir, # disabled, because it exposes a bug in ext3+dir_index
- subdirsof100,
- subdirsof250,
- subdirsof1000,
- subsubdirsof100,
- subsubdirsof250,
- subsubdirsof1000,
- ]
-
-def measure(func, count, size):
- cleanup()
- gc.collect()
- timer = timeit.Timer(stmt='%s(%d, %d)' % (func.func_name, count, size),
- setup="from __main__ import %s" % func.func_name)
- return timer.timeit(number=1)
-
-def check(func, count, size):
- cleanup()
- func(count, size)
- actual_count = 0
- for dirname, _, filenames in os.walk(ROOT):
- actual_count += len(filenames)
- for name in [os.path.join(dirname, x) for x in filenames]:
- actual_size = os.path.getsize(name)
- assert size == actual_size, \
- "size is %d, should be %d" % (actual_size, size)
- assert count == actual_count, \
- "count=%d and actual_count=%d should be identical" % \
- (count, actual_count)
-
-def parse_args(args):
- p = optparse.OptionParser()
-
- p.add_option("-c", "--check", action="store_true", default=False,
- help="Check that all functions actually work")
- p.add_option("-s", "--size", action="store", type="int", metavar="SIZE",
- default=0, help="Create files of SIZE bytes (default: 0)")
-
- return p.parse_args(args)
-
-
-def main():
- options, counts = parse_args(sys.argv[1:])
-
- if counts:
- counts = [int(x) for x in counts]
- else:
- counts = [10000]
-
- for count in counts:
- print "Measuring %d functions for creating %d files of %d bytes" % \
- (len(funcs), count, options.size)
- print
-
- namelen = max(len(func.func_name) for func in funcs)
-
- for func in funcs:
- if options.check:
- check(func, count, options.size)
- secs = measure(func, count, options.size)
- speed = count/secs
- print "%6.0f files/s %-*s %s" % \
- (speed, namelen, func.func_name, func.__doc__)
- sys.stdout.flush()
- print
- print
-
- cleanup()
- os.rmdir(ROOT)
-
-if __name__ == "__main__":
- main()
diff --git a/genbackupdata b/genbackupdata
index 8467396..becd5e9 100755
--- a/genbackupdata
+++ b/genbackupdata
@@ -1,6 +1,107 @@
#!/usr/bin/python
-#
-# Run genbackupdata.Application as a command line utility.
+# Copyright 2011 Lars Wirzenius
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+
+import cliapp
+import os
+import sys
+import ttystatus
+
+import genbackupdatalib
+
+
+class GenbackupdataApp(cliapp.Application):
+
+ def add_settings(self):
+ self.add_bytesize_setting(['create', 'c'],
+ 'how much data to create '
+ '(default: %default)')
+ self.add_bytesize_setting(['file-size'],
+ 'size of one file (default: %default)',
+ default=16*1024)
+ self.add_bytesize_setting(['chunk-size'],
+ 'generate data in chunks of this size '
+ '(default: %default)',
+ default=16*1024)
+ self.add_integer_setting(['depth'],
+ 'depth of directory tree (default: %default)',
+ default=3)
+ self.add_integer_setting(['max-files'],
+ 'max files/dirs per dir (default: %default)',
+ default=128)
+ self.add_integer_setting(['seed'],
+ 'seed for random number generator '
+ '(default: %default)',
+ default=0)
+ self.add_boolean_setting(['quiet'], 'do not report progress')
+
+ def process_args(self, args):
+ outputdir = args[0]
+ bytes = self['create']
+ self.gen = genbackupdatalib.DataGenerator(self['seed'])
+ self.names = genbackupdatalib.NameGenerator(outputdir, self['depth'],
+ self['max-files'])
+
+ self.setup_ttystatus()
+ self.status['total'] = bytes
+ while bytes > 0:
+ n = min(self['file-size'], bytes)
+ self.create_file(n)
+ bytes -= n
+ self.status.finish()
+
+ def create_file(self, bytes):
+ '''Generate one output file.'''
+
+ file_size = self['file-size']
+ chunk_size = self['chunk-size']
+ pathname = self.names.new()
+ dirname = os.path.dirname(pathname)
+ if not os.path.exists(dirname):
+ os.makedirs(dirname)
+ f = open(pathname, 'wb')
+ while bytes >= chunk_size:
+ self.write_bytes(f, chunk_size)
+ bytes -= chunk_size
+ if bytes > 0:
+ self.write_bytes(f, bytes)
+ f.close()
+
+ def write_bytes(self, f, bytes):
+ chunk = self.gen.generate(bytes)
+ f.write(chunk)
+ self.status['written'] += bytes
+
+ def setup_ttystatus(self):
+ self.status = ttystatus.TerminalStatus(period=0.1)
+ if self['quiet']:
+ self.status.disable()
+ self.status['written'] = 0
+ self.status['total'] = 0
+ self.status.add(ttystatus.Literal('Generating: '))
+ self.status.add(ttystatus.ByteSize('written'))
+ self.status.add(ttystatus.Literal(' of '))
+ self.status.add(ttystatus.ByteSize('total'))
+ self.status.add(ttystatus.Literal(' '))
+ self.status.add(ttystatus.PercentDone('written', 'total'))
+ self.status.add(ttystatus.Literal(' ('))
+ self.status.add(ttystatus.ByteSpeed('written'))
+ self.status.add(ttystatus.Literal(')'))
+
+
+if __name__ == '__main__':
+ GenbackupdataApp().run()
-import sys, genbackupdata
-genbackupdata.Application(sys.argv[1:]).run()
diff --git a/genbackupdata.py b/genbackupdata.py
deleted file mode 100644
index 2fbb416..0000000
--- a/genbackupdata.py
+++ /dev/null
@@ -1,660 +0,0 @@
-# Copyright (C) 2007 Lars Wirzenius <liw@iki.fi>
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-
-"""Generate backup test data"""
-
-
-import hashlib
-import optparse
-import os
-import random
-import sys
-
-
-KiB = 2 ** 10 # A kibibyte
-MiB = 2 ** 20 # A mebibyte
-GiB = 2 ** 30 # A gibibyte
-TiB = 2 ** 40 # A tebibyte
-
-# Defaults for various settings in the BackupData class.
-DEFAULT_SEED = 0
-DEFAULT_BINARY_CHUNK_SIZE = KiB
-DEFAULT_TEXT_FILE_SIZE = 10 * KiB
-DEFAULT_BINARY_FILE_SIZE = 10 * MiB
-DEFAULT_TEXT_DATA_PERCENTAGE = 10.0
-DEFAULT_MAX_FILES_PER_DIRECTORY = 256
-DEFAULT_MODIFY_PERCENTAGE = 10
-
-# Random filler text for generating text data.
-if True: #pragma: no cover:
- LOREM_IPSUM = """
-Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod
-tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim
-veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea
-commodo consequat. Duis aute irure dolor in reprehenderit in voluptate
-velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint
-occaecat cupidatat non proident, sunt in culpa qui officia deserunt
-mollit anim id est laborum.
-"""
-
-
-class BackupData:
-
- """This class represents the directory with backup data"""
-
- def __init__(self):
- self._dirname = None
- self._seed = 0
- self._prng = None
- self._chunk_size = DEFAULT_BINARY_CHUNK_SIZE
- self._text_file_size = DEFAULT_TEXT_FILE_SIZE
- self._binary_file_size = DEFAULT_BINARY_FILE_SIZE
- self._text_data_percentage = DEFAULT_TEXT_DATA_PERCENTAGE
- self._max_files_per_directory = DEFAULT_MAX_FILES_PER_DIRECTORY
- self._modify_percentage = DEFAULT_MODIFY_PERCENTAGE
- self._preexisting_file_count = 0
- self._preexisting_data_size = 0
- self._filename_counter = 0
- self._current_dir_no = 0
- self._next_filecount = 0
-
- # The zlib compression algorithm gives up if it gets a block of
- # 32 KiB bytes it can't find in its dictionary. It completely
- # ignores such a block, meaning that if it is repeated, then
- # it ignores it repeatedly. Most importantly for us, it doesn't
- # compress the repeats, either. Thus, to generate lots of
- # uncompressible binary data, we can generate a blob and repeat
- # that. Thanks to Richard Braakman for the idea.
- self._binary_blob_size = 64 * 1024 # Safety factor of 2
- self._binary_blob = None
-
- def set_directory(self, dirname):
- """Set the directory to be operated on
-
- Note that this must be set exactly once. Setting it twice will cause
- an assertion error, and not setting it will cause other errors.
-
- """
-
- assert self._dirname is None
- self._dirname = dirname
-
- def get_directory(self):
- """Return the directory being operated on, or None if not set"""
- return self._dirname
-
- def create_directory(self):
- """Create the backup data directory, if it doesn't exist already"""
- if not os.path.exists(self._dirname):
- os.mkdir(self._dirname)
-
- def get_seed(self):
- """Return the initial seed for the pseudo-random number generator"""
- return self._seed
-
- def set_seed(self, seed):
- """Set the initial seed for the pseudo-random number generator
-
- The seed will be used when the generator is first initialized.
- It is initialized implicitly as soon as something in this class
- needs randomness. Setting the seed after the generator has been
- initialized causes an assertion failure.
-
- """
-
- assert self.get_prng() is None
- self._seed = seed
-
- def get_prng(self):
- """Return reference to the psuedo-random number generator being used
-
- Return None, if one hasn't be initialized yet.
-
- """
-
- return self._prng
-
- def init_prng(self):
- """Initialize the psuedo-random number generator (using seed)"""
- if self._prng is None:
- self._prng = random.Random()
- self._prng.seed(self._seed)
-
- def get_text_file_size(self):
- """Return size of newly created text files"""
- return self._text_file_size
-
- def set_text_file_size(self, size):
- """Set size of newly created text files"""
- self._text_file_size = size
-
- def get_binary_file_size(self):
- """Return size of newly created binary files"""
- return self._binary_file_size
-
- def set_binary_file_size(self, size):
- """Set size of newly created binary files"""
- self._binary_file_size = size
-
- def get_text_data_percentage(self):
- """Return percentage of text data of new data that gets created"""
- return self._text_data_percentage
-
- def set_text_data_percentage(self, percent):
- """Set percentage of text data of new data that gets created"""
- self._text_data_percentage = percent
-
- def get_max_files_per_directory(self):
- """Return current setting of maximum number of files per directory"""
- return self._max_files_per_directory
-
- def set_max_files_per_directory(self, count):
- """Set maximum number of files per directory"""
- self._max_files_per_directory = count
-
- def get_preexisting_file_count(self):
- """Return count of files that existed in directory in the beginning"""
- return self._preexisting_file_count
-
- def set_preexisting_file_count(self, count):
- """Set count of files that existed in directory in the beginning
-
- This is useful only for unit tests.
-
- """
- self._preexisting_file_count = count
-
- def get_preexisting_data_size(self):
- """Return size of data that existed in directory in the beginning"""
- return self._preexisting_data_size
-
- def set_preexisting_data_size(self, size):
- """Set size of data that existed in directory in the beginning
-
- This is useful only for unit tests.
-
- """
- self._preexisting_data_size = size
-
- def get_relative_file_count(self, percent):
- """Return PERCENT percent of pre-existing file count"""
- return int(0.01 * percent * self.get_preexisting_file_count())
-
- def get_relative_data_size(self, percent):
- """Return PERCENT percent of pre-existing data"""
- return int(0.01 * percent * self.get_preexisting_data_size())
-
- def find_preexisting_files(self):
- """Find all the files that exists in the directory right now"""
- count = 0
- size = 0
- if os.path.exists(self._dirname):
- for root, dirs, filenames in os.walk(self._dirname):
- count += len(filenames)
- for filename in filenames:
- size += os.path.getsize(os.path.join(root, filename))
- self.set_preexisting_file_count(count)
- self.set_preexisting_data_size(size)
-
- def _choose_directory(self):
- """Choose directory in which to create the next file"""
-
- while True:
- dirname = os.path.join(self._dirname,
- "dir%d" % self._current_dir_no)
- if not os.path.exists(dirname):
- self._next_filecount = 0
- break
- if (self._next_filecount > 0 and
- self._next_filecount < self._max_files_per_directory):
- break
- self._current_dir_no += 1
- return dirname
-
- def next_filename(self):
- """Choose the name of the next filename
-
- The file does not currently exist. This is not, however, a guarantee
- that no other process won't create it before we do. Thus, this
- is NOT a secure way to create temporary files. But it's good enough
- for our intended purpose.
-
- For simplified unit testing, the names are very easily predictable,
- but it is probably a bad idea for external code to rely on this.
-
- """
-
- dirname = self._choose_directory()
- while True:
- filename = os.path.join(dirname,
- "file%d" % self._filename_counter)
- if not os.path.exists(filename):
- self._next_filecount += 1
- return filename
- self._filename_counter += 1
-
- def generate_text_data(self, size):
- """Generate SIZE characters of text data"""
- if size <= len(LOREM_IPSUM):
- return LOREM_IPSUM[:size]
- else:
- full = size / len(LOREM_IPSUM)
- rest = size % len(LOREM_IPSUM)
- return "".join(([LOREM_IPSUM] * full) + [LOREM_IPSUM[:rest]])
-
- def generate_binary_data_well(self, size):
- """Generate SIZE bytes of more or less random binary junk"""
-
- # The following code has had some fine manual fine tuning done
- # to it. This has made it a bit ugly, but faster. On a
- # "Intel(R) Core(TM)2 Duo CPU L9400 @ 1.86GHz", it produces
- # about 25 MB/s.
-
- chunks = []
- sum = hashlib.sha1()
- chunk_size = len(sum.digest())
-
- initial_bytes = min(size, 128)
- for i in range(initial_bytes / chunk_size):
- sum.update(chr(random.getrandbits(8)))
- chunk = sum.digest()
- chunks.append(chunk)
-
- size -= len(chunks) * chunk_size
- for i in range(size / chunk_size):
- sum.update("a")
- chunk = sum.digest()
- chunks.append(chunk)
-
- if size % chunk_size > 0:
- sum.update(chr(random.getrandbits(8)))
- chunk = sum.digest()
- chunks.append(chunk[:size % chunk_size])
-
- return "".join(chunks)
-
-
- def generate_binary_data(self, size):
- """Generate SIZE bytes of binary junk.
-
- This is different from generate_binary_data_well in that
- it makes use of _binary_blob (and generates that if it does
- not yet exist).
-
- """
-
- if self._binary_blob is None:
- self._binary_blob = self.generate_binary_data_well(
- self._binary_blob_size)
- if size <= len(self._binary_blob):
- return self._binary_blob[:size]
- else:
- full = size / len(self._binary_blob)
- rest = size % len(self._binary_blob)
- return "".join(([self._binary_blob] * full) +
- [self._binary_blob[:rest]])
-
- def create_subdirectories(self, filename):
- """Create the sub-directories that are needed to create filename"""
- subdir = os.path.dirname(filename)
- if not os.path.exists(subdir):
- os.makedirs(subdir)
-
- def create_text_file(self, size):
- """Create a new text file of the desired size"""
- filename = self.next_filename()
- self.create_subdirectories(filename)
- f = file(filename, "w")
- f.write(self.generate_text_data(size))
- f.close()
-
- def get_binary_chunk_size(self):
- """Return the size of chunks used when writing binary data"""
- return self._chunk_size
-
- def set_binary_chunk_size(self, size):
- """Set the size of chunks used when writing binary data"""
- self._chunk_size = size
-
- def create_binary_file(self, size):
- """Create a new binary file of the desired size"""
- filename = self.next_filename()
- self.create_subdirectories(filename)
- f = file(filename, "w")
- # We write the data in chunks, so as not to keep the entire file
- # contents in memory at a time. Since the size may be very large,
- # we might otherwise run out of swap.
- while size >= self._chunk_size:
- f.write(self.generate_binary_data(self._chunk_size))
- size -= self._chunk_size
- f.write(self.generate_binary_data(size))
- f.close()
-
- def _create_files_of_a_kind(self, size, file_size, create_one):
- """Create files with create_one"""
- while size > 0:
- this_size = min(size, file_size)
- create_one(this_size)
- size -= this_size
-
- def create_files(self, size):
- """Create new files, totalling SIZE bytes in size"""
- text_size = int(0.01 * self._text_data_percentage * size)
- bin_size = size - text_size
-
- self._create_files_of_a_kind(text_size, self.get_text_file_size(),
- self.create_text_file)
- self._create_files_of_a_kind(bin_size, self.get_binary_file_size(),
- self.create_binary_file)
-
- def find_files(self):
- """Find all non-directory files in the test data set"""
- files = []
- for root, dirs, filenames in os.walk(self._dirname):
- for filename in filenames:
- files.append(os.path.join(root, filename))
- return files
-
- def choose_files_randomly(self, count):
- """Choose COUNT files randomly"""
- files = self.find_files()
- if len(files) >= count:
- self.init_prng()
- files = self._prng.sample(files, count)
- return files
-
- def delete_files(self, count):
- """Delete COUNT files"""
- if os.path.exists(self._dirname):
- for file in self.choose_files_randomly(count):
- os.remove(file)
-
- def rename_files(self, count):
- """Rename COUNT files to new names"""
- if os.path.exists(self._dirname):
- for file in self.choose_files_randomly(count):
- new_file = self.next_filename()
- self.create_subdirectories(new_file)
- os.rename(file, new_file)
-
- def link_files(self, count):
- """Create COUNT new filenames that are hard links to existing files"""
- if os.path.exists(self._dirname):
- for file in self.choose_files_randomly(count):
- new_file = self.next_filename()
- self.create_subdirectories(new_file)
- os.link(file, new_file)
-
- def get_modify_percentage(self):
- """Return how many percent to grow each file with modify_files()"""
- return self._modify_percentage
-
- def set_modify_percentage(self, percent):
- """Set how many percent to grow each file with modify_files()"""
- self._modify_percentage = percent
-
- def append_data(self, filename, data):
- """Append data to a file"""
- f = file(filename, "a")
- f.write(data)
- f.close()
-
- def _modify_files_of_a_kind(self, filenames, size, generate_data):
- """Modify files by appending data to them"""
- while size > 0:
- filename = self._prng.choice(filenames)
- this_size = os.path.getsize(filename)
- amount = min(int(0.01 * self._modify_percentage * this_size),
- size)
- self.append_data(filename, generate_data(amount))
- size -= amount
-
- def modify_files(self, size):
- """Modify existing files by appending to them
-
- SIZE gives the total amount of new data for all files.
- Files are chosen at random, and new data is appended to them.
- The amount appended to each file is set by
- set_modify_percentage. The data is split between text and
- binary data according to set_text_data_percentage.
-
- """
-
- if os.path.exists(self._dirname):
- files = self.find_files()
-
- text_size = int(0.01 * self._text_data_percentage * size)
- bin_size = size - text_size
-
- self.init_prng()
- self._modify_files_of_a_kind(files, text_size,
- self.generate_text_data)
- self._modify_files_of_a_kind(files, bin_size,
- self.generate_binary_data)
-
-
-
-class CommandLineParser:
-
- """Parse the command line for the genbackupdata utility"""
-
- def __init__(self, backup_data):
- self._bd = backup_data
- self._parser = self._create_option_parser()
-
- def _create_option_parser(self):
- """Create the OptionParser we need"""
-
- p = optparse.OptionParser()
-
- p.add_option("--seed",
- help="Set pseudo-random number generator seed to SEED")
-
- p.add_option("--max-count",
- action="store",
- metavar="COUNT",
- help="Allow at most COUNT files per directory")
-
- p.add_option("-p", "--percentage-text-data",
- action="store",
- metavar="PERCENT",
- help="Make PERCENT of new data textual, not binary")
-
- p.add_option("-t", "--text-file-size",
- action="store",
- metavar="SIZE",
- help="Make new text files be of size SIZE")
-
- p.add_option("-b", "--binary-file-size",
- action="store",
- metavar="SIZE",
- help="Make new binary files be of size SIZE")
-
- p.add_option("-c", "--create",
- action="store",
- metavar="SIZE",
- help="Create SIZE amount of new files")
-
- p.add_option("-d", "--delete",
- action="store",
- metavar="COUNT",
- help="Delete COUNT files")
-
- p.add_option("-r", "--rename",
- action="store",
- metavar="COUNT",
- help="Rename COUNT files")
-
- p.add_option("-l", "--link",
- action="store",
- metavar="COUNT",
- help="Create COUNT new hard links")
-
- p.add_option("-m", "--modify",
- action="store",
- metavar="SIZE",
- help="Grow total data size by SIZE")
-
- p.add_option("--modify-percentage",
- action="store",
- metavar="PERCENT",
- help="Increase file size by PERCENT")
-
- return p
-
- def parse_size(self, size, base_size=None):
- """Parse a SIZE argument (absolute, relative, with/without suffix)"""
-
- suffixes = (("k", KiB), ("m", MiB), ("g", GiB), ("t", TiB))
-
- for suffix, factor in suffixes:
- if size.lower().endswith(suffix):
- return int(float(size[:-len(suffix)]) * factor)
-
- if size.endswith("%"):
- if base_size is None:
- return 0
- else:
- return int(float(size[:-1]) * 0.01 * base_size)
-
- return int(size)
-
- def parse_count(self, count, base_count=None):
- """Parse a COUNT argument (absolute, relative, with/without suffix)"""
-
- suffixes = (("k", 10**3), ("m", 10**6), ("g", 10**9), ("t", 10**12))
-
- for suffix, factor in suffixes:
- if count.lower().endswith(suffix):
- return int(float(count[:-len(suffix)]) * factor)
-
- if count.endswith("%"):
- if base_count is None:
- return 0
- else:
- return int(float(count[:-1]) * 0.01 * base_count)
-
- return int(count)
-
- def parse(self, args):
- """Parse command line arguments"""
- options, args = self._parser.parse_args(args)
-
- if len(args) == 1:
- self._bd.set_directory(args[0])
- self._bd.find_preexisting_files()
-
- if options.seed:
- self._bd.set_seed(int(options.seed))
-
- if options.max_count:
- self._bd.set_max_files_per_directory(int(options.max_count))
-
- if options.percentage_text_data:
- self._bd.set_text_data_percentage(
- float(options.percentage_text_data))
-
- if options.modify_percentage:
- self._bd.set_modify_percentage(float(options.modify_percentage))
-
- if options.text_file_size:
- self._bd.set_text_file_size(
- self.parse_size(options.text_file_size))
-
- if options.binary_file_size:
- self._bd.set_binary_file_size(
- self.parse_size(options.binary_file_size))
-
- if options.create:
- options.create = self.parse_size(options.create,
- self._bd.get_preexisting_data_size())
-
- if options.modify:
- options.modify = self.parse_size(options.modify,
- self._bd.get_preexisting_data_size())
-
- if options.delete:
- options.delete = self.parse_count(options.delete,
- self._bd.get_preexisting_file_count())
-
- if options.rename:
- options.rename = self.parse_count(options.rename,
- self._bd.get_preexisting_file_count())
-
- if options.link:
- options.link = self.parse_count(options.link,
- self._bd.get_preexisting_file_count())
-
- return options, args
-
-
-class AppException(Exception):
-
- def __str__(self):
- return self._str
-
-
-class NeedExactlyOneDirectoryName(AppException):
-
- def __init__(self):
- self._str = ("Need exactly one command line argument, "
- "giving directory name")
-
-
-class Application:
-
- """The main program"""
-
- def __init__(self, args):
- self._args = args
- self._bd = BackupData()
- self._clp = CommandLineParser(self._bd)
- self._error = sys.stderr.write
-
- def set_error_writer(self, writer):
- self._error = writer
-
- def run(self):
- """Execute the desired operations"""
- try:
- options, args = self._clp.parse(self._args)
-
- if len(args) != 1:
- raise NeedExactlyOneDirectoryName()
-
- if options.delete:
- self._bd.delete_files(options.delete)
-
- if options.rename:
- self._bd.rename_files(options.rename)
-
- if options.link:
- self._bd.link_files(options.link)
-
- if options.modify:
- self._bd.modify_files(options.modify)
-
- if options.create:
- self._bd.create_files(options.create)
-
- except AppException, e:
- self._error(str(e) + "\n")
- sys.exit(1)
-
-
-if __name__ == "__main__": #pragma: no cover
- Application(sys.argv[1:]).run()
diff --git a/genbackupdatalib/__init__.py b/genbackupdatalib/__init__.py
new file mode 100644
index 0000000..b7771db
--- /dev/null
+++ b/genbackupdatalib/__init__.py
@@ -0,0 +1,20 @@
+# Copyright 2010 Lars Wirzenius
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+
+version = '1.3'
+
+from generator import DataGenerator
+from names import NameGenerator
diff --git a/genbackupdatalib/generator.py b/genbackupdatalib/generator.py
new file mode 100644
index 0000000..8cf349c
--- /dev/null
+++ b/genbackupdatalib/generator.py
@@ -0,0 +1,62 @@
+# Copyright 2010 Lars Wirzenius
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+
+import random
+import struct
+
+
+class DataGenerator(object):
+
+ '''Generate random binary data.'''
+
+ # We generate data by using a blob of suitable size. The output
+ # sequence repeats the blob, where each repetition is preceded by
+ # a 64-bit counter.
+ #
+ # We need to be relatively prime with obnam's chunk size, which
+ # defaults to 64 KiB (65536 bytes). This is so that obnam does not
+ # notice a lot of duplicated data, resulting in unrealistically
+ # high amounts of compression in the backup store.
+ #
+ # Ideally, we would not generate any repeating data, but the random
+ # number generator is not fast enough for that. We need to generate
+ # data about as fast as the disk can write it, and the random number
+ # generator is orders of magnitude slower than that.
+
+ _blob_size = 65521
+ _blob_size = 1021
+
+ def __init__(self, seed):
+ self._random = random.Random(seed)
+ self._blob = self._generate_blob()
+ self._counter = 0
+ self._buffer = ''
+
+ def _generate_blob(self):
+ return ''.join(chr(self._random.randint(0, 255))
+ for i in range(self._blob_size))
+
+ def generate(self, size):
+ while size > len(self._buffer):
+ self._buffer += self._generate_more_data()
+ data = self._buffer[:size]
+ self._buffer = self._buffer[size:]
+ return data
+
+ def _generate_more_data(self):
+ self._counter += 1
+ return struct.pack('!Q', self._counter) + self._blob
+
diff --git a/genbackupdatalib/generator_tests.py b/genbackupdatalib/generator_tests.py
new file mode 100644
index 0000000..80d12b4
--- /dev/null
+++ b/genbackupdatalib/generator_tests.py
@@ -0,0 +1,42 @@
+# Copyright 2010 Lars Wirzenius
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+
+import unittest
+
+import genbackupdatalib
+
+
+class DataGeneratorTests(unittest.TestCase):
+
+ def setUp(self):
+ self.g1 = genbackupdatalib.DataGenerator(0)
+ self.g2 = genbackupdatalib.DataGenerator(0)
+
+ def test_every_generator_returns_same_sequence(self):
+ amount = 1024
+ self.assertEqual(self.g1.generate(amount), self.g2.generate(amount))
+
+ def test_returns_different_sequence_for_different_seed(self):
+ amount = 1024
+ g3 = genbackupdatalib.DataGenerator(1)
+ self.assertNotEqual(self.g1.generate(amount), g3.generate(amount))
+
+ def test_returns_distinct_64k_chunks(self):
+ size = 64 * 1024
+ chunk1 = self.g1.generate(size)
+ num_chunks = 100
+ for i in range(num_chunks):
+ self.assertNotEqual(self.g1.generate(size), chunk1)
diff --git a/genbackupdatalib/names.py b/genbackupdatalib/names.py
new file mode 100644
index 0000000..287112d
--- /dev/null
+++ b/genbackupdatalib/names.py
@@ -0,0 +1,75 @@
+# Copyright 2011 Lars Wirzenius
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+
+import os
+
+
+class NameGenerator(object):
+
+ '''Generate names for new output files.
+
+ If the target directory is empty, the sequence of output files is
+ always the same for the same parameters.
+
+ A directory structure is also generated. The shape of the tree is
+ defined by two parameters: 'max' and 'depth'. 'depth' is the number
+ of levels of subdirectories to create, and 'max' is the maximum
+ number of files/dirs to allow per output directory. Thus, if max is
+ 3 and depth is 2, the output files are: 0/0/0, 0/0/1, 0/0/2,
+ 0/1/0, 0/1/1, etc.
+
+ If depth is zero, all output files go directly to the target
+ directory, and max is ignored.
+
+ '''
+
+ def __init__(self, dirname, depth, max):
+ self.dirname = dirname
+ self.depth = depth
+ self.max = max
+ self.counter = 0
+
+ def _path_tuple(self, n):
+ '''Return tuple for dir/file numbers for nth output file.
+
+ The last item in the tuple gives the file number, the precding
+ items the directory numbers. Thus, a tuple (1, 2, 3) would
+ mean path '1/2/3', but it is given as a tuple for easier
+ manipulation.
+
+ '''
+
+ if self.depth == 0:
+ return (n,)
+ else:
+ items = []
+ for i in range(self.depth):
+ items.append(n % self.max)
+ n /= self.max
+ items.append(n)
+ items.reverse()
+ return tuple(items)
+
+ def _next_candidate_name(self):
+ items = self._path_tuple(self.counter)
+ self.counter += 1
+ return os.path.join(self.dirname, *[str(i) for i in items])
+
+ def new(self):
+ while True:
+ name = self._next_candidate_name()
+ if not os.path.exists(name):
+ return name
diff --git a/genbackupdatalib/names_tests.py b/genbackupdatalib/names_tests.py
new file mode 100644
index 0000000..60b4d79
--- /dev/null
+++ b/genbackupdatalib/names_tests.py
@@ -0,0 +1,84 @@
+# Copyright 2011 Lars Wirzenius
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+
+import os
+import shutil
+import tempfile
+import unittest
+
+import genbackupdatalib
+
+
+class NameGeneratorTests(unittest.TestCase):
+
+ def setUp(self):
+ self.tempdir = tempfile.mkdtemp()
+ self.depth = 2
+ self.max = 3
+ self.names = self.new()
+
+ def tearDown(self):
+ shutil.rmtree(self.tempdir)
+
+ def new(self):
+ return genbackupdatalib.NameGenerator(self.tempdir, self.depth,
+ self.max)
+
+ def test_generates_name_that_is_inside_target_directory(self):
+ name = self.names.new()
+ self.assert_(name.startswith(self.tempdir + os.sep))
+
+ def test_generates_different_names_every_time(self):
+ names = set(self.names.new() for i in range(10))
+ self.assertEqual(len(names), 10)
+
+ def test_generates_names_that_do_not_exist(self):
+ for i in range(10):
+ name = self.names.new()
+ self.assertFalse(os.path.exists(name))
+
+ def test_generates_the_same_sequence_with_every_instance(self):
+ n = 10
+ first = [self.names.new() for i in range(n)]
+ names2 = self.new()
+ second = [names2.new() for i in range(n)]
+ self.assertEqual(first, second)
+
+ def test_does_not_generate_names_of_existing_files(self):
+ name = self.names.new()
+ os.makedirs(os.path.dirname(name))
+ file(name, 'w').close()
+ names2 = self.new()
+ name2 = names2.new()
+ self.assertNotEqual(name, name2)
+ self.assertFalse(os.path.exists(name2))
+
+ def test_converts_file_sequence_number_into_right_path_tuple(self):
+ self.assertEqual(self.names._path_tuple(0), (0, 0, 0))
+ self.assertEqual(self.names._path_tuple(1), (0, 0, 1))
+ self.assertEqual(self.names._path_tuple(2), (0, 0, 2))
+ self.assertEqual(self.names._path_tuple(3), (0, 1, 0))
+ self.assertEqual(self.names._path_tuple(4), (0, 1, 1))
+ self.assertEqual(self.names._path_tuple(5), (0, 1, 2))
+ self.assertEqual(self.names._path_tuple(6), (0, 2, 0))
+ self.assertEqual(self.names._path_tuple(9), (1, 0, 0))
+ self.assertEqual(self.names._path_tuple(18), (2, 0, 0))
+ self.assertEqual(self.names._path_tuple(27), (3, 0, 0))
+
+ def test_returns_1tuple_for_depth_zero(self):
+ names = genbackupdatalib.NameGenerator(self.tempdir, 0, 1)
+ self.assertEqual(names._path_tuple(42), (42,))
+
diff --git a/generate-speed b/generate-speed
new file mode 100755
index 0000000..472c8f2
--- /dev/null
+++ b/generate-speed
@@ -0,0 +1,70 @@
+#!/usr/bin/python
+# Copyright 2010 Lars Wirzenius
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+
+import cProfile
+import sys
+import time
+
+import genbackupdatalib
+
+
+def measure(repeats, func, arg, do_profile, profname):
+ def helper():
+ for i in range(repeats):
+ func(arg)
+
+ print 'measuring', profname
+ start_time = time.time()
+ start = time.clock()
+ if do_profile:
+ globaldict = globals().copy()
+ localdict = locals().copy()
+ cProfile.runctx('helper()', globaldict, localdict,
+ '%s.prof' % profname)
+ else:
+ helper()
+ end = time.clock()
+ end_time = time.time()
+ return end - start, end_time - start_time
+
+
+def nop(size):
+ pass
+
+
+def main():
+ repeats = int(sys.argv[1])
+ size1 = int(sys.argv[2])
+ do_profile = sys.argv[3] == 'yes'
+ looptime = measure(repeats, nop, None, do_profile, 'calibrate')
+
+ g = genbackupdatalib.DataGenerator(0)
+ result = measure(repeats, g.generate, size1, do_profile, 'generate')
+
+ def speed(result, i):
+ total_data = repeats * size1
+ return total_data / (result[i] - looptime[i])
+ def humansize(size):
+ return '%4.1f MiB/s' % (size / 1024 / 1024)
+ def report(label, result):
+ cpu, wall = result
+ print '%-12s: %5.3f s (%8s)' % \
+ (label, cpu, humansize(speed(result, 0)))
+ report('generate', result)
+
+if __name__ == '__main__':
+ main()
diff --git a/tests.py b/tests.py
deleted file mode 100755
index 4466fc0..0000000
--- a/tests.py
+++ /dev/null
@@ -1,565 +0,0 @@
-#!/usr/bin/python
-#
-# Copyright (C) 2007 Lars Wirzenius <liw@iki.fi>
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-
-"""Unit tests for genbackupdata.py"""
-
-
-import os
-import shutil
-import unittest
-import zlib
-
-import genbackupdata
-
-
-class BackupDataTests(unittest.TestCase):
-
- dirname = "tests.dir"
-
- def remove_dir(self):
- """Remove a directory, if it exists"""
- if os.path.exists(self.dirname):
- shutil.rmtree(self.dirname)
-
- def create(self, filename, contents):
- """Create a new file with the desired contents"""
- dirname = os.path.dirname(filename)
- if not os.path.exists(dirname):
- os.makedirs(dirname)
- f = file(filename, "w")
- f.write(contents)
- f.close()
-
- def read_file(self, filename):
- """Return the entire contents of a file"""
- f = file(filename)
- data = f.read()
- f.close()
- return data
-
- def setUp(self):
- self.remove_dir()
- self.bd = genbackupdata.BackupData()
- self.bd.set_directory(self.dirname)
-
- def tearDown(self):
- del self.bd
- self.remove_dir()
-
- def testSetsDirectoryCorrect(self):
- self.failUnlessEqual(self.bd.get_directory(), self.dirname)
-
- def testDoesNotCreateDirectoryAtOnce(self):
- self.failIf(os.path.exists(self.dirname))
-
- def testCreatesDirectory(self):
- self.bd.create_directory()
- self.failUnless(os.path.exists(self.dirname))
-
- def testDoesNotFailWhenCreatingDirectoryWhenItExistsAlready(self):
- self.bd.create_directory()
- self.bd.create_directory()
-
- def testHasCorrectDefaultSeed(self):
- self.failUnlessEqual(self.bd.get_seed(), genbackupdata.DEFAULT_SEED)
-
- def testCorrectlySetsSeed(self):
- self.bd.set_seed(12765)
- self.failUnlessEqual(self.bd.get_seed(), 12765)
-
- def testHasNoPRNGInitially(self):
- self.failUnlessEqual(self.bd.get_prng(), None)
-
- def testCreatesPRNGWhenRequested(self):
- self.bd.init_prng()
- self.failIfEqual(self.bd.get_prng(), None)
-
- def testFailsIfSeedGetsSetAfterPRNGHasBeenCreatedTwice(self):
- self.bd.init_prng()
- self.failUnlessRaises(AssertionError, self.bd.set_seed, 12765)
-
- def testHasCorrectDefaultBinaryChunkSize(self):
- self.failUnlessEqual(self.bd.get_binary_chunk_size(),
- genbackupdata.DEFAULT_BINARY_CHUNK_SIZE)
-
- def testCorrectlySetsBinaryChunkSize(self):
- self.bd.set_binary_chunk_size(12765)
- self.failUnlessEqual(self.bd.get_binary_chunk_size(), 12765)
-
- def testHasCorrectDefaultTextFileSize(self):
- self.failUnlessEqual(self.bd.get_text_file_size(),
- genbackupdata.DEFAULT_TEXT_FILE_SIZE)
-
- def testCorrectlySetsTextFileSize(self):
- self.bd.set_text_file_size(12765)
- self.failUnlessEqual(self.bd.get_text_file_size(), 12765)
-
- def testHasCorrectDefaultBinaryFileSize(self):
- self.failUnlessEqual(self.bd.get_binary_file_size(),
- genbackupdata.DEFAULT_BINARY_FILE_SIZE)
-
- def testCorrectlySetsBinaryFileSize(self):
- self.bd.set_binary_file_size(12765)
- self.failUnlessEqual(self.bd.get_binary_file_size(), 12765)
-
- def testHasCorrectDefaultTextDataPercentage(self):
- self.failUnlessEqual(self.bd.get_text_data_percentage(),
- genbackupdata.DEFAULT_TEXT_DATA_PERCENTAGE)
-
- def testCorrectlySetsTextDataPercentage(self):
- self.bd.set_text_data_percentage(42.0)
- self.failUnlessEqual(self.bd.get_text_data_percentage(), 42.0)
-
- def testHasCorrectDefaultMaxFilesPerDirectory(self):
- self.failUnlessEqual(self.bd.get_max_files_per_directory(),
- genbackupdata.DEFAULT_MAX_FILES_PER_DIRECTORY)
-
- def testCorrectlySetsMaxFilesPerDirectory(self):
- self.bd.set_max_files_per_directory(12765)
- self.failUnlessEqual(self.bd.get_max_files_per_directory(), 12765)
-
- def testSetsPreExistingFileCountToZeroByDefault(self):
- self.failUnlessEqual(self.bd.get_preexisting_file_count(), 0)
-
- def testCanFakePreExistingFileCount(self):
- self.bd.set_preexisting_file_count(12765)
- self.failUnlessEqual(self.bd.get_preexisting_file_count(), 12765)
-
- def testSetsPreExistingDataSizeToZeroByDefault(self):
- self.failUnlessEqual(self.bd.get_preexisting_data_size(), 0)
-
- def testCanFakePreExistingDataSize(self):
- self.bd.set_preexisting_data_size(12765)
- self.failUnlessEqual(self.bd.get_preexisting_data_size(), 12765)
-
- def testFindsPreExistingFilesAndDAtaCorrectly(self):
- file_count = 10
- file_size = self.bd.get_text_file_size()
- data = self.bd.generate_text_data(file_size)
- for i in range(file_count):
- self.create(self.bd.next_filename(), data)
- self.bd.find_preexisting_files()
- self.failUnlessEqual(self.bd.get_preexisting_file_count(), file_count)
- self.failUnlessEqual(self.bd.get_preexisting_data_size(),
- file_count * file_size)
-
- def testComputesRelativeFileCountCorrectly(self):
- self.bd.set_preexisting_file_count(12765)
- self.failUnlessEqual(self.bd.get_relative_file_count(10), 1276)
-
- def testComputesRelativeDataSizeCorrectly(self):
- self.bd.set_preexisting_data_size(12765)
- self.failUnlessEqual(self.bd.get_relative_data_size(10), 1276)
-
- def testFindsNoPreExistingFilesWhenDirectoryDoesNotExist(self):
- self.bd.find_preexisting_files()
- self.failUnlessEqual(self.bd.get_preexisting_file_count(), 0)
-
- def testFindsNoPreExistingDataWhenDirectoryDoesNotExist(self):
- self.bd.find_preexisting_files()
- self.failUnlessEqual(self.bd.get_preexisting_data_size(), 0)
-
- def testChoosesFirstFilenameCorrectly(self):
- filename = self.bd.next_filename()
- self.failUnless(filename.startswith(self.dirname))
-
- def testChoosesFirstFilenameCorrectlyTwice(self):
- filename1 = self.bd.next_filename()
- filename2 = self.bd.next_filename()
- self.failUnlessEqual(filename1, filename2)
-
- def testChoosesFilenameCorrectlyWhenFirstOneExistsAlready(self):
- self.bd.create_directory()
- filename1 = self.bd.next_filename()
- self.create(filename1, "")
- filename2 = self.bd.next_filename()
- self.failIfEqual(filename1, filename2)
-
- def testChoosesSameDirectoryUntilMaxFileLimitIsReached(self):
- self.bd.set_max_files_per_directory(10) # For speed
- self.bd.create_directory()
- chosen = self.bd._choose_directory()
- for i in range(self.bd.get_max_files_per_directory()):
- self.failUnlessEqual(self.bd._choose_directory(), chosen)
- self.create(self.bd.next_filename(), "")
-
- def testChoosesNewDirectoryWhenMaxFileLimitIsReached(self):
- self.bd.set_max_files_per_directory(10) # For speed
- self.bd.create_directory()
- chosen = self.bd._choose_directory()
- for i in range(self.bd.get_max_files_per_directory()):
- self.create(self.bd.next_filename(), "")
- self.failIfEqual(self.bd._choose_directory(), chosen)
-
- def testGeneratesSmallAmountOfTextDataCorrectly(self):
- n = 128
- self.failUnlessEqual(self.bd.generate_text_data(n),
- genbackupdata.LOREM_IPSUM[:n])
-
- def testGeneratesLargeAmountOfTextDataCorrectly(self):
- n = len(genbackupdata.LOREM_IPSUM)
- self.failUnlessEqual(self.bd.generate_text_data(n * 2),
- genbackupdata.LOREM_IPSUM * 2)
-
- def testGeneratesRequestedAmountOfBinaryDataWell(self):
- n = 37
- self.failUnlessEqual(len(self.bd.generate_binary_data_well(n)), n)
-
- def testGeneratesRequestedAmountOfBinaryDataQuickly(self):
- n = 128
- self.failUnlessEqual(len(self.bd.generate_binary_data(n)), n)
-
- def testGeneratesRequestedLargeAmountOfBinaryData(self):
- n = self.bd._binary_blob_size + 1
- self.failUnlessEqual(len(self.bd.generate_binary_data(n)), n)
-
- def testGeneratesBinaryDataWhichDoesNotCompressWell(self):
- n = self.bd._binary_blob_size * 4
- data = zlib.compress(self.bd.generate_binary_data(n))
- self.failUnless(len(data) > 0.95* n)
-
- def testCreatesSubdirectoriesCorrectly(self):
- filename = os.path.join(self.dirname, "subdir", "filename")
- self.bd.create_subdirectories(filename)
- self.failUnless(os.path.isdir(os.path.dirname(filename)))
-
- def testCreatesTextFileCorrectly(self):
- size = self.bd.get_text_file_size()
- filename = self.bd.next_filename()
- self.bd.create_text_file(size)
- self.failUnless(os.path.isfile(filename))
- self.failUnlessEqual(self.read_file(filename),
- self.bd.generate_text_data(size))
-
- def testCreatesBinaryFileCorrectly(self):
- size = self.bd.get_binary_chunk_size() + 1
- filename = self.bd.next_filename()
- self.bd.create_binary_file(size)
- self.failUnless(os.path.isfile(filename))
- self.failUnlessEqual(os.path.getsize(filename), size)
-
- def failUnlessTextFile(self, filename):
- data = self.read_file(filename)
- self.failUnlessEqual(data, self.bd.generate_text_data(len(data)))
-
- def testCreatesTextFilesOnlyCorrectly(self):
- self.bd.set_text_data_percentage(100)
- count = 10
- size = count * self.bd.get_text_file_size()
- self.bd.create_files(size)
- self.bd.find_preexisting_files()
- self.failUnlessEqual(self.bd.get_preexisting_file_count(), count)
- self.failUnlessEqual(self.bd.get_preexisting_data_size(), size)
- for root, dirs, filenames in os.walk(self.dirname):
- for filename in filenames:
- pathname = os.path.join(root, filename)
- self.failUnlessTextFile(pathname)
-
- def failIfTextFile(self, filename):
- data = self.read_file(filename)
- self.failIfEqual(data, self.bd.generate_text_data(len(data)))
-
- def testCreatesNoTextFilesCorrectly(self):
- self.bd.set_text_data_percentage(0)
- count = 1
- size = count * self.bd.get_text_file_size()
- self.bd.create_files(size)
- self.bd.find_preexisting_files()
- self.failUnlessEqual(self.bd.get_preexisting_file_count(), 1)
- self.failUnlessEqual(self.bd.get_preexisting_data_size(), size)
- for root, dirs, filenames in os.walk(self.dirname):
- for filename in filenames:
- pathname = os.path.join(root, filename)
- self.failIfTextFile(pathname)
-
- def testDeletesFilesCorrectly(self):
- size = 100
- self.bd.set_text_file_size(1)
- self.bd.set_binary_file_size(1)
- self.bd.create_files(size)
- self.bd.find_preexisting_files()
- count = self.bd.get_preexisting_file_count()
- to_delete = count/3
- remaining = count - to_delete
- self.bd.delete_files(to_delete)
- self.bd.find_preexisting_files()
- self.failUnlessEqual(self.bd.get_preexisting_file_count(), remaining)
-
- def testRenamesFilesCorrectly(self):
- self.bd.create_directory()
- filename = self.bd.next_filename()
- self.create(filename, "")
- new_filename = self.bd.next_filename()
- self.bd.rename_files(1)
- self.failIf(os.path.exists(filename))
- self.failUnless(os.path.exists(new_filename))
-
- def testCreatesLinksCorrectly(self):
- self.bd.create_directory()
- filename = self.bd.next_filename()
- self.create(filename, "")
- new_filename = self.bd.next_filename()
- self.bd.link_files(1)
- self.failUnless(os.path.samefile(filename, new_filename))
-
- def testHasCorrectDefaultModifyPercentage(self):
- self.failUnlessEqual(self.bd.get_modify_percentage(),
- genbackupdata.DEFAULT_MODIFY_PERCENTAGE)
-
- def testCorrectlySetsModifyPercentage(self):
- self.bd.set_modify_percentage(42.0)
- self.failUnlessEqual(self.bd.get_modify_percentage(), 42.0)
-
- def testModifyFiles(self):
- size = 100
- self.bd.create_directory()
- filename = self.bd.next_filename()
- orig_data = "x" * size
- self.create(filename, orig_data)
- self.bd.modify_files(size)
- self.failUnlessEqual(len(self.read_file(filename)), 2 * size)
-
-
-class CommandLineParserTests(unittest.TestCase):
-
- dirname = "tests.dir"
-
- def setUp(self):
- self.bd = genbackupdata.BackupData()
- self.bd.set_directory(self.dirname)
- self.clp = genbackupdata.CommandLineParser(self.bd)
-
- def tearDown(self):
- del self.bd
- del self.clp
-
- def testDoesNotTouchDefaultsWithEmptyCommandLine(self):
- self.failUnlessEqual(self.bd.get_seed(), genbackupdata.DEFAULT_SEED)
- self.failUnlessEqual(self.bd.get_binary_chunk_size(),
- genbackupdata.DEFAULT_BINARY_CHUNK_SIZE)
- self.failUnlessEqual(self.bd.get_text_file_size(),
- genbackupdata.DEFAULT_TEXT_FILE_SIZE)
- self.failUnlessEqual(self.bd.get_binary_file_size(),
- genbackupdata.DEFAULT_BINARY_FILE_SIZE)
- self.failUnlessEqual(self.bd.get_text_data_percentage(),
- genbackupdata.DEFAULT_TEXT_DATA_PERCENTAGE)
- self.failUnlessEqual(self.bd.get_max_files_per_directory(),
- genbackupdata.DEFAULT_MAX_FILES_PER_DIRECTORY)
- self.failUnlessEqual(self.bd.get_modify_percentage(),
- genbackupdata.DEFAULT_MODIFY_PERCENTAGE)
-
- def testParsesPlainSizeCorrectly(self):
- self.failUnlessEqual(self.clp.parse_size("12765"), 12765)
-
- def testParsesAbsoluteSizeSuffixesCorrectly(self):
- self.failUnlessEqual(self.clp.parse_size("3k"), 3 * genbackupdata.KiB)
- self.failUnlessEqual(self.clp.parse_size("3K"), 3 * genbackupdata.KiB)
- self.failUnlessEqual(self.clp.parse_size("3m"), 3 * genbackupdata.MiB)
- self.failUnlessEqual(self.clp.parse_size("3M"), 3 * genbackupdata.MiB)
- self.failUnlessEqual(self.clp.parse_size("3g"), 3 * genbackupdata.GiB)
- self.failUnlessEqual(self.clp.parse_size("3G"), 3 * genbackupdata.GiB)
- self.failUnlessEqual(self.clp.parse_size("3t"), 3 * genbackupdata.TiB)
- self.failUnlessEqual(self.clp.parse_size("3T"), 3 * genbackupdata.TiB)
-
- def testParsesRelativeSizeSuffixCorrectly(self):
- self.failUnlessEqual(self.clp.parse_size("10%", 12765), 1276)
-
- def testParsesRelativeSizeSuffixCorrectlyWithMissingBaseSize(self):
- self.failUnlessEqual(self.clp.parse_size("10%"), 0)
-
- def testParsesPlainCountCorrectly(self):
- self.failUnlessEqual(self.clp.parse_count("12765"), 12765)
-
- def testParsesAbsoluteCountSuffixesCorrectly(self):
- self.failUnlessEqual(self.clp.parse_count("3k"), 3 * 10**3)
- self.failUnlessEqual(self.clp.parse_count("3K"), 3 * 10**3)
- self.failUnlessEqual(self.clp.parse_count("3m"), 3 * 10**6)
- self.failUnlessEqual(self.clp.parse_count("3M"), 3 * 10**6)
- self.failUnlessEqual(self.clp.parse_count("3g"), 3 * 10**9)
- self.failUnlessEqual(self.clp.parse_count("3G"), 3 * 10**9)
- self.failUnlessEqual(self.clp.parse_count("3t"), 3 * 10**12)
- self.failUnlessEqual(self.clp.parse_count("3T"), 3 * 10**12)
-
- def testParsesRelativeCountSuffixCorrectly(self):
- self.failUnlessEqual(self.clp.parse_count("10%", 12765), 1276)
-
- def testParsesRelativeCountSuffixCorrectlyWithMissingBaseSize(self):
- self.failUnlessEqual(self.clp.parse_count("10%"), 0)
-
- def testHandlesOptionForSeed(self):
- optons, args = self.clp.parse(["--seed=12765"])
- self.failUnlessEqual(args, [])
- self.failUnlessEqual(self.bd.get_seed(), 12765)
-
- def testHandlesOptionForMaxCount(self):
- options, args = self.clp.parse(["--max-count=12765"])
- self.failUnlessEqual(args, [])
- self.failUnlessEqual(self.bd.get_max_files_per_directory(), 12765)
-
- def testHandlesOptionForPercentageTextData(self):
- options, args = self.clp.parse(["--percentage-text-data=4.2"])
- self.failUnlessEqual(args, [])
- self.failUnlessEqual(self.bd.get_text_data_percentage(), 4.2)
-
- def testHandlesOptionForTextFileSize(self):
- options, args = self.clp.parse(["--text-file-size=12765"])
- self.failUnlessEqual(args, [])
- self.failUnlessEqual(self.bd.get_text_file_size(), 12765)
-
- def testHandlesOptionForTextFileSizeWithSuffix(self):
- options, args = self.clp.parse(["--text-file-size=1t"])
- self.failUnlessEqual(args, [])
- self.failUnlessEqual(self.bd.get_text_file_size(), genbackupdata.TiB)
-
- def testHandlesOptionForBinaryFileSizeWithSuffix(self):
- options, args = self.clp.parse(["--binary-file-size=1t"])
- self.failUnlessEqual(args, [])
- self.failUnlessEqual(self.bd.get_binary_file_size(),
- genbackupdata.TiB)
-
- def testHandlesOptionForCreate(self):
- options, args = self.clp.parse(["--create=1t"])
- self.failUnlessEqual(args, [])
- self.failUnlessEqual(options.create, genbackupdata.TiB)
-
- def testHandlesOptionForCreateWithRelativeSize(self):
- self.bd.set_preexisting_data_size(12765)
- options, args = self.clp.parse(["--create=10%"])
- self.failUnlessEqual(args, [])
- self.failUnlessEqual(options.create, 1276)
-
- def testHandlesOptionForDelete(self):
- options, args = self.clp.parse(["--delete=12765"])
- self.failUnlessEqual(args, [])
- self.failUnlessEqual(options.delete, 12765)
-
- def testHandlesOptionForDeleteWithRelativeCount(self):
- self.bd.set_preexisting_file_count(12765)
- options, args = self.clp.parse(["--delete=10%"])
- self.failUnlessEqual(args, [])
- self.failUnlessEqual(options.delete, 1276)
-
- def testHandlesOptionForRename(self):
- options, args = self.clp.parse(["--rename=12765"])
- self.failUnlessEqual(args, [])
- self.failUnlessEqual(options.rename, 12765)
-
- def testHandlesOptionForLink(self):
- options, args = self.clp.parse(["--link=12765"])
- self.failUnlessEqual(args, [])
- self.failUnlessEqual(options.link, 12765)
-
- def testHandlesOptionForModify(self):
- options, args = self.clp.parse(["--modify=12765"])
- self.failUnlessEqual(args, [])
- self.failUnlessEqual(options.modify, 12765)
-
- def testHandlesOptionForModifyPercentage(self):
- options, args = self.clp.parse(["--modify-percentage=4.2"])
- self.failUnlessEqual(args, [])
- self.failUnlessEqual(self.bd.get_modify_percentage(), 4.2)
-
-
-class ApplicationTests(unittest.TestCase):
-
- dirname = "tests.dir"
-
- def remove_dir(self):
- """Remove a directory, if it exists"""
- if os.path.exists(self.dirname):
- shutil.rmtree(self.dirname)
-
- def setUp(self):
- self.remove_dir()
-
- def tearDown(self):
- self.remove_dir()
-
- def file_count(self):
- count = 0
- for root, dirs, filenames in os.walk(self.dirname):
- count += len(filenames)
- return count
-
- def data_size(self):
- size = 0
- for root, dirs, filenames in os.walk(self.dirname):
- for filename in filenames:
- size += os.path.getsize(os.path.join(root, filename))
- return size
-
- def file_list(self):
- files = []
- for root, dirs, filenames in os.walk(self.dirname):
- for filename in filenames:
- files.append(os.path.join(root, filename))
- return files
-
- def nop(self, *args):
- pass
-
- def testTerminatesWithoutDirectory(self):
- app = genbackupdata.Application([])
- app.set_error_writer(self.nop)
- self.failUnlessRaises(SystemExit, app.run)
-
- def apprun(self, args):
- app = genbackupdata.Application(args)
- app.run()
-
- def testCreatesFirstGenerationCorrectly(self):
- self.apprun(["-c10k", self.dirname])
- self.failUnlessEqual(self.data_size(), 10 * genbackupdata.KiB)
-
- def testIncreasesSecondGenerationCorrectly(self):
- self.apprun(["-c10k", self.dirname])
- self.apprun(["-c10k", self.dirname])
- self.failUnlessEqual(self.data_size(), 20 * genbackupdata.KiB)
-
- def testModifiesSecondGenerationCorrectly(self):
- self.apprun(["-c10k", self.dirname])
- count = self.file_count()
- self.apprun(["-m1k", self.dirname])
- self.failUnlessEqual(self.data_size(), 11 * genbackupdata.KiB)
- self.failUnlessEqual(self.file_count(), count)
-
- def testDeletesFilesForSecondGenerationCorrectly(self):
- self.apprun(["-c10k", self.dirname])
- count = self.file_count()
- self.apprun(["-d2", self.dirname])
- self.failUnlessEqual(self.file_count(), count - 2)
-
- def testRenamesFilesForSecondGenerationCorrectly(self):
- self.apprun(["-c10k", self.dirname])
- files1 = self.file_list()
- self.apprun(["-r2", self.dirname])
- files2 = self.file_list()
- self.failIfEqual(sorted(files1), sorted(files2))
- self.failUnlessEqual(len(files1), len(files2))
-
- def testLinksFilesForSecondGenerationCorrectly(self):
- self.apprun(["-c10k", self.dirname])
- files1 = self.file_list()
- self.apprun(["-l2", self.dirname])
- files2 = self.file_list()
- self.failUnlessEqual(len(files1) + 2, len(files2))
-
-
-if __name__ == "__main__":
- unittest.main()
diff --git a/without-tests b/without-tests
new file mode 100644
index 0000000..f849124
--- /dev/null
+++ b/without-tests
@@ -0,0 +1,2 @@
+./setup.py
+./genbackupdatalib/__init__.py