summaryrefslogtreecommitdiff
path: root/create-files
diff options
context:
space:
mode:
authorLars Wirzenius <liw@liw.fi>2019-05-04 20:14:05 +0300
committerLars Wirzenius <liw@liw.fi>2019-05-04 20:14:05 +0300
commit2fb5fe3aa68d039a73ddc6fd4602e0ae7e5b8fb9 (patch)
tree9f2628fe4b5f585f66e5d331d414dab12b95b0ab /create-files
downloadbillion-files-2fb5fe3aa68d039a73ddc6fd4602e0ae7e5b8fb9.tar.gz
Add: create-files
Diffstat (limited to 'create-files')
-rwxr-xr-xcreate-files78
1 files changed, 78 insertions, 0 deletions
diff --git a/create-files b/create-files
new file mode 100755
index 0000000..93b50e0
--- /dev/null
+++ b/create-files
@@ -0,0 +1,78 @@
+#!/usr/bin/env python
+
+import json
+import os
+import sys
+import time
+
+
+billion = 10 ** 9
+per_level = 10 ** 3
+interval = 2
+state_filename = 'state.json'
+
+
+def level(i, n):
+ return (i % (per_level ** n)) / (per_level ** (n-1))
+
+
+def pathname(i):
+ fullname = '{:09d}'.format(i)
+ dir1 = fullname[-9:-6]
+ dir2 = fullname[-6:-3]
+ dirname = '{}/{}'.format(dir1, dir2)
+ basename = fullname[-3:]
+ return dirname, basename
+
+
+def create_dir(root, dirname):
+ try:
+ os.makedirs(os.path.join(root, dirname))
+ except OSError:
+ pass
+
+
+def create_file(root, dirname, basename):
+ try:
+ open(os.path.join(root, dirname, basename), 'w')
+ except OSError:
+ return
+
+
+def load_state():
+ try:
+ return json.load(open(state_filename))
+ except IOError:
+ return {}
+
+
+def save_state(i):
+ state = {
+ 'current': i,
+ }
+ with open(state_filename, 'w') as f:
+ json.dump(state, f)
+
+
+
+root = sys.argv[1]
+start = int(sys.argv[2])
+end = int(sys.argv[3])
+
+i = load_state().get('current', start)
+
+curdir = None
+last_checkpoint = time.time()
+
+for i in range(i, end+1):
+ dirname, basename = pathname(i)
+ if dirname != curdir:
+ create_dir(root, dirname)
+ curdir = dirname
+ create_file(root, dirname, basename)
+ now = time.time()
+ if now > last_checkpoint + interval:
+ save_state(i)
+ last_checkpoint = now
+
+save_state(i)