#!/usr/bin/env python import json import os import sys import time billion = 10 ** 9 per_level = 10 ** 3 interval = 2 state_filename = 'state.json' def level(i, n): return (i % (per_level ** n)) / (per_level ** (n-1)) def pathname(i): fullname = '{:09d}'.format(i) dir1 = fullname[-9:-6] dir2 = fullname[-6:-3] dirname = '{}/{}'.format(dir1, dir2) basename = fullname[-3:] return dirname, basename def create_dir(root, dirname): try: os.makedirs(os.path.join(root, dirname)) except OSError: pass def create_file(root, dirname, basename): try: open(os.path.join(root, dirname, basename), 'w') except OSError: return def load_state(): try: return json.load(open(state_filename)) except IOError: return {} def save_state(i): state = { 'current': i, } with open(state_filename, 'w') as f: json.dump(state, f) root = sys.argv[1] start = int(sys.argv[2]) end = int(sys.argv[3]) i = load_state().get('current', start) curdir = None last_checkpoint = time.time() for i in range(i, end+1): dirname, basename = pathname(i) if dirname != curdir: create_dir(root, dirname) curdir = dirname create_file(root, dirname, basename) now = time.time() if now > last_checkpoint + interval: save_state(i) last_checkpoint = now save_state(i)