From affce30bc41659bbfd60646db94fde37152f20c6 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Mon, 18 Dec 2017 12:28:42 +0200 Subject: Add: ql-ikiwiki-preprocess --- ql-ikiwiki-preprocess | 236 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 236 insertions(+) create mode 100755 ql-ikiwiki-preprocess diff --git a/ql-ikiwiki-preprocess b/ql-ikiwiki-preprocess new file mode 100755 index 0000000..2803043 --- /dev/null +++ b/ql-ikiwiki-preprocess @@ -0,0 +1,236 @@ +#!/usr/bin/env python3 +# Copyright 2017 QvarnLabs Ab +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# =*= License: GPL-3+ =*= + + +import os +import shutil +import subprocess +import sys +import tempfile + +import cliapp + + +def write_temp(content, suffix, tgtdir): + fd, filename = tempfile.mkstemp(dir=tgtdir, suffix=suffix) + os.close(fd) + with open(filename, 'w') as f: + f.write(content) + return filename + + +def get_uml_title(lines): + prefix = 'title ' + for line in lines: + line = line.strip() + if line.startswith(prefix): + return line[len(prefix):] + return 'title' + + +def format_uml(filename): + subprocess.check_call(['plantuml', '-tpng', '-output', '.', filename]) + + +def remove_prefixes(lines): + prefix = ' ' * 4 + result = [] + for line in lines: + if line.startswith(prefix): + line = line[len(prefix):] + result.append(line) + return ''.join(result) + + +def process_uml(lines, tgtdir): + title = get_uml_title(lines) + uml_text = remove_prefixes(lines) + filename = write_temp(uml_text, '.uml', tgtdir) + format_uml(filename) + prefix, _ = os.path.splitext(filename) + image = prefix + '.png' + relative = make_relative(os.path.abspath(tgtdir), image) + include = '[[!img {}]]\n'.format(relative) + return [include] + + +def format_roadmap(text): + input = text.encode('utf-8') + output = cliapp.runcmd(['projgraph'], feed_stdin=input) + output_text = output.decode('utf-8') + output_lines = output_text.splitlines() + return '\n'.join(output_lines[1:-1]) + + +def process_roadmap(lines, tgtdir): + roadmap_text = remove_prefixes(lines[1:-1]) + include = format_roadmap(roadmap_text) + return ['[[!graph src="""\n', include, '"""]]\n'] + + +def read_lines(filename): + with open(filename) as f: + return f.readlines() + + +def write_lines(filename, lines): + with open(filename, 'w') as f: + f.write(''.join(lines)) + + +def line_matches(line, patterns): + prefix = ' ' * 4 + for i, pattern in enumerate(patterns): + if line in [pattern, prefix + pattern]: + return i + return None + + +def find_match(lines, patterns): + for i, line in enumerate(lines): + j = line_matches(line, patterns) + if j is not None: + return i, j + else: + return None, None + + +def find_section(lines, sections): + starters = [x for x, _, _ in sections] + enders = [x for _, x, _ in sections] + funcs = [x for _, _, x in sections] + + first, section = find_match(lines, starters) + if first is not None: + last, _ = find_match(lines, enders[section:section+1]) + if last is not None: + return first, last+1, funcs[section] + return 0, len(lines), None + + +def preprocess_lines(lines, tgtdir): + sections = [ + ('@startuml\n', '@enduml\n', process_uml), + ('@startroadmap\n', '@endroadmap\n', process_roadmap), + ] + + result = [] + while lines: + start, after, func = find_section(lines, sections) + result.extend(lines[:start]) + section = lines[start:after] + if func: + include = func(section, tgtdir) + result.extend(include) + else: + result.extend(section) + lines = lines[after:] + return result + + +def preprocess_mdwn(src, tgt): + lines = read_lines(src) + tgtdir = os.path.dirname(tgt) + lines = preprocess_lines(lines, tgtdir) + write_lines(tgt, lines) + shutil.copystat(src, tgt) + + +def copy_file(src, tgt): + shutil.copy(src, tgt) + shutil.copystat(src, tgt) + + +def preprocess_file(src, tgt): + if src.endswith('.mdwn'): + preprocess_mdwn(src, tgt) + else: + copy_file(src, tgt) + + +def copy_dir(src, tgt): + os.makedirs(tgt) + return lambda: shutil.copystat(src, tgt) + + +def acceptable(basename): + return basename != '.git' + + +def make_relative(dirname, pathname): + prefix = dirname + if not prefix.endswith('/'): + prefix += '/' + + assert pathname == dirname or pathname.startswith(prefix) + if pathname == dirname: + return '.' + else: + return pathname[len(prefix):] + + +def find_all(root): + for dirpath, subdirs, filenames in os.walk(root): + yield dirpath + for x in subdirs[:]: + if not acceptable(x): + subdirs.remove(x) + for basename in filenames: + yield os.path.join(dirpath, basename) + + +def find_relative(dirname): + for pathname in find_all(dirname): + yield make_relative(dirname, pathname) + + +def make_absolute(root, relative): + return os.path.join(root, relative) + + +def preprocess(srcdir, tgtdir): + preprocessors = [ + (os.path.isfile, preprocess_file), + (os.path.isdir, copy_dir), + ] + + assert os.path.isdir(srcdir) + assert not os.path.exists(tgtdir) + todo = [] + for pathname in find_relative(srcdir): + src = make_absolute(srcdir, pathname) + tgt = make_absolute(tgtdir, pathname) + + for func, callback in preprocessors: + if func(src): + do = callback(src, tgt) + if do is not None: + todo.append(do) + + for do in todo: + do() + + +def main(): + srcdir = sys.argv[1] + tgtdir = sys.argv[2] + preprocess(srcdir, tgtdir) + + +if __name__ == '__main__': + main() -- cgit v1.2.1 From b13331e6a6e42a2219890224ff1b170b8f8785ec Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Mon, 18 Dec 2017 12:29:09 +0200 Subject: Add: projgraph --- projgraph | 96 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 96 insertions(+) create mode 100755 projgraph diff --git a/projgraph b/projgraph new file mode 100755 index 0000000..c3f5757 --- /dev/null +++ b/projgraph @@ -0,0 +1,96 @@ +#!/usr/bin/python3 + +import sys + +import yaml + + +unknown = 0 +blocked = 1 +finished = 2 +ready = 3 +next = 4 + +statuses = { + 'blocked': blocked, + 'finished': finished, + 'ready': ready, + 'next': next, +} + +def nodeattrs(done): + attrs = { + unknown: {'shape': 'diamond'}, + blocked: {'color': '#777777', 'shape': 'rectangle'}, + finished: {'color': '#eeeeee'}, + ready: {'color': '#bbbbbb'}, + next: {'color': '#00cc00'}, + } + + a = dict(attrs[done]) + if 'style' not in a: + a['style'] = 'filled' + return ' '.join('{}="{}"'.format(key, a[key]) for key in a) + + +def find_unknown(tasklist): + return [t for t in tasklist if t['status'] == unknown] + + +def all_deps(tasks, task, status): + for dep_name in task.get('depends', []): + dep = tasks[dep_name] + if dep['status'] != status: + return False + return True + + +def any_dep(tasks, task, status): + for dep_name in task.get('depends', []): + dep = tasks[dep_name] + if dep['status'] == status: + return True + return False + + +def set_status(tasks): + tasklist = list(tasks.values()) + for task in tasklist: + if 'status' not in task: + task['status'] = unknown + else: + task['status'] = statuses[task['status']] + unknown_tasks = find_unknown(tasklist) + while unknown_tasks: + for t in unknown_tasks: + if not t.get('depends', []): + t['status'] = ready + else: + if all_deps(tasks, t, finished): + t['status'] = ready + else: + t['status'] = blocked + unknown_tasks = find_unknown(tasklist) + + +obj = yaml.safe_load(sys.stdin) +ikiwiki = sys.argv[1:] == ['ikiwiki'] + +if ikiwiki: + print('[[!graph src="""') +else: + print('digraph "project" {') + +tasks = obj['tasks'] +set_status(tasks) +for name, task in tasks.items(): + print('{} [label="{}"]'.format(name, task['label'])) + status = task['status'] + print('{} [{}]'.format(name, nodeattrs(status))) + for dep in task.get('depends', []): + print('{} -> {}'.format(name, dep)) + +if ikiwiki: + print('"""]]') +else: + print('}') -- cgit v1.2.1 From 67f26c9248291b7332a166f91dafde0ae99573e3 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Mon, 18 Dec 2017 12:29:42 +0200 Subject: Add: new scripts to setup.py --- setup.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 680c5f3..e04ee39 100644 --- a/setup.py +++ b/setup.py @@ -27,6 +27,10 @@ setup( description='Build and publish a static site with ikiwiki', author='QvarnLabs Ab', author_email='info@qvarnlabs.com', - scripts=['ql-ikiwiki-publish'], + scripts=[ + 'ql-ikiwiki-publish', + 'ql-ikiwiki-preprocess', + 'projgraph', + ], packages=['ql_ikiwiki_publish'], ) -- cgit v1.2.1 From 0dbfa477ea9ba569c7edd21bf83f335dedf5a092 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Mon, 18 Dec 2017 13:08:47 +0200 Subject: Refactor: ql-ikiwiki-publish to be simpler --- ql-ikiwiki-publish | 159 +++++++++++++++++------------------------------------ 1 file changed, 49 insertions(+), 110 deletions(-) diff --git a/ql-ikiwiki-publish b/ql-ikiwiki-publish index 6745f69..4067af7 100755 --- a/ql-ikiwiki-publish +++ b/ql-ikiwiki-publish @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright 2016 QvarnLabs Ab # # This program is free software: you can redistribute it and/or modify @@ -26,128 +26,67 @@ import fnmatch import re -def process_uml_blocks(mdwnpath, mdwndir, filename): - blocks = [] - tmproot = os.path.join(T, os.path.splitext(filename)[0] + '_uml') - - with open(mdwnpath) as f: - content = f.read() - - # Find UML code blocks to be replaced. - blocks = re.findall( - r"(?:\t| {4})@startuml[\s\S]*?(?:\t| {4})@enduml", - content - ) - - for i, block in enumerate(blocks, start=1): - umlpath = '%s%d.puml' % (tmproot, i) - imgpath = '%s%d.png' % (tmproot, i) - - # Remove whitespace and save UML file. - with open(umlpath, 'w') as f: - f.write(re.sub(r"(?m)^(\t| {4})", "", block) + '\n') - - # Create a diagram image at imgpath and copy it. - subprocess.check_call(['plantuml', umlpath]) - shutil.copy2(imgpath, mdwndir) - - # Replace diagram code with a link to the image. - content = '' - with open(mdwnpath) as f: - content = f.read() - with open(mdwnpath, 'w') as f: - link = '[[!img %s]]' % os.path.basename(imgpath) - f.write(re.sub(re.escape(block), link, content)) - - -def process_uml_includes(mdwnpath, mdwndir): - includes = [] - - with open(mdwnpath) as f: - content = f.read() - - # Find include statements, - # put file name in a separate group. - includes = re.findall( - r"(?m)^()", - content - ) - - for inc in includes: - # Don't need to copy an image this time - subprocess.check_call(['plantuml', os.path.join(mdwndir, inc[1])]) - - # Replace include statement with a link to the image. - content = '' - with open(mdwnpath) as f: - content = f.read() - with open(mdwnpath, 'w') as f: - link = '[[!img %s.png]]' % os.path.splitext(inc[1])[0] - f.write(re.sub('(?m)^' + inc[0], link, content)) - - -def process_uml(S, T, dirname='.', ignore=['.git*', '.ikiwiki']): - shutil.copytree( - os.path.abspath('.'), S, ignore=shutil.ignore_patterns(*ignore) - ) +def preprocess(srcdir, tgtdir): + argv = [ + './ql-ikiwiki-preprocess', + srcdir, + tgtdir, + ] + subprocess.check_call(argv) - for topdir, dirs, files in os.walk(S, topdown=True): - for filename in fnmatch.filter(files, '*.mdwn'): - mdwnpath = os.path.join(topdir, filename) - process_uml_blocks(mdwnpath, topdir, filename) - process_uml_includes(mdwnpath, topdir) +def wanted(line): + prefixes = ['srcdir:', 'destdir:'] + for prefix in prefixes: + if line.startswith(prefix): + return False + return True -def mangle_setup(src, dest, html, mdwn): - with open(src) as f: +def mangle_setup(setup, tgtdir, htmldir): + with open(setup) as f: text = f.read() lines = [ line for line in text.splitlines() - if (not line.startswith('srcdir:') - or not line.startswith('destdir:')) + if wanted(line) ] - lines.append('srcdir: {}'.format(mdwn)) - lines.append('destdir: {}'.format(html)) + lines.append('srcdir: {}'.format(tgtdir)) + lines.append('destdir: {}'.format(htmldir)) mangled = ''.join(line + '\n' for line in lines) - - with open(dest, 'w') as f: + with open(setup, 'w') as f: f.write(mangled) -def run_ikiwiki(S, T, locally=False): - setup = os.path.join(T, 'ikiwiki.setup') - html = os.path.join(T, 'html') - mangle_setup('ikiwiki.setup', setup, html, S) - subprocess.check_call(['ikiwiki', '--setup', setup, '--gettime']) - if not locally: - subprocess.check_call( - ['rsync', '-ahHSvs', '--delete', html + '/.', rsync_target]) - - -rsync_target = None - -if len(sys.argv) == 3: - static_http = sys.argv[1] - dirname = sys.argv[2] - rsync_target = 'static@{}:/srv/http/{}/.'.format(static_http, dirname) - -S = '.publish' -T = tempfile.mkdtemp() -try: - process_uml(S, T) - run_ikiwiki(S, T, locally=rsync_target is None) -except BaseException as e: - shutil.rmtree(S) - shutil.rmtree(T) - raise -else: - shutil.rmtree(S) - if rsync_target is None: - print("Open wiki: firefox %s/html/index.html" % T) - else: - shutil.rmtree(T) +def run_ikiwiki(setup): + argv = ['ikiwiki', '--setup', setup] + subprocess.check_call(argv) + + +def run_rsync(htmldir, target): + argv = ['rsync', '-a', '--delete', htmldir + '/.', target + '/.'] + subprocess.check_call(argv) + + +srcdir = sys.argv[1] +rsync_target = sys.argv[2] + +tempdir = tempfile.mkdtemp() +tgtdir = os.path.join(tempdir, 'tgt') +htmldir = os.path.join(tempdir, 'html') +setup = os.path.join(tgtdir, 'ikiwiki.setup') + +print('tempdir', tempdir) +print('srcdir', srcdir) +print('tgtdir', tgtdir) +print('htmldir', htmldir) +print('setup', setup) +print('rsync_target', rsync_target) + +preprocess(srcdir, tgtdir) +mangle_setup(setup, tgtdir, htmldir) +run_ikiwiki(setup) +run_rsync(htmldir, rsync_target) -- cgit v1.2.1 From 3d9a63366e441c749c2eb92202c95893370a11eb Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Mon, 18 Dec 2017 13:10:42 +0200 Subject: Add: update NEWS --- NEWS | 3 +++ 1 file changed, 3 insertions(+) diff --git a/NEWS b/NEWS index 1d5aeb6..47db032 100644 --- a/NEWS +++ b/NEWS @@ -4,6 +4,9 @@ NEWS for ql-ikiwiki-publish Version 0.7+git, not yet released --------------------------------- +* Add new scripts `projgraph`, `ql-ikiwiki-preprocess` Rewrite + `ql-ikiwiki-publish` to use them and simplify it massively in the + process. Command line interface has changed incompatibly. Version 0.7, released 2017-10-06 --------------------------------- -- cgit v1.2.1