summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--NEWS3
-rwxr-xr-xprojgraph96
-rwxr-xr-xql-ikiwiki-preprocess236
-rwxr-xr-xql-ikiwiki-publish159
-rw-r--r--setup.py6
5 files changed, 389 insertions, 111 deletions
diff --git a/NEWS b/NEWS
index 1d5aeb6..47db032 100644
--- a/NEWS
+++ b/NEWS
@@ -4,6 +4,9 @@ NEWS for ql-ikiwiki-publish
Version 0.7+git, not yet released
---------------------------------
+* Add new scripts `projgraph`, `ql-ikiwiki-preprocess` Rewrite
+ `ql-ikiwiki-publish` to use them and simplify it massively in the
+ process. Command line interface has changed incompatibly.
Version 0.7, released 2017-10-06
---------------------------------
diff --git a/projgraph b/projgraph
new file mode 100755
index 0000000..c3f5757
--- /dev/null
+++ b/projgraph
@@ -0,0 +1,96 @@
+#!/usr/bin/python3
+
+import sys
+
+import yaml
+
+
+unknown = 0
+blocked = 1
+finished = 2
+ready = 3
+next = 4
+
+statuses = {
+ 'blocked': blocked,
+ 'finished': finished,
+ 'ready': ready,
+ 'next': next,
+}
+
+def nodeattrs(done):
+ attrs = {
+ unknown: {'shape': 'diamond'},
+ blocked: {'color': '#777777', 'shape': 'rectangle'},
+ finished: {'color': '#eeeeee'},
+ ready: {'color': '#bbbbbb'},
+ next: {'color': '#00cc00'},
+ }
+
+ a = dict(attrs[done])
+ if 'style' not in a:
+ a['style'] = 'filled'
+ return ' '.join('{}="{}"'.format(key, a[key]) for key in a)
+
+
+def find_unknown(tasklist):
+ return [t for t in tasklist if t['status'] == unknown]
+
+
+def all_deps(tasks, task, status):
+ for dep_name in task.get('depends', []):
+ dep = tasks[dep_name]
+ if dep['status'] != status:
+ return False
+ return True
+
+
+def any_dep(tasks, task, status):
+ for dep_name in task.get('depends', []):
+ dep = tasks[dep_name]
+ if dep['status'] == status:
+ return True
+ return False
+
+
+def set_status(tasks):
+ tasklist = list(tasks.values())
+ for task in tasklist:
+ if 'status' not in task:
+ task['status'] = unknown
+ else:
+ task['status'] = statuses[task['status']]
+ unknown_tasks = find_unknown(tasklist)
+ while unknown_tasks:
+ for t in unknown_tasks:
+ if not t.get('depends', []):
+ t['status'] = ready
+ else:
+ if all_deps(tasks, t, finished):
+ t['status'] = ready
+ else:
+ t['status'] = blocked
+ unknown_tasks = find_unknown(tasklist)
+
+
+obj = yaml.safe_load(sys.stdin)
+ikiwiki = sys.argv[1:] == ['ikiwiki']
+
+if ikiwiki:
+ print('[[!graph src="""')
+else:
+ print('digraph "project" {')
+
+tasks = obj['tasks']
+set_status(tasks)
+for name, task in tasks.items():
+ print('{} [label="{}"]'.format(name, task['label']))
+ status = task['status']
+ print('{} [{}]'.format(name, nodeattrs(status)))
+ for dep in task.get('depends', []):
+ print('{} -> {}'.format(name, dep))
+
+if ikiwiki:
+ print('"""]]')
+else:
+ print('}')
diff --git a/ql-ikiwiki-preprocess b/ql-ikiwiki-preprocess
new file mode 100755
index 0000000..2803043
--- /dev/null
+++ b/ql-ikiwiki-preprocess
@@ -0,0 +1,236 @@
+#!/usr/bin/env python3
+# Copyright 2017 QvarnLabs Ab
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+# =*= License: GPL-3+ =*=
+
+
+import os
+import shutil
+import subprocess
+import sys
+import tempfile
+
+import cliapp
+
+
+def write_temp(content, suffix, tgtdir):
+ fd, filename = tempfile.mkstemp(dir=tgtdir, suffix=suffix)
+ os.close(fd)
+ with open(filename, 'w') as f:
+ f.write(content)
+ return filename
+
+
+def get_uml_title(lines):
+ prefix = 'title '
+ for line in lines:
+ line = line.strip()
+ if line.startswith(prefix):
+ return line[len(prefix):]
+ return 'title'
+
+
+def format_uml(filename):
+ subprocess.check_call(['plantuml', '-tpng', '-output', '.', filename])
+
+
+def remove_prefixes(lines):
+ prefix = ' ' * 4
+ result = []
+ for line in lines:
+ if line.startswith(prefix):
+ line = line[len(prefix):]
+ result.append(line)
+ return ''.join(result)
+
+
+def process_uml(lines, tgtdir):
+ title = get_uml_title(lines)
+ uml_text = remove_prefixes(lines)
+ filename = write_temp(uml_text, '.uml', tgtdir)
+ format_uml(filename)
+ prefix, _ = os.path.splitext(filename)
+ image = prefix + '.png'
+ relative = make_relative(os.path.abspath(tgtdir), image)
+ include = '[[!img {}]]\n'.format(relative)
+ return [include]
+
+
+def format_roadmap(text):
+ input = text.encode('utf-8')
+ output = cliapp.runcmd(['projgraph'], feed_stdin=input)
+ output_text = output.decode('utf-8')
+ output_lines = output_text.splitlines()
+ return '\n'.join(output_lines[1:-1])
+
+
+def process_roadmap(lines, tgtdir):
+ roadmap_text = remove_prefixes(lines[1:-1])
+ include = format_roadmap(roadmap_text)
+ return ['[[!graph src="""\n', include, '"""]]\n']
+
+
+def read_lines(filename):
+ with open(filename) as f:
+ return f.readlines()
+
+
+def write_lines(filename, lines):
+ with open(filename, 'w') as f:
+ f.write(''.join(lines))
+
+
+def line_matches(line, patterns):
+ prefix = ' ' * 4
+ for i, pattern in enumerate(patterns):
+ if line in [pattern, prefix + pattern]:
+ return i
+ return None
+
+
+def find_match(lines, patterns):
+ for i, line in enumerate(lines):
+ j = line_matches(line, patterns)
+ if j is not None:
+ return i, j
+ else:
+ return None, None
+
+
+def find_section(lines, sections):
+ starters = [x for x, _, _ in sections]
+ enders = [x for _, x, _ in sections]
+ funcs = [x for _, _, x in sections]
+
+ first, section = find_match(lines, starters)
+ if first is not None:
+ last, _ = find_match(lines, enders[section:section+1])
+ if last is not None:
+ return first, last+1, funcs[section]
+ return 0, len(lines), None
+
+
+def preprocess_lines(lines, tgtdir):
+ sections = [
+ ('@startuml\n', '@enduml\n', process_uml),
+ ('@startroadmap\n', '@endroadmap\n', process_roadmap),
+ ]
+
+ result = []
+ while lines:
+ start, after, func = find_section(lines, sections)
+ result.extend(lines[:start])
+ section = lines[start:after]
+ if func:
+ include = func(section, tgtdir)
+ result.extend(include)
+ else:
+ result.extend(section)
+ lines = lines[after:]
+ return result
+
+
+def preprocess_mdwn(src, tgt):
+ lines = read_lines(src)
+ tgtdir = os.path.dirname(tgt)
+ lines = preprocess_lines(lines, tgtdir)
+ write_lines(tgt, lines)
+ shutil.copystat(src, tgt)
+
+
+def copy_file(src, tgt):
+ shutil.copy(src, tgt)
+ shutil.copystat(src, tgt)
+
+
+def preprocess_file(src, tgt):
+ if src.endswith('.mdwn'):
+ preprocess_mdwn(src, tgt)
+ else:
+ copy_file(src, tgt)
+
+
+def copy_dir(src, tgt):
+ os.makedirs(tgt)
+ return lambda: shutil.copystat(src, tgt)
+
+
+def acceptable(basename):
+ return basename != '.git'
+
+
+def make_relative(dirname, pathname):
+ prefix = dirname
+ if not prefix.endswith('/'):
+ prefix += '/'
+
+ assert pathname == dirname or pathname.startswith(prefix)
+ if pathname == dirname:
+ return '.'
+ else:
+ return pathname[len(prefix):]
+
+
+def find_all(root):
+ for dirpath, subdirs, filenames in os.walk(root):
+ yield dirpath
+ for x in subdirs[:]:
+ if not acceptable(x):
+ subdirs.remove(x)
+ for basename in filenames:
+ yield os.path.join(dirpath, basename)
+
+
+def find_relative(dirname):
+ for pathname in find_all(dirname):
+ yield make_relative(dirname, pathname)
+
+
+def make_absolute(root, relative):
+ return os.path.join(root, relative)
+
+
+def preprocess(srcdir, tgtdir):
+ preprocessors = [
+ (os.path.isfile, preprocess_file),
+ (os.path.isdir, copy_dir),
+ ]
+
+ assert os.path.isdir(srcdir)
+ assert not os.path.exists(tgtdir)
+ todo = []
+ for pathname in find_relative(srcdir):
+ src = make_absolute(srcdir, pathname)
+ tgt = make_absolute(tgtdir, pathname)
+
+ for func, callback in preprocessors:
+ if func(src):
+ do = callback(src, tgt)
+ if do is not None:
+ todo.append(do)
+
+ for do in todo:
+ do()
+
+
+def main():
+ srcdir = sys.argv[1]
+ tgtdir = sys.argv[2]
+ preprocess(srcdir, tgtdir)
+
+
+if __name__ == '__main__':
+ main()
diff --git a/ql-ikiwiki-publish b/ql-ikiwiki-publish
index 6745f69..4067af7 100755
--- a/ql-ikiwiki-publish
+++ b/ql-ikiwiki-publish
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# Copyright 2016 QvarnLabs Ab
#
# This program is free software: you can redistribute it and/or modify
@@ -26,128 +26,67 @@ import fnmatch
import re
-def process_uml_blocks(mdwnpath, mdwndir, filename):
- blocks = []
- tmproot = os.path.join(T, os.path.splitext(filename)[0] + '_uml')
-
- with open(mdwnpath) as f:
- content = f.read()
-
- # Find UML code blocks to be replaced.
- blocks = re.findall(
- r"(?:\t| {4})@startuml[\s\S]*?(?:\t| {4})@enduml",
- content
- )
-
- for i, block in enumerate(blocks, start=1):
- umlpath = '%s%d.puml' % (tmproot, i)
- imgpath = '%s%d.png' % (tmproot, i)
-
- # Remove whitespace and save UML file.
- with open(umlpath, 'w') as f:
- f.write(re.sub(r"(?m)^(\t| {4})", "", block) + '\n')
-
- # Create a diagram image at imgpath and copy it.
- subprocess.check_call(['plantuml', umlpath])
- shutil.copy2(imgpath, mdwndir)
-
- # Replace diagram code with a link to the image.
- content = ''
- with open(mdwnpath) as f:
- content = f.read()
- with open(mdwnpath, 'w') as f:
- link = '[[!img %s]]' % os.path.basename(imgpath)
- f.write(re.sub(re.escape(block), link, content))
-
-
-def process_uml_includes(mdwnpath, mdwndir):
- includes = []
-
- with open(mdwnpath) as f:
- content = f.read()
-
- # Find include statements,
- # put file name in a separate group.
- includes = re.findall(
- r"(?m)^(<!-{2,3} INCLUDE_UML: (.+\.puml) -->)",
- content
- )
-
- for inc in includes:
- # Don't need to copy an image this time
- subprocess.check_call(['plantuml', os.path.join(mdwndir, inc[1])])
-
- # Replace include statement with a link to the image.
- content = ''
- with open(mdwnpath) as f:
- content = f.read()
- with open(mdwnpath, 'w') as f:
- link = '[[!img %s.png]]' % os.path.splitext(inc[1])[0]
- f.write(re.sub('(?m)^' + inc[0], link, content))
-
-
-def process_uml(S, T, dirname='.', ignore=['.git*', '.ikiwiki']):
- shutil.copytree(
- os.path.abspath('.'), S, ignore=shutil.ignore_patterns(*ignore)
- )
+def preprocess(srcdir, tgtdir):
+ argv = [
+ './ql-ikiwiki-preprocess',
+ srcdir,
+ tgtdir,
+ ]
+ subprocess.check_call(argv)
- for topdir, dirs, files in os.walk(S, topdown=True):
- for filename in fnmatch.filter(files, '*.mdwn'):
- mdwnpath = os.path.join(topdir, filename)
- process_uml_blocks(mdwnpath, topdir, filename)
- process_uml_includes(mdwnpath, topdir)
+def wanted(line):
+ prefixes = ['srcdir:', 'destdir:']
+ for prefix in prefixes:
+ if line.startswith(prefix):
+ return False
+ return True
-def mangle_setup(src, dest, html, mdwn):
- with open(src) as f:
+def mangle_setup(setup, tgtdir, htmldir):
+ with open(setup) as f:
text = f.read()
lines = [
line
for line in text.splitlines()
- if (not line.startswith('srcdir:')
- or not line.startswith('destdir:'))
+ if wanted(line)
]
- lines.append('srcdir: {}'.format(mdwn))
- lines.append('destdir: {}'.format(html))
+ lines.append('srcdir: {}'.format(tgtdir))
+ lines.append('destdir: {}'.format(htmldir))
mangled = ''.join(line + '\n' for line in lines)
-
- with open(dest, 'w') as f:
+ with open(setup, 'w') as f:
f.write(mangled)
-def run_ikiwiki(S, T, locally=False):
- setup = os.path.join(T, 'ikiwiki.setup')
- html = os.path.join(T, 'html')
- mangle_setup('ikiwiki.setup', setup, html, S)
- subprocess.check_call(['ikiwiki', '--setup', setup, '--gettime'])
- if not locally:
- subprocess.check_call(
- ['rsync', '-ahHSvs', '--delete', html + '/.', rsync_target])
-
-
-rsync_target = None
-
-if len(sys.argv) == 3:
- static_http = sys.argv[1]
- dirname = sys.argv[2]
- rsync_target = 'static@{}:/srv/http/{}/.'.format(static_http, dirname)
-
-S = '.publish'
-T = tempfile.mkdtemp()
-try:
- process_uml(S, T)
- run_ikiwiki(S, T, locally=rsync_target is None)
-except BaseException as e:
- shutil.rmtree(S)
- shutil.rmtree(T)
- raise
-else:
- shutil.rmtree(S)
- if rsync_target is None:
- print("Open wiki: firefox %s/html/index.html" % T)
- else:
- shutil.rmtree(T)
+def run_ikiwiki(setup):
+ argv = ['ikiwiki', '--setup', setup]
+ subprocess.check_call(argv)
+
+
+def run_rsync(htmldir, target):
+ argv = ['rsync', '-a', '--delete', htmldir + '/.', target + '/.']
+ subprocess.check_call(argv)
+
+
+srcdir = sys.argv[1]
+rsync_target = sys.argv[2]
+
+tempdir = tempfile.mkdtemp()
+tgtdir = os.path.join(tempdir, 'tgt')
+htmldir = os.path.join(tempdir, 'html')
+setup = os.path.join(tgtdir, 'ikiwiki.setup')
+
+print('tempdir', tempdir)
+print('srcdir', srcdir)
+print('tgtdir', tgtdir)
+print('htmldir', htmldir)
+print('setup', setup)
+print('rsync_target', rsync_target)
+
+preprocess(srcdir, tgtdir)
+mangle_setup(setup, tgtdir, htmldir)
+run_ikiwiki(setup)
+run_rsync(htmldir, rsync_target)
diff --git a/setup.py b/setup.py
index 680c5f3..e04ee39 100644
--- a/setup.py
+++ b/setup.py
@@ -27,6 +27,10 @@ setup(
description='Build and publish a static site with ikiwiki',
author='QvarnLabs Ab',
author_email='info@qvarnlabs.com',
- scripts=['ql-ikiwiki-publish'],
+ scripts=[
+ 'ql-ikiwiki-publish',
+ 'ql-ikiwiki-preprocess',
+ 'projgraph',
+ ],
packages=['ql_ikiwiki_publish'],
)