aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorbnewbold <bnewbold@robocracy.org>2017-07-15 23:41:39 -0700
committerbnewbold <bnewbold@robocracy.org>2017-07-15 23:41:39 -0700
commit3a720729fc2ec1b147aa15b07aa2bed8346e02bf (patch)
treea5c9edef911a2e603f5b506609a28c34ef23a7a2
parent6a1b9c499e2fb4c9a444a0f0c75ea43c33d7f28b (diff)
downloaddivergence-3a720729fc2ec1b147aa15b07aa2bed8346e02bf.zip
divergence-3a720729fc2ec1b147aa15b07aa2bed8346e02bf.tar.gz
cleanups
-rwxr-xr-xdivergence151
-rw-r--r--pandoc_confluence.lua346
2 files changed, 497 insertions, 0 deletions
diff --git a/divergence b/divergence
new file mode 100755
index 0000000..f72d4db
--- /dev/null
+++ b/divergence
@@ -0,0 +1,151 @@
+#!/usr/bin/env python3
+"""
+License: MIT
+Author: Bryan Newbold <bnewbold@archive.org>
+Date: July 2017
+
+See README
+"""
+
+from __future__ import print_function
+import sys, os
+import argparse
+import requests
+import subprocess
+import logging as log
+
+
+class DivergenceProgram:
+
+ def __init__(self, user, password, url, space):
+ self.api = requests.Session()
+ self.api.auth = (user, password)
+ self.api.headers.update({'Content-Type': 'application/json'})
+ self.base_url = url
+ self.space = space
+
+ # TODO: find lua path?
+
+ def get_page(self, title):
+ """
+ Returns None if not found, otherwise a dict with id, space, and body (in storage format)
+ """
+ resp = self.api.get(self.base_url + "/rest/api/content",
+ params={"spaceKey": self.space,
+ "title": title,
+ "expand": "body.storage,version,space",
+ "type": "page"})
+
+ log.debug(resp)
+ log.debug(resp.content)
+ assert resp.status_code == 200
+ respj = resp.json()
+ if respj['size'] == 0:
+ return None
+ assert respj['size'] == 1, "Expect single result for title lookup"
+ page = respj['results'][0]
+ assert page['space']['key'].upper() == self.space.upper(), "Expect spaces to match"
+
+ return {"id": int(page['id']),
+ "version": int(page['version']['number']),
+ "space": page['space']['key'],
+ "body": page['body']['storage']['value']}
+
+ def create_page(self, title, body):
+ resp = self.api.post(self.base_url + "/rest/api/content",
+ json={"space": { "key": self.space },
+ "type": "page",
+ "title": title,
+ "body": {
+ "storage": {
+ "representation": "storage",
+ "value": body } } } )
+ log.debug(resp)
+ log.debug(resp.content)
+ assert resp.status_code == 200
+
+ def update_page(self, title, body, page_id, prev_version):
+ resp = self.api.put(self.base_url + "/rest/api/content/%d" % page_id,
+ json={"type": "page",
+ "title": title,
+ "version": {"number": prev_version+1},
+ "body": {
+ "storage": {
+ "representation": "storage",
+ "value": body } } } )
+ log.debug(resp)
+ log.debug(resp.content)
+ assert resp.status_code == 200
+
+ def title_from_path(self, path):
+ title = path.split('.')[0].replace('_', ' ')
+ # TODO: only alphanum and spaces?
+ return title
+
+ def convert(self, f):
+ proc = subprocess.run(["pandoc", "-t", "pandoc_confluence.lua", f],
+ stdout=subprocess.PIPE)
+ assert proc.returncode == 0
+ return proc.stdout.decode('UTF-8')
+
+ def run(self, files):
+
+ for f in files:
+ title = self.title_from_path(f)
+ log.debug(title)
+ body = self.convert(f)
+ prev = self.get_page(title)
+ log.debug(prev)
+ if prev is None:
+ self.create_page(title, body)
+ print(f + ": created")
+ else:
+ if prev['body'] != body:
+ self.update_page(title, body, prev['id'], prev['version'])
+ print(f + ": updated")
+ else:
+ print(f + ": no change")
+
+def main():
+ parser = argparse.ArgumentParser(
+ description="""
+Simple Markdown-to-Confluence uploader, using pandoc and the Confluence REST
+API.
+
+Specify credentials and site URL with environment variables:
+ CONFLUENCE_USER
+ CONFLUENCE_PASSWORD
+ CONFLUENCE_URL
+""")
+ #usage="%(prog)s [options] -s <space-key> <files>")
+ parser.add_argument("-v", "--verbose",
+ action="count",
+ default=0,
+ help="Show more debugging statements (can be repeated)")
+ parser.add_argument("-s", "--space-key",
+ required=True,
+ help='Confluence Space Key (usually like "PROJ" or "~username")')
+ parser.add_argument("FILE", nargs='+')
+
+ args = parser.parse_args()
+
+ if args.verbose > 0:
+ log.basicConfig(format="%(levelname)s: %(message)s", level=log.DEBUG)
+ else:
+ log.basicConfig(format="%(levelname)s: %(message)s")
+
+ try:
+ user = os.environ['CONFLUENCE_USER']
+ password = os.environ['CONFLUENCE_PASSWORD']
+ url = os.environ['CONFLUENCE_URL']
+ except KeyError:
+ parser.exit(-1, "Need to pass environment variable configs")
+
+ if url.endswith('/'):
+ url = url[:-1]
+
+ dp = DivergenceProgram(user, password, url, args.space_key)
+ dp.run(args.FILE)
+
+if __name__ == '__main__':
+ main()
diff --git a/pandoc_confluence.lua b/pandoc_confluence.lua
new file mode 100644
index 0000000..83c60e2
--- /dev/null
+++ b/pandoc_confluence.lua
@@ -0,0 +1,346 @@
+-- This is a custom writer for Pandoc to output Confluence Storage Format.
+-- It is based on the sample custom HTML writer packaged with Pandoc.
+--
+-- Note: you need not have lua installed on your system to use this
+-- custom writer. However, if you do have lua installed, you can
+-- use it to test changes to the script. 'lua sample.lua' will
+-- produce informative error messages if your code contains
+-- syntax errors.
+
+-- Character escaping
+local function escape(s, in_attribute)
+ return s:gsub("[<>&\"']",
+ function(x)
+ if x == '<' then
+ return '&lt;'
+ elseif x == '>' then
+ return '&gt;'
+ elseif x == '&' then
+ return '&amp;'
+ elseif x == '"' then
+ return '&quot;'
+ elseif x == "'" then
+ return '&#39;'
+ else
+ return x
+ end
+ end)
+end
+
+-- Helper function to convert an attributes table into
+-- a string that can be put into HTML tags.
+local function attributes(attr)
+ local attr_table = {}
+ for x,y in pairs(attr) do
+ if y and y ~= "" then
+ table.insert(attr_table, ' ' .. x .. '="' .. escape(y,true) .. '"')
+ end
+ end
+ return table.concat(attr_table)
+end
+
+local function anchor(id)
+ if id ~= "" then
+ return '<ac:structured-macro ac:name="anchor"><ac:parameter ac:name="">' .. id .. '</ac:parameter></ac:structured-macro>'
+ else
+ return ""
+ end
+end
+
+-- Run cmd on a temporary file containing inp and return result.
+local function pipe(cmd, inp)
+ local tmp = os.tmpname()
+ local tmph = io.open(tmp, "w")
+ tmph:write(inp)
+ tmph:close()
+ local outh = io.popen(cmd .. " " .. tmp,"r")
+ local result = outh:read("*all")
+ outh:close()
+ os.remove(tmp)
+ return result
+end
+
+-- Table to store footnotes, so they can be included at the end.
+local notes = {}
+
+-- Blocksep is used to separate block elements.
+function Blocksep()
+ return "\n\n"
+end
+
+-- This function is called once for the whole document. Parameters:
+-- body is a string, metadata is a table, variables is a table.
+-- This gives you a fragment. You could use the metadata table to
+-- fill variables in a custom lua template. Or, pass `--template=...`
+-- to pandoc, and pandoc will add do the template processing as
+-- usual.
+function Doc(body, metadata, variables)
+ local buffer = {}
+ local function add(s)
+ table.insert(buffer, s)
+ end
+ add(body)
+ if #notes > 0 then
+ add('<ol class="footnotes">')
+ for _,note in pairs(notes) do
+ add(note)
+ end
+ add('</ol>')
+ end
+ return table.concat(buffer,'\n')
+end
+
+-- The functions that follow render corresponding pandoc elements.
+-- s is always a string, attr is always a table of attributes, and
+-- items is always an array of strings (the items in a list).
+-- Comments indicate the types of other variables.
+
+function Str(s)
+ return escape(s)
+end
+
+function Space()
+ return " "
+end
+
+function SoftBreak()
+ return "\n"
+end
+
+function RawBlock(format, str)
+ -- TODO
+ return str
+end
+
+function LineBreak()
+ return "<br/>"
+end
+
+function Emph(s)
+ return "<em>" .. s .. "</em>"
+end
+
+function Strong(s)
+ return "<strong>" .. s .. "</strong>"
+end
+
+function Subscript(s)
+ return "<sub>" .. s .. "</sub>"
+end
+
+function Superscript(s)
+ return "<sup>" .. s .. "</sup>"
+end
+
+function SmallCaps(s)
+ return '<span style="font-variant: small-caps;">' .. s .. '</span>'
+end
+
+function Strikeout(s)
+ return '<span style="text-decoration: line-through;">' .. s .. '</span>'
+end
+
+function Link(s, src, tit)
+ if string.find(src, "^#") then
+ return '<ac:link ac:anchor="' .. string.sub(src, 2) .. '"><ac:link-body>' .. s .. '</ac:link-body></ac:link>'
+ else
+ return "<a href='" .. escape(src, true) .. "'>" .. s .. "</a>"
+ end
+end
+
+function imageRi(path)
+ if string.find(path, "^media/") ~= nil then
+ local _, _, _, name = string.find(path, "(.*/)(.*)$")
+ return "<ri:attachment ri:filename='" .. escape(name, true) .. "' />"
+ else
+ return "<ri:url ri:value='" .. escape(path, true) .. "' />"
+ end
+end
+
+function Image(s, src, tit)
+ return "<ac:image ac:alt='" .. escape(tit, true) .. "'>" .. imageRi(src) .."</ac:image>"
+end
+
+function CaptionedImage(src, tit, txt)
+ return "<ac:image ac:alt='" .. escape(txt, true) .. "'>" .. imageRi(src) .."</ac:image>"
+end
+
+function Code(s, attr)
+ return anchor(attr.id) .. "<code" .. attributes(attr) .. ">" .. escape(s) .. "</code>"
+end
+
+function InlineMath(s)
+ return "\\(" .. escape(s) .. "\\)"
+end
+
+function DisplayMath(s)
+ return "\\[" .. escape(s) .. "\\]"
+end
+
+-- TODO use anchors for footnote references
+function Note(s)
+ local num = #notes + 1
+ -- insert the back reference right before the final closing tag.
+ s = string.gsub(s,
+ '(.*)</', '%1 <a href="#fnref' .. num .. '">&#8617;</a></')
+ -- add a list item with the note to the note table.
+ table.insert(notes, '<li id="fn' .. num .. '">' .. s .. '</li>')
+ -- return the footnote reference, linked to the note.
+ return '<a id="fnref' .. num .. '" href="#fn' .. num ..
+ '"><sup>' .. num .. '</sup></a>'
+end
+
+function Span(s, attr)
+ return anchor(attr.id) .. "<span" .. attributes(attr) .. ">" .. s .. "</span>"
+end
+
+function Cite(s, cs)
+ local ids = {}
+ for _,cit in ipairs(cs) do
+ table.insert(ids, cit.citationId)
+ end
+ return "<span class=\"cite\" data-citation-ids=\"" .. table.concat(ids, ",") ..
+ "\">" .. s .. "</span>"
+end
+
+function Plain(s)
+ return s
+end
+
+function Para(s)
+ return "<p>" .. s .. "</p>"
+end
+
+-- lev is an integer, the header level.
+function Header(lev, s, attr)
+ return anchor(attr.id) .. "<h" .. lev .. attributes(attr) .. ">" .. s .. "</h" .. lev .. ">"
+end
+
+function BlockQuote(s)
+ return "<blockquote><p>" .. s .. "</p></blockquote>"
+end
+
+function HorizontalRule()
+ return "<hr/>"
+end
+
+function CodeBlock(s, attr)
+ -- If code block has class 'dot', pipe the contents through dot
+ -- and base64, and include the base64-encoded png as a data: URL.
+ if attr.class and string.match(' ' .. attr.class .. ' ',' dot ') then
+ local png = pipe("base64", pipe("dot -Tpng", s))
+ return anchor(attr.id) .. '<img src="data:image/png;base64,' .. png .. '"/>'
+ -- otherwise treat as code (one could pipe through a highlighter)
+ else
+ -- TODO: could try to parse attr and insert a lang here
+ -- XXX: "<pre>" .. escape(s) .. "</pre>"
+ return anchor(attr.id) .. '<ac:structured-macro ac:name="code">' ..
+ '<ac:parameter ac:name="language">text</ac:parameter>' ..
+ '<ac:plain-text-body> <![CDATA[' .. s .. ']]></ac:plain-text-body>' ..
+ '</ac:structured-macro>'
+ end
+end
+
+function BulletList(items)
+ local buffer = {}
+ for _, item in pairs(items) do
+ table.insert(buffer, "<li>" .. item .. "</li>")
+ end
+ return "<ul>\n" .. table.concat(buffer, "\n") .. "\n</ul>"
+end
+
+function OrderedList(items)
+ local buffer = {}
+ for _, item in pairs(items) do
+ table.insert(buffer, "<li>" .. item .. "</li>")
+ end
+ return "<ol>\n" .. table.concat(buffer, "\n") .. "\n</ol>"
+end
+
+-- Revisit association list STackValue instance.
+function DefinitionList(items)
+ local buffer = {}
+ for _,item in pairs(items) do
+ for k, v in pairs(item) do
+ table.insert(buffer,"<dt>" .. k .. "</dt>\n<dd>" ..
+ table.concat(v,"</dd>\n<dd>") .. "</dd>")
+ end
+ end
+ return "<dl>\n" .. table.concat(buffer, "\n") .. "\n</dl>"
+end
+
+-- Convert pandoc alignment to something HTML can use.
+-- align is AlignLeft, AlignRight, AlignCenter, or AlignDefault.
+function html_align(align)
+ if align == 'AlignLeft' then
+ return 'left'
+ elseif align == 'AlignRight' then
+ return 'right'
+ elseif align == 'AlignCenter' then
+ return 'center'
+ else
+ return 'left'
+ end
+end
+
+-- Caption is a string, aligns is an array of strings,
+-- widths is an array of floats, headers is an array of
+-- strings, rows is an array of arrays of strings.
+function Table(caption, aligns, widths, headers, rows)
+ local buffer = {}
+ local function add(s)
+ table.insert(buffer, s)
+ end
+ add("<table>")
+ if caption ~= "" then
+ add("<caption>" .. caption .. "</caption>")
+ end
+ if widths and widths[1] ~= 0 then
+ for _, w in pairs(widths) do
+ add('<col width="' .. string.format("%d%%", w * 100) .. '" />')
+ end
+ end
+ local header_row = {}
+ local empty_header = true
+ for i, h in pairs(headers) do
+ local align = html_align(aligns[i])
+ table.insert(header_row,'<th align="' .. align .. '">' .. h .. '</th>')
+ empty_header = empty_header and h == ""
+ end
+ if empty_header then
+ head = ""
+ else
+ add('<tr class="header">')
+ for _,h in pairs(header_row) do
+ add(h)
+ end
+ add('</tr>')
+ end
+ local class = "even"
+ for _, row in pairs(rows) do
+ class = (class == "even" and "odd") or "even"
+ add('<tr class="' .. class .. '">')
+ for i,c in pairs(row) do
+ add('<td align="' .. html_align(aligns[i]) .. '">' .. c .. '</td>')
+ end
+ add('</tr>')
+ end
+ add('</table>')
+ return table.concat(buffer,'\n')
+end
+
+function Div(s, attr)
+ return anchor(attr.id) .. "<div" .. attributes(attr) .. ">\n" .. s .. "</div>"
+end
+
+-- The following code will produce runtime warnings when you haven't defined
+-- all of the functions you need for the custom writer, so it's useful
+-- to include when you're working on a writer.
+local meta = {}
+meta.__index =
+ function(_, key)
+ io.stderr:write(string.format("WARNING: Undefined function '%s'\n",key))
+ return function() return "" end
+ end
+setmetatable(_G, meta)
+