From 3a720729fc2ec1b147aa15b07aa2bed8346e02bf Mon Sep 17 00:00:00 2001 From: bnewbold Date: Sat, 15 Jul 2017 23:41:39 -0700 Subject: cleanups --- divergence | 151 ++++++++++++++++++++++ pandoc_confluence.lua | 346 ++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 497 insertions(+) create mode 100755 divergence create mode 100644 pandoc_confluence.lua diff --git a/divergence b/divergence new file mode 100755 index 0000000..f72d4db --- /dev/null +++ b/divergence @@ -0,0 +1,151 @@ +#!/usr/bin/env python3 +""" +License: MIT +Author: Bryan Newbold +Date: July 2017 + +See README +""" + +from __future__ import print_function +import sys, os +import argparse +import requests +import subprocess +import logging as log + + +class DivergenceProgram: + + def __init__(self, user, password, url, space): + self.api = requests.Session() + self.api.auth = (user, password) + self.api.headers.update({'Content-Type': 'application/json'}) + self.base_url = url + self.space = space + + # TODO: find lua path? + + def get_page(self, title): + """ + Returns None if not found, otherwise a dict with id, space, and body (in storage format) + """ + resp = self.api.get(self.base_url + "/rest/api/content", + params={"spaceKey": self.space, + "title": title, + "expand": "body.storage,version,space", + "type": "page"}) + + log.debug(resp) + log.debug(resp.content) + assert resp.status_code == 200 + respj = resp.json() + if respj['size'] == 0: + return None + assert respj['size'] == 1, "Expect single result for title lookup" + page = respj['results'][0] + assert page['space']['key'].upper() == self.space.upper(), "Expect spaces to match" + + return {"id": int(page['id']), + "version": int(page['version']['number']), + "space": page['space']['key'], + "body": page['body']['storage']['value']} + + def create_page(self, title, body): + resp = self.api.post(self.base_url + "/rest/api/content", + json={"space": { "key": self.space }, + "type": "page", + "title": title, + "body": { + "storage": { + "representation": "storage", + "value": body } } } ) + log.debug(resp) + log.debug(resp.content) + assert resp.status_code == 200 + + def update_page(self, title, body, page_id, prev_version): + resp = self.api.put(self.base_url + "/rest/api/content/%d" % page_id, + json={"type": "page", + "title": title, + "version": {"number": prev_version+1}, + "body": { + "storage": { + "representation": "storage", + "value": body } } } ) + log.debug(resp) + log.debug(resp.content) + assert resp.status_code == 200 + + def title_from_path(self, path): + title = path.split('.')[0].replace('_', ' ') + # TODO: only alphanum and spaces? + return title + + def convert(self, f): + proc = subprocess.run(["pandoc", "-t", "pandoc_confluence.lua", f], + stdout=subprocess.PIPE) + assert proc.returncode == 0 + return proc.stdout.decode('UTF-8') + + def run(self, files): + + for f in files: + title = self.title_from_path(f) + log.debug(title) + body = self.convert(f) + prev = self.get_page(title) + log.debug(prev) + if prev is None: + self.create_page(title, body) + print(f + ": created") + else: + if prev['body'] != body: + self.update_page(title, body, prev['id'], prev['version']) + print(f + ": updated") + else: + print(f + ": no change") + +def main(): + parser = argparse.ArgumentParser( + description=""" +Simple Markdown-to-Confluence uploader, using pandoc and the Confluence REST +API. + +Specify credentials and site URL with environment variables: + CONFLUENCE_USER + CONFLUENCE_PASSWORD + CONFLUENCE_URL +""") + #usage="%(prog)s [options] -s ") + parser.add_argument("-v", "--verbose", + action="count", + default=0, + help="Show more debugging statements (can be repeated)") + parser.add_argument("-s", "--space-key", + required=True, + help='Confluence Space Key (usually like "PROJ" or "~username")') + parser.add_argument("FILE", nargs='+') + + args = parser.parse_args() + + if args.verbose > 0: + log.basicConfig(format="%(levelname)s: %(message)s", level=log.DEBUG) + else: + log.basicConfig(format="%(levelname)s: %(message)s") + + try: + user = os.environ['CONFLUENCE_USER'] + password = os.environ['CONFLUENCE_PASSWORD'] + url = os.environ['CONFLUENCE_URL'] + except KeyError: + parser.exit(-1, "Need to pass environment variable configs") + + if url.endswith('/'): + url = url[:-1] + + dp = DivergenceProgram(user, password, url, args.space_key) + dp.run(args.FILE) + +if __name__ == '__main__': + main() diff --git a/pandoc_confluence.lua b/pandoc_confluence.lua new file mode 100644 index 0000000..83c60e2 --- /dev/null +++ b/pandoc_confluence.lua @@ -0,0 +1,346 @@ +-- This is a custom writer for Pandoc to output Confluence Storage Format. +-- It is based on the sample custom HTML writer packaged with Pandoc. +-- +-- Note: you need not have lua installed on your system to use this +-- custom writer. However, if you do have lua installed, you can +-- use it to test changes to the script. 'lua sample.lua' will +-- produce informative error messages if your code contains +-- syntax errors. + +-- Character escaping +local function escape(s, in_attribute) + return s:gsub("[<>&\"']", + function(x) + if x == '<' then + return '<' + elseif x == '>' then + return '>' + elseif x == '&' then + return '&' + elseif x == '"' then + return '"' + elseif x == "'" then + return ''' + else + return x + end + end) +end + +-- Helper function to convert an attributes table into +-- a string that can be put into HTML tags. +local function attributes(attr) + local attr_table = {} + for x,y in pairs(attr) do + if y and y ~= "" then + table.insert(attr_table, ' ' .. x .. '="' .. escape(y,true) .. '"') + end + end + return table.concat(attr_table) +end + +local function anchor(id) + if id ~= "" then + return '' .. id .. '' + else + return "" + end +end + +-- Run cmd on a temporary file containing inp and return result. +local function pipe(cmd, inp) + local tmp = os.tmpname() + local tmph = io.open(tmp, "w") + tmph:write(inp) + tmph:close() + local outh = io.popen(cmd .. " " .. tmp,"r") + local result = outh:read("*all") + outh:close() + os.remove(tmp) + return result +end + +-- Table to store footnotes, so they can be included at the end. +local notes = {} + +-- Blocksep is used to separate block elements. +function Blocksep() + return "\n\n" +end + +-- This function is called once for the whole document. Parameters: +-- body is a string, metadata is a table, variables is a table. +-- This gives you a fragment. You could use the metadata table to +-- fill variables in a custom lua template. Or, pass `--template=...` +-- to pandoc, and pandoc will add do the template processing as +-- usual. +function Doc(body, metadata, variables) + local buffer = {} + local function add(s) + table.insert(buffer, s) + end + add(body) + if #notes > 0 then + add('
    ') + for _,note in pairs(notes) do + add(note) + end + add('
') + end + return table.concat(buffer,'\n') +end + +-- The functions that follow render corresponding pandoc elements. +-- s is always a string, attr is always a table of attributes, and +-- items is always an array of strings (the items in a list). +-- Comments indicate the types of other variables. + +function Str(s) + return escape(s) +end + +function Space() + return " " +end + +function SoftBreak() + return "\n" +end + +function RawBlock(format, str) + -- TODO + return str +end + +function LineBreak() + return "
" +end + +function Emph(s) + return "" .. s .. "" +end + +function Strong(s) + return "" .. s .. "" +end + +function Subscript(s) + return "" .. s .. "" +end + +function Superscript(s) + return "" .. s .. "" +end + +function SmallCaps(s) + return '' .. s .. '' +end + +function Strikeout(s) + return '' .. s .. '' +end + +function Link(s, src, tit) + if string.find(src, "^#") then + return '' .. s .. '' + else + return "" .. s .. "" + end +end + +function imageRi(path) + if string.find(path, "^media/") ~= nil then + local _, _, _, name = string.find(path, "(.*/)(.*)$") + return "" + else + return "" + end +end + +function Image(s, src, tit) + return "" .. imageRi(src) .."" +end + +function CaptionedImage(src, tit, txt) + return "" .. imageRi(src) .."" +end + +function Code(s, attr) + return anchor(attr.id) .. "" .. escape(s) .. "" +end + +function InlineMath(s) + return "\\(" .. escape(s) .. "\\)" +end + +function DisplayMath(s) + return "\\[" .. escape(s) .. "\\]" +end + +-- TODO use anchors for footnote references +function Note(s) + local num = #notes + 1 + -- insert the back reference right before the final closing tag. + s = string.gsub(s, + '(.*)↩' .. s .. '') + -- return the footnote reference, linked to the note. + return '' .. num .. '' +end + +function Span(s, attr) + return anchor(attr.id) .. "" .. s .. "" +end + +function Cite(s, cs) + local ids = {} + for _,cit in ipairs(cs) do + table.insert(ids, cit.citationId) + end + return "" .. s .. "" +end + +function Plain(s) + return s +end + +function Para(s) + return "

" .. s .. "

" +end + +-- lev is an integer, the header level. +function Header(lev, s, attr) + return anchor(attr.id) .. "" .. s .. "" +end + +function BlockQuote(s) + return "

" .. s .. "

" +end + +function HorizontalRule() + return "
" +end + +function CodeBlock(s, attr) + -- If code block has class 'dot', pipe the contents through dot + -- and base64, and include the base64-encoded png as a data: URL. + if attr.class and string.match(' ' .. attr.class .. ' ',' dot ') then + local png = pipe("base64", pipe("dot -Tpng", s)) + return anchor(attr.id) .. '' + -- otherwise treat as code (one could pipe through a highlighter) + else + -- TODO: could try to parse attr and insert a lang here + -- XXX: "
" .. escape(s) .. "
" + return anchor(attr.id) .. '' .. + 'text' .. + ' ' .. + '' + end +end + +function BulletList(items) + local buffer = {} + for _, item in pairs(items) do + table.insert(buffer, "
  • " .. item .. "
    • ") + end + return "
      \n" .. table.concat(buffer, "\n") .. "\n
    " +end + +function OrderedList(items) + local buffer = {} + for _, item in pairs(items) do + table.insert(buffer, "
  • " .. item .. "
    • ") + end + return "
      \n" .. table.concat(buffer, "\n") .. "\n
    " +end + +-- Revisit association list STackValue instance. +function DefinitionList(items) + local buffer = {} + for _,item in pairs(items) do + for k, v in pairs(item) do + table.insert(buffer,"
    " .. k .. "
    \n
    " .. + table.concat(v,"
    \n
    ") .. "
    ") + end + end + return "
    \n" .. table.concat(buffer, "\n") .. "\n
    " +end + +-- Convert pandoc alignment to something HTML can use. +-- align is AlignLeft, AlignRight, AlignCenter, or AlignDefault. +function html_align(align) + if align == 'AlignLeft' then + return 'left' + elseif align == 'AlignRight' then + return 'right' + elseif align == 'AlignCenter' then + return 'center' + else + return 'left' + end +end + +-- Caption is a string, aligns is an array of strings, +-- widths is an array of floats, headers is an array of +-- strings, rows is an array of arrays of strings. +function Table(caption, aligns, widths, headers, rows) + local buffer = {} + local function add(s) + table.insert(buffer, s) + end + add("") + if caption ~= "" then + add("") + end + if widths and widths[1] ~= 0 then + for _, w in pairs(widths) do + add('') + end + end + local header_row = {} + local empty_header = true + for i, h in pairs(headers) do + local align = html_align(aligns[i]) + table.insert(header_row,'') + empty_header = empty_header and h == "" + end + if empty_header then + head = "" + else + add('') + for _,h in pairs(header_row) do + add(h) + end + add('') + end + local class = "even" + for _, row in pairs(rows) do + class = (class == "even" and "odd") or "even" + add('') + for i,c in pairs(row) do + add('') + end + add('') + end + add('
    " .. caption .. "
    ' .. h .. '
    ' .. c .. '
    ') + return table.concat(buffer,'\n') +end + +function Div(s, attr) + return anchor(attr.id) .. "\n" .. s .. "" +end + +-- The following code will produce runtime warnings when you haven't defined +-- all of the functions you need for the custom writer, so it's useful +-- to include when you're working on a writer. +local meta = {} +meta.__index = + function(_, key) + io.stderr:write(string.format("WARNING: Undefined function '%s'\n",key)) + return function() return "" end + end +setmetatable(_G, meta) + -- cgit v1.2.3