aboutsummaryrefslogtreecommitdiffstats
path: root/pycco
blob: 7a17a11a8d083ad616813385acf9385bdf22e8ac (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
#!/usr/bin/env python

# **Pycco** is a Python port of [Docco](http://jashkenas.github.com/docco/ ):
# the original quick-and-dirty, hundred-line-long, literate-programming-style
# documentation generator. It produces HTML that displays your comments
# alongside your code. Comments are passed through
# [Markdown](http://daringfireball.net/projects/markdown/syntax), and code is
# passed through [Pygments](http://pygments.org/) syntax highlighting.  This
# page is the result of running Pycco against its own source file.
#
# If you install Pycco, you can run it from the command-line:
#
#     pycco src/*.py
#
# ...will generate linked HTML documentation for the named source files, saving
# it into a `docs` folder.
#
# To install Pycco, simply
#
#     sudo setup.py install
#

#### Main Documentation Generation Functions

# Generate the documentation for a source file by reading it in, splitting it
# up into comment/code sections, highlighting them for the appropriate language,
# and merging them into an HTML template.
def generate_documentation(source):
    fh = open(source, "r")
    sections = parse(source, preparse(fh.read()))
    highlight(source,
              sections)
    generate_html(source, sections)

license_header = """// This code is under the XYZ License""";

# Preparse /*blah*/-style comments into //-style comments
def preparse(code):
    c = re.compile('(/\*.*?\*/)', re.DOTALL)
    d = re.compile('^\s?([ \t\*]+?)', re.MULTILINE)
    l = []
    for block in c.split(code):
        if(block.count("THE SOFTWARE IS PROVIDED")):
            l.append(license_header);
        elif(block.strip().startswith("/*")):
            print [block, d.sub(r'// ', block.strip()[2:-2])]
            l.append(d.sub(r'// ', block.strip()[2:-2]))
        else:
            l.append(block)

    return "".join(l)

# Given a string of source code, parse out each comment and the code that
# follows it, and create an individual **section** for it.
# Sections take the form:
#
#     { "docs_text": ...,
#       "docs_html": ...,
#       "code_text": ...,
#       "code_html": ...,
#       "num":       ...
#     }
#
def parse(source, code):
    lines = code.split("\n")
    sections = []
    language = get_language(source)
    has_code = docs_text = code_text = ""

    if lines[0].startswith("#!"):
        lines.pop(0)

    def save(docs, code):
        sections.append({
            "docs_text": docs,
            "code_text": code
        })

    for line in lines:
        if re.match(language["comment_matcher"], line):
            if has_code:
                save(docs_text, code_text)
                has_code = docs_text = code_text = ''
            docs_text += re.sub(language["comment_matcher"], "", line) + "\n"
        else:
            has_code = True
            code_text += line + '\n'
    save(docs_text, code_text)
    return sections

# Highlights a single chunk of code using the **Pygments** module, and runs the
# text of its corresponding comment through **Markdown**.
#
# We process the entire file in a single call to Pygments by inserting little
# marker comments between each section and then splitting the result string
# wherever our markers occur.
def highlight(source, sections):
    language = get_language(source)

    output = pygments.highlight(language["divider_text"].join(section["code_text"] for section in sections),
                                language["lexer"],
                                formatters.get_formatter_by_name("html"))

    output = output.replace(highlight_start, "").replace(highlight_end, "")
    fragments = re.split(language["divider_html"], output)
    for i, section in enumerate(sections):
        section["code_html"] = highlight_start + shift(fragments, "") + highlight_end
        section["docs_html"] = markdown(section["docs_text"])
        section["num"] = i

# Once all of the code is finished highlighting, we can generate the HTML file
# and write out the documentation. Pass the completed sections into the template
# found in `resources/pycco.html`
def generate_html(source, sections):
    title = path.basename(source)
    dest = destination(source)
    html = pycco_template({
        "title":       title,
        "sections":    sections,
        "sources":     sources,
        "path":        path,
        "destination": destination
    })
    print "pycco = %s -> %s" % (source, dest)
    fh = open(dest, "w")
    fh.write(html)
    fh.close()

#### Helpers & Setup

# This module contains all of our static resources.
import pycco_resources

# Import our external dependencies.
import pygments
import pystache
import re
import sys
from markdown import markdown
from os import path
from pygments import lexers, formatters
from subprocess import Popen, PIPE

# A list of the languages that Pycco supports, mapping the file extension to
# the name of the Pygments lexer and the symbol that indicates a comment. To
# add another language to Pycco's repertoire, add it here.
languages = {
    #".coffee": { "name": "coffee-script", "symbol": "#" },
    ".js":     { "name": "javascript",    "symbol": "//" },
    ".rb":     { "name": "ruby",          "symbol": "#" },
    ".py":     { "name": "python",        "symbol": "#" },
    #".scm":    { "name": "scheme",        "symbol": ";;" },
    #".lua":    { "name": "lua",           "symbol": "--" },
    ".c":      { "name": "c",             "symbol": "//"},
}

# Build out the appropriate matchers and delimiters for each language.
for ext, l in languages.items():
    # Does the line begin with a comment?
    l["comment_matcher"] = re.compile(r"^\s*" + l["symbol"] + "\s?")

    # The dividing token we feed into Pygments, to delimit the boundaries between
    # sections.
    l["divider_text"] = "\n" + l["symbol"] + "DIVIDER\n"

    # The mirror of `divider_text` that we expect Pygments to return. We can split
    # on this to recover the original sections.
    l["divider_html"] = re.compile(r'\n*<span class="c[1]?">' + l["symbol"] + 'DIVIDER</span>\n*')

    # Get the Pygments Lexer for this language.
    l["lexer"] = lexers.get_lexer_by_name(l["name"])

# Get the current language we're documenting, based on the extension.
def get_language(source):
    try:
        return languages[ source[source.rindex("."):] ]
    except ValueError:
        source = open(source, "r")
        code = source.read()
        source.close()
        lang = lexers.guess_lexer(code).name.lower()
        for l in languages.values():
            if l["name"] == lang:
                return l
        else:
            raise ValueError("Can't figure out the language!")

# Compute the destination HTML path for an input source file path. If the source
# is `lib/example.py`, the HTML will be at `docs/example.html`
def destination(filepath):
    try:
        name = filepath.replace(filepath[ filepath.rindex("."): ], "")
    except ValueError:
        name = filepath
    return "docs/" + path.basename(name) + ".html"

# Shift items off the front of the `list` until it is empty, then return
# `default`.
def shift(list, default):
    try:
        return list.pop(0)
    except IndexError:
        return default

# Ensure that the destination directory exists.
def ensure_directory():
    Popen(["mkdir", "-p", "docs"]).wait()

def template(source):
    return lambda context: pystache.render(source, context)

# Create the template that we will use to generate the Pycco HTML page.
pycco_template = template(pycco_resources.html)

# The CSS styles we"d like to apply to the documentation.
pycco_styles = pycco_resources.css

# The start of each Pygments highlight block.
highlight_start = "<div class=\"highlight\"><pre>"

# The end of each Pygments highlight block.
highlight_end = "</pre></div>"

# Run the script.
# For each source file passed in as an argument, generate the documentation.
if __name__ == "__main__":
    sources = list(sys.argv[1:])
    sources.sort()
    if sources:
        ensure_directory()
        css = open("docs/pycco.css", "w")
        css.write(pycco_styles)
        css.close()

        def next_file():
            generate_documentation(sources.pop(0))
            if sources:
                next_file()
        next_file()