From 8d0b14c2aa854a45c0aeff8b9d888f3d9ea1b003 Mon Sep 17 00:00:00 2001
From: Alexander M Pickering <alex@cogarr.net>
Date: Thu, 1 Dec 2022 22:24:17 -0600
Subject: prepare mdoc for linux

---
 src/init.lua | 1138 +++++++++++++++++++++++++++++-----------------------------
 1 file changed, 569 insertions(+), 569 deletions(-)

diff --git a/src/init.lua b/src/init.lua
index 0d30148..2a8d282 100644
--- a/src/init.lua
+++ b/src/init.lua
@@ -1,569 +1,569 @@
---[[
-Mdoc creats documentation from comments in source files.
-Mdoc works in several steps:
-	1. Create "chunks" from input source files
-	2. Create a dependnecy graph, so we only update the things that
-	   need updating
-	3. Output html documentation
-]]
-local lfs = require("lfs")
-local et = require("etlua")
-require("mdoc.ext")
-local opt = require("mdoc.opt")
-
-local args = {...}
-local options = opt.parse_options(args)
-if options.help then
-	print(opt.help())	
-	return
-end
-if not options.paths or #options.paths == 0 then
-	print(opt.help())
-	return
-end
-
-local state = {
-	title = "",
-	sections = {},
-	files = {},
-	documents = {},
-	known_names = {},
-	index = {},
-}
-
-local function log(...)
-	if options.verbose then
-		print(...)
-	end
-end
-local function logf(...)
-	if options.verbose then
-		printf(...)
-	end
-end
-
-local toskip = {["."] = true,[".."] = true}
-local function scan(path,callback)
-	assert(path,"Cannot scan without a path to scan (was nil)")
-	assert(callback,"Cannot scan without a callback to call (was nil)")
-	for item in lfs.dir(path) do
-		log("looking at:",item)
-		if toskip[item] then
-			goto nextitem
-		end
-		local fullpath = path .. "/" .. item
-		local attributes = lfs.attributes(fullpath)
-		if attributes.mode == "directory" or attributes.mode == "link" then
-			log("found directory:",directory)
-			scan(fullpath,callback)
-		elseif attributes.mode == "file" then
-			log("found item:",item)
-			callback(fullpath)
-		end
-		::nextitem::
-	end
-end
-for _,path in pairs(options.paths) do
-	scan(path,function(p)
-		local attributes = lfs.attributes(p)
-		table.insert(state.files,{
-			path = p,
-			relpath = p:match(string.format("%s/(.*)",path)),
-			lastmod = attributes.modification,
-		})
-	end)
-end
-logf("Found %d files.",#state.files)
-
-for _,path in pairs(options.document_paths) do
-	scan(path,function(p)
-		local attributes = lfs.attributes(p)
-		table.insert(state.documents,{
-			path = p,
-			lastmod = attributes.modification,
-		})
-	end)
-end
-logf("Found %d documents.",#state.documents)
-
-
-local function parse(text,filename)
-	assert(options.parser,"Failed to find a parser option")
-	assert(text,"Failed to get text")
-	text = text:gsub("@{(.+)}",function(data)
-		return string.format("[%s](%s.html)",data,data)
-	end)
-	log("Using parser:",options.parser)
-	local tmp = options.output .. "/" .. os.tmpname()
-	log("using temp name:",tmp)
-	local pd = io.popen(options.parser .. " > " .. tmp,"w")
-	pd:write(text)
-	pd:close()
-	local id = assert(io.open(tmp,"r"))
-	local ret = id:read("*a")
-	id:close()
-	assert(os.remove(tmp))
-	return ret
-end
-
-local function table_to_string(tbl)
-	--Collect all of our tables first,
-	--so that we don't break when we have recursive tables.
-	local tables = {}
-	local table_order = {}
-	local function tables_helper(t)
-		tables[t] = #table_order + 1
-		table_order[#table_order + 1] = t
-		for k,v in pairs(t) do
-			if type(k) == "table" and not tables[k] then
-				tables_helper(k)
-			end
-			if type(v) == "table" and not tables[v] then
-				tables_helper(v)
-			end
-		end
-	end
-	tables_helper(tbl)
-	
-	--Get the string representation of an element
-	local errfun = function(e) error("Cannot format a " .. type(e)) end
-	local rep_map = {
-		--things we can format
-		number = function(e)
-			if e % 1 == 0 then return string.format("%d",e)
-			else return string.format("%f",e) end
-		end,
-		string = function(e) return string.format("%q",e) end,
-		boolean = function(e) return e and "true" or "false" end,
-		table = function(e)
-			assertf(tables[e] ~= nil,"Could not find dependency table %s",tostring(e))
-			return string.format("table_%d",tables[e])
-		end,
-		--things we can't format
-		["function"] = errfun,
-		coroutine = errfun,
-		file = errfun,
-		userdata = errfun,
-		--nil can never happen
-	}
-	local sb = {}
-	--Create all the variables first, so that recursive tables will work
-	for n,_ in pairs(table_order) do
-		sb[#sb + 1] = string.format("local table_%d = {}",n)
-	end
-	--Go backwards through tables, since that should be the
-	--"dependency" order
-	for i = #table_order, 1, -1 do -- -1 is needed in case #table_order == 0
-		local tstr = {}
-		local this_table = table_order[i]
-		for k,v in pairs(this_table) do
-			tstr[#tstr + 1] = string.format("table_%d[%s] = %s",i,rep_map[type(k)](k), rep_map[type(v)](v))
-		end
-		sb[#sb + 1] = table.concat(tstr,"\n")
-	end
-	sb[#sb + 1] = "return table_1"
-	return table.concat(sb,"\n\n");
-end
-
---io.open automatically creates parent directories
-local oldopen = io.open
-function io.open(path,mode)
-	if mode == "w" or mode == "w+" then
-		local path_so_far = ""
-		for folder in path:gmatch("([^/]+)/") do
-			path_so_far = path_so_far .. folder .. "/"
-			if lfs.attributes(path_so_far) == nil then
-				lfs.mkdir(path_so_far)
-			end
-		end
-	end
-	return oldopen(path,mode)
-end
-
---Create a cache directory
-if not lfs.attributes(options.output .. "/cache") then
-	assert(lfs.mkdir(options.output .. "/cache"))
-	assert(lfs.mkdir(options.output .. "/cache/documents"))
-	assert(lfs.mkdir(options.output .. "/cache/files"))
-	assert(lfs.mkdir(options.output .. "/cache/chunks"))
-end
-
---Get the data for the reference documents
-for dn, data in pairs(state.documents) do
-	--local file_name = data.path
-	local file_name = data.path:match("([^./]+)%.[^.]+$")
-	local cachefile = string.format("%s/cache/documents/%s",options.output,file_name)
-	local out_attrs = lfs.attributes(cachefile)
-	if not out_attrs or out_attrs.modification < data.lastmod or options.nocache then
-		--we're out of date, update
-		local fd = assert(io.open(data.path,"r"))
-		local file_text = fd:read("*a")
-		fd:close()
-		local parsed_text = parse(file_text, data.path)
-		local od = assert(io.open(cachefile,"w"))
-		od:write(parsed_text)
-		od:close()
-		state.sections[file_name] = {
-			type = "reference",
-			data_file = cachefile
-		}
-	end
-end
-
---Openers and closers
-local parse_between = {
-	["/***"] = "*/",
-	["--[[**"] = "]]"
-}
-for _,file in pairs(state.files) do
-	local cachefilename = string.format("%s/cache/files/%s",options.output,file.relpath)
-	local cacheattrs = lfs.attributes(cachefilename)
-	if (not cacheattrs) or cacheattrs.modification < file.lastmod or options.nocache then
-		local chunks = {}
-		local in_chunk = false
-		local line_num = 0
-		local closer
-		local fd = assert(io.open(file.path,"r"))
-		for line in fd:lines() do
-			line_num = line_num + 1
-			if parse_between[line] then
-				closer = parse_between[line]
-				table.insert(chunks,{})
-				in_chunk = true
-			end
-			if in_chunk then
-				table.insert(chunks[#chunks],line)
-			end
-			if in_chunk and line == closer then
-				log("found end, chunks:",chunks)
-				chunks[#chunks] = {
-					text = chunks[#chunks],
-					file = file.path,
-					line = line_num,
-					relpath = file.relpath,
-				}
-				in_chunk = false
-			end
-		end
-		file.chunks = chunks
-		local cachefile = assert(io.open(cachefilename,"w"))
-		cachefile:write(table_to_string(chunks))
-		cachefile:close()
-	else
-		file.chunks = loadfile(cachefilename)()
-	end
-end
-
-for _,file in pairs(state.files) do
-	local function process_chunk(chunk)
-		assert(chunk.file and chunk.line, "Chunk without file or line num:")
-		local section = nil
-		local partname = nil
-		local sectiontype = nil
-		local short_desc
-		local desc = {}
-		--Parameters are {
-		--	name = "string",
-		--	type = "string",
-		--	optional = bool
-		--	optchain = bool
-		--	default = nil | "string"
-		--}
-		local ret = {}
-		for num,line in pairs(chunk.text) do
-			if num == 2 then
-				if line:match("^@.*") then
-					short_desc = "no short description provided"
-				else
-					short_desc = line
-				end
-			end
-			if num > 2 and type(desc) == "table" and not line:match("^@.*") then
-				table.insert(desc,line)
-			end
-			if line:match("^@") then
-				local command, rest = line:match("^@([^ ]+) (.*)")
-				if command == "function" then
-					sectiontype = "function"
-					if rest:find(":") then
-						local classname, functionname = rest:match("^([^:]+):([^%(]+)")
-						ret = {
-						--state.sections[classname] = state.sections[classname] or {type = "class"}
-						--state.sections[classname][functionname] = state.sections[classname][functionname] or {
-							type="method",
-							name = functionname,
-							path = {classname,functionname},
-							short_desc = parse(short_desc),
-							line = chunk.line,
-							desc = parse(table.concat(desc,"\n")),
-							params = {},
-							returns = {},
-							file = chunk.relpath
-						}
-						section = classname
-						partname = functionname
-					else
-						local namespace, functionname
-						local c = rest:match("^([^.%(]+)%(")
-						if c then
-							namespace = "_G"
-							functionname = c
-						else
-							namespace, functionname = rest:match("([^.]+)%.([^%(]+)")
-						end
-						log("namespace:",namespace,"functionname:",functionname)
-						ret = {
-						--state.sections[namespace] = state.sections[namespace] or {type = "namespace"}
-						--state.sections[namespace][functionname] = state.sections[namespace][functionname] or {
-							type="function",
-							name = functionname,
-							short_desc = parse(short_desc),
-							path = {namespace,functionname},
-							line = chunk.line,
-							desc = parse(table.concat(desc,"\n")),
-							params = {},
-							returns = {},
-							references = {},
-							file = chunk.relpath
-						}
-						section = namespace
-						partname = functionname
-					end
-				elseif command == "tparam" or command == "tparam?" then
-					local parts = {}
-					local vartype, name, description
-					for word in rest:gmatch("([^ ]+)") do table.insert(parts,word) end
-					assertf(#parts >= 2,"@tparam at %s:%d requires at least a type and a variable name",chunk.file,chunk.line_num)
-					if parts[1] then
-						vartype = parts[1]
-						table.remove(parts,1)
-					end
-					if parts[1] then
-						name = parts[1]
-						table.remove(parts,1)
-					end
-					description = table.concat(parts,"\n")
-
-					assertf(section and partname and sectiontype == "function","Tried to specify a tparam for something that wasn't a function at %s:%d",chunk.file,chunk.line)
-					local param = {
-						type = vartype,
-						name = name,
-						description = parse(description),
-						optional = command == "tparam?"
-					}
-					table.insert(ret.params,param)
-					--table.insert(state.sections[section][partname].params,param)
-				elseif command == "treturn" then
-					local vartype, description = rest:match("([^ ]+) (.*)")
-					assertf(section and partname and sectiontype == "function","Tried to specify a treturn for something that wasn't a function at %s:%d",chunk.file,chunk.line)
-					local tret = {
-						type = vartype,
-						description = description and parse(description)
-					}
-					table.insert(ret.returns,tret)
-					--table.insert(state.sections[section][partname].returns,{
-						--type = vartype,
-						--description = description and parse(description)
-					--})
-				elseif command == "field" then
-					local namespace, field, value
-					local c,v = rest:match("^([^ ]+) ([^ ]+)$")
-					if c then
-						namespace = "_G"
-						field = c
-						value = v
-					else
-						namespace, field, value = rest:match("([^.]+).([^ ]+) ?(.*)")
-					end
-					assertf(namespace and field and value, "Improperly defined @field at %s:%d, should be '@field module.field Description here'",chunk.file,chunk.line)
-					section = namespace
-					partname = field
-					state.sections[namespace] = state.sections[namespace] or {type="namespace"}
-					assertf(not state.sections[namespace][field],"2 or more definitions for %s.%s",namespace,field)
-					ret = {
-					--state.sections[namespace][field] = {
-						type = "field",
-						name = field,
-						path = {namespace,field},
-						short_desc = parse(short_desc),
-						desc = desc and parse(table.concat(desc,"\n")),
-						line = chunk.line,
-						file = chunk.relpath
-					}
-				elseif command == "class" then
-					local classname = rest
-					section = classname
-					ret = {
-					--state.sections[classname] = state.sections[classname] or {
-						type = "class",
-						name = classname,
-						short_desc = parse(short_desc),
-						desc = parse(table.concat(desc,"\n")),
-						path = {classname}
-					}
-					partname = nil
-				elseif command == "inherits" then
-					assertf(ret and ret.type == "class","Don't know what is using @inherits at %s:%d, add a @class definition above it",chunk.file,chunk.line)
-					ret.inherits = ret.inherits or {}
-					table.insert(ret.inherits,rest)
-					--state.sections[section].inherits = state.sections[section].inherits or {}
-					--table.insert(state.sections[section].inherits,rest)
-				else
-					assertf(ret and ret.path[1] ,"Don't know where to put @%s, add a @function, @field or @table marker before it.",command)
-					if partname then
-						assertf(ret.path[2] ,"Don't know where to put @%s, add a @function, @field, or @table marker before it")
-						ret[command] = rest
-						--state.sections[section][partname][command] = rest
-					else
-						ret[command] = rest
-						--state.sections[section][command] = rest
-					end
-				end
-			end
-		end
-		return ret
-	end
-	local cachefilename = string.format("%s/cache/chunks/%s",options.output,file.relpath)
-	local cachefileattrs = lfs.attributes(cachefilename)
-	if (not cachefileattrs) or cachefileattrs.modification < file.lastmod or options.nocache then
-		for k,v in pairs(file.chunks) do
-			file.chunks[k] = process_chunk(v)
-		end
-		local cachefile = assert(io.open(cachefilename,"w"))
-		cachefile:write(table_to_string(file.chunks))
-		cachefile:close()
-	else
-		file.chunks = loadfile(cachefilename)()
-	end
-end
-
-for _,file in pairs(state.files) do
-	for _, chunk in pairs(file.chunks) do
-		assertf(chunk.path, "Chunk at %s:%d didn't have a path.", file.name, chunk.line)
-		local cursor = state.sections
-		for _, path_part in pairs(chunk.path) do
-			if not cursor[path_part] then
-				cursor[path_part] = {}
-			end
-			cursor = cursor[path_part]
-		end
-		for k,v in pairs(chunk) do
-			cursor[k] = v
-		end
-	end
-end
-
---Get the data for the reference documents
-if options.index ~= nil then
-	local file_mod = lfs.attributes(options.index).modification
-	local file_name = options.index:match("([^/.]+)%.[^.]+$")
-	local cachefile = string.format("%s/cache/documents/%s",options.output,file_name)
-	local out_attrs = lfs.attributes(cachefile)
-	if not out_attrs or out_attrs.modification < file_mod or options.nocache then
-		--we're out of date, update
-		local fd = assert(io.open(options.index,"r"))
-		local file_text = fd:read("*a")
-		fd:close()
-		local parsed_text = parse(file_text, options.index)
-		local od = assert(io.open(cachefile,"w"))
-		od:write(parsed_text)
-		od:close()
-	end
-	state.sections[file_name] = {
-		type = "reference",
-		data_file = cachefile,
-	}
-	state.index = state.sections[file_name]
-end
-
-log("After copying everything in, state.sections is:")
-log(state.sections)
-
-local headers = {}
-for name,section in pairs(state.sections) do
-	log("section:",name,"type:",section.type,"data:")
-	local ust = "namespace"
-	for _,v in pairs(section) do
-		if v.type == "method" then
-			ust = "class"
-		end
-	end
-	section.type = section.type or ust
-	headers[section.type] = headers[section.type] or {}
-	section.name = name
-	table.insert(headers[section.type],section)
-end
-for _,group in pairs(headers) do
-	table.sort(group,function(a,b)
-		--print("sorting:",a.type,"against",b.type)
-		return a.type < b.type
-	end)
-end
---print("headers:",headers)
-
---local nvfd = assert(io.open("../share/navbar.etlua","r"))
-local navbar = assert(et.compile(require("mdoc.files.navbar")))
-local navbarhtml = navbar{headers = headers}
---local pagefd = assert(io.open("../share/page.etlua","r"))
-local page = assert(et.compile(require("mdoc.files.page")))
---local funcsignaturefd = assert(io.open("../share/funcsignature.etlua","r"))
-local funcsignature = assert(et.compile(require("mdoc.files.funcsignature")))
-local sorted_headers = {}
-for name,_ in pairs(headers) do
-	table.insert(sorted_headers,name)
-end
-table.sort(sorted_headers)
-for _,name in pairs(sorted_headers) do
-	local header = headers[name]
-	for _,section in pairs(header) do
-		log("About to render section:",section.name)
-		log(section)
-		local pagehtml = assert(page{
-			header = section,
-			navbar = navbarhtml,
-			options = options,
-			et = et,
-			funcsig = funcsignature,
-		})
-		log("Done rendering pagehtml")
-		log("section name:",section.name)
-		log("section:",section)
-		local ofd = assert(io.open(options.output .. "/" .. section.name .. ".html","w"))
-		ofd:write(pagehtml)
-		ofd:close()
-	end
-end
-
-if options.index ~= nil then
-	--local indexfd = assert(io.open("../share/index.etlua","r"))
-	local index = assert(et.compile(require("mdoc.files.index")))
-	log("state.index:",state.index)
-	local indextextfd = assert(io.open(state.index.data_file,"r"))
-	local indextext = indextextfd:read("*a")
-	indextextfd:close()
-	local indexhtml = index{
-		navbar = navbarhtml,
-		options = options,
-		et = et,
-		text = indextext,
-	}
-	local ofd = assert(io.open(options.output .. "/index.html", "w"))
-	ofd:write(indexhtml)
-	ofd:close()
-end
-
---Copy style css
---local css = assert(io.open("style.css","r"))
-local css = assert(require("mdoc.files.style"))
-local css_out = assert(io.open(options.output .. "/style.css","w"))
-css_out:write(css)
-css_out:close()
-
---Generate html
---for header, sections in pairs(headers) do
-	
-	----local ofd = assert(io.open(options.output .. "/" .. name,"w"))
-	--print("want to output section:",name,section.type)
---end
-
---print(state)
+#!/usr/bin/env lua
+
+--[[
+Mdoc creats documentation from comments in source files.
+Mdoc works in several steps:
+	1. Create "chunks" from input source files
+	2. Create a dependnecy graph, so we only update the things that
+	   need updating
+	3. Output html documentation
+]]
+local lfs = require("lfs")
+local et = require("etlua")
+require("mdoc.ext")
+local opt = require("mdoc.opt")
+
+local args = {...}
+local options = opt.parse_options(args)
+if options.help then
+	print(opt.help())	
+	return
+end
+if not options.paths or #options.paths == 0 then
+	print(opt.help())
+	return
+end
+
+local state = {
+	title = "",
+	sections = {},
+	files = {},
+	documents = {},
+	known_names = {},
+	index = {},
+}
+
+local function log(...)
+	if options.verbose then
+		print(...)
+	end
+end
+local function logf(...)
+	if options.verbose then
+		printf(...)
+	end
+end
+
+local toskip = {["."] = true,[".."] = true}
+local function scan(path,callback)
+	assert(path,"Cannot scan without a path to scan (was nil)")
+	assert(callback,"Cannot scan without a callback to call (was nil)")
+	for item in lfs.dir(path) do
+		log("looking at:",item)
+		if not toskip[item] then
+			local fullpath = path .. "/" .. item
+			local attributes = lfs.attributes(fullpath)
+			if attributes.mode == "directory" or attributes.mode == "link" then
+				log("found directory:",directory)
+				scan(fullpath,callback)
+			elseif attributes.mode == "file" then
+				log("found item:",item)
+				callback(fullpath)
+			end
+		end
+	end
+end
+for _,path in pairs(options.paths) do
+	scan(path,function(p)
+		local attributes = lfs.attributes(p)
+		table.insert(state.files,{
+			path = p,
+			relpath = p:match(string.format("%s/(.*)",path)),
+			lastmod = attributes.modification,
+		})
+	end)
+end
+logf("Found %d files.",#state.files)
+
+for _,path in pairs(options.document_paths) do
+	scan(path,function(p)
+		local attributes = lfs.attributes(p)
+		table.insert(state.documents,{
+			path = p,
+			lastmod = attributes.modification,
+		})
+	end)
+end
+logf("Found %d documents.",#state.documents)
+
+
+local function parse(text,filename)
+	assert(options.parser,"Failed to find a parser option")
+	assert(text,"Failed to get text")
+	text = text:gsub("@{(.+)}",function(data)
+		return string.format("[%s](%s.html)",data,data)
+	end)
+	log("Using parser:",options.parser)
+	local tmp = options.output .. "/" .. os.tmpname()
+	log("using temp name:",tmp)
+	local pd = io.popen(options.parser .. " > " .. tmp,"w")
+	pd:write(text)
+	pd:close()
+	local id = assert(io.open(tmp,"r"))
+	local ret = id:read("*a")
+	id:close()
+	assert(os.remove(tmp))
+	return ret
+end
+
+local function table_to_string(tbl)
+	--Collect all of our tables first,
+	--so that we don't break when we have recursive tables.
+	local tables = {}
+	local table_order = {}
+	local function tables_helper(t)
+		tables[t] = #table_order + 1
+		table_order[#table_order + 1] = t
+		for k,v in pairs(t) do
+			if type(k) == "table" and not tables[k] then
+				tables_helper(k)
+			end
+			if type(v) == "table" and not tables[v] then
+				tables_helper(v)
+			end
+		end
+	end
+	tables_helper(tbl)
+	
+	--Get the string representation of an element
+	local errfun = function(e) error("Cannot format a " .. type(e)) end
+	local rep_map = {
+		--things we can format
+		number = function(e)
+			if e % 1 == 0 then return string.format("%d",e)
+			else return string.format("%f",e) end
+		end,
+		string = function(e) return string.format("%q",e) end,
+		boolean = function(e) return e and "true" or "false" end,
+		table = function(e)
+			assertf(tables[e] ~= nil,"Could not find dependency table %s",tostring(e))
+			return string.format("table_%d",tables[e])
+		end,
+		--things we can't format
+		["function"] = errfun,
+		coroutine = errfun,
+		file = errfun,
+		userdata = errfun,
+		--nil can never happen
+	}
+	local sb = {}
+	--Create all the variables first, so that recursive tables will work
+	for n,_ in pairs(table_order) do
+		sb[#sb + 1] = string.format("local table_%d = {}",n)
+	end
+	--Go backwards through tables, since that should be the
+	--"dependency" order
+	for i = #table_order, 1, -1 do -- -1 is needed in case #table_order == 0
+		local tstr = {}
+		local this_table = table_order[i]
+		for k,v in pairs(this_table) do
+			tstr[#tstr + 1] = string.format("table_%d[%s] = %s",i,rep_map[type(k)](k), rep_map[type(v)](v))
+		end
+		sb[#sb + 1] = table.concat(tstr,"\n")
+	end
+	sb[#sb + 1] = "return table_1"
+	return table.concat(sb,"\n\n");
+end
+
+--io.open automatically creates parent directories
+local oldopen = io.open
+function io.open(path,mode)
+	if mode == "w" or mode == "w+" then
+		local path_so_far = ""
+		for folder in path:gmatch("([^/]+)/") do
+			path_so_far = path_so_far .. folder .. "/"
+			if lfs.attributes(path_so_far) == nil then
+				lfs.mkdir(path_so_far)
+			end
+		end
+	end
+	return oldopen(path,mode)
+end
+
+--Create a cache directory
+if not lfs.attributes(options.output .. "/cache") then
+	assert(lfs.mkdir(options.output .. "/cache"))
+	assert(lfs.mkdir(options.output .. "/cache/documents"))
+	assert(lfs.mkdir(options.output .. "/cache/files"))
+	assert(lfs.mkdir(options.output .. "/cache/chunks"))
+end
+
+--Get the data for the reference documents
+for dn, data in pairs(state.documents) do
+	--local file_name = data.path
+	local file_name = data.path:match("([^./]+)%.[^.]+$")
+	local cachefile = string.format("%s/cache/documents/%s",options.output,file_name)
+	local out_attrs = lfs.attributes(cachefile)
+	if not out_attrs or out_attrs.modification < data.lastmod or options.nocache then
+		--we're out of date, update
+		local fd = assert(io.open(data.path,"r"))
+		local file_text = fd:read("*a")
+		fd:close()
+		local parsed_text = parse(file_text, data.path)
+		local od = assert(io.open(cachefile,"w"))
+		od:write(parsed_text)
+		od:close()
+		state.sections[file_name] = {
+			type = "reference",
+			data_file = cachefile
+		}
+	end
+end
+
+--Openers and closers
+local parse_between = {
+	["/***"] = "*/",
+	["--[[**"] = "]]"
+}
+for _,file in pairs(state.files) do
+	local cachefilename = string.format("%s/cache/files/%s",options.output,file.relpath)
+	local cacheattrs = lfs.attributes(cachefilename)
+	if (not cacheattrs) or cacheattrs.modification < file.lastmod or options.nocache then
+		local chunks = {}
+		local in_chunk = false
+		local line_num = 0
+		local closer
+		local fd = assert(io.open(file.path,"r"))
+		for line in fd:lines() do
+			line_num = line_num + 1
+			if parse_between[line] then
+				closer = parse_between[line]
+				table.insert(chunks,{})
+				in_chunk = true
+			end
+			if in_chunk then
+				table.insert(chunks[#chunks],line)
+			end
+			if in_chunk and line == closer then
+				log("found end, chunks:",chunks)
+				chunks[#chunks] = {
+					text = chunks[#chunks],
+					file = file.path,
+					line = line_num,
+					relpath = file.relpath,
+				}
+				in_chunk = false
+			end
+		end
+		file.chunks = chunks
+		local cachefile = assert(io.open(cachefilename,"w"))
+		cachefile:write(table_to_string(chunks))
+		cachefile:close()
+	else
+		file.chunks = loadfile(cachefilename)()
+	end
+end
+
+for _,file in pairs(state.files) do
+	local function process_chunk(chunk)
+		assert(chunk.file and chunk.line, "Chunk without file or line num:")
+		local section = nil
+		local partname = nil
+		local sectiontype = nil
+		local short_desc
+		local desc = {}
+		--Parameters are {
+		--	name = "string",
+		--	type = "string",
+		--	optional = bool
+		--	optchain = bool
+		--	default = nil | "string"
+		--}
+		local ret = {}
+		for num,line in pairs(chunk.text) do
+			if num == 2 then
+				if line:match("^@.*") then
+					short_desc = "no short description provided"
+				else
+					short_desc = line
+				end
+			end
+			if num > 2 and type(desc) == "table" and not line:match("^@.*") then
+				table.insert(desc,line)
+			end
+			if line:match("^@") then
+				local command, rest = line:match("^@([^ ]+) (.*)")
+				if command == "function" then
+					sectiontype = "function"
+					if rest:find(":") then
+						local classname, functionname = rest:match("^([^:]+):([^%(]+)")
+						ret = {
+						--state.sections[classname] = state.sections[classname] or {type = "class"}
+						--state.sections[classname][functionname] = state.sections[classname][functionname] or {
+							type="method",
+							name = functionname,
+							path = {classname,functionname},
+							short_desc = parse(short_desc),
+							line = chunk.line,
+							desc = parse(table.concat(desc,"\n")),
+							params = {},
+							returns = {},
+							file = chunk.relpath
+						}
+						section = classname
+						partname = functionname
+					else
+						local namespace, functionname
+						local c = rest:match("^([^.%(]+)%(")
+						if c then
+							namespace = "_G"
+							functionname = c
+						else
+							namespace, functionname = rest:match("([^.]+)%.([^%(]+)")
+						end
+						log("namespace:",namespace,"functionname:",functionname)
+						ret = {
+						--state.sections[namespace] = state.sections[namespace] or {type = "namespace"}
+						--state.sections[namespace][functionname] = state.sections[namespace][functionname] or {
+							type="function",
+							name = functionname,
+							short_desc = parse(short_desc),
+							path = {namespace,functionname},
+							line = chunk.line,
+							desc = parse(table.concat(desc,"\n")),
+							params = {},
+							returns = {},
+							references = {},
+							file = chunk.relpath
+						}
+						section = namespace
+						partname = functionname
+					end
+				elseif command == "tparam" or command == "tparam?" then
+					local parts = {}
+					local vartype, name, description
+					for word in rest:gmatch("([^ ]+)") do table.insert(parts,word) end
+					assertf(#parts >= 2,"@tparam at %s:%d requires at least a type and a variable name",chunk.file,chunk.line_num)
+					if parts[1] then
+						vartype = parts[1]
+						table.remove(parts,1)
+					end
+					if parts[1] then
+						name = parts[1]
+						table.remove(parts,1)
+					end
+					description = table.concat(parts,"\n")
+
+					assertf(section and partname and sectiontype == "function","Tried to specify a tparam for something that wasn't a function at %s:%d",chunk.file,chunk.line)
+					local param = {
+						type = vartype,
+						name = name,
+						description = parse(description),
+						optional = command == "tparam?"
+					}
+					table.insert(ret.params,param)
+					--table.insert(state.sections[section][partname].params,param)
+				elseif command == "treturn" then
+					local vartype, description = rest:match("([^ ]+) (.*)")
+					assertf(section and partname and sectiontype == "function","Tried to specify a treturn for something that wasn't a function at %s:%d",chunk.file,chunk.line)
+					local tret = {
+						type = vartype,
+						description = description and parse(description)
+					}
+					table.insert(ret.returns,tret)
+					--table.insert(state.sections[section][partname].returns,{
+						--type = vartype,
+						--description = description and parse(description)
+					--})
+				elseif command == "field" then
+					local namespace, field, value
+					local c,v = rest:match("^([^ ]+) ([^ ]+)$")
+					if c then
+						namespace = "_G"
+						field = c
+						value = v
+					else
+						namespace, field, value = rest:match("([^.]+).([^ ]+) ?(.*)")
+					end
+					assertf(namespace and field and value, "Improperly defined @field at %s:%d, should be '@field module.field Description here'",chunk.file,chunk.line)
+					section = namespace
+					partname = field
+					state.sections[namespace] = state.sections[namespace] or {type="namespace"}
+					assertf(not state.sections[namespace][field],"2 or more definitions for %s.%s",namespace,field)
+					ret = {
+					--state.sections[namespace][field] = {
+						type = "field",
+						name = field,
+						path = {namespace,field},
+						short_desc = parse(short_desc),
+						desc = desc and parse(table.concat(desc,"\n")),
+						line = chunk.line,
+						file = chunk.relpath
+					}
+				elseif command == "class" then
+					local classname = rest
+					section = classname
+					ret = {
+					--state.sections[classname] = state.sections[classname] or {
+						type = "class",
+						name = classname,
+						short_desc = parse(short_desc),
+						desc = parse(table.concat(desc,"\n")),
+						path = {classname}
+					}
+					partname = nil
+				elseif command == "inherits" then
+					assertf(ret and ret.type == "class","Don't know what is using @inherits at %s:%d, add a @class definition above it",chunk.file,chunk.line)
+					ret.inherits = ret.inherits or {}
+					table.insert(ret.inherits,rest)
+					--state.sections[section].inherits = state.sections[section].inherits or {}
+					--table.insert(state.sections[section].inherits,rest)
+				else
+					assertf(ret and ret.path[1] ,"Don't know where to put @%s, add a @function, @field or @table marker before it.",command)
+					if partname then
+						assertf(ret.path[2] ,"Don't know where to put @%s, add a @function, @field, or @table marker before it")
+						ret[command] = rest
+						--state.sections[section][partname][command] = rest
+					else
+						ret[command] = rest
+						--state.sections[section][command] = rest
+					end
+				end
+			end
+		end
+		return ret
+	end
+	local cachefilename = string.format("%s/cache/chunks/%s",options.output,file.relpath)
+	local cachefileattrs = lfs.attributes(cachefilename)
+	if (not cachefileattrs) or cachefileattrs.modification < file.lastmod or options.nocache then
+		for k,v in pairs(file.chunks) do
+			file.chunks[k] = process_chunk(v)
+		end
+		local cachefile = assert(io.open(cachefilename,"w"))
+		cachefile:write(table_to_string(file.chunks))
+		cachefile:close()
+	else
+		file.chunks = loadfile(cachefilename)()
+	end
+end
+
+for _,file in pairs(state.files) do
+	for _, chunk in pairs(file.chunks) do
+		assertf(chunk.path, "Chunk at %s:%d didn't have a path.", file.name, chunk.line)
+		local cursor = state.sections
+		for _, path_part in pairs(chunk.path) do
+			if not cursor[path_part] then
+				cursor[path_part] = {}
+			end
+			cursor = cursor[path_part]
+		end
+		for k,v in pairs(chunk) do
+			cursor[k] = v
+		end
+	end
+end
+
+--Get the data for the reference documents
+if options.index ~= nil then
+	local file_mod = lfs.attributes(options.index).modification
+	local file_name = options.index:match("([^/.]+)%.[^.]+$")
+	local cachefile = string.format("%s/cache/documents/%s",options.output,file_name)
+	local out_attrs = lfs.attributes(cachefile)
+	if not out_attrs or out_attrs.modification < file_mod or options.nocache then
+		--we're out of date, update
+		local fd = assert(io.open(options.index,"r"))
+		local file_text = fd:read("*a")
+		fd:close()
+		local parsed_text = parse(file_text, options.index)
+		local od = assert(io.open(cachefile,"w"))
+		od:write(parsed_text)
+		od:close()
+	end
+	state.sections[file_name] = {
+		type = "reference",
+		data_file = cachefile,
+	}
+	state.index = state.sections[file_name]
+end
+
+log("After copying everything in, state.sections is:")
+log(state.sections)
+
+local headers = {}
+for name,section in pairs(state.sections) do
+	log("section:",name,"type:",section.type,"data:")
+	local ust = "namespace"
+	for _,v in pairs(section) do
+		if v.type == "method" then
+			ust = "class"
+		end
+	end
+	section.type = section.type or ust
+	headers[section.type] = headers[section.type] or {}
+	section.name = name
+	table.insert(headers[section.type],section)
+end
+for _,group in pairs(headers) do
+	table.sort(group,function(a,b)
+		--print("sorting:",a.type,"against",b.type)
+		return a.type < b.type
+	end)
+end
+--print("headers:",headers)
+
+--local nvfd = assert(io.open("../share/navbar.etlua","r"))
+local navbar = assert(et.compile(require("mdoc.files.navbar")))
+local navbarhtml = navbar{headers = headers}
+--local pagefd = assert(io.open("../share/page.etlua","r"))
+local page = assert(et.compile(require("mdoc.files.page")))
+--local funcsignaturefd = assert(io.open("../share/funcsignature.etlua","r"))
+local funcsignature = assert(et.compile(require("mdoc.files.funcsignature")))
+local sorted_headers = {}
+for name,_ in pairs(headers) do
+	table.insert(sorted_headers,name)
+end
+table.sort(sorted_headers)
+for _,name in pairs(sorted_headers) do
+	local header = headers[name]
+	for _,section in pairs(header) do
+		log("About to render section:",section.name)
+		log(section)
+		local pagehtml = assert(page{
+			header = section,
+			navbar = navbarhtml,
+			options = options,
+			et = et,
+			funcsig = funcsignature,
+		})
+		log("Done rendering pagehtml")
+		log("section name:",section.name)
+		log("section:",section)
+		local ofd = assert(io.open(options.output .. "/" .. section.name .. ".html","w"))
+		ofd:write(pagehtml)
+		ofd:close()
+	end
+end
+
+if options.index ~= nil then
+	--local indexfd = assert(io.open("../share/index.etlua","r"))
+	local index = assert(et.compile(require("mdoc.files.index")))
+	log("state.index:",state.index)
+	local indextextfd = assert(io.open(state.index.data_file,"r"))
+	local indextext = indextextfd:read("*a")
+	indextextfd:close()
+	local indexhtml = index{
+		navbar = navbarhtml,
+		options = options,
+		et = et,
+		text = indextext,
+	}
+	local ofd = assert(io.open(options.output .. "/index.html", "w"))
+	ofd:write(indexhtml)
+	ofd:close()
+end
+
+--Copy style css
+--local css = assert(io.open("style.css","r"))
+local css = assert(require("mdoc.files.style"))
+local css_out = assert(io.open(options.output .. "/style.css","w"))
+css_out:write(css)
+css_out:close()
+
+--Generate html
+--for header, sections in pairs(headers) do
+	
+	----local ofd = assert(io.open(options.output .. "/" .. name,"w"))
+	--print("want to output section:",name,section.type)
+--end
+
+--print(state)
-- 
cgit v1.2.3-70-g09d2