path: root/init.lua

--[[
Mdoc creats documentation from comments in source files.
Mdoc works in several steps:
	1. Create "chunks" from input source files
	2. Create a dependnecy graph, so we only update the things that
	   need updating
	3. Output html documentation
]]
local lpeg = require("lpeg")
local lfs = require("lfs")
local et = require("etlua")
require("ext")
local opt = require("opt_parser")

local args = {...}
local options = opt.parse_options(args)
if options.help then
	print(opt.help())	
	return
end
print("options:",options)

local state = {
	title = "",
	sections = {},
	files = {},
	documents = {},
	known_names = {},
	index = {},
}

local function log(...)
	if options.verbose then
		print(...)
	end
end
local function logf(...)
	if options.verbose then
		printf(...)
	end
end

local toskip = {["."] = true,[".."] = true}
local function scan(path,callback)
	assert(path,"Cannot scan without a path to scan (was nil)")
	assert(callback,"Cannot scan without a callback to call (was nil)")
	for item in lfs.dir(path) do
		log("looking at:",item)
		if toskip[item] then
			goto nextitem
		end
		local fullpath = path .. "/" .. item
		local attributes = lfs.attributes(fullpath)
		if attributes.mode == "directory" or attributes.mode == "link" then
			log("found directory:",directory)
			scan(fullpath,callback)
		elseif attributes.mode == "file" then
			log("found item:",item)
			callback(fullpath)
		end
		::nextitem::
	end
end
for _,path in pairs(options.paths) do
	scan(path,function(p)
		local attributes = lfs.attributes(p)
		table.insert(state.files,{
			path = p,
			relpath = p:match(string.format("%s/(.*)",path)),
			lastmod = attributes.modification,
		})
	end)
end
logf("Found %d files.",#state.files)

for _,path in pairs(options.document_paths) do
	scan(path,function(p)
		local attributes = lfs.attributes(p)
		table.insert(state.documents,{
			path = p,
			lastmod = attributes.modification,
		})
	end)
end
logf("Found %d documents.",#state.documents)


local function parse(text,filename)
	assert(options.parser,"Failed to find a parser option")
	assert(text,"Failed to get text")
	text = text:gsub("@{(.+)}",function(data)
		return string.format("[%s](%s.html)",data,data)
	end)
	log("Using parser:",options.parser)
	local tmp = options.output .. "/" .. os.tmpname()
	log("using temp name:",tmp)
	local pd = io.popen(options.parser .. " > " .. tmp,"w")
	pd:write(text)
	pd:close()
	local id = assert(io.open(tmp,"r"))
	local ret = id:read("*a")
	id:close()
	assert(os.remove(tmp))
	return ret
end

local function table_to_string(tbl)
	--Collect all of our tables first,
	--so that we don't break when we have recursive tables.
	local tables = {}
	local table_order = {}
	local function tables_helper(t)
		tables[t] = #table_order + 1
		table_order[#table_order + 1] = t
		for k,v in pairs(t) do
			if type(k) == "table" and not tables[k] then
				tables_helper(k)
			end
			if type(v) == "table" and not tables[v] then
				tables_helper(v)
			end
		end
	end
	tables_helper(tbl)
	
	--Get the string representation of an element
	local errfun = function(e) error("Cannot format a " .. type(e)) end
	local rep_map = {
		--things we can format
		number = function(e)
			if e % 1 == 0 then return string.format("%d",e)
			else return string.format("%f",e) end
		end,
		string = function(e) return string.format("%q",e) end,
		boolean = function(e) return e and "true" or "false" end,
		table = function(e)
			assertf(tables[e] ~= nil,"Could not find dependency table %s",tostring(e))
			return string.format("table_%d",tables[e])
		end,
		--things we can't format
		["function"] = errfun,
		coroutine = errfun,
		file = errfun,
		userdata = errfun,
		--nil can never happen
	}
	local sb = {}
	--Create all the variables first, so that recursive tables will work
	for n,_ in pairs(table_order) do
		sb[#sb + 1] = string.format("local table_%d = {}",n)
	end
	--Go backwards through tables, since that should be the
	--"dependency" order
	for i = #table_order, 1, -1 do -- -1 is needed in case #table_order == 0
		local tstr = {}
		local this_table = table_order[i]
		for k,v in pairs(this_table) do
			tstr[#tstr + 1] = string.format("table_%d[%s] = %s",i,rep_map[type(k)](k), rep_map[type(v)](v))
		end
		sb[#sb + 1] = table.concat(tstr,"\n")
	end
	sb[#sb + 1] = "return table_1"
	return table.concat(sb,"\n\n");
end

--io.open automatically creates parent directories
local oldopen = io.open
function io.open(path,mode)
	if mode == "w" or mode == "w+" then
		local path_so_far = ""
		for folder in path:gmatch("([^/]+)/") do
			path_so_far = path_so_far .. folder .. "/"
			if lfs.attributes(path_so_far) == nil then
				lfs.mkdir(path_so_far)
			end
		end
	end
	return oldopen(path,mode)
end

--Create a cache directory
if not lfs.attributes(options.output .. "/cache") then
	assert(lfs.mkdir(options.output .. "/cache"))
	assert(lfs.mkdir(options.output .. "/cache/documents"))
	assert(lfs.mkdir(options.output .. "/cache/files"))
	assert(lfs.mkdir(options.output .. "/cache/chunks"))
end

--Get the data for the reference documents
for dn, data in pairs(state.documents) do
	local file_name = data.path:match(".*/([^.]+)%.[^.]+$")
	local cachefile = string.format("%s/cache/documents/%s",options.output,file_name)
	local out_attrs = lfs.attributes(cachefile)
	if not out_attrs or out_attrs.modification < data.lastmod then
		--we're out of date, update
		local fd = assert(io.open(data.path,"r"))
		local file_text = fd:read("*a")
		fd:close()
		local parsed_text = parse(file_text, data.path)
		local od = assert(io.open(cachefile,"w"))
		od:write(parsed_text)
		od:close()
		state.sections[file_name] = {
			type = "reference",
			data_file = cachefile
		}
	end
end

--Openers and closers
local parse_between = {
	["/***"] = "*/",
	["--[[**"] = "]]"
}
for _,file in pairs(state.files) do
	local cachefilename = string.format("%s/cache/files/%s",options.output,file.relpath)
	local cacheattrs = lfs.attributes(cachefilename)
	if (not cacheattrs) or cacheattrs.modification < file.lastmod then
		local chunks = {}
		local in_chunk = false
		local line_num = 0
		local closer
		local fd = assert(io.open(file.path,"r"))
		for line in fd:lines() do
			line_num = line_num + 1
			if parse_between[line] then
				closer = parse_between[line]
				table.insert(chunks,{})
				in_chunk = true
			end
			if in_chunk then
				table.insert(chunks[#chunks],line)
			end
			if in_chunk and line == closer then
				log("found end, chunks:",chunks)
				chunks[#chunks] = {
					text = chunks[#chunks],
					file = file.path,
					line = line_num,
					relpath = file.relpath,
				}
				in_chunk = false
			end
		end
		file.chunks = chunks
		local cachefile = assert(io.open(cachefilename,"w"))
		cachefile:write(table_to_string(chunks))
		cachefile:close()
	else
		file.chunks = loadfile(cachefilename)()
	end
end

for _,file in pairs(state.files) do
	local function process_chunk(chunk)
		assert(chunk.file and chunk.line, "Chunk without file or line num:")
		local section = nil
		local partname = nil
		local sectiontype = nil
		local short_desc
		local desc = {}
		--Parameters are {
		--	name = "string",
		--	type = "string",
		--	optional = bool
		--	optchain = bool
		--	default = nil | "string"
		--}
		local ret = {}
		for num,line in pairs(chunk.text) do
			if num == 2 then
				if line:match("^@.*") then
					short_desc = "no short description provided"
				else
					short_desc = line
				end
			end
			if num > 2 and type(desc) == "table" and not line:match("^@.*") then
				table.insert(desc,line)
			end
			if line:match("^@") then
				local command, rest = line:match("^@([^ ]+) (.*)")
				if command == "function" then
					sectiontype = "function"
					if rest:find(":") then
						local classname, functionname = rest:match("^([^:]+):([^%(]+)")
						ret = {
						--state.sections[classname] = state.sections[classname] or {type = "class"}
						--state.sections[classname][functionname] = state.sections[classname][functionname] or {
							type="method",
							name = functionname,
							path = {classname,functionname},
							short_desc = parse(short_desc),
							line = chunk.line,
							desc = parse(table.concat(desc,"\n")),
							params = {},
							returns = {},
							file = chunk.relpath
						}
						section = classname
						partname = functionname
					else
						local namespace, functionname
						local c = rest:match("^([^.%(]+)%(")
						if c then
							namespace = "_G"
							functionname = c
						else
							namespace, functionname = rest:match("([^.]+)%.([^%(]+)")
						end
						log("namespace:",namespace,"functionname:",functionname)
						ret = {
						--state.sections[namespace] = state.sections[namespace] or {type = "namespace"}
						--state.sections[namespace][functionname] = state.sections[namespace][functionname] or {
							type="function",
							name = functionname,
							short_desc = parse(short_desc),
							path = {namespace,functionname},
							line = chunk.line,
							desc = parse(table.concat(desc,"\n")),
							params = {},
							returns = {},
							references = {},
							file = chunk.relpath
						}
						section = namespace
						partname = functionname
					end
				elseif command == "tparam" or command == "tparam?" then
					local parts = {}
					local vartype, name, description
					for word in rest:gmatch("([^ ]+)") do table.insert(parts,word) end
					assertf(#parts >= 2,"@tparam at %s:%d requires at least a type and a variable name",chunk.file,chunk.line_num)
					if parts[1] then
						vartype = parts[1]
						table.remove(parts,1)
					end
					if parts[1] then
						name = parts[1]
						table.remove(parts,1)
					end
					description = table.concat(parts,"\n")

					assertf(section and partname and sectiontype == "function","Tried to specify a tparam for something that wasn't a function at %s:%d",chunk.file,chunk.line)
					local param = {
						type = vartype,
						name = name,
						description = parse(description),
						optional = command == "tparam?"
					}
					table.insert(ret.params,param)
					--table.insert(state.sections[section][partname].params,param)
				elseif command == "treturn" then
					local vartype, description = rest:match("([^ ]+) (.*)")
					assertf(section and partname and sectiontype == "function","Tried to specify a treturn for something that wasn't a function at %s:%d",chunk.file,chunk.line)
					local tret = {
						type = vartype,
						description = description and parse(description)
					}
					table.insert(ret.returns,tret)
					--table.insert(state.sections[section][partname].returns,{
						--type = vartype,
						--description = description and parse(description)
					--})
				elseif command == "field" then
					local namespace, field, value
					local c,v = rest:match("^([^ ]+) ([^ ]+)$")
					if c then
						namespace = "_G"
						field = c
						value = v
					else
						namespace, field, value = rest:match("([^.]+).([^ ]+) ?(.*)")
					end
					assertf(namespace and field and value, "Improperly defined @field at %s:%d, should be '@field module.field Description here'",chunk.file,chunk.line)
					section = namespace
					partname = field
					state.sections[namespace] = state.sections[namespace] or {type="namespace"}
					assertf(not state.sections[namespace][field],"2 or more definitions for %s.%s",namespace,field)
					ret = {
					--state.sections[namespace][field] = {
						type = "field",
						name = field,
						path = {namespace,field},
						short_desc = parse(short_desc),
						desc = desc and parse(table.concat(desc,"\n")),
						line = chunk.line,
						file = chunk.relpath
					}
				elseif command == "class" then
					local classname = rest
					section = classname
					ret = {
					--state.sections[classname] = state.sections[classname] or {
						type = "class",
						name = classname,
						short_desc = parse(short_desc),
						desc = parse(table.concat(desc,"\n")),
						path = {classname}
					}
					partname = nil
				elseif command == "inherits" then
					assertf(ret and ret.type == "class","Don't know what is using @inherits at %s:%d, add a @class definition above it",chunk.file,chunk.line)
					ret.inherits = ret.inherits or {}
					table.insert(ret.inherits,rest)
					--state.sections[section].inherits = state.sections[section].inherits or {}
					--table.insert(state.sections[section].inherits,rest)
				else
					assertf(ret and ret.path[1] ,"Don't know where to put @%s, add a @function, @field or @table marker before it.",command)
					if partname then
						assertf(ret.path[2] ,"Don't know where to put @%s, add a @function, @field, or @table marker before it")
						ret[command] = rest
						--state.sections[section][partname][command] = rest
					else
						ret[command] = rest
						--state.sections[section][command] = rest
					end
				end
			end
		end
		return ret
	end
	local cachefilename = string.format("%s/cache/chunks/%s",options.output,file.relpath)
	local cachefileattrs = lfs.attributes(cachefilename)
	if (not cachefileattrs) or cachefileattrs.modification < file.lastmod then
		for k,v in pairs(file.chunks) do
			file.chunks[k] = process_chunk(v)
		end
		local cachefile = assert(io.open(cachefilename,"w"))
		cachefile:write(table_to_string(file.chunks))
		cachefile:close()
	else
		file.chunks = loadfile(cachefilename)()
	end
end

for _,file in pairs(state.files) do
	for _, chunk in pairs(file.chunks) do
		assertf(chunk.path, "Chunk at %s:%d didn't have a path.", file.name, chunk.line)
		local cursor = state.sections
		for _, path_part in pairs(chunk.path) do
			if not cursor[path_part] then
				cursor[path_part] = {}
			end
			cursor = cursor[path_part]
		end
		for k,v in pairs(chunk) do
			cursor[k] = v
		end
	end
end

--Get the data for the reference documents
if options.index ~= nil then
	local file_mod = lfs.attributes(options.index).modification
	local file_name = options.index:match(".*/([^.]+)%.[^.]+$")
	local cachefile = string.format("%s/cache/documents/%s",options.output,file_name)
	local out_attrs = lfs.attributes(cachefile)
	if not out_attrs or out_attrs.modification < file_mod then
		--we're out of date, update
		local fd = assert(io.open(options.index,"r"))
		local file_text = fd:read("*a")
		fd:close()
		local parsed_text = parse(file_text, options.index)
		local od = assert(io.open(cachefile,"w"))
		od:write(parsed_text)
		od:close()
	end
	state.sections[file_name] = {
		type = "reference",
		data_file = cachefile
	}
	state.index = state.sections[file_name]
end

log("After copying everything in, state.sections is:")
log(state.sections)

local headers = {}
for name,section in pairs(state.sections) do
	log("section:",name,"type:",section.type,"data:")
	local ust = "namespace"
	for _,v in pairs(section) do
		if v.type == "method" then
			ust = "class"
		end
	end
	section.type = section.type or ust
	headers[section.type] = headers[section.type] or {}
	section.name = name
	table.insert(headers[section.type],section)
end
for _,group in pairs(headers) do
	table.sort(group,function(a,b)
		--print("sorting:",a.type,"against",b.type)
		return a.type < b.type
	end)
end
--print("headers:",headers)

local nvfd = assert(io.open("navbar.etlua","r"))
local navbar = assert(et.compile(nvfd:read("*a")))
nvfd:close()
local navbarhtml = navbar{headers = headers}
local pagefd = assert(io.open("page.etlua","r"))
local page = assert(et.compile(pagefd:read("*a")))
pagefd:close()
local funcsignaturefd = assert(io.open("funcsignature.etlua","r"))
local funcsignature = assert(et.compile(funcsignaturefd:read("*a")))
funcsignaturefd:close()
local sorted_headers = {}
for name,_ in pairs(headers) do
	table.insert(sorted_headers,name)
end
table.sort(sorted_headers)
for _,name in pairs(sorted_headers) do
	local header = headers[name]
	for _,section in pairs(header) do
		log("About to render section:",section.name)
		log(section)
		local pagehtml = assert(page{
			header = section,
			navbar = navbarhtml,
			options = options,
			et = et,
			funcsig = funcsignature,
		})
		log("Done rendering pagehtml")
		log("section name:",section.name)
		log("section:",section)
		local ofd = assert(io.open(options.output .. "/" .. section.name .. ".html","w"))
		ofd:write(pagehtml)
		ofd:close()
	end
end

if options.index ~= nil then
	local indexfd = assert(io.open("index.etlua","r"))
	local index = assert(et.compile(indexfd:read("*a")))
	indexfd:close()
	log("state.index:",state.index)
	local indextextfd = assert(io.open(state.index.data_file,"r"))
	local indextext = indextextfd:read("*a")
	indextextfd:close()
	local indexhtml = index{
		navbar = navbarhtml,
		options = options,
		et = et,
		text = indextext,
	}
	local ofd = assert(io.open(options.output .. "/index.html", "w"))
	ofd:write(indexhtml)
	ofd:close()
end

--Copy style css
local css = assert(io.open("style.css","r"))
local css_out = assert(io.open(options.output .. "/style.css","w"))
for line in css:lines() do
	css_out:write(line)
end
css:close()
css_out:close()

--Generate html
--for header, sections in pairs(headers) do
	
	----local ofd = assert(io.open(options.output .. "/" .. name,"w"))
	--print("want to output section:",name,section.type)
--end

--print(state)