path: root/trbldoc.sh

#!/bin/sh

# trbldoc.sh
# 	A really awful documentation tool
# Installation
# 	Simply drop into /usr/local/bin/trbldoc and mark executable
# 	$ sudo cp trbldoc.sh /usr/local/bin/trbldoc
# 	$ sudo chmod +x /usr/local/bin/trbldoc
# Usage:
#	trbldoc [options]
# Description:
# 	trbldoc works in several passes, first, it looks at each file in the
# 	<source> folders, and for each file it looks for comments
# 	and extracts them as a series of "chunks" - 
# 	unbroken lines of comments. Each chunk should declare
# 	a program. The text of the chunk will be fed into this program
# 	on standard in, and trbldoc expects html for this chunk on standard out.
#
# 	Then, it groups chunks according to their "@name <doc_name>" line,
# 	and orders them according to their "@priority n" line.
# 	If a comment doesn't have a name, <doc_name> is the file path the
# 	comment was found in. If a comment doesn't have a priority, it's priority
# 	is the line number the chunk started on.
#
# 	Chunks may additionaly have the following "documentation identifiers":
# 	@name <name/with/slashes> - Override the file path this chunk was found
# 		in
# 	@exec <program> <arg1> <arg2> ... - Override the program we use to
# 		process this chunk
# 	@file <name> - Pretend this chunk came from another file
# 		e.g. if the file being documented is built from another
# 		(source) file.
# 	@ref <name> - This chunk should be referenced by this name (instead of
# 		of it's @name).
# 	@priority <number> - Higher values render first when assembling pages.
# 		helps keep important information at the top.
#
# 	The following template variables may be used anywhere in chunk body text
# 	and are substituted before the chunk's program is run:
# 	{{src_repo}} - source repository URL (see TRBLDOC_SRC_REPO). Useful
# 		for building links back to source files.
# 	{{ref_string}} - HTML link for this chunk's @ref directive, resolved
# 		via the matching ref_resolvers script. Empty if @ref is absent
# 		or the resolver produces no output.
# 	{{toc}} - full site navigation rendered as HTML list items.
#
# 	Then, it resolves refrences, replacing text of "@<name>" with a link
# 	to the first chunk declaring "@ref <name>" (by priority/source order).
# 	If no internal ref matches, it falls back to external resolvers:
# 	"@<script>/<path>" runs ref_resolvers/<script>.sh with <path> as input.
#
# Options:
# 	-R '<prefix>=<script>;<options>' - When a document tag (@<prefix>/ref/path)
# 		contains the prefix <prefix>, strip it, and pass the rest as
# 		stdin to <script>, run with <options>. The following environment
# 		variables are available to <script>:
# 		$TRBLCACHE - a cache folder for this project.
# 		$FILEPATH - The file the reference came from, extension included
# 		$LINENUM - The line number the reference was found on
# 	-s <source> - Add <source> to files or folders scanned
# 	-o <dest> - Set the destination path that documentation is output to.
# 	-l <layout> - Use an alternative main.layout template file instead of
# 		the built-in default.
# 	-f 'file_ext;start_regex;end_regex' - Add regexes for file extensions to
# 		extract comments. Comments begin with "start_regex", followed
# 		by a newline-terminated command to run the body with.
# 	-C - Force a clean rebuild: clears all cached chunks and last-processed
# 		timestamps before scanning. Equivalent to manually deleting
# 		.trblcache/chunks/ and .trblcache/last_processed/.
# Environment variables:
# 	TRBLDOC_MD - Override the markdown renderer used for .md files and
# 		"@exec md" chunks. Defaults to:
# 		"md2html --github --fpermissive-autolinks --ftables"
# 	TRBLDOC_SRC_REPO - Repository URL substituted for {{src_repo}} in
# 		doc comments. Falls back to "git remote get-url origin".
# Parse command-line options
# -s <source>         : source folder to scan (repeatable)
# -o <dest>           : final output directory (default: .trblcache/built)
# -l <layout>         : alternative main.layout template path
# -f ext;start;end    : register a custom file extension with AWK-regex delimiters
# -R prefix=script;opts : reference-resolver override
NL='
'
US=$(printf '\037')
ref_resolver_overrides=""
extra_formats=""
infolders=""
out_dir=""
layout_template=""
clean_build=0
while getopts ":s:o:l:R:f:C" opt; do
	case "$opt" in
		s)	infolders="${infolders}${OPTARG}${NL}" ;;
		o)	out_dir="$OPTARG" ;;
		l)	layout_template="$OPTARG" ;;
		R)	ref_resolver_overrides="${ref_resolver_overrides}${OPTARG}${NL}" ;;
		f)	extra_formats="${extra_formats}${OPTARG}${NL}" ;;
		C)	clean_build=1 ;;
		:)	echo "trbldoc: option -$OPTARG requires an argument" >&2; exit 1 ;;
		?)	echo "trbldoc: unknown option: -$OPTARG" >&2; exit 1 ;;
	esac
done
shift $((OPTIND - 1))
if [ -z "$infolders" ]; then
	echo "trbldoc: no source folders specified; use -s <folder>" >&2
	exit 1
fi
resolver_overrides_dir=$(mktemp -d)
# Parse -R overrides: prefix=script;options
printf '%s' "$ref_resolver_overrides" | while IFS= read -r override_spec; do
	[ -z "$override_spec" ] && continue
	override_prefix="${override_spec%%=*}"
	override_rhs="${override_spec#*=}"
	override_script="${override_rhs%%;*}"
	override_opts=""
	if [ "$override_rhs" != "$override_script" ]; then
		override_opts="${override_rhs#*;}"
	fi
	if [ -n "$override_prefix" ] && [ -n "$override_script" ]; then
		printf '%s' "$override_script" > "$resolver_overrides_dir/${override_prefix}.script"
		printf '%s' "$override_opts" > "$resolver_overrides_dir/${override_prefix}.opts"
	fi
done
TRBLDOC_RESOLVER_OVERRIDES_RAW="$ref_resolver_overrides"
export TRBLDOC_RESOLVER_OVERRIDES_RAW
configure_renderer_aliases() { :; }

SCRIPT_DIR="${SCRIPT_DIR:-$(cd "$(dirname "$0")" && pwd)}"

sed_pattern_literal='s/[][\\.^$*+?(){}|]/\\&/g'
sed_replacement_literal='s/[\\&|]/\\&/g'
validate_namespace_path() {
	local namespace="$1"
	if ! pathchk -P "$namespace" >/dev/null 2>&1; then
		echo "trbldoc: Invalid @name, got $namespace"
		return 1
	fi

	local part _old_ifs
	set -f
	_old_ifs="$IFS"
	IFS='/'
	set -- $namespace
	IFS="$_old_ifs"
	set +f
	for part do
		if [ -z "$part" ]; then
			echo "trbldoc: @name must not contain empty path segments, got '$namespace'" >&2
			return 1
		fi
		if [ "$part" = "." ] || [ "$part" = ".." ]; then
			echo "trbldoc: @name must not contain '.' or '..' path segments, got '$namespace'" >&2
			return 1
		fi
	done
}
namespace_for_chunk() {
	local source_path="$1"
	local explicit_namespace="$2"
	local source_basename="${source_path##*/}"
	local namespace="$explicit_namespace"

	if [ -z "$namespace" ]; then
		case $(printf '%s' "$source_basename" | tr '[:upper:]' '[:lower:]') in
			readme.md) return 0 ;;
		esac
	fi
	if [ -z "$namespace" ]; then
		namespace="$source_path"
	fi
	validate_namespace_path "$namespace" || return 1
	printf '%s' "$namespace"
}

parse_chunk_metadata() {
	local chunk_file="$1"
	awk '
BEGIN {
	us = sprintf("%c", 31)
	from_p = ""
	namespace_p = ""
	priority_p = ""
	program_p = ""
	ref_p = ""
}
{sub(/\r$/, "", $0)}
/^[[:blank:]]*@file / && from_p == "" {
	from_p = $0
	sub(/^[[:blank:]]*@file /, "", from_p)
}
/^[[:blank:]]*@name / && namespace_p == "" { namespace_p = $2 }
/^[[:blank:]]*@priority / && priority_p == "" { priority_p = $2 }
/^[[:blank:]]*@exec / && program_p == "" {
	program_p = $0
	sub(/^[[:blank:]]*@exec /, "", program_p)
}
/^[[:blank:]]*@ref / && ref_p == "" { ref_p = $2 }
END {
	printf "%s%s%s%s%s%s%s%s%s\n", from_p, us, namespace_p, us, priority_p, us, program_p, us, ref_p
}
' "$chunk_file"
}

get_parallel_jobs() {
	nproc 2>/dev/null || \
	getconf _NPROCESSORS_ONLN 2>/dev/null || \
	grep -c '^processor' /proc/cpuinfo 2>/dev/null || \
	echo 4
}

# Apply many literal replacements by streaming file content through
# chained sed processes and writing back once at the end.
# replacements_file format: <search><US><replacement> one pair per line.
replace_literal_pairs_in_file() {
	local target_file="$1"
	local replacements_file="$2"
	local token_sep
	local tmp_file
	local sed_chain

	[ ! -s "$replacements_file" ] && return 0
	token_sep=$(printf '\037')
	tmp_file=$(mktemp) || return 1

	# Single-quote a string for safe eval embedding.
	_quote_sq() {
		printf "'%s'" "$(printf '%s' "$1" | sed "s/'/'\\\\''/g")"
	}

	local cmd="cat $(_quote_sq "$target_file")"
	sed_chain=$(awk -v us="$token_sep" '
function escape_pattern(v) { gsub(/[][\\.^$*+?(){}|]/, "\\\\&", v); return v }
function escape_replacement(v) { gsub(/[\\&|]/, "\\\\&", v); return v }
function escape_dquote(v) { gsub(/["\\`$]/, "\\\\&", v); return v }
{
	sep = index($0, us)
	if (sep == 0) next
	search = substr($0, 1, sep - 1)
	repl = substr($0, sep + 1)
	search = escape_pattern(search)
	repl = escape_replacement(repl)
	expr = "s|" search "|" repl "|g"
	expr = escape_dquote(expr)
	printf " | sed -r \"%s\"", expr
}
' "$replacements_file")

	cmd="$cmd$sed_chain > $(_quote_sq "$tmp_file")"
	eval "$cmd" || {
		rm -f "$tmp_file"
		return 1
	}
	mv "$tmp_file" "$target_file"
}

# Extract first "From ..." citation source (without HTML tags) for resolver context.
extract_source_from_cite() {
	local html_file="$1"
	awk '
match($0, /<cite>From .*<\/cite>/) {
	source = substr($0, RSTART + 11, RLENGTH - 18)
	gsub(/<[^>]*>/, "", source)
	print source
	exit
}
' "$html_file"
}

# Emit unique reference tokens from HTML:
# I<US>@token for internal refs, E<US>@prefix/path for external refs.
collect_html_ref_tokens() {
	local html_file="$1"
	awk '
{
	internal_line = $0
	while (match(internal_line, /@[a-zA-Z0-9_.\/-]+/)) {
		token = substr(internal_line, RSTART, RLENGTH)
		if (index(token, "|") == 0 && !seen_internal[token]++) {
			printf "I%c%s\n", 31, token
		}
		internal_line = substr(internal_line, RSTART + RLENGTH)
	}

	external_line = $0
	while (match(external_line, /@[a-zA-Z0-9_-]+\/[^[:space:]<>"]+/)) {
		token = substr(external_line, RSTART, RLENGTH)
		if (!seen_external[token]++) {
			printf "E%c%s\n", 31, token
		}
		external_line = substr(external_line, RSTART + RLENGTH)
	}
}
' "$html_file"
}

# render_layout_page $layout_file $nav_file $body_file $page_title $page_breadcrub $output_file
render_layout_page() {
	# Read nav content (may contain newlines, so flatten for sed)
	local nav_html
	local escaped_title
	local escaped_breadcrumb
	local escaped_nav
	nav_html=$(tr '\n' ' ' < "$2")
	escaped_title=$(escape_sed_replacement_literal "$4")
	escaped_breadcrumb=$(escape_sed_replacement_literal "$5")
	escaped_nav=$(escape_sed_replacement_literal "$nav_html")

	# Render layout to output in a single sed pass.
	# {{ yield }} is expected to be on its own line.
	sed -E \
		-e "s|\\{\\{ title \\}\\}|$escaped_title|g" \
		-e "s|\\{\\{ breadcrumb \\}\\}|$escaped_breadcrumb|g" \
		-e "s|\\{\\{ nav \\}\\}|$escaped_nav|g" \
		-e "/\\{\\{ yield \\}\\}/r $3" \
		-e "/\\{\\{ yield \\}\\}/d" \
		"$1" > "$6"
}

assemble_site() {
	local namespace_root="$1"
	local destination_root="$2"
	local layout_file="$namespace_root/main.layout"
	local nav_file="$namespace_root/nav.partial"
	local rendered_page
	local rel_path
	local namespace_path
	local page_title="index"
	local page_breadcrumb="index"
	local target_dir
	local output_file

	rm -rf "$destination_root"
	mkdir -p "$destination_root"

	if [ -d "$namespace_root/stylesheets" ]; then
		mkdir -p "$destination_root/stylesheets"
		cp -R "$namespace_root/stylesheets/." "$destination_root/stylesheets/"
	fi

	_asm_find_tmp=$(mktemp)
	find "$namespace_root" -name "index.html" > "$_asm_find_tmp"
	while IFS= read -r rendered_page; do
		rel_path="${rendered_page#$namespace_root/}"
		namespace_path="${rel_path%/index.html}"
		if [ "$rel_path" = "index.html" ]; then
			namespace_path=""
		fi
		if [ -n "$namespace_path" ]; then
			target_dir="$destination_root/$namespace_path"
			page_title="$namespace_path"
			page_breadcrumb="$namespace_path"
		else
			target_dir="$destination_root"
		fi
		mkdir -p "$target_dir"
		output_file="$target_dir/index.html"
		render_layout_page "$layout_file" "$nav_file" "$rendered_page" "$page_title" "$page_breadcrumb" "$output_file"
	done < "$_asm_find_tmp"
	rm -f "$_asm_find_tmp"
}

# Cache directory
cache_dir=./.trblcache
export TRBLCACHE="$cache_dir"
# Resolve output directory now that cache_dir is known
out_dir="${out_dir:-$cache_dir/built}"
mkdir -p "$cache_dir"
common_functions_sh="$cache_dir/_common_functions.sh"
cat > "$common_functions_sh" << 'COMMON_SH'
escape_sed_pattern_literal() {
	local value="$1"
	printf '%s' "$value" | sed -e 's/[][\\.^$*+?(){}|]/\\&/g'
}

escape_sed_replacement_literal() {
	local value="$1"
	printf '%s' "$value" | sed -e 's/[\\&|]/\\&/g'
}

md() { ${TRBLDOC_MD:-md2html --github --fpermissive-autolinks --ftables} "$@"; }
rst() { ${TRBLDOC_RST:-docutils --template ./.trblcache/rsttemplate.txt} "$@"; }

# resolve_ref PREFIX PATH FILEPATH LINENUM
# Resolves a reference by looking up the resolver script,
# exports TRBLCACHE/FILEPATH/LINENUM env vars, and runs the script with PATH as stdin.
# Prints the resolved URL to stdout, or nothing if resolution fails.
resolve_ref() {
	local prefix="$1"
	local resolver_script="$SCRIPT_DIR/ref_resolvers/${prefix}.sh"
	local resolver_opts=""

	if [ -f "$resolver_overrides_dir/${prefix}.script" ]; then
		resolver_script=$(cat "$resolver_overrides_dir/${prefix}.script")
		resolver_opts=$(cat "$resolver_overrides_dir/${prefix}.opts" 2>/dev/null || true)
	elif [ -n "${TRBLDOC_RESOLVER_OVERRIDES_RAW:-}" ]; then
		printf '%s' "$TRBLDOC_RESOLVER_OVERRIDES_RAW" | while IFS= read -r override_spec; do
			[ -z "$override_spec" ] && continue
			override_prefix="${override_spec%%=*}"
			[ "$override_prefix" != "$prefix" ] && continue
			override_rhs="${override_spec#*=}"
			resolver_script="${override_rhs%%;*}"
			resolver_opts=""
			if [ "$override_rhs" != "$resolver_script" ]; then
				resolver_opts="${override_rhs#*;}"
			fi
			break
		done
	fi

	if [ ! -f "$resolver_script" ]; then
		return 1
	fi

	export TRBLCACHE="$cache_dir"
	export FILEPATH="$3"
	export LINENUM="$4"

	if [ -n "$resolver_opts" ]; then
		printf '%s\n' "$2" | eval "\"$resolver_script\" $resolver_opts" 2>/dev/null || true
	else
		printf '%s\n' "$2" | "$resolver_script" 2>/dev/null || true
	fi
}
COMMON_SH
. "$common_functions_sh"
export common_functions_sh

parallel_jobs="${TRBLDOC_JOBS:-$(get_parallel_jobs)}"

# `$cache/last_processed` stores empty files to represent the last updated date (so we don't reprocess files if they haven't been updated)
# `$cache/namespace` is the output of documentation execution, to be aggregated and copied into build/
# `$cache/ref` references, things that should be resolved after all the pages are generated
# `$cache/chunks` # Chunks extracted from files, kept seperate so we can just delete the chunks for one file if it's changed.
# `$cache/comment_forms` stores awk programs that extract chunks from files based on their extension
# Weirdly, this is one of the slower parts of trbldoc.
xargs -n 1 -P $parallel_jobs mkdir -p <<EOF
$cache_dir/last_processed
$cache_dir/namespace
$cache_dir/ref
$cache_dir/chunks
$cache_dir/comment_forms
$out_dir
EOF


# Various opening and closing tags for different languages (based on file extension)
# Each entry: ext,end_regex,start_regex  (AWK extended regex patterns, no slashes)
comment_forms="
c,\*\/,\/\*
lua,\]\],\-\-\[\[
"

# Helper: write comment-extraction awk script from start/end regex pair.
# Uses single-quote splicing to inject shell variables into awk source.
_write_comment_form() {
	printf '%s\n' '
BEGIN{}
/'"$2"'/ {
	if(in_comment) {print "EOF"}
	in_comment=0
}
in_comment {print}
/'"$3"' .+/ && !/'"$2"'$/ {
	print "chunkfile=$(mktemp -p '"${cache_dir}"'/chunks)"
	print "cat > $chunkfile << \"EOF\"\n@file " name ":" NR ;
	if($2) {$1=""; print "@exec " $0}
	in_comment=1;
}' > "$1"
}

comment_form_dir="$cache_dir/comment_forms"

# Built-in comment forms from the table
printf '%s' "$comment_forms" | while IFS= read -r _cf_line; do
	case "$_cf_line" in *[!" 	"]*) ;; *) continue ;; esac
	_cf_lang="${_cf_line%%,*}"
	_cf_rest="${_cf_line#*,}"
	_cf_end="${_cf_rest%%,*}"
	_cf_start="${_cf_rest#*,}"
	_write_comment_form "$comment_form_dir/${_cf_lang}.awk" "$_cf_end" "$_cf_start"
done

# Aliases for same-syntax languages
for _alias_ext in h cpp hpp sql dot ts js; do
	cp "$comment_form_dir/c.awk" "$comment_form_dir/${_alias_ext}.awk" 2>/dev/null || true
done
for _alias_ext in moon tl; do
	cp "$comment_form_dir/lua.awk" "$comment_form_dir/${_alias_ext}.awk" 2>/dev/null || true
done

# Python: ''' is both the open and close delimiter. Renderer name is
# glued directly after the opening ''' with no space required (e.g. '''md).
# Custom AWK uses sub() for renderer extraction instead of field splitting.
sed "s|__PY_CACHE__|${cache_dir}|g" << 'PYAWK' > "$comment_form_dir/py.awk"
BEGIN{}
/'''/ {
	if(in_comment) {print "EOF"}
	in_comment=0
}
in_comment {print}
/'''.+/ && !/'''$/ {
	print "chunkfile=$(mktemp -p __PY_CACHE__/chunks)"
	print "cat > $chunkfile << \"EOF\"\n@file " name ":" NR ;
	renderer = $0
	sub(/'''/, "", renderer)
	sub(/^[[:space:]]+/, "", renderer)
	sub(/[[:space:]]+$/, "", renderer)
	if (renderer != "") {print "@exec " renderer}
	in_comment=1;
}
PYAWK

# .md files only have 1 chunk
printf '%s\n' '
BEGIN{
	print "chunkfile=$(mktemp -p '"${cache_dir}"'/chunks)"
	print "cat > $chunkfile << \"EOF\"\n@file " name ":" NR "\n@exec md" ;
}
{print $0}
END{print "EOF"}' > "$comment_form_dir/md.awk"
echo "%(body)s" > "$cache_dir/rsttemplate.txt"

# Apply -f extra file format definitions from command line.
# Format: ext;start_regex;end_regex (AWK regex patterns without slashes).
printf '%s' "$extra_formats" | while IFS= read -r fmt_spec; do
	[ -z "$fmt_spec" ] && continue
	_ef_ext="${fmt_spec%%;*}"
	_ef_rest="${fmt_spec#*;}"
	_ef_start="${_ef_rest%%;*}"
	_ef_end="${_ef_rest#*;}"
	if [ -n "$_ef_ext" ] && [ -n "$_ef_start" ] && [ -n "$_ef_end" ]; then
		_write_comment_form "$comment_form_dir/${_ef_ext}.awk" "$_ef_end" "$_ef_start"
	fi
done


echo "" > "$cache_dir/namespace/global.meta"
if [ -n "$layout_template" ]; then
	if [ ! -f "$layout_template" ]; then
		echo "trbldoc: layout template not found: $layout_template" >&2
		exit 1
	fi
	cp "$layout_template" "$cache_dir/namespace/main.layout"
else
cat > $cache_dir/namespace/main.layout << EOF
<!DOCTYPE html>
<html lang="en">
	<head>
		<meta charset="utf-8">
		<title>{{ title }}</title>
		<link href="/stylesheets/style.css" rel="stylesheet">
	</head>
	<body>
		<header>
			<h1><a href="#">{{ breadcrumb }}</a></h1>
		</header>
		<nav><ul>{{ nav }}</ul></nav>
		<article>
			{{ yield }}
		</article>
		<footer>
			<p>Generated by trbldoc</p>
		</footer>
	</body>
</html>
EOF
fi
mkdir -p "$cache_dir/namespace/stylesheets"
cat > "$cache_dir/namespace/stylesheets/style.css" << EOF
body{
	margin:40px auto;
	max-width:80ch;
	line-height:1.6;
	font-size:18px;
	color:#444;
	padding:0 10px
}
h1,h2,h3{line-height:1.2}
caption,td{border-bottom:1px solid black;}
nav{float:left;position:absolute;left:10%;}
table{width:100%;}
EOF

# Find all the source files
# Build extension filter from awk files in comment_form_dir
ext_filter=""
for _awk_file in "$comment_form_dir"/*.awk; do
	[ -f "$_awk_file" ] || continue
	_ext=$(basename "$_awk_file" .awk)
	ext_filter="${ext_filter}${ext_filter:+|}${_ext}"
done
# Populate source files list (one per line)
source_files_list="$cache_dir/source_files.list"
printf '%s' "$infolders" \
| xargs --no-run-if-empty -I '{}' find "{}" -type f \
| grep -E "\.(${ext_filter})$" \
> "$source_files_list"

# Analyze source files for chunks
# -C wipes the chunk cache and all timestamps for a full re-scan.
# Otherwise only source files newer than their last-processed marker are
# re-extracted; chunks for unchanged files are reused as-is.
if [ "$clean_build" -eq 1 ]; then
	rm -rf "$cache_dir/chunks" "$cache_dir/last_processed"
fi
mkdir -p "$cache_dir/chunks"
mkdir -p "$cache_dir/last_processed"

# Rendered output is always rebuilt from the full set of chunks.
# This has to be the case because the `@exec` may change.
# `@exec` is just a program, we can't garentee it'll do the same stuff next run.
# See `docs/trblcache.md for more info
find "$cache_dir/namespace" -name "index.html" -delete 2>/dev/null || true
> "$cache_dir/namespace/nav.partial"
export cache_dir comment_form_dir
changed_sources_list="$cache_dir/changed_sources.list"
tracked_sources_list="$cache_dir/tracked_sources.list"
removed_sources_list="$cache_dir/removed_sources.list"
purge_sources_list="$cache_dir/purge_sources.list"
> "$changed_sources_list"

# Build changed source list (only files newer than their marker, or missing marker).
if [ -s "$source_files_list" ]; then
	while IFS= read -r source_file; do
		last="$cache_dir/last_processed/$source_file"
		if [ -f "$last" ] && [ ! "$source_file" -nt "$last" ]; then
			continue
		fi
		printf '%s\n' "$source_file" >> "$changed_sources_list"
	done < "$source_files_list"
fi

# Compute removed sources from stale markers, then remove those markers.
find "$cache_dir/last_processed" -type f 2>/dev/null \
	| sed "s|^$cache_dir/last_processed/||" > "$tracked_sources_list"
grep -Fvx -f "$source_files_list" "$tracked_sources_list" 2>/dev/null > "$removed_sources_list" || true
if [ -s "$removed_sources_list" ]; then
	awk -v base="$cache_dir/last_processed/" '{ print base $0 }' "$removed_sources_list" \
		| xargs rm -f 2>/dev/null || true
fi

# Purge stale chunks once (changed + removed), instead of per-worker scans.
{
	cat "$changed_sources_list" 2>/dev/null
	cat "$removed_sources_list" 2>/dev/null
} | awk 'NF && !seen[$0]++' > "$purge_sources_list"
if [ -s "$purge_sources_list" ]; then
	_purge_patterns=$(mktemp)
	awk '{ printf "@file %s:\n", $0 }' "$purge_sources_list" > "$_purge_patterns"
	grep -rlFf "$_purge_patterns" "$cache_dir/chunks/" 2>/dev/null \
		| xargs rm -f 2>/dev/null || true
	rm -f "$_purge_patterns"
fi
# Create helper script for parallel extraction
_extract_helper="$cache_dir/_extract_helper.sh"
cat > "$_extract_helper" << 'EXTRACT_SH'
#!/bin/sh
source_file="$1"
file_name=$(basename -- "$source_file")
file_ext=${file_name##*.}
awk_prog_file="$comment_form_dir/$file_ext.awk"
if [ ! -f "$awk_prog_file" ]; then
	exit 0
fi
chunk_script=$(awk -v name="$source_file" -f "$awk_prog_file" "$source_file")
eval "$chunk_script"
last="$cache_dir/last_processed/$source_file"
mkdir -p "${last%/*}" && touch "$last"
EXTRACT_SH
chmod +x "$_extract_helper"
if [ -s "$changed_sources_list" ]; then
	tr '\n' '\0' < "$changed_sources_list" \
		| xargs -0 -n 1 -P "$parallel_jobs" "$_extract_helper"
fi

# Resolve src_repo once before the chunk loop.
# Prefers the TRBLDOC_SRC_REPO environment variable; falls back to the git
# remote origin URL; defaults to empty string if neither is available.
src_repo_val="${TRBLDOC_SRC_REPO:-$(git remote get-url origin 2>/dev/null || echo '')}"

_chunk_files_list=$(mktemp)
find "$cache_dir/chunks" -type f > "$_chunk_files_list"
chunk_order_file="$cache_dir/chunk.render_order"
> "$chunk_order_file"
chunk_order_delim="$US"
while IFS= read -r chunk_file; do
	IFS="$US" read -r chunk_from_p chunk_namespace_p chunk_priority_p chunk_program_p chunk_ref_p <<_USEOF_
$(parse_chunk_metadata "$chunk_file")
_USEOF_
	chunk_source_path="${chunk_from_p%:*}"
	chunk_source_line="${chunk_from_p##*:}"
	if ! printf '%s' "$chunk_source_line" | grep -qE '^-?[0-9]+$'; then
		chunk_source_line=0
	fi
	chunk_namespace="$(namespace_for_chunk "$chunk_source_path" "$chunk_namespace_p")" || exit 1
	chunk_priority=0
	if printf '%s' "$chunk_priority_p" | grep -qE '^-?[0-9]+$'; then
		chunk_priority="$chunk_priority_p"
	fi
	printf "%s${chunk_order_delim}%s${chunk_order_delim}%s${chunk_order_delim}%s${chunk_order_delim}%s${chunk_order_delim}%s${chunk_order_delim}%s${chunk_order_delim}%s\n" \
		"$chunk_priority" \
		"$chunk_namespace" \
		"$chunk_source_path" \
		"$chunk_source_line" \
		"$chunk_file" \
		"$chunk_from_p" \
		"$chunk_program_p" \
		"$chunk_ref_p" >> "$chunk_order_file"
done < "$_chunk_files_list"
rm -f "$_chunk_files_list"
# Deterministic render order:
# 1 @priority (higher numbers first)
# 2 namespace
# 3 source path
# 4 source line
# 5 chunk temp file path (stable tie-breaker)
sorted_chunk_order_file="$cache_dir/chunk.render_order.sorted"
LC_ALL=C sort -t "$US" -k1,1nr -k2,2 -k3,3 -k4,4n -k5,5 "$chunk_order_file" > "$sorted_chunk_order_file"
chunk_render_records_file="$cache_dir/chunk.render_records"
> "$chunk_render_records_file"
chunk_seq=0
while IFS="$US" read -r _chunk_priority namespace _chunk_source_path _chunk_source_line chunk_file from_p program_p ref_p; do
	chunk_seq=$((chunk_seq + 1))
	printf "%s${chunk_order_delim}%s${chunk_order_delim}%s${chunk_order_delim}%s${chunk_order_delim}%s${chunk_order_delim}%s${chunk_order_delim}%s${chunk_order_delim}%s${chunk_order_delim}%s\n" \
		"$chunk_seq" \
		"$_chunk_priority" \
		"$namespace" \
		"$_chunk_source_path" \
		"$_chunk_source_line" \
		"$chunk_file" \
		"$from_p" \
		"$program_p" \
		"$ref_p" >> "$chunk_render_records_file"
done < "$sorted_chunk_order_file"
render_artifacts_dir="$cache_dir/rendered_chunks"
rm -rf "$render_artifacts_dir"
mkdir -p "$render_artifacts_dir"
export SCRIPT_DIR src_repo_val render_artifacts_dir US
export TRBLDOC_RESOLVER_OVERRIDES_RAW resolver_overrides_dir
# Pre-compute the escaped sed patterns/replacements for template variables
# once in the parent so child renderers avoid repeated escape subshells.
# {{ref_string}} pattern is constant; {{src_repo}} pattern+replacement are constant.
_esc_ref_pat=$(escape_sed_pattern_literal '{{ ref_string }}')
_esc_src_pat=$(escape_sed_pattern_literal '{{ src_repo }}')
_esc_src_repl=$(escape_sed_replacement_literal "$src_repo_val")
export _esc_ref_pat _esc_src_pat _esc_src_repl
_render_helper="$cache_dir/_render_helper.sh"
cat > "$_render_helper" << 'RENDER_SH'
#!/bin/sh
render_record="$1"
IFS="$US" read -r chunk_seq _chunk_priority namespace _chunk_source_path _chunk_source_line chunk_file from_p program_p ref_p <<_USEOF_
$render_record
_USEOF_
program="${program_p:-cat}"
. "$common_functions_sh"
# Compute the {{ref_string}} replacement.  Only chunks with @ref need the
# full resolve_ref call.
_esc_ref_repl=""
if [ -n "$ref_p" ]; then
	_rs_prefix="${ref_p%%/*}"
	_rs_path="${ref_p#*/}"
	_rs_url=$(resolve_ref "$_rs_prefix" "$_rs_path" "$_chunk_source_path" "$_chunk_source_line")
	if [ -n "$_rs_url" ]; then
		_esc_ref_repl=$(escape_sed_replacement_literal "<a href='$_rs_url'>$ref_p</a>")
	fi
fi
export TRBLCACHE="$cache_dir" FILEPATH="$_chunk_source_path" LINENUM="$_chunk_source_line"
artifact_prefix=$(printf '%s/%08d' "$render_artifacts_dir" "$chunk_seq")
# One sed pass: strip directives + substitute both template variables, piped
# directly into the renderer.  The escaped patterns and the {{src_repo}}
# replacement are pre-computed in the parent; only {{ref_string}}'s
# replacement (when @ref is present) is computed per-chunk.
render_rc=0
sed -E \
	-e "s/^@.*//g" \
	-e "s|${_esc_ref_pat}|${_esc_ref_repl}|g" \
	-e "s|${_esc_src_pat}|${_esc_src_repl}|g" \
	"$chunk_file" | eval "$program" > "${artifact_prefix}.body" || render_rc=$?
if [ "$render_rc" -ne 0 ]; then
	echo "trbldoc: renderer failed (exit $render_rc): $program" >&2
	echo "trbldoc: chunk source: $from_p" >&2
fi
RENDER_SH
chmod +x "$_render_helper"
if [ -s "$chunk_render_records_file" ]; then
	while IFS= read -r render_record; do
		printf '%s\0' "$render_record"
	done < "$chunk_render_records_file" \
		| xargs -0 -n 1 -P "$parallel_jobs" "$_render_helper"
fi
while IFS="$US" read -r chunk_seq _chunk_priority namespace _chunk_source_path _chunk_source_line _chunk_file from_p _program_p ref_p; do
	ref="${ref_p:-$namespace}"
	artifact_prefix=$(printf '%s/%08d' "$render_artifacts_dir" "$chunk_seq")
	output=""
	if [ -f "${artifact_prefix}.body" ]; then
		output=$(cat "${artifact_prefix}.body")
	fi
	if [ -n "$output" ]; then
		from_html="<cite>From $from_p</cite>"
		if [ -n "$src_repo_val" ]; then
			src_file_path="$_chunk_source_path"
			src_file_line=""
			if printf '%s' "$_chunk_source_line" | grep -qE '^[1-9][0-9]*$'; then
				src_file_line="$_chunk_source_line"
			fi
			src_file_url="${src_repo_val%/}/$src_file_path"
			if [ -n "$src_file_line" ]; then
				src_file_url="${src_file_url}#L${src_file_line}"
			fi
			from_html="<cite>From <a href='$src_file_url'>$from_p</a></cite>"
		fi
		if [ -n "$namespace" ]; then
			outfolder="$cache_dir/namespace/$namespace"
		else
			outfolder="$cache_dir/namespace"
		fi
		if [ ! -d "$outfolder" ]; then
			mkdir -p "$outfolder"
		fi
		echo "$output" "$from_html" >> "$outfolder/index.html"
	fi
	# internal links for generated cross references
	if [ "$ref" = "$namespace" ]; then
		if [ -n "$ref" ]; then
			echo "$ref: <a href='/$ref'>$ref</a>" >> "$cache_dir/namespace/global.meta"
		else
			echo "index: <a href='/'>index</a>" >> "$cache_dir/namespace/global.meta"
		fi
	else
		echo "$ref: <a href='/$namespace#$ref'>$ref</a>" >> "$cache_dir/namespace/global.meta"
	fi
done < "$chunk_render_records_file"

> "$cache_dir/namespace/nav.partial"
find "$cache_dir/namespace" -name "index.html" \
| sed \
	-e 's;/index\.html$;;' \
	-e "s;^$cache_dir/namespace/*;;" \
| sort -u \
| awk '
function emit(path, nxt,   n, c, i) {
	if (path == "") { print "<li><a href=\047/\047>index</a></li>"; return }
	n = split(path, p, "/")
	c = 0
	while (c < depth && c < n - 1 && q[c+1] == p[c+1]) c++
	for (; depth > c; depth--) print "</ul></li>"
	for (i = c + 1; i < n; i++) print "<li>" p[i] "/<ul>"
	if (index(nxt, path "/") == 1) { print "<li><a href=\047/" path "\047>" p[n] "</a>/<ul>"; depth = n }
	else { print "<li><a href=\047/" path "\047>" p[n] "</a></li>"; depth = n - 1 }
	split(path, q, "/")
}
NR > 1 { emit(prev, $0) }
{ prev = $0 }
END {
	if (NR) emit(prev, "")
	for (; depth > 0; depth--) print "</ul></li>"
}
' > "$cache_dir/namespace/nav.partial"

# Second pass: resolve inline references now that global.meta is complete.
# Deduplicate global.meta (keep first occurrence per ref name, which has highest priority)
_deduped_meta="$cache_dir/namespace/global.meta.deduped"
awk -F': ' '!seen[$1]++' "$cache_dir/namespace/global.meta" > "$_deduped_meta"

# Lookup ref link HTML from the deduplicated file
_ref_link_lookup() {
	awk -v name="$1" '{
		p = index($0, ": ")
		if (p > 0 && substr($0, 1, p - 1) == name) {
			print substr($0, p + 2)
			exit
		}
	}' "$_deduped_meta"
}

token_sep="$US"
_ref_find_tmp=$(mktemp)
find "$cache_dir/namespace" -name "index.html" > "$_ref_find_tmp"
while IFS= read -r htmlfile; do
	replacement_pairs_file=$(mktemp)
	# Extract source file info for external resolver env vars
	_html_from=$(extract_source_from_cite "$htmlfile")
	_html_filepath="${_html_from%:*}"
	_html_linenum="${_html_from##*:}"
	if ! printf '%s' "$_html_linenum" | grep -qE '^[0-9]+$'; then
		_html_linenum=0
	fi
	collect_html_ref_tokens "$htmlfile" | while IFS="$token_sep" read -r ref_token_type ref_token; do
		[ -z "$ref_token" ] && continue
		if [ "$ref_token_type" = "I" ]; then
			ref_name="${ref_token#@}"  # strip leading @
			_link_html=$(_ref_link_lookup "$ref_name")
			if [ -n "$_link_html" ]; then
				# Found in global.meta - use the pre-built HTML link
				printf '%s%s%s\n' "$ref_token" "$token_sep" "$_link_html" >> "$replacement_pairs_file"
			fi
			continue
		fi
		if [ "$ref_token_type" != "E" ]; then
			continue
		fi

		# Resolve @script/path tokens (external refs) - only if not matched above
		ref_name="${ref_token#@}"
		_link_html=$(_ref_link_lookup "$ref_name")
		if [ -n "$_link_html" ]; then
			continue
		fi
		script_name="${ref_name%%/*}"      # get script prefix before /
		ref_path="${ref_name#*/}"          # get path after @prefix/
		resolved_url=$(resolve_ref "$script_name" "$ref_path" "$_html_filepath" "$_html_linenum")
		if [ -n "$resolved_url" ]; then
			replacement_link="<a href='${resolved_url}'>$script_name&#128279;${ref_path}</a>"
			printf '%s%s%s\n' "$ref_token" "$token_sep" "$replacement_link" >> "$replacement_pairs_file"
		else
			echo "trbldoc: unresolved ref: $ref_token (at $_html_filepath:$_html_linenum)" >&2
		fi
	done

	replace_literal_pairs_in_file "$htmlfile" "$replacement_pairs_file"
	rm -f "$replacement_pairs_file"

done < "$_ref_find_tmp"
rm -f "$_ref_find_tmp"

# Post-processing pass: substitute {{ toc }} in all rendered index.html files
# now that `nav.partial` is finalised.  Lines are joined into one so the sed
# substitution works without needing multi-line mode.
toc_oneline=$( \
    cat "$cache_dir/namespace/nav.partial" \
    | tr '\n' ' ' \
    | sed -e "$sed_replacement_literal" \
)
toc_repl=$( \
    printf "%s" "{{ toc }}" \
    | sed -e "$sed_pattern_literal" \
)
find "$cache_dir/namespace" -name "index.html" \
| xargs -n 1 -P "$parallel_jobs" sed -E -i -e "s|$toc_repl|$toc_oneline|g"
assemble_site "$cache_dir/namespace" "$out_dir"