diff options
Diffstat (limited to 'trbldoc.sh')
| -rw-r--r-- | trbldoc.sh | 940 |
1 files changed, 940 insertions, 0 deletions
diff --git a/trbldoc.sh b/trbldoc.sh new file mode 100644 index 0000000..99e89be --- /dev/null +++ b/trbldoc.sh @@ -0,0 +1,940 @@ +#!/bin/sh + +# trbldoc.sh +# A really awful documentation tool +# Installation +# Simply drop into /usr/local/bin/trbldoc and mark executable +# $ sudo cp trbldoc.sh /usr/local/bin/trbldoc +# $ sudo chmod +x /usr/local/bin/trbldoc +# Usage: +# trbldoc [options] +# Description: +# trbldoc works in several passes, first, it looks at each file in the +# <source> folders, and for each file it looks for comments +# and extracts them as a series of "chunks" - +# unbroken lines of comments. Each chunk should declare +# a program. The text of the chunk will be fed into this program +# on standard in, and trbldoc expects html for this chunk on standard out. +# +# Then, it groups chunks according to their "@name <doc_name>" line, +# and orders them according to their "@priority n" line. +# If a comment doesn't have a name, <doc_name> is the file path the +# comment was found in. If a comment doesn't have a priority, it's priority +# is the line number the chunk started on. +# +# Chunks may additionaly have the following "documentation identifiers": +# @name <name/with/slashes> - Override the file path this chunk was found +# in +# @exec <program> <arg1> <arg2> ... - Override the program we use to +# process this chunk +# @file <name> - Pretend this chunk came from another file +# e.g. if the file being documented is built from another +# (source) file. +# @ref <name> - This chunk should be referenced by this name (instead of +# of it's @name). +# @priority <number> - Higher values render first when assembling pages. +# helps keep important information at the top. +# +# The following template variables may be used anywhere in chunk body text +# and are substituted before the chunk's program is run: +# {{src_repo}} - source repository URL (see TRBLDOC_SRC_REPO). Useful +# for building links back to source files. +# {{ref_string}} - HTML link for this chunk's @ref directive, resolved +# via the matching ref_resolvers script. Empty if @ref is absent +# or the resolver produces no output. +# {{toc}} - full site navigation rendered as HTML list items. +# +# Then, it resolves refrences, replacing text of "@<name>" with a link +# to the first chunk declaring "@ref <name>" (by priority/source order). +# If no internal ref matches, it falls back to external resolvers: +# "@<script>/<path>" runs ref_resolvers/<script>.sh with <path> as input. +# +# Options: +# -R '<prefix>=<script>;<options>' - When a document tag (@<prefix>/ref/path) +# contains the prefix <prefix>, strip it, and pass the rest as +# stdin to <script>, run with <options>. The following environment +# variables are available to <script>: +# $TRBLCACHE - a cache folder for this project. +# $FILEPATH - The file the reference came from, extension included +# $LINENUM - The line number the reference was found on +# -s <source> - Add <source> to files or folders scanned +# -o <dest> - Set the destination path that documentation is output to. +# -l <layout> - Use an alternative main.layout template file instead of +# the built-in default. +# -f 'file_ext;start_regex;end_regex' - Add regexes for file extensions to +# extract comments. Comments begin with "start_regex", followed +# by a newline-terminated command to run the body with. +# -C - Force a clean rebuild: clears all cached chunks and last-processed +# timestamps before scanning. Equivalent to manually deleting +# .trblcache/chunks/ and .trblcache/last_processed/. +# Environment variables: +# TRBLDOC_MD - Override the markdown renderer used for .md files and +# "@exec md" chunks. Defaults to: +# "md2html --github --fpermissive-autolinks --ftables" +# TRBLDOC_SRC_REPO - Repository URL substituted for {{src_repo}} in +# doc comments. Falls back to "git remote get-url origin". +# Parse command-line options +# -s <source> : source folder to scan (repeatable) +# -o <dest> : final output directory (default: .trblcache/built) +# -l <layout> : alternative main.layout template path +# -f ext;start;end : register a custom file extension with AWK-regex delimiters +# -R prefix=script;opts : reference-resolver override +NL=' +' +US=$(printf '\037') +ref_resolver_overrides="" +extra_formats="" +infolders="" +out_dir="" +layout_template="" +clean_build=0 +while getopts ":s:o:l:R:f:C" opt; do + case "$opt" in + s) infolders="${infolders}${OPTARG}${NL}" ;; + o) out_dir="$OPTARG" ;; + l) layout_template="$OPTARG" ;; + R) ref_resolver_overrides="${ref_resolver_overrides}${OPTARG}${NL}" ;; + f) extra_formats="${extra_formats}${OPTARG}${NL}" ;; + C) clean_build=1 ;; + :) echo "trbldoc: option -$OPTARG requires an argument" >&2; exit 1 ;; + ?) echo "trbldoc: unknown option: -$OPTARG" >&2; exit 1 ;; + esac +done +shift $((OPTIND - 1)) +if [ -z "$infolders" ]; then + echo "trbldoc: no source folders specified; use -s <folder>" >&2 + exit 1 +fi +resolver_overrides_dir=$(mktemp -d) +# Parse -R overrides: prefix=script;options +printf '%s' "$ref_resolver_overrides" | while IFS= read -r override_spec; do + [ -z "$override_spec" ] && continue + override_prefix="${override_spec%%=*}" + override_rhs="${override_spec#*=}" + override_script="${override_rhs%%;*}" + override_opts="" + if [ "$override_rhs" != "$override_script" ]; then + override_opts="${override_rhs#*;}" + fi + if [ -n "$override_prefix" ] && [ -n "$override_script" ]; then + printf '%s' "$override_script" > "$resolver_overrides_dir/${override_prefix}.script" + printf '%s' "$override_opts" > "$resolver_overrides_dir/${override_prefix}.opts" + fi +done +TRBLDOC_RESOLVER_OVERRIDES_RAW="$ref_resolver_overrides" +export TRBLDOC_RESOLVER_OVERRIDES_RAW +configure_renderer_aliases() { :; } + +SCRIPT_DIR="${SCRIPT_DIR:-$(cd "$(dirname "$0")" && pwd)}" + +sed_pattern_literal='s/[][\\.^$*+?(){}|]/\\&/g' +sed_replacement_literal='s/[\\&|]/\\&/g' +validate_namespace_path() { + local namespace="$1" + if ! pathchk -P "$namespace" >/dev/null 2>&1; then + echo "trbldoc: Invalid @name, got $namespace" + return 1 + fi + + local part _old_ifs + set -f + _old_ifs="$IFS" + IFS='/' + set -- $namespace + IFS="$_old_ifs" + set +f + for part do + if [ -z "$part" ]; then + echo "trbldoc: @name must not contain empty path segments, got '$namespace'" >&2 + return 1 + fi + if [ "$part" = "." ] || [ "$part" = ".." ]; then + echo "trbldoc: @name must not contain '.' or '..' path segments, got '$namespace'" >&2 + return 1 + fi + done +} +namespace_for_chunk() { + local source_path="$1" + local explicit_namespace="$2" + local source_basename="${source_path##*/}" + local namespace="$explicit_namespace" + + if [ -z "$namespace" ]; then + case $(printf '%s' "$source_basename" | tr '[:upper:]' '[:lower:]') in + readme.md) return 0 ;; + esac + fi + if [ -z "$namespace" ]; then + namespace="$source_path" + fi + validate_namespace_path "$namespace" || return 1 + printf '%s' "$namespace" +} + +parse_chunk_metadata() { + local chunk_file="$1" + awk ' +BEGIN { + us = sprintf("%c", 31) + from_p = "" + namespace_p = "" + priority_p = "" + program_p = "" + ref_p = "" +} +{sub(/\r$/, "", $0)} +/^[[:blank:]]*@file / && from_p == "" { + from_p = $0 + sub(/^[[:blank:]]*@file /, "", from_p) +} +/^[[:blank:]]*@name / && namespace_p == "" { namespace_p = $2 } +/^[[:blank:]]*@priority / && priority_p == "" { priority_p = $2 } +/^[[:blank:]]*@exec / && program_p == "" { + program_p = $0 + sub(/^[[:blank:]]*@exec /, "", program_p) +} +/^[[:blank:]]*@ref / && ref_p == "" { ref_p = $2 } +END { + printf "%s%s%s%s%s%s%s%s%s\n", from_p, us, namespace_p, us, priority_p, us, program_p, us, ref_p +} +' "$chunk_file" +} + +get_parallel_jobs() { + nproc 2>/dev/null || \ + getconf _NPROCESSORS_ONLN 2>/dev/null || \ + grep -c '^processor' /proc/cpuinfo 2>/dev/null || \ + echo 4 +} + +# Apply many literal replacements by streaming file content through +# chained sed processes and writing back once at the end. +# replacements_file format: <search><US><replacement> one pair per line. +replace_literal_pairs_in_file() { + local target_file="$1" + local replacements_file="$2" + local token_sep + local tmp_file + local sed_chain + + [ ! -s "$replacements_file" ] && return 0 + token_sep=$(printf '\037') + tmp_file=$(mktemp) || return 1 + + # Single-quote a string for safe eval embedding. + _quote_sq() { + printf "'%s'" "$(printf '%s' "$1" | sed "s/'/'\\\\''/g")" + } + + local cmd="cat $(_quote_sq "$target_file")" + sed_chain=$(awk -v us="$token_sep" ' +function escape_pattern(v) { gsub(/[][\\.^$*+?(){}|]/, "\\\\&", v); return v } +function escape_replacement(v) { gsub(/[\\&|]/, "\\\\&", v); return v } +function escape_dquote(v) { gsub(/["\\`$]/, "\\\\&", v); return v } +{ + sep = index($0, us) + if (sep == 0) next + search = substr($0, 1, sep - 1) + repl = substr($0, sep + 1) + search = escape_pattern(search) + repl = escape_replacement(repl) + expr = "s|" search "|" repl "|g" + expr = escape_dquote(expr) + printf " | sed -r \"%s\"", expr +} +' "$replacements_file") + + cmd="$cmd$sed_chain > $(_quote_sq "$tmp_file")" + eval "$cmd" || { + rm -f "$tmp_file" + return 1 + } + mv "$tmp_file" "$target_file" +} + +# Extract first "From ..." citation source (without HTML tags) for resolver context. +extract_source_from_cite() { + local html_file="$1" + awk ' +match($0, /<cite>From .*<\/cite>/) { + source = substr($0, RSTART + 11, RLENGTH - 18) + gsub(/<[^>]*>/, "", source) + print source + exit +} +' "$html_file" +} + +# Emit unique reference tokens from HTML: +# I<US>@token for internal refs, E<US>@prefix/path for external refs. +collect_html_ref_tokens() { + local html_file="$1" + awk ' +{ + internal_line = $0 + while (match(internal_line, /@[a-zA-Z0-9_.\/-]+/)) { + token = substr(internal_line, RSTART, RLENGTH) + if (index(token, "|") == 0 && !seen_internal[token]++) { + printf "I%c%s\n", 31, token + } + internal_line = substr(internal_line, RSTART + RLENGTH) + } + + external_line = $0 + while (match(external_line, /@[a-zA-Z0-9_-]+\/[^[:space:]<>"]+/)) { + token = substr(external_line, RSTART, RLENGTH) + if (!seen_external[token]++) { + printf "E%c%s\n", 31, token + } + external_line = substr(external_line, RSTART + RLENGTH) + } +} +' "$html_file" +} + +# render_layout_page $layout_file $nav_file $body_file $page_title $page_breadcrub $output_file +render_layout_page() { + # Read nav content (may contain newlines, so flatten for sed) + local nav_html + local escaped_title + local escaped_breadcrumb + local escaped_nav + nav_html=$(tr '\n' ' ' < "$2") + escaped_title=$(escape_sed_replacement_literal "$4") + escaped_breadcrumb=$(escape_sed_replacement_literal "$5") + escaped_nav=$(escape_sed_replacement_literal "$nav_html") + + # Render layout to output in a single sed pass. + # {{ yield }} is expected to be on its own line. + sed -E \ + -e "s|\\{\\{ title \\}\\}|$escaped_title|g" \ + -e "s|\\{\\{ breadcrumb \\}\\}|$escaped_breadcrumb|g" \ + -e "s|\\{\\{ nav \\}\\}|$escaped_nav|g" \ + -e "/\\{\\{ yield \\}\\}/r $3" \ + -e "/\\{\\{ yield \\}\\}/d" \ + "$1" > "$6" +} + +assemble_site() { + local namespace_root="$1" + local destination_root="$2" + local layout_file="$namespace_root/main.layout" + local nav_file="$namespace_root/nav.partial" + local rendered_page + local rel_path + local namespace_path + local page_title="index" + local page_breadcrumb="index" + local target_dir + local output_file + + rm -rf "$destination_root" + mkdir -p "$destination_root" + + if [ -d "$namespace_root/stylesheets" ]; then + mkdir -p "$destination_root/stylesheets" + cp -R "$namespace_root/stylesheets/." "$destination_root/stylesheets/" + fi + + _asm_find_tmp=$(mktemp) + find "$namespace_root" -name "index.html" > "$_asm_find_tmp" + while IFS= read -r rendered_page; do + rel_path="${rendered_page#$namespace_root/}" + namespace_path="${rel_path%/index.html}" + if [ "$rel_path" = "index.html" ]; then + namespace_path="" + fi + if [ -n "$namespace_path" ]; then + target_dir="$destination_root/$namespace_path" + page_title="$namespace_path" + page_breadcrumb="$namespace_path" + else + target_dir="$destination_root" + fi + mkdir -p "$target_dir" + output_file="$target_dir/index.html" + render_layout_page "$layout_file" "$nav_file" "$rendered_page" "$page_title" "$page_breadcrumb" "$output_file" + done < "$_asm_find_tmp" + rm -f "$_asm_find_tmp" +} + +# Cache directory +cache_dir=./.trblcache +export TRBLCACHE="$cache_dir" +# Resolve output directory now that cache_dir is known +out_dir="${out_dir:-$cache_dir/built}" +mkdir -p "$cache_dir" +common_functions_sh="$cache_dir/_common_functions.sh" +cat > "$common_functions_sh" << 'COMMON_SH' +escape_sed_pattern_literal() { + local value="$1" + printf '%s' "$value" | sed -e 's/[][\\.^$*+?(){}|]/\\&/g' +} + +escape_sed_replacement_literal() { + local value="$1" + printf '%s' "$value" | sed -e 's/[\\&|]/\\&/g' +} + +md() { ${TRBLDOC_MD:-md2html --github --fpermissive-autolinks --ftables} "$@"; } +rst() { ${TRBLDOC_RST:-docutils --template ./.trblcache/rsttemplate.txt} "$@"; } + +# resolve_ref PREFIX PATH FILEPATH LINENUM +# Resolves a reference by looking up the resolver script, +# exports TRBLCACHE/FILEPATH/LINENUM env vars, and runs the script with PATH as stdin. +# Prints the resolved URL to stdout, or nothing if resolution fails. +resolve_ref() { + local prefix="$1" + local resolver_script="$SCRIPT_DIR/ref_resolvers/${prefix}.sh" + local resolver_opts="" + + if [ -f "$resolver_overrides_dir/${prefix}.script" ]; then + resolver_script=$(cat "$resolver_overrides_dir/${prefix}.script") + resolver_opts=$(cat "$resolver_overrides_dir/${prefix}.opts" 2>/dev/null || true) + elif [ -n "${TRBLDOC_RESOLVER_OVERRIDES_RAW:-}" ]; then + printf '%s' "$TRBLDOC_RESOLVER_OVERRIDES_RAW" | while IFS= read -r override_spec; do + [ -z "$override_spec" ] && continue + override_prefix="${override_spec%%=*}" + [ "$override_prefix" != "$prefix" ] && continue + override_rhs="${override_spec#*=}" + resolver_script="${override_rhs%%;*}" + resolver_opts="" + if [ "$override_rhs" != "$resolver_script" ]; then + resolver_opts="${override_rhs#*;}" + fi + break + done + fi + + if [ ! -f "$resolver_script" ]; then + return 1 + fi + + export TRBLCACHE="$cache_dir" + export FILEPATH="$3" + export LINENUM="$4" + + if [ -n "$resolver_opts" ]; then + printf '%s\n' "$2" | eval "\"$resolver_script\" $resolver_opts" 2>/dev/null || true + else + printf '%s\n' "$2" | "$resolver_script" 2>/dev/null || true + fi +} +COMMON_SH +. "$common_functions_sh" +export common_functions_sh + +parallel_jobs="${TRBLDOC_JOBS:-$(get_parallel_jobs)}" + +# `$cache/last_processed` stores empty files to represent the last updated date (so we don't reprocess files if they haven't been updated) +# `$cache/namespace` is the output of documentation execution, to be aggregated and copied into build/ +# `$cache/ref` references, things that should be resolved after all the pages are generated +# `$cache/chunks` # Chunks extracted from files, kept seperate so we can just delete the chunks for one file if it's changed. +# `$cache/comment_forms` stores awk programs that extract chunks from files based on their extension +# Weirdly, this is one of the slower parts of trbldoc. +xargs -n 1 -P $parallel_jobs mkdir -p <<EOF +$cache_dir/last_processed +$cache_dir/namespace +$cache_dir/ref +$cache_dir/chunks +$cache_dir/comment_forms +$out_dir +EOF + + +# Various opening and closing tags for different languages (based on file extension) +# Each entry: ext,end_regex,start_regex (AWK extended regex patterns, no slashes) +comment_forms=" +c,\*\/,\/\* +lua,\]\],\-\-\[\[ +" + +# Helper: write comment-extraction awk script from start/end regex pair. +# Uses single-quote splicing to inject shell variables into awk source. +_write_comment_form() { + printf '%s\n' ' +BEGIN{} +/'"$2"'/ { + if(in_comment) {print "EOF"} + in_comment=0 +} +in_comment {print} +/'"$3"' .+/ && !/'"$2"'$/ { + print "chunkfile=$(mktemp -p '"${cache_dir}"'/chunks)" + print "cat > $chunkfile << \"EOF\"\n@file " name ":" NR ; + if($2) {$1=""; print "@exec " $0} + in_comment=1; +}' > "$1" +} + +comment_form_dir="$cache_dir/comment_forms" + +# Built-in comment forms from the table +printf '%s' "$comment_forms" | while IFS= read -r _cf_line; do + case "$_cf_line" in *[!" "]*) ;; *) continue ;; esac + _cf_lang="${_cf_line%%,*}" + _cf_rest="${_cf_line#*,}" + _cf_end="${_cf_rest%%,*}" + _cf_start="${_cf_rest#*,}" + _write_comment_form "$comment_form_dir/${_cf_lang}.awk" "$_cf_end" "$_cf_start" +done + +# Aliases for same-syntax languages +for _alias_ext in h cpp hpp sql dot ts js; do + cp "$comment_form_dir/c.awk" "$comment_form_dir/${_alias_ext}.awk" 2>/dev/null || true +done +for _alias_ext in moon tl; do + cp "$comment_form_dir/lua.awk" "$comment_form_dir/${_alias_ext}.awk" 2>/dev/null || true +done + +# Python: ''' is both the open and close delimiter. Renderer name is +# glued directly after the opening ''' with no space required (e.g. '''md). +# Custom AWK uses sub() for renderer extraction instead of field splitting. +sed "s|__PY_CACHE__|${cache_dir}|g" << 'PYAWK' > "$comment_form_dir/py.awk" +BEGIN{} +/'''/ { + if(in_comment) {print "EOF"} + in_comment=0 +} +in_comment {print} +/'''.+/ && !/'''$/ { + print "chunkfile=$(mktemp -p __PY_CACHE__/chunks)" + print "cat > $chunkfile << \"EOF\"\n@file " name ":" NR ; + renderer = $0 + sub(/'''/, "", renderer) + sub(/^[[:space:]]+/, "", renderer) + sub(/[[:space:]]+$/, "", renderer) + if (renderer != "") {print "@exec " renderer} + in_comment=1; +} +PYAWK + +# .md files only have 1 chunk +printf '%s\n' ' +BEGIN{ + print "chunkfile=$(mktemp -p '"${cache_dir}"'/chunks)" + print "cat > $chunkfile << \"EOF\"\n@file " name ":" NR "\n@exec md" ; +} +{print $0} +END{print "EOF"}' > "$comment_form_dir/md.awk" +echo "%(body)s" > "$cache_dir/rsttemplate.txt" + +# Apply -f extra file format definitions from command line. +# Format: ext;start_regex;end_regex (AWK regex patterns without slashes). +printf '%s' "$extra_formats" | while IFS= read -r fmt_spec; do + [ -z "$fmt_spec" ] && continue + _ef_ext="${fmt_spec%%;*}" + _ef_rest="${fmt_spec#*;}" + _ef_start="${_ef_rest%%;*}" + _ef_end="${_ef_rest#*;}" + if [ -n "$_ef_ext" ] && [ -n "$_ef_start" ] && [ -n "$_ef_end" ]; then + _write_comment_form "$comment_form_dir/${_ef_ext}.awk" "$_ef_end" "$_ef_start" + fi +done + + +echo "" > "$cache_dir/namespace/global.meta" +if [ -n "$layout_template" ]; then + if [ ! -f "$layout_template" ]; then + echo "trbldoc: layout template not found: $layout_template" >&2 + exit 1 + fi + cp "$layout_template" "$cache_dir/namespace/main.layout" +else +cat > $cache_dir/namespace/main.layout << EOF +<!DOCTYPE html> +<html lang="en"> + <head> + <meta charset="utf-8"> + <title>{{ title }}</title> + <link href="/stylesheets/style.css" rel="stylesheet"> + </head> + <body> + <header> + <h1><a href="#">{{ breadcrumb }}</a></h1> + </header> + <nav><ul>{{ nav }}</ul></nav> + <article> + {{ yield }} + </article> + <footer> + <p>Generated by trbldoc</p> + </footer> + </body> +</html> +EOF +fi +mkdir -p "$cache_dir/namespace/stylesheets" +cat > "$cache_dir/namespace/stylesheets/style.css" << EOF +body{ + margin:40px auto; + max-width:80ch; + line-height:1.6; + font-size:18px; + color:#444; + padding:0 10px +} +h1,h2,h3{line-height:1.2} +caption,td{border-bottom:1px solid black;} +nav{float:left;position:absolute;left:10%;} +table{width:100%;} +EOF + +# Find all the source files +# Build extension filter from awk files in comment_form_dir +ext_filter="" +for _awk_file in "$comment_form_dir"/*.awk; do + [ -f "$_awk_file" ] || continue + _ext=$(basename "$_awk_file" .awk) + ext_filter="${ext_filter}${ext_filter:+|}${_ext}" +done +# Populate source files list (one per line) +source_files_list="$cache_dir/source_files.list" +printf '%s' "$infolders" \ +| xargs --no-run-if-empty -I '{}' find "{}" -type f \ +| grep -E "\.(${ext_filter})$" \ +> "$source_files_list" + +# Analyze source files for chunks +# -C wipes the chunk cache and all timestamps for a full re-scan. +# Otherwise only source files newer than their last-processed marker are +# re-extracted; chunks for unchanged files are reused as-is. +if [ "$clean_build" -eq 1 ]; then + rm -rf "$cache_dir/chunks" "$cache_dir/last_processed" +fi +mkdir -p "$cache_dir/chunks" +mkdir -p "$cache_dir/last_processed" + +# Rendered output is always rebuilt from the full set of chunks. +# This has to be the case because the `@exec` may change. +# `@exec` is just a program, we can't garentee it'll do the same stuff next run. +# See `docs/trblcache.md for more info +find "$cache_dir/namespace" -name "index.html" -delete 2>/dev/null || true +> "$cache_dir/namespace/nav.partial" +export cache_dir comment_form_dir +changed_sources_list="$cache_dir/changed_sources.list" +tracked_sources_list="$cache_dir/tracked_sources.list" +removed_sources_list="$cache_dir/removed_sources.list" +purge_sources_list="$cache_dir/purge_sources.list" +> "$changed_sources_list" + +# Build changed source list (only files newer than their marker, or missing marker). +if [ -s "$source_files_list" ]; then + while IFS= read -r source_file; do + last="$cache_dir/last_processed/$source_file" + if [ -f "$last" ] && [ ! "$source_file" -nt "$last" ]; then + continue + fi + printf '%s\n' "$source_file" >> "$changed_sources_list" + done < "$source_files_list" +fi + +# Compute removed sources from stale markers, then remove those markers. +find "$cache_dir/last_processed" -type f 2>/dev/null \ + | sed "s|^$cache_dir/last_processed/||" > "$tracked_sources_list" +grep -Fvx -f "$source_files_list" "$tracked_sources_list" 2>/dev/null > "$removed_sources_list" || true +if [ -s "$removed_sources_list" ]; then + awk -v base="$cache_dir/last_processed/" '{ print base $0 }' "$removed_sources_list" \ + | xargs rm -f 2>/dev/null || true +fi + +# Purge stale chunks once (changed + removed), instead of per-worker scans. +{ + cat "$changed_sources_list" 2>/dev/null + cat "$removed_sources_list" 2>/dev/null +} | awk 'NF && !seen[$0]++' > "$purge_sources_list" +if [ -s "$purge_sources_list" ]; then + _purge_patterns=$(mktemp) + awk '{ printf "@file %s:\n", $0 }' "$purge_sources_list" > "$_purge_patterns" + grep -rlFf "$_purge_patterns" "$cache_dir/chunks/" 2>/dev/null \ + | xargs rm -f 2>/dev/null || true + rm -f "$_purge_patterns" +fi +# Create helper script for parallel extraction +_extract_helper="$cache_dir/_extract_helper.sh" +cat > "$_extract_helper" << 'EXTRACT_SH' +#!/bin/sh +source_file="$1" +file_name=$(basename -- "$source_file") +file_ext=${file_name##*.} +awk_prog_file="$comment_form_dir/$file_ext.awk" +if [ ! -f "$awk_prog_file" ]; then + exit 0 +fi +chunk_script=$(awk -v name="$source_file" -f "$awk_prog_file" "$source_file") +eval "$chunk_script" +last="$cache_dir/last_processed/$source_file" +mkdir -p "${last%/*}" && touch "$last" +EXTRACT_SH +chmod +x "$_extract_helper" +if [ -s "$changed_sources_list" ]; then + tr '\n' '\0' < "$changed_sources_list" \ + | xargs -0 -n 1 -P "$parallel_jobs" "$_extract_helper" +fi + +# Resolve src_repo once before the chunk loop. +# Prefers the TRBLDOC_SRC_REPO environment variable; falls back to the git +# remote origin URL; defaults to empty string if neither is available. +src_repo_val="${TRBLDOC_SRC_REPO:-$(git remote get-url origin 2>/dev/null || echo '')}" + +_chunk_files_list=$(mktemp) +find "$cache_dir/chunks" -type f > "$_chunk_files_list" +chunk_order_file="$cache_dir/chunk.render_order" +> "$chunk_order_file" +chunk_order_delim="$US" +while IFS= read -r chunk_file; do + IFS="$US" read -r chunk_from_p chunk_namespace_p chunk_priority_p chunk_program_p chunk_ref_p <<_USEOF_ +$(parse_chunk_metadata "$chunk_file") +_USEOF_ + chunk_source_path="${chunk_from_p%:*}" + chunk_source_line="${chunk_from_p##*:}" + if ! printf '%s' "$chunk_source_line" | grep -qE '^-?[0-9]+$'; then + chunk_source_line=0 + fi + chunk_namespace="$(namespace_for_chunk "$chunk_source_path" "$chunk_namespace_p")" || exit 1 + chunk_priority=0 + if printf '%s' "$chunk_priority_p" | grep -qE '^-?[0-9]+$'; then + chunk_priority="$chunk_priority_p" + fi + printf "%s${chunk_order_delim}%s${chunk_order_delim}%s${chunk_order_delim}%s${chunk_order_delim}%s${chunk_order_delim}%s${chunk_order_delim}%s${chunk_order_delim}%s\n" \ + "$chunk_priority" \ + "$chunk_namespace" \ + "$chunk_source_path" \ + "$chunk_source_line" \ + "$chunk_file" \ + "$chunk_from_p" \ + "$chunk_program_p" \ + "$chunk_ref_p" >> "$chunk_order_file" +done < "$_chunk_files_list" +rm -f "$_chunk_files_list" +# Deterministic render order: +# 1 @priority (higher numbers first) +# 2 namespace +# 3 source path +# 4 source line +# 5 chunk temp file path (stable tie-breaker) +sorted_chunk_order_file="$cache_dir/chunk.render_order.sorted" +LC_ALL=C sort -t "$US" -k1,1nr -k2,2 -k3,3 -k4,4n -k5,5 "$chunk_order_file" > "$sorted_chunk_order_file" +chunk_render_records_file="$cache_dir/chunk.render_records" +> "$chunk_render_records_file" +chunk_seq=0 +while IFS="$US" read -r _chunk_priority namespace _chunk_source_path _chunk_source_line chunk_file from_p program_p ref_p; do + chunk_seq=$((chunk_seq + 1)) + printf "%s${chunk_order_delim}%s${chunk_order_delim}%s${chunk_order_delim}%s${chunk_order_delim}%s${chunk_order_delim}%s${chunk_order_delim}%s${chunk_order_delim}%s${chunk_order_delim}%s\n" \ + "$chunk_seq" \ + "$_chunk_priority" \ + "$namespace" \ + "$_chunk_source_path" \ + "$_chunk_source_line" \ + "$chunk_file" \ + "$from_p" \ + "$program_p" \ + "$ref_p" >> "$chunk_render_records_file" +done < "$sorted_chunk_order_file" +render_artifacts_dir="$cache_dir/rendered_chunks" +rm -rf "$render_artifacts_dir" +mkdir -p "$render_artifacts_dir" +export SCRIPT_DIR src_repo_val render_artifacts_dir US +export TRBLDOC_RESOLVER_OVERRIDES_RAW resolver_overrides_dir +# Pre-compute the escaped sed patterns/replacements for template variables +# once in the parent so child renderers avoid repeated escape subshells. +# {{ref_string}} pattern is constant; {{src_repo}} pattern+replacement are constant. +_esc_ref_pat=$(escape_sed_pattern_literal '{{ ref_string }}') +_esc_src_pat=$(escape_sed_pattern_literal '{{ src_repo }}') +_esc_src_repl=$(escape_sed_replacement_literal "$src_repo_val") +export _esc_ref_pat _esc_src_pat _esc_src_repl +_render_helper="$cache_dir/_render_helper.sh" +cat > "$_render_helper" << 'RENDER_SH' +#!/bin/sh +render_record="$1" +IFS="$US" read -r chunk_seq _chunk_priority namespace _chunk_source_path _chunk_source_line chunk_file from_p program_p ref_p <<_USEOF_ +$render_record +_USEOF_ +program="${program_p:-cat}" +. "$common_functions_sh" +# Compute the {{ref_string}} replacement. Only chunks with @ref need the +# full resolve_ref call. +_esc_ref_repl="" +if [ -n "$ref_p" ]; then + _rs_prefix="${ref_p%%/*}" + _rs_path="${ref_p#*/}" + _rs_url=$(resolve_ref "$_rs_prefix" "$_rs_path" "$_chunk_source_path" "$_chunk_source_line") + if [ -n "$_rs_url" ]; then + _esc_ref_repl=$(escape_sed_replacement_literal "<a href='$_rs_url'>$ref_p</a>") + fi +fi +export TRBLCACHE="$cache_dir" FILEPATH="$_chunk_source_path" LINENUM="$_chunk_source_line" +artifact_prefix=$(printf '%s/%08d' "$render_artifacts_dir" "$chunk_seq") +# One sed pass: strip directives + substitute both template variables, piped +# directly into the renderer. The escaped patterns and the {{src_repo}} +# replacement are pre-computed in the parent; only {{ref_string}}'s +# replacement (when @ref is present) is computed per-chunk. +render_rc=0 +sed -E \ + -e "s/^@.*//g" \ + -e "s|${_esc_ref_pat}|${_esc_ref_repl}|g" \ + -e "s|${_esc_src_pat}|${_esc_src_repl}|g" \ + "$chunk_file" | eval "$program" > "${artifact_prefix}.body" || render_rc=$? +if [ "$render_rc" -ne 0 ]; then + echo "trbldoc: renderer failed (exit $render_rc): $program" >&2 + echo "trbldoc: chunk source: $from_p" >&2 +fi +RENDER_SH +chmod +x "$_render_helper" +if [ -s "$chunk_render_records_file" ]; then + while IFS= read -r render_record; do + printf '%s\0' "$render_record" + done < "$chunk_render_records_file" \ + | xargs -0 -n 1 -P "$parallel_jobs" "$_render_helper" +fi +while IFS="$US" read -r chunk_seq _chunk_priority namespace _chunk_source_path _chunk_source_line _chunk_file from_p _program_p ref_p; do + ref="${ref_p:-$namespace}" + artifact_prefix=$(printf '%s/%08d' "$render_artifacts_dir" "$chunk_seq") + output="" + if [ -f "${artifact_prefix}.body" ]; then + output=$(cat "${artifact_prefix}.body") + fi + if [ -n "$output" ]; then + from_html="<cite>From $from_p</cite>" + if [ -n "$src_repo_val" ]; then + src_file_path="$_chunk_source_path" + src_file_line="" + if printf '%s' "$_chunk_source_line" | grep -qE '^[1-9][0-9]*$'; then + src_file_line="$_chunk_source_line" + fi + src_file_url="${src_repo_val%/}/$src_file_path" + if [ -n "$src_file_line" ]; then + src_file_url="${src_file_url}#L${src_file_line}" + fi + from_html="<cite>From <a href='$src_file_url'>$from_p</a></cite>" + fi + if [ -n "$namespace" ]; then + outfolder="$cache_dir/namespace/$namespace" + else + outfolder="$cache_dir/namespace" + fi + if [ ! -d "$outfolder" ]; then + mkdir -p "$outfolder" + fi + echo "$output" "$from_html" >> "$outfolder/index.html" + fi + # internal links for generated cross references + if [ "$ref" = "$namespace" ]; then + if [ -n "$ref" ]; then + echo "$ref: <a href='/$ref'>$ref</a>" >> "$cache_dir/namespace/global.meta" + else + echo "index: <a href='/'>index</a>" >> "$cache_dir/namespace/global.meta" + fi + else + echo "$ref: <a href='/$namespace#$ref'>$ref</a>" >> "$cache_dir/namespace/global.meta" + fi +done < "$chunk_render_records_file" + +> "$cache_dir/namespace/nav.partial" +find "$cache_dir/namespace" -name "index.html" \ +| sed \ + -e 's;/index\.html$;;' \ + -e "s;^$cache_dir/namespace/*;;" \ +| sort -u \ +| awk ' +function emit(path, nxt, n, c, i) { + if (path == "") { print "<li><a href=\047/\047>index</a></li>"; return } + n = split(path, p, "/") + c = 0 + while (c < depth && c < n - 1 && q[c+1] == p[c+1]) c++ + for (; depth > c; depth--) print "</ul></li>" + for (i = c + 1; i < n; i++) print "<li>" p[i] "/<ul>" + if (index(nxt, path "/") == 1) { print "<li><a href=\047/" path "\047>" p[n] "</a>/<ul>"; depth = n } + else { print "<li><a href=\047/" path "\047>" p[n] "</a></li>"; depth = n - 1 } + split(path, q, "/") +} +NR > 1 { emit(prev, $0) } +{ prev = $0 } +END { + if (NR) emit(prev, "") + for (; depth > 0; depth--) print "</ul></li>" +} +' > "$cache_dir/namespace/nav.partial" + +# Second pass: resolve inline references now that global.meta is complete. +# Deduplicate global.meta (keep first occurrence per ref name, which has highest priority) +_deduped_meta="$cache_dir/namespace/global.meta.deduped" +awk -F': ' '!seen[$1]++' "$cache_dir/namespace/global.meta" > "$_deduped_meta" + +# Lookup ref link HTML from the deduplicated file +_ref_link_lookup() { + awk -v name="$1" '{ + p = index($0, ": ") + if (p > 0 && substr($0, 1, p - 1) == name) { + print substr($0, p + 2) + exit + } + }' "$_deduped_meta" +} + +token_sep="$US" +_ref_find_tmp=$(mktemp) +find "$cache_dir/namespace" -name "index.html" > "$_ref_find_tmp" +while IFS= read -r htmlfile; do + replacement_pairs_file=$(mktemp) + # Extract source file info for external resolver env vars + _html_from=$(extract_source_from_cite "$htmlfile") + _html_filepath="${_html_from%:*}" + _html_linenum="${_html_from##*:}" + if ! printf '%s' "$_html_linenum" | grep -qE '^[0-9]+$'; then + _html_linenum=0 + fi + collect_html_ref_tokens "$htmlfile" | while IFS="$token_sep" read -r ref_token_type ref_token; do + [ -z "$ref_token" ] && continue + if [ "$ref_token_type" = "I" ]; then + ref_name="${ref_token#@}" # strip leading @ + _link_html=$(_ref_link_lookup "$ref_name") + if [ -n "$_link_html" ]; then + # Found in global.meta - use the pre-built HTML link + printf '%s%s%s\n' "$ref_token" "$token_sep" "$_link_html" >> "$replacement_pairs_file" + fi + continue + fi + if [ "$ref_token_type" != "E" ]; then + continue + fi + + # Resolve @script/path tokens (external refs) - only if not matched above + ref_name="${ref_token#@}" + _link_html=$(_ref_link_lookup "$ref_name") + if [ -n "$_link_html" ]; then + continue + fi + script_name="${ref_name%%/*}" # get script prefix before / + ref_path="${ref_name#*/}" # get path after @prefix/ + resolved_url=$(resolve_ref "$script_name" "$ref_path" "$_html_filepath" "$_html_linenum") + if [ -n "$resolved_url" ]; then + replacement_link="<a href='${resolved_url}'>$script_name🔗${ref_path}</a>" + printf '%s%s%s\n' "$ref_token" "$token_sep" "$replacement_link" >> "$replacement_pairs_file" + else + echo "trbldoc: unresolved ref: $ref_token (at $_html_filepath:$_html_linenum)" >&2 + fi + done + + replace_literal_pairs_in_file "$htmlfile" "$replacement_pairs_file" + rm -f "$replacement_pairs_file" + +done < "$_ref_find_tmp" +rm -f "$_ref_find_tmp" + +# Post-processing pass: substitute {{ toc }} in all rendered index.html files +# now that `nav.partial` is finalised. Lines are joined into one so the sed +# substitution works without needing multi-line mode. +toc_oneline=$( \ + cat "$cache_dir/namespace/nav.partial" \ + | tr '\n' ' ' \ + | sed -e "$sed_replacement_literal" \ +) +toc_repl=$( \ + printf "%s" "{{ toc }}" \ + | sed -e "$sed_pattern_literal" \ +) +find "$cache_dir/namespace" -name "index.html" \ +| xargs -n 1 -P "$parallel_jobs" sed -E -i -e "s|$toc_repl|$toc_oneline|g" +assemble_site "$cache_dir/namespace" "$out_dir" |