From 657fb0007f39f07cc0401e0c5d03e25df6234aa4 Mon Sep 17 00:00:00 2001 From: Alexander M Pickering Date: Tue, 21 Jul 2026 20:19:46 -0500 Subject: Inital commit. --- .gitignore | 1 + docs/coverage.md | 55 ++ docs/profiling.md | 81 +++ docs/trblcache.md | 83 +++ readme.md | 275 ++++++++++ ref_resolvers/lua.sh | 342 +++++++++++++ scripts/README.md | 50 ++ scripts/coverage-tests.sh | 143 ++++++ scripts/generate-reports.sh | 58 +++ scripts/hotspot-tests.sh | 204 ++++++++ scripts/lib/common.sh | 105 ++++ scripts/lib/coverage-report.awk | 84 +++ scripts/lib/render-coverage-index.awk | 48 ++ scripts/lib/render-hotspots.awk | 56 ++ scripts/lib/render-phases.awk | 41 ++ scripts/lib/render-timing.awk | 41 ++ scripts/lib/timing-extract.awk | 32 ++ scripts/lib/trace-extract.awk | 71 +++ scripts/lib/trace-init.sh | 42 ++ scripts/lib/trace-phase-extract.awk | 75 +++ scripts/profile-run.sh | 180 +++++++ scripts/profile-tests.sh | 184 +++++++ tests/fixtures/c/directives.c | 7 + tests/fixtures/c/inline_ref.c | 5 + tests/fixtures/c/multi_chunk.c | 11 + tests/fixtures/c/no_comments.c | 3 + tests/fixtures/c/ref_string.c | 6 + tests/fixtures/c/rst_exec.c | 8 + tests/fixtures/c/shared_namespace.c | 11 + tests/fixtures/c/simple.c | 5 + tests/fixtures/c/src_repo.c | 5 + tests/fixtures/c/toc.c | 5 + tests/fixtures/lua/simple.lua | 5 + tests/fixtures/lua/single_line.lua | 2 + tests/fixtures/md/readme.md | 2 + tests/fixtures/md/simple.md | 3 + tests/fixtures/py/simple.py | 6 + tests/fixtures/py/single_line.py | 2 + tests/helpers/common.bash | 263 ++++++++++ tests/helpers/profiler.bash | 53 ++ tests/integration/cli.bats | 215 ++++++++ tests/integration/pipeline.bats | 842 ++++++++++++++++++++++++++++++ tests/run_tests.sh | 57 +++ tests/unit/directives.bats | 235 +++++++++ tests/unit/extraction.bats | 171 +++++++ tests/unit/template_vars.bats | 160 ++++++ trbldoc.sh | 940 ++++++++++++++++++++++++++++++++++ 47 files changed, 5273 insertions(+) create mode 100644 .gitignore create mode 100644 docs/coverage.md create mode 100644 docs/profiling.md create mode 100644 docs/trblcache.md create mode 100644 readme.md create mode 100644 ref_resolvers/lua.sh create mode 100644 scripts/README.md create mode 100644 scripts/coverage-tests.sh create mode 100644 scripts/generate-reports.sh create mode 100644 scripts/hotspot-tests.sh create mode 100644 scripts/lib/common.sh create mode 100644 scripts/lib/coverage-report.awk create mode 100644 scripts/lib/render-coverage-index.awk create mode 100644 scripts/lib/render-hotspots.awk create mode 100644 scripts/lib/render-phases.awk create mode 100644 scripts/lib/render-timing.awk create mode 100644 scripts/lib/timing-extract.awk create mode 100644 scripts/lib/trace-extract.awk create mode 100644 scripts/lib/trace-init.sh create mode 100644 scripts/lib/trace-phase-extract.awk create mode 100644 scripts/profile-run.sh create mode 100644 scripts/profile-tests.sh create mode 100644 tests/fixtures/c/directives.c create mode 100644 tests/fixtures/c/inline_ref.c create mode 100644 tests/fixtures/c/multi_chunk.c create mode 100644 tests/fixtures/c/no_comments.c create mode 100644 tests/fixtures/c/ref_string.c create mode 100644 tests/fixtures/c/rst_exec.c create mode 100644 tests/fixtures/c/shared_namespace.c create mode 100644 tests/fixtures/c/simple.c create mode 100644 tests/fixtures/c/src_repo.c create mode 100644 tests/fixtures/c/toc.c create mode 100644 tests/fixtures/lua/simple.lua create mode 100644 tests/fixtures/lua/single_line.lua create mode 100644 tests/fixtures/md/readme.md create mode 100644 tests/fixtures/md/simple.md create mode 100644 tests/fixtures/py/simple.py create mode 100644 tests/fixtures/py/single_line.py create mode 100644 tests/helpers/common.bash create mode 100644 tests/helpers/profiler.bash create mode 100644 tests/integration/cli.bats create mode 100644 tests/integration/pipeline.bats create mode 100644 tests/run_tests.sh create mode 100644 tests/unit/directives.bats create mode 100644 tests/unit/extraction.bats create mode 100644 tests/unit/template_vars.bats create mode 100644 trbldoc.sh diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..cbdc0e0 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +reports/* diff --git a/docs/coverage.md b/docs/coverage.md new file mode 100644 index 0000000..c6371d0 --- /dev/null +++ b/docs/coverage.md @@ -0,0 +1,55 @@ +# Test-suite coverage + +`scripts/coverage-tests.sh` reports which lines of trbldoc's shell sources the +test suite actually executes. + +## Usage + +```sh +# Coverage of trbldoc.sh from the integration suite (default) +sh scripts/coverage-tests.sh + +# Whole suite, also cover a resolver, fail under 70% +sh scripts/coverage-tests.sh -s all -t "trbldoc.sh ref_resolvers/lua.sh" -m 70 +``` + +Options: `-s all|unit|integration`, `-o OUTDIR`, `-t "file.sh ..."` (targets), +`-m MIN_PERCENT` (exit non-zero below this; default `0` = no gate). + +## Output + +Written to `reports/coverage-/` (or `-o`): + +- `index.html` — per-file coverage with bars and an overall percentage. +- `html/.html` — annotated source; green = executed, red = missed, + grey = non-executable. +- `summary.csv` — `file,executable,covered,percent`. +- `covered.tsv` — raw union of executed `file,line` pairs. + +## How it works + +The bats tests run the tool as `bash "$TRBLDOC"`. Coverage exports +`BASH_ENV=scripts/lib/trace-init.sh`, so each target process enables `set -x` +with a `PS4` that stamps `BASH_SOURCE:LINENO` to a per-PID trace file. After +the run, the union of executed lines is compared against the executable lines +in each target source to compute coverage. No changes to `trbldoc.sh` or the +tests are needed. + +Only scripts whose basename is listed in `-t` are traced, so bats internals +and the `md2html` stub stay out of the report. + +## Determining "executable" lines + +The denominator is a heuristic: a line counts as executable when, after +trimming, it is non-blank, not a comment, and not a pure structural token +(`then`, `do`, `done`, `else`, `fi`, `esac`, `{`, `}`, `(`, `)`, `;;`). + +## Limitations + +- Coverage is collected under `bash` (the shell the suite already uses). + Lines only reachable under a different shell are not distinguished. +- xtrace reports the line where a command *starts*. Interior lines of + heredocs and multi-line strings are not individually reported and may show + as missed; exclude such files or read the numbers with that caveat. +- Code in subprocesses spawned via `ash`/`xargs` is not traced (not bash), + so it is not counted toward coverage of the main script. diff --git a/docs/profiling.md b/docs/profiling.md new file mode 100644 index 0000000..9978347 --- /dev/null +++ b/docs/profiling.md @@ -0,0 +1,81 @@ +# Profiling the test suite + +`scripts/profile-tests.sh` reports where time goes when running the bats +suite, at two levels of detail. + +## What it produces + +1. **Per-test timing** (always). Runs bats with `--timing` and the JUnit + report formatter, then ranks every test slowest-first. Answers *which + tests take a long time*. +2. **Per-line hotspots** (with `-p`). Re-runs the suite with `trbldoc.sh` + under `bash` xtrace and attributes elapsed wall-time to individual source + lines. Answers *which code takes a long time*. + +## Usage + +```sh +# Per-test timing for the integration suite (default) +sh scripts/profile-tests.sh + +# Include per-line hotspots inside trbldoc.sh +sh scripts/profile-tests.sh -p + +# Profile the whole suite, show the 30 slowest tests +sh scripts/profile-tests.sh -s all -n 30 +``` + +Options: `-s all|unit|integration`, `-o OUTDIR`, `-n TOPN`, `-p` (hotspots), +`-t "file.sh ..."` (hotspot target scripts). + +## Output + +Written to `reports/profile-/` (or `-o`): + +- `timings.html` — ranked per-test bar chart. +- `timings.csv` / `timings.sorted.tsv` — raw per-test data. +- `hotspots.html` / `hotspots.csv` — per-line self-time (only with `-p`). +- `junit/`, `bats-output.txt` — raw bats artifacts. + +## Profiling real trbldoc runs (outside tests) + +Use `scripts/profile-run.sh` when you want hotspot data for real workloads +rather than test execution. + +```sh +# Single direct run +sh scripts/profile-run.sh -- -s src -o docs + +# Repeat 3 clean runs to stabilize hotspot ranking +sh scripts/profile-run.sh -n 3 -C -- -s src -o docs +``` + +Outputs under `reports/profile-run-/`: + +- `run-summary.csv` — wall-clock runtime per run. +- `hotspots.html` — per-line self-time aggregated from xtrace markers. +- `phase-hotspots.html` — coarse pipeline phase self-time (discovery, + extraction, render ordering, rendering, nav, ref resolution, toc, assemble), + inferred from traced line ranges in `trbldoc.sh`. + +## How the hotspot profiler works + +The bats tests invoke the tool as `bash "$TRBLDOC"`. The profiler exports +`BASH_ENV=scripts/lib/trace-init.sh`, which every non-interactive bash sources +at startup. For scripts whose basename is in `TRBLDOC_TRACE_TARGETS` +(default `trbldoc.sh`) it enables `set -x` with a `PS4` that stamps +`BASH_SOURCE:LINENO` and `EPOCHREALTIME` to a per-PID trace file. The +aggregator computes the delta between consecutive traced lines *within each +process*, so trbldoc's parallel `xargs -P` workers are not cross-attributed. + +## Limitations + +- Hotspot timing needs `bash` ≥ 5 (`EPOCHREALTIME`). The report scripts + themselves are POSIX sh and run under mingw bash, WSL bash and busybox ash. +- Subprocesses trbldoc spawns via `ash`/`xargs` are not bash and are not + traced; hotspots reflect the main-script control flow, not the render + helper's internals. +- Per-line self-time is derived from trace timestamps and includes tracing + overhead, so treat the numbers as relative, not absolute. +- Phase hotspots are inferred from source-line boundaries and are intended for + prioritisation, not exact accounting. diff --git a/docs/trblcache.md b/docs/trblcache.md new file mode 100644 index 0000000..7351fb0 --- /dev/null +++ b/docs/trblcache.md @@ -0,0 +1,83 @@ +# Cacheing + +Nearly all caching happens in `$(pwd)/.trblcache` by default (hereafter `$CACHE`), +here's a reminder of the process `trblcache` uses when building documentation: + +0. Scan: find all source files under the input folders whose extension has + a registered comment form (`c`, `h`, `cpp`, `hpp`, `lua`, `py`, `sql`, + `md`, `dot`, plus any `-f` additions). +0. Extract: run each file through an AWK program that emits chunk files + into `$CACHE/chunks/`. +0. Render: for each chunk, strip directives, run the renderer, and append + output to `$CACHE/namespace//index.html`. +0. Assemble: write `main.layout`, `nav.partial`, `global.meta`, and CSS + into `$CACHE/namespace/`. +0. Build: apply the layout/template tokens to each namespace page and + write the final site to `-o ` (default `$CACHE/built`). + +Of these, only chunk extraction can be cached. Cacheing works by first +creating a file under `$CACHE/last_processed/` when a file is processed, +and then on subsequent processing, ignore files if their `$CACHE/last_processed/` +file timestamp is newer than the `` timestamp. If your filesystem does +not support file timestamps, `trbldoc` will never assume the cache is invalid, +and will serve stale data. If you are on such a system, always run with `-C` to +force `trbldoc` to generate new chunks. + +## Notes + +### No cached chunks +The rendered chunks, generally, cannot be cached. If the user provided their +own tooling (i.e. bash scripts) for `trbldoc`, `trbldoc` cannot check for updates +to the scripts in order to invalidate the cached rendered chunk. Checking +file last updated times won't work, if a script invokes another script or relies +on any system environment the user's updates won't take. Example: + +``` +project +| src/ +| | file1.c - last updated 1d ago +| tools/ +| | checker.sh - last updated 1d ago +| | graph.sh - last updated 30s ago +``` + +file1.c + +```c +... +/* tools/checker.sh tools/graph.sh +strict graph { + a -- b + a -- b + b -- a [color=blue] +} +*/ +... +``` + +checker.sh + +```sh +#!/bin/sh + +if [ -z "$(command $1 -V)" ]; then + echo "Could not find $1" + exit -1 +fi + +exec $1 +``` + +graph.sh + +```sh +#!/bin/sh + +exec dot +``` + +It would be impossible for `trbldoc` to determine that `tools/checker.sh` uses +`tools/graph.sh` without forcing it to run in a hermetically sealed sandbox that +carefully controls what environment, filesystem, and other programs may be run. +Such heavy sandboxing is against the vision of `trbldoc` as a simple, single-file +documentation generator. diff --git a/readme.md b/readme.md new file mode 100644 index 0000000..29cd403 --- /dev/null +++ b/readme.md @@ -0,0 +1,275 @@ +# `trbldoc` + +A really awful documentation generator. + +Trbldoc scans source files for comments, extracts them as content chunks, +passes each chunk through a renderer, groups the results by namespace, +and assembles an HTML site. + +### Dependencies + +| Tool | Purpose | +|------|---------| +| Any POSIX shell (`ash`, `dash`, `sh`, `bash`, ...) | Runtime | +| `awk`, `sed` (with `-E`), `grep`, `find`, `mktemp`, `xargs` | Standard Unix tools | + +That's it. Really. Runs under busybox systems, no GNU required! + +## Installation + +Drop `trbldoc.sh` into your `$PATH` and mark it executable: + +```bash +sudo cp trbldoc.sh /usr/local/bin/trbldoc +sudo chmod +x /usr/local/bin/trbldoc +``` + +## Usage + +``` +trbldoc [-s -s ...] [-o ] [-l ] [-f ext;start;end] [-R prefix=script;opts] [-C] [source ...] +``` + +* `-s ` - Add a folder or file to scan. Repeatable. +* `-o ` - Output directory (default: `.trblcache/built`). +* `-l ` - Use an alternative `main.layout` template file instead of the built-in default. +* `-f ext;start;end` - Register a custom file extension with AWK-regex delimiters. +* `-R prefix=script;opts` - Set a resolver used for `@prefix/...` reference tags. +* `-C` - Force a clean rebuild — wipes cached chunks and timestamps before scanning. + +### Environment variables + +`trbldoc` allows you to use some environment variables to provide more information +while rendering. + +| Variable | Description | +|----------|-------------| +| `TRBLDOC_SRC_REPO` | Repository URL substituted for `{{ src_repo }}` in doc comments. Falls back to `git remote get-url origin` if not provided. | +| `TRBLDOC_MD` | Override the Markdown renderer used for `.md` files and `@exec md` chunks. Defaults to `md2html --github --fpermissive-autolinks --ftables`. | +| `TRBLDOC_RST` | Override the ReStructured Text renderer used for `.rst` files and `@exec rst` chunks. Defaults to `%(body)s`. | +| `TRBLDOC_JOBS` | Number of parallel jobs used for extraction and rendering. Defaults to the detected processor count. | + +### Quick start + + +Scan the `src/` folder and write output to `docs/` + +```bash +trbldoc -s src -o docs +``` + +Include the source repository URL in generated pages + +``` +TRBLDOC_SRC_REPO=https://github.com/you/repo trbldoc -s src -o docs +``` + +## Writing doc comments + +A doc comment is a block comment whose opening tag is followed _on the same +line_ by the renderer command to use for that chunk. `trbldoc` includes the +following markup format -> html mapping: + +| alias | program | +| --- | --- | +| `md` | `md2html` | +| `rst` | `docutils` | + +### C / C++ / SQL + +```c +/* md +# My Function + +Does something useful. +*/ +``` + +### Lua + +```lua +--[[ md +## Overview + +Brief description here. +]] +``` + +### Python + +```python +'''md +## Helper + +Processes the input. +''' +``` + +`trbldoc` lets you call _arbitrary bash_(**!**) from source code comments. +Some users may be alarmed, or believe this is a security vulnerability. +The reality at the time of publishing is that it is almost all +software documentation is generated on computers controlled by +the same person or people that wrote the source code, and published on the internet. +If you can trust someone to write some source code that's running on your machine, +you can probably trust them to run some more code to generate documentation. +If you are interested in letting third parties generate documentation +without reading the code, or you want to generate documentation from untrusted input, +`trbldoc` is not right for your project. + +With that said, allowing people to generate arbitrary html to explain their +code can be *extremely powerful*: + + +```lua +--[[ cat +
+---
+title: querying a resource
+---
+sequenceDiagram
+    participant db@{"type": "database"}
+    actor client
+    client ->> host : REQ resource 123
+    host ->> db : select * from resources where id = 123
+    db ->> host : 0 rows
+    host ->> client : 404 - File not Found
+
+]] +``` + +In addition, you can add new kinds of comment blocks, here's how you would +add comment blocks for [Nim](https://nim-lang.org/) + +```bash +$ trbldoc -s src -f 'nim;#[;#]' +``` + +If you wanted to cite the [Nim standard library](https://nim-lang.org/docs/lib.html) in your comments like + +```nim +#[ md +This function uses @nim/ropes under the hood... +#] +``` + +You need a way to turn `ropes` into the nim link, `https://nim-lang.org/docs/ropes.html`. +You might chose to use `sed`: + +```bash +$ trbldoc -s src -f 'nim;#[;#]' -R 'nim=sed;-e s|^|https://nim-lang.org/docs/|g -e s|$|.html|g' +``` + +As your prefix to link resolution gets more complex, you may chose to use a +simple bash program to generate links. You may also chose to call `trbldoc` +from a bash script or makefile. + +### Markdown (`.md`) + +The whole file is treated as a single chunk and rendered with `md`. + +--- + +Single-line comments (`/* ... */`, `--[[ ]]`, `''' '''`) on one line are +intentionally ignored, trbldoc only extracts multiline blocks. + +Comment blocks that do not specify a program on it's first line are likewise ignored. + +## In-chunk directives + +Directives are lines inside a doc comment that control how the chunk is +processed. They are stripped before rendering. + +| Directive | Effect | +|-----------|--------| +| `@name ` | Group this chunk under `` using slash-separated paths (e.g. `lua/string`). Defaults to the relative source file path. Use slashes (`/`) as the separator. | +| `@priority ` | Controls chunk ordering when rendering pages. Higher values are rendered first. Falls back to alphabetical `@name` ordering. | +| `@exec ` | Override the program used to turn text into html for this chunk. | +| `@file :` | Override the displayed source file attribution. Useful if the file seen by `trbldoc` has been built from some other source file. | +| `@ref ` | Publish this chunk under a custom reference name for cross-linking. (e.g. if you want to shorten a long name like `@interface/composites/menus/dropdown` to just `@ui/dropdown`) | + +## Template variables + +The following `{{ }}` markers are expanded in chunk bodies before the renderer +runs (or, for `toc`, in a post-render pass): + +| Variable | Expands to | +|----------|------------| +| `{{ src_repo }}` | Source repository URL (`TRBLDOC_SRC_REPO` env var or `git remote get-url origin`). | +| `{{ ref_string }}` | HTML link for this chunk's `@ref` value, resolved from the built-in references map. | +| `{{ toc }}` | Navigation list (`nav.partial` content) - substituted after all chunks are rendered. | +| `@prefix/path/to/resolve` | Create a link to another document in this or another project. | + +The default page layout (`main.layout`) also supports: + +| Variable | Expands to | +|----------|------------| +| `{{ title }}` | page title in ``. | +| `{{ breadcrumb }}` | heading text for the current page. | +| `{{ nav }}` | navigation list. | +| `{{ yield }}` | rendered page body. | + +Example: + +```c +/* md +@name http/api/public/connect +@ref db/connect + +Establishes a database connection. +*/ +``` + +## Pipeline overview + +`trbldoc` has several steps, in each step multiple actions are done in parallel, +as your `xargs` allows. + +1. Finds all source files under the input folders whose extension has + a registered comment form (`c`, `h`, `cpp`, `hpp`, `lua`, `py`, `sql`, + `md`, `dot`, plus any `-f` additions). +2. Run each file through an AWK program that emits chunk files + into `.trblcache/chunks/`. +3. For each chunk, strip directives, run the renderer, and append + output to `.trblcache/namespace/<namespace>/index.html`. +4. Write `main.layout`, `nav.partial`, `global.meta`, and CSS + into `.trblcache/namespace/`. +5. Apply the layout/template tokens to each namespace page and + write the final site to `-o <dest>` (default `.trblcache/built`). + +Chunk files in `.trblcache/chunks/` and last-processed timestamps in +`.trblcache/last_processed/` persist between runs. Only source files that have +changed since the last run are re-extracted. Use `-C` to force a full rebuild. +Rendered output (`index.html`, `nav.partial`, etc.) are always regenerated from +all chunks on every run. See [docs/trblcache](./docs/trblcache.md) +for why they have to be. + +## Tests + +The test suite uses [bats-core](https://github.com/bats-core/bats-core) ≥ 1.5. + +```bash +npm install -g bats # or: brew install bats-core +bats tests/unit tests/integration +``` + +Tests are organised as: + +- `tests/unit/extraction.bats` AWK comment extraction per language +- `tests/unit/directives.bats` directive parsing and stripping +- `tests/unit/template_vars.bats` template variable pattern matching +- `tests/integration/pipeline.bats` end-to-end pipeline assertions +- `tests/integration/cli.bats` CLI option parsing + +## Repository layout + +``` +trbldoc.sh Main pipeline script +ref_resolvers/lua.sh Reference Lua API prefix resolver +tests/ + helpers/ Shared test helpers, stubs, AWK program builders + common.bash Common test helpers + profiler.bash Helpers for profileing + fixtures/ Small files used as test inputs + unit/ Unit tests + integration/ Integration tests +``` diff --git a/ref_resolvers/lua.sh b/ref_resolvers/lua.sh new file mode 100644 index 0000000..ec65080 --- /dev/null +++ b/ref_resolvers/lua.sh @@ -0,0 +1,342 @@ +#!/bin/bash +# ref_resolvers/lua.sh +# Resolves a Lua manual section reference to its full URL. +# The reference name is read from stdin. +# Outputs the URL to stdout. +reference="$(cat)" +webroot="https://www.lua.org/manual/5.2/manual.html" + +case $(echo $reference | tr '[:upper:]' '[:lower:]') in + + "introduction") + printf "%s#%s" "$webroot" "1" + exit 0 + ;; + + "basic concepts") + printf "%s#%s" "$webroot" "2" + exit 0 + ;; + + "values and types") + printf "%s#%s" "$webroot" "2.1" + exit 0 + ;; + + "environments and the global environment") + printf "%s#%s" "$webroot" "2.2" + exit 0 + ;; + + "error handling") + printf "%s#%s" "$webroot" "2.3" + exit 0 + ;; + + "metatables and metamethods") + printf "%s#%s" "$webroot" "2.4" + exit 0 + ;; + + "garbage collection") + printf "%s#%s" "$webroot" "2.5" + exit 0 + ;; + + "garbage-collection metamethods") + printf "%s#%s" "$webroot" "2.5.1" + exit 0 + ;; + + "weak tables") + printf "%s#%s" "$webroot" "2.5.2" + exit 0 + ;; + + "coroutines") + printf "%s#%s" "$webroot" "2.6" + exit 0 + ;; + + "the language") + printf "%s#%s" "$webroot" "3" + exit 0 + ;; + + "lexical conventions") + printf "%s#%s" "$webroot" "3.1" + exit 0 + ;; + + "variables") + printf "%s#%s" "$webroot" "3.2" + exit 0 + ;; + + "statements") + printf "%s#%s" "$webroot" "3.3" + exit 0 + ;; + + "blocks") + printf "%s#%s" "$webroot" "3.3.1" + exit 0 + ;; + + "chunks") + printf "%s#%s" "$webroot" "3.3.2" + exit 0 + ;; + + "assignment") + printf "%s#%s" "$webroot" "3.3.3" + exit 0 + ;; + + "control structures") + printf "%s#%s" "$webroot" "3.3.4" + exit 0 + ;; + + "for statement") + printf "%s#%s" "$webroot" "3.3.5" + exit 0 + ;; + + "function calls as statements") + printf "%s#%s" "$webroot" "3.3.6" + exit 0 + ;; + + "local declarations") + printf "%s#%s" "$webroot" "3.3.7" + exit 0 + ;; + + "expressions") + printf "%s#%s" "$webroot" "3.4" + exit 0 + ;; + + "arithmetic operators") + printf "%s#%s" "$webroot" "3.4.1" + exit 0 + ;; + + "coercion") + printf "%s#%s" "$webroot" "3.4.2" + exit 0 + ;; + + "relational operators") + printf "%s#%s" "$webroot" "3.4.3" + exit 0 + ;; + + "logical operators") + printf "%s#%s" "$webroot" "3.4.4" + exit 0 + ;; + + "concatenation") + printf "%s#%s" "$webroot" "3.4.5" + exit 0 + ;; + + "the length operator") + printf "%s#%s" "$webroot" "3.4.6" + exit 0 + ;; + + "precedence") + printf "%s#%s" "$webroot" "3.4.7" + exit 0 + ;; + + "table constructors") + printf "%s#%s" "$webroot" "3.4.8" + exit 0 + ;; + + "function calls") + printf "%s#%s" "$webroot" "3.4.9" + exit 0 + ;; + + "function definitions") + printf "%s#%s" "$webroot" "3.4.10" + exit 0 + ;; + + "visibility rules") + printf "%s#%s" "$webroot" "3.5" + exit 0 + ;; + + "the application program interface") + printf "%s#%s" "$webroot" "4" + exit 0 + ;; + + "the stack") + printf "%s#%s" "$webroot" "4.1" + exit 0 + ;; + + "stack size") + printf "%s#%s" "$webroot" "4.2" + exit 0 + ;; + + "valid and acceptable indices") + printf "%s#%s" "$webroot" "4.3" + exit 0 + ;; + + "c closures") + printf "%s#%s" "$webroot" "4.4" + exit 0 + ;; + + "registry") + printf "%s#%s" "$webroot" "4.5" + exit 0 + ;; + + "error handling in c") + printf "%s#%s" "$webroot" "4.6" + exit 0 + ;; + + "handling yields in c") + printf "%s#%s" "$webroot" "4.7" + exit 0 + ;; + + "functions and types") + # Matches section 4.8 (C API). Section 5.1 (Auxiliary Library) has the + # same title; this entry resolves to the first occurrence. + printf "%s#%s" "$webroot" "4.8" + exit 0 + ;; + + "the debug interface") + printf "%s#%s" "$webroot" "4.9" + exit 0 + ;; + + "the auxiliary library") + printf "%s#%s" "$webroot" "5" + exit 0 + ;; + + "standard libraries") + printf "%s#%s" "$webroot" "6" + exit 0 + ;; + + "basic functions") + printf "%s#%s" "$webroot" "6.1" + exit 0 + ;; + + "coroutine manipulation") + printf "%s#%s" "$webroot" "6.2" + exit 0 + ;; + + "modules") + printf "%s#%s" "$webroot" "6.3" + exit 0 + ;; + + "string manipulation") + printf "%s#%s" "$webroot" "6.4" + exit 0 + ;; + + "patterns") + printf "%s#%s" "$webroot" "6.4.1" + exit 0 + ;; + + "table manipulation") + printf "%s#%s" "$webroot" "6.5" + exit 0 + ;; + + "mathematical functions") + printf "%s#%s" "$webroot" "6.6" + exit 0 + ;; + + "bitwise operations") + printf "%s#%s" "$webroot" "6.7" + exit 0 + ;; + + "input and output facilities") + printf "%s#%s" "$webroot" "6.8" + exit 0 + ;; + + "operating system facilities") + printf "%s#%s" "$webroot" "6.9" + exit 0 + ;; + + "the debug library") + printf "%s#%s" "$webroot" "6.10" + exit 0 + ;; + + "lua standalone") + printf "%s#%s" "$webroot" "7" + exit 0 + ;; + + "incompatibilities with the previous version") + printf "%s#%s" "$webroot" "8" + exit 0 + ;; + + "changes in the language") + printf "%s#%s" "$webroot" "8.1" + exit 0 + ;; + + "changes in the libraries") + printf "%s#%s" "$webroot" "8.2" + exit 0 + ;; + + "changes in the api") + printf "%s#%s" "$webroot" "8.3" + exit 0 + ;; + + "the complete syntax of lua") + printf "%s#%s" "$webroot" "9" + exit 0 + ;; + + # Lua type names — resolve to the Values and Types section (2.1) + "nil" \ + | "boolean" \ + | "number" \ + | "string" \ + | "function" \ + | "userdata" \ + | "thread" \ + | "table") + printf "%s#%s" "$webroot" "2.1" + exit 0 + ;; + + # Alternative lowercase spelling for coroutines + "coroutines") + printf "%s#%s" "$webroot" "2.6" + exit 0 + ;; + +esac diff --git a/scripts/README.md b/scripts/README.md new file mode 100644 index 0000000..91adbd3 --- /dev/null +++ b/scripts/README.md @@ -0,0 +1,50 @@ +# Report scripts + +Portable POSIX-sh tooling to generate reports from the bats test suite. All +entrypoints run under mingw bash, WSL bash and Alpine/busybox ash, and depend +only on `bats`, `awk`, `sort` and coreutils. + +## `profile-tests.sh` + +Per-test timing (and optional per-line hotspots). See `doc/profiling.md`. +## `profile-run.sh` + +Direct profiling for real `trbldoc.sh` invocations (outside bats). Produces: + +- per-run wall-clock summary (`run-summary.csv`) +- line hotspots (`hotspots.html`) +- pipeline phase hotspots inferred from trace line ranges (`phase-hotspots.html`) + +Example: + +```sh +sh scripts/profile-run.sh -n 3 -C -- -s tests/fixtures/pipeline -o .trblcache/built +``` +## `hotspot-tests.sh` + +Repeatable hotspot analysis for the bats suite without relying on JUnit output. +It captures: + +- per-test timing from raw `bats --timing` output +- line hotspots from xtrace logs +- optional phase hotspots for `trbldoc.sh` (when helper scripts are present) + +```sh +sh scripts/hotspot-tests.sh -s all -n 30 +``` + +## `coverage-tests.sh` + +Shell line coverage of `trbldoc.sh`. See `doc/coverage.md`. + +## `generate-reports.sh` + +Run both into one timestamped `reports/` directory. Puts reports into a +`reports/run-<timestamp>/` directory. + +```sh +sh scripts/generate-reports.sh -p +``` + +`lib/` holds shared helpers: `common.sh` (sh helpers), `trace-init.sh` (the +`BASH_ENV` xtrace hook), and the `*.awk` extractors/renderers. diff --git a/scripts/coverage-tests.sh b/scripts/coverage-tests.sh new file mode 100644 index 0000000..cf03825 --- /dev/null +++ b/scripts/coverage-tests.sh @@ -0,0 +1,143 @@ +#!/bin/sh +# scripts/coverage-tests.sh +# Generate line-coverage reports for trbldoc's shell sources from the test +# suite. Answers "what is the test suite not exercising". +# +# Mechanism: the suite runs the tool via `bash "$TRBLDOC"`, so exporting +# BASH_ENV=scripts/lib/trace-init.sh makes each target process emit an xtrace +# of every executed line. We union the executed lines across the whole run and +# compare against the executable lines in each target source. No modification +# of trbldoc.sh or the tests is required. +# +# POSIX sh; works under mingw bash, WSL bash and Alpine/busybox ash. The traced +# run itself uses bash (the suite already invokes bash), which is one of the +# supported target shells. +# +# Usage: +# scripts/coverage-tests.sh [-s all|unit|integration] [-o OUTDIR] +# [-t "file.sh ..."] [-m MIN_PERCENT] +# +# -s SUITE suite to run (default: integration) +# -o OUTDIR output directory (default: reports/coverage-<timestamp>) +# -t LIST space-separated target source files (default: trbldoc.sh) +# -m MIN_PERCENT fail (exit 2) if overall coverage is below this (default: 0) + +set -u + +SELF="$0" +. "$(dirname -- "$0")/lib/common.sh" +tb_resolve_paths "$SELF" + +SUITE=integration +OUTDIR="" +TARGETS="trbldoc.sh" +MIN_PERCENT=0 + +while getopts ":s:o:t:m:h" opt; do + case "$opt" in + s) SUITE="$OPTARG" ;; + o) OUTDIR="$OPTARG" ;; + t) TARGETS="$OPTARG" ;; + m) MIN_PERCENT="$OPTARG" ;; + h) sed -n '2,30p' "$SELF"; exit 0 ;; + :) tb_die "option -$OPTARG requires an argument" ;; + \?) tb_die "unknown option -$OPTARG" ;; + esac +done + +tb_require awk +tb_require sort + +[ -n "$OUTDIR" ] || OUTDIR="$REPO_ROOT/reports/coverage-$(tb_timestamp)" +case "$OUTDIR" in + /*|[A-Za-z]:[\\/]*) + ;; + *) + OUTDIR="$REPO_ROOT/$OUTDIR" + ;; +esac +mkdir -p "$OUTDIR/html" || tb_die "cannot create output dir: $OUTDIR" +JUNIT_DIR="$OUTDIR/junit"; mkdir -p "$JUNIT_DIR" +TRACE_DIR="$OUTDIR/trace"; rm -rf "$TRACE_DIR"; mkdir -p "$TRACE_DIR" + +SUITE_DIRS="$(tb_suite_dirs "$SUITE")" +SUITE_DIRS_REL="$(tb_suite_dirs_rel "$SUITE")" + +# Basename filter for the trace hook + absolute source paths for reporting. +TRACE_TARGETS="" +SRC_FILES="" +for tf in $TARGETS; do + TRACE_TARGETS="$TRACE_TARGETS ${tf##*/}" + case "$tf" in + /*) p="$tf" ;; + *) p="$REPO_ROOT/$tf" ;; + esac + [ -f "$p" ] || tb_die "target source not found: $p" + SRC_FILES="$SRC_FILES $p" +done + +# ── run suite under xtrace ─────────────────────────────────────────────────── +tb_info "running bats ($SUITE) under coverage instrumentation..." +if tb_bats_usable "$SUITE"; then + BASH_ENV="$LIB_DIR/trace-init.sh" + TRBLDOC_TRACE_DIR="$TRACE_DIR" + TRBLDOC_TRACE_MODE="cov" + TRBLDOC_TRACE_TARGETS="$TRACE_TARGETS" + export BASH_ENV TRBLDOC_TRACE_DIR TRBLDOC_TRACE_MODE TRBLDOC_TRACE_TARGETS + # shellcheck disable=SC2086 + bats --report-formatter junit --output "$JUNIT_DIR" $SUITE_DIRS \ + > "$OUTDIR/bats-output.txt" 2>&1 || tb_info "bats reported failures (see bats-output.txt)" + unset BASH_ENV TRBLDOC_TRACE_DIR TRBLDOC_TRACE_MODE TRBLDOC_TRACE_TARGETS +elif command -v wsl >/dev/null 2>&1; then + tb_info "local bats appears unusable; retrying via WSL bats..." + WSL_REPO="$(tb_to_wsl_path "$REPO_ROOT")" + WSL_LIB="$(tb_to_wsl_path "$LIB_DIR")" + WSL_TRACE="$(tb_to_wsl_path "$TRACE_DIR")" + WSL_JUNIT="$(tb_to_wsl_path "$JUNIT_DIR")" + wsl -- bash -lc \ + "cd '$WSL_REPO' && export BASH_ENV='$WSL_LIB/trace-init.sh' TRBLDOC_TRACE_DIR='$WSL_TRACE' TRBLDOC_TRACE_MODE='cov' TRBLDOC_TRACE_TARGETS='$TRACE_TARGETS'; bats --report-formatter junit --output '$WSL_JUNIT' $SUITE_DIRS_REL" \ + > "$OUTDIR/bats-output.txt" 2>&1 || tb_info "bats (WSL) reported failures (see bats-output.txt)" +else + tb_die "local bats command appears unusable and no WSL fallback is available" +fi + +if ! ls "$TRACE_DIR"/trace.*.log >/dev/null 2>&1; then + if grep -qi "bash\\\\r" "$OUTDIR/bats-output.txt" 2>/dev/null; then + tb_die "bats appears broken (CRLF shebang: 'bash\\r'). Reinstall bats or run dos2unix on the bats scripts, then retry." + fi + tb_die "no trace data captured; ensure bats actually runs tests (see $OUTDIR/bats-output.txt)" +fi + +# ── union executed lines ───────────────────────────────────────────────────── +awk -f "$LIB_DIR/trace-extract.awk" -v MODE=cov "$TRACE_DIR"/trace.*.log \ + | sort -u > "$OUTDIR/covered.tsv" + +# ── per-file coverage report + summary ─────────────────────────────────────── +# shellcheck disable=SC2086 +awk -f "$LIB_DIR/coverage-report.awk" \ + -v COVERED="$OUTDIR/covered.tsv" \ + -v HTMLDIR="$OUTDIR/html" \ + $SRC_FILES > "$OUTDIR/summary.rows" + +{ + printf 'file,executable,covered,percent\n' + cat "$OUTDIR/summary.rows" +} > "$OUTDIR/summary.csv" + +awk -f "$LIB_DIR/render-coverage-index.awk" \ + -v TITLE="trbldoc coverage ($SUITE)" \ + "$OUTDIR/summary.rows" > "$OUTDIR/index.html" + +# ── overall percent + threshold gate ───────────────────────────────────────── +OVERALL="$(awk -F',' '{t+=$2; c+=$3} END{ if(t>0) printf "%.1f", c*100.0/t; else print "0.0" }' "$OUTDIR/summary.rows")" + +printf '\n==== coverage summary (%s) ====\n' "$SUITE" +awk -F',' '{ printf "%6.1f%% %-20s (%d/%d)\n", $4, $1, $3, $2 }' "$OUTDIR/summary.rows" +printf ' overall: %s%%\n' "$OVERALL" +printf '\nReports written to: %s\n' "$OUTDIR" +printf ' coverage index: %s\n' "$OUTDIR/index.html" + +# Threshold enforcement (integer or decimal compare via awk). +if awk -v o="$OVERALL" -v m="$MIN_PERCENT" 'BEGIN{ exit !(m>0 && o+0 < m+0) }'; then + tb_die "coverage ${OVERALL}% is below the required minimum ${MIN_PERCENT}%" +fi diff --git a/scripts/generate-reports.sh b/scripts/generate-reports.sh new file mode 100644 index 0000000..77c4b0f --- /dev/null +++ b/scripts/generate-reports.sh @@ -0,0 +1,58 @@ +#!/bin/sh +# scripts/generate-reports.sh +# Convenience wrapper: run both the profiler and the coverage report for a +# suite into a single timestamped directory under reports/. +# +# POSIX sh; works under mingw bash, WSL bash and Alpine/busybox ash. +# +# Usage: +# scripts/generate-reports.sh [-s all|unit|integration] [-o OUTDIR] +# [-p] [-m MIN_PERCENT] [-t "file.sh ..."] +# +# -s SUITE suite to run (default: integration) +# -o OUTDIR base output directory (default: reports/run-<timestamp>) +# -p include per-line hotspots in the profile report +# -m MIN_PERCENT coverage threshold to enforce (default: 0 = no gate) +# -t LIST space-separated target source files (default: trbldoc.sh) + +set -u + +SELF="$0" +. "$(dirname -- "$0")/lib/common.sh" +tb_resolve_paths "$SELF" + +SUITE=integration +OUTDIR="" +HOTSPOTS=0 +MIN_PERCENT=0 +TARGETS="trbldoc.sh" + +while getopts ":s:o:pm:t:h" opt; do + case "$opt" in + s) SUITE="$OPTARG" ;; + o) OUTDIR="$OPTARG" ;; + p) HOTSPOTS=1 ;; + m) MIN_PERCENT="$OPTARG" ;; + t) TARGETS="$OPTARG" ;; + h) sed -n '2,20p' "$SELF"; exit 0 ;; + :) tb_die "option -$OPTARG requires an argument" ;; + \?) tb_die "unknown option -$OPTARG" ;; + esac +done + +[ -n "$OUTDIR" ] || OUTDIR="$REPO_ROOT/reports/run-$(tb_timestamp)" +mkdir -p "$OUTDIR" || tb_die "cannot create output dir: $OUTDIR" + +PROFILE_ARGS="-s $SUITE -o $OUTDIR/profile -t $TARGETS" +[ "$HOTSPOTS" -eq 1 ] && PROFILE_ARGS="$PROFILE_ARGS -p" + +tb_info "=== profiling ===" +# shellcheck disable=SC2086 +sh "$SCRIPTS_DIR/profile-tests.sh" $PROFILE_ARGS + +tb_info "=== coverage ===" +# shellcheck disable=SC2086 +sh "$SCRIPTS_DIR/coverage-tests.sh" -s "$SUITE" -o "$OUTDIR/coverage" \ + -t "$TARGETS" -m "$MIN_PERCENT" + +printf '\nAll reports under: %s\n' "$OUTDIR" diff --git a/scripts/hotspot-tests.sh b/scripts/hotspot-tests.sh new file mode 100644 index 0000000..8973e6b --- /dev/null +++ b/scripts/hotspot-tests.sh @@ -0,0 +1,204 @@ +#!/bin/sh +# scripts/hotspot-tests.sh +# Repeatable hotspot analysis for the trbldoc test suite. +# +# Produces: +# - per-test timings (parsed from `bats --timing`) +# - line-level hotspots for target scripts from xtrace logs +# - optional phase hotspots for trbldoc.sh +# +# Usage: +# scripts/hotspot-tests.sh [-s all|unit|integration] [-o OUTDIR] +# [-n TOPN] [-t "file.sh ..."] +# +# -s SUITE suite to run (default: all) +# -o OUTDIR output directory (default: reports/hotspots-<timestamp>) +# -n TOPN number of slowest tests to print (default: 25) +# -t LIST space-separated target scripts for trace hotspots (default: trbldoc.sh) + +set -u + +SELF="$0" +. "$(dirname -- "$0")/lib/common.sh" +tb_resolve_paths "$SELF" + +SUITE=all +OUTDIR="" +TOPN=25 +TARGETS="trbldoc.sh" + +while getopts ":s:o:n:t:h" opt; do + case "$opt" in + s) SUITE="$OPTARG" ;; + o) OUTDIR="$OPTARG" ;; + n) TOPN="$OPTARG" ;; + t) TARGETS="$OPTARG" ;; + h) sed -n '2,24p' "$SELF"; exit 0 ;; + :) tb_die "option -$OPTARG requires an argument" ;; + \?) tb_die "unknown option -$OPTARG" ;; + esac +done + +tb_require awk +tb_require sort + +case "$TOPN" in + ''|*[!0-9]*|0) tb_die "-n TOPN must be a positive integer" ;; +esac + +[ -n "$OUTDIR" ] || OUTDIR="$REPO_ROOT/reports/hotspots-$(tb_timestamp)" +case "$OUTDIR" in + /*|[A-Za-z]:[\\/]*) ;; + *) OUTDIR="$REPO_ROOT/$OUTDIR" ;; +esac +mkdir -p "$OUTDIR" || tb_die "cannot create output dir: $OUTDIR" + +SUITE_DIRS="$(tb_suite_dirs "$SUITE")" +SUITE_DIRS_REL="$(tb_suite_dirs_rel "$SUITE")" +TIMING_RAW="$OUTDIR/bats-timing-output.txt" +TRACE_DIR="$OUTDIR/trace" +rm -rf "$TRACE_DIR"; mkdir -p "$TRACE_DIR" + +# Build trace target filter and resolve first target source for HTML context. +TRACE_TARGETS="" +FIRST_TARGET="" +for tf in $TARGETS; do + TRACE_TARGETS="$TRACE_TARGETS ${tf##*/}" + if [ -z "$FIRST_TARGET" ]; then + FIRST_TARGET="$tf" + fi +done + +tb_info "running bats ($SUITE) with --timing output..." +if tb_bats_usable "$SUITE"; then + # shellcheck disable=SC2086 + bats --timing $SUITE_DIRS > "$TIMING_RAW" 2>&1 || tb_info "bats reported failures (timing output still collected)" + BATS_MODE="local" +elif command -v wsl >/dev/null 2>&1; then + tb_info "local bats appears unusable; retrying timing run via WSL bats..." + WSL_REPO="$(tb_to_wsl_path "$REPO_ROOT")" + wsl -- bash -lc \ + "cd '$WSL_REPO' && bats --timing $SUITE_DIRS_REL" \ + > "$TIMING_RAW" 2>&1 || tb_info "bats (WSL) reported failures (timing output still collected)" + BATS_MODE="wsl" +else + tb_die "local bats command appears unusable and no WSL fallback is available" +fi + +# Parse bats timing lines: "ok N <name> in <ms>ms" +awk ' +match($0, /^ok [0-9]+ (.*) in ([0-9]+)ms$/, m) { + name = m[1] + ms = m[2] + 0 + suite = name + sub(/:.*/, "", suite) + printf "%.3f\t%s\t%s\t%d\n", ms / 1000.0, suite, name, ms +} +' "$TIMING_RAW" > "$OUTDIR/timings.tsv" + +if [ ! -s "$OUTDIR/timings.tsv" ]; then + tb_die "no per-test timing rows parsed (check $TIMING_RAW)" +fi + +sort -t "$(printf '\t')" -k1,1nr "$OUTDIR/timings.tsv" > "$OUTDIR/timings.sorted.tsv" +{ + printf 'seconds,suite,test,milliseconds\n' + awk -F '\t' '{ gsub(/"/,"\"\"",$3); printf "%s,%s,\"%s\",%s\n",$1,$2,$3,$4 }' "$OUTDIR/timings.sorted.tsv" +} > "$OUTDIR/timings.csv" + +awk -f "$LIB_DIR/render-timing.awk" \ + -v TITLE="trbldoc test timings ($SUITE)" \ + "$OUTDIR/timings.sorted.tsv" > "$OUTDIR/timings.html" + +tb_info "running bats ($SUITE) under xtrace timing hook..." +if [ "$BATS_MODE" = "local" ]; then + BASH_ENV="$LIB_DIR/trace-init.sh" + TRBLDOC_TRACE_DIR="$TRACE_DIR" + TRBLDOC_TRACE_MODE="time" + TRBLDOC_TRACE_TARGETS="$TRACE_TARGETS" + export BASH_ENV TRBLDOC_TRACE_DIR TRBLDOC_TRACE_MODE TRBLDOC_TRACE_TARGETS + # shellcheck disable=SC2086 + bats $SUITE_DIRS > "$OUTDIR/bats-trace-output.txt" 2>&1 || tb_info "bats reported failures during trace run" + unset BASH_ENV TRBLDOC_TRACE_DIR TRBLDOC_TRACE_MODE TRBLDOC_TRACE_TARGETS +else + WSL_REPO="$(tb_to_wsl_path "$REPO_ROOT")" + WSL_LIB="$(tb_to_wsl_path "$LIB_DIR")" + WSL_TRACE="$(tb_to_wsl_path "$TRACE_DIR")" + wsl -- bash -lc \ + "cd '$WSL_REPO' && export BASH_ENV='$WSL_LIB/trace-init.sh' TRBLDOC_TRACE_DIR='$WSL_TRACE' TRBLDOC_TRACE_MODE='time' TRBLDOC_TRACE_TARGETS='$TRACE_TARGETS'; bats $SUITE_DIRS_REL" \ + > "$OUTDIR/bats-trace-output.txt" 2>&1 || tb_info "bats (WSL) reported failures during trace run" +fi + +if ls "$TRACE_DIR"/trace.*.log >/dev/null 2>&1; then + awk -f "$LIB_DIR/trace-extract.awk" -v MODE=time "$TRACE_DIR"/trace.*.log \ + | sort -t "$(printf '\t')" -k1 -rn > "$OUTDIR/hotspots.tsv" + { + printf 'total_ms,hits,file,line\n' + awk -F '\t' '{ printf "%s,%s,%s,%s\n",$1,$2,$3,$4 }' "$OUTDIR/hotspots.tsv" + } > "$OUTDIR/hotspots.csv" + + case "$FIRST_TARGET" in + /*|[A-Za-z]:[\\/]*) SRC_PATH="$FIRST_TARGET" ;; + *) SRC_PATH="$REPO_ROOT/$FIRST_TARGET" ;; + esac + awk -f "$LIB_DIR/render-hotspots.awk" \ + -v TITLE="trbldoc line hotspots ($SUITE)" \ + -v SRC="$SRC_PATH" \ + "$OUTDIR/hotspots.tsv" > "$OUTDIR/hotspots.html" +else + tb_info "no trace logs captured for target(s):$TRACE_TARGETS" +fi + +# Optional phase hotspots when trbldoc.sh + helper scripts are available. +if [ -f "$OUTDIR/hotspots.tsv" ] && [ -f "$LIB_DIR/trace-phase-extract.awk" ] && [ -f "$LIB_DIR/render-phases.awk" ] && [ -f "$REPO_ROOT/trbldoc.sh" ]; then + _find_line() { + awk -v p="$1" 'index($0, p) { print NR; exit }' "$REPO_ROOT/trbldoc.sh" + } + L_DISCOVER_END="$(_find_line '> "$source_files_list"')" + L_EXTRACT_END="$(_find_line '# Resolve src_repo once before the chunk loop.')" + L_ORDER_END="$(_find_line 'done < "$sorted_chunk_order_file"')" + L_RENDER_END="$(_find_line 'done < "$chunk_render_records_file"')" + L_NAV_END="$(_find_line '# Second pass: resolve inline references now that global.meta is complete.')" + L_REFS_END="$(_find_line 'rm -f "$_ref_find_tmp"')" + L_TOC_END="$(_find_line '# Post-processing pass: substitute {{ toc }} in all rendered index.html files')" + L_ASSEMBLE_END="$(_find_line 'assemble_site "$cache_dir/namespace" "$out_dir"')" + + if [ -n "$L_DISCOVER_END" ] && [ -n "$L_EXTRACT_END" ] && [ -n "$L_ORDER_END" ] \ + && [ -n "$L_RENDER_END" ] && [ -n "$L_NAV_END" ] && [ -n "$L_REFS_END" ] \ + && [ -n "$L_TOC_END" ] && [ -n "$L_ASSEMBLE_END" ]; then + awk -f "$LIB_DIR/trace-phase-extract.awk" \ + -v SRC_BASE="trbldoc.sh" \ + -v L_DISCOVER_END="$L_DISCOVER_END" \ + -v L_EXTRACT_END="$L_EXTRACT_END" \ + -v L_ORDER_END="$L_ORDER_END" \ + -v L_RENDER_END="$L_RENDER_END" \ + -v L_NAV_END="$L_NAV_END" \ + -v L_REFS_END="$L_REFS_END" \ + -v L_TOC_END="$L_TOC_END" \ + -v L_ASSEMBLE_END="$L_ASSEMBLE_END" \ + "$TRACE_DIR"/trace.*.log \ + | sort -t "$(printf '\t')" -k1 -rn > "$OUTDIR/phase-hotspots.tsv" + + { + printf 'total_ms,hits,phase\n' + awk -F '\t' '{ printf "%s,%s,%s\n",$1,$2,$3 }' "$OUTDIR/phase-hotspots.tsv" + } > "$OUTDIR/phase-hotspots.csv" + + awk -f "$LIB_DIR/render-phases.awk" \ + -v TITLE="trbldoc phase hotspots ($SUITE)" \ + "$OUTDIR/phase-hotspots.tsv" > "$OUTDIR/phase-hotspots.html" + fi +fi + +printf '\n==== slowest %s tests ====\n' "$TOPN" +awk -F '\t' -v n="$TOPN" 'NR<=n { printf "%8.3fs %s\n", $1, $3 }' "$OUTDIR/timings.sorted.tsv" +printf '\nReports written to: %s\n' "$OUTDIR" +printf ' per-test timing : %s\n' "$OUTDIR/timings.html" +if [ -f "$OUTDIR/hotspots.html" ]; then + printf ' line hotspots : %s\n' "$OUTDIR/hotspots.html" +else + printf ' line hotspots : not generated (target not exercised)\n' +fi +if [ -f "$OUTDIR/phase-hotspots.html" ]; then + printf ' phase hotspots : %s\n' "$OUTDIR/phase-hotspots.html" +fi diff --git a/scripts/lib/common.sh b/scripts/lib/common.sh new file mode 100644 index 0000000..0fdf98a --- /dev/null +++ b/scripts/lib/common.sh @@ -0,0 +1,105 @@ +# scripts/lib/common.sh +# Shared helpers for the trbldoc report scripts. +# +# POSIX sh compatible. Verified to work under mingw bash, WSL bash and +# Alpine/busybox ash. Sourced by the entrypoint scripts; do not execute +# directly. + +# Resolve the repository root from the location of the scripts/ directory. +# Sets: REPO_ROOT, SCRIPTS_DIR, LIB_DIR +tb_resolve_paths() { + # $1: path to the calling script ($0) + script_path="$1" + # Directory containing the calling script (scripts/). + SCRIPTS_DIR="$(CDPATH= cd -- "$(dirname -- "$script_path")" && pwd)" + LIB_DIR="$SCRIPTS_DIR/lib" + REPO_ROOT="$(CDPATH= cd -- "$SCRIPTS_DIR/.." && pwd)" + export REPO_ROOT SCRIPTS_DIR LIB_DIR +} + +# Print an ISO-8601 UTC timestamp suitable for directory names. +# Falls back to epoch seconds when the platform date(1) cannot format UTC. +tb_timestamp() { + if date -u +%Y%m%dT%H%M%SZ 2>/dev/null; then + return 0 + fi + printf '%s\n' "$(date +%s)" +} + +# Emit an informational message to stderr. +tb_info() { + printf '[trbldoc-report] %s\n' "$*" >&2 +} + +# Emit an error message to stderr and return non-zero. +tb_die() { + printf '[trbldoc-report] ERROR: %s\n' "$*" >&2 + exit 1 +} + +# Verify a command exists on PATH, otherwise abort. +tb_require() { + command -v "$1" >/dev/null 2>&1 || tb_die "required tool not found on PATH: $1" +} + +# Resolve the bats suite directories for a target keyword. +# $1: all|unit|integration ; echoes space-separated dirs relative to REPO_ROOT. +tb_suite_dirs() { + case "$1" in + unit) printf '%s\n' "$REPO_ROOT/tests/unit" ;; + integration) printf '%s\n' "$REPO_ROOT/tests/integration" ;; + all) printf '%s\n' "$REPO_ROOT/tests/unit $REPO_ROOT/tests/integration" ;; + *) tb_die "unknown suite '$1' (expected: all|unit|integration)" ;; + esac +} + +# Resolve bats suite directories relative to repository root. +# $1: all|unit|integration ; echoes a space-separated relative path list. +tb_suite_dirs_rel() { + case "$1" in + unit) printf '%s\n' "tests/unit" ;; + integration) printf '%s\n' "tests/integration" ;; + all) printf '%s\n' "tests/unit tests/integration" ;; + *) tb_die "unknown suite '$1' (expected: all|unit|integration)" ;; + esac +} + +# Convert a POSIX-style MSYS path (/d/foo/bar) to a WSL Linux path +# (/mnt/d/foo/bar). Other paths are returned unchanged. +tb_to_wsl_path() { + p="$1" + case "$p" in + [A-Za-z]:\\*|[A-Za-z]:/*) + d="$(printf '%s' "$p" | cut -c1 | tr 'A-Z' 'a-z')" + rest="$(printf '%s' "$p" | cut -c3- | sed 's#\\\\#/#g')" + rest="$(printf '%s' "$rest" | sed 's#^/*##')" + printf '/mnt/%s/%s\n' "$d" "$rest" + ;; + /[A-Za-z]/*) + d="$(printf '%s' "$p" | cut -c2 | tr 'A-Z' 'a-z')" + rest="$(printf '%s' "$p" | cut -c4-)" + printf '/mnt/%s/%s\n' "$d" "$rest" + ;; + *) + printf '%s\n' "$p" + ;; + esac +} + +# Returns success if the local `bats` command appears usable for this repo. +# Some mixed Windows/MSYS setups expose a broken bats wrapper that exits 0 +# without running tests or producing output; this guard catches that case. +tb_bats_usable() { + suite="${1:-integration}" + dirs="$(tb_suite_dirs "$suite")" + # shellcheck disable=SC2086 + count="$(bats --count $dirs 2>/dev/null | tr -d '[:space:]')" + case "$count" in + ''|*[!0-9]*) + return 1 + ;; + *) + return 0 + ;; + esac +} diff --git a/scripts/lib/coverage-report.awk b/scripts/lib/coverage-report.awk new file mode 100644 index 0000000..982eb45 --- /dev/null +++ b/scripts/lib/coverage-report.awk @@ -0,0 +1,84 @@ +# scripts/lib/coverage-report.awk +# Produce line-coverage reports for shell sources from an executed-line set. +# +# Inputs: +# -v COVERED=<file> TSV of "<base>\t<line>" executed lines (sorted -u). +# -v HTMLDIR=<dir> directory to write per-file <base>.html reports into. +# ARGV the source files to report on (absolute paths). +# +# Output (stdout): one CSV row per source file: +# file,total_executable,covered,percent +# +# "Executable" lines are determined heuristically: non-blank, non-comment +# lines that are not pure structural tokens (then/do/done/else/fi/esac/{/}). +# Heredoc/multiline string bodies are approximated and may read as missed; +# see docs/coverage.md for the documented limitations. + +function trim(s) { sub(/^[ \t]+/, "", s); sub(/[ \t]+$/, "", s); return s } + +function is_exec(line, t) { + t = trim(line) + if (t == "") return 0 + if (t ~ /^#/) return 0 + if (t ~ /^(then|do|done|else|fi|esac)([ \t].*)?$/) return 0 + if (t ~ /^[{}]([ \t;].*)?$/) return 0 + if (t ~ /^[()]([ \t].*)?$/) return 0 + if (t ~ /^;;/) return 0 + return 1 +} + +function esc(s) { + gsub(/&/, "\\&", s) + gsub(/</, "\\<", s) + gsub(/>/, "\\>", s) + return s +} + +function basename(p) { sub(/.*\//, "", p); sub(/.*\\/, "", p); return p } + +BEGIN { + FS = "\t" + # Load the executed-line set. + while ((getline ln < COVERED) > 0) { + nf = split(ln, a, "\t") + if (nf >= 2) covered[a[1] SUBSEP a[2]] = 1 + } + close(COVERED) +} + +# New source file: open its HTML report and reset counters. +FNR == 1 { + if (cur != "") finish_file() + cur = FILENAME + curbase = basename(FILENAME) + total = 0; hit = 0 + html = HTMLDIR "/" curbase ".html" + print "<!DOCTYPE html><html lang=\"en\"><head><meta charset=\"utf-8\">" > html + print "<title>coverage: " esc(curbase) "" > html + print "" > html + print "

coverage: " esc(curbase) "

" > html +} + +{ + cls = "na" + if (is_exec($0)) { + total++ + if ((curbase SUBSEP FNR) in covered) { hit++; cls = "hit" } else { cls = "miss" } + } + print "" > html +} + +function finish_file( pct) { + print "
" FNR "" esc($0) "
" > html + close(html) + pct = (total > 0 ? (hit * 100.0 / total) : 0) + printf "%s,%d,%d,%.1f\n", curbase, total, hit, pct +} + +END { if (cur != "") finish_file() } diff --git a/scripts/lib/render-coverage-index.awk b/scripts/lib/render-coverage-index.awk new file mode 100644 index 0000000..cc5af1b --- /dev/null +++ b/scripts/lib/render-coverage-index.awk @@ -0,0 +1,48 @@ +# scripts/lib/render-coverage-index.awk +# Render the coverage index page from summary rows. +# Input CSV rows (no header): ",,,". +# Per-file detail pages are expected at html/.html. +# Variables: -v TITLE="..." + +function esc(s) { + gsub(/&/, "\\&", s); gsub(//, "\\>", s) + return s +} + +function color(p) { + if (p + 0 >= 80) return "#2da44e" + if (p + 0 >= 50) return "#bf8700" + return "#cf222e" +} + +BEGIN { FS = ","; n = 0; tot = 0; cov = 0 } + +{ + n++ + f[n] = $1; ex[n] = $2 + 0; cv[n] = $3 + 0; pc[n] = $4 + 0 + tot += $2 + 0; cov += $3 + 0 +} + +END { + overall = (tot > 0 ? cov * 100.0 / tot : 0) + print "" + print "" esc(TITLE) "" + print "" + print "

" esc(TITLE) "

" + printf "
Overall: %.1f%% (%d/%d executable lines)
\n", color(overall), overall, cov, tot + print "" + for (i = 1; i <= n; i++) { + printf "", esc(f[i]), esc(f[i]) + printf "", pc[i], color(pc[i]) + printf "\n", pc[i], cv[i], ex[i] + } + print "
filecoverage%coveredexecutable
%s
%.1f%%%d%d
" +} diff --git a/scripts/lib/render-hotspots.awk b/scripts/lib/render-hotspots.awk new file mode 100644 index 0000000..0eb2b6d --- /dev/null +++ b/scripts/lib/render-hotspots.awk @@ -0,0 +1,56 @@ +# scripts/lib/render-hotspots.awk +# Render a per-line hotspot HTML report from sorted TSV rows. +# Input TSV (sorted slowest-first): "\t\t\t". +# Variables: +# -v TITLE="..." +# -v SRC= source file for the primary target, used to show line text. + +function esc(s) { + gsub(/&/, "\\&", s); gsub(//, "\\>", s) + return s +} + +BEGIN { + FS = "\t"; n = 0 + # Load source lines for text display (best effort). + if (SRC != "") { + i = 0 + while ((getline ln < SRC) > 0) { i++; srctext[i] = ln } + close(SRC) + srcbase = SRC; sub(/.*\//, "", srcbase); sub(/.*\\/, "", srcbase) + } +} + +{ + n++ + ms[n] = $1 + 0 + hits[n] = $2 + 0 + file[n] = $3 + line[n] = $4 + 0 + if (n == 1) maxv = $1 + 0 +} + +END { + print "" + print "" esc(TITLE) "" + print "" + print "

" esc(TITLE) "

" + print "
cumulative self-time per source line, slowest first
" + print "" + for (i = 1; i <= n; i++) { + w = (maxv > 0 ? (ms[i] * 100.0 / maxv) : 0) + txt = "" + if (file[i] == srcbase && (line[i] in srctext)) txt = srctext[line[i]] + printf "", i, ms[i], hits[i] + printf "", esc(file[i]), line[i], esc(txt) + printf "\n", w + } + print "
#mshitsfile:linesource
%d%.1f%d%s:%d%s
" +} diff --git a/scripts/lib/render-phases.awk b/scripts/lib/render-phases.awk new file mode 100644 index 0000000..fcb11ec --- /dev/null +++ b/scripts/lib/render-phases.awk @@ -0,0 +1,41 @@ +# scripts/lib/render-phases.awk +# Render per-phase hotspot HTML from sorted TSV rows. +# Input TSV: "\t\t". +# Variables: +# -v TITLE="..." + +function esc(s) { + gsub(/&/, "\\&", s); gsub(//, "\\>", s) + return s +} + +BEGIN { FS = "\t"; n = 0 } + +{ + n++ + ms[n] = $1 + 0 + hits[n] = $2 + 0 + phase[n] = $3 + if (n == 1) maxv = ms[n] +} + +END { + print "" + print "" esc(TITLE) "" + print "" + print "

" esc(TITLE) "

" + print "
cumulative self-time by pipeline phase, slowest first
" + print "" + for (i = 1; i <= n; i++) { + w = (maxv > 0 ? (ms[i] * 100.0 / maxv) : 0) + printf "", i, ms[i], hits[i], esc(phase[i]) + printf "\n", w + } + print "
#mshitsphase
%d%.1f%d%s
" +} diff --git a/scripts/lib/render-timing.awk b/scripts/lib/render-timing.awk new file mode 100644 index 0000000..ffd807f --- /dev/null +++ b/scripts/lib/render-timing.awk @@ -0,0 +1,41 @@ +# scripts/lib/render-timing.awk +# Render a per-test timing HTML report from sorted TSV rows. +# Input TSV (already sorted slowest-first): "\t\t". +# Variables: -v TITLE="..." + +function esc(s) { + gsub(/&/, "\\&", s); gsub(//, "\\>", s) + return s +} + +BEGIN { FS = "\t"; n = 0; total = 0 } + +{ + n++ + sec[n] = $1 + 0 + suite[n] = $2 + name[n] = $3 + total += $1 + 0 + if (n == 1) maxv = $1 + 0 +} + +END { + print "" + print "" esc(TITLE) "" + print "" + print "

" esc(TITLE) "

" + printf "
%d tests · %.3f s total
\n", n, total + print "" + for (i = 1; i <= n; i++) { + w = (maxv > 0 ? (sec[i] * 100.0 / maxv) : 0) + printf "", i, sec[i], esc(name[i]) + printf "\n", w + } + print "
#secondstest
%d%.3f%s
" +} diff --git a/scripts/lib/timing-extract.awk b/scripts/lib/timing-extract.awk new file mode 100644 index 0000000..9cb15d7 --- /dev/null +++ b/scripts/lib/timing-extract.awk @@ -0,0 +1,32 @@ +# scripts/lib/timing-extract.awk +# Extract per-test durations from bats JUnit report XML. +# +# Emits TSV: "\t\t" for each . +# The caller sorts and renders. Handles testcase elements that span the line +# regardless of whether they are self-closing or contain children. + +function attr(line, key, re, s) { + re = key "=\"" + if (index(line, re) == 0) return "" + s = line + sub(".*" re, "", s) # drop up to opening quote + sub(/".*/, "", s) # drop from closing quote + return s +} + +function unesc(s) { + gsub(/"/, "\"", s) + gsub(/'/, "'", s) + gsub(/</, "<", s) + gsub(/>/, ">", s) + gsub(/&/, "\\&", s) + return s +} + +/ prints "\t" per marker line +# MODE=time -> prints "\t\t\t" at END, +# computing per-line durations from consecutive markers +# WITHIN each trace file (one file == one process, so +# parallel xargs workers are not cross-attributed). + +function basename(p) { + sub(/.*\//, "", p) # strip up to last forward slash + sub(/.*\\/, "", p) # strip up to last backslash (defensive) + return p +} + +# Parse a marker line into globals g_base and g_line (and g_epoch for time). +# Returns 1 on success, 0 if the line is not a usable marker. +function parse(line, s, n, f, loc, rest) { + if (line !~ /@+TBL@@ /) return 0 + s = line + sub(/^.*TBL@@ /, "", s) # drop marker and everything before it + n = split(s, f, " ") + if (n < 1) return 0 + loc = f[1] # : + if (!match(loc, /:[0-9]+$/)) return 0 + g_line = substr(loc, RSTART + 1) + g_base = basename(substr(loc, 1, RSTART - 1)) + if (MODE == "time") { + g_epoch = (n >= 2 ? f[2] : "") + } + return 1 +} + +BEGIN { if (MODE == "") MODE = "cov" } + +# Reset per-process state at the start of each trace file. +FNR == 1 { have_prev = 0 } + +{ + if (!parse($0)) next + if (MODE == "cov") { + print g_base "\t" g_line + next + } + # MODE == time: attribute elapsed time to the PREVIOUS marker. + if (have_prev && g_epoch != "" && prev_epoch != "") { + d = g_epoch - prev_epoch + if (d < 0) d = 0 + key = prev_base SUBSEP prev_line + ms[key] += d * 1000.0 + hits[key] += 1 + base_of[key] = prev_base + line_of[key] = prev_line + } + prev_base = g_base + prev_line = g_line + prev_epoch = g_epoch + have_prev = 1 +} + +END { + if (MODE != "time") exit + for (k in ms) { + printf "%.3f\t%d\t%s\t%s\n", ms[k], hits[k], base_of[k], line_of[k] + } +} diff --git a/scripts/lib/trace-init.sh b/scripts/lib/trace-init.sh new file mode 100644 index 0000000..80f1ab0 --- /dev/null +++ b/scripts/lib/trace-init.sh @@ -0,0 +1,42 @@ +# scripts/lib/trace-init.sh +# Instrumentation hook sourced automatically by every non-interactive bash +# invocation via the BASH_ENV environment variable. +# +# The bats suite runs the tool with `bash "$TRBLDOC"`, so exporting +# BASH_ENV= causes each trbldoc.sh process to enable xtrace at +# startup WITHOUT modifying trbldoc.sh or the test files. The emitted trace is +# consumed by the coverage and hotspot aggregators. +# +# Activated only when TRBLDOC_TRACE_DIR is set, and only for scripts whose +# basename appears in TRBLDOC_TRACE_TARGETS (default: trbldoc.sh). This keeps +# unrelated bash helpers (md2html stub, bats internals) out of the trace. +# +# TRBLDOC_TRACE_MODE selects the PS4 format: +# cov -> " :" (line coverage) +# time -> " : " (line timing; bash>=5) +# +# Every traced line is prefixed with the marker token @@TBL@@. bash repeats the +# FIRST PS4 character once per call-nesting level, so the aggregators match the +# marker with a tolerant pattern rather than a fixed prefix. + +if [ -n "${TRBLDOC_TRACE_DIR:-}" ]; then + __tb_self="$0" + __tb_base="${__tb_self##*/}" + case " ${TRBLDOC_TRACE_TARGETS:-trbldoc.sh} " in + *" ${__tb_base} "*) + # Dedicated append fd per process; PID keeps parallel xargs workers apart. + __tb_trace_file="${TRBLDOC_TRACE_DIR}/trace.$$.log" + # shellcheck disable=SC3023 + exec 7>>"$__tb_trace_file" + export BASH_XTRACEFD=7 + if [ "${TRBLDOC_TRACE_MODE:-cov}" = "time" ]; then + # EPOCHREALTIME is bash>=5; used only for the hotspot profiler. + PS4='@@TBL@@ ${BASH_SOURCE}:${LINENO} ${EPOCHREALTIME} ' + else + PS4='@@TBL@@ ${BASH_SOURCE}:${LINENO} ' + fi + export PS4 + set -x + ;; + esac +fi diff --git a/scripts/lib/trace-phase-extract.awk b/scripts/lib/trace-phase-extract.awk new file mode 100644 index 0000000..429f8cf --- /dev/null +++ b/scripts/lib/trace-phase-extract.awk @@ -0,0 +1,75 @@ +# scripts/lib/trace-phase-extract.awk +# Aggregate xtrace timing markers into coarse trbldoc pipeline phases. +# +# Input: trace.*.log files emitted by scripts/lib/trace-init.sh in time mode. +# Output rows: "\t\t" at END. +# +# Required -v vars: +# SRC_BASE +# L_DISCOVER_END +# L_EXTRACT_END +# L_ORDER_END +# L_RENDER_END +# L_NAV_END +# L_REFS_END +# L_TOC_END +# L_ASSEMBLE_END + +function basename(p) { + sub(/.*\//, "", p) + sub(/.*\\/, "", p) + return p +} + +function parse(line, s, n, f, loc) { + if (line !~ /@+TBL@@ /) return 0 + s = line + sub(/^.*TBL@@ /, "", s) + n = split(s, f, " ") + if (n < 2) return 0 + loc = f[1] + if (!match(loc, /:[0-9]+$/)) return 0 + g_line = substr(loc, RSTART + 1) + 0 + g_base = basename(substr(loc, 1, RSTART - 1)) + g_epoch = f[2] + return 1 +} + +function phase_for(line) { + if (line <= L_DISCOVER_END) return "discover_sources" + if (line <= L_EXTRACT_END) return "extract_chunks" + if (line <= L_ORDER_END) return "build_render_order" + if (line <= L_RENDER_END) return "render_chunks" + if (line <= L_NAV_END) return "build_nav" + if (line <= L_REFS_END) return "resolve_refs" + if (line <= L_TOC_END) return "apply_toc" + if (line <= L_ASSEMBLE_END) return "assemble_site" + return "other" +} + +BEGIN { + if (SRC_BASE == "") SRC_BASE = "trbldoc.sh" +} + +FNR == 1 { have_prev = 0 } + +{ + if (!parse($0)) next + if (g_base != SRC_BASE) next + if (have_prev && g_epoch != "" && prev_epoch != "") { + d = g_epoch - prev_epoch + if (d < 0) d = 0 + ph = phase_for(prev_line) + ms[ph] += d * 1000.0 + hits[ph] += 1 + } + prev_line = g_line + prev_epoch = g_epoch + have_prev = 1 +} + +END { + for (ph in ms) { + printf "%.3f\t%d\t%s\n", ms[ph], hits[ph], ph + } +} diff --git a/scripts/profile-run.sh b/scripts/profile-run.sh new file mode 100644 index 0000000..74ab425 --- /dev/null +++ b/scripts/profile-run.sh @@ -0,0 +1,180 @@ +#!/bin/sh +# scripts/profile-run.sh +# Profile direct trbldoc invocations (outside bats) using the existing xtrace +# hook and report pipeline. +# +# Outputs: +# - run-summary.tsv/csv wall-clock per run +# - hotspots.tsv/csv/html per-line cumulative self-time +# - phase-hotspots.tsv/csv/html per-phase cumulative self-time +# +# Usage: +# scripts/profile-run.sh [-o OUTDIR] [-n RUNS] [-C] [-t "file.sh ..."] -- [trbldoc args...] +# +# -o OUTDIR output directory (default: reports/profile-run-) +# -n RUNS number of repeated runs (default: 1) +# -C force clean rebuild (-C) on every run +# -t LIST trace target script basenames (default: trbldoc.sh) + +set -u + +SELF="$0" +. "$(dirname -- "$0")/lib/common.sh" +tb_resolve_paths "$SELF" + +OUTDIR="" +RUNS=1 +CLEAN=0 +TARGETS="trbldoc.sh" + +while getopts ":o:n:Ct:h" opt; do + case "$opt" in + o) OUTDIR="$OPTARG" ;; + n) RUNS="$OPTARG" ;; + C) CLEAN=1 ;; + t) TARGETS="$OPTARG" ;; + h) sed -n '2,24p' "$SELF"; exit 0 ;; + :) tb_die "option -$OPTARG requires an argument" ;; + \?) tb_die "unknown option -$OPTARG" ;; + esac +done +shift $((OPTIND - 1)) + +[ "${1:-}" = "--" ] && shift + +tb_require awk +tb_require sort +tb_require bash + +case "$RUNS" in + ''|*[!0-9]*|0) tb_die "-n RUNS must be a positive integer" ;; +esac + +[ -n "$OUTDIR" ] || OUTDIR="$REPO_ROOT/reports/profile-run-$(tb_timestamp)" +case "$OUTDIR" in + /*|[A-Za-z]:[\\/]*) ;; + *) OUTDIR="$REPO_ROOT/$OUTDIR" ;; +esac +mkdir -p "$OUTDIR" || tb_die "cannot create output dir: $OUTDIR" + +TRACE_DIR="$OUTDIR/trace" +RUN_DIR="$OUTDIR/runs" +mkdir -p "$TRACE_DIR" "$RUN_DIR" + +TRBLDOC_PATH="$REPO_ROOT/trbldoc.sh" +[ -f "$TRBLDOC_PATH" ] || tb_die "trbldoc source not found: $TRBLDOC_PATH" + +TRACE_TARGETS="" +for tf in $TARGETS; do + TRACE_TARGETS="$TRACE_TARGETS ${tf##*/}" +done + +# Resolve line anchors once for phase aggregation. If any anchor is missing, +# skip phase aggregation rather than failing the whole profiling run. +tb_find_line() { + _pat="$1" + awk -v p="$_pat" 'index($0, p) { print NR; exit }' "$TRBLDOC_PATH" +} +L_DISCOVER_END="$(tb_find_line '> "$source_files_list"')" +L_EXTRACT_END="$(tb_find_line '# Resolve src_repo once before the chunk loop.')" +L_ORDER_END="$(tb_find_line 'done < "$sorted_chunk_order_file"')" +L_RENDER_END="$(tb_find_line 'done < "$chunk_render_records_file"')" +L_NAV_END="$(tb_find_line '# Second pass: resolve inline references now that global.meta is complete.')" +L_REFS_END="$(tb_find_line 'rm -f "$_ref_find_tmp"')" +L_TOC_END="$(tb_find_line '# Post-processing pass: substitute {{ toc }} in all rendered index.html files')" +L_ASSEMBLE_END="$(tb_find_line 'assemble_site "$cache_dir/namespace" "$out_dir"')" + +: > "$OUTDIR/run-summary.tsv" + +run_idx=1 +while [ "$run_idx" -le "$RUNS" ]; do + tb_info "profiling run $run_idx/$RUNS..." + run_out="$RUN_DIR/run-$run_idx" + mkdir -p "$run_out" + run_stdout="$run_out/stdout.txt" + run_stderr="$run_out/stderr.txt" + run_start_s="$(date +%s)" + + BASH_ENV="$LIB_DIR/trace-init.sh" + TRBLDOC_TRACE_DIR="$TRACE_DIR" + TRBLDOC_TRACE_MODE="time" + TRBLDOC_TRACE_TARGETS="$TRACE_TARGETS" + export BASH_ENV TRBLDOC_TRACE_DIR TRBLDOC_TRACE_MODE TRBLDOC_TRACE_TARGETS + + if [ "$CLEAN" -eq 1 ]; then + (cd "$REPO_ROOT" && bash "$TRBLDOC_PATH" -C "$@") >"$run_stdout" 2>"$run_stderr" + else + (cd "$REPO_ROOT" && bash "$TRBLDOC_PATH" "$@") >"$run_stdout" 2>"$run_stderr" + fi + run_exit=$? + unset BASH_ENV TRBLDOC_TRACE_DIR TRBLDOC_TRACE_MODE TRBLDOC_TRACE_TARGETS + + run_end_s="$(date +%s)" + run_ms=$(( (run_end_s - run_start_s) * 1000 )) + printf '%s\t%s\t%s\n' "$run_idx" "$run_ms" "$run_exit" >> "$OUTDIR/run-summary.tsv" + run_idx=$((run_idx + 1)) +done + +sort -t "$(printf '\t')" -k2 -rn "$OUTDIR/run-summary.tsv" > "$OUTDIR/run-summary.sorted.tsv" +{ + printf 'run,total_ms,exit_code\n' + awk -F '\t' '{ printf "%s,%s,%s\n",$1,$2,$3 }' "$OUTDIR/run-summary.sorted.tsv" +} > "$OUTDIR/run-summary.csv" + +if ls "$TRACE_DIR"/trace.*.log >/dev/null 2>&1; then + awk -f "$LIB_DIR/trace-extract.awk" -v MODE=time "$TRACE_DIR"/trace.*.log \ + | sort -t "$(printf '\t')" -k1 -rn > "$OUTDIR/hotspots.tsv" + { + printf 'total_ms,hits,file,line\n' + awk -F '\t' '{ printf "%s,%s,%s,%s\n",$1,$2,$3,$4 }' "$OUTDIR/hotspots.tsv" + } > "$OUTDIR/hotspots.csv" + + FIRST_TARGET="${TARGETS%% *}" + case "$FIRST_TARGET" in + /*|[A-Za-z]:[\\/]*) SRC_PATH="$FIRST_TARGET" ;; + *) SRC_PATH="$REPO_ROOT/$FIRST_TARGET" ;; + esac + awk -f "$LIB_DIR/render-hotspots.awk" \ + -v TITLE="trbldoc line hotspots (direct runs)" \ + -v SRC="$SRC_PATH" \ + "$OUTDIR/hotspots.tsv" > "$OUTDIR/hotspots.html" +else + tb_die "no trace logs captured (check run stderr logs under $RUN_DIR)" +fi + +if [ -n "$L_DISCOVER_END" ] && [ -n "$L_EXTRACT_END" ] && [ -n "$L_ORDER_END" ] \ + && [ -n "$L_RENDER_END" ] && [ -n "$L_NAV_END" ] && [ -n "$L_REFS_END" ] \ + && [ -n "$L_TOC_END" ] && [ -n "$L_ASSEMBLE_END" ]; then + awk -f "$LIB_DIR/trace-phase-extract.awk" \ + -v SRC_BASE="trbldoc.sh" \ + -v L_DISCOVER_END="$L_DISCOVER_END" \ + -v L_EXTRACT_END="$L_EXTRACT_END" \ + -v L_ORDER_END="$L_ORDER_END" \ + -v L_RENDER_END="$L_RENDER_END" \ + -v L_NAV_END="$L_NAV_END" \ + -v L_REFS_END="$L_REFS_END" \ + -v L_TOC_END="$L_TOC_END" \ + -v L_ASSEMBLE_END="$L_ASSEMBLE_END" \ + "$TRACE_DIR"/trace.*.log \ + | sort -t "$(printf '\t')" -k1 -rn > "$OUTDIR/phase-hotspots.tsv" + + { + printf 'total_ms,hits,phase\n' + awk -F '\t' '{ printf "%s,%s,%s\n",$1,$2,$3 }' "$OUTDIR/phase-hotspots.tsv" + } > "$OUTDIR/phase-hotspots.csv" + + awk -f "$LIB_DIR/render-phases.awk" \ + -v TITLE="trbldoc phase hotspots (direct runs)" \ + "$OUTDIR/phase-hotspots.tsv" > "$OUTDIR/phase-hotspots.html" +else + tb_info "phase anchor discovery incomplete; skipping phase-hotspots report" +fi + +printf '\n==== slowest runs ====\n' +awk -F '\t' '{ printf "run %s %8.1f ms exit=%s\n",$1,$2,$3 }' "$OUTDIR/run-summary.sorted.tsv" +printf '\nReports written to: %s\n' "$OUTDIR" +printf ' run summary : %s\n' "$OUTDIR/run-summary.csv" +printf ' line hotspots : %s\n' "$OUTDIR/hotspots.html" +if [ -f "$OUTDIR/phase-hotspots.html" ]; then + printf ' phase hotspots : %s\n' "$OUTDIR/phase-hotspots.html" +fi diff --git a/scripts/profile-tests.sh b/scripts/profile-tests.sh new file mode 100644 index 0000000..498ae25 --- /dev/null +++ b/scripts/profile-tests.sh @@ -0,0 +1,184 @@ +#!/bin/sh +# scripts/profile-tests.sh +# Generate timing reports for the trbldoc test suite. +# +# Two levels of detail: +# 1. Per-test wall-clock timings from bats (--timing + JUnit report). Always +# produced. Answers "which tests take a long time". +# 2. Per-line hotspots inside trbldoc.sh (optional, --hotspots). Uses the +# BASH_ENV xtrace hook to time individual source lines across the whole +# run. Answers "which code takes a long time". +# +# POSIX sh; works under mingw bash, WSL bash and Alpine/busybox ash. +# +# Usage: +# scripts/profile-tests.sh [-s all|unit|integration] [-o OUTDIR] +# [-n TOPN] [-p] [-t "file.sh ..."] +# +# -s SUITE suite to run (default: integration) +# -o OUTDIR output directory (default: reports/profile-) +# -n TOPN number of rows in the printed "slowest" summary (default: 20) +# -p also collect per-line hotspots inside the target script(s) +# -t LIST space-separated target source files for -p (default: trbldoc.sh) + +set -u + +SELF="$0" +. "$(dirname -- "$0")/lib/common.sh" +tb_resolve_paths "$SELF" + +SUITE=integration +OUTDIR="" +TOPN=20 +HOTSPOTS=0 +TARGETS="trbldoc.sh" + +while getopts ":s:o:n:pt:h" opt; do + case "$opt" in + s) SUITE="$OPTARG" ;; + o) OUTDIR="$OPTARG" ;; + n) TOPN="$OPTARG" ;; + p) HOTSPOTS=1 ;; + t) TARGETS="$OPTARG" ;; + h) sed -n '2,30p' "$SELF"; exit 0 ;; + :) tb_die "option -$OPTARG requires an argument" ;; + \?) tb_die "unknown option -$OPTARG" ;; + esac +done + +tb_require awk +tb_require sort + +[ -n "$OUTDIR" ] || OUTDIR="$REPO_ROOT/reports/profile-$(tb_timestamp)" +case "$OUTDIR" in + /*|[A-Za-z]:[\\/]*) + ;; + *) + OUTDIR="$REPO_ROOT/$OUTDIR" + ;; +esac +mkdir -p "$OUTDIR" || tb_die "cannot create output dir: $OUTDIR" +JUNIT_DIR="$OUTDIR/junit" +mkdir -p "$JUNIT_DIR" + +SUITE_DIRS="$(tb_suite_dirs "$SUITE")" +SUITE_DIRS_REL="$(tb_suite_dirs_rel "$SUITE")" + +# ── 1. per-test timing via bats JUnit ─────────────────────────────────────── +tb_info "running bats ($SUITE) with timing..." +# --timing adds per-test durations; the junit report captures them in XML. +# We do not abort on test failures: timing is still useful. +if tb_bats_usable "$SUITE"; then + # shellcheck disable=SC2086 + bats --timing --report-formatter junit --output "$JUNIT_DIR" $SUITE_DIRS \ + > "$OUTDIR/bats-output.txt" 2>&1 || tb_info "bats reported failures (see bats-output.txt)" + BATS_MODE="local" +elif command -v wsl >/dev/null 2>&1; then + tb_info "local bats appears unusable; retrying via WSL bats..." + WSL_REPO="$(tb_to_wsl_path "$REPO_ROOT")" + WSL_JUNIT="$(tb_to_wsl_path "$JUNIT_DIR")" + wsl -- bash -lc \ + "cd '$WSL_REPO' && bats --timing --report-formatter junit --output '$WSL_JUNIT' $SUITE_DIRS_REL" \ + > "$OUTDIR/bats-output.txt" 2>&1 || tb_info "bats (WSL) reported failures (see bats-output.txt)" + BATS_MODE="wsl" +else + tb_die "local bats command appears unusable and no WSL fallback is available" +fi + +# Collect all JUnit XML the formatter produced. +: > "$OUTDIR/timings.tsv" +for xml in "$JUNIT_DIR"/*.xml; do + [ -f "$xml" ] || continue + awk -f "$LIB_DIR/timing-extract.awk" "$xml" >> "$OUTDIR/timings.tsv" +done + +if [ ! -s "$OUTDIR/timings.tsv" ]; then + if grep -qi "bash\\\\r" "$OUTDIR/bats-output.txt" 2>/dev/null; then + tb_die "bats appears broken (CRLF shebang: 'bash\\r'). Reinstall bats or run dos2unix on the bats scripts, then retry." + fi + tb_die "no per-test timings parsed (check $OUTDIR/bats-output.txt)" +fi + +# Sort slowest first. +sort -t "$(printf '\t')" -k1 -rn "$OUTDIR/timings.tsv" > "$OUTDIR/timings.sorted.tsv" + +# CSV summary. +{ + printf 'seconds,suite,test\n' + awk -F '\t' '{ gsub(/"/,"\"\"",$3); printf "%s,%s,\"%s\"\n",$1,$2,$3 }' \ + "$OUTDIR/timings.sorted.tsv" +} > "$OUTDIR/timings.csv" + +# HTML report. +awk -f "$LIB_DIR/render-timing.awk" \ + -v TITLE="trbldoc test timings ($SUITE)" \ + "$OUTDIR/timings.sorted.tsv" > "$OUTDIR/timings.html" + +# ── 2. optional per-line hotspots ──────────────────────────────────────────── +if [ "$HOTSPOTS" -eq 1 ]; then + tb_info "collecting per-line hotspots (this reruns the suite under xtrace)..." + TRACE_DIR="$OUTDIR/trace" + rm -rf "$TRACE_DIR"; mkdir -p "$TRACE_DIR" + + # Build the trace-target basename filter. + TRACE_TARGETS="" + for tf in $TARGETS; do + TRACE_TARGETS="$TRACE_TARGETS ${tf##*/}" + done + + BASH_ENV="$LIB_DIR/trace-init.sh" + TRBLDOC_TRACE_DIR="$TRACE_DIR" + TRBLDOC_TRACE_MODE="time" + TRBLDOC_TRACE_TARGETS="$TRACE_TARGETS" + if [ "$BATS_MODE" = "local" ]; then + export BASH_ENV TRBLDOC_TRACE_DIR TRBLDOC_TRACE_MODE TRBLDOC_TRACE_TARGETS + # shellcheck disable=SC2086 + bats --report-formatter junit --output "$JUNIT_DIR" $SUITE_DIRS \ + > "$OUTDIR/bats-hotspots-output.txt" 2>&1 \ + || tb_info "bats reported failures during hotspot run" + unset BASH_ENV TRBLDOC_TRACE_DIR TRBLDOC_TRACE_MODE TRBLDOC_TRACE_TARGETS + else + WSL_REPO="$(tb_to_wsl_path "$REPO_ROOT")" + WSL_LIB="$(tb_to_wsl_path "$LIB_DIR")" + WSL_TRACE="$(tb_to_wsl_path "$TRACE_DIR")" + WSL_JUNIT="$(tb_to_wsl_path "$JUNIT_DIR")" + wsl -- bash -lc \ + "cd '$WSL_REPO' && export BASH_ENV='$WSL_LIB/trace-init.sh' TRBLDOC_TRACE_DIR='$WSL_TRACE' TRBLDOC_TRACE_MODE='time' TRBLDOC_TRACE_TARGETS='$TRACE_TARGETS'; bats --report-formatter junit --output '$WSL_JUNIT' $SUITE_DIRS_REL" \ + > "$OUTDIR/bats-hotspots-output.txt" 2>&1 \ + || tb_info "bats (WSL) reported failures during hotspot run" + fi + + if ls "$TRACE_DIR"/trace.*.log >/dev/null 2>&1; then + awk -f "$LIB_DIR/trace-extract.awk" -v MODE=time "$TRACE_DIR"/trace.*.log \ + | sort -t "$(printf '\t')" -k1 -rn > "$OUTDIR/hotspots.tsv" + { + printf 'total_ms,hits,file,line\n' + awk -F '\t' '{ printf "%s,%s,%s,%s\n",$1,$2,$3,$4 }' "$OUTDIR/hotspots.tsv" + } > "$OUTDIR/hotspots.csv" + # Resolve first target path for source display in the HTML. + FIRST_TARGET="${TARGETS%% *}" + case "$FIRST_TARGET" in + /*) SRC_PATH="$FIRST_TARGET" ;; + *) SRC_PATH="$REPO_ROOT/$FIRST_TARGET" ;; + esac + awk -f "$LIB_DIR/render-hotspots.awk" \ + -v TITLE="trbldoc line hotspots ($SUITE)" \ + -v SRC="$SRC_PATH" \ + "$OUTDIR/hotspots.tsv" > "$OUTDIR/hotspots.html" + else + if grep -qi "bash\\\\r" "$OUTDIR/bats-hotspots-output.txt" 2>/dev/null; then + tb_die "bats appears broken (CRLF shebang: 'bash\\r') during hotspot run. Reinstall bats or run dos2unix on the bats scripts." + fi + tb_info "no trace data captured; skipping hotspots report" + HOTSPOTS=0 + fi +fi + +# ── printed summary ────────────────────────────────────────────────────────── +printf '\n==== slowest %s tests ====\n' "$TOPN" +awk -F '\t' -v n="$TOPN" 'NR<=n { printf "%8.3fs %s\n", $1, $3 }' \ + "$OUTDIR/timings.sorted.tsv" + +printf '\nReports written to: %s\n' "$OUTDIR" +printf ' per-test timing : %s\n' "$OUTDIR/timings.html" +[ "$HOTSPOTS" -eq 1 ] && printf ' line hotspots : %s\n' "$OUTDIR/hotspots.html" diff --git a/tests/fixtures/c/directives.c b/tests/fixtures/c/directives.c new file mode 100644 index 0000000..4becc88 --- /dev/null +++ b/tests/fixtures/c/directives.c @@ -0,0 +1,7 @@ +/* md +@name test/directives +@file generated/source.c +@ref test/directives-ref +Directive test body content. +*/ +void directives_demo() {} diff --git a/tests/fixtures/c/inline_ref.c b/tests/fixtures/c/inline_ref.c new file mode 100644 index 0000000..72ef395 --- /dev/null +++ b/tests/fixtures/c/inline_ref.c @@ -0,0 +1,5 @@ +/* md +@name test/inline-ref +See @lua/Coroutines for details on cooperative multitasking. +*/ +void inline_ref_func() {} diff --git a/tests/fixtures/c/multi_chunk.c b/tests/fixtures/c/multi_chunk.c new file mode 100644 index 0000000..57fd463 --- /dev/null +++ b/tests/fixtures/c/multi_chunk.c @@ -0,0 +1,11 @@ +/* md +@name test/alpha +First chunk content. +*/ +void foo() {} + +/* md +@name test/beta +Second chunk content. +*/ +void bar() {} diff --git a/tests/fixtures/c/no_comments.c b/tests/fixtures/c/no_comments.c new file mode 100644 index 0000000..a667cef --- /dev/null +++ b/tests/fixtures/c/no_comments.c @@ -0,0 +1,3 @@ +/* This is a regular comment, not a doc comment */ +// Another non-doc comment +int noop() { return 42; } diff --git a/tests/fixtures/c/ref_string.c b/tests/fixtures/c/ref_string.c new file mode 100644 index 0000000..711ac20 --- /dev/null +++ b/tests/fixtures/c/ref_string.c @@ -0,0 +1,6 @@ +/* md +@name test/ref-string +@ref lua/Coroutines +Return type: {{ ref_string }} +*/ +void ref_string_func() {} diff --git a/tests/fixtures/c/rst_exec.c b/tests/fixtures/c/rst_exec.c new file mode 100644 index 0000000..3aabf9f --- /dev/null +++ b/tests/fixtures/c/rst_exec.c @@ -0,0 +1,8 @@ +/* rst +@name test/rst-exec +RST Heading +=========== + +This chunk should be rendered by the rst renderer. +*/ +int noop() { return 0; } diff --git a/tests/fixtures/c/shared_namespace.c b/tests/fixtures/c/shared_namespace.c new file mode 100644 index 0000000..25f064a --- /dev/null +++ b/tests/fixtures/c/shared_namespace.c @@ -0,0 +1,11 @@ +/* md +@name test/shared +Part one of the shared page. +*/ +void part_one() {} + +/* md +@name test/shared +Part two of the shared page. +*/ +void part_two() {} diff --git a/tests/fixtures/c/simple.c b/tests/fixtures/c/simple.c new file mode 100644 index 0000000..2595929 --- /dev/null +++ b/tests/fixtures/c/simple.c @@ -0,0 +1,5 @@ +/* md +@name test/simple +Hello from a C comment. +*/ +int main() { return 0; } diff --git a/tests/fixtures/c/src_repo.c b/tests/fixtures/c/src_repo.c new file mode 100644 index 0000000..fb0d8b6 --- /dev/null +++ b/tests/fixtures/c/src_repo.c @@ -0,0 +1,5 @@ +/* md +@name test/src-repo +Generated from {{ src_repo }}/src.c +*/ +void src_repo_func() {} diff --git a/tests/fixtures/c/toc.c b/tests/fixtures/c/toc.c new file mode 100644 index 0000000..bb0782b --- /dev/null +++ b/tests/fixtures/c/toc.c @@ -0,0 +1,5 @@ +/* md +@name test/toc-chunk +Navigation: {{ toc }} +*/ +void toc_func() {} diff --git a/tests/fixtures/lua/simple.lua b/tests/fixtures/lua/simple.lua new file mode 100644 index 0000000..de527a9 --- /dev/null +++ b/tests/fixtures/lua/simple.lua @@ -0,0 +1,5 @@ +--[[ md +@name test/lua-simple +Hello from a Lua comment. +]] +local x = 1 diff --git a/tests/fixtures/lua/single_line.lua b/tests/fixtures/lua/single_line.lua new file mode 100644 index 0000000..f541825 --- /dev/null +++ b/tests/fixtures/lua/single_line.lua @@ -0,0 +1,2 @@ +--[[ md This comment opens and closes on one line — no chunk. ]] +local x = 1 diff --git a/tests/fixtures/md/readme.md b/tests/fixtures/md/readme.md new file mode 100644 index 0000000..5833cab --- /dev/null +++ b/tests/fixtures/md/readme.md @@ -0,0 +1,2 @@ +# Project Overview +Hello from the readme. diff --git a/tests/fixtures/md/simple.md b/tests/fixtures/md/simple.md new file mode 100644 index 0000000..494ade7 --- /dev/null +++ b/tests/fixtures/md/simple.md @@ -0,0 +1,3 @@ +@name test/md-simple +# Markdown Page +Hello from a markdown file. diff --git a/tests/fixtures/py/simple.py b/tests/fixtures/py/simple.py new file mode 100644 index 0000000..b8e2ce9 --- /dev/null +++ b/tests/fixtures/py/simple.py @@ -0,0 +1,6 @@ +'''md +@name test/py-simple +Hello from a Python doc comment. +''' +def noop(): + pass diff --git a/tests/fixtures/py/single_line.py b/tests/fixtures/py/single_line.py new file mode 100644 index 0000000..baafa10 --- /dev/null +++ b/tests/fixtures/py/single_line.py @@ -0,0 +1,2 @@ +'''This comment opens and closes on one line — no chunk.''' +x = 1 diff --git a/tests/helpers/common.bash b/tests/helpers/common.bash new file mode 100644 index 0000000..779cc96 --- /dev/null +++ b/tests/helpers/common.bash @@ -0,0 +1,263 @@ +# tests/helpers/common.bash +# Sourced by every bats test file via `load`. +# Provides: workspace isolation, PATH stubs, assertion helpers. + +# ── workspace setup/teardown ──────────────────────────────────────────────── + +# Call in setup(): creates an isolated temp directory and cds into it. +# A stub for md2html is pre-installed in a fake bin/ on PATH. +setup_workspace() { + TEST_TMPDIR="$(mktemp -d)" + # trap 'rm -rf $TEST_TMPDIR' EXIT + # Fake bin injected before the real PATH so stubs take priority + STUB_BIN="$TEST_TMPDIR/bin" + mkdir -p "$STUB_BIN" + install_stub_md2html "$STUB_BIN" + ORIG_DIR="$(pwd)" + export PATH="$STUB_BIN:$PATH" + # Work inside the temp dir so .trblcache is isolated + pushd "$TEST_TMPDIR" >/dev/null +} + +# Call in teardown(): removes the temp directory. +teardown_workspace() { + popd >/dev/null 2>&1 || true + rm -rf "$TEST_TMPDIR" +} + +# ── stubs ─────────────────────────────────────────────────────────────────── + +# md2html stub: echoes a predictable HTML marker for easy assertion. +# Accepts any flags; always succeeds. +install_stub_md2html() { + local bin_dir="$1" + cat > "$bin_dir/md2html" <<'EOF' +#!/usr/bin/env bash +# Stub: find the last non-flag argument as the input file, or read stdin. +# Flags (args starting with -) from the md alias are ignored. +input="" +for arg in "$@"; do + [[ "$arg" == -* ]] || input="$arg" +done +if [[ -n "$input" ]]; then + content="$(cat "$input")" +else + content="$(cat)" +fi +echo "

$content

" +EOF + chmod +x "$bin_dir/md2html" +} + + +# ── AWK program builders ───────────────────────────────────────────────────── +# These mirror the comment_form patterns from trbldoc.sh so extraction +# logic can be unit-tested without sourcing the whole script. + +CACHE_DIR=".trblcache" + +# Returns the AWK program string for C-style /* */ comments. +awk_prog_c() { + local cache="$1" + cat < \$chunkfile << \"EOF\"\\n@file " name ":" NR ; + if(\$2) {\$1=""; print "@exec " \$0} + in_comment=1; +} +EOF +} + +# Returns the AWK program string for Lua +awk_prog_lua() { + local cache="$1" + cat < \$chunkfile << \"EOF\"\\n@file " name ":" NR ; + if(\$2) {\$1=""; print "@exec " \$0} + in_comment=1; +} +EOF +} + +# Returns the AWK program string for Python. +# Handles '''renderer format (renderer glued directly to ''', no space required). +awk_prog_py() { + local cache="$1" + sed "s|__PY_CACHE__|${cache}|g" << 'PYEOF' +BEGIN{} +/'''/ { + if(in_comment) {print "EOF"} + in_comment=0 +} +in_comment {print} +/'''.+/ && !/'''$/ { + print "chunkfile=$(mktemp -p __PY_CACHE__/chunks)" + print "cat > $chunkfile << \"EOF\"\n@file " name ":" NR ; + renderer = $0 + sub(/'''/, "", renderer) + sub(/^[[:space:]]+/, "", renderer) + sub(/[[:space:]]+$/, "", renderer) + if (renderer != "") {print "@exec " renderer} + in_comment=1; +} +PYEOF +} + +# Returns the AWK program string for .md files +awk_prog_md() { + local cache="$1" + cat < \$chunkfile << \"EOF\"\\n@file " name ":1\\n@exec md" ; +} +{print \$0} +END{print "EOF"} +EOF +} + +# ── extraction helper ──────────────────────────────────────────────────────── + +# run_extraction FILE AWK_PROG CACHE_DIR +# Runs AWK against FILE, evals the output shell script, and populates CACHE_DIR/chunks. +run_extraction() { + local file="$1" + local awk_prog="$2" + local cache="$3" + mkdir -p "$cache/chunks" + local chunk_script + chunk_script=$(awk -v name="$file" "$awk_prog" < "$file") + eval "$chunk_script" +} + +# chunk_count CACHE_DIR +# Prints the number of chunk files produced. +chunk_count() { + find "$1/chunks" -type f 2>/dev/null | wc -l | tr -d ' ' +} + +# chunk_contains CACHE_DIR PATTERN +# Succeeds if any chunk file contains a line matching PATTERN (grep -q). +chunk_contains() { + grep -rl "$2" "$1/chunks" >/dev/null 2>&1 +} + +# ── assertion helpers ──────────────────────────────────────────────────────── + +# assert_file_exists PATH +assert_file_exists() { + if [[ ! -f "$1" ]]; then + echo "ASSERTION FAILED: expected file to exist: $1" >&2 + return 1 + fi +} + +# assert_dir_exists PATH +assert_dir_exists() { + if [[ ! -d "$1" ]]; then + echo "ASSERTION FAILED: expected directory to exist: $1" >&2 + return 1 + fi +} + +# assert_contains FILE PATTERN +assert_contains() { + if ! grep -q "$2" "$1" 2>/dev/null; then + echo "ASSERTION FAILED: '$2' not found in $1" >&2 + echo "--- file contents ---" >&2 + cat "$1" >&2 + return 1 + fi +} + +# assert_not_contains FILE PATTERN +assert_not_contains() { + if grep -q "$2" "$1" 2>/dev/null; then + echo "ASSERTION FAILED: '$2' unexpectedly found in $1" >&2 + return 1 + fi +} + +# assert_equal ACTUAL EXPECTED +assert_equal() { + if [[ "$1" != "$2" ]]; then + echo "ASSERTION FAILED: expected '$2', got '$1'" >&2 + return 1 + fi +} + +# ── HTML validation helpers ────────────────────────────────────────────────── + +# assert_valid_html FILE +# Validates FILE as a complete HTML5 document using tidy. +# Mustache template markers ({{...}}, {{{...}}}) are substituted with +# structurally valid HTML before linting to avoid false positives from +# template syntax that tidy cannot parse. +# Skips automatically if tidy is not installed. +# tidy exit codes: 0 = clean, 1 = warnings only, 2 = errors. +# This helper fails only on exit code 2 (actual errors). +assert_valid_html() { + local file="$1" + if ! command -v tidy >/dev/null 2>&1; then + skip "tidy not installed (apt install tidy)" + fi + local tmp + tmp=$(mktemp --suffix=.html) + # Replace {{ var }} template markers with valid HTML equivalents so tidy + # sees a structurally correct document. + sed \ + -e 's/{{[^}]*}}/placeholder/g' \ + "$file" > "$tmp" + local tidy_out rc + tidy_out=$(tidy -errors -quiet --show-warnings no -utf8 "$tmp" 2>&1) + rc=$? + rm -f "$tmp" + if [ "$rc" -ge 2 ]; then + echo "HTML VALIDATION FAILED: $file" >&2 + echo "$tidy_out" >&2 + return 1 + fi +} + +# assert_valid_html_fragment FILE +# Wraps FILE in a minimal HTML5 document shell, then validates with tidy. +# Use for HTML fragment files (e.g. rendered chunk index.html output) that +# are not complete documents on their own. +# Skips automatically if tidy is not installed. +assert_valid_html_fragment() { + local file="$1" + if ! command -v tidy >/dev/null 2>&1; then + skip "tidy not installed (apt install tidy)" + fi + local tmp + tmp=$(mktemp --suffix=.html) + { + echo 'fragment' + cat "$file" + echo '' + } > "$tmp" + local tidy_out rc + tidy_out=$(tidy -errors -quiet --show-warnings no -utf8 "$tmp" 2>&1) + rc=$? + rm -f "$tmp" + if [ "$rc" -ge 2 ]; then + echo "HTML VALIDATION FAILED: $file" >&2 + echo "$tidy_out" >&2 + return 1 + fi +} diff --git a/tests/helpers/profiler.bash b/tests/helpers/profiler.bash new file mode 100644 index 0000000..c3d5207 --- /dev/null +++ b/tests/helpers/profiler.bash @@ -0,0 +1,53 @@ +# Profiler implementation +profile() { + # Set default options + local file=profiler.log + + # Open a file descriptor for writing to $file and save it in $tracefd + exec {tracefd}>"$file" + # Send trace output to $tracefd + export BASH_XTRACEFD="$tracefd" + # Print microsecond time in trace output + export PS4='+ $EPOCHREALTIME ' + # Enable tracing, run script, and disable tracing + set -x + bash -x -- "$@" + set +x + # Un-redirect the trace output. This also closes the file descriptor. + unset BASH_XTRACEFD + export -n BASH_XTRACEFD PS4 + + # Remove "source" line from output and change last line to only include the + # timestamp with no name. + sed -i -e 1d -e '$s/\(+\+ [0-9\.]\+\) .*$/\1/' "$file" +} + +analyze() { + # Set defaults + local file=profiler.log + local -a sortcmd=(cat) tablecmd=(cat) + + # Declare variables + local timestamp nestlvl cmd next_timestamp next_nestlvl next_cmd duration + # Open file as a file descriptor so we can re-use the same stream + exec {fd}<"$file"; + # Read first line + read -r nestlvl timestamp cmd <&"$fd" + # Process each line + while read -r next_nestlvl next_timestamp next_cmd + do + duration="$(echo "scale=6; $next_timestamp" - "$timestamp" | bc)" + # Prepend leading zero + if [ "${duration:0:1}" = . ] + then + duration="0$duration" + fi + echo "$duration" "$nestlvl" "$cmd" + + timestamp="$next_timestamp" + nestlvl="$next_nestlvl" + cmd="$next_cmd" + done <&"$fd" | "${sortcmd[@]}" | "${tablecmd[@]}" + # Close file descriptor + exec {fd}<&- +} diff --git a/tests/integration/cli.bats b/tests/integration/cli.bats new file mode 100644 index 0000000..5ffc55c --- /dev/null +++ b/tests/integration/cli.bats @@ -0,0 +1,215 @@ +#!/usr/bin/env bats +# tests/integration/cli.bats +# Tests for trbldoc.sh command-line option parsing. +# +# Each test runs the full script (with md2html stubs) and asserts that +# the correct sources are scanned and outputs produced, based on the options +# passed. + +load "../helpers/common.bash" +load "../helpers/profiler.bash" + +FIXTURES="$BATS_TEST_DIRNAME/../fixtures" +_REPO_ROOT="${REPO_ROOT:-$(cd "$BATS_TEST_DIRNAME/../.." && pwd)}" +TRBLDOC="$_REPO_ROOT/trbldoc.sh" + +setup() { + setup_workspace + CACHE=".trblcache" + NS="$CACHE/namespace" +} + +teardown() { + teardown_workspace +} + +# ── helpers ────────────────────────────────────────────────────────────────── + +run_pipeline() { + bash "$TRBLDOC" "$@" # 2>/dev/null + #profile "$TRBLDOC" "$@" 2>/dev/null + #analyze >> profile.txt +} + +stage_fixture() { + local src="$1" + local dest="$2" + mkdir -p "$(dirname "$dest")" + cp -r "$src" "$dest" +} + +# ── error cases ────────────────────────────────────────────────────────────── + +@test "cli: no arguments exits non-zero with an error message" { + run bash "$TRBLDOC" 2>&1 + [ "$status" -ne 0 ] + [[ "$output" == *"trbldoc:"* ]] +} + +@test "cli: unknown option exits non-zero" { + run bash "$TRBLDOC" -z 2>&1 + [ "$status" -ne 0 ] +} + +@test "cli: option missing its required argument exits non-zero" { + run bash "$TRBLDOC" -s 2>&1 + [ "$status" -ne 0 ] +} + +# ── -s (source folder) ─────────────────────────────────────────────────────── + +@test "cli: -s flag specifies the source folder to scan" { + stage_fixture "$FIXTURES/c/simple.c" "src/simple.c" + run_pipeline -s src + assert_dir_exists "$NS/test/simple" +} + +@test "cli: multiple -s flags scan all specified folders" { + stage_fixture "$FIXTURES/c/simple.c" "src1/simple.c" + stage_fixture "$FIXTURES/c/multi_chunk.c" "src2/multi_chunk.c" + run_pipeline -s src1 -s src2 + assert_dir_exists "$NS/test/simple" + assert_dir_exists "$NS/test/alpha" +} + +# ── -o (output directory) ──────────────────────────────────────────────────── + +@test "cli: -o flag creates the specified output directory" { + stage_fixture "$FIXTURES/c/simple.c" "src/simple.c" + run_pipeline -s src -o my_output + assert_dir_exists "my_output" + assert_file_exists "my_output/test/simple/index.html" +} + +@test "cli: without -o the default output directory .trblcache/built is created" { + stage_fixture "$FIXTURES/c/simple.c" "src/simple.c" + run_pipeline -s src + assert_dir_exists "$CACHE/built" +} + +# ── -f (custom file extension) ─────────────────────────────────────────────── + +@test "cli: -f flag registers a custom file extension for scanning" { + mkdir -p src + # Create a .myext file with C-style doc comments. + # The -f option takes ext;start_regex;end_regex (AWK patterns without slashes). + cat > src/custom.myext <<'SRCEOF' +/* md +@name test/custom-ext +Hello from a custom extension. +*/ +int noop() {} +SRCEOF + run_pipeline -s src -f 'myext;\/\*;\*\/' + assert_dir_exists "$NS/test/custom-ext" +} + +@test "cli: -f registered extension produces correct chunk content" { + mkdir -p src + cat > src/custom.myext <<'SRCEOF' +/* md +@name test/custom-content +Content from custom ext. +*/ +int noop() {} +SRCEOF + run_pipeline -s src -f 'myext;\/\*;\*\/' + assert_contains "$NS/test/custom-content/index.html" "Content from custom ext" +} + +# ── TRBLDOC_MD (markdown renderer override) ────────────────────────────────── + +@test "cli: TRBLDOC_MD overrides the markdown renderer for .md files" { + mkdir -p src + cat > src/doc.md <<'EOF' +@name test/md-override +# Heading +Some content. +EOF + cat > custom_md.sh <<'EOF' +#!/usr/bin/env bash +# Stub: emit a recognisable marker instead of real HTML. +cat "$@" > /dev/null +echo "

custom renderer was here

" +EOF + chmod +x custom_md.sh + TRBLDOC_MD="./custom_md.sh" run_pipeline -s src + assert_contains "$NS/test/md-override/index.html" "custom-md-renderer" +} + +@test "cli: TRBLDOC_MD override is not used when unset (default md2html stub runs)" { + stage_fixture "$FIXTURES/md/simple.md" "src/simple.md" + run_pipeline -s src + # The default stub wraps content in class=stub-md; confirm it ran. + assert_contains "$NS/test/md-simple/index.html" "stub-md" +} + +# ── built-in site assembly ─────────────────────────────────────────────────── + +@test "cli: built output is assembled by default without external zod" { + mkdir -p src + cat > src/simple.c <<'EOF' +/* md +@name test/simple +Hello from a C comment. +*/ +int main() { return 0; } +EOF + run_pipeline -s src + assert_file_exists "$CACHE/built/test/simple/index.html" + assert_contains "$CACHE/built/test/simple/index.html" "" + assert_contains "$CACHE/built/test/simple/index.html" "Hello from a C comment" + assert_not_contains "$CACHE/built/test/simple/index.html" "{{{yield}}}" +} + +@test "cli: default main.layout includes stylesheet and excludes highlightjs mermaid and mathjax" { + stage_fixture "$FIXTURES/c/simple.c" "src/simple.c" + run_pipeline -s src + assert_contains "$NS/main.layout" '' + assert_not_contains "$NS/main.layout" "highlight.min.js" + assert_not_contains "$NS/main.layout" "highlightAll()" + assert_not_contains "$NS/main.layout" "mermaid.min.js" + assert_not_contains "$NS/main.layout" "mermaid.initialize" + assert_not_contains "$NS/main.layout" "mathjax" +} + +@test "cli: -l uses the provided main.layout template" { + stage_fixture "$FIXTURES/c/simple.c" "src/simple.c" + cat > alt.layout <<'EOF' + + +{{ title }} + +
ALT {{ breadcrumb }}
+ +
{{ yield }}
+ + +EOF + run_pipeline -s src -l alt.layout + assert_contains "$NS/main.layout" "ALT {{ breadcrumb }}" + assert_not_contains "$NS/main.layout" "Generated by trbldoc" +} + +@test "cli: -l fails when layout template file does not exist" { + stage_fixture "$FIXTURES/c/simple.c" "src/simple.c" + run bash "$TRBLDOC" -s src -l does-not-exist.layout 2>&1 + [ "$status" -ne 0 ] + [[ "$output" == *"layout template not found"* ]] +} + +@test "cli: files without a registered extension are not scanned" { + mkdir -p src + # .xyz has no registered comment form; no output namespace should appear. + cat > src/ignored.xyz <<'SRCEOF' +/* md +@name test/should-not-exist +This should not be processed. +*/ +SRCEOF + run_pipeline -s src + if [[ -d "$NS/test/should-not-exist" ]]; then + echo "FAILED: .xyz file was unexpectedly processed" >&2 + return 1 + fi +} diff --git a/tests/integration/pipeline.bats b/tests/integration/pipeline.bats new file mode 100644 index 0000000..2f43304 --- /dev/null +++ b/tests/integration/pipeline.bats @@ -0,0 +1,842 @@ +#!/usr/bin/env bats +# tests/integration/pipeline.bats +# End-to-end tests for trbldoc.sh. +# +# External markdown rendering is stubbed via helpers/common.bash. +# Tests inspect both .trblcache/namespace intermediate output and the assembled +# .trblcache/built pages produced by trbldoc itself. +# +# Tests marked `skip` document known bugs from todo.md. Remove `skip` after +# the corresponding bug is fixed. + +load "../helpers/common.bash" + +FIXTURES="$BATS_TEST_DIRNAME/../fixtures" +# REPO_ROOT is exported by run_tests.sh; fall back to relative path when bats +# is invoked directly on this file. +_REPO_ROOT="${REPO_ROOT:-$(cd "$BATS_TEST_DIRNAME/../.." && pwd)}" +TRBLDOC="$_REPO_ROOT/trbldoc.sh" + +setup() { + setup_workspace + CACHE=".trblcache" + NS="$CACHE/namespace" +} + +teardown() { + teardown_workspace +} + +# ── helpers ───────────────────────────────────────────────────────────────── + +# Copy a fixture file into a temporary source tree and run the pipeline. +# Usage: run_pipeline SRC_DIR [additional source dirs...] +run_pipeline() { + bash "$TRBLDOC" "$@" +} + +# Copy a fixture into a local source dir for a clean run. +stage_fixture() { + local src="$1" # path to fixture file or dir + local dest="$2" # destination relative path in tmp workspace + mkdir -p "$(dirname "$dest")" + cp -r "$src" "$dest" +} + +install_busybox_tool_wrapper() { + local tool="$1" + local busybox_bin="$2" + cat > "$STUB_BIN/$tool" <> "\$TRBLDOC_TEST_TOOL_LOG" +fi +exec "$busybox_bin" $tool "\$@" +EOF + chmod +x "$STUB_BIN/$tool" +} + +# ── cache structure ────────────────────────────────────────────────────────── + +@test "pipeline: .trblcache directory is created on run" { + stage_fixture "$FIXTURES/c/simple.c" "src/simple.c" +run_pipeline -s src + assert_dir_exists ".trblcache" +} + +@test "pipeline: namespace subdirectory is created on run" { + stage_fixture "$FIXTURES/c/simple.c" "src/simple.c" + run_pipeline -s src + assert_dir_exists ".trblcache/namespace" +} + +@test "pipeline: chunks subdirectory is created on run" { + stage_fixture "$FIXTURES/c/simple.c" "src/simple.c" + run_pipeline -s src + assert_dir_exists ".trblcache/chunks" +} + +@test "pipeline: global.meta is created" { + stage_fixture "$FIXTURES/c/simple.c" "src/simple.c" + run_pipeline -s src + assert_file_exists "$NS/global.meta" +} + +@test "pipeline: main.layout is created" { + stage_fixture "$FIXTURES/c/simple.c" "src/simple.c" + run_pipeline -s src + assert_file_exists "$NS/main.layout" +} + +@test "pipeline: main.layout passes HTML validation" { + stage_fixture "$FIXTURES/c/simple.c" "src/simple.c" + run_pipeline -s src + assert_valid_html "$NS/main.layout" +} + +@test "pipeline: main.layout has well-formed HTML structure" { + stage_fixture "$FIXTURES/c/simple.c" "src/simple.c" + run_pipeline -s src + local layout="$NS/main.layout" + # must appear before + local html_line head_line body_line footer_line close_body_line + html_line=$(grep -n '' "$layout" | head -1 | cut -d: -f1) + # before + [ "$html_line" -lt "$head_line" ] + # exactly one + local head_count + head_count=$(grep -c '' "$layout") + assert_equal "$head_count" "1" + #