From 657fb0007f39f07cc0401e0c5d03e25df6234aa4 Mon Sep 17 00:00:00 2001 From: Alexander M Pickering Date: Tue, 21 Jul 2026 20:19:46 -0500 Subject: Inital commit. --- scripts/coverage-tests.sh | 143 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 143 insertions(+) create mode 100644 scripts/coverage-tests.sh (limited to 'scripts/coverage-tests.sh') diff --git a/scripts/coverage-tests.sh b/scripts/coverage-tests.sh new file mode 100644 index 0000000..cf03825 --- /dev/null +++ b/scripts/coverage-tests.sh @@ -0,0 +1,143 @@ +#!/bin/sh +# scripts/coverage-tests.sh +# Generate line-coverage reports for trbldoc's shell sources from the test +# suite. Answers "what is the test suite not exercising". +# +# Mechanism: the suite runs the tool via `bash "$TRBLDOC"`, so exporting +# BASH_ENV=scripts/lib/trace-init.sh makes each target process emit an xtrace +# of every executed line. We union the executed lines across the whole run and +# compare against the executable lines in each target source. No modification +# of trbldoc.sh or the tests is required. +# +# POSIX sh; works under mingw bash, WSL bash and Alpine/busybox ash. The traced +# run itself uses bash (the suite already invokes bash), which is one of the +# supported target shells. +# +# Usage: +# scripts/coverage-tests.sh [-s all|unit|integration] [-o OUTDIR] +# [-t "file.sh ..."] [-m MIN_PERCENT] +# +# -s SUITE suite to run (default: integration) +# -o OUTDIR output directory (default: reports/coverage-) +# -t LIST space-separated target source files (default: trbldoc.sh) +# -m MIN_PERCENT fail (exit 2) if overall coverage is below this (default: 0) + +set -u + +SELF="$0" +. "$(dirname -- "$0")/lib/common.sh" +tb_resolve_paths "$SELF" + +SUITE=integration +OUTDIR="" +TARGETS="trbldoc.sh" +MIN_PERCENT=0 + +while getopts ":s:o:t:m:h" opt; do + case "$opt" in + s) SUITE="$OPTARG" ;; + o) OUTDIR="$OPTARG" ;; + t) TARGETS="$OPTARG" ;; + m) MIN_PERCENT="$OPTARG" ;; + h) sed -n '2,30p' "$SELF"; exit 0 ;; + :) tb_die "option -$OPTARG requires an argument" ;; + \?) tb_die "unknown option -$OPTARG" ;; + esac +done + +tb_require awk +tb_require sort + +[ -n "$OUTDIR" ] || OUTDIR="$REPO_ROOT/reports/coverage-$(tb_timestamp)" +case "$OUTDIR" in + /*|[A-Za-z]:[\\/]*) + ;; + *) + OUTDIR="$REPO_ROOT/$OUTDIR" + ;; +esac +mkdir -p "$OUTDIR/html" || tb_die "cannot create output dir: $OUTDIR" +JUNIT_DIR="$OUTDIR/junit"; mkdir -p "$JUNIT_DIR" +TRACE_DIR="$OUTDIR/trace"; rm -rf "$TRACE_DIR"; mkdir -p "$TRACE_DIR" + +SUITE_DIRS="$(tb_suite_dirs "$SUITE")" +SUITE_DIRS_REL="$(tb_suite_dirs_rel "$SUITE")" + +# Basename filter for the trace hook + absolute source paths for reporting. +TRACE_TARGETS="" +SRC_FILES="" +for tf in $TARGETS; do + TRACE_TARGETS="$TRACE_TARGETS ${tf##*/}" + case "$tf" in + /*) p="$tf" ;; + *) p="$REPO_ROOT/$tf" ;; + esac + [ -f "$p" ] || tb_die "target source not found: $p" + SRC_FILES="$SRC_FILES $p" +done + +# ── run suite under xtrace ─────────────────────────────────────────────────── +tb_info "running bats ($SUITE) under coverage instrumentation..." +if tb_bats_usable "$SUITE"; then + BASH_ENV="$LIB_DIR/trace-init.sh" + TRBLDOC_TRACE_DIR="$TRACE_DIR" + TRBLDOC_TRACE_MODE="cov" + TRBLDOC_TRACE_TARGETS="$TRACE_TARGETS" + export BASH_ENV TRBLDOC_TRACE_DIR TRBLDOC_TRACE_MODE TRBLDOC_TRACE_TARGETS + # shellcheck disable=SC2086 + bats --report-formatter junit --output "$JUNIT_DIR" $SUITE_DIRS \ + > "$OUTDIR/bats-output.txt" 2>&1 || tb_info "bats reported failures (see bats-output.txt)" + unset BASH_ENV TRBLDOC_TRACE_DIR TRBLDOC_TRACE_MODE TRBLDOC_TRACE_TARGETS +elif command -v wsl >/dev/null 2>&1; then + tb_info "local bats appears unusable; retrying via WSL bats..." + WSL_REPO="$(tb_to_wsl_path "$REPO_ROOT")" + WSL_LIB="$(tb_to_wsl_path "$LIB_DIR")" + WSL_TRACE="$(tb_to_wsl_path "$TRACE_DIR")" + WSL_JUNIT="$(tb_to_wsl_path "$JUNIT_DIR")" + wsl -- bash -lc \ + "cd '$WSL_REPO' && export BASH_ENV='$WSL_LIB/trace-init.sh' TRBLDOC_TRACE_DIR='$WSL_TRACE' TRBLDOC_TRACE_MODE='cov' TRBLDOC_TRACE_TARGETS='$TRACE_TARGETS'; bats --report-formatter junit --output '$WSL_JUNIT' $SUITE_DIRS_REL" \ + > "$OUTDIR/bats-output.txt" 2>&1 || tb_info "bats (WSL) reported failures (see bats-output.txt)" +else + tb_die "local bats command appears unusable and no WSL fallback is available" +fi + +if ! ls "$TRACE_DIR"/trace.*.log >/dev/null 2>&1; then + if grep -qi "bash\\\\r" "$OUTDIR/bats-output.txt" 2>/dev/null; then + tb_die "bats appears broken (CRLF shebang: 'bash\\r'). Reinstall bats or run dos2unix on the bats scripts, then retry." + fi + tb_die "no trace data captured; ensure bats actually runs tests (see $OUTDIR/bats-output.txt)" +fi + +# ── union executed lines ───────────────────────────────────────────────────── +awk -f "$LIB_DIR/trace-extract.awk" -v MODE=cov "$TRACE_DIR"/trace.*.log \ + | sort -u > "$OUTDIR/covered.tsv" + +# ── per-file coverage report + summary ─────────────────────────────────────── +# shellcheck disable=SC2086 +awk -f "$LIB_DIR/coverage-report.awk" \ + -v COVERED="$OUTDIR/covered.tsv" \ + -v HTMLDIR="$OUTDIR/html" \ + $SRC_FILES > "$OUTDIR/summary.rows" + +{ + printf 'file,executable,covered,percent\n' + cat "$OUTDIR/summary.rows" +} > "$OUTDIR/summary.csv" + +awk -f "$LIB_DIR/render-coverage-index.awk" \ + -v TITLE="trbldoc coverage ($SUITE)" \ + "$OUTDIR/summary.rows" > "$OUTDIR/index.html" + +# ── overall percent + threshold gate ───────────────────────────────────────── +OVERALL="$(awk -F',' '{t+=$2; c+=$3} END{ if(t>0) printf "%.1f", c*100.0/t; else print "0.0" }' "$OUTDIR/summary.rows")" + +printf '\n==== coverage summary (%s) ====\n' "$SUITE" +awk -F',' '{ printf "%6.1f%% %-20s (%d/%d)\n", $4, $1, $3, $2 }' "$OUTDIR/summary.rows" +printf ' overall: %s%%\n' "$OVERALL" +printf '\nReports written to: %s\n' "$OUTDIR" +printf ' coverage index: %s\n' "$OUTDIR/index.html" + +# Threshold enforcement (integer or decimal compare via awk). +if awk -v o="$OVERALL" -v m="$MIN_PERCENT" 'BEGIN{ exit !(m>0 && o+0 < m+0) }'; then + tb_die "coverage ${OVERALL}% is below the required minimum ${MIN_PERCENT}%" +fi -- cgit v1.2.3-70-g09d2