From f5f2fe7d83ca2f6b9b54b6a566080581771f710c Mon Sep 17 00:00:00 2001 From: Julian Noble Date: Tue, 21 Jul 2026 13:10:46 +1000 Subject: [PATCH] G-107: per-library testsuite steps with gate/record/skip policy tiers The external libraries the suite builds and ships now run their own testsuites under the suite-built shell as named recipe steps - evidence for the exact zig-built combinations nobody upstream tests. tools/test_gate.tcl generalized into the shared engine: -mode gate|record (gate = parsed totals diffed against the tracked dispositioned baseline, the core-gate mechanism; record = run must complete, failures recorded not fatal), -driver selection (tcllib-family trees run via their own support/devel/all.tcl - sak's underlying driver, standard aggregate totals, per-file child interps), -summaryfile/-library line-record evidence summaries for the G-103 artifact-metadata consumer (shape documented in the goal detail file), file-error evidence capture (@+ blocks, runAllTests error-exit sections). A run with no parseable totals FAILS the step in every mode - infrastructure breakage must never masquerade as recorded results. Core gate semantics unchanged; the core gate step now also writes out/testreports/tclcore.summary. build905.zig: steps test-thread test-tclvfs test-tcllib test-tklib test-tk + composite test-libraries (tk excluded - opt-in: interactive desktop, maps windows, tens of minutes; intended before publishing bi-family artifacts). Per-library -Dtestpolicy-=gate|record|skip and -Dtestargs- invocation overrides - no recipe edits needed. Baselines staged in both flows (suite.tcl item list + bootstrap recipe_items). suite.tcl: test action takes -steps (default test-gate) and a -zigargs passthrough (both actions); passes -j1 so combined test invocations serialize (zig runs independent steps concurrently; thread's suite is timing-sensitive). Census results (suite tclsh90s.exe 9.0.5, zig 0.16.0): thread 143/0 failed, EMPTY baseline, GATE PASS x3 identical consecutive runs; tclvfs 51/7 failed, all dispositioned (3x tcl9 glob-nomatch semantics through vfs handlers, 4x dead-FTP network tests), GATE PASS x3 identical; tcllib WITH tcllibc accelerators engaged (E tcllibc markers, crc32-critcl-* variants running) RECORDED 62328/54 failed; tklib RECORDED 796/18 failed (widgetPlus drift under Tk 9). No-totals failure path verified in both modes. Assisted-by: harness=claude; primary-model=claude-fable-5; api-location=anthropic.com --- src/buildsuites/suite_tcl90/README.md | 57 +++++- src/buildsuites/suite_tcl90/build905.zig | 95 ++++++++++ .../expected_test_failures_tclvfs.txt | 25 +++ .../expected_test_failures_thread.txt | 10 + src/buildsuites/suite_tcl90/suite.tcl | 39 +++- .../suite_tcl90/tools/test_gate.tcl | 172 ++++++++++++++++-- 6 files changed, 368 insertions(+), 30 deletions(-) create mode 100644 src/buildsuites/suite_tcl90/expected_test_failures_tclvfs.txt create mode 100644 src/buildsuites/suite_tcl90/expected_test_failures_thread.txt diff --git a/src/buildsuites/suite_tcl90/README.md b/src/buildsuites/suite_tcl90/README.md index 2cbbca92..50ed38d4 100644 --- a/src/buildsuites/suite_tcl90/README.md +++ b/src/buildsuites/suite_tcl90/README.md @@ -47,6 +47,58 @@ See `suite.tcl`'s header for options (`-zig`, `-optimize`, `-tclbranch`, `-refre `-steps`). Zig resolution: `-zig` > `$env(PUNK_ZIG)` > the pinned toolchain at `bin/tools/zig-x86_64-windows-0.16.0/zig.exe` > `zig` on PATH. +## Library test runs (G-107) + +The external libraries the suite builds and ships run their OWN testsuites under +the suite-built shell, as named recipe steps with a per-library policy tier - +evidence for the exact zig-built combinations nobody upstream tests: + + tclsh suite.tcl test # core gate (test-gate), as before + tclsh suite.tcl test -steps {test-libraries} # thread+tclvfs GATED, tcllib+tklib RECORDED + tclsh suite.tcl test -steps {test-thread} # individual: test-thread test-tclvfs + # test-tcllib test-tklib test-tk + tclsh suite.tcl test -steps {test-tcllib} -zigargs {-Dtestpolicy-tcllib=gate} + +Policy tiers (`-Dtestpolicy-=gate|record|skip`, defaults in the recipe, +overridable at invocation without recipe edits): + +- **gate** (default: thread, tclvfs) - parsed totals diffed against the tracked + dispositioned baseline `expected_test_failures_.txt` (core-gate + semantics including stale-entry notes). Small, load-bearing for every kit. +- **record** (default: tcllib, tklib, tk) - the run must complete; failures are + recorded, not fatal. Emits `out/testreports/.log` (full output) and + `out/testreports/.summary` (line-record evidence summary - shape + documented in `goals/G-107-buildsuite-library-tests.md`, consumed by the + G-103 artifact metadata; the core gate writes `tclcore.summary` there too). + Promote a library to gate once its census is dispositioned and two-run + deterministic. +- **skip** - the named step becomes a no-op (composite tuning). + +EVERY mode fails the step when the run produces no parseable totals line - +infrastructure breakage must never masquerade as recorded results. Per-library +driver args: `-Dtestargs-="..."` (e.g `-notfile x.test` to disposition a +problem file at invocation level; recorded in the evidence summary). + +`test-libraries` runs thread/tclvfs/tcllib/tklib. The full Tk suite is +deliberately OUTSIDE it: **`test-tk` is opt-in** (needs an interactive desktop, +maps real windows, steals focus, runs tens of minutes) - intended before +PUBLISHING bi-family artifacts, not on every build. + +Drivers: thread/tclvfs/tk use their trees' own `tests/all.tcl`. tcllib/tklib +are driven via their own `support/devel/all.tcl` (sak's underlying driver, +bypassing sak's log format): per-file child interps, aggregated standard +totals, and - because the installed `tcllibc` is on the shell's package path - +the tcllib suites run WITH the critcl accelerators engaged (`E tcllibc ` +markers in the log). A test file that errors out is recorded in the summary +(`errored_files` / `errorexit_files`) as evidence; the gate criterion itself +stays totals-vs-baseline. Known residual: the old-style thread/tclvfs drivers +catch a mid-file abort without a structured marker, so such an abort shows up +only as a failure-count drop plus stale-baseline notes. + +Timing-sensitive suites (thread) must not run in parallel with heavy ones: +`suite.tcl test` passes `-j1` so combined step invocations serialize (zig +otherwise runs independent steps concurrently). + ## Pinned zig The suite pin is **zig 0.16.0** (current official release; adopted 2026-07-20 - @@ -82,7 +134,10 @@ producing stale/duplicate artifacts at link time). `stagetree.zig` (bootstrap-mode source/recipe materializer), `zipfs_mkzip.tcl` / `zipfs_mkimg.tcl` (make-zipfs drivers), and the suite-built-shell step scripts `test_gate.tcl` / `suite_smoke.tcl` / - `pkg_smoke.tcl` (G-102). + `pkg_smoke.tcl` (G-102). `test_gate.tcl` is the shared testsuite engine + (G-107): gate/record modes, driver selection, evidence summaries. +- `expected_test_failures.txt` (tcl core) / `expected_test_failures_thread.txt` / + `expected_test_failures_tclvfs.txt` - tracked dispositioned gate baselines. - `patches/` - recovered 2024 experiment patches, including TCLSH_PIPEREPL (see `patches/README.md`) - compiled into the SEPARATE `tclsh90spr`/ `tclsh90sprzip` products; the stock shells stay unpatched. diff --git a/src/buildsuites/suite_tcl90/build905.zig b/src/buildsuites/suite_tcl90/build905.zig index ac52e356..3c5e614d 100644 --- a/src/buildsuites/suite_tcl90/build905.zig +++ b/src/buildsuites/suite_tcl90/build905.zig @@ -2329,6 +2329,7 @@ pub fn build(b: *std.Build) !void { //-- test-gate: the core testsuite under the built shell, gated on parsed //totals vs the tracked dispositioned baseline (all.tcl's exit code lies) + const testreports_dir = common.replaceAll(b, b.pathJoin(&.{ b.install_path, "testreports" }), "\\", "/"); const gate_step = b.step("test-gate", "tcl core testsuite under the built shell, gated on parsed totals vs expected_test_failures.txt"); const testargs_opt = b.option([]const u8, "testargs", "extra tcltest args for test-gate (e.g -file http.test)") orelse ""; const gate_run = b.addRunArtifact(tclsh_exe); @@ -2336,17 +2337,109 @@ pub fn build(b: *std.Build) !void { gate_run.setEnvironmentVariable("TCL_LIBRARY", tcl_library_src); gate_run.has_side_effects = true; gate_run.addFileArg(b.path("tools/test_gate.tcl")); + gate_run.addArgs(&.{ "-library", "tclcore" }); gate_run.addArg("-testsdir"); gate_run.addArg(common.replaceAll(b, b.pathFromRoot(tcl_source_folder ++ "/tests"), "\\", "/")); gate_run.addArg("-baseline"); gate_run.addFileArg(b.path("expected_test_failures.txt")); gate_run.addArg("-logfile"); gate_run.addArg(common.replaceAll(b, b.pathFromRoot("../tcltest.log"), "\\", "/")); + gate_run.addArg("-summaryfile"); + gate_run.addArg(b.fmt("{s}/tclcore.summary", .{testreports_dir})); if (testargs_opt.len != 0) { gate_run.addArgs(&.{ "-testargs", testargs_opt }); } gate_run.step.dependOn(b.getInstallStep()); gate_step.dependOn(&gate_run.step); + + //-- per-library testsuites (G-107): the external libraries the suite builds + //and ships run their own testsuites under the suite-built shell, with a + //per-library policy tier resolved at invocation (no recipe edits needed): + // gate parsed totals diffed against the tracked dispositioned baseline + // (core-gate semantics; default for the small load-bearing libs) + // record run must complete and emit report artifacts under + // /testreports/ (evidence for the exact zig-built + // combinations nobody upstream tests); failures recorded, not fatal + // skip named step becomes a no-op (composite tuning) + //Override per-library: -Dtestpolicy-=gate|record|skip. Per-library + //driver/tcltest args: -Dtestargs-="..." (e.g '-notfile x.test' to + //disposition a problem file; recorded in the evidence summary). + //'test-libraries' runs the default set (thread tclvfs tcllib tklib). tk's + //full suite is deliberately OUTSIDE it: opt-in step 'test-tk' (needs an + //interactive desktop, maps real windows, runs tens of minutes; intended + //before PUBLISHING bi-family artifacts, not on every build). + //NOTE zig runs independent steps concurrently - for gate determinism of + //timing-sensitive suites drive combined test invocations with -j1 + //(suite.tcl's test action does). + const LibSuite = struct { + name: []const u8, //step suffix, policy/testargs option suffix, report basename + testsdir: []const u8, //run cwd (from build root) + driver: []const u8, //driver script, relative to testsdir + default_policy: []const u8, + baseline: []const u8, //tracked baseline filename ("" = none tracked yet) + deps: []const *std.Build.Step, //beyond the top-level install step + }; + const thread_test_deps = [_]*std.Build.Step{install_libraries}; + const tclvfs_test_deps = [_]*std.Build.Step{install_libraries}; + const tcllib_test_deps = [_]*std.Build.Step{ install_libraries, &tcllib_install.step, &tcllibc_install.step }; + const tklib_test_deps = [_]*std.Build.Step{ install_libraries, &tklib_install.step, &tcllib_install.step }; + const tk_test_deps = [_]*std.Build.Step{install_libraries}; + const lib_suites = [_]LibSuite{ + //thread's testsuite requires the tree's exact version - the recipe derives + //the installed package version from the same tree (build_tclthread). + .{ .name = "thread", .testsdir = "../tclthread/tests", .driver = "all.tcl", .default_policy = "gate", .baseline = "expected_test_failures_thread.txt", .deps = &thread_test_deps }, + .{ .name = "tclvfs", .testsdir = "../tclvfs/tests", .driver = "all.tcl", .default_policy = "gate", .baseline = "expected_test_failures_tclvfs.txt", .deps = &tclvfs_test_deps }, + //tcllib via its own support/devel/all.tcl (sak's underlying driver - + //standard aggregate totals, per-file child interps). The installed + //tcllibc is on the shell's package path, so accelerator-aware suites + //engage the critcl accelerators ('E tcllibc ' markers in the log) - + //the single highest-value run here: zig-built accelerators against a + //static tclsh is a combination nobody upstream tests. + .{ .name = "tcllib", .testsdir = "../tcllib", .driver = "support/devel/all.tcl", .default_policy = "record", .baseline = "", .deps = &tcllib_test_deps }, + .{ .name = "tklib", .testsdir = "../tklib", .driver = "support/devel/all.tcl", .default_policy = "record", .baseline = "", .deps = &tklib_test_deps }, + .{ .name = "tk", .testsdir = "../tk9/tests", .driver = "all.tcl", .default_policy = "record", .baseline = "", .deps = &tk_test_deps }, + }; + const libtests_step = b.step("test-libraries", "library testsuites under the built shell per policy tier (default: thread+tclvfs gate, tcllib+tklib record; tk excluded - see test-tk)"); + for (lib_suites) |ls| { + const policy = b.option([]const u8, b.fmt("testpolicy-{s}", .{ls.name}), b.fmt("{s} testsuite policy: gate|record|skip (default {s})", .{ ls.name, ls.default_policy })) orelse ls.default_policy; + const lib_testargs = b.option([]const u8, b.fmt("testargs-{s}", .{ls.name}), b.fmt("extra tcltest/driver args for the {s} testsuite", .{ls.name})) orelse ""; + const is_tk = std.mem.eql(u8, ls.name, "tk"); + const step = b.step(b.fmt("test-{s}", .{ls.name}), b.fmt("{s} testsuite under the built shell (policy: {s})", .{ ls.name, policy })); + if (!is_tk) libtests_step.dependOn(step); + if (std.mem.eql(u8, policy, "skip")) continue; + if (!std.mem.eql(u8, policy, "gate") and !std.mem.eql(u8, policy, "record")) { + std.debug.panic("-Dtestpolicy-{s}: unknown policy '{s}' (expected gate|record|skip)", .{ ls.name, policy }); + } + const run = b.addRunArtifact(tclsh_exe); + common.scrubTclEnv(run); + run.setEnvironmentVariable("TCL_LIBRARY", tcl_library_src); + run.has_side_effects = true; + run.addFileArg(b.path("tools/test_gate.tcl")); + run.addArgs(&.{ "-library", ls.name, "-mode", policy }); + run.addArg("-testsdir"); + run.addArg(common.replaceAll(b, b.pathFromRoot(ls.testsdir), "\\", "/")); + run.addArgs(&.{ "-driver", ls.driver }); + if (std.mem.eql(u8, policy, "gate")) { + run.addArg("-baseline"); + if (ls.baseline.len != 0) { + run.addFileArg(b.path(ls.baseline)); + } else { + //no tracked baseline yet: pass the conventional name so the + //engine's missing-baseline message names the file to create + run.addArg(common.replaceAll(b, b.pathFromRoot(b.fmt("expected_test_failures_{s}.txt", .{ls.name})), "\\", "/")); + } + } + run.addArg("-logfile"); + run.addArg(b.fmt("{s}/{s}.log", .{ testreports_dir, ls.name })); + run.addArg("-summaryfile"); + run.addArg(b.fmt("{s}/{s}.summary", .{ testreports_dir, ls.name })); + if (lib_testargs.len != 0) { + run.addArgs(&.{ "-testargs", lib_testargs }); + } + run.step.dependOn(b.getInstallStep()); + for (ls.deps) |dep| run.step.dependOn(dep); + step.dependOn(&run.step); + } } // ================== } @@ -2425,6 +2518,8 @@ fn bootstrapMode(b: *std.Build) !void { "build_tk", "critcl_zig.config", "expected_test_failures.txt", + "expected_test_failures_thread.txt", + "expected_test_failures_tclvfs.txt", "patches", "src", "tools", diff --git a/src/buildsuites/suite_tcl90/expected_test_failures_tclvfs.txt b/src/buildsuites/suite_tcl90/expected_test_failures_tclvfs.txt new file mode 100644 index 00000000..cba0e59b --- /dev/null +++ b/src/buildsuites/suite_tcl90/expected_test_failures_tclvfs.txt @@ -0,0 +1,25 @@ +# suite_tcl90 expected tclvfs-testsuite failures (G-107 library test gate baseline). +# One test name per line; comments carry the disposition reason. The test-tclvfs +# step (policy gate) fails on any failed test NOT listed here, and notes listed +# tests that no longer fail (candidates for removal). +# +# Baseline established 2026-07-21 against tclvfs trunk (vfs 1.4.2), suite-built +# tclsh90s.exe 9.0.5, machine superbee (windows 11). Census totals at capture: +# 51 run / 39 passed / 5 skipped / 7 failed, all dispositioned below. + +# -- tcl9 glob-nomatch semantics through vfs handlers: glob on a non-matching +# pattern / non-present directory inside a mounted vfs returns an empty result +# where the tests (written for tcl8-era vfs) expect the "no files matched" +# error. Consistent behaviour drift, not a build defect; candidate upstream +# test/behaviour reconciliation. +vfs-4.2 +vfsZip-1.6 +vfsZip-1.7 + +# -- network-reaching tests (the suite census runs offline-tolerant): remote +# FTP hosts (ftp.ucsd.edu, ftp.tcl.tk) unreachable/retired. The goal survey +# (G-107) anticipated skipping/dispositioning vfsFtp/vfsUrl. +vfsFtp-1.1 +vfsFtp-1.2 +vfsFtp-1.3 +vfsUrl-2.1 diff --git a/src/buildsuites/suite_tcl90/expected_test_failures_thread.txt b/src/buildsuites/suite_tcl90/expected_test_failures_thread.txt new file mode 100644 index 00000000..f87d209f --- /dev/null +++ b/src/buildsuites/suite_tcl90/expected_test_failures_thread.txt @@ -0,0 +1,10 @@ +# suite_tcl90 expected thread-testsuite failures (G-107 library test gate baseline). +# One test name per line; comments carry the disposition reason. The test-thread +# step (policy gate) fails on any failed test NOT listed here, and notes listed +# tests that no longer fail (candidates for removal). +# +# Baseline established 2026-07-21 against thread trunk (3.0.7), suite-built +# tclsh90s.exe 9.0.5, machine superbee (windows 11). Census totals at capture: +# 143 run / 117 passed / 26 skipped / 0 failed - EMPTY baseline; skips are +# constraint-driven (chanTransfer windows, have_gdbm/have_lmdb backends not +# built into the suite's psGdbm/psLmdb compile). diff --git a/src/buildsuites/suite_tcl90/suite.tcl b/src/buildsuites/suite_tcl90/suite.tcl index 7e9a8453..e6144e81 100644 --- a/src/buildsuites/suite_tcl90/suite.tcl +++ b/src/buildsuites/suite_tcl90/suite.tcl @@ -21,8 +21,18 @@ # -repofolder fossil clone store (default ~/.punkshell/fossils; env PUNK_FOSSIL_STORE) # -seedfossils read-only seed collection for missing clones (default ~/.fossils; "" disables) # -all <0|1> with 'clean': reserved (the clone store lives outside the stage) -# -steps zig build steps (default {install install-libraries make-zipfs -# smoke tklib tcllib tcllibc} - the full pipeline short of test-gate) +# -steps zig build steps. 'build' default: {install install-libraries +# make-zipfs smoke tklib tcllib tcllibc} - the full pipeline +# short of the test steps. 'test' default: {test-gate} (the +# core suite gate); other test steps (G-107 library runs): +# test-libraries (thread+tclvfs gated, tcllib+tklib recorded), +# test-thread test-tclvfs test-tcllib test-tklib, and the +# OPT-IN test-tk (full Tk suite: interactive desktop, maps +# windows, tens of minutes - intended before publishing +# bi-family artifacts, not on every build) +# -zigargs extra args appended to the zig build invocation, e.g +# {-Dtestpolicy-tcllib=gate -Dtestargs-thread=-notfile\ x.test} +# (per-library policy/testargs overrides without recipe edits) # #The build area lives entirely under src/buildsuites/_build (VCS-ignored via the #existing _build globs). Fossil clones live in the PUNKSHELL-OWNED machine-level @@ -68,9 +78,11 @@ array set opt { -refresh 0 -steps {install install-libraries make-zipfs smoke tklib tcllib tcllibc} -testargs {} + -zigargs {} -repofolder {} -all 0 } +set explicitopts {} set opt(-seedfossils) [file normalize [file join $env(HOME) .fossils]] #zig resolution: -zig option > PUNK_ZIG env > the suite's pinned toolchain under #bin/tools (zig 0.16.0 official release - hosted on punkbin, minisign-verified; @@ -87,6 +99,7 @@ if {[info exists env(PUNK_ZIG)]} { foreach {k v} [lrange $argv 1 end] { if {![info exists opt($k)]} {fail "unknown option '$k'"} set opt($k) $v + lappend explicitopts $k } #suite-owned clone store resolution: -repofolder > PUNK_FOSSIL_STORE > ~/.punkshell/fossils @@ -148,14 +161,22 @@ if {$action eq "clean"} { } if {$action eq "test"} { - #Delegates to the recipe's test-gate step (tools/test_gate.tcl under the - #suite-built shell): core testsuite gated on parsed totals vs the tracked - #expected_test_failures.txt baseline. Full output: /tcltest.log. - set extraopts {} + #Delegates to the recipe's test steps (tools/test_gate.tcl under the + #suite-built shell). Default step: test-gate (core testsuite gated on parsed + #totals vs the tracked expected_test_failures.txt baseline; full output + #/tcltest.log). -steps overrides for the G-107 library runs + #(test-libraries, test-, opt-in test-tk); record-mode report artifacts + #land under /out/testreports/. -j1: zig runs independent steps + #concurrently - combined test invocations must not run timing-sensitive + #suites in parallel with heavy ones. + set extraopts {-j1} if {[llength $opt(-testargs)]} { lappend extraopts -Dtestargs=$opt(-testargs) } - run_zig_steps {test-gate} $extraopts + lappend extraopts {*}$opt(-zigargs) + set teststeps [expr {"-steps" in $explicitopts ? $opt(-steps) : {test-gate}}] + run_zig_steps $teststeps $extraopts + log "test steps done ([join $teststeps {, }]) - evidence summaries/logs (when written): [file join $stage out testreports]" exit 0 } @@ -226,7 +247,7 @@ proc git_source {name url ref dir} { #bootstrap mode's recipe_items (build905.zig bootstrapMode). log "staging recipe -> $builddir" file mkdir $builddir -foreach item {build905.zig build_common.zig build_libtommath build_zlib build_tclvfs build_tclthread build_tk critcl_zig.config expected_test_failures.txt patches src tools} { +foreach item {build905.zig build_common.zig build_libtommath build_zlib build_tclvfs build_tclthread build_tk critcl_zig.config expected_test_failures.txt expected_test_failures_thread.txt expected_test_failures_tclvfs.txt patches src tools} { file delete -force [file join $builddir $item] file copy [file join $suiteroot $item] $builddir } @@ -287,6 +308,6 @@ if {![regexp {#\s*define\s+TCL_PATCH_LEVEL\s+"([^"]+)"} $tclh -> patchlevel]} {f log "sources: tcl $patchlevel (checkout [string range $uuid 0 11]...)" #-- build (staged recipe steps) ------------------------------------------- -run_zig_steps $opt(-steps) +run_zig_steps $opt(-steps) $opt(-zigargs) log "PASS - built $patchlevel (checkout [string range $uuid 0 11]...) -> $stage/out" diff --git a/src/buildsuites/suite_tcl90/tools/test_gate.tcl b/src/buildsuites/suite_tcl90/tools/test_gate.tcl index 18e5cad1..04e3d43d 100644 --- a/src/buildsuites/suite_tcl90/tools/test_gate.tcl +++ b/src/buildsuites/suite_tcl90/tools/test_gate.tcl @@ -1,43 +1,173 @@ -#test_gate.tcl (G-102): run the Tcl core testsuite under THIS interpreter's own -#shell ([info nameofexecutable] - the suite-built tclsh when driven as the -#'zig build test-gate' step) and GATE on the parsed totals: all.tcl's exit code -#does NOT reflect failures, so every failed test must appear in the tracked -#dispositioned baseline or the gate fails listing the unexplained names. -#Formerly suite.tcl's 'test' action; suite.tcl now delegates here. +#test_gate.tcl (G-102 core gate; generalized for the G-107 library test runs): +#run a tcltest-driven testsuite under THIS interpreter's own shell +#([info nameofexecutable] - the suite-built tclsh when driven as a +#'zig build test-*' step) and act on the PARSED totals: tcltest drivers' exit +#codes do NOT reflect failures, so results are judged from the captured log. # -#args: -testsdir -baseline -logfile ?-testargs {tcltest args}? +#Modes (-mode, default gate): +# gate every failed test must appear in the tracked dispositioned baseline +# or the gate fails listing the unexplained names (the core test-gate +# mechanism, G-098); notes baseline entries that no longer fail. +# record the run must COMPLETE (totals parsed) but failures do not fail the +# step - results are recorded as evidence (G-107 library policy tier +# for suites without a dispositioned baseline yet). +#EVERY mode: a run with no parseable aggregate totals line FAILS the step - +#infrastructure breakage must never masquerade as recorded results. +# +#Driver: /all.tcl by default. tcllib-family trees are driven via +#their own support/devel/all.tcl (-driver), which runs every module's test +#files in per-file child interps and aggregates counts into the same standard +#totals line ('all.tcl: Total N Passed N Skipped N Failed N'). That driver +#marks a test file that errors out with '@+'/'@|' errorInfo blocks - counted +#here as errored_files evidence (recorded in the summary, not a gate criterion; +#the gate criterion stays totals-vs-baseline, core-gate semantics). The +#runAllTests-vintage drivers (tcl core, tk) report 'Test files exiting with +#errors:' - captured as errorexit_files evidence the same way. +# +#Evidence summary (-summaryfile): a line-record file for artifact-metadata +#consumers (G-103). Each non-comment line is a Tcl list: ' ' +#(value may itself be a list). Shape documented in +#goals/G-107-buildsuite-library-tests.md. +# +#args: -testsdir -logfile ?-mode gate|record? +# ?-baseline ? (required in gate mode) +# ?-driver ? (default all.tcl; relative paths resolve against -testsdir) +# ?-summaryfile ? ?-library ? ?-testargs {tcltest args}? proc fail {msg} {puts stderr "test_gate ERROR: $msg"; flush stderr; exit 1} -array set opt {-testsdir {} -baseline {} -logfile {} -testargs {}} +array set opt {-testsdir {} -baseline {} -logfile {} -testargs {} -mode gate -driver all.tcl -summaryfile {} -library {}} foreach {k v} $argv { - if {![info exists opt($k)]} {fail "unknown option '$k' (expected -testsdir -baseline -logfile -testargs)"} + if {![info exists opt($k)]} {fail "unknown option '$k' (expected -testsdir -baseline -logfile -mode -driver -summaryfile -library -testargs)"} set opt($k) $v } -foreach req {-testsdir -baseline -logfile} { +foreach req {-testsdir -logfile} { if {$opt($req) eq ""} {fail "missing required option $req"} } +if {$opt(-mode) ni {gate record}} {fail "unknown -mode '$opt(-mode)' (expected gate|record)"} if {![file isdirectory $opt(-testsdir)]} {fail "no tests dir at $opt(-testsdir) - build first"} -if {![file exists $opt(-baseline)]} {fail "no baseline file at $opt(-baseline)"} +if {$opt(-mode) eq "gate"} { + if {$opt(-baseline) eq ""} {fail "-mode gate requires -baseline"} + if {![file exists $opt(-baseline)]} {fail "no baseline file at $opt(-baseline) - for a first census run the step in record mode (-Dtestpolicy-=record), disposition the recorded failures into the baseline, then gate"} +} +set library $opt(-library) +if {$library eq ""} {set library [file tail [file normalize $opt(-testsdir)]]} set exe [info nameofexecutable] set logfile [file normalize $opt(-logfile)] -puts "test_gate: tcl testsuite under $exe [expr {[llength $opt(-testargs)] ? $opt(-testargs) : "(full suite)"}] -> $logfile" +file mkdir [file dirname $logfile] +set driver $opt(-driver) +if {[file pathtype $driver] ne "absolute"} {set driver [file join $opt(-testsdir) $driver]} +set driver [file normalize $driver] +if {![file exists $driver]} {fail "no driver script at $driver"} + +#-- evidence summary (written on every outcome once the run has been attempted) +set summary [dict create] +proc sumset {key value} {global summary; dict set summary $key $value} +proc write_summary {} { + global opt summary + if {$opt(-summaryfile) eq ""} return + set sf [file normalize $opt(-summaryfile)] + file mkdir [file dirname $sf] + set f [open $sf w] + fconfigure $f -translation lf + puts $f "# library-test evidence summary (G-107; consumed by G-103 artifact metadata)" + puts $f "# line records: - parse each non-comment line as a Tcl list." + puts $f "# shape: goals/G-107-buildsuite-library-tests.md (punkshell repo)" + dict for {k v} $summary {puts $f [list $k $v]} + close $f +} +proc finish {result failreason msg} { + #single exit point after the run: record outcome, write summary, report + global opt logfile + sumset result $result + if {$failreason ne ""} {sumset failreason $failreason} + write_summary + if {$opt(-summaryfile) ne ""} { + puts "test_gate: evidence: [file normalize $opt(-summaryfile)] (log: $logfile)" + } + if {$result eq "pass"} { + puts "test_gate: $msg" + exit 0 + } + puts stderr "test_gate $msg" + flush stderr + exit 1 +} + +sumset library $library +sumset mode $opt(-mode) +sumset shell $exe +sumset driver $driver +sumset testsdir [file normalize $opt(-testsdir)] +sumset testargs $opt(-testargs) +sumset baseline [expr {$opt(-baseline) ne "" ? [file normalize $opt(-baseline)] : ""}] +sumset logfile $logfile + +puts "test_gate: $library testsuite under $exe [expr {[llength $opt(-testargs)] ? $opt(-testargs) : "(full suite)"}] (mode $opt(-mode)) -> $logfile" flush stdout set savedpwd [pwd] cd $opt(-testsdir) -catch {exec $exe all.tcl {*}$opt(-testargs) > $logfile 2>@1} +catch {exec $exe $driver {*}$opt(-testargs) > $logfile 2>@1} cd $savedpwd set f [open $logfile r]; set testout [read $f]; close $f -if {![regexp {all\.tcl:\s+Total\s+(\d+)\s+Passed\s+(\d+)\s+Skipped\s+(\d+)\s+Failed\s+(\d+)} $testout -> ntotal npassed nskipped nfailed]} { - fail "no all.tcl totals line found in $logfile - run did not complete" + +#aggregate totals: the master driver's final cleanupTests/runAllTests line. +#Take the LAST match - per-file summary lines carry the file's own name, but a +#driver arrangement that reuses 'all.tcl' for a nested stage must not shadow +#the final aggregate. +set quads [regexp -all -inline {all\.tcl:\s+Total\s+(\d+)\s+Passed\s+(\d+)\s+Skipped\s+(\d+)\s+Failed\s+(\d+)} $testout] +if {![llength $quads]} { + finish fail no-totals "FAIL ($library): no all.tcl totals line found in $logfile - run did not complete" } +lassign [lrange $quads end-3 end] ntotal npassed nskipped nfailed puts "test_gate: totals: $ntotal run, $npassed passed, $nskipped skipped, $nfailed failed" +sumset total $ntotal +sumset passed $npassed +sumset skipped $nskipped +sumset failed $nfailed + set failednames {} foreach {- tname} [regexp -all -inline -line {^==== (\S+) FAILED} $testout] { if {$tname ni $failednames} {lappend failednames $tname} } +sumset failednames $failednames + +#file-level completeness evidence (recorded, not gate criteria - the gate +#criterion is totals-vs-baseline, matching the core gate): +#tcllib-family driver: a test file that errors mid-source is caught and marked +#with an '@+' line followed by '@|' errorInfo lines. +set errored_count 0 +foreach line [split $testout \n] { + if {$line eq "@+"} {incr errored_count} +} +sumset errored_files $errored_count +#runAllTests-vintage drivers (tcl core, tk): 'Test files exiting with errors:' +#section lists files whose child run errored. +set errorexit_files {} +if {[regexp -indices {Test files exiting with errors:} $testout headidx]} { + set tailtext [string range $testout [lindex $headidx 1]+1 end] + foreach line [split $tailtext \n] { + set line [string trim $line] + if {$line eq ""} continue + if {[regexp {^(\S+\.test)$} $line -> fname]} { + lappend errorexit_files $fname + } else { + break + } + } +} +sumset errorexit_files $errorexit_files +if {$errored_count || [llength $errorexit_files]} { + puts "test_gate: note: file-level errors recorded: errored_files=$errored_count errorexit_files=[llength $errorexit_files] ([join $errorexit_files { }])" +} + +if {$opt(-mode) eq "record"} { + finish pass "" "RECORDED ($library): $ntotal run, $nfailed failed ([llength $failednames] distinct names) - evidence artifacts written" +} + +#-- gate mode: diff failed names against the dispositioned baseline #baseline: one test name per line; '#' comments carry the disposition reasons set baseline {} set f [open $opt(-baseline) r] @@ -51,9 +181,10 @@ set unexplained {} foreach tname $failednames { if {$tname ni $baseline} {lappend unexplained $tname} } +sumset unexplained $unexplained +set stale {} if {![llength $opt(-testargs)]} { #only meaningful for full runs: baseline entries that no longer fail - set stale {} foreach tname $baseline { if {$tname ni $failednames} {lappend stale $tname} } @@ -61,11 +192,12 @@ if {![llength $opt(-testargs)]} { puts "test_gate: note: [llength $stale] baseline entries did not fail this run (candidates for removal): $stale" } } +sumset stalebaseline $stale if {[llength $unexplained]} { - puts stderr "test_gate GATE FAIL: [llength $unexplained] failed test(s) not in the expected-failure baseline:" + puts stderr "test_gate GATE FAIL ($library): [llength $unexplained] failed test(s) not in the expected-failure baseline:" foreach tname $unexplained {puts stderr " $tname"} puts stderr "(full output: $logfile; baseline: $opt(-baseline))" - exit 1 + flush stderr + finish fail unexplained "GATE FAIL ($library): [llength $unexplained] unexplained failure(s) - see above" } -puts "test_gate: GATE PASS ([llength $failednames] failures, all baselined)" -exit 0 +finish pass "" "GATE PASS ($library): [llength $failednames] failures, all baselined"