From df9fdbd1d6ae543fba9358a31977e39a378d4367 Mon Sep 17 00:00:00 2001 From: Julian Noble Date: Mon, 20 Jul 2026 13:46:31 +1000 Subject: [PATCH] G-098 test gate: parsed totals vs tracked baseline (GATE PASS); thread/registry/vfs runtime plumbing suite.tcl 'test' is now a gate: full tcltest output logged, totals parsed (all.tcl's exit code does not reflect failures), failed test names compared against expected_test_failures.txt - unexplained failures fail the action; stale baseline entries are noted on full runs. Totals on core-9-0-branch 9.0.5 / zig 0.16.0: 67281 run / 55835 passed / 11435 skipped / 11 failed, GATE PASS, deterministic across two full runs. The 11 are dispositioned in the baseline: 6 fCmd (OS username vs profile-dir machine trait), 2 filename (admin-share self-globbing), 2 safe-stock (install-layout accessPath tokens), 1 console-bug-f10d91c2 (powershell driver yields empty result in batch context; semantics verified independently by G-076 kill procedure). Runtime package plumbing gaps closed (built artifacts without install wiring): thread pkgIndex generated (lib/thread3.0.1 -> bin dll), registry dll install artifact hooked to install-libraries (was orphaned - registry 1.3.7 + dde 1.4.6 now load), tclvfs script package installed with vfs.tcl + pkgIndex.tcl generated from .in templates (vfs 1.4.2 + vfs::zip 1.0.4). punkshell runtests under the suite runtime: checkbugs 5/5, opunk backends.test PASS (Tk-widget console suite runs unskipped on the suite-built Tk), repl consolebackends 2/3 - repl_console_backend_tkconsole open (code-interp console_pending_out junction init missing; DRIVER-DONE timeout-failsafe). Assisted-by: harness=claude; primary-model=claude-fable-5; api-location=anthropic.com --- goals/G-098-suite-tk-tcllib-critcl.md | 42 +++++++- src/buildsuites/suite_tcl90/build905.zig | 4 + .../suite_tcl90/expected_test_failures.txt | 40 +++++++ src/buildsuites/suite_tcl90/suite.tcl | 102 ++++++++++++++++-- 4 files changed, 178 insertions(+), 10 deletions(-) create mode 100644 src/buildsuites/suite_tcl90/expected_test_failures.txt diff --git a/goals/G-098-suite-tk-tcllib-critcl.md b/goals/G-098-suite-tk-tcllib-critcl.md index 8483bfc9..cad34ffc 100644 --- a/goals/G-098-suite-tk-tcllib-critcl.md +++ b/goals/G-098-suite-tk-tcllib-critcl.md @@ -121,11 +121,47 @@ user direction 2026-07-20: an in-place rename does not hide a package) = ~248x speedup, confirming the user-reported punkcheck hashing pain and its fix. +### 2026-07-20 increment 4: test gate live (72 -> 11 dispositioned, GATE PASS); thread/registry/vfs runtime gaps closed; runtests on the suite runtime + +- suite.tcl test rewritten as a GATE: full output to /tcltest.log, totals + PARSED (all.tcl's exit code does not reflect failures), failed test names collected + and compared against the tracked expected_test_failures.txt baseline - unexplained + failures fail the action listing them; stale baseline entries are noted on full + runs. Verified failing correctly (11 listed pre-baseline) and passing after + baselining: totals 67281 run / 55835 passed / 11435 skipped / 11 failed, + GATE PASS, deterministic across two full runs. +- The 72 initial failures collapsed to 11 via earlier recipe fixes (http tm + filename/provide mismatch -> 44; missing library subdirs -> 8 msgcat; library-tree + install also cleared clock-vm/winTime/cmdAH). The 11 are dispositioned in the + baseline with reasons: 6 fCmd (machine trait: OS user jnoble vs profile dir + C:/Users/sleek - 'file home ' assumptions), 2 filename (windows admin-share + self-globbing //hostname/c), 2 safe-stock (install-layout-sensitive safe-interp + accessPath tokens - candidate deeper look), 1 console-bug-f10d91c2 (upstream + regression test credited to the punkshell author; expects the FIXED behaviour via a + powershell conhost-kill driver but yields an empty result in batch context - + execution-policy interaction suspected; semantics verified independently by G-076's + kill procedure; candidate upstream feedback). +- Runtime package gaps closed (all same class - built artifacts without installed + package plumbing): thread pkgIndex generated (lib/thread3.0.1 -> bin dll; version + derived from the dll name), registry dll install artifact existed but was hooked to + nothing (now under install-libraries; registry 1.3.7 + dde 1.4.6 load), tclvfs + script package installed with its two configure-products generated (vfs.tcl + + pkgIndex.tcl from .in templates, version from AC_INIT; dll alongside) - vfs 1.4.2 + + vfs::zip 1.0.4 load. +- punkshell runtests UNDER the suite runtime (TCLLIBPATH cleared): checkbugs 5/5; + opunk console backends.test PASS - the Tk-widget console suite runs unskipped on + the suite-built Tk; repl consolebackends: testconsole + sshconsole backends PASS, + repl_console_backend_tkconsole FAILED (child repl reaches the prompt but + repl_process_data errors reading ::codeinterp::console_pending_out - the + foreign-console var-junction init in the code interp did not run; DRIVER-DONE + timeout-failsafe). The vfs install did not change it. OPEN ITEM: diagnose the + TkConsole child's code-interp init under the plain suite runtime (candidates: + init-order/timing vs kit environments, twapi absence side-path). + Remaining for acceptance: -- suite.tcl test gate: parse all.tcl totals vs a recorded expected-failure baseline; - disposition the 72 current failures. -- punkshell runtests against the suite-built runtime with Tk suites unskipped. +- Diagnose/fix repl_console_backend_tkconsole under the suite runtime, then a full + punkshell runtests sweep on it (Tk suites unskipped) recorded here. - (superseded note kept for history below: the clone store subsequently moved from the suite-owned _build location to the machine-level ~/.punkshell/fossils - see increment 1 and the suite README for current truth) diff --git a/src/buildsuites/suite_tcl90/build905.zig b/src/buildsuites/suite_tcl90/build905.zig index 45ae1840..d7d1ddad 100644 --- a/src/buildsuites/suite_tcl90/build905.zig +++ b/src/buildsuites/suite_tcl90/build905.zig @@ -1916,6 +1916,10 @@ pub fn build(b: *std.Build) !void { if (target.result.os.tag == .windows) { install_libraries.dependOn(&install_dde_dll.step); + //G-098: the registry install artifact existed but was hooked to nothing - + //library/registry/pkgIndex.tcl then pointed at a dll never installed + //(punkshell's punk module probes 'package require registry' at load). + install_libraries.dependOn(&install_reg_dll.step); } //**************************** diff --git a/src/buildsuites/suite_tcl90/expected_test_failures.txt b/src/buildsuites/suite_tcl90/expected_test_failures.txt new file mode 100644 index 00000000..1f05753f --- /dev/null +++ b/src/buildsuites/suite_tcl90/expected_test_failures.txt @@ -0,0 +1,40 @@ +# suite_tcl90 expected Tcl-core-test failures (G-098 test gate baseline). +# One test name per line; comments carry the disposition reason. 'suite.tcl test' +# fails on any failed test NOT listed here, and notes listed tests that no longer +# fail (candidates for removal). +# +# Baseline established 2026-07-20 against tcl core-9-0-branch 1a9c3b9d96 (9.0.5), +# zig 0.16.0 suite build, machine superbee (windows 11). Full-run totals at capture: +# 66350 run / 200-class skips constraint-driven / 11 failed, all dispositioned below. +# History: the initial 0.14.1-era run had 72 failures; 61 were FIXED by suite recipe +# corrections (http tm filename/provide mismatch -> 44 http; missing library subdirs +# -> 8 msgcat; library-tree install also cleared clock vm/winTime/cmdAH classes). + +# -- machine trait: OS username (jnoble) differs from profile dir (C:/Users/sleek). +# 'file home ' tests assume they match; fails on any runtime on this machine. +fCmd-31.6 +fCmd-31.9 +fCmd-32.5 +fCmd-32.9 +fCmd-32.16 +fCmd-32.17 + +# -- environment: windows admin-share self-globbing (///c/...) requires +# accessible administrative shares; not available/permitted on this machine. +filename-16.12 +filename-16.13 + +# -- install-layout sensitivity: safe-interp accessPath token expectations differ +# under the suite's install layout (tm dirs + whole library tree). Candidate for a +# deeper look; behaviour of safe interps otherwise exercised by passing safe-* tests. +safe-stock-7.2 +safe-stock-18.2 + +# -- upstream regression test for ticket f10d91c2 (test script credited to the +# punkshell author). It EXPECTS the fixed behaviour (payload exits, 'readable fired') +# via a powershell conhost-kill driver, but in the suite's batch context the driver +# yields an empty result (powershell -File environment/execution-policy interaction +# suspected). The semantics it guards are verified independently by punkshell's +# G-076 kill-procedure verification on this same runtime. Needs isolated +# investigation; candidate upstream test-harness feedback. +console-bug-f10d91c2 diff --git a/src/buildsuites/suite_tcl90/suite.tcl b/src/buildsuites/suite_tcl90/suite.tcl index 64df2fc8..88ce3c24 100644 --- a/src/buildsuites/suite_tcl90/suite.tcl +++ b/src/buildsuites/suite_tcl90/suite.tcl @@ -108,20 +108,63 @@ if {$action eq "clean"} { } if {$action eq "test"} { - #Exercise the Tcl core test suite on the suite-built static shell (all.tcl in the - #staged checkout's tests dir; pass tcltest selection via -testargs). all.tcl's - #exit code reflects test failures on modern tcl; output is streamed regardless. + #Exercise the Tcl core test suite on the suite-built static shell and GATE on the + #parsed totals (all.tcl's exit code does NOT reflect failures): every failed test + #must appear in the tracked expected-failure baseline + #(suite_tcl90/expected_test_failures.txt) or the action fails listing the + #unexplained names. Full output is written to /tcltest.log. set exe [file join $stage out bin tclsh90s.exe] set testsdir [file join $stage tcl905 tests] if {![file exists $exe]} {fail "no built shell at $exe - run 'suite.tcl build' first"} if {![file isdirectory $testsdir]} {fail "no tcl tests dir at $testsdir - run 'suite.tcl build' first"} - log "tcl testsuite on $exe [expr {[llength $opt(-testargs)] ? $opt(-testargs) : "(full suite)"}]" + set logfile [file join $stage tcltest.log] + log "tcl testsuite on $exe [expr {[llength $opt(-testargs)] ? $opt(-testargs) : "(full suite)"}] -> $logfile" set savedpwd [pwd] cd $testsdir - set rc [catch {exec $exe all.tcl {*}$opt(-testargs) >@stdout 2>@stderr} err] + catch {exec $exe all.tcl {*}$opt(-testargs) > $logfile 2>@1} cd $savedpwd - if {$rc} {fail "tcl testsuite reported failure: $err"} - log "tcl testsuite run complete (see output above for any FAILED lines)" + set f [open $logfile r]; set testout [read $f]; close $f + if {![regexp {all\.tcl:\s+Total\s+(\d+)\s+Passed\s+(\d+)\s+Skipped\s+(\d+)\s+Failed\s+(\d+)} $testout -> ntotal npassed nskipped nfailed]} { + fail "no all.tcl totals line found in $logfile - run did not complete" + } + log "totals: $ntotal run, $npassed passed, $nskipped skipped, $nfailed failed" + set failednames {} + foreach {- tname} [regexp -all -inline -line {^==== (\S+) FAILED} $testout] { + if {$tname ni $failednames} {lappend failednames $tname} + } + #baseline: one test name per line; '#' comments carry the disposition reasons + set baseline {} + set basefile [file join $suiteroot expected_test_failures.txt] + if {[file exists $basefile]} { + set f [open $basefile r] + foreach line [split [read $f] \n] { + set line [string trim $line] + if {$line eq "" || [string index $line 0] eq "#"} continue + lappend baseline [lindex $line 0] + } + close $f + } + set unexplained {} + foreach tname $failednames { + if {$tname ni $baseline} {lappend unexplained $tname} + } + set stale {} + if {![llength $opt(-testargs)]} { + #only meaningful for full runs: baseline entries that no longer fail + foreach tname $baseline { + if {$tname ni $failednames} {lappend stale $tname} + } + } + if {[llength $stale]} { + log "note: [llength $stale] baseline entries did not fail this run (candidates for removal): $stale" + } + if {[llength $unexplained]} { + puts stderr "suite_tcl90 TEST GATE FAIL: [llength $unexplained] failed test(s) not in the expected-failure baseline:" + foreach tname $unexplained {puts stderr " $tname"} + puts stderr "(full output: $logfile; baseline: $basefile)" + exit 1 + } + log "TEST GATE PASS ([llength $failednames] failures, all baselined)" exit 0 } @@ -276,6 +319,51 @@ cd $builddir run $opt(-zig) build --build-file build905.zig --cache-dir $cachedir --prefix [file join $stage out] -Doptimize=$opt(-optimize) {*}$opt(-steps) cd $savedpwd +#-- thread pkgIndex (G-098: the recipe installs tcl9thread.dll but no index; +#punkshell runtests/-jobs and codethread need 'package require Thread' to resolve) -- +set threaddlls [glob -nocomplain [file join $stage out bin tcl9thread*.dll]] +if {[llength $threaddlls]} { + set tdll [file tail [lindex $threaddlls 0]] + if {[regexp {tcl9thread(\d)(\d)(\d)\.dll} $tdll -> tj tn tp]} { + set threadver "$tj.$tn.$tp" + set tpkgdir [file join $stage out lib thread$threadver] + file mkdir $tpkgdir + set f [open [file join $tpkgdir pkgIndex.tcl] w] + fconfigure $f -translation lf + puts $f "package ifneeded Thread $threadver \[list load \[file normalize \[file join \$dir .. .. bin $tdll\]\] Thread\]" + close $f + log "thread pkgIndex.tcl written ($tpkgdir -> $tdll)" + } +} + +#-- vfs package install (G-098): the recipe builds tcl9vfs142.dll only; punkshell's +#repl code interp wants vfs/vfs::zip (modpod mounting). Install the tclvfs script +#package with its two configure-products generated (vfs.tcl, pkgIndex.tcl) and the +#dll alongside (vfs.tcl loads from its own dir via ::vfs::self). +set vfssrc [file join $stage tclvfs] +if {[file isdirectory $vfssrc]} { + set f [open [file join $vfssrc configure.ac] r]; set cac [read $f]; close $f + if {![regexp {AC_INIT\(\[vfs\],\[([^\]]+)\]} $cac -> vfsver]} {fail "vfs version not found in tclvfs configure.ac"} + set vpkgdir [file join $stage out lib vfs$vfsver] + if {![file isdirectory $vpkgdir]} { + log "installing tclvfs $vfsver package -> $vpkgdir" + file mkdir $vpkgdir + foreach tf [glob -directory [file join $vfssrc library] *.tcl] { + file copy -force $tf $vpkgdir + } + file copy -force [file join $vfssrc library template] $vpkgdir + set map [list @PACKAGE_VERSION@ $vfsver @PKG_LIB_FILE9@ tcl9vfs142.dll @PKG_LIB_FILE8@ tclvfs142.dll] + foreach {intmpl outname} [list [file join $vfssrc library vfs.tcl.in] vfs.tcl [file join $vfssrc pkgIndex.tcl.in] pkgIndex.tcl] { + set f [open $intmpl r]; set t [read $f]; close $f + set f [open [file join $vpkgdir $outname] w] + fconfigure $f -translation lf + puts -nonewline $f [string map $map $t] + close $f + } + file copy -force [file join $stage out bin tcl9vfs142.dll] $vpkgdir + } +} + #-- tk pkgIndex (makefile.vc install-binaries convention) ------------------ set tkpkgdir [file join $stage out lib tk$tkversion] if {[file isdirectory $tkpkgdir]} {