diff --git a/CHANGELOG.md b/CHANGELOG.md index 7c6d8229..2b6dfd2b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,10 @@ The latest `## [X.Y.Z]` header must match the `version` field in `punkproject.to Entries are newest-first; one bullet per notable change. See the root `AGENTS.md` "Project Versioning" section for the bump policy. +## [0.10.1] - 2026-07-11 + +- punk::libunknown: pkgIndex.tcl scripts are now executed in an isolated `source_pkgindex` frame providing the documented `$dir` variable plus `auto_path`/`env` global links (matching stock tclPkgUnknown's contract), instead of at `::` scope with a `global dir`. Fixes any user global named `dir` being silently overwritten whenever a `package require` fell through to the pkg unknown handler, and stops index scripts' helper variables (`ver`, `pkg`, `script`, `_CawtSubDirs`, ...) leaking into the global namespace. Verified against tcllib 2.0's index behaviours (auto_path extension, apply-scoped subindex sweep, critcl-style loaders); regression tests pin the `$dir`/auto_path/no-leak contract. + ## [0.10.0] - 2026-07-11 - `dev lib.search` now performs deep module discovery by default: new `punk::libunknown::register_all_tm` registers `.tm` modules at every namespace depth across all tm paths (reusing/populating the per-epoch directory index cache, at most one scan per `package epoch` per interp), so namespaced modules in never-requested subfolders (e.g. `test::*`) appear in search results without `-refresh` (punk::mix::commandset::loadedlib 0.2.0). diff --git a/punkproject.toml b/punkproject.toml index 81d86325..91c14505 100644 --- a/punkproject.toml +++ b/punkproject.toml @@ -1,3 +1,3 @@ [project] name = "punkshell" -version = "0.10.0" +version = "0.10.1" diff --git a/src/modules/punk/libunknown-0.1.tm b/src/modules/punk/libunknown-0.1.tm index a119a618..4d9468a6 100644 --- a/src/modules/punk/libunknown-0.1.tm +++ b/src/modules/punk/libunknown-0.1.tm @@ -104,6 +104,21 @@ tcl::namespace::eval ::punk::libunknown { } } + #Execute a pkgIndex.tcl script in this proc's frame (tcl_Pkg_source uplevels + #the actual 'source' into its caller). + #The pkgIndex.tcl contract is that $dir holds the index file's directory, and + #stock tclPkgUnknown additionally exposes the auto_path and env globals (some + #indexes, e.g tcllib's, extend auto_path with an unqualified lappend). + #Providing exactly that environment here means anything ELSE the script sets + #stays local to this frame and is discarded - previously indexes were sourced + #via 'namespace eval ::', which clobbered any user global named 'dir' (via the + #handler's since-removed 'global dir') and leaked each index's helper + #variables (ver, pkg, script, ...) into the global namespace. + proc source_pkgindex {dir indexfile} { + global auto_path env + tcl_Pkg_source $indexfile + } + namespace eval argdoc { variable PUNKARGS lappend PUNKARGS [list { @@ -496,6 +511,12 @@ tcl::namespace::eval ::punk::libunknown { Key differences from Tcl's standard tclPkgUnknown: - Uses an epoch-based cache to avoid re-sourcing pkgIndex.tcl files that have already been processed in the current epoch. + - pkgIndex.tcl scripts execute in an isolated frame (see source_pkgindex) + providing the documented \$dir variable plus auto_path/env global links. + Stray unqualified variables set by index scripts stay local to that frame + instead of leaking into the global namespace, and user globals (notably + 'dir') are not clobbered. Stock tclPkgUnknown instead exposes all of its + own proc locals to the index scripts. - Processes auto_path entries from end to front (same as tclPkgUnknown), but tracks which packages and versions were added or changed by each pkgIndex.tcl so that ifneeded scripts from earlier (higher-priority) paths can be reverted @@ -518,7 +539,10 @@ tcl::namespace::eval ::punk::libunknown { proc zipfs_tclPkgUnknown {name args} { #puts "-> zipfs_tclPkgUnknown $name $args EXPERIMENTAL" - global dir + #Note: no 'global dir' here (an earlier revision had one so that pkgIndex.tcl + #scripts sourced at :: scope could read $dir - at the cost of clobbering any + #user global named 'dir'). Index scripts now execute in a source_pkgindex + #frame which provides $dir locally - see source_pkgindex. variable epoch set pkg_epoch [dict get $epoch pkg current] @@ -683,9 +707,8 @@ tcl::namespace::eval ::punk::libunknown { # puts stderr "----->0 sourcing zipfs file $file" #} incr sourced ;#count as sourced even if source fails; keep before actual source action - #::tcl::Pkg::source $file #lappend sourced_files $file - namespace eval :: [list ::punk::libunknown::tcl_Pkg_source $file] + source_pkgindex $dir $file } trap {POSIX EACCES} {} { # $file was not readable; silently ignore puts stderr "zipfs_tclPkgUnknown file unreadable '$file' while trying to load $name (1)" @@ -715,8 +738,7 @@ tcl::namespace::eval ::punk::libunknown { #puts "----->2 sourcing $file" incr sourced #lappend sourced_files $file - #::tcl::Pkg::source $file - namespace eval :: [list punk::libunknown::tcl_Pkg_source $file] + source_pkgindex $dir $file } trap {POSIX EACCES} {} { # $file was not readable; silently ignore puts stderr "zipfs_tclPkgUnknown file unreadable '$file' while trying to load $name (2)" diff --git a/src/tests/modules/AGENTS.md b/src/tests/modules/AGENTS.md index 11bd9bcd..53c2e0cf 100644 --- a/src/tests/modules/AGENTS.md +++ b/src/tests/modules/AGENTS.md @@ -45,5 +45,5 @@ Unit tests for editable source modules under `src/modules/`, `src/modules_tcl8/` - `punk/mix/` — punk::mix::cli tests (prune helpers, punkcheck virtual sources), punk::mix::commandset::repo fossil move/rename characterization tests (`testsuites/repo/`, FOSSIL_HOME-isolated; GAP-marked tests pin behaviour G-022 will change), punk::mix::commandset::loadedlib tests (`testsuites/loadedlib/libsearch.test`: 'dev lib.search' match semantics via -return list — wrap-glob default, =exact prefix, case rules, explicit globs, version aggregation — plus the loadedlib 0.2.0 contract: deep discovery by default (deep .tm modules found without -refresh, registration persists), -refresh = genuine re-scan (epoch incr + rediscovery picks up .tm files added to already-scanned dirs), and highlight working without the shell-global a+ alias; shared provisioned child interp sourcing the source-tree libunknown directly — see the file's ORDERING NOTE), and the MULTISHELL polyglot build machinery (`testsuites/scriptwrap/multishell.test`: scriptset wrap via the punk.multishell.cmd template - structure/LF-only/determinism, checkfile 512-byte label validation of fresh wraps AND the committed bin/runtime.cmd, the runtime scriptset round-trip byte-identity pin, and platform-gated execution smoke: cmd.exe→powershell payload on windows, sh payload on unix or via the `wsllinux` capability constraint from `src/tests/testsupport/wslprobe.tcl` - staged to the WSL distro's native filesystem, G-059) - `punk/lib/` — punk::lib tests (`testsuites/lib/`): range/index/parse/compat/interp_sync utilities, G-058 static-baseline seeding (`staticseed.test`: interp_sync_package_paths/snapshot_package_paths propagate a simulated ::punkboot static baseline and seed `load {} ` ifneeded mappings; no-op without a baseline), and the repl command-completeness engine (`commandcomplete.test`: punk::lib::system::incomplete pending-opener stacks - the info-complete quoting quirk progression (`set x "{*}{"` standalone vs in-proc-body), single openers, tabs, escapes, incomplete<->info-complete parity property; pre-repl-refactor characterization, see goals/G-044 detail preserve-list) - `punk/packagepreference/` — punk::packagepreference tests (`testsuites/packagepreference/`): G-058 static-vs-bundled policy (`staticpolicy.test`: require of a baseline package triggers the index scan before resolution so a newer bundled copy wins, static beats older bundled, exact requires of bundled versions stay reachable, missing static mappings get seeded) -- `punk/libunknown/` — .tm same-version shadowing pin-tests (`testsuites/shadowing/`): tcl::tm::add prepend rule, head-of-tm-list wins exact-version ties, version beats order, punk::libunknown parity — shipped behaviour depends on these (runtests tm ordering, punk_main package-mode precedence, G-033); mixed .tm/pkgIndex.tcl characterization is goal G-035; discovery/epoch-cache characterization (`testsuites/discovery/discovery.test`): sibling .tm registration happens only at the requested namespace depth (deeper modules invisible to 'package names' until requested), register_all_tm deep discovery (all-depths registration, per-epoch cached no-op, head-of-tm-list precedence parity), 'package epoch' command shape, trace-driven epoch increments on tm/auto_path changes, the epoch-index short-circuit (a .tm added to an already-scanned dir needs 'package epoch incr'), plus a GAP pin of zipfs_tclPkgUnknown clobbering a user global 'dir' (stock tclPkgUnknown keeps dir proc-local; fix candidate). Suite conventions learned the hard way: child probes source the SOURCE-TREE libunknown directly by path — 'package require punk::libunknown' tie-breaks the same-version copies (src/modules vs bootsupport) by tm path order, which favours bootsupport in the testinterp and machine env paths in children; and probe scripts must avoid global variable names the handlers use (notably 'dir') — that clobbering misdirected a fixture write into the real Tcl install during test development +- `punk/libunknown/` — .tm same-version shadowing pin-tests (`testsuites/shadowing/`): tcl::tm::add prepend rule, head-of-tm-list wins exact-version ties, version beats order, punk::libunknown parity — shipped behaviour depends on these (runtests tm ordering, punk_main package-mode precedence, G-033); mixed .tm/pkgIndex.tcl characterization is goal G-035; discovery/epoch-cache characterization (`testsuites/discovery/discovery.test`): sibling .tm registration happens only at the requested namespace depth (deeper modules invisible to 'package names' until requested), register_all_tm deep discovery (all-depths registration, per-epoch cached no-op, head-of-tm-list precedence parity), 'package epoch' command shape, trace-driven epoch increments on tm/auto_path changes, the epoch-index short-circuit (a .tm added to an already-scanned dir needs 'package epoch incr'), and the pkgIndex.tcl sourcing-scope contract (source_pkgindex, fixed 2026-07-11 - formerly a GAP pin of the global-'dir' clobber): user global 'dir' survives pkg-unknown fallthrough, index scripts see $dir and reach the real ::auto_path (tcllib extension pattern), and their stray unqualified sets no longer leak to ::. Suite conventions learned the hard way: child probes source the SOURCE-TREE libunknown directly by path — 'package require punk::libunknown' tie-breaks the same-version copies (src/modules vs bootsupport) by tm path order, which favours bootsupport in the testinterp and machine env paths in children; and probe scripts must avoid global variable names the handlers use (notably 'dir') — that clobbering misdirected a fixture write into the real Tcl install during test development diff --git a/src/tests/modules/punk/libunknown/testsuites/discovery/discovery.test b/src/tests/modules/punk/libunknown/testsuites/discovery/discovery.test index 0b57013a..e4836591 100644 --- a/src/tests/modules/punk/libunknown/testsuites/discovery/discovery.test +++ b/src/tests/modules/punk/libunknown/testsuites/discovery/discovery.test @@ -38,6 +38,13 @@ namespace eval ::testspace { # sdirA/pklu_shadow-1.0.tm (impl A - register_all_tm precedence pin) # sdirB/pklu_shadow-1.0.tm (impl B - register_all_tm precedence pin) # epochextra/ (empty - target for tm::add) + # libbase/ (auto_path fixture - pkgIndex.tcl contract) + # pklu_pkgidx/pkgIndex.tcl + pklu_pkgidx.tcl ($dir-based ifneeded/source) + # pklu_extender/pkgIndex.tcl (unqualified 'lappend auto_path' + # tcllib pattern + stray unqualified + # 'set pklu_leak_probe' that must NOT + # reach the global namespace) + # pklu_extender/sublib/pklu_ext/pkgIndex.tcl + pklu_ext.tcl # ------------------------------------------------------------------------- variable fixture_error "" variable fixdir "" @@ -46,6 +53,7 @@ namespace eval ::testspace { variable sdirA "" variable sdirB "" variable epochextra "" + variable libbase "" try { set base [punk::lib::tempdir_newfolder -prefix pkludiscovery] set fixdir [file join $base fix] @@ -54,7 +62,10 @@ namespace eval ::testspace { set sdirA [file join $base sdirA] set sdirB [file join $base sdirB] set epochextra [file join $base epochextra] - file mkdir [file join $fixdir pkluns subns] $mutdir $mut2dir $sdirA $sdirB $epochextra + set libbase [file join $base libbase] + file mkdir [file join $fixdir pkluns subns] $mutdir $mut2dir $sdirA $sdirB $epochextra\ + [file join $libbase pklu_pkgidx]\ + [file join $libbase pklu_extender sublib pklu_ext] foreach {relpath content} { {pklu_top-1.0.tm} {package provide pklu_top 1.0} {pklu_sib-1.0.tm} {package provide pklu_sib 1.0} @@ -78,6 +89,24 @@ namespace eval ::testspace { set fd [open [file join $sdirB pklu_shadow-1.0.tm] w] puts $fd {package provide pklu_shadow 1.0; proc ::pklu_shadow_whoami {} {return B}} close $fd + #auto_path/pkgIndex fixture - content braced so $dir stays literal + foreach {relpath content} { + {pklu_pkgidx/pkgIndex.tcl} + {package ifneeded pklu_pkgidx 1.0 [list source [file join $dir pklu_pkgidx.tcl]]} + {pklu_pkgidx/pklu_pkgidx.tcl} + {package provide pklu_pkgidx 1.0} + {pklu_extender/pkgIndex.tcl} + {lappend auto_path [file join $dir sublib] + set pklu_leak_probe leaked_value} + {pklu_extender/sublib/pklu_ext/pkgIndex.tcl} + {package ifneeded pklu_ext 1.0 [list source [file join $dir pklu_ext.tcl]]} + {pklu_extender/sublib/pklu_ext/pklu_ext.tcl} + {package provide pklu_ext 1.0} + } { + set fd [open [file join $libbase {*}[file split $relpath]] w] + puts $fd $content + close $fd + } variable tempbase $base } on error {result} { set fixture_error $result @@ -291,14 +320,16 @@ namespace eval ::testspace { list $r1 $r2 } -result {B A} - #GAP: pins a divergence from stock Tcl that is a candidate for fixing. - #Stock tclPkgUnknown keeps 'dir' proc-local (pkgIndex.tcl files are sourced - #in the proc's scope). punk::libunknown::zipfs_tclPkgUnknown sources index - #files at :: scope and therefore declares 'global dir' - clobbering any - #user global named 'dir' whenever a package require falls through to the - #pkg unknown handler. If that is fixed (e.g save/restore ::dir), the - #expected result flips to 1. - test libunknown_GAP_pkgunknown_clobbers_global_dir {a global 'dir' variable is overwritten by a package unknown fallthrough}\ + # -- pkgIndex.tcl sourcing scope (source_pkgindex) -------------------------------- + + #fixed 2026-07-11 (agent): was libunknown_GAP_pkgunknown_clobbers_global_dir + #(-result 0). zipfs_tclPkgUnknown previously declared 'global dir' and sourced + #pkgIndex.tcl scripts at :: scope - clobbering any user global named 'dir' and + #leaking each index's helper variables into the global namespace. Index scripts + #now execute in a source_pkgindex frame ($dir local, auto_path/env global links) + #matching stock tclPkgUnknown's documented contract without stock's exposure of + #its own proc locals. + test libunknown_pkgunknown_preserves_global_dir {a global 'dir' variable survives a package unknown fallthrough}\ -constraints {discoveryfixture libunknownavailable} -body { variable fixdir libu_probe [list $fixdir] { @@ -306,7 +337,36 @@ namespace eval ::testspace { catch {package require pklu_nonexistent_zzz} expr {$::dir eq "preserved_value"} } - } -result 0 + } -result 1 + + #added 2026-07-11 (agent) - pkgIndex.tcl contract under source_pkgindex + test pkgindex_dir_and_isolation {pkgIndex.tcl sees $dir; its stray unqualified sets don't leak to ::}\ + -constraints {discoveryfixture libunknownavailable} -body { + variable fixdir + variable libbase + libu_probe [list $fixdir] [string map [list %LIBBASE% [list $libbase]] { + set ::auto_path [list %LIBBASE%] + set ::t_result [list] + #the ifneeded script and its source path are built from $dir by the index + lappend ::t_result [package require pklu_pkgidx] + #pklu_extender/pkgIndex.tcl was swept in the same pass - its unqualified + #'set pklu_leak_probe' must have stayed local to the sourcing frame + lappend ::t_result [info exists ::pklu_leak_probe] + }] + } -result {1.0 0} + + test pkgindex_auto_path_extension {a pkgIndex extending auto_path with an unqualified lappend still works (tcllib pattern)}\ + -constraints {discoveryfixture libunknownavailable} -body { + variable fixdir + variable libbase + libu_probe [list $fixdir] [string map [list %LIBBASE% [list $libbase]] { + set ::auto_path [list %LIBBASE%] + #pklu_ext lives below sublib, reachable only via pklu_extender/pkgIndex.tcl's + #'lappend auto_path' - which must reach the real ::auto_path through the + #sourcing frame's global link + package require pklu_ext + }] + } -result 1.0 # cleanup fixture variable tempbase diff --git a/src/vfs/_vfscommon.vfs/modules/punk/libunknown-0.1.tm b/src/vfs/_vfscommon.vfs/modules/punk/libunknown-0.1.tm index a119a618..4d9468a6 100644 --- a/src/vfs/_vfscommon.vfs/modules/punk/libunknown-0.1.tm +++ b/src/vfs/_vfscommon.vfs/modules/punk/libunknown-0.1.tm @@ -104,6 +104,21 @@ tcl::namespace::eval ::punk::libunknown { } } + #Execute a pkgIndex.tcl script in this proc's frame (tcl_Pkg_source uplevels + #the actual 'source' into its caller). + #The pkgIndex.tcl contract is that $dir holds the index file's directory, and + #stock tclPkgUnknown additionally exposes the auto_path and env globals (some + #indexes, e.g tcllib's, extend auto_path with an unqualified lappend). + #Providing exactly that environment here means anything ELSE the script sets + #stays local to this frame and is discarded - previously indexes were sourced + #via 'namespace eval ::', which clobbered any user global named 'dir' (via the + #handler's since-removed 'global dir') and leaked each index's helper + #variables (ver, pkg, script, ...) into the global namespace. + proc source_pkgindex {dir indexfile} { + global auto_path env + tcl_Pkg_source $indexfile + } + namespace eval argdoc { variable PUNKARGS lappend PUNKARGS [list { @@ -496,6 +511,12 @@ tcl::namespace::eval ::punk::libunknown { Key differences from Tcl's standard tclPkgUnknown: - Uses an epoch-based cache to avoid re-sourcing pkgIndex.tcl files that have already been processed in the current epoch. + - pkgIndex.tcl scripts execute in an isolated frame (see source_pkgindex) + providing the documented \$dir variable plus auto_path/env global links. + Stray unqualified variables set by index scripts stay local to that frame + instead of leaking into the global namespace, and user globals (notably + 'dir') are not clobbered. Stock tclPkgUnknown instead exposes all of its + own proc locals to the index scripts. - Processes auto_path entries from end to front (same as tclPkgUnknown), but tracks which packages and versions were added or changed by each pkgIndex.tcl so that ifneeded scripts from earlier (higher-priority) paths can be reverted @@ -518,7 +539,10 @@ tcl::namespace::eval ::punk::libunknown { proc zipfs_tclPkgUnknown {name args} { #puts "-> zipfs_tclPkgUnknown $name $args EXPERIMENTAL" - global dir + #Note: no 'global dir' here (an earlier revision had one so that pkgIndex.tcl + #scripts sourced at :: scope could read $dir - at the cost of clobbering any + #user global named 'dir'). Index scripts now execute in a source_pkgindex + #frame which provides $dir locally - see source_pkgindex. variable epoch set pkg_epoch [dict get $epoch pkg current] @@ -683,9 +707,8 @@ tcl::namespace::eval ::punk::libunknown { # puts stderr "----->0 sourcing zipfs file $file" #} incr sourced ;#count as sourced even if source fails; keep before actual source action - #::tcl::Pkg::source $file #lappend sourced_files $file - namespace eval :: [list ::punk::libunknown::tcl_Pkg_source $file] + source_pkgindex $dir $file } trap {POSIX EACCES} {} { # $file was not readable; silently ignore puts stderr "zipfs_tclPkgUnknown file unreadable '$file' while trying to load $name (1)" @@ -715,8 +738,7 @@ tcl::namespace::eval ::punk::libunknown { #puts "----->2 sourcing $file" incr sourced #lappend sourced_files $file - #::tcl::Pkg::source $file - namespace eval :: [list punk::libunknown::tcl_Pkg_source $file] + source_pkgindex $dir $file } trap {POSIX EACCES} {} { # $file was not readable; silently ignore puts stderr "zipfs_tclPkgUnknown file unreadable '$file' while trying to load $name (2)"