diff --git a/CHANGELOG.md b/CHANGELOG.md index fc919c4e..9bc7cb8b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,12 @@ The latest `## [X.Y.Z]` header must match the `version` field in `punkproject.to Entries are newest-first; one bullet per notable change. See the root `AGENTS.md` "Project Versioning" section for the bump policy. +## [0.6.0] - 2026-07-10 + +- G-058: punkshell boot honours statically-linked runtime packages. Kits built on runtimes with static extensions (e.g `tclsfe-x64.exe`: Thread/twapi/sqlite3/tdbc — the punk91 kit) previously lost `package require` access to them in every interp/thread punkshell fabricates: the appended vfs replaces the runtime's own `//zipfs:/app` mount (taking its pkgIndex files with it) and punkshell's boot path replacement discards what remains — the repl code interp failed with `can't find package Thread` and punk::console couldn't load. Now `punk_main.tcl` captures the static baseline at boot (probe-loading each empty-filename `info loaded` prefix in a throwaway interp and recording only packages the load actually provides; configurable denylist for side-effecting/composite inits), and the baseline seeds `package ifneeded {load {} }` mappings in the boot interp, the repl codethread and code interp (punk::lib 0.4.0 `interp_sync_package_paths`/`snapshot_package_paths`, punk::repl 0.3.0), and shellthread 1.7.0 workers — introspectable at the repl via `::punkboot::static_packages`. +- punk::packagepreference 0.2.0: static-vs-bundled resolution is version-aware — requiring a baseline package triggers the package-index scan before resolution so all candidates register and the highest version wins (a static copy is no longer masked by an older bundled dll, and a genuinely newer bundled copy remains reachable); static *registrations* (empty-filename `info loaded` entries) no longer trip the loaded-shared-object same-version pinning. Composite statics whose script layer lived in the runtime's replaced zipfs (e.g. static twapi) provide nothing under the probe and correctly defer to bundled complete copies. +- Tests: `staticseed.test` + `staticpolicy.test` (un-gated, simulated baselines) and `staticruntime.test` (kit integration behind the capability-probed `statickitavailable` constraint). The verification runtime `tclsfe-x64.exe` is pinned in the punkbin artifact repository (win32-x86_64 + sha1sums) so the constraint is satisfiable on other machines. + ## [0.5.0] - 2026-07-10 - punk::args 0.5.0 (G-049): new `punk::args::parse_status` - runs a parse attempt and returns a documented parse-status structure instead of raising on validation failure: overall `ok`/`status` (valid|invalid|incomplete)/`scheme`/`message`/`errorcode`/`failureclass`/`badarg`/`form`/`receivednames` plus per-argument `argstatus` entries (class, status ok|bad|unparsed, received count/positions, value-in-effect including `-default` fill). `arg_error` accepts the structure via `-parsestatus` and both its renderers derive goodarg/badarg row marking and choice value-in-effect highlighting from it. Scheme colours now resolve per-render: the documented `-scheme` value `nocolour` takes effect (previously it fell through to whatever colours the last scheme render left behind) and scheme renders no longer mutate the shared colour array (the strike-only goodarg style no longer leaks into later renders). `missingrequiredvalue`/`missingrequiredleader` failures now carry `-badarg`, and `punk::args::parse` gained `-caller` to control the `%caller%` attribution in failure messages. diff --git a/GOALS.md b/GOALS.md index 931e7fc3..b8ae7590 100644 --- a/GOALS.md +++ b/GOALS.md @@ -392,7 +392,7 @@ Detail: goals/G-057-kit-icon-embedding.md Goal: Windows kit/zipkit builds produce executables carrying an embedded icon chosen at build time - defaulting to the project icon src/runtime/punk1.ico, overridable per kit by its .vfs folder - by replacing the icon resources in the built executable using the twapi-based mechanism demonstrated in tcl-sfe (TEMP_REFERENCE/tcl-sfe, by twapi author and Tcl core member Ashok P. Nadkarni): RT_ICON/RT_GROUP_ICON replacement via twapi resource-update APIs, applied so the appended vfs payload stays intact (icon the stub before appending, or sfe-style split/update/reattach). Acceptance: a Windows `make.tcl project` build produces kit executables whose embedded icon resources are the project default punk1.ico, and a kit whose .vfs supplies an override icon gets that icon instead (verified by resource inspection, e.g twapi::extract_resources, not just Explorer eyeballing); the icon-replaced executables still boot to a working punk shell reading their vfs payload for the kit types we build (kit, zip, zipcat per mapvfs.config); runtimes under src/runtime are never modified - replacement applies to the built copies only; twapi unavailable or non-Windows platform skips the icon step with a notice and the build otherwise completes unchanged; rebuilds are idempotent (re-wrapping an already-iconed build copy converges, no resource accumulation); the override convention (filename/location in the kit's custom .vfs folder vs a mapvfs.config element) and the stub-vs-split ordering decision are recorded in the detail file with the tcl-sfe attribution. -### G-058 [proposed] Boot honours statically-linked runtime packages (static baseline seeding + packagepreference static-awareness) +### G-058 [achieved 2026-07-10] Boot honours statically-linked runtime packages (static baseline seeding + packagepreference static-awareness) Scope: src/vfs/_config/punk_main.tcl (boot auto_path/tm path filtering), src/modules/punk/packagepreference-999999.0a1.0.tm, src/modules/punk/repl-999999.0a1.0.tm + src/modules/punk/repl/codethread-999999.0a1.0.tm (code interp / codethread bootstrap), src/modules/shellthread-999999.0a1.0.tm (punkshell-created worker threads) as applicable, src/tests/ (un-gated unit tests + constraint-gated shell/kit integration tests), punkbin artifact repo (separate git repo, local checkout c:/repo/jn/punkbin - pinned runtime additions) Detail: goals/G-058-static-runtime-packages.md Goal: punkshell running on a runtime with statically-linked/builtin packages (tclsfe-x64: Thread/twapi/sqlite3/tdbc; punkbin runtimes' builtin Thread/tcllibc/vfs/vlerq; the expected shape of future zig-built static runtimes, G-005) keeps those packages resolvable in every interp and thread punkshell fabricates: boot captures the static baseline (info loaded entries with empty filename, plus their provided versions) before replacing package search paths, code interps and punkshell-created threads are seeded with ifneeded scripts mapping each static package to 'load {} ', and punk::packagepreference resolves static-vs-bundled by a documented version-aware policy instead of blindly loading a bundled dll over an already-provided static package. diff --git a/goals/G-058-static-runtime-packages.md b/goals/G-058-static-runtime-packages.md index 26891e88..3446c94d 100644 --- a/goals/G-058-static-runtime-packages.md +++ b/goals/G-058-static-runtime-packages.md @@ -1,6 +1,6 @@ # G-058 Boot honours statically-linked runtime packages (static baseline seeding + packagepreference static-awareness) -Status: proposed +Status: achieved 2026-07-10 Scope: src/vfs/_config/punk_main.tcl (boot auto_path/tm path filtering), src/modules/punk/packagepreference-999999.0a1.0.tm, src/modules/punk/repl-999999.0a1.0.tm + src/modules/punk/repl/codethread-999999.0a1.0.tm (code interp / codethread bootstrap), src/modules/shellthread-999999.0a1.0.tm (punkshell-created worker threads) as applicable, src/tests/ (un-gated unit tests + constraint-gated shell/kit integration tests), punkbin artifact repo (separate git repo, local checkout c:/repo/jn/punkbin - pinned runtime additions) Acceptance: see GOALS.md index entry (canonical). @@ -93,3 +93,50 @@ with NO directory to re-add - ifneeded seeding is required instead. - Related: [[G-005]] (zig static runtimes make this foundational), [[G-006]] (artifact retrieval), the tk-loading experiment that surfaced it (punk91 = tclsfe-x64 + punk9win.vfs per src/runtime/mapvfs.config). + +## Outcome (achieved 2026-07-10) + +Shipped as punk::lib 0.4.0, punk::repl 0.3.0, punk::packagepreference 0.2.0, +shellthread 1.7.0 + punk_main.tcl capture (punkshell 0.6.0). + +Implementation-discovered refinements to the Approach: + +- The tcl-sfe stub does NOT seed ifneeded entries for its statics: it registers + them process-wide via Tcl_StaticLibrary(NULL,...) and relies on pkgIndex.tcl + files inside its own //zipfs:/app zip, resolved through a per-interp PostInit + hook (tclSfeMainHook.c) forcing auto_path to //zipfs:/app. A punk kit's appended + vfs REPLACES that mount, so even the pristine boot interp has no name->load + mapping - the capture therefore probe-loads each empty-filename [info loaded] + prefix in a throwaway interp and records only packages the load actually + PROVIDES (diff on provided packages, not 'package names' - a probe load can + trigger an index scan that registers many names without providing them). +- Composite statics exclude themselves: static twapi's C init provides nothing + (its script layer lived in the replaced stub zip), so it is never seeded and the + bundled complete twapi is correctly the resolution - the observed + "static Twapi + twapi dll" double listing was registration+load, not a double + load, and the bundled copy was the only usable one. A configurable denylist + (::punkboot::static_probe_denylist, default: tk* vfs mk4tcl vlerq tdbc*) guards + side-effecting (tk creates '.') and known-composite inits from probing at all. +- Policy mechanics: seeding alone would HIDE newer bundled copies (a satisfiable + ifneeded suppresses the package unknown scan), so packagepreference triggers the + index scan for baseline names before resolution; and static REGISTRATIONS + (empty-filename info loaded entries) are excluded from the pre-existing + loaded-shared-object same-version pinning, which would otherwise force the + static version even when bundled is newer. Net policy: all candidates + registered, highest acceptable version wins; identical versions resolve to the + bundled copy (later-registered ifneeded entry). +- vfs/vfs::zip re-diagnosis: NOT a seeding defect - the tclsfe runtime doesn't + bundle tclvfs at all (tclkit-family runtimes carry vfs1.5.0+vlerq in their + tcl_library; sfe carries thread/twapi/sqlite3/tdbc only), so the punk91 kit + genuinely lacks the package and the repl's existing soft-fail warning stands. + Candidate follow-up if wanted: bundle a tcl9 tclvfs build in punk9win.vfs, or + accept vfs-less zipfs kits (vfs is primarily needed for .kit interop). +- Verification: capture confirmed on the stub (thread/Thread 3.0.5, sqlite3 + 3.53.0, dde 1.5b0 seeded; twapi excluded); punk91 rebuilt - repl boots with no + "can't find package Thread", code interp resolves Thread and introspects + ::punkboot::static_packages, fabricated interp + snapshot-seeded worker resolve + Thread, twapi resolves (staticruntime.test 4/4 behind the capability probe); + staticseed.test 4/4 + staticpolicy.test 4/4 un-gated (9.0.3 and 8.7); + punk902z + full shell tree unchanged (23 pass/2 constraint skips); full punk/lib + tree green. tclsfe-x64.exe pinned in punkbin (commit b2dbbe6) with sha1sums and + new AGENTS.md scaffolding. diff --git a/punkproject.toml b/punkproject.toml index 7aab4a78..c23d3e9e 100644 --- a/punkproject.toml +++ b/punkproject.toml @@ -1,3 +1,3 @@ [project] name = "punkshell" -version = "0.5.0" +version = "0.6.0" diff --git a/src/modules/punk/lib-999999.0a1.0.tm b/src/modules/punk/lib-999999.0a1.0.tm index aa93ddb6..2537a2dc 100644 --- a/src/modules/punk/lib-999999.0a1.0.tm +++ b/src/modules/punk/lib-999999.0a1.0.tm @@ -8197,6 +8197,12 @@ namespace eval punk::lib { is used, in which case punk::libunknown is sourced from the tm paths that were already set up by the earlier part of the script. + When the kit boot captured runtime static/builtin packages + (::punkboot::static_packages - see punk_main.tcl), the script also copies + that baseline and seeds 'package ifneeded {load {} }' + entries so statically-linked packages (e.g Thread/twapi on a tcl-sfe + runtime) resolve in the target. + See also: punk::lib::interp_sync_package_paths" @opts -libunknown -type boolean -default 0 -help\ @@ -8235,6 +8241,18 @@ namespace eval punk::lib { append script "tcl::tm::add \{*\}\[list [lreverse $tmlist]\]\n" append script "set ::auto_path \[list $ap\]\n" append script "package prefer $prefer\n" + #G-058: propagate runtime static/builtin package mappings captured at kit boot + #(see punk_main.tcl) so 'package require' of a statically-linked package (e.g + #Thread/twapi on a tcl-sfe runtime) resolves in the new thread/interp even though + #no pkgIndex for it exists on the (replaced) package search paths. + if {[info exists ::punkboot::static_packages] && [dict size $::punkboot::static_packages]} { + append script "namespace eval ::punkboot {}\n" + append script [list set ::punkboot::static_prefixes $::punkboot::static_prefixes] \n + append script [list set ::punkboot::static_packages $::punkboot::static_packages] \n + dict for {pkgname pinfo} $::punkboot::static_packages { + append script "if \{\[package provide [list $pkgname]\] eq \"\"\} \{[list package ifneeded $pkgname [dict get $pinfo version] [list load {} [dict get $pinfo prefix]]]\}\n" + } + } if {$do_libunknown} { if {[info exists ::punk::libunknown::epoch]} { append script "namespace eval ::punk::libunknown {}\n" @@ -8299,6 +8317,12 @@ namespace eval punk::lib { package unknown handler and scan cache as the parent. This is recommended when the child will do significant package loading. + When the kit boot captured runtime static/builtin packages + (::punkboot::static_packages - see punk_main.tcl), the child also receives + that baseline plus 'package ifneeded {load {} }' + seeding so statically-linked packages (e.g Thread/twapi on a tcl-sfe + runtime) resolve in the child. + This is an opt-in helper — call sites that don't use it keep their current behavior. It does not load packages, install punk::packagepreference, share channels, or configure safe-interp restrictions — those remain caller @@ -8340,6 +8364,20 @@ namespace eval punk::lib { interp eval $interp {tcl::tm::remove {*}[tcl::tm::list]} interp eval $interp [list tcl::tm::add {*}[lreverse [tcl::tm::list]]] interp eval $interp [list package prefer [package prefer]] + #G-058: propagate runtime static/builtin package mappings captured at kit boot + #(see punk_main.tcl) so 'package require' of a statically-linked package (e.g + #Thread/twapi on a tcl-sfe runtime) resolves in the child interp even though no + #pkgIndex for it exists on the (replaced) package search paths. + if {[info exists ::punkboot::static_packages] && [dict size $::punkboot::static_packages]} { + interp eval $interp [list namespace eval ::punkboot {}] + interp eval $interp [list set ::punkboot::static_prefixes $::punkboot::static_prefixes] + interp eval $interp [list set ::punkboot::static_packages $::punkboot::static_packages] + dict for {pkgname pinfo} $::punkboot::static_packages { + if {[interp eval $interp [list package provide $pkgname]] eq ""} { + interp eval $interp [list package ifneeded $pkgname [dict get $pinfo version] [list load {} [dict get $pinfo prefix]]] + } + } + } #extended: copy epoch + source libunknown + call init if {$do_libunknown} { if {[info exists ::punk::libunknown::epoch]} { diff --git a/src/modules/punk/lib-buildversion.txt b/src/modules/punk/lib-buildversion.txt index ee96c9b4..d742dffb 100644 --- a/src/modules/punk/lib-buildversion.txt +++ b/src/modules/punk/lib-buildversion.txt @@ -1,6 +1,7 @@ -0.3.1 +0.4.0 #First line must be a semantic version number #all other lines are ignored. +#0.4.0 - G-058: snapshot_package_paths and interp_sync_package_paths now also propagate the runtime static/builtin package baseline captured at kit boot (::punkboot::static_prefixes/static_packages - see punk_main.tcl) and seed 'package ifneeded {load {} }' entries in the target thread/interp, so statically-linked runtime packages (e.g Thread/twapi on a tcl-sfe runtime) resolve in fabricated interps whose package search paths punkshell controls. No-op when no baseline was captured (plain tclsh dev launches). #0.3.1 - Tcl 9.1 compatibility (TIP 746 removed expr behaviour from lseq operands): range (lseq branch) now normalizes int[+-]int offset operands itself via offset_expr - callers such as `range 0 [llength $list]-1` keep working; also aligned the lseq branch with the fallback contract: default by now infers direction (range 5 1 -> descending, previously empty under tcl9) and by 0 returns empty (Tcl 9.1 lseq changed by-0 to return one element). lzipn_tcl9b/lzipn_tcl9c/cols/cols2: lseq expression operands wrapped in expr. check::has_tclbug_safeinterp_compile: Tcl 9.1 safe interps hide tcl::unsupported::* - falls back to interp invokehidden tcl:unsupported:disassemble #0.3.0 - new: punk::lib::check::has_libbug_udp_threadexit + libbug_udp_threadexit_applies classifier - version-based detection of tcludp < 1.0.13 on Tcl 9 Windows (per-thread exit handler closes process-global event handles; G-036). has_libbug_* is the new check family for bundled/vendored library bugs, surfaced through 'help tcl' alongside has_tclbug_*; buginfo dicts may carry a full 'url' reference key (non tcl-core trackers) #0.2.1 - call-site update for punk::console 0.6.0 tsv array rename: console -> punk_console (is_raw); no behaviour change diff --git a/src/modules/punk/packagepreference-999999.0a1.0.tm b/src/modules/punk/packagepreference-999999.0a1.0.tm index ff58a76b..fbb2c717 100644 --- a/src/modules/punk/packagepreference-999999.0a1.0.tm +++ b/src/modules/punk/packagepreference-999999.0a1.0.tm @@ -171,6 +171,24 @@ tcl::namespace::eval punk::packagepreference { return [uplevel 1 [list $COMMANDSTACKNEXT {*}$args]] } + #G-058 static-vs-bundled policy for runtime static/builtin packages + #(baseline captured at kit boot into ::punkboot::static_packages - see punk_main.tcl). + #Ensure the static mapping exists in this interp (seeding is normally done at + #interp/thread creation - this covers stragglers), then trigger the package + #unknown index scan so any bundled vfs/module copies are ALSO registered before + #resolution. With all candidates registered, standard resolution compares + #versions (highest acceptable wins) - a static copy is not blindly masked by an + #older bundled dll, and a genuinely newer bundled copy remains reachable. + #(On identical versions the bundled copy's ifneeded entry, registered by the + # later scan, wins - same version either way.) + if {[info exists ::punkboot::static_packages] && [dict exists $::punkboot::static_packages $pkg]} { + set spinfo [dict get $::punkboot::static_packages $pkg] + if {[dict get $spinfo version] ni [$COMMANDSTACKNEXT_ORIGINAL versions $pkg]} { + $COMMANDSTACKNEXT_ORIGINAL ifneeded $pkg [dict get $spinfo version] [list load {} [dict get $spinfo prefix]] + } + catch {uplevel 1 [list {*}[$COMMANDSTACKNEXT_ORIGINAL unknown] $pkg]} + } + if {!$is_exact && [llength $vwant] <= 1 } { #required version unspecified - or specified singularly set available_versions [$COMMANDSTACKNEXT_ORIGINAL versions $pkg] @@ -179,7 +197,12 @@ tcl::namespace::eval punk::packagepreference { #An attempt to detect dll/so loaded and try to load same version #dll/so files are often named with version numbers that don't contain dots or a version number at all #e.g sqlite3400.dll Thread288.dll - set pkgloadedinfo [lsearch -nocase -inline -index 1 [info loaded] $pkg] + #G-058: exclude empty-filename entries - those are static REGISTRATIONS + #(process-global, not necessarily loaded anywhere) and forcing their + #version here would defeat the version-aware static-vs-bundled policy + #above (e.g pinning to the static version even when a bundled copy is + #genuinely newer) + set pkgloadedinfo [lsearch -nocase -inline -index 1 [lsearch -all -inline -not -exact -index 0 [info loaded] {}] $pkg] if {[llength $pkgloadedinfo]} { if {[llength $available_versions] > 1} { diff --git a/src/modules/punk/packagepreference-buildversion.txt b/src/modules/punk/packagepreference-buildversion.txt index b0df4036..ba459500 100644 --- a/src/modules/punk/packagepreference-buildversion.txt +++ b/src/modules/punk/packagepreference-buildversion.txt @@ -1,4 +1,5 @@ -0.1.1 +0.2.0 #First line must be a semantic version number #all other lines are ignored. +#0.2.0 - G-058 static-vs-bundled policy: on require of a package in the runtime static baseline (::punkboot::static_packages), ensure the static ifneeded mapping exists and trigger the package unknown index scan so bundled vfs/module copies register BEFORE resolution - version comparison then applies (highest wins; identical versions resolve to the bundled copy). Static REGISTRATIONS (empty-filename info loaded entries) are now excluded from the loaded-shared-object same-version forcing, which would otherwise pin resolution to the static version even when a bundled copy is genuinely newer. #0.1.1 - moduledoc auto-load success notice moved from stdout to stderr (stdout must stay clean for script/exec contexts e.g the punk 'script' subcommand; failure branch already used stderr) diff --git a/src/modules/punk/repl-999999.0a1.0.tm b/src/modules/punk/repl-999999.0a1.0.tm index 9b8e4ceb..a736bdac 100644 --- a/src/modules/punk/repl-999999.0a1.0.tm +++ b/src/modules/punk/repl-999999.0a1.0.tm @@ -3241,6 +3241,8 @@ namespace eval repl { } %autopath% [list $::auto_path] {*}{ } %lib_epoch% [list $::punk::libunknown::epoch] {*}{ } %packageprefer% [list [package prefer]] {*}{ + } %staticprefixes% [list [expr {[info exists ::punkboot::static_prefixes] ? $::punkboot::static_prefixes : ""}]] {*}{ + } %staticpackages% [list [expr {[info exists ::punkboot::static_packages] ? $::punkboot::static_packages : ""}]] {*}{ } ] #scriptmap applied at end to satisfy silly editor highlighting. @@ -3256,6 +3258,20 @@ namespace eval repl { #It will also need to be added in that interp set ::auto_path %autopath% package prefer %packageprefer% + #G-058: runtime static/builtin package baseline captured at kit boot (punk_main.tcl). + #Make it available in this codethread and seed ifneeded mappings so statically-linked + #packages (e.g Thread/twapi on a tcl-sfe runtime) resolve here - and so + #punk::lib::interp_sync_package_paths can propagate the same into the code interp. + namespace eval ::punkboot {} + set ::punkboot::static_prefixes %staticprefixes% + set ::punkboot::static_packages %staticpackages% + if {[dict size $::punkboot::static_packages]} { + dict for {pkgname pinfo} $::punkboot::static_packages { + if {[package provide $pkgname] eq ""} { + package ifneeded $pkgname [dict get $pinfo version] [list load {} [dict get $pinfo prefix]] + } + } + } set tclmajorv [lindex [split [tcl::info::tclversion] .] 0] #jmn #puts stdout "CODETHREAD tm list" diff --git a/src/modules/punk/repl-buildversion.txt b/src/modules/punk/repl-buildversion.txt index 700f4255..b61cb517 100644 --- a/src/modules/punk/repl-buildversion.txt +++ b/src/modules/punk/repl-buildversion.txt @@ -1,6 +1,7 @@ -0.2.2 +0.3.0 #First line must be a semantic version number #all other lines are ignored. +#0.3.0 - G-058: codethread init script receives the runtime static/builtin package baseline (::punkboot::static_prefixes/static_packages via new %staticprefixes%/%staticpackages% scriptmap entries) and seeds ifneeded mappings in the codethread interp, so the code interp (seeded in turn via punk::lib::interp_sync_package_paths) can package-require statically-linked runtime packages - fixes 'can't find package Thread' booting a punk9win.vfs kit on the tclsfe-x64 static runtime (punk91) #0.2.2 - repl_handler line-mode waiting-chunks path no longer performs its opportunistic unsized read on tcl 8.6 windows console channels (no -inputmode key, real twapi console handle): the path is entered via 'after idle' with the channel usually drained, and on the 8.6 console driver a drained read parks a blocking cooked-mode ReadConsole that a later raw flip cannot cancel - after typed-ahead was stashed during a terminal-query raw window, that parked read swallowed every subsequent query response in the session until Enter (companion to the punk::console 0.7.1 three-site console-misdetection fix). On such consoles the path now consumes only data already in the Tcl channel buffer (chan pending input + sized read - no driver probe); with nothing buffered it processes the stashed complete lines directly and arms the readable handler for any remaining partial line (replacing an after-idle reinvoke that could never progress). Behaviour on tcl 9/8.7 (-inputmode consoles) and in raw mode is unchanged. #0.2.1 - call-site update for punk::console 0.6.0 tsv array rename: console -> punk_console (is_raw); no behaviour change #0.2.0 - codethread init_script now propagates package prefer via %packageprefer% scriptmap entry diff --git a/src/modules/shellthread-999999.0a1.0.tm b/src/modules/shellthread-999999.0a1.0.tm index 42767dc3..601f8a65 100644 --- a/src/modules/shellthread-999999.0a1.0.tm +++ b/src/modules/shellthread-999999.0a1.0.tm @@ -617,7 +617,9 @@ namespace eval shellthread::manager { #set ts_start [::shellthread::iso8601] set tidworker [thread::create -preserved] - set init_script [string map [list %ts_start% $ts_start %mp% [tcl::tm::list] %ap% $::auto_path %tidcli% $tidclient %sd% $settingsdict] { + set static_prefixes [expr {[info exists ::punkboot::static_prefixes] ? $::punkboot::static_prefixes : ""}] + set static_packages [expr {[info exists ::punkboot::static_packages] ? $::punkboot::static_packages : ""}] + set init_script [string map [list %ts_start% $ts_start %mp% [tcl::tm::list] %ap% $::auto_path %tidcli% $tidclient %sd% $settingsdict %spfx% [list $static_prefixes] %spkg% [list $static_packages]] { #set tclbase [file dirname [file dirname [info nameofexecutable]]] #set tcllib $tclbase/lib #if {$tcllib ni $::auto_path} { @@ -642,6 +644,19 @@ namespace eval shellthread::manager { set ::auto_path [dict get $::settingsinfo auto_path] } + #G-058: runtime static/builtin package baseline captured at kit boot (punk_main.tcl) + #seed ifneeded mappings so statically-linked packages resolve in this worker + namespace eval ::punkboot {} + set ::punkboot::static_prefixes %spfx% + set ::punkboot::static_packages %spkg% + if {[dict size $::punkboot::static_packages]} { + dict for {pkgname pinfo} $::punkboot::static_packages { + if {[package provide $pkgname] eq ""} { + package ifneeded $pkgname [dict get $pinfo version] [list load {} [dict get $pinfo prefix]] + } + } + } + package require punk::packagepreference punk::packagepreference::install diff --git a/src/modules/shellthread-buildversion.txt b/src/modules/shellthread-buildversion.txt index a34e44ca..f74ef5af 100644 --- a/src/modules/shellthread-buildversion.txt +++ b/src/modules/shellthread-buildversion.txt @@ -1,4 +1,5 @@ -1.6.3 +1.7.0 #First line must be a semantic version number #all other lines are ignored. +#1.7.0 - G-058: worker thread init script seeds the runtime static/builtin package baseline (::punkboot::static_prefixes/static_packages captured at kit boot) with ifneeded {load {} } mappings before packagepreference install, so workers can require statically-linked runtime packages #1.6.3 - manager teardown must not block on a wedged worker (G-036: Tcl 9 console+udp worker event-loop wedge): unsubscribe's settings-reset thread::send is now -async (was bare synchronous - froze exit/quit when the shellfilter-run syslog worker had wedged); shutdown_free_threads keeps its timeout timer armed across all vwait iterations (was cancelled on first response, leaving later waits unbounded); get_tag_config's bare synchronous send flagged as the same hazard class (needs a result - convert to async+vwait-with-timeout if it shows in a hang) diff --git a/src/tests/modules/AGENTS.md b/src/tests/modules/AGENTS.md index c7d0c552..d34364dc 100644 --- a/src/tests/modules/AGENTS.md +++ b/src/tests/modules/AGENTS.md @@ -42,5 +42,7 @@ Unit tests for editable source modules under `src/modules/`, `src/modules_tcl8/` - `punk/args/` — punk::args tests (`testsuites/args/`): parsing, choices/choicegroups, forms, rendering/indentation characterization, usage-marking characterization (`usagemarking.test`: -parsedargs/-badarg/-parsestatus/-scheme marking primitives plus goodchoice highlighting of selected/default-in-effect choice words, asserted by SGR-parameter subset against the live colour arrays; the G-049 nocolour/colour-leak GAP pins flipped 2026-07-10 to scheme-statelessness assertions), and the G-049 parse-status structure (`parsestatus.test`: punk::args::parse_status overall/per-argument statuses, badarg for type/allocation failures, -caller attribution, errorcode -argspecs stripping) - `punk/ns/` — punk::ns tests (`testsuites/ns/`): cmdwhich/cmdinfo/cmd_traverse doc-lookup flow (`cmdflow.test`, G-040 parity) and cmdhelp usage-rendering integration (`cmdhelp.test`: scheme selection, goodarg/badarg marking incl type/allocation failures, goodchoice highlighting of supplied/default choice words, alias path, cmdinfo result shape, queried-command failure attribution, and `-return dict` parse-status returns (G-049 - its GAP pins flipped 2026-07-10); remaining GAP pins for pseudo-command cmdtype + space-form docid prefixes (G-051, real `string is` pins behind the have_tclcoredocs constraint), TclOO undocumented-method fallback (G-052), and synopsis marking absence (G-050)) - `punk/mix/` — punk::mix::cli tests (prune helpers, punkcheck virtual sources) and punk::mix::commandset::repo fossil move/rename characterization tests (`testsuites/repo/`, FOSSIL_HOME-isolated; GAP-marked tests pin behaviour G-022 will change) +- `punk/lib/` — punk::lib tests (`testsuites/lib/`): range/index/parse/compat/interp_sync utilities, and G-058 static-baseline seeding (`staticseed.test`: interp_sync_package_paths/snapshot_package_paths propagate a simulated ::punkboot static baseline and seed `load {} ` ifneeded mappings; no-op without a baseline) +- `punk/packagepreference/` — punk::packagepreference tests (`testsuites/packagepreference/`): G-058 static-vs-bundled policy (`staticpolicy.test`: require of a baseline package triggers the index scan before resolution so a newer bundled copy wins, static beats older bundled, exact requires of bundled versions stay reachable, missing static mappings get seeded) - `punk/libunknown/` — .tm same-version shadowing pin-tests (`testsuites/shadowing/`): tcl::tm::add prepend rule, head-of-tm-list wins exact-version ties, version beats order, punk::libunknown parity — shipped behaviour depends on these (runtests tm ordering, punk_main package-mode precedence, G-033); mixed .tm/pkgIndex.tcl characterization is goal G-035 diff --git a/src/tests/modules/punk/lib/testsuites/lib/staticseed.test b/src/tests/modules/punk/lib/testsuites/lib/staticseed.test new file mode 100644 index 00000000..078e9cc3 --- /dev/null +++ b/src/tests/modules/punk/lib/testsuites/lib/staticseed.test @@ -0,0 +1,131 @@ +package require tcltest + +package require punk::lib + +#G-058 unit coverage: propagation/seeding of the runtime static/builtin package baseline +#(::punkboot::static_prefixes / ::punkboot::static_packages, captured at kit boot by +#punk_main.tcl) through punk::lib::interp_sync_package_paths and +#punk::lib::snapshot_package_paths. These tests run un-gated under plain tclsh using a +#SIMULATED baseline - the ifneeded entries they assert reference a fake static prefix +#that is never actually loaded. Kit-level integration (real static runtime) is covered +#by src/tests/shell/testsuites/punkexe/staticruntime.test behind a capability constraint. + +namespace eval ::testspace { + namespace import ::tcltest::* + variable common { + set result "" + } + + #save/restore any real ::punkboot baseline around each test (shared test interp) + proc baseline_save {} { + variable saved + set saved [dict create] + foreach v {static_prefixes static_packages} { + if {[info exists ::punkboot::$v]} { + dict set saved $v [set ::punkboot::$v] + } + } + } + proc baseline_restore {} { + variable saved + foreach v {static_prefixes static_packages} { + if {[dict exists $saved $v]} { + set ::punkboot::$v [dict get $saved $v] + } else { + catch {unset ::punkboot::$v} + } + } + } + proc baseline_fake {} { + namespace eval ::punkboot {} + set ::punkboot::static_prefixes [list Fakestat] + set ::punkboot::static_packages [dict create {*}{ + fakestatic {version 1.2 prefix Fakestat} + Fakestatic {version 1.2 prefix Fakestat} + }] + } + proc baseline_none {} { + catch {unset ::punkboot::static_prefixes} + catch {unset ::punkboot::static_packages} + } + + test staticseed_interp_sync_seeds {interp_sync_package_paths seeds baseline ifneeded mappings and copies the ::punkboot vars into the child}\ + -setup { + set result "" + baseline_save + baseline_fake + interp create staticseed_child + }\ + -body { + punk::lib::interp_sync_package_paths staticseed_child + lappend result [interp eval staticseed_child {package ifneeded fakestatic 1.2}] + lappend result [interp eval staticseed_child {package ifneeded Fakestatic 1.2}] + lappend result [interp eval staticseed_child {dict keys $::punkboot::static_packages}] + lappend result [interp eval staticseed_child {set ::punkboot::static_prefixes}] + }\ + -cleanup { + interp delete staticseed_child + baseline_restore + }\ + -result [list {load {} Fakestat} {load {} Fakestat} {fakestatic Fakestatic} Fakestat] + + test staticseed_interp_sync_no_overwrite_provided {a package already provided in the child is not re-mapped; other baseline entries still seed}\ + -setup { + set result "" + baseline_save + baseline_fake + interp create staticseed_child + }\ + -body { + interp eval staticseed_child {package provide fakestatic 9.9} + punk::lib::interp_sync_package_paths staticseed_child + lappend result [interp eval staticseed_child {package ifneeded fakestatic 1.2}] + lappend result [interp eval staticseed_child {package ifneeded Fakestatic 1.2}] + }\ + -cleanup { + interp delete staticseed_child + baseline_restore + }\ + -result [list {} {load {} Fakestat}] + + test staticseed_snapshot_script_seeds {snapshot_package_paths returns a script that reproduces the baseline and seeding in a fresh interp}\ + -setup { + set result "" + baseline_save + baseline_fake + interp create staticseed_child + }\ + -body { + set scr [punk::lib::snapshot_package_paths] + lappend result [string match "*::punkboot::static_packages*" $scr] + interp eval staticseed_child $scr + lappend result [interp eval staticseed_child {package ifneeded fakestatic 1.2}] + lappend result [interp eval staticseed_child {dict get $::punkboot::static_packages fakestatic prefix}] + }\ + -cleanup { + interp delete staticseed_child + baseline_restore + }\ + -result [list 1 {load {} Fakestat} Fakestat] + + test staticseed_noop_without_baseline {with no captured baseline (plain tclsh) neither helper adds punkboot state or ifneeded entries}\ + -setup { + set result "" + baseline_save + baseline_none + interp create staticseed_child + }\ + -body { + punk::lib::interp_sync_package_paths staticseed_child + lappend result [interp eval staticseed_child {info exists ::punkboot::static_packages}] + lappend result [interp eval staticseed_child {package ifneeded fakestatic 1.2}] + set scr [punk::lib::snapshot_package_paths] + lappend result [string match "*static_packages*" $scr] + }\ + -cleanup { + interp delete staticseed_child + baseline_restore + }\ + -result [list 0 {} 0] +} +tcltest::cleanupTests ;#needed to produce test summary line. diff --git a/src/tests/modules/punk/packagepreference/testsuites/packagepreference/staticpolicy.test b/src/tests/modules/punk/packagepreference/testsuites/packagepreference/staticpolicy.test new file mode 100644 index 00000000..821d338c --- /dev/null +++ b/src/tests/modules/punk/packagepreference/testsuites/packagepreference/staticpolicy.test @@ -0,0 +1,119 @@ +package require tcltest + +package require punk::lib + +#G-058 unit coverage: punk::packagepreference static-vs-bundled resolution policy. +#On 'package require' of a package present in the runtime static baseline +#(::punkboot::static_packages), the overloaded require ensures the static ifneeded +#mapping exists and triggers the package unknown index scan so bundled copies register +#BEFORE resolution - then standard version comparison applies (highest acceptable wins). +#Without the scan trigger, a seeded static version would satisfy the require immediately +#and a genuinely newer bundled copy would never be seen. +# +#These tests run un-gated under plain tclsh: the "static" package is simulated with a +#pre-registered working ifneeded script (the policy block only adds its own +#'load {} ' mapping when the baseline version is absent from package versions), +#and the "bundled" copies are pkgIndex.tcl fixtures on the child interp's auto_path. +#Real static runtime integration: src/tests/shell/testsuites/punkexe/staticruntime.test. + +namespace eval ::testspace { + namespace import ::tcltest::* + variable common { + set result "" + } + + variable fixdir_new [makeDirectory staticpolicy_newer] + variable fixdir_old [makeDirectory staticpolicy_older] + makeFile {package ifneeded fakestatic 1.5 {package provide fakestatic 1.5}} pkgIndex.tcl $fixdir_new + makeFile {package ifneeded fakestatic 1.0 {package provide fakestatic 1.0}} pkgIndex.tcl $fixdir_old + + #child interp with punk::packagepreference installed, a simulated static baseline for + #'fakestatic 1.2', and auto_path pointing at the given bundled-fixture dir + proc policy_child {name bundledir} { + interp create $name + #tm paths so the child can require punk::packagepreference from the source tree + punk::lib::interp_sync_package_paths $name + interp eval $name {package require punk::packagepreference} + interp eval $name {punk::packagepreference::install} + interp eval $name [list set ::auto_path [list $bundledir]] + interp eval $name { + namespace eval ::punkboot {} + set ::punkboot::static_prefixes [list Fakestat] + set ::punkboot::static_packages [dict create fakestatic {version 1.2 prefix Fakestat}] + #simulate a WORKING static loader for the baseline version (a real runtime + #would satisfy this via 'load {} Fakestat') + package ifneeded fakestatic 1.2 {package provide fakestatic 1.2} + } + } + + test staticpolicy_newer_bundled_wins {a bundled copy newer than the static baseline version is found (scan triggered before resolution) and wins}\ + -setup { + set result "" + variable fixdir_new + policy_child pol_child $fixdir_new + }\ + -body { + lappend result [interp eval pol_child {package require fakestatic}] + lappend result [interp eval pol_child {package provide fakestatic}] + }\ + -cleanup { + interp delete pol_child + }\ + -result [list 1.5 1.5] + + test staticpolicy_static_beats_older_bundled {the static baseline version wins over an older bundled copy}\ + -setup { + set result "" + variable fixdir_old + policy_child pol_child $fixdir_old + }\ + -body { + lappend result [interp eval pol_child {package require fakestatic}] + lappend result [interp eval pol_child {package provide fakestatic}] + }\ + -cleanup { + interp delete pol_child + }\ + -result [list 1.2 1.2] + + test staticpolicy_exact_older_bundled_reachable {an exact require of the older bundled version still resolves (scan applies to exact requires too)}\ + -setup { + set result "" + variable fixdir_old + policy_child pol_child $fixdir_old + }\ + -body { + lappend result [interp eval pol_child {package require -exact fakestatic 1.0}] + }\ + -cleanup { + interp delete pol_child + }\ + -result [list 1.0] + + test staticpolicy_seeds_missing_mapping {when the baseline version has no ifneeded entry the policy block adds the 'load {} ' mapping}\ + -setup { + set result "" + variable fixdir_old + interp create pol_child + punk::lib::interp_sync_package_paths pol_child + interp eval pol_child {package require punk::packagepreference} + interp eval pol_child {punk::packagepreference::install} + interp eval pol_child [list set ::auto_path [list $fixdir_old]] + interp eval pol_child { + namespace eval ::punkboot {} + set ::punkboot::static_prefixes [list Fakestat] + set ::punkboot::static_packages [dict create fakestatic {version 1.2 prefix Fakestat}] + #no working ifneeded for 1.2 - the require will fail at load time + #(no real static Fakestat exists) but must have REGISTERED the mapping + } + }\ + -body { + catch {interp eval pol_child {package require fakestatic 1.2}} + lappend result [interp eval pol_child {package ifneeded fakestatic 1.2}] + }\ + -cleanup { + interp delete pol_child + }\ + -result [list {load {} Fakestat}] +} +tcltest::cleanupTests ;#needed to produce test summary line. diff --git a/src/tests/shell/AGENTS.md b/src/tests/shell/AGENTS.md index 3e5f85b8..bdb79a14 100644 --- a/src/tests/shell/AGENTS.md +++ b/src/tests/shell/AGENTS.md @@ -19,6 +19,7 @@ Tests for shell-level behavior, command-line execution, and stdin/stdout interac - `testsuites/punkexe/shellexit.test` guards piped-stdin termination behaviour of the built executable (no `invalid thread handle` on shutdown races, no console-reopen hang on eof/error paths, exitcode propagation); do not weaken these tests - a red result means a regression in punk::repl shutdown or app-punkshell eof handling. - `testsuites/punkexe/scriptexec.test` covers the `script` subcommand (app-punkscript, G-015): piped execution + honest exit codes, result echo, file-form argv, the `lib:` scriptlib-resolution mechanism, and the tclsh-matching Tk main-loop behaviour. The `lib:` and Tk cases resolve committed fixtures under `scriptlib/_punktest/` (test-owned; a `lib:`-resolvable fixture cannot live in a tcltest temp dir since resolution only searches scriptlib locations relative to the exe). Tk cases are gated by the `punktk` constraint, auto-detected by probing whether the kit can load Tk. - punkexe tests run against the built binary: after changing punk::repl or app-punkshell source, rebuild via `make.tcl packages`, `make.tcl vfscommonupdate` (requires interactive `y`), then `make.tcl project` before trusting results; `make.tcl project` alone does not refresh `_vfscommon.vfs`. +- `testsuites/punkexe/staticruntime.test` (G-058) targets a kit built on a runtime with statically-linked packages: resolved from `env(PUNK_STATICKIT_TEST_EXE)`, else `/bin/punk91.exe`, gated by the capability-probed `statickitavailable` constraint (a probe run must show a captured `::punkboot::static_packages` baseline including Thread - not mere file existence). It asserts repl boot without the codethread Thread failure, Thread resolution in the code interp / fabricated interps / snapshot-seeded threads, and twapi resolution per the static-vs-bundled policy. The verification runtime (tclsfe-x64.exe) is pinned in the punkbin artifact repo so the constraint is satisfiable off the original dev machine. Kit script-exec paths must be passed with forward slashes. - GOAL tests (constraint `punkgoals`, enabled by env `PUNK_TEST_GOALS=1`) cover the piped-stdin-then-interactive-shell drop-in (`shellpipe_eof_then_interactive_shell`, `shellpipe_script_and_eof_then_interactive_shell`); they are excluded from normal runs because they require an openable console in the test environment and briefly attach a live shell to it - they pass in console-attached environments against a current build. - The eof-policy discriminator is `env(PUNK_PIPE_EOF)` (`exit`|`interactive`|unset=console heuristic) implemented in app-punkshell; termination-guard tests spawn children with `exit`, goal tests spawn with it unset. diff --git a/src/tests/shell/testsuites/punkexe/staticruntime.test b/src/tests/shell/testsuites/punkexe/staticruntime.test new file mode 100644 index 00000000..af842cc4 --- /dev/null +++ b/src/tests/shell/testsuites/punkexe/staticruntime.test @@ -0,0 +1,209 @@ +package require tcltest + +#G-058 integration: a punkshell kit built on a runtime with statically-linked packages +#(e.g punk91 = tclsfe-x64.exe + punk9win.vfs: static Thread/twapi/sqlite3/tdbc) must keep +#those packages resolvable in the interps/threads punkshell fabricates - the repl's +#codethread/code interp, punk::lib::interp_sync_package_paths children, and +#snapshot_package_paths threads. Seeding/policy logic has un-gated unit coverage in +#modules/punk/lib/.../staticseed.test and modules/punk/packagepreference/.../staticpolicy.test; +#these tests exercise the real kit end to end. +# +#CONSTRAINT (capability-probed, not filename-hardcoded): the target executable is resolved +#from env(PUNK_STATICKIT_TEST_EXE) if set, else /bin/punk91.exe - and is only +#used if a probe run shows a captured static baseline including Thread +#(::punkboot::static_packages populated by the kit boot). Tests skip cleanly otherwise. +#To make the constraint satisfiable on other machines the verification runtime +#(tclsfe-x64.exe) is pinned in the punkbin artifact repository (win32-x86_64/ + +#sha1sums.txt) - retrieve to bin/runtime// and rebuild. +# +#NOTE: tests exercise the BUILT executable - after changing punk_main.tcl or the seeding +#modules, rebuild with 'make.tcl modules', 'make.tcl vfscommonupdate', 'make.tcl project' +#or these tests will run against a stale binary. + +namespace eval ::testspace { + namespace import ::tcltest::* + + variable testdir [file dirname [file normalize [info script]]] + #/src/tests/shell/testsuites/punkexe -> 5 levels up to + variable projectroot [file normalize [file join $testdir .. .. .. .. ..]] + + variable statickitexe "" + if {[info exists ::env(PUNK_STATICKIT_TEST_EXE)] && $::env(PUNK_STATICKIT_TEST_EXE) ne ""} { + set statickitexe [file normalize $::env(PUNK_STATICKIT_TEST_EXE)] + } else { + set candidate [file join $projectroot bin punk91.exe] + if {[file exists $candidate]} { + set statickitexe $candidate + } + } + + variable punk_run_timeout_ms 30000 + variable runstate + array set runstate {} + + proc punk_run_read {chan} { + variable runstate + append runstate(output) [read $chan] + if {[chan eof $chan]} { + chan event $chan readable {} + set runstate(done) eof + } + } + #Run the target executable with args, feed stdin_data via pipe then half-close. + #Returns dict: timedout 0|1, exitcode , output + proc punk_run {exe cmdargs stdin_data} { + variable runstate + variable punk_run_timeout_ms + array unset runstate + set runstate(output) "" + set runstate(done) "" + set env_had [info exists ::env(PUNK_PIPE_EOF)] + if {$env_had} { + set env_save $::env(PUNK_PIPE_EOF) + } + set ::env(PUNK_PIPE_EOF) exit + set chan [open |[list $exe {*}$cmdargs 2>@1] r+] + if {$env_had} { + set ::env(PUNK_PIPE_EOF) $env_save + } else { + unset ::env(PUNK_PIPE_EOF) + } + set pids [pid $chan] + chan configure $chan -blocking 0 -translation binary + catch { + puts -nonewline $chan $stdin_data + flush $chan + chan close $chan write + } + set timerid [after $punk_run_timeout_ms [list set [namespace current]::runstate(done) timeout]] + chan event $chan readable [list [namespace current]::punk_run_read $chan] + vwait [namespace current]::runstate(done) + after cancel $timerid + set timedout [expr {$runstate(done) eq "timeout"}] + set exitcode "" + if {$timedout} { + foreach p $pids { + catch {exec {*}[auto_execok taskkill] /F /T /PID $p} killres + lappend exitcode [list killed $p] + } + catch {chan close $chan} + } else { + chan configure $chan -blocking 1 + if {[catch {chan close $chan} errM errOpts]} { + set details [dict get $errOpts -errorcode] + if {[lindex $details 0] eq "CHILDSTATUS"} { + set exitcode [lindex $details 2] + } else { + set exitcode [list error $errM] + } + } else { + set exitcode 0 + } + } + return [dict create timedout $timedout exitcode $exitcode output $runstate(output)] + } + + #capability probe: run the candidate with a script that reports the captured baseline. + #Constraint requires a non-empty ::punkboot::static_packages including Thread. + variable probe_ok 0 + if {$statickitexe ne "" && [file exists $statickitexe]} { + variable probescript [makeFile { + if {[info exists ::punkboot::static_packages]} { + puts "STATICBASELINE=[dict keys $::punkboot::static_packages]" + } else { + puts "STATICBASELINE=" + } + exit 0 + } staticruntime_probe.tcl] + #kit script exec requires forward slashes (the shellspy dispatch mangles backslashes) + set proberesult [punk_run $statickitexe [list [string map {\\ /} $probescript]] ""] + if {![dict get $proberesult timedout] && [regexp {STATICBASELINE=([^\r\n]*)} [dict get $proberesult output] -> blkeys]} { + if {"Thread" in $blkeys} { + set probe_ok 1 + } + } + } + testConstraint statickitavailable $probe_ok + + test staticruntime_shell_boot_no_thread_error {the repl boots on the static kit without the codethread Thread resolution failure}\ + -constraints statickitavailable\ + -setup { + set result "" + }\ + -body { + variable statickitexe + set r [punk_run $statickitexe [list shell] "exit\n"] + lappend result [dict get $r timedout] + lappend result [string match "*can't find package Thread*" [dict get $r output]] + lappend result [string match "*failed to load punk::console*" [dict get $r output]] + }\ + -cleanup { + }\ + -result [list 0 0 0] + + test staticruntime_codeinterp_thread_and_baseline {the code interp resolves Thread (static version) and can introspect the captured baseline}\ + -constraints statickitavailable\ + -setup { + set result "" + }\ + -body { + variable statickitexe + set stdin_data "puts THREADVER=\[package require Thread\]\nputs BASELINE=\[expr {\[dict size \$::punkboot::static_packages\] > 0}\]\nexit\n" + set r [punk_run $statickitexe [list shell] $stdin_data] + lappend result [dict get $r timedout] + lappend result [regexp {THREADVER=[0-9][0-9.abp]*} [dict get $r output]] + lappend result [regexp {BASELINE=1} [dict get $r output]] + }\ + -cleanup { + }\ + -result [list 0 1 1] + + test staticruntime_fabricated_interp_and_thread {a fabricated child interp and a snapshot-seeded worker thread resolve Thread inside the kit}\ + -constraints statickitavailable\ + -setup { + set result "" + variable fabscript [makeFile { + package require punk::lib + interp create fab + punk::lib::interp_sync_package_paths fab + puts "FABINTERP=[interp eval fab {catch {package require Thread} r; set r}]" + interp delete fab + package require Thread + #worker must stay alive for the sends (a bare create waits in its event loop) + set wid [thread::create] + thread::send $wid [punk::lib::snapshot_package_paths] + thread::send $wid {catch {package require Thread} r; set r} wr + puts "FABTHREAD=$wr" + thread::release $wid + exit 0 + } staticruntime_fab.tcl] + }\ + -body { + variable statickitexe + variable fabscript + set r [punk_run $statickitexe [list [string map {\\ /} $fabscript]] ""] + lappend result [dict get $r timedout] + lappend result [regexp {FABINTERP=[0-9][0-9.abp]*} [dict get $r output]] + lappend result [regexp {FABTHREAD=[0-9][0-9.abp]*} [dict get $r output]] + }\ + -cleanup { + }\ + -result [list 0 1 1] + + test staticruntime_twapi_resolves {twapi resolves in the kit per the static-vs-bundled policy (composite static twapi defers to the bundled complete copy)}\ + -constraints statickitavailable\ + -setup { + set result "" + }\ + -body { + variable statickitexe + set stdin_data "puts TWAPIVER=\[package require twapi\]\nexit\n" + set r [punk_run $statickitexe [list shell] $stdin_data] + lappend result [dict get $r timedout] + lappend result [regexp {TWAPIVER=[0-9][0-9.abp]*} [dict get $r output]] + }\ + -cleanup { + }\ + -result [list 0 1] +} +tcltest::cleanupTests ;#needed to produce test summary line. diff --git a/src/vfs/AGENTS.md b/src/vfs/AGENTS.md index 276aa342..4b4a4580 100644 --- a/src/vfs/AGENTS.md +++ b/src/vfs/AGENTS.md @@ -16,6 +16,7 @@ VFS (Virtual File System) folders define the runtime payloads that get wrapped i - VFS content must be compatible with the target platform runtime. - `_vfscommon.vfs/` is an auto-generated merge of common libraries; manually editing it will be overwritten. - `_config/punk_main.tcl` accepts an optional package_mode argument before the subcommand. Valid modes are `dev`, `os`, `internal`, and `src`, dash-delimited in priority order (e.g `src-internal`). `internal` is always appended if omitted. +- `_config/punk_main.tcl` captures the runtime's static/builtin packages at boot (G-058): empty-filename `info loaded` prefixes are probe-loaded in a throwaway interp (denylist `::punkboot::static_probe_denylist`, default `tk* vfs mk4tcl vlerq tdbc*` - side-effecting or composite inits) and each package a probe-load actually PROVIDES is recorded in `::punkboot::static_packages` {pkgname {version prefix

}} with `::punkboot::static_prefixes` alongside. The boot interp and every punkshell-fabricated interp/thread (punk::lib::interp_sync_package_paths / snapshot_package_paths, repl codethread, shellthread workers) seed `package ifneeded {load {} }` from it, and punk::packagepreference resolves static-vs-bundled version-aware. Composite statics whose script layer lived in the runtime's own (replaced) zipfs provide nothing under the probe and are correctly left to bundled copies. - `src` mode loads unbuilt modules and libraries from the project's `src/` tree (`src/modules`, `src/lib`, `src/bootsupport`, `src/vendormodules`) instead of built output. It also runs an inline `#modpod` package-ifneeded scanner (using Tcl builtins only, since no punk modules are loaded at boot time) and sets `package prefer latest` so `999999.0a1.0` dev modules are preferred over stable bootsupport/vendored copies. - `dev` mode loads built output from `/modules` and `/lib`; `src` mode loads unbuilt source from `/src/modules` etc. Use `src-dev-internal` to combine both with src priority. diff --git a/src/vfs/_config/punk_main.tcl b/src/vfs/_config/punk_main.tcl index 34e833db..b3c4f232 100644 --- a/src/vfs/_config/punk_main.tcl +++ b/src/vfs/_config/punk_main.tcl @@ -25,6 +25,100 @@ apply { args { set ::punkargv $args set tclmajorv [lindex [split [info tclversion] .] 0] + + # -- runtime static/builtin package capture (G-058) -------------------------------------- + #Runtimes may statically link packages (e.g tcl-sfe: Thread,twapi,sqlite3,tdbc). + #Static registrations are process-global ('load {} ' works in any interp/thread) + #but 'package require' needs a package-name -> load mapping in EACH interp. A kit's + #appended vfs replaces the runtime's own //zipfs:/app mount, so any pkgIndex.tcl the + #runtime shipped for its statics is gone - and the path setup below controls resolution + #anyway. Capture the static prefixes now, discover the package names/versions each + #provides by loading into a throwaway interp, and record the results in ::punkboot so + #that (a) this interp and (b) every interp/thread punkshell fabricates can seed + #'package ifneeded {load {} }' entries. + #(see punk::lib::interp_sync_package_paths / snapshot_package_paths and the + # punk::packagepreference static-vs-bundled policy) + #Probe-loading is safe for self-contained extensions; static_probe_denylist excludes + #prefixes whose init has side effects (tk* creates '.') or whose package is known to be + #composite (C part + on-disk scripts: vfs, tdbc*, and kit machinery mk4tcl/vlerq) - + #those keep their existing resolution behaviour. Composite statics are ALSO excluded + #naturally: only packages a probe-load actually PROVIDES are recorded, so an init that + #defers to script files which no longer exist (e.g static twapi whose script layer + #lived in the runtime's replaced zip) is never seeded, and any bundled complete copy + #resolves as before. + namespace eval ::punkboot { + variable static_prefixes [list] + variable static_packages [dict create] ;#pkgname -> {version prefix

} + variable static_probe_denylist [list tk* vfs mk4tcl vlerq tdbc*] + } + apply {{} { + foreach rec [info loaded] { + lassign $rec fpath prefix + if {$fpath eq "" && $prefix ne ""} { + lappend ::punkboot::static_prefixes $prefix + } + } + if {![llength $::punkboot::static_prefixes]} { + return + } + set probeable [list] + foreach prefix $::punkboot::static_prefixes { + set denied 0 + foreach dpat $::punkboot::static_probe_denylist { + if {[string match -nocase $dpat $prefix]} { + set denied 1 + break + } + } + if {!$denied} { + lappend probeable $prefix + } + } + set probe __punkboot_staticprobe + catch {interp delete $probe} + interp create $probe + #diff on PROVIDED packages (not 'package names' - a probe load can trigger an index + #scan that registers many names as ifneeded without providing them) + set provided_in_probe {{probe} { + set pdict [dict create] + foreach n [interp eval $probe {package names}] { + set pv [interp eval $probe [list package provide $n]] + if {$pv ne ""} { + dict set pdict $n $pv + } + } + return $pdict + }} + #an init may depend on another static - retry failures once + set attempts [list {*}$probeable {*}$probeable] + set done [list] + foreach prefix $attempts { + if {$prefix in $done} { + continue + } + set before_prov [apply $provided_in_probe $probe] + if {[catch {load {} $prefix $probe}]} { + continue + } + lappend done $prefix + dict for {pkgname v} [apply $provided_in_probe $probe] { + if {![dict exists $before_prov $pkgname]} { + dict set ::punkboot::static_packages $pkgname [dict create version $v prefix $prefix] + } + } + } + interp delete $probe + #seed this interp's package db so requires here can resolve to the runtime's own + #copy. Coexisting ifneeded entries (static + any vfs/module copies registered by + #index scans) resolve version-aware under the standard package machinery. + dict for {pkgname pinfo} $::punkboot::static_packages { + if {[package provide $pkgname] eq ""} { + package ifneeded $pkgname [dict get $pinfo version] [list load {} [dict get $pinfo prefix]] + } + } + }} + # -- end runtime static/builtin package capture ------------------------------------------- + namespace eval ::punkboot { #This is somewhat ugly - but we don't want to do any 'package require' operations at this stage # even for something that is available in tcl_library.