From d3d81c1c39bd27a0791b0a4cda2ff20d920a9153 Mon Sep 17 00:00:00 2001 From: Julian Noble Date: Fri, 10 Jul 2026 20:26:08 +1000 Subject: [PATCH] G-058 achieved: boot honours statically-linked runtime packages (punk::lib 0.4.0, punk::repl 0.3.0, punk::packagepreference 0.2.0, shellthread 1.7.0) - punk_main.tcl captures the runtime static baseline at kit boot: empty-filename [info loaded] prefixes probe-loaded in a throwaway interp (configurable denylist tk*/vfs/mk4tcl/vlerq/tdbc* for side-effecting/composite inits), recording only packages a load actually PROVIDES (diff on provides, not package names - probe loads can trigger index scans) into ::punkboot::static_packages/static_prefixes; the boot interp is seeded with ifneeded {load {} } mappings - punk::lib interp_sync_package_paths/snapshot_package_paths propagate the baseline and seed fabricated interps/threads; punk::repl codethread init (new %staticprefixes%/%staticpackages% scriptmap) and shellthread worker init do the same - the repl code interp on a static runtime resolves Thread again (punk91 = tclsfe-x64 + punk9win.vfs failed with "can't find package Thread"; the appended vfs replaces the stub's //zipfs:/app mount taking its static pkgIndex files with it, and the tclsfe PostInit hook seeds per-interp auto_path only) - punk::packagepreference static-vs-bundled policy: requiring a baseline package ensures the static mapping and triggers the package unknown scan BEFORE resolution so bundled copies register and the highest version wins (newer bundled reachable, static not masked by older bundled); static registrations excluded from the loaded-shared-object same-version pinning - composite statics defer correctly: static twapi provides nothing under the probe (script layer lived in the replaced stub zip) so the bundled complete twapi is the resolution; punk91 vfs/vfs::zip warning re-diagnosed as kit content (tclsfe bundles no tclvfs) - recorded in the goal detail, not a seeding defect - tests: staticseed.test (4) + staticpolicy.test (4) un-gated simulated-baseline units (green 9.0.3 + 8.7); staticruntime.test (4) kit integration behind the capability-probed statickitavailable constraint (probes ::punkboot baseline incl Thread; env PUNK_STATICKIT_TEST_EXE override, default bin/punk91.exe) - 4/4 against the rebuilt punk91; punk902z/full shell tree/full source-tree suite baselines unchanged (exec-14.3 only) - verification runtime tclsfe-x64.exe pinned in the punkbin artifact repo (separate repo, commit b2dbbe6) with sha1sums + new AGENTS.md scaffolding - punkshell 0.6.0: CHANGELOG entry; AGENTS updates (src/vfs capture contract, tests indexes incl new punk/lib + punk/packagepreference entries, shell staticruntime contract) Assisted-by: harness=claude; primary-model=claude-fable-5; api-location=anthropic.com --- CHANGELOG.md | 6 + GOALS.md | 2 +- goals/G-058-static-runtime-packages.md | 49 +++- punkproject.toml | 2 +- src/modules/punk/lib-999999.0a1.0.tm | 38 ++++ src/modules/punk/lib-buildversion.txt | 3 +- .../punk/packagepreference-999999.0a1.0.tm | 25 ++- .../punk/packagepreference-buildversion.txt | 3 +- src/modules/punk/repl-999999.0a1.0.tm | 16 ++ src/modules/punk/repl-buildversion.txt | 3 +- src/modules/shellthread-999999.0a1.0.tm | 17 +- src/modules/shellthread-buildversion.txt | 3 +- src/tests/modules/AGENTS.md | 2 + .../punk/lib/testsuites/lib/staticseed.test | 131 +++++++++++ .../packagepreference/staticpolicy.test | 119 ++++++++++ src/tests/shell/AGENTS.md | 1 + .../testsuites/punkexe/staticruntime.test | 209 ++++++++++++++++++ src/vfs/AGENTS.md | 1 + src/vfs/_config/punk_main.tcl | 94 ++++++++ 19 files changed, 715 insertions(+), 9 deletions(-) create mode 100644 src/tests/modules/punk/lib/testsuites/lib/staticseed.test create mode 100644 src/tests/modules/punk/packagepreference/testsuites/packagepreference/staticpolicy.test create mode 100644 src/tests/shell/testsuites/punkexe/staticruntime.test diff --git a/CHANGELOG.md b/CHANGELOG.md index fc919c4e..9bc7cb8b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,12 @@ The latest `## [X.Y.Z]` header must match the `version` field in `punkproject.to Entries are newest-first; one bullet per notable change. See the root `AGENTS.md` "Project Versioning" section for the bump policy. +## [0.6.0] - 2026-07-10 + +- G-058: punkshell boot honours statically-linked runtime packages. Kits built on runtimes with static extensions (e.g `tclsfe-x64.exe`: Thread/twapi/sqlite3/tdbc — the punk91 kit) previously lost `package require` access to them in every interp/thread punkshell fabricates: the appended vfs replaces the runtime's own `//zipfs:/app` mount (taking its pkgIndex files with it) and punkshell's boot path replacement discards what remains — the repl code interp failed with `can't find package Thread` and punk::console couldn't load. Now `punk_main.tcl` captures the static baseline at boot (probe-loading each empty-filename `info loaded` prefix in a throwaway interp and recording only packages the load actually provides; configurable denylist for side-effecting/composite inits), and the baseline seeds `package ifneeded {load {} }` mappings in the boot interp, the repl codethread and code interp (punk::lib 0.4.0 `interp_sync_package_paths`/`snapshot_package_paths`, punk::repl 0.3.0), and shellthread 1.7.0 workers — introspectable at the repl via `::punkboot::static_packages`. +- punk::packagepreference 0.2.0: static-vs-bundled resolution is version-aware — requiring a baseline package triggers the package-index scan before resolution so all candidates register and the highest version wins (a static copy is no longer masked by an older bundled dll, and a genuinely newer bundled copy remains reachable); static *registrations* (empty-filename `info loaded` entries) no longer trip the loaded-shared-object same-version pinning. Composite statics whose script layer lived in the runtime's replaced zipfs (e.g. static twapi) provide nothing under the probe and correctly defer to bundled complete copies. +- Tests: `staticseed.test` + `staticpolicy.test` (un-gated, simulated baselines) and `staticruntime.test` (kit integration behind the capability-probed `statickitavailable` constraint). The verification runtime `tclsfe-x64.exe` is pinned in the punkbin artifact repository (win32-x86_64 + sha1sums) so the constraint is satisfiable on other machines. + ## [0.5.0] - 2026-07-10 - punk::args 0.5.0 (G-049): new `punk::args::parse_status` - runs a parse attempt and returns a documented parse-status structure instead of raising on validation failure: overall `ok`/`status` (valid|invalid|incomplete)/`scheme`/`message`/`errorcode`/`failureclass`/`badarg`/`form`/`receivednames` plus per-argument `argstatus` entries (class, status ok|bad|unparsed, received count/positions, value-in-effect including `-default` fill). `arg_error` accepts the structure via `-parsestatus` and both its renderers derive goodarg/badarg row marking and choice value-in-effect highlighting from it. Scheme colours now resolve per-render: the documented `-scheme` value `nocolour` takes effect (previously it fell through to whatever colours the last scheme render left behind) and scheme renders no longer mutate the shared colour array (the strike-only goodarg style no longer leaks into later renders). `missingrequiredvalue`/`missingrequiredleader` failures now carry `-badarg`, and `punk::args::parse` gained `-caller` to control the `%caller%` attribution in failure messages. diff --git a/GOALS.md b/GOALS.md index 931e7fc3..b8ae7590 100644 --- a/GOALS.md +++ b/GOALS.md @@ -392,7 +392,7 @@ Detail: goals/G-057-kit-icon-embedding.md Goal: Windows kit/zipkit builds produce executables carrying an embedded icon chosen at build time - defaulting to the project icon src/runtime/punk1.ico, overridable per kit by its .vfs folder - by replacing the icon resources in the built executable using the twapi-based mechanism demonstrated in tcl-sfe (TEMP_REFERENCE/tcl-sfe, by twapi author and Tcl core member Ashok P. Nadkarni): RT_ICON/RT_GROUP_ICON replacement via twapi resource-update APIs, applied so the appended vfs payload stays intact (icon the stub before appending, or sfe-style split/update/reattach). Acceptance: a Windows `make.tcl project` build produces kit executables whose embedded icon resources are the project default punk1.ico, and a kit whose .vfs supplies an override icon gets that icon instead (verified by resource inspection, e.g twapi::extract_resources, not just Explorer eyeballing); the icon-replaced executables still boot to a working punk shell reading their vfs payload for the kit types we build (kit, zip, zipcat per mapvfs.config); runtimes under src/runtime are never modified - replacement applies to the built copies only; twapi unavailable or non-Windows platform skips the icon step with a notice and the build otherwise completes unchanged; rebuilds are idempotent (re-wrapping an already-iconed build copy converges, no resource accumulation); the override convention (filename/location in the kit's custom .vfs folder vs a mapvfs.config element) and the stub-vs-split ordering decision are recorded in the detail file with the tcl-sfe attribution. -### G-058 [proposed] Boot honours statically-linked runtime packages (static baseline seeding + packagepreference static-awareness) +### G-058 [achieved 2026-07-10] Boot honours statically-linked runtime packages (static baseline seeding + packagepreference static-awareness) Scope: src/vfs/_config/punk_main.tcl (boot auto_path/tm path filtering), src/modules/punk/packagepreference-999999.0a1.0.tm, src/modules/punk/repl-999999.0a1.0.tm + src/modules/punk/repl/codethread-999999.0a1.0.tm (code interp / codethread bootstrap), src/modules/shellthread-999999.0a1.0.tm (punkshell-created worker threads) as applicable, src/tests/ (un-gated unit tests + constraint-gated shell/kit integration tests), punkbin artifact repo (separate git repo, local checkout c:/repo/jn/punkbin - pinned runtime additions) Detail: goals/G-058-static-runtime-packages.md Goal: punkshell running on a runtime with statically-linked/builtin packages (tclsfe-x64: Thread/twapi/sqlite3/tdbc; punkbin runtimes' builtin Thread/tcllibc/vfs/vlerq; the expected shape of future zig-built static runtimes, G-005) keeps those packages resolvable in every interp and thread punkshell fabricates: boot captures the static baseline (info loaded entries with empty filename, plus their provided versions) before replacing package search paths, code interps and punkshell-created threads are seeded with ifneeded scripts mapping each static package to 'load {} ', and punk::packagepreference resolves static-vs-bundled by a documented version-aware policy instead of blindly loading a bundled dll over an already-provided static package. diff --git a/goals/G-058-static-runtime-packages.md b/goals/G-058-static-runtime-packages.md index 26891e88..3446c94d 100644 --- a/goals/G-058-static-runtime-packages.md +++ b/goals/G-058-static-runtime-packages.md @@ -1,6 +1,6 @@ # G-058 Boot honours statically-linked runtime packages (static baseline seeding + packagepreference static-awareness) -Status: proposed +Status: achieved 2026-07-10 Scope: src/vfs/_config/punk_main.tcl (boot auto_path/tm path filtering), src/modules/punk/packagepreference-999999.0a1.0.tm, src/modules/punk/repl-999999.0a1.0.tm + src/modules/punk/repl/codethread-999999.0a1.0.tm (code interp / codethread bootstrap), src/modules/shellthread-999999.0a1.0.tm (punkshell-created worker threads) as applicable, src/tests/ (un-gated unit tests + constraint-gated shell/kit integration tests), punkbin artifact repo (separate git repo, local checkout c:/repo/jn/punkbin - pinned runtime additions) Acceptance: see GOALS.md index entry (canonical). @@ -93,3 +93,50 @@ with NO directory to re-add - ifneeded seeding is required instead. - Related: [[G-005]] (zig static runtimes make this foundational), [[G-006]] (artifact retrieval), the tk-loading experiment that surfaced it (punk91 = tclsfe-x64 + punk9win.vfs per src/runtime/mapvfs.config). + +## Outcome (achieved 2026-07-10) + +Shipped as punk::lib 0.4.0, punk::repl 0.3.0, punk::packagepreference 0.2.0, +shellthread 1.7.0 + punk_main.tcl capture (punkshell 0.6.0). + +Implementation-discovered refinements to the Approach: + +- The tcl-sfe stub does NOT seed ifneeded entries for its statics: it registers + them process-wide via Tcl_StaticLibrary(NULL,...) and relies on pkgIndex.tcl + files inside its own //zipfs:/app zip, resolved through a per-interp PostInit + hook (tclSfeMainHook.c) forcing auto_path to //zipfs:/app. A punk kit's appended + vfs REPLACES that mount, so even the pristine boot interp has no name->load + mapping - the capture therefore probe-loads each empty-filename [info loaded] + prefix in a throwaway interp and records only packages the load actually + PROVIDES (diff on provided packages, not 'package names' - a probe load can + trigger an index scan that registers many names without providing them). +- Composite statics exclude themselves: static twapi's C init provides nothing + (its script layer lived in the replaced stub zip), so it is never seeded and the + bundled complete twapi is correctly the resolution - the observed + "static Twapi + twapi dll" double listing was registration+load, not a double + load, and the bundled copy was the only usable one. A configurable denylist + (::punkboot::static_probe_denylist, default: tk* vfs mk4tcl vlerq tdbc*) guards + side-effecting (tk creates '.') and known-composite inits from probing at all. +- Policy mechanics: seeding alone would HIDE newer bundled copies (a satisfiable + ifneeded suppresses the package unknown scan), so packagepreference triggers the + index scan for baseline names before resolution; and static REGISTRATIONS + (empty-filename info loaded entries) are excluded from the pre-existing + loaded-shared-object same-version pinning, which would otherwise force the + static version even when bundled is newer. Net policy: all candidates + registered, highest acceptable version wins; identical versions resolve to the + bundled copy (later-registered ifneeded entry). +- vfs/vfs::zip re-diagnosis: NOT a seeding defect - the tclsfe runtime doesn't + bundle tclvfs at all (tclkit-family runtimes carry vfs1.5.0+vlerq in their + tcl_library; sfe carries thread/twapi/sqlite3/tdbc only), so the punk91 kit + genuinely lacks the package and the repl's existing soft-fail warning stands. + Candidate follow-up if wanted: bundle a tcl9 tclvfs build in punk9win.vfs, or + accept vfs-less zipfs kits (vfs is primarily needed for .kit interop). +- Verification: capture confirmed on the stub (thread/Thread 3.0.5, sqlite3 + 3.53.0, dde 1.5b0 seeded; twapi excluded); punk91 rebuilt - repl boots with no + "can't find package Thread", code interp resolves Thread and introspects + ::punkboot::static_packages, fabricated interp + snapshot-seeded worker resolve + Thread, twapi resolves (staticruntime.test 4/4 behind the capability probe); + staticseed.test 4/4 + staticpolicy.test 4/4 un-gated (9.0.3 and 8.7); + punk902z + full shell tree unchanged (23 pass/2 constraint skips); full punk/lib + tree green. tclsfe-x64.exe pinned in punkbin (commit b2dbbe6) with sha1sums and + new AGENTS.md scaffolding. diff --git a/punkproject.toml b/punkproject.toml index 7aab4a78..c23d3e9e 100644 --- a/punkproject.toml +++ b/punkproject.toml @@ -1,3 +1,3 @@ [project] name = "punkshell" -version = "0.5.0" +version = "0.6.0" diff --git a/src/modules/punk/lib-999999.0a1.0.tm b/src/modules/punk/lib-999999.0a1.0.tm index aa93ddb6..2537a2dc 100644 --- a/src/modules/punk/lib-999999.0a1.0.tm +++ b/src/modules/punk/lib-999999.0a1.0.tm @@ -8197,6 +8197,12 @@ namespace eval punk::lib { is used, in which case punk::libunknown is sourced from the tm paths that were already set up by the earlier part of the script. + When the kit boot captured runtime static/builtin packages + (::punkboot::static_packages - see punk_main.tcl), the script also copies + that baseline and seeds 'package ifneeded {load {} }' + entries so statically-linked packages (e.g Thread/twapi on a tcl-sfe + runtime) resolve in the target. + See also: punk::lib::interp_sync_package_paths" @opts -libunknown -type boolean -default 0 -help\ @@ -8235,6 +8241,18 @@ namespace eval punk::lib { append script "tcl::tm::add \{*\}\[list [lreverse $tmlist]\]\n" append script "set ::auto_path \[list $ap\]\n" append script "package prefer $prefer\n" + #G-058: propagate runtime static/builtin package mappings captured at kit boot + #(see punk_main.tcl) so 'package require' of a statically-linked package (e.g + #Thread/twapi on a tcl-sfe runtime) resolves in the new thread/interp even though + #no pkgIndex for it exists on the (replaced) package search paths. + if {[info exists ::punkboot::static_packages] && [dict size $::punkboot::static_packages]} { + append script "namespace eval ::punkboot {}\n" + append script [list set ::punkboot::static_prefixes $::punkboot::static_prefixes] \n + append script [list set ::punkboot::static_packages $::punkboot::static_packages] \n + dict for {pkgname pinfo} $::punkboot::static_packages { + append script "if \{\[package provide [list $pkgname]\] eq \"\"\} \{[list package ifneeded $pkgname [dict get $pinfo version] [list load {} [dict get $pinfo prefix]]]\}\n" + } + } if {$do_libunknown} { if {[info exists ::punk::libunknown::epoch]} { append script "namespace eval ::punk::libunknown {}\n" @@ -8299,6 +8317,12 @@ namespace eval punk::lib { package unknown handler and scan cache as the parent. This is recommended when the child will do significant package loading. + When the kit boot captured runtime static/builtin packages + (::punkboot::static_packages - see punk_main.tcl), the child also receives + that baseline plus 'package ifneeded {load {} }' + seeding so statically-linked packages (e.g Thread/twapi on a tcl-sfe + runtime) resolve in the child. + This is an opt-in helper — call sites that don't use it keep their current behavior. It does not load packages, install punk::packagepreference, share channels, or configure safe-interp restrictions — those remain caller @@ -8340,6 +8364,20 @@ namespace eval punk::lib { interp eval $interp {tcl::tm::remove {*}[tcl::tm::list]} interp eval $interp [list tcl::tm::add {*}[lreverse [tcl::tm::list]]] interp eval $interp [list package prefer [package prefer]] + #G-058: propagate runtime static/builtin package mappings captured at kit boot + #(see punk_main.tcl) so 'package require' of a statically-linked package (e.g + #Thread/twapi on a tcl-sfe runtime) resolves in the child interp even though no + #pkgIndex for it exists on the (replaced) package search paths. + if {[info exists ::punkboot::static_packages] && [dict size $::punkboot::static_packages]} { + interp eval $interp [list namespace eval ::punkboot {}] + interp eval $interp [list set ::punkboot::static_prefixes $::punkboot::static_prefixes] + interp eval $interp [list set ::punkboot::static_packages $::punkboot::static_packages] + dict for {pkgname pinfo} $::punkboot::static_packages { + if {[interp eval $interp [list package provide $pkgname]] eq ""} { + interp eval $interp [list package ifneeded $pkgname [dict get $pinfo version] [list load {} [dict get $pinfo prefix]]] + } + } + } #extended: copy epoch + source libunknown + call init if {$do_libunknown} { if {[info exists ::punk::libunknown::epoch]} { diff --git a/src/modules/punk/lib-buildversion.txt b/src/modules/punk/lib-buildversion.txt index ee96c9b4..d742dffb 100644 --- a/src/modules/punk/lib-buildversion.txt +++ b/src/modules/punk/lib-buildversion.txt @@ -1,6 +1,7 @@ -0.3.1 +0.4.0 #First line must be a semantic version number #all other lines are ignored. +#0.4.0 - G-058: snapshot_package_paths and interp_sync_package_paths now also propagate the runtime static/builtin package baseline captured at kit boot (::punkboot::static_prefixes/static_packages - see punk_main.tcl) and seed 'package ifneeded {load {} }' entries in the target thread/interp, so statically-linked runtime packages (e.g Thread/twapi on a tcl-sfe runtime) resolve in fabricated interps whose package search paths punkshell controls. No-op when no baseline was captured (plain tclsh dev launches). #0.3.1 - Tcl 9.1 compatibility (TIP 746 removed expr behaviour from lseq operands): range (lseq branch) now normalizes int[+-]int offset operands itself via offset_expr - callers such as `range 0 [llength $list]-1` keep working; also aligned the lseq branch with the fallback contract: default by now infers direction (range 5 1 -> descending, previously empty under tcl9) and by 0 returns empty (Tcl 9.1 lseq changed by-0 to return one element). lzipn_tcl9b/lzipn_tcl9c/cols/cols2: lseq expression operands wrapped in expr. check::has_tclbug_safeinterp_compile: Tcl 9.1 safe interps hide tcl::unsupported::* - falls back to interp invokehidden tcl:unsupported:disassemble #0.3.0 - new: punk::lib::check::has_libbug_udp_threadexit + libbug_udp_threadexit_applies classifier - version-based detection of tcludp < 1.0.13 on Tcl 9 Windows (per-thread exit handler closes process-global event handles; G-036). has_libbug_* is the new check family for bundled/vendored library bugs, surfaced through 'help tcl' alongside has_tclbug_*; buginfo dicts may carry a full 'url' reference key (non tcl-core trackers) #0.2.1 - call-site update for punk::console 0.6.0 tsv array rename: console -> punk_console (is_raw); no behaviour change diff --git a/src/modules/punk/packagepreference-999999.0a1.0.tm b/src/modules/punk/packagepreference-999999.0a1.0.tm index ff58a76b..fbb2c717 100644 --- a/src/modules/punk/packagepreference-999999.0a1.0.tm +++ b/src/modules/punk/packagepreference-999999.0a1.0.tm @@ -171,6 +171,24 @@ tcl::namespace::eval punk::packagepreference { return [uplevel 1 [list $COMMANDSTACKNEXT {*}$args]] } + #G-058 static-vs-bundled policy for runtime static/builtin packages + #(baseline captured at kit boot into ::punkboot::static_packages - see punk_main.tcl). + #Ensure the static mapping exists in this interp (seeding is normally done at + #interp/thread creation - this covers stragglers), then trigger the package + #unknown index scan so any bundled vfs/module copies are ALSO registered before + #resolution. With all candidates registered, standard resolution compares + #versions (highest acceptable wins) - a static copy is not blindly masked by an + #older bundled dll, and a genuinely newer bundled copy remains reachable. + #(On identical versions the bundled copy's ifneeded entry, registered by the + # later scan, wins - same version either way.) + if {[info exists ::punkboot::static_packages] && [dict exists $::punkboot::static_packages $pkg]} { + set spinfo [dict get $::punkboot::static_packages $pkg] + if {[dict get $spinfo version] ni [$COMMANDSTACKNEXT_ORIGINAL versions $pkg]} { + $COMMANDSTACKNEXT_ORIGINAL ifneeded $pkg [dict get $spinfo version] [list load {} [dict get $spinfo prefix]] + } + catch {uplevel 1 [list {*}[$COMMANDSTACKNEXT_ORIGINAL unknown] $pkg]} + } + if {!$is_exact && [llength $vwant] <= 1 } { #required version unspecified - or specified singularly set available_versions [$COMMANDSTACKNEXT_ORIGINAL versions $pkg] @@ -179,7 +197,12 @@ tcl::namespace::eval punk::packagepreference { #An attempt to detect dll/so loaded and try to load same version #dll/so files are often named with version numbers that don't contain dots or a version number at all #e.g sqlite3400.dll Thread288.dll - set pkgloadedinfo [lsearch -nocase -inline -index 1 [info loaded] $pkg] + #G-058: exclude empty-filename entries - those are static REGISTRATIONS + #(process-global, not necessarily loaded anywhere) and forcing their + #version here would defeat the version-aware static-vs-bundled policy + #above (e.g pinning to the static version even when a bundled copy is + #genuinely newer) + set pkgloadedinfo [lsearch -nocase -inline -index 1 [lsearch -all -inline -not -exact -index 0 [info loaded] {}] $pkg] if {[llength $pkgloadedinfo]} { if {[llength $available_versions] > 1} { diff --git a/src/modules/punk/packagepreference-buildversion.txt b/src/modules/punk/packagepreference-buildversion.txt index b0df4036..ba459500 100644 --- a/src/modules/punk/packagepreference-buildversion.txt +++ b/src/modules/punk/packagepreference-buildversion.txt @@ -1,4 +1,5 @@ -0.1.1 +0.2.0 #First line must be a semantic version number #all other lines are ignored. +#0.2.0 - G-058 static-vs-bundled policy: on require of a package in the runtime static baseline (::punkboot::static_packages), ensure the static ifneeded mapping exists and trigger the package unknown index scan so bundled vfs/module copies register BEFORE resolution - version comparison then applies (highest wins; identical versions resolve to the bundled copy). Static REGISTRATIONS (empty-filename info loaded entries) are now excluded from the loaded-shared-object same-version forcing, which would otherwise pin resolution to the static version even when a bundled copy is genuinely newer. #0.1.1 - moduledoc auto-load success notice moved from stdout to stderr (stdout must stay clean for script/exec contexts e.g the punk 'script' subcommand; failure branch already used stderr) diff --git a/src/modules/punk/repl-999999.0a1.0.tm b/src/modules/punk/repl-999999.0a1.0.tm index 9b8e4ceb..a736bdac 100644 --- a/src/modules/punk/repl-999999.0a1.0.tm +++ b/src/modules/punk/repl-999999.0a1.0.tm @@ -3241,6 +3241,8 @@ namespace eval repl { } %autopath% [list $::auto_path] {*}{ } %lib_epoch% [list $::punk::libunknown::epoch] {*}{ } %packageprefer% [list [package prefer]] {*}{ + } %staticprefixes% [list [expr {[info exists ::punkboot::static_prefixes] ? $::punkboot::static_prefixes : ""}]] {*}{ + } %staticpackages% [list [expr {[info exists ::punkboot::static_packages] ? $::punkboot::static_packages : ""}]] {*}{ } ] #scriptmap applied at end to satisfy silly editor highlighting. @@ -3256,6 +3258,20 @@ namespace eval repl { #It will also need to be added in that interp set ::auto_path %autopath% package prefer %packageprefer% + #G-058: runtime static/builtin package baseline captured at kit boot (punk_main.tcl). + #Make it available in this codethread and seed ifneeded mappings so statically-linked + #packages (e.g Thread/twapi on a tcl-sfe runtime) resolve here - and so + #punk::lib::interp_sync_package_paths can propagate the same into the code interp. + namespace eval ::punkboot {} + set ::punkboot::static_prefixes %staticprefixes% + set ::punkboot::static_packages %staticpackages% + if {[dict size $::punkboot::static_packages]} { + dict for {pkgname pinfo} $::punkboot::static_packages { + if {[package provide $pkgname] eq ""} { + package ifneeded $pkgname [dict get $pinfo version] [list load {} [dict get $pinfo prefix]] + } + } + } set tclmajorv [lindex [split [tcl::info::tclversion] .] 0] #jmn #puts stdout "CODETHREAD tm list" diff --git a/src/modules/punk/repl-buildversion.txt b/src/modules/punk/repl-buildversion.txt index 700f4255..b61cb517 100644 --- a/src/modules/punk/repl-buildversion.txt +++ b/src/modules/punk/repl-buildversion.txt @@ -1,6 +1,7 @@ -0.2.2 +0.3.0 #First line must be a semantic version number #all other lines are ignored. +#0.3.0 - G-058: codethread init script receives the runtime static/builtin package baseline (::punkboot::static_prefixes/static_packages via new %staticprefixes%/%staticpackages% scriptmap entries) and seeds ifneeded mappings in the codethread interp, so the code interp (seeded in turn via punk::lib::interp_sync_package_paths) can package-require statically-linked runtime packages - fixes 'can't find package Thread' booting a punk9win.vfs kit on the tclsfe-x64 static runtime (punk91) #0.2.2 - repl_handler line-mode waiting-chunks path no longer performs its opportunistic unsized read on tcl 8.6 windows console channels (no -inputmode key, real twapi console handle): the path is entered via 'after idle' with the channel usually drained, and on the 8.6 console driver a drained read parks a blocking cooked-mode ReadConsole that a later raw flip cannot cancel - after typed-ahead was stashed during a terminal-query raw window, that parked read swallowed every subsequent query response in the session until Enter (companion to the punk::console 0.7.1 three-site console-misdetection fix). On such consoles the path now consumes only data already in the Tcl channel buffer (chan pending input + sized read - no driver probe); with nothing buffered it processes the stashed complete lines directly and arms the readable handler for any remaining partial line (replacing an after-idle reinvoke that could never progress). Behaviour on tcl 9/8.7 (-inputmode consoles) and in raw mode is unchanged. #0.2.1 - call-site update for punk::console 0.6.0 tsv array rename: console -> punk_console (is_raw); no behaviour change #0.2.0 - codethread init_script now propagates package prefer via %packageprefer% scriptmap entry diff --git a/src/modules/shellthread-999999.0a1.0.tm b/src/modules/shellthread-999999.0a1.0.tm index 42767dc3..601f8a65 100644 --- a/src/modules/shellthread-999999.0a1.0.tm +++ b/src/modules/shellthread-999999.0a1.0.tm @@ -617,7 +617,9 @@ namespace eval shellthread::manager { #set ts_start [::shellthread::iso8601] set tidworker [thread::create -preserved] - set init_script [string map [list %ts_start% $ts_start %mp% [tcl::tm::list] %ap% $::auto_path %tidcli% $tidclient %sd% $settingsdict] { + set static_prefixes [expr {[info exists ::punkboot::static_prefixes] ? $::punkboot::static_prefixes : ""}] + set static_packages [expr {[info exists ::punkboot::static_packages] ? $::punkboot::static_packages : ""}] + set init_script [string map [list %ts_start% $ts_start %mp% [tcl::tm::list] %ap% $::auto_path %tidcli% $tidclient %sd% $settingsdict %spfx% [list $static_prefixes] %spkg% [list $static_packages]] { #set tclbase [file dirname [file dirname [info nameofexecutable]]] #set tcllib $tclbase/lib #if {$tcllib ni $::auto_path} { @@ -642,6 +644,19 @@ namespace eval shellthread::manager { set ::auto_path [dict get $::settingsinfo auto_path] } + #G-058: runtime static/builtin package baseline captured at kit boot (punk_main.tcl) + #seed ifneeded mappings so statically-linked packages resolve in this worker + namespace eval ::punkboot {} + set ::punkboot::static_prefixes %spfx% + set ::punkboot::static_packages %spkg% + if {[dict size $::punkboot::static_packages]} { + dict for {pkgname pinfo} $::punkboot::static_packages { + if {[package provide $pkgname] eq ""} { + package ifneeded $pkgname [dict get $pinfo version] [list load {} [dict get $pinfo prefix]] + } + } + } + package require punk::packagepreference punk::packagepreference::install diff --git a/src/modules/shellthread-buildversion.txt b/src/modules/shellthread-buildversion.txt index a34e44ca..f74ef5af 100644 --- a/src/modules/shellthread-buildversion.txt +++ b/src/modules/shellthread-buildversion.txt @@ -1,4 +1,5 @@ -1.6.3 +1.7.0 #First line must be a semantic version number #all other lines are ignored. +#1.7.0 - G-058: worker thread init script seeds the runtime static/builtin package baseline (::punkboot::static_prefixes/static_packages captured at kit boot) with ifneeded {load {} } mappings before packagepreference install, so workers can require statically-linked runtime packages #1.6.3 - manager teardown must not block on a wedged worker (G-036: Tcl 9 console+udp worker event-loop wedge): unsubscribe's settings-reset thread::send is now -async (was bare synchronous - froze exit/quit when the shellfilter-run syslog worker had wedged); shutdown_free_threads keeps its timeout timer armed across all vwait iterations (was cancelled on first response, leaving later waits unbounded); get_tag_config's bare synchronous send flagged as the same hazard class (needs a result - convert to async+vwait-with-timeout if it shows in a hang) diff --git a/src/tests/modules/AGENTS.md b/src/tests/modules/AGENTS.md index c7d0c552..d34364dc 100644 --- a/src/tests/modules/AGENTS.md +++ b/src/tests/modules/AGENTS.md @@ -42,5 +42,7 @@ Unit tests for editable source modules under `src/modules/`, `src/modules_tcl8/` - `punk/args/` — punk::args tests (`testsuites/args/`): parsing, choices/choicegroups, forms, rendering/indentation characterization, usage-marking characterization (`usagemarking.test`: -parsedargs/-badarg/-parsestatus/-scheme marking primitives plus goodchoice highlighting of selected/default-in-effect choice words, asserted by SGR-parameter subset against the live colour arrays; the G-049 nocolour/colour-leak GAP pins flipped 2026-07-10 to scheme-statelessness assertions), and the G-049 parse-status structure (`parsestatus.test`: punk::args::parse_status overall/per-argument statuses, badarg for type/allocation failures, -caller attribution, errorcode -argspecs stripping) - `punk/ns/` — punk::ns tests (`testsuites/ns/`): cmdwhich/cmdinfo/cmd_traverse doc-lookup flow (`cmdflow.test`, G-040 parity) and cmdhelp usage-rendering integration (`cmdhelp.test`: scheme selection, goodarg/badarg marking incl type/allocation failures, goodchoice highlighting of supplied/default choice words, alias path, cmdinfo result shape, queried-command failure attribution, and `-return dict` parse-status returns (G-049 - its GAP pins flipped 2026-07-10); remaining GAP pins for pseudo-command cmdtype + space-form docid prefixes (G-051, real `string is` pins behind the have_tclcoredocs constraint), TclOO undocumented-method fallback (G-052), and synopsis marking absence (G-050)) - `punk/mix/` — punk::mix::cli tests (prune helpers, punkcheck virtual sources) and punk::mix::commandset::repo fossil move/rename characterization tests (`testsuites/repo/`, FOSSIL_HOME-isolated; GAP-marked tests pin behaviour G-022 will change) +- `punk/lib/` — punk::lib tests (`testsuites/lib/`): range/index/parse/compat/interp_sync utilities, and G-058 static-baseline seeding (`staticseed.test`: interp_sync_package_paths/snapshot_package_paths propagate a simulated ::punkboot static baseline and seed `load {} ` ifneeded mappings; no-op without a baseline) +- `punk/packagepreference/` — punk::packagepreference tests (`testsuites/packagepreference/`): G-058 static-vs-bundled policy (`staticpolicy.test`: require of a baseline package triggers the index scan before resolution so a newer bundled copy wins, static beats older bundled, exact requires of bundled versions stay reachable, missing static mappings get seeded) - `punk/libunknown/` — .tm same-version shadowing pin-tests (`testsuites/shadowing/`): tcl::tm::add prepend rule, head-of-tm-list wins exact-version ties, version beats order, punk::libunknown parity — shipped behaviour depends on these (runtests tm ordering, punk_main package-mode precedence, G-033); mixed .tm/pkgIndex.tcl characterization is goal G-035 diff --git a/src/tests/modules/punk/lib/testsuites/lib/staticseed.test b/src/tests/modules/punk/lib/testsuites/lib/staticseed.test new file mode 100644 index 00000000..078e9cc3 --- /dev/null +++ b/src/tests/modules/punk/lib/testsuites/lib/staticseed.test @@ -0,0 +1,131 @@ +package require tcltest + +package require punk::lib + +#G-058 unit coverage: propagation/seeding of the runtime static/builtin package baseline +#(::punkboot::static_prefixes / ::punkboot::static_packages, captured at kit boot by +#punk_main.tcl) through punk::lib::interp_sync_package_paths and +#punk::lib::snapshot_package_paths. These tests run un-gated under plain tclsh using a +#SIMULATED baseline - the ifneeded entries they assert reference a fake static prefix +#that is never actually loaded. Kit-level integration (real static runtime) is covered +#by src/tests/shell/testsuites/punkexe/staticruntime.test behind a capability constraint. + +namespace eval ::testspace { + namespace import ::tcltest::* + variable common { + set result "" + } + + #save/restore any real ::punkboot baseline around each test (shared test interp) + proc baseline_save {} { + variable saved + set saved [dict create] + foreach v {static_prefixes static_packages} { + if {[info exists ::punkboot::$v]} { + dict set saved $v [set ::punkboot::$v] + } + } + } + proc baseline_restore {} { + variable saved + foreach v {static_prefixes static_packages} { + if {[dict exists $saved $v]} { + set ::punkboot::$v [dict get $saved $v] + } else { + catch {unset ::punkboot::$v} + } + } + } + proc baseline_fake {} { + namespace eval ::punkboot {} + set ::punkboot::static_prefixes [list Fakestat] + set ::punkboot::static_packages [dict create {*}{ + fakestatic {version 1.2 prefix Fakestat} + Fakestatic {version 1.2 prefix Fakestat} + }] + } + proc baseline_none {} { + catch {unset ::punkboot::static_prefixes} + catch {unset ::punkboot::static_packages} + } + + test staticseed_interp_sync_seeds {interp_sync_package_paths seeds baseline ifneeded mappings and copies the ::punkboot vars into the child}\ + -setup { + set result "" + baseline_save + baseline_fake + interp create staticseed_child + }\ + -body { + punk::lib::interp_sync_package_paths staticseed_child + lappend result [interp eval staticseed_child {package ifneeded fakestatic 1.2}] + lappend result [interp eval staticseed_child {package ifneeded Fakestatic 1.2}] + lappend result [interp eval staticseed_child {dict keys $::punkboot::static_packages}] + lappend result [interp eval staticseed_child {set ::punkboot::static_prefixes}] + }\ + -cleanup { + interp delete staticseed_child + baseline_restore + }\ + -result [list {load {} Fakestat} {load {} Fakestat} {fakestatic Fakestatic} Fakestat] + + test staticseed_interp_sync_no_overwrite_provided {a package already provided in the child is not re-mapped; other baseline entries still seed}\ + -setup { + set result "" + baseline_save + baseline_fake + interp create staticseed_child + }\ + -body { + interp eval staticseed_child {package provide fakestatic 9.9} + punk::lib::interp_sync_package_paths staticseed_child + lappend result [interp eval staticseed_child {package ifneeded fakestatic 1.2}] + lappend result [interp eval staticseed_child {package ifneeded Fakestatic 1.2}] + }\ + -cleanup { + interp delete staticseed_child + baseline_restore + }\ + -result [list {} {load {} Fakestat}] + + test staticseed_snapshot_script_seeds {snapshot_package_paths returns a script that reproduces the baseline and seeding in a fresh interp}\ + -setup { + set result "" + baseline_save + baseline_fake + interp create staticseed_child + }\ + -body { + set scr [punk::lib::snapshot_package_paths] + lappend result [string match "*::punkboot::static_packages*" $scr] + interp eval staticseed_child $scr + lappend result [interp eval staticseed_child {package ifneeded fakestatic 1.2}] + lappend result [interp eval staticseed_child {dict get $::punkboot::static_packages fakestatic prefix}] + }\ + -cleanup { + interp delete staticseed_child + baseline_restore + }\ + -result [list 1 {load {} Fakestat} Fakestat] + + test staticseed_noop_without_baseline {with no captured baseline (plain tclsh) neither helper adds punkboot state or ifneeded entries}\ + -setup { + set result "" + baseline_save + baseline_none + interp create staticseed_child + }\ + -body { + punk::lib::interp_sync_package_paths staticseed_child + lappend result [interp eval staticseed_child {info exists ::punkboot::static_packages}] + lappend result [interp eval staticseed_child {package ifneeded fakestatic 1.2}] + set scr [punk::lib::snapshot_package_paths] + lappend result [string match "*static_packages*" $scr] + }\ + -cleanup { + interp delete staticseed_child + baseline_restore + }\ + -result [list 0 {} 0] +} +tcltest::cleanupTests ;#needed to produce test summary line. diff --git a/src/tests/modules/punk/packagepreference/testsuites/packagepreference/staticpolicy.test b/src/tests/modules/punk/packagepreference/testsuites/packagepreference/staticpolicy.test new file mode 100644 index 00000000..821d338c --- /dev/null +++ b/src/tests/modules/punk/packagepreference/testsuites/packagepreference/staticpolicy.test @@ -0,0 +1,119 @@ +package require tcltest + +package require punk::lib + +#G-058 unit coverage: punk::packagepreference static-vs-bundled resolution policy. +#On 'package require' of a package present in the runtime static baseline +#(::punkboot::static_packages), the overloaded require ensures the static ifneeded +#mapping exists and triggers the package unknown index scan so bundled copies register +#BEFORE resolution - then standard version comparison applies (highest acceptable wins). +#Without the scan trigger, a seeded static version would satisfy the require immediately +#and a genuinely newer bundled copy would never be seen. +# +#These tests run un-gated under plain tclsh: the "static" package is simulated with a +#pre-registered working ifneeded script (the policy block only adds its own +#'load {} ' mapping when the baseline version is absent from package versions), +#and the "bundled" copies are pkgIndex.tcl fixtures on the child interp's auto_path. +#Real static runtime integration: src/tests/shell/testsuites/punkexe/staticruntime.test. + +namespace eval ::testspace { + namespace import ::tcltest::* + variable common { + set result "" + } + + variable fixdir_new [makeDirectory staticpolicy_newer] + variable fixdir_old [makeDirectory staticpolicy_older] + makeFile {package ifneeded fakestatic 1.5 {package provide fakestatic 1.5}} pkgIndex.tcl $fixdir_new + makeFile {package ifneeded fakestatic 1.0 {package provide fakestatic 1.0}} pkgIndex.tcl $fixdir_old + + #child interp with punk::packagepreference installed, a simulated static baseline for + #'fakestatic 1.2', and auto_path pointing at the given bundled-fixture dir + proc policy_child {name bundledir} { + interp create $name + #tm paths so the child can require punk::packagepreference from the source tree + punk::lib::interp_sync_package_paths $name + interp eval $name {package require punk::packagepreference} + interp eval $name {punk::packagepreference::install} + interp eval $name [list set ::auto_path [list $bundledir]] + interp eval $name { + namespace eval ::punkboot {} + set ::punkboot::static_prefixes [list Fakestat] + set ::punkboot::static_packages [dict create fakestatic {version 1.2 prefix Fakestat}] + #simulate a WORKING static loader for the baseline version (a real runtime + #would satisfy this via 'load {} Fakestat') + package ifneeded fakestatic 1.2 {package provide fakestatic 1.2} + } + } + + test staticpolicy_newer_bundled_wins {a bundled copy newer than the static baseline version is found (scan triggered before resolution) and wins}\ + -setup { + set result "" + variable fixdir_new + policy_child pol_child $fixdir_new + }\ + -body { + lappend result [interp eval pol_child {package require fakestatic}] + lappend result [interp eval pol_child {package provide fakestatic}] + }\ + -cleanup { + interp delete pol_child + }\ + -result [list 1.5 1.5] + + test staticpolicy_static_beats_older_bundled {the static baseline version wins over an older bundled copy}\ + -setup { + set result "" + variable fixdir_old + policy_child pol_child $fixdir_old + }\ + -body { + lappend result [interp eval pol_child {package require fakestatic}] + lappend result [interp eval pol_child {package provide fakestatic}] + }\ + -cleanup { + interp delete pol_child + }\ + -result [list 1.2 1.2] + + test staticpolicy_exact_older_bundled_reachable {an exact require of the older bundled version still resolves (scan applies to exact requires too)}\ + -setup { + set result "" + variable fixdir_old + policy_child pol_child $fixdir_old + }\ + -body { + lappend result [interp eval pol_child {package require -exact fakestatic 1.0}] + }\ + -cleanup { + interp delete pol_child + }\ + -result [list 1.0] + + test staticpolicy_seeds_missing_mapping {when the baseline version has no ifneeded entry the policy block adds the 'load {} ' mapping}\ + -setup { + set result "" + variable fixdir_old + interp create pol_child + punk::lib::interp_sync_package_paths pol_child + interp eval pol_child {package require punk::packagepreference} + interp eval pol_child {punk::packagepreference::install} + interp eval pol_child [list set ::auto_path [list $fixdir_old]] + interp eval pol_child { + namespace eval ::punkboot {} + set ::punkboot::static_prefixes [list Fakestat] + set ::punkboot::static_packages [dict create fakestatic {version 1.2 prefix Fakestat}] + #no working ifneeded for 1.2 - the require will fail at load time + #(no real static Fakestat exists) but must have REGISTERED the mapping + } + }\ + -body { + catch {interp eval pol_child {package require fakestatic 1.2}} + lappend result [interp eval pol_child {package ifneeded fakestatic 1.2}] + }\ + -cleanup { + interp delete pol_child + }\ + -result [list {load {} Fakestat}] +} +tcltest::cleanupTests ;#needed to produce test summary line. diff --git a/src/tests/shell/AGENTS.md b/src/tests/shell/AGENTS.md index 3e5f85b8..bdb79a14 100644 --- a/src/tests/shell/AGENTS.md +++ b/src/tests/shell/AGENTS.md @@ -19,6 +19,7 @@ Tests for shell-level behavior, command-line execution, and stdin/stdout interac - `testsuites/punkexe/shellexit.test` guards piped-stdin termination behaviour of the built executable (no `invalid thread handle` on shutdown races, no console-reopen hang on eof/error paths, exitcode propagation); do not weaken these tests - a red result means a regression in punk::repl shutdown or app-punkshell eof handling. - `testsuites/punkexe/scriptexec.test` covers the `script` subcommand (app-punkscript, G-015): piped execution + honest exit codes, result echo, file-form argv, the `lib:` scriptlib-resolution mechanism, and the tclsh-matching Tk main-loop behaviour. The `lib:` and Tk cases resolve committed fixtures under `scriptlib/_punktest/` (test-owned; a `lib:`-resolvable fixture cannot live in a tcltest temp dir since resolution only searches scriptlib locations relative to the exe). Tk cases are gated by the `punktk` constraint, auto-detected by probing whether the kit can load Tk. - punkexe tests run against the built binary: after changing punk::repl or app-punkshell source, rebuild via `make.tcl packages`, `make.tcl vfscommonupdate` (requires interactive `y`), then `make.tcl project` before trusting results; `make.tcl project` alone does not refresh `_vfscommon.vfs`. +- `testsuites/punkexe/staticruntime.test` (G-058) targets a kit built on a runtime with statically-linked packages: resolved from `env(PUNK_STATICKIT_TEST_EXE)`, else `/bin/punk91.exe`, gated by the capability-probed `statickitavailable` constraint (a probe run must show a captured `::punkboot::static_packages` baseline including Thread - not mere file existence). It asserts repl boot without the codethread Thread failure, Thread resolution in the code interp / fabricated interps / snapshot-seeded threads, and twapi resolution per the static-vs-bundled policy. The verification runtime (tclsfe-x64.exe) is pinned in the punkbin artifact repo so the constraint is satisfiable off the original dev machine. Kit script-exec paths must be passed with forward slashes. - GOAL tests (constraint `punkgoals`, enabled by env `PUNK_TEST_GOALS=1`) cover the piped-stdin-then-interactive-shell drop-in (`shellpipe_eof_then_interactive_shell`, `shellpipe_script_and_eof_then_interactive_shell`); they are excluded from normal runs because they require an openable console in the test environment and briefly attach a live shell to it - they pass in console-attached environments against a current build. - The eof-policy discriminator is `env(PUNK_PIPE_EOF)` (`exit`|`interactive`|unset=console heuristic) implemented in app-punkshell; termination-guard tests spawn children with `exit`, goal tests spawn with it unset. diff --git a/src/tests/shell/testsuites/punkexe/staticruntime.test b/src/tests/shell/testsuites/punkexe/staticruntime.test new file mode 100644 index 00000000..af842cc4 --- /dev/null +++ b/src/tests/shell/testsuites/punkexe/staticruntime.test @@ -0,0 +1,209 @@ +package require tcltest + +#G-058 integration: a punkshell kit built on a runtime with statically-linked packages +#(e.g punk91 = tclsfe-x64.exe + punk9win.vfs: static Thread/twapi/sqlite3/tdbc) must keep +#those packages resolvable in the interps/threads punkshell fabricates - the repl's +#codethread/code interp, punk::lib::interp_sync_package_paths children, and +#snapshot_package_paths threads. Seeding/policy logic has un-gated unit coverage in +#modules/punk/lib/.../staticseed.test and modules/punk/packagepreference/.../staticpolicy.test; +#these tests exercise the real kit end to end. +# +#CONSTRAINT (capability-probed, not filename-hardcoded): the target executable is resolved +#from env(PUNK_STATICKIT_TEST_EXE) if set, else /bin/punk91.exe - and is only +#used if a probe run shows a captured static baseline including Thread +#(::punkboot::static_packages populated by the kit boot). Tests skip cleanly otherwise. +#To make the constraint satisfiable on other machines the verification runtime +#(tclsfe-x64.exe) is pinned in the punkbin artifact repository (win32-x86_64/ + +#sha1sums.txt) - retrieve to bin/runtime// and rebuild. +# +#NOTE: tests exercise the BUILT executable - after changing punk_main.tcl or the seeding +#modules, rebuild with 'make.tcl modules', 'make.tcl vfscommonupdate', 'make.tcl project' +#or these tests will run against a stale binary. + +namespace eval ::testspace { + namespace import ::tcltest::* + + variable testdir [file dirname [file normalize [info script]]] + #/src/tests/shell/testsuites/punkexe -> 5 levels up to + variable projectroot [file normalize [file join $testdir .. .. .. .. ..]] + + variable statickitexe "" + if {[info exists ::env(PUNK_STATICKIT_TEST_EXE)] && $::env(PUNK_STATICKIT_TEST_EXE) ne ""} { + set statickitexe [file normalize $::env(PUNK_STATICKIT_TEST_EXE)] + } else { + set candidate [file join $projectroot bin punk91.exe] + if {[file exists $candidate]} { + set statickitexe $candidate + } + } + + variable punk_run_timeout_ms 30000 + variable runstate + array set runstate {} + + proc punk_run_read {chan} { + variable runstate + append runstate(output) [read $chan] + if {[chan eof $chan]} { + chan event $chan readable {} + set runstate(done) eof + } + } + #Run the target executable with args, feed stdin_data via pipe then half-close. + #Returns dict: timedout 0|1, exitcode , output + proc punk_run {exe cmdargs stdin_data} { + variable runstate + variable punk_run_timeout_ms + array unset runstate + set runstate(output) "" + set runstate(done) "" + set env_had [info exists ::env(PUNK_PIPE_EOF)] + if {$env_had} { + set env_save $::env(PUNK_PIPE_EOF) + } + set ::env(PUNK_PIPE_EOF) exit + set chan [open |[list $exe {*}$cmdargs 2>@1] r+] + if {$env_had} { + set ::env(PUNK_PIPE_EOF) $env_save + } else { + unset ::env(PUNK_PIPE_EOF) + } + set pids [pid $chan] + chan configure $chan -blocking 0 -translation binary + catch { + puts -nonewline $chan $stdin_data + flush $chan + chan close $chan write + } + set timerid [after $punk_run_timeout_ms [list set [namespace current]::runstate(done) timeout]] + chan event $chan readable [list [namespace current]::punk_run_read $chan] + vwait [namespace current]::runstate(done) + after cancel $timerid + set timedout [expr {$runstate(done) eq "timeout"}] + set exitcode "" + if {$timedout} { + foreach p $pids { + catch {exec {*}[auto_execok taskkill] /F /T /PID $p} killres + lappend exitcode [list killed $p] + } + catch {chan close $chan} + } else { + chan configure $chan -blocking 1 + if {[catch {chan close $chan} errM errOpts]} { + set details [dict get $errOpts -errorcode] + if {[lindex $details 0] eq "CHILDSTATUS"} { + set exitcode [lindex $details 2] + } else { + set exitcode [list error $errM] + } + } else { + set exitcode 0 + } + } + return [dict create timedout $timedout exitcode $exitcode output $runstate(output)] + } + + #capability probe: run the candidate with a script that reports the captured baseline. + #Constraint requires a non-empty ::punkboot::static_packages including Thread. + variable probe_ok 0 + if {$statickitexe ne "" && [file exists $statickitexe]} { + variable probescript [makeFile { + if {[info exists ::punkboot::static_packages]} { + puts "STATICBASELINE=[dict keys $::punkboot::static_packages]" + } else { + puts "STATICBASELINE=" + } + exit 0 + } staticruntime_probe.tcl] + #kit script exec requires forward slashes (the shellspy dispatch mangles backslashes) + set proberesult [punk_run $statickitexe [list [string map {\\ /} $probescript]] ""] + if {![dict get $proberesult timedout] && [regexp {STATICBASELINE=([^\r\n]*)} [dict get $proberesult output] -> blkeys]} { + if {"Thread" in $blkeys} { + set probe_ok 1 + } + } + } + testConstraint statickitavailable $probe_ok + + test staticruntime_shell_boot_no_thread_error {the repl boots on the static kit without the codethread Thread resolution failure}\ + -constraints statickitavailable\ + -setup { + set result "" + }\ + -body { + variable statickitexe + set r [punk_run $statickitexe [list shell] "exit\n"] + lappend result [dict get $r timedout] + lappend result [string match "*can't find package Thread*" [dict get $r output]] + lappend result [string match "*failed to load punk::console*" [dict get $r output]] + }\ + -cleanup { + }\ + -result [list 0 0 0] + + test staticruntime_codeinterp_thread_and_baseline {the code interp resolves Thread (static version) and can introspect the captured baseline}\ + -constraints statickitavailable\ + -setup { + set result "" + }\ + -body { + variable statickitexe + set stdin_data "puts THREADVER=\[package require Thread\]\nputs BASELINE=\[expr {\[dict size \$::punkboot::static_packages\] > 0}\]\nexit\n" + set r [punk_run $statickitexe [list shell] $stdin_data] + lappend result [dict get $r timedout] + lappend result [regexp {THREADVER=[0-9][0-9.abp]*} [dict get $r output]] + lappend result [regexp {BASELINE=1} [dict get $r output]] + }\ + -cleanup { + }\ + -result [list 0 1 1] + + test staticruntime_fabricated_interp_and_thread {a fabricated child interp and a snapshot-seeded worker thread resolve Thread inside the kit}\ + -constraints statickitavailable\ + -setup { + set result "" + variable fabscript [makeFile { + package require punk::lib + interp create fab + punk::lib::interp_sync_package_paths fab + puts "FABINTERP=[interp eval fab {catch {package require Thread} r; set r}]" + interp delete fab + package require Thread + #worker must stay alive for the sends (a bare create waits in its event loop) + set wid [thread::create] + thread::send $wid [punk::lib::snapshot_package_paths] + thread::send $wid {catch {package require Thread} r; set r} wr + puts "FABTHREAD=$wr" + thread::release $wid + exit 0 + } staticruntime_fab.tcl] + }\ + -body { + variable statickitexe + variable fabscript + set r [punk_run $statickitexe [list [string map {\\ /} $fabscript]] ""] + lappend result [dict get $r timedout] + lappend result [regexp {FABINTERP=[0-9][0-9.abp]*} [dict get $r output]] + lappend result [regexp {FABTHREAD=[0-9][0-9.abp]*} [dict get $r output]] + }\ + -cleanup { + }\ + -result [list 0 1 1] + + test staticruntime_twapi_resolves {twapi resolves in the kit per the static-vs-bundled policy (composite static twapi defers to the bundled complete copy)}\ + -constraints statickitavailable\ + -setup { + set result "" + }\ + -body { + variable statickitexe + set stdin_data "puts TWAPIVER=\[package require twapi\]\nexit\n" + set r [punk_run $statickitexe [list shell] $stdin_data] + lappend result [dict get $r timedout] + lappend result [regexp {TWAPIVER=[0-9][0-9.abp]*} [dict get $r output]] + }\ + -cleanup { + }\ + -result [list 0 1] +} +tcltest::cleanupTests ;#needed to produce test summary line. diff --git a/src/vfs/AGENTS.md b/src/vfs/AGENTS.md index 276aa342..4b4a4580 100644 --- a/src/vfs/AGENTS.md +++ b/src/vfs/AGENTS.md @@ -16,6 +16,7 @@ VFS (Virtual File System) folders define the runtime payloads that get wrapped i - VFS content must be compatible with the target platform runtime. - `_vfscommon.vfs/` is an auto-generated merge of common libraries; manually editing it will be overwritten. - `_config/punk_main.tcl` accepts an optional package_mode argument before the subcommand. Valid modes are `dev`, `os`, `internal`, and `src`, dash-delimited in priority order (e.g `src-internal`). `internal` is always appended if omitted. +- `_config/punk_main.tcl` captures the runtime's static/builtin packages at boot (G-058): empty-filename `info loaded` prefixes are probe-loaded in a throwaway interp (denylist `::punkboot::static_probe_denylist`, default `tk* vfs mk4tcl vlerq tdbc*` - side-effecting or composite inits) and each package a probe-load actually PROVIDES is recorded in `::punkboot::static_packages` {pkgname {version prefix

}} with `::punkboot::static_prefixes` alongside. The boot interp and every punkshell-fabricated interp/thread (punk::lib::interp_sync_package_paths / snapshot_package_paths, repl codethread, shellthread workers) seed `package ifneeded {load {} }` from it, and punk::packagepreference resolves static-vs-bundled version-aware. Composite statics whose script layer lived in the runtime's own (replaced) zipfs provide nothing under the probe and are correctly left to bundled copies. - `src` mode loads unbuilt modules and libraries from the project's `src/` tree (`src/modules`, `src/lib`, `src/bootsupport`, `src/vendormodules`) instead of built output. It also runs an inline `#modpod` package-ifneeded scanner (using Tcl builtins only, since no punk modules are loaded at boot time) and sets `package prefer latest` so `999999.0a1.0` dev modules are preferred over stable bootsupport/vendored copies. - `dev` mode loads built output from `/modules` and `/lib`; `src` mode loads unbuilt source from `/src/modules` etc. Use `src-dev-internal` to combine both with src priority. diff --git a/src/vfs/_config/punk_main.tcl b/src/vfs/_config/punk_main.tcl index 34e833db..b3c4f232 100644 --- a/src/vfs/_config/punk_main.tcl +++ b/src/vfs/_config/punk_main.tcl @@ -25,6 +25,100 @@ apply { args { set ::punkargv $args set tclmajorv [lindex [split [info tclversion] .] 0] + + # -- runtime static/builtin package capture (G-058) -------------------------------------- + #Runtimes may statically link packages (e.g tcl-sfe: Thread,twapi,sqlite3,tdbc). + #Static registrations are process-global ('load {} ' works in any interp/thread) + #but 'package require' needs a package-name -> load mapping in EACH interp. A kit's + #appended vfs replaces the runtime's own //zipfs:/app mount, so any pkgIndex.tcl the + #runtime shipped for its statics is gone - and the path setup below controls resolution + #anyway. Capture the static prefixes now, discover the package names/versions each + #provides by loading into a throwaway interp, and record the results in ::punkboot so + #that (a) this interp and (b) every interp/thread punkshell fabricates can seed + #'package ifneeded {load {} }' entries. + #(see punk::lib::interp_sync_package_paths / snapshot_package_paths and the + # punk::packagepreference static-vs-bundled policy) + #Probe-loading is safe for self-contained extensions; static_probe_denylist excludes + #prefixes whose init has side effects (tk* creates '.') or whose package is known to be + #composite (C part + on-disk scripts: vfs, tdbc*, and kit machinery mk4tcl/vlerq) - + #those keep their existing resolution behaviour. Composite statics are ALSO excluded + #naturally: only packages a probe-load actually PROVIDES are recorded, so an init that + #defers to script files which no longer exist (e.g static twapi whose script layer + #lived in the runtime's replaced zip) is never seeded, and any bundled complete copy + #resolves as before. + namespace eval ::punkboot { + variable static_prefixes [list] + variable static_packages [dict create] ;#pkgname -> {version prefix

} + variable static_probe_denylist [list tk* vfs mk4tcl vlerq tdbc*] + } + apply {{} { + foreach rec [info loaded] { + lassign $rec fpath prefix + if {$fpath eq "" && $prefix ne ""} { + lappend ::punkboot::static_prefixes $prefix + } + } + if {![llength $::punkboot::static_prefixes]} { + return + } + set probeable [list] + foreach prefix $::punkboot::static_prefixes { + set denied 0 + foreach dpat $::punkboot::static_probe_denylist { + if {[string match -nocase $dpat $prefix]} { + set denied 1 + break + } + } + if {!$denied} { + lappend probeable $prefix + } + } + set probe __punkboot_staticprobe + catch {interp delete $probe} + interp create $probe + #diff on PROVIDED packages (not 'package names' - a probe load can trigger an index + #scan that registers many names as ifneeded without providing them) + set provided_in_probe {{probe} { + set pdict [dict create] + foreach n [interp eval $probe {package names}] { + set pv [interp eval $probe [list package provide $n]] + if {$pv ne ""} { + dict set pdict $n $pv + } + } + return $pdict + }} + #an init may depend on another static - retry failures once + set attempts [list {*}$probeable {*}$probeable] + set done [list] + foreach prefix $attempts { + if {$prefix in $done} { + continue + } + set before_prov [apply $provided_in_probe $probe] + if {[catch {load {} $prefix $probe}]} { + continue + } + lappend done $prefix + dict for {pkgname v} [apply $provided_in_probe $probe] { + if {![dict exists $before_prov $pkgname]} { + dict set ::punkboot::static_packages $pkgname [dict create version $v prefix $prefix] + } + } + } + interp delete $probe + #seed this interp's package db so requires here can resolve to the runtime's own + #copy. Coexisting ifneeded entries (static + any vfs/module copies registered by + #index scans) resolve version-aware under the standard package machinery. + dict for {pkgname pinfo} $::punkboot::static_packages { + if {[package provide $pkgname] eq ""} { + package ifneeded $pkgname [dict get $pinfo version] [list load {} [dict get $pinfo prefix]] + } + } + }} + # -- end runtime static/builtin package capture ------------------------------------------- + namespace eval ::punkboot { #This is somewhat ugly - but we don't want to do any 'package require' operations at this stage # even for something that is available in tcl_library.