From 5c1e69270019c3ee04dff846189dbdd132e6de1c Mon Sep 17 00:00:00 2001 From: Julian Noble Date: Sun, 26 Jul 2026 16:34:51 +1000 Subject: [PATCH] build outputs: thin-layout make.tcl sync + include_modules.config punk::platform entry Regenerated by 'make.tcl modules'/'packages' after the G-122 make.tcl changes - the punkcheck-managed thin-layout copies (src/project_layouts/vendor/punk/{basic,project-0.1} and the punk.project payload inside the punk::mix templates modpod) plus the pending punk::platform entry in the bootsupport include_modules.config copies. Assisted-by: harness=claude; primary-model=claude-opus-5[1m]; api-location=anthropic.com --- .../modules/include_modules.config | 1 + .../vendor/punk/project-0.1/src/make.tcl | 865 +++++++++++++----- .../modules/include_modules.config | 1 + .../vendor/punk/basic/src/make.tcl | 865 +++++++++++++----- .../modules/include_modules.config | 1 + .../vendor/punk/project-0.1/src/make.tcl | 865 +++++++++++++----- 6 files changed, 1866 insertions(+), 732 deletions(-) diff --git a/src/modules/punk/mix/#modpod-templates-999999.0a1.0/templates/project_layouts/vendor/punk/project-0.1/src/bootsupport/modules/include_modules.config b/src/modules/punk/mix/#modpod-templates-999999.0a1.0/templates/project_layouts/vendor/punk/project-0.1/src/bootsupport/modules/include_modules.config index fac8ef64..95c2f4dc 100644 --- a/src/modules/punk/mix/#modpod-templates-999999.0a1.0/templates/project_layouts/vendor/punk/project-0.1/src/bootsupport/modules/include_modules.config +++ b/src/modules/punk/mix/#modpod-templates-999999.0a1.0/templates/project_layouts/vendor/punk/project-0.1/src/bootsupport/modules/include_modules.config @@ -104,6 +104,7 @@ set bootsupport_modules [list\ modules punk::ns\ modules punk::overlay\ modules punk::path\ + modules punk::platform\ modules punk::packagepreference\ modules punk::repl\ modules punk::repl::codethread\ diff --git a/src/modules/punk/mix/#modpod-templates-999999.0a1.0/templates/project_layouts/vendor/punk/project-0.1/src/make.tcl b/src/modules/punk/mix/#modpod-templates-999999.0a1.0/templates/project_layouts/vendor/punk/project-0.1/src/make.tcl index 83930762..7d08906c 100644 --- a/src/modules/punk/mix/#modpod-templates-999999.0a1.0/templates/project_layouts/vendor/punk/project-0.1/src/make.tcl +++ b/src/modules/punk/mix/#modpod-templates-999999.0a1.0/templates/project_layouts/vendor/punk/project-0.1/src/make.tcl @@ -444,16 +444,29 @@ namespace eval ::punkboot::lib { return "${plat}-${cpu}" } + proc platform_normalize_os {os} { + #INLINE COPY of punk::platform::normalize_os - see platform_punk below. + switch -glob -- [string tolower $os] { + macos {return macosx} + msys - msys_nt-* {return msys} + mingw32 - mingw64 - mingw_nt-* - mingw32_nt-* - mingw64_nt-* {return msys} + ucrt64 - ucrt64_nt-* {return msys} + clang32 - clang64 - clangarm64 - clang32_nt-* - clang64_nt-* - clangarm64_nt-* {return msys} + cygwin_nt-* {return cygwin} + } + return $os + } proc platform_punk {} { #canonical punkshell platform-dir name: platform_generic normalized. #INLINE COPY of punk::platform::normalize (src/modules/punk/platform-*.tm; #'help platforms' documents the canon) - the boot stage cannot package #require, so keep this mapping in sync with that module: - #amd64->x86_64, aarch64->arm64, macos->macosx, macosx arm->arm64. + #amd64->x86_64, aarch64->arm64, macos->macosx, macosx arm->arm64, + #msys2 family (mingw64/mingw32/ucrt64/clang*) -> msys, CYGWIN_NT -> cygwin. set parts [split [platform_generic] -] set cpu [lindex $parts end] set os [join [lrange $parts 0 end-1] -] - if {$os eq "macos"} {set os macosx} + set os [platform_normalize_os $os] switch -- $cpu { amd64 {set cpu x86_64} aarch64 {set cpu arm64} @@ -462,6 +475,93 @@ namespace eval ::punkboot::lib { return "${os}-${cpu}" } + # -- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- + # G-122 host/target platform split. + # platform_punk answers "what is THIS tclsh" (the HOST canon). Everything a + # build EMITS is a separate question keyed by TARGET: which bin/runtime/ + # store holds the runtime, whether artifacts carry .exe, and which process + # tooling can see/kill a running artifact. The two diverge for the whole + # cygwin family: an msys2/cygwin-runtime tclsh on windows reports + # tcl_platform(platform) 'unix' and canonizes as msys-x86_64/cygwin-x86_64, + # yet the kits it bakes are ordinary win32-x86_64 .exe binaries that only + # tasklist/taskkill can manage (msys 'ps' sees only msys-descendant + # processes). Host process semantics - copy commands, path handling, prompts, + # filesystem case rules - stay keyed to the host. + # -- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- + + #os token of a canonical platform-dir name (win32-x86_64 -> win32; macosx -> macosx) + proc platform_os {platform} { + set parts [split $platform -] + if {[llength $parts] < 2} { + return $platform + } + return [join [lrange $parts 0 end-1] -] + } + #cpu token of a canonical platform-dir name ("" for single-token names like macosx) + proc platform_cpu {platform} { + set parts [split $platform -] + if {[llength $parts] < 2} { + return "" + } + return [lindex $parts end] + } + #posix-personality runtimes hosted on windows (msys2 / cygwin) + proc platform_is_cygwin_family {platform} { + return [expr {[platform_os $platform] in {msys cygwin}}] + } + #Windows-family platforms: native win32 AND the cygwin-family personalities. + #They share what matters to a build: executables are .exe files and running + #processes are windows processes (tasklist/taskkill), whatever tcl_platform says. + proc platform_is_windows_family {platform} { + return [expr {[platform_os $platform] in {win32 msys cygwin}}] + } + #Default TARGET platform for a host canon. A cygwin-family host builds native + #windows kits (there is no separate msys kit lineage - third-party msys + #runtimes are addressed by an explicit per-entry target instead). + proc platform_target_default {hostplatform} { + if {[platform_is_cygwin_family $hostplatform]} { + set cpu [platform_cpu $hostplatform] + if {$cpu eq ""} {set cpu x86_64} + return "win32-$cpu" + } + return $hostplatform + } + #bin/runtime store tier folder name for a target: macOS runtimes are universal + #(multi-arch) binaries kept in one 'macosx' folder - see punk::platform. + proc platform_store_tier {target} { + if {[string match "macosx-*" $target]} { + return macosx + } + return $target + } + #executable filename suffix artifacts for this target carry + proc platform_exe_suffix {target} { + if {[platform_is_windows_family $target]} { + return ".exe" + } + return "" + } + #which process tooling manages a running artifact of this platform + proc platform_process_family {platform} { + if {[platform_is_windows_family $platform]} { + return windows + } + return posix + } + #A target platform name must look like a platform-dir name so it can address a + #store tier. Returns "" when acceptable, else a reason string. + proc platform_name_problem {platform} { + if {$platform eq ""} { + return "empty" + } + if {[regexp {[\\/:*?\"<>|]} $platform]} { + return "not a platform-dir name (contains a path/wildcard character)" + } + if {[llength [split $platform -]] < 2 && $platform ne "macosx"} { + return "not an - platform-dir name ('help platforms' lists the canon)" + } + return "" + } } #------------------------------------------------------------------------------ @@ -533,27 +633,35 @@ proc ::punkboot::lib::bootsupport_prompt_yesno {prompt} { # directly, so a config-format change (G-024 toml direction) swaps in underneath # by reimplementing mapvfs_parse alone. # -# mapvfs_parse: pure parse - no output, no exits. Returns a dict: +# mapvfs_parse: pure parse - no output, no exits. Every store address, presence +# probe and artifact suffix it computes is keyed by the entry's TARGET platform +# (G-122), never by the driving tclsh's personality: $rtbase is the store ROOT +# (bin/runtime) and each entry addresses $rtbase/. Returns a dict: # exists 0|1 (mapfile present) # mapfile path as supplied -# runtime_vfs_map runtime -> raw vfs_specs as listed (appname/type may be "") -# vfs_runtime_map vfstail -> list of {runtime appname type} (defaults applied; -# only vfs folders that exist on disk - matching the historical -# inline parse the kit loop was built around) +# default_target target platform for entries that declare none +# runtime_vfs_map runtime -> raw vfs_specs as listed (appname/type/target may be "") +# vfs_runtime_map vfstail -> list of {runtime appname type target} (defaults +# applied; only vfs folders that exist on disk - matching the +# historical inline parse the kit loop was built around) +# runtime_target runtime -> resolved target platform (all runtimes, mapped order) # missing names of referenced-but-absent runtimes/vfs folders # warnings warning lines in encounter order, ready to print verbatim # (missing-item lines carry their historical WARNING: prefix) # badentries subset of warnings: malformed entry lines (these have no kit # record to carry them, so bakelist prints them; missing-item # warnings are instead surfaced as per-row state there) -# configerrors fatal config problems (duplicate runtime line; caller decides -# whether to abort - the bake path keeps its historical exit 3) -proc ::punkboot::lib::mapvfs_parse {mapfile rtfolder sourcefolder} { +# configerrors fatal config problems (duplicate runtime line; unusable or +# self-contradicting target platform; caller decides whether to +# abort - the bake path keeps its historical exit 3) +proc ::punkboot::lib::mapvfs_parse {mapfile rtbase sourcefolder default_target} { set model [dict create\ exists 0\ mapfile $mapfile\ + default_target $default_target\ runtime_vfs_map [dict create]\ vfs_runtime_map [dict create]\ + runtime_target [dict create]\ missing [list]\ warnings [list]\ badentries [list]\ @@ -570,6 +678,7 @@ proc ::punkboot::lib::mapvfs_parse {mapfile rtfolder sourcefolder} { set mapdata [string map [list \r\n \n] $mapdata] set runtime_vfs_map [dict create] set vfs_runtime_map [dict create] + set runtime_target [dict create] set missing [list] set warnings [list] set badentries [list] @@ -585,19 +694,45 @@ proc ::punkboot::lib::mapvfs_parse {mapfile rtfolder sourcefolder} { #drop windows .exe suffix so same config can work cross platform - extension will be re-added if necessary later set runtime [string range $runtime 0 end-4] } - if {$runtime ne "-"} { - set runtime_test $runtime - if {"windows" eq $::tcl_platform(platform)} { - set runtime_test $runtime.exe + #Target platform is declared per kit-config entry (4th element) but is a + #property of the RUNTIME: one binary lives in one store tier and has one + #executable convention. Entries on a line may therefore repeat the + #declaration but must not disagree. + set line_target "" + foreach vfsconfig $vfs_specs { + if {[llength $vfsconfig] < 4} { + continue + } + set declared [lindex $vfsconfig 3] + if {$declared eq ""} { + continue } - if {![file exists [file join $rtfolder $runtime_test]]} { - lappend warnings "WARNING: Missing runtime file $rtfolder/$runtime_test (line in mapvfs.config: $ln)" + if {$line_target eq ""} { + set problem [platform_name_problem $declared] + if {$problem ne ""} { + lappend configerrors "CONFIG FILE ERROR. runtime: $runtime declares target platform '$declared' - $problem." + continue + } + set line_target $declared + } elseif {$declared ne $line_target} { + lappend configerrors "CONFIG FILE ERROR. runtime: $runtime declares conflicting target platforms ('$line_target' and '$declared') in $mapfile - a runtime has one target." + } + } + if {$line_target eq ""} { + set line_target $default_target + } + set rtdir [file join $rtbase [platform_store_tier $line_target]] + set exe_suffix [platform_exe_suffix $line_target] + if {$runtime ne "-"} { + set runtime_test $runtime$exe_suffix + if {![file exists [file join $rtdir $runtime_test]]} { + lappend warnings "WARNING: Missing runtime file $rtdir/$runtime_test (line in mapvfs.config: $ln)" lappend missing $runtime } } foreach vfsconfig $vfs_specs { switch -- [llength $vfsconfig] { - 1 - 2 - 3 { + 1 - 2 - 3 - 4 { lassign $vfsconfig vfstail appname target_kit_type if {![file isdirectory [file join $sourcefolder vfs $vfstail]]} { lappend warnings "WARNING: Missing vfs folder [file join $sourcefolder vfs $vfstail] specified in mapvfs.config for runtime $runtime" @@ -606,11 +741,11 @@ proc ::punkboot::lib::mapvfs_parse {mapfile rtfolder sourcefolder} { if {$appname eq ""} { set appname [file rootname $vfstail] } - dict lappend vfs_runtime_map $vfstail [list $runtime $appname $target_kit_type] + dict lappend vfs_runtime_map $vfstail [list $runtime $appname $target_kit_type $line_target] } } default { - set badline "bad entry in mapvfs.config - expected each entry after the runtime name to be of length 1 or length 2. got: $vfsconfig ([llength $vfsconfig])" + set badline "bad entry in mapvfs.config - expected each entry after the runtime name to be a list of 1 to 4 elements {vfsfolder ?kitname? ?kittype? ?targetplatform?}. got: $vfsconfig ([llength $vfsconfig])" lappend warnings $badline lappend badentries $badline } @@ -620,10 +755,12 @@ proc ::punkboot::lib::mapvfs_parse {mapfile rtfolder sourcefolder} { lappend configerrors "CONFIG FILE ERROR. runtime: $runtime was specified more than once in $mapfile." } else { dict set runtime_vfs_map $runtime $vfs_specs + dict set runtime_target $runtime $line_target } } dict set model runtime_vfs_map $runtime_vfs_map dict set model vfs_runtime_map $vfs_runtime_map + dict set model runtime_target $runtime_target dict set model missing $missing dict set model warnings $warnings dict set model badentries $badentries @@ -635,24 +772,30 @@ proc ::punkboot::lib::mapvfs_parse {mapfile rtfolder sourcefolder} { # Replicates the kit loop's iteration + naming (vfs_tails glob order with map # extras appended, per-vfs runtime order from the mapping, appname defaulting to # the vfs rootname, kit_type defaulting to 'kit', '-' runtime -> .kit, -# windows .exe suffix, duplicate appname -> _) so listed names +# TARGET .exe suffix, duplicate appname -> _) so listed names # match what a bake produces. Entries whose vfs folder is missing on disk never # reach the kit loop - they are appended after the buildable set so 'bakelist' # can surface them as broken config. Duplicate-name disambiguation assumes the # config keeps appnames unique (as the kit loop itself effectively does). -# Returns a list of record dicts: +# $rtbase is the store ROOT (bin/runtime): each record resolves its own tier +# under it, so entries targeting different platforms report their own store +# presence (G-122). Returns a list of record dicts: # kitname base output name (selection key, e.g punk91) # targetkit platform artifact name (e.g punk91.exe, or .kit for '-') # kit_type kit|zip|zipcat|cookfs... as configured (defaulted to kit) # runtime runtime name as configured (unsuffixed), or "-" -# runtime_file platform runtime filename ("" for "-") +# runtime_file target runtime filename ("" for "-") # runtime_present 1|0 (1 for "-": no runtime needed) +# runtime_dir store tier folder holding the runtime ("" for "-") +# target target platform (e.g win32-x86_64) +# store_tier bin/runtime tier folder name for the target # vfs vfs folder tail (e.g punk9wintk903.vfs) # vfs_present 1|0 -proc ::punkboot::lib::mapvfs_kit_outputs {model rtfolder sourcefolder} { +proc ::punkboot::lib::mapvfs_kit_outputs {model rtbase sourcefolder} { set runtime_vfs_map [dict get $model runtime_vfs_map] set vfs_runtime_map [dict get $model vfs_runtime_map] - set on_windows [expr {"windows" eq $::tcl_platform(platform)}] + set runtime_target [dict get $model runtime_target] + set default_target [dict get $model default_target] set vfs_tails [glob -nocomplain -dir $sourcefolder/vfs -types d -tail *.vfs] dict for {vfstail -} $vfs_runtime_map { if {$vfstail ni $vfs_tails} { @@ -662,33 +805,37 @@ proc ::punkboot::lib::mapvfs_kit_outputs {model rtfolder sourcefolder} { set records [list] set exe_names_seen [list] set record_pair {{rtname vfstail appname target_kit_type} { - upvar 1 records records exe_names_seen exe_names_seen rtfolder rtfolder on_windows on_windows sourcefolder sourcefolder + upvar 1 records records exe_names_seen exe_names_seen rtbase rtbase sourcefolder sourcefolder\ + runtime_target runtime_target default_target default_target if {$appname eq ""} { set appname [file rootname $vfstail] } if {$target_kit_type eq ""} { set target_kit_type "kit" } + if {[dict exists $runtime_target $rtname]} { + set target [dict get $runtime_target $rtname] + } else { + set target $default_target + } + set store_tier [::punkboot::lib::platform_store_tier $target] if {$rtname eq "-"} { set targetkit $appname.kit set kitname $appname set runtime_file "" + set runtime_dir "" set runtime_present 1 ;#no runtime needed } else { - if {$on_windows} { - set runtime_file $rtname.exe - set targetkit ${appname}.exe - } else { - set runtime_file $rtname - set targetkit $appname - } + set runtime_dir [file join $rtbase $store_tier] + set runtime_file $rtname[::punkboot::lib::platform_exe_suffix $target] + set targetkit ${appname}[::punkboot::lib::platform_exe_suffix $target] set kitname $appname if {$targetkit in $exe_names_seen} { #duplicate appname configured - kit loop disambiguates the same way set targetkit ${appname}_$rtname set kitname ${appname}_$rtname } - set runtime_present [file exists [file join $rtfolder $runtime_file]] + set runtime_present [file exists [file join $runtime_dir $runtime_file]] } lappend exe_names_seen $targetkit lappend records [dict create\ @@ -698,6 +845,9 @@ proc ::punkboot::lib::mapvfs_kit_outputs {model rtfolder sourcefolder} { runtime $rtname\ runtime_file $runtime_file\ runtime_present $runtime_present\ + runtime_dir $runtime_dir\ + target $target\ + store_tier $store_tier\ vfs $vfstail\ vfs_present [file isdirectory [file join $sourcefolder vfs $vfstail]]\ ] @@ -723,7 +873,7 @@ proc ::punkboot::lib::mapvfs_kit_outputs {model rtfolder sourcefolder} { #entries whose vfs folder is missing (excluded from vfs_runtime_map by the parse) dict for {rtname vfs_specs} $runtime_vfs_map { foreach vfsconfig $vfs_specs { - if {[llength $vfsconfig] < 1 || [llength $vfsconfig] > 3} { + if {[llength $vfsconfig] < 1 || [llength $vfsconfig] > 4} { continue } lassign $vfsconfig vfstail appname target_kit_type @@ -738,11 +888,13 @@ proc ::punkboot::lib::mapvfs_kit_outputs {model rtfolder sourcefolder} { # Match user-supplied kit names against mapvfs_kit_outputs records. # Accepts the base kit name (punk91) or the platform artifact name (punk91.exe, -# name.kit); matching is case-insensitive on windows. Returns a dict: +# name.kit); matching is case-insensitive when the HOST filesystem is (a name +# the user types is a host-side convenience - a cygwin-family host is as +# case-insensitive as a native windows one, whatever it targets). Returns a dict: # selected matched records (kit-output order, deduplicated) # unknown supplied names that matched nothing proc ::punkboot::lib::mapvfs_match_outputs {kit_outputs names} { - set on_windows [expr {"windows" eq $::tcl_platform(platform)}] + set on_windows [expr {[info exists ::punkboot::host_windows] ? $::punkboot::host_windows : ("windows" eq $::tcl_platform(platform))}] set selected_idx [list] set unknown [list] foreach name $names { @@ -918,6 +1070,21 @@ set startdir [pwd] set bootsupport_module_paths [list] set bootsupport_library_paths [list] set this_platform_generic [punkboot::lib::platform_punk] ;#normalized punkshell platform-dir name (punk::platform canon) +# -- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- +#G-122 host/target split. $this_platform_generic is the HOST canon and stays the +#key for everything that describes THIS interpreter (bootsupport/vendorlib +#platform dirs below, diagnostics). What the build EMITS is keyed by target: +# host_platform - canon of the driving tclsh (same value, named for clarity) +# target_platform - default target for kit outputs; a cygwin-family host +# (msys2/cygwin runtime on windows) builds win32 kits. +# Per-entry mapvfs.config declarations override it per runtime. +# host_windows - host process/filesystem semantics are windows-like +# (native windows OR cygwin family): drives copy commands, +# path case comparison, prompt behaviour - NOT artifact naming. +set ::punkboot::host_platform $this_platform_generic +set ::punkboot::target_platform [punkboot::lib::platform_target_default $this_platform_generic] +set ::punkboot::host_windows [expr {[punkboot::lib::platform_process_family $this_platform_generic] eq "windows"}] +# -- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- #we always create these lists in order of desired precedence. # - this is the same order when adding to auto_path - but will need to be reversed when using tcl:tm::add if {[file exists [file join $::punkboot::scriptfolder bootsupport]]} { @@ -931,6 +1098,17 @@ if {[file exists [file join $::punkboot::scriptfolder bootsupport]]} { lappend bootsupport_library_paths [file join $bootsupportdir lib] } else { puts stderr "No bootsupport dir for script [info script] at [file join $::punkboot::scriptfolder bootsupport]" + #G-122 host-personality diagnosis: a cygwin-family tclsh (msys2/cygwin runtime on + #windows) is a POSIX Tcl - 'C:/path/make.tcl' is not an absolute path to it, so + #'file normalize' silently prefixes the cwd and every derived path is wrong. The + #failure that follows (missing bootsupport, then an unresolvable project root) is + #unrecognisable without this note. Caller-side spelling, not something make.tcl can + #safely rewrite - name the script in the host's own path syntax. + if {[punkboot::lib::platform_is_cygwin_family $this_platform_generic] && [regexp {^([A-Za-z]):[/\\](.*)$} [info script] -> _cygdrive _cygrest]} { + puts stderr " NOTE: this tclsh is a $this_platform_generic (msys2/cygwin-runtime) build - a POSIX Tcl, to which the windows-style path '[info script]' is NOT absolute, so it was resolved relative to the current directory." + puts stderr " Re-run naming the script in this shell's own path syntax: /[string tolower $_cygdrive]/[string map {\\ /} $_cygrest]" + unset -nocomplain _cygdrive _cygrest + } #lappend bootsupport_module_paths [file join $startdir bootsupport modules_tcl$::tclmajorv] #lappend bootsupport_module_paths [file join $startdir bootsupport modules] #lappend bootsupport_library_paths [file join $startdir bootsupport lib_tcl$::tclmajorv/allplatforms] @@ -2086,12 +2264,14 @@ DIAGRAM 1b - THIN-LAYOUT SYNC (part of modules/libs/packages/bakehouse runs) DIAGRAM 2 - KIT ASSEMBLY DETAIL (the 'make.tcl bake' stage; incl. vfslibs phase) ---------------------------------------------------------------------------- - src/runtime/mapvfs.config which vfs folder pairs with which runtime -> kit name + src/runtime/mapvfs.config which vfs folder pairs with which runtime -> kit name, + and optionally the TARGET PLATFORM of that runtime [K9] 'make.tcl bakelist ?kitname ...?' reports the configured kit outputs (runtime/vfs presence + deployed state); 'make.tcl bake ?kitname ...?' bakes only the named kits (bare bake = all; unknown names error before any build) - src/runtime/.exe bare Tcl runtime (tclkit / tclsfe build) + bin/runtime// bare Tcl runtime (tclkit / tclsfe / suite build), in the + store tier of the TARGET platform it builds kits for [K9] src/vfs/_vfscommon.vfs/ common payload (built modules + libs, main boot support) src/vfs/.vfs/ per-kit overlay (main.tcl, kit-only modules/config) @@ -2102,7 +2282,9 @@ DIAGRAM 2 - KIT ASSEMBLY DETAIL (the 'make.tcl bake' stage; incl. vfslibs phase) v src/_build/.exe.vfs | - | tcl::zipfs::mkimg (vfs image appended to runtime exe) + | zip image appended to the runtime exe: tcl::zipfs::mkimg, + | or (driving tcl without zipfs, e.g 8.6) raw-runtime split + | + punk::zip::mkzip + concatenation - both mount the same v src/_build/.exe | @@ -2160,6 +2342,18 @@ KEY / NOTES untracked: modules*/ lib*/ at projectroot, src/_build/, bin/.exe (bin/ scripts/tools are tracked; kit executables are not) +[K9] HOST vs TARGET platform (G-122). What a bake EMITS is keyed by the target + platform, never by the driving tclsh's personality: which bin/runtime/ + holds the runtime, whether the runtime file and the built kit carry .exe, and + whether the pre-deploy sweep uses tasklist/taskkill or ps/kill. The default + target is this host's own platform - except a cygwin-family host (an + msys2/cygwin-runtime tclsh, which reports platform 'unix' on windows), which + targets win32. A mapvfs.config entry may declare its own target as a 4th + element and is then read from that platform's tier, named with its executable + convention, and (being unrunnable here) skipped by the process sweep. + 'make.tcl check' prints the derivation; 'make.tcl bakelist' shows per-kit + targets, with a target= note on rows that are not on the default target. + MAINTENANCE (agents take note) ---------------------------------------------------------------------------- This text is embedded in make.tcl (::punkboot::workflow_text). When the build @@ -2339,7 +2533,8 @@ if {[info exists ::env(PUNKBOOT_PLAIN)] && [string is true -strict $::env(PUNKBO List the kit outputs configured in src/runtime/mapvfs.config (the same parsed mapping the bake subcommand consumes): kit name, kit type (kit|zip|zipcat...), runtime with presence in the runtime - store (/bin/runtime/), vfs folder, and the + store of that kit's TARGET platform + (/bin/runtime/), vfs folder, and the deployed state of /bin/ vs the src/_build build product: current - deployed copy is identical to the build product @@ -2349,9 +2544,12 @@ if {[info exists ::env(PUNKBOOT_PLAIN)] && [string is true -strict $::env(PUNKBO A trailing notes column flags anomalies (runtime=missing, vfs=missing, rtrev=r when the runtime working copy is materialized from an older revision than a -r artifact beside - it). + it) and marks cross-target entries (target= on any kit + whose mapvfs entry declares a target other than this host's + default - 'make.tcl check' shows that default). kitname arguments filter the report to the named entries (per-kit - detail, with resolved paths). Reporting only - never builds." + detail, with resolved store tier, target and paths). Reporting + only - never builds." vfslibs " Propagate declared vendored platform-library packages into kit vfs @@ -2394,8 +2592,11 @@ if {[info exists ::env(PUNKBOOT_PLAIN)] && [string is true -strict $::env(PUNKBO " Show module/library paths and any potentially problematic packages for running this script. - Also reports bootsupport staleness and the src provenance status (what - the build/promotion commands would do)." + Also reports the colour policy, the host/target platform derivation + (this tclsh's platform canon vs the default kit target, its runtime + store tier, executable suffix and process tooling), bootsupport + staleness and the src provenance status (what the build/promotion + commands would do)." projectversion " Advisory check: verify CHANGELOG.md matches punkproject.toml and warn @@ -2527,7 +2728,7 @@ if {[info exists ::env(PUNKBOOT_PLAIN)] && [string is true -strict $::env(PUNKBO #still works when the mapping cannot be parsed here) stays authoritative. variable KITNAME_CHOICES [list] catch { - set _mapmodel [punkboot::lib::mapvfs_parse [file join $::punkboot::scriptfolder runtime mapvfs.config] "" $::punkboot::scriptfolder] + set _mapmodel [punkboot::lib::mapvfs_parse [file join $::punkboot::scriptfolder runtime mapvfs.config] "" $::punkboot::scriptfolder $::punkboot::target_platform] foreach _rec [punkboot::lib::mapvfs_kit_outputs $_mapmodel "" $::punkboot::scriptfolder] { lappend KITNAME_CHOICES [dict get $_rec kitname] } @@ -2955,23 +3156,15 @@ set ::punkboot::bake_selected_targetkits [list] ;#platform artifact names (e.g p set ::punkboot::bake_selected_vfs [list] ;#vfs folder tails the selected kits use set ::punkboot::bake_selected_runtimes [list] ;#unsuffixed runtime names the selected kits wrap if {$::punkboot::command eq "bake" && [llength $::punkboot::opt_kitnames]} { - switch -glob -- $this_platform_generic { - macosx-* { - set _rt_os_arch macosx - } - default { - set _rt_os_arch $this_platform_generic - } - } - set _rtfolder $binfolder/runtime/$_rt_os_arch - set _mapmodel [punkboot::lib::mapvfs_parse $sourcefolder/runtime/mapvfs.config $_rtfolder $sourcefolder] + set _rtbase $binfolder/runtime + set _mapmodel [punkboot::lib::mapvfs_parse $sourcefolder/runtime/mapvfs.config $_rtbase $sourcefolder $::punkboot::target_platform] if {[llength [dict get $_mapmodel configerrors]]} { foreach _errline [dict get $_mapmodel configerrors] { puts stderr $_errline } exit 3 } - set _kit_outputs [punkboot::lib::mapvfs_kit_outputs $_mapmodel $_rtfolder $sourcefolder] + set _kit_outputs [punkboot::lib::mapvfs_kit_outputs $_mapmodel $_rtbase $sourcefolder] set _matchinfo [punkboot::lib::mapvfs_match_outputs $_kit_outputs $::punkboot::opt_kitnames] if {[llength [dict get $_matchinfo unknown]]} { puts stderr "make.tcl bake: unknown kit name(s): [join [dict get $_matchinfo unknown] {, }] - nothing built" @@ -2995,7 +3188,7 @@ if {$::punkboot::command eq "bake" && [llength $::punkboot::opt_kitnames]} { exit 1 } if {[dict get $_rec runtime] ne "-" && ![dict get $_rec runtime_present]} { - puts stderr "make.tcl bake: kit '[dict get $_rec kitname]' cannot be built - runtime [dict get $_rec runtime_file] not present in bin/runtime/$_rt_os_arch - nothing built" + puts stderr "make.tcl bake: kit '[dict get $_rec kitname]' cannot be built - runtime [dict get $_rec runtime_file] not present in bin/runtime/[dict get $_rec store_tier] (target [dict get $_rec target]) - nothing built" exit 1 } } @@ -3010,7 +3203,7 @@ if {$::punkboot::command eq "bake" && [llength $::punkboot::opt_kitnames]} { } } puts stdout "selective bake (G-121): [join $::punkboot::bake_selected_kitnames {, }]" - unset -nocomplain _rt_os_arch _rtfolder _mapmodel _kit_outputs _matchinfo _rec _r _errline + unset -nocomplain _rtbase _mapmodel _kit_outputs _matchinfo _rec _r _errline } # ---------------------------------------- @@ -3140,6 +3333,15 @@ if {$::punkboot::command eq "check"} { puts stdout "colour policy (G-113): mode=$::punkboot::colour_mode colour_disabled=$::punk::console::colour_disabled ansistrip=[expr {[llength $::punkboot::ansistrip_pushed] ? [join $::punkboot::ansistrip_pushed +] : 0}]" puts stdout " (precedence: NO_COLOR > PUNK_FORCE_COLOR/FORCE_COLOR > stdout tty probe incl. the 8.6 windows console-encoding signature; piped/unknown output is fully ESC-stripped per channel unless forced - utf-16 console channels are never wrapped)" puts stdout $sep + #G-122 host/target platform split - the self-diagnosis for "why is my kit set + #addressed the way it is". One stable single line; the columns are key=value. + set _chk_suffix [punkboot::lib::platform_exe_suffix $::punkboot::target_platform] + if {$_chk_suffix eq ""} {set _chk_suffix "(none)"} + puts stdout "platform (G-122): host=$::punkboot::host_platform target=$::punkboot::target_platform store=bin/runtime/[punkboot::lib::platform_store_tier $::punkboot::target_platform] exe-suffix=$_chk_suffix process-tooling=[punkboot::lib::platform_process_family $::punkboot::target_platform]" + puts stdout " (host = this tclsh's punk::platform canon, from tcl_platform(os)='$::tcl_platform(os)' platform='$::tcl_platform(platform)'; target = default kit target - a cygwin-family host (msys2/cygwin runtime on windows) bakes win32 kits)" + puts stdout " (mapvfs.config entries may declare their own target platform as a 4th element - 'make.tcl bakelist' shows each kit's target, store tier and artifact name)" + unset -nocomplain _chk_suffix + puts stdout $sep puts stdout "module/library checks - paths from bootsupport only" puts stdout $sep puts stdout "- tcl::tm::list" @@ -3493,18 +3695,17 @@ if {$::punkboot::command eq "bakelist"} { #grepping; expected state sits in the columns, anomalies surface in a trailing #sparse notes column (key=value tags). Kit name arguments filter the report and #add a per-kit detail block (resolved paths/sizes/mtimes). - switch -glob -- $this_platform_generic { - macosx-* { - set rt_os_arch macosx - } - default { - set rt_os_arch $this_platform_generic - } - } - set rtfolder $binfolder/runtime/$rt_os_arch + #G-122: store addressing is keyed by TARGET platform. The default target is + #this host's own platform (a cygwin-family host targets win32) and mapvfs + #entries may declare their own - so each row resolves its own tier under + #bin/runtime and the report names the tiers it consulted. + set rt_default_target $::punkboot::target_platform + set rt_os_arch [punkboot::lib::platform_store_tier $rt_default_target] + set rtbase $binfolder/runtime + set rtfolder $rtbase/$rt_os_arch set buildfolder $sourcefolder/_build set mapfile $sourcefolder/runtime/mapvfs.config - set mapmodel [punkboot::lib::mapvfs_parse $mapfile $rtfolder $sourcefolder] + set mapmodel [punkboot::lib::mapvfs_parse $mapfile $rtbase $sourcefolder $rt_default_target] foreach errline [dict get $mapmodel configerrors] { puts stderr $errline } @@ -3515,11 +3716,32 @@ if {$::punkboot::command eq "bakelist"} { puts stdout "no kit mapping config at $mapfile - no kit outputs configured" exit 0 } - set kit_outputs [punkboot::lib::mapvfs_kit_outputs $mapmodel $rtfolder $sourcefolder] + set kit_outputs [punkboot::lib::mapvfs_kit_outputs $mapmodel $rtbase $sourcefolder] if {![llength $kit_outputs]} { puts stdout "no kit outputs configured in $mapfile" exit 0 } + #Loud store-tier guard: a nonexistent store folder makes every row addressing it + #read runtime=missing for a reason that is not per-entry. Reported per tier + #actually referenced, since entries may target different platforms (G-122). + set report_tiers [list] + foreach rec $kit_outputs { + if {[dict get $rec runtime] eq "-"} {continue} + set t [dict get $rec store_tier] + if {$t ni $report_tiers} {lappend report_tiers $t} + } + set rt_store_missing [expr {![file isdirectory $rtfolder]}] + if {$rt_os_arch ni $report_tiers} {lappend report_tiers $rt_os_arch} + foreach t $report_tiers { + if {[file isdirectory $rtbase/$t]} {continue} + if {$t eq $rt_os_arch} { + puts stderr "WARNING: runtime store folder bin/runtime/$t does not exist - every configured runtime on the default target will read runtime=missing." + puts stderr " Default target '$rt_default_target' derives from THIS tclsh: tcl_platform(os)='$::tcl_platform(os)' tcl_platform(platform)='$::tcl_platform(platform)' -> host canon '$::punkboot::host_platform' -> target '$rt_default_target'." + puts stderr " Populate it with 'bin/punk-runtime.cmd' (or by hand), or declare a per-entry target platform in mapvfs.config for runtimes stored elsewhere." + } else { + puts stderr "WARNING: runtime store folder bin/runtime/$t does not exist - configured entries targeting $t will read runtime=missing." + } + } set reportset $kit_outputs if {[llength $::punkboot::opt_kitnames]} { set matchinfo [punkboot::lib::mapvfs_match_outputs $kit_outputs $::punkboot::opt_kitnames] @@ -3545,8 +3767,13 @@ if {$::punkboot::command eq "bakelist"} { if {![dict get $rec vfs_present]} { lappend notes vfs=missing } + if {[dict get $rec target] ne $rt_default_target} { + #cross-target entry: its runtime, store tier and artifact naming follow + #its own platform, not this host's default + lappend notes target=[dict get $rec target] + } if {$runtime ne "-" && [dict get $rec runtime_present]} { - set rtstale [punkboot::lib::runtime_materialization_check $rtfolder [dict get $rec runtime_file]] + set rtstale [punkboot::lib::runtime_materialization_check [dict get $rec runtime_dir] [dict get $rec runtime_file]] if {[dict size $rtstale]} { lappend notes rtrev=r[dict get $rtstale current_rev] vs src/_build/" + set rt_store_note [expr {$rt_store_missing ? " (FOLDER MISSING)" : ""}] + puts stdout "# mapvfs: src/runtime/mapvfs.config runtimes: bin/runtime/$rt_os_arch$rt_store_note deployed: bin/ vs src/_build/" set headline "" foreach h $headings w $widths { append headline [format "%-*s " $w $h] @@ -3594,10 +3822,11 @@ if {$::punkboot::command eq "bakelist"} { if {$runtime eq "-"} { puts stdout " runtime: (none - unwrapped .kit output)" } else { - puts stdout " runtime file: bin/runtime/$rt_os_arch/[dict get $rec runtime_file] [expr {[dict get $rec runtime_present] ? {(present)} : {(MISSING)}}]" + puts stdout " runtime file: bin/runtime/[dict get $rec store_tier]/[dict get $rec runtime_file] [expr {[dict get $rec runtime_present] ? {(present)} : {(MISSING)}}]" } puts stdout " vfs folder: src/vfs/[dict get $rec vfs] [expr {[dict get $rec vfs_present] ? {(present)} : {(MISSING)}}]" puts stdout " kit type: [dict get $rec kit_type]" + puts stdout " target: [dict get $rec target][expr {[dict get $rec target] eq $rt_default_target ? " (host default)" : " (declared in mapvfs.config)"}]" puts stdout " build product: src/_build/[dict get $rec targetkit] [punkboot::lib::bakelist_file_detail $buildfolder/[dict get $rec targetkit]]" puts stdout " deployed: bin/[dict get $rec targetkit] [punkboot::lib::bakelist_file_detail $binfolder/[dict get $rec targetkit]] state=[lindex $row 4]" } @@ -4706,28 +4935,40 @@ if {$::punkboot::command eq "bin"} { #set rtfolder $sourcefolder/runtime #AAA -switch -glob -- $this_platform_generic { - macosx-* { - #assuming universal binaries x86_64 and arm - set rt_os_arch macosx - } - default { - set rt_os_arch $this_platform_generic - } -} -set rtfolder $binfolder/runtime/$rt_os_arch +#G-122: the runtime store is addressed by TARGET platform, not by the driving +#tclsh's personality. $rt_default_target is this host's default target (a +#cygwin-family host targets win32; macOS collapses to the universal 'macosx' +#tier) and $rtfolder is that target's store. mapvfs.config entries may declare +#their own target platform - those runtimes live in their own tier under $rtbase +#and are resolved through $rtfolder_of below. +set rt_default_target $::punkboot::target_platform +set rt_os_arch [punkboot::lib::platform_store_tier $rt_default_target] +set rtbase $binfolder/runtime +set rtfolder $rtbase/$rt_os_arch +set rt_exe_suffix [punkboot::lib::platform_exe_suffix $rt_default_target] set rt_sourcefolder $sourcefolder/runtime ;#where our config lives -#review - when building kits for other platforms - it's unlikely runtime will be marked as executable - we should probably process all files in runtime folder except those with certain extensions +#G-122: derive the BUILDCOPY name from the runtime FILENAME rather than from the +#driving tclsh's platform. The copy is the same binary as its source, so it must +#keep the source's executable convention - an .exe stays an .exe whether a native +#windows tclsh, an msys/cygwin-runtime tclsh (platform 'unix', .exe files) or a +#cross-target host is driving the build. +#Mapping key for a runtime FILE name: the name as mapvfs.config spells it, i.e +#with a windows .exe suffix removed. Only .exe may be stripped - 'file rootname' +#would eat the last dotted segment of version-named runtimes such as tclsh9.0.5, +#which carry no suffix at all on non-windows targets (G-122). +proc ::punkboot::lib::runtime_mapkey {runtimefile} { + if {[string match -nocase *.exe $runtimefile]} { + return [string range $runtimefile 0 end-4] + } + return $runtimefile +} proc ::punkboot::lib::runtime_buildcopyname {runtime} { - if {$::tcl_platform(platform) eq "windows"} { - set rtname [file rootname [file tail $runtime]] - set bcname ${rtname}_BUILDCOPY.exe - } else { - set rtname [file tail $runtime] - set bcname ${rtname}_BUILDCOPY + set rttail [file tail $runtime] + if {[string match -nocase *.exe $rttail]} { + return "[file rootname $rttail]_BUILDCOPY.exe" } - return $bcname + return "${rttail}_BUILDCOPY" } #Local runtime-materialization staleness check (warn-only, no network). The kit @@ -4741,6 +4982,31 @@ proc ::punkboot::lib::runtime_buildcopyname {runtime} { #toml) and direct -r references stay silent; one warning per runtime per run. #G-121: the comparison core lives in ::punkboot::lib::runtime_materialization_check, #shared with the bakelist report's rtrev= note. +#G-122: run a command whose arguments must reach a NATIVE WINDOWS program +#verbatim. The msys2 runtime rewrites any argument that looks like an absolute +#posix path when it spawns a native windows executable, so 'taskkill /PID 1234' +#arrives as 'taskkill C:/msys64-root/PID 1234' - field-verified 2026-07-26: the +#pre-deploy process sweep found the running kit with tasklist and then could not +#kill it, failing the whole kit. MSYS2_ARG_CONV_EXCL=* is msys2's documented +#opt-out and is inert on every other host (real cygwin does not convert arguments +#at all), so one command spelling stays correct from every host personality. +proc ::punkboot::exec_nativeargs {cmdlist} { + set had [info exists ::env(MSYS2_ARG_CONV_EXCL)] + if {$had} { + set prev $::env(MSYS2_ARG_CONV_EXCL) + } + set ::env(MSYS2_ARG_CONV_EXCL) * + try { + return [exec {*}$cmdlist] + } finally { + if {$had} { + set ::env(MSYS2_ARG_CONV_EXCL) $prev + } else { + unset -nocomplain ::env(MSYS2_ARG_CONV_EXCL) + } + } +} + set ::punkboot::runtime_rev_warned [dict create] proc ::punkboot::runtime_materialization_warning {rtfolder runtime_fullname} { if {[dict exists $::punkboot::runtime_rev_warned $runtime_fullname]} { @@ -4757,16 +5023,75 @@ proc ::punkboot::runtime_materialization_warning {rtfolder runtime_fullname} { return } +# -- --- --- --- --- --- --- --- --- --- +#load mapvfs.config file (if any) in runtime folder to map runtimes to vfs folders. +#build a dict keyed on runtime executable name. +#If no mapfile (or no mapfile entry for that runtime) - the runtime will be paired with a matching .vfs folder in src folder. e.g punk.exe to src/punk.vfs +#If vfs folders or runtime executables which are explicitly listed in the mapfile don't exist - warn on stderr - but continue. if such nonexistants found; prompt user for whether to continue or abort. +#G-121: parsing lives in ::punkboot::lib::mapvfs_parse - the shared model consumed here, +#by 'bakelist' and by the selective-bake name resolution (never the file format directly). +#G-122: parsed BEFORE runtime discovery, because the mapping is what says which +#store tiers this bake must look in - an entry declaring a target platform other +#than the host default addresses its own bin/runtime/. +set mapfile $rt_sourcefolder/mapvfs.config +set mapmodel [punkboot::lib::mapvfs_parse $mapfile $rtbase $sourcefolder $rt_default_target] +foreach warnline [dict get $mapmodel warnings] { + puts stderr $warnline +} +if {[llength [dict get $mapmodel configerrors]]} { + foreach errline [dict get $mapmodel configerrors] { + puts stderr $errline + } + exit 3 +} +set runtime_vfs_map [dict get $mapmodel runtime_vfs_map] +set vfs_runtime_map [dict get $mapmodel vfs_runtime_map] +set runtime_target [dict get $mapmodel runtime_target] +set missing [dict get $mapmodel missing] +if {[llength $missing]} { + puts stderr "WARNING [llength $missing] missing items from $mapfile. (TODO - prompt user to continue/abort)" + foreach m $missing { + puts stderr " $m" + } + puts stderr "continuing..." +} +# -- --- --- --- --- --- --- --- --- --- +puts "-- runtime_vfs_map --" +punk::lib::pdict runtime_vfs_map +puts "---------------------" +puts "-- vfs_runtime_map--" +punk::lib::pdict vfs_runtime_map +puts "---------------------" + +# -- --- --- --- --- --- --- --- --- --- +#Runtime discovery. The default-target store is scanned wholesale (a runtime +#there needs no mapping entry - it can pair with a same-named .vfs folder); +#cross-target runtimes are only picked up when the mapping names them, since +#scanning every tier would pull in runtimes nothing asked to be built. +#$rtfolder_of resolves a discovered runtime FILE name to the store folder it +#came from (runtime_target_of gives its target platform) - every later store +#address, suffix and process-tooling decision goes through these rather than +#assuming the driving tclsh's platform (G-122). set runtimes [list] +set rtfolder_of [dict create] +proc ::punkboot::lib::runtime_target_of {rtname} { + #target platform of a runtime name (unsuffixed), from the parsed mapping + upvar #0 runtime_target runtime_target rt_default_target rt_default_target + if {[info exists runtime_target] && [dict exists $runtime_target $rtname]} { + return [dict get $runtime_target $rtname] + } + return $rt_default_target +} if {[file dirname [info nameofexecutable]] eq $rtfolder && [file isdirectory [info nameofexecutable]]} { #info name will only appear to be a directory when the runtime is a tclkit that is mounted at the same path as the physical file. #This is not a problem for zip based kits. #fix so that we can find and use a runtime tclkit that is currently in use as the runtime that is running this script. # - when the current runtime is using a runtime in the rtfolder that is a tclkit, it is mounted at the same path as the physical file and so appears to Tcl as if it's a directory. # - use external filesystem tools to make a copy of the file - if {$::tcl_platform(platform) eq "windows"} { + #(copy tooling is a HOST concern - the running executable is by definition a host binary) + if {$::punkboot::host_windows} { #exec cmd /c copy [info nameofexecutable] $rtfolder/[file rootname [file tail [info nameofexecutable]]]_BUILDCOPY.exe - exec cmd /c copy [file nativename [info nameofexecutable]] [file nativename $rtfolder/[punkboot::lib::runtime_buildcopyname [info nameofexecutable]]] + ::punkboot::exec_nativeargs [list cmd /c copy [file nativename [info nameofexecutable]] [file nativename $rtfolder/[punkboot::lib::runtime_buildcopyname [info nameofexecutable]]]] } else { #exec cp [info nameofexecutable] $rtfolder/[file tail [info nameofexecutable]]_BUILDCOPY exec cp [info nameofexecutable] $rtfolder/[punkboot::lib::runtime_buildcopyname [info nameofexecutable]] @@ -4774,12 +5099,14 @@ if {[file dirname [info nameofexecutable]] eq $rtfolder && [file isdirectory [in #we need to add the base name of the current exe to the runtimes list because the next section will look for files in the runtime folder #and won't find the mounted runtime since it's mounted as a directory. lappend runtimes [file tail [info nameofexecutable]] + dict set rtfolder_of [file tail [info nameofexecutable]] $rtfolder } set rtfolder_files [glob -nocomplain -dir $rtfolder -types {f} -tail *] set exclusions {.config .md .ico .txt .doc .pdf .htm .html} ;#we don't encourage other files in runtime folder aside from mapvfs.config - but lets ignore some common possibilities lappend exclusions .zip .7z .pea .bz2 .tar .gz .tgz .z .xz ;#don't allow archives to directly be treated as runtimes - tolerate presence but require user to unpack or rename if they're to be used as runtimes lappend exclusions .tail ;#result of running sdx mksplit on a kit - in theory the .head could be used - review/test +lappend exclusions .toml ;#G-103 runtime artifact metadata sits beside its runtime foreach f $rtfolder_files { if {[string match "*_BUILDCOPY*" $f]} { continue @@ -4789,11 +5116,43 @@ foreach f $rtfolder_files { } if {$f ni $runtimes} { lappend runtimes $f + dict set rtfolder_of $f $rtfolder } } +#cross-target runtimes named by the mapping (their own tier, their own suffix) +set cross_target_tiers [list] +dict for {_rtname _rttarget} $runtime_target { + if {$_rtname eq "-" || $_rttarget eq $rt_default_target} { + continue + } + set _tier [punkboot::lib::platform_store_tier $_rttarget] + set _dir $rtbase/$_tier + set _file $_rtname[punkboot::lib::platform_exe_suffix $_rttarget] + if {$_tier ni $cross_target_tiers} { + lappend cross_target_tiers $_tier + } + if {![file exists [file join $_dir $_file]]} { + continue ;#absent cross-target runtime already warned by the parse + } + if {[dict exists $rtfolder_of $_file]} { + puts stderr "WARNING: runtime file $_file is present in more than one store tier ([dict get $rtfolder_of $_file] and $_dir) - using the first found. Rename one of them to build both." + continue + } + lappend runtimes $_file + dict set rtfolder_of $_file $_dir +} +if {[llength $cross_target_tiers]} { + puts stdout "cross-target runtime tiers in use (mapvfs.config target platforms): [join $cross_target_tiers {, }]" +} +unset -nocomplain _rtname _rttarget _tier _dir _file if {![llength $runtimes]} { puts stderr "No executable runtimes found in $rtfolder - unable to build any .vfs folders into executables." puts stderr "Add runtimes to $rtfolder if required" + if {![file isdirectory $rtfolder]} { + #same store-tier self-diagnosis as bakelist + puts stderr "NOTE: that runtime store folder does not exist. Default target '$rt_default_target' derives from THIS tclsh: tcl_platform(os)='$::tcl_platform(os)' tcl_platform(platform)='$::tcl_platform(platform)' -> host canon '$::punkboot::host_platform' -> target '$rt_default_target'." + puts stderr " Populate it with 'bin/punk-runtime.cmd' (or by hand), or declare a per-entry target platform in mapvfs.config for runtimes stored elsewhere." + } #todo - don't exit - it is valid to use runtime of - to just build a .kit/.zipkit ? exit 0 } @@ -4801,10 +5160,10 @@ if {![llength $runtimes]} { #other runtimes get no BUILDCOPY refresh, no capability probe and no punkcheck events. if {[llength $::punkboot::bake_selected_kitnames]} { set runtimes [lmap _rtfile $runtimes { - set _rtkey $_rtfile - if {[string match *.exe $_rtkey]} { - set _rtkey [string range $_rtkey 0 end-4] - } + #NOTE: only an .exe suffix may be stripped to recover the mapping key - + #'file rootname' would eat the last dotted version segment of names like + #tclsh9.0.5 (suffixless on non-windows targets). + set _rtkey [punkboot::lib::runtime_mapkey $_rtfile] if {$_rtkey ni $::punkboot::bake_selected_runtimes} { continue } @@ -4814,41 +5173,6 @@ if {[llength $::punkboot::bake_selected_kitnames]} { } #previous have_sdx test location -# -- --- --- --- --- --- --- --- --- --- -#load mapvfs.config file (if any) in runtime folder to map runtimes to vfs folders. -#build a dict keyed on runtime executable name. -#If no mapfile (or no mapfile entry for that runtime) - the runtime will be paired with a matching .vfs folder in src folder. e.g punk.exe to src/punk.vfs -#If vfs folders or runtime executables which are explicitly listed in the mapfile don't exist - warn on stderr - but continue. if such nonexistants found; prompt user for whether to continue or abort. -#G-121: parsing lives in ::punkboot::lib::mapvfs_parse - the shared model consumed here, -#by 'bakelist' and by the selective-bake name resolution (never the file format directly). -set mapfile $rt_sourcefolder/mapvfs.config -set mapmodel [punkboot::lib::mapvfs_parse $mapfile $rtfolder $sourcefolder] -foreach warnline [dict get $mapmodel warnings] { - puts stderr $warnline -} -if {[llength [dict get $mapmodel configerrors]]} { - foreach errline [dict get $mapmodel configerrors] { - puts stderr $errline - } - exit 3 -} -set runtime_vfs_map [dict get $mapmodel runtime_vfs_map] -set vfs_runtime_map [dict get $mapmodel vfs_runtime_map] -set missing [dict get $mapmodel missing] -if {[llength $missing]} { - puts stderr "WARNING [llength $missing] missing items from $mapfile. (TODO - prompt user to continue/abort)" - foreach m $missing { - puts stderr " $m" - } - puts stderr "continuing..." -} -# -- --- --- --- --- --- --- --- --- --- -puts "-- runtime_vfs_map --" -punk::lib::pdict runtime_vfs_map -puts "---------------------" -puts "-- vfs_runtime_map--" -punk::lib::pdict vfs_runtime_map -puts "---------------------" #only test the runtime capabilities for runtimes that are actually in our map @@ -4897,27 +5221,34 @@ foreach runtime [dict keys $runtime_vfs_map] { exit 0 } - #temp - #build for current platform only for now - if {![file exists $rtfolder/$runtime]} { - dict set runtime_caps $runtime exitcode -1 error "find-fail $rtfolder/$runtime" + #G-122: address the runtime in ITS target's store tier, with ITS target's + #executable suffix. (Before the split this probed a suffixless name in the + #host-derived folder, so on windows every runtime silently reported + #find-fail and no capability was ever learned.) + set rt_target [punkboot::lib::runtime_target_of $runtime] + set rt_dir $rtbase/[punkboot::lib::platform_store_tier $rt_target] + set rt_file $runtime[punkboot::lib::platform_exe_suffix $rt_target] + if {![file exists $rt_dir/$rt_file]} { + dict set runtime_caps $runtime exitcode -1 error "find-fail $rt_dir/$rt_file" + continue + } + if {[punkboot::lib::platform_process_family $rt_target] ne [punkboot::lib::platform_process_family $::punkboot::host_platform]} { + #cross-target runtime: this host cannot execute it, so its capabilities + #stay unknown (the kit machinery falls back to trying the configured type) + dict set runtime_caps $runtime exitcode -1 error "cross-target ($rt_target) - not executable on this host" continue } #invoke can fail if runtime not an executable file for the current platform - if {[file type $rtfolder/$runtime] eq "directory"} { + if {[file type $rt_dir/$rt_file] eq "directory"} { #assume it's a mounted tclkit (because tcl kits mount in same place as their underlying path and mask the executable as a directory) #use the BUILDCOPY created above - REVIEW - if {$::tcl_platform(platform) eq "windows"} { - set useruntime [file rootname $runtime]_BUILDCOPY.exe - } else { - set useruntime ${runtime}_BUILDCOPY - } + set useruntime [punkboot::lib::runtime_buildcopyname $rt_file] } else { - set useruntime $runtime + set useruntime $rt_file } if {![catch { - lassign [punk::lib::invoke [list $rtfolder/$useruntime <<$capscript]] stdout stderr exitcode + lassign [punk::lib::invoke [list $rt_dir/$useruntime <<$capscript]] stdout stderr exitcode } errM]} { if {$exitcode == 0} { dict set runtime_caps $runtime $stdout @@ -4959,7 +5290,7 @@ set vfs_folder_changes [dict create] ;#cache whether each .vfs folder has change # -- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- #make build copy for all runtimes - not just those in the map - because even without a mapvfs.config file entry we build an exe for a runtime that matches .vfs folder name - REVIEW. foreach runtimefile $runtimes { - #runtimefile e.g tclkit86bi.exe on windows tclkit86bi on other platforms + #runtimefile e.g tclkit86bi.exe for a win32 target, tclkit86bi for a posix one #sdx *may* be pointed to use the runtime we use to build the kit, or the user may manually use this runtime if they don't have tclsh #sdx will complain if the same runtime is used for the shell as is used in the -runtime argument - so we make a copy (REVIEW) @@ -4967,23 +5298,19 @@ foreach runtimefile $runtimes { # file copy $rtfolder/$runtimefile $buildfolder/buildruntime.exe #} - #---------------------- - #todo - #temp - only build for current platform - need changes to runtime/mapvfs.config etc - if {![file exists $rtfolder/$runtimefile]} { - puts stderr " >> skipping $runtimefile not present in $rtfolder (wrong platform?) x-platform kit build TODO." + #G-122: each runtime is read from the tier its target platform names, not + #from one host-derived folder. Build copies keep the flat build_ name - + #the mapping cannot hold one runtime name in two tiers, so they cannot collide. + set rt_dir [dict get $rtfolder_of $runtimefile] + if {![file exists $rt_dir/$runtimefile]} { + puts stderr " >> skipping $runtimefile not present in $rt_dir" continue } - #---------------------- - if {[file type $rtfolder/$runtimefile] eq "directory"} { + if {[file type $rt_dir/$runtimefile] eq "directory"} { #assume it's a mounted tclkit (because tcl kits mount in same place as their underlying path and mask the executable) #use the BUILDCOPY created above - REVIEW - if {$::tcl_platform(platform) eq "windows"} { - set useruntimefile [file tail $runtimefile]_BUILDCOPY.exe - } else { - set useruntimefile ${runtimefile}_BUILDCOPY - } + set useruntimefile [punkboot::lib::runtime_buildcopyname $runtimefile] } else { set useruntimefile $runtimefile } @@ -4994,11 +5321,11 @@ foreach runtimefile $runtimes { }] #---------- set installer [punkcheck::installtrack new $installername $basedir/.punkcheck] - $installer set_source_target $rtfolder $buildfolder + $installer set_source_target $rt_dir $buildfolder set event [$installer start_event $config] $event targetset_init INSTALL $buildfolder/build_$runtimefile - #$event targetset_addsource $rtfolder/$runtimefile - $event targetset_addsource $rtfolder/$useruntimefile ;#possibly the _BUILDCOPY in the runtime folder (created for when mounted in current executable as a tclkit) + #$event targetset_addsource $rt_dir/$runtimefile + $event targetset_addsource $rt_dir/$useruntimefile ;#possibly the _BUILDCOPY in the runtime folder (created for when mounted in current executable as a tclkit) $event targetset_addsource $buildfolder/build_$runtimefile ;#self as source for change detection #---------- @@ -5010,9 +5337,9 @@ foreach runtimefile $runtimes { $event targetset_started # -- --- --- --- --- --- #This is the full runtime - *possibly* with some sort of vfs attached. - puts stdout "Copying runtime (as is) from $rtfolder/$useruntimefile to $buildfolder/build_$runtimefile" + puts stdout "Copying runtime (as is) from $rt_dir/$useruntimefile to $buildfolder/build_$runtimefile" if {[catch { - file copy -force $rtfolder/$useruntimefile $buildfolder/build_$runtimefile ;#becomes building_runtime + file copy -force $rt_dir/$useruntimefile $buildfolder/build_$runtimefile ;#becomes building_runtime } errM]} { puts stderr " >> copy runtime to $buildfolder/build_$runtimefile FAILED" $event targetset_end FAILED @@ -5050,12 +5377,16 @@ if {$sdxpath eq ""} { #see if the runtime (or any runtime *for current platform*) has tclkit in the name or has_starkit(?) and use that #todo - review #picking first found for now - fix + #G-122: only a runtime this HOST can execute can drive sdx - the + #capability probe already skipped cross-target runtimes, so a + #has_starkit record means the runtime ran here. dict for {rtname rtprops} $runtime_caps { if {[dict exists $rtprops has_starkit] && [dict get $rtprops has_starkit]} { - set testpath $rtfolder/$rtname + set _rt_target [punkboot::lib::runtime_target_of $rtname] + set testpath $rtbase/[punkboot::lib::platform_store_tier $_rt_target]/$rtname[punkboot::lib::platform_exe_suffix $_rt_target] if {[file exists $testpath]} { if {[file type $testpath] eq "directory"} { - set alt_tclkitpath $rtfolder/[punkboot::lib::runtime_buildcopyname $testpath] + set alt_tclkitpath [file dirname $testpath]/[punkboot::lib::runtime_buildcopyname $testpath] } else { set alt_tclkitpath $testpath } @@ -5246,6 +5577,43 @@ proc vfs_startup_script_warning {vfsfolder} { } return "vfs folder $vfsfolder has NO STARTUP SCRIPT: no root main.tcl and no root fauxlink resolving to main.tcl with an existing .tcl target - kits built from it will have no boot script (build proceeding anyway)$detail" } +#Cat-style zipkit assembly: a raw (vfs-free) runtime followed by a freshly built +#zip, concatenated. This is the zipfs-less assembly path for BOTH zip-type kits +#(G-122): Tcl's zipfs locates an image by scanning back from the END of the file +#and reads the central directory with ARCHIVE-START-RELATIVE offsets, so a plain +#concatenation mounts. 'zipfs mkimg's file-relative offset rewrite is one +#convention, not a mounting requirement - the zipcat kit type is the in-repo +#proof, and this lets a driving tcl with no zipfs at all (any 8.6) assemble zip +#kits. The zip itself is written by tcl::zipfs::mkzip when the driving tcl has +#it, else by punk::zip::mkzip (pure Tcl). +#Returns the number of zip bytes appended. +proc ::punkboot::assemble_zipcat_image {raw_runtime wrapvfs outfile zipfile} { + file copy -force $raw_runtime $outfile + #runtime in runtime folder may not have write perm set - ensure the copy does as we need to append + catch {exec chmod +w $outfile} + file delete $zipfile + if {[info commands ::tcl::zipfs] ne ""} { + puts stdout "tcl::zipfs::mkzip $zipfile $wrapvfs $wrapvfs" + ::tcl::zipfs::mkzip $zipfile $wrapvfs $wrapvfs + } else { + puts stdout "punk::zip::mkzip -directory $wrapvfs -base $wrapvfs $zipfile *" + package require punk::zip + punk::zip::mkzip -directory $wrapvfs -base $wrapvfs $zipfile * + } + puts stderr "concatenating zip to executable.." + set fdout [open $outfile a] + chan conf $fdout -translation binary + puts stderr "runtime bytes: [tell $fdout]" + set fdzip [open $zipfile r] + chan conf $fdzip -translation binary + set zipbytes [fcopy $fdzip $fdout] + close $fdzip + puts stderr "zip bytes: $zipbytes" + puts stderr "exezip bytes: [tell $fdout]" + close $fdout + return $zipbytes +} + set startdir [pwd] puts stdout "Found [llength $vfs_tails] .vfs folders - checking each for executables that may need to be built" cd [file dirname $buildfolder] @@ -5275,24 +5643,26 @@ foreach vfstail $vfs_tails { }] + #G-122: a runtime's filename and store tier come from ITS target platform. + #The '-' pseudo-runtime (unwrapped .kit output) carries no file at all. set runtimes [list] if {[dict exists $vfs_runtime_map $vfstail]} { - #set runtimes [dict get $vfs_runtime_map $vfstail] + #set runtimes [dict get $vfs_runtime_map $vfstail] #runtimes in vfs_runtime_map_vfs dict are unsuffixed (.exe stripped or was not present) set applist [dict get $vfs_runtime_map $vfstail] - if {"windows" eq $::tcl_platform(platform)} { - foreach rt_app $applist { - set rt [lindex $rt_app 0] - if {$rt ne "-" && [file exists $rtfolder/$rt.exe]} { - lappend runtimes $rt.exe - } + foreach rt_app $applist { + set rt [lindex $rt_app 0] + if {$rt eq "-"} { + #the '-' (runtime-less .kit) pseudo-runtime has never reached this + #list - both historical branches filtered it out - so the loop + #below's '-' handling is currently unreachable. Left as-is: + #enabling it is a separate, untested change. + continue } - } else { - foreach rt_app $applist { - set rt [lindex $rt_app 0] - if {[file exists $rtfolder/$rt]} { - lappend runtimes $rt - } + set rt_target [punkboot::lib::runtime_target_of $rt] + set rt_file $rt[punkboot::lib::platform_exe_suffix $rt_target] + if {[file exists $rtbase/[punkboot::lib::platform_store_tier $rt_target]/$rt_file]} { + lappend runtimes $rt_file } } } else { @@ -5301,18 +5671,14 @@ foreach vfstail $vfs_tails { #but conversely, adding an extra entry shouldn't stop default builds that used to run.. set matchrt [file rootname [file tail $vfstail]] ;#e.g project.vfs -> project if {![dict exists $runtime_vfs_map $matchrt]} { - if {"windows" eq $::tcl_platform(platform)} { - if {[file exists $rtfolder/$matchrt.exe]} { - lappend runtimes $matchrt.exe - } - } else { - if {[file exists $rtfolder/$matchrt]} { - lappend runtimes $matchrt - } + #unmapped: the default-target store is the only place a name-match is looked for + if {[file exists $rtfolder/$matchrt$rt_exe_suffix]} { + lappend runtimes $matchrt$rt_exe_suffix } } } - #assert $runtimes is a list of executable names suffixed with .exe if on windows - whether or not specified with .exe in the mapvfs.config + #assert $runtimes is a list of runtime filenames as they exist in their target's store tier + #(suffixed with .exe for windows-family targets, whether or not mapvfs.config spelled it that way) puts " vfs: $vfstail runtimes to process ([llength $runtimes]): $runtimes" if {[llength $runtimes]} { @@ -5326,13 +5692,17 @@ foreach vfstail $vfs_tails { #todo - non kit based - zipkit? # $runtimes may now include a dash entry "-" (from mapvfs.config file) foreach runtime_fullname $runtimes { - set rtname [file rootname $runtime_fullname] + set rtname [punkboot::lib::runtime_mapkey $runtime_fullname] if {[llength $::punkboot::bake_selected_kitnames] && $rtname ni $::punkboot::bake_selected_runtimes} { continue ;#G-121 selective bake - no requested kit wraps this runtime } + #G-122: everything this iteration emits is keyed by the runtime's TARGET + set rt_target [punkboot::lib::runtime_target_of $rtname] + set rt_tierdir $rtbase/[punkboot::lib::platform_store_tier $rt_target] + set rt_suffix [punkboot::lib::platform_exe_suffix $rt_target] #rtname of "-" indicates build a kit without a runtime if {$runtime_fullname ne "-"} { - ::punkboot::runtime_materialization_warning $rtfolder $runtime_fullname + ::punkboot::runtime_materialization_warning $rt_tierdir $runtime_fullname } #first configured runtime will be the one to use the same name as .vfs folder for output executable. Additional runtimes on this .vfs will need to suffix the runtime name to disambiguate. @@ -5356,11 +5726,7 @@ foreach vfstail $vfs_tails { if {$rtname eq "-"} { set targetkit $appname.kit } else { - if {$::tcl_platform(platform) eq "windows"} { - set targetkit ${appname}.exe - } else { - set targetkit $appname - } + set targetkit ${appname}$rt_suffix if {$targetkit in $exe_names_seen} { #duplicate appname configured? #todo - consider creating as ${appname}(2) etc? @@ -5384,9 +5750,10 @@ foreach vfstail $vfs_tails { #cannot replace a running image anyway. Informational/update subcommands are #unaffected (they exit before the kit machinery), so driving make.tcl from a built #punk executable remains supported for everything except rebuilding that executable. + #(a HOST filesystem comparison - case rules follow the host, not the target) set self_exe [file normalize [info nameofexecutable]] set self_deploy_target [file normalize $binfolder/$targetkit] - if {$::tcl_platform(platform) eq "windows"} { + if {$::punkboot::host_windows} { set is_self_build [string equal -nocase $self_exe $self_deploy_target] } else { set is_self_build [string equal $self_exe $self_deploy_target] @@ -5451,15 +5818,23 @@ foreach vfstail $vfs_tails { #to consider: - allow specifying runtime kits as if they are vfs folders in the normal xxx.vfs list - and autodetect and extract #would need to detect UPX, cookfs,zipfs,tclkit set rtmountpoint "" + #Test the capability VALUE, not just its presence: a probe that ran + #reports has_starkit/has_cookfs 0 for a plain zipfs runtime, and + #trying to unwrap such a runtime as a kit ends in sdx mksplit + #failing and REPLACING raw_runtime with the un-split original - + #which silently yields a zip kit with two zips and no tcl_library. + #(Latent until G-122 made the capability probe actually run on + #windows: before that every probe reported find-fail, so this list + #was always empty.) set caps [dict get $runtime_caps $rtname] set source_kit_caps [list] - if {[dict exists $caps has_zipfs]} { + if {[dict exists $caps has_zipfs] && [dict get $caps has_zipfs]} { lappend source_kit_caps zip } - if {[dict exists $caps has_starkit]} { + if {[dict exists $caps has_starkit] && [dict get $caps has_starkit]} { lappend source_kit_caps kit } - if {[dict exists $caps has_cookfs]} { + if {[dict exists $caps has_cookfs] && [dict get $caps has_cookfs]} { lappend source_kit_caps cookfs } @@ -5576,11 +5951,18 @@ foreach vfstail $vfs_tails { zipfile::decode::open $extractedzipfile set archiveinfo [zipfile::decode::archive] zipfile::decode::unzip $archiveinfo $extractedzipfolder - }]} { + } extracterr]} { set extraction_done 1 set extract_kit_type $extract_kit_try #todo - verify that init.tcl etc are present? merge_over $extractedzipfolder $targetvfs + } else { + #Do not swallow the reason: without the runtime's own + #zip contents the built kit has no tcl_library. Note + #zipfile::decode is a system tcllib package, not a + #vendored one - a tclsh without tcllib (e.g the msys2 + #/usr/bin/tclsh8.6) cannot take this path at all. + puts stderr "zipfs-less extraction of $runtime_fullname FAILED: $extracterr" } } @@ -5668,10 +6050,11 @@ foreach vfstail $vfs_tails { if {!$extraction_done} { #TODO: if not extracted - use a default tcl_library for patchlevel and platform? - puts stderr "--------------------------------------------" - puts stderr "[punkboot::sgr 31]WARNING: No extraction done from runtime $runtime_fullname[punkboot::sgr]" - puts stderr "If no init.tcl provided in the vfs at the proper location (containing init.tcl) - the resulting kit will probably not initialise properly!" - puts stderr "--------------------------------------------" + #Recapped BUILD-WARNING (not just an inline note): the build continues + #and produces an artifact, but unless the .vfs itself supplies a + #tcl_library that artifact cannot boot - and this line would otherwise + #scroll away behind thousands of merge lines. + ::punkboot::print_build_warnings [list "No extraction done from runtime $runtime_fullname (tried: $target_kit_type[expr {[llength $source_kit_caps] ? " + runtime caps $source_kit_caps" : ""}]) - kit $targetkit gets no tcl_library from the runtime and will not initialise unless src/vfs/$vfstail supplies one"] file mkdir $targetvfs } @@ -5756,9 +6139,17 @@ foreach vfstail $vfs_tails { puts stderr "RUNTIME capabilities unknown. Unsure if zip supported. trying anyway.." } } - #note - as at 2024-08 - there is some discussion about the interface to mkimg - it is considered unstable (may change to -option value syntax) - puts stderr "calling: tcl::zipfs::mkimg $buildfolder/$vfsname.new $wrapvfs $wrapvfs \"\" $raw_runtime" - tcl::zipfs::mkimg $buildfolder/$vfsname.new $wrapvfs $wrapvfs "" $raw_runtime + if {[info commands ::tcl::zipfs::mkimg] ne ""} { + #note - as at 2024-08 - there is some discussion about the interface to mkimg - it is considered unstable (may change to -option value syntax) + puts stderr "calling: tcl::zipfs::mkimg $buildfolder/$vfsname.new $wrapvfs $wrapvfs \"\" $raw_runtime" + tcl::zipfs::mkimg $buildfolder/$vfsname.new $wrapvfs $wrapvfs "" $raw_runtime + } else { + #G-122: the driving tcl has no zipfs (8.6) - assemble the same + #kind of image by concatenation. The result mounts on any + #zipfs-capable runtime (archive-start-relative offsets). + puts stderr "WARNING: tcl shell '[info nameofexecutable]' has no zipfs - assembling zip kit $targetkit by concatenation (raw runtime + mkzip)" + ::punkboot::assemble_zipcat_image $raw_runtime $wrapvfs $buildfolder/$vfsname.new $buildfolder/$vfsname.zip + } } result ]} { set failmsg "zipfs mkimg failed with msg: $result" puts stderr "tcl::zipfs::mkimg $targetkit failed" @@ -5768,7 +6159,7 @@ foreach vfstail $vfs_tails { $vfs_installer destroy continue } else { - puts stdout "ok - finished tcl::zipfs::mkimg" + puts stdout "ok - finished zip image assembly" set separator [string repeat = 40] puts stdout $separator puts stdout $result @@ -5794,32 +6185,7 @@ foreach vfstail $vfs_tails { } #'archive' based zip offsets - editable in 7z,peazip - file copy $raw_runtime $buildfolder/$vfsname.new - #runtime in runtime folder may not have write perm set - ensure the copy does as we need to append - catch {exec chmod +w $buildfolder/$vfsname.new} - file delete $buildfolder/$vfsname.zip - - if {[info commands ::tcl::zipfs] ne ""} { - puts stdout "tcl::zipfs::mkzip $buildfolder/$vfsname.zip $wrapvfs $wrapvfs" - ::tcl::zipfs::mkzip $buildfolder/$vfsname.zip $wrapvfs $wrapvfs - } else { - puts stdout "punk::zip::mkzip -directory $wrapvfs -base $wrapvfs $buildfolder/$vfsname.zip *" - package require punk::zip - punk::zip::mkzip -directory $wrapvfs -base $wrapvfs $buildfolder/$vfsname.zip * - } - - - puts stderr "concatenating executable to zip.." - set fdout [open $buildfolder/$vfsname.new a] - chan conf $fdout -translation binary - puts stderr "runtime bytes: [tell $fdout]" - set fdzip [open $buildfolder/$vfsname.zip r] - chan conf $fdzip -translation binary - set zipbytes [fcopy $fdzip $fdout] - close $fdzip - puts stderr "zip bytes: $zipbytes" - puts stderr "exezip bytes: [tell $fdout]" - close $fdout + ::punkboot::assemble_zipcat_image $raw_runtime $wrapvfs $buildfolder/$vfsname.new $buildfolder/$vfsname.zip } result ]} { set failmsg "creating zipcat image failed with msg: $result" puts stderr "creating image (zipcat) $targetkit failed" @@ -5942,18 +6308,34 @@ foreach vfstail $vfs_tails { #still - seems to be under 1s on a 2018 era i9 running in 2025 - which is not a big problem in this context. #using a filter in tasklist or get-command can make things a little faster - if {$::tcl_platform(platform) eq "windows"} { + #G-122: the tooling follows the TARGET, not the driving tclsh. + #A win32 kit is a windows process whatever baked it - and msys 'ps' + #sees only msys-descendant processes, so an msys-hosted build using + #'ps' would report a natively-launched kit as not running and then + #fail to replace it. tasklist/taskkill exec fine from an msys tclsh. + #When target and host process worlds differ (a cross-target bake) no + #local process can be holding the artifact - skip the sweep rather + #than run tooling that cannot see the target's processes anyway. + set target_proc_family [punkboot::lib::platform_process_family $rt_target] + set host_proc_family [punkboot::lib::platform_process_family $::punkboot::host_platform] + if {$target_proc_family eq "windows"} { set pscmd "tasklist" + set pid_field 1 } else { set pscmd "ps" + set pid_field 0 + } + set sweep_applicable [expr {$target_proc_family eq $host_proc_family}] + if {!$sweep_applicable} { + puts stdout " (skipping running-process sweep for $targetkit - target $rt_target processes are not visible to this $::punkboot::host_platform host)" } - #killing process doesn't apply to .kit build - if {$rtname ne "-"} { + #killing process doesn't apply to .kit build + if {$rtname ne "-" && $sweep_applicable} { #exec $pscmd | grep $targetkit ;#wrong - fails on at least some OSes because grep itself is in the list #exec ps | grep $targetkit | grep -v grep ;#exclude grep process if {![catch { - exec $pscmd + ::punkboot::exec_nativeargs [list $pscmd] } ps_lines]} { #we will process without grep for for now - may not be avail on windows? set still_running_lines [list] @@ -5966,11 +6348,7 @@ foreach vfstail $vfs_tails { #itself (e.g a copy of the target executable driving this build from #a path other than the deploy target, which the self-build guard #above doesn't catch). - if {$::tcl_platform(platform) eq "windows"} { - set ln_pid [lindex $ln 1] - } else { - set ln_pid [lindex $ln 0] - } + set ln_pid [lindex $ln $pid_field] if {$ln_pid eq [pid]} {continue} lappend still_running_lines $ln } @@ -5983,15 +6361,14 @@ foreach vfstail $vfs_tails { foreach ln $still_running_lines { puts stdout " $ln" - if {$::tcl_platform(platform) eq "windows"} { - set pid [lindex $ln 1] + set pid [lindex $ln $pid_field] + if {$target_proc_family eq "windows"} { if {$forcekill} { set killcmd [list taskkill /F /PID $pid] } else { set killcmd [list taskkill /PID $pid] } } else { - set pid [lindex $ln 0] #review! if {$forcekill} { set killcmd [list kill -9 $pid] @@ -6001,7 +6378,7 @@ foreach vfstail $vfs_tails { } puts stdout " pid: $pid (attempting to kill now using '$killcmd')" if {[catch { - exec {*}$killcmd + ::punkboot::exec_nativeargs $killcmd } errMsg]} { puts stderr "$killcmd returned an error:" puts stderr $errMsg diff --git a/src/project_layouts/vendor/punk/basic/src/bootsupport/modules/include_modules.config b/src/project_layouts/vendor/punk/basic/src/bootsupport/modules/include_modules.config index fac8ef64..95c2f4dc 100644 --- a/src/project_layouts/vendor/punk/basic/src/bootsupport/modules/include_modules.config +++ b/src/project_layouts/vendor/punk/basic/src/bootsupport/modules/include_modules.config @@ -104,6 +104,7 @@ set bootsupport_modules [list\ modules punk::ns\ modules punk::overlay\ modules punk::path\ + modules punk::platform\ modules punk::packagepreference\ modules punk::repl\ modules punk::repl::codethread\ diff --git a/src/project_layouts/vendor/punk/basic/src/make.tcl b/src/project_layouts/vendor/punk/basic/src/make.tcl index 83930762..7d08906c 100644 --- a/src/project_layouts/vendor/punk/basic/src/make.tcl +++ b/src/project_layouts/vendor/punk/basic/src/make.tcl @@ -444,16 +444,29 @@ namespace eval ::punkboot::lib { return "${plat}-${cpu}" } + proc platform_normalize_os {os} { + #INLINE COPY of punk::platform::normalize_os - see platform_punk below. + switch -glob -- [string tolower $os] { + macos {return macosx} + msys - msys_nt-* {return msys} + mingw32 - mingw64 - mingw_nt-* - mingw32_nt-* - mingw64_nt-* {return msys} + ucrt64 - ucrt64_nt-* {return msys} + clang32 - clang64 - clangarm64 - clang32_nt-* - clang64_nt-* - clangarm64_nt-* {return msys} + cygwin_nt-* {return cygwin} + } + return $os + } proc platform_punk {} { #canonical punkshell platform-dir name: platform_generic normalized. #INLINE COPY of punk::platform::normalize (src/modules/punk/platform-*.tm; #'help platforms' documents the canon) - the boot stage cannot package #require, so keep this mapping in sync with that module: - #amd64->x86_64, aarch64->arm64, macos->macosx, macosx arm->arm64. + #amd64->x86_64, aarch64->arm64, macos->macosx, macosx arm->arm64, + #msys2 family (mingw64/mingw32/ucrt64/clang*) -> msys, CYGWIN_NT -> cygwin. set parts [split [platform_generic] -] set cpu [lindex $parts end] set os [join [lrange $parts 0 end-1] -] - if {$os eq "macos"} {set os macosx} + set os [platform_normalize_os $os] switch -- $cpu { amd64 {set cpu x86_64} aarch64 {set cpu arm64} @@ -462,6 +475,93 @@ namespace eval ::punkboot::lib { return "${os}-${cpu}" } + # -- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- + # G-122 host/target platform split. + # platform_punk answers "what is THIS tclsh" (the HOST canon). Everything a + # build EMITS is a separate question keyed by TARGET: which bin/runtime/ + # store holds the runtime, whether artifacts carry .exe, and which process + # tooling can see/kill a running artifact. The two diverge for the whole + # cygwin family: an msys2/cygwin-runtime tclsh on windows reports + # tcl_platform(platform) 'unix' and canonizes as msys-x86_64/cygwin-x86_64, + # yet the kits it bakes are ordinary win32-x86_64 .exe binaries that only + # tasklist/taskkill can manage (msys 'ps' sees only msys-descendant + # processes). Host process semantics - copy commands, path handling, prompts, + # filesystem case rules - stay keyed to the host. + # -- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- + + #os token of a canonical platform-dir name (win32-x86_64 -> win32; macosx -> macosx) + proc platform_os {platform} { + set parts [split $platform -] + if {[llength $parts] < 2} { + return $platform + } + return [join [lrange $parts 0 end-1] -] + } + #cpu token of a canonical platform-dir name ("" for single-token names like macosx) + proc platform_cpu {platform} { + set parts [split $platform -] + if {[llength $parts] < 2} { + return "" + } + return [lindex $parts end] + } + #posix-personality runtimes hosted on windows (msys2 / cygwin) + proc platform_is_cygwin_family {platform} { + return [expr {[platform_os $platform] in {msys cygwin}}] + } + #Windows-family platforms: native win32 AND the cygwin-family personalities. + #They share what matters to a build: executables are .exe files and running + #processes are windows processes (tasklist/taskkill), whatever tcl_platform says. + proc platform_is_windows_family {platform} { + return [expr {[platform_os $platform] in {win32 msys cygwin}}] + } + #Default TARGET platform for a host canon. A cygwin-family host builds native + #windows kits (there is no separate msys kit lineage - third-party msys + #runtimes are addressed by an explicit per-entry target instead). + proc platform_target_default {hostplatform} { + if {[platform_is_cygwin_family $hostplatform]} { + set cpu [platform_cpu $hostplatform] + if {$cpu eq ""} {set cpu x86_64} + return "win32-$cpu" + } + return $hostplatform + } + #bin/runtime store tier folder name for a target: macOS runtimes are universal + #(multi-arch) binaries kept in one 'macosx' folder - see punk::platform. + proc platform_store_tier {target} { + if {[string match "macosx-*" $target]} { + return macosx + } + return $target + } + #executable filename suffix artifacts for this target carry + proc platform_exe_suffix {target} { + if {[platform_is_windows_family $target]} { + return ".exe" + } + return "" + } + #which process tooling manages a running artifact of this platform + proc platform_process_family {platform} { + if {[platform_is_windows_family $platform]} { + return windows + } + return posix + } + #A target platform name must look like a platform-dir name so it can address a + #store tier. Returns "" when acceptable, else a reason string. + proc platform_name_problem {platform} { + if {$platform eq ""} { + return "empty" + } + if {[regexp {[\\/:*?\"<>|]} $platform]} { + return "not a platform-dir name (contains a path/wildcard character)" + } + if {[llength [split $platform -]] < 2 && $platform ne "macosx"} { + return "not an - platform-dir name ('help platforms' lists the canon)" + } + return "" + } } #------------------------------------------------------------------------------ @@ -533,27 +633,35 @@ proc ::punkboot::lib::bootsupport_prompt_yesno {prompt} { # directly, so a config-format change (G-024 toml direction) swaps in underneath # by reimplementing mapvfs_parse alone. # -# mapvfs_parse: pure parse - no output, no exits. Returns a dict: +# mapvfs_parse: pure parse - no output, no exits. Every store address, presence +# probe and artifact suffix it computes is keyed by the entry's TARGET platform +# (G-122), never by the driving tclsh's personality: $rtbase is the store ROOT +# (bin/runtime) and each entry addresses $rtbase/. Returns a dict: # exists 0|1 (mapfile present) # mapfile path as supplied -# runtime_vfs_map runtime -> raw vfs_specs as listed (appname/type may be "") -# vfs_runtime_map vfstail -> list of {runtime appname type} (defaults applied; -# only vfs folders that exist on disk - matching the historical -# inline parse the kit loop was built around) +# default_target target platform for entries that declare none +# runtime_vfs_map runtime -> raw vfs_specs as listed (appname/type/target may be "") +# vfs_runtime_map vfstail -> list of {runtime appname type target} (defaults +# applied; only vfs folders that exist on disk - matching the +# historical inline parse the kit loop was built around) +# runtime_target runtime -> resolved target platform (all runtimes, mapped order) # missing names of referenced-but-absent runtimes/vfs folders # warnings warning lines in encounter order, ready to print verbatim # (missing-item lines carry their historical WARNING: prefix) # badentries subset of warnings: malformed entry lines (these have no kit # record to carry them, so bakelist prints them; missing-item # warnings are instead surfaced as per-row state there) -# configerrors fatal config problems (duplicate runtime line; caller decides -# whether to abort - the bake path keeps its historical exit 3) -proc ::punkboot::lib::mapvfs_parse {mapfile rtfolder sourcefolder} { +# configerrors fatal config problems (duplicate runtime line; unusable or +# self-contradicting target platform; caller decides whether to +# abort - the bake path keeps its historical exit 3) +proc ::punkboot::lib::mapvfs_parse {mapfile rtbase sourcefolder default_target} { set model [dict create\ exists 0\ mapfile $mapfile\ + default_target $default_target\ runtime_vfs_map [dict create]\ vfs_runtime_map [dict create]\ + runtime_target [dict create]\ missing [list]\ warnings [list]\ badentries [list]\ @@ -570,6 +678,7 @@ proc ::punkboot::lib::mapvfs_parse {mapfile rtfolder sourcefolder} { set mapdata [string map [list \r\n \n] $mapdata] set runtime_vfs_map [dict create] set vfs_runtime_map [dict create] + set runtime_target [dict create] set missing [list] set warnings [list] set badentries [list] @@ -585,19 +694,45 @@ proc ::punkboot::lib::mapvfs_parse {mapfile rtfolder sourcefolder} { #drop windows .exe suffix so same config can work cross platform - extension will be re-added if necessary later set runtime [string range $runtime 0 end-4] } - if {$runtime ne "-"} { - set runtime_test $runtime - if {"windows" eq $::tcl_platform(platform)} { - set runtime_test $runtime.exe + #Target platform is declared per kit-config entry (4th element) but is a + #property of the RUNTIME: one binary lives in one store tier and has one + #executable convention. Entries on a line may therefore repeat the + #declaration but must not disagree. + set line_target "" + foreach vfsconfig $vfs_specs { + if {[llength $vfsconfig] < 4} { + continue + } + set declared [lindex $vfsconfig 3] + if {$declared eq ""} { + continue } - if {![file exists [file join $rtfolder $runtime_test]]} { - lappend warnings "WARNING: Missing runtime file $rtfolder/$runtime_test (line in mapvfs.config: $ln)" + if {$line_target eq ""} { + set problem [platform_name_problem $declared] + if {$problem ne ""} { + lappend configerrors "CONFIG FILE ERROR. runtime: $runtime declares target platform '$declared' - $problem." + continue + } + set line_target $declared + } elseif {$declared ne $line_target} { + lappend configerrors "CONFIG FILE ERROR. runtime: $runtime declares conflicting target platforms ('$line_target' and '$declared') in $mapfile - a runtime has one target." + } + } + if {$line_target eq ""} { + set line_target $default_target + } + set rtdir [file join $rtbase [platform_store_tier $line_target]] + set exe_suffix [platform_exe_suffix $line_target] + if {$runtime ne "-"} { + set runtime_test $runtime$exe_suffix + if {![file exists [file join $rtdir $runtime_test]]} { + lappend warnings "WARNING: Missing runtime file $rtdir/$runtime_test (line in mapvfs.config: $ln)" lappend missing $runtime } } foreach vfsconfig $vfs_specs { switch -- [llength $vfsconfig] { - 1 - 2 - 3 { + 1 - 2 - 3 - 4 { lassign $vfsconfig vfstail appname target_kit_type if {![file isdirectory [file join $sourcefolder vfs $vfstail]]} { lappend warnings "WARNING: Missing vfs folder [file join $sourcefolder vfs $vfstail] specified in mapvfs.config for runtime $runtime" @@ -606,11 +741,11 @@ proc ::punkboot::lib::mapvfs_parse {mapfile rtfolder sourcefolder} { if {$appname eq ""} { set appname [file rootname $vfstail] } - dict lappend vfs_runtime_map $vfstail [list $runtime $appname $target_kit_type] + dict lappend vfs_runtime_map $vfstail [list $runtime $appname $target_kit_type $line_target] } } default { - set badline "bad entry in mapvfs.config - expected each entry after the runtime name to be of length 1 or length 2. got: $vfsconfig ([llength $vfsconfig])" + set badline "bad entry in mapvfs.config - expected each entry after the runtime name to be a list of 1 to 4 elements {vfsfolder ?kitname? ?kittype? ?targetplatform?}. got: $vfsconfig ([llength $vfsconfig])" lappend warnings $badline lappend badentries $badline } @@ -620,10 +755,12 @@ proc ::punkboot::lib::mapvfs_parse {mapfile rtfolder sourcefolder} { lappend configerrors "CONFIG FILE ERROR. runtime: $runtime was specified more than once in $mapfile." } else { dict set runtime_vfs_map $runtime $vfs_specs + dict set runtime_target $runtime $line_target } } dict set model runtime_vfs_map $runtime_vfs_map dict set model vfs_runtime_map $vfs_runtime_map + dict set model runtime_target $runtime_target dict set model missing $missing dict set model warnings $warnings dict set model badentries $badentries @@ -635,24 +772,30 @@ proc ::punkboot::lib::mapvfs_parse {mapfile rtfolder sourcefolder} { # Replicates the kit loop's iteration + naming (vfs_tails glob order with map # extras appended, per-vfs runtime order from the mapping, appname defaulting to # the vfs rootname, kit_type defaulting to 'kit', '-' runtime -> .kit, -# windows .exe suffix, duplicate appname -> _) so listed names +# TARGET .exe suffix, duplicate appname -> _) so listed names # match what a bake produces. Entries whose vfs folder is missing on disk never # reach the kit loop - they are appended after the buildable set so 'bakelist' # can surface them as broken config. Duplicate-name disambiguation assumes the # config keeps appnames unique (as the kit loop itself effectively does). -# Returns a list of record dicts: +# $rtbase is the store ROOT (bin/runtime): each record resolves its own tier +# under it, so entries targeting different platforms report their own store +# presence (G-122). Returns a list of record dicts: # kitname base output name (selection key, e.g punk91) # targetkit platform artifact name (e.g punk91.exe, or .kit for '-') # kit_type kit|zip|zipcat|cookfs... as configured (defaulted to kit) # runtime runtime name as configured (unsuffixed), or "-" -# runtime_file platform runtime filename ("" for "-") +# runtime_file target runtime filename ("" for "-") # runtime_present 1|0 (1 for "-": no runtime needed) +# runtime_dir store tier folder holding the runtime ("" for "-") +# target target platform (e.g win32-x86_64) +# store_tier bin/runtime tier folder name for the target # vfs vfs folder tail (e.g punk9wintk903.vfs) # vfs_present 1|0 -proc ::punkboot::lib::mapvfs_kit_outputs {model rtfolder sourcefolder} { +proc ::punkboot::lib::mapvfs_kit_outputs {model rtbase sourcefolder} { set runtime_vfs_map [dict get $model runtime_vfs_map] set vfs_runtime_map [dict get $model vfs_runtime_map] - set on_windows [expr {"windows" eq $::tcl_platform(platform)}] + set runtime_target [dict get $model runtime_target] + set default_target [dict get $model default_target] set vfs_tails [glob -nocomplain -dir $sourcefolder/vfs -types d -tail *.vfs] dict for {vfstail -} $vfs_runtime_map { if {$vfstail ni $vfs_tails} { @@ -662,33 +805,37 @@ proc ::punkboot::lib::mapvfs_kit_outputs {model rtfolder sourcefolder} { set records [list] set exe_names_seen [list] set record_pair {{rtname vfstail appname target_kit_type} { - upvar 1 records records exe_names_seen exe_names_seen rtfolder rtfolder on_windows on_windows sourcefolder sourcefolder + upvar 1 records records exe_names_seen exe_names_seen rtbase rtbase sourcefolder sourcefolder\ + runtime_target runtime_target default_target default_target if {$appname eq ""} { set appname [file rootname $vfstail] } if {$target_kit_type eq ""} { set target_kit_type "kit" } + if {[dict exists $runtime_target $rtname]} { + set target [dict get $runtime_target $rtname] + } else { + set target $default_target + } + set store_tier [::punkboot::lib::platform_store_tier $target] if {$rtname eq "-"} { set targetkit $appname.kit set kitname $appname set runtime_file "" + set runtime_dir "" set runtime_present 1 ;#no runtime needed } else { - if {$on_windows} { - set runtime_file $rtname.exe - set targetkit ${appname}.exe - } else { - set runtime_file $rtname - set targetkit $appname - } + set runtime_dir [file join $rtbase $store_tier] + set runtime_file $rtname[::punkboot::lib::platform_exe_suffix $target] + set targetkit ${appname}[::punkboot::lib::platform_exe_suffix $target] set kitname $appname if {$targetkit in $exe_names_seen} { #duplicate appname configured - kit loop disambiguates the same way set targetkit ${appname}_$rtname set kitname ${appname}_$rtname } - set runtime_present [file exists [file join $rtfolder $runtime_file]] + set runtime_present [file exists [file join $runtime_dir $runtime_file]] } lappend exe_names_seen $targetkit lappend records [dict create\ @@ -698,6 +845,9 @@ proc ::punkboot::lib::mapvfs_kit_outputs {model rtfolder sourcefolder} { runtime $rtname\ runtime_file $runtime_file\ runtime_present $runtime_present\ + runtime_dir $runtime_dir\ + target $target\ + store_tier $store_tier\ vfs $vfstail\ vfs_present [file isdirectory [file join $sourcefolder vfs $vfstail]]\ ] @@ -723,7 +873,7 @@ proc ::punkboot::lib::mapvfs_kit_outputs {model rtfolder sourcefolder} { #entries whose vfs folder is missing (excluded from vfs_runtime_map by the parse) dict for {rtname vfs_specs} $runtime_vfs_map { foreach vfsconfig $vfs_specs { - if {[llength $vfsconfig] < 1 || [llength $vfsconfig] > 3} { + if {[llength $vfsconfig] < 1 || [llength $vfsconfig] > 4} { continue } lassign $vfsconfig vfstail appname target_kit_type @@ -738,11 +888,13 @@ proc ::punkboot::lib::mapvfs_kit_outputs {model rtfolder sourcefolder} { # Match user-supplied kit names against mapvfs_kit_outputs records. # Accepts the base kit name (punk91) or the platform artifact name (punk91.exe, -# name.kit); matching is case-insensitive on windows. Returns a dict: +# name.kit); matching is case-insensitive when the HOST filesystem is (a name +# the user types is a host-side convenience - a cygwin-family host is as +# case-insensitive as a native windows one, whatever it targets). Returns a dict: # selected matched records (kit-output order, deduplicated) # unknown supplied names that matched nothing proc ::punkboot::lib::mapvfs_match_outputs {kit_outputs names} { - set on_windows [expr {"windows" eq $::tcl_platform(platform)}] + set on_windows [expr {[info exists ::punkboot::host_windows] ? $::punkboot::host_windows : ("windows" eq $::tcl_platform(platform))}] set selected_idx [list] set unknown [list] foreach name $names { @@ -918,6 +1070,21 @@ set startdir [pwd] set bootsupport_module_paths [list] set bootsupport_library_paths [list] set this_platform_generic [punkboot::lib::platform_punk] ;#normalized punkshell platform-dir name (punk::platform canon) +# -- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- +#G-122 host/target split. $this_platform_generic is the HOST canon and stays the +#key for everything that describes THIS interpreter (bootsupport/vendorlib +#platform dirs below, diagnostics). What the build EMITS is keyed by target: +# host_platform - canon of the driving tclsh (same value, named for clarity) +# target_platform - default target for kit outputs; a cygwin-family host +# (msys2/cygwin runtime on windows) builds win32 kits. +# Per-entry mapvfs.config declarations override it per runtime. +# host_windows - host process/filesystem semantics are windows-like +# (native windows OR cygwin family): drives copy commands, +# path case comparison, prompt behaviour - NOT artifact naming. +set ::punkboot::host_platform $this_platform_generic +set ::punkboot::target_platform [punkboot::lib::platform_target_default $this_platform_generic] +set ::punkboot::host_windows [expr {[punkboot::lib::platform_process_family $this_platform_generic] eq "windows"}] +# -- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- #we always create these lists in order of desired precedence. # - this is the same order when adding to auto_path - but will need to be reversed when using tcl:tm::add if {[file exists [file join $::punkboot::scriptfolder bootsupport]]} { @@ -931,6 +1098,17 @@ if {[file exists [file join $::punkboot::scriptfolder bootsupport]]} { lappend bootsupport_library_paths [file join $bootsupportdir lib] } else { puts stderr "No bootsupport dir for script [info script] at [file join $::punkboot::scriptfolder bootsupport]" + #G-122 host-personality diagnosis: a cygwin-family tclsh (msys2/cygwin runtime on + #windows) is a POSIX Tcl - 'C:/path/make.tcl' is not an absolute path to it, so + #'file normalize' silently prefixes the cwd and every derived path is wrong. The + #failure that follows (missing bootsupport, then an unresolvable project root) is + #unrecognisable without this note. Caller-side spelling, not something make.tcl can + #safely rewrite - name the script in the host's own path syntax. + if {[punkboot::lib::platform_is_cygwin_family $this_platform_generic] && [regexp {^([A-Za-z]):[/\\](.*)$} [info script] -> _cygdrive _cygrest]} { + puts stderr " NOTE: this tclsh is a $this_platform_generic (msys2/cygwin-runtime) build - a POSIX Tcl, to which the windows-style path '[info script]' is NOT absolute, so it was resolved relative to the current directory." + puts stderr " Re-run naming the script in this shell's own path syntax: /[string tolower $_cygdrive]/[string map {\\ /} $_cygrest]" + unset -nocomplain _cygdrive _cygrest + } #lappend bootsupport_module_paths [file join $startdir bootsupport modules_tcl$::tclmajorv] #lappend bootsupport_module_paths [file join $startdir bootsupport modules] #lappend bootsupport_library_paths [file join $startdir bootsupport lib_tcl$::tclmajorv/allplatforms] @@ -2086,12 +2264,14 @@ DIAGRAM 1b - THIN-LAYOUT SYNC (part of modules/libs/packages/bakehouse runs) DIAGRAM 2 - KIT ASSEMBLY DETAIL (the 'make.tcl bake' stage; incl. vfslibs phase) ---------------------------------------------------------------------------- - src/runtime/mapvfs.config which vfs folder pairs with which runtime -> kit name + src/runtime/mapvfs.config which vfs folder pairs with which runtime -> kit name, + and optionally the TARGET PLATFORM of that runtime [K9] 'make.tcl bakelist ?kitname ...?' reports the configured kit outputs (runtime/vfs presence + deployed state); 'make.tcl bake ?kitname ...?' bakes only the named kits (bare bake = all; unknown names error before any build) - src/runtime/.exe bare Tcl runtime (tclkit / tclsfe build) + bin/runtime// bare Tcl runtime (tclkit / tclsfe / suite build), in the + store tier of the TARGET platform it builds kits for [K9] src/vfs/_vfscommon.vfs/ common payload (built modules + libs, main boot support) src/vfs/.vfs/ per-kit overlay (main.tcl, kit-only modules/config) @@ -2102,7 +2282,9 @@ DIAGRAM 2 - KIT ASSEMBLY DETAIL (the 'make.tcl bake' stage; incl. vfslibs phase) v src/_build/.exe.vfs | - | tcl::zipfs::mkimg (vfs image appended to runtime exe) + | zip image appended to the runtime exe: tcl::zipfs::mkimg, + | or (driving tcl without zipfs, e.g 8.6) raw-runtime split + | + punk::zip::mkzip + concatenation - both mount the same v src/_build/.exe | @@ -2160,6 +2342,18 @@ KEY / NOTES untracked: modules*/ lib*/ at projectroot, src/_build/, bin/.exe (bin/ scripts/tools are tracked; kit executables are not) +[K9] HOST vs TARGET platform (G-122). What a bake EMITS is keyed by the target + platform, never by the driving tclsh's personality: which bin/runtime/ + holds the runtime, whether the runtime file and the built kit carry .exe, and + whether the pre-deploy sweep uses tasklist/taskkill or ps/kill. The default + target is this host's own platform - except a cygwin-family host (an + msys2/cygwin-runtime tclsh, which reports platform 'unix' on windows), which + targets win32. A mapvfs.config entry may declare its own target as a 4th + element and is then read from that platform's tier, named with its executable + convention, and (being unrunnable here) skipped by the process sweep. + 'make.tcl check' prints the derivation; 'make.tcl bakelist' shows per-kit + targets, with a target= note on rows that are not on the default target. + MAINTENANCE (agents take note) ---------------------------------------------------------------------------- This text is embedded in make.tcl (::punkboot::workflow_text). When the build @@ -2339,7 +2533,8 @@ if {[info exists ::env(PUNKBOOT_PLAIN)] && [string is true -strict $::env(PUNKBO List the kit outputs configured in src/runtime/mapvfs.config (the same parsed mapping the bake subcommand consumes): kit name, kit type (kit|zip|zipcat...), runtime with presence in the runtime - store (/bin/runtime/), vfs folder, and the + store of that kit's TARGET platform + (/bin/runtime/), vfs folder, and the deployed state of /bin/ vs the src/_build build product: current - deployed copy is identical to the build product @@ -2349,9 +2544,12 @@ if {[info exists ::env(PUNKBOOT_PLAIN)] && [string is true -strict $::env(PUNKBO A trailing notes column flags anomalies (runtime=missing, vfs=missing, rtrev=r when the runtime working copy is materialized from an older revision than a -r artifact beside - it). + it) and marks cross-target entries (target= on any kit + whose mapvfs entry declares a target other than this host's + default - 'make.tcl check' shows that default). kitname arguments filter the report to the named entries (per-kit - detail, with resolved paths). Reporting only - never builds." + detail, with resolved store tier, target and paths). Reporting + only - never builds." vfslibs " Propagate declared vendored platform-library packages into kit vfs @@ -2394,8 +2592,11 @@ if {[info exists ::env(PUNKBOOT_PLAIN)] && [string is true -strict $::env(PUNKBO " Show module/library paths and any potentially problematic packages for running this script. - Also reports bootsupport staleness and the src provenance status (what - the build/promotion commands would do)." + Also reports the colour policy, the host/target platform derivation + (this tclsh's platform canon vs the default kit target, its runtime + store tier, executable suffix and process tooling), bootsupport + staleness and the src provenance status (what the build/promotion + commands would do)." projectversion " Advisory check: verify CHANGELOG.md matches punkproject.toml and warn @@ -2527,7 +2728,7 @@ if {[info exists ::env(PUNKBOOT_PLAIN)] && [string is true -strict $::env(PUNKBO #still works when the mapping cannot be parsed here) stays authoritative. variable KITNAME_CHOICES [list] catch { - set _mapmodel [punkboot::lib::mapvfs_parse [file join $::punkboot::scriptfolder runtime mapvfs.config] "" $::punkboot::scriptfolder] + set _mapmodel [punkboot::lib::mapvfs_parse [file join $::punkboot::scriptfolder runtime mapvfs.config] "" $::punkboot::scriptfolder $::punkboot::target_platform] foreach _rec [punkboot::lib::mapvfs_kit_outputs $_mapmodel "" $::punkboot::scriptfolder] { lappend KITNAME_CHOICES [dict get $_rec kitname] } @@ -2955,23 +3156,15 @@ set ::punkboot::bake_selected_targetkits [list] ;#platform artifact names (e.g p set ::punkboot::bake_selected_vfs [list] ;#vfs folder tails the selected kits use set ::punkboot::bake_selected_runtimes [list] ;#unsuffixed runtime names the selected kits wrap if {$::punkboot::command eq "bake" && [llength $::punkboot::opt_kitnames]} { - switch -glob -- $this_platform_generic { - macosx-* { - set _rt_os_arch macosx - } - default { - set _rt_os_arch $this_platform_generic - } - } - set _rtfolder $binfolder/runtime/$_rt_os_arch - set _mapmodel [punkboot::lib::mapvfs_parse $sourcefolder/runtime/mapvfs.config $_rtfolder $sourcefolder] + set _rtbase $binfolder/runtime + set _mapmodel [punkboot::lib::mapvfs_parse $sourcefolder/runtime/mapvfs.config $_rtbase $sourcefolder $::punkboot::target_platform] if {[llength [dict get $_mapmodel configerrors]]} { foreach _errline [dict get $_mapmodel configerrors] { puts stderr $_errline } exit 3 } - set _kit_outputs [punkboot::lib::mapvfs_kit_outputs $_mapmodel $_rtfolder $sourcefolder] + set _kit_outputs [punkboot::lib::mapvfs_kit_outputs $_mapmodel $_rtbase $sourcefolder] set _matchinfo [punkboot::lib::mapvfs_match_outputs $_kit_outputs $::punkboot::opt_kitnames] if {[llength [dict get $_matchinfo unknown]]} { puts stderr "make.tcl bake: unknown kit name(s): [join [dict get $_matchinfo unknown] {, }] - nothing built" @@ -2995,7 +3188,7 @@ if {$::punkboot::command eq "bake" && [llength $::punkboot::opt_kitnames]} { exit 1 } if {[dict get $_rec runtime] ne "-" && ![dict get $_rec runtime_present]} { - puts stderr "make.tcl bake: kit '[dict get $_rec kitname]' cannot be built - runtime [dict get $_rec runtime_file] not present in bin/runtime/$_rt_os_arch - nothing built" + puts stderr "make.tcl bake: kit '[dict get $_rec kitname]' cannot be built - runtime [dict get $_rec runtime_file] not present in bin/runtime/[dict get $_rec store_tier] (target [dict get $_rec target]) - nothing built" exit 1 } } @@ -3010,7 +3203,7 @@ if {$::punkboot::command eq "bake" && [llength $::punkboot::opt_kitnames]} { } } puts stdout "selective bake (G-121): [join $::punkboot::bake_selected_kitnames {, }]" - unset -nocomplain _rt_os_arch _rtfolder _mapmodel _kit_outputs _matchinfo _rec _r _errline + unset -nocomplain _rtbase _mapmodel _kit_outputs _matchinfo _rec _r _errline } # ---------------------------------------- @@ -3140,6 +3333,15 @@ if {$::punkboot::command eq "check"} { puts stdout "colour policy (G-113): mode=$::punkboot::colour_mode colour_disabled=$::punk::console::colour_disabled ansistrip=[expr {[llength $::punkboot::ansistrip_pushed] ? [join $::punkboot::ansistrip_pushed +] : 0}]" puts stdout " (precedence: NO_COLOR > PUNK_FORCE_COLOR/FORCE_COLOR > stdout tty probe incl. the 8.6 windows console-encoding signature; piped/unknown output is fully ESC-stripped per channel unless forced - utf-16 console channels are never wrapped)" puts stdout $sep + #G-122 host/target platform split - the self-diagnosis for "why is my kit set + #addressed the way it is". One stable single line; the columns are key=value. + set _chk_suffix [punkboot::lib::platform_exe_suffix $::punkboot::target_platform] + if {$_chk_suffix eq ""} {set _chk_suffix "(none)"} + puts stdout "platform (G-122): host=$::punkboot::host_platform target=$::punkboot::target_platform store=bin/runtime/[punkboot::lib::platform_store_tier $::punkboot::target_platform] exe-suffix=$_chk_suffix process-tooling=[punkboot::lib::platform_process_family $::punkboot::target_platform]" + puts stdout " (host = this tclsh's punk::platform canon, from tcl_platform(os)='$::tcl_platform(os)' platform='$::tcl_platform(platform)'; target = default kit target - a cygwin-family host (msys2/cygwin runtime on windows) bakes win32 kits)" + puts stdout " (mapvfs.config entries may declare their own target platform as a 4th element - 'make.tcl bakelist' shows each kit's target, store tier and artifact name)" + unset -nocomplain _chk_suffix + puts stdout $sep puts stdout "module/library checks - paths from bootsupport only" puts stdout $sep puts stdout "- tcl::tm::list" @@ -3493,18 +3695,17 @@ if {$::punkboot::command eq "bakelist"} { #grepping; expected state sits in the columns, anomalies surface in a trailing #sparse notes column (key=value tags). Kit name arguments filter the report and #add a per-kit detail block (resolved paths/sizes/mtimes). - switch -glob -- $this_platform_generic { - macosx-* { - set rt_os_arch macosx - } - default { - set rt_os_arch $this_platform_generic - } - } - set rtfolder $binfolder/runtime/$rt_os_arch + #G-122: store addressing is keyed by TARGET platform. The default target is + #this host's own platform (a cygwin-family host targets win32) and mapvfs + #entries may declare their own - so each row resolves its own tier under + #bin/runtime and the report names the tiers it consulted. + set rt_default_target $::punkboot::target_platform + set rt_os_arch [punkboot::lib::platform_store_tier $rt_default_target] + set rtbase $binfolder/runtime + set rtfolder $rtbase/$rt_os_arch set buildfolder $sourcefolder/_build set mapfile $sourcefolder/runtime/mapvfs.config - set mapmodel [punkboot::lib::mapvfs_parse $mapfile $rtfolder $sourcefolder] + set mapmodel [punkboot::lib::mapvfs_parse $mapfile $rtbase $sourcefolder $rt_default_target] foreach errline [dict get $mapmodel configerrors] { puts stderr $errline } @@ -3515,11 +3716,32 @@ if {$::punkboot::command eq "bakelist"} { puts stdout "no kit mapping config at $mapfile - no kit outputs configured" exit 0 } - set kit_outputs [punkboot::lib::mapvfs_kit_outputs $mapmodel $rtfolder $sourcefolder] + set kit_outputs [punkboot::lib::mapvfs_kit_outputs $mapmodel $rtbase $sourcefolder] if {![llength $kit_outputs]} { puts stdout "no kit outputs configured in $mapfile" exit 0 } + #Loud store-tier guard: a nonexistent store folder makes every row addressing it + #read runtime=missing for a reason that is not per-entry. Reported per tier + #actually referenced, since entries may target different platforms (G-122). + set report_tiers [list] + foreach rec $kit_outputs { + if {[dict get $rec runtime] eq "-"} {continue} + set t [dict get $rec store_tier] + if {$t ni $report_tiers} {lappend report_tiers $t} + } + set rt_store_missing [expr {![file isdirectory $rtfolder]}] + if {$rt_os_arch ni $report_tiers} {lappend report_tiers $rt_os_arch} + foreach t $report_tiers { + if {[file isdirectory $rtbase/$t]} {continue} + if {$t eq $rt_os_arch} { + puts stderr "WARNING: runtime store folder bin/runtime/$t does not exist - every configured runtime on the default target will read runtime=missing." + puts stderr " Default target '$rt_default_target' derives from THIS tclsh: tcl_platform(os)='$::tcl_platform(os)' tcl_platform(platform)='$::tcl_platform(platform)' -> host canon '$::punkboot::host_platform' -> target '$rt_default_target'." + puts stderr " Populate it with 'bin/punk-runtime.cmd' (or by hand), or declare a per-entry target platform in mapvfs.config for runtimes stored elsewhere." + } else { + puts stderr "WARNING: runtime store folder bin/runtime/$t does not exist - configured entries targeting $t will read runtime=missing." + } + } set reportset $kit_outputs if {[llength $::punkboot::opt_kitnames]} { set matchinfo [punkboot::lib::mapvfs_match_outputs $kit_outputs $::punkboot::opt_kitnames] @@ -3545,8 +3767,13 @@ if {$::punkboot::command eq "bakelist"} { if {![dict get $rec vfs_present]} { lappend notes vfs=missing } + if {[dict get $rec target] ne $rt_default_target} { + #cross-target entry: its runtime, store tier and artifact naming follow + #its own platform, not this host's default + lappend notes target=[dict get $rec target] + } if {$runtime ne "-" && [dict get $rec runtime_present]} { - set rtstale [punkboot::lib::runtime_materialization_check $rtfolder [dict get $rec runtime_file]] + set rtstale [punkboot::lib::runtime_materialization_check [dict get $rec runtime_dir] [dict get $rec runtime_file]] if {[dict size $rtstale]} { lappend notes rtrev=r[dict get $rtstale current_rev] vs src/_build/" + set rt_store_note [expr {$rt_store_missing ? " (FOLDER MISSING)" : ""}] + puts stdout "# mapvfs: src/runtime/mapvfs.config runtimes: bin/runtime/$rt_os_arch$rt_store_note deployed: bin/ vs src/_build/" set headline "" foreach h $headings w $widths { append headline [format "%-*s " $w $h] @@ -3594,10 +3822,11 @@ if {$::punkboot::command eq "bakelist"} { if {$runtime eq "-"} { puts stdout " runtime: (none - unwrapped .kit output)" } else { - puts stdout " runtime file: bin/runtime/$rt_os_arch/[dict get $rec runtime_file] [expr {[dict get $rec runtime_present] ? {(present)} : {(MISSING)}}]" + puts stdout " runtime file: bin/runtime/[dict get $rec store_tier]/[dict get $rec runtime_file] [expr {[dict get $rec runtime_present] ? {(present)} : {(MISSING)}}]" } puts stdout " vfs folder: src/vfs/[dict get $rec vfs] [expr {[dict get $rec vfs_present] ? {(present)} : {(MISSING)}}]" puts stdout " kit type: [dict get $rec kit_type]" + puts stdout " target: [dict get $rec target][expr {[dict get $rec target] eq $rt_default_target ? " (host default)" : " (declared in mapvfs.config)"}]" puts stdout " build product: src/_build/[dict get $rec targetkit] [punkboot::lib::bakelist_file_detail $buildfolder/[dict get $rec targetkit]]" puts stdout " deployed: bin/[dict get $rec targetkit] [punkboot::lib::bakelist_file_detail $binfolder/[dict get $rec targetkit]] state=[lindex $row 4]" } @@ -4706,28 +4935,40 @@ if {$::punkboot::command eq "bin"} { #set rtfolder $sourcefolder/runtime #AAA -switch -glob -- $this_platform_generic { - macosx-* { - #assuming universal binaries x86_64 and arm - set rt_os_arch macosx - } - default { - set rt_os_arch $this_platform_generic - } -} -set rtfolder $binfolder/runtime/$rt_os_arch +#G-122: the runtime store is addressed by TARGET platform, not by the driving +#tclsh's personality. $rt_default_target is this host's default target (a +#cygwin-family host targets win32; macOS collapses to the universal 'macosx' +#tier) and $rtfolder is that target's store. mapvfs.config entries may declare +#their own target platform - those runtimes live in their own tier under $rtbase +#and are resolved through $rtfolder_of below. +set rt_default_target $::punkboot::target_platform +set rt_os_arch [punkboot::lib::platform_store_tier $rt_default_target] +set rtbase $binfolder/runtime +set rtfolder $rtbase/$rt_os_arch +set rt_exe_suffix [punkboot::lib::platform_exe_suffix $rt_default_target] set rt_sourcefolder $sourcefolder/runtime ;#where our config lives -#review - when building kits for other platforms - it's unlikely runtime will be marked as executable - we should probably process all files in runtime folder except those with certain extensions +#G-122: derive the BUILDCOPY name from the runtime FILENAME rather than from the +#driving tclsh's platform. The copy is the same binary as its source, so it must +#keep the source's executable convention - an .exe stays an .exe whether a native +#windows tclsh, an msys/cygwin-runtime tclsh (platform 'unix', .exe files) or a +#cross-target host is driving the build. +#Mapping key for a runtime FILE name: the name as mapvfs.config spells it, i.e +#with a windows .exe suffix removed. Only .exe may be stripped - 'file rootname' +#would eat the last dotted segment of version-named runtimes such as tclsh9.0.5, +#which carry no suffix at all on non-windows targets (G-122). +proc ::punkboot::lib::runtime_mapkey {runtimefile} { + if {[string match -nocase *.exe $runtimefile]} { + return [string range $runtimefile 0 end-4] + } + return $runtimefile +} proc ::punkboot::lib::runtime_buildcopyname {runtime} { - if {$::tcl_platform(platform) eq "windows"} { - set rtname [file rootname [file tail $runtime]] - set bcname ${rtname}_BUILDCOPY.exe - } else { - set rtname [file tail $runtime] - set bcname ${rtname}_BUILDCOPY + set rttail [file tail $runtime] + if {[string match -nocase *.exe $rttail]} { + return "[file rootname $rttail]_BUILDCOPY.exe" } - return $bcname + return "${rttail}_BUILDCOPY" } #Local runtime-materialization staleness check (warn-only, no network). The kit @@ -4741,6 +4982,31 @@ proc ::punkboot::lib::runtime_buildcopyname {runtime} { #toml) and direct -r references stay silent; one warning per runtime per run. #G-121: the comparison core lives in ::punkboot::lib::runtime_materialization_check, #shared with the bakelist report's rtrev= note. +#G-122: run a command whose arguments must reach a NATIVE WINDOWS program +#verbatim. The msys2 runtime rewrites any argument that looks like an absolute +#posix path when it spawns a native windows executable, so 'taskkill /PID 1234' +#arrives as 'taskkill C:/msys64-root/PID 1234' - field-verified 2026-07-26: the +#pre-deploy process sweep found the running kit with tasklist and then could not +#kill it, failing the whole kit. MSYS2_ARG_CONV_EXCL=* is msys2's documented +#opt-out and is inert on every other host (real cygwin does not convert arguments +#at all), so one command spelling stays correct from every host personality. +proc ::punkboot::exec_nativeargs {cmdlist} { + set had [info exists ::env(MSYS2_ARG_CONV_EXCL)] + if {$had} { + set prev $::env(MSYS2_ARG_CONV_EXCL) + } + set ::env(MSYS2_ARG_CONV_EXCL) * + try { + return [exec {*}$cmdlist] + } finally { + if {$had} { + set ::env(MSYS2_ARG_CONV_EXCL) $prev + } else { + unset -nocomplain ::env(MSYS2_ARG_CONV_EXCL) + } + } +} + set ::punkboot::runtime_rev_warned [dict create] proc ::punkboot::runtime_materialization_warning {rtfolder runtime_fullname} { if {[dict exists $::punkboot::runtime_rev_warned $runtime_fullname]} { @@ -4757,16 +5023,75 @@ proc ::punkboot::runtime_materialization_warning {rtfolder runtime_fullname} { return } +# -- --- --- --- --- --- --- --- --- --- +#load mapvfs.config file (if any) in runtime folder to map runtimes to vfs folders. +#build a dict keyed on runtime executable name. +#If no mapfile (or no mapfile entry for that runtime) - the runtime will be paired with a matching .vfs folder in src folder. e.g punk.exe to src/punk.vfs +#If vfs folders or runtime executables which are explicitly listed in the mapfile don't exist - warn on stderr - but continue. if such nonexistants found; prompt user for whether to continue or abort. +#G-121: parsing lives in ::punkboot::lib::mapvfs_parse - the shared model consumed here, +#by 'bakelist' and by the selective-bake name resolution (never the file format directly). +#G-122: parsed BEFORE runtime discovery, because the mapping is what says which +#store tiers this bake must look in - an entry declaring a target platform other +#than the host default addresses its own bin/runtime/. +set mapfile $rt_sourcefolder/mapvfs.config +set mapmodel [punkboot::lib::mapvfs_parse $mapfile $rtbase $sourcefolder $rt_default_target] +foreach warnline [dict get $mapmodel warnings] { + puts stderr $warnline +} +if {[llength [dict get $mapmodel configerrors]]} { + foreach errline [dict get $mapmodel configerrors] { + puts stderr $errline + } + exit 3 +} +set runtime_vfs_map [dict get $mapmodel runtime_vfs_map] +set vfs_runtime_map [dict get $mapmodel vfs_runtime_map] +set runtime_target [dict get $mapmodel runtime_target] +set missing [dict get $mapmodel missing] +if {[llength $missing]} { + puts stderr "WARNING [llength $missing] missing items from $mapfile. (TODO - prompt user to continue/abort)" + foreach m $missing { + puts stderr " $m" + } + puts stderr "continuing..." +} +# -- --- --- --- --- --- --- --- --- --- +puts "-- runtime_vfs_map --" +punk::lib::pdict runtime_vfs_map +puts "---------------------" +puts "-- vfs_runtime_map--" +punk::lib::pdict vfs_runtime_map +puts "---------------------" + +# -- --- --- --- --- --- --- --- --- --- +#Runtime discovery. The default-target store is scanned wholesale (a runtime +#there needs no mapping entry - it can pair with a same-named .vfs folder); +#cross-target runtimes are only picked up when the mapping names them, since +#scanning every tier would pull in runtimes nothing asked to be built. +#$rtfolder_of resolves a discovered runtime FILE name to the store folder it +#came from (runtime_target_of gives its target platform) - every later store +#address, suffix and process-tooling decision goes through these rather than +#assuming the driving tclsh's platform (G-122). set runtimes [list] +set rtfolder_of [dict create] +proc ::punkboot::lib::runtime_target_of {rtname} { + #target platform of a runtime name (unsuffixed), from the parsed mapping + upvar #0 runtime_target runtime_target rt_default_target rt_default_target + if {[info exists runtime_target] && [dict exists $runtime_target $rtname]} { + return [dict get $runtime_target $rtname] + } + return $rt_default_target +} if {[file dirname [info nameofexecutable]] eq $rtfolder && [file isdirectory [info nameofexecutable]]} { #info name will only appear to be a directory when the runtime is a tclkit that is mounted at the same path as the physical file. #This is not a problem for zip based kits. #fix so that we can find and use a runtime tclkit that is currently in use as the runtime that is running this script. # - when the current runtime is using a runtime in the rtfolder that is a tclkit, it is mounted at the same path as the physical file and so appears to Tcl as if it's a directory. # - use external filesystem tools to make a copy of the file - if {$::tcl_platform(platform) eq "windows"} { + #(copy tooling is a HOST concern - the running executable is by definition a host binary) + if {$::punkboot::host_windows} { #exec cmd /c copy [info nameofexecutable] $rtfolder/[file rootname [file tail [info nameofexecutable]]]_BUILDCOPY.exe - exec cmd /c copy [file nativename [info nameofexecutable]] [file nativename $rtfolder/[punkboot::lib::runtime_buildcopyname [info nameofexecutable]]] + ::punkboot::exec_nativeargs [list cmd /c copy [file nativename [info nameofexecutable]] [file nativename $rtfolder/[punkboot::lib::runtime_buildcopyname [info nameofexecutable]]]] } else { #exec cp [info nameofexecutable] $rtfolder/[file tail [info nameofexecutable]]_BUILDCOPY exec cp [info nameofexecutable] $rtfolder/[punkboot::lib::runtime_buildcopyname [info nameofexecutable]] @@ -4774,12 +5099,14 @@ if {[file dirname [info nameofexecutable]] eq $rtfolder && [file isdirectory [in #we need to add the base name of the current exe to the runtimes list because the next section will look for files in the runtime folder #and won't find the mounted runtime since it's mounted as a directory. lappend runtimes [file tail [info nameofexecutable]] + dict set rtfolder_of [file tail [info nameofexecutable]] $rtfolder } set rtfolder_files [glob -nocomplain -dir $rtfolder -types {f} -tail *] set exclusions {.config .md .ico .txt .doc .pdf .htm .html} ;#we don't encourage other files in runtime folder aside from mapvfs.config - but lets ignore some common possibilities lappend exclusions .zip .7z .pea .bz2 .tar .gz .tgz .z .xz ;#don't allow archives to directly be treated as runtimes - tolerate presence but require user to unpack or rename if they're to be used as runtimes lappend exclusions .tail ;#result of running sdx mksplit on a kit - in theory the .head could be used - review/test +lappend exclusions .toml ;#G-103 runtime artifact metadata sits beside its runtime foreach f $rtfolder_files { if {[string match "*_BUILDCOPY*" $f]} { continue @@ -4789,11 +5116,43 @@ foreach f $rtfolder_files { } if {$f ni $runtimes} { lappend runtimes $f + dict set rtfolder_of $f $rtfolder } } +#cross-target runtimes named by the mapping (their own tier, their own suffix) +set cross_target_tiers [list] +dict for {_rtname _rttarget} $runtime_target { + if {$_rtname eq "-" || $_rttarget eq $rt_default_target} { + continue + } + set _tier [punkboot::lib::platform_store_tier $_rttarget] + set _dir $rtbase/$_tier + set _file $_rtname[punkboot::lib::platform_exe_suffix $_rttarget] + if {$_tier ni $cross_target_tiers} { + lappend cross_target_tiers $_tier + } + if {![file exists [file join $_dir $_file]]} { + continue ;#absent cross-target runtime already warned by the parse + } + if {[dict exists $rtfolder_of $_file]} { + puts stderr "WARNING: runtime file $_file is present in more than one store tier ([dict get $rtfolder_of $_file] and $_dir) - using the first found. Rename one of them to build both." + continue + } + lappend runtimes $_file + dict set rtfolder_of $_file $_dir +} +if {[llength $cross_target_tiers]} { + puts stdout "cross-target runtime tiers in use (mapvfs.config target platforms): [join $cross_target_tiers {, }]" +} +unset -nocomplain _rtname _rttarget _tier _dir _file if {![llength $runtimes]} { puts stderr "No executable runtimes found in $rtfolder - unable to build any .vfs folders into executables." puts stderr "Add runtimes to $rtfolder if required" + if {![file isdirectory $rtfolder]} { + #same store-tier self-diagnosis as bakelist + puts stderr "NOTE: that runtime store folder does not exist. Default target '$rt_default_target' derives from THIS tclsh: tcl_platform(os)='$::tcl_platform(os)' tcl_platform(platform)='$::tcl_platform(platform)' -> host canon '$::punkboot::host_platform' -> target '$rt_default_target'." + puts stderr " Populate it with 'bin/punk-runtime.cmd' (or by hand), or declare a per-entry target platform in mapvfs.config for runtimes stored elsewhere." + } #todo - don't exit - it is valid to use runtime of - to just build a .kit/.zipkit ? exit 0 } @@ -4801,10 +5160,10 @@ if {![llength $runtimes]} { #other runtimes get no BUILDCOPY refresh, no capability probe and no punkcheck events. if {[llength $::punkboot::bake_selected_kitnames]} { set runtimes [lmap _rtfile $runtimes { - set _rtkey $_rtfile - if {[string match *.exe $_rtkey]} { - set _rtkey [string range $_rtkey 0 end-4] - } + #NOTE: only an .exe suffix may be stripped to recover the mapping key - + #'file rootname' would eat the last dotted version segment of names like + #tclsh9.0.5 (suffixless on non-windows targets). + set _rtkey [punkboot::lib::runtime_mapkey $_rtfile] if {$_rtkey ni $::punkboot::bake_selected_runtimes} { continue } @@ -4814,41 +5173,6 @@ if {[llength $::punkboot::bake_selected_kitnames]} { } #previous have_sdx test location -# -- --- --- --- --- --- --- --- --- --- -#load mapvfs.config file (if any) in runtime folder to map runtimes to vfs folders. -#build a dict keyed on runtime executable name. -#If no mapfile (or no mapfile entry for that runtime) - the runtime will be paired with a matching .vfs folder in src folder. e.g punk.exe to src/punk.vfs -#If vfs folders or runtime executables which are explicitly listed in the mapfile don't exist - warn on stderr - but continue. if such nonexistants found; prompt user for whether to continue or abort. -#G-121: parsing lives in ::punkboot::lib::mapvfs_parse - the shared model consumed here, -#by 'bakelist' and by the selective-bake name resolution (never the file format directly). -set mapfile $rt_sourcefolder/mapvfs.config -set mapmodel [punkboot::lib::mapvfs_parse $mapfile $rtfolder $sourcefolder] -foreach warnline [dict get $mapmodel warnings] { - puts stderr $warnline -} -if {[llength [dict get $mapmodel configerrors]]} { - foreach errline [dict get $mapmodel configerrors] { - puts stderr $errline - } - exit 3 -} -set runtime_vfs_map [dict get $mapmodel runtime_vfs_map] -set vfs_runtime_map [dict get $mapmodel vfs_runtime_map] -set missing [dict get $mapmodel missing] -if {[llength $missing]} { - puts stderr "WARNING [llength $missing] missing items from $mapfile. (TODO - prompt user to continue/abort)" - foreach m $missing { - puts stderr " $m" - } - puts stderr "continuing..." -} -# -- --- --- --- --- --- --- --- --- --- -puts "-- runtime_vfs_map --" -punk::lib::pdict runtime_vfs_map -puts "---------------------" -puts "-- vfs_runtime_map--" -punk::lib::pdict vfs_runtime_map -puts "---------------------" #only test the runtime capabilities for runtimes that are actually in our map @@ -4897,27 +5221,34 @@ foreach runtime [dict keys $runtime_vfs_map] { exit 0 } - #temp - #build for current platform only for now - if {![file exists $rtfolder/$runtime]} { - dict set runtime_caps $runtime exitcode -1 error "find-fail $rtfolder/$runtime" + #G-122: address the runtime in ITS target's store tier, with ITS target's + #executable suffix. (Before the split this probed a suffixless name in the + #host-derived folder, so on windows every runtime silently reported + #find-fail and no capability was ever learned.) + set rt_target [punkboot::lib::runtime_target_of $runtime] + set rt_dir $rtbase/[punkboot::lib::platform_store_tier $rt_target] + set rt_file $runtime[punkboot::lib::platform_exe_suffix $rt_target] + if {![file exists $rt_dir/$rt_file]} { + dict set runtime_caps $runtime exitcode -1 error "find-fail $rt_dir/$rt_file" + continue + } + if {[punkboot::lib::platform_process_family $rt_target] ne [punkboot::lib::platform_process_family $::punkboot::host_platform]} { + #cross-target runtime: this host cannot execute it, so its capabilities + #stay unknown (the kit machinery falls back to trying the configured type) + dict set runtime_caps $runtime exitcode -1 error "cross-target ($rt_target) - not executable on this host" continue } #invoke can fail if runtime not an executable file for the current platform - if {[file type $rtfolder/$runtime] eq "directory"} { + if {[file type $rt_dir/$rt_file] eq "directory"} { #assume it's a mounted tclkit (because tcl kits mount in same place as their underlying path and mask the executable as a directory) #use the BUILDCOPY created above - REVIEW - if {$::tcl_platform(platform) eq "windows"} { - set useruntime [file rootname $runtime]_BUILDCOPY.exe - } else { - set useruntime ${runtime}_BUILDCOPY - } + set useruntime [punkboot::lib::runtime_buildcopyname $rt_file] } else { - set useruntime $runtime + set useruntime $rt_file } if {![catch { - lassign [punk::lib::invoke [list $rtfolder/$useruntime <<$capscript]] stdout stderr exitcode + lassign [punk::lib::invoke [list $rt_dir/$useruntime <<$capscript]] stdout stderr exitcode } errM]} { if {$exitcode == 0} { dict set runtime_caps $runtime $stdout @@ -4959,7 +5290,7 @@ set vfs_folder_changes [dict create] ;#cache whether each .vfs folder has change # -- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- #make build copy for all runtimes - not just those in the map - because even without a mapvfs.config file entry we build an exe for a runtime that matches .vfs folder name - REVIEW. foreach runtimefile $runtimes { - #runtimefile e.g tclkit86bi.exe on windows tclkit86bi on other platforms + #runtimefile e.g tclkit86bi.exe for a win32 target, tclkit86bi for a posix one #sdx *may* be pointed to use the runtime we use to build the kit, or the user may manually use this runtime if they don't have tclsh #sdx will complain if the same runtime is used for the shell as is used in the -runtime argument - so we make a copy (REVIEW) @@ -4967,23 +5298,19 @@ foreach runtimefile $runtimes { # file copy $rtfolder/$runtimefile $buildfolder/buildruntime.exe #} - #---------------------- - #todo - #temp - only build for current platform - need changes to runtime/mapvfs.config etc - if {![file exists $rtfolder/$runtimefile]} { - puts stderr " >> skipping $runtimefile not present in $rtfolder (wrong platform?) x-platform kit build TODO." + #G-122: each runtime is read from the tier its target platform names, not + #from one host-derived folder. Build copies keep the flat build_ name - + #the mapping cannot hold one runtime name in two tiers, so they cannot collide. + set rt_dir [dict get $rtfolder_of $runtimefile] + if {![file exists $rt_dir/$runtimefile]} { + puts stderr " >> skipping $runtimefile not present in $rt_dir" continue } - #---------------------- - if {[file type $rtfolder/$runtimefile] eq "directory"} { + if {[file type $rt_dir/$runtimefile] eq "directory"} { #assume it's a mounted tclkit (because tcl kits mount in same place as their underlying path and mask the executable) #use the BUILDCOPY created above - REVIEW - if {$::tcl_platform(platform) eq "windows"} { - set useruntimefile [file tail $runtimefile]_BUILDCOPY.exe - } else { - set useruntimefile ${runtimefile}_BUILDCOPY - } + set useruntimefile [punkboot::lib::runtime_buildcopyname $runtimefile] } else { set useruntimefile $runtimefile } @@ -4994,11 +5321,11 @@ foreach runtimefile $runtimes { }] #---------- set installer [punkcheck::installtrack new $installername $basedir/.punkcheck] - $installer set_source_target $rtfolder $buildfolder + $installer set_source_target $rt_dir $buildfolder set event [$installer start_event $config] $event targetset_init INSTALL $buildfolder/build_$runtimefile - #$event targetset_addsource $rtfolder/$runtimefile - $event targetset_addsource $rtfolder/$useruntimefile ;#possibly the _BUILDCOPY in the runtime folder (created for when mounted in current executable as a tclkit) + #$event targetset_addsource $rt_dir/$runtimefile + $event targetset_addsource $rt_dir/$useruntimefile ;#possibly the _BUILDCOPY in the runtime folder (created for when mounted in current executable as a tclkit) $event targetset_addsource $buildfolder/build_$runtimefile ;#self as source for change detection #---------- @@ -5010,9 +5337,9 @@ foreach runtimefile $runtimes { $event targetset_started # -- --- --- --- --- --- #This is the full runtime - *possibly* with some sort of vfs attached. - puts stdout "Copying runtime (as is) from $rtfolder/$useruntimefile to $buildfolder/build_$runtimefile" + puts stdout "Copying runtime (as is) from $rt_dir/$useruntimefile to $buildfolder/build_$runtimefile" if {[catch { - file copy -force $rtfolder/$useruntimefile $buildfolder/build_$runtimefile ;#becomes building_runtime + file copy -force $rt_dir/$useruntimefile $buildfolder/build_$runtimefile ;#becomes building_runtime } errM]} { puts stderr " >> copy runtime to $buildfolder/build_$runtimefile FAILED" $event targetset_end FAILED @@ -5050,12 +5377,16 @@ if {$sdxpath eq ""} { #see if the runtime (or any runtime *for current platform*) has tclkit in the name or has_starkit(?) and use that #todo - review #picking first found for now - fix + #G-122: only a runtime this HOST can execute can drive sdx - the + #capability probe already skipped cross-target runtimes, so a + #has_starkit record means the runtime ran here. dict for {rtname rtprops} $runtime_caps { if {[dict exists $rtprops has_starkit] && [dict get $rtprops has_starkit]} { - set testpath $rtfolder/$rtname + set _rt_target [punkboot::lib::runtime_target_of $rtname] + set testpath $rtbase/[punkboot::lib::platform_store_tier $_rt_target]/$rtname[punkboot::lib::platform_exe_suffix $_rt_target] if {[file exists $testpath]} { if {[file type $testpath] eq "directory"} { - set alt_tclkitpath $rtfolder/[punkboot::lib::runtime_buildcopyname $testpath] + set alt_tclkitpath [file dirname $testpath]/[punkboot::lib::runtime_buildcopyname $testpath] } else { set alt_tclkitpath $testpath } @@ -5246,6 +5577,43 @@ proc vfs_startup_script_warning {vfsfolder} { } return "vfs folder $vfsfolder has NO STARTUP SCRIPT: no root main.tcl and no root fauxlink resolving to main.tcl with an existing .tcl target - kits built from it will have no boot script (build proceeding anyway)$detail" } +#Cat-style zipkit assembly: a raw (vfs-free) runtime followed by a freshly built +#zip, concatenated. This is the zipfs-less assembly path for BOTH zip-type kits +#(G-122): Tcl's zipfs locates an image by scanning back from the END of the file +#and reads the central directory with ARCHIVE-START-RELATIVE offsets, so a plain +#concatenation mounts. 'zipfs mkimg's file-relative offset rewrite is one +#convention, not a mounting requirement - the zipcat kit type is the in-repo +#proof, and this lets a driving tcl with no zipfs at all (any 8.6) assemble zip +#kits. The zip itself is written by tcl::zipfs::mkzip when the driving tcl has +#it, else by punk::zip::mkzip (pure Tcl). +#Returns the number of zip bytes appended. +proc ::punkboot::assemble_zipcat_image {raw_runtime wrapvfs outfile zipfile} { + file copy -force $raw_runtime $outfile + #runtime in runtime folder may not have write perm set - ensure the copy does as we need to append + catch {exec chmod +w $outfile} + file delete $zipfile + if {[info commands ::tcl::zipfs] ne ""} { + puts stdout "tcl::zipfs::mkzip $zipfile $wrapvfs $wrapvfs" + ::tcl::zipfs::mkzip $zipfile $wrapvfs $wrapvfs + } else { + puts stdout "punk::zip::mkzip -directory $wrapvfs -base $wrapvfs $zipfile *" + package require punk::zip + punk::zip::mkzip -directory $wrapvfs -base $wrapvfs $zipfile * + } + puts stderr "concatenating zip to executable.." + set fdout [open $outfile a] + chan conf $fdout -translation binary + puts stderr "runtime bytes: [tell $fdout]" + set fdzip [open $zipfile r] + chan conf $fdzip -translation binary + set zipbytes [fcopy $fdzip $fdout] + close $fdzip + puts stderr "zip bytes: $zipbytes" + puts stderr "exezip bytes: [tell $fdout]" + close $fdout + return $zipbytes +} + set startdir [pwd] puts stdout "Found [llength $vfs_tails] .vfs folders - checking each for executables that may need to be built" cd [file dirname $buildfolder] @@ -5275,24 +5643,26 @@ foreach vfstail $vfs_tails { }] + #G-122: a runtime's filename and store tier come from ITS target platform. + #The '-' pseudo-runtime (unwrapped .kit output) carries no file at all. set runtimes [list] if {[dict exists $vfs_runtime_map $vfstail]} { - #set runtimes [dict get $vfs_runtime_map $vfstail] + #set runtimes [dict get $vfs_runtime_map $vfstail] #runtimes in vfs_runtime_map_vfs dict are unsuffixed (.exe stripped or was not present) set applist [dict get $vfs_runtime_map $vfstail] - if {"windows" eq $::tcl_platform(platform)} { - foreach rt_app $applist { - set rt [lindex $rt_app 0] - if {$rt ne "-" && [file exists $rtfolder/$rt.exe]} { - lappend runtimes $rt.exe - } + foreach rt_app $applist { + set rt [lindex $rt_app 0] + if {$rt eq "-"} { + #the '-' (runtime-less .kit) pseudo-runtime has never reached this + #list - both historical branches filtered it out - so the loop + #below's '-' handling is currently unreachable. Left as-is: + #enabling it is a separate, untested change. + continue } - } else { - foreach rt_app $applist { - set rt [lindex $rt_app 0] - if {[file exists $rtfolder/$rt]} { - lappend runtimes $rt - } + set rt_target [punkboot::lib::runtime_target_of $rt] + set rt_file $rt[punkboot::lib::platform_exe_suffix $rt_target] + if {[file exists $rtbase/[punkboot::lib::platform_store_tier $rt_target]/$rt_file]} { + lappend runtimes $rt_file } } } else { @@ -5301,18 +5671,14 @@ foreach vfstail $vfs_tails { #but conversely, adding an extra entry shouldn't stop default builds that used to run.. set matchrt [file rootname [file tail $vfstail]] ;#e.g project.vfs -> project if {![dict exists $runtime_vfs_map $matchrt]} { - if {"windows" eq $::tcl_platform(platform)} { - if {[file exists $rtfolder/$matchrt.exe]} { - lappend runtimes $matchrt.exe - } - } else { - if {[file exists $rtfolder/$matchrt]} { - lappend runtimes $matchrt - } + #unmapped: the default-target store is the only place a name-match is looked for + if {[file exists $rtfolder/$matchrt$rt_exe_suffix]} { + lappend runtimes $matchrt$rt_exe_suffix } } } - #assert $runtimes is a list of executable names suffixed with .exe if on windows - whether or not specified with .exe in the mapvfs.config + #assert $runtimes is a list of runtime filenames as they exist in their target's store tier + #(suffixed with .exe for windows-family targets, whether or not mapvfs.config spelled it that way) puts " vfs: $vfstail runtimes to process ([llength $runtimes]): $runtimes" if {[llength $runtimes]} { @@ -5326,13 +5692,17 @@ foreach vfstail $vfs_tails { #todo - non kit based - zipkit? # $runtimes may now include a dash entry "-" (from mapvfs.config file) foreach runtime_fullname $runtimes { - set rtname [file rootname $runtime_fullname] + set rtname [punkboot::lib::runtime_mapkey $runtime_fullname] if {[llength $::punkboot::bake_selected_kitnames] && $rtname ni $::punkboot::bake_selected_runtimes} { continue ;#G-121 selective bake - no requested kit wraps this runtime } + #G-122: everything this iteration emits is keyed by the runtime's TARGET + set rt_target [punkboot::lib::runtime_target_of $rtname] + set rt_tierdir $rtbase/[punkboot::lib::platform_store_tier $rt_target] + set rt_suffix [punkboot::lib::platform_exe_suffix $rt_target] #rtname of "-" indicates build a kit without a runtime if {$runtime_fullname ne "-"} { - ::punkboot::runtime_materialization_warning $rtfolder $runtime_fullname + ::punkboot::runtime_materialization_warning $rt_tierdir $runtime_fullname } #first configured runtime will be the one to use the same name as .vfs folder for output executable. Additional runtimes on this .vfs will need to suffix the runtime name to disambiguate. @@ -5356,11 +5726,7 @@ foreach vfstail $vfs_tails { if {$rtname eq "-"} { set targetkit $appname.kit } else { - if {$::tcl_platform(platform) eq "windows"} { - set targetkit ${appname}.exe - } else { - set targetkit $appname - } + set targetkit ${appname}$rt_suffix if {$targetkit in $exe_names_seen} { #duplicate appname configured? #todo - consider creating as ${appname}(2) etc? @@ -5384,9 +5750,10 @@ foreach vfstail $vfs_tails { #cannot replace a running image anyway. Informational/update subcommands are #unaffected (they exit before the kit machinery), so driving make.tcl from a built #punk executable remains supported for everything except rebuilding that executable. + #(a HOST filesystem comparison - case rules follow the host, not the target) set self_exe [file normalize [info nameofexecutable]] set self_deploy_target [file normalize $binfolder/$targetkit] - if {$::tcl_platform(platform) eq "windows"} { + if {$::punkboot::host_windows} { set is_self_build [string equal -nocase $self_exe $self_deploy_target] } else { set is_self_build [string equal $self_exe $self_deploy_target] @@ -5451,15 +5818,23 @@ foreach vfstail $vfs_tails { #to consider: - allow specifying runtime kits as if they are vfs folders in the normal xxx.vfs list - and autodetect and extract #would need to detect UPX, cookfs,zipfs,tclkit set rtmountpoint "" + #Test the capability VALUE, not just its presence: a probe that ran + #reports has_starkit/has_cookfs 0 for a plain zipfs runtime, and + #trying to unwrap such a runtime as a kit ends in sdx mksplit + #failing and REPLACING raw_runtime with the un-split original - + #which silently yields a zip kit with two zips and no tcl_library. + #(Latent until G-122 made the capability probe actually run on + #windows: before that every probe reported find-fail, so this list + #was always empty.) set caps [dict get $runtime_caps $rtname] set source_kit_caps [list] - if {[dict exists $caps has_zipfs]} { + if {[dict exists $caps has_zipfs] && [dict get $caps has_zipfs]} { lappend source_kit_caps zip } - if {[dict exists $caps has_starkit]} { + if {[dict exists $caps has_starkit] && [dict get $caps has_starkit]} { lappend source_kit_caps kit } - if {[dict exists $caps has_cookfs]} { + if {[dict exists $caps has_cookfs] && [dict get $caps has_cookfs]} { lappend source_kit_caps cookfs } @@ -5576,11 +5951,18 @@ foreach vfstail $vfs_tails { zipfile::decode::open $extractedzipfile set archiveinfo [zipfile::decode::archive] zipfile::decode::unzip $archiveinfo $extractedzipfolder - }]} { + } extracterr]} { set extraction_done 1 set extract_kit_type $extract_kit_try #todo - verify that init.tcl etc are present? merge_over $extractedzipfolder $targetvfs + } else { + #Do not swallow the reason: without the runtime's own + #zip contents the built kit has no tcl_library. Note + #zipfile::decode is a system tcllib package, not a + #vendored one - a tclsh without tcllib (e.g the msys2 + #/usr/bin/tclsh8.6) cannot take this path at all. + puts stderr "zipfs-less extraction of $runtime_fullname FAILED: $extracterr" } } @@ -5668,10 +6050,11 @@ foreach vfstail $vfs_tails { if {!$extraction_done} { #TODO: if not extracted - use a default tcl_library for patchlevel and platform? - puts stderr "--------------------------------------------" - puts stderr "[punkboot::sgr 31]WARNING: No extraction done from runtime $runtime_fullname[punkboot::sgr]" - puts stderr "If no init.tcl provided in the vfs at the proper location (containing init.tcl) - the resulting kit will probably not initialise properly!" - puts stderr "--------------------------------------------" + #Recapped BUILD-WARNING (not just an inline note): the build continues + #and produces an artifact, but unless the .vfs itself supplies a + #tcl_library that artifact cannot boot - and this line would otherwise + #scroll away behind thousands of merge lines. + ::punkboot::print_build_warnings [list "No extraction done from runtime $runtime_fullname (tried: $target_kit_type[expr {[llength $source_kit_caps] ? " + runtime caps $source_kit_caps" : ""}]) - kit $targetkit gets no tcl_library from the runtime and will not initialise unless src/vfs/$vfstail supplies one"] file mkdir $targetvfs } @@ -5756,9 +6139,17 @@ foreach vfstail $vfs_tails { puts stderr "RUNTIME capabilities unknown. Unsure if zip supported. trying anyway.." } } - #note - as at 2024-08 - there is some discussion about the interface to mkimg - it is considered unstable (may change to -option value syntax) - puts stderr "calling: tcl::zipfs::mkimg $buildfolder/$vfsname.new $wrapvfs $wrapvfs \"\" $raw_runtime" - tcl::zipfs::mkimg $buildfolder/$vfsname.new $wrapvfs $wrapvfs "" $raw_runtime + if {[info commands ::tcl::zipfs::mkimg] ne ""} { + #note - as at 2024-08 - there is some discussion about the interface to mkimg - it is considered unstable (may change to -option value syntax) + puts stderr "calling: tcl::zipfs::mkimg $buildfolder/$vfsname.new $wrapvfs $wrapvfs \"\" $raw_runtime" + tcl::zipfs::mkimg $buildfolder/$vfsname.new $wrapvfs $wrapvfs "" $raw_runtime + } else { + #G-122: the driving tcl has no zipfs (8.6) - assemble the same + #kind of image by concatenation. The result mounts on any + #zipfs-capable runtime (archive-start-relative offsets). + puts stderr "WARNING: tcl shell '[info nameofexecutable]' has no zipfs - assembling zip kit $targetkit by concatenation (raw runtime + mkzip)" + ::punkboot::assemble_zipcat_image $raw_runtime $wrapvfs $buildfolder/$vfsname.new $buildfolder/$vfsname.zip + } } result ]} { set failmsg "zipfs mkimg failed with msg: $result" puts stderr "tcl::zipfs::mkimg $targetkit failed" @@ -5768,7 +6159,7 @@ foreach vfstail $vfs_tails { $vfs_installer destroy continue } else { - puts stdout "ok - finished tcl::zipfs::mkimg" + puts stdout "ok - finished zip image assembly" set separator [string repeat = 40] puts stdout $separator puts stdout $result @@ -5794,32 +6185,7 @@ foreach vfstail $vfs_tails { } #'archive' based zip offsets - editable in 7z,peazip - file copy $raw_runtime $buildfolder/$vfsname.new - #runtime in runtime folder may not have write perm set - ensure the copy does as we need to append - catch {exec chmod +w $buildfolder/$vfsname.new} - file delete $buildfolder/$vfsname.zip - - if {[info commands ::tcl::zipfs] ne ""} { - puts stdout "tcl::zipfs::mkzip $buildfolder/$vfsname.zip $wrapvfs $wrapvfs" - ::tcl::zipfs::mkzip $buildfolder/$vfsname.zip $wrapvfs $wrapvfs - } else { - puts stdout "punk::zip::mkzip -directory $wrapvfs -base $wrapvfs $buildfolder/$vfsname.zip *" - package require punk::zip - punk::zip::mkzip -directory $wrapvfs -base $wrapvfs $buildfolder/$vfsname.zip * - } - - - puts stderr "concatenating executable to zip.." - set fdout [open $buildfolder/$vfsname.new a] - chan conf $fdout -translation binary - puts stderr "runtime bytes: [tell $fdout]" - set fdzip [open $buildfolder/$vfsname.zip r] - chan conf $fdzip -translation binary - set zipbytes [fcopy $fdzip $fdout] - close $fdzip - puts stderr "zip bytes: $zipbytes" - puts stderr "exezip bytes: [tell $fdout]" - close $fdout + ::punkboot::assemble_zipcat_image $raw_runtime $wrapvfs $buildfolder/$vfsname.new $buildfolder/$vfsname.zip } result ]} { set failmsg "creating zipcat image failed with msg: $result" puts stderr "creating image (zipcat) $targetkit failed" @@ -5942,18 +6308,34 @@ foreach vfstail $vfs_tails { #still - seems to be under 1s on a 2018 era i9 running in 2025 - which is not a big problem in this context. #using a filter in tasklist or get-command can make things a little faster - if {$::tcl_platform(platform) eq "windows"} { + #G-122: the tooling follows the TARGET, not the driving tclsh. + #A win32 kit is a windows process whatever baked it - and msys 'ps' + #sees only msys-descendant processes, so an msys-hosted build using + #'ps' would report a natively-launched kit as not running and then + #fail to replace it. tasklist/taskkill exec fine from an msys tclsh. + #When target and host process worlds differ (a cross-target bake) no + #local process can be holding the artifact - skip the sweep rather + #than run tooling that cannot see the target's processes anyway. + set target_proc_family [punkboot::lib::platform_process_family $rt_target] + set host_proc_family [punkboot::lib::platform_process_family $::punkboot::host_platform] + if {$target_proc_family eq "windows"} { set pscmd "tasklist" + set pid_field 1 } else { set pscmd "ps" + set pid_field 0 + } + set sweep_applicable [expr {$target_proc_family eq $host_proc_family}] + if {!$sweep_applicable} { + puts stdout " (skipping running-process sweep for $targetkit - target $rt_target processes are not visible to this $::punkboot::host_platform host)" } - #killing process doesn't apply to .kit build - if {$rtname ne "-"} { + #killing process doesn't apply to .kit build + if {$rtname ne "-" && $sweep_applicable} { #exec $pscmd | grep $targetkit ;#wrong - fails on at least some OSes because grep itself is in the list #exec ps | grep $targetkit | grep -v grep ;#exclude grep process if {![catch { - exec $pscmd + ::punkboot::exec_nativeargs [list $pscmd] } ps_lines]} { #we will process without grep for for now - may not be avail on windows? set still_running_lines [list] @@ -5966,11 +6348,7 @@ foreach vfstail $vfs_tails { #itself (e.g a copy of the target executable driving this build from #a path other than the deploy target, which the self-build guard #above doesn't catch). - if {$::tcl_platform(platform) eq "windows"} { - set ln_pid [lindex $ln 1] - } else { - set ln_pid [lindex $ln 0] - } + set ln_pid [lindex $ln $pid_field] if {$ln_pid eq [pid]} {continue} lappend still_running_lines $ln } @@ -5983,15 +6361,14 @@ foreach vfstail $vfs_tails { foreach ln $still_running_lines { puts stdout " $ln" - if {$::tcl_platform(platform) eq "windows"} { - set pid [lindex $ln 1] + set pid [lindex $ln $pid_field] + if {$target_proc_family eq "windows"} { if {$forcekill} { set killcmd [list taskkill /F /PID $pid] } else { set killcmd [list taskkill /PID $pid] } } else { - set pid [lindex $ln 0] #review! if {$forcekill} { set killcmd [list kill -9 $pid] @@ -6001,7 +6378,7 @@ foreach vfstail $vfs_tails { } puts stdout " pid: $pid (attempting to kill now using '$killcmd')" if {[catch { - exec {*}$killcmd + ::punkboot::exec_nativeargs $killcmd } errMsg]} { puts stderr "$killcmd returned an error:" puts stderr $errMsg diff --git a/src/project_layouts/vendor/punk/project-0.1/src/bootsupport/modules/include_modules.config b/src/project_layouts/vendor/punk/project-0.1/src/bootsupport/modules/include_modules.config index fac8ef64..95c2f4dc 100644 --- a/src/project_layouts/vendor/punk/project-0.1/src/bootsupport/modules/include_modules.config +++ b/src/project_layouts/vendor/punk/project-0.1/src/bootsupport/modules/include_modules.config @@ -104,6 +104,7 @@ set bootsupport_modules [list\ modules punk::ns\ modules punk::overlay\ modules punk::path\ + modules punk::platform\ modules punk::packagepreference\ modules punk::repl\ modules punk::repl::codethread\ diff --git a/src/project_layouts/vendor/punk/project-0.1/src/make.tcl b/src/project_layouts/vendor/punk/project-0.1/src/make.tcl index 83930762..7d08906c 100644 --- a/src/project_layouts/vendor/punk/project-0.1/src/make.tcl +++ b/src/project_layouts/vendor/punk/project-0.1/src/make.tcl @@ -444,16 +444,29 @@ namespace eval ::punkboot::lib { return "${plat}-${cpu}" } + proc platform_normalize_os {os} { + #INLINE COPY of punk::platform::normalize_os - see platform_punk below. + switch -glob -- [string tolower $os] { + macos {return macosx} + msys - msys_nt-* {return msys} + mingw32 - mingw64 - mingw_nt-* - mingw32_nt-* - mingw64_nt-* {return msys} + ucrt64 - ucrt64_nt-* {return msys} + clang32 - clang64 - clangarm64 - clang32_nt-* - clang64_nt-* - clangarm64_nt-* {return msys} + cygwin_nt-* {return cygwin} + } + return $os + } proc platform_punk {} { #canonical punkshell platform-dir name: platform_generic normalized. #INLINE COPY of punk::platform::normalize (src/modules/punk/platform-*.tm; #'help platforms' documents the canon) - the boot stage cannot package #require, so keep this mapping in sync with that module: - #amd64->x86_64, aarch64->arm64, macos->macosx, macosx arm->arm64. + #amd64->x86_64, aarch64->arm64, macos->macosx, macosx arm->arm64, + #msys2 family (mingw64/mingw32/ucrt64/clang*) -> msys, CYGWIN_NT -> cygwin. set parts [split [platform_generic] -] set cpu [lindex $parts end] set os [join [lrange $parts 0 end-1] -] - if {$os eq "macos"} {set os macosx} + set os [platform_normalize_os $os] switch -- $cpu { amd64 {set cpu x86_64} aarch64 {set cpu arm64} @@ -462,6 +475,93 @@ namespace eval ::punkboot::lib { return "${os}-${cpu}" } + # -- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- + # G-122 host/target platform split. + # platform_punk answers "what is THIS tclsh" (the HOST canon). Everything a + # build EMITS is a separate question keyed by TARGET: which bin/runtime/ + # store holds the runtime, whether artifacts carry .exe, and which process + # tooling can see/kill a running artifact. The two diverge for the whole + # cygwin family: an msys2/cygwin-runtime tclsh on windows reports + # tcl_platform(platform) 'unix' and canonizes as msys-x86_64/cygwin-x86_64, + # yet the kits it bakes are ordinary win32-x86_64 .exe binaries that only + # tasklist/taskkill can manage (msys 'ps' sees only msys-descendant + # processes). Host process semantics - copy commands, path handling, prompts, + # filesystem case rules - stay keyed to the host. + # -- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- + + #os token of a canonical platform-dir name (win32-x86_64 -> win32; macosx -> macosx) + proc platform_os {platform} { + set parts [split $platform -] + if {[llength $parts] < 2} { + return $platform + } + return [join [lrange $parts 0 end-1] -] + } + #cpu token of a canonical platform-dir name ("" for single-token names like macosx) + proc platform_cpu {platform} { + set parts [split $platform -] + if {[llength $parts] < 2} { + return "" + } + return [lindex $parts end] + } + #posix-personality runtimes hosted on windows (msys2 / cygwin) + proc platform_is_cygwin_family {platform} { + return [expr {[platform_os $platform] in {msys cygwin}}] + } + #Windows-family platforms: native win32 AND the cygwin-family personalities. + #They share what matters to a build: executables are .exe files and running + #processes are windows processes (tasklist/taskkill), whatever tcl_platform says. + proc platform_is_windows_family {platform} { + return [expr {[platform_os $platform] in {win32 msys cygwin}}] + } + #Default TARGET platform for a host canon. A cygwin-family host builds native + #windows kits (there is no separate msys kit lineage - third-party msys + #runtimes are addressed by an explicit per-entry target instead). + proc platform_target_default {hostplatform} { + if {[platform_is_cygwin_family $hostplatform]} { + set cpu [platform_cpu $hostplatform] + if {$cpu eq ""} {set cpu x86_64} + return "win32-$cpu" + } + return $hostplatform + } + #bin/runtime store tier folder name for a target: macOS runtimes are universal + #(multi-arch) binaries kept in one 'macosx' folder - see punk::platform. + proc platform_store_tier {target} { + if {[string match "macosx-*" $target]} { + return macosx + } + return $target + } + #executable filename suffix artifacts for this target carry + proc platform_exe_suffix {target} { + if {[platform_is_windows_family $target]} { + return ".exe" + } + return "" + } + #which process tooling manages a running artifact of this platform + proc platform_process_family {platform} { + if {[platform_is_windows_family $platform]} { + return windows + } + return posix + } + #A target platform name must look like a platform-dir name so it can address a + #store tier. Returns "" when acceptable, else a reason string. + proc platform_name_problem {platform} { + if {$platform eq ""} { + return "empty" + } + if {[regexp {[\\/:*?\"<>|]} $platform]} { + return "not a platform-dir name (contains a path/wildcard character)" + } + if {[llength [split $platform -]] < 2 && $platform ne "macosx"} { + return "not an - platform-dir name ('help platforms' lists the canon)" + } + return "" + } } #------------------------------------------------------------------------------ @@ -533,27 +633,35 @@ proc ::punkboot::lib::bootsupport_prompt_yesno {prompt} { # directly, so a config-format change (G-024 toml direction) swaps in underneath # by reimplementing mapvfs_parse alone. # -# mapvfs_parse: pure parse - no output, no exits. Returns a dict: +# mapvfs_parse: pure parse - no output, no exits. Every store address, presence +# probe and artifact suffix it computes is keyed by the entry's TARGET platform +# (G-122), never by the driving tclsh's personality: $rtbase is the store ROOT +# (bin/runtime) and each entry addresses $rtbase/. Returns a dict: # exists 0|1 (mapfile present) # mapfile path as supplied -# runtime_vfs_map runtime -> raw vfs_specs as listed (appname/type may be "") -# vfs_runtime_map vfstail -> list of {runtime appname type} (defaults applied; -# only vfs folders that exist on disk - matching the historical -# inline parse the kit loop was built around) +# default_target target platform for entries that declare none +# runtime_vfs_map runtime -> raw vfs_specs as listed (appname/type/target may be "") +# vfs_runtime_map vfstail -> list of {runtime appname type target} (defaults +# applied; only vfs folders that exist on disk - matching the +# historical inline parse the kit loop was built around) +# runtime_target runtime -> resolved target platform (all runtimes, mapped order) # missing names of referenced-but-absent runtimes/vfs folders # warnings warning lines in encounter order, ready to print verbatim # (missing-item lines carry their historical WARNING: prefix) # badentries subset of warnings: malformed entry lines (these have no kit # record to carry them, so bakelist prints them; missing-item # warnings are instead surfaced as per-row state there) -# configerrors fatal config problems (duplicate runtime line; caller decides -# whether to abort - the bake path keeps its historical exit 3) -proc ::punkboot::lib::mapvfs_parse {mapfile rtfolder sourcefolder} { +# configerrors fatal config problems (duplicate runtime line; unusable or +# self-contradicting target platform; caller decides whether to +# abort - the bake path keeps its historical exit 3) +proc ::punkboot::lib::mapvfs_parse {mapfile rtbase sourcefolder default_target} { set model [dict create\ exists 0\ mapfile $mapfile\ + default_target $default_target\ runtime_vfs_map [dict create]\ vfs_runtime_map [dict create]\ + runtime_target [dict create]\ missing [list]\ warnings [list]\ badentries [list]\ @@ -570,6 +678,7 @@ proc ::punkboot::lib::mapvfs_parse {mapfile rtfolder sourcefolder} { set mapdata [string map [list \r\n \n] $mapdata] set runtime_vfs_map [dict create] set vfs_runtime_map [dict create] + set runtime_target [dict create] set missing [list] set warnings [list] set badentries [list] @@ -585,19 +694,45 @@ proc ::punkboot::lib::mapvfs_parse {mapfile rtfolder sourcefolder} { #drop windows .exe suffix so same config can work cross platform - extension will be re-added if necessary later set runtime [string range $runtime 0 end-4] } - if {$runtime ne "-"} { - set runtime_test $runtime - if {"windows" eq $::tcl_platform(platform)} { - set runtime_test $runtime.exe + #Target platform is declared per kit-config entry (4th element) but is a + #property of the RUNTIME: one binary lives in one store tier and has one + #executable convention. Entries on a line may therefore repeat the + #declaration but must not disagree. + set line_target "" + foreach vfsconfig $vfs_specs { + if {[llength $vfsconfig] < 4} { + continue + } + set declared [lindex $vfsconfig 3] + if {$declared eq ""} { + continue } - if {![file exists [file join $rtfolder $runtime_test]]} { - lappend warnings "WARNING: Missing runtime file $rtfolder/$runtime_test (line in mapvfs.config: $ln)" + if {$line_target eq ""} { + set problem [platform_name_problem $declared] + if {$problem ne ""} { + lappend configerrors "CONFIG FILE ERROR. runtime: $runtime declares target platform '$declared' - $problem." + continue + } + set line_target $declared + } elseif {$declared ne $line_target} { + lappend configerrors "CONFIG FILE ERROR. runtime: $runtime declares conflicting target platforms ('$line_target' and '$declared') in $mapfile - a runtime has one target." + } + } + if {$line_target eq ""} { + set line_target $default_target + } + set rtdir [file join $rtbase [platform_store_tier $line_target]] + set exe_suffix [platform_exe_suffix $line_target] + if {$runtime ne "-"} { + set runtime_test $runtime$exe_suffix + if {![file exists [file join $rtdir $runtime_test]]} { + lappend warnings "WARNING: Missing runtime file $rtdir/$runtime_test (line in mapvfs.config: $ln)" lappend missing $runtime } } foreach vfsconfig $vfs_specs { switch -- [llength $vfsconfig] { - 1 - 2 - 3 { + 1 - 2 - 3 - 4 { lassign $vfsconfig vfstail appname target_kit_type if {![file isdirectory [file join $sourcefolder vfs $vfstail]]} { lappend warnings "WARNING: Missing vfs folder [file join $sourcefolder vfs $vfstail] specified in mapvfs.config for runtime $runtime" @@ -606,11 +741,11 @@ proc ::punkboot::lib::mapvfs_parse {mapfile rtfolder sourcefolder} { if {$appname eq ""} { set appname [file rootname $vfstail] } - dict lappend vfs_runtime_map $vfstail [list $runtime $appname $target_kit_type] + dict lappend vfs_runtime_map $vfstail [list $runtime $appname $target_kit_type $line_target] } } default { - set badline "bad entry in mapvfs.config - expected each entry after the runtime name to be of length 1 or length 2. got: $vfsconfig ([llength $vfsconfig])" + set badline "bad entry in mapvfs.config - expected each entry after the runtime name to be a list of 1 to 4 elements {vfsfolder ?kitname? ?kittype? ?targetplatform?}. got: $vfsconfig ([llength $vfsconfig])" lappend warnings $badline lappend badentries $badline } @@ -620,10 +755,12 @@ proc ::punkboot::lib::mapvfs_parse {mapfile rtfolder sourcefolder} { lappend configerrors "CONFIG FILE ERROR. runtime: $runtime was specified more than once in $mapfile." } else { dict set runtime_vfs_map $runtime $vfs_specs + dict set runtime_target $runtime $line_target } } dict set model runtime_vfs_map $runtime_vfs_map dict set model vfs_runtime_map $vfs_runtime_map + dict set model runtime_target $runtime_target dict set model missing $missing dict set model warnings $warnings dict set model badentries $badentries @@ -635,24 +772,30 @@ proc ::punkboot::lib::mapvfs_parse {mapfile rtfolder sourcefolder} { # Replicates the kit loop's iteration + naming (vfs_tails glob order with map # extras appended, per-vfs runtime order from the mapping, appname defaulting to # the vfs rootname, kit_type defaulting to 'kit', '-' runtime -> .kit, -# windows .exe suffix, duplicate appname -> _) so listed names +# TARGET .exe suffix, duplicate appname -> _) so listed names # match what a bake produces. Entries whose vfs folder is missing on disk never # reach the kit loop - they are appended after the buildable set so 'bakelist' # can surface them as broken config. Duplicate-name disambiguation assumes the # config keeps appnames unique (as the kit loop itself effectively does). -# Returns a list of record dicts: +# $rtbase is the store ROOT (bin/runtime): each record resolves its own tier +# under it, so entries targeting different platforms report their own store +# presence (G-122). Returns a list of record dicts: # kitname base output name (selection key, e.g punk91) # targetkit platform artifact name (e.g punk91.exe, or .kit for '-') # kit_type kit|zip|zipcat|cookfs... as configured (defaulted to kit) # runtime runtime name as configured (unsuffixed), or "-" -# runtime_file platform runtime filename ("" for "-") +# runtime_file target runtime filename ("" for "-") # runtime_present 1|0 (1 for "-": no runtime needed) +# runtime_dir store tier folder holding the runtime ("" for "-") +# target target platform (e.g win32-x86_64) +# store_tier bin/runtime tier folder name for the target # vfs vfs folder tail (e.g punk9wintk903.vfs) # vfs_present 1|0 -proc ::punkboot::lib::mapvfs_kit_outputs {model rtfolder sourcefolder} { +proc ::punkboot::lib::mapvfs_kit_outputs {model rtbase sourcefolder} { set runtime_vfs_map [dict get $model runtime_vfs_map] set vfs_runtime_map [dict get $model vfs_runtime_map] - set on_windows [expr {"windows" eq $::tcl_platform(platform)}] + set runtime_target [dict get $model runtime_target] + set default_target [dict get $model default_target] set vfs_tails [glob -nocomplain -dir $sourcefolder/vfs -types d -tail *.vfs] dict for {vfstail -} $vfs_runtime_map { if {$vfstail ni $vfs_tails} { @@ -662,33 +805,37 @@ proc ::punkboot::lib::mapvfs_kit_outputs {model rtfolder sourcefolder} { set records [list] set exe_names_seen [list] set record_pair {{rtname vfstail appname target_kit_type} { - upvar 1 records records exe_names_seen exe_names_seen rtfolder rtfolder on_windows on_windows sourcefolder sourcefolder + upvar 1 records records exe_names_seen exe_names_seen rtbase rtbase sourcefolder sourcefolder\ + runtime_target runtime_target default_target default_target if {$appname eq ""} { set appname [file rootname $vfstail] } if {$target_kit_type eq ""} { set target_kit_type "kit" } + if {[dict exists $runtime_target $rtname]} { + set target [dict get $runtime_target $rtname] + } else { + set target $default_target + } + set store_tier [::punkboot::lib::platform_store_tier $target] if {$rtname eq "-"} { set targetkit $appname.kit set kitname $appname set runtime_file "" + set runtime_dir "" set runtime_present 1 ;#no runtime needed } else { - if {$on_windows} { - set runtime_file $rtname.exe - set targetkit ${appname}.exe - } else { - set runtime_file $rtname - set targetkit $appname - } + set runtime_dir [file join $rtbase $store_tier] + set runtime_file $rtname[::punkboot::lib::platform_exe_suffix $target] + set targetkit ${appname}[::punkboot::lib::platform_exe_suffix $target] set kitname $appname if {$targetkit in $exe_names_seen} { #duplicate appname configured - kit loop disambiguates the same way set targetkit ${appname}_$rtname set kitname ${appname}_$rtname } - set runtime_present [file exists [file join $rtfolder $runtime_file]] + set runtime_present [file exists [file join $runtime_dir $runtime_file]] } lappend exe_names_seen $targetkit lappend records [dict create\ @@ -698,6 +845,9 @@ proc ::punkboot::lib::mapvfs_kit_outputs {model rtfolder sourcefolder} { runtime $rtname\ runtime_file $runtime_file\ runtime_present $runtime_present\ + runtime_dir $runtime_dir\ + target $target\ + store_tier $store_tier\ vfs $vfstail\ vfs_present [file isdirectory [file join $sourcefolder vfs $vfstail]]\ ] @@ -723,7 +873,7 @@ proc ::punkboot::lib::mapvfs_kit_outputs {model rtfolder sourcefolder} { #entries whose vfs folder is missing (excluded from vfs_runtime_map by the parse) dict for {rtname vfs_specs} $runtime_vfs_map { foreach vfsconfig $vfs_specs { - if {[llength $vfsconfig] < 1 || [llength $vfsconfig] > 3} { + if {[llength $vfsconfig] < 1 || [llength $vfsconfig] > 4} { continue } lassign $vfsconfig vfstail appname target_kit_type @@ -738,11 +888,13 @@ proc ::punkboot::lib::mapvfs_kit_outputs {model rtfolder sourcefolder} { # Match user-supplied kit names against mapvfs_kit_outputs records. # Accepts the base kit name (punk91) or the platform artifact name (punk91.exe, -# name.kit); matching is case-insensitive on windows. Returns a dict: +# name.kit); matching is case-insensitive when the HOST filesystem is (a name +# the user types is a host-side convenience - a cygwin-family host is as +# case-insensitive as a native windows one, whatever it targets). Returns a dict: # selected matched records (kit-output order, deduplicated) # unknown supplied names that matched nothing proc ::punkboot::lib::mapvfs_match_outputs {kit_outputs names} { - set on_windows [expr {"windows" eq $::tcl_platform(platform)}] + set on_windows [expr {[info exists ::punkboot::host_windows] ? $::punkboot::host_windows : ("windows" eq $::tcl_platform(platform))}] set selected_idx [list] set unknown [list] foreach name $names { @@ -918,6 +1070,21 @@ set startdir [pwd] set bootsupport_module_paths [list] set bootsupport_library_paths [list] set this_platform_generic [punkboot::lib::platform_punk] ;#normalized punkshell platform-dir name (punk::platform canon) +# -- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- +#G-122 host/target split. $this_platform_generic is the HOST canon and stays the +#key for everything that describes THIS interpreter (bootsupport/vendorlib +#platform dirs below, diagnostics). What the build EMITS is keyed by target: +# host_platform - canon of the driving tclsh (same value, named for clarity) +# target_platform - default target for kit outputs; a cygwin-family host +# (msys2/cygwin runtime on windows) builds win32 kits. +# Per-entry mapvfs.config declarations override it per runtime. +# host_windows - host process/filesystem semantics are windows-like +# (native windows OR cygwin family): drives copy commands, +# path case comparison, prompt behaviour - NOT artifact naming. +set ::punkboot::host_platform $this_platform_generic +set ::punkboot::target_platform [punkboot::lib::platform_target_default $this_platform_generic] +set ::punkboot::host_windows [expr {[punkboot::lib::platform_process_family $this_platform_generic] eq "windows"}] +# -- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- #we always create these lists in order of desired precedence. # - this is the same order when adding to auto_path - but will need to be reversed when using tcl:tm::add if {[file exists [file join $::punkboot::scriptfolder bootsupport]]} { @@ -931,6 +1098,17 @@ if {[file exists [file join $::punkboot::scriptfolder bootsupport]]} { lappend bootsupport_library_paths [file join $bootsupportdir lib] } else { puts stderr "No bootsupport dir for script [info script] at [file join $::punkboot::scriptfolder bootsupport]" + #G-122 host-personality diagnosis: a cygwin-family tclsh (msys2/cygwin runtime on + #windows) is a POSIX Tcl - 'C:/path/make.tcl' is not an absolute path to it, so + #'file normalize' silently prefixes the cwd and every derived path is wrong. The + #failure that follows (missing bootsupport, then an unresolvable project root) is + #unrecognisable without this note. Caller-side spelling, not something make.tcl can + #safely rewrite - name the script in the host's own path syntax. + if {[punkboot::lib::platform_is_cygwin_family $this_platform_generic] && [regexp {^([A-Za-z]):[/\\](.*)$} [info script] -> _cygdrive _cygrest]} { + puts stderr " NOTE: this tclsh is a $this_platform_generic (msys2/cygwin-runtime) build - a POSIX Tcl, to which the windows-style path '[info script]' is NOT absolute, so it was resolved relative to the current directory." + puts stderr " Re-run naming the script in this shell's own path syntax: /[string tolower $_cygdrive]/[string map {\\ /} $_cygrest]" + unset -nocomplain _cygdrive _cygrest + } #lappend bootsupport_module_paths [file join $startdir bootsupport modules_tcl$::tclmajorv] #lappend bootsupport_module_paths [file join $startdir bootsupport modules] #lappend bootsupport_library_paths [file join $startdir bootsupport lib_tcl$::tclmajorv/allplatforms] @@ -2086,12 +2264,14 @@ DIAGRAM 1b - THIN-LAYOUT SYNC (part of modules/libs/packages/bakehouse runs) DIAGRAM 2 - KIT ASSEMBLY DETAIL (the 'make.tcl bake' stage; incl. vfslibs phase) ---------------------------------------------------------------------------- - src/runtime/mapvfs.config which vfs folder pairs with which runtime -> kit name + src/runtime/mapvfs.config which vfs folder pairs with which runtime -> kit name, + and optionally the TARGET PLATFORM of that runtime [K9] 'make.tcl bakelist ?kitname ...?' reports the configured kit outputs (runtime/vfs presence + deployed state); 'make.tcl bake ?kitname ...?' bakes only the named kits (bare bake = all; unknown names error before any build) - src/runtime/.exe bare Tcl runtime (tclkit / tclsfe build) + bin/runtime// bare Tcl runtime (tclkit / tclsfe / suite build), in the + store tier of the TARGET platform it builds kits for [K9] src/vfs/_vfscommon.vfs/ common payload (built modules + libs, main boot support) src/vfs/.vfs/ per-kit overlay (main.tcl, kit-only modules/config) @@ -2102,7 +2282,9 @@ DIAGRAM 2 - KIT ASSEMBLY DETAIL (the 'make.tcl bake' stage; incl. vfslibs phase) v src/_build/.exe.vfs | - | tcl::zipfs::mkimg (vfs image appended to runtime exe) + | zip image appended to the runtime exe: tcl::zipfs::mkimg, + | or (driving tcl without zipfs, e.g 8.6) raw-runtime split + | + punk::zip::mkzip + concatenation - both mount the same v src/_build/.exe | @@ -2160,6 +2342,18 @@ KEY / NOTES untracked: modules*/ lib*/ at projectroot, src/_build/, bin/.exe (bin/ scripts/tools are tracked; kit executables are not) +[K9] HOST vs TARGET platform (G-122). What a bake EMITS is keyed by the target + platform, never by the driving tclsh's personality: which bin/runtime/ + holds the runtime, whether the runtime file and the built kit carry .exe, and + whether the pre-deploy sweep uses tasklist/taskkill or ps/kill. The default + target is this host's own platform - except a cygwin-family host (an + msys2/cygwin-runtime tclsh, which reports platform 'unix' on windows), which + targets win32. A mapvfs.config entry may declare its own target as a 4th + element and is then read from that platform's tier, named with its executable + convention, and (being unrunnable here) skipped by the process sweep. + 'make.tcl check' prints the derivation; 'make.tcl bakelist' shows per-kit + targets, with a target= note on rows that are not on the default target. + MAINTENANCE (agents take note) ---------------------------------------------------------------------------- This text is embedded in make.tcl (::punkboot::workflow_text). When the build @@ -2339,7 +2533,8 @@ if {[info exists ::env(PUNKBOOT_PLAIN)] && [string is true -strict $::env(PUNKBO List the kit outputs configured in src/runtime/mapvfs.config (the same parsed mapping the bake subcommand consumes): kit name, kit type (kit|zip|zipcat...), runtime with presence in the runtime - store (/bin/runtime/), vfs folder, and the + store of that kit's TARGET platform + (/bin/runtime/), vfs folder, and the deployed state of /bin/ vs the src/_build build product: current - deployed copy is identical to the build product @@ -2349,9 +2544,12 @@ if {[info exists ::env(PUNKBOOT_PLAIN)] && [string is true -strict $::env(PUNKBO A trailing notes column flags anomalies (runtime=missing, vfs=missing, rtrev=r when the runtime working copy is materialized from an older revision than a -r artifact beside - it). + it) and marks cross-target entries (target= on any kit + whose mapvfs entry declares a target other than this host's + default - 'make.tcl check' shows that default). kitname arguments filter the report to the named entries (per-kit - detail, with resolved paths). Reporting only - never builds." + detail, with resolved store tier, target and paths). Reporting + only - never builds." vfslibs " Propagate declared vendored platform-library packages into kit vfs @@ -2394,8 +2592,11 @@ if {[info exists ::env(PUNKBOOT_PLAIN)] && [string is true -strict $::env(PUNKBO " Show module/library paths and any potentially problematic packages for running this script. - Also reports bootsupport staleness and the src provenance status (what - the build/promotion commands would do)." + Also reports the colour policy, the host/target platform derivation + (this tclsh's platform canon vs the default kit target, its runtime + store tier, executable suffix and process tooling), bootsupport + staleness and the src provenance status (what the build/promotion + commands would do)." projectversion " Advisory check: verify CHANGELOG.md matches punkproject.toml and warn @@ -2527,7 +2728,7 @@ if {[info exists ::env(PUNKBOOT_PLAIN)] && [string is true -strict $::env(PUNKBO #still works when the mapping cannot be parsed here) stays authoritative. variable KITNAME_CHOICES [list] catch { - set _mapmodel [punkboot::lib::mapvfs_parse [file join $::punkboot::scriptfolder runtime mapvfs.config] "" $::punkboot::scriptfolder] + set _mapmodel [punkboot::lib::mapvfs_parse [file join $::punkboot::scriptfolder runtime mapvfs.config] "" $::punkboot::scriptfolder $::punkboot::target_platform] foreach _rec [punkboot::lib::mapvfs_kit_outputs $_mapmodel "" $::punkboot::scriptfolder] { lappend KITNAME_CHOICES [dict get $_rec kitname] } @@ -2955,23 +3156,15 @@ set ::punkboot::bake_selected_targetkits [list] ;#platform artifact names (e.g p set ::punkboot::bake_selected_vfs [list] ;#vfs folder tails the selected kits use set ::punkboot::bake_selected_runtimes [list] ;#unsuffixed runtime names the selected kits wrap if {$::punkboot::command eq "bake" && [llength $::punkboot::opt_kitnames]} { - switch -glob -- $this_platform_generic { - macosx-* { - set _rt_os_arch macosx - } - default { - set _rt_os_arch $this_platform_generic - } - } - set _rtfolder $binfolder/runtime/$_rt_os_arch - set _mapmodel [punkboot::lib::mapvfs_parse $sourcefolder/runtime/mapvfs.config $_rtfolder $sourcefolder] + set _rtbase $binfolder/runtime + set _mapmodel [punkboot::lib::mapvfs_parse $sourcefolder/runtime/mapvfs.config $_rtbase $sourcefolder $::punkboot::target_platform] if {[llength [dict get $_mapmodel configerrors]]} { foreach _errline [dict get $_mapmodel configerrors] { puts stderr $_errline } exit 3 } - set _kit_outputs [punkboot::lib::mapvfs_kit_outputs $_mapmodel $_rtfolder $sourcefolder] + set _kit_outputs [punkboot::lib::mapvfs_kit_outputs $_mapmodel $_rtbase $sourcefolder] set _matchinfo [punkboot::lib::mapvfs_match_outputs $_kit_outputs $::punkboot::opt_kitnames] if {[llength [dict get $_matchinfo unknown]]} { puts stderr "make.tcl bake: unknown kit name(s): [join [dict get $_matchinfo unknown] {, }] - nothing built" @@ -2995,7 +3188,7 @@ if {$::punkboot::command eq "bake" && [llength $::punkboot::opt_kitnames]} { exit 1 } if {[dict get $_rec runtime] ne "-" && ![dict get $_rec runtime_present]} { - puts stderr "make.tcl bake: kit '[dict get $_rec kitname]' cannot be built - runtime [dict get $_rec runtime_file] not present in bin/runtime/$_rt_os_arch - nothing built" + puts stderr "make.tcl bake: kit '[dict get $_rec kitname]' cannot be built - runtime [dict get $_rec runtime_file] not present in bin/runtime/[dict get $_rec store_tier] (target [dict get $_rec target]) - nothing built" exit 1 } } @@ -3010,7 +3203,7 @@ if {$::punkboot::command eq "bake" && [llength $::punkboot::opt_kitnames]} { } } puts stdout "selective bake (G-121): [join $::punkboot::bake_selected_kitnames {, }]" - unset -nocomplain _rt_os_arch _rtfolder _mapmodel _kit_outputs _matchinfo _rec _r _errline + unset -nocomplain _rtbase _mapmodel _kit_outputs _matchinfo _rec _r _errline } # ---------------------------------------- @@ -3140,6 +3333,15 @@ if {$::punkboot::command eq "check"} { puts stdout "colour policy (G-113): mode=$::punkboot::colour_mode colour_disabled=$::punk::console::colour_disabled ansistrip=[expr {[llength $::punkboot::ansistrip_pushed] ? [join $::punkboot::ansistrip_pushed +] : 0}]" puts stdout " (precedence: NO_COLOR > PUNK_FORCE_COLOR/FORCE_COLOR > stdout tty probe incl. the 8.6 windows console-encoding signature; piped/unknown output is fully ESC-stripped per channel unless forced - utf-16 console channels are never wrapped)" puts stdout $sep + #G-122 host/target platform split - the self-diagnosis for "why is my kit set + #addressed the way it is". One stable single line; the columns are key=value. + set _chk_suffix [punkboot::lib::platform_exe_suffix $::punkboot::target_platform] + if {$_chk_suffix eq ""} {set _chk_suffix "(none)"} + puts stdout "platform (G-122): host=$::punkboot::host_platform target=$::punkboot::target_platform store=bin/runtime/[punkboot::lib::platform_store_tier $::punkboot::target_platform] exe-suffix=$_chk_suffix process-tooling=[punkboot::lib::platform_process_family $::punkboot::target_platform]" + puts stdout " (host = this tclsh's punk::platform canon, from tcl_platform(os)='$::tcl_platform(os)' platform='$::tcl_platform(platform)'; target = default kit target - a cygwin-family host (msys2/cygwin runtime on windows) bakes win32 kits)" + puts stdout " (mapvfs.config entries may declare their own target platform as a 4th element - 'make.tcl bakelist' shows each kit's target, store tier and artifact name)" + unset -nocomplain _chk_suffix + puts stdout $sep puts stdout "module/library checks - paths from bootsupport only" puts stdout $sep puts stdout "- tcl::tm::list" @@ -3493,18 +3695,17 @@ if {$::punkboot::command eq "bakelist"} { #grepping; expected state sits in the columns, anomalies surface in a trailing #sparse notes column (key=value tags). Kit name arguments filter the report and #add a per-kit detail block (resolved paths/sizes/mtimes). - switch -glob -- $this_platform_generic { - macosx-* { - set rt_os_arch macosx - } - default { - set rt_os_arch $this_platform_generic - } - } - set rtfolder $binfolder/runtime/$rt_os_arch + #G-122: store addressing is keyed by TARGET platform. The default target is + #this host's own platform (a cygwin-family host targets win32) and mapvfs + #entries may declare their own - so each row resolves its own tier under + #bin/runtime and the report names the tiers it consulted. + set rt_default_target $::punkboot::target_platform + set rt_os_arch [punkboot::lib::platform_store_tier $rt_default_target] + set rtbase $binfolder/runtime + set rtfolder $rtbase/$rt_os_arch set buildfolder $sourcefolder/_build set mapfile $sourcefolder/runtime/mapvfs.config - set mapmodel [punkboot::lib::mapvfs_parse $mapfile $rtfolder $sourcefolder] + set mapmodel [punkboot::lib::mapvfs_parse $mapfile $rtbase $sourcefolder $rt_default_target] foreach errline [dict get $mapmodel configerrors] { puts stderr $errline } @@ -3515,11 +3716,32 @@ if {$::punkboot::command eq "bakelist"} { puts stdout "no kit mapping config at $mapfile - no kit outputs configured" exit 0 } - set kit_outputs [punkboot::lib::mapvfs_kit_outputs $mapmodel $rtfolder $sourcefolder] + set kit_outputs [punkboot::lib::mapvfs_kit_outputs $mapmodel $rtbase $sourcefolder] if {![llength $kit_outputs]} { puts stdout "no kit outputs configured in $mapfile" exit 0 } + #Loud store-tier guard: a nonexistent store folder makes every row addressing it + #read runtime=missing for a reason that is not per-entry. Reported per tier + #actually referenced, since entries may target different platforms (G-122). + set report_tiers [list] + foreach rec $kit_outputs { + if {[dict get $rec runtime] eq "-"} {continue} + set t [dict get $rec store_tier] + if {$t ni $report_tiers} {lappend report_tiers $t} + } + set rt_store_missing [expr {![file isdirectory $rtfolder]}] + if {$rt_os_arch ni $report_tiers} {lappend report_tiers $rt_os_arch} + foreach t $report_tiers { + if {[file isdirectory $rtbase/$t]} {continue} + if {$t eq $rt_os_arch} { + puts stderr "WARNING: runtime store folder bin/runtime/$t does not exist - every configured runtime on the default target will read runtime=missing." + puts stderr " Default target '$rt_default_target' derives from THIS tclsh: tcl_platform(os)='$::tcl_platform(os)' tcl_platform(platform)='$::tcl_platform(platform)' -> host canon '$::punkboot::host_platform' -> target '$rt_default_target'." + puts stderr " Populate it with 'bin/punk-runtime.cmd' (or by hand), or declare a per-entry target platform in mapvfs.config for runtimes stored elsewhere." + } else { + puts stderr "WARNING: runtime store folder bin/runtime/$t does not exist - configured entries targeting $t will read runtime=missing." + } + } set reportset $kit_outputs if {[llength $::punkboot::opt_kitnames]} { set matchinfo [punkboot::lib::mapvfs_match_outputs $kit_outputs $::punkboot::opt_kitnames] @@ -3545,8 +3767,13 @@ if {$::punkboot::command eq "bakelist"} { if {![dict get $rec vfs_present]} { lappend notes vfs=missing } + if {[dict get $rec target] ne $rt_default_target} { + #cross-target entry: its runtime, store tier and artifact naming follow + #its own platform, not this host's default + lappend notes target=[dict get $rec target] + } if {$runtime ne "-" && [dict get $rec runtime_present]} { - set rtstale [punkboot::lib::runtime_materialization_check $rtfolder [dict get $rec runtime_file]] + set rtstale [punkboot::lib::runtime_materialization_check [dict get $rec runtime_dir] [dict get $rec runtime_file]] if {[dict size $rtstale]} { lappend notes rtrev=r[dict get $rtstale current_rev] vs src/_build/" + set rt_store_note [expr {$rt_store_missing ? " (FOLDER MISSING)" : ""}] + puts stdout "# mapvfs: src/runtime/mapvfs.config runtimes: bin/runtime/$rt_os_arch$rt_store_note deployed: bin/ vs src/_build/" set headline "" foreach h $headings w $widths { append headline [format "%-*s " $w $h] @@ -3594,10 +3822,11 @@ if {$::punkboot::command eq "bakelist"} { if {$runtime eq "-"} { puts stdout " runtime: (none - unwrapped .kit output)" } else { - puts stdout " runtime file: bin/runtime/$rt_os_arch/[dict get $rec runtime_file] [expr {[dict get $rec runtime_present] ? {(present)} : {(MISSING)}}]" + puts stdout " runtime file: bin/runtime/[dict get $rec store_tier]/[dict get $rec runtime_file] [expr {[dict get $rec runtime_present] ? {(present)} : {(MISSING)}}]" } puts stdout " vfs folder: src/vfs/[dict get $rec vfs] [expr {[dict get $rec vfs_present] ? {(present)} : {(MISSING)}}]" puts stdout " kit type: [dict get $rec kit_type]" + puts stdout " target: [dict get $rec target][expr {[dict get $rec target] eq $rt_default_target ? " (host default)" : " (declared in mapvfs.config)"}]" puts stdout " build product: src/_build/[dict get $rec targetkit] [punkboot::lib::bakelist_file_detail $buildfolder/[dict get $rec targetkit]]" puts stdout " deployed: bin/[dict get $rec targetkit] [punkboot::lib::bakelist_file_detail $binfolder/[dict get $rec targetkit]] state=[lindex $row 4]" } @@ -4706,28 +4935,40 @@ if {$::punkboot::command eq "bin"} { #set rtfolder $sourcefolder/runtime #AAA -switch -glob -- $this_platform_generic { - macosx-* { - #assuming universal binaries x86_64 and arm - set rt_os_arch macosx - } - default { - set rt_os_arch $this_platform_generic - } -} -set rtfolder $binfolder/runtime/$rt_os_arch +#G-122: the runtime store is addressed by TARGET platform, not by the driving +#tclsh's personality. $rt_default_target is this host's default target (a +#cygwin-family host targets win32; macOS collapses to the universal 'macosx' +#tier) and $rtfolder is that target's store. mapvfs.config entries may declare +#their own target platform - those runtimes live in their own tier under $rtbase +#and are resolved through $rtfolder_of below. +set rt_default_target $::punkboot::target_platform +set rt_os_arch [punkboot::lib::platform_store_tier $rt_default_target] +set rtbase $binfolder/runtime +set rtfolder $rtbase/$rt_os_arch +set rt_exe_suffix [punkboot::lib::platform_exe_suffix $rt_default_target] set rt_sourcefolder $sourcefolder/runtime ;#where our config lives -#review - when building kits for other platforms - it's unlikely runtime will be marked as executable - we should probably process all files in runtime folder except those with certain extensions +#G-122: derive the BUILDCOPY name from the runtime FILENAME rather than from the +#driving tclsh's platform. The copy is the same binary as its source, so it must +#keep the source's executable convention - an .exe stays an .exe whether a native +#windows tclsh, an msys/cygwin-runtime tclsh (platform 'unix', .exe files) or a +#cross-target host is driving the build. +#Mapping key for a runtime FILE name: the name as mapvfs.config spells it, i.e +#with a windows .exe suffix removed. Only .exe may be stripped - 'file rootname' +#would eat the last dotted segment of version-named runtimes such as tclsh9.0.5, +#which carry no suffix at all on non-windows targets (G-122). +proc ::punkboot::lib::runtime_mapkey {runtimefile} { + if {[string match -nocase *.exe $runtimefile]} { + return [string range $runtimefile 0 end-4] + } + return $runtimefile +} proc ::punkboot::lib::runtime_buildcopyname {runtime} { - if {$::tcl_platform(platform) eq "windows"} { - set rtname [file rootname [file tail $runtime]] - set bcname ${rtname}_BUILDCOPY.exe - } else { - set rtname [file tail $runtime] - set bcname ${rtname}_BUILDCOPY + set rttail [file tail $runtime] + if {[string match -nocase *.exe $rttail]} { + return "[file rootname $rttail]_BUILDCOPY.exe" } - return $bcname + return "${rttail}_BUILDCOPY" } #Local runtime-materialization staleness check (warn-only, no network). The kit @@ -4741,6 +4982,31 @@ proc ::punkboot::lib::runtime_buildcopyname {runtime} { #toml) and direct -r references stay silent; one warning per runtime per run. #G-121: the comparison core lives in ::punkboot::lib::runtime_materialization_check, #shared with the bakelist report's rtrev= note. +#G-122: run a command whose arguments must reach a NATIVE WINDOWS program +#verbatim. The msys2 runtime rewrites any argument that looks like an absolute +#posix path when it spawns a native windows executable, so 'taskkill /PID 1234' +#arrives as 'taskkill C:/msys64-root/PID 1234' - field-verified 2026-07-26: the +#pre-deploy process sweep found the running kit with tasklist and then could not +#kill it, failing the whole kit. MSYS2_ARG_CONV_EXCL=* is msys2's documented +#opt-out and is inert on every other host (real cygwin does not convert arguments +#at all), so one command spelling stays correct from every host personality. +proc ::punkboot::exec_nativeargs {cmdlist} { + set had [info exists ::env(MSYS2_ARG_CONV_EXCL)] + if {$had} { + set prev $::env(MSYS2_ARG_CONV_EXCL) + } + set ::env(MSYS2_ARG_CONV_EXCL) * + try { + return [exec {*}$cmdlist] + } finally { + if {$had} { + set ::env(MSYS2_ARG_CONV_EXCL) $prev + } else { + unset -nocomplain ::env(MSYS2_ARG_CONV_EXCL) + } + } +} + set ::punkboot::runtime_rev_warned [dict create] proc ::punkboot::runtime_materialization_warning {rtfolder runtime_fullname} { if {[dict exists $::punkboot::runtime_rev_warned $runtime_fullname]} { @@ -4757,16 +5023,75 @@ proc ::punkboot::runtime_materialization_warning {rtfolder runtime_fullname} { return } +# -- --- --- --- --- --- --- --- --- --- +#load mapvfs.config file (if any) in runtime folder to map runtimes to vfs folders. +#build a dict keyed on runtime executable name. +#If no mapfile (or no mapfile entry for that runtime) - the runtime will be paired with a matching .vfs folder in src folder. e.g punk.exe to src/punk.vfs +#If vfs folders or runtime executables which are explicitly listed in the mapfile don't exist - warn on stderr - but continue. if such nonexistants found; prompt user for whether to continue or abort. +#G-121: parsing lives in ::punkboot::lib::mapvfs_parse - the shared model consumed here, +#by 'bakelist' and by the selective-bake name resolution (never the file format directly). +#G-122: parsed BEFORE runtime discovery, because the mapping is what says which +#store tiers this bake must look in - an entry declaring a target platform other +#than the host default addresses its own bin/runtime/. +set mapfile $rt_sourcefolder/mapvfs.config +set mapmodel [punkboot::lib::mapvfs_parse $mapfile $rtbase $sourcefolder $rt_default_target] +foreach warnline [dict get $mapmodel warnings] { + puts stderr $warnline +} +if {[llength [dict get $mapmodel configerrors]]} { + foreach errline [dict get $mapmodel configerrors] { + puts stderr $errline + } + exit 3 +} +set runtime_vfs_map [dict get $mapmodel runtime_vfs_map] +set vfs_runtime_map [dict get $mapmodel vfs_runtime_map] +set runtime_target [dict get $mapmodel runtime_target] +set missing [dict get $mapmodel missing] +if {[llength $missing]} { + puts stderr "WARNING [llength $missing] missing items from $mapfile. (TODO - prompt user to continue/abort)" + foreach m $missing { + puts stderr " $m" + } + puts stderr "continuing..." +} +# -- --- --- --- --- --- --- --- --- --- +puts "-- runtime_vfs_map --" +punk::lib::pdict runtime_vfs_map +puts "---------------------" +puts "-- vfs_runtime_map--" +punk::lib::pdict vfs_runtime_map +puts "---------------------" + +# -- --- --- --- --- --- --- --- --- --- +#Runtime discovery. The default-target store is scanned wholesale (a runtime +#there needs no mapping entry - it can pair with a same-named .vfs folder); +#cross-target runtimes are only picked up when the mapping names them, since +#scanning every tier would pull in runtimes nothing asked to be built. +#$rtfolder_of resolves a discovered runtime FILE name to the store folder it +#came from (runtime_target_of gives its target platform) - every later store +#address, suffix and process-tooling decision goes through these rather than +#assuming the driving tclsh's platform (G-122). set runtimes [list] +set rtfolder_of [dict create] +proc ::punkboot::lib::runtime_target_of {rtname} { + #target platform of a runtime name (unsuffixed), from the parsed mapping + upvar #0 runtime_target runtime_target rt_default_target rt_default_target + if {[info exists runtime_target] && [dict exists $runtime_target $rtname]} { + return [dict get $runtime_target $rtname] + } + return $rt_default_target +} if {[file dirname [info nameofexecutable]] eq $rtfolder && [file isdirectory [info nameofexecutable]]} { #info name will only appear to be a directory when the runtime is a tclkit that is mounted at the same path as the physical file. #This is not a problem for zip based kits. #fix so that we can find and use a runtime tclkit that is currently in use as the runtime that is running this script. # - when the current runtime is using a runtime in the rtfolder that is a tclkit, it is mounted at the same path as the physical file and so appears to Tcl as if it's a directory. # - use external filesystem tools to make a copy of the file - if {$::tcl_platform(platform) eq "windows"} { + #(copy tooling is a HOST concern - the running executable is by definition a host binary) + if {$::punkboot::host_windows} { #exec cmd /c copy [info nameofexecutable] $rtfolder/[file rootname [file tail [info nameofexecutable]]]_BUILDCOPY.exe - exec cmd /c copy [file nativename [info nameofexecutable]] [file nativename $rtfolder/[punkboot::lib::runtime_buildcopyname [info nameofexecutable]]] + ::punkboot::exec_nativeargs [list cmd /c copy [file nativename [info nameofexecutable]] [file nativename $rtfolder/[punkboot::lib::runtime_buildcopyname [info nameofexecutable]]]] } else { #exec cp [info nameofexecutable] $rtfolder/[file tail [info nameofexecutable]]_BUILDCOPY exec cp [info nameofexecutable] $rtfolder/[punkboot::lib::runtime_buildcopyname [info nameofexecutable]] @@ -4774,12 +5099,14 @@ if {[file dirname [info nameofexecutable]] eq $rtfolder && [file isdirectory [in #we need to add the base name of the current exe to the runtimes list because the next section will look for files in the runtime folder #and won't find the mounted runtime since it's mounted as a directory. lappend runtimes [file tail [info nameofexecutable]] + dict set rtfolder_of [file tail [info nameofexecutable]] $rtfolder } set rtfolder_files [glob -nocomplain -dir $rtfolder -types {f} -tail *] set exclusions {.config .md .ico .txt .doc .pdf .htm .html} ;#we don't encourage other files in runtime folder aside from mapvfs.config - but lets ignore some common possibilities lappend exclusions .zip .7z .pea .bz2 .tar .gz .tgz .z .xz ;#don't allow archives to directly be treated as runtimes - tolerate presence but require user to unpack or rename if they're to be used as runtimes lappend exclusions .tail ;#result of running sdx mksplit on a kit - in theory the .head could be used - review/test +lappend exclusions .toml ;#G-103 runtime artifact metadata sits beside its runtime foreach f $rtfolder_files { if {[string match "*_BUILDCOPY*" $f]} { continue @@ -4789,11 +5116,43 @@ foreach f $rtfolder_files { } if {$f ni $runtimes} { lappend runtimes $f + dict set rtfolder_of $f $rtfolder } } +#cross-target runtimes named by the mapping (their own tier, their own suffix) +set cross_target_tiers [list] +dict for {_rtname _rttarget} $runtime_target { + if {$_rtname eq "-" || $_rttarget eq $rt_default_target} { + continue + } + set _tier [punkboot::lib::platform_store_tier $_rttarget] + set _dir $rtbase/$_tier + set _file $_rtname[punkboot::lib::platform_exe_suffix $_rttarget] + if {$_tier ni $cross_target_tiers} { + lappend cross_target_tiers $_tier + } + if {![file exists [file join $_dir $_file]]} { + continue ;#absent cross-target runtime already warned by the parse + } + if {[dict exists $rtfolder_of $_file]} { + puts stderr "WARNING: runtime file $_file is present in more than one store tier ([dict get $rtfolder_of $_file] and $_dir) - using the first found. Rename one of them to build both." + continue + } + lappend runtimes $_file + dict set rtfolder_of $_file $_dir +} +if {[llength $cross_target_tiers]} { + puts stdout "cross-target runtime tiers in use (mapvfs.config target platforms): [join $cross_target_tiers {, }]" +} +unset -nocomplain _rtname _rttarget _tier _dir _file if {![llength $runtimes]} { puts stderr "No executable runtimes found in $rtfolder - unable to build any .vfs folders into executables." puts stderr "Add runtimes to $rtfolder if required" + if {![file isdirectory $rtfolder]} { + #same store-tier self-diagnosis as bakelist + puts stderr "NOTE: that runtime store folder does not exist. Default target '$rt_default_target' derives from THIS tclsh: tcl_platform(os)='$::tcl_platform(os)' tcl_platform(platform)='$::tcl_platform(platform)' -> host canon '$::punkboot::host_platform' -> target '$rt_default_target'." + puts stderr " Populate it with 'bin/punk-runtime.cmd' (or by hand), or declare a per-entry target platform in mapvfs.config for runtimes stored elsewhere." + } #todo - don't exit - it is valid to use runtime of - to just build a .kit/.zipkit ? exit 0 } @@ -4801,10 +5160,10 @@ if {![llength $runtimes]} { #other runtimes get no BUILDCOPY refresh, no capability probe and no punkcheck events. if {[llength $::punkboot::bake_selected_kitnames]} { set runtimes [lmap _rtfile $runtimes { - set _rtkey $_rtfile - if {[string match *.exe $_rtkey]} { - set _rtkey [string range $_rtkey 0 end-4] - } + #NOTE: only an .exe suffix may be stripped to recover the mapping key - + #'file rootname' would eat the last dotted version segment of names like + #tclsh9.0.5 (suffixless on non-windows targets). + set _rtkey [punkboot::lib::runtime_mapkey $_rtfile] if {$_rtkey ni $::punkboot::bake_selected_runtimes} { continue } @@ -4814,41 +5173,6 @@ if {[llength $::punkboot::bake_selected_kitnames]} { } #previous have_sdx test location -# -- --- --- --- --- --- --- --- --- --- -#load mapvfs.config file (if any) in runtime folder to map runtimes to vfs folders. -#build a dict keyed on runtime executable name. -#If no mapfile (or no mapfile entry for that runtime) - the runtime will be paired with a matching .vfs folder in src folder. e.g punk.exe to src/punk.vfs -#If vfs folders or runtime executables which are explicitly listed in the mapfile don't exist - warn on stderr - but continue. if such nonexistants found; prompt user for whether to continue or abort. -#G-121: parsing lives in ::punkboot::lib::mapvfs_parse - the shared model consumed here, -#by 'bakelist' and by the selective-bake name resolution (never the file format directly). -set mapfile $rt_sourcefolder/mapvfs.config -set mapmodel [punkboot::lib::mapvfs_parse $mapfile $rtfolder $sourcefolder] -foreach warnline [dict get $mapmodel warnings] { - puts stderr $warnline -} -if {[llength [dict get $mapmodel configerrors]]} { - foreach errline [dict get $mapmodel configerrors] { - puts stderr $errline - } - exit 3 -} -set runtime_vfs_map [dict get $mapmodel runtime_vfs_map] -set vfs_runtime_map [dict get $mapmodel vfs_runtime_map] -set missing [dict get $mapmodel missing] -if {[llength $missing]} { - puts stderr "WARNING [llength $missing] missing items from $mapfile. (TODO - prompt user to continue/abort)" - foreach m $missing { - puts stderr " $m" - } - puts stderr "continuing..." -} -# -- --- --- --- --- --- --- --- --- --- -puts "-- runtime_vfs_map --" -punk::lib::pdict runtime_vfs_map -puts "---------------------" -puts "-- vfs_runtime_map--" -punk::lib::pdict vfs_runtime_map -puts "---------------------" #only test the runtime capabilities for runtimes that are actually in our map @@ -4897,27 +5221,34 @@ foreach runtime [dict keys $runtime_vfs_map] { exit 0 } - #temp - #build for current platform only for now - if {![file exists $rtfolder/$runtime]} { - dict set runtime_caps $runtime exitcode -1 error "find-fail $rtfolder/$runtime" + #G-122: address the runtime in ITS target's store tier, with ITS target's + #executable suffix. (Before the split this probed a suffixless name in the + #host-derived folder, so on windows every runtime silently reported + #find-fail and no capability was ever learned.) + set rt_target [punkboot::lib::runtime_target_of $runtime] + set rt_dir $rtbase/[punkboot::lib::platform_store_tier $rt_target] + set rt_file $runtime[punkboot::lib::platform_exe_suffix $rt_target] + if {![file exists $rt_dir/$rt_file]} { + dict set runtime_caps $runtime exitcode -1 error "find-fail $rt_dir/$rt_file" + continue + } + if {[punkboot::lib::platform_process_family $rt_target] ne [punkboot::lib::platform_process_family $::punkboot::host_platform]} { + #cross-target runtime: this host cannot execute it, so its capabilities + #stay unknown (the kit machinery falls back to trying the configured type) + dict set runtime_caps $runtime exitcode -1 error "cross-target ($rt_target) - not executable on this host" continue } #invoke can fail if runtime not an executable file for the current platform - if {[file type $rtfolder/$runtime] eq "directory"} { + if {[file type $rt_dir/$rt_file] eq "directory"} { #assume it's a mounted tclkit (because tcl kits mount in same place as their underlying path and mask the executable as a directory) #use the BUILDCOPY created above - REVIEW - if {$::tcl_platform(platform) eq "windows"} { - set useruntime [file rootname $runtime]_BUILDCOPY.exe - } else { - set useruntime ${runtime}_BUILDCOPY - } + set useruntime [punkboot::lib::runtime_buildcopyname $rt_file] } else { - set useruntime $runtime + set useruntime $rt_file } if {![catch { - lassign [punk::lib::invoke [list $rtfolder/$useruntime <<$capscript]] stdout stderr exitcode + lassign [punk::lib::invoke [list $rt_dir/$useruntime <<$capscript]] stdout stderr exitcode } errM]} { if {$exitcode == 0} { dict set runtime_caps $runtime $stdout @@ -4959,7 +5290,7 @@ set vfs_folder_changes [dict create] ;#cache whether each .vfs folder has change # -- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- #make build copy for all runtimes - not just those in the map - because even without a mapvfs.config file entry we build an exe for a runtime that matches .vfs folder name - REVIEW. foreach runtimefile $runtimes { - #runtimefile e.g tclkit86bi.exe on windows tclkit86bi on other platforms + #runtimefile e.g tclkit86bi.exe for a win32 target, tclkit86bi for a posix one #sdx *may* be pointed to use the runtime we use to build the kit, or the user may manually use this runtime if they don't have tclsh #sdx will complain if the same runtime is used for the shell as is used in the -runtime argument - so we make a copy (REVIEW) @@ -4967,23 +5298,19 @@ foreach runtimefile $runtimes { # file copy $rtfolder/$runtimefile $buildfolder/buildruntime.exe #} - #---------------------- - #todo - #temp - only build for current platform - need changes to runtime/mapvfs.config etc - if {![file exists $rtfolder/$runtimefile]} { - puts stderr " >> skipping $runtimefile not present in $rtfolder (wrong platform?) x-platform kit build TODO." + #G-122: each runtime is read from the tier its target platform names, not + #from one host-derived folder. Build copies keep the flat build_ name - + #the mapping cannot hold one runtime name in two tiers, so they cannot collide. + set rt_dir [dict get $rtfolder_of $runtimefile] + if {![file exists $rt_dir/$runtimefile]} { + puts stderr " >> skipping $runtimefile not present in $rt_dir" continue } - #---------------------- - if {[file type $rtfolder/$runtimefile] eq "directory"} { + if {[file type $rt_dir/$runtimefile] eq "directory"} { #assume it's a mounted tclkit (because tcl kits mount in same place as their underlying path and mask the executable) #use the BUILDCOPY created above - REVIEW - if {$::tcl_platform(platform) eq "windows"} { - set useruntimefile [file tail $runtimefile]_BUILDCOPY.exe - } else { - set useruntimefile ${runtimefile}_BUILDCOPY - } + set useruntimefile [punkboot::lib::runtime_buildcopyname $runtimefile] } else { set useruntimefile $runtimefile } @@ -4994,11 +5321,11 @@ foreach runtimefile $runtimes { }] #---------- set installer [punkcheck::installtrack new $installername $basedir/.punkcheck] - $installer set_source_target $rtfolder $buildfolder + $installer set_source_target $rt_dir $buildfolder set event [$installer start_event $config] $event targetset_init INSTALL $buildfolder/build_$runtimefile - #$event targetset_addsource $rtfolder/$runtimefile - $event targetset_addsource $rtfolder/$useruntimefile ;#possibly the _BUILDCOPY in the runtime folder (created for when mounted in current executable as a tclkit) + #$event targetset_addsource $rt_dir/$runtimefile + $event targetset_addsource $rt_dir/$useruntimefile ;#possibly the _BUILDCOPY in the runtime folder (created for when mounted in current executable as a tclkit) $event targetset_addsource $buildfolder/build_$runtimefile ;#self as source for change detection #---------- @@ -5010,9 +5337,9 @@ foreach runtimefile $runtimes { $event targetset_started # -- --- --- --- --- --- #This is the full runtime - *possibly* with some sort of vfs attached. - puts stdout "Copying runtime (as is) from $rtfolder/$useruntimefile to $buildfolder/build_$runtimefile" + puts stdout "Copying runtime (as is) from $rt_dir/$useruntimefile to $buildfolder/build_$runtimefile" if {[catch { - file copy -force $rtfolder/$useruntimefile $buildfolder/build_$runtimefile ;#becomes building_runtime + file copy -force $rt_dir/$useruntimefile $buildfolder/build_$runtimefile ;#becomes building_runtime } errM]} { puts stderr " >> copy runtime to $buildfolder/build_$runtimefile FAILED" $event targetset_end FAILED @@ -5050,12 +5377,16 @@ if {$sdxpath eq ""} { #see if the runtime (or any runtime *for current platform*) has tclkit in the name or has_starkit(?) and use that #todo - review #picking first found for now - fix + #G-122: only a runtime this HOST can execute can drive sdx - the + #capability probe already skipped cross-target runtimes, so a + #has_starkit record means the runtime ran here. dict for {rtname rtprops} $runtime_caps { if {[dict exists $rtprops has_starkit] && [dict get $rtprops has_starkit]} { - set testpath $rtfolder/$rtname + set _rt_target [punkboot::lib::runtime_target_of $rtname] + set testpath $rtbase/[punkboot::lib::platform_store_tier $_rt_target]/$rtname[punkboot::lib::platform_exe_suffix $_rt_target] if {[file exists $testpath]} { if {[file type $testpath] eq "directory"} { - set alt_tclkitpath $rtfolder/[punkboot::lib::runtime_buildcopyname $testpath] + set alt_tclkitpath [file dirname $testpath]/[punkboot::lib::runtime_buildcopyname $testpath] } else { set alt_tclkitpath $testpath } @@ -5246,6 +5577,43 @@ proc vfs_startup_script_warning {vfsfolder} { } return "vfs folder $vfsfolder has NO STARTUP SCRIPT: no root main.tcl and no root fauxlink resolving to main.tcl with an existing .tcl target - kits built from it will have no boot script (build proceeding anyway)$detail" } +#Cat-style zipkit assembly: a raw (vfs-free) runtime followed by a freshly built +#zip, concatenated. This is the zipfs-less assembly path for BOTH zip-type kits +#(G-122): Tcl's zipfs locates an image by scanning back from the END of the file +#and reads the central directory with ARCHIVE-START-RELATIVE offsets, so a plain +#concatenation mounts. 'zipfs mkimg's file-relative offset rewrite is one +#convention, not a mounting requirement - the zipcat kit type is the in-repo +#proof, and this lets a driving tcl with no zipfs at all (any 8.6) assemble zip +#kits. The zip itself is written by tcl::zipfs::mkzip when the driving tcl has +#it, else by punk::zip::mkzip (pure Tcl). +#Returns the number of zip bytes appended. +proc ::punkboot::assemble_zipcat_image {raw_runtime wrapvfs outfile zipfile} { + file copy -force $raw_runtime $outfile + #runtime in runtime folder may not have write perm set - ensure the copy does as we need to append + catch {exec chmod +w $outfile} + file delete $zipfile + if {[info commands ::tcl::zipfs] ne ""} { + puts stdout "tcl::zipfs::mkzip $zipfile $wrapvfs $wrapvfs" + ::tcl::zipfs::mkzip $zipfile $wrapvfs $wrapvfs + } else { + puts stdout "punk::zip::mkzip -directory $wrapvfs -base $wrapvfs $zipfile *" + package require punk::zip + punk::zip::mkzip -directory $wrapvfs -base $wrapvfs $zipfile * + } + puts stderr "concatenating zip to executable.." + set fdout [open $outfile a] + chan conf $fdout -translation binary + puts stderr "runtime bytes: [tell $fdout]" + set fdzip [open $zipfile r] + chan conf $fdzip -translation binary + set zipbytes [fcopy $fdzip $fdout] + close $fdzip + puts stderr "zip bytes: $zipbytes" + puts stderr "exezip bytes: [tell $fdout]" + close $fdout + return $zipbytes +} + set startdir [pwd] puts stdout "Found [llength $vfs_tails] .vfs folders - checking each for executables that may need to be built" cd [file dirname $buildfolder] @@ -5275,24 +5643,26 @@ foreach vfstail $vfs_tails { }] + #G-122: a runtime's filename and store tier come from ITS target platform. + #The '-' pseudo-runtime (unwrapped .kit output) carries no file at all. set runtimes [list] if {[dict exists $vfs_runtime_map $vfstail]} { - #set runtimes [dict get $vfs_runtime_map $vfstail] + #set runtimes [dict get $vfs_runtime_map $vfstail] #runtimes in vfs_runtime_map_vfs dict are unsuffixed (.exe stripped or was not present) set applist [dict get $vfs_runtime_map $vfstail] - if {"windows" eq $::tcl_platform(platform)} { - foreach rt_app $applist { - set rt [lindex $rt_app 0] - if {$rt ne "-" && [file exists $rtfolder/$rt.exe]} { - lappend runtimes $rt.exe - } + foreach rt_app $applist { + set rt [lindex $rt_app 0] + if {$rt eq "-"} { + #the '-' (runtime-less .kit) pseudo-runtime has never reached this + #list - both historical branches filtered it out - so the loop + #below's '-' handling is currently unreachable. Left as-is: + #enabling it is a separate, untested change. + continue } - } else { - foreach rt_app $applist { - set rt [lindex $rt_app 0] - if {[file exists $rtfolder/$rt]} { - lappend runtimes $rt - } + set rt_target [punkboot::lib::runtime_target_of $rt] + set rt_file $rt[punkboot::lib::platform_exe_suffix $rt_target] + if {[file exists $rtbase/[punkboot::lib::platform_store_tier $rt_target]/$rt_file]} { + lappend runtimes $rt_file } } } else { @@ -5301,18 +5671,14 @@ foreach vfstail $vfs_tails { #but conversely, adding an extra entry shouldn't stop default builds that used to run.. set matchrt [file rootname [file tail $vfstail]] ;#e.g project.vfs -> project if {![dict exists $runtime_vfs_map $matchrt]} { - if {"windows" eq $::tcl_platform(platform)} { - if {[file exists $rtfolder/$matchrt.exe]} { - lappend runtimes $matchrt.exe - } - } else { - if {[file exists $rtfolder/$matchrt]} { - lappend runtimes $matchrt - } + #unmapped: the default-target store is the only place a name-match is looked for + if {[file exists $rtfolder/$matchrt$rt_exe_suffix]} { + lappend runtimes $matchrt$rt_exe_suffix } } } - #assert $runtimes is a list of executable names suffixed with .exe if on windows - whether or not specified with .exe in the mapvfs.config + #assert $runtimes is a list of runtime filenames as they exist in their target's store tier + #(suffixed with .exe for windows-family targets, whether or not mapvfs.config spelled it that way) puts " vfs: $vfstail runtimes to process ([llength $runtimes]): $runtimes" if {[llength $runtimes]} { @@ -5326,13 +5692,17 @@ foreach vfstail $vfs_tails { #todo - non kit based - zipkit? # $runtimes may now include a dash entry "-" (from mapvfs.config file) foreach runtime_fullname $runtimes { - set rtname [file rootname $runtime_fullname] + set rtname [punkboot::lib::runtime_mapkey $runtime_fullname] if {[llength $::punkboot::bake_selected_kitnames] && $rtname ni $::punkboot::bake_selected_runtimes} { continue ;#G-121 selective bake - no requested kit wraps this runtime } + #G-122: everything this iteration emits is keyed by the runtime's TARGET + set rt_target [punkboot::lib::runtime_target_of $rtname] + set rt_tierdir $rtbase/[punkboot::lib::platform_store_tier $rt_target] + set rt_suffix [punkboot::lib::platform_exe_suffix $rt_target] #rtname of "-" indicates build a kit without a runtime if {$runtime_fullname ne "-"} { - ::punkboot::runtime_materialization_warning $rtfolder $runtime_fullname + ::punkboot::runtime_materialization_warning $rt_tierdir $runtime_fullname } #first configured runtime will be the one to use the same name as .vfs folder for output executable. Additional runtimes on this .vfs will need to suffix the runtime name to disambiguate. @@ -5356,11 +5726,7 @@ foreach vfstail $vfs_tails { if {$rtname eq "-"} { set targetkit $appname.kit } else { - if {$::tcl_platform(platform) eq "windows"} { - set targetkit ${appname}.exe - } else { - set targetkit $appname - } + set targetkit ${appname}$rt_suffix if {$targetkit in $exe_names_seen} { #duplicate appname configured? #todo - consider creating as ${appname}(2) etc? @@ -5384,9 +5750,10 @@ foreach vfstail $vfs_tails { #cannot replace a running image anyway. Informational/update subcommands are #unaffected (they exit before the kit machinery), so driving make.tcl from a built #punk executable remains supported for everything except rebuilding that executable. + #(a HOST filesystem comparison - case rules follow the host, not the target) set self_exe [file normalize [info nameofexecutable]] set self_deploy_target [file normalize $binfolder/$targetkit] - if {$::tcl_platform(platform) eq "windows"} { + if {$::punkboot::host_windows} { set is_self_build [string equal -nocase $self_exe $self_deploy_target] } else { set is_self_build [string equal $self_exe $self_deploy_target] @@ -5451,15 +5818,23 @@ foreach vfstail $vfs_tails { #to consider: - allow specifying runtime kits as if they are vfs folders in the normal xxx.vfs list - and autodetect and extract #would need to detect UPX, cookfs,zipfs,tclkit set rtmountpoint "" + #Test the capability VALUE, not just its presence: a probe that ran + #reports has_starkit/has_cookfs 0 for a plain zipfs runtime, and + #trying to unwrap such a runtime as a kit ends in sdx mksplit + #failing and REPLACING raw_runtime with the un-split original - + #which silently yields a zip kit with two zips and no tcl_library. + #(Latent until G-122 made the capability probe actually run on + #windows: before that every probe reported find-fail, so this list + #was always empty.) set caps [dict get $runtime_caps $rtname] set source_kit_caps [list] - if {[dict exists $caps has_zipfs]} { + if {[dict exists $caps has_zipfs] && [dict get $caps has_zipfs]} { lappend source_kit_caps zip } - if {[dict exists $caps has_starkit]} { + if {[dict exists $caps has_starkit] && [dict get $caps has_starkit]} { lappend source_kit_caps kit } - if {[dict exists $caps has_cookfs]} { + if {[dict exists $caps has_cookfs] && [dict get $caps has_cookfs]} { lappend source_kit_caps cookfs } @@ -5576,11 +5951,18 @@ foreach vfstail $vfs_tails { zipfile::decode::open $extractedzipfile set archiveinfo [zipfile::decode::archive] zipfile::decode::unzip $archiveinfo $extractedzipfolder - }]} { + } extracterr]} { set extraction_done 1 set extract_kit_type $extract_kit_try #todo - verify that init.tcl etc are present? merge_over $extractedzipfolder $targetvfs + } else { + #Do not swallow the reason: without the runtime's own + #zip contents the built kit has no tcl_library. Note + #zipfile::decode is a system tcllib package, not a + #vendored one - a tclsh without tcllib (e.g the msys2 + #/usr/bin/tclsh8.6) cannot take this path at all. + puts stderr "zipfs-less extraction of $runtime_fullname FAILED: $extracterr" } } @@ -5668,10 +6050,11 @@ foreach vfstail $vfs_tails { if {!$extraction_done} { #TODO: if not extracted - use a default tcl_library for patchlevel and platform? - puts stderr "--------------------------------------------" - puts stderr "[punkboot::sgr 31]WARNING: No extraction done from runtime $runtime_fullname[punkboot::sgr]" - puts stderr "If no init.tcl provided in the vfs at the proper location (containing init.tcl) - the resulting kit will probably not initialise properly!" - puts stderr "--------------------------------------------" + #Recapped BUILD-WARNING (not just an inline note): the build continues + #and produces an artifact, but unless the .vfs itself supplies a + #tcl_library that artifact cannot boot - and this line would otherwise + #scroll away behind thousands of merge lines. + ::punkboot::print_build_warnings [list "No extraction done from runtime $runtime_fullname (tried: $target_kit_type[expr {[llength $source_kit_caps] ? " + runtime caps $source_kit_caps" : ""}]) - kit $targetkit gets no tcl_library from the runtime and will not initialise unless src/vfs/$vfstail supplies one"] file mkdir $targetvfs } @@ -5756,9 +6139,17 @@ foreach vfstail $vfs_tails { puts stderr "RUNTIME capabilities unknown. Unsure if zip supported. trying anyway.." } } - #note - as at 2024-08 - there is some discussion about the interface to mkimg - it is considered unstable (may change to -option value syntax) - puts stderr "calling: tcl::zipfs::mkimg $buildfolder/$vfsname.new $wrapvfs $wrapvfs \"\" $raw_runtime" - tcl::zipfs::mkimg $buildfolder/$vfsname.new $wrapvfs $wrapvfs "" $raw_runtime + if {[info commands ::tcl::zipfs::mkimg] ne ""} { + #note - as at 2024-08 - there is some discussion about the interface to mkimg - it is considered unstable (may change to -option value syntax) + puts stderr "calling: tcl::zipfs::mkimg $buildfolder/$vfsname.new $wrapvfs $wrapvfs \"\" $raw_runtime" + tcl::zipfs::mkimg $buildfolder/$vfsname.new $wrapvfs $wrapvfs "" $raw_runtime + } else { + #G-122: the driving tcl has no zipfs (8.6) - assemble the same + #kind of image by concatenation. The result mounts on any + #zipfs-capable runtime (archive-start-relative offsets). + puts stderr "WARNING: tcl shell '[info nameofexecutable]' has no zipfs - assembling zip kit $targetkit by concatenation (raw runtime + mkzip)" + ::punkboot::assemble_zipcat_image $raw_runtime $wrapvfs $buildfolder/$vfsname.new $buildfolder/$vfsname.zip + } } result ]} { set failmsg "zipfs mkimg failed with msg: $result" puts stderr "tcl::zipfs::mkimg $targetkit failed" @@ -5768,7 +6159,7 @@ foreach vfstail $vfs_tails { $vfs_installer destroy continue } else { - puts stdout "ok - finished tcl::zipfs::mkimg" + puts stdout "ok - finished zip image assembly" set separator [string repeat = 40] puts stdout $separator puts stdout $result @@ -5794,32 +6185,7 @@ foreach vfstail $vfs_tails { } #'archive' based zip offsets - editable in 7z,peazip - file copy $raw_runtime $buildfolder/$vfsname.new - #runtime in runtime folder may not have write perm set - ensure the copy does as we need to append - catch {exec chmod +w $buildfolder/$vfsname.new} - file delete $buildfolder/$vfsname.zip - - if {[info commands ::tcl::zipfs] ne ""} { - puts stdout "tcl::zipfs::mkzip $buildfolder/$vfsname.zip $wrapvfs $wrapvfs" - ::tcl::zipfs::mkzip $buildfolder/$vfsname.zip $wrapvfs $wrapvfs - } else { - puts stdout "punk::zip::mkzip -directory $wrapvfs -base $wrapvfs $buildfolder/$vfsname.zip *" - package require punk::zip - punk::zip::mkzip -directory $wrapvfs -base $wrapvfs $buildfolder/$vfsname.zip * - } - - - puts stderr "concatenating executable to zip.." - set fdout [open $buildfolder/$vfsname.new a] - chan conf $fdout -translation binary - puts stderr "runtime bytes: [tell $fdout]" - set fdzip [open $buildfolder/$vfsname.zip r] - chan conf $fdzip -translation binary - set zipbytes [fcopy $fdzip $fdout] - close $fdzip - puts stderr "zip bytes: $zipbytes" - puts stderr "exezip bytes: [tell $fdout]" - close $fdout + ::punkboot::assemble_zipcat_image $raw_runtime $wrapvfs $buildfolder/$vfsname.new $buildfolder/$vfsname.zip } result ]} { set failmsg "creating zipcat image failed with msg: $result" puts stderr "creating image (zipcat) $targetkit failed" @@ -5942,18 +6308,34 @@ foreach vfstail $vfs_tails { #still - seems to be under 1s on a 2018 era i9 running in 2025 - which is not a big problem in this context. #using a filter in tasklist or get-command can make things a little faster - if {$::tcl_platform(platform) eq "windows"} { + #G-122: the tooling follows the TARGET, not the driving tclsh. + #A win32 kit is a windows process whatever baked it - and msys 'ps' + #sees only msys-descendant processes, so an msys-hosted build using + #'ps' would report a natively-launched kit as not running and then + #fail to replace it. tasklist/taskkill exec fine from an msys tclsh. + #When target and host process worlds differ (a cross-target bake) no + #local process can be holding the artifact - skip the sweep rather + #than run tooling that cannot see the target's processes anyway. + set target_proc_family [punkboot::lib::platform_process_family $rt_target] + set host_proc_family [punkboot::lib::platform_process_family $::punkboot::host_platform] + if {$target_proc_family eq "windows"} { set pscmd "tasklist" + set pid_field 1 } else { set pscmd "ps" + set pid_field 0 + } + set sweep_applicable [expr {$target_proc_family eq $host_proc_family}] + if {!$sweep_applicable} { + puts stdout " (skipping running-process sweep for $targetkit - target $rt_target processes are not visible to this $::punkboot::host_platform host)" } - #killing process doesn't apply to .kit build - if {$rtname ne "-"} { + #killing process doesn't apply to .kit build + if {$rtname ne "-" && $sweep_applicable} { #exec $pscmd | grep $targetkit ;#wrong - fails on at least some OSes because grep itself is in the list #exec ps | grep $targetkit | grep -v grep ;#exclude grep process if {![catch { - exec $pscmd + ::punkboot::exec_nativeargs [list $pscmd] } ps_lines]} { #we will process without grep for for now - may not be avail on windows? set still_running_lines [list] @@ -5966,11 +6348,7 @@ foreach vfstail $vfs_tails { #itself (e.g a copy of the target executable driving this build from #a path other than the deploy target, which the self-build guard #above doesn't catch). - if {$::tcl_platform(platform) eq "windows"} { - set ln_pid [lindex $ln 1] - } else { - set ln_pid [lindex $ln 0] - } + set ln_pid [lindex $ln $pid_field] if {$ln_pid eq [pid]} {continue} lappend still_running_lines $ln } @@ -5983,15 +6361,14 @@ foreach vfstail $vfs_tails { foreach ln $still_running_lines { puts stdout " $ln" - if {$::tcl_platform(platform) eq "windows"} { - set pid [lindex $ln 1] + set pid [lindex $ln $pid_field] + if {$target_proc_family eq "windows"} { if {$forcekill} { set killcmd [list taskkill /F /PID $pid] } else { set killcmd [list taskkill /PID $pid] } } else { - set pid [lindex $ln 0] #review! if {$forcekill} { set killcmd [list kill -9 $pid] @@ -6001,7 +6378,7 @@ foreach vfstail $vfs_tails { } puts stdout " pid: $pid (attempting to kill now using '$killcmd')" if {[catch { - exec {*}$killcmd + ::punkboot::exec_nativeargs $killcmd } errMsg]} { puts stderr "$killcmd returned an error:" puts stderr $errMsg