From d88e1c1759071c902e1c7157111beedb572853ac Mon Sep 17 00:00:00 2001 From: Julian Noble Date: Mon, 6 Jul 2026 18:24:16 +1000 Subject: [PATCH] punkboot::utils 0.1.1: move vendorupdate dirty-checkout check into module Move vendor_source_dirty_warnings out of make.tcl into punkboot::utils as vcs_dirty_warnings (PUNKARGS-documented, caller-label param so the bootsupport update path can share it per G-026). make.tcl now uses a guarded lazy require and degrades to skipped-with-warning if the bootsupport snapshot lacks the proc - a stale/broken snapshot must never brick the make.tcl commands used to repair it. Chicken-and-egg-safe ordering used and now documented in src/bootsupport/AGENTS.md ("Moving make.tcl functionality into punkboot::utils"): edit src module + bump buildversion, make.tcl modules, make.tcl bootsupport, only then repoint make.tcl call sites. Boot-phase code (path setup, punkboot::lib prompts, package-availability checks) stays self-contained in make.tcl. Bootsupport snapshot refresh from the sanctioned pipeline: punkboot::utils 0.1.0->0.1.1, shellrun 0.1.2->0.1.4 (punk::args require fix catch-up), punk::console 0.7.1 content catch-up; layout mirrors via the punkcheck-tracked sync steps. New tests: src/tests/modules/punkboot/utils/testsuites/utils/vcsdirty.test (git fixture: dirty/clean/dedupe/label/unversioned/missing) - 25/25 across punkboot suites; vendorupdate verified end-to-end through the module path. Assisted-by: harness=claude; primary-model=claude-fable-5; api-location=anthropic.com --- CHANGELOG.md | 1 + src/bootsupport/AGENTS.md | 13 + src/bootsupport/modules/punk/console-0.7.1.tm | 27 +- .../{utils-0.1.0.tm => utils-0.1.1.tm} | 90 +++- .../modules/shellrun-0.1.4.tm} | 421 +++++++++++++----- src/make.tcl | 71 +-- src/modules/punkboot/utils-999999.0a1.0.tm | 84 +++- src/modules/punkboot/utils-buildversion.txt | 3 +- .../custom/_project/punk.basic/src/make.tcl | 65 +++ .../bootsupport/modules/punk/console-0.7.1.tm | 27 +- .../modules/punkboot/utils-0.1.1.tm} | 90 +++- .../bootsupport/modules/shellrun-0.1.4.tm} | 421 +++++++++++++----- .../_project/punk.project-0.1/src/make.tcl | 65 +++ .../bootsupport/modules/punk/console-0.7.1.tm | 27 +- .../modules/punkboot/utils-0.1.1.tm} | 90 +++- .../bootsupport/modules/shellrun-0.1.4.tm} | 421 +++++++++++++----- .../_project/punk.shell-0.1/src/make.tcl | 65 +++ .../utils/testsuites/utils/vcsdirty.test | 98 ++++ 18 files changed, 1683 insertions(+), 396 deletions(-) rename src/bootsupport/modules/punkboot/{utils-0.1.0.tm => utils-0.1.1.tm} (50%) rename src/{project_layouts/custom/_project/punk.shell-0.1/src/bootsupport/modules/shellrun-0.1.2.tm => bootsupport/modules/shellrun-0.1.4.tm} (66%) rename src/project_layouts/custom/_project/{punk.shell-0.1/src/bootsupport/modules/punkboot/utils-0.1.0.tm => punk.project-0.1/src/bootsupport/modules/punkboot/utils-0.1.1.tm} (50%) rename src/{bootsupport/modules/shellrun-0.1.2.tm => project_layouts/custom/_project/punk.project-0.1/src/bootsupport/modules/shellrun-0.1.4.tm} (66%) rename src/project_layouts/custom/_project/{punk.project-0.1/src/bootsupport/modules/punkboot/utils-0.1.0.tm => punk.shell-0.1/src/bootsupport/modules/punkboot/utils-0.1.1.tm} (50%) rename src/project_layouts/custom/_project/{punk.project-0.1/src/bootsupport/modules/shellrun-0.1.2.tm => punk.shell-0.1/src/bootsupport/modules/shellrun-0.1.4.tm} (66%) create mode 100644 src/tests/modules/punkboot/utils/testsuites/utils/vcsdirty.test diff --git a/CHANGELOG.md b/CHANGELOG.md index fbf03d85..fdd38a10 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ Entries are newest-first; one bullet per notable change. See the root `AGENTS.md ## [0.2.5] - 2026-07-06 - `make.tcl vendorupdate` now checks each vendor source project's fossil/git checkout state and warns (non-fatal, once per VCS root) when pulling from a dirty tree — vendored artifacts built from uncommitted source have no committed provenance. Enforcement policy (abort/override, bootsupport coverage) tracked by goal G-026. +- punkboot::utils 0.1.1: the dirty-checkout check lives in the module as `vcs_dirty_warnings` (make.tcl calls it via a guarded lazy require and degrades to skipped if the bootsupport snapshot lacks it); bootsupport snapshots refreshed (punkboot::utils 0.1.1, shellrun 0.1.4, punk::console 0.7.1 content catch-up). ## [0.2.4] - 2026-07-06 diff --git a/src/bootsupport/AGENTS.md b/src/bootsupport/AGENTS.md index 0ec12731..a22953c6 100644 --- a/src/bootsupport/AGENTS.md +++ b/src/bootsupport/AGENTS.md @@ -50,6 +50,19 @@ After changing any build-critical module (`punkcheck`, `punk::repo`, `punk::mix` - Do not add compiled extensions or packages requiring dynamic loading to bootsupport. - Check `src/bootsupport/modules/README.md` for the copy-as-module workflow. +### Moving make.tcl functionality into punkboot::utils + +`punkboot::utils` (source of truth `src/modules/punkboot/utils-999999.0a1.0.tm`) exists to keep make.tcl from growing unwieldy. Only post-boot functionality may move there — make.tcl loads it lazily from the bootsupport snapshot after boot has set the module paths; anything used during early boot (path setup, `::punkboot::lib` prompts/classifiers, package-availability checks) must stay self-contained in make.tcl. + +Chicken-and-egg-safe ordering when adding to punkboot::utils (make.tcl must never reference a proc its loaded snapshot lacks): + +1. Edit `src/modules/punkboot/utils-999999.0a1.0.tm` and bump its buildversion. +2. `tclsh src/make.tcl modules` (build to root `modules/`). +3. `tclsh src/make.tcl bootsupport` (pull the new build into `src/bootsupport/modules/`). +4. Only then repoint make.tcl call sites to the new proc. + +make.tcl call sites must use a guarded `package require punkboot::utils` and degrade gracefully (skip the feature with a warning) when the proc is unavailable — a stale or broken bootsupport snapshot must never brick the make.tcl commands used to repair it (`modules`, `bootsupport`). + ## Verification - `tclsh src/make.tcl packages` succeeds after any modification. diff --git a/src/bootsupport/modules/punk/console-0.7.1.tm b/src/bootsupport/modules/punk/console-0.7.1.tm index aac663a3..7a9936af 100644 --- a/src/bootsupport/modules/punk/console-0.7.1.tm +++ b/src/bootsupport/modules/punk/console-0.7.1.tm @@ -581,7 +581,10 @@ namespace eval punk::console { try { return [thread::send $route_owner [list ::punk::console::internal::get_ansi_response_payload -console $inoutchannels -expected_ms $expected -ignoreok $ignoreok -return $returntype -passthrough $passthrough $query $capturingendregex]] } on error {errM erropts} { - return -code error "get_ansi_response_payload query routed to console-owning thread $route_owner failed: $errM" + #preserve the owner-side errorcode (where the transport propagates it) so + #callers can discriminate, e.g. {PUNK CONSOLE QUERY HOSTAGE_COOKED_READ} + #from the idle-reader guard + return -code error -errorcode [dict get $erropts -errorcode] "get_ansi_response_payload query routed to console-owning thread $route_owner failed: $errM" } } @@ -611,6 +614,28 @@ namespace eval punk::console { error "punk::console::get_ansi_response_payload input channel '$input' is closed or at eof - cannot receive terminal response" } + #tcl 8.6 windows console idle-reader hostage guard (fail fast, no emission). + #With the console in cooked (line) mode and a readable handler armed on the input + #channel, the 8.6 channel driver has a blocking cooked-mode ReadConsole parked + #(arming is what posts it), and driver reads sample the console mode at issue time - + #the raw cycle below cannot rescue it. Any response would be swallowed by that read + #until the user presses Enter, then leak to the line reader as phantom input + #(~500ms timeout now, input corruption later). This is the idle-at-a-line-mode-prompt + #condition: the repl disarms its reader during command dispatch, so mid-command + #queries are unaffected, and raw mode reads cooperatively and is exempt. Typical + #trigger: a query from an after-script or worker thread firing while the shell sits + #at a line-mode prompt. Best-effort by design: a parked read can outlive a removed + #handler, so this catches the systematic case, not every conceivable one. + if {![tsv::get punk_console is_raw] && $input eq "stdin" && $::punk::console::has_twapi} { + set guard_conf [chan configure $input] + if {![dict exists $guard_conf -inputmode] && ![dict exists $guard_conf -mode] + && [chan event $input readable] ne "" + && ![catch {twapi::get_console_handle stdin}]} { + return -code error -errorcode [list PUNK CONSOLE QUERY HOSTAGE_COOKED_READ]\ + "punk::console::get_ansi_response_payload refusing to emit query on '$input': tcl 8.6 console is in cooked (line) mode with an armed readable handler - the driver's parked cooked read would swallow the response until Enter (query issued from an idle line-mode prompt?). Use raw mode, or issue queries while the reader is disarmed (e.g. during repl command evaluation)." + } + } + #chunks from input that need to be handled by readers upvar ::punk::console::input_chunks_waiting input_chunks_waiting diff --git a/src/bootsupport/modules/punkboot/utils-0.1.0.tm b/src/bootsupport/modules/punkboot/utils-0.1.1.tm similarity index 50% rename from src/bootsupport/modules/punkboot/utils-0.1.0.tm rename to src/bootsupport/modules/punkboot/utils-0.1.1.tm index fa10b93e..be23f56e 100644 --- a/src/bootsupport/modules/punkboot/utils-0.1.0.tm +++ b/src/bootsupport/modules/punkboot/utils-0.1.1.tm @@ -7,7 +7,7 @@ # (C) 2023 # # @@ Meta Begin -# Application punkboot::utils 0.1.0 +# Application punkboot::utils 0.1.1 # Meta platform tcl # Meta license BSD # @@ Meta End @@ -23,7 +23,7 @@ package require Tcl 8.6- # ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++ namespace eval punkboot::utils { variable version 0.1.0 - namespace export parse_punkproject_version read_punkproject_version read_changelog_latest_version + namespace export parse_punkproject_version read_punkproject_version read_changelog_latest_version vcs_dirty_warnings namespace eval argdoc { @@ -115,6 +115,88 @@ namespace eval punkboot::utils { } return "" } + + + namespace eval argdoc { + lappend PUNKARGS [list { + @id -id ::punkboot::utils::vcs_dirty_warnings + @cmd -name "::punkboot::utils::vcs_dirty_warnings"\ + -summary\ + "Warning lines for uncommitted VCS changes above a path"\ + -help\ + "Walks up from path to the nearest fossil and/or git root and + returns warning lines describing uncommitted changes. + Artifacts built from a dirty source tree have no committed + provenance - callers such as make.tcl vendorupdate use this + to flag source projects that should be committed first. + + checkedrootsvar names a dict variable in the caller's scope + used to report each VCS root at most once across calls. + label, if supplied, is prefixed into each warning to name + the calling operation (e.g. 'vendorupdate'). + + Returns an empty list if the path is clean, unversioned, + missing, already reported, or state cannot be determined." + @leaders + path -type string -optional 0 -help\ + "Path to check; the nearest enclosing fossil/git root is examined" + checkedrootsvar -type string -optional 0 -help\ + "Name of a dict variable in the caller's scope for dedupe across calls" + label -type string -optional 1 -default "" -help\ + "Operation name prefixed into warnings" + }] + } + proc vcs_dirty_warnings {path checkedrootsvar {label ""}} { + upvar 1 $checkedrootsvar checkedroots + if {![info exists checkedroots]} {set checkedroots [dict create]} + if {$label ne ""} {set label "$label "} + set warnings [list] + set dir [file normalize $path] + if {![file isdirectory $dir]} { + return $warnings ;#missing path - not this proc's business to report + } + set fossilroot "" + set gitroot "" + while {1} { + if {$fossilroot eq "" && ([file exists [file join $dir .fslckout]] || [file exists [file join $dir _FOSSIL_]])} { + set fossilroot $dir + } + if {$gitroot eq "" && [file exists [file join $dir .git]]} { + set gitroot $dir + } + if {$fossilroot ne "" || $gitroot ne ""} { + break ;#report at the nearest VCS root(s); a dual git+fossil root is caught in one pass + } + set parent [file dirname $dir] + if {$parent eq $dir} { break } + set dir $parent + } + if {$fossilroot ne "" && ![dict exists $checkedroots fossil,$fossilroot] && [llength [auto_execok fossil]]} { + dict set checkedroots fossil,$fossilroot 1 + #fossil changes must run from within the checkout + set original_cwd [pwd] + if {[catch { + cd $fossilroot + set fchanges [string trim [exec {*}[auto_execok fossil] changes]] + } errM]} { + lappend warnings "WARNING: ${label}could not determine fossil state of source project at $fossilroot ($errM)" + } elseif {$fchanges ne ""} { + lappend warnings "WARNING: ${label}source project at $fossilroot has uncommitted fossil changes ([llength [split $fchanges \n]] file(s)) - artifacts built from a dirty tree have no committed provenance" + } + cd $original_cwd + } + if {$gitroot ne "" && ![dict exists $checkedroots git,$gitroot] && [llength [auto_execok git]]} { + dict set checkedroots git,$gitroot 1 + if {[catch { + set gchanges [string trim [exec {*}[auto_execok git] -C $gitroot status --porcelain]] + } errM]} { + lappend warnings "WARNING: ${label}could not determine git state of source project at $gitroot ($errM)" + } elseif {$gchanges ne ""} { + lappend warnings "WARNING: ${label}source project at $gitroot has uncommitted git changes ([llength [split $gchanges \n]] file(s)) - artifacts built from a dirty tree have no committed provenance" + } + } + return $warnings + } } namespace eval ::punk::args::register { @@ -125,8 +207,8 @@ namespace eval ::punk::args::register { ## Ready package provide punkboot::utils [tcl::namespace::eval punkboot::utils { variable version - #- this version number, exactly 0.1.0, is a literal used in src module folders + #- this version number, exactly 0.1.1, is a literal used in src module folders #- we refer to this sometimes as the magic version number - set version 0.1.0 + set version 0.1.1 }] return diff --git a/src/project_layouts/custom/_project/punk.shell-0.1/src/bootsupport/modules/shellrun-0.1.2.tm b/src/bootsupport/modules/shellrun-0.1.4.tm similarity index 66% rename from src/project_layouts/custom/_project/punk.shell-0.1/src/bootsupport/modules/shellrun-0.1.2.tm rename to src/bootsupport/modules/shellrun-0.1.4.tm index 07bca217..08538dec 100644 --- a/src/project_layouts/custom/_project/punk.shell-0.1/src/bootsupport/modules/shellrun-0.1.2.tm +++ b/src/bootsupport/modules/shellrun-0.1.4.tm @@ -5,6 +5,7 @@ package require shellfilter package require punk::ansi +package require punk::args #NOTE: the run,runout,runerr,runx commands only produce an error if the command didn't run. # - If it did run, but there was a non-zero exitcode it is up to the application to check that. @@ -79,76 +80,45 @@ namespace eval shellrun { - #maintenance: similar used in punk::ns & punk::winrun - #todo - take runopts + aliases as args - #longopts must be passed as a single item ie --timeout=100 not --timeout 100 - proc get_run_opts {arglist} { - if {[catch { - set callerinfo [info level -1] - } errM]} { - set caller "" - } else { - set caller [lindex $callerinfo 0] - } - - #we provide -nonewline even for 'run' even though run doesn't deliver stderr or stdout to the tcl return value - #This is for compatibility with other runX commands, and the difference is also visible when calling from repl. - set known_runopts [list "-echo" "-e" "-nonewline" "-n" "-tcl" "-debug"] - set known_longopts [list "--timeout"] - set known_longopts_msg "" - foreach lng $known_longopts { - append known_longopts_msg "${lng}=val " - } - set aliases [list "-e" "-echo" "-echo" "-echo" "-n" "-nonewline" "-nonewline" "-nonewline" "-tcl" "-tcl" "-debug" "-debug"] ;#include map to self - set runopts [list] - set runoptslong [list] - set cmdargs [list] - - set idx_first_cmdarg [lsearch -not $arglist "-*"] - - set allopts [lrange $arglist 0 $idx_first_cmdarg-1] - set cmdargs [lrange $arglist $idx_first_cmdarg end] - foreach o $allopts { - if {[string match --* $o]} { - lassign [split $o =] flagpart valpart - if {$valpart eq ""} { - error "$caller: longopt $o seems to be missing a value - must be of form --option=value" - } - if {$flagpart ni $known_longopts} { - error "$caller: Unknown runoption $o - known options $known_runopts $known_longopts_msg" - } - lappend runoptslong $flagpart $valpart - } else { - if {$o ni $known_runopts} { - error "$caller: Unknown runoption $o - known options $known_runopts $known_longopts_msg" - } - lappend runopts [dict get $aliases $o] - } - } - return [list runopts $runopts runoptslong $runoptslong cmdargs $cmdargs] - } - - #todo - investigate cause of punk86 run hanging sometimes. An 'after 500' before exit in the called script fixes the issue. punk87 doesn't seem to be affected. lappend PUNKARGS [list { @id -id ::shellrun::run + @cmd -name "shellrun::run"\ + -summary "Run an external command, streaming stdout/stderr to the console in real-time"\ + -help\ + "Run an external command and return the shellfilter exitinfo dict. + + stdout and stderr are written to the console as they arrive; + nothing is captured for the return value. The exitcode is + part of the returned dict — a non-zero exitcode is NOT raised + as a Tcl error, the caller must check it explicitly. + + Use -tcl to run a Tcl script/command instead of an external + process; in that case the returned dict carries a 'result' + key (or error/errorCode/errorInfo on failure) instead of an + exitcode." @leaders -min 0 -max 0 @opts - -nonewline -type none - -tcl -type none -default 0 - -debug -type none -default 0 - --timeout= -type integer + -nonewline -type none -help\ + "Accepted for compatibility with runout/runerr/runx. Has no + effect on run because run does not capture or return stdout." + -tcl -type none -default 0 -help\ + "Interpret cmdname (and cmdarg) as a Tcl command/script rather + than an external process." + -debug -type none -default 0 -help\ + "Enable debug diagnostics from shellfilter::run." + --timeout= -type integer -help\ + "Timeout in milliseconds for the external process." @values -min 1 -max -1 - cmdname -type string - cmdarg -type any -multiple 1 -optional 1 + cmdname -type string -help\ + "Name of the external command (or Tcl command when -tcl is + given)." + cmdarg -type any -multiple 1 -optional 1 -help\ + "Additional arguments passed through to the command." }] proc run {args} { #set_last_run_display [list] - #set splitargs [get_run_opts $args] - #set runopts [dict get $splitargs runopts] - #set runoptslong [dict get $splitargs runoptslong] - #set cmdargs [dict get $splitargs cmdargs] set argd [punk::args::parse $args withid ::shellrun::run] lassign [dict values $argd] leaders opts values received @@ -206,11 +176,25 @@ namespace eval shellrun { lappend PUNKARGS [list { @id -id ::shellrun::runconsole + @cmd -name "shellrun::runconsole"\ + -summary "Run a command via exec with stdout/stderr directed to the console, mimicking Tcl's unknown handler"\ + -help\ + "Resolve cmdname via auto_execok and exec it with stdout + and stderr redirected to the active console. The result is + returned via uplevel with the exec result/options dict, + mirroring the way Tcl's unknown proc handles external + commands. + + Unlike run, this uses exec semantics — there is no + exitcode in the return value and stdout/stderr are not + captured separately." @leaders -min 0 -max 0 @opts @values -min 1 -max -1 - cmdname -type string - cmdarg -type any -multiple 1 -optional 1 + cmdname -type string -help\ + "Name of the command to resolve via auto_execok and run." + cmdarg -type any -multiple 1 -optional 1 -help\ + "Additional arguments passed to the resolved command." }] #run in the way tcl unknown does - but without regard to auto_noexec proc runconsole {args} { @@ -256,16 +240,42 @@ namespace eval shellrun { } lappend PUNKARGS [list { @id -id ::shellrun::runout + @cmd -name "shellrun::runout"\ + -summary "Run an external command and return its stdout"\ + -help\ + "Run an external command, capture stdout and stderr, and + return stdout as the result. stderr and the exitcode are + emitted via repl telemetry chunks but are not part of the + return value. + + By default no console output is produced while the command + runs; use -echo to tee stdout and stderr to the console in + real-time. Use -nonewline to strip a trailing newline from + the returned stdout. + + Use -tcl to run a Tcl script/command instead of an external + process." @leaders -min 0 -max 0 @opts - -echo -type none - -nonewline -type none - -tcl -type none -default 0 - -debug -type none -default 0 - --timeout= -type integer + -echo -type none -help\ + "Tee stdout and stderr to the console in real-time while still + capturing them for the return value." + -nonewline -type none -help\ + "Strip a trailing newline (\\r\\n or \\n) from the returned + stdout." + -tcl -type none -default 0 -help\ + "Interpret cmdname (and cmdarg) as a Tcl command/script rather + than an external process." + -debug -type none -default 0 -help\ + "Enable debug diagnostics from shellfilter::run." + --timeout= -type integer -help\ + "Timeout in milliseconds for the external process." @values -min 1 -max -1 - cmdname -type string - cmdarg -type any -multiple 1 -optional 1 + cmdname -type string -help\ + "Name of the external command (or Tcl command when -tcl is + given)." + cmdarg -type any -multiple 1 -optional 1 -help\ + "Additional arguments passed through to the command." }] proc runout {args} { set argd [punk::args::parse $args withid ::shellrun::runout] @@ -284,11 +294,6 @@ namespace eval shellrun { set runerr "" set RST [a] - #set splitargs [get_run_opts $args] - #set runopts [dict get $splitargs runopts] - #set cmdargs [dict get $splitargs cmdargs] - - #puts stdout "RUNOUT cmdargs: $cmdargs" #todo add -data boolean and -data lastwrite to -settings with default being -data all @@ -422,16 +427,42 @@ namespace eval shellrun { lappend PUNKARGS [list { @id -id ::shellrun::runerr + @cmd -name "shellrun::runerr"\ + -summary "Run an external command and return its stderr"\ + -help\ + "Run an external command, capture stdout and stderr, and + return stderr as the result. stdout and the exitcode are + emitted via repl telemetry chunks but are not part of the + return value. + + By default no console output is produced while the command + runs; use -echo to tee stdout and stderr to the console in + real-time. Use -nonewline to strip a trailing newline from + the returned stderr. + + Use -tcl to run a Tcl script/command instead of an external + process." @leaders -min 0 -max 0 @opts - -echo -type none - -nonewline -type none - -tcl -type none -default 0 - -debug -type none -default 0 - --timeout= -type integer + -echo -type none -help\ + "Tee stdout and stderr to the console in real-time while still + capturing them for the return value." + -nonewline -type none -help\ + "Strip a trailing newline (\\r\\n or \\n) from the returned + stderr." + -tcl -type none -default 0 -help\ + "Interpret cmdname (and cmdarg) as a Tcl command/script rather + than an external process." + -debug -type none -default 0 -help\ + "Enable debug diagnostics from shellfilter::run." + --timeout= -type integer -help\ + "Timeout in milliseconds for the external process." @values -min 1 -max -1 - cmdname -type string - cmdarg -type any -multiple 1 -optional 1 + cmdname -type string -help\ + "Name of the external command (or Tcl command when -tcl is + given)." + cmdarg -type any -multiple 1 -optional 1 -help\ + "Additional arguments passed through to the command." }] proc runerr {args} { set argd [punk::args::parse $args withid ::shellrun::runerr] @@ -449,10 +480,6 @@ namespace eval shellrun { set runout "" set runerr "" - #set splitargs [get_run_opts $args] - #set runopts [dict get $splitargs runopts] - #set cmdargs [dict get $splitargs cmdargs] - set callopts [dict create] if {[dict exists $received "-tcl"]} { dict set callopts -tclscript 1 @@ -567,6 +594,43 @@ namespace eval shellrun { } + lappend PUNKARGS [list { + @id -id ::shellrun::runx + @cmd -name "shellrun::runx"\ + -summary "Run an external command and return a dict of stdout, stderr, and exitcode"\ + -help\ + "Run an external command, capture stdout and stderr, and + return a dict containing stdout, stderr, and exitcode (or + result/error/errorCode/errorInfo for -tcl calls). + + No console output is produced unless -echo is given. Use + -nonewline to strip trailing newlines from captured stdout + and stderr before returning. + + Use -tcl to run a Tcl script/command instead of an external + process." + @leaders -min 0 -max 0 + @opts + -echo -type none -help\ + "Tee stdout and stderr to the console in real-time while still + capturing them for the return dict." + -nonewline -type none -help\ + "Strip a trailing newline (\\r\\n or \\n) from captured stdout + and stderr in the returned dict." + -tcl -type none -default 0 -help\ + "Interpret cmdname (and cmdarg) as a Tcl command/script rather + than an external process." + -debug -type none -default 0 -help\ + "Enable debug diagnostics from shellfilter::run." + --timeout= -type integer -help\ + "Timeout in milliseconds for the external process." + @values -min 1 -max -1 + cmdname -type string -help\ + "Name of the external command (or Tcl command when -tcl is + given)." + cmdarg -type any -multiple 1 -optional 1 -help\ + "Additional arguments passed through to the command." + }] proc runx {args} { #set_last_run_display [list] variable runout @@ -574,11 +638,10 @@ namespace eval shellrun { set runout "" set runerr "" - set splitargs [get_run_opts $args] - set runopts [dict get $splitargs runopts] - set cmdargs [dict get $splitargs cmdargs] + set argd [punk::args::parse $args withid ::shellrun::runx] + lassign [dict values $argd] leaders opts values received - if {"-nonewline" in $runopts} { + if {[dict exists $received "-nonewline"]} { set nonewline 1 } else { set nonewline 0 @@ -586,7 +649,7 @@ namespace eval shellrun { #shellfilter::stack::remove stdout $::repl::id_outstack - if {"-echo" in $runopts} { + if {[dict exists $received "-echo"]} { #float to ensure repl transform doesn't interfere with the output data set stderr_stackid [shellfilter::stack::add stderr tee_to_var -action float -settings {-varname ::shellrun::runerr}] set stdout_stackid [shellfilter::stack::add stdout tee_to_var -action float -settings {-varname ::shellrun::runout}] @@ -601,10 +664,23 @@ namespace eval shellrun { set stdout_stackid [shellfilter::stack::add stdout var -action float-locked -junction 1 -settings {-varname ::shellrun::runout}] } - set callopts "" - if {"-tcl" in $runopts} { - append callopts " -tclscript 1" + set callopts [dict create] + if {[dict exists $received "-tcl"]} { + dict set callopts -tclscript 1 + } + if {[dict exists $received "-debug"]} { + dict set callopts -debug 1 + } + if {[dict exists $received --timeout]} { + dict set callopts -timeout [dict get $opts --timeout] + } + set cmdname [dict get $values cmdname] + if {[dict exists $received cmdarg]} { + set cmdarglist [dict get $values cmdarg] + } else { + set cmdarglist {} } + set cmdargs [concat $cmdname $cmdarglist] #set exitinfo [shellfilter::run $cmdargs -teehandle punksh -inbuffering none -outbuffering none -stdinhandler ::repl::repl_handler] set exitinfo [shellfilter::run $cmdargs {*}$callopts -teehandle punksh -inbuffering none -outbuffering none] @@ -616,7 +692,7 @@ namespace eval shellrun { flush stdout if {[dict exists $exitinfo error]} { - if {"-tcl" in $runopts} { + if {[dict exists $received "-tcl"]} { } else { #todo - check errorInfo makes sense.. return -code? tailcall? @@ -826,30 +902,161 @@ namespace eval shellrun { return $exitinfo } + lappend PUNKARGS [list { + @id -id ::shellrun::sh_run + @cmd -name "shellrun::sh_run"\ + -summary "Run a command line via sh -c, streaming stdout/stderr to the console"\ + -help\ + "Wrapper around shellrun::run that invokes the command line + via sh -c. The entire command line after options is passed + as a single argument to sh -c. + + Accepts the same options as shellrun::run except -tcl, + which is not meaningful for the sh_* wrappers because they + always invoke an external shell." + @leaders -min 0 -max 0 + @opts + -nonewline -type none -help\ + "Accepted for compatibility. Has no effect on run-based + commands because run does not capture output." + -debug -type none -default 0 -help\ + "Enable debug diagnostics from shellfilter::run." + --timeout= -type integer -help\ + "Timeout in milliseconds for the external process." + @values -min 1 -max -1 + cmdargs -type any -multiple 1 -help\ + "Command line to pass to sh -c as a single string argument." + }] proc sh_run {args} { - set splitargs [get_run_opts $args] - set runopts [dict get $splitargs runopts] - set cmdargs [dict get $splitargs cmdargs] + set argd [punk::args::parse $args withid ::shellrun::sh_run] + lassign [dict values $argd] leaders opts values received + set cmdargs [dict get $values cmdargs] + set runopts [list] + if {[dict exists $received -nonewline]} { lappend runopts -nonewline } + if {[dict exists $received -debug]} { lappend runopts -debug } + if {[dict exists $received --timeout]} { lappend runopts --timeout=[dict get $opts --timeout] } #e.g sh -c "ls -l *" #we pass cmdargs to sh -c as a list, not individually tailcall shellrun::run {*}$runopts sh -c $cmdargs } + + lappend PUNKARGS [list { + @id -id ::shellrun::sh_runout + @cmd -name "shellrun::sh_runout"\ + -summary "Run a command line via sh -c and return its stdout"\ + -help\ + "Wrapper around shellrun::runout that invokes the command + line via sh -c. The entire command line after options is + passed as a single argument to sh -c. + + Accepts the same options as shellrun::runout except -tcl, + which is not meaningful for the sh_* wrappers because they + always invoke an external shell." + @leaders -min 0 -max 0 + @opts + -echo -type none -help\ + "Tee stdout and stderr to the console in real-time while still + capturing them for the return value." + -nonewline -type none -help\ + "Strip a trailing newline (\\r\\n or \\n) from the returned + stdout." + -debug -type none -default 0 -help\ + "Enable debug diagnostics from shellfilter::run." + --timeout= -type integer -help\ + "Timeout in milliseconds for the external process." + @values -min 1 -max -1 + cmdargs -type any -multiple 1 -help\ + "Command line to pass to sh -c as a single string argument." + }] proc sh_runout {args} { - set splitargs [get_run_opts $args] - set runopts [dict get $splitargs runopts] - set cmdargs [dict get $splitargs cmdargs] + set argd [punk::args::parse $args withid ::shellrun::sh_runout] + lassign [dict values $argd] leaders opts values received + set cmdargs [dict get $values cmdargs] + set runopts [list] + if {[dict exists $received -echo]} { lappend runopts -echo } + if {[dict exists $received -nonewline]} { lappend runopts -nonewline } + if {[dict exists $received -debug]} { lappend runopts -debug } + if {[dict exists $received --timeout]} { lappend runopts --timeout=[dict get $opts --timeout] } tailcall shellrun::runout {*}$runopts sh -c $cmdargs } + + lappend PUNKARGS [list { + @id -id ::shellrun::sh_runerr + @cmd -name "shellrun::sh_runerr"\ + -summary "Run a command line via sh -c and return its stderr"\ + -help\ + "Wrapper around shellrun::runerr that invokes the command + line via sh -c. The entire command line after options is + passed as a single argument to sh -c. + + Accepts the same options as shellrun::runerr except -tcl, + which is not meaningful for the sh_* wrappers because they + always invoke an external shell." + @leaders -min 0 -max 0 + @opts + -echo -type none -help\ + "Tee stdout and stderr to the console in real-time while still + capturing them for the return value." + -nonewline -type none -help\ + "Strip a trailing newline (\\r\\n or \\n) from the returned + stderr." + -debug -type none -default 0 -help\ + "Enable debug diagnostics from shellfilter::run." + --timeout= -type integer -help\ + "Timeout in milliseconds for the external process." + @values -min 1 -max -1 + cmdargs -type any -multiple 1 -help\ + "Command line to pass to sh -c as a single string argument." + }] proc sh_runerr {args} { - set splitargs [get_run_opts $args] - set runopts [dict get $splitargs runopts] - set cmdargs [dict get $splitargs cmdargs] + set argd [punk::args::parse $args withid ::shellrun::sh_runerr] + lassign [dict values $argd] leaders opts values received + set cmdargs [dict get $values cmdargs] + set runopts [list] + if {[dict exists $received -echo]} { lappend runopts -echo } + if {[dict exists $received -nonewline]} { lappend runopts -nonewline } + if {[dict exists $received -debug]} { lappend runopts -debug } + if {[dict exists $received --timeout]} { lappend runopts --timeout=[dict get $opts --timeout] } tailcall shellrun::runerr {*}$runopts sh -c $cmdargs } + + lappend PUNKARGS [list { + @id -id ::shellrun::sh_runx + @cmd -name "shellrun::sh_runx"\ + -summary "Run a command line via sh -c and return a dict of stdout, stderr, and exitcode"\ + -help\ + "Wrapper around shellrun::runx that invokes the command + line via sh -c. The entire command line after options is + passed as a single argument to sh -c. + + Accepts the same options as shellrun::runx except -tcl, + which is not meaningful for the sh_* wrappers because they + always invoke an external shell." + @leaders -min 0 -max 0 + @opts + -echo -type none -help\ + "Tee stdout and stderr to the console in real-time while still + capturing them for the return dict." + -nonewline -type none -help\ + "Strip a trailing newline (\\r\\n or \\n) from captured stdout + and stderr in the returned dict." + -debug -type none -default 0 -help\ + "Enable debug diagnostics from shellfilter::run." + --timeout= -type integer -help\ + "Timeout in milliseconds for the external process." + @values -min 1 -max -1 + cmdargs -type any -multiple 1 -help\ + "Command line to pass to sh -c as a single string argument." + }] proc sh_runx {args} { - set splitargs [get_run_opts $args] - set runopts [dict get $splitargs runopts] - set cmdargs [dict get $splitargs cmdargs] + set argd [punk::args::parse $args withid ::shellrun::sh_runx] + lassign [dict values $argd] leaders opts values received + set cmdargs [dict get $values cmdargs] + set runopts [list] + if {[dict exists $received -echo]} { lappend runopts -echo } + if {[dict exists $received -nonewline]} { lappend runopts -nonewline } + if {[dict exists $received -debug]} { lappend runopts -debug } + if {[dict exists $received --timeout]} { lappend runopts --timeout=[dict get $opts --timeout] } tailcall shellrun::runx {*}$runopts sh -c $cmdargs } } @@ -895,5 +1102,5 @@ namespace eval ::punk::args::register { package provide shellrun [namespace eval shellrun { variable version - set version 0.1.2 + set version 0.1.4 }] diff --git a/src/make.tcl b/src/make.tcl index 4a936c7b..a45272bf 100644 --- a/src/make.tcl +++ b/src/make.tcl @@ -1972,59 +1972,12 @@ if {$::punkboot::command eq "vendorupdate"} { #puts "-- [tcl::tm::list] --" puts stdout "Updating vendor modules in src folder" - #Walk up from a vendor source path to the nearest fossil and/or git root and report uncommitted changes. - #Vendored artifacts built from a dirty source tree have no committed provenance (the built module files - #may correspond to no checked-in state) - warn so the source project can be committed first. - #checkedrootsvar names a dict in the caller used to report each VCS root at most once per run. - #Returns a list of warning lines (empty if clean, unversioned, already reported, or state cannot be determined). - proc vendor_source_dirty_warnings {srcpath checkedrootsvar} { - upvar 1 $checkedrootsvar checkedroots - set warnings [list] - set dir [file normalize $srcpath] - if {![file isdirectory $dir]} { - return $warnings ;#missing source location warned about elsewhere - } - set fossilroot "" - set gitroot "" - while {1} { - if {$fossilroot eq "" && ([file exists [file join $dir .fslckout]] || [file exists [file join $dir _FOSSIL_]])} { - set fossilroot $dir - } - if {$gitroot eq "" && [file exists [file join $dir .git]]} { - set gitroot $dir - } - if {$fossilroot ne "" || $gitroot ne ""} { - break ;#report at the nearest VCS root(s); a dual git+fossil root is caught in one pass - } - set parent [file dirname $dir] - if {$parent eq $dir} { break } - set dir $parent - } - if {$fossilroot ne "" && ![dict exists $checkedroots fossil,$fossilroot] && [llength [auto_execok fossil]]} { - dict set checkedroots fossil,$fossilroot 1 - #fossil changes must run from within the checkout - set original_cwd [pwd] - if {[catch { - cd $fossilroot - set fchanges [string trim [exec {*}[auto_execok fossil] changes]] - } errM]} { - lappend warnings "WARNING: vendorupdate could not determine fossil state of source project at $fossilroot ($errM)" - } elseif {$fchanges ne ""} { - lappend warnings "WARNING: vendorupdate source project at $fossilroot has uncommitted fossil changes ([llength [split $fchanges \n]] file(s)) - vendored artifacts built from a dirty tree have no committed provenance" - } - cd $original_cwd - } - if {$gitroot ne "" && ![dict exists $checkedroots git,$gitroot] && [llength [auto_execok git]]} { - dict set checkedroots git,$gitroot 1 - if {[catch { - set gchanges [string trim [exec {*}[auto_execok git] -C $gitroot status --porcelain]] - } errM]} { - lappend warnings "WARNING: vendorupdate could not determine git state of source project at $gitroot ($errM)" - } elseif {$gchanges ne ""} { - lappend warnings "WARNING: vendorupdate source project at $gitroot has uncommitted git changes ([llength [split $gchanges \n]] file(s)) - vendored artifacts built from a dirty tree have no committed provenance" - } - } - return $warnings + #dirty-checkout provenance warnings via punkboot::utils::vcs_dirty_warnings (loaded from bootsupport). + #Guarded require: vendorupdate must keep working even if the bootsupport punkboot::utils copy is + #missing or stale (it may be the very thing being repaired) - the check degrades to skipped. + set have_vcs_dirty_check [expr {![catch {package require punkboot::utils}] && [llength [info commands ::punkboot::utils::vcs_dirty_warnings]]}] + if {!$have_vcs_dirty_check} { + puts stderr "WARNING: vendorupdate dirty-checkout provenance check unavailable (punkboot::utils vcs_dirty_warnings not loadable from bootsupport) - continuing without it" } proc vendor_localupdate {projectroot} { @@ -2079,11 +2032,13 @@ if {$::punkboot::command eq "vendorupdate"} { set srclocation [file join $localpath $module_subpath] } #dirty-checkout provenance check - each VCS root reported at most once per run - set normsrclocation [file normalize $srclocation] - if {![dict exists $checked_source_paths $normsrclocation]} { - dict set checked_source_paths $normsrclocation 1 - foreach w [vendor_source_dirty_warnings $normsrclocation checked_vcs_roots] { - puts stderr $w + if {$::have_vcs_dirty_check} { + set normsrclocation [file normalize $srclocation] + if {![dict exists $checked_source_paths $normsrclocation]} { + dict set checked_source_paths $normsrclocation 1 + foreach w [::punkboot::utils::vcs_dirty_warnings $normsrclocation checked_vcs_roots vendorupdate] { + puts stderr $w + } } } #puts stdout "$relpath $module $module_subpath $srclocation" diff --git a/src/modules/punkboot/utils-999999.0a1.0.tm b/src/modules/punkboot/utils-999999.0a1.0.tm index 67d3db03..b65abd3e 100644 --- a/src/modules/punkboot/utils-999999.0a1.0.tm +++ b/src/modules/punkboot/utils-999999.0a1.0.tm @@ -23,7 +23,7 @@ package require Tcl 8.6- # ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++ namespace eval punkboot::utils { variable version 0.1.0 - namespace export parse_punkproject_version read_punkproject_version read_changelog_latest_version + namespace export parse_punkproject_version read_punkproject_version read_changelog_latest_version vcs_dirty_warnings namespace eval argdoc { @@ -115,6 +115,88 @@ namespace eval punkboot::utils { } return "" } + + + namespace eval argdoc { + lappend PUNKARGS [list { + @id -id ::punkboot::utils::vcs_dirty_warnings + @cmd -name "::punkboot::utils::vcs_dirty_warnings"\ + -summary\ + "Warning lines for uncommitted VCS changes above a path"\ + -help\ + "Walks up from path to the nearest fossil and/or git root and + returns warning lines describing uncommitted changes. + Artifacts built from a dirty source tree have no committed + provenance - callers such as make.tcl vendorupdate use this + to flag source projects that should be committed first. + + checkedrootsvar names a dict variable in the caller's scope + used to report each VCS root at most once across calls. + label, if supplied, is prefixed into each warning to name + the calling operation (e.g. 'vendorupdate'). + + Returns an empty list if the path is clean, unversioned, + missing, already reported, or state cannot be determined." + @leaders + path -type string -optional 0 -help\ + "Path to check; the nearest enclosing fossil/git root is examined" + checkedrootsvar -type string -optional 0 -help\ + "Name of a dict variable in the caller's scope for dedupe across calls" + label -type string -optional 1 -default "" -help\ + "Operation name prefixed into warnings" + }] + } + proc vcs_dirty_warnings {path checkedrootsvar {label ""}} { + upvar 1 $checkedrootsvar checkedroots + if {![info exists checkedroots]} {set checkedroots [dict create]} + if {$label ne ""} {set label "$label "} + set warnings [list] + set dir [file normalize $path] + if {![file isdirectory $dir]} { + return $warnings ;#missing path - not this proc's business to report + } + set fossilroot "" + set gitroot "" + while {1} { + if {$fossilroot eq "" && ([file exists [file join $dir .fslckout]] || [file exists [file join $dir _FOSSIL_]])} { + set fossilroot $dir + } + if {$gitroot eq "" && [file exists [file join $dir .git]]} { + set gitroot $dir + } + if {$fossilroot ne "" || $gitroot ne ""} { + break ;#report at the nearest VCS root(s); a dual git+fossil root is caught in one pass + } + set parent [file dirname $dir] + if {$parent eq $dir} { break } + set dir $parent + } + if {$fossilroot ne "" && ![dict exists $checkedroots fossil,$fossilroot] && [llength [auto_execok fossil]]} { + dict set checkedroots fossil,$fossilroot 1 + #fossil changes must run from within the checkout + set original_cwd [pwd] + if {[catch { + cd $fossilroot + set fchanges [string trim [exec {*}[auto_execok fossil] changes]] + } errM]} { + lappend warnings "WARNING: ${label}could not determine fossil state of source project at $fossilroot ($errM)" + } elseif {$fchanges ne ""} { + lappend warnings "WARNING: ${label}source project at $fossilroot has uncommitted fossil changes ([llength [split $fchanges \n]] file(s)) - artifacts built from a dirty tree have no committed provenance" + } + cd $original_cwd + } + if {$gitroot ne "" && ![dict exists $checkedroots git,$gitroot] && [llength [auto_execok git]]} { + dict set checkedroots git,$gitroot 1 + if {[catch { + set gchanges [string trim [exec {*}[auto_execok git] -C $gitroot status --porcelain]] + } errM]} { + lappend warnings "WARNING: ${label}could not determine git state of source project at $gitroot ($errM)" + } elseif {$gchanges ne ""} { + lappend warnings "WARNING: ${label}source project at $gitroot has uncommitted git changes ([llength [split $gchanges \n]] file(s)) - artifacts built from a dirty tree have no committed provenance" + } + } + return $warnings + } } namespace eval ::punk::args::register { diff --git a/src/modules/punkboot/utils-buildversion.txt b/src/modules/punkboot/utils-buildversion.txt index d0b26fee..98bd2835 100644 --- a/src/modules/punkboot/utils-buildversion.txt +++ b/src/modules/punkboot/utils-buildversion.txt @@ -1,4 +1,5 @@ -0.1.0 +0.1.1 #First line must be a semantic version number #all other lines are ignored. +#0.1.1 - added vcs_dirty_warnings (dirty fossil/git checkout provenance warnings; used by make.tcl vendorupdate, shared with bootsupport update per goal G-026) #0.1.0 - initial: parse_punkproject_version, read_punkproject_version, read_changelog_latest_version extracted from make.tcl inline procs diff --git a/src/project_layouts/custom/_project/punk.basic/src/make.tcl b/src/project_layouts/custom/_project/punk.basic/src/make.tcl index 19e78adb..4a936c7b 100644 --- a/src/project_layouts/custom/_project/punk.basic/src/make.tcl +++ b/src/project_layouts/custom/_project/punk.basic/src/make.tcl @@ -1972,10 +1972,67 @@ if {$::punkboot::command eq "vendorupdate"} { #puts "-- [tcl::tm::list] --" puts stdout "Updating vendor modules in src folder" + #Walk up from a vendor source path to the nearest fossil and/or git root and report uncommitted changes. + #Vendored artifacts built from a dirty source tree have no committed provenance (the built module files + #may correspond to no checked-in state) - warn so the source project can be committed first. + #checkedrootsvar names a dict in the caller used to report each VCS root at most once per run. + #Returns a list of warning lines (empty if clean, unversioned, already reported, or state cannot be determined). + proc vendor_source_dirty_warnings {srcpath checkedrootsvar} { + upvar 1 $checkedrootsvar checkedroots + set warnings [list] + set dir [file normalize $srcpath] + if {![file isdirectory $dir]} { + return $warnings ;#missing source location warned about elsewhere + } + set fossilroot "" + set gitroot "" + while {1} { + if {$fossilroot eq "" && ([file exists [file join $dir .fslckout]] || [file exists [file join $dir _FOSSIL_]])} { + set fossilroot $dir + } + if {$gitroot eq "" && [file exists [file join $dir .git]]} { + set gitroot $dir + } + if {$fossilroot ne "" || $gitroot ne ""} { + break ;#report at the nearest VCS root(s); a dual git+fossil root is caught in one pass + } + set parent [file dirname $dir] + if {$parent eq $dir} { break } + set dir $parent + } + if {$fossilroot ne "" && ![dict exists $checkedroots fossil,$fossilroot] && [llength [auto_execok fossil]]} { + dict set checkedroots fossil,$fossilroot 1 + #fossil changes must run from within the checkout + set original_cwd [pwd] + if {[catch { + cd $fossilroot + set fchanges [string trim [exec {*}[auto_execok fossil] changes]] + } errM]} { + lappend warnings "WARNING: vendorupdate could not determine fossil state of source project at $fossilroot ($errM)" + } elseif {$fchanges ne ""} { + lappend warnings "WARNING: vendorupdate source project at $fossilroot has uncommitted fossil changes ([llength [split $fchanges \n]] file(s)) - vendored artifacts built from a dirty tree have no committed provenance" + } + cd $original_cwd + } + if {$gitroot ne "" && ![dict exists $checkedroots git,$gitroot] && [llength [auto_execok git]]} { + dict set checkedroots git,$gitroot 1 + if {[catch { + set gchanges [string trim [exec {*}[auto_execok git] -C $gitroot status --porcelain]] + } errM]} { + lappend warnings "WARNING: vendorupdate could not determine git state of source project at $gitroot ($errM)" + } elseif {$gchanges ne ""} { + lappend warnings "WARNING: vendorupdate source project at $gitroot has uncommitted git changes ([llength [split $gchanges \n]] file(s)) - vendored artifacts built from a dirty tree have no committed provenance" + } + } + return $warnings + } + proc vendor_localupdate {projectroot} { set local_modules [list] set git_modules [list] set fossil_modules [list] + set checked_source_paths [dict create] ;#skip repeat walk-ups for identical source paths + set checked_vcs_roots [dict create] ;#report each VCS root at most once per run set sourcefolder $projectroot/src #todo vendor/lib set vendorlibfolders [glob -nocomplain -dir $sourcefolder -type d -tails vendorlib_tcl*] @@ -2021,6 +2078,14 @@ if {$::punkboot::command eq "vendorupdate"} { } else { set srclocation [file join $localpath $module_subpath] } + #dirty-checkout provenance check - each VCS root reported at most once per run + set normsrclocation [file normalize $srclocation] + if {![dict exists $checked_source_paths $normsrclocation]} { + dict set checked_source_paths $normsrclocation 1 + foreach w [vendor_source_dirty_warnings $normsrclocation checked_vcs_roots] { + puts stderr $w + } + } #puts stdout "$relpath $module $module_subpath $srclocation" #todo - check if requested_module has version extension and allow explicit versions instead of just latest diff --git a/src/project_layouts/custom/_project/punk.project-0.1/src/bootsupport/modules/punk/console-0.7.1.tm b/src/project_layouts/custom/_project/punk.project-0.1/src/bootsupport/modules/punk/console-0.7.1.tm index aac663a3..7a9936af 100644 --- a/src/project_layouts/custom/_project/punk.project-0.1/src/bootsupport/modules/punk/console-0.7.1.tm +++ b/src/project_layouts/custom/_project/punk.project-0.1/src/bootsupport/modules/punk/console-0.7.1.tm @@ -581,7 +581,10 @@ namespace eval punk::console { try { return [thread::send $route_owner [list ::punk::console::internal::get_ansi_response_payload -console $inoutchannels -expected_ms $expected -ignoreok $ignoreok -return $returntype -passthrough $passthrough $query $capturingendregex]] } on error {errM erropts} { - return -code error "get_ansi_response_payload query routed to console-owning thread $route_owner failed: $errM" + #preserve the owner-side errorcode (where the transport propagates it) so + #callers can discriminate, e.g. {PUNK CONSOLE QUERY HOSTAGE_COOKED_READ} + #from the idle-reader guard + return -code error -errorcode [dict get $erropts -errorcode] "get_ansi_response_payload query routed to console-owning thread $route_owner failed: $errM" } } @@ -611,6 +614,28 @@ namespace eval punk::console { error "punk::console::get_ansi_response_payload input channel '$input' is closed or at eof - cannot receive terminal response" } + #tcl 8.6 windows console idle-reader hostage guard (fail fast, no emission). + #With the console in cooked (line) mode and a readable handler armed on the input + #channel, the 8.6 channel driver has a blocking cooked-mode ReadConsole parked + #(arming is what posts it), and driver reads sample the console mode at issue time - + #the raw cycle below cannot rescue it. Any response would be swallowed by that read + #until the user presses Enter, then leak to the line reader as phantom input + #(~500ms timeout now, input corruption later). This is the idle-at-a-line-mode-prompt + #condition: the repl disarms its reader during command dispatch, so mid-command + #queries are unaffected, and raw mode reads cooperatively and is exempt. Typical + #trigger: a query from an after-script or worker thread firing while the shell sits + #at a line-mode prompt. Best-effort by design: a parked read can outlive a removed + #handler, so this catches the systematic case, not every conceivable one. + if {![tsv::get punk_console is_raw] && $input eq "stdin" && $::punk::console::has_twapi} { + set guard_conf [chan configure $input] + if {![dict exists $guard_conf -inputmode] && ![dict exists $guard_conf -mode] + && [chan event $input readable] ne "" + && ![catch {twapi::get_console_handle stdin}]} { + return -code error -errorcode [list PUNK CONSOLE QUERY HOSTAGE_COOKED_READ]\ + "punk::console::get_ansi_response_payload refusing to emit query on '$input': tcl 8.6 console is in cooked (line) mode with an armed readable handler - the driver's parked cooked read would swallow the response until Enter (query issued from an idle line-mode prompt?). Use raw mode, or issue queries while the reader is disarmed (e.g. during repl command evaluation)." + } + } + #chunks from input that need to be handled by readers upvar ::punk::console::input_chunks_waiting input_chunks_waiting diff --git a/src/project_layouts/custom/_project/punk.shell-0.1/src/bootsupport/modules/punkboot/utils-0.1.0.tm b/src/project_layouts/custom/_project/punk.project-0.1/src/bootsupport/modules/punkboot/utils-0.1.1.tm similarity index 50% rename from src/project_layouts/custom/_project/punk.shell-0.1/src/bootsupport/modules/punkboot/utils-0.1.0.tm rename to src/project_layouts/custom/_project/punk.project-0.1/src/bootsupport/modules/punkboot/utils-0.1.1.tm index fa10b93e..be23f56e 100644 --- a/src/project_layouts/custom/_project/punk.shell-0.1/src/bootsupport/modules/punkboot/utils-0.1.0.tm +++ b/src/project_layouts/custom/_project/punk.project-0.1/src/bootsupport/modules/punkboot/utils-0.1.1.tm @@ -7,7 +7,7 @@ # (C) 2023 # # @@ Meta Begin -# Application punkboot::utils 0.1.0 +# Application punkboot::utils 0.1.1 # Meta platform tcl # Meta license BSD # @@ Meta End @@ -23,7 +23,7 @@ package require Tcl 8.6- # ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++ namespace eval punkboot::utils { variable version 0.1.0 - namespace export parse_punkproject_version read_punkproject_version read_changelog_latest_version + namespace export parse_punkproject_version read_punkproject_version read_changelog_latest_version vcs_dirty_warnings namespace eval argdoc { @@ -115,6 +115,88 @@ namespace eval punkboot::utils { } return "" } + + + namespace eval argdoc { + lappend PUNKARGS [list { + @id -id ::punkboot::utils::vcs_dirty_warnings + @cmd -name "::punkboot::utils::vcs_dirty_warnings"\ + -summary\ + "Warning lines for uncommitted VCS changes above a path"\ + -help\ + "Walks up from path to the nearest fossil and/or git root and + returns warning lines describing uncommitted changes. + Artifacts built from a dirty source tree have no committed + provenance - callers such as make.tcl vendorupdate use this + to flag source projects that should be committed first. + + checkedrootsvar names a dict variable in the caller's scope + used to report each VCS root at most once across calls. + label, if supplied, is prefixed into each warning to name + the calling operation (e.g. 'vendorupdate'). + + Returns an empty list if the path is clean, unversioned, + missing, already reported, or state cannot be determined." + @leaders + path -type string -optional 0 -help\ + "Path to check; the nearest enclosing fossil/git root is examined" + checkedrootsvar -type string -optional 0 -help\ + "Name of a dict variable in the caller's scope for dedupe across calls" + label -type string -optional 1 -default "" -help\ + "Operation name prefixed into warnings" + }] + } + proc vcs_dirty_warnings {path checkedrootsvar {label ""}} { + upvar 1 $checkedrootsvar checkedroots + if {![info exists checkedroots]} {set checkedroots [dict create]} + if {$label ne ""} {set label "$label "} + set warnings [list] + set dir [file normalize $path] + if {![file isdirectory $dir]} { + return $warnings ;#missing path - not this proc's business to report + } + set fossilroot "" + set gitroot "" + while {1} { + if {$fossilroot eq "" && ([file exists [file join $dir .fslckout]] || [file exists [file join $dir _FOSSIL_]])} { + set fossilroot $dir + } + if {$gitroot eq "" && [file exists [file join $dir .git]]} { + set gitroot $dir + } + if {$fossilroot ne "" || $gitroot ne ""} { + break ;#report at the nearest VCS root(s); a dual git+fossil root is caught in one pass + } + set parent [file dirname $dir] + if {$parent eq $dir} { break } + set dir $parent + } + if {$fossilroot ne "" && ![dict exists $checkedroots fossil,$fossilroot] && [llength [auto_execok fossil]]} { + dict set checkedroots fossil,$fossilroot 1 + #fossil changes must run from within the checkout + set original_cwd [pwd] + if {[catch { + cd $fossilroot + set fchanges [string trim [exec {*}[auto_execok fossil] changes]] + } errM]} { + lappend warnings "WARNING: ${label}could not determine fossil state of source project at $fossilroot ($errM)" + } elseif {$fchanges ne ""} { + lappend warnings "WARNING: ${label}source project at $fossilroot has uncommitted fossil changes ([llength [split $fchanges \n]] file(s)) - artifacts built from a dirty tree have no committed provenance" + } + cd $original_cwd + } + if {$gitroot ne "" && ![dict exists $checkedroots git,$gitroot] && [llength [auto_execok git]]} { + dict set checkedroots git,$gitroot 1 + if {[catch { + set gchanges [string trim [exec {*}[auto_execok git] -C $gitroot status --porcelain]] + } errM]} { + lappend warnings "WARNING: ${label}could not determine git state of source project at $gitroot ($errM)" + } elseif {$gchanges ne ""} { + lappend warnings "WARNING: ${label}source project at $gitroot has uncommitted git changes ([llength [split $gchanges \n]] file(s)) - artifacts built from a dirty tree have no committed provenance" + } + } + return $warnings + } } namespace eval ::punk::args::register { @@ -125,8 +207,8 @@ namespace eval ::punk::args::register { ## Ready package provide punkboot::utils [tcl::namespace::eval punkboot::utils { variable version - #- this version number, exactly 0.1.0, is a literal used in src module folders + #- this version number, exactly 0.1.1, is a literal used in src module folders #- we refer to this sometimes as the magic version number - set version 0.1.0 + set version 0.1.1 }] return diff --git a/src/bootsupport/modules/shellrun-0.1.2.tm b/src/project_layouts/custom/_project/punk.project-0.1/src/bootsupport/modules/shellrun-0.1.4.tm similarity index 66% rename from src/bootsupport/modules/shellrun-0.1.2.tm rename to src/project_layouts/custom/_project/punk.project-0.1/src/bootsupport/modules/shellrun-0.1.4.tm index 07bca217..08538dec 100644 --- a/src/bootsupport/modules/shellrun-0.1.2.tm +++ b/src/project_layouts/custom/_project/punk.project-0.1/src/bootsupport/modules/shellrun-0.1.4.tm @@ -5,6 +5,7 @@ package require shellfilter package require punk::ansi +package require punk::args #NOTE: the run,runout,runerr,runx commands only produce an error if the command didn't run. # - If it did run, but there was a non-zero exitcode it is up to the application to check that. @@ -79,76 +80,45 @@ namespace eval shellrun { - #maintenance: similar used in punk::ns & punk::winrun - #todo - take runopts + aliases as args - #longopts must be passed as a single item ie --timeout=100 not --timeout 100 - proc get_run_opts {arglist} { - if {[catch { - set callerinfo [info level -1] - } errM]} { - set caller "" - } else { - set caller [lindex $callerinfo 0] - } - - #we provide -nonewline even for 'run' even though run doesn't deliver stderr or stdout to the tcl return value - #This is for compatibility with other runX commands, and the difference is also visible when calling from repl. - set known_runopts [list "-echo" "-e" "-nonewline" "-n" "-tcl" "-debug"] - set known_longopts [list "--timeout"] - set known_longopts_msg "" - foreach lng $known_longopts { - append known_longopts_msg "${lng}=val " - } - set aliases [list "-e" "-echo" "-echo" "-echo" "-n" "-nonewline" "-nonewline" "-nonewline" "-tcl" "-tcl" "-debug" "-debug"] ;#include map to self - set runopts [list] - set runoptslong [list] - set cmdargs [list] - - set idx_first_cmdarg [lsearch -not $arglist "-*"] - - set allopts [lrange $arglist 0 $idx_first_cmdarg-1] - set cmdargs [lrange $arglist $idx_first_cmdarg end] - foreach o $allopts { - if {[string match --* $o]} { - lassign [split $o =] flagpart valpart - if {$valpart eq ""} { - error "$caller: longopt $o seems to be missing a value - must be of form --option=value" - } - if {$flagpart ni $known_longopts} { - error "$caller: Unknown runoption $o - known options $known_runopts $known_longopts_msg" - } - lappend runoptslong $flagpart $valpart - } else { - if {$o ni $known_runopts} { - error "$caller: Unknown runoption $o - known options $known_runopts $known_longopts_msg" - } - lappend runopts [dict get $aliases $o] - } - } - return [list runopts $runopts runoptslong $runoptslong cmdargs $cmdargs] - } - - #todo - investigate cause of punk86 run hanging sometimes. An 'after 500' before exit in the called script fixes the issue. punk87 doesn't seem to be affected. lappend PUNKARGS [list { @id -id ::shellrun::run + @cmd -name "shellrun::run"\ + -summary "Run an external command, streaming stdout/stderr to the console in real-time"\ + -help\ + "Run an external command and return the shellfilter exitinfo dict. + + stdout and stderr are written to the console as they arrive; + nothing is captured for the return value. The exitcode is + part of the returned dict — a non-zero exitcode is NOT raised + as a Tcl error, the caller must check it explicitly. + + Use -tcl to run a Tcl script/command instead of an external + process; in that case the returned dict carries a 'result' + key (or error/errorCode/errorInfo on failure) instead of an + exitcode." @leaders -min 0 -max 0 @opts - -nonewline -type none - -tcl -type none -default 0 - -debug -type none -default 0 - --timeout= -type integer + -nonewline -type none -help\ + "Accepted for compatibility with runout/runerr/runx. Has no + effect on run because run does not capture or return stdout." + -tcl -type none -default 0 -help\ + "Interpret cmdname (and cmdarg) as a Tcl command/script rather + than an external process." + -debug -type none -default 0 -help\ + "Enable debug diagnostics from shellfilter::run." + --timeout= -type integer -help\ + "Timeout in milliseconds for the external process." @values -min 1 -max -1 - cmdname -type string - cmdarg -type any -multiple 1 -optional 1 + cmdname -type string -help\ + "Name of the external command (or Tcl command when -tcl is + given)." + cmdarg -type any -multiple 1 -optional 1 -help\ + "Additional arguments passed through to the command." }] proc run {args} { #set_last_run_display [list] - #set splitargs [get_run_opts $args] - #set runopts [dict get $splitargs runopts] - #set runoptslong [dict get $splitargs runoptslong] - #set cmdargs [dict get $splitargs cmdargs] set argd [punk::args::parse $args withid ::shellrun::run] lassign [dict values $argd] leaders opts values received @@ -206,11 +176,25 @@ namespace eval shellrun { lappend PUNKARGS [list { @id -id ::shellrun::runconsole + @cmd -name "shellrun::runconsole"\ + -summary "Run a command via exec with stdout/stderr directed to the console, mimicking Tcl's unknown handler"\ + -help\ + "Resolve cmdname via auto_execok and exec it with stdout + and stderr redirected to the active console. The result is + returned via uplevel with the exec result/options dict, + mirroring the way Tcl's unknown proc handles external + commands. + + Unlike run, this uses exec semantics — there is no + exitcode in the return value and stdout/stderr are not + captured separately." @leaders -min 0 -max 0 @opts @values -min 1 -max -1 - cmdname -type string - cmdarg -type any -multiple 1 -optional 1 + cmdname -type string -help\ + "Name of the command to resolve via auto_execok and run." + cmdarg -type any -multiple 1 -optional 1 -help\ + "Additional arguments passed to the resolved command." }] #run in the way tcl unknown does - but without regard to auto_noexec proc runconsole {args} { @@ -256,16 +240,42 @@ namespace eval shellrun { } lappend PUNKARGS [list { @id -id ::shellrun::runout + @cmd -name "shellrun::runout"\ + -summary "Run an external command and return its stdout"\ + -help\ + "Run an external command, capture stdout and stderr, and + return stdout as the result. stderr and the exitcode are + emitted via repl telemetry chunks but are not part of the + return value. + + By default no console output is produced while the command + runs; use -echo to tee stdout and stderr to the console in + real-time. Use -nonewline to strip a trailing newline from + the returned stdout. + + Use -tcl to run a Tcl script/command instead of an external + process." @leaders -min 0 -max 0 @opts - -echo -type none - -nonewline -type none - -tcl -type none -default 0 - -debug -type none -default 0 - --timeout= -type integer + -echo -type none -help\ + "Tee stdout and stderr to the console in real-time while still + capturing them for the return value." + -nonewline -type none -help\ + "Strip a trailing newline (\\r\\n or \\n) from the returned + stdout." + -tcl -type none -default 0 -help\ + "Interpret cmdname (and cmdarg) as a Tcl command/script rather + than an external process." + -debug -type none -default 0 -help\ + "Enable debug diagnostics from shellfilter::run." + --timeout= -type integer -help\ + "Timeout in milliseconds for the external process." @values -min 1 -max -1 - cmdname -type string - cmdarg -type any -multiple 1 -optional 1 + cmdname -type string -help\ + "Name of the external command (or Tcl command when -tcl is + given)." + cmdarg -type any -multiple 1 -optional 1 -help\ + "Additional arguments passed through to the command." }] proc runout {args} { set argd [punk::args::parse $args withid ::shellrun::runout] @@ -284,11 +294,6 @@ namespace eval shellrun { set runerr "" set RST [a] - #set splitargs [get_run_opts $args] - #set runopts [dict get $splitargs runopts] - #set cmdargs [dict get $splitargs cmdargs] - - #puts stdout "RUNOUT cmdargs: $cmdargs" #todo add -data boolean and -data lastwrite to -settings with default being -data all @@ -422,16 +427,42 @@ namespace eval shellrun { lappend PUNKARGS [list { @id -id ::shellrun::runerr + @cmd -name "shellrun::runerr"\ + -summary "Run an external command and return its stderr"\ + -help\ + "Run an external command, capture stdout and stderr, and + return stderr as the result. stdout and the exitcode are + emitted via repl telemetry chunks but are not part of the + return value. + + By default no console output is produced while the command + runs; use -echo to tee stdout and stderr to the console in + real-time. Use -nonewline to strip a trailing newline from + the returned stderr. + + Use -tcl to run a Tcl script/command instead of an external + process." @leaders -min 0 -max 0 @opts - -echo -type none - -nonewline -type none - -tcl -type none -default 0 - -debug -type none -default 0 - --timeout= -type integer + -echo -type none -help\ + "Tee stdout and stderr to the console in real-time while still + capturing them for the return value." + -nonewline -type none -help\ + "Strip a trailing newline (\\r\\n or \\n) from the returned + stderr." + -tcl -type none -default 0 -help\ + "Interpret cmdname (and cmdarg) as a Tcl command/script rather + than an external process." + -debug -type none -default 0 -help\ + "Enable debug diagnostics from shellfilter::run." + --timeout= -type integer -help\ + "Timeout in milliseconds for the external process." @values -min 1 -max -1 - cmdname -type string - cmdarg -type any -multiple 1 -optional 1 + cmdname -type string -help\ + "Name of the external command (or Tcl command when -tcl is + given)." + cmdarg -type any -multiple 1 -optional 1 -help\ + "Additional arguments passed through to the command." }] proc runerr {args} { set argd [punk::args::parse $args withid ::shellrun::runerr] @@ -449,10 +480,6 @@ namespace eval shellrun { set runout "" set runerr "" - #set splitargs [get_run_opts $args] - #set runopts [dict get $splitargs runopts] - #set cmdargs [dict get $splitargs cmdargs] - set callopts [dict create] if {[dict exists $received "-tcl"]} { dict set callopts -tclscript 1 @@ -567,6 +594,43 @@ namespace eval shellrun { } + lappend PUNKARGS [list { + @id -id ::shellrun::runx + @cmd -name "shellrun::runx"\ + -summary "Run an external command and return a dict of stdout, stderr, and exitcode"\ + -help\ + "Run an external command, capture stdout and stderr, and + return a dict containing stdout, stderr, and exitcode (or + result/error/errorCode/errorInfo for -tcl calls). + + No console output is produced unless -echo is given. Use + -nonewline to strip trailing newlines from captured stdout + and stderr before returning. + + Use -tcl to run a Tcl script/command instead of an external + process." + @leaders -min 0 -max 0 + @opts + -echo -type none -help\ + "Tee stdout and stderr to the console in real-time while still + capturing them for the return dict." + -nonewline -type none -help\ + "Strip a trailing newline (\\r\\n or \\n) from captured stdout + and stderr in the returned dict." + -tcl -type none -default 0 -help\ + "Interpret cmdname (and cmdarg) as a Tcl command/script rather + than an external process." + -debug -type none -default 0 -help\ + "Enable debug diagnostics from shellfilter::run." + --timeout= -type integer -help\ + "Timeout in milliseconds for the external process." + @values -min 1 -max -1 + cmdname -type string -help\ + "Name of the external command (or Tcl command when -tcl is + given)." + cmdarg -type any -multiple 1 -optional 1 -help\ + "Additional arguments passed through to the command." + }] proc runx {args} { #set_last_run_display [list] variable runout @@ -574,11 +638,10 @@ namespace eval shellrun { set runout "" set runerr "" - set splitargs [get_run_opts $args] - set runopts [dict get $splitargs runopts] - set cmdargs [dict get $splitargs cmdargs] + set argd [punk::args::parse $args withid ::shellrun::runx] + lassign [dict values $argd] leaders opts values received - if {"-nonewline" in $runopts} { + if {[dict exists $received "-nonewline"]} { set nonewline 1 } else { set nonewline 0 @@ -586,7 +649,7 @@ namespace eval shellrun { #shellfilter::stack::remove stdout $::repl::id_outstack - if {"-echo" in $runopts} { + if {[dict exists $received "-echo"]} { #float to ensure repl transform doesn't interfere with the output data set stderr_stackid [shellfilter::stack::add stderr tee_to_var -action float -settings {-varname ::shellrun::runerr}] set stdout_stackid [shellfilter::stack::add stdout tee_to_var -action float -settings {-varname ::shellrun::runout}] @@ -601,10 +664,23 @@ namespace eval shellrun { set stdout_stackid [shellfilter::stack::add stdout var -action float-locked -junction 1 -settings {-varname ::shellrun::runout}] } - set callopts "" - if {"-tcl" in $runopts} { - append callopts " -tclscript 1" + set callopts [dict create] + if {[dict exists $received "-tcl"]} { + dict set callopts -tclscript 1 + } + if {[dict exists $received "-debug"]} { + dict set callopts -debug 1 + } + if {[dict exists $received --timeout]} { + dict set callopts -timeout [dict get $opts --timeout] + } + set cmdname [dict get $values cmdname] + if {[dict exists $received cmdarg]} { + set cmdarglist [dict get $values cmdarg] + } else { + set cmdarglist {} } + set cmdargs [concat $cmdname $cmdarglist] #set exitinfo [shellfilter::run $cmdargs -teehandle punksh -inbuffering none -outbuffering none -stdinhandler ::repl::repl_handler] set exitinfo [shellfilter::run $cmdargs {*}$callopts -teehandle punksh -inbuffering none -outbuffering none] @@ -616,7 +692,7 @@ namespace eval shellrun { flush stdout if {[dict exists $exitinfo error]} { - if {"-tcl" in $runopts} { + if {[dict exists $received "-tcl"]} { } else { #todo - check errorInfo makes sense.. return -code? tailcall? @@ -826,30 +902,161 @@ namespace eval shellrun { return $exitinfo } + lappend PUNKARGS [list { + @id -id ::shellrun::sh_run + @cmd -name "shellrun::sh_run"\ + -summary "Run a command line via sh -c, streaming stdout/stderr to the console"\ + -help\ + "Wrapper around shellrun::run that invokes the command line + via sh -c. The entire command line after options is passed + as a single argument to sh -c. + + Accepts the same options as shellrun::run except -tcl, + which is not meaningful for the sh_* wrappers because they + always invoke an external shell." + @leaders -min 0 -max 0 + @opts + -nonewline -type none -help\ + "Accepted for compatibility. Has no effect on run-based + commands because run does not capture output." + -debug -type none -default 0 -help\ + "Enable debug diagnostics from shellfilter::run." + --timeout= -type integer -help\ + "Timeout in milliseconds for the external process." + @values -min 1 -max -1 + cmdargs -type any -multiple 1 -help\ + "Command line to pass to sh -c as a single string argument." + }] proc sh_run {args} { - set splitargs [get_run_opts $args] - set runopts [dict get $splitargs runopts] - set cmdargs [dict get $splitargs cmdargs] + set argd [punk::args::parse $args withid ::shellrun::sh_run] + lassign [dict values $argd] leaders opts values received + set cmdargs [dict get $values cmdargs] + set runopts [list] + if {[dict exists $received -nonewline]} { lappend runopts -nonewline } + if {[dict exists $received -debug]} { lappend runopts -debug } + if {[dict exists $received --timeout]} { lappend runopts --timeout=[dict get $opts --timeout] } #e.g sh -c "ls -l *" #we pass cmdargs to sh -c as a list, not individually tailcall shellrun::run {*}$runopts sh -c $cmdargs } + + lappend PUNKARGS [list { + @id -id ::shellrun::sh_runout + @cmd -name "shellrun::sh_runout"\ + -summary "Run a command line via sh -c and return its stdout"\ + -help\ + "Wrapper around shellrun::runout that invokes the command + line via sh -c. The entire command line after options is + passed as a single argument to sh -c. + + Accepts the same options as shellrun::runout except -tcl, + which is not meaningful for the sh_* wrappers because they + always invoke an external shell." + @leaders -min 0 -max 0 + @opts + -echo -type none -help\ + "Tee stdout and stderr to the console in real-time while still + capturing them for the return value." + -nonewline -type none -help\ + "Strip a trailing newline (\\r\\n or \\n) from the returned + stdout." + -debug -type none -default 0 -help\ + "Enable debug diagnostics from shellfilter::run." + --timeout= -type integer -help\ + "Timeout in milliseconds for the external process." + @values -min 1 -max -1 + cmdargs -type any -multiple 1 -help\ + "Command line to pass to sh -c as a single string argument." + }] proc sh_runout {args} { - set splitargs [get_run_opts $args] - set runopts [dict get $splitargs runopts] - set cmdargs [dict get $splitargs cmdargs] + set argd [punk::args::parse $args withid ::shellrun::sh_runout] + lassign [dict values $argd] leaders opts values received + set cmdargs [dict get $values cmdargs] + set runopts [list] + if {[dict exists $received -echo]} { lappend runopts -echo } + if {[dict exists $received -nonewline]} { lappend runopts -nonewline } + if {[dict exists $received -debug]} { lappend runopts -debug } + if {[dict exists $received --timeout]} { lappend runopts --timeout=[dict get $opts --timeout] } tailcall shellrun::runout {*}$runopts sh -c $cmdargs } + + lappend PUNKARGS [list { + @id -id ::shellrun::sh_runerr + @cmd -name "shellrun::sh_runerr"\ + -summary "Run a command line via sh -c and return its stderr"\ + -help\ + "Wrapper around shellrun::runerr that invokes the command + line via sh -c. The entire command line after options is + passed as a single argument to sh -c. + + Accepts the same options as shellrun::runerr except -tcl, + which is not meaningful for the sh_* wrappers because they + always invoke an external shell." + @leaders -min 0 -max 0 + @opts + -echo -type none -help\ + "Tee stdout and stderr to the console in real-time while still + capturing them for the return value." + -nonewline -type none -help\ + "Strip a trailing newline (\\r\\n or \\n) from the returned + stderr." + -debug -type none -default 0 -help\ + "Enable debug diagnostics from shellfilter::run." + --timeout= -type integer -help\ + "Timeout in milliseconds for the external process." + @values -min 1 -max -1 + cmdargs -type any -multiple 1 -help\ + "Command line to pass to sh -c as a single string argument." + }] proc sh_runerr {args} { - set splitargs [get_run_opts $args] - set runopts [dict get $splitargs runopts] - set cmdargs [dict get $splitargs cmdargs] + set argd [punk::args::parse $args withid ::shellrun::sh_runerr] + lassign [dict values $argd] leaders opts values received + set cmdargs [dict get $values cmdargs] + set runopts [list] + if {[dict exists $received -echo]} { lappend runopts -echo } + if {[dict exists $received -nonewline]} { lappend runopts -nonewline } + if {[dict exists $received -debug]} { lappend runopts -debug } + if {[dict exists $received --timeout]} { lappend runopts --timeout=[dict get $opts --timeout] } tailcall shellrun::runerr {*}$runopts sh -c $cmdargs } + + lappend PUNKARGS [list { + @id -id ::shellrun::sh_runx + @cmd -name "shellrun::sh_runx"\ + -summary "Run a command line via sh -c and return a dict of stdout, stderr, and exitcode"\ + -help\ + "Wrapper around shellrun::runx that invokes the command + line via sh -c. The entire command line after options is + passed as a single argument to sh -c. + + Accepts the same options as shellrun::runx except -tcl, + which is not meaningful for the sh_* wrappers because they + always invoke an external shell." + @leaders -min 0 -max 0 + @opts + -echo -type none -help\ + "Tee stdout and stderr to the console in real-time while still + capturing them for the return dict." + -nonewline -type none -help\ + "Strip a trailing newline (\\r\\n or \\n) from captured stdout + and stderr in the returned dict." + -debug -type none -default 0 -help\ + "Enable debug diagnostics from shellfilter::run." + --timeout= -type integer -help\ + "Timeout in milliseconds for the external process." + @values -min 1 -max -1 + cmdargs -type any -multiple 1 -help\ + "Command line to pass to sh -c as a single string argument." + }] proc sh_runx {args} { - set splitargs [get_run_opts $args] - set runopts [dict get $splitargs runopts] - set cmdargs [dict get $splitargs cmdargs] + set argd [punk::args::parse $args withid ::shellrun::sh_runx] + lassign [dict values $argd] leaders opts values received + set cmdargs [dict get $values cmdargs] + set runopts [list] + if {[dict exists $received -echo]} { lappend runopts -echo } + if {[dict exists $received -nonewline]} { lappend runopts -nonewline } + if {[dict exists $received -debug]} { lappend runopts -debug } + if {[dict exists $received --timeout]} { lappend runopts --timeout=[dict get $opts --timeout] } tailcall shellrun::runx {*}$runopts sh -c $cmdargs } } @@ -895,5 +1102,5 @@ namespace eval ::punk::args::register { package provide shellrun [namespace eval shellrun { variable version - set version 0.1.2 + set version 0.1.4 }] diff --git a/src/project_layouts/custom/_project/punk.project-0.1/src/make.tcl b/src/project_layouts/custom/_project/punk.project-0.1/src/make.tcl index 19e78adb..4a936c7b 100644 --- a/src/project_layouts/custom/_project/punk.project-0.1/src/make.tcl +++ b/src/project_layouts/custom/_project/punk.project-0.1/src/make.tcl @@ -1972,10 +1972,67 @@ if {$::punkboot::command eq "vendorupdate"} { #puts "-- [tcl::tm::list] --" puts stdout "Updating vendor modules in src folder" + #Walk up from a vendor source path to the nearest fossil and/or git root and report uncommitted changes. + #Vendored artifacts built from a dirty source tree have no committed provenance (the built module files + #may correspond to no checked-in state) - warn so the source project can be committed first. + #checkedrootsvar names a dict in the caller used to report each VCS root at most once per run. + #Returns a list of warning lines (empty if clean, unversioned, already reported, or state cannot be determined). + proc vendor_source_dirty_warnings {srcpath checkedrootsvar} { + upvar 1 $checkedrootsvar checkedroots + set warnings [list] + set dir [file normalize $srcpath] + if {![file isdirectory $dir]} { + return $warnings ;#missing source location warned about elsewhere + } + set fossilroot "" + set gitroot "" + while {1} { + if {$fossilroot eq "" && ([file exists [file join $dir .fslckout]] || [file exists [file join $dir _FOSSIL_]])} { + set fossilroot $dir + } + if {$gitroot eq "" && [file exists [file join $dir .git]]} { + set gitroot $dir + } + if {$fossilroot ne "" || $gitroot ne ""} { + break ;#report at the nearest VCS root(s); a dual git+fossil root is caught in one pass + } + set parent [file dirname $dir] + if {$parent eq $dir} { break } + set dir $parent + } + if {$fossilroot ne "" && ![dict exists $checkedroots fossil,$fossilroot] && [llength [auto_execok fossil]]} { + dict set checkedroots fossil,$fossilroot 1 + #fossil changes must run from within the checkout + set original_cwd [pwd] + if {[catch { + cd $fossilroot + set fchanges [string trim [exec {*}[auto_execok fossil] changes]] + } errM]} { + lappend warnings "WARNING: vendorupdate could not determine fossil state of source project at $fossilroot ($errM)" + } elseif {$fchanges ne ""} { + lappend warnings "WARNING: vendorupdate source project at $fossilroot has uncommitted fossil changes ([llength [split $fchanges \n]] file(s)) - vendored artifacts built from a dirty tree have no committed provenance" + } + cd $original_cwd + } + if {$gitroot ne "" && ![dict exists $checkedroots git,$gitroot] && [llength [auto_execok git]]} { + dict set checkedroots git,$gitroot 1 + if {[catch { + set gchanges [string trim [exec {*}[auto_execok git] -C $gitroot status --porcelain]] + } errM]} { + lappend warnings "WARNING: vendorupdate could not determine git state of source project at $gitroot ($errM)" + } elseif {$gchanges ne ""} { + lappend warnings "WARNING: vendorupdate source project at $gitroot has uncommitted git changes ([llength [split $gchanges \n]] file(s)) - vendored artifacts built from a dirty tree have no committed provenance" + } + } + return $warnings + } + proc vendor_localupdate {projectroot} { set local_modules [list] set git_modules [list] set fossil_modules [list] + set checked_source_paths [dict create] ;#skip repeat walk-ups for identical source paths + set checked_vcs_roots [dict create] ;#report each VCS root at most once per run set sourcefolder $projectroot/src #todo vendor/lib set vendorlibfolders [glob -nocomplain -dir $sourcefolder -type d -tails vendorlib_tcl*] @@ -2021,6 +2078,14 @@ if {$::punkboot::command eq "vendorupdate"} { } else { set srclocation [file join $localpath $module_subpath] } + #dirty-checkout provenance check - each VCS root reported at most once per run + set normsrclocation [file normalize $srclocation] + if {![dict exists $checked_source_paths $normsrclocation]} { + dict set checked_source_paths $normsrclocation 1 + foreach w [vendor_source_dirty_warnings $normsrclocation checked_vcs_roots] { + puts stderr $w + } + } #puts stdout "$relpath $module $module_subpath $srclocation" #todo - check if requested_module has version extension and allow explicit versions instead of just latest diff --git a/src/project_layouts/custom/_project/punk.shell-0.1/src/bootsupport/modules/punk/console-0.7.1.tm b/src/project_layouts/custom/_project/punk.shell-0.1/src/bootsupport/modules/punk/console-0.7.1.tm index aac663a3..7a9936af 100644 --- a/src/project_layouts/custom/_project/punk.shell-0.1/src/bootsupport/modules/punk/console-0.7.1.tm +++ b/src/project_layouts/custom/_project/punk.shell-0.1/src/bootsupport/modules/punk/console-0.7.1.tm @@ -581,7 +581,10 @@ namespace eval punk::console { try { return [thread::send $route_owner [list ::punk::console::internal::get_ansi_response_payload -console $inoutchannels -expected_ms $expected -ignoreok $ignoreok -return $returntype -passthrough $passthrough $query $capturingendregex]] } on error {errM erropts} { - return -code error "get_ansi_response_payload query routed to console-owning thread $route_owner failed: $errM" + #preserve the owner-side errorcode (where the transport propagates it) so + #callers can discriminate, e.g. {PUNK CONSOLE QUERY HOSTAGE_COOKED_READ} + #from the idle-reader guard + return -code error -errorcode [dict get $erropts -errorcode] "get_ansi_response_payload query routed to console-owning thread $route_owner failed: $errM" } } @@ -611,6 +614,28 @@ namespace eval punk::console { error "punk::console::get_ansi_response_payload input channel '$input' is closed or at eof - cannot receive terminal response" } + #tcl 8.6 windows console idle-reader hostage guard (fail fast, no emission). + #With the console in cooked (line) mode and a readable handler armed on the input + #channel, the 8.6 channel driver has a blocking cooked-mode ReadConsole parked + #(arming is what posts it), and driver reads sample the console mode at issue time - + #the raw cycle below cannot rescue it. Any response would be swallowed by that read + #until the user presses Enter, then leak to the line reader as phantom input + #(~500ms timeout now, input corruption later). This is the idle-at-a-line-mode-prompt + #condition: the repl disarms its reader during command dispatch, so mid-command + #queries are unaffected, and raw mode reads cooperatively and is exempt. Typical + #trigger: a query from an after-script or worker thread firing while the shell sits + #at a line-mode prompt. Best-effort by design: a parked read can outlive a removed + #handler, so this catches the systematic case, not every conceivable one. + if {![tsv::get punk_console is_raw] && $input eq "stdin" && $::punk::console::has_twapi} { + set guard_conf [chan configure $input] + if {![dict exists $guard_conf -inputmode] && ![dict exists $guard_conf -mode] + && [chan event $input readable] ne "" + && ![catch {twapi::get_console_handle stdin}]} { + return -code error -errorcode [list PUNK CONSOLE QUERY HOSTAGE_COOKED_READ]\ + "punk::console::get_ansi_response_payload refusing to emit query on '$input': tcl 8.6 console is in cooked (line) mode with an armed readable handler - the driver's parked cooked read would swallow the response until Enter (query issued from an idle line-mode prompt?). Use raw mode, or issue queries while the reader is disarmed (e.g. during repl command evaluation)." + } + } + #chunks from input that need to be handled by readers upvar ::punk::console::input_chunks_waiting input_chunks_waiting diff --git a/src/project_layouts/custom/_project/punk.project-0.1/src/bootsupport/modules/punkboot/utils-0.1.0.tm b/src/project_layouts/custom/_project/punk.shell-0.1/src/bootsupport/modules/punkboot/utils-0.1.1.tm similarity index 50% rename from src/project_layouts/custom/_project/punk.project-0.1/src/bootsupport/modules/punkboot/utils-0.1.0.tm rename to src/project_layouts/custom/_project/punk.shell-0.1/src/bootsupport/modules/punkboot/utils-0.1.1.tm index fa10b93e..be23f56e 100644 --- a/src/project_layouts/custom/_project/punk.project-0.1/src/bootsupport/modules/punkboot/utils-0.1.0.tm +++ b/src/project_layouts/custom/_project/punk.shell-0.1/src/bootsupport/modules/punkboot/utils-0.1.1.tm @@ -7,7 +7,7 @@ # (C) 2023 # # @@ Meta Begin -# Application punkboot::utils 0.1.0 +# Application punkboot::utils 0.1.1 # Meta platform tcl # Meta license BSD # @@ Meta End @@ -23,7 +23,7 @@ package require Tcl 8.6- # ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++ namespace eval punkboot::utils { variable version 0.1.0 - namespace export parse_punkproject_version read_punkproject_version read_changelog_latest_version + namespace export parse_punkproject_version read_punkproject_version read_changelog_latest_version vcs_dirty_warnings namespace eval argdoc { @@ -115,6 +115,88 @@ namespace eval punkboot::utils { } return "" } + + + namespace eval argdoc { + lappend PUNKARGS [list { + @id -id ::punkboot::utils::vcs_dirty_warnings + @cmd -name "::punkboot::utils::vcs_dirty_warnings"\ + -summary\ + "Warning lines for uncommitted VCS changes above a path"\ + -help\ + "Walks up from path to the nearest fossil and/or git root and + returns warning lines describing uncommitted changes. + Artifacts built from a dirty source tree have no committed + provenance - callers such as make.tcl vendorupdate use this + to flag source projects that should be committed first. + + checkedrootsvar names a dict variable in the caller's scope + used to report each VCS root at most once across calls. + label, if supplied, is prefixed into each warning to name + the calling operation (e.g. 'vendorupdate'). + + Returns an empty list if the path is clean, unversioned, + missing, already reported, or state cannot be determined." + @leaders + path -type string -optional 0 -help\ + "Path to check; the nearest enclosing fossil/git root is examined" + checkedrootsvar -type string -optional 0 -help\ + "Name of a dict variable in the caller's scope for dedupe across calls" + label -type string -optional 1 -default "" -help\ + "Operation name prefixed into warnings" + }] + } + proc vcs_dirty_warnings {path checkedrootsvar {label ""}} { + upvar 1 $checkedrootsvar checkedroots + if {![info exists checkedroots]} {set checkedroots [dict create]} + if {$label ne ""} {set label "$label "} + set warnings [list] + set dir [file normalize $path] + if {![file isdirectory $dir]} { + return $warnings ;#missing path - not this proc's business to report + } + set fossilroot "" + set gitroot "" + while {1} { + if {$fossilroot eq "" && ([file exists [file join $dir .fslckout]] || [file exists [file join $dir _FOSSIL_]])} { + set fossilroot $dir + } + if {$gitroot eq "" && [file exists [file join $dir .git]]} { + set gitroot $dir + } + if {$fossilroot ne "" || $gitroot ne ""} { + break ;#report at the nearest VCS root(s); a dual git+fossil root is caught in one pass + } + set parent [file dirname $dir] + if {$parent eq $dir} { break } + set dir $parent + } + if {$fossilroot ne "" && ![dict exists $checkedroots fossil,$fossilroot] && [llength [auto_execok fossil]]} { + dict set checkedroots fossil,$fossilroot 1 + #fossil changes must run from within the checkout + set original_cwd [pwd] + if {[catch { + cd $fossilroot + set fchanges [string trim [exec {*}[auto_execok fossil] changes]] + } errM]} { + lappend warnings "WARNING: ${label}could not determine fossil state of source project at $fossilroot ($errM)" + } elseif {$fchanges ne ""} { + lappend warnings "WARNING: ${label}source project at $fossilroot has uncommitted fossil changes ([llength [split $fchanges \n]] file(s)) - artifacts built from a dirty tree have no committed provenance" + } + cd $original_cwd + } + if {$gitroot ne "" && ![dict exists $checkedroots git,$gitroot] && [llength [auto_execok git]]} { + dict set checkedroots git,$gitroot 1 + if {[catch { + set gchanges [string trim [exec {*}[auto_execok git] -C $gitroot status --porcelain]] + } errM]} { + lappend warnings "WARNING: ${label}could not determine git state of source project at $gitroot ($errM)" + } elseif {$gchanges ne ""} { + lappend warnings "WARNING: ${label}source project at $gitroot has uncommitted git changes ([llength [split $gchanges \n]] file(s)) - artifacts built from a dirty tree have no committed provenance" + } + } + return $warnings + } } namespace eval ::punk::args::register { @@ -125,8 +207,8 @@ namespace eval ::punk::args::register { ## Ready package provide punkboot::utils [tcl::namespace::eval punkboot::utils { variable version - #- this version number, exactly 0.1.0, is a literal used in src module folders + #- this version number, exactly 0.1.1, is a literal used in src module folders #- we refer to this sometimes as the magic version number - set version 0.1.0 + set version 0.1.1 }] return diff --git a/src/project_layouts/custom/_project/punk.project-0.1/src/bootsupport/modules/shellrun-0.1.2.tm b/src/project_layouts/custom/_project/punk.shell-0.1/src/bootsupport/modules/shellrun-0.1.4.tm similarity index 66% rename from src/project_layouts/custom/_project/punk.project-0.1/src/bootsupport/modules/shellrun-0.1.2.tm rename to src/project_layouts/custom/_project/punk.shell-0.1/src/bootsupport/modules/shellrun-0.1.4.tm index 07bca217..08538dec 100644 --- a/src/project_layouts/custom/_project/punk.project-0.1/src/bootsupport/modules/shellrun-0.1.2.tm +++ b/src/project_layouts/custom/_project/punk.shell-0.1/src/bootsupport/modules/shellrun-0.1.4.tm @@ -5,6 +5,7 @@ package require shellfilter package require punk::ansi +package require punk::args #NOTE: the run,runout,runerr,runx commands only produce an error if the command didn't run. # - If it did run, but there was a non-zero exitcode it is up to the application to check that. @@ -79,76 +80,45 @@ namespace eval shellrun { - #maintenance: similar used in punk::ns & punk::winrun - #todo - take runopts + aliases as args - #longopts must be passed as a single item ie --timeout=100 not --timeout 100 - proc get_run_opts {arglist} { - if {[catch { - set callerinfo [info level -1] - } errM]} { - set caller "" - } else { - set caller [lindex $callerinfo 0] - } - - #we provide -nonewline even for 'run' even though run doesn't deliver stderr or stdout to the tcl return value - #This is for compatibility with other runX commands, and the difference is also visible when calling from repl. - set known_runopts [list "-echo" "-e" "-nonewline" "-n" "-tcl" "-debug"] - set known_longopts [list "--timeout"] - set known_longopts_msg "" - foreach lng $known_longopts { - append known_longopts_msg "${lng}=val " - } - set aliases [list "-e" "-echo" "-echo" "-echo" "-n" "-nonewline" "-nonewline" "-nonewline" "-tcl" "-tcl" "-debug" "-debug"] ;#include map to self - set runopts [list] - set runoptslong [list] - set cmdargs [list] - - set idx_first_cmdarg [lsearch -not $arglist "-*"] - - set allopts [lrange $arglist 0 $idx_first_cmdarg-1] - set cmdargs [lrange $arglist $idx_first_cmdarg end] - foreach o $allopts { - if {[string match --* $o]} { - lassign [split $o =] flagpart valpart - if {$valpart eq ""} { - error "$caller: longopt $o seems to be missing a value - must be of form --option=value" - } - if {$flagpart ni $known_longopts} { - error "$caller: Unknown runoption $o - known options $known_runopts $known_longopts_msg" - } - lappend runoptslong $flagpart $valpart - } else { - if {$o ni $known_runopts} { - error "$caller: Unknown runoption $o - known options $known_runopts $known_longopts_msg" - } - lappend runopts [dict get $aliases $o] - } - } - return [list runopts $runopts runoptslong $runoptslong cmdargs $cmdargs] - } - - #todo - investigate cause of punk86 run hanging sometimes. An 'after 500' before exit in the called script fixes the issue. punk87 doesn't seem to be affected. lappend PUNKARGS [list { @id -id ::shellrun::run + @cmd -name "shellrun::run"\ + -summary "Run an external command, streaming stdout/stderr to the console in real-time"\ + -help\ + "Run an external command and return the shellfilter exitinfo dict. + + stdout and stderr are written to the console as they arrive; + nothing is captured for the return value. The exitcode is + part of the returned dict — a non-zero exitcode is NOT raised + as a Tcl error, the caller must check it explicitly. + + Use -tcl to run a Tcl script/command instead of an external + process; in that case the returned dict carries a 'result' + key (or error/errorCode/errorInfo on failure) instead of an + exitcode." @leaders -min 0 -max 0 @opts - -nonewline -type none - -tcl -type none -default 0 - -debug -type none -default 0 - --timeout= -type integer + -nonewline -type none -help\ + "Accepted for compatibility with runout/runerr/runx. Has no + effect on run because run does not capture or return stdout." + -tcl -type none -default 0 -help\ + "Interpret cmdname (and cmdarg) as a Tcl command/script rather + than an external process." + -debug -type none -default 0 -help\ + "Enable debug diagnostics from shellfilter::run." + --timeout= -type integer -help\ + "Timeout in milliseconds for the external process." @values -min 1 -max -1 - cmdname -type string - cmdarg -type any -multiple 1 -optional 1 + cmdname -type string -help\ + "Name of the external command (or Tcl command when -tcl is + given)." + cmdarg -type any -multiple 1 -optional 1 -help\ + "Additional arguments passed through to the command." }] proc run {args} { #set_last_run_display [list] - #set splitargs [get_run_opts $args] - #set runopts [dict get $splitargs runopts] - #set runoptslong [dict get $splitargs runoptslong] - #set cmdargs [dict get $splitargs cmdargs] set argd [punk::args::parse $args withid ::shellrun::run] lassign [dict values $argd] leaders opts values received @@ -206,11 +176,25 @@ namespace eval shellrun { lappend PUNKARGS [list { @id -id ::shellrun::runconsole + @cmd -name "shellrun::runconsole"\ + -summary "Run a command via exec with stdout/stderr directed to the console, mimicking Tcl's unknown handler"\ + -help\ + "Resolve cmdname via auto_execok and exec it with stdout + and stderr redirected to the active console. The result is + returned via uplevel with the exec result/options dict, + mirroring the way Tcl's unknown proc handles external + commands. + + Unlike run, this uses exec semantics — there is no + exitcode in the return value and stdout/stderr are not + captured separately." @leaders -min 0 -max 0 @opts @values -min 1 -max -1 - cmdname -type string - cmdarg -type any -multiple 1 -optional 1 + cmdname -type string -help\ + "Name of the command to resolve via auto_execok and run." + cmdarg -type any -multiple 1 -optional 1 -help\ + "Additional arguments passed to the resolved command." }] #run in the way tcl unknown does - but without regard to auto_noexec proc runconsole {args} { @@ -256,16 +240,42 @@ namespace eval shellrun { } lappend PUNKARGS [list { @id -id ::shellrun::runout + @cmd -name "shellrun::runout"\ + -summary "Run an external command and return its stdout"\ + -help\ + "Run an external command, capture stdout and stderr, and + return stdout as the result. stderr and the exitcode are + emitted via repl telemetry chunks but are not part of the + return value. + + By default no console output is produced while the command + runs; use -echo to tee stdout and stderr to the console in + real-time. Use -nonewline to strip a trailing newline from + the returned stdout. + + Use -tcl to run a Tcl script/command instead of an external + process." @leaders -min 0 -max 0 @opts - -echo -type none - -nonewline -type none - -tcl -type none -default 0 - -debug -type none -default 0 - --timeout= -type integer + -echo -type none -help\ + "Tee stdout and stderr to the console in real-time while still + capturing them for the return value." + -nonewline -type none -help\ + "Strip a trailing newline (\\r\\n or \\n) from the returned + stdout." + -tcl -type none -default 0 -help\ + "Interpret cmdname (and cmdarg) as a Tcl command/script rather + than an external process." + -debug -type none -default 0 -help\ + "Enable debug diagnostics from shellfilter::run." + --timeout= -type integer -help\ + "Timeout in milliseconds for the external process." @values -min 1 -max -1 - cmdname -type string - cmdarg -type any -multiple 1 -optional 1 + cmdname -type string -help\ + "Name of the external command (or Tcl command when -tcl is + given)." + cmdarg -type any -multiple 1 -optional 1 -help\ + "Additional arguments passed through to the command." }] proc runout {args} { set argd [punk::args::parse $args withid ::shellrun::runout] @@ -284,11 +294,6 @@ namespace eval shellrun { set runerr "" set RST [a] - #set splitargs [get_run_opts $args] - #set runopts [dict get $splitargs runopts] - #set cmdargs [dict get $splitargs cmdargs] - - #puts stdout "RUNOUT cmdargs: $cmdargs" #todo add -data boolean and -data lastwrite to -settings with default being -data all @@ -422,16 +427,42 @@ namespace eval shellrun { lappend PUNKARGS [list { @id -id ::shellrun::runerr + @cmd -name "shellrun::runerr"\ + -summary "Run an external command and return its stderr"\ + -help\ + "Run an external command, capture stdout and stderr, and + return stderr as the result. stdout and the exitcode are + emitted via repl telemetry chunks but are not part of the + return value. + + By default no console output is produced while the command + runs; use -echo to tee stdout and stderr to the console in + real-time. Use -nonewline to strip a trailing newline from + the returned stderr. + + Use -tcl to run a Tcl script/command instead of an external + process." @leaders -min 0 -max 0 @opts - -echo -type none - -nonewline -type none - -tcl -type none -default 0 - -debug -type none -default 0 - --timeout= -type integer + -echo -type none -help\ + "Tee stdout and stderr to the console in real-time while still + capturing them for the return value." + -nonewline -type none -help\ + "Strip a trailing newline (\\r\\n or \\n) from the returned + stderr." + -tcl -type none -default 0 -help\ + "Interpret cmdname (and cmdarg) as a Tcl command/script rather + than an external process." + -debug -type none -default 0 -help\ + "Enable debug diagnostics from shellfilter::run." + --timeout= -type integer -help\ + "Timeout in milliseconds for the external process." @values -min 1 -max -1 - cmdname -type string - cmdarg -type any -multiple 1 -optional 1 + cmdname -type string -help\ + "Name of the external command (or Tcl command when -tcl is + given)." + cmdarg -type any -multiple 1 -optional 1 -help\ + "Additional arguments passed through to the command." }] proc runerr {args} { set argd [punk::args::parse $args withid ::shellrun::runerr] @@ -449,10 +480,6 @@ namespace eval shellrun { set runout "" set runerr "" - #set splitargs [get_run_opts $args] - #set runopts [dict get $splitargs runopts] - #set cmdargs [dict get $splitargs cmdargs] - set callopts [dict create] if {[dict exists $received "-tcl"]} { dict set callopts -tclscript 1 @@ -567,6 +594,43 @@ namespace eval shellrun { } + lappend PUNKARGS [list { + @id -id ::shellrun::runx + @cmd -name "shellrun::runx"\ + -summary "Run an external command and return a dict of stdout, stderr, and exitcode"\ + -help\ + "Run an external command, capture stdout and stderr, and + return a dict containing stdout, stderr, and exitcode (or + result/error/errorCode/errorInfo for -tcl calls). + + No console output is produced unless -echo is given. Use + -nonewline to strip trailing newlines from captured stdout + and stderr before returning. + + Use -tcl to run a Tcl script/command instead of an external + process." + @leaders -min 0 -max 0 + @opts + -echo -type none -help\ + "Tee stdout and stderr to the console in real-time while still + capturing them for the return dict." + -nonewline -type none -help\ + "Strip a trailing newline (\\r\\n or \\n) from captured stdout + and stderr in the returned dict." + -tcl -type none -default 0 -help\ + "Interpret cmdname (and cmdarg) as a Tcl command/script rather + than an external process." + -debug -type none -default 0 -help\ + "Enable debug diagnostics from shellfilter::run." + --timeout= -type integer -help\ + "Timeout in milliseconds for the external process." + @values -min 1 -max -1 + cmdname -type string -help\ + "Name of the external command (or Tcl command when -tcl is + given)." + cmdarg -type any -multiple 1 -optional 1 -help\ + "Additional arguments passed through to the command." + }] proc runx {args} { #set_last_run_display [list] variable runout @@ -574,11 +638,10 @@ namespace eval shellrun { set runout "" set runerr "" - set splitargs [get_run_opts $args] - set runopts [dict get $splitargs runopts] - set cmdargs [dict get $splitargs cmdargs] + set argd [punk::args::parse $args withid ::shellrun::runx] + lassign [dict values $argd] leaders opts values received - if {"-nonewline" in $runopts} { + if {[dict exists $received "-nonewline"]} { set nonewline 1 } else { set nonewline 0 @@ -586,7 +649,7 @@ namespace eval shellrun { #shellfilter::stack::remove stdout $::repl::id_outstack - if {"-echo" in $runopts} { + if {[dict exists $received "-echo"]} { #float to ensure repl transform doesn't interfere with the output data set stderr_stackid [shellfilter::stack::add stderr tee_to_var -action float -settings {-varname ::shellrun::runerr}] set stdout_stackid [shellfilter::stack::add stdout tee_to_var -action float -settings {-varname ::shellrun::runout}] @@ -601,10 +664,23 @@ namespace eval shellrun { set stdout_stackid [shellfilter::stack::add stdout var -action float-locked -junction 1 -settings {-varname ::shellrun::runout}] } - set callopts "" - if {"-tcl" in $runopts} { - append callopts " -tclscript 1" + set callopts [dict create] + if {[dict exists $received "-tcl"]} { + dict set callopts -tclscript 1 + } + if {[dict exists $received "-debug"]} { + dict set callopts -debug 1 + } + if {[dict exists $received --timeout]} { + dict set callopts -timeout [dict get $opts --timeout] + } + set cmdname [dict get $values cmdname] + if {[dict exists $received cmdarg]} { + set cmdarglist [dict get $values cmdarg] + } else { + set cmdarglist {} } + set cmdargs [concat $cmdname $cmdarglist] #set exitinfo [shellfilter::run $cmdargs -teehandle punksh -inbuffering none -outbuffering none -stdinhandler ::repl::repl_handler] set exitinfo [shellfilter::run $cmdargs {*}$callopts -teehandle punksh -inbuffering none -outbuffering none] @@ -616,7 +692,7 @@ namespace eval shellrun { flush stdout if {[dict exists $exitinfo error]} { - if {"-tcl" in $runopts} { + if {[dict exists $received "-tcl"]} { } else { #todo - check errorInfo makes sense.. return -code? tailcall? @@ -826,30 +902,161 @@ namespace eval shellrun { return $exitinfo } + lappend PUNKARGS [list { + @id -id ::shellrun::sh_run + @cmd -name "shellrun::sh_run"\ + -summary "Run a command line via sh -c, streaming stdout/stderr to the console"\ + -help\ + "Wrapper around shellrun::run that invokes the command line + via sh -c. The entire command line after options is passed + as a single argument to sh -c. + + Accepts the same options as shellrun::run except -tcl, + which is not meaningful for the sh_* wrappers because they + always invoke an external shell." + @leaders -min 0 -max 0 + @opts + -nonewline -type none -help\ + "Accepted for compatibility. Has no effect on run-based + commands because run does not capture output." + -debug -type none -default 0 -help\ + "Enable debug diagnostics from shellfilter::run." + --timeout= -type integer -help\ + "Timeout in milliseconds for the external process." + @values -min 1 -max -1 + cmdargs -type any -multiple 1 -help\ + "Command line to pass to sh -c as a single string argument." + }] proc sh_run {args} { - set splitargs [get_run_opts $args] - set runopts [dict get $splitargs runopts] - set cmdargs [dict get $splitargs cmdargs] + set argd [punk::args::parse $args withid ::shellrun::sh_run] + lassign [dict values $argd] leaders opts values received + set cmdargs [dict get $values cmdargs] + set runopts [list] + if {[dict exists $received -nonewline]} { lappend runopts -nonewline } + if {[dict exists $received -debug]} { lappend runopts -debug } + if {[dict exists $received --timeout]} { lappend runopts --timeout=[dict get $opts --timeout] } #e.g sh -c "ls -l *" #we pass cmdargs to sh -c as a list, not individually tailcall shellrun::run {*}$runopts sh -c $cmdargs } + + lappend PUNKARGS [list { + @id -id ::shellrun::sh_runout + @cmd -name "shellrun::sh_runout"\ + -summary "Run a command line via sh -c and return its stdout"\ + -help\ + "Wrapper around shellrun::runout that invokes the command + line via sh -c. The entire command line after options is + passed as a single argument to sh -c. + + Accepts the same options as shellrun::runout except -tcl, + which is not meaningful for the sh_* wrappers because they + always invoke an external shell." + @leaders -min 0 -max 0 + @opts + -echo -type none -help\ + "Tee stdout and stderr to the console in real-time while still + capturing them for the return value." + -nonewline -type none -help\ + "Strip a trailing newline (\\r\\n or \\n) from the returned + stdout." + -debug -type none -default 0 -help\ + "Enable debug diagnostics from shellfilter::run." + --timeout= -type integer -help\ + "Timeout in milliseconds for the external process." + @values -min 1 -max -1 + cmdargs -type any -multiple 1 -help\ + "Command line to pass to sh -c as a single string argument." + }] proc sh_runout {args} { - set splitargs [get_run_opts $args] - set runopts [dict get $splitargs runopts] - set cmdargs [dict get $splitargs cmdargs] + set argd [punk::args::parse $args withid ::shellrun::sh_runout] + lassign [dict values $argd] leaders opts values received + set cmdargs [dict get $values cmdargs] + set runopts [list] + if {[dict exists $received -echo]} { lappend runopts -echo } + if {[dict exists $received -nonewline]} { lappend runopts -nonewline } + if {[dict exists $received -debug]} { lappend runopts -debug } + if {[dict exists $received --timeout]} { lappend runopts --timeout=[dict get $opts --timeout] } tailcall shellrun::runout {*}$runopts sh -c $cmdargs } + + lappend PUNKARGS [list { + @id -id ::shellrun::sh_runerr + @cmd -name "shellrun::sh_runerr"\ + -summary "Run a command line via sh -c and return its stderr"\ + -help\ + "Wrapper around shellrun::runerr that invokes the command + line via sh -c. The entire command line after options is + passed as a single argument to sh -c. + + Accepts the same options as shellrun::runerr except -tcl, + which is not meaningful for the sh_* wrappers because they + always invoke an external shell." + @leaders -min 0 -max 0 + @opts + -echo -type none -help\ + "Tee stdout and stderr to the console in real-time while still + capturing them for the return value." + -nonewline -type none -help\ + "Strip a trailing newline (\\r\\n or \\n) from the returned + stderr." + -debug -type none -default 0 -help\ + "Enable debug diagnostics from shellfilter::run." + --timeout= -type integer -help\ + "Timeout in milliseconds for the external process." + @values -min 1 -max -1 + cmdargs -type any -multiple 1 -help\ + "Command line to pass to sh -c as a single string argument." + }] proc sh_runerr {args} { - set splitargs [get_run_opts $args] - set runopts [dict get $splitargs runopts] - set cmdargs [dict get $splitargs cmdargs] + set argd [punk::args::parse $args withid ::shellrun::sh_runerr] + lassign [dict values $argd] leaders opts values received + set cmdargs [dict get $values cmdargs] + set runopts [list] + if {[dict exists $received -echo]} { lappend runopts -echo } + if {[dict exists $received -nonewline]} { lappend runopts -nonewline } + if {[dict exists $received -debug]} { lappend runopts -debug } + if {[dict exists $received --timeout]} { lappend runopts --timeout=[dict get $opts --timeout] } tailcall shellrun::runerr {*}$runopts sh -c $cmdargs } + + lappend PUNKARGS [list { + @id -id ::shellrun::sh_runx + @cmd -name "shellrun::sh_runx"\ + -summary "Run a command line via sh -c and return a dict of stdout, stderr, and exitcode"\ + -help\ + "Wrapper around shellrun::runx that invokes the command + line via sh -c. The entire command line after options is + passed as a single argument to sh -c. + + Accepts the same options as shellrun::runx except -tcl, + which is not meaningful for the sh_* wrappers because they + always invoke an external shell." + @leaders -min 0 -max 0 + @opts + -echo -type none -help\ + "Tee stdout and stderr to the console in real-time while still + capturing them for the return dict." + -nonewline -type none -help\ + "Strip a trailing newline (\\r\\n or \\n) from captured stdout + and stderr in the returned dict." + -debug -type none -default 0 -help\ + "Enable debug diagnostics from shellfilter::run." + --timeout= -type integer -help\ + "Timeout in milliseconds for the external process." + @values -min 1 -max -1 + cmdargs -type any -multiple 1 -help\ + "Command line to pass to sh -c as a single string argument." + }] proc sh_runx {args} { - set splitargs [get_run_opts $args] - set runopts [dict get $splitargs runopts] - set cmdargs [dict get $splitargs cmdargs] + set argd [punk::args::parse $args withid ::shellrun::sh_runx] + lassign [dict values $argd] leaders opts values received + set cmdargs [dict get $values cmdargs] + set runopts [list] + if {[dict exists $received -echo]} { lappend runopts -echo } + if {[dict exists $received -nonewline]} { lappend runopts -nonewline } + if {[dict exists $received -debug]} { lappend runopts -debug } + if {[dict exists $received --timeout]} { lappend runopts --timeout=[dict get $opts --timeout] } tailcall shellrun::runx {*}$runopts sh -c $cmdargs } } @@ -895,5 +1102,5 @@ namespace eval ::punk::args::register { package provide shellrun [namespace eval shellrun { variable version - set version 0.1.2 + set version 0.1.4 }] diff --git a/src/project_layouts/custom/_project/punk.shell-0.1/src/make.tcl b/src/project_layouts/custom/_project/punk.shell-0.1/src/make.tcl index 19e78adb..4a936c7b 100644 --- a/src/project_layouts/custom/_project/punk.shell-0.1/src/make.tcl +++ b/src/project_layouts/custom/_project/punk.shell-0.1/src/make.tcl @@ -1972,10 +1972,67 @@ if {$::punkboot::command eq "vendorupdate"} { #puts "-- [tcl::tm::list] --" puts stdout "Updating vendor modules in src folder" + #Walk up from a vendor source path to the nearest fossil and/or git root and report uncommitted changes. + #Vendored artifacts built from a dirty source tree have no committed provenance (the built module files + #may correspond to no checked-in state) - warn so the source project can be committed first. + #checkedrootsvar names a dict in the caller used to report each VCS root at most once per run. + #Returns a list of warning lines (empty if clean, unversioned, already reported, or state cannot be determined). + proc vendor_source_dirty_warnings {srcpath checkedrootsvar} { + upvar 1 $checkedrootsvar checkedroots + set warnings [list] + set dir [file normalize $srcpath] + if {![file isdirectory $dir]} { + return $warnings ;#missing source location warned about elsewhere + } + set fossilroot "" + set gitroot "" + while {1} { + if {$fossilroot eq "" && ([file exists [file join $dir .fslckout]] || [file exists [file join $dir _FOSSIL_]])} { + set fossilroot $dir + } + if {$gitroot eq "" && [file exists [file join $dir .git]]} { + set gitroot $dir + } + if {$fossilroot ne "" || $gitroot ne ""} { + break ;#report at the nearest VCS root(s); a dual git+fossil root is caught in one pass + } + set parent [file dirname $dir] + if {$parent eq $dir} { break } + set dir $parent + } + if {$fossilroot ne "" && ![dict exists $checkedroots fossil,$fossilroot] && [llength [auto_execok fossil]]} { + dict set checkedroots fossil,$fossilroot 1 + #fossil changes must run from within the checkout + set original_cwd [pwd] + if {[catch { + cd $fossilroot + set fchanges [string trim [exec {*}[auto_execok fossil] changes]] + } errM]} { + lappend warnings "WARNING: vendorupdate could not determine fossil state of source project at $fossilroot ($errM)" + } elseif {$fchanges ne ""} { + lappend warnings "WARNING: vendorupdate source project at $fossilroot has uncommitted fossil changes ([llength [split $fchanges \n]] file(s)) - vendored artifacts built from a dirty tree have no committed provenance" + } + cd $original_cwd + } + if {$gitroot ne "" && ![dict exists $checkedroots git,$gitroot] && [llength [auto_execok git]]} { + dict set checkedroots git,$gitroot 1 + if {[catch { + set gchanges [string trim [exec {*}[auto_execok git] -C $gitroot status --porcelain]] + } errM]} { + lappend warnings "WARNING: vendorupdate could not determine git state of source project at $gitroot ($errM)" + } elseif {$gchanges ne ""} { + lappend warnings "WARNING: vendorupdate source project at $gitroot has uncommitted git changes ([llength [split $gchanges \n]] file(s)) - vendored artifacts built from a dirty tree have no committed provenance" + } + } + return $warnings + } + proc vendor_localupdate {projectroot} { set local_modules [list] set git_modules [list] set fossil_modules [list] + set checked_source_paths [dict create] ;#skip repeat walk-ups for identical source paths + set checked_vcs_roots [dict create] ;#report each VCS root at most once per run set sourcefolder $projectroot/src #todo vendor/lib set vendorlibfolders [glob -nocomplain -dir $sourcefolder -type d -tails vendorlib_tcl*] @@ -2021,6 +2078,14 @@ if {$::punkboot::command eq "vendorupdate"} { } else { set srclocation [file join $localpath $module_subpath] } + #dirty-checkout provenance check - each VCS root reported at most once per run + set normsrclocation [file normalize $srclocation] + if {![dict exists $checked_source_paths $normsrclocation]} { + dict set checked_source_paths $normsrclocation 1 + foreach w [vendor_source_dirty_warnings $normsrclocation checked_vcs_roots] { + puts stderr $w + } + } #puts stdout "$relpath $module $module_subpath $srclocation" #todo - check if requested_module has version extension and allow explicit versions instead of just latest diff --git a/src/tests/modules/punkboot/utils/testsuites/utils/vcsdirty.test b/src/tests/modules/punkboot/utils/testsuites/utils/vcsdirty.test new file mode 100644 index 00000000..71d194e0 --- /dev/null +++ b/src/tests/modules/punkboot/utils/testsuites/utils/vcsdirty.test @@ -0,0 +1,98 @@ +# -*- tcl -*- +# Tests for punkboot::utils::vcs_dirty_warnings: +# - walk-up VCS root discovery, dirty/clean reporting, per-root dedupe via the +# caller's checkedroots dict, label prefixing, silence for unversioned or +# missing paths. +# Uses a scratch git repo fixture (git side); the fossil branch is symmetric +# code exercised end-to-end by make.tcl vendorupdate and the FOSSIL_HOME +# isolation pattern in modules/punk/mix/testsuites/repo/fossilmove.test. +# Run: tclsh src/tests/runtests.tcl -report compact -show-passes 0 -include-paths modules/punkboot/utils/** vcsdirty.test + +package require tcltest +package require punk::lib +package require punkboot::utils + +namespace eval ::testspace { + namespace import ::tcltest::* + + set gitcmd [auto_execok git] + tcltest::testConstraint git_available [expr {[llength $gitcmd] > 0}] + + variable fixture_error "" + variable gitroot "" + variable subdir "" + if {[tcltest::testConstraint git_available]} { + try { + set gitroot [punk::lib::tempdir_newfolder -prefix pkvcsdirtytest] + exec {*}$gitcmd -C $gitroot init -q + set subdir [file join $gitroot deep nested] + file mkdir $subdir + set fd [open [file join $subdir f.txt] w] + try { puts $fd "uncommitted content" } finally { close $fd } + } on error {result} { + set fixture_error $result + } + } + tcltest::testConstraint git_fixture [expr { + [tcltest::testConstraint git_available] && $fixture_error eq "" + }] + if {$fixture_error ne ""} { + puts stderr "vcsdirty.test fixture setup failed (tests will be skipped): $fixture_error" + } + + test vcsdirty-1.1 {dirty git repo produces one warning naming the root} -constraints git_fixture -body { + variable subdir + variable gitroot + set checked [dict create] + set warnings [::punkboot::utils::vcs_dirty_warnings $subdir checked] + list [llength $warnings] \ + [string match "WARNING: source project at [file normalize $gitroot] has uncommitted git changes*" [lindex $warnings 0]] + } -result {1 1} + + test vcsdirty-1.2 {label is prefixed into the warning} -constraints git_fixture -body { + variable subdir + set checked [dict create] + set warnings [::punkboot::utils::vcs_dirty_warnings $subdir checked someop] + string match "WARNING: someop source project at *" [lindex $warnings 0] + } -result 1 + + test vcsdirty-2.1 {checkedroots dict dedupes repeat reports of the same root} -constraints git_fixture -body { + variable subdir + variable gitroot + set checked [dict create] + set first [::punkboot::utils::vcs_dirty_warnings $subdir checked] + set second [::punkboot::utils::vcs_dirty_warnings $gitroot checked] + list [llength $first] [llength $second] + } -result {1 0} + + test vcsdirty-3.1 {unversioned path is silent} -constraints git_available -body { + set checked [dict create] + set dir [punk::lib::tempdir_newfolder -prefix pkvcsdirtynone] + set result [::punkboot::utils::vcs_dirty_warnings $dir checked] + file delete -force $dir + set result + } -result {} + + test vcsdirty-3.2 {missing path is silent} -constraints git_available -body { + set checked [dict create] + ::punkboot::utils::vcs_dirty_warnings [file join [punk::lib::tempdir] no_such_dir_xyz_12345] checked + } -result {} + + test vcsdirty-4.1 {clean git repo is silent} -constraints git_fixture -body { + variable gitroot + set gitcmd [auto_execok git] + exec {*}$gitcmd -C $gitroot add -A + exec {*}$gitcmd -C $gitroot -c user.email=t@t -c user.name=t commit -q -m fixture + set checked [dict create] + ::punkboot::utils::vcs_dirty_warnings $gitroot checked + } -result {} + + # cleanup fixture + variable gitroot + if {$gitroot ne "" && [file isdirectory $gitroot]} { + catch {file delete -force $gitroot} + } +} + +tcltest::cleanupTests +namespace delete ::testspace