diff --git a/CHANGELOG.md b/CHANGELOG.md index f34ab8bd..48267a8b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,20 @@ The latest `## [X.Y.Z]` header must match the `version` field in `punkproject.to Entries are newest-first; one bullet per notable change. See the root `AGENTS.md` "Project Versioning" section for the bump policy. +## [0.49.4] - 2026-08-03 + +- shellfilter 0.2.5: fixed unix repl result-echo corruption - large multibyte-dense + results (e.g `i ` help tables) displayed with contiguous missing ranges, + fused mid-word seam lines or truncated bottom rows on unix ttys (WSL and native). + Root cause: `shellfilter::chan::tee_to_pipe` declared the transchan `clear` op - + the Tcl core delivers `clear` before every write when declared (the G-145 defect + class) - and its `clear` discarded the held partial multi-byte character, leaving + the next chunk starting with orphan continuation bytes; whole chunks then + accumulated undecodable and successive clears discarded them. `clear` is no longer + declared (method kept as state-preserving no-op). Also repaired `flush` in + tee_to_pipe/tee_to_log/logonly (referenced an undeclared variable and threw on + every invocation). Windows rendering paths unchanged. + ## [0.49.3] - 2026-08-03 - commandstack 0.6.0 (G-160 hygiene pass): tokenids are unique and monotonic diff --git a/goals/G-090-shellfilter-shellthread-audit.md b/goals/G-090-shellfilter-shellthread-audit.md index 616c88e5..05479324 100644 --- a/goals/G-090-shellfilter-shellthread-audit.md +++ b/goals/G-090-shellfilter-shellthread-audit.md @@ -48,6 +48,19 @@ none of this is currently pinned by committed tests): PUNK_TEST_UDPTEE tk watch app) - workaround until resolved: timer-driven non-blocking reads or a non-tcludp receiver. +6. (root-caused and FIXED 2026-08-03, shellfilter 0.2.5 - recorded here as audit evidence): + tee_to_pipe declared the transchan 'clear' op, which the core delivers before EVERY + write when declared (G-145 defect class - see goals/archive/G-145-piped-usage-ansi-remnants.md); + its clear discarded o_encbuf (held partial multi-byte char), cascading into contiguous + multi-KB loss of multibyte-dense output on the unix repl console stack ('i ' help + tables displayed with missing ranges/fused seam lines; isolation repro: 393/600 lines + lost with clear-per-write vs 0/600 without). Additionally the flush methods of + tee_to_pipe, tee_to_log and logonly referenced an undeclared o_buffered variable and + threw on every invocation (copy-paste from the o_buffered-bearing ansiwrap/ansistrip + classes). Both fixed; no characterization tests exist yet for these transforms - a + chunk-boundary/lifecycle suite (clear/flush/multibyte-split cases) belongs in this + audit's scope. + ## Notes - The shipped workaround that motivated the probes: shellrun 0.2.0 runx -teelog implements diff --git a/punkproject.toml b/punkproject.toml index 78798d0f..2678dc1e 100644 --- a/punkproject.toml +++ b/punkproject.toml @@ -1,6 +1,6 @@ [project] name = "punkshell" -version = "0.49.3" +version = "0.49.4" license = "BSD-2-Clause" url = "https://www.gitea1.intx.com.au/jn/punkshell" #packager: declared identity for published artifacts (declarative, not proof - diff --git a/src/modules/shellfilter-999999.0a1.0.tm b/src/modules/shellfilter-999999.0a1.0.tm index d13a2329..5d4e57ac 100644 --- a/src/modules/shellfilter-999999.0a1.0.tm +++ b/src/modules/shellfilter-999999.0a1.0.tm @@ -419,7 +419,10 @@ namespace eval shellfilter::chan { } method initialize {transform_handle mode} { #return [list initialize read drain write flush clear finalize] - return [list initialize write flush clear finalize] + #'clear' deliberately NOT declared: when a transform declares it, the core + #delivers a 'clear' op before EVERY write (tclIORTrans.c ReflectOutput), not + #just on seek - see goals/archive/G-145-piped-usage-ansi-remnants.md. + return [list initialize write flush finalize] } method finalize {transform_handle} { #Note that an error in the finalize can stop 'chan pop' from running properly. @@ -430,8 +433,15 @@ namespace eval shellfilter::chan { # must be present but we ignore it because we do not # post any events } + #G-145 defect class: 'clear' must NOT discard o_encbuf (held partial multi-byte + #character). With clear declared, the core calls it before every write; dropping + #the carry leaves the next chunk starting with orphan continuation bytes, making + #whole chunks unconvertible - they accumulate in o_encbuf and successive clears + #discard them, eating contiguous ranges of multibyte-dense output (the 2026-08-03 + #unix repl result-echo corruption). Kept as a state-preserving no-op in case + #'clear' is ever re-declared; this write-only transform has no read-side state + #(the documented scope of 'clear'). method clear {transform_handle} { - set o_encbuf "" return } #method drain {transform_handle} { @@ -471,16 +481,22 @@ namespace eval shellfilter::chan { # return $clear #} method flush {transform_handle} { - set clear $o_buffered$o_encbuf - if {[catch {tcl::encoding::convertfrom $o_enc $clear} stringdata]} { - #if we can't convert the buffer contents to a string - does it make sense to emit the raw bytes? - # - probably not. - #REVIEW? + #this class holds only o_encbuf (raw bytes of a trailing incomplete + #multi-byte character) - there is no o_buffered ansi-carry here (that + #belongs to the ansiwrap/ansistrip style transforms). + #An incomplete char is not decodable on its own: hold it for the next write + #rather than emitting garbage or discarding (G-145: dropping held stream + #state corrupts content split across write chunks). finalize dropping it at + #true end of stream is acceptable. + if {$o_encbuf eq ""} { + return "" + } + if {[catch {tcl::encoding::convertfrom $o_enc $o_encbuf} stringdata]} { return "" } - set o_buffered "" set o_encbuf "" - return $stringdata + puts -nonewline $o_localchan $stringdata + return [tcl::encoding::convertto $o_enc $stringdata] } method write {transform_handle bytes} { #set logdata [tcl::encoding::convertfrom $o_enc $bytes] @@ -570,16 +586,19 @@ namespace eval shellfilter::chan { # return $clear #} method flush {transform_handle} { - set clear $o_buffered$o_encbuf - if {[catch {tcl::encoding::convertfrom $o_enc $clear} stringdata]} { - #if we can't convert the buffer contents to a string - does it make sense to emit the raw bytes? - # - probably not. - #REVIEW? + #only o_encbuf exists in this class (no o_buffered ansi-carry as in ansiwrap). + #An incomplete multi-byte char is not decodable on its own: hold it for the + #next write rather than discarding (G-145: dropping held stream state + #corrupts content split across write chunks). + if {$o_encbuf eq ""} { + return "" + } + if {[catch {tcl::encoding::convertfrom $o_enc $o_encbuf} stringdata]} { return "" } - set o_buffered "" set o_encbuf "" - return $stringdata + ::shellfilter::log::write $o_logsource $stringdata + return [tcl::encoding::convertto $o_enc $stringdata] } method write {ch bytes} { #set logdata [tcl::encoding::convertfrom $o_enc $bytes] @@ -668,16 +687,19 @@ namespace eval shellfilter::chan { # return #} method flush {transform_handle} { - set clear $o_buffered$o_encbuf - if {[catch {tcl::encoding::convertfrom $o_enc $clear} stringdata]} { - #if we have data in the buffer that we haven't been able to convert to a string - #- then we probably have some kind of encoding mismatch. Is it safer to discard it than to emit garbage chars to the log? - #REVIEW. - we are writing the raw bytes to the log here because we can't convert them to a string. - #This may be useful for debugging issues, but it may also result in garbage data in the log. - ::shellfilter::log::write $o_logsource $o_encbuf + #only o_encbuf exists in this class (no o_buffered ansi-carry as in ansiwrap). + #logonly emits nothing downstream. An incomplete multi-byte char is not + #decodable on its own: hold it for the next write rather than discarding + #(G-145: dropping held stream state corrupts content split across write + #chunks); if it decodes anyway (e.g after an encoding change), log it now. + if {$o_encbuf eq ""} { + return "" + } + if {![catch {tcl::encoding::convertfrom $o_enc $o_encbuf} stringdata]} { + ::shellfilter::log::write $o_logsource $stringdata set o_encbuf "" } - return + return "" } method write {transform_handle bytes} { #set logdata [encoding convertfrom $o_enc $bytes] diff --git a/src/modules/shellfilter-buildversion.txt b/src/modules/shellfilter-buildversion.txt index 7f0143d3..12b36f58 100644 --- a/src/modules/shellfilter-buildversion.txt +++ b/src/modules/shellfilter-buildversion.txt @@ -1,5 +1,6 @@ -0.2.4 +0.2.5 #First line must be a semantic version number #all other lines are ignored. +#0.2.5 - tee_to_pipe: no longer declares transchan 'clear' (core delivers clear before EVERY write when declared - G-145 defect class; its o_encbuf discard cascaded into contiguous multi-KB loss of multibyte-dense output, the unix repl result-echo corruption in 'i ' tables); clear method kept as state-preserving no-op; flush repaired (referenced undeclared o_buffered -> threw on every call; now holds undecodable o_encbuf for next write, emits decodable remainder to both pipe and downstream as bytes). Same undeclared-o_buffered flush throw repaired in tee_to_log and logonly (o_encbuf-only, per-class semantics preserved; neither declares clear) #0.2.4 - shellfilter::run honors its -syslog option (default empty -> the shellfilter-run log worker is a noop, no thread created) instead of the hardcoded 127.0.0.1:514 debug leftover; syslog remains supported when explicitly configured. Part of the piped-stdin exit/quit hang fix - the always-on syslog worker exposed the G-036 Tcl 9 console+udp worker wedge on the exit teardown path (see shellthread 1.6.3) #0.2.3 - shellfilter::run: try/finally guarantees tee-stack removal (stdout/stderr restoration) even when the body errors; decorative stack-status table rendering in log::critical blocks is catch-guarded so a rendering failure (e.g mismatched module snapshots - stale textblock vs new punk::ansi sgr_merge_singles options) degrades to a log note instead of aborting the run and leaving the process's stdout/stderr diverted (silent-output-loss failure mode found via tomlish runtests 2026-07-06)