punk::path and punk::lib fixes

src/bootsupport/modules/fauxlink-0.1.1.tm

@@ -205,7 +205,7 @@ namespace eval fauxlink {
     #    %2F     "/"
     #    %2f     "/"
     #    %7f     (del)
-    #we exlude the forward slash because we already have + for that - and multiple ways to specify it obscure intention.
+    #we exclude the forward slash because we already have + for that - and multiple ways to specify it obscure intention.
     #
     set decode_map [dict merge $decode_map [dict create\
         %09     \t\

src/bootsupport/modules/funcl-0.1.tm

@@ -1,3 +1,6 @@
+
+#experimental.
+
 package provide funcl [namespace eval funcl {
     variable version
     set version 0.1
@@ -235,7 +238,7 @@ namespace eval funcl {
         }
         set comp [list] ;#composition list
         set end [lindex $args end]
-        if {[lindex $end 0] in {_fn _call}]} {
+        if {[lindex $end 0] in {_fn _call}} {
             #is_funcl
             set endfunc [lindex $args end]
         } else {

src/bootsupport/modules/oolib-0.1.3.tm

@@ -0,0 +1,200 @@
+#JMN - api should be kept in sync with package patternlib where possible
+#
+
+namespace eval oolib {
+    oo::class create collection {
+        variable o_data ;#dict
+        #variable o_alias
+        constructor {} {
+            set o_data [dict create]
+        }
+        method info {} {
+            return [dict info $o_data]
+        }
+        method count {} {
+            return [dict size $o_data]
+        }
+        method isEmpty {} {
+            expr {[dict size $o_data] == 0}
+        }
+        method names {{globOrIdx {}}} {
+            if {[llength $globOrIdx]} {
+                if {[string is integer -strict $globOrIdx]} {
+                    set idx $globOrIdx
+                    if {$idx < 0} {
+                        set idx "end-[expr {abs($idx + 1)}]"
+                    }
+                    if {[catch {lindex [dict keys $o_data] $idx} result]} {
+                        error "[self object] no such index : '$idx'"
+                    } else {
+                        return $result
+                    }
+                } else {
+                    #glob
+                    return [lsearch -glob -all -inline [dict keys $o_data] $globOrIdx]
+                }
+            } else {
+                return [dict keys $o_data]
+            }
+        }
+        #like names but without globbing
+        method keys {} {
+            dict keys $o_data
+        }
+        method key {{posn 0}} {
+            if {$posn < 0} {
+                set posn "end-[expr {abs($posn + 1)}]"
+            }
+            if {[catch {lindex [dict keys $o_data] $posn} result]} {
+                error "[self object] no such index : '$posn'"
+            } else {
+                return $result
+            }
+        }
+        method hasKey {key} {
+            dict exists $o_data $key
+        }
+        method get {} {
+            return $o_data
+        }
+        method items {} {
+            return [dict values $o_data]
+        }
+        method item {key} {
+            if {[string is integer -strict $key]} {
+                if {$key >= 0} {
+                    set valposn [expr {(2*$key) +1}]
+                    return [lindex $o_data $valposn]
+                } else {
+                    set key "end-[expr {abs($key + 1)}]"
+                    return [lindex $o_data $key]
+                    #return [lindex [dict keys $o_data] $key]
+                }
+            }
+            if {[dict exists $o_data $key]} {
+                return [dict get $o_data $key]
+            }
+        }
+        #inverse lookup
+        method itemKeys {value} {
+            set value_indices [lsearch -all [dict values $o_data] $value]
+            set keylist [list]
+            foreach i $value_indices {
+                set idx [expr {(($i + 1) *2) -2}]
+                lappend keylist [lindex $o_data $idx]
+            }
+            return $keylist
+        }
+        method search {value args} {
+            set matches [lsearch {*}$args [dict values $o_data] $value]
+            if {"-inline" in $args} {
+                return $matches
+            } else {
+                set keylist [list]
+                foreach i $matches {
+                    set idx [expr {(($i + 1) *2) -2}]
+                    lappend keylist [lindex $o_data $idx]
+                }
+                return $keylist
+            }
+        }
+        #review - see patternlib. Is the intention for aliases to be configurable independent of whether the target exists?
+        #review - what is the point of alias anyway? - why slow down other operations when a variable can hold a keyname perfectly well?
+        #method alias {newAlias existingKeyOrAlias} {
+        #    if {[string is integer -strict $newAlias]} {
+        #        error "[self object] collection key alias cannot be integer"
+        #    }
+        #    if {[string length $existingKeyOrAlias]} {
+        #        set o_alias($newAlias) $existingKeyOrAlias
+        #    } else {
+        #        unset o_alias($newAlias)
+        #    }
+        #}
+        #method aliases {{key ""}} {
+        #    if {[string length $key]} {
+        #        set result [list]
+        #        foreach {n v} [array get o_alias] {
+        #            if {$v eq $key} {
+        #                lappend result $n $v
+        #            }
+        #        }
+        #        return $result
+        #    } else {
+        #        return [array get o_alias]
+        #    }
+        #}
+        ##if the supplied index is an alias, return the underlying key; else return the index supplied.
+        #method realKey {idx} {
+        #    if {[catch {set o_alias($idx)} key]} {
+        #        return $idx
+        #    } else {
+        #        return $key
+        #    }
+        #}
+        method add {value key} {
+            if {[string is integer -strict $key]} {
+                error "[self object] collection key must not be an integer. Use another structure if integer keys required"
+            }
+            if {[dict exists $o_data $key]} {
+                error "[self object] col_processors object error: key '$key' already exists in collection"
+            }
+            dict set o_data $key $value
+            return [expr {[dict size $o_data] - 1}] ;#return index of item
+        }
+        method remove {idx {endRange ""}} {
+            if {[string length $endRange]} {
+                error "[self object] collection error: ranged removal not yet implemented.. remove one item at a time"
+            }
+            if {[string is integer -strict $idx]} {
+                if {$idx < 0} {
+                    set idx "end-[expr {abs($idx+1)}]"
+                }
+                set key [lindex [dict keys $o_data] $idx]
+                set posn $idx
+            } else {
+                set key $idx
+                set posn [lsearch -exact [dict keys $o_data] $key]
+                if {$posn < 0} {
+                    error "[self object] no such index: '$idx' in this collection"
+                }
+            }
+            dict unset o_data $key
+            return
+        }
+        method clear {} {
+            set o_data [dict create]
+            return
+        }
+        method reverse_the_collection {} {
+            #named slightly obtusely because reversing the data when there may be references held is a potential source of bugs
+            #the name reverse_the_collection should make it clear that the object is being modified in place as opposed to simply 'reverse' which may imply a view/copy.
+            #todo - consider implementing a get_reverse which provides an interface to the same collection without affecting original references, yet both allowing delete/edit operations.
+            set dictnew [dict create]
+            foreach k [lreverse [dict keys $o_data]] {
+                dict set dictnew $k [dict get $o_data $k]
+            }
+            set o_data $dictnew
+            return
+        }
+        #review - cmd as list vs cmd as script?
+        method map {cmd} {
+            set seed [list]
+            dict for {k v} $o_data {
+                lappend seed [uplevel #0 [list {*}$cmd $v]]
+            }
+            return $seed
+        }
+        method objectmap {cmd} {
+            set seed [list]
+            dict for {k v} $o_data {
+                lappend seed [uplevel #0 [list $v {*}$cmd]]
+            }
+            return $seed
+        }
+    }
+}
+
+package provide oolib [namespace eval oolib {
+    variable version
+    set version 0.1.3
+}]

src/bootsupport/modules/punk/aliascore-0.1.0.tm

@@ -123,6 +123,7 @@ tcl::namespace::eval punk::aliascore {
         ansistrip     ::punk::ansi::ansistrip
         stripansi     ::punk::ansi::ansistrip
         ansiwrap      ::punk::ansi::ansiwrap
+        ansisplit     ::punk::ansi::ta::split_codes_single
         grepstr       ::punk::ansi::grepstr
         untabify      ::punk::ansi::untabify
         colour        ::punk::console::colour

src/bootsupport/modules/punk/ansi/sauce-0.1.0.tm

@@ -39,32 +39,35 @@ tcl::namespace::eval punk::ansi::sauce {
 
     proc from_file {fname} {
         if {[file size $fname] < 128} {
-            return
+            return [dict create posn -1]
         }
         set fd [open $fname r]
         chan conf $fd -translation binary
         chan seek $fd -128 end
+        set sauce_block_posn [expr {[chan tell $fd] -1}] ;#entire sauce block including ctrl-z and any comments - initial value assuming no comments
+        #If we treat the ctrl-z (\x1a) as part of the sauce - actual start of entire sauce info is 1 before sauce_header_posn,
+        #or further back if there are comments.
         set srec [read $fd]
         set srec_len 128 ;#This is the normal length of a SAUCE record - we may need to set it shorter if truncation detected
         if {[catch {set sdict [to_dict $srec]}]} {
             #review - have seen truncated SAUCE records < 128 bytes
             #we could search for SAUCE00 in the tail and see what records can be parsed?
             #specifically  publicdomain roysac images sometimes only 99 Bytes of sauce - suspect remaining were null \x0 padded and trimmed
-            set sauceposn [string first SAUCE00 $srec]
+            set saucestart [string first SAUCE00 $srec]
-            if {$sauceposn <= 0} {
+            if {$saucestart <= 0} {
                 close $fd
-                return
+                return [dict create posn -1]
             }
             #emit something to give user an indication something isn't right
             puts stderr "punk::ansi::sauce::from_file WARNING SAUCE record seems to be truncated - padding rhs with nulls and trying again.."
             #SAUCE00 is not at the beginning
             #pad the tail with nulls and try again
-            set srec [string range $srec $sauceposn end]
+            set srec [string range $srec $saucestart end]
             set srec_len [string length $srec]
             set srec ${srec}[string repeat \x0 [expr {128 - [string length $srec]}]]
             if {[catch {set sdict [to_dict $srec]}]} {
                 close $fd
-                return
+                return [dict create posn -1]
             }
             dict set sdict warning "SAUCE truncation to $srec_len bytes detected"
         }
@@ -73,6 +76,7 @@ tcl::namespace::eval punk::ansi::sauce {
             #Use srec_len instead of 128 - in case we had truncated source record which we padded and were able to parse
             set offset [expr {-1 *($srec_len + ($clines * 64) + 5)}]
             chan seek $fd $offset end
+            set sauce_block_posn [expr {[chan tell $fd] -1}] ;#entire sauce block including ctrl-z and any comments
             set tag [chan read $fd 5]
             if {$tag eq "COMNT"} {
                 #'character' data - shouldn't be null terminated c-style string - but can be
@@ -95,6 +99,7 @@ tcl::namespace::eval punk::ansi::sauce {
                 dict set sdict commentlines $commentlines
             }
         }
+        dict set sdict posn $sauce_block_posn
         close $fd
         return $sdict
     }
@@ -213,7 +218,9 @@ tcl::namespace::eval punk::ansi::sauce {
 
 
 
-
+    #---------------------------------------------------------------------------------------------------------------------------------------------
+    # This data comes from the sauce spec.
+    #---------------------------------------------------------------------------------------------------------------------------------------------
     #todo - fontName - which can also specify e.g code page 437
     ## Font name [1]	Font size	Resolution [2]	Aspect ratio [3]        Vertical stretch [6]	Description
     ##                                              Display [4]	Pixel [5]
@@ -221,7 +228,14 @@ tcl::namespace::eval punk::ansi::sauce {
     set fontnames [dict create]
 
     ## IBM VGA	9×16 [7]	720×400	4:3	20:27 (1:1.35)	35%	Standard hardware font on VGA cards for 80×25 text mode (code page 437)
-    dict set fontnames "IBM VGA" [list fontsize "9x16" resolution "720x400" aspect_ratio_display "4:3" aspect_ratio_pixel "20:27 (1:1.35)" vertical_stretch "35%" description "Standard hardware font on VGA cards for 80×25 text mode (code page 437)"]
+    dict set fontnames "IBM VGA" [list {*}{
+        fontsize                "9x16"
+        resolution              "720x400"
+        aspect_ratio_display    "4:3"
+        aspect_ratio_pixel      "20:27 (1:1.35)"
+        vertical_stretch        "35%"
+        description             "Standard hardware font on VGA cards for 80×25 text mode (code page 437)"
+    }]
     ## IBM VGA ### [8]	9×16 [7]	720×400	4:3	20:27 (1:1.35)	35%	Software installed code page font for VGA 80×25 text mode
     # - where ### is placeholder for 437,720,737 etc
 
@@ -247,6 +261,7 @@ tcl::namespace::eval punk::ansi::sauce {
     ## C64 PETSCII unshifted	8×8 [10]	320×200	4:3	5:6 (1:1.2)	20%	Original Commodore PETSCII font (PET, VIC-20, C64, CBM-II, Plus/4, C16, C116 and C128) in the unshifted mode. Unshifted mode (graphics) only has uppercase letters and additional graphic characters. This is the normal boot font.
     ## C64 PETSCII shifted	8×8 [10]	320×200	4:3	5:6 (1:1.2)	20%	Original PETSCII font in shifted mode. Shifted mode (text) has both uppercase and lowercase letters. This mode is actuated by pressing Shift+Commodore key.
     ## Atari ATASCII	8×8 [11]	320×192	4:3	4:5 (1:1.25)	25%	Original ATASCII font (Atari 400, 800, XL, XE)
+    #---------------------------------------------------------------------------------------------------------------------------------------------
 
 
     #expect a 128 Byte sauce record
@@ -256,6 +271,7 @@ tcl::namespace::eval punk::ansi::sauce {
         variable datatypes
         variable filetypes
         variable encodings
+        set warnings [list]
         if {[string length $saucerecord] != 128} {
             error "punk::ansi::sauce::to_dict: Unable to interpret data as a SAUCE record - length != 128"
         }
@@ -321,6 +337,8 @@ tcl::namespace::eval punk::ansi::sauce {
                 dict set sdict filetype_name ""
             }
         } else {
+            #how can a byte fail to scan with cu? is this even reachable?
+            puts stderr "punk::ansi::sauce::to_dict filetype byte failed to scan - setting filetype and filetype_name to empty string byte: [ansistring VIEW -lf 1 [string range $saucerecord 95 95]]"
             dict set sdict filetype ""
             dict set sdict filetype_name ""
         }
@@ -417,25 +435,40 @@ tcl::namespace::eval punk::ansi::sauce {
             5 {
                 #binarytext
                 #filetype is supposed to represent half the characterwidth (only widths with multiples of 2 can be specified)
-                #HOWEVER - in the wild we may see width/height specified in tinfo1/tinfo2 with some other value in filetype (eg 1)
+                #HOWEVER - in the wild we may see width/height specified in tinfo1/tinfo2 with some apparently unrelated value in filetype (eg 0 or 1) that doesn't match the intended image dimensions.
-                #If both tinfo1 and tinfo2 are non zero - we will use them, even though it's not in spec.
+                #If both tinfo1 and tinfo2 are non zero - we *could* use them, even though it's not in spec.
-                set t1 [dict get $sdict tinfo1]
+                #An example file (us-used1.bin) has filetype 0 and tinfo1/tinfo2 640/350
-                if {$t1 eq ""} {
+                #It's possible tinfo1/tinfo2 represent pixel dimensions for a 'standard' 8x16 font, but this image is 160 columns wide, so we would expect tinfo1 to be 1280.
-                    set t1 0
+                #The sauce spec seems to indicate we should ignore tinfo1/tinfo2 for binarytext and only use filetype to determine width.
-                }
+                #the default for binarytext is 160 columns.
-                set t2 [dict get $sdict tinfo2]
+
-                if {$t2 eq ""} {
+                #filetype 1 is theoretically possible, representing 2 columns
-                    set t2 0
+                #in practice we see this value for binarytext images that are definitely not intended to be 2 columns wide. Why?
+                #is there some assumption that that images are at least a certain width, and filetype has been repurposed to indicate something else?
+                #The spec would seem to rule out images of a single column due to filetype being half the character width but a value of 0.5 isn't supported.
+                #It specifically mentions that only even widths up to 510 can be specified. ($filetype * 2 where filetype is 1-255?)
+
+
+                #proper mechanism to specify columns for binarytext is the datatype field.
+                set cols [expr {2*[dict get $sdict filetype]}]
+                if {$cols == 0} {
+                    lappend warnings "binarytext filetype value of [dict get $sdict filetype] - using binarytext default cols of 160"
+                     #default for binarytext is 160 columns
+                    set cols 160
                 }
-                if {$t1 != 0 && $t2 != 0} {
+                if {$cols == 2 && [dict get $sdict tinfo1] != 0 && [dict get $sdict tinfo2] != 0} {
                     #not to spec - but we will assume these have values for a reason..
-                    puts stderr "punk::ansi::sauce::to_dict using tinfo1/tinfo2 data for columns/rows (non-compliant SAUCE data)"
+                    #---------------------------------------------------------------------------------------------------------------------
-                    dict set sdict columns [expr {2 * $t1}]
+                    #The sample file src/testansi/formatsamples/image/binaryText/test.bin has a filetype 1 and tinfo1 40 and tinfo2 25.
-                    dict set sdict rows $t2
+                    #(similarly ppe-ansi.bin has tinfo1 80 and tinfo2 26)
+                    #They seem to use the 1 in filetype to indicate that the tinfo1/tinfo2 values should be used.
+                    #(The 80 cols wide test.bin binaryText image matches the xbin sample file src/testansi/formatsamples/image/xbin/test.xb which is a more fully specified format using a header)
+                    #---------------------------------------------------------------------------------------------------------------------
+                    lappend warnings "binarytext filetype of 1 with non-zero tinfo1/tinfo2 - using tinfo1/tinfo2 data for columns/rows (possibly non-conforming SAUCE data - matching observed data in the wild)"
+                    set cols [expr {2 * [dict get $sdict tinfo1]}]
+                    dict set sdict columns $cols
+                    dict set sdict rows [dict get $sdict tinfo2]
                 } else {
-                    #proper mechanism to specify columns for binarytext is the datatype field.
-
-                    set cols [expr {2*[dict get $sdict filetype]}]
                     dict set sdict columns $cols
                     #rows must be calculated from file size
                     #rows = (filesize - sauceinfosize)/ filetype * 2 * 2
@@ -447,11 +480,13 @@ tcl::namespace::eval punk::ansi::sauce {
 
             }
             6 {
-                #xbin - only filtype is 0
+                #xbin - only filetype is 0
                 #https://web.archive.org/web/20120204063040/http://www.acid.org/info/xbin/x_spec.htm
                 dict set sdict columns [dict get $sdict tinfo1]
                 dict set sdict rows    [dict get $sdict tinfo2]
                 dict set sdict fontname [dict get $sdict tinfos]
+                #Values from sauce record are probably only informational, because xbin has an 11-byte header with width,height,fontsize and flags.
+                #presumably the header-info should take precedence over all sauce data (? review)
             }
         }
         if {[dict exists $sdict fontname]} {
@@ -474,6 +509,9 @@ tcl::namespace::eval punk::ansi::sauce {
                 }
             }
         }
+        if {[llength $warnings]} {
+            dict set sdict warnings $warnings
+        }
         return $sdict
     }
 

src/bootsupport/modules/punk/args-0.2.1.tm

@@ -9086,7 +9086,7 @@ tcl::namespace::eval punk::args {
                     }
                     if {!([tcl::string::first " " $a]>=0 || [tcl::string::first \t $a]>=0 || [tcl::string::last \n $a]>=0)} {
                         if {$OPT_ANY} {
-                            #exlude argument with whitespace from being a possible option e.g dict
+                            #exclude argument with whitespace from being a possible option e.g dict
                             #todo - passthrough of unrecognised --longopt=xxx without looking for following flag-value
                             set eposn [string first = $a]
                             if {$eposn > 2 && [string match --* $a]} {

src/bootsupport/modules/punk/char-0.1.0.tm

@@ -3033,14 +3033,18 @@ tcl::namespace::eval punk::char {
     #This still leaves a whole class of clusters.. korean etc unhandled :/
     #todo - tk::startOfCluster / tk::endOfCluster - try to get it brought into Tcl
     #https://core.tcl-lang.org/tcl/tktview/a4c7eeaf63
-    proc grapheme_split {text} {
+    proc grapheme_split {text {return list}} {
         #we should treat \r\n as a single grapheme cluster (as tk::endOfCluster does)
         set components [list]
         set csplits [combiner_split $text]
         foreach {pt combiners} [lrange $csplits 0 end-1] {
             set clist [split $pt ""]
-            lappend components {*}[lrange $clist 0 end-1]
+            #review
-            lappend components [tcl::string::cat [lindex $clist end] $combiners]
+            #lset clist end [tcl::string::cat [lindex $clist end] $combiners]
+            ledit clist end end [tcl::string::cat [lindex $clist end] $combiners]
+            lappend components {*}$clist
+            #lappend components {*}[lrange $clist 0 end-1]
+            #lappend components [tcl::string::cat [lindex $clist end] $combiners]
         }
         #last csplit never has a combiner (_perlish_split style) - and may be empty - in which case we don't append it as a grapheme
         if {[lindex $csplits end] ne ""} {
@@ -3066,7 +3070,6 @@ tcl::namespace::eval punk::char {
 
         #review \uFE0F variation selector 16 - forces emoji presentation for preceding char
 
-        if 1 {
         #This is a basic implementation that does not check that all combinations are valid.
         set graphemes [list]
         set current_cluster ""
@@ -3085,13 +3088,13 @@ tcl::namespace::eval punk::char {
                 lappend graphemes "\r\n"
                 incr i ;#skip the \n as we've already processed it as part of the cluster
                 set current_cluster ""
-                    grapheme_split::reset_base
+                set cluster_base 0; set cluster_base_RI 0 ;#grapheme_split::reset_base
                 set current_cluster_is_extensible 0
             } elseif {$component eq "\u200d"} {
                 if {$current_cluster eq ""} {
                     #ZWJ at start of string - treat as separate grapheme cluster - but isn't a valid base for further combining with more ZWJs or modifiers
                     set current_cluster $component
-                        grapheme_split::reset_base
+                    set cluster_base 0; set cluster_base_RI 0 ;#grapheme_split::reset_base
                     set current_cluster_is_extensible 0
                 } else {
                     if {$cluster_base} {
@@ -3103,7 +3106,7 @@ tcl::namespace::eval punk::char {
                             append current_cluster $component
                             if {$cluster_base_RI} {
                                 #regional indicators can only combine in pairs, and don't combine with modifiers or ZWJs to form longer clusters - so not extensible.
-                                    grapheme_split::reset_base
+                                set cluster_base 0; set cluster_base_RI 0 ;#grapheme_split::reset_base
                                 set current_cluster_is_extensible 0
                                 #we can keep adding ZWJs or modifiers though
                             } else {
@@ -3122,13 +3125,13 @@ tcl::namespace::eval punk::char {
                 if {$current_cluster eq ""} {
                     #modifier at start of string -  not a valid base for further combining with more modifiers or ZWJs - but we continue appending modifiers to the current cluster.
                     set current_cluster $component
-                        grapheme_split::reset_base
+                    set cluster_base 0; set cluster_base_RI 0 ;#grapheme_split::reset_base
                 } else {
                     if {$cluster_base} {
                         if {$current_cluster_is_extensible} {
                             append current_cluster $component
                             #invalidate the base!
-                                grapheme_split::reset_base
+                            set cluster_base 0; set cluster_base_RI 0 ;#grapheme_split::reset_base
                         } else {
                             append current_cluster $component
                         }
@@ -3155,7 +3158,7 @@ tcl::namespace::eval punk::char {
                         } else {
                             lappend graphemes $current_cluster
                             set current_cluster $component
-                                grapheme_split::reset_base
+                            set cluster_base 0; set cluster_base_RI 0 ;#grapheme_split::reset_base
                         }
                         set current_cluster_is_extensible 0
                     } elseif {$cluster_base_RI} {
@@ -3165,7 +3168,7 @@ tcl::namespace::eval punk::char {
 
                             #invalidate the base - we can't combine more than 2 RIs in a cluster, and they don't combine with modifiers or ZWJs to form longer clusters.
                             #we can however add more ZWJs or modifiers to the cluster - but they don't make it extensible for combining with more RIs
-                                grapheme_split::reset_base
+                            set cluster_base 0; set cluster_base_RI 0 ;#grapheme_split::reset_base
                         } else {
                             #something else while RI cluster is open - end the current cluster and start a new one with the current char.
                             lappend graphemes $current_cluster
@@ -3182,11 +3185,11 @@ tcl::namespace::eval punk::char {
         if {$current_cluster ne ""} {
             lappend graphemes $current_cluster
         }
+        if {$return eq "list"} {
+             return $graphemes
         } else {
-            set graphemes $components
+             return [dict create list $graphemes last_extensible $current_cluster_is_extensible base $cluster_base RI_base $cluster_base_RI]
         }
-
-        return $graphemes
     }
     namespace eval grapheme_split {
         proc about {} {

src/bootsupport/modules/punk/console-0.1.1.tm

@@ -71,11 +71,6 @@ package require punk::args
 
 
 
-#if {"windows" eq $::tcl_platform(platform)} {
-#    #package require zzzload
-#    #zzzload::pkg_require twapi
-#}
-
 #see https://learn.microsoft.com/en-us/windows/console/classic-vs-vt
 #https://learn.microsoft.com/en-us/windows/console/creating-a-pseudoconsole-session
 

src/bootsupport/modules/punk/du-0.1.0.tm

@@ -2529,21 +2529,30 @@ namespace eval punk::du {
                     #jmn disable twapi
                     #tailcall du_dirlisting_generic $folderpath {*}$args
 
-                    package require zzzload
+                    #package require zzzload
-                    set loadstate [zzzload::pkg_require twapi]
+                    #set loadstate [zzzload::pkg_require twapi]
-
+
-                    if {$loadstate ni [list loading failed]} {
+                    #if {$loadstate ni [list loading failed]} {
-                        #either already loaded by zzload or ordinary package require
+                    #    #either already loaded by zzload or ordinary package require
-                        package require twapi ;#should be fast once twapi dll loaded in zzzload thread
+                    #    package require twapi ;#should be fast once twapi dll loaded in zzzload thread
+                    #    set ::punk::du::has_twapi 1
+                    #    punk::du::active::set_active_function du_dirlisting du_dirlisting_twapi
+                    #    tailcall du_dirlisting_twapi $folderpath {*}$args
+                    #} else {
+                    #    if {$loadstate eq "failed"} {
+                    #        puts stderr "punk::du defaulting to du_dirlisting_generic because twapi load failed"
+                    #        punk::du::active::set_active_function du_dirlisting du_dirlisting_generic
+                    #    }
+                    #    tailcall du_dirlisting_generic $folderpath {*}$args
+                    #}
+                    if {[catch {package require twapi} errM]} {
+                        puts stderr "punk::du defaulting to du_dirlisting_generic because twapi load failed: $errM"
+                        punk::du::active::set_active_function du_dirlisting du_dirlisting_generic
+                        tailcall du_dirlisting_generic $folderpath {*}$args
+                    } else {
                         set ::punk::du::has_twapi 1
                         punk::du::active::set_active_function du_dirlisting du_dirlisting_twapi
                         tailcall du_dirlisting_twapi $folderpath {*}$args
-                    } else {
-                        if {$loadstate eq "failed"} {
-                            puts stderr "punk::du defaulting to du_dirlisting_generic because twapi load failed"
-                            punk::du::active::set_active_function du_dirlisting du_dirlisting_generic
-                        }
-                        tailcall du_dirlisting_generic $folderpath {*}$args
                     }
                 }
                 default {

src/bootsupport/modules/punk/lib-0.1.6.tm

@@ -138,25 +138,10 @@ tcl::namespace::eval punk::lib::check {
         if {"windows" ne $::tcl_platform(platform)} {
             set bug 0
         } else {
-            if {![catch {file tempdir} tmpdir]} {
+            set tmpdir [punk::lib::tempdir_newfolder] ;#uses 'file tempdir' on tcl9+ or fallback to env vars or current directory on older versions
-                #tcl 9+ has 'file tempdir'
-                set testfile [file join $tmpdir "bugtest"]
-            } else {
-                #fallback for older tcl versions - use env TEMP/TMP or current directory
-                set tmpdir ""
-                foreach e {TEMP TMP} {
-                    if {[info exists ::env($e)] && [file isdirectory ::env($e)]} {
-                        set tmpdir ::env($e)
-                        break
-                    }
-                }
-                if {$tmpdir eq ""} {
-                    #no env vars - fallback to current directory
-                    set tmpdir [pwd]
-                }
             set testfile [file join $tmpdir "bugtest"]
-            }
 
+            try {
                 set fd [open $testfile w]
                 puts $fd test
                 close $fd
@@ -170,6 +155,14 @@ tcl::namespace::eval punk::lib::check {
                         break
                     }
                 }
+            } finally {
+                if {[file exists $testfile]} {
+                    file delete $testfile
+                }
+                if {[file exists $tmpdir]} {
+                    file delete -force $tmpdir
+                }
+            }
         }
         #possibly related anomaly is that a returned result with case that doesn't match that of the file in the underlying filesystem can't be normalized
         # to the correct case without doing some sort of string manipulation on the result such as 'string range $f 0 end' to affect the internal representation.
@@ -679,7 +672,207 @@ namespace eval punk::lib {
         }
     }
 
+    namespace eval argdoc {
+        variable PUNKARGS
+        lappend PUNKARGS [list {
+            @id     -id     ::punk::lib::tempdir
+            @cmd    -name     punk::lib::tempdir\
+                -summary\
+                "Determine an appropriate temp dir for the process we are running under."\
+                -help\
+                "On windows:
+                  If the process is running under the system account use the modern windows location: %SystemRoot%\SystemTemp
+                  Detection of the system account relies on either twapi, or a combination of the whoami command and the 
+                  registry package.
+                  Proceeds with same fallback logic as for other platforms if we fail to detect the system account or its temp location.
+
+
+                For other platforms we use the environment variables TMPDIR/TEMP/TMP or fallback to /tmp or /var/tmp if those
+                env vars aren't set or aren't writable directories.
+
+                Final fallback attempt is the current working directory.
+                Result is normalized so resulting path will have forward slashes on all platforms.
+
+                Alternatives: see the tcllib fileutil::tempdir function.
+                "
+            @values -min 0 -max 0
+        }]
+    }
+    proc tempdir {} {
+        set trydirs [list]
+        if {"windows" eq $::tcl_platform(platform)} {
+            #review.
+            #consider also checking for whether running under various service accounts
+
+            if {![catch {package require twapi}]} {
+                set tok [twapi::open_process_token] ;#first call is a little pricy.
+                set sid [twapi::get_token_user $tok]
+                if {$sid eq "S-1-5-18"} {
+                    #system account - use system account temp location
+                    set sysroot [twapi::get_shell_folder csidl_windows] ;#first call is a little pricy.
+                    lappend trydirs [file join $sysroot "SystemTemp"]
+                }
+                #if not system account - use env vars as first choice.
+            } else {
+                #twapi not available - try to use builtin windows whoami to detect if we're running under system account - but this is less reliable than using twapi to check the process token - so we warn about it.
+                set whoami_exe [auto_execok whoami]
+                #test that system32 is somewhere in the whoami path as a basic attempt to avoid non-windows whoami commands that might be present in the path
+                set whoami_exe_parts [file split $whoami_exe]
+                if {"system32" in [string tolower $whoami_exe_parts]} {
+                    set whoamiresult [string trim [exec {*}$whoami_exe /USER /FO LIST] \n\r]
+                    set whoamiresult [string map {\r\n \n} $whoamiresult]
+                    set whoamiresult_lines [split $whoamiresult \n]
+                    set sid ""
+                    foreach line $whoamiresult_lines {
+                        if {[string match "SID:*" $line]} {
+                            set sid [lindex $line 1]
+                            break
+                        }
+                    }
+                    set has_registry [expr {![catch {package require registry}]}]
+                    if {$sid eq "S-1-5-18"} {
+                        #system account - use system account temp location
+                        set sysroot ""
+                        if {$has_registry} {
+                            #registry path is case-insensitive.
+                            catch {
+                                set sysroot [registry get {HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Windows NT\CurrentVersion} SystemRoot]
+                            }
+                        } else {
+                            if {[info exists ::env(SystemRoot)]} {
+                                set sysroot [set ::env(SystemRoot)]
+                            }
+                        }
+                        if {$sysroot ne ""} {
+                            lappend trydirs [file join $sysroot "SystemTemp"]
+                        }
+                    }
+                    #if not system account - use env vars as first choice.
+                }
+            }
+        }
+
+        foreach t {TMPDIR TEMP TMP} {
+            #TMPDIR is the posix standard as first choice for temp dir env var.
+            if {[info exists ::env($t)]} {
+                lappend trydirs $::env($t)
+            }
+        }
+
+        if {"windows" ne $::tcl_platform(platform)} {
+            #suitable for macos,linux and freebsd at least.
+            lappend trydirs [file join / tmp] [file join / var tmp]
+            #/usr/tmp is probably not a common location for a temp dir on modern unix-based systems.
+        }
+
+        foreach d $trydirs {
+            if {[file isdirectory $d] && [file writable $d]} {
+                return [file normalize $d]
+            }
+        }
+
+        #only even call 'pwd' as a last resort (mildly slow on first call).
+        set cwd [pwd]
+        if {[file isdirectory $cwd] && [file writable $cwd]} {
+            return $cwd
+        }
+
+        return -code error "punk::lib::tempdir unable to determine a suitable temp directory - tried: $trydirs"
+    }
+
+    namespace eval argdoc {
+        variable PUNKARGS
+        lappend PUNKARGS [list {
+            @id     -id     ::punk::lib::tempdir_newfolder
+            @cmd    -name     punk::lib::tempdir_newfolder\
+                -summary\
+                "Create unique folder within temp dir (or cwd as last resort)"\
+                -help\
+                "Creates a new unique folder within the temp dir determined by punk::lib::tempdir.
+                The folder is created before returning its full path and will be empty.
+                The folder is named with a tcl_ prefix followed by a random string.
+
+                See also: 'file tempdir' in tcl 9+ or fileutil::maketempdir from tcllib"
+            @opts
+                -dir -type string -default "" -help\
+                    "Base directory to create the temp folder in - defaults to the result of punk::lib::tempdir"
+                -prefix -type string -default tcl -help\
+                    "Prefix for the temp folder name
+                    An underscore is automatically appended to the prefix in the generated folder name.
+                    If prefix is the empty string - then the generated folder name will still be autoprefixed
+                    with tcl_  (consistent with tcl9 'file tempdir')"
+            @values -min 0 -max 0
+        }]
+    }
+    proc tempdir_newfolder {args} {
+        set argd [punk::args::parse $args withid ::punk::lib::tempdir_newfolder]
+        set opt_dir [dict get $argd opts -dir]
+        set opt_prefix [dict get $argd opts -prefix]
+        puts "opt_prefix: $opt_prefix"
+        if {[llength [file split $opt_prefix]] > 1} {
+            error "punk::lib::tempdir_newfolder -prefix option should not contain any path separators"
+        }
+        if {$opt_prefix eq ""} {
+            #don't allow empty prefix - for consistency with tcl9 'file tempdir' - which would still prefix with 'tcl_' even if prefix is empty string.
+            set opt_prefix "tcl"
+        }
+
+        if {$opt_dir ne ""} {
+            if {[file isdirectory $opt_dir] && [file writable $opt_dir]} {
+                set tmpbase [file normalize $opt_dir]
+            } else {
+                error "punk::lib::tempdir_newfolder -dir option '$opt_dir' is not a writable directory"
+            }
+        } else {
+            set tmpbase [punk::lib::tempdir] ;#will raise an error if no writable temp dir or cwd found.
+        }
+        #assert: tmpbase has no trailing slash - unless it is a root dir (e.g / on unix, or C:/ on windows)
+        #assert: tmpbase is normalized with forward slashes on all platforms.
 
+        set tcl9_template_base [string trimright $tmpbase "/"] ;#set it to a form that can join along with a forward slash to prefix for tcl 9 'file tempdir' style template.
+        #tcl9 'file tempdir' separates the prefix in the template from the random string with an underscore.
+
+        #now form template by always joining with a slash (even if opt_prefix is empty)
+        #(avoiding file join and file normalize to ensure template is properly formed)
+        #whether or not opt_prefix ends with an underscore - another will be added. (by file tempdir, or by our own code if tcl9 'file tempdir' isn't available)
+        #assert: opt_prefix is not empty string (will have defaulted to 'tcl') and does not contain any path separators.
+        set tcl9_template "$tcl9_template_base/$opt_prefix"
+
+
+        #tcl 9+ has 'file tempdir'
+        #we don't support the same template as 'file tempdir'
+        if {[catch {file tempdir $tcl9_template} tmpdir]} {
+
+            set prefix tcl_ ;#todo - accept option: -prefix
+
+            set chars abcddefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789
+            set nrand_chars 8
+            set maxtries 100
+            for {set i 0} {$i < $maxtries} {incr i} {
+                set dirname ${opt_prefix}_  ;#always add underscore for consistency with tcl9 'file tempdir'.
+                for {set j 0} {$j < $nrand_chars} {incr j} {
+                    append dirname [string index $chars [expr {int(rand()*62)}]]
+                }
+                set path [file join $tmpbase $dirname]
+                if {[file exists $path]} {
+                    continue
+                }
+                if {[catch {
+                        file mkdir $path
+                        if {"windows" ne $::tcl_platform(platform)} {
+                            file attributes $path -permissions 0o700
+                        }
+                    }]} {
+                    continue
+                }
+                return $path
+            }
+            return -code error "punk::lib::tempdir_newfolder unable to create unique tempdir in $tmpbase after $maxtries attempts - too many collisions - aborting"
+        }
+        #tcl 9 'file tempdir' return.
+        #normalize because on early tcl 9 versions on windows it can return with mixed forward and backward slashes when a template is supplied with forward slashes.
+        return [file normalize $tmpdir]
+    }
 
     # == == == == == == == == == == == == == == == == == == == == == == == == == == == == == == ==
     # Maintenance - This is the primary source for tm_version... functions
@@ -814,6 +1007,89 @@ namespace eval punk::lib {
             error "tm_version_required_canonical should have already returned a canonicalised versionspec - or produced an error with reason before this point"
         }
     }
+    proc tm_version_magic {} {
+        #maintenance instruction:
+        #  This specific 12 character nnnnnn.0a1.0 version number is used in text replacement when installing modules.
+        #  It should not be modified without also considering the impact on existing projects based on punkshell that may have used this version number in their own code or documentation as a placeholder for tm version handling.
+        #  A copy of this is also present in punk::lib.
+        #  Essentially this should probably never be changed - and instead deprecated if we want to stop using it and replaced with a different mechanism/placeholder.
+
+        #tm versioning is not quite the same as semantic versioning - but similarly we should see extremely high major version numbers rarely if ever in real use,
+        #even over decades of development.
+        set magicbase 999999  ;#deliberately large so given load-preference when testing!
+        #we split the literal to avoid the literal appearing here - reduce risk of accidentally converting to a release version
+        return ${magicbase}.0a1.0
+    }
+
+    namespace eval argdoc {
+        lappend PUNKARGS [list {
+            @id     -id     ::punk::lib::tm_split_name
+            @cmd    -name     punk::lib::tm_split_name\
+                -summary\
+                "Split a versioned module name into name and version parts, dropping trailing .tcl/.tm if any."\
+                -help\
+                "Splits a versioned module name (as present in a filename or namespaced name) into name and version parts,
+                Ignores any trailing .tm or .tcl file extension.
+
+                If the fullmodulename given is a namespaced name - then the returned modulename will also be namespaced,
+                but with any leading :: removed.
+
+                Returns a two element list - with the first element being the modulename and the second element being the version.
+
+                Tcl module version numbers are understood with leading zeros in each dotted part, but leading zeros are not canonical.
+
+                This split does not canonicalise the version number.
+                If the last dash-separated segment of the name doesn't look like a valid version number
+                - then it is treated as part of the modulename and an empty version string is returned.
+                e.g
+                  mymod-1.2.3.tm -> mymod 1.2.3
+                  mymod-1aa2.3.tm -> mymod-1aa2.3 {}
+                  (repeated a is not a valid version segment - so the entire '1aa2.3' is treated as part of the modulename)
+
+                see also: tm_version_canonical
+                "
+            @values -min 1 -max 1
+            fullmodulename -type string -help\
+                "The full module name to split - as present in a filename or namespaced name. E.g:
+                mymod-1.2.3
+                mymod-1.2.3.tm
+                mymod-1.2.3.tcl
+                /some/where/mymod-123.0a4.0.tm
+                mymod
+                mymod.tm
+                mymod.tcl
+                ns1::ns2::mymod-1.2.3
+                ::ns1::ns2::mymod"
+        }]
+    }
+    proc tm_split_name {fullmodulename} {
+        #split modulename (as present in a filename or namespaced name) into name/version ignoring leading namespace path
+        #ignore trailing .tm .TM if present
+        #if version doesn't pass validation - treat it as part of the modulename and return empty version string without error
+        #Up to caller to validate.
+        set lastpart [namespace tail $fullmodulename]
+        set lastpart [file tail $lastpart] ;# should be ok to use file tail now that we've ensured no namespace components
+        if {[string tolower [file extension $fullmodulename]] in {.tcl .tm}} {
+            set fileparts [split [file rootname $lastpart] -]
+        } else {
+            set fileparts [split $lastpart -]
+        }
+        if {[tm_version_isvalid [lindex $fileparts end]]} {
+            set versionsegment [lindex $fileparts end]
+            set namesegment [join [lrange $fileparts 0 end-1] -];#re-stitch
+        } else {
+            set namesegment [join $fileparts -]
+            set versionsegment ""
+        }
+        set base [string trimleft [namespace qualifiers $fullmodulename] :]
+        if {$base ne ""} {
+            set modulename "${base}::$namesegment"
+        } else {
+            set modulename $namesegment
+        }
+        return [list $modulename $versionsegment]
+    }
+
     # end tm_version... functions
     # == == == == == == == == == == == == == == == == == == == == == == == == == == == == == == ==
 
@@ -4210,6 +4486,9 @@ namespace eval punk::lib {
                                                 if {[string index $key 0] ne "%"} {
                                                     set key %$key
                                                 }
+                                                #puts "---key:'$key'"
+                                                set key [string map {; \\;} $key] ;#review
+                                                #puts "---key:'$key'"
                                                 #pipeline - use punk patterns.
                                                 % thisval.= $key= $thisval
                                             }

src/bootsupport/modules/punk/mix/cli-0.3.1.tm

@@ -499,7 +499,7 @@ namespace eval punk::mix::cli {
             # -- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- ---
             set opt_punkcheck_eventobj                   [dict get $opts -punkcheck_eventobj]
 
-            set magicversion [punk::mix::util::magic_tm_version] ;#deliberately large so given load-preference when testing
+            set magicversion [punk::mix::util::tm_version_magic] ;#deliberately large so given load-preference when testing
             set module_list [list]
 
             if {[file tail [file dirname $srcdir]] ne "src"} {

src/bootsupport/modules/punk/mix/commandset/loadedlib-0.1.0.tm

@@ -54,13 +54,18 @@ namespace eval punk::mix::commandset::loadedlib {
         if {$opt_refresh} {
             catch {package require frobznodule666} ;#ensure pkg system has loaded/searched for everything REVIEW - this doesn't result in full scans
             foreach tm_path [tcl::tm::list] {
+                #review - todo - adjust punk::path::subfolders to take arguments to do some filtering itself rather than recurse down unnecessary branches.
                 set paths_below [punk::path::subfolders -recursive $tm_path]
                 foreach folder $paths_below {
+                    if {[string match */_build/* $folder]} {continue}
                     set tail [file tail $folder]
-                    if {[string match #modpod-* $tail] || [string match #tarjar-* $tail]} {
+                    if {[string match #tarjar-* $tail]} {
+                        continue
+                    }
+                    if {[string match #modpod-* $tail]} {
+                        #manually do a 'package ifneeded' fore each module found here.
                         continue
                     }
-                    if {[string match */_build/* $folder]} {continue}
                     set relpath [string tolower [punk::path::relative $tm_path $folder]]
                     set modpath [string map {/ ::} $relpath]
                     catch {package require ${modpath}::flobrudder99}

src/bootsupport/modules/punk/mix/commandset/module-0.1.0.tm

@@ -362,7 +362,7 @@ namespace eval punk::mix::commandset::module {
         file mkdir $modulefolder
 
         set moduletail [namespace tail $modulename]
-        set magicversion [punk::mix::util::magic_tm_version] ;#deliberately large so given load-preference when testing
+        set magicversion [punk::mix::util::tm_version_magic] ;#deliberately large so given load-preference when testing
 
 
 

src/bootsupport/modules/punk/mix/commandset/project-0.1.0.tm

@@ -380,25 +380,25 @@ namespace eval punk::mix::commandset::project {
 
         puts stdout ">>> about to call punkcheck::install $layout_path $projectdir"
         set resultdict [dict create]
-        set antipaths [list\
+        set antipaths [list         {*}{
-            src/doc/*\
+            src/doc/*
-            src/doc/include/*\
+            src/doc/include/*
-            src/PROJECT_LAYOUTS_*\
+            src/PROJECT_LAYOUTS_*
-        ]
+        }]
 
-        #set antiglob_dir [list\
+        #set exclude_dirsegments [list {*}{
-        #    _ignore_*\
+        #    _ignore_*
-        #]
+        #}]
-        set antiglob_dir [list\
+        set exclude_dirsegments [list {*}{
-        ]
+        }]
 
-        #default antiglob_dir_core will stop .fossil* from being updated - which is generally desirable as these are likely to be customized
+        #default -exclude-dirsegments_core will stop .fossil* from being updated - which is generally desirable as these are likely to be customized
         if {$opt_force} {
             puts stdout "copying layout files - with force applied - overwrite all-targets"
-            set resultdict [punkcheck::install $layout_path $projectdir -installer project.new -createempty 1  -overwrite ALL-TARGETS -antiglob_paths $antipaths -antiglob_dir $antiglob_dir]
+            set resultdict [punkcheck::install $layout_path $projectdir -installer project.new -createempty 1  -overwrite ALL-TARGETS -antiglob_paths $antipaths -exclude-dirsegments $exclude_dirsegments]
         } else {
             puts stdout "copying layout files - (if source file changed)"
-            set resultdict [punkcheck::install $layout_path $projectdir -installer project.new -createempty 1 -overwrite installedsourcechanged-targets -antiglob_paths $antipaths -antiglob_dir $antiglob_dir]
+            set resultdict [punkcheck::install $layout_path $projectdir -installer project.new -createempty 1 -overwrite installedsourcechanged-targets -antiglob_paths $antipaths -exclude-dirsegments $exclude_dirsegments]
         }
         puts stdout [punkcheck::summarize_install_resultdict $resultdict]
 
@@ -412,11 +412,11 @@ namespace eval punk::mix::commandset::project {
 
         #target folders .fossil-custom and .fossil-settings may not exist. use -createdir 1 to ensure existence.
         #In this case we need to override the default dir antiglob - as .fossil-xxx folders need to be installed from template if missing, or if target is uncustomized.
-        ## default_antiglob_dir_core  [list "#*" "_aside" ".git" ".fossil*"]
+        ## default_exclude_dirsegments_core  [list "#*" "_aside" ".git" ".fossil*"]
-        set override_antiglob_dir_core [list #* _aside .git]
+        set override_exclude_dirsegments_core [list #* _aside .git]
         if {[file exists $layout_path/.fossil-custom]} {
             puts stdout "copying layout src/.fossil-custom files (if target missing or uncustomised)"
-            set resultdict [punkcheck::install $layout_path/.fossil-custom $projectdir/.fossil-custom -createdir 1 -createempty 1 -punkcheck_folder $projectdir -installer project.new -antiglob_dir_core $override_antiglob_dir_core -overwrite SYNCED-TARGETS]
+            set resultdict [punkcheck::install $layout_path/.fossil-custom $projectdir/.fossil-custom -createdir 1 -createempty 1 -punkcheck_folder $projectdir -installer project.new -exclude-dirsegments_core $override_exclude_dirsegments_core -overwrite SYNCED-TARGETS]
             puts stdout [punkcheck::summarize_install_resultdict $resultdict]
         } else {
             puts stdout "no .fossil-custom in source template - update not required"
@@ -424,7 +424,7 @@ namespace eval punk::mix::commandset::project {
 
         if {[file exists $layout_path/.fossil-settings]} {
             puts stdout "copying layout src/.fossil-settings files (if target missing or uncustomised)"
-            set resultdict [punkcheck::install $layout_path/.fossil-settings $projectdir/.fossil-settings -createdir 1 -createempty 1 -punkcheck_folder $projectdir -installer project.new -antiglob_dir_core $override_antiglob_dir_core -overwrite SYNCED-TARGETS]
+            set resultdict [punkcheck::install $layout_path/.fossil-settings $projectdir/.fossil-settings -createdir 1 -createempty 1 -punkcheck_folder $projectdir -installer project.new -exclude-dirsegments_core $override_exclude_dirsegments_core -overwrite SYNCED-TARGETS]
             puts stdout [punkcheck::summarize_install_resultdict $resultdict]
         } else {
             puts stdout "no .fossil-settings in source template - update not required"
@@ -470,8 +470,8 @@ namespace eval punk::mix::commandset::project {
         if {[file exists $projectdir/src/modules]} {
             foreach m $opt_modules {
                 #check if mod-ver.tm file or #modpod-mod-ver folder exist
-                set tmfile $projectdir/src/modules/$m-[punk::mix::util::magic_tm_version].tm
+                set tmfile $projectdir/src/modules/$m-[punk::mix::util::tm_version_magic].tm
-                set podfile $projectdir/src/modules/#modpod-$m-[punk::mix::util::magic_tm_version]/$m-[punk::mix::util::magic_tm_version].tm
+                set podfile $projectdir/src/modules/#modpod-$m-[punk::mix::util::tm_version_magic]/$m-[punk::mix::util::tm_version_magic].tm
 
                 set has_tm  [file exists $tmfile]
                 set has_pod [file exists $podfile]

src/bootsupport/modules/punk/mix/util-0.1.0.tm

@@ -367,7 +367,16 @@ namespace eval punk::mix::util {
     }
     #todo - semver conversion/validation for other systems?
 
-    proc magic_tm_version {} {
+    proc tm_version_magic {} {
+        #maintenance instruction:
+        #  This specific 12 character nnnnnn.0a1.0 version number is used in text replacement when installing modules.
+        #  It should not be modified without also considering the impact on existing projects based on punkshell that may have used this version number in their own code or documentation as a placeholder for tm version handling.
+        #  A copy of this is also present in punk::lib to aid in dependency management.
+        #  These 2 copies should be kept in sync.
+        #  Essentially this should probably never be changed - and instead deprecated if we want to stop using it and replaced with a different mechanism/placeholder.
+
+        #tm versioning is not quite the same as semantic versioning - but similarly we should see extremely high major version numbers rarely if ever in real use,
+        #even over decades of development.
         set magicbase 999999  ;#deliberately large so given load-preference when testing!
         #we split the literal to avoid the literal appearing here - reduce risk of accidentally converting to a release version
         return ${magicbase}.0a1.0

src/bootsupport/modules/punk/mod-0.1.1.tm

@@ -0,0 +1,158 @@
+#punkapps app manager
+# deck cli
+
+namespace eval punk::mod::cli {
+    namespace export help list run
+    namespace ensemble create
+
+   # namespace ensemble configure [namespace current] -unknown punk::mod::cli::_unknown
+    if 0 {
+    proc _unknown {ns args} {
+        puts stderr "punk::mod::cli::_unknown '$ns' '$args'"
+        puts stderr "punk::mod::cli::help $args"
+        puts stderr "arglen:[llength $args]"
+        punk::mod::cli::help {*}$args
+    }
+    }
+
+    #cli must have _init method - usually used to load commandsets lazily
+    #
+    variable initialised 0
+    proc _init {args} {
+        variable initialised
+        if {$initialised} {
+            return
+        }
+        #...
+        set initialised 1
+    }
+
+    proc help {args} {
+        set basehelp [punk::mix::base help {*}$args]
+        #namespace export
+        return $basehelp
+    }
+    proc getraw {appname} {
+        set app_folders [punk::config::configure running apps]
+        #todo search each app folder
+        set bases [::list]
+        set versions [::list]
+        set mains [::list]
+        set appinfo [::list bases {} mains {} versions {}]
+
+        foreach containerfolder $app_folders {
+            lappend bases $containerfolder
+            if {[file exists $containerfolder]} {
+                if {[file exists $containerfolder/$appname/main.tcl]} {
+                    #exact match - only return info for the exact one specified
+                    set namematches $appname
+                    set parts [split $appname -]
+                } else {
+                    set namematches [glob -nocomplain -dir $containerfolder -type d -tail ${appname}-*]
+                    set namematches [lsort $namematches] ;#todo - -ascii? -dictionary? natsort?
+                }
+                foreach nm $namematches {
+                    set mainfile $containerfolder/$nm/main.tcl
+                    set parts [split $nm -]
+                    if {[llength $parts] == 1} {
+                        set ver ""
+                    } else {
+                        set ver [lindex $parts end]
+                    }
+                    if {$ver ni $versions} {
+                        lappend versions $ver
+                        lappend mains $ver $mainfile
+                    } else {
+                        puts stderr "punk::apps::app version '$ver' of app '$appname' already encountered at $mainfile. (will use earliest encountered in running-config apps and ignore others of same version)"
+                    }
+                }
+            } else {
+                puts stderr "punk::apps::app missing apps_folder:'$containerfolder' Ensure apps_folder is set in punk::config"
+            }
+        }
+        dict set appinfo versions $versions
+        #todo - natsort!
+        set sorted_versions [lsort $versions]
+        set latest [lindex $sorted_versions 0]
+        if {$latest eq "" && [llength $sorted_versions] > 1} {
+            set latest [lindex $sorted_versions 1]
+        }
+        dict set appinfo latest $latest
+
+        dict set appinfo bases $bases
+        dict set appinfo mains $mains
+        return $appinfo
+    }
+
+    proc list {{glob *}} {
+        set apps_folder [punk::config::configure running apps]
+        if {[file exists $apps_folder]} {
+            if {[file exists $apps_folder/$glob]} {
+                #tailcall source $apps_folder/$glob/main.tcl
+                return $glob
+            }
+            set apps [glob -nocomplain -dir $apps_folder -type d -tail $glob]
+            if {[llength $apps] == 0} {
+                if {[string first * $glob] <0 && [string first ? $glob] <0} {
+                    #no glob chars supplied - only launch if exact match for name part
+                    set namematches [glob -nocomplain -dir $apps_folder -type d -tail ${glob}-*]
+                    set namematches [lsort $namematches] ;#todo - -ascii? -dictionary? natsort?
+                    if {[llength $namematches] > 0} {
+                        set latest [lindex $namematches end]
+                        lassign $latest nm ver
+                        #tailcall source $apps_folder/$latest/main.tcl
+                    }
+                }
+            }
+            return $apps
+        }
+    }
+
+    #todo - way to launch as separate process
+    # solo-opts only before appname - args following appname are passed to the app
+    proc run {args} {
+        set nameposn [lsearch -not $args -*]
+        if {$nameposn < 0} {
+            error "punkapp::run unable to determine application name"
+        }
+        set appname [lindex $args $nameposn]
+        set controlargs [lrange $args 0 $nameposn-1]
+        set appargs [lrange $args $nameposn+1 end]
+
+        set appinfo [punk::mod::cli::getraw $appname]
+        if {[llength [dict get $appinfo versions]]} {
+            set ver [dict get $appinfo latest]
+            puts stdout "info: $appinfo"
+            set ::argc [llength $appargs]
+            set ::argv $appargs
+            source [dict get $appinfo mains $ver]
+            if {"-hideconsole" in $controlargs} {
+                puts stderr "attempting console hide"
+                #todo - something better - a callback when window mapped?
+                after 500 {::punkapp::hide_console}
+            }
+            return $appinfo
+        } else {
+            error "punk::mod::cli unable to run '$appname'. main.tcl not found in [dict get $appinfo bases]"
+        }
+    }
+}
+
+namespace eval punk::mod::cli {
+    proc _cli {args} {
+        #don't use tailcall - base uses info level to determine caller
+        ::punk::mix::base::_cli {*}$args
+    }
+    variable default_command help
+    package require punk::mix::base
+    package require punk::overlay
+    punk::overlay::custom_from_base [namespace current] ::punk::mix::base
+}
+
+package provide punk::mod [namespace eval punk::mod {
+    variable version
+    set version 0.1.1
+}]
+
+
+

src/bootsupport/modules/punk/nav/fs-0.1.0.tm

@@ -847,7 +847,7 @@ tcl::namespace::eval punk::nav::fs {
 
               Regardless of whether -nonportable is supplied or not, some characters are not
               suitable for windows or most other platforms and will be rejected with an error.
-              An example of this is the null character (\0)."
+              An example of this is the null character (\\0)."
         @values -min 1 -max -1 -type string
         path -type string -multiple 1 -help\
             "Path(s) to create. Can be absolute or relative.

src/bootsupport/modules/punk/ns-0.1.0.tm

@@ -721,6 +721,22 @@ tcl::namespace::eval punk::ns {
     }
 
 
+    punk::args::define {
+        @id     -id ::punk::ns::nstree_list
+        @cmd    -name punk::ns::nstree_list\
+            -summary\
+            ""\
+            -help\
+            ""
+        @leaders
+            location -type path -optional 0
+        @opts
+            -subnslist -type list -default {} -help\
+            ""
+            -allbelow -type boolean -default 1 -help\
+            ""
+        @values -min 0 -max 0
+    }
     #important: add tests for tricky cases - e.g punk::m**::util  vs punk::m*::util vs punk::m*::**::util  - these should all be able to return different results depending on namespace structure.
     #e.g punk::m**::util will return punk::mix::util  but punk::m*::**::util will not because punk::mix::util is too short to match. Both will return deeper matches such as: punk::mix::commandset::repo::util
     proc nstree_list {location args} {
@@ -775,13 +791,8 @@ tcl::namespace::eval punk::ns {
         #set parent [nsprefix $ns_absolute]
         #set tail [nstail $ns_absolute]
 
-        #jjj
         #set allchildren [lsort [nseval $base [list ::namespace children]]]
-        #set allchildren [lsort [tcl::namespace::eval $base [list ::namespace children]]]
-        set allchildren [lsort [nseval $base [list ::namespace children]]]
 
-        #puts "->base:$base tailparts:$tailparts allchildren: $allchildren"
-        #puts "->base:$base tailparts:$tailparts childcount: [llength $allchildren]"
 
         #** only significant when it is the trailing part of a segment eg ::**::xxx  ::a**::xxx
         if {[llength $tailparts]} {
@@ -790,6 +801,7 @@ tcl::namespace::eval punk::ns {
                 set nslist [nstree_list $base -subnslist {} -allbelow 1]
             } elseif {[regexp {[*]{2}$} $nextglob]} {
                 set nslist [list]
+                set allchildren [lsort [nseval $base [list ::namespace children]]]
                 lappend nsmatches {*}[lsearch -all -inline -glob $allchildren [nsjoin ${base} $nextglob]]
                 foreach ch $nsmatches {
                     lappend nslist $ch
@@ -799,6 +811,7 @@ tcl::namespace::eval punk::ns {
             } else {
                 #lsearch with -glob ok even if nextglob has no globchars (no discernable speed diff, and earlier parts may have globchars anyway)
                 set nslist [list]
+                set allchildren [lsort [nseval $base [list ::namespace children]]]
                 lappend nsmatches {*}[lsearch -all -inline -glob $allchildren [nsjoin ${base} $nextglob]]
                 if {[llength $tailparts] >1 || $allbelow} {
                     foreach ch $nsmatches {
@@ -812,6 +825,7 @@ tcl::namespace::eval punk::ns {
             }
         } else {
             #puts "nstree_list: no tailparts base:$base"
+            set allchildren [lsort [nseval $base [list ::namespace children]]]
             if {$allbelow} {
                 set nsmatches $allchildren
                 set nslist [list]
@@ -2134,8 +2148,8 @@ y" {return quirkykeyscript}
         tcl::dict::set tinfo($target) procoffset 0
         tcl::dict::set tinfo($target) level [expr {[::tcl::info::level]+1}]
         tcl::dict::set tinfo($target) subcmds 0
-        puts "enter: $target -- $args"
+        puts stderr "enter: $target -- $args"
-        puts "frame-2: [::tcl::info::frame -2]"
+        #puts stderr "frame-2: [::tcl::info::frame -2]"
 
         set _cmdtrace_disabled false
     }
@@ -2481,7 +2495,7 @@ y" {return quirkykeyscript}
                         set line $traceline
                         dict set linedict $target eval_base $traceline
                         dict set linedict $target eval_offset 1
-                        puts "  step type: proc traceline:$traceline ** $args"
+                        puts "  step type: proc traceline:$traceline ** $args\x1b\[m"
                         #puts "** $callinfo"
                         if {[dict exists $callinfo cmd]} {
                             #set cmd [string trim [dict get $callinfo cmd]] ;#raw 'unexpanded' script from the stack frame
@@ -2504,8 +2518,8 @@ y" {return quirkykeyscript}
                         set eval_base   [dict get $linedict $target eval_base]
                         set eval_offset [dict get $linedict $target eval_offset]
                         set line [expr {$eval_base + ($eval_offset-1) + ($traceline-1)}]
-                        puts "stack-- $callinfo"
+                        #puts "stack-- $callinfo"
-                        puts "  step type: eval traceline: $traceline -- "
+                        puts stderr "  step type: eval traceline: $traceline -- "
                         if {[dict exists $callinfo cmd]} {
                             #set cmd [string trim [dict get $callinfo cmd]]
                             set cmdlist [lindex $args 0]
@@ -2627,6 +2641,8 @@ y" {return quirkykeyscript}
         }]
     }
     proc cmdtrace {args} {
+        #review - displaying argument values has to be done carefully. Small values are ok, but large lists or dicts can be overwhelming.
+        #Potentially we could apply some heuristics to truncate or summarise them.
         package require dictn ;#convenience to allow dictn::incr d {key subkey}
         variable tinfo
         array unset tinfo
@@ -2676,7 +2692,7 @@ y" {return quirkykeyscript}
         #if the target command has a leading colon (e.g expr alternative :) we can't put a trace directly on a fully qualified name with a triple colon such as :::
         #we will need to evaluate in the namespace
         foreach {tgt_cmd ns nscmd} $resolved_targets {
-            puts "tracing target: $tgt_cmd whilst running: $origin $arglist"
+            puts stderr "tracing target: $tgt_cmd whilst running: $origin $arglist"
 
             #::tcl::namespace::eval $ns [list ::trace add execution $nscmd enter       [list ::punk::ns::_cmdtrace_enter       ::punk::ns::linedict $tgt_cmd]]
             #::tcl::namespace::eval $ns [list ::trace add execution $nscmd enterstep   [list ::punk::ns::_cmdtrace_enterstep   ::punk::ns::linedict $tgt_cmd]]

src/bootsupport/modules/punk/overlay-0.1.1.tm

@@ -0,0 +1,192 @@
+
+
+package require punk::mix::util
+package require punk::args
+
+tcl::namespace::eval ::punk::overlay {
+    #based *loosely* on: wiki.tcl-lang.org/page/ensemble+extend
+    # extend an ensemble-like routine with the routines in some namespace
+    #
+    # e.g custom_from_base ::punk::mix::cli ::punk::mix::base
+    #
+    proc custom_from_base {routine base} {
+        if {![tcl::string::match ::* $routine]} {
+            set resolved [uplevel 1 [list ::tcl::namespace::which $routine]]
+            if {$resolved eq {}} {
+                error [list {no such routine} $routine]
+            }
+            set routine $resolved
+        }
+        set routinens [tcl::namespace::qualifiers $routine]
+        if {$routinens eq {::}} {
+            set routinens {}
+        }
+        set routinetail [tcl::namespace::tail $routine]
+
+        if {![tcl::string::match ::* $base]} {
+            set base [uplevel 1 [
+                list [tcl::namespace::which namespace] current]]::$base
+        }
+
+        if {![tcl::namespace::exists $base]} {
+            error [list {no such namespace} $base]
+        }
+
+        set base [tcl::namespace::eval $base [
+            list [tcl::namespace::which namespace] current]]
+
+
+        #while 1 {
+        #    set renamed ${routinens}::${routinetail}_[info cmdcount]
+        #    if {[namespace which $renamed] eq {}} break
+        #}
+
+        tcl::namespace::eval $routine [
+            ::list tcl::namespace::ensemble configure  $routine -unknown [
+                ::list ::apply {{base ensemble subcommand args} {
+                    ::list ${base}::_redirected $ensemble $subcommand 
+                }} $base
+            ]
+        ]
+
+        punk::mix::util::namespace_import_pattern_to_namespace_noclobber ::punk::mix::util::* ${routine}::util
+        #namespace eval ${routine}::util {
+            #::namespace import ::punk::mix::util::*
+        #}
+        punk::mix::util::namespace_import_pattern_to_namespace_noclobber ${base}::lib::* ${routine}::lib
+        #namespace eval ${routine}::lib [string map [list <base> $base] {
+        #    ::namespace import <base>::lib::*
+        #}]
+
+        tcl::namespace::eval ${routine}::lib [tcl::string::map [list <base> $base <routine> $routine] {
+            if {[tcl::namespace::exists <base>::lib]} {
+                ::set current_paths [tcl::namespace::path]
+                if {"<routine>" ni $current_paths} {
+                    ::lappend current_paths <routine>
+                }
+                tcl::namespace::path $current_paths
+            }
+        }]
+
+        tcl::namespace::eval $routine {
+            ::set exportlist [::list]
+            ::foreach cmd [tcl::info::commands [tcl::namespace::current]::*] {
+                ::set c [tcl::namespace::tail $cmd]
+                if {![tcl::string::match _* $c]} {
+                    ::lappend exportlist $c
+                }
+            }
+            tcl::namespace::export {*}$exportlist
+        }
+
+        return $routine
+    }
+    punk::args::define {
+        @id     -id     ::punk::overlay::import_commandset
+        @cmd    -name     punk::overlay::import_commandset\
+            -summary\
+            "Import commands into caller's namespace with optional prefix and separator."\
+            -help\
+            "Import commands that have been exported by another namespace into the caller's
+            namespace. Usually a prefix and optionally a separator should be used.
+            This is part of the punk::mix CLI commandset infrastructure - design in flux.
+            Todo - .toml configuration files for defining CLI configurations."
+        @values
+        prefix          -type string
+        separator       -type string -help\
+            "A string, usually punctuation, to separate the prefix and the command name
+              of the final imported command. The value \"::\" is disallowed in this context."
+        cmdnamespace    -type string -help\
+            "Namespace from which to import commands. Commands are those that have been exported."
+    }
+    #load *exported* commands from cmdnamespace into caller's namespace - prefixing each command with $prefix
+    #Note: commandset may be imported by different CLIs with different bases *at the same time*
+    #so we don't make commands from the cli or its base available automatically (will generally require fully-qualified commands to use code from cli/base) 
+    #we do load punk::mix::util::*  into the util subnamespace even though the commandset might not be loaded in a cli using punk::mix::base  i.e punk::mix::util is a common dependency for CLIs.
+    #commandsets designed to be used with a specific cli/base may choose to do their own import e.g with util::namespace_import_pattern_to_namespace_noclobber and/or set namespace path if they 
+    #want the convenience of using lib:xxx with commands coming from those packages.
+    #This won't stop the commandset being used with other cli/bases unless the import is done by looking up the callers namespace.
+    #The basic principle is that the commandset is loaded into the caller(s) with a prefix 
+    #- but commandsets should explicitly package require if they have any backwards dependencies on cli/base (which they may or may not be loaded into)
+    proc import_commandset {prefix separator cmdnamespace} {
+        set bad_seps [list "::"]
+        if {$separator in $bad_seps} {
+            error "import_commandset invalid separator '$separator'"
+        }
+        if {$prefix in $bad_seps} {
+            error "import_commandset invalid prefix '$prefix'"
+        }
+        if {"$prefix$separator" in $bad_seps} {
+            error "import_commandset invalid prefix/separator combination '$prefix$separator'"
+        }
+        if {"[string index $prefix end][string index $separator 0]" in $bad_seps} {
+            error "import_commandset invalid prefix/separator combination '$prefix$separator'"
+        }
+        #review - do we allow prefixes/separators such as a::b?
+
+        #namespace may or may not be a package
+        # allow with or without leading ::
+        if {[tcl::string::range $cmdnamespace 0 1] eq "::"} {
+            set cmdpackage [tcl::string::range $cmdnamespace 2 end]
+        } else {
+            set cmdpackage $cmdnamespace
+            set cmdnamespace ::$cmdnamespace
+        }
+
+        if {![tcl::namespace::exists $cmdnamespace]} {
+            #only do package require if the namespace not already present
+            catch {package require $cmdpackage} pkg_load_info 
+            #recheck
+            if {![tcl::namespace::exists $cmdnamespace]} {
+                set prov [package provide $cmdpackage]
+                if {[tcl::string::length $prov]} {
+                    set provinfo "(package $cmdpackage is present with version $prov)"
+                } else {
+                    set provinfo "(package $cmdpackage not present)"
+                }
+                error "punk::overlay::import_commandset supplied namespace '$cmdnamespace' doesn't exist. $provinfo  Pkg_load_result: $pkg_load_info  Usage: import_commandset prefix separator namespace"
+            }
+        }
+     
+        punk::mix::util::namespace_import_pattern_to_namespace_noclobber ::punk::mix::util::* ${cmdnamespace}::util
+
+        #let child namespace 'lib' resolve parent namespace and thus util::xxx
+        tcl::namespace::eval ${cmdnamespace}::lib [tcl::string::map [list <cmdns> $cmdnamespace] {
+            ::set nspaths [tcl::namespace::path]
+            if {"<cmdns>" ni $nspaths} {
+                ::lappend nspaths <cmdns>
+            }
+            tcl::namespace::path $nspaths
+        }]
+
+        set imported_commands [list]
+        set imported_tails [list]
+        set nscaller [uplevel 1 [list tcl::namespace::current]]
+        if {[catch {
+            #review - noclobber?
+            tcl::namespace::eval ${nscaller}::temp_import [list tcl::namespace::import ${cmdnamespace}::*]
+            foreach cmd [tcl::info::commands ${nscaller}::temp_import::*] {
+                set cmdtail [tcl::namespace::tail $cmd]
+                if {$cmdtail eq "_default"} {
+                    set import_as ${nscaller}::${prefix}
+                } else {
+                    set import_as ${nscaller}::${prefix}${separator}${cmdtail}
+                }
+                rename $cmd  $import_as
+                lappend imported_commands $import_as
+                lappend imported_tails [namespace tail $import_as]
+            }
+            #make imported commands exported so they are available to the ensemble
+            tcl::namespace::eval ${nscaller} [list namespace export {*}$imported_tails]
+        } errM]} {
+            puts stderr "Error loading commandset $prefix $separator $cmdnamespace"
+            puts stderr "err: $errM"
+        }
+        return $imported_commands
+    }
+}
+
+package provide punk::overlay [tcl::namespace::eval punk::overlay {
+    variable version
+    set version 0.1.1
+}]

src/bootsupport/modules/punk/path-0.1.0.tm

@@ -565,10 +565,45 @@ namespace eval punk::path {
                 end]]
     }
 
+
+    ## for comparison
+    #proc nsglob_as_re {glob} {
+    #    #any segment that is not just * must match exactly one segment in the path
+    #    set pats [list]
+    #    foreach seg [nsparts_cached $glob] {
+    #        switch -exact -- $seg {
+    #            "" {
+    #                lappend pats ""
+    #            }
+    #            * {
+    #                #review - ::g*t  will not find ::got:it   (won't match single inner colon) - this should be fixed
+    #                #lappend pats {[^:]*}
+    #                #negative lookahead
+    #                #any number of chars not followed by ::, followed by any number of non :
+    #                lappend pats {(?:.(?!::))*[^:]*}
+    #            }
+    #            ** {
+    #                lappend pats {.*}
+    #            }
+    #            default {
+    #                set seg [string map {. [.]} $seg]
+    #                if {[regexp {[*?]} $seg]} {
+    #                    #set pat [string map [list ** {.*} * {[^:]*} ? {[^:]}] $seg]
+    #                    set pat [string map [list ** {.*} * {(?:.(?!::))*[^:]*} ? {[^:]}] $seg]
+    #                    lappend pats "$pat"
+    #                } else {
+    #                    lappend pats "$seg"
+    #                }
+    #            }
+    #        }
+    #    }
+    #    return  "^[join $pats ::]\$"
+    #}
     proc pathglob_as_re {pathglob} {
         #*** !doctools
         #[call [fun pathglob_as_re] [arg pathglob]]
         #[para] Returns a regular expression for matching a path to a glob pattern which can contain glob chars *|? in any segment of the path structure
+        #[para] Does not support square bracket globs or character classes.
         #[para] ** matches any number of subdirectories.
         #[para] e.g /etc/**/*.txt will match any .txt files at any depth below /etc  (except directly within /etc itself)
         #[para] e.g /etc/**.txt will match any .txt files at any depth below /etc
@@ -589,7 +624,7 @@ namespace eval punk::path {
                 *   {lappend pats {[^/]*}}
                 **  {lappend pats {.*}}
                 default {
-                    set seg [string map [list {^ {\^} $ {\$} [} {\[} ( {\(} \{ \\\{ \\ {\\}] $seg] ;#treat regex characters in the input as literals
+                    set seg [string map [list ^ {\^} $ {\$} \[ {\[} \] {\]} ( {\(} ) {\)} \{ \\\{ \\ {\\}] $seg] ;#treat regex characters (or tcl glob square bracket chars) in the input as literals
                     #set seg [string map [list  . {[.]}] $seg]
                     set seg [string map {. [.]} $seg]
                     if {[regexp {[*?]} $seg]} {
@@ -603,6 +638,52 @@ namespace eval punk::path {
         }
         return  "^[join $pats /]\$"
     }
+
+    punk::args::define {
+        @id     -id     ::punk::path::globmatchpath
+        @cmd    -name     punk::path::globmatchpath\
+            -summary\
+            "Match path to *|**|? glob patterns"\
+            -help\
+            "Return a boolean indicating whether the path matches the specialised glob pattern.
+            A pattern such as /usr/*/bin will match any path that has /usr as the first segment and bin as the third segment,
+            with any single segment in between.
+            A pattern such as /usr/**/bin will match any path that has /usr as the first segment and bin as the last segment,
+            with 1 or more segments in between (so it will not match /usr/bin).
+            A pattern such as /usr/** will match any path that has /usr as the first segment, with 1 or more segments
+            following (so it will not match /usr itself).
+            A pattern such as **/*.txt will match any path that ends with .txt, with 1 or more leading segments 
+            (so it will not match test.txt or .txt).
+            A pattern such as ** will match any path.
+            The glob characters * and ? are the only special characters in the pathglob syntax.
+            - they are treated as glob characters regardless of where they appear in the pathglob string.
+            Note that this is different from other Tcl glob contexts where square brackets can be used.
+            The pathglob syntax treats other characters, including square brackets as literals.
+            For example, the pattern /usr/te?t will match /usr/test and /usr/text but not /usr/texxt, and the pattern /usr/te*t
+            will match /usr/test, /usr/teat, and /usr/teeeet but not /usr/te/t.
+            The pathglob syntax does not support escaping of glob characters - any glob characters in the pathglob are treated
+            as glob characters. For example, the pattern /usr/* will match any path that has /usr as the first segment and any
+            single segment as the second segment, but there is no way to specify a pattern that matches any path that has /usr
+            as the first segment and a literal * as the second segment.
+            Caller must ensure that file separator is forward slash. (e.g use file normalize on windows)
+
+            options:
+                -nocase 0|1  (default 0 - case sensitive)
+             If -nocase is not supplied - default to case sensitive *except for driveletter*
+             ie - the driveletter alone in paths such as c:/etc will still be case insensitive. (ie c:/ETC/* will match C:/ETC/blah but not C:/etc/blah)
+             Explicitly specifying -nocase 0 will require the entire case to match including the driveletter.
+            "
+        @leaders
+            pathglob -type string -help "glob pattern to match path against. See [fun pathglob_as_re] for syntax of glob patterns"
+            path -type string -help "path to match against glob pattern"
+        @opts
+            -nocase -type boolean -default 0 -help\
+                "case insensitive matching (default false - case sensitive)
+                - except for driveletter on windows which is always case insensitive
+                unless -nocase 0 is explicitly specified"
+        @values -min 0 -max 0 
+    }
+    # -id
     proc globmatchpath {pathglob path args} {
         #*** !doctools
         #[call [fun globmatchpath] [arg pathglob] [arg path] [opt {option value...}]]
@@ -659,294 +740,576 @@ namespace eval punk::path {
         return $ismatch
     }
     punk::args::define {
-        @id     -id     ::punk::path::subfolders
+        @id     -id     ::punk::path::subfolders1
-        @cmd    -name     punk::path::subfolders\
+        @cmd    -name     punk::path::subfolders1\
             -summary\
-            "Listing of directories within supplied path."\
+            "Listing of directories below supplied path."\
             -help\
             "List of folders below path.
             The resulting list is unsorted."
         @opts
         -recursive -type none -help\
             ""
+        -exclude-paths -type list -default {} -help\
+            "list of path patterns to exclude from results.
+            May include * and ** path segments e.g /usr/**
+            A single /*/ will match any single segment in the path, and a single /**/ will match any number of segments in the path.
+
+            e.g to exclude any path with _aside as a segment in the middle: -exclude-paths **/_aside/**
+            i.e this would exclude /usr/_aside/etc and /usr/x/_aside/etc but not /usr/x/_aside or _aside/etc
+
+            To exclude all paths with _aside as a segment anywhere: -exclude-paths { **/_aside/** **/_aside _aside/**}
+            "
         #todo -depth
         @values -min 0 -max 1
         path -type directory -optional 1 -help\
-            "Path of folder. If not supplied current directory is used."
+            "Path of folder. If not supplied current directory is used.
-    }
+            This may be a relative or absolute path. Relative paths are treated as relative to current directory.
-    proc subfolders {args} {
+            When using relative paths - the result will also be relative paths with the same relative prefix.
-        set argd [punk::args::parse $args withid ::punk::path::subfolders]
+            (e.g if path is ../test - the results will be ../test/subfolder1 ../test/subfolder2 etc)
+            Patterns in -exclude-paths are matched against the resulting paths
+            (so should be written to match the same relative prefix if path is relative)"
+    }
+
+    proc subfolders1 {args} {
+        #NOTE - this algorithm based on omit_only_patterns and prune_base_patterns was suggested by a 2026 AI model - it is apparent to this programmer that it is inadequate for the purpose.
+        #e.g consider subfolders1 -recursion -exclude {**/vfs/** **/src/**}
+        #This can still return something like c:/repo/etc/src/vfs - which should be excluded by the pattern **/src/**
+        #todo - review and fix properly.
+        set argd [punk::args::parse $args withid ::punk::path::subfolders1]
         lassign [dict values $argd] leaders opts values received
         set do_recursion    [dict exists $received -recursive]
+        set exclude_paths   [dict get $opts -exclude-paths]
+        if {"**" in $exclude_paths} {
+            #if ** is in exclude_paths - then we can skip all glob matching and just return empty list
+            #This is likely user error - so we'll be loud about it for now but will still return empty list rather than erroring.
+            #If user code is building exclude_paths dynamically - they can check for this case themselves and avoid the call to subfolders1 to suppress this message.
+            puts stderr "punk::path::subfolders1 Warning - exclude_paths contains '**' - all paths will be excluded"
+            return [list]
+        }
         if {[dict exists $received path]} {
             set path [dict get $values path]
         } else {
             set path [pwd]
         }
-        set folders [glob -nocomplain -directory $path -types d *]
+
+        set all_subfolders [glob -nocomplain -directory $path -types d *]
+
+
+        #example of expected exclude_paths pattern behaviour when recursion is enabled:
+        # **/dirname    -> omit /x/y/dirname, but still visit /x/y/dirname/*
+
+        # **/dirname/*   -> include /x/y/dirname and /x/y/dirname/a/b but omit directories that are a single level below /x/y/dirname such as /x/y/dirname/a
+
+        #c:/** - would exclude all subfolders below c: but not c: itself
+
+        # **/test/** - would exclude any path with test as a segment and all its subfolders
+        #- but not paths with test as a segment that is the final segment
+
+        set folders [list]
+        set recurse_subdirs [list]
+
+        foreach f $all_subfolders {
+            set include_in_results 1
+            set allow_recurse 1
+            foreach pat $exclude_paths {
+                set pat_parts [file split $pat] ;#note file split c:/test gives {c:/ test}  but file split **/test gives {** test}
+                #also note that file split on windows treats forward slashes and backslashes the same.
+                #by using file split, we gain some flexibility in syntax of paths and patterns,
+                #but lose the ability to use backslashes as escapes to allow literal glob characters in path segments.
+                #This is almost always a non-issue on windows since * and ? are not valid in path segments there, and is rarely an issue on unix even though
+                # * and ? are technically valid in path segments, but it is inadvisable there anyway for compatibility with shells etc.
+                if {[llength $pat_parts] >= 2 && [lindex $pat_parts end] eq "**"} {
+                    set base_pat [file join {*}[lrange $pat_parts 0 end-1]]
+                    if {[globmatchpath $pat $f]} {
+                        set include_in_results 0
+                        set allow_recurse 0
+                    } elseif {[globmatchpath $base_pat $f]} {
+                        set allow_recurse 0
+                    }
+                } elseif {[globmatchpath $pat $f]} {
+                    set include_in_results 0
+                }
+                if {!$include_in_results && !$allow_recurse} {
+                    break
+                }
+            }
+            if {$include_in_results} {
+                lappend folders $f
+            }
+            if {$allow_recurse} {
+                lappend recurse_subdirs $f
+            }
+        }
         if {$do_recursion} {
-            foreach subdir $folders {
+            foreach subdir $recurse_subdirs {
-                lappend folders {*}[subfolders -recursive $subdir]
+                lappend folders {*}[subfolders1 -exclude-paths $exclude_paths -recursive $subdir]
             }
         }
         return $folders
     }
 
-    #todo - treefolders with similar search caps as treefilenames
+    namespace eval subfolder_priv {
-
+        proc classify_exclude_pattern {pat} {
-    punk::args::define {
+            set parts [file split $pat]
-        @id     -id ::punk::path::treefilenames
+            if {[llength $parts] >= 2 && [lindex $parts end] eq "**"} {
-        -directory       -type directory  -help\
+                set boundary_pat [file join {*}[lrange $parts 0 end-1]]
-            "folder in which to begin recursive scan for files."
+                return [dict create \
-        -call-depth-internal    -default 0 -type integer
+                    pattern $pat \
-        -sort -type any -default natural -choices {none ascii dictionary natural}
+                    kind subtree \
-        -antiglob_paths         -default {} -help\
+                    boundary_pat $boundary_pat \
-            "list of path patterns to exclude
+                    descend_pat $pat]
-              may include * and ** path segments e.g
+            }
-              /usr/**       (exlude subfolders based at /usr but not
+            if {[llength $parts] >= 2 && [lindex $parts end] eq "*"} {
-                             files within /usr itself)
+                return [dict create \
-              **/_aside     (exlude  files where _aside is last segment)
+                    pattern $pat \
-              **/_aside/*   (exclude folders one below an _aside folder)
+                    kind child_only \
-              **/_aside/**  (exclude all folders with _aside as a segment)"
+                    match_pat $pat]
-        -antiglob_files         -default {}
+            }
-        @values -min 0 -max -1 -optional 1 -type string
+            return [dict create \
-        tailglobs -default * -multiple 1   -help\
+                pattern $pat \
-            "Patterns to match against filename portion (last segment) of each file path
+                kind exact \
-              within the directory tree being searched."
+                match_pat $pat]
         }
 
-    #todo - implement treefiles which acts like dirfiles but allows path globbing in the same way as punk::ns::ns/
+        proc compile_exclude_rules {exclude_paths} {
-    #then review if treefiles can replace dirfiles or if both should exist (dirfiles can have literal glob chars in path segments - but that is a rare usecase)
+            set rules [list]
-    proc treefilenames {args} {
+            foreach pat $exclude_paths {
-        #*** !doctools
+                lappend rules [classify_exclude_pattern $pat]
-        #[call [fun treefilenames]  [opt {option value...}] [opt {globpattern...}]]
+            }
-        #[para]basic (glob based) list of filenames matching each pattern in tailglobs - recursive
+            return $rules
-        #[para] options:
+        }
-        #[para] [opt -dir] <path>
-        #[para]     defaults to [lb]pwd[rb] - base path for tree to search
-        #[para] [opt -antiglob_paths] <list>
-        #[para]     list of path patterns to exclude - may include * and ** path segments e.g /usr/**
-        #[para]no natsorting - so order is dependent on filesystem
 
-        set argd [punk::args::parse $args withid ::punk::path::treefilenames]
+        proc match_rule_at_node {rule path} {
-        lassign [dict values $argd] leaders opts values received
+            set kind [dict get $rule kind]
-        set tailglobs [dict get $values tailglobs]
+            switch -- $kind {
-        # -- --- --- --- --- --- ---
+                exact - child_only {
-        set opt_sort           [dict get $opts -sort]
+                    if {[::punk::path::globmatchpath [dict get $rule match_pat] $path]} {
-        set opt_antiglob_paths [dict get $opts -antiglob_paths]
+                        return [dict create include_current 0 recurse_below 1 child_rules [list $rule]]
-        set opt_antiglob_files [dict get $opts -antiglob_files]
+                    }
-        set CALLDEPTH [dict get $opts -call-depth-internal]
+                    return [dict create include_current 1 recurse_below 1 child_rules [list $rule]]
-        # -- --- --- --- --- --- ---
+                }
-        # -- --- --- --- --- --- ---
+                subtree {
+                    set descend_pat  [dict get $rule descend_pat]
+                    set boundary_pat [dict get $rule boundary_pat]
+                    if {[::punk::path::globmatchpath $descend_pat $path]} {
+                        return [dict create include_current 0 recurse_below 0 child_rules [list]]
+                    }
+                    if {[::punk::path::globmatchpath $boundary_pat $path]} {
+                        return [dict create include_current 1 recurse_below 0 child_rules [list]]
+                    }
+                    return [dict create include_current 1 recurse_below 1 child_rules [list $rule]]
+                }
+                default {
+                    error "Unknown exclude rule kind '$kind'"
+                }
+            }
+        }
 
-        set files [list]
+        proc walk_subfolders {path rules do_recursion} {
-        if {$CALLDEPTH == 0} {
+            set all_subfolders [glob -nocomplain -directory $path -types d *]
-            if {$opt_sort eq "natural"} {
+            set folders [list]
-                package require natsort
+            foreach f $all_subfolders {
+                set include_current 1
+                set recurse_below $do_recursion
+                set child_rules [list]
+                foreach rule $rules {
+                    set outcome [match_rule_at_node $rule $f]
+                    if {![dict get $outcome include_current]} {
+                        set include_current 0
                     }
-            #set opts [dict merge $opts [list -directory $opt_dir]]
+                    if {![dict get $outcome recurse_below]} {
-            if {![dict exists $received -directory]} {
+                        set recurse_below 0
-                set opt_dir [pwd]
-            } else {
-                set opt_dir [dict get $opts -directory]
                     }
-            if {![file isdirectory $opt_dir]} {
+                    if {$do_recursion} {
+                        lappend child_rules {*}[dict get $outcome child_rules]
+                    }
+                    if {!$include_current && !$recurse_below} {
+                        break
+                    }
+                }
+                if {$include_current} {
+                    lappend folders $f
+                }
+                if {$do_recursion && $recurse_below} {
+                    lappend folders {*}[walk_subfolders $f $child_rules $do_recursion]
+                }
+            }
+            return $folders
+        }
+    }
+
+    punk::args::define {
+        @id     -id     ::punk::path::subfolders
+        @cmd    -name     punk::path::subfolders\
+            -summary\
+            "Listing of directories below supplied path."\
+            -help\
+            "List of folders below path.
+            The resulting list is unsorted.
+            "
+        @opts
+        -recursive -type none -help\
+            ""
+        -exclude-paths -type list -default {} -help\
+            "list of path patterns to exclude from results.
+            May include * and ** path segments e.g /usr/**
+            A single /*/ will match any single segment in the path, and a single /**/ will match any number of segments in the path.
+
+            e.g to exclude any path with _aside as a segment in the middle: -exclude-paths **/_aside/**
+            i.e this would exclude /usr/_aside/etc and /usr/x/_aside/etc but not /usr/x/_aside or _aside/etc
+
+            To exclude all paths with _aside as a segment anywhere: -exclude-paths { **/_aside/** **/_aside ./_aside/**}
+            "
+        #todo -depth
+        @values -min 0 -max 1
+        path -type directory -optional 1 -help\
+            "Path of base folder. If not supplied current directory is used.
+            This may be a relative or absolute path. Relative paths are treated as relative to current directory.
+            When using relative paths - the result will also be relative paths with the same relative prefix.
+            (e.g if path is ../test - the results will be ../test/subfolder1 ../test/subfolder2 etc)
+            Patterns in -exclude-paths are matched against the resulting paths
+            (so should be written to match the same relative prefix if path is relative)"
+    }
+
+    proc subfolders {args} {
+        set argd [punk::args::parse $args withid ::punk::path::subfolders]
+        lassign [dict values $argd] leaders opts values received
+        set do_recursion    [dict exists $received -recursive]
+        set exclude_paths   [dict get $opts -exclude-paths]
+        if {"**" in $exclude_paths} {
+            puts stderr "punk::path::subfolders Warning - exclude_paths contains '**' - all paths will be excluded"
             return [list]
         }
+        if {[dict exists $received path]} {
+            set path [dict get $values path]
         } else {
-            #assume/require to exist in any recursive call
+            set path [pwd]
-            set opt_dir [dict get $opts -directory]
+        }
+        set compiled_rules [subfolder_priv::compile_exclude_rules $exclude_paths]
+        return [subfolder_priv::walk_subfolders $path $compiled_rules $do_recursion]
     }
 
-        #comment out to compare timings with treefilenames_zipfs
+    namespace eval treefile_priv {
-        if {[string match //zipfs:/* $opt_dir]} {
+        proc _pattern_prefix_viable_parts {pattern_parts path_parts} {
-            return [treefilenames_zipfs {*}$args]
+            if {![llength $path_parts]} {
+                return 1
+            }
+            if {![llength $pattern_parts]} {
+                return 0
             }
 
-        set skip 0
+            set pattern_head [lindex $pattern_parts 0]
-        foreach anti $opt_antiglob_paths {
+            set path_head [lindex $path_parts 0]
-            if {[globmatchpath $anti $opt_dir]} {
+
-                set skip 1
+            if {$pattern_head eq "**"} {
+                if {[_pattern_prefix_viable_parts [lrange $pattern_parts 1 end] $path_parts]} {
+                    return 1
+                }
+                return [_pattern_prefix_viable_parts $pattern_parts [lrange $path_parts 1 end]]
+            }
+
+            if {[::punk::path::globmatchpath $pattern_head $path_head]} {
+                return [_pattern_prefix_viable_parts [lrange $pattern_parts 1 end] [lrange $path_parts 1 end]]
+            }
+            return 0
+        }
+
+        proc pattern_prefix_viable {pattern path} {
+            return [_pattern_prefix_viable_parts [file split $pattern] [file split $path]]
+        }
+
+        proc pattern_boundary {pattern} {
+            set parts [file split $pattern]
+            if {[llength $parts] >= 2 && [lindex $parts end] eq "**"} {
+                return [file join {*}[lrange $parts 0 end-1]]
+            }
+            return ""
+        }
+
+        proc directory_state {glob_paths path inherited_allbelow} {
+            if {$inherited_allbelow} {
+                return [dict create include_files 1 recurse_below 1 next_allbelow 1]
+            }
+
+            set include_files 0
+            set recurse_below 0
+            set next_allbelow 0
+
+            foreach gp $glob_paths {
+                if {[::punk::path::globmatchpath $gp $path]} {
+                    set include_files 1
+                    set recurse_below 1
+                    set next_allbelow 1
                     break
                 }
+
+                set boundary [pattern_boundary $gp]
+                if {$boundary ne "" && [::punk::path::globmatchpath $boundary $path]} {
+                    set recurse_below 1
+                    set next_allbelow 1
+                    continue
+                }
+
+                if {[pattern_prefix_viable $gp $path]} {
+                    set recurse_below 1
                 }
-        if {$skip} {
-            return [list]
             }
 
-        #todo - account for vfs where matched path could appear to be a directory but is mounted so could be a desired match?
+            return [dict create                     {*}{
-        if {[catch {glob -nocomplain -dir $opt_dir -type f {*}$tailglobs} matches]} {
+                    } include_files $include_files  {*}{
-            #we can get for example a permissions error
+                    } recurse_below $recurse_below  {*}{
-            puts stderr "treefilenames error while listing files in dir $opt_dir\n $matches"
+                    } next_allbelow $next_allbelow  {*}{
-            set dirfiles [list]
-        } else {
-            set retained [list]
-            if {[llength $opt_antiglob_files]} {
-                foreach m $matches {
-                    set skip 0
-                    set ftail [file tail $m]
-                    foreach anti $opt_antiglob_files {
-                        if {[string match $anti $ftail]} {
-                            set skip 1; break
                     }
+                ]
         }
-                    if {!$skip} {
+
-                        lappend retained $m
+        proc child_path_state {glob_paths child_path inherited_allbelow} {
+            if {$inherited_allbelow} {
+                return 1
             }
+            foreach gp $glob_paths {
+                if {[pattern_prefix_viable $gp $child_path]} {
+                    return 1
                 }
-            } else {
-                set retained $matches
             }
-            switch -- $opt_sort {
+            return 0
+        }
+
+        proc _sort_paths {paths sortmode} {
+            switch -- $sortmode {
                 ascii {
-                    set dirfiles [lsort $retained]
+                    return [lsort $paths]
                 }
                 dictionary {
-                    set dirfiles [lsort -dictionary $retained]
+                    return [lsort -dictionary $paths]
                 }
                 natural {
-                    set dirfiles [natsort::sort $retained]
+                    return [natsort::sort $paths]
                 }
                 default {
-                    set dirfiles $retained
+                    return $paths
                 }
             }
         }
 
-        lappend files  {*}$dirfiles
+        proc _path_matches_any {patterns path} {
-        if {[catch {glob -nocomplain -dir $opt_dir -type d *} dirdirs]} {
+            foreach pattern $patterns {
-            puts stderr "treefilenames error while listing subdirs in dir $opt_dir\n $dirdirs"
+                if {[::punk::path::globmatchpath $pattern $path]} {
-            set dirdirs [list]
+                    return 1
                 }
-        set okdirs [list]
+            }
-        foreach dir $dirdirs {
+            return 0
+        }
+
+        proc _tailbase_relative {tailbase path} {
+            if {$tailbase eq ""} {
+                return $path
+            }
+            return [::punk::path::relative $tailbase $path]
+        }
+
+        proc _tailbase_match_path {tailbase path} {
+            set match_path [_tailbase_relative $tailbase $path]
+            if {$match_path eq "."} {
+                return ""
+            }
+            return $match_path
+        }
+
+        proc _tailbase_relative_list {tailbase paths} {
+            if {$tailbase eq ""} {
+                return $paths
+            }
+            set relative_paths [list]
+            foreach path $paths {
+                lappend relative_paths [_tailbase_relative $tailbase $path]
+            }
+            return $relative_paths
+        }
+
+        proc _retain_files {matches exclude_files sortmode} {
+            set retained [list]
+            foreach match $matches {
                 set skip 0
-            foreach anti $opt_antiglob_paths {
+                set file_tail [file tail $match]
-                if {[globmatchpath $anti $dir]} {
+                foreach anti $exclude_files {
+                    if {[string match $anti $file_tail]} {
                         set skip 1
                         break
                     }
                 }
                 if {!$skip} {
-                lappend okdirs $dir
+                    lappend retained $match
                 }
             }
-        if {[llength $okdirs]} {
+            return [_sort_paths $retained $sortmode]
-            switch -- $opt_sort {
-                ascii {
-                    set finaldirs [lsort $okdirs]
         }
-                dictionary {
+
-                    set finaldirs [lsort -dictionary $okdirs]
+        proc _state_from_argd {argd} {
+            set opts        [dict get $argd opts]
+            set values      [dict get $argd values]
+            set received    [dict get $argd received]
+
+            if {[dict exists $received -directory]} {
+                set directory [dict get $opts -directory]
+            } else {
+                set directory [pwd]
             }
-                natural {
+
-                    set finaldirs [natsort::sort $okdirs]
+            set glob_paths [dict get $opts -include-paths]
+            if {"*" in $glob_paths} {
+                set glob_paths {*}
             }
-                default {
+
-                    set finaldirs $okdirs
+            set sortmode [dict get $opts -sort]
+            if {$sortmode eq "natural"} {
+                package require natsort
             }
+
+            return [dict create {*}{
+                    depth           0
+                    subvector       {}
+                    allbelow        0
+                }   sort            $sortmode                             {*}{
+                }   directory       $directory                            {*}{
+                }   tailbase        [dict get $opts -tailbase]            {*}{
+                }   exclude_paths   [dict get $opts -exclude-paths]       {*}{
+                }   exclude_files   [dict get $opts -exclude-files]       {*}{
+                }   glob_paths      $glob_paths                           {*}{
+                }   tailglobs       [dict get $values tailglobs]          {*}{
+                }
+            ]
+        }
+
+        proc walk_treefilenames {state} {
+            set opt_dir             [dict get $state directory]
+            set opt_tailbase        [dict get $state tailbase]
+            set depth               [dict get $state depth]
+            set subvector           [dict get $state subvector]
+            set callallbelow        [dict get $state allbelow]
+            set opt_sort            [dict get $state sort]
+            set opt_exclude_paths   [dict get $state exclude_paths]
+            set opt_exclude_files   [dict get $state exclude_files]
+            set opt_glob_paths      [dict get $state glob_paths]
+            set tailglobs           [dict get $state tailglobs]
+
+            if {![file isdirectory $opt_dir]} {
+                return [list]
             }
-            foreach dir $finaldirs {
+            if {[string match //zipfs:/* $opt_dir]} {
-                set nextopts [dict merge $opts [list -directory $dir -call-depth-internal [incr CALLDEPTH]]]
+                return [walk_treefilenames_zipfs $state]
-                lappend files {*}[treefilenames {*}$nextopts {*}$tailglobs]
             }
+            set opt_dir_match [_tailbase_match_path $opt_tailbase $opt_dir]
+            if {[_path_matches_any $opt_exclude_paths $opt_dir_match]} {
+                return [list]
             }
+
+            set files [list]
+            set dir_state [directory_state $opt_glob_paths $opt_dir_match $callallbelow]
+            if {[dict get $dir_state include_files]} {
+                if {[catch {glob -nocomplain -dir $opt_dir -type f {*}$tailglobs} matches]} {
+                    puts stderr "treefilenames error while listing files in dir $opt_dir\n $matches"
+                    set dirfiles [list]
+                } else {
+                    set dirfiles [_retain_files $matches $opt_exclude_files $opt_sort]
+                }
+                lappend files {*}[_tailbase_relative_list $opt_tailbase $dirfiles]
+            }
+
+            if {![dict get $dir_state recurse_below]} {
                 return $files
             }
-    proc treefilenames_zipfs {args} {
-        #seems to be 2 or 3 times faster than treefilenames for //zipfs:/ paths  - REVIEW
-        # is sort order the same?
-        set argd [punk::args::parse $args withid ::punk::path::treefilenames]
-        lassign [dict values $argd] leaders opts values received
-        set tailglobs [dict get $values tailglobs]
-        # -- --- --- --- --- --- ---
-        set opt_antiglob_paths [dict get $opts -antiglob_paths]
-        set opt_antiglob_files [dict get $opts -antiglob_files]
-        set opt_sort           [dict get $opts -sort]
-        set CALLDEPTH [dict get $opts -call-depth-internal]
-        # -- --- --- --- --- --- ---
-        # -- --- --- --- --- --- ---
 
-        set files [list]
+            if {[catch {glob -nocomplain -dir $opt_dir -type d *} dirdirs]} {
-        if {$CALLDEPTH == 0} {
+                puts stderr "treefilenames error while listing subdirs in dir $opt_dir\n $dirdirs"
-            if {$opt_sort eq "natural"} {
+                set dirdirs [list]
-                package require natsort
             }
-            #set opts [dict merge $opts [list -directory $opt_dir]]
+            set okdirs [list]
-            if {![dict exists $received -directory]} {
+            foreach dir $dirdirs {
-                set opt_dir [pwd]
+                if {![_path_matches_any $opt_exclude_paths [_tailbase_match_path $opt_tailbase $dir]]} {
-            } else {
+                    lappend okdirs $dir
-                set opt_dir [dict get $opts -directory]
                 }
-            if {![file isdirectory $opt_dir]} {
-                return [list]
             }
+
+            if {$opt_glob_paths eq "*"} {
+                set matchdirs $okdirs
             } else {
-            #assume/require to exist in any recursive call
+                set matchdirs [list]
-            set opt_dir [dict get $opts -directory]
+                foreach dir $okdirs {
+                    if {$callallbelow || [child_path_state $opt_glob_paths [_tailbase_match_path $opt_tailbase $dir] $callallbelow]} {
+                        lappend matchdirs $dir
+                    }
                 }
-        if {![string match [zipfs root]* $opt_dir]} {
-            error "treefilenames_zipfs can only be used on paths beginning with [zipfs root] on this systems"
             }
-        set dir [string trimright $opt_dir "/"] ;#e.g normalize //zipfs:/x/ to //zipfs:/x
-        set dirlen [string length $dir]
 
-        set skip 0
+            set finaldirs [_sort_paths $matchdirs $opt_sort]
-        foreach anti $opt_antiglob_paths {
+            set childallbelow [expr {$callallbelow || [dict get $dir_state next_allbelow]}]
-            if {[globmatchpath $anti $dir]} {
+            set nextsubvector [list {*}$subvector [file tail $opt_dir]]
-                set skip 1
+            foreach dir $finaldirs {
-                break
+                set child_state [dict merge $state [dict create {*}{} \
+                    directory   $dir \
+                    depth       [expr {$depth + 1}] \
+                    subvector   $nextsubvector \
+                    allbelow    $childallbelow]]
+                lappend files {*}[walk_treefilenames $child_state]
+            }
+            return $files
         }
+
+        proc walk_treefilenames_zipfs {state} {
+            set opt_dir             [dict get $state directory]
+            set opt_tailbase        [dict get $state tailbase]
+            set opt_exclude_paths   [dict get $state exclude_paths]
+            set opt_exclude_files   [dict get $state exclude_files]
+            set opt_glob_paths      [dict get $state glob_paths]
+            set opt_sort            [dict get $state sort]
+            set tailglobs           [dict get $state tailglobs]
+
+            if {![string match [zipfs root]* $opt_dir]} {
+                error "treefilenames_zipfs can only be used on paths beginning with [zipfs root] on this systems"
             }
-        if {$skip} {
+            set dir [string trimright $opt_dir "/"]
+            if {[_path_matches_any $opt_exclude_paths [_tailbase_match_path $opt_tailbase $dir]]} {
                 return [list]
             }
+            set dirlen [string length $dir]
             set subpaths [zipfs list $dir/*]
             set dirlist [list]
             set skipdirs [list]
             set filelist [list]
-        #process in the order they came - sorting large list more expensive?? review
             foreach sub $subpaths {
-            set tail [string range $sub $dirlen+1 end] ;#dirlen is without trailing slash
+                set tail [string range $sub $dirlen+1 end]
                 set tailparts [file split $tail]
                 set accum ""
                 set skipdir 0
-            foreach tp [lrange $tailparts 0 end-1] {
+                foreach tailpart [lrange $tailparts 0 end-1] {
-                append accum "/$tp"
+                    append accum "/$tailpart"
                     set superpath "${dir}${accum}"
                     if {$superpath in $skipdirs} {
-                    #subpart already in skipdirs
                         set skipdir 1
                         break
                     }
                     if {$superpath ni $dirlist} {
-                    set skip2 0
+                        if {[_path_matches_any $opt_exclude_paths [_tailbase_match_path $opt_tailbase $superpath]]} {
-                    foreach anti $opt_antiglob_paths {
-                        if {[globmatchpath $anti $superpath]} {
-                            set skip2 1
                             lappend skipdirs $superpath
-                            break
-                        }
-                    }
-                    if {!$skip2} {
-                        lappend dirlist $superpath
-                    } else {
                             set skipdir 1
                             break
+                        } else {
+                            lappend dirlist $superpath
                         }
                     }
                 }
                 if {!$skipdir} {
-                #process final part of path
                     append accum "/[lindex $tailparts end]"
                     set finalpart "${dir}${accum}"
                     if {$finalpart ni $dirlist} {
                         if {[file type $finalpart] eq "file"} {
-                        set ftail [lindex $tailparts end]
+                            set file_tail [lindex $tailparts end]
                             set match 0
                             if {"*" ni $tailglobs} {
-                            foreach tg $tailglobs {
+                                foreach tailglob $tailglobs {
-                                if {[string match $tg $ftail]} {
+                                    if {[string match $tailglob $file_tail]} {
                                         set match 1
                                         break
                                     }
@@ -955,31 +1318,29 @@ namespace eval punk::path {
                                 set match 1
                             }
                             if {$match} {
-                            if {[llength $opt_antiglob_files]} {
+                                if {$opt_glob_paths ne "*"} {
+                                    set file_dir_match [_tailbase_match_path $opt_tailbase [file dirname $finalpart]]
+                                    set file_dir_state [directory_state $opt_glob_paths $file_dir_match 0]
+                                    set match [dict get $file_dir_state include_files]
+                                }
+                            }
+                            if {$match} {
                                 set skipfile 0
-                                foreach anti $opt_antiglob_files {
+                                foreach anti $opt_exclude_files {
-                                    if {[string match $anti $ftail]} {
+                                    if {[string match $anti $file_tail]} {
-                                        set skipfile 1; break
+                                        set skipfile 1
+                                        break
                                     }
                                 }
                                 if {!$skipfile} {
-                                    lappend filelist $finalpart
+                                    lappend filelist [_tailbase_relative $opt_tailbase $finalpart]
-                                }
-                            } else {
-                                lappend filelist $finalpart
                                 }
                             }
                         } else {
                             if {$finalpart ni $dirlist} {
-                            set skip2 0
+                                if {[_path_matches_any $opt_exclude_paths [_tailbase_match_path $opt_tailbase $finalpart]]} {
-                            foreach anti $opt_antiglob_paths {
-                                if {[globmatchpath $anti $finalpart]} {
-                                    set skip2 1
                                     lappend skipdirs $finalpart
-                                    break
+                                } else {
-                                }
-                            }
-                            if {!$skip2} {
                                     lappend dirlist $finalpart
                                 }
                             }
@@ -987,21 +1348,81 @@ namespace eval punk::path {
                     }
                 }
             }
-        switch -- $opt_sort {
+            return [_sort_paths $filelist $opt_sort]
-            ascii {
-                set finalfilelist [lsort $filelist]
         }
-            dictionary {
-                set finalfilelist [lsort -dictionary $filelist]
     }
-            natural {
+
-                set finalfilelist [natsort::sort $filelist]
+    #todo - treefolders with similar search caps as treefilenames
+
+    punk::args::define {
+        @id     -id     ::punk::path::treefilenames
+        @cmd    -name     punk::path::treefilenames\
+            -summary\
+            "List of filenames below supplied path."\
+            -help\
+            "List of filenames below path.
+            The resulting list is unsorted.
+
+            The path globbing syntax supports *, ** and ? as glob characters in any segment of the path, with the following semantics:
+            * matches any single segment in the path
+            ** matches 1 or more segments in the path (so /usr/**/bin will match /usr/x/bin and user/x/y/bin but not /usr/bin )
+            ? matches any single character in a single segment of the path (so /usr/te?t will match /usr/test and /usr/text but not /usr/texxt)
+            "
+        -directory       -type directory  -help\
+            "folder in which to begin recursive scan for files."
+        -tailbase         -type string    -default "" -help\
+            "if supplied, only the relative path compared to the tailbase will be returned for each file.
+            So if tailbase is /usr and a file is found at /usr/x/y/file.txt, the returned path for that file would be x/y/file.txt.
+            If tailbase is not supplied, the full path to each file will be returned.
+
+            If tailbase is supplied, it should be a prefix of the directory supplied (or the directory itself)
+            The patterns in -exclude-paths should be written to match the returned paths (i.e with the tailbase prefix removed) if -tailbase is supplied.
+            If the tailbase is not a prefix of the directory supplied, the resulting paths may have /../ components in them to account for the difference,
+            but the behaviour is not well defined in this case and it is recommended to ensure tailbase is a prefix of the directory supplied if using -tailbase.
+
+            see: punk::path::relative to compute relative paths
+            "
+        -sort -type any         -default natural -choices {none ascii dictionary natural}
+        -exclude-paths         -default {} -help\
+            "list of path patterns to exclude
+              may include * and ** path segments e.g
+              /usr/**       (exclude subfolders based at /usr but not
+                             files within /usr itself)
+              **/_aside     (exclude  files where _aside is last segment)
+              **/_aside/*   (exclude folders one below an _aside folder)
+              **/_aside/**  (exclude files in all folders with _aside as a segment)"
+        -exclude-files         -default {}
+        -include-paths     -default {**} -help\
+            "list of path patterns to include
+              may include * and ** path segments e.g
+              /usr/**       (include files in subfolders based at /usr but not
+                             files within /usr itself)
+              **/_aside     (include  files where _aside is last segment in the folder)
+              **/_aside/*   (include files in folders one below an _aside folder)
+              **/_aside/**  (include all files in folders with _aside as a segment)"
+        @values -min 0 -max -1 -optional 1 -type string
+        tailglobs -default * -multiple 1   -help\
+            "Patterns to match against filename portion (last segment) of each file path
+              within the directory tree being searched."
     }
-            default {
+
-                set finalfilelist $filelist
+    #todo - implement treefiles which acts like dirfiles but allows path globbing in the same way as punk::ns::ns/
+    #then review if treefiles can replace dirfiles or if both should exist (dirfiles can have literal glob chars in path segments - but that is a rare usecase)
+    proc treefilenames {args} {
+        set argd [punk::args::parse $args withid ::punk::path::treefilenames]
+        set state [treefile_priv::_state_from_argd $argd]
+        return [treefile_priv::walk_treefilenames $state]
     }
+    punk::args::set_idalias ::punk::path::treefilenames_zipfs ::punk::path::treefilenames
+    proc treefilenames_zipfs {args} {
+        #seems to be 2 or 3 times faster than treefilenames for //zipfs:/ paths  - REVIEW
+        # is sort order the same?
+        set argd [punk::args::parse $args withid ::punk::path::treefilenames]
+        set state [treefile_priv::_state_from_argd $argd]
+        if {![file isdirectory [dict get $state directory]]} {
+            return [list]
         }
-        return $finalfilelist
+        return [treefile_priv::walk_treefilenames_zipfs $state]
     }
 
     #maint warning - also in punkcheck

src/bootsupport/modules/punk/pipe-1.0.tm

@@ -169,8 +169,8 @@ tcl::namespace::eval punk::pipe::lib {
         #This stops us matching {/@**@x x}  vs {/@**@x  x}
         #---
 
-        set rhs [tcl::string::map {: <c> ? <q> * <star> [ <lb> ] <rb> \\ <bsl> {"} <dq> " " <sp>} $rhs] 
+        set rhs [tcl::string::map {: <c> ; <sc> ? <q> * <star> [ <lb> ] <rb> \\ <bsl> {"} <dq> " " <sp>} $rhs] 
-        #review - we don't expect other command-incompatible chars such as colon?
+        #review - we don't expect other command-incompatible chars?
         return $rhs
     }
 
@@ -187,6 +187,7 @@ tcl::namespace::eval punk::pipe::lib {
 
     #exclude quoted whitespace
     proc arg_is_script_shaped {arg} {
+        set arg [string map {\\; "<escaped_semicolon>"} $arg]
         if {[tcl::string::first \n $arg] >= 0} {
             return 1
         } elseif {[tcl::string::first ";" $arg] >= 0} {

src/bootsupport/modules/punk/repo-0.1.1.tm

@@ -1817,17 +1817,13 @@ namespace eval punk::repo {
         error "unimplemented"
     }
 
-    #file normalize is expensive so this is too
+    #file normalize can be a little expensive so this is too
     proc norm {path {platform env}} {
-        #kettle::path::norm
-        #see also wiki
-        #full path normalization
-
-        set platform [string tolower $platform]
-        if {$platform eq "env"} {
-            set platform $::tcl_platform(platform)
-        }
 
+        #set platform [string tolower $platform]
+        #if {$platform eq "env"} {
+        #    set platform $::tcl_platform(platform)
+        #}
         #No - don't do this sort of path translation here - leave as option for specific utils only such as ./
         #Windows volume-relative syntax with specific volume specified is somewhat broken in Tcl - but leading slash volume-relative does work
         #We shouldn't break it totally just because accessing WSL/mingw paths is slightly more useful
@@ -1835,6 +1831,9 @@ namespace eval punk::repo {
             #return [file dirname [file normalize [punk::unixywindows::towinpath $path]/__]]
         #}
 
+        #kettle::path::norm
+        #see also wiki
+        #full path normalization
         return [file dirname [file normalize $path/__]]
     }
 

src/bootsupport/modules/punkapp-0.1.1.tm

@@ -0,0 +1,240 @@
+#utilities for punk apps to call
+
+
+namespace eval punkapp {
+    variable result
+    variable waiting "no"
+    proc hide_dot_window {} {
+        #alternative to wm withdraw .
+        #see https://wiki.tcl-lang.org/page/wm+withdraw
+        wm geometry . 1x1+0+0
+        wm overrideredirect . 1
+        wm transient .
+    }
+    proc is_toplevel {w} {
+        if {![llength [info commands winfo]]} {
+            return 0
+        }
+        expr {[winfo toplevel $w] eq $w && ![catch {$w cget -menu}]}
+    }
+    proc get_toplevels {{w .}} {
+        if {![llength [info commands winfo]]} {
+            return [list]
+        }
+        set list {}
+        if {[is_toplevel $w]} {
+            lappend list $w
+        }
+        foreach w [winfo children $w] {
+            lappend list {*}[get_toplevels $w]
+        }
+        return $list
+    }
+
+    proc make_toplevel_next {prefix} {
+        set top [get_toplevel_next $prefix]
+        return [toplevel $top]
+    }
+    #possible race condition if multiple calls made without actually creating the toplevel, or gap if highest existing closed in the meantime
+    #todo - reserve_toplevel_next ? keep list of toplevels considered 'allocated' even if never created or already destroyed? what usecase?
+    #can call wm withdraw to to reserve newly created toplevel. To stop re-use of existing names after destruction would require a list or at least a record of highest created for each prefix
+    proc get_toplevel_next {prefix} {
+        set base [string trim $prefix .] ;# .myapp -> myapp  .myapp.somewindow  -> myapp.somewindow   . -> ""
+
+
+
+    }
+    proc exit {{toplevel ""}} {
+        variable waiting
+        variable result
+        variable default_result
+        set toplevels [get_toplevels]
+        if {[string length $toplevel]} {
+            set wposn [lsearch $toplevels $toplevel]
+            if {$wposn > 0} {
+                destroy $toplevel
+            }
+        } else {
+            #review
+            if {[package provide punk::repl::codethread] ne "" &&  [punk::repl::codethread::is_running]} {
+                puts stderr "punkapp::exit called without toplevel - showing console"
+                show_console
+                return 0
+            } else {
+                puts stderr "punkapp::exit called without toplevel - exiting"
+                if {$waiting ne "no"} {
+                    if {[info exists result(shell)]} {
+                        set temp [set result(shell)]
+                        unset result(shell)
+                        set waiting $temp
+                    } else {
+                        set waiting ""
+                    }
+                } else {
+                    ::exit
+                }
+            }
+        }
+
+        set controllable [get_user_controllable_toplevels]
+        if {![llength $controllable]} {
+            if {[package provide punk::repl::codethread] ne "" &&  [punk::repl::codethread::is_running]} {
+                show_console
+            } else {
+                if {$waiting ne "no"} {
+                    if {[info exists result(shell)]} {
+                        set temp [set result(shell)]
+                        unset result(shell)
+                        set waiting $temp
+                    } elseif {[info exists result($toplevel)]} {
+                        set temp [set result($toplevel)]
+                        unset result($toplevel)
+                        set waiting $temp
+                    } elseif {[info exists default_result]} {
+                        set temp $default_result
+                        unset default_result
+                        set waiting $temp
+                    } else {
+                        set waiting ""
+                    }
+                } else {
+                    ::exit
+                }
+            }
+        }
+    }
+    proc close_window {toplevel} {
+        wm withdraw $toplevel
+        if {![llength [get_user_controllable_toplevels]]} {
+            punkapp::exit $toplevel
+        }
+        destroy $toplevel
+    }
+    proc wait {args} {
+        variable waiting
+        variable default_result
+        if {[dict exists $args -defaultresult]} {
+            set default_result [dict get $args -defaultresult]
+        }
+        foreach t [punkapp::get_toplevels] {
+            if {[wm protocol $t WM_DELETE_WINDOW] eq ""} {
+                wm protocol $t WM_DELETE_WINDOW [list punkapp::close_window $t]
+            }
+        }
+        if {[package provide punk::repl::codethread] ne "" &&  [punk::repl::codethread::is_running]} {
+            puts stderr "repl eventloop seems to be running - punkapp::wait not required"
+        } else {
+            if {$waiting eq "no"} {
+                set waiting "waiting"
+                vwait ::punkapp::waiting
+                return $::punkapp::waiting
+            }
+        }
+    }
+
+    #A window can be 'visible' according to this - but underneath other windows etc
+    #REVIEW - change name?
+    proc get_visible_toplevels {{w .}} {
+        if {![llength [info commands winfo]]} {
+            return [list]
+        }
+        set list [get_toplevels $w]
+        set mapped [lmap v $list {expr {[winfo ismapped $v] ? $v : {}}}]
+        set mapped [concat {*}$mapped] ;#ignore {}
+        set visible [list]
+        foreach m $mapped {
+            if {[wm overrideredirect $m] == 0 } {
+                lappend visible $m
+            } else {
+                if {[winfo height $m] >1 && [winfo width $m] > 1} {
+                    #technically even a 1x1 is visible.. but in practice even a 10x10 is hardly likely to be noticeable when overrideredirect == 1
+                    #as a convention - 1x1 with no controls is used to make a window invisible so we'll treat anything larger as visible
+                    lappend visible $m
+                }
+            }
+        }
+        return $visible
+    }
+    proc get_user_controllable_toplevels {{w .}} {
+        set visible [get_visible_toplevels $w]
+        set controllable [list]
+        foreach v $visible {
+            if {[wm overrideredirect $v] == 0} {
+                lappend controllable $v
+            }
+        }
+        #only return visible windows with overrideredirect == 0 because there exists some user control.
+        #todo - review.. consider checking if position is outside screen areas? Technically controllable.. but not easily
+        return $controllable
+    }
+    proc hide_console {args} {
+        set opts [dict create -force 0]
+        if {([llength $args] % 2) != 0} {
+            error "hide_console expects pairs of arguments. e.g -force 1"
+        }
+        #set known_opts [dict keys $defaults]
+        foreach {k v} $args {
+            switch -- $k {
+                -force {
+                    dict set opts $k $v
+                }
+                default {
+                    error "Unrecognised options '$k'  known options: [dict keys $opts]"
+                }
+            }
+        }
+        set force [dict get $opts -force]
+
+        if {!$force} {
+            if {![llength [get_user_controllable_toplevels]]} {
+                puts stderr "Cannot hide console while no user-controllable windows available"
+                return 0
+            }
+        }
+        if {$::tcl_platform(platform) eq "windows"} {
+            #hide won't work for certain consoles cush as conemu,wezterm - and doesn't really make sense for tabbed windows anyway.
+            #It would be nice if we could tell the console window to hide just the relevant tab - or the whole window if only one tab present - but this is unlikely to be possible in any standard way.
+            #an ordinary cmd.exe or pwsh.exe or powershell.exe window can be hidden ok though.
+            #(but with wezterm - process is cmd.exe - but it has style popup and can't be hidden with a twapi::hide_window call)
+            package require twapi
+            set h [twapi::get_console_window]
+            set pid [twapi::get_window_process $h]
+            set pinfo [twapi::get_process_info $pid -name]
+            set pname [dict get $pinfo -name]
+            set wstyle [twapi::get_window_style $h]
+            #tclkitsh/tclsh?
+            if {($pname in [list cmd.exe pwsh.exe powershell.exe] || [string match punk*.exe $pname]) && "popup" ni $wstyle} {
+                twapi::hide_window $h
+                return 1
+            } else {
+                puts stderr "punkapp::hide_console unable to hide this type of console window"
+                return 0
+            }
+        } else {
+            #todo
+            puts stderr "punkapp::hide_console unimplemented on this platform (todo)"
+            return 0
+        }
+    }
+
+    proc show_console {} {
+        if {$::tcl_platform(platform) eq "windows"} {
+            package require twapi
+            if {![catch {set h [twapi::get_console_window]} errM]} {
+                twapi::show_window $h -activate -normal
+            } else {
+                #no console - assume launched from something like wish?
+                catch {console show}
+            }
+        } else {
+            #todo
+            puts stderr "punkapp::show_console unimplemented on this platform"
+        }
+    }
+
+}
+
+package provide punkapp [namespace eval punkapp {
+    variable version
+    set version 0.1.1
+}]

src/bootsupport/modules/punkcheck/cli-0.1.0.tm

@@ -259,7 +259,6 @@ namespace eval punkcheck::cli {
 }
 
 
-
 # ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++
 namespace eval punkcheck::cli::lib {
     namespace path ::punk::mix::util ;#askuser, do_in_path, foreach-file etc
@@ -307,7 +306,6 @@ namespace eval punkcheck::cli::lib {
         }
         return {}
     }
-    
 }
 
 

src/bootsupport/modules/shellfilter-0.2.2.tm

@@ -326,18 +326,34 @@ namespace eval shellfilter::chan {
         #method flush {ch} {
         #    return ""
         #}
+        #method flush {transform_handle} {
+        #    #puts stdout "<flush>"
+        #    #review - just clear o_encbuf and emit nothing?
+        #    #we wouldn't have a value there if it was convertable from the channel encoding?
+        #    if {[string length $o_encbuf]} {
+        #        #if we have data in the buffer that we haven't been able to convert to a string
+        #        #- then we probably have some kind of encoding mismatch. Is it safer to discard it than to emit garbage chars to the channel or var?
+        #        #REVIEW - log that we are discarding the buffer contents on flush?
+        #        puts stderr "WARNING: flush called on tee_to_var with non-empty buffer. This probably indicates an encoding mismatch between the channel encoding and the encoding expected by the transform. Discarding buffer contents: '$o_encbuf'"
+        #    }
+        #    set clear $o_encbuf
+        #    set o_encbuf ""
+        #    return $clear
+        #}
         method flush {transform_handle} {
-            #puts stdout "<flush>"
+            #puts stderr "<flush> $transform_handle o_encbuf: '$o_encbuf' datavars: $o_datavars"
-            #review - just clear o_encbuf and emit nothing?
+            set clear $o_encbuf
-            #we wouldn't have a value there if it was convertable from the channel encoding?
+            if {[catch {tcl::encoding::convertfrom $o_enc $clear} stringdata]} {
-            if {[string length $o_encbuf]} {
+                #if we can't convert the buffer contents to a string - does it make sense to emit the raw bytes?
-                #if we have data in the buffer that we haven't been able to convert to a string
+                # - probably not.
-                #- then we probably have some kind of encoding mismatch. Is it safer to discard it than to emit garbage chars to the channel or var?
+                #REVIEW?
-                #REVIEW - log that we are discarding the buffer contents on flush?
+                return ""
-                puts stderr "WARNING: flush called on tee_to_var with non-empty buffer. This probably indicates an encoding mismatch between the channel encoding and the encoding expected by the transform. Discarding buffer contents: '$o_encbuf'"
             }
             set o_encbuf ""
-            return ""
+            foreach v $o_datavars {
+                append $v $stringdata
+            }
+            return $stringdata
         }
         method write {ch bytes} {
             #test with set x [string repeat " \U1f6c8" 2043]
@@ -442,16 +458,29 @@ namespace eval shellfilter::chan {
         #    flush $o_localchan
         #    return $clear
         #}
+        #method flush {transform_handle} {
+        #    #we wouldn't have a value in o_encbuf if it was convertable from the channel encoding?
+        #    if {[string length $o_encbuf]} {
+        #        #if we have data in the buffer that we haven't been able to convert to a string
+        #        #- then we probably have some kind of encoding mismatch. Is it safer to discard it than to emit garbage chars to the channel or var?
+        #        #REVIEW - log that we are discarding the buffer contents on flush?
+        #        puts stderr "WARNING: flush called on tee_to_pipe with non-empty buffer. This probably indicates an encoding mismatch between the channel encoding and the encoding expected by the transform. Discarding buffer contents: '$o_encbuf'"
+        #    }
+        #    set clear $o_encbuf
+        #    set o_encbuf ""
+        #    return $clear
+        #}
         method flush {transform_handle} {
-            #we wouldn't have a value in o_encbuf if it was convertable from the channel encoding?
+            set clear $o_buffered$o_encbuf
-            if {[string length $o_encbuf]} {
+            if {[catch {tcl::encoding::convertfrom $o_enc $clear} stringdata]} {
-                #if we have data in the buffer that we haven't been able to convert to a string
+                #if we can't convert the buffer contents to a string - does it make sense to emit the raw bytes?
-                #- then we probably have some kind of encoding mismatch. Is it safer to discard it than to emit garbage chars to the channel or var?
+                # - probably not.
-                #REVIEW - log that we are discarding the buffer contents on flush?
+                #REVIEW?
-                puts stderr "WARNING: flush called on tee_to_pipe with non-empty buffer. This probably indicates an encoding mismatch between the channel encoding and the encoding expected by the transform. Discarding buffer contents: '$o_encbuf'"
+                return ""
             }
+            set o_buffered ""
             set o_encbuf ""
-            return ""
+            return $stringdata
         }
         method write {transform_handle bytes} {
             #set logdata [tcl::encoding::convertfrom $o_enc $bytes]
@@ -533,11 +562,24 @@ namespace eval shellfilter::chan {
             ::shellfilter::log::write $o_logsource $logdata
             return  $bytes
         }
+        #method flush {transform_handle} {
+        #    #return ""
+        #    set clear $o_encbuf
+        #    set o_encbuf ""
+        #    #review
+        #    return $clear
+        #}
         method flush {transform_handle} {
-            #return ""
+            set clear $o_buffered$o_encbuf
-            set clear $o_encbuf
+            if {[catch {tcl::encoding::convertfrom $o_enc $clear} stringdata]} {
+                #if we can't convert the buffer contents to a string - does it make sense to emit the raw bytes?
+                # - probably not.
+                #REVIEW?
+                return ""
+            }
+            set o_buffered ""
             set o_encbuf ""
-            return $o_encbuf
+            return $stringdata
         }
         method write {ch bytes} {
             #set logdata [tcl::encoding::convertfrom $o_enc $bytes]
@@ -613,9 +655,21 @@ namespace eval shellfilter::chan {
             my destroy
         }
         #clear?
+        #method flush {transform_handle} {
+        #    #we wouldn't have a value in o_encbuf if it was convertable from the channel encoding?
+        #    if {[string length $o_encbuf]} {
+        #        #if we have data in the buffer that we haven't been able to convert to a string
+        #        #- then we probably have some kind of encoding mismatch. Is it safer to discard it than to emit garbage chars to the log?
+        #        #REVIEW. - we are writing the raw bytes to the log here because we can't convert them to a string.
+        #        #This may be useful for debugging issues, but it may also result in garbage data in the log.
+        #        ::shellfilter::log::write $o_logsource $o_encbuf
+        #        set o_encbuf ""
+        #    }
+        #    return
+        #}
         method flush {transform_handle} {
-            #we wouldn't have a value in o_encbuf if it was convertable from the channel encoding?
+            set clear $o_buffered$o_encbuf
-            if {[string length $o_encbuf]} {
+            if {[catch {tcl::encoding::convertfrom $o_enc $clear} stringdata]} {
                 #if we have data in the buffer that we haven't been able to convert to a string
                 #- then we probably have some kind of encoding mismatch. Is it safer to discard it than to emit garbage chars to the log?
                 #REVIEW. - we are writing the raw bytes to the log here because we can't convert them to a string.
@@ -755,6 +809,110 @@ namespace eval shellfilter::chan {
         }
     }
 
+
+    #experimental
+    #applying this to stdout breaks console query/responses - why?
+    #- probably because we are splitting graphemes across writes and flushes and the console doesn't know how to handle that?
+    oo::class create unicode_normalize {
+        variable o_trecord
+        variable o_enc
+        variable o_encbuf
+        variable o_graphemebuf
+        variable o_mode
+        variable o_is_junction
+        constructor {tf} {
+            package require punk::ansi
+            set o_trecord $tf
+            set o_enc [::tcl::dict::get $tf -encoding]
+            set o_encbuf ""
+            set o_graphemebuf ""
+            set settingsdict [tcl::dict::get $tf -settings]
+            if {[dict exists $settingsdict -mode]} {
+                set o_mode [dict get $settingsdict -mode]
+                if {$o_mode ni {nfc nfd nfkc nfkd none}} {
+                    error "unicode_normalize transform - invalid mode '$o_mode' in settings"
+                }
+                if {$o_mode ne "none"} {
+                    #we get dll dependent load errors on windows sometimes - but still seems to work - REVIEW/FIX.
+                    catch {::tcl::unsupported::loadIcu}
+                }
+            } else {
+                #if no mode specified - default to 'none' which just does grapheme splitting without any unicode normalization
+                set o_mode "none"
+            }
+            if {[::tcl::dict::exists $tf -junction]} {
+                set o_is_junction [::tcl::dict::get $tf -junction]
+            } else {
+                set o_is_junction 0
+            }
+        }
+        method initialize {transform_handle mode} {
+            return [list initialize write flush finalize]
+        }
+        method finalize {transform_handle} {
+            my destroy
+        }
+        method flush {transform_handle} {
+            #flush seems to do nothing - why?
+            set clear $o_encbuf[unset o_encbuf]
+            if {[catch {tcl::encoding::convertfrom $o_enc $clear} stringdata]} {
+                #if we can't convert the buffer contents to a string - put it back and try again with more data later
+                #REVIEW?
+                set o_encbuf $clear
+                return ""
+            }
+            #review
+
+            set graphemes [punk::char::grapheme_split $o_graphemebuf$stringdata]
+            set outstring [join $graphemes ""]
+            #puts "outstring: '$outstring' graphemes: $graphemes"
+            if {$o_mode ne "none"} {
+                set outstring [::tcl::unsupported::icu::normalize -mode $o_mode $outstring]
+            }
+            set o_graphemebuf ""
+            return [tcl::encoding::convertto $o_enc $outstring]
+        }
+        method write {transform_handle bytes} {
+            set inputbytes $o_encbuf$bytes
+            set o_encbuf ""
+            set tail_offset 0
+
+            if {$inputbytes eq ""} {
+                #review - do we even get empty writes?
+                puts stderr "WARNING: write called on unicode_normalize with empty inputbytes. This may be a no-op, but it may also indicate an issue with the upstream transform or channel. Emitting no data for this write."
+                set stringdata ""
+            }
+
+            while {$tail_offset < [::tcl::string::length $inputbytes] && [catch {tcl::encoding::convertfrom $o_enc [::tcl::string::range $inputbytes 0 end-$tail_offset]} stringdata]} {
+                incr tail_offset
+            }
+            if {$tail_offset > 0} {
+                if {$tail_offset < [::tcl::string::length $inputbytes]} {
+                    #stringdata from catch statement must be a valid result
+                    set t [expr {$tail_offset - 1}]
+                    set o_encbuf [::tcl::string::range $inputbytes end-$t end]
+                } else {
+                    set stringdata ""
+                    set o_encbuf $inputbytes
+                    return ""
+                }
+            }
+            set graphemes [punk::char::grapheme_split $o_graphemebuf$stringdata]
+            set outstring [join [lrange $graphemes 0 end-1] ""]
+            set o_graphemebuf [lindex $graphemes end]
+
+            if {$o_mode ne "none"} {
+                set outstring [::tcl::unsupported::icu::normalize -mode $o_mode $outstring]
+            }
+
+            return [tcl::encoding::convertto $o_enc $outstring]
+        }
+        method meta_is_redirection {} {
+            return $o_is_junction
+        }
+    }
+
+
     #a test
     oo::class create reconvert {
         variable o_trecord
@@ -1114,7 +1272,6 @@ namespace eval shellfilter::chan {
         #    return $emit
         #}
         method flush {transform_handle} {
-            #return ""
             set clear $o_buffered$o_encbuf
             if {[catch {tcl::encoding::convertfrom $o_enc $clear} stringdata]} {
                 #if we can't convert the buffer contents to a string - does it make sense to emit the raw bytes?

src/bootsupport/modules/shellrun-0.1.2.tm

@@ -0,0 +1,897 @@
+# vim: set ft=tcl
+#
+#purpose: handle the run commands that call shellfilter::run
+#e.g run,runout,runerr,runx
+
+package require shellfilter
+package require punk::ansi
+
+#NOTE: the run,runout,runerr,runx commands only produce an error if the command didn't run.
+# - If it did run, but there was a non-zero exitcode it is up to the application to check that.
+#This is deliberate, but means 'catch' doesn't catch errors within the command itself -  the exitcode has to be checked.
+#The user can always use exec for different process error semantics (they don't get exitcode with exec)
+
+namespace eval shellrun {
+    variable PUNKARGS
+    variable runout
+    variable runerr
+
+    #do we need these?
+    #variable punkout
+    #variable punkerr
+
+    #some ugly coupling with punk/punk::config for now
+    #todo - something better
+    if {[info exists ::punk::config::configdata]} {
+        set conf_running    [punk::config::configure running]
+        set syslog_stdout   [dict get $conf_running syslog_stdout]
+        set syslog_stderr   [dict get $conf_running syslog_stderr]
+        set logfile_stdout  [dict get $conf_running logfile_stdout]
+        set logfile_stderr  [dict get $conf_running logfile_stderr]
+    } else {
+        lassign [list "" "" "" ""] syslog_stdout syslog_stderr logfile_stdout logfile_stderr
+    }
+    if {"punkshout" ni [shellfilter::stack::items]} {
+        set outdevice [shellfilter::stack::new punkshout -settings [list -tag "punkshout" -buffering none -raw 1 -syslog $syslog_stdout -file $logfile_stdout]]
+        set out [dict get $outdevice localchan]
+    } else {
+        set out [dict get [shellfilter::stack::item punkshout] device localchan]
+    }
+    if {"punksherr" ni [shellfilter::stack::items]} {
+        set errdevice [shellfilter::stack::new punksherr -settings [list -tag "punksherr" -buffering none -raw 1 -syslog $syslog_stderr -file $logfile_stderr]]
+        set err [dict get $errdevice localchan]
+    } else {
+        set err  [dict get [shellfilter::stack::item punksherr] device localchan]
+    }
+
+    namespace import ::punk::ansi::a+
+    namespace import ::punk::ansi::a
+
+
+
+
+    #repltelemetry - additional/alternative display info used in a repl context i.e info directed towards the screen
+    #todo - package up in repltelemetry module and rewrite proc based on whether the module was found/loaded.
+    #somewhat strong coupling to punk - but let's try to behave decently if it's not loaded
+    #The last_run_display is actually intended for the repl - but is resident in the punk namespace with a view to the possibility of a different repl being in use.
+    proc set_last_run_display {chunklist} {
+        #chunklist as understood by the
+        if {![info exists ::punk::repltelemetry_emmitters]} {
+            namespace eval ::punk {
+                variable repltelemetry_emmitters
+                set repltelemetry_emmitters "shellrun"
+            }
+        } else {
+            if {"shellrun" ni $::punk::repltelemetry_emmitters} {
+                lappend punk::repltelemetry_emmitters "shellrun"
+            }
+        }
+
+        #most basic of validity tests here.. just that it is a list (can be empty). We don't want to duplicate or over-constrain the way repls/shells/terminals interpet the info
+        if {[catch {llength $chunklist} errMsg]} {
+            error  "set_last_run_display expects a list. Value supplied doesn't appear to be a well formed tcl list. '$errMsg'"
+        }
+        #todo -
+        tsv::lappend repl runchunks-[tsv::get repl runid] {*}$chunklist
+    }
+
+
+
+    #maintenance: similar used in punk::ns & punk::winrun
+    #todo - take runopts + aliases as args
+    #longopts must be passed as a single item ie  --timeout=100  not --timeout 100
+    proc get_run_opts {arglist} {
+        if {[catch {
+            set callerinfo [info level -1]
+        } errM]} {
+            set caller ""
+        } else {
+            set caller [lindex $callerinfo 0]
+        }
+
+        #we provide -nonewline even for 'run' even though run doesn't deliver stderr or stdout to the tcl return value
+        #This is for compatibility with other runX commands, and the difference is also visible when calling from repl.
+        set known_runopts [list "-echo" "-e" "-nonewline" "-n" "-tcl" "-debug"]
+        set known_longopts [list "--timeout"]
+        set known_longopts_msg ""
+        foreach lng $known_longopts {
+            append known_longopts_msg "${lng}=val "
+        }
+        set aliases [list "-e" "-echo" "-echo" "-echo" "-n" "-nonewline" "-nonewline" "-nonewline" "-tcl" "-tcl" "-debug" "-debug"] ;#include map to self
+        set runopts [list]
+        set runoptslong [list]
+        set cmdargs [list]
+
+        set idx_first_cmdarg [lsearch -not $arglist "-*"]
+
+        set allopts [lrange $arglist 0 $idx_first_cmdarg-1]
+        set cmdargs [lrange $arglist $idx_first_cmdarg end]
+        foreach o $allopts {
+            if {[string match --* $o]} {
+                lassign [split $o =] flagpart valpart
+                if {$valpart eq ""} {
+                    error "$caller: longopt $o seems to be missing a value - must be of form --option=value"
+                }
+                if {$flagpart ni $known_longopts} {
+                    error "$caller: Unknown runoption $o - known options $known_runopts $known_longopts_msg"
+                }
+                lappend runoptslong $flagpart $valpart
+            } else {
+                if {$o ni $known_runopts} {
+                    error "$caller: Unknown runoption $o - known options $known_runopts $known_longopts_msg"
+                }
+                lappend runopts [dict get $aliases $o]
+            }
+        }
+        return [list runopts $runopts runoptslong $runoptslong cmdargs $cmdargs]
+    }
+
+
+    #todo - investigate cause of punk86 run hanging sometimes. An 'after 500' before exit in the called script fixes the issue. punk87 doesn't seem to be affected.
+    lappend PUNKARGS [list {
+        @id     -id ::shellrun::run
+        @leaders -min 0 -max 0
+        @opts
+        -nonewline  -type none
+        -tcl        -type none -default 0
+        -debug      -type none -default 0
+        --timeout=  -type integer
+        @values -min 1 -max -1
+        cmdname -type string
+        cmdarg -type any -multiple 1 -optional 1
+    }]
+    proc run {args} {
+        #set_last_run_display [list]
+
+        #set splitargs [get_run_opts $args]
+        #set runopts [dict get $splitargs runopts]
+        #set runoptslong [dict get $splitargs runoptslong]
+        #set cmdargs [dict get $splitargs cmdargs]
+        set argd [punk::args::parse $args withid ::shellrun::run]
+        lassign [dict values $argd] leaders opts values received
+
+        if {[dict exists $received "-nonewline"]} {
+            set nonewline 1
+        } else {
+            set nonewline 0
+        }
+        #review nonewline does nothing here..
+
+        set idlist_stderr [list]
+        #we leave stdout without imposed ansi colouring - because the source may be colourised and because ansi-wrapping a stream at whatever boundaries it comes in at isn't a really nice thing to do.
+        #stderr might have source colouring - but it usually doesn't seem to, and the visual distiction of red stderr can be very handy for the run command.
+        #A further enhancement could be to detect well-known options such as --color and/or use a configuration for specific commands that have useful colourised stderr,
+        #but having an option to configure stderr to red is a compromise.
+        #Note that the other run commands, runout,runerr, runx don't emit in real-time - so for those commands there may be options to detect and/or post-process stdout and stderr.
+        #TODO - fix. This has no effect if/when the repl adds an ansiwrap transform
+        # what we probably want to do is 'aside' that transform for runxxx commands only.
+        #lappend idlist_stderr [shellfilter::stack::add stderr ansiwrap -settings {-colour {red bold}}]
+
+        set callopts [dict create]
+        if {[dict exists $received "-tcl"]} {
+            dict set callopts -tclscript 1 
+        }
+        if {[dict exists $received "-debug"]} {
+            dict set callopts -debug 1
+        }
+        if {[dict exists $received --timeout]} {
+            dict set callopts -timeout [dict get $opts --timeout] ;#convert to single dash
+        }
+        set cmdname     [dict get $values cmdname]
+        if {[dict exists $received cmdarg]} {
+            set cmdarglist  [dict get $values cmdarg]
+        } else {
+            set cmdarglist {}
+        }
+        set cmdargs [concat $cmdname $cmdarglist]
+        #---------------------------------------------------------------------------------------------
+        set exitinfo [shellfilter::run $cmdargs {*}$callopts -teehandle punksh -inbuffering none -outbuffering none ]
+        #---------------------------------------------------------------------------------------------
+        foreach id $idlist_stderr {
+            shellfilter::stack::remove stderr $id
+        }
+        #puts stderr "shellrun::run exitinfo: $exitinfo"
+
+        flush stderr
+        flush stdout
+
+        if {[dict exists $exitinfo error]} {
+            error "[dict get $exitinfo error]\n$exitinfo"
+        }
+
+        return $exitinfo
+    }
+
+    lappend PUNKARGS [list {
+        @id     -id ::shellrun::runconsole
+        @leaders -min 0 -max 0
+        @opts
+        @values -min 1 -max -1
+        cmdname -type string
+        cmdarg -type any -multiple 1 -optional 1
+    }]
+    #run in the way tcl unknown does - but without regard to auto_noexec
+    proc runconsole {args} {
+        set argd [punk::args::parse $args withid ::shellrun::runconsole]
+        lassign [dict values $argd] leaders opts values received
+        set cmdname     [dict get $values cmdname]
+        if {[dict exists $received cmdarg]} {
+            set arglist     [dict get $values cmdarg]
+        } else {
+            set arglist {}
+        }
+
+        set resolved_cmdname [auto_execok $cmdname]
+        if {$resolved_cmdname eq ""} {
+            error "Cannot find path for executable '$cmdname'"
+        }
+        set repl_runid [punk::get_repl_runid]
+        #set ::punk::last_run_display [list]
+
+        set redir ">&@stdout <@stdin"
+        uplevel 1 [list ::catch [concat exec $redir $resolved_cmdname $arglist] ::tcl::UnknownResult ::tcl::UnknownOptions]
+        #we can't detect stdout/stderr output from the exec
+        #for now emit an extra \n on stderr
+        #todo - there is probably no way around this but to somehow exec in the context of a completely separate console
+        #This is probably a tricky problem - especially to do cross-platform
+        #
+        # - use [dict get $::tcl::UnknownOptions -code]  (0|1)  exit
+        if {[dict get $::tcl::UnknownOptions -code] == 0} {
+            set c green
+            set m "ok"
+        } else {
+            set c yellow
+            set m "errorCode $::errorCode"
+        }
+        set chunklist [list]
+        lappend chunklist [list "info" "[a $c]$m[a] " ]
+        if {$repl_runid != 0} {
+            tsv::lappend repl runchunks-$repl_runid {*}$chunklist
+        }
+
+        dict incr ::tcl::UnknownOptions -level
+        return -options $::tcl::UnknownOptions $::tcl::UnknownResult
+    }
+    lappend PUNKARGS [list {
+        @id     -id ::shellrun::runout
+        @leaders -min 0 -max 0
+        @opts
+        -echo       -type none
+        -nonewline  -type none
+        -tcl        -type none -default 0
+        -debug      -type none -default 0
+        --timeout=  -type integer
+        @values -min 1 -max -1
+        cmdname -type string
+        cmdarg -type any -multiple 1 -optional 1
+    }]
+    proc runout {args} {
+        set argd [punk::args::parse $args withid ::shellrun::runout]
+        lassign [dict values $argd] leaders opts values received
+
+        if {[dict exists $received "-nonewline"]} {
+            set nonewline 1
+        } else {
+            set nonewline 0
+        }
+
+        #set_last_run_display [list]
+        variable runout
+        variable runerr
+        set runout ""
+        set runerr ""
+        set RST [a]
+
+        #set splitargs [get_run_opts $args]
+        #set runopts [dict get $splitargs runopts]
+        #set cmdargs [dict get $splitargs cmdargs]
+
+
+        #puts stdout "RUNOUT cmdargs: $cmdargs"
+
+        #todo   add -data boolean and -data lastwrite to -settings with default being -data all
+        # because sometimes we're only interested in last char (e.g to detect something was output)
+
+        #set outvar_stackid [shellfilter::stack::add commandout tee_to_var  -action float -settings {-varname ::runout}]
+        #
+        #when not echoing - use float-locked so that the repl's stack is bypassed
+        if {[dict exists $received "-echo"]} {
+            set stdout_stackid [shellfilter::stack::add stdout tee_to_var -action float-locked -settings {-varname ::shellrun::runout}]
+            set stderr_stackid [shellfilter::stack::add stderr tee_to_var -action float-locked -settings {-varname ::shellrun::runerr}]
+            #set stderr_stackid [shellfilter::stack::add stderr tee_to_var -action sink-locked -settings {-varname ::shellrun::runerr}]
+        } else {
+            set stdout_stackid [shellfilter::stack::add stdout var -action float-locked -settings {-varname ::shellrun::runout}]
+            set stderr_stackid [shellfilter::stack::add stderr var -action float-locked -settings {-varname ::shellrun::runerr}]
+        }
+
+        set callopts [dict create]
+        if {[dict exists $received "-tcl"]} {
+            dict set callopts -tclscript 1 
+        }
+        if {[dict exists $received "-debug"]} {
+            dict set callopts -debug 1
+        }
+        if {[dict exists $received --timeout]} {
+            dict set callopts -timeout [dict get $opts --timeout] ;#convert to single dash
+        }
+        set cmdname     [dict get $values cmdname]
+        if {[dict exists $received cmdarg]} {
+            set cmdarglist  [dict get $values cmdarg]
+        } else {
+            set cmdarglist {}
+        }
+        set cmdargs [concat $cmdname $cmdarglist]
+
+        #shellfilter::run [lrange $args 1 end] -teehandle punksh -outchan stdout -inbuffering none -outbuffering none -stdinhandler ::repl::repl_handler
+        set exitinfo [shellfilter::run $cmdargs {*}$callopts -teehandle punksh -inbuffering none -outbuffering none ]
+
+        flush stderr
+        flush stdout
+
+        shellfilter::stack::remove stdout $stdout_stackid
+        shellfilter::stack::remove stderr $stderr_stackid
+
+        #shellfilter::stack::remove commandout $outvar_stackid
+
+        if {[dict exists $exitinfo error]} {
+            if {[dict exists $received "-tcl"]} {
+
+            } else {
+                #we must raise an error.
+                #todo - check errorInfo makes sense.. return -code? tailcall?
+                #
+                set msg ""
+                append msg [dict get $exitinfo error]
+                append msg "\n(add -tcl option to run as a tcl command/script instead of an external command)"
+                error $msg
+            }
+        }
+
+        set chunklist [list]
+
+        #exitcode not part of return value for runout - colourcode appropriately
+        set n $RST
+        set c ""
+
+        if {[dict exists $exitinfo exitcode]} {
+            set code [dict get $exitinfo exitcode]
+            if {$code == 0} {
+                set c [a+ green]
+            } else {
+                set c [a+ white bold]
+            }
+            lappend chunklist [list "info" "$c$exitinfo$n"]
+        } elseif {[dict exists $exitinfo error]} {
+            # -tcl (with error)
+            set c [a+ yellow bold]
+            lappend chunklist [list "info" "${c}error [dict get $exitinfo error]$n"]
+            lappend chunklist [list "info" "errorCode [dict get $exitinfo errorCode]"]
+            #lappend chunklist [list "info" "errorInfo [list [dict get $exitinfo errorInfo]]"]
+            lappend chunklist [list "info" errorInfo]
+            lappend chunklist [list "stderr" [dict get $exitinfo errorInfo]]
+        } else {
+            # -tcl (without error)
+            set c [a+ Green white bold]
+            #lappend chunklist [list "info" "$c$exitinfo$n"]
+            lappend chunklist [list "info" [punk::ansi::ansiwrap_raw $c \x1b\[m "" $exitinfo]]
+        }
+
+
+        set chunk "[a+ red bold]stderr$RST"
+        lappend chunklist [list "info" $chunk]
+
+        set chunk ""
+        if {[string length $::shellrun::runerr]} {
+            if {$nonewline} {
+                set e [string trimright $::shellrun::runerr \r\n]
+            } else {
+                set e $::shellrun::runerr
+            }
+            #append chunk "[a+ red normal]$e$RST\n"
+            append chunk "[a+ red normal]$e$RST"
+        }
+        lappend chunklist [list stderr $chunk]
+
+
+
+
+        lappend chunklist [list "info" "[a+ white bold]stdout$RST"]
+        set chunk ""
+        if {[string length $::shellrun::runout]} {
+            if {$nonewline} {
+                set o [string trimright $::shellrun::runout \r\n]
+            } else {
+                set o $::shellrun::runout
+            }
+            append chunk "$o"
+        }
+        lappend chunklist [list result $chunk]
+
+
+        #set_last_run_display $chunklist
+        tsv::lappend repl runchunks-[tsv::get repl runid] {*}$chunklist
+
+        if {$nonewline} {
+            return [string trimright $::shellrun::runout \r\n]
+        } else {
+            return $::shellrun::runout
+        }
+    }
+
+    lappend PUNKARGS [list {
+        @id     -id ::shellrun::runerr
+        @leaders -min 0 -max 0
+        @opts
+        -echo       -type none
+        -nonewline  -type none
+        -tcl        -type none -default 0
+        -debug      -type none -default 0
+        --timeout=  -type integer
+        @values -min 1 -max -1
+        cmdname -type string
+        cmdarg -type any -multiple 1 -optional 1
+    }]
+    proc runerr {args} {
+        set argd [punk::args::parse $args withid ::shellrun::runerr]
+        lassign [dict values $argd] leaders opts values received
+
+        if {[dict exists $received "-nonewline"]} {
+            set nonewline 1
+        } else {
+            set nonewline 0
+        }
+
+        #set_last_run_display [list]
+        variable runout
+        variable runerr
+        set runout ""
+        set runerr ""
+
+        #set splitargs [get_run_opts $args]
+        #set runopts [dict get $splitargs runopts]
+        #set cmdargs [dict get $splitargs cmdargs]
+
+        set callopts [dict create]
+        if {[dict exists $received "-tcl"]} {
+            dict set callopts -tclscript 1 
+        }
+        if {[dict exists $received "-debug"]} {
+            dict set callopts -debug 1
+        }
+        if {[dict exists $received --timeout]} {
+            dict set callopts -timeout [dict get $opts --timeout] ;#convert to single dash
+        }
+        set cmdname     [dict get $values cmdname]
+        if {[dict exists $received cmdarg]} {
+            set cmdarglist  [dict get $values cmdarg]
+        } else {
+            set cmdarglist {}
+        }
+        set cmdargs [concat $cmdname $cmdarglist]
+
+        if {[dict exists $received "-tcl"]} {
+            append callopts " -tclscript 1"
+        }
+        if {[dict exists $received "-echo"]} {
+            set stderr_stackid [shellfilter::stack::add stderr tee_to_var -action float-locked -settings {-varname ::shellrun::runerr}]
+            set stdout_stackid [shellfilter::stack::add stdout tee_to_var -action float-locked -settings {-varname ::shellrun::runout}]
+        } else {
+            set stderr_stackid [shellfilter::stack::add stderr var -action float-locked -settings {-varname ::shellrun::runerr}]
+            set stdout_stackid [shellfilter::stack::add stdout var -action float-locked -settings {-varname ::shellrun::runout}]
+        }
+
+
+        set exitinfo [shellfilter::run $cmdargs {*}$callopts -teehandle punksh -inbuffering none -outbuffering none -stdinhandler ::repl::repl_handler]
+        shellfilter::stack::remove stderr $stderr_stackid
+        shellfilter::stack::remove stdout $stdout_stackid
+
+
+        flush stderr
+        flush stdout
+
+        #we raise an error because an error during calling is different to collecting stderr from a command, and the caller should be able to wrap in a catch
+        # to determine something other than just a nonzero exit code or output  on stderr.
+        if {[dict exists $exitinfo error]} {
+            if {[dict exists $received "-tcl"]} {
+
+            } else {
+                #todo - check errorInfo makes sense.. return -code? tailcall?
+                error [dict get $exitinfo error]
+            }
+        }
+
+        set chunklist [list]
+
+        set n [a]
+        set c ""
+        if {[dict exists $exitinfo exitcode]} {
+            set code [dict get $exitinfo exitcode]
+            if {$code == 0} {
+                set c [a+ green]
+            } else {
+                set c [a+ white bold]
+            }
+            lappend chunklist [list "info" "$c$exitinfo$n"]
+        } elseif {[dict exists $exitinfo error]} {
+            # -tcl (with error)
+            set c [a+ yellow bold]
+            lappend chunklist [list "info" "error [dict get $exitinfo error]"]
+            lappend chunklist [list "info" "errorCode [dict get $exitinfo errorCode]"]
+            lappend chunklist [list "info" "errorInfo [list [dict get $exitinfo errorInfo]]"]
+        } else {
+            # -tcl (without error)
+            set c [a+ Green white bold]
+            #lappend chunklist [list "info" "$c$exitinfo$n"]
+            lappend chunklist [list "info" [punk::ansi::ansiwrap_raw $c "\x1b\[m" "" $exitinfo]]
+        }
+
+
+        lappend chunklist [list "info" "[a+ white bold]stdout[a]"]
+        set chunk ""
+        if {[string length $::shellrun::runout]} {
+            if {$nonewline} {
+                set o [string trimright $::shellrun::runout \r\n]
+            } else {
+                set o $::shellrun::runout
+            }
+            append chunk "[a+ white normal]$o[a]\n"  ;#this newline is the display output separator - always there whether data has trailing newline or not.
+        }
+        lappend chunklist [list stdout $chunk]
+
+
+        #set c_stderr [punk::config]
+        set chunk "[a+ red bold]stderr[a]"
+        lappend chunklist [list "info" $chunk]
+
+        set chunk ""
+        if {[string length $::shellrun::runerr]} {
+            if {$nonewline} {
+                set e [string trimright $::shellrun::runerr \r\n]
+            } else {
+                set e $::shellrun::runerr
+            }
+            append chunk "$e"
+        }
+        lappend chunklist [list resulterr $chunk]
+
+
+        #set_last_run_display $chunklist
+        tsv::lappend repl runchunks-[tsv::get repl runid] {*}$chunklist
+
+        if {$nonewline} {
+            return [string trimright $::shellrun::runerr \r\n]
+        }
+        return $::shellrun::runerr
+    }
+
+
+    proc runx {args} {
+        #set_last_run_display [list]
+        variable runout
+        variable runerr
+        set runout ""
+        set runerr ""
+
+        set splitargs [get_run_opts $args]
+        set runopts [dict get $splitargs runopts]
+        set cmdargs [dict get $splitargs cmdargs]
+
+        if {"-nonewline" in $runopts} {
+            set nonewline 1
+        } else {
+            set nonewline 0
+        }
+
+        #shellfilter::stack::remove stdout $::repl::id_outstack
+
+        if {"-echo" in $runopts} {
+            #float to ensure repl transform doesn't interfere with the output data
+            set stderr_stackid [shellfilter::stack::add stderr tee_to_var -action float -settings {-varname ::shellrun::runerr}]
+            set stdout_stackid [shellfilter::stack::add stdout tee_to_var -action float -settings {-varname ::shellrun::runout}]
+        } else {
+            #set stderr_stackid [shellfilter::stack::add stderr var -action sink-locked -settings {-varname ::shellrun::runerr}]
+            #set stdout_stackid [shellfilter::stack::add stdout var -action sink-locked -settings {-varname ::shellrun::runout}]
+
+            #float above the repl's tee_to_var to deliberately block it.
+            #a var transform is naturally a junction point because there is no flow-through..
+            # - but mark it with -junction 1 just to be explicit
+            set stderr_stackid [shellfilter::stack::add stderr var -action float-locked -junction 1 -settings {-varname ::shellrun::runerr}]
+            set stdout_stackid [shellfilter::stack::add stdout var -action float-locked -junction 1 -settings {-varname ::shellrun::runout}]
+        }
+
+        set callopts ""
+        if {"-tcl" in $runopts} {
+            append callopts " -tclscript 1"
+        }
+        #set exitinfo [shellfilter::run $cmdargs -teehandle punksh -inbuffering none -outbuffering none -stdinhandler ::repl::repl_handler]
+        set exitinfo [shellfilter::run $cmdargs {*}$callopts -teehandle punksh -inbuffering none -outbuffering none]
+
+        shellfilter::stack::remove stdout $stdout_stackid
+        shellfilter::stack::remove stderr $stderr_stackid
+
+
+        flush stderr
+        flush stdout
+
+        if {[dict exists $exitinfo error]} {
+            if {"-tcl" in $runopts} {
+
+            } else {
+                #todo - check errorInfo makes sense.. return -code? tailcall?
+                error [dict get $exitinfo error]
+            }
+        }
+
+        #set x [shellfilter::stack::add stdout var -action sink-locked -settings {-varname ::repl::runxoutput}]
+
+        set chunk ""
+        if {[string length $::shellrun::runout]} {
+            if {$nonewline} {
+                set o [string trimright $::shellrun::runout \r\n]
+            } else {
+                set o $::shellrun::runout
+            }
+            set chunk $o
+        }
+        set chunklist [list]
+        lappend chunklist [list "info" " "]
+        lappend chunklist [list "result" stdout] ;#key 'stdout' forms part of the resulting dictionary output
+        lappend chunklist [list "info" "[a+ white bold]stdout[a]"]
+        lappend chunklist [list result $chunk]   ;#value corresponding to 'stdout' key in resulting dict
+
+
+        lappend chunklist [list "info" " "]
+        set chunk "[a+ red bold]stderr[a]"
+        lappend chunklist [list "result" $chunk]
+        lappend chunklist [list "info" stderr]
+
+        set chunk ""
+        if {[string length $::shellrun::runerr]} {
+            if {$nonewline} {
+                set e [string trimright $::shellrun::runerr \r\n]
+            } else {
+                set e $::shellrun::runerr
+            }
+            set chunk $e
+        }
+        #stderr is part of the result
+        lappend chunklist [list "resulterr" $chunk]
+
+
+
+        set n [a]
+        set c ""
+        if {[dict exists $exitinfo exitcode]} {
+            set code [dict get $exitinfo exitcode]
+            if {$code == 0} {
+                set c [a+ green]
+            } else {
+                set c [a+ yellow bold]
+            }
+            lappend chunklist [list "info" " "]
+            lappend chunklist [list "result" exitcode]
+            lappend chunklist [list "info" "exitcode $code"]
+            lappend chunklist [list "result" "$c$code$n"]
+            set exitdict [list exitcode $code]
+        } elseif {[dict exists $exitinfo result]} {
+            # presumably from a -tcl call
+            set val [dict get $exitinfo result]
+            lappend chunklist [list "info" " "]
+            lappend chunklist [list "result" result]
+            lappend chunklist [list "info"   result]
+            lappend chunklist [list "result" $val]
+            set exitdict [list result $val]
+        } elseif {[dict exists $exitinfo error]} {
+            # -tcl call with error
+            #set exitdict [dict create]
+            lappend chunklist [list "info" " "]
+            lappend chunklist [list "result" error]
+            lappend chunklist [list "info"   error]
+            lappend chunklist [list "result" [dict get $exitinfo error]]
+
+            lappend chunklist [list "info" " "]
+            lappend chunklist [list "result" errorCode]
+            lappend chunklist [list "info"   errorCode]
+            lappend chunklist [list "result" [dict get $exitinfo errorCode]]
+
+            lappend chunklist [list "info" " "]
+            lappend chunklist [list "result" errorInfo]
+            lappend chunklist [list "info"   errorInfo]
+            lappend chunklist [list "result" [dict get $exitinfo errorInfo]]
+
+            set exitdict $exitinfo
+        } else {
+            #review - if no exitcode or result. then what is it?
+            lappend chunklist [list "info" exitinfo]
+            set c [a+ yellow bold]
+            lappend chunklist [list result "$c$exitinfo$n"]
+            set exitdict [list exitinfo $exitinfo]
+        }
+
+        #set_last_run_display $chunklist
+        tsv::lappend repl runchunks-[tsv::get repl runid] {*}$chunklist
+
+        #set ::repl::result_print 0
+        #return [lindex [list [list stdout $::runout stderr $::runerr {*}$exitinfo] [shellfilter::stack::remove stdout $x][puts -nonewline stdout $pretty][set ::repl::output ""]] 0]
+
+
+        if {$nonewline} {
+            return [list {*}$exitdict stdout [string trimright $::shellrun::runout \r\n] stderr [string trimright $::shellrun::runerr \r\n]]
+        }
+        #always return exitinfo $code at beginning of dict (so that punk unknown can interpret the exit code as a unix-style bool if double evaluated)
+        return [list {*}$exitdict stdout $::shellrun::runout stderr $::shellrun::runerr]
+    }
+
+    #an experiment
+    #
+    #run as raw string instead of tcl-list - no variable subst  etc
+    #
+    #dummy repl_runraw that repl will intercept
+    proc repl_runraw {args} {
+        error "runraw: only available in repl as direct call - not from script"
+    }
+    #we can only call runraw with a single (presumably braced) string if we want to use it from both repl and tcl scripts  (why? todo with unbalanced quotes/braces?)
+    proc runraw {commandline} {
+        #runraw fails as intended - because we can't bypass exec/open interference quoting :/
+        #set_last_run_display [list]
+        variable runout
+        variable runerr
+        set runout ""
+        set runerr ""
+
+        #return [shellfilter::run [lrange $args 1 end] -teehandle punksh -inbuffering none -outbuffering none -stdinhandler ::repl::repl_handler]
+        puts stdout ">>runraw got: $commandline"
+
+        #run always echoes anyway.. as we aren't diverting stdout/stderr off for capturing
+        #for consistency with other runxxx commands - we'll just consume it. (review)
+
+        set reallyraw 1
+        if {$reallyraw} {
+            set wordparts [regexp -inline -all {\S+} $commandline]
+            set runwords $wordparts
+        } else {
+            #shell style args parsing not suitable for windows where we can't assume matched quotes etc.
+            package require string::token::shell
+            set parts [string token shell -indices -- $commandline]
+            puts stdout ">>shellparts: $parts"
+            set runwords [list]
+            foreach p $parts {
+                set ptype [lindex $p 0]
+                set pval [lindex $p 3]
+                if {$ptype eq "PLAIN"} {
+                    lappend runwords [lindex $p 3]
+                } elseif {$ptype eq "D:QUOTED"} {
+                    set v {"}
+                    append v $pval
+                    append v {"}
+                    lappend runwords $v
+                } elseif {$ptype eq "S:QUOTED"} {
+                    set v {'}
+                    append v $pval
+                    append v {'}
+                    lappend runwords $v
+                }
+            }
+        }
+
+        puts stdout ">>runraw runwords: $runwords"
+        set runwords [lrange $runwords 1 end]
+
+        puts stdout ">>runraw runwords: $runwords"
+        #set args [lrange $args 1 end]
+        #set runwords [lrange $wordparts 1 end]
+
+        set known_runopts [list "-echo" "-e" "-terminal" "-t"]
+        set aliases [list "-e" "-echo" "-echo" "-echo" "-t" "-terminal" "-terminal" "-terminal"] ;#include map to self
+        set runopts [list]
+        set cmdwords [list]
+        set idx_first_cmdarg [lsearch -not $runwords "-*"]
+        set runopts [lrange $runwords 0 $idx_first_cmdarg-1]
+        set cmdwords [lrange $runwords $idx_first_cmdarg end]
+
+        foreach o $runopts {
+            if {$o ni $known_runopts} {
+                error "runraw: Unknown runoption $o"
+            }
+        }
+        set runopts [lmap o $runopts {dict get $aliases $o}]
+
+        set cmd_as_string [join $cmdwords " "]
+        puts stdout ">>cmd_as_string: $cmd_as_string"
+
+        if {"-terminal" in $runopts} {
+            #fake terminal using 'script' command.
+            #not ideal: smushes stdout & stderr together amongst other problems
+            set tcmd [shellfilter::get_scriptrun_from_cmdlist_dquote_if_not $cmdwords]
+            puts stdout ">>tcmd: $tcmd"
+            set exitinfo [shellfilter::run $tcmd -teehandle punksh -inbuffering line -outbuffering none ]
+            set exitinfo "exitcode not-implemented"
+        } else {
+            set exitinfo [shellfilter::run $cmdwords -teehandle punksh -inbuffering line -outbuffering none ]
+        }
+
+        if {[dict exists $exitinfo error]} {
+            #todo - check errorInfo makes sense.. return -code? tailcall?
+            error [dict get $exitinfo error]
+        }
+        set code [dict get $exitinfo exitcode]
+        if {$code == 0} {
+            set c [a+ green]
+        } else {
+            set c [a+ white bold]
+        }
+        puts stderr $c
+        return $exitinfo
+    }
+
+    proc sh_run {args} {
+        set splitargs [get_run_opts $args]
+        set runopts [dict get $splitargs runopts]
+        set cmdargs [dict get $splitargs cmdargs]
+        #e.g sh -c "ls -l *"
+        #we pass cmdargs to sh -c as a list, not individually
+        tailcall shellrun::run {*}$runopts sh -c $cmdargs
+    }
+    proc sh_runout {args} {
+        set splitargs [get_run_opts $args]
+        set runopts [dict get $splitargs runopts]
+        set cmdargs [dict get $splitargs cmdargs]
+        tailcall shellrun::runout {*}$runopts sh -c $cmdargs
+    }
+    proc sh_runerr {args} {
+        set splitargs [get_run_opts $args]
+        set runopts [dict get $splitargs runopts]
+        set cmdargs [dict get $splitargs cmdargs]
+        tailcall shellrun::runerr {*}$runopts sh -c $cmdargs
+    }
+    proc sh_runx {args} {
+        set splitargs [get_run_opts $args]
+        set runopts [dict get $splitargs runopts]
+        set cmdargs [dict get $splitargs cmdargs]
+        tailcall shellrun::runx {*}$runopts sh -c $cmdargs
+    }
+}
+
+namespace eval shellrun {
+    interp alias {} run     {} shellrun::run
+    interp alias {} sh_run  {} shellrun::sh_run
+    interp alias {} runout  {} shellrun::runout
+    interp alias {} sh_runout  {} shellrun::sh_runout
+    interp alias {} runerr  {} shellrun::runerr
+    interp alias {} sh_runerr  {} shellrun::sh_runerr
+    interp alias {} runx    {} shellrun::runx
+    interp alias {} sh_runx  {} shellrun::sh_runx
+
+    interp alias {} runc    {} shellrun::runconsole
+    interp alias {} runraw {} shellrun::runraw
+
+
+    #the shortened versions deliberately don't get pretty output from the repl
+    interp alias {} r       {} shellrun::run
+    interp alias {} ro      {} shellrun::runout
+    interp alias {} re      {} shellrun::runerr
+    interp alias {} rx      {} shellrun::runx
+
+
+}
+
+namespace eval shellrun {
+    proc test_cffi {} {
+        package require test_cffi
+        cffi::Wrapper create ::shellrun::kernel32 [file join $env(windir) system32 Kernel32.dll]
+        ::shellrun::kernel32 stdcall CreateProcessA
+        #todo - stuff.
+        return ::shellrun::kernel32
+    }
+
+}
+namespace eval ::punk::args::register {
+    #use fully qualified so 8.6 doesn't find existing var in global namespace
+    lappend ::punk::args::register::NAMESPACES ::shellrun
+}
+
+
+package provide shellrun [namespace eval shellrun {
+    variable version
+    set version 0.1.2
+}]

src/bootsupport/modules/zzzload-0.1.0.tm

@@ -20,6 +20,7 @@
 
 package require Thread
 
+#EXPERIMENTAL.
 
 # ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++
 namespace eval zzzload {
@@ -63,6 +64,8 @@ namespace eval zzzload {
         }
         if {$loader_tid eq ""} {
             set loader_tid [thread::create -joinable -preserved]
+            #todo - set tcl::tm::list and ::auto_path in the loader thread to match the main thread.
+            #(startup process may have modified these paths)
         }
         if {![tsv::exists zzzload_pkg $pkgname]} {
             #puts stderr "zzzload pkg_require $pkgname"
@@ -85,7 +88,7 @@ namespace eval zzzload {
         }
     }
     proc pkg_wait {pkgname} {
-        if {[set ver [package provide twapi]] ne ""} {
+        if {[set ver [package provide $pkgname]] ne ""} {
             return $ver
         }
 
@@ -116,18 +119,6 @@ namespace eval zzzload {
 }
 
 
-
-
-
-
-
-
-
-
-
-
-
-
 # ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++
 ##  Ready
 package provide zzzload [namespace eval zzzload {

src/make.tcl

@@ -2093,12 +2093,16 @@ if {$::punkboot::command in {project packages modules}} {
 
         #install .tm *and other files*
         puts stdout "VENDORMODULES$which: copying from $sourcefolder/$vf to $target_module_folder (if source file changed)"
-        set resultdict [punkcheck::install $sourcefolder/$vf  $target_module_folder\
+        set resultdict [punkcheck::install $sourcefolder/$vf  $target_module_folder {*}{
-            -installer make.tcl\
+            -installer make.tcl
-            -overwrite installedsourcechanged-targets\
+            -overwrite installedsourcechanged-targets
-            -antiglob_paths {README.md AGENTS.md **/AGENTS.md include_modules.config}\
+            -progresschannel stdout
-            -progresschannel stdout\
+            -exclude-filetails {AGENTS.md include_modules.config}
-            ]
+            -antiglob_paths {README.md}
+        }]
+        # -antiglob_paths {README.md AGENTS.md **/AGENTS.md include_modules.config}
+        #-exclude-filetails {AGENTS.md include_modules.config}
+        #-antiglob_paths {README.md}
         puts stdout [punkcheck::summarize_install_resultdict $resultdict]
     }
     if {![llength $vendormodulefolders]} {

src/modules/punk/args-999999.0a1.0.tm

@@ -9086,7 +9086,7 @@ tcl::namespace::eval punk::args {
                     }
                     if {!([tcl::string::first " " $a]>=0 || [tcl::string::first \t $a]>=0 || [tcl::string::last \n $a]>=0)} {
                         if {$OPT_ANY} {
-                            #exlude argument with whitespace from being a possible option e.g dict
+                            #exclude argument with whitespace from being a possible option e.g dict
                             #todo - passthrough of unrecognised --longopt=xxx without looking for following flag-value
                             set eposn [string first = $a]
                             if {$eposn > 2 && [string match --* $a]} {

src/modules/punk/path-999999.0a1.0.tm

@@ -1048,7 +1048,7 @@ namespace eval punk::path {
                 if {$boundary ne "" && [::punk::path::globmatchpath $boundary $path]} {
                     set recurse_below 1
                     set next_allbelow 1
-                    break
+                    continue
                 }
 
                 if {[pattern_prefix_viable $gp $path]} {

src/modules/punkcheck-999999.0a1.0.tm

@@ -42,7 +42,7 @@ namespace eval punkcheck {
         }
 
         #exclude-dir & exclude-file entries match the pattern at any level - should not contain path separators
-        variable default_exludedirseg_core  [list "#*" "_aside" "_build" ".git" ".fossil*"]
+        variable default_excludedirseg_core  [list "#*" "_aside" "_build" ".git" ".fossil*"]
         variable default_excludefiletail_core ""
 
         set has_twapi 0
@@ -1285,7 +1285,7 @@ namespace eval punkcheck {
             set defaults [list  {*}{
                 }   -glob                       *                                           {*}{
                 }   -exclude-filetails          [list "*.tm" "*-buildversion.txt" "*.exe"]  {*}{
-                }   -exclude-dirsegment_core  $excludedirseg_core                          {*}{
+                }   -exclude-dirsegments_core   $excludedirseg_core                         {*}{
                 }   -installer                  punkcheck::install_non_tm_files             {*}{
                 }
             ]

src/project_layouts/custom/_project/punk.basic/src/make.tcl

@@ -2093,12 +2093,16 @@ if {$::punkboot::command in {project packages modules}} {
 
         #install .tm *and other files*
         puts stdout "VENDORMODULES$which: copying from $sourcefolder/$vf to $target_module_folder (if source file changed)"
-        set resultdict [punkcheck::install $sourcefolder/$vf  $target_module_folder\
+        set resultdict [punkcheck::install $sourcefolder/$vf  $target_module_folder {*}{
-            -installer make.tcl\
+            -installer make.tcl
-            -overwrite installedsourcechanged-targets\
+            -overwrite installedsourcechanged-targets
-            -antiglob_paths {README.md AGENTS.md **/AGENTS.md include_modules.config}\
+            -progresschannel stdout
-            -progresschannel stdout\
+            -exclude-filetails {AGENTS.md include_modules.config}
-            ]
+            -antiglob_paths {README.md}
+        }]
+        # -antiglob_paths {README.md AGENTS.md **/AGENTS.md include_modules.config}
+        #-exclude-filetails {AGENTS.md include_modules.config}
+        #-antiglob_paths {README.md}
         puts stdout [punkcheck::summarize_install_resultdict $resultdict]
     }
     if {![llength $vendormodulefolders]} {

src/project_layouts/custom/_project/punk.project-0.1/src/bootsupport/modules/fauxlink-0.1.1.tm

@@ -205,7 +205,7 @@ namespace eval fauxlink {
     #    %2F     "/"
     #    %2f     "/"
     #    %7f     (del)
-    #we exlude the forward slash because we already have + for that - and multiple ways to specify it obscure intention.
+    #we exclude the forward slash because we already have + for that - and multiple ways to specify it obscure intention.
     #
     set decode_map [dict merge $decode_map [dict create\
         %09     \t\

src/project_layouts/custom/_project/punk.project-0.1/src/bootsupport/modules/funcl-0.1.tm

@@ -1,3 +1,6 @@
+
+#experimental.
+
 package provide funcl [namespace eval funcl {
     variable version
     set version 0.1
@@ -235,7 +238,7 @@ namespace eval funcl {
         }
         set comp [list] ;#composition list
         set end [lindex $args end]
-        if {[lindex $end 0] in {_fn _call}]} {
+        if {[lindex $end 0] in {_fn _call}} {
             #is_funcl
             set endfunc [lindex $args end]
         } else {

src/project_layouts/custom/_project/punk.project-0.1/src/bootsupport/modules/oolib-0.1.3.tm

@@ -0,0 +1,200 @@
+#JMN - api should be kept in sync with package patternlib where possible
+#
+
+namespace eval oolib {
+    oo::class create collection {
+        variable o_data ;#dict
+        #variable o_alias
+        constructor {} {
+            set o_data [dict create]
+        }
+        method info {} {
+            return [dict info $o_data]
+        }
+        method count {} {
+            return [dict size $o_data]
+        }
+        method isEmpty {} {
+            expr {[dict size $o_data] == 0}
+        }
+        method names {{globOrIdx {}}} {
+            if {[llength $globOrIdx]} {
+                if {[string is integer -strict $globOrIdx]} {
+                    set idx $globOrIdx
+                    if {$idx < 0} {
+                        set idx "end-[expr {abs($idx + 1)}]"
+                    }
+                    if {[catch {lindex [dict keys $o_data] $idx} result]} {
+                        error "[self object] no such index : '$idx'"
+                    } else {
+                        return $result
+                    }
+                } else {
+                    #glob
+                    return [lsearch -glob -all -inline [dict keys $o_data] $globOrIdx]
+                }
+            } else {
+                return [dict keys $o_data]
+            }
+        }
+        #like names but without globbing
+        method keys {} {
+            dict keys $o_data
+        }
+        method key {{posn 0}} {
+            if {$posn < 0} {
+                set posn "end-[expr {abs($posn + 1)}]"
+            }
+            if {[catch {lindex [dict keys $o_data] $posn} result]} {
+                error "[self object] no such index : '$posn'"
+            } else {
+                return $result
+            }
+        }
+        method hasKey {key} {
+            dict exists $o_data $key
+        }
+        method get {} {
+            return $o_data
+        }
+        method items {} {
+            return [dict values $o_data]
+        }
+        method item {key} {
+            if {[string is integer -strict $key]} {
+                if {$key >= 0} {
+                    set valposn [expr {(2*$key) +1}]
+                    return [lindex $o_data $valposn]
+                } else {
+                    set key "end-[expr {abs($key + 1)}]"
+                    return [lindex $o_data $key]
+                    #return [lindex [dict keys $o_data] $key]
+                }
+            }
+            if {[dict exists $o_data $key]} {
+                return [dict get $o_data $key]
+            }
+        }
+        #inverse lookup
+        method itemKeys {value} {
+            set value_indices [lsearch -all [dict values $o_data] $value]
+            set keylist [list]
+            foreach i $value_indices {
+                set idx [expr {(($i + 1) *2) -2}]
+                lappend keylist [lindex $o_data $idx]
+            }
+            return $keylist
+        }
+        method search {value args} {
+            set matches [lsearch {*}$args [dict values $o_data] $value]
+            if {"-inline" in $args} {
+                return $matches
+            } else {
+                set keylist [list]
+                foreach i $matches {
+                    set idx [expr {(($i + 1) *2) -2}]
+                    lappend keylist [lindex $o_data $idx]
+                }
+                return $keylist
+            }
+        }
+        #review - see patternlib. Is the intention for aliases to be configurable independent of whether the target exists?
+        #review - what is the point of alias anyway? - why slow down other operations when a variable can hold a keyname perfectly well?
+        #method alias {newAlias existingKeyOrAlias} {
+        #    if {[string is integer -strict $newAlias]} {
+        #        error "[self object] collection key alias cannot be integer"
+        #    }
+        #    if {[string length $existingKeyOrAlias]} {
+        #        set o_alias($newAlias) $existingKeyOrAlias
+        #    } else {
+        #        unset o_alias($newAlias)
+        #    }
+        #}
+        #method aliases {{key ""}} {
+        #    if {[string length $key]} {
+        #        set result [list]
+        #        foreach {n v} [array get o_alias] {
+        #            if {$v eq $key} {
+        #                lappend result $n $v
+        #            }
+        #        }
+        #        return $result
+        #    } else {
+        #        return [array get o_alias]
+        #    }
+        #}
+        ##if the supplied index is an alias, return the underlying key; else return the index supplied.
+        #method realKey {idx} {
+        #    if {[catch {set o_alias($idx)} key]} {
+        #        return $idx
+        #    } else {
+        #        return $key
+        #    }
+        #}
+        method add {value key} {
+            if {[string is integer -strict $key]} {
+                error "[self object] collection key must not be an integer. Use another structure if integer keys required"
+            }
+            if {[dict exists $o_data $key]} {
+                error "[self object] col_processors object error: key '$key' already exists in collection"
+            }
+            dict set o_data $key $value
+            return [expr {[dict size $o_data] - 1}] ;#return index of item
+        }
+        method remove {idx {endRange ""}} {
+            if {[string length $endRange]} {
+                error "[self object] collection error: ranged removal not yet implemented.. remove one item at a time"
+            }
+            if {[string is integer -strict $idx]} {
+                if {$idx < 0} {
+                    set idx "end-[expr {abs($idx+1)}]"
+                }
+                set key [lindex [dict keys $o_data] $idx]
+                set posn $idx
+            } else {
+                set key $idx
+                set posn [lsearch -exact [dict keys $o_data] $key]
+                if {$posn < 0} {
+                    error "[self object] no such index: '$idx' in this collection"
+                }
+            }
+            dict unset o_data $key
+            return
+        }
+        method clear {} {
+            set o_data [dict create]
+            return
+        }
+        method reverse_the_collection {} {
+            #named slightly obtusely because reversing the data when there may be references held is a potential source of bugs
+            #the name reverse_the_collection should make it clear that the object is being modified in place as opposed to simply 'reverse' which may imply a view/copy.
+            #todo - consider implementing a get_reverse which provides an interface to the same collection without affecting original references, yet both allowing delete/edit operations.
+            set dictnew [dict create]
+            foreach k [lreverse [dict keys $o_data]] {
+                dict set dictnew $k [dict get $o_data $k]
+            }
+            set o_data $dictnew
+            return
+        }
+        #review - cmd as list vs cmd as script?
+        method map {cmd} {
+            set seed [list]
+            dict for {k v} $o_data {
+                lappend seed [uplevel #0 [list {*}$cmd $v]]
+            }
+            return $seed
+        }
+        method objectmap {cmd} {
+            set seed [list]
+            dict for {k v} $o_data {
+                lappend seed [uplevel #0 [list $v {*}$cmd]]
+            }
+            return $seed
+        }
+    }
+}
+
+package provide oolib [namespace eval oolib {
+    variable version
+    set version 0.1.3
+}]

src/project_layouts/custom/_project/punk.project-0.1/src/bootsupport/modules/punk/aliascore-0.1.0.tm

@@ -123,6 +123,7 @@ tcl::namespace::eval punk::aliascore {
         ansistrip     ::punk::ansi::ansistrip
         stripansi     ::punk::ansi::ansistrip
         ansiwrap      ::punk::ansi::ansiwrap
+        ansisplit     ::punk::ansi::ta::split_codes_single
         grepstr       ::punk::ansi::grepstr
         untabify      ::punk::ansi::untabify
         colour        ::punk::console::colour

src/project_layouts/custom/_project/punk.project-0.1/src/bootsupport/modules/punk/ansi/sauce-0.1.0.tm

@@ -39,32 +39,35 @@ tcl::namespace::eval punk::ansi::sauce {
 
     proc from_file {fname} {
         if {[file size $fname] < 128} {
-            return
+            return [dict create posn -1]
         }
         set fd [open $fname r]
         chan conf $fd -translation binary
         chan seek $fd -128 end
+        set sauce_block_posn [expr {[chan tell $fd] -1}] ;#entire sauce block including ctrl-z and any comments - initial value assuming no comments
+        #If we treat the ctrl-z (\x1a) as part of the sauce - actual start of entire sauce info is 1 before sauce_header_posn,
+        #or further back if there are comments.
         set srec [read $fd]
         set srec_len 128 ;#This is the normal length of a SAUCE record - we may need to set it shorter if truncation detected
         if {[catch {set sdict [to_dict $srec]}]} {
             #review - have seen truncated SAUCE records < 128 bytes
             #we could search for SAUCE00 in the tail and see what records can be parsed?
             #specifically  publicdomain roysac images sometimes only 99 Bytes of sauce - suspect remaining were null \x0 padded and trimmed
-            set sauceposn [string first SAUCE00 $srec]
+            set saucestart [string first SAUCE00 $srec]
-            if {$sauceposn <= 0} {
+            if {$saucestart <= 0} {
                 close $fd
-                return
+                return [dict create posn -1]
             }
             #emit something to give user an indication something isn't right
             puts stderr "punk::ansi::sauce::from_file WARNING SAUCE record seems to be truncated - padding rhs with nulls and trying again.."
             #SAUCE00 is not at the beginning
             #pad the tail with nulls and try again
-            set srec [string range $srec $sauceposn end]
+            set srec [string range $srec $saucestart end]
             set srec_len [string length $srec]
             set srec ${srec}[string repeat \x0 [expr {128 - [string length $srec]}]]
             if {[catch {set sdict [to_dict $srec]}]} {
                 close $fd
-                return
+                return [dict create posn -1]
             }
             dict set sdict warning "SAUCE truncation to $srec_len bytes detected"
         }
@@ -73,6 +76,7 @@ tcl::namespace::eval punk::ansi::sauce {
             #Use srec_len instead of 128 - in case we had truncated source record which we padded and were able to parse
             set offset [expr {-1 *($srec_len + ($clines * 64) + 5)}]
             chan seek $fd $offset end
+            set sauce_block_posn [expr {[chan tell $fd] -1}] ;#entire sauce block including ctrl-z and any comments
             set tag [chan read $fd 5]
             if {$tag eq "COMNT"} {
                 #'character' data - shouldn't be null terminated c-style string - but can be
@@ -95,6 +99,7 @@ tcl::namespace::eval punk::ansi::sauce {
                 dict set sdict commentlines $commentlines
             }
         }
+        dict set sdict posn $sauce_block_posn
         close $fd
         return $sdict
     }
@@ -213,7 +218,9 @@ tcl::namespace::eval punk::ansi::sauce {
 
 
 
-
+    #---------------------------------------------------------------------------------------------------------------------------------------------
+    # This data comes from the sauce spec.
+    #---------------------------------------------------------------------------------------------------------------------------------------------
     #todo - fontName - which can also specify e.g code page 437
     ## Font name [1]	Font size	Resolution [2]	Aspect ratio [3]        Vertical stretch [6]	Description
     ##                                              Display [4]	Pixel [5]
@@ -221,7 +228,14 @@ tcl::namespace::eval punk::ansi::sauce {
     set fontnames [dict create]
 
     ## IBM VGA	9×16 [7]	720×400	4:3	20:27 (1:1.35)	35%	Standard hardware font on VGA cards for 80×25 text mode (code page 437)
-    dict set fontnames "IBM VGA" [list fontsize "9x16" resolution "720x400" aspect_ratio_display "4:3" aspect_ratio_pixel "20:27 (1:1.35)" vertical_stretch "35%" description "Standard hardware font on VGA cards for 80×25 text mode (code page 437)"]
+    dict set fontnames "IBM VGA" [list {*}{
+        fontsize                "9x16"
+        resolution              "720x400"
+        aspect_ratio_display    "4:3"
+        aspect_ratio_pixel      "20:27 (1:1.35)"
+        vertical_stretch        "35%"
+        description             "Standard hardware font on VGA cards for 80×25 text mode (code page 437)"
+    }]
     ## IBM VGA ### [8]	9×16 [7]	720×400	4:3	20:27 (1:1.35)	35%	Software installed code page font for VGA 80×25 text mode
     # - where ### is placeholder for 437,720,737 etc
 
@@ -247,6 +261,7 @@ tcl::namespace::eval punk::ansi::sauce {
     ## C64 PETSCII unshifted	8×8 [10]	320×200	4:3	5:6 (1:1.2)	20%	Original Commodore PETSCII font (PET, VIC-20, C64, CBM-II, Plus/4, C16, C116 and C128) in the unshifted mode. Unshifted mode (graphics) only has uppercase letters and additional graphic characters. This is the normal boot font.
     ## C64 PETSCII shifted	8×8 [10]	320×200	4:3	5:6 (1:1.2)	20%	Original PETSCII font in shifted mode. Shifted mode (text) has both uppercase and lowercase letters. This mode is actuated by pressing Shift+Commodore key.
     ## Atari ATASCII	8×8 [11]	320×192	4:3	4:5 (1:1.25)	25%	Original ATASCII font (Atari 400, 800, XL, XE)
+    #---------------------------------------------------------------------------------------------------------------------------------------------
 
 
     #expect a 128 Byte sauce record
@@ -256,6 +271,7 @@ tcl::namespace::eval punk::ansi::sauce {
         variable datatypes
         variable filetypes
         variable encodings
+        set warnings [list]
         if {[string length $saucerecord] != 128} {
             error "punk::ansi::sauce::to_dict: Unable to interpret data as a SAUCE record - length != 128"
         }
@@ -321,6 +337,8 @@ tcl::namespace::eval punk::ansi::sauce {
                 dict set sdict filetype_name ""
             }
         } else {
+            #how can a byte fail to scan with cu? is this even reachable?
+            puts stderr "punk::ansi::sauce::to_dict filetype byte failed to scan - setting filetype and filetype_name to empty string byte: [ansistring VIEW -lf 1 [string range $saucerecord 95 95]]"
             dict set sdict filetype ""
             dict set sdict filetype_name ""
         }
@@ -417,25 +435,40 @@ tcl::namespace::eval punk::ansi::sauce {
             5 {
                 #binarytext
                 #filetype is supposed to represent half the characterwidth (only widths with multiples of 2 can be specified)
-                #HOWEVER - in the wild we may see width/height specified in tinfo1/tinfo2 with some other value in filetype (eg 1)
+                #HOWEVER - in the wild we may see width/height specified in tinfo1/tinfo2 with some apparently unrelated value in filetype (eg 0 or 1) that doesn't match the intended image dimensions.
-                #If both tinfo1 and tinfo2 are non zero - we will use them, even though it's not in spec.
+                #If both tinfo1 and tinfo2 are non zero - we *could* use them, even though it's not in spec.
-                set t1 [dict get $sdict tinfo1]
+                #An example file (us-used1.bin) has filetype 0 and tinfo1/tinfo2 640/350
-                if {$t1 eq ""} {
+                #It's possible tinfo1/tinfo2 represent pixel dimensions for a 'standard' 8x16 font, but this image is 160 columns wide, so we would expect tinfo1 to be 1280.
-                    set t1 0
+                #The sauce spec seems to indicate we should ignore tinfo1/tinfo2 for binarytext and only use filetype to determine width.
-                }
+                #the default for binarytext is 160 columns.
-                set t2 [dict get $sdict tinfo2]
+
-                if {$t2 eq ""} {
+                #filetype 1 is theoretically possible, representing 2 columns
-                    set t2 0
+                #in practice we see this value for binarytext images that are definitely not intended to be 2 columns wide. Why?
+                #is there some assumption that that images are at least a certain width, and filetype has been repurposed to indicate something else?
+                #The spec would seem to rule out images of a single column due to filetype being half the character width but a value of 0.5 isn't supported.
+                #It specifically mentions that only even widths up to 510 can be specified. ($filetype * 2 where filetype is 1-255?)
+
+
+                #proper mechanism to specify columns for binarytext is the datatype field.
+                set cols [expr {2*[dict get $sdict filetype]}]
+                if {$cols == 0} {
+                    lappend warnings "binarytext filetype value of [dict get $sdict filetype] - using binarytext default cols of 160"
+                     #default for binarytext is 160 columns
+                    set cols 160
                 }
-                if {$t1 != 0 && $t2 != 0} {
+                if {$cols == 2 && [dict get $sdict tinfo1] != 0 && [dict get $sdict tinfo2] != 0} {
                     #not to spec - but we will assume these have values for a reason..
-                    puts stderr "punk::ansi::sauce::to_dict using tinfo1/tinfo2 data for columns/rows (non-compliant SAUCE data)"
+                    #---------------------------------------------------------------------------------------------------------------------
-                    dict set sdict columns [expr {2 * $t1}]
+                    #The sample file src/testansi/formatsamples/image/binaryText/test.bin has a filetype 1 and tinfo1 40 and tinfo2 25.
-                    dict set sdict rows $t2
+                    #(similarly ppe-ansi.bin has tinfo1 80 and tinfo2 26)
+                    #They seem to use the 1 in filetype to indicate that the tinfo1/tinfo2 values should be used.
+                    #(The 80 cols wide test.bin binaryText image matches the xbin sample file src/testansi/formatsamples/image/xbin/test.xb which is a more fully specified format using a header)
+                    #---------------------------------------------------------------------------------------------------------------------
+                    lappend warnings "binarytext filetype of 1 with non-zero tinfo1/tinfo2 - using tinfo1/tinfo2 data for columns/rows (possibly non-conforming SAUCE data - matching observed data in the wild)"
+                    set cols [expr {2 * [dict get $sdict tinfo1]}]
+                    dict set sdict columns $cols
+                    dict set sdict rows [dict get $sdict tinfo2]
                 } else {
-                    #proper mechanism to specify columns for binarytext is the datatype field.
-
-                    set cols [expr {2*[dict get $sdict filetype]}]
                     dict set sdict columns $cols
                     #rows must be calculated from file size
                     #rows = (filesize - sauceinfosize)/ filetype * 2 * 2
@@ -447,11 +480,13 @@ tcl::namespace::eval punk::ansi::sauce {
 
             }
             6 {
-                #xbin - only filtype is 0
+                #xbin - only filetype is 0
                 #https://web.archive.org/web/20120204063040/http://www.acid.org/info/xbin/x_spec.htm
                 dict set sdict columns [dict get $sdict tinfo1]
                 dict set sdict rows    [dict get $sdict tinfo2]
                 dict set sdict fontname [dict get $sdict tinfos]
+                #Values from sauce record are probably only informational, because xbin has an 11-byte header with width,height,fontsize and flags.
+                #presumably the header-info should take precedence over all sauce data (? review)
             }
         }
         if {[dict exists $sdict fontname]} {
@@ -474,6 +509,9 @@ tcl::namespace::eval punk::ansi::sauce {
                 }
             }
         }
+        if {[llength $warnings]} {
+            dict set sdict warnings $warnings
+        }
         return $sdict
     }
 

src/project_layouts/custom/_project/punk.project-0.1/src/bootsupport/modules/punk/args-0.2.1.tm

@@ -9086,7 +9086,7 @@ tcl::namespace::eval punk::args {
                     }
                     if {!([tcl::string::first " " $a]>=0 || [tcl::string::first \t $a]>=0 || [tcl::string::last \n $a]>=0)} {
                         if {$OPT_ANY} {
-                            #exlude argument with whitespace from being a possible option e.g dict
+                            #exclude argument with whitespace from being a possible option e.g dict
                             #todo - passthrough of unrecognised --longopt=xxx without looking for following flag-value
                             set eposn [string first = $a]
                             if {$eposn > 2 && [string match --* $a]} {

src/project_layouts/custom/_project/punk.project-0.1/src/bootsupport/modules/punk/char-0.1.0.tm

@@ -3033,14 +3033,18 @@ tcl::namespace::eval punk::char {
     #This still leaves a whole class of clusters.. korean etc unhandled :/
     #todo - tk::startOfCluster / tk::endOfCluster - try to get it brought into Tcl
     #https://core.tcl-lang.org/tcl/tktview/a4c7eeaf63
-    proc grapheme_split {text} {
+    proc grapheme_split {text {return list}} {
         #we should treat \r\n as a single grapheme cluster (as tk::endOfCluster does)
         set components [list]
         set csplits [combiner_split $text]
         foreach {pt combiners} [lrange $csplits 0 end-1] {
             set clist [split $pt ""]
-            lappend components {*}[lrange $clist 0 end-1]
+            #review
-            lappend components [tcl::string::cat [lindex $clist end] $combiners]
+            #lset clist end [tcl::string::cat [lindex $clist end] $combiners]
+            ledit clist end end [tcl::string::cat [lindex $clist end] $combiners]
+            lappend components {*}$clist
+            #lappend components {*}[lrange $clist 0 end-1]
+            #lappend components [tcl::string::cat [lindex $clist end] $combiners]
         }
         #last csplit never has a combiner (_perlish_split style) - and may be empty - in which case we don't append it as a grapheme
         if {[lindex $csplits end] ne ""} {
@@ -3066,7 +3070,6 @@ tcl::namespace::eval punk::char {
 
         #review \uFE0F variation selector 16 - forces emoji presentation for preceding char
 
-        if 1 {
         #This is a basic implementation that does not check that all combinations are valid.
         set graphemes [list]
         set current_cluster ""
@@ -3085,13 +3088,13 @@ tcl::namespace::eval punk::char {
                 lappend graphemes "\r\n"
                 incr i ;#skip the \n as we've already processed it as part of the cluster
                 set current_cluster ""
-                    grapheme_split::reset_base
+                set cluster_base 0; set cluster_base_RI 0 ;#grapheme_split::reset_base
                 set current_cluster_is_extensible 0
             } elseif {$component eq "\u200d"} {
                 if {$current_cluster eq ""} {
                     #ZWJ at start of string - treat as separate grapheme cluster - but isn't a valid base for further combining with more ZWJs or modifiers
                     set current_cluster $component
-                        grapheme_split::reset_base
+                    set cluster_base 0; set cluster_base_RI 0 ;#grapheme_split::reset_base
                     set current_cluster_is_extensible 0
                 } else {
                     if {$cluster_base} {
@@ -3103,7 +3106,7 @@ tcl::namespace::eval punk::char {
                             append current_cluster $component
                             if {$cluster_base_RI} {
                                 #regional indicators can only combine in pairs, and don't combine with modifiers or ZWJs to form longer clusters - so not extensible.
-                                    grapheme_split::reset_base
+                                set cluster_base 0; set cluster_base_RI 0 ;#grapheme_split::reset_base
                                 set current_cluster_is_extensible 0
                                 #we can keep adding ZWJs or modifiers though
                             } else {
@@ -3122,13 +3125,13 @@ tcl::namespace::eval punk::char {
                 if {$current_cluster eq ""} {
                     #modifier at start of string -  not a valid base for further combining with more modifiers or ZWJs - but we continue appending modifiers to the current cluster.
                     set current_cluster $component
-                        grapheme_split::reset_base
+                    set cluster_base 0; set cluster_base_RI 0 ;#grapheme_split::reset_base
                 } else {
                     if {$cluster_base} {
                         if {$current_cluster_is_extensible} {
                             append current_cluster $component
                             #invalidate the base!
-                                grapheme_split::reset_base
+                            set cluster_base 0; set cluster_base_RI 0 ;#grapheme_split::reset_base
                         } else {
                             append current_cluster $component
                         }
@@ -3155,7 +3158,7 @@ tcl::namespace::eval punk::char {
                         } else {
                             lappend graphemes $current_cluster
                             set current_cluster $component
-                                grapheme_split::reset_base
+                            set cluster_base 0; set cluster_base_RI 0 ;#grapheme_split::reset_base
                         }
                         set current_cluster_is_extensible 0
                     } elseif {$cluster_base_RI} {
@@ -3165,7 +3168,7 @@ tcl::namespace::eval punk::char {
 
                             #invalidate the base - we can't combine more than 2 RIs in a cluster, and they don't combine with modifiers or ZWJs to form longer clusters.
                             #we can however add more ZWJs or modifiers to the cluster - but they don't make it extensible for combining with more RIs
-                                grapheme_split::reset_base
+                            set cluster_base 0; set cluster_base_RI 0 ;#grapheme_split::reset_base
                         } else {
                             #something else while RI cluster is open - end the current cluster and start a new one with the current char.
                             lappend graphemes $current_cluster
@@ -3182,11 +3185,11 @@ tcl::namespace::eval punk::char {
         if {$current_cluster ne ""} {
             lappend graphemes $current_cluster
         }
+        if {$return eq "list"} {
+             return $graphemes
         } else {
-            set graphemes $components
+             return [dict create list $graphemes last_extensible $current_cluster_is_extensible base $cluster_base RI_base $cluster_base_RI]
         }
-
-        return $graphemes
     }
     namespace eval grapheme_split {
         proc about {} {

src/project_layouts/custom/_project/punk.project-0.1/src/bootsupport/modules/punk/console-0.1.1.tm

@@ -71,11 +71,6 @@ package require punk::args
 
 
 
-#if {"windows" eq $::tcl_platform(platform)} {
-#    #package require zzzload
-#    #zzzload::pkg_require twapi
-#}
-
 #see https://learn.microsoft.com/en-us/windows/console/classic-vs-vt
 #https://learn.microsoft.com/en-us/windows/console/creating-a-pseudoconsole-session
 

src/project_layouts/custom/_project/punk.project-0.1/src/bootsupport/modules/punk/du-0.1.0.tm

@@ -2529,21 +2529,30 @@ namespace eval punk::du {
                     #jmn disable twapi
                     #tailcall du_dirlisting_generic $folderpath {*}$args
 
-                    package require zzzload
+                    #package require zzzload
-                    set loadstate [zzzload::pkg_require twapi]
+                    #set loadstate [zzzload::pkg_require twapi]
-
+
-                    if {$loadstate ni [list loading failed]} {
+                    #if {$loadstate ni [list loading failed]} {
-                        #either already loaded by zzload or ordinary package require
+                    #    #either already loaded by zzload or ordinary package require
-                        package require twapi ;#should be fast once twapi dll loaded in zzzload thread
+                    #    package require twapi ;#should be fast once twapi dll loaded in zzzload thread
+                    #    set ::punk::du::has_twapi 1
+                    #    punk::du::active::set_active_function du_dirlisting du_dirlisting_twapi
+                    #    tailcall du_dirlisting_twapi $folderpath {*}$args
+                    #} else {
+                    #    if {$loadstate eq "failed"} {
+                    #        puts stderr "punk::du defaulting to du_dirlisting_generic because twapi load failed"
+                    #        punk::du::active::set_active_function du_dirlisting du_dirlisting_generic
+                    #    }
+                    #    tailcall du_dirlisting_generic $folderpath {*}$args
+                    #}
+                    if {[catch {package require twapi} errM]} {
+                        puts stderr "punk::du defaulting to du_dirlisting_generic because twapi load failed: $errM"
+                        punk::du::active::set_active_function du_dirlisting du_dirlisting_generic
+                        tailcall du_dirlisting_generic $folderpath {*}$args
+                    } else {
                         set ::punk::du::has_twapi 1
                         punk::du::active::set_active_function du_dirlisting du_dirlisting_twapi
                         tailcall du_dirlisting_twapi $folderpath {*}$args
-                    } else {
-                        if {$loadstate eq "failed"} {
-                            puts stderr "punk::du defaulting to du_dirlisting_generic because twapi load failed"
-                            punk::du::active::set_active_function du_dirlisting du_dirlisting_generic
-                        }
-                        tailcall du_dirlisting_generic $folderpath {*}$args
                     }
                 }
                 default {

src/project_layouts/custom/_project/punk.project-0.1/src/bootsupport/modules/punk/lib-0.1.6.tm

@@ -138,25 +138,10 @@ tcl::namespace::eval punk::lib::check {
         if {"windows" ne $::tcl_platform(platform)} {
             set bug 0
         } else {
-            if {![catch {file tempdir} tmpdir]} {
+            set tmpdir [punk::lib::tempdir_newfolder] ;#uses 'file tempdir' on tcl9+ or fallback to env vars or current directory on older versions
-                #tcl 9+ has 'file tempdir'
-                set testfile [file join $tmpdir "bugtest"]
-            } else {
-                #fallback for older tcl versions - use env TEMP/TMP or current directory
-                set tmpdir ""
-                foreach e {TEMP TMP} {
-                    if {[info exists ::env($e)] && [file isdirectory ::env($e)]} {
-                        set tmpdir ::env($e)
-                        break
-                    }
-                }
-                if {$tmpdir eq ""} {
-                    #no env vars - fallback to current directory
-                    set tmpdir [pwd]
-                }
             set testfile [file join $tmpdir "bugtest"]
-            }
 
+            try {
                 set fd [open $testfile w]
                 puts $fd test
                 close $fd
@@ -170,6 +155,14 @@ tcl::namespace::eval punk::lib::check {
                         break
                     }
                 }
+            } finally {
+                if {[file exists $testfile]} {
+                    file delete $testfile
+                }
+                if {[file exists $tmpdir]} {
+                    file delete -force $tmpdir
+                }
+            }
         }
         #possibly related anomaly is that a returned result with case that doesn't match that of the file in the underlying filesystem can't be normalized
         # to the correct case without doing some sort of string manipulation on the result such as 'string range $f 0 end' to affect the internal representation.
@@ -679,7 +672,207 @@ namespace eval punk::lib {
         }
     }
 
+    namespace eval argdoc {
+        variable PUNKARGS
+        lappend PUNKARGS [list {
+            @id     -id     ::punk::lib::tempdir
+            @cmd    -name     punk::lib::tempdir\
+                -summary\
+                "Determine an appropriate temp dir for the process we are running under."\
+                -help\
+                "On windows:
+                  If the process is running under the system account use the modern windows location: %SystemRoot%\SystemTemp
+                  Detection of the system account relies on either twapi, or a combination of the whoami command and the 
+                  registry package.
+                  Proceeds with same fallback logic as for other platforms if we fail to detect the system account or its temp location.
+
+
+                For other platforms we use the environment variables TMPDIR/TEMP/TMP or fallback to /tmp or /var/tmp if those
+                env vars aren't set or aren't writable directories.
+
+                Final fallback attempt is the current working directory.
+                Result is normalized so resulting path will have forward slashes on all platforms.
+
+                Alternatives: see the tcllib fileutil::tempdir function.
+                "
+            @values -min 0 -max 0
+        }]
+    }
+    proc tempdir {} {
+        set trydirs [list]
+        if {"windows" eq $::tcl_platform(platform)} {
+            #review.
+            #consider also checking for whether running under various service accounts
+
+            if {![catch {package require twapi}]} {
+                set tok [twapi::open_process_token] ;#first call is a little pricy.
+                set sid [twapi::get_token_user $tok]
+                if {$sid eq "S-1-5-18"} {
+                    #system account - use system account temp location
+                    set sysroot [twapi::get_shell_folder csidl_windows] ;#first call is a little pricy.
+                    lappend trydirs [file join $sysroot "SystemTemp"]
+                }
+                #if not system account - use env vars as first choice.
+            } else {
+                #twapi not available - try to use builtin windows whoami to detect if we're running under system account - but this is less reliable than using twapi to check the process token - so we warn about it.
+                set whoami_exe [auto_execok whoami]
+                #test that system32 is somewhere in the whoami path as a basic attempt to avoid non-windows whoami commands that might be present in the path
+                set whoami_exe_parts [file split $whoami_exe]
+                if {"system32" in [string tolower $whoami_exe_parts]} {
+                    set whoamiresult [string trim [exec {*}$whoami_exe /USER /FO LIST] \n\r]
+                    set whoamiresult [string map {\r\n \n} $whoamiresult]
+                    set whoamiresult_lines [split $whoamiresult \n]
+                    set sid ""
+                    foreach line $whoamiresult_lines {
+                        if {[string match "SID:*" $line]} {
+                            set sid [lindex $line 1]
+                            break
+                        }
+                    }
+                    set has_registry [expr {![catch {package require registry}]}]
+                    if {$sid eq "S-1-5-18"} {
+                        #system account - use system account temp location
+                        set sysroot ""
+                        if {$has_registry} {
+                            #registry path is case-insensitive.
+                            catch {
+                                set sysroot [registry get {HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Windows NT\CurrentVersion} SystemRoot]
+                            }
+                        } else {
+                            if {[info exists ::env(SystemRoot)]} {
+                                set sysroot [set ::env(SystemRoot)]
+                            }
+                        }
+                        if {$sysroot ne ""} {
+                            lappend trydirs [file join $sysroot "SystemTemp"]
+                        }
+                    }
+                    #if not system account - use env vars as first choice.
+                }
+            }
+        }
+
+        foreach t {TMPDIR TEMP TMP} {
+            #TMPDIR is the posix standard as first choice for temp dir env var.
+            if {[info exists ::env($t)]} {
+                lappend trydirs $::env($t)
+            }
+        }
+
+        if {"windows" ne $::tcl_platform(platform)} {
+            #suitable for macos,linux and freebsd at least.
+            lappend trydirs [file join / tmp] [file join / var tmp]
+            #/usr/tmp is probably not a common location for a temp dir on modern unix-based systems.
+        }
+
+        foreach d $trydirs {
+            if {[file isdirectory $d] && [file writable $d]} {
+                return [file normalize $d]
+            }
+        }
+
+        #only even call 'pwd' as a last resort (mildly slow on first call).
+        set cwd [pwd]
+        if {[file isdirectory $cwd] && [file writable $cwd]} {
+            return $cwd
+        }
+
+        return -code error "punk::lib::tempdir unable to determine a suitable temp directory - tried: $trydirs"
+    }
+
+    namespace eval argdoc {
+        variable PUNKARGS
+        lappend PUNKARGS [list {
+            @id     -id     ::punk::lib::tempdir_newfolder
+            @cmd    -name     punk::lib::tempdir_newfolder\
+                -summary\
+                "Create unique folder within temp dir (or cwd as last resort)"\
+                -help\
+                "Creates a new unique folder within the temp dir determined by punk::lib::tempdir.
+                The folder is created before returning its full path and will be empty.
+                The folder is named with a tcl_ prefix followed by a random string.
+
+                See also: 'file tempdir' in tcl 9+ or fileutil::maketempdir from tcllib"
+            @opts
+                -dir -type string -default "" -help\
+                    "Base directory to create the temp folder in - defaults to the result of punk::lib::tempdir"
+                -prefix -type string -default tcl -help\
+                    "Prefix for the temp folder name
+                    An underscore is automatically appended to the prefix in the generated folder name.
+                    If prefix is the empty string - then the generated folder name will still be autoprefixed
+                    with tcl_  (consistent with tcl9 'file tempdir')"
+            @values -min 0 -max 0
+        }]
+    }
+    proc tempdir_newfolder {args} {
+        set argd [punk::args::parse $args withid ::punk::lib::tempdir_newfolder]
+        set opt_dir [dict get $argd opts -dir]
+        set opt_prefix [dict get $argd opts -prefix]
+        puts "opt_prefix: $opt_prefix"
+        if {[llength [file split $opt_prefix]] > 1} {
+            error "punk::lib::tempdir_newfolder -prefix option should not contain any path separators"
+        }
+        if {$opt_prefix eq ""} {
+            #don't allow empty prefix - for consistency with tcl9 'file tempdir' - which would still prefix with 'tcl_' even if prefix is empty string.
+            set opt_prefix "tcl"
+        }
+
+        if {$opt_dir ne ""} {
+            if {[file isdirectory $opt_dir] && [file writable $opt_dir]} {
+                set tmpbase [file normalize $opt_dir]
+            } else {
+                error "punk::lib::tempdir_newfolder -dir option '$opt_dir' is not a writable directory"
+            }
+        } else {
+            set tmpbase [punk::lib::tempdir] ;#will raise an error if no writable temp dir or cwd found.
+        }
+        #assert: tmpbase has no trailing slash - unless it is a root dir (e.g / on unix, or C:/ on windows)
+        #assert: tmpbase is normalized with forward slashes on all platforms.
 
+        set tcl9_template_base [string trimright $tmpbase "/"] ;#set it to a form that can join along with a forward slash to prefix for tcl 9 'file tempdir' style template.
+        #tcl9 'file tempdir' separates the prefix in the template from the random string with an underscore.
+
+        #now form template by always joining with a slash (even if opt_prefix is empty)
+        #(avoiding file join and file normalize to ensure template is properly formed)
+        #whether or not opt_prefix ends with an underscore - another will be added. (by file tempdir, or by our own code if tcl9 'file tempdir' isn't available)
+        #assert: opt_prefix is not empty string (will have defaulted to 'tcl') and does not contain any path separators.
+        set tcl9_template "$tcl9_template_base/$opt_prefix"
+
+
+        #tcl 9+ has 'file tempdir'
+        #we don't support the same template as 'file tempdir'
+        if {[catch {file tempdir $tcl9_template} tmpdir]} {
+
+            set prefix tcl_ ;#todo - accept option: -prefix
+
+            set chars abcddefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789
+            set nrand_chars 8
+            set maxtries 100
+            for {set i 0} {$i < $maxtries} {incr i} {
+                set dirname ${opt_prefix}_  ;#always add underscore for consistency with tcl9 'file tempdir'.
+                for {set j 0} {$j < $nrand_chars} {incr j} {
+                    append dirname [string index $chars [expr {int(rand()*62)}]]
+                }
+                set path [file join $tmpbase $dirname]
+                if {[file exists $path]} {
+                    continue
+                }
+                if {[catch {
+                        file mkdir $path
+                        if {"windows" ne $::tcl_platform(platform)} {
+                            file attributes $path -permissions 0o700
+                        }
+                    }]} {
+                    continue
+                }
+                return $path
+            }
+            return -code error "punk::lib::tempdir_newfolder unable to create unique tempdir in $tmpbase after $maxtries attempts - too many collisions - aborting"
+        }
+        #tcl 9 'file tempdir' return.
+        #normalize because on early tcl 9 versions on windows it can return with mixed forward and backward slashes when a template is supplied with forward slashes.
+        return [file normalize $tmpdir]
+    }
 
     # == == == == == == == == == == == == == == == == == == == == == == == == == == == == == == ==
     # Maintenance - This is the primary source for tm_version... functions
@@ -814,6 +1007,89 @@ namespace eval punk::lib {
             error "tm_version_required_canonical should have already returned a canonicalised versionspec - or produced an error with reason before this point"
         }
     }
+    proc tm_version_magic {} {
+        #maintenance instruction:
+        #  This specific 12 character nnnnnn.0a1.0 version number is used in text replacement when installing modules.
+        #  It should not be modified without also considering the impact on existing projects based on punkshell that may have used this version number in their own code or documentation as a placeholder for tm version handling.
+        #  A copy of this is also present in punk::lib.
+        #  Essentially this should probably never be changed - and instead deprecated if we want to stop using it and replaced with a different mechanism/placeholder.
+
+        #tm versioning is not quite the same as semantic versioning - but similarly we should see extremely high major version numbers rarely if ever in real use,
+        #even over decades of development.
+        set magicbase 999999  ;#deliberately large so given load-preference when testing!
+        #we split the literal to avoid the literal appearing here - reduce risk of accidentally converting to a release version
+        return ${magicbase}.0a1.0
+    }
+
+    namespace eval argdoc {
+        lappend PUNKARGS [list {
+            @id     -id     ::punk::lib::tm_split_name
+            @cmd    -name     punk::lib::tm_split_name\
+                -summary\
+                "Split a versioned module name into name and version parts, dropping trailing .tcl/.tm if any."\
+                -help\
+                "Splits a versioned module name (as present in a filename or namespaced name) into name and version parts,
+                Ignores any trailing .tm or .tcl file extension.
+
+                If the fullmodulename given is a namespaced name - then the returned modulename will also be namespaced,
+                but with any leading :: removed.
+
+                Returns a two element list - with the first element being the modulename and the second element being the version.
+
+                Tcl module version numbers are understood with leading zeros in each dotted part, but leading zeros are not canonical.
+
+                This split does not canonicalise the version number.
+                If the last dash-separated segment of the name doesn't look like a valid version number
+                - then it is treated as part of the modulename and an empty version string is returned.
+                e.g
+                  mymod-1.2.3.tm -> mymod 1.2.3
+                  mymod-1aa2.3.tm -> mymod-1aa2.3 {}
+                  (repeated a is not a valid version segment - so the entire '1aa2.3' is treated as part of the modulename)
+
+                see also: tm_version_canonical
+                "
+            @values -min 1 -max 1
+            fullmodulename -type string -help\
+                "The full module name to split - as present in a filename or namespaced name. E.g:
+                mymod-1.2.3
+                mymod-1.2.3.tm
+                mymod-1.2.3.tcl
+                /some/where/mymod-123.0a4.0.tm
+                mymod
+                mymod.tm
+                mymod.tcl
+                ns1::ns2::mymod-1.2.3
+                ::ns1::ns2::mymod"
+        }]
+    }
+    proc tm_split_name {fullmodulename} {
+        #split modulename (as present in a filename or namespaced name) into name/version ignoring leading namespace path
+        #ignore trailing .tm .TM if present
+        #if version doesn't pass validation - treat it as part of the modulename and return empty version string without error
+        #Up to caller to validate.
+        set lastpart [namespace tail $fullmodulename]
+        set lastpart [file tail $lastpart] ;# should be ok to use file tail now that we've ensured no namespace components
+        if {[string tolower [file extension $fullmodulename]] in {.tcl .tm}} {
+            set fileparts [split [file rootname $lastpart] -]
+        } else {
+            set fileparts [split $lastpart -]
+        }
+        if {[tm_version_isvalid [lindex $fileparts end]]} {
+            set versionsegment [lindex $fileparts end]
+            set namesegment [join [lrange $fileparts 0 end-1] -];#re-stitch
+        } else {
+            set namesegment [join $fileparts -]
+            set versionsegment ""
+        }
+        set base [string trimleft [namespace qualifiers $fullmodulename] :]
+        if {$base ne ""} {
+            set modulename "${base}::$namesegment"
+        } else {
+            set modulename $namesegment
+        }
+        return [list $modulename $versionsegment]
+    }
+
     # end tm_version... functions
     # == == == == == == == == == == == == == == == == == == == == == == == == == == == == == == ==
 
@@ -4210,6 +4486,9 @@ namespace eval punk::lib {
                                                 if {[string index $key 0] ne "%"} {
                                                     set key %$key
                                                 }
+                                                #puts "---key:'$key'"
+                                                set key [string map {; \\;} $key] ;#review
+                                                #puts "---key:'$key'"
                                                 #pipeline - use punk patterns.
                                                 % thisval.= $key= $thisval
                                             }

src/project_layouts/custom/_project/punk.project-0.1/src/bootsupport/modules/punk/mix/cli-0.3.1.tm

@@ -499,7 +499,7 @@ namespace eval punk::mix::cli {
             # -- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- ---
             set opt_punkcheck_eventobj                   [dict get $opts -punkcheck_eventobj]
 
-            set magicversion [punk::mix::util::magic_tm_version] ;#deliberately large so given load-preference when testing
+            set magicversion [punk::mix::util::tm_version_magic] ;#deliberately large so given load-preference when testing
             set module_list [list]
 
             if {[file tail [file dirname $srcdir]] ne "src"} {

src/project_layouts/custom/_project/punk.project-0.1/src/bootsupport/modules/punk/mix/commandset/loadedlib-0.1.0.tm

@@ -54,13 +54,18 @@ namespace eval punk::mix::commandset::loadedlib {
         if {$opt_refresh} {
             catch {package require frobznodule666} ;#ensure pkg system has loaded/searched for everything REVIEW - this doesn't result in full scans
             foreach tm_path [tcl::tm::list] {
+                #review - todo - adjust punk::path::subfolders to take arguments to do some filtering itself rather than recurse down unnecessary branches.
                 set paths_below [punk::path::subfolders -recursive $tm_path]
                 foreach folder $paths_below {
+                    if {[string match */_build/* $folder]} {continue}
                     set tail [file tail $folder]
-                    if {[string match #modpod-* $tail] || [string match #tarjar-* $tail]} {
+                    if {[string match #tarjar-* $tail]} {
+                        continue
+                    }
+                    if {[string match #modpod-* $tail]} {
+                        #manually do a 'package ifneeded' fore each module found here.
                         continue
                     }
-                    if {[string match */_build/* $folder]} {continue}
                     set relpath [string tolower [punk::path::relative $tm_path $folder]]
                     set modpath [string map {/ ::} $relpath]
                     catch {package require ${modpath}::flobrudder99}

src/project_layouts/custom/_project/punk.project-0.1/src/bootsupport/modules/punk/mix/commandset/module-0.1.0.tm

@@ -362,7 +362,7 @@ namespace eval punk::mix::commandset::module {
         file mkdir $modulefolder
 
         set moduletail [namespace tail $modulename]
-        set magicversion [punk::mix::util::magic_tm_version] ;#deliberately large so given load-preference when testing
+        set magicversion [punk::mix::util::tm_version_magic] ;#deliberately large so given load-preference when testing
 
 
 

src/project_layouts/custom/_project/punk.project-0.1/src/bootsupport/modules/punk/mix/commandset/project-0.1.0.tm

@@ -380,25 +380,25 @@ namespace eval punk::mix::commandset::project {
 
         puts stdout ">>> about to call punkcheck::install $layout_path $projectdir"
         set resultdict [dict create]
-        set antipaths [list\
+        set antipaths [list         {*}{
-            src/doc/*\
+            src/doc/*
-            src/doc/include/*\
+            src/doc/include/*
-            src/PROJECT_LAYOUTS_*\
+            src/PROJECT_LAYOUTS_*
-        ]
+        }]
 
-        #set antiglob_dir [list\
+        #set exclude_dirsegments [list {*}{
-        #    _ignore_*\
+        #    _ignore_*
-        #]
+        #}]
-        set antiglob_dir [list\
+        set exclude_dirsegments [list {*}{
-        ]
+        }]
 
-        #default antiglob_dir_core will stop .fossil* from being updated - which is generally desirable as these are likely to be customized
+        #default -exclude-dirsegments_core will stop .fossil* from being updated - which is generally desirable as these are likely to be customized
         if {$opt_force} {
             puts stdout "copying layout files - with force applied - overwrite all-targets"
-            set resultdict [punkcheck::install $layout_path $projectdir -installer project.new -createempty 1  -overwrite ALL-TARGETS -antiglob_paths $antipaths -antiglob_dir $antiglob_dir]
+            set resultdict [punkcheck::install $layout_path $projectdir -installer project.new -createempty 1  -overwrite ALL-TARGETS -antiglob_paths $antipaths -exclude-dirsegments $exclude_dirsegments]
         } else {
             puts stdout "copying layout files - (if source file changed)"
-            set resultdict [punkcheck::install $layout_path $projectdir -installer project.new -createempty 1 -overwrite installedsourcechanged-targets -antiglob_paths $antipaths -antiglob_dir $antiglob_dir]
+            set resultdict [punkcheck::install $layout_path $projectdir -installer project.new -createempty 1 -overwrite installedsourcechanged-targets -antiglob_paths $antipaths -exclude-dirsegments $exclude_dirsegments]
         }
         puts stdout [punkcheck::summarize_install_resultdict $resultdict]
 
@@ -412,11 +412,11 @@ namespace eval punk::mix::commandset::project {
 
         #target folders .fossil-custom and .fossil-settings may not exist. use -createdir 1 to ensure existence.
         #In this case we need to override the default dir antiglob - as .fossil-xxx folders need to be installed from template if missing, or if target is uncustomized.
-        ## default_antiglob_dir_core  [list "#*" "_aside" ".git" ".fossil*"]
+        ## default_exclude_dirsegments_core  [list "#*" "_aside" ".git" ".fossil*"]
-        set override_antiglob_dir_core [list #* _aside .git]
+        set override_exclude_dirsegments_core [list #* _aside .git]
         if {[file exists $layout_path/.fossil-custom]} {
             puts stdout "copying layout src/.fossil-custom files (if target missing or uncustomised)"
-            set resultdict [punkcheck::install $layout_path/.fossil-custom $projectdir/.fossil-custom -createdir 1 -createempty 1 -punkcheck_folder $projectdir -installer project.new -antiglob_dir_core $override_antiglob_dir_core -overwrite SYNCED-TARGETS]
+            set resultdict [punkcheck::install $layout_path/.fossil-custom $projectdir/.fossil-custom -createdir 1 -createempty 1 -punkcheck_folder $projectdir -installer project.new -exclude-dirsegments_core $override_exclude_dirsegments_core -overwrite SYNCED-TARGETS]
             puts stdout [punkcheck::summarize_install_resultdict $resultdict]
         } else {
             puts stdout "no .fossil-custom in source template - update not required"
@@ -424,7 +424,7 @@ namespace eval punk::mix::commandset::project {
 
         if {[file exists $layout_path/.fossil-settings]} {
             puts stdout "copying layout src/.fossil-settings files (if target missing or uncustomised)"
-            set resultdict [punkcheck::install $layout_path/.fossil-settings $projectdir/.fossil-settings -createdir 1 -createempty 1 -punkcheck_folder $projectdir -installer project.new -antiglob_dir_core $override_antiglob_dir_core -overwrite SYNCED-TARGETS]
+            set resultdict [punkcheck::install $layout_path/.fossil-settings $projectdir/.fossil-settings -createdir 1 -createempty 1 -punkcheck_folder $projectdir -installer project.new -exclude-dirsegments_core $override_exclude_dirsegments_core -overwrite SYNCED-TARGETS]
             puts stdout [punkcheck::summarize_install_resultdict $resultdict]
         } else {
             puts stdout "no .fossil-settings in source template - update not required"
@@ -470,8 +470,8 @@ namespace eval punk::mix::commandset::project {
         if {[file exists $projectdir/src/modules]} {
             foreach m $opt_modules {
                 #check if mod-ver.tm file or #modpod-mod-ver folder exist
-                set tmfile $projectdir/src/modules/$m-[punk::mix::util::magic_tm_version].tm
+                set tmfile $projectdir/src/modules/$m-[punk::mix::util::tm_version_magic].tm
-                set podfile $projectdir/src/modules/#modpod-$m-[punk::mix::util::magic_tm_version]/$m-[punk::mix::util::magic_tm_version].tm
+                set podfile $projectdir/src/modules/#modpod-$m-[punk::mix::util::tm_version_magic]/$m-[punk::mix::util::tm_version_magic].tm
 
                 set has_tm  [file exists $tmfile]
                 set has_pod [file exists $podfile]

src/project_layouts/custom/_project/punk.project-0.1/src/bootsupport/modules/punk/mix/util-0.1.0.tm

@@ -367,7 +367,16 @@ namespace eval punk::mix::util {
     }
     #todo - semver conversion/validation for other systems?
 
-    proc magic_tm_version {} {
+    proc tm_version_magic {} {
+        #maintenance instruction:
+        #  This specific 12 character nnnnnn.0a1.0 version number is used in text replacement when installing modules.
+        #  It should not be modified without also considering the impact on existing projects based on punkshell that may have used this version number in their own code or documentation as a placeholder for tm version handling.
+        #  A copy of this is also present in punk::lib to aid in dependency management.
+        #  These 2 copies should be kept in sync.
+        #  Essentially this should probably never be changed - and instead deprecated if we want to stop using it and replaced with a different mechanism/placeholder.
+
+        #tm versioning is not quite the same as semantic versioning - but similarly we should see extremely high major version numbers rarely if ever in real use,
+        #even over decades of development.
         set magicbase 999999  ;#deliberately large so given load-preference when testing!
         #we split the literal to avoid the literal appearing here - reduce risk of accidentally converting to a release version
         return ${magicbase}.0a1.0

src/project_layouts/custom/_project/punk.project-0.1/src/bootsupport/modules/punk/mod-0.1.1.tm

@@ -0,0 +1,158 @@
+#punkapps app manager
+# deck cli
+
+namespace eval punk::mod::cli {
+    namespace export help list run
+    namespace ensemble create
+
+   # namespace ensemble configure [namespace current] -unknown punk::mod::cli::_unknown
+    if 0 {
+    proc _unknown {ns args} {
+        puts stderr "punk::mod::cli::_unknown '$ns' '$args'"
+        puts stderr "punk::mod::cli::help $args"
+        puts stderr "arglen:[llength $args]"
+        punk::mod::cli::help {*}$args
+    }
+    }
+
+    #cli must have _init method - usually used to load commandsets lazily
+    #
+    variable initialised 0
+    proc _init {args} {
+        variable initialised
+        if {$initialised} {
+            return
+        }
+        #...
+        set initialised 1
+    }
+
+    proc help {args} {
+        set basehelp [punk::mix::base help {*}$args]
+        #namespace export
+        return $basehelp
+    }
+    proc getraw {appname} {
+        set app_folders [punk::config::configure running apps]
+        #todo search each app folder
+        set bases [::list]
+        set versions [::list]
+        set mains [::list]
+        set appinfo [::list bases {} mains {} versions {}]
+
+        foreach containerfolder $app_folders {
+            lappend bases $containerfolder
+            if {[file exists $containerfolder]} {
+                if {[file exists $containerfolder/$appname/main.tcl]} {
+                    #exact match - only return info for the exact one specified
+                    set namematches $appname
+                    set parts [split $appname -]
+                } else {
+                    set namematches [glob -nocomplain -dir $containerfolder -type d -tail ${appname}-*]
+                    set namematches [lsort $namematches] ;#todo - -ascii? -dictionary? natsort?
+                }
+                foreach nm $namematches {
+                    set mainfile $containerfolder/$nm/main.tcl
+                    set parts [split $nm -]
+                    if {[llength $parts] == 1} {
+                        set ver ""
+                    } else {
+                        set ver [lindex $parts end]
+                    }
+                    if {$ver ni $versions} {
+                        lappend versions $ver
+                        lappend mains $ver $mainfile
+                    } else {
+                        puts stderr "punk::apps::app version '$ver' of app '$appname' already encountered at $mainfile. (will use earliest encountered in running-config apps and ignore others of same version)"
+                    }
+                }
+            } else {
+                puts stderr "punk::apps::app missing apps_folder:'$containerfolder' Ensure apps_folder is set in punk::config"
+            }
+        }
+        dict set appinfo versions $versions
+        #todo - natsort!
+        set sorted_versions [lsort $versions]
+        set latest [lindex $sorted_versions 0]
+        if {$latest eq "" && [llength $sorted_versions] > 1} {
+            set latest [lindex $sorted_versions 1]
+        }
+        dict set appinfo latest $latest
+
+        dict set appinfo bases $bases
+        dict set appinfo mains $mains
+        return $appinfo
+    }
+
+    proc list {{glob *}} {
+        set apps_folder [punk::config::configure running apps]
+        if {[file exists $apps_folder]} {
+            if {[file exists $apps_folder/$glob]} {
+                #tailcall source $apps_folder/$glob/main.tcl
+                return $glob
+            }
+            set apps [glob -nocomplain -dir $apps_folder -type d -tail $glob]
+            if {[llength $apps] == 0} {
+                if {[string first * $glob] <0 && [string first ? $glob] <0} {
+                    #no glob chars supplied - only launch if exact match for name part
+                    set namematches [glob -nocomplain -dir $apps_folder -type d -tail ${glob}-*]
+                    set namematches [lsort $namematches] ;#todo - -ascii? -dictionary? natsort?
+                    if {[llength $namematches] > 0} {
+                        set latest [lindex $namematches end]
+                        lassign $latest nm ver
+                        #tailcall source $apps_folder/$latest/main.tcl
+                    }
+                }
+            }
+            return $apps
+        }
+    }
+
+    #todo - way to launch as separate process
+    # solo-opts only before appname - args following appname are passed to the app
+    proc run {args} {
+        set nameposn [lsearch -not $args -*]
+        if {$nameposn < 0} {
+            error "punkapp::run unable to determine application name"
+        }
+        set appname [lindex $args $nameposn]
+        set controlargs [lrange $args 0 $nameposn-1]
+        set appargs [lrange $args $nameposn+1 end]
+
+        set appinfo [punk::mod::cli::getraw $appname]
+        if {[llength [dict get $appinfo versions]]} {
+            set ver [dict get $appinfo latest]
+            puts stdout "info: $appinfo"
+            set ::argc [llength $appargs]
+            set ::argv $appargs
+            source [dict get $appinfo mains $ver]
+            if {"-hideconsole" in $controlargs} {
+                puts stderr "attempting console hide"
+                #todo - something better - a callback when window mapped?
+                after 500 {::punkapp::hide_console}
+            }
+            return $appinfo
+        } else {
+            error "punk::mod::cli unable to run '$appname'. main.tcl not found in [dict get $appinfo bases]"
+        }
+    }
+}
+
+namespace eval punk::mod::cli {
+    proc _cli {args} {
+        #don't use tailcall - base uses info level to determine caller
+        ::punk::mix::base::_cli {*}$args
+    }
+    variable default_command help
+    package require punk::mix::base
+    package require punk::overlay
+    punk::overlay::custom_from_base [namespace current] ::punk::mix::base
+}
+
+package provide punk::mod [namespace eval punk::mod {
+    variable version
+    set version 0.1.1
+}]
+
+
+

src/project_layouts/custom/_project/punk.project-0.1/src/bootsupport/modules/punk/nav/fs-0.1.0.tm

@@ -847,7 +847,7 @@ tcl::namespace::eval punk::nav::fs {
 
               Regardless of whether -nonportable is supplied or not, some characters are not
               suitable for windows or most other platforms and will be rejected with an error.
-              An example of this is the null character (\0)."
+              An example of this is the null character (\\0)."
         @values -min 1 -max -1 -type string
         path -type string -multiple 1 -help\
             "Path(s) to create. Can be absolute or relative.

src/project_layouts/custom/_project/punk.project-0.1/src/bootsupport/modules/punk/ns-0.1.0.tm

@@ -721,6 +721,22 @@ tcl::namespace::eval punk::ns {
     }
 
 
+    punk::args::define {
+        @id     -id ::punk::ns::nstree_list
+        @cmd    -name punk::ns::nstree_list\
+            -summary\
+            ""\
+            -help\
+            ""
+        @leaders
+            location -type path -optional 0
+        @opts
+            -subnslist -type list -default {} -help\
+            ""
+            -allbelow -type boolean -default 1 -help\
+            ""
+        @values -min 0 -max 0
+    }
     #important: add tests for tricky cases - e.g punk::m**::util  vs punk::m*::util vs punk::m*::**::util  - these should all be able to return different results depending on namespace structure.
     #e.g punk::m**::util will return punk::mix::util  but punk::m*::**::util will not because punk::mix::util is too short to match. Both will return deeper matches such as: punk::mix::commandset::repo::util
     proc nstree_list {location args} {
@@ -775,13 +791,8 @@ tcl::namespace::eval punk::ns {
         #set parent [nsprefix $ns_absolute]
         #set tail [nstail $ns_absolute]
 
-        #jjj
         #set allchildren [lsort [nseval $base [list ::namespace children]]]
-        #set allchildren [lsort [tcl::namespace::eval $base [list ::namespace children]]]
-        set allchildren [lsort [nseval $base [list ::namespace children]]]
 
-        #puts "->base:$base tailparts:$tailparts allchildren: $allchildren"
-        #puts "->base:$base tailparts:$tailparts childcount: [llength $allchildren]"
 
         #** only significant when it is the trailing part of a segment eg ::**::xxx  ::a**::xxx
         if {[llength $tailparts]} {
@@ -790,6 +801,7 @@ tcl::namespace::eval punk::ns {
                 set nslist [nstree_list $base -subnslist {} -allbelow 1]
             } elseif {[regexp {[*]{2}$} $nextglob]} {
                 set nslist [list]
+                set allchildren [lsort [nseval $base [list ::namespace children]]]
                 lappend nsmatches {*}[lsearch -all -inline -glob $allchildren [nsjoin ${base} $nextglob]]
                 foreach ch $nsmatches {
                     lappend nslist $ch
@@ -799,6 +811,7 @@ tcl::namespace::eval punk::ns {
             } else {
                 #lsearch with -glob ok even if nextglob has no globchars (no discernable speed diff, and earlier parts may have globchars anyway)
                 set nslist [list]
+                set allchildren [lsort [nseval $base [list ::namespace children]]]
                 lappend nsmatches {*}[lsearch -all -inline -glob $allchildren [nsjoin ${base} $nextglob]]
                 if {[llength $tailparts] >1 || $allbelow} {
                     foreach ch $nsmatches {
@@ -812,6 +825,7 @@ tcl::namespace::eval punk::ns {
             }
         } else {
             #puts "nstree_list: no tailparts base:$base"
+            set allchildren [lsort [nseval $base [list ::namespace children]]]
             if {$allbelow} {
                 set nsmatches $allchildren
                 set nslist [list]
@@ -2134,8 +2148,8 @@ y" {return quirkykeyscript}
         tcl::dict::set tinfo($target) procoffset 0
         tcl::dict::set tinfo($target) level [expr {[::tcl::info::level]+1}]
         tcl::dict::set tinfo($target) subcmds 0
-        puts "enter: $target -- $args"
+        puts stderr "enter: $target -- $args"
-        puts "frame-2: [::tcl::info::frame -2]"
+        #puts stderr "frame-2: [::tcl::info::frame -2]"
 
         set _cmdtrace_disabled false
     }
@@ -2481,7 +2495,7 @@ y" {return quirkykeyscript}
                         set line $traceline
                         dict set linedict $target eval_base $traceline
                         dict set linedict $target eval_offset 1
-                        puts "  step type: proc traceline:$traceline ** $args"
+                        puts "  step type: proc traceline:$traceline ** $args\x1b\[m"
                         #puts "** $callinfo"
                         if {[dict exists $callinfo cmd]} {
                             #set cmd [string trim [dict get $callinfo cmd]] ;#raw 'unexpanded' script from the stack frame
@@ -2504,8 +2518,8 @@ y" {return quirkykeyscript}
                         set eval_base   [dict get $linedict $target eval_base]
                         set eval_offset [dict get $linedict $target eval_offset]
                         set line [expr {$eval_base + ($eval_offset-1) + ($traceline-1)}]
-                        puts "stack-- $callinfo"
+                        #puts "stack-- $callinfo"
-                        puts "  step type: eval traceline: $traceline -- "
+                        puts stderr "  step type: eval traceline: $traceline -- "
                         if {[dict exists $callinfo cmd]} {
                             #set cmd [string trim [dict get $callinfo cmd]]
                             set cmdlist [lindex $args 0]
@@ -2627,6 +2641,8 @@ y" {return quirkykeyscript}
         }]
     }
     proc cmdtrace {args} {
+        #review - displaying argument values has to be done carefully. Small values are ok, but large lists or dicts can be overwhelming.
+        #Potentially we could apply some heuristics to truncate or summarise them.
         package require dictn ;#convenience to allow dictn::incr d {key subkey}
         variable tinfo
         array unset tinfo
@@ -2676,7 +2692,7 @@ y" {return quirkykeyscript}
         #if the target command has a leading colon (e.g expr alternative :) we can't put a trace directly on a fully qualified name with a triple colon such as :::
         #we will need to evaluate in the namespace
         foreach {tgt_cmd ns nscmd} $resolved_targets {
-            puts "tracing target: $tgt_cmd whilst running: $origin $arglist"
+            puts stderr "tracing target: $tgt_cmd whilst running: $origin $arglist"
 
             #::tcl::namespace::eval $ns [list ::trace add execution $nscmd enter       [list ::punk::ns::_cmdtrace_enter       ::punk::ns::linedict $tgt_cmd]]
             #::tcl::namespace::eval $ns [list ::trace add execution $nscmd enterstep   [list ::punk::ns::_cmdtrace_enterstep   ::punk::ns::linedict $tgt_cmd]]

src/project_layouts/custom/_project/punk.project-0.1/src/bootsupport/modules/punk/overlay-0.1.1.tm

@@ -0,0 +1,192 @@
+
+
+package require punk::mix::util
+package require punk::args
+
+tcl::namespace::eval ::punk::overlay {
+    #based *loosely* on: wiki.tcl-lang.org/page/ensemble+extend
+    # extend an ensemble-like routine with the routines in some namespace
+    #
+    # e.g custom_from_base ::punk::mix::cli ::punk::mix::base
+    #
+    proc custom_from_base {routine base} {
+        if {![tcl::string::match ::* $routine]} {
+            set resolved [uplevel 1 [list ::tcl::namespace::which $routine]]
+            if {$resolved eq {}} {
+                error [list {no such routine} $routine]
+            }
+            set routine $resolved
+        }
+        set routinens [tcl::namespace::qualifiers $routine]
+        if {$routinens eq {::}} {
+            set routinens {}
+        }
+        set routinetail [tcl::namespace::tail $routine]
+
+        if {![tcl::string::match ::* $base]} {
+            set base [uplevel 1 [
+                list [tcl::namespace::which namespace] current]]::$base
+        }
+
+        if {![tcl::namespace::exists $base]} {
+            error [list {no such namespace} $base]
+        }
+
+        set base [tcl::namespace::eval $base [
+            list [tcl::namespace::which namespace] current]]
+
+
+        #while 1 {
+        #    set renamed ${routinens}::${routinetail}_[info cmdcount]
+        #    if {[namespace which $renamed] eq {}} break
+        #}
+
+        tcl::namespace::eval $routine [
+            ::list tcl::namespace::ensemble configure  $routine -unknown [
+                ::list ::apply {{base ensemble subcommand args} {
+                    ::list ${base}::_redirected $ensemble $subcommand 
+                }} $base
+            ]
+        ]
+
+        punk::mix::util::namespace_import_pattern_to_namespace_noclobber ::punk::mix::util::* ${routine}::util
+        #namespace eval ${routine}::util {
+            #::namespace import ::punk::mix::util::*
+        #}
+        punk::mix::util::namespace_import_pattern_to_namespace_noclobber ${base}::lib::* ${routine}::lib
+        #namespace eval ${routine}::lib [string map [list <base> $base] {
+        #    ::namespace import <base>::lib::*
+        #}]
+
+        tcl::namespace::eval ${routine}::lib [tcl::string::map [list <base> $base <routine> $routine] {
+            if {[tcl::namespace::exists <base>::lib]} {
+                ::set current_paths [tcl::namespace::path]
+                if {"<routine>" ni $current_paths} {
+                    ::lappend current_paths <routine>
+                }
+                tcl::namespace::path $current_paths
+            }
+        }]
+
+        tcl::namespace::eval $routine {
+            ::set exportlist [::list]
+            ::foreach cmd [tcl::info::commands [tcl::namespace::current]::*] {
+                ::set c [tcl::namespace::tail $cmd]
+                if {![tcl::string::match _* $c]} {
+                    ::lappend exportlist $c
+                }
+            }
+            tcl::namespace::export {*}$exportlist
+        }
+
+        return $routine
+    }
+    punk::args::define {
+        @id     -id     ::punk::overlay::import_commandset
+        @cmd    -name     punk::overlay::import_commandset\
+            -summary\
+            "Import commands into caller's namespace with optional prefix and separator."\
+            -help\
+            "Import commands that have been exported by another namespace into the caller's
+            namespace. Usually a prefix and optionally a separator should be used.
+            This is part of the punk::mix CLI commandset infrastructure - design in flux.
+            Todo - .toml configuration files for defining CLI configurations."
+        @values
+        prefix          -type string
+        separator       -type string -help\
+            "A string, usually punctuation, to separate the prefix and the command name
+              of the final imported command. The value \"::\" is disallowed in this context."
+        cmdnamespace    -type string -help\
+            "Namespace from which to import commands. Commands are those that have been exported."
+    }
+    #load *exported* commands from cmdnamespace into caller's namespace - prefixing each command with $prefix
+    #Note: commandset may be imported by different CLIs with different bases *at the same time*
+    #so we don't make commands from the cli or its base available automatically (will generally require fully-qualified commands to use code from cli/base) 
+    #we do load punk::mix::util::*  into the util subnamespace even though the commandset might not be loaded in a cli using punk::mix::base  i.e punk::mix::util is a common dependency for CLIs.
+    #commandsets designed to be used with a specific cli/base may choose to do their own import e.g with util::namespace_import_pattern_to_namespace_noclobber and/or set namespace path if they 
+    #want the convenience of using lib:xxx with commands coming from those packages.
+    #This won't stop the commandset being used with other cli/bases unless the import is done by looking up the callers namespace.
+    #The basic principle is that the commandset is loaded into the caller(s) with a prefix 
+    #- but commandsets should explicitly package require if they have any backwards dependencies on cli/base (which they may or may not be loaded into)
+    proc import_commandset {prefix separator cmdnamespace} {
+        set bad_seps [list "::"]
+        if {$separator in $bad_seps} {
+            error "import_commandset invalid separator '$separator'"
+        }
+        if {$prefix in $bad_seps} {
+            error "import_commandset invalid prefix '$prefix'"
+        }
+        if {"$prefix$separator" in $bad_seps} {
+            error "import_commandset invalid prefix/separator combination '$prefix$separator'"
+        }
+        if {"[string index $prefix end][string index $separator 0]" in $bad_seps} {
+            error "import_commandset invalid prefix/separator combination '$prefix$separator'"
+        }
+        #review - do we allow prefixes/separators such as a::b?
+
+        #namespace may or may not be a package
+        # allow with or without leading ::
+        if {[tcl::string::range $cmdnamespace 0 1] eq "::"} {
+            set cmdpackage [tcl::string::range $cmdnamespace 2 end]
+        } else {
+            set cmdpackage $cmdnamespace
+            set cmdnamespace ::$cmdnamespace
+        }
+
+        if {![tcl::namespace::exists $cmdnamespace]} {
+            #only do package require if the namespace not already present
+            catch {package require $cmdpackage} pkg_load_info 
+            #recheck
+            if {![tcl::namespace::exists $cmdnamespace]} {
+                set prov [package provide $cmdpackage]
+                if {[tcl::string::length $prov]} {
+                    set provinfo "(package $cmdpackage is present with version $prov)"
+                } else {
+                    set provinfo "(package $cmdpackage not present)"
+                }
+                error "punk::overlay::import_commandset supplied namespace '$cmdnamespace' doesn't exist. $provinfo  Pkg_load_result: $pkg_load_info  Usage: import_commandset prefix separator namespace"
+            }
+        }
+     
+        punk::mix::util::namespace_import_pattern_to_namespace_noclobber ::punk::mix::util::* ${cmdnamespace}::util
+
+        #let child namespace 'lib' resolve parent namespace and thus util::xxx
+        tcl::namespace::eval ${cmdnamespace}::lib [tcl::string::map [list <cmdns> $cmdnamespace] {
+            ::set nspaths [tcl::namespace::path]
+            if {"<cmdns>" ni $nspaths} {
+                ::lappend nspaths <cmdns>
+            }
+            tcl::namespace::path $nspaths
+        }]
+
+        set imported_commands [list]
+        set imported_tails [list]
+        set nscaller [uplevel 1 [list tcl::namespace::current]]
+        if {[catch {
+            #review - noclobber?
+            tcl::namespace::eval ${nscaller}::temp_import [list tcl::namespace::import ${cmdnamespace}::*]
+            foreach cmd [tcl::info::commands ${nscaller}::temp_import::*] {
+                set cmdtail [tcl::namespace::tail $cmd]
+                if {$cmdtail eq "_default"} {
+                    set import_as ${nscaller}::${prefix}
+                } else {
+                    set import_as ${nscaller}::${prefix}${separator}${cmdtail}
+                }
+                rename $cmd  $import_as
+                lappend imported_commands $import_as
+                lappend imported_tails [namespace tail $import_as]
+            }
+            #make imported commands exported so they are available to the ensemble
+            tcl::namespace::eval ${nscaller} [list namespace export {*}$imported_tails]
+        } errM]} {
+            puts stderr "Error loading commandset $prefix $separator $cmdnamespace"
+            puts stderr "err: $errM"
+        }
+        return $imported_commands
+    }
+}
+
+package provide punk::overlay [tcl::namespace::eval punk::overlay {
+    variable version
+    set version 0.1.1
+}]

src/project_layouts/custom/_project/punk.project-0.1/src/bootsupport/modules/punk/path-0.1.0.tm

@@ -565,10 +565,45 @@ namespace eval punk::path {
                 end]]
     }
 
+
+    ## for comparison
+    #proc nsglob_as_re {glob} {
+    #    #any segment that is not just * must match exactly one segment in the path
+    #    set pats [list]
+    #    foreach seg [nsparts_cached $glob] {
+    #        switch -exact -- $seg {
+    #            "" {
+    #                lappend pats ""
+    #            }
+    #            * {
+    #                #review - ::g*t  will not find ::got:it   (won't match single inner colon) - this should be fixed
+    #                #lappend pats {[^:]*}
+    #                #negative lookahead
+    #                #any number of chars not followed by ::, followed by any number of non :
+    #                lappend pats {(?:.(?!::))*[^:]*}
+    #            }
+    #            ** {
+    #                lappend pats {.*}
+    #            }
+    #            default {
+    #                set seg [string map {. [.]} $seg]
+    #                if {[regexp {[*?]} $seg]} {
+    #                    #set pat [string map [list ** {.*} * {[^:]*} ? {[^:]}] $seg]
+    #                    set pat [string map [list ** {.*} * {(?:.(?!::))*[^:]*} ? {[^:]}] $seg]
+    #                    lappend pats "$pat"
+    #                } else {
+    #                    lappend pats "$seg"
+    #                }
+    #            }
+    #        }
+    #    }
+    #    return  "^[join $pats ::]\$"
+    #}
     proc pathglob_as_re {pathglob} {
         #*** !doctools
         #[call [fun pathglob_as_re] [arg pathglob]]
         #[para] Returns a regular expression for matching a path to a glob pattern which can contain glob chars *|? in any segment of the path structure
+        #[para] Does not support square bracket globs or character classes.
         #[para] ** matches any number of subdirectories.
         #[para] e.g /etc/**/*.txt will match any .txt files at any depth below /etc  (except directly within /etc itself)
         #[para] e.g /etc/**.txt will match any .txt files at any depth below /etc
@@ -589,7 +624,7 @@ namespace eval punk::path {
                 *   {lappend pats {[^/]*}}
                 **  {lappend pats {.*}}
                 default {
-                    set seg [string map [list {^ {\^} $ {\$} [} {\[} ( {\(} \{ \\\{ \\ {\\}] $seg] ;#treat regex characters in the input as literals
+                    set seg [string map [list ^ {\^} $ {\$} \[ {\[} \] {\]} ( {\(} ) {\)} \{ \\\{ \\ {\\}] $seg] ;#treat regex characters (or tcl glob square bracket chars) in the input as literals
                     #set seg [string map [list  . {[.]}] $seg]
                     set seg [string map {. [.]} $seg]
                     if {[regexp {[*?]} $seg]} {
@@ -603,6 +638,52 @@ namespace eval punk::path {
         }
         return  "^[join $pats /]\$"
     }
+
+    punk::args::define {
+        @id     -id     ::punk::path::globmatchpath
+        @cmd    -name     punk::path::globmatchpath\
+            -summary\
+            "Match path to *|**|? glob patterns"\
+            -help\
+            "Return a boolean indicating whether the path matches the specialised glob pattern.
+            A pattern such as /usr/*/bin will match any path that has /usr as the first segment and bin as the third segment,
+            with any single segment in between.
+            A pattern such as /usr/**/bin will match any path that has /usr as the first segment and bin as the last segment,
+            with 1 or more segments in between (so it will not match /usr/bin).
+            A pattern such as /usr/** will match any path that has /usr as the first segment, with 1 or more segments
+            following (so it will not match /usr itself).
+            A pattern such as **/*.txt will match any path that ends with .txt, with 1 or more leading segments 
+            (so it will not match test.txt or .txt).
+            A pattern such as ** will match any path.
+            The glob characters * and ? are the only special characters in the pathglob syntax.
+            - they are treated as glob characters regardless of where they appear in the pathglob string.
+            Note that this is different from other Tcl glob contexts where square brackets can be used.
+            The pathglob syntax treats other characters, including square brackets as literals.
+            For example, the pattern /usr/te?t will match /usr/test and /usr/text but not /usr/texxt, and the pattern /usr/te*t
+            will match /usr/test, /usr/teat, and /usr/teeeet but not /usr/te/t.
+            The pathglob syntax does not support escaping of glob characters - any glob characters in the pathglob are treated
+            as glob characters. For example, the pattern /usr/* will match any path that has /usr as the first segment and any
+            single segment as the second segment, but there is no way to specify a pattern that matches any path that has /usr
+            as the first segment and a literal * as the second segment.
+            Caller must ensure that file separator is forward slash. (e.g use file normalize on windows)
+
+            options:
+                -nocase 0|1  (default 0 - case sensitive)
+             If -nocase is not supplied - default to case sensitive *except for driveletter*
+             ie - the driveletter alone in paths such as c:/etc will still be case insensitive. (ie c:/ETC/* will match C:/ETC/blah but not C:/etc/blah)
+             Explicitly specifying -nocase 0 will require the entire case to match including the driveletter.
+            "
+        @leaders
+            pathglob -type string -help "glob pattern to match path against. See [fun pathglob_as_re] for syntax of glob patterns"
+            path -type string -help "path to match against glob pattern"
+        @opts
+            -nocase -type boolean -default 0 -help\
+                "case insensitive matching (default false - case sensitive)
+                - except for driveletter on windows which is always case insensitive
+                unless -nocase 0 is explicitly specified"
+        @values -min 0 -max 0 
+    }
+    # -id
     proc globmatchpath {pathglob path args} {
         #*** !doctools
         #[call [fun globmatchpath] [arg pathglob] [arg path] [opt {option value...}]]
@@ -659,294 +740,576 @@ namespace eval punk::path {
         return $ismatch
     }
     punk::args::define {
-        @id     -id     ::punk::path::subfolders
+        @id     -id     ::punk::path::subfolders1
-        @cmd    -name     punk::path::subfolders\
+        @cmd    -name     punk::path::subfolders1\
             -summary\
-            "Listing of directories within supplied path."\
+            "Listing of directories below supplied path."\
             -help\
             "List of folders below path.
             The resulting list is unsorted."
         @opts
         -recursive -type none -help\
             ""
+        -exclude-paths -type list -default {} -help\
+            "list of path patterns to exclude from results.
+            May include * and ** path segments e.g /usr/**
+            A single /*/ will match any single segment in the path, and a single /**/ will match any number of segments in the path.
+
+            e.g to exclude any path with _aside as a segment in the middle: -exclude-paths **/_aside/**
+            i.e this would exclude /usr/_aside/etc and /usr/x/_aside/etc but not /usr/x/_aside or _aside/etc
+
+            To exclude all paths with _aside as a segment anywhere: -exclude-paths { **/_aside/** **/_aside _aside/**}
+            "
         #todo -depth
         @values -min 0 -max 1
         path -type directory -optional 1 -help\
-            "Path of folder. If not supplied current directory is used."
+            "Path of folder. If not supplied current directory is used.
-    }
+            This may be a relative or absolute path. Relative paths are treated as relative to current directory.
-    proc subfolders {args} {
+            When using relative paths - the result will also be relative paths with the same relative prefix.
-        set argd [punk::args::parse $args withid ::punk::path::subfolders]
+            (e.g if path is ../test - the results will be ../test/subfolder1 ../test/subfolder2 etc)
+            Patterns in -exclude-paths are matched against the resulting paths
+            (so should be written to match the same relative prefix if path is relative)"
+    }
+
+    proc subfolders1 {args} {
+        #NOTE - this algorithm based on omit_only_patterns and prune_base_patterns was suggested by a 2026 AI model - it is apparent to this programmer that it is inadequate for the purpose.
+        #e.g consider subfolders1 -recursion -exclude {**/vfs/** **/src/**}
+        #This can still return something like c:/repo/etc/src/vfs - which should be excluded by the pattern **/src/**
+        #todo - review and fix properly.
+        set argd [punk::args::parse $args withid ::punk::path::subfolders1]
         lassign [dict values $argd] leaders opts values received
         set do_recursion    [dict exists $received -recursive]
+        set exclude_paths   [dict get $opts -exclude-paths]
+        if {"**" in $exclude_paths} {
+            #if ** is in exclude_paths - then we can skip all glob matching and just return empty list
+            #This is likely user error - so we'll be loud about it for now but will still return empty list rather than erroring.
+            #If user code is building exclude_paths dynamically - they can check for this case themselves and avoid the call to subfolders1 to suppress this message.
+            puts stderr "punk::path::subfolders1 Warning - exclude_paths contains '**' - all paths will be excluded"
+            return [list]
+        }
         if {[dict exists $received path]} {
             set path [dict get $values path]
         } else {
             set path [pwd]
         }
-        set folders [glob -nocomplain -directory $path -types d *]
+
+        set all_subfolders [glob -nocomplain -directory $path -types d *]
+
+
+        #example of expected exclude_paths pattern behaviour when recursion is enabled:
+        # **/dirname    -> omit /x/y/dirname, but still visit /x/y/dirname/*
+
+        # **/dirname/*   -> include /x/y/dirname and /x/y/dirname/a/b but omit directories that are a single level below /x/y/dirname such as /x/y/dirname/a
+
+        #c:/** - would exclude all subfolders below c: but not c: itself
+
+        # **/test/** - would exclude any path with test as a segment and all its subfolders
+        #- but not paths with test as a segment that is the final segment
+
+        set folders [list]
+        set recurse_subdirs [list]
+
+        foreach f $all_subfolders {
+            set include_in_results 1
+            set allow_recurse 1
+            foreach pat $exclude_paths {
+                set pat_parts [file split $pat] ;#note file split c:/test gives {c:/ test}  but file split **/test gives {** test}
+                #also note that file split on windows treats forward slashes and backslashes the same.
+                #by using file split, we gain some flexibility in syntax of paths and patterns,
+                #but lose the ability to use backslashes as escapes to allow literal glob characters in path segments.
+                #This is almost always a non-issue on windows since * and ? are not valid in path segments there, and is rarely an issue on unix even though
+                # * and ? are technically valid in path segments, but it is inadvisable there anyway for compatibility with shells etc.
+                if {[llength $pat_parts] >= 2 && [lindex $pat_parts end] eq "**"} {
+                    set base_pat [file join {*}[lrange $pat_parts 0 end-1]]
+                    if {[globmatchpath $pat $f]} {
+                        set include_in_results 0
+                        set allow_recurse 0
+                    } elseif {[globmatchpath $base_pat $f]} {
+                        set allow_recurse 0
+                    }
+                } elseif {[globmatchpath $pat $f]} {
+                    set include_in_results 0
+                }
+                if {!$include_in_results && !$allow_recurse} {
+                    break
+                }
+            }
+            if {$include_in_results} {
+                lappend folders $f
+            }
+            if {$allow_recurse} {
+                lappend recurse_subdirs $f
+            }
+        }
         if {$do_recursion} {
-            foreach subdir $folders {
+            foreach subdir $recurse_subdirs {
-                lappend folders {*}[subfolders -recursive $subdir]
+                lappend folders {*}[subfolders1 -exclude-paths $exclude_paths -recursive $subdir]
             }
         }
         return $folders
     }
 
-    #todo - treefolders with similar search caps as treefilenames
+    namespace eval subfolder_priv {
-
+        proc classify_exclude_pattern {pat} {
-    punk::args::define {
+            set parts [file split $pat]
-        @id     -id ::punk::path::treefilenames
+            if {[llength $parts] >= 2 && [lindex $parts end] eq "**"} {
-        -directory       -type directory  -help\
+                set boundary_pat [file join {*}[lrange $parts 0 end-1]]
-            "folder in which to begin recursive scan for files."
+                return [dict create \
-        -call-depth-internal    -default 0 -type integer
+                    pattern $pat \
-        -sort -type any -default natural -choices {none ascii dictionary natural}
+                    kind subtree \
-        -antiglob_paths         -default {} -help\
+                    boundary_pat $boundary_pat \
-            "list of path patterns to exclude
+                    descend_pat $pat]
-              may include * and ** path segments e.g
+            }
-              /usr/**       (exlude subfolders based at /usr but not
+            if {[llength $parts] >= 2 && [lindex $parts end] eq "*"} {
-                             files within /usr itself)
+                return [dict create \
-              **/_aside     (exlude  files where _aside is last segment)
+                    pattern $pat \
-              **/_aside/*   (exclude folders one below an _aside folder)
+                    kind child_only \
-              **/_aside/**  (exclude all folders with _aside as a segment)"
+                    match_pat $pat]
-        -antiglob_files         -default {}
+            }
-        @values -min 0 -max -1 -optional 1 -type string
+            return [dict create \
-        tailglobs -default * -multiple 1   -help\
+                pattern $pat \
-            "Patterns to match against filename portion (last segment) of each file path
+                kind exact \
-              within the directory tree being searched."
+                match_pat $pat]
         }
 
-    #todo - implement treefiles which acts like dirfiles but allows path globbing in the same way as punk::ns::ns/
+        proc compile_exclude_rules {exclude_paths} {
-    #then review if treefiles can replace dirfiles or if both should exist (dirfiles can have literal glob chars in path segments - but that is a rare usecase)
+            set rules [list]
-    proc treefilenames {args} {
+            foreach pat $exclude_paths {
-        #*** !doctools
+                lappend rules [classify_exclude_pattern $pat]
-        #[call [fun treefilenames]  [opt {option value...}] [opt {globpattern...}]]
+            }
-        #[para]basic (glob based) list of filenames matching each pattern in tailglobs - recursive
+            return $rules
-        #[para] options:
+        }
-        #[para] [opt -dir] <path>
-        #[para]     defaults to [lb]pwd[rb] - base path for tree to search
-        #[para] [opt -antiglob_paths] <list>
-        #[para]     list of path patterns to exclude - may include * and ** path segments e.g /usr/**
-        #[para]no natsorting - so order is dependent on filesystem
 
-        set argd [punk::args::parse $args withid ::punk::path::treefilenames]
+        proc match_rule_at_node {rule path} {
-        lassign [dict values $argd] leaders opts values received
+            set kind [dict get $rule kind]
-        set tailglobs [dict get $values tailglobs]
+            switch -- $kind {
-        # -- --- --- --- --- --- ---
+                exact - child_only {
-        set opt_sort           [dict get $opts -sort]
+                    if {[::punk::path::globmatchpath [dict get $rule match_pat] $path]} {
-        set opt_antiglob_paths [dict get $opts -antiglob_paths]
+                        return [dict create include_current 0 recurse_below 1 child_rules [list $rule]]
-        set opt_antiglob_files [dict get $opts -antiglob_files]
+                    }
-        set CALLDEPTH [dict get $opts -call-depth-internal]
+                    return [dict create include_current 1 recurse_below 1 child_rules [list $rule]]
-        # -- --- --- --- --- --- ---
+                }
-        # -- --- --- --- --- --- ---
+                subtree {
+                    set descend_pat  [dict get $rule descend_pat]
+                    set boundary_pat [dict get $rule boundary_pat]
+                    if {[::punk::path::globmatchpath $descend_pat $path]} {
+                        return [dict create include_current 0 recurse_below 0 child_rules [list]]
+                    }
+                    if {[::punk::path::globmatchpath $boundary_pat $path]} {
+                        return [dict create include_current 1 recurse_below 0 child_rules [list]]
+                    }
+                    return [dict create include_current 1 recurse_below 1 child_rules [list $rule]]
+                }
+                default {
+                    error "Unknown exclude rule kind '$kind'"
+                }
+            }
+        }
 
-        set files [list]
+        proc walk_subfolders {path rules do_recursion} {
-        if {$CALLDEPTH == 0} {
+            set all_subfolders [glob -nocomplain -directory $path -types d *]
-            if {$opt_sort eq "natural"} {
+            set folders [list]
-                package require natsort
+            foreach f $all_subfolders {
+                set include_current 1
+                set recurse_below $do_recursion
+                set child_rules [list]
+                foreach rule $rules {
+                    set outcome [match_rule_at_node $rule $f]
+                    if {![dict get $outcome include_current]} {
+                        set include_current 0
                     }
-            #set opts [dict merge $opts [list -directory $opt_dir]]
+                    if {![dict get $outcome recurse_below]} {
-            if {![dict exists $received -directory]} {
+                        set recurse_below 0
-                set opt_dir [pwd]
-            } else {
-                set opt_dir [dict get $opts -directory]
                     }
-            if {![file isdirectory $opt_dir]} {
+                    if {$do_recursion} {
+                        lappend child_rules {*}[dict get $outcome child_rules]
+                    }
+                    if {!$include_current && !$recurse_below} {
+                        break
+                    }
+                }
+                if {$include_current} {
+                    lappend folders $f
+                }
+                if {$do_recursion && $recurse_below} {
+                    lappend folders {*}[walk_subfolders $f $child_rules $do_recursion]
+                }
+            }
+            return $folders
+        }
+    }
+
+    punk::args::define {
+        @id     -id     ::punk::path::subfolders
+        @cmd    -name     punk::path::subfolders\
+            -summary\
+            "Listing of directories below supplied path."\
+            -help\
+            "List of folders below path.
+            The resulting list is unsorted.
+            "
+        @opts
+        -recursive -type none -help\
+            ""
+        -exclude-paths -type list -default {} -help\
+            "list of path patterns to exclude from results.
+            May include * and ** path segments e.g /usr/**
+            A single /*/ will match any single segment in the path, and a single /**/ will match any number of segments in the path.
+
+            e.g to exclude any path with _aside as a segment in the middle: -exclude-paths **/_aside/**
+            i.e this would exclude /usr/_aside/etc and /usr/x/_aside/etc but not /usr/x/_aside or _aside/etc
+
+            To exclude all paths with _aside as a segment anywhere: -exclude-paths { **/_aside/** **/_aside ./_aside/**}
+            "
+        #todo -depth
+        @values -min 0 -max 1
+        path -type directory -optional 1 -help\
+            "Path of base folder. If not supplied current directory is used.
+            This may be a relative or absolute path. Relative paths are treated as relative to current directory.
+            When using relative paths - the result will also be relative paths with the same relative prefix.
+            (e.g if path is ../test - the results will be ../test/subfolder1 ../test/subfolder2 etc)
+            Patterns in -exclude-paths are matched against the resulting paths
+            (so should be written to match the same relative prefix if path is relative)"
+    }
+
+    proc subfolders {args} {
+        set argd [punk::args::parse $args withid ::punk::path::subfolders]
+        lassign [dict values $argd] leaders opts values received
+        set do_recursion    [dict exists $received -recursive]
+        set exclude_paths   [dict get $opts -exclude-paths]
+        if {"**" in $exclude_paths} {
+            puts stderr "punk::path::subfolders Warning - exclude_paths contains '**' - all paths will be excluded"
             return [list]
         }
+        if {[dict exists $received path]} {
+            set path [dict get $values path]
         } else {
-            #assume/require to exist in any recursive call
+            set path [pwd]
-            set opt_dir [dict get $opts -directory]
+        }
+        set compiled_rules [subfolder_priv::compile_exclude_rules $exclude_paths]
+        return [subfolder_priv::walk_subfolders $path $compiled_rules $do_recursion]
     }
 
-        #comment out to compare timings with treefilenames_zipfs
+    namespace eval treefile_priv {
-        if {[string match //zipfs:/* $opt_dir]} {
+        proc _pattern_prefix_viable_parts {pattern_parts path_parts} {
-            return [treefilenames_zipfs {*}$args]
+            if {![llength $path_parts]} {
+                return 1
+            }
+            if {![llength $pattern_parts]} {
+                return 0
             }
 
-        set skip 0
+            set pattern_head [lindex $pattern_parts 0]
-        foreach anti $opt_antiglob_paths {
+            set path_head [lindex $path_parts 0]
-            if {[globmatchpath $anti $opt_dir]} {
+
-                set skip 1
+            if {$pattern_head eq "**"} {
+                if {[_pattern_prefix_viable_parts [lrange $pattern_parts 1 end] $path_parts]} {
+                    return 1
+                }
+                return [_pattern_prefix_viable_parts $pattern_parts [lrange $path_parts 1 end]]
+            }
+
+            if {[::punk::path::globmatchpath $pattern_head $path_head]} {
+                return [_pattern_prefix_viable_parts [lrange $pattern_parts 1 end] [lrange $path_parts 1 end]]
+            }
+            return 0
+        }
+
+        proc pattern_prefix_viable {pattern path} {
+            return [_pattern_prefix_viable_parts [file split $pattern] [file split $path]]
+        }
+
+        proc pattern_boundary {pattern} {
+            set parts [file split $pattern]
+            if {[llength $parts] >= 2 && [lindex $parts end] eq "**"} {
+                return [file join {*}[lrange $parts 0 end-1]]
+            }
+            return ""
+        }
+
+        proc directory_state {glob_paths path inherited_allbelow} {
+            if {$inherited_allbelow} {
+                return [dict create include_files 1 recurse_below 1 next_allbelow 1]
+            }
+
+            set include_files 0
+            set recurse_below 0
+            set next_allbelow 0
+
+            foreach gp $glob_paths {
+                if {[::punk::path::globmatchpath $gp $path]} {
+                    set include_files 1
+                    set recurse_below 1
+                    set next_allbelow 1
                     break
                 }
+
+                set boundary [pattern_boundary $gp]
+                if {$boundary ne "" && [::punk::path::globmatchpath $boundary $path]} {
+                    set recurse_below 1
+                    set next_allbelow 1
+                    continue
+                }
+
+                if {[pattern_prefix_viable $gp $path]} {
+                    set recurse_below 1
                 }
-        if {$skip} {
-            return [list]
             }
 
-        #todo - account for vfs where matched path could appear to be a directory but is mounted so could be a desired match?
+            return [dict create                     {*}{
-        if {[catch {glob -nocomplain -dir $opt_dir -type f {*}$tailglobs} matches]} {
+                    } include_files $include_files  {*}{
-            #we can get for example a permissions error
+                    } recurse_below $recurse_below  {*}{
-            puts stderr "treefilenames error while listing files in dir $opt_dir\n $matches"
+                    } next_allbelow $next_allbelow  {*}{
-            set dirfiles [list]
-        } else {
-            set retained [list]
-            if {[llength $opt_antiglob_files]} {
-                foreach m $matches {
-                    set skip 0
-                    set ftail [file tail $m]
-                    foreach anti $opt_antiglob_files {
-                        if {[string match $anti $ftail]} {
-                            set skip 1; break
                     }
+                ]
         }
-                    if {!$skip} {
+
-                        lappend retained $m
+        proc child_path_state {glob_paths child_path inherited_allbelow} {
+            if {$inherited_allbelow} {
+                return 1
             }
+            foreach gp $glob_paths {
+                if {[pattern_prefix_viable $gp $child_path]} {
+                    return 1
                 }
-            } else {
-                set retained $matches
             }
-            switch -- $opt_sort {
+            return 0
+        }
+
+        proc _sort_paths {paths sortmode} {
+            switch -- $sortmode {
                 ascii {
-                    set dirfiles [lsort $retained]
+                    return [lsort $paths]
                 }
                 dictionary {
-                    set dirfiles [lsort -dictionary $retained]
+                    return [lsort -dictionary $paths]
                 }
                 natural {
-                    set dirfiles [natsort::sort $retained]
+                    return [natsort::sort $paths]
                 }
                 default {
-                    set dirfiles $retained
+                    return $paths
                 }
             }
         }
 
-        lappend files  {*}$dirfiles
+        proc _path_matches_any {patterns path} {
-        if {[catch {glob -nocomplain -dir $opt_dir -type d *} dirdirs]} {
+            foreach pattern $patterns {
-            puts stderr "treefilenames error while listing subdirs in dir $opt_dir\n $dirdirs"
+                if {[::punk::path::globmatchpath $pattern $path]} {
-            set dirdirs [list]
+                    return 1
                 }
-        set okdirs [list]
+            }
-        foreach dir $dirdirs {
+            return 0
+        }
+
+        proc _tailbase_relative {tailbase path} {
+            if {$tailbase eq ""} {
+                return $path
+            }
+            return [::punk::path::relative $tailbase $path]
+        }
+
+        proc _tailbase_match_path {tailbase path} {
+            set match_path [_tailbase_relative $tailbase $path]
+            if {$match_path eq "."} {
+                return ""
+            }
+            return $match_path
+        }
+
+        proc _tailbase_relative_list {tailbase paths} {
+            if {$tailbase eq ""} {
+                return $paths
+            }
+            set relative_paths [list]
+            foreach path $paths {
+                lappend relative_paths [_tailbase_relative $tailbase $path]
+            }
+            return $relative_paths
+        }
+
+        proc _retain_files {matches exclude_files sortmode} {
+            set retained [list]
+            foreach match $matches {
                 set skip 0
-            foreach anti $opt_antiglob_paths {
+                set file_tail [file tail $match]
-                if {[globmatchpath $anti $dir]} {
+                foreach anti $exclude_files {
+                    if {[string match $anti $file_tail]} {
                         set skip 1
                         break
                     }
                 }
                 if {!$skip} {
-                lappend okdirs $dir
+                    lappend retained $match
                 }
             }
-        if {[llength $okdirs]} {
+            return [_sort_paths $retained $sortmode]
-            switch -- $opt_sort {
-                ascii {
-                    set finaldirs [lsort $okdirs]
         }
-                dictionary {
+
-                    set finaldirs [lsort -dictionary $okdirs]
+        proc _state_from_argd {argd} {
+            set opts        [dict get $argd opts]
+            set values      [dict get $argd values]
+            set received    [dict get $argd received]
+
+            if {[dict exists $received -directory]} {
+                set directory [dict get $opts -directory]
+            } else {
+                set directory [pwd]
             }
-                natural {
+
-                    set finaldirs [natsort::sort $okdirs]
+            set glob_paths [dict get $opts -include-paths]
+            if {"*" in $glob_paths} {
+                set glob_paths {*}
             }
-                default {
+
-                    set finaldirs $okdirs
+            set sortmode [dict get $opts -sort]
+            if {$sortmode eq "natural"} {
+                package require natsort
             }
+
+            return [dict create {*}{
+                    depth           0
+                    subvector       {}
+                    allbelow        0
+                }   sort            $sortmode                             {*}{
+                }   directory       $directory                            {*}{
+                }   tailbase        [dict get $opts -tailbase]            {*}{
+                }   exclude_paths   [dict get $opts -exclude-paths]       {*}{
+                }   exclude_files   [dict get $opts -exclude-files]       {*}{
+                }   glob_paths      $glob_paths                           {*}{
+                }   tailglobs       [dict get $values tailglobs]          {*}{
+                }
+            ]
+        }
+
+        proc walk_treefilenames {state} {
+            set opt_dir             [dict get $state directory]
+            set opt_tailbase        [dict get $state tailbase]
+            set depth               [dict get $state depth]
+            set subvector           [dict get $state subvector]
+            set callallbelow        [dict get $state allbelow]
+            set opt_sort            [dict get $state sort]
+            set opt_exclude_paths   [dict get $state exclude_paths]
+            set opt_exclude_files   [dict get $state exclude_files]
+            set opt_glob_paths      [dict get $state glob_paths]
+            set tailglobs           [dict get $state tailglobs]
+
+            if {![file isdirectory $opt_dir]} {
+                return [list]
             }
-            foreach dir $finaldirs {
+            if {[string match //zipfs:/* $opt_dir]} {
-                set nextopts [dict merge $opts [list -directory $dir -call-depth-internal [incr CALLDEPTH]]]
+                return [walk_treefilenames_zipfs $state]
-                lappend files {*}[treefilenames {*}$nextopts {*}$tailglobs]
             }
+            set opt_dir_match [_tailbase_match_path $opt_tailbase $opt_dir]
+            if {[_path_matches_any $opt_exclude_paths $opt_dir_match]} {
+                return [list]
             }
+
+            set files [list]
+            set dir_state [directory_state $opt_glob_paths $opt_dir_match $callallbelow]
+            if {[dict get $dir_state include_files]} {
+                if {[catch {glob -nocomplain -dir $opt_dir -type f {*}$tailglobs} matches]} {
+                    puts stderr "treefilenames error while listing files in dir $opt_dir\n $matches"
+                    set dirfiles [list]
+                } else {
+                    set dirfiles [_retain_files $matches $opt_exclude_files $opt_sort]
+                }
+                lappend files {*}[_tailbase_relative_list $opt_tailbase $dirfiles]
+            }
+
+            if {![dict get $dir_state recurse_below]} {
                 return $files
             }
-    proc treefilenames_zipfs {args} {
-        #seems to be 2 or 3 times faster than treefilenames for //zipfs:/ paths  - REVIEW
-        # is sort order the same?
-        set argd [punk::args::parse $args withid ::punk::path::treefilenames]
-        lassign [dict values $argd] leaders opts values received
-        set tailglobs [dict get $values tailglobs]
-        # -- --- --- --- --- --- ---
-        set opt_antiglob_paths [dict get $opts -antiglob_paths]
-        set opt_antiglob_files [dict get $opts -antiglob_files]
-        set opt_sort           [dict get $opts -sort]
-        set CALLDEPTH [dict get $opts -call-depth-internal]
-        # -- --- --- --- --- --- ---
-        # -- --- --- --- --- --- ---
 
-        set files [list]
+            if {[catch {glob -nocomplain -dir $opt_dir -type d *} dirdirs]} {
-        if {$CALLDEPTH == 0} {
+                puts stderr "treefilenames error while listing subdirs in dir $opt_dir\n $dirdirs"
-            if {$opt_sort eq "natural"} {
+                set dirdirs [list]
-                package require natsort
             }
-            #set opts [dict merge $opts [list -directory $opt_dir]]
+            set okdirs [list]
-            if {![dict exists $received -directory]} {
+            foreach dir $dirdirs {
-                set opt_dir [pwd]
+                if {![_path_matches_any $opt_exclude_paths [_tailbase_match_path $opt_tailbase $dir]]} {
-            } else {
+                    lappend okdirs $dir
-                set opt_dir [dict get $opts -directory]
                 }
-            if {![file isdirectory $opt_dir]} {
-                return [list]
             }
+
+            if {$opt_glob_paths eq "*"} {
+                set matchdirs $okdirs
             } else {
-            #assume/require to exist in any recursive call
+                set matchdirs [list]
-            set opt_dir [dict get $opts -directory]
+                foreach dir $okdirs {
+                    if {$callallbelow || [child_path_state $opt_glob_paths [_tailbase_match_path $opt_tailbase $dir] $callallbelow]} {
+                        lappend matchdirs $dir
+                    }
                 }
-        if {![string match [zipfs root]* $opt_dir]} {
-            error "treefilenames_zipfs can only be used on paths beginning with [zipfs root] on this systems"
             }
-        set dir [string trimright $opt_dir "/"] ;#e.g normalize //zipfs:/x/ to //zipfs:/x
-        set dirlen [string length $dir]
 
-        set skip 0
+            set finaldirs [_sort_paths $matchdirs $opt_sort]
-        foreach anti $opt_antiglob_paths {
+            set childallbelow [expr {$callallbelow || [dict get $dir_state next_allbelow]}]
-            if {[globmatchpath $anti $dir]} {
+            set nextsubvector [list {*}$subvector [file tail $opt_dir]]
-                set skip 1
+            foreach dir $finaldirs {
-                break
+                set child_state [dict merge $state [dict create {*}{} \
+                    directory   $dir \
+                    depth       [expr {$depth + 1}] \
+                    subvector   $nextsubvector \
+                    allbelow    $childallbelow]]
+                lappend files {*}[walk_treefilenames $child_state]
+            }
+            return $files
         }
+
+        proc walk_treefilenames_zipfs {state} {
+            set opt_dir             [dict get $state directory]
+            set opt_tailbase        [dict get $state tailbase]
+            set opt_exclude_paths   [dict get $state exclude_paths]
+            set opt_exclude_files   [dict get $state exclude_files]
+            set opt_glob_paths      [dict get $state glob_paths]
+            set opt_sort            [dict get $state sort]
+            set tailglobs           [dict get $state tailglobs]
+
+            if {![string match [zipfs root]* $opt_dir]} {
+                error "treefilenames_zipfs can only be used on paths beginning with [zipfs root] on this systems"
             }
-        if {$skip} {
+            set dir [string trimright $opt_dir "/"]
+            if {[_path_matches_any $opt_exclude_paths [_tailbase_match_path $opt_tailbase $dir]]} {
                 return [list]
             }
+            set dirlen [string length $dir]
             set subpaths [zipfs list $dir/*]
             set dirlist [list]
             set skipdirs [list]
             set filelist [list]
-        #process in the order they came - sorting large list more expensive?? review
             foreach sub $subpaths {
-            set tail [string range $sub $dirlen+1 end] ;#dirlen is without trailing slash
+                set tail [string range $sub $dirlen+1 end]
                 set tailparts [file split $tail]
                 set accum ""
                 set skipdir 0
-            foreach tp [lrange $tailparts 0 end-1] {
+                foreach tailpart [lrange $tailparts 0 end-1] {
-                append accum "/$tp"
+                    append accum "/$tailpart"
                     set superpath "${dir}${accum}"
                     if {$superpath in $skipdirs} {
-                    #subpart already in skipdirs
                         set skipdir 1
                         break
                     }
                     if {$superpath ni $dirlist} {
-                    set skip2 0
+                        if {[_path_matches_any $opt_exclude_paths [_tailbase_match_path $opt_tailbase $superpath]]} {
-                    foreach anti $opt_antiglob_paths {
-                        if {[globmatchpath $anti $superpath]} {
-                            set skip2 1
                             lappend skipdirs $superpath
-                            break
-                        }
-                    }
-                    if {!$skip2} {
-                        lappend dirlist $superpath
-                    } else {
                             set skipdir 1
                             break
+                        } else {
+                            lappend dirlist $superpath
                         }
                     }
                 }
                 if {!$skipdir} {
-                #process final part of path
                     append accum "/[lindex $tailparts end]"
                     set finalpart "${dir}${accum}"
                     if {$finalpart ni $dirlist} {
                         if {[file type $finalpart] eq "file"} {
-                        set ftail [lindex $tailparts end]
+                            set file_tail [lindex $tailparts end]
                             set match 0
                             if {"*" ni $tailglobs} {
-                            foreach tg $tailglobs {
+                                foreach tailglob $tailglobs {
-                                if {[string match $tg $ftail]} {
+                                    if {[string match $tailglob $file_tail]} {
                                         set match 1
                                         break
                                     }
@@ -955,31 +1318,29 @@ namespace eval punk::path {
                                 set match 1
                             }
                             if {$match} {
-                            if {[llength $opt_antiglob_files]} {
+                                if {$opt_glob_paths ne "*"} {
+                                    set file_dir_match [_tailbase_match_path $opt_tailbase [file dirname $finalpart]]
+                                    set file_dir_state [directory_state $opt_glob_paths $file_dir_match 0]
+                                    set match [dict get $file_dir_state include_files]
+                                }
+                            }
+                            if {$match} {
                                 set skipfile 0
-                                foreach anti $opt_antiglob_files {
+                                foreach anti $opt_exclude_files {
-                                    if {[string match $anti $ftail]} {
+                                    if {[string match $anti $file_tail]} {
-                                        set skipfile 1; break
+                                        set skipfile 1
+                                        break
                                     }
                                 }
                                 if {!$skipfile} {
-                                    lappend filelist $finalpart
+                                    lappend filelist [_tailbase_relative $opt_tailbase $finalpart]
-                                }
-                            } else {
-                                lappend filelist $finalpart
                                 }
                             }
                         } else {
                             if {$finalpart ni $dirlist} {
-                            set skip2 0
+                                if {[_path_matches_any $opt_exclude_paths [_tailbase_match_path $opt_tailbase $finalpart]]} {
-                            foreach anti $opt_antiglob_paths {
-                                if {[globmatchpath $anti $finalpart]} {
-                                    set skip2 1
                                     lappend skipdirs $finalpart
-                                    break
+                                } else {
-                                }
-                            }
-                            if {!$skip2} {
                                     lappend dirlist $finalpart
                                 }
                             }
@@ -987,21 +1348,81 @@ namespace eval punk::path {
                     }
                 }
             }
-        switch -- $opt_sort {
+            return [_sort_paths $filelist $opt_sort]
-            ascii {
-                set finalfilelist [lsort $filelist]
         }
-            dictionary {
-                set finalfilelist [lsort -dictionary $filelist]
     }
-            natural {
+
-                set finalfilelist [natsort::sort $filelist]
+    #todo - treefolders with similar search caps as treefilenames
+
+    punk::args::define {
+        @id     -id     ::punk::path::treefilenames
+        @cmd    -name     punk::path::treefilenames\
+            -summary\
+            "List of filenames below supplied path."\
+            -help\
+            "List of filenames below path.
+            The resulting list is unsorted.
+
+            The path globbing syntax supports *, ** and ? as glob characters in any segment of the path, with the following semantics:
+            * matches any single segment in the path
+            ** matches 1 or more segments in the path (so /usr/**/bin will match /usr/x/bin and user/x/y/bin but not /usr/bin )
+            ? matches any single character in a single segment of the path (so /usr/te?t will match /usr/test and /usr/text but not /usr/texxt)
+            "
+        -directory       -type directory  -help\
+            "folder in which to begin recursive scan for files."
+        -tailbase         -type string    -default "" -help\
+            "if supplied, only the relative path compared to the tailbase will be returned for each file.
+            So if tailbase is /usr and a file is found at /usr/x/y/file.txt, the returned path for that file would be x/y/file.txt.
+            If tailbase is not supplied, the full path to each file will be returned.
+
+            If tailbase is supplied, it should be a prefix of the directory supplied (or the directory itself)
+            The patterns in -exclude-paths should be written to match the returned paths (i.e with the tailbase prefix removed) if -tailbase is supplied.
+            If the tailbase is not a prefix of the directory supplied, the resulting paths may have /../ components in them to account for the difference,
+            but the behaviour is not well defined in this case and it is recommended to ensure tailbase is a prefix of the directory supplied if using -tailbase.
+
+            see: punk::path::relative to compute relative paths
+            "
+        -sort -type any         -default natural -choices {none ascii dictionary natural}
+        -exclude-paths         -default {} -help\
+            "list of path patterns to exclude
+              may include * and ** path segments e.g
+              /usr/**       (exclude subfolders based at /usr but not
+                             files within /usr itself)
+              **/_aside     (exclude  files where _aside is last segment)
+              **/_aside/*   (exclude folders one below an _aside folder)
+              **/_aside/**  (exclude files in all folders with _aside as a segment)"
+        -exclude-files         -default {}
+        -include-paths     -default {**} -help\
+            "list of path patterns to include
+              may include * and ** path segments e.g
+              /usr/**       (include files in subfolders based at /usr but not
+                             files within /usr itself)
+              **/_aside     (include  files where _aside is last segment in the folder)
+              **/_aside/*   (include files in folders one below an _aside folder)
+              **/_aside/**  (include all files in folders with _aside as a segment)"
+        @values -min 0 -max -1 -optional 1 -type string
+        tailglobs -default * -multiple 1   -help\
+            "Patterns to match against filename portion (last segment) of each file path
+              within the directory tree being searched."
     }
-            default {
+
-                set finalfilelist $filelist
+    #todo - implement treefiles which acts like dirfiles but allows path globbing in the same way as punk::ns::ns/
+    #then review if treefiles can replace dirfiles or if both should exist (dirfiles can have literal glob chars in path segments - but that is a rare usecase)
+    proc treefilenames {args} {
+        set argd [punk::args::parse $args withid ::punk::path::treefilenames]
+        set state [treefile_priv::_state_from_argd $argd]
+        return [treefile_priv::walk_treefilenames $state]
     }
+    punk::args::set_idalias ::punk::path::treefilenames_zipfs ::punk::path::treefilenames
+    proc treefilenames_zipfs {args} {
+        #seems to be 2 or 3 times faster than treefilenames for //zipfs:/ paths  - REVIEW
+        # is sort order the same?
+        set argd [punk::args::parse $args withid ::punk::path::treefilenames]
+        set state [treefile_priv::_state_from_argd $argd]
+        if {![file isdirectory [dict get $state directory]]} {
+            return [list]
         }
-        return $finalfilelist
+        return [treefile_priv::walk_treefilenames_zipfs $state]
     }
 
     #maint warning - also in punkcheck

src/project_layouts/custom/_project/punk.project-0.1/src/bootsupport/modules/punk/pipe-1.0.tm

@@ -169,8 +169,8 @@ tcl::namespace::eval punk::pipe::lib {
         #This stops us matching {/@**@x x}  vs {/@**@x  x}
         #---
 
-        set rhs [tcl::string::map {: <c> ? <q> * <star> [ <lb> ] <rb> \\ <bsl> {"} <dq> " " <sp>} $rhs] 
+        set rhs [tcl::string::map {: <c> ; <sc> ? <q> * <star> [ <lb> ] <rb> \\ <bsl> {"} <dq> " " <sp>} $rhs] 
-        #review - we don't expect other command-incompatible chars such as colon?
+        #review - we don't expect other command-incompatible chars?
         return $rhs
     }
 
@@ -187,6 +187,7 @@ tcl::namespace::eval punk::pipe::lib {
 
     #exclude quoted whitespace
     proc arg_is_script_shaped {arg} {
+        set arg [string map {\\; "<escaped_semicolon>"} $arg]
         if {[tcl::string::first \n $arg] >= 0} {
             return 1
         } elseif {[tcl::string::first ";" $arg] >= 0} {

src/project_layouts/custom/_project/punk.project-0.1/src/bootsupport/modules/punk/repo-0.1.1.tm

@@ -1817,17 +1817,13 @@ namespace eval punk::repo {
         error "unimplemented"
     }
 
-    #file normalize is expensive so this is too
+    #file normalize can be a little expensive so this is too
     proc norm {path {platform env}} {
-        #kettle::path::norm
-        #see also wiki
-        #full path normalization
-
-        set platform [string tolower $platform]
-        if {$platform eq "env"} {
-            set platform $::tcl_platform(platform)
-        }
 
+        #set platform [string tolower $platform]
+        #if {$platform eq "env"} {
+        #    set platform $::tcl_platform(platform)
+        #}
         #No - don't do this sort of path translation here - leave as option for specific utils only such as ./
         #Windows volume-relative syntax with specific volume specified is somewhat broken in Tcl - but leading slash volume-relative does work
         #We shouldn't break it totally just because accessing WSL/mingw paths is slightly more useful
@@ -1835,6 +1831,9 @@ namespace eval punk::repo {
             #return [file dirname [file normalize [punk::unixywindows::towinpath $path]/__]]
         #}
 
+        #kettle::path::norm
+        #see also wiki
+        #full path normalization
         return [file dirname [file normalize $path/__]]
     }
 

src/project_layouts/custom/_project/punk.project-0.1/src/bootsupport/modules/punkapp-0.1.1.tm

@@ -0,0 +1,240 @@
+#utilities for punk apps to call
+
+
+namespace eval punkapp {
+    variable result
+    variable waiting "no"
+    proc hide_dot_window {} {
+        #alternative to wm withdraw .
+        #see https://wiki.tcl-lang.org/page/wm+withdraw
+        wm geometry . 1x1+0+0
+        wm overrideredirect . 1
+        wm transient .
+    }
+    proc is_toplevel {w} {
+        if {![llength [info commands winfo]]} {
+            return 0
+        }
+        expr {[winfo toplevel $w] eq $w && ![catch {$w cget -menu}]}
+    }
+    proc get_toplevels {{w .}} {
+        if {![llength [info commands winfo]]} {
+            return [list]
+        }
+        set list {}
+        if {[is_toplevel $w]} {
+            lappend list $w
+        }
+        foreach w [winfo children $w] {
+            lappend list {*}[get_toplevels $w]
+        }
+        return $list
+    }
+
+    proc make_toplevel_next {prefix} {
+        set top [get_toplevel_next $prefix]
+        return [toplevel $top]
+    }
+    #possible race condition if multiple calls made without actually creating the toplevel, or gap if highest existing closed in the meantime
+    #todo - reserve_toplevel_next ? keep list of toplevels considered 'allocated' even if never created or already destroyed? what usecase?
+    #can call wm withdraw to to reserve newly created toplevel. To stop re-use of existing names after destruction would require a list or at least a record of highest created for each prefix
+    proc get_toplevel_next {prefix} {
+        set base [string trim $prefix .] ;# .myapp -> myapp  .myapp.somewindow  -> myapp.somewindow   . -> ""
+
+
+
+    }
+    proc exit {{toplevel ""}} {
+        variable waiting
+        variable result
+        variable default_result
+        set toplevels [get_toplevels]
+        if {[string length $toplevel]} {
+            set wposn [lsearch $toplevels $toplevel]
+            if {$wposn > 0} {
+                destroy $toplevel
+            }
+        } else {
+            #review
+            if {[package provide punk::repl::codethread] ne "" &&  [punk::repl::codethread::is_running]} {
+                puts stderr "punkapp::exit called without toplevel - showing console"
+                show_console
+                return 0
+            } else {
+                puts stderr "punkapp::exit called without toplevel - exiting"
+                if {$waiting ne "no"} {
+                    if {[info exists result(shell)]} {
+                        set temp [set result(shell)]
+                        unset result(shell)
+                        set waiting $temp
+                    } else {
+                        set waiting ""
+                    }
+                } else {
+                    ::exit
+                }
+            }
+        }
+
+        set controllable [get_user_controllable_toplevels]
+        if {![llength $controllable]} {
+            if {[package provide punk::repl::codethread] ne "" &&  [punk::repl::codethread::is_running]} {
+                show_console
+            } else {
+                if {$waiting ne "no"} {
+                    if {[info exists result(shell)]} {
+                        set temp [set result(shell)]
+                        unset result(shell)
+                        set waiting $temp
+                    } elseif {[info exists result($toplevel)]} {
+                        set temp [set result($toplevel)]
+                        unset result($toplevel)
+                        set waiting $temp
+                    } elseif {[info exists default_result]} {
+                        set temp $default_result
+                        unset default_result
+                        set waiting $temp
+                    } else {
+                        set waiting ""
+                    }
+                } else {
+                    ::exit
+                }
+            }
+        }
+    }
+    proc close_window {toplevel} {
+        wm withdraw $toplevel
+        if {![llength [get_user_controllable_toplevels]]} {
+            punkapp::exit $toplevel
+        }
+        destroy $toplevel
+    }
+    proc wait {args} {
+        variable waiting
+        variable default_result
+        if {[dict exists $args -defaultresult]} {
+            set default_result [dict get $args -defaultresult]
+        }
+        foreach t [punkapp::get_toplevels] {
+            if {[wm protocol $t WM_DELETE_WINDOW] eq ""} {
+                wm protocol $t WM_DELETE_WINDOW [list punkapp::close_window $t]
+            }
+        }
+        if {[package provide punk::repl::codethread] ne "" &&  [punk::repl::codethread::is_running]} {
+            puts stderr "repl eventloop seems to be running - punkapp::wait not required"
+        } else {
+            if {$waiting eq "no"} {
+                set waiting "waiting"
+                vwait ::punkapp::waiting
+                return $::punkapp::waiting
+            }
+        }
+    }
+
+    #A window can be 'visible' according to this - but underneath other windows etc
+    #REVIEW - change name?
+    proc get_visible_toplevels {{w .}} {
+        if {![llength [info commands winfo]]} {
+            return [list]
+        }
+        set list [get_toplevels $w]
+        set mapped [lmap v $list {expr {[winfo ismapped $v] ? $v : {}}}]
+        set mapped [concat {*}$mapped] ;#ignore {}
+        set visible [list]
+        foreach m $mapped {
+            if {[wm overrideredirect $m] == 0 } {
+                lappend visible $m
+            } else {
+                if {[winfo height $m] >1 && [winfo width $m] > 1} {
+                    #technically even a 1x1 is visible.. but in practice even a 10x10 is hardly likely to be noticeable when overrideredirect == 1
+                    #as a convention - 1x1 with no controls is used to make a window invisible so we'll treat anything larger as visible
+                    lappend visible $m
+                }
+            }
+        }
+        return $visible
+    }
+    proc get_user_controllable_toplevels {{w .}} {
+        set visible [get_visible_toplevels $w]
+        set controllable [list]
+        foreach v $visible {
+            if {[wm overrideredirect $v] == 0} {
+                lappend controllable $v
+            }
+        }
+        #only return visible windows with overrideredirect == 0 because there exists some user control.
+        #todo - review.. consider checking if position is outside screen areas? Technically controllable.. but not easily
+        return $controllable
+    }
+    proc hide_console {args} {
+        set opts [dict create -force 0]
+        if {([llength $args] % 2) != 0} {
+            error "hide_console expects pairs of arguments. e.g -force 1"
+        }
+        #set known_opts [dict keys $defaults]
+        foreach {k v} $args {
+            switch -- $k {
+                -force {
+                    dict set opts $k $v
+                }
+                default {
+                    error "Unrecognised options '$k'  known options: [dict keys $opts]"
+                }
+            }
+        }
+        set force [dict get $opts -force]
+
+        if {!$force} {
+            if {![llength [get_user_controllable_toplevels]]} {
+                puts stderr "Cannot hide console while no user-controllable windows available"
+                return 0
+            }
+        }
+        if {$::tcl_platform(platform) eq "windows"} {
+            #hide won't work for certain consoles cush as conemu,wezterm - and doesn't really make sense for tabbed windows anyway.
+            #It would be nice if we could tell the console window to hide just the relevant tab - or the whole window if only one tab present - but this is unlikely to be possible in any standard way.
+            #an ordinary cmd.exe or pwsh.exe or powershell.exe window can be hidden ok though.
+            #(but with wezterm - process is cmd.exe - but it has style popup and can't be hidden with a twapi::hide_window call)
+            package require twapi
+            set h [twapi::get_console_window]
+            set pid [twapi::get_window_process $h]
+            set pinfo [twapi::get_process_info $pid -name]
+            set pname [dict get $pinfo -name]
+            set wstyle [twapi::get_window_style $h]
+            #tclkitsh/tclsh?
+            if {($pname in [list cmd.exe pwsh.exe powershell.exe] || [string match punk*.exe $pname]) && "popup" ni $wstyle} {
+                twapi::hide_window $h
+                return 1
+            } else {
+                puts stderr "punkapp::hide_console unable to hide this type of console window"
+                return 0
+            }
+        } else {
+            #todo
+            puts stderr "punkapp::hide_console unimplemented on this platform (todo)"
+            return 0
+        }
+    }
+
+    proc show_console {} {
+        if {$::tcl_platform(platform) eq "windows"} {
+            package require twapi
+            if {![catch {set h [twapi::get_console_window]} errM]} {
+                twapi::show_window $h -activate -normal
+            } else {
+                #no console - assume launched from something like wish?
+                catch {console show}
+            }
+        } else {
+            #todo
+            puts stderr "punkapp::show_console unimplemented on this platform"
+        }
+    }
+
+}
+
+package provide punkapp [namespace eval punkapp {
+    variable version
+    set version 0.1.1
+}]

src/project_layouts/custom/_project/punk.project-0.1/src/bootsupport/modules/punkcheck/cli-0.1.0.tm

@@ -259,7 +259,6 @@ namespace eval punkcheck::cli {
 }
 
 
-
 # ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++
 namespace eval punkcheck::cli::lib {
     namespace path ::punk::mix::util ;#askuser, do_in_path, foreach-file etc
@@ -307,7 +306,6 @@ namespace eval punkcheck::cli::lib {
         }
         return {}
     }
-    
 }
 
 

src/project_layouts/custom/_project/punk.project-0.1/src/bootsupport/modules/shellfilter-0.2.2.tm

@@ -326,18 +326,34 @@ namespace eval shellfilter::chan {
         #method flush {ch} {
         #    return ""
         #}
+        #method flush {transform_handle} {
+        #    #puts stdout "<flush>"
+        #    #review - just clear o_encbuf and emit nothing?
+        #    #we wouldn't have a value there if it was convertable from the channel encoding?
+        #    if {[string length $o_encbuf]} {
+        #        #if we have data in the buffer that we haven't been able to convert to a string
+        #        #- then we probably have some kind of encoding mismatch. Is it safer to discard it than to emit garbage chars to the channel or var?
+        #        #REVIEW - log that we are discarding the buffer contents on flush?
+        #        puts stderr "WARNING: flush called on tee_to_var with non-empty buffer. This probably indicates an encoding mismatch between the channel encoding and the encoding expected by the transform. Discarding buffer contents: '$o_encbuf'"
+        #    }
+        #    set clear $o_encbuf
+        #    set o_encbuf ""
+        #    return $clear
+        #}
         method flush {transform_handle} {
-            #puts stdout "<flush>"
+            #puts stderr "<flush> $transform_handle o_encbuf: '$o_encbuf' datavars: $o_datavars"
-            #review - just clear o_encbuf and emit nothing?
+            set clear $o_encbuf
-            #we wouldn't have a value there if it was convertable from the channel encoding?
+            if {[catch {tcl::encoding::convertfrom $o_enc $clear} stringdata]} {
-            if {[string length $o_encbuf]} {
+                #if we can't convert the buffer contents to a string - does it make sense to emit the raw bytes?
-                #if we have data in the buffer that we haven't been able to convert to a string
+                # - probably not.
-                #- then we probably have some kind of encoding mismatch. Is it safer to discard it than to emit garbage chars to the channel or var?
+                #REVIEW?
-                #REVIEW - log that we are discarding the buffer contents on flush?
+                return ""
-                puts stderr "WARNING: flush called on tee_to_var with non-empty buffer. This probably indicates an encoding mismatch between the channel encoding and the encoding expected by the transform. Discarding buffer contents: '$o_encbuf'"
             }
             set o_encbuf ""
-            return ""
+            foreach v $o_datavars {
+                append $v $stringdata
+            }
+            return $stringdata
         }
         method write {ch bytes} {
             #test with set x [string repeat " \U1f6c8" 2043]
@@ -442,16 +458,29 @@ namespace eval shellfilter::chan {
         #    flush $o_localchan
         #    return $clear
         #}
+        #method flush {transform_handle} {
+        #    #we wouldn't have a value in o_encbuf if it was convertable from the channel encoding?
+        #    if {[string length $o_encbuf]} {
+        #        #if we have data in the buffer that we haven't been able to convert to a string
+        #        #- then we probably have some kind of encoding mismatch. Is it safer to discard it than to emit garbage chars to the channel or var?
+        #        #REVIEW - log that we are discarding the buffer contents on flush?
+        #        puts stderr "WARNING: flush called on tee_to_pipe with non-empty buffer. This probably indicates an encoding mismatch between the channel encoding and the encoding expected by the transform. Discarding buffer contents: '$o_encbuf'"
+        #    }
+        #    set clear $o_encbuf
+        #    set o_encbuf ""
+        #    return $clear
+        #}
         method flush {transform_handle} {
-            #we wouldn't have a value in o_encbuf if it was convertable from the channel encoding?
+            set clear $o_buffered$o_encbuf
-            if {[string length $o_encbuf]} {
+            if {[catch {tcl::encoding::convertfrom $o_enc $clear} stringdata]} {
-                #if we have data in the buffer that we haven't been able to convert to a string
+                #if we can't convert the buffer contents to a string - does it make sense to emit the raw bytes?
-                #- then we probably have some kind of encoding mismatch. Is it safer to discard it than to emit garbage chars to the channel or var?
+                # - probably not.
-                #REVIEW - log that we are discarding the buffer contents on flush?
+                #REVIEW?
-                puts stderr "WARNING: flush called on tee_to_pipe with non-empty buffer. This probably indicates an encoding mismatch between the channel encoding and the encoding expected by the transform. Discarding buffer contents: '$o_encbuf'"
+                return ""
             }
+            set o_buffered ""
             set o_encbuf ""
-            return ""
+            return $stringdata
         }
         method write {transform_handle bytes} {
             #set logdata [tcl::encoding::convertfrom $o_enc $bytes]
@@ -533,11 +562,24 @@ namespace eval shellfilter::chan {
             ::shellfilter::log::write $o_logsource $logdata
             return  $bytes
         }
+        #method flush {transform_handle} {
+        #    #return ""
+        #    set clear $o_encbuf
+        #    set o_encbuf ""
+        #    #review
+        #    return $clear
+        #}
         method flush {transform_handle} {
-            #return ""
+            set clear $o_buffered$o_encbuf
-            set clear $o_encbuf
+            if {[catch {tcl::encoding::convertfrom $o_enc $clear} stringdata]} {
+                #if we can't convert the buffer contents to a string - does it make sense to emit the raw bytes?
+                # - probably not.
+                #REVIEW?
+                return ""
+            }
+            set o_buffered ""
             set o_encbuf ""
-            return $o_encbuf
+            return $stringdata
         }
         method write {ch bytes} {
             #set logdata [tcl::encoding::convertfrom $o_enc $bytes]
@@ -613,9 +655,21 @@ namespace eval shellfilter::chan {
             my destroy
         }
         #clear?
+        #method flush {transform_handle} {
+        #    #we wouldn't have a value in o_encbuf if it was convertable from the channel encoding?
+        #    if {[string length $o_encbuf]} {
+        #        #if we have data in the buffer that we haven't been able to convert to a string
+        #        #- then we probably have some kind of encoding mismatch. Is it safer to discard it than to emit garbage chars to the log?
+        #        #REVIEW. - we are writing the raw bytes to the log here because we can't convert them to a string.
+        #        #This may be useful for debugging issues, but it may also result in garbage data in the log.
+        #        ::shellfilter::log::write $o_logsource $o_encbuf
+        #        set o_encbuf ""
+        #    }
+        #    return
+        #}
         method flush {transform_handle} {
-            #we wouldn't have a value in o_encbuf if it was convertable from the channel encoding?
+            set clear $o_buffered$o_encbuf
-            if {[string length $o_encbuf]} {
+            if {[catch {tcl::encoding::convertfrom $o_enc $clear} stringdata]} {
                 #if we have data in the buffer that we haven't been able to convert to a string
                 #- then we probably have some kind of encoding mismatch. Is it safer to discard it than to emit garbage chars to the log?
                 #REVIEW. - we are writing the raw bytes to the log here because we can't convert them to a string.
@@ -755,6 +809,110 @@ namespace eval shellfilter::chan {
         }
     }
 
+
+    #experimental
+    #applying this to stdout breaks console query/responses - why?
+    #- probably because we are splitting graphemes across writes and flushes and the console doesn't know how to handle that?
+    oo::class create unicode_normalize {
+        variable o_trecord
+        variable o_enc
+        variable o_encbuf
+        variable o_graphemebuf
+        variable o_mode
+        variable o_is_junction
+        constructor {tf} {
+            package require punk::ansi
+            set o_trecord $tf
+            set o_enc [::tcl::dict::get $tf -encoding]
+            set o_encbuf ""
+            set o_graphemebuf ""
+            set settingsdict [tcl::dict::get $tf -settings]
+            if {[dict exists $settingsdict -mode]} {
+                set o_mode [dict get $settingsdict -mode]
+                if {$o_mode ni {nfc nfd nfkc nfkd none}} {
+                    error "unicode_normalize transform - invalid mode '$o_mode' in settings"
+                }
+                if {$o_mode ne "none"} {
+                    #we get dll dependent load errors on windows sometimes - but still seems to work - REVIEW/FIX.
+                    catch {::tcl::unsupported::loadIcu}
+                }
+            } else {
+                #if no mode specified - default to 'none' which just does grapheme splitting without any unicode normalization
+                set o_mode "none"
+            }
+            if {[::tcl::dict::exists $tf -junction]} {
+                set o_is_junction [::tcl::dict::get $tf -junction]
+            } else {
+                set o_is_junction 0
+            }
+        }
+        method initialize {transform_handle mode} {
+            return [list initialize write flush finalize]
+        }
+        method finalize {transform_handle} {
+            my destroy
+        }
+        method flush {transform_handle} {
+            #flush seems to do nothing - why?
+            set clear $o_encbuf[unset o_encbuf]
+            if {[catch {tcl::encoding::convertfrom $o_enc $clear} stringdata]} {
+                #if we can't convert the buffer contents to a string - put it back and try again with more data later
+                #REVIEW?
+                set o_encbuf $clear
+                return ""
+            }
+            #review
+
+            set graphemes [punk::char::grapheme_split $o_graphemebuf$stringdata]
+            set outstring [join $graphemes ""]
+            #puts "outstring: '$outstring' graphemes: $graphemes"
+            if {$o_mode ne "none"} {
+                set outstring [::tcl::unsupported::icu::normalize -mode $o_mode $outstring]
+            }
+            set o_graphemebuf ""
+            return [tcl::encoding::convertto $o_enc $outstring]
+        }
+        method write {transform_handle bytes} {
+            set inputbytes $o_encbuf$bytes
+            set o_encbuf ""
+            set tail_offset 0
+
+            if {$inputbytes eq ""} {
+                #review - do we even get empty writes?
+                puts stderr "WARNING: write called on unicode_normalize with empty inputbytes. This may be a no-op, but it may also indicate an issue with the upstream transform or channel. Emitting no data for this write."
+                set stringdata ""
+            }
+
+            while {$tail_offset < [::tcl::string::length $inputbytes] && [catch {tcl::encoding::convertfrom $o_enc [::tcl::string::range $inputbytes 0 end-$tail_offset]} stringdata]} {
+                incr tail_offset
+            }
+            if {$tail_offset > 0} {
+                if {$tail_offset < [::tcl::string::length $inputbytes]} {
+                    #stringdata from catch statement must be a valid result
+                    set t [expr {$tail_offset - 1}]
+                    set o_encbuf [::tcl::string::range $inputbytes end-$t end]
+                } else {
+                    set stringdata ""
+                    set o_encbuf $inputbytes
+                    return ""
+                }
+            }
+            set graphemes [punk::char::grapheme_split $o_graphemebuf$stringdata]
+            set outstring [join [lrange $graphemes 0 end-1] ""]
+            set o_graphemebuf [lindex $graphemes end]
+
+            if {$o_mode ne "none"} {
+                set outstring [::tcl::unsupported::icu::normalize -mode $o_mode $outstring]
+            }
+
+            return [tcl::encoding::convertto $o_enc $outstring]
+        }
+        method meta_is_redirection {} {
+            return $o_is_junction
+        }
+    }
+
+
     #a test
     oo::class create reconvert {
         variable o_trecord
@@ -1114,7 +1272,6 @@ namespace eval shellfilter::chan {
         #    return $emit
         #}
         method flush {transform_handle} {
-            #return ""
             set clear $o_buffered$o_encbuf
             if {[catch {tcl::encoding::convertfrom $o_enc $clear} stringdata]} {
                 #if we can't convert the buffer contents to a string - does it make sense to emit the raw bytes?

src/project_layouts/custom/_project/punk.project-0.1/src/bootsupport/modules/shellrun-0.1.2.tm

@@ -0,0 +1,897 @@
+# vim: set ft=tcl
+#
+#purpose: handle the run commands that call shellfilter::run
+#e.g run,runout,runerr,runx
+
+package require shellfilter
+package require punk::ansi
+
+#NOTE: the run,runout,runerr,runx commands only produce an error if the command didn't run.
+# - If it did run, but there was a non-zero exitcode it is up to the application to check that.
+#This is deliberate, but means 'catch' doesn't catch errors within the command itself -  the exitcode has to be checked.
+#The user can always use exec for different process error semantics (they don't get exitcode with exec)
+
+namespace eval shellrun {
+    variable PUNKARGS
+    variable runout
+    variable runerr
+
+    #do we need these?
+    #variable punkout
+    #variable punkerr
+
+    #some ugly coupling with punk/punk::config for now
+    #todo - something better
+    if {[info exists ::punk::config::configdata]} {
+        set conf_running    [punk::config::configure running]
+        set syslog_stdout   [dict get $conf_running syslog_stdout]
+        set syslog_stderr   [dict get $conf_running syslog_stderr]
+        set logfile_stdout  [dict get $conf_running logfile_stdout]
+        set logfile_stderr  [dict get $conf_running logfile_stderr]
+    } else {
+        lassign [list "" "" "" ""] syslog_stdout syslog_stderr logfile_stdout logfile_stderr
+    }
+    if {"punkshout" ni [shellfilter::stack::items]} {
+        set outdevice [shellfilter::stack::new punkshout -settings [list -tag "punkshout" -buffering none -raw 1 -syslog $syslog_stdout -file $logfile_stdout]]
+        set out [dict get $outdevice localchan]
+    } else {
+        set out [dict get [shellfilter::stack::item punkshout] device localchan]
+    }
+    if {"punksherr" ni [shellfilter::stack::items]} {
+        set errdevice [shellfilter::stack::new punksherr -settings [list -tag "punksherr" -buffering none -raw 1 -syslog $syslog_stderr -file $logfile_stderr]]
+        set err [dict get $errdevice localchan]
+    } else {
+        set err  [dict get [shellfilter::stack::item punksherr] device localchan]
+    }
+
+    namespace import ::punk::ansi::a+
+    namespace import ::punk::ansi::a
+
+
+
+
+    #repltelemetry - additional/alternative display info used in a repl context i.e info directed towards the screen
+    #todo - package up in repltelemetry module and rewrite proc based on whether the module was found/loaded.
+    #somewhat strong coupling to punk - but let's try to behave decently if it's not loaded
+    #The last_run_display is actually intended for the repl - but is resident in the punk namespace with a view to the possibility of a different repl being in use.
+    proc set_last_run_display {chunklist} {
+        #chunklist as understood by the
+        if {![info exists ::punk::repltelemetry_emmitters]} {
+            namespace eval ::punk {
+                variable repltelemetry_emmitters
+                set repltelemetry_emmitters "shellrun"
+            }
+        } else {
+            if {"shellrun" ni $::punk::repltelemetry_emmitters} {
+                lappend punk::repltelemetry_emmitters "shellrun"
+            }
+        }
+
+        #most basic of validity tests here.. just that it is a list (can be empty). We don't want to duplicate or over-constrain the way repls/shells/terminals interpet the info
+        if {[catch {llength $chunklist} errMsg]} {
+            error  "set_last_run_display expects a list. Value supplied doesn't appear to be a well formed tcl list. '$errMsg'"
+        }
+        #todo -
+        tsv::lappend repl runchunks-[tsv::get repl runid] {*}$chunklist
+    }
+
+
+
+    #maintenance: similar used in punk::ns & punk::winrun
+    #todo - take runopts + aliases as args
+    #longopts must be passed as a single item ie  --timeout=100  not --timeout 100
+    proc get_run_opts {arglist} {
+        if {[catch {
+            set callerinfo [info level -1]
+        } errM]} {
+            set caller ""
+        } else {
+            set caller [lindex $callerinfo 0]
+        }
+
+        #we provide -nonewline even for 'run' even though run doesn't deliver stderr or stdout to the tcl return value
+        #This is for compatibility with other runX commands, and the difference is also visible when calling from repl.
+        set known_runopts [list "-echo" "-e" "-nonewline" "-n" "-tcl" "-debug"]
+        set known_longopts [list "--timeout"]
+        set known_longopts_msg ""
+        foreach lng $known_longopts {
+            append known_longopts_msg "${lng}=val "
+        }
+        set aliases [list "-e" "-echo" "-echo" "-echo" "-n" "-nonewline" "-nonewline" "-nonewline" "-tcl" "-tcl" "-debug" "-debug"] ;#include map to self
+        set runopts [list]
+        set runoptslong [list]
+        set cmdargs [list]
+
+        set idx_first_cmdarg [lsearch -not $arglist "-*"]
+
+        set allopts [lrange $arglist 0 $idx_first_cmdarg-1]
+        set cmdargs [lrange $arglist $idx_first_cmdarg end]
+        foreach o $allopts {
+            if {[string match --* $o]} {
+                lassign [split $o =] flagpart valpart
+                if {$valpart eq ""} {
+                    error "$caller: longopt $o seems to be missing a value - must be of form --option=value"
+                }
+                if {$flagpart ni $known_longopts} {
+                    error "$caller: Unknown runoption $o - known options $known_runopts $known_longopts_msg"
+                }
+                lappend runoptslong $flagpart $valpart
+            } else {
+                if {$o ni $known_runopts} {
+                    error "$caller: Unknown runoption $o - known options $known_runopts $known_longopts_msg"
+                }
+                lappend runopts [dict get $aliases $o]
+            }
+        }
+        return [list runopts $runopts runoptslong $runoptslong cmdargs $cmdargs]
+    }
+
+
+    #todo - investigate cause of punk86 run hanging sometimes. An 'after 500' before exit in the called script fixes the issue. punk87 doesn't seem to be affected.
+    lappend PUNKARGS [list {
+        @id     -id ::shellrun::run
+        @leaders -min 0 -max 0
+        @opts
+        -nonewline  -type none
+        -tcl        -type none -default 0
+        -debug      -type none -default 0
+        --timeout=  -type integer
+        @values -min 1 -max -1
+        cmdname -type string
+        cmdarg -type any -multiple 1 -optional 1
+    }]
+    proc run {args} {
+        #set_last_run_display [list]
+
+        #set splitargs [get_run_opts $args]
+        #set runopts [dict get $splitargs runopts]
+        #set runoptslong [dict get $splitargs runoptslong]
+        #set cmdargs [dict get $splitargs cmdargs]
+        set argd [punk::args::parse $args withid ::shellrun::run]
+        lassign [dict values $argd] leaders opts values received
+
+        if {[dict exists $received "-nonewline"]} {
+            set nonewline 1
+        } else {
+            set nonewline 0
+        }
+        #review nonewline does nothing here..
+
+        set idlist_stderr [list]
+        #we leave stdout without imposed ansi colouring - because the source may be colourised and because ansi-wrapping a stream at whatever boundaries it comes in at isn't a really nice thing to do.
+        #stderr might have source colouring - but it usually doesn't seem to, and the visual distiction of red stderr can be very handy for the run command.
+        #A further enhancement could be to detect well-known options such as --color and/or use a configuration for specific commands that have useful colourised stderr,
+        #but having an option to configure stderr to red is a compromise.
+        #Note that the other run commands, runout,runerr, runx don't emit in real-time - so for those commands there may be options to detect and/or post-process stdout and stderr.
+        #TODO - fix. This has no effect if/when the repl adds an ansiwrap transform
+        # what we probably want to do is 'aside' that transform for runxxx commands only.
+        #lappend idlist_stderr [shellfilter::stack::add stderr ansiwrap -settings {-colour {red bold}}]
+
+        set callopts [dict create]
+        if {[dict exists $received "-tcl"]} {
+            dict set callopts -tclscript 1 
+        }
+        if {[dict exists $received "-debug"]} {
+            dict set callopts -debug 1
+        }
+        if {[dict exists $received --timeout]} {
+            dict set callopts -timeout [dict get $opts --timeout] ;#convert to single dash
+        }
+        set cmdname     [dict get $values cmdname]
+        if {[dict exists $received cmdarg]} {
+            set cmdarglist  [dict get $values cmdarg]
+        } else {
+            set cmdarglist {}
+        }
+        set cmdargs [concat $cmdname $cmdarglist]
+        #---------------------------------------------------------------------------------------------
+        set exitinfo [shellfilter::run $cmdargs {*}$callopts -teehandle punksh -inbuffering none -outbuffering none ]
+        #---------------------------------------------------------------------------------------------
+        foreach id $idlist_stderr {
+            shellfilter::stack::remove stderr $id
+        }
+        #puts stderr "shellrun::run exitinfo: $exitinfo"
+
+        flush stderr
+        flush stdout
+
+        if {[dict exists $exitinfo error]} {
+            error "[dict get $exitinfo error]\n$exitinfo"
+        }
+
+        return $exitinfo
+    }
+
+    lappend PUNKARGS [list {
+        @id     -id ::shellrun::runconsole
+        @leaders -min 0 -max 0
+        @opts
+        @values -min 1 -max -1
+        cmdname -type string
+        cmdarg -type any -multiple 1 -optional 1
+    }]
+    #run in the way tcl unknown does - but without regard to auto_noexec
+    proc runconsole {args} {
+        set argd [punk::args::parse $args withid ::shellrun::runconsole]
+        lassign [dict values $argd] leaders opts values received
+        set cmdname     [dict get $values cmdname]
+        if {[dict exists $received cmdarg]} {
+            set arglist     [dict get $values cmdarg]
+        } else {
+            set arglist {}
+        }
+
+        set resolved_cmdname [auto_execok $cmdname]
+        if {$resolved_cmdname eq ""} {
+            error "Cannot find path for executable '$cmdname'"
+        }
+        set repl_runid [punk::get_repl_runid]
+        #set ::punk::last_run_display [list]
+
+        set redir ">&@stdout <@stdin"
+        uplevel 1 [list ::catch [concat exec $redir $resolved_cmdname $arglist] ::tcl::UnknownResult ::tcl::UnknownOptions]
+        #we can't detect stdout/stderr output from the exec
+        #for now emit an extra \n on stderr
+        #todo - there is probably no way around this but to somehow exec in the context of a completely separate console
+        #This is probably a tricky problem - especially to do cross-platform
+        #
+        # - use [dict get $::tcl::UnknownOptions -code]  (0|1)  exit
+        if {[dict get $::tcl::UnknownOptions -code] == 0} {
+            set c green
+            set m "ok"
+        } else {
+            set c yellow
+            set m "errorCode $::errorCode"
+        }
+        set chunklist [list]
+        lappend chunklist [list "info" "[a $c]$m[a] " ]
+        if {$repl_runid != 0} {
+            tsv::lappend repl runchunks-$repl_runid {*}$chunklist
+        }
+
+        dict incr ::tcl::UnknownOptions -level
+        return -options $::tcl::UnknownOptions $::tcl::UnknownResult
+    }
+    lappend PUNKARGS [list {
+        @id     -id ::shellrun::runout
+        @leaders -min 0 -max 0
+        @opts
+        -echo       -type none
+        -nonewline  -type none
+        -tcl        -type none -default 0
+        -debug      -type none -default 0
+        --timeout=  -type integer
+        @values -min 1 -max -1
+        cmdname -type string
+        cmdarg -type any -multiple 1 -optional 1
+    }]
+    proc runout {args} {
+        set argd [punk::args::parse $args withid ::shellrun::runout]
+        lassign [dict values $argd] leaders opts values received
+
+        if {[dict exists $received "-nonewline"]} {
+            set nonewline 1
+        } else {
+            set nonewline 0
+        }
+
+        #set_last_run_display [list]
+        variable runout
+        variable runerr
+        set runout ""
+        set runerr ""
+        set RST [a]
+
+        #set splitargs [get_run_opts $args]
+        #set runopts [dict get $splitargs runopts]
+        #set cmdargs [dict get $splitargs cmdargs]
+
+
+        #puts stdout "RUNOUT cmdargs: $cmdargs"
+
+        #todo   add -data boolean and -data lastwrite to -settings with default being -data all
+        # because sometimes we're only interested in last char (e.g to detect something was output)
+
+        #set outvar_stackid [shellfilter::stack::add commandout tee_to_var  -action float -settings {-varname ::runout}]
+        #
+        #when not echoing - use float-locked so that the repl's stack is bypassed
+        if {[dict exists $received "-echo"]} {
+            set stdout_stackid [shellfilter::stack::add stdout tee_to_var -action float-locked -settings {-varname ::shellrun::runout}]
+            set stderr_stackid [shellfilter::stack::add stderr tee_to_var -action float-locked -settings {-varname ::shellrun::runerr}]
+            #set stderr_stackid [shellfilter::stack::add stderr tee_to_var -action sink-locked -settings {-varname ::shellrun::runerr}]
+        } else {
+            set stdout_stackid [shellfilter::stack::add stdout var -action float-locked -settings {-varname ::shellrun::runout}]
+            set stderr_stackid [shellfilter::stack::add stderr var -action float-locked -settings {-varname ::shellrun::runerr}]
+        }
+
+        set callopts [dict create]
+        if {[dict exists $received "-tcl"]} {
+            dict set callopts -tclscript 1 
+        }
+        if {[dict exists $received "-debug"]} {
+            dict set callopts -debug 1
+        }
+        if {[dict exists $received --timeout]} {
+            dict set callopts -timeout [dict get $opts --timeout] ;#convert to single dash
+        }
+        set cmdname     [dict get $values cmdname]
+        if {[dict exists $received cmdarg]} {
+            set cmdarglist  [dict get $values cmdarg]
+        } else {
+            set cmdarglist {}
+        }
+        set cmdargs [concat $cmdname $cmdarglist]
+
+        #shellfilter::run [lrange $args 1 end] -teehandle punksh -outchan stdout -inbuffering none -outbuffering none -stdinhandler ::repl::repl_handler
+        set exitinfo [shellfilter::run $cmdargs {*}$callopts -teehandle punksh -inbuffering none -outbuffering none ]
+
+        flush stderr
+        flush stdout
+
+        shellfilter::stack::remove stdout $stdout_stackid
+        shellfilter::stack::remove stderr $stderr_stackid
+
+        #shellfilter::stack::remove commandout $outvar_stackid
+
+        if {[dict exists $exitinfo error]} {
+            if {[dict exists $received "-tcl"]} {
+
+            } else {
+                #we must raise an error.
+                #todo - check errorInfo makes sense.. return -code? tailcall?
+                #
+                set msg ""
+                append msg [dict get $exitinfo error]
+                append msg "\n(add -tcl option to run as a tcl command/script instead of an external command)"
+                error $msg
+            }
+        }
+
+        set chunklist [list]
+
+        #exitcode not part of return value for runout - colourcode appropriately
+        set n $RST
+        set c ""
+
+        if {[dict exists $exitinfo exitcode]} {
+            set code [dict get $exitinfo exitcode]
+            if {$code == 0} {
+                set c [a+ green]
+            } else {
+                set c [a+ white bold]
+            }
+            lappend chunklist [list "info" "$c$exitinfo$n"]
+        } elseif {[dict exists $exitinfo error]} {
+            # -tcl (with error)
+            set c [a+ yellow bold]
+            lappend chunklist [list "info" "${c}error [dict get $exitinfo error]$n"]
+            lappend chunklist [list "info" "errorCode [dict get $exitinfo errorCode]"]
+            #lappend chunklist [list "info" "errorInfo [list [dict get $exitinfo errorInfo]]"]
+            lappend chunklist [list "info" errorInfo]
+            lappend chunklist [list "stderr" [dict get $exitinfo errorInfo]]
+        } else {
+            # -tcl (without error)
+            set c [a+ Green white bold]
+            #lappend chunklist [list "info" "$c$exitinfo$n"]
+            lappend chunklist [list "info" [punk::ansi::ansiwrap_raw $c \x1b\[m "" $exitinfo]]
+        }
+
+
+        set chunk "[a+ red bold]stderr$RST"
+        lappend chunklist [list "info" $chunk]
+
+        set chunk ""
+        if {[string length $::shellrun::runerr]} {
+            if {$nonewline} {
+                set e [string trimright $::shellrun::runerr \r\n]
+            } else {
+                set e $::shellrun::runerr
+            }
+            #append chunk "[a+ red normal]$e$RST\n"
+            append chunk "[a+ red normal]$e$RST"
+        }
+        lappend chunklist [list stderr $chunk]
+
+
+
+
+        lappend chunklist [list "info" "[a+ white bold]stdout$RST"]
+        set chunk ""
+        if {[string length $::shellrun::runout]} {
+            if {$nonewline} {
+                set o [string trimright $::shellrun::runout \r\n]
+            } else {
+                set o $::shellrun::runout
+            }
+            append chunk "$o"
+        }
+        lappend chunklist [list result $chunk]
+
+
+        #set_last_run_display $chunklist
+        tsv::lappend repl runchunks-[tsv::get repl runid] {*}$chunklist
+
+        if {$nonewline} {
+            return [string trimright $::shellrun::runout \r\n]
+        } else {
+            return $::shellrun::runout
+        }
+    }
+
+    lappend PUNKARGS [list {
+        @id     -id ::shellrun::runerr
+        @leaders -min 0 -max 0
+        @opts
+        -echo       -type none
+        -nonewline  -type none
+        -tcl        -type none -default 0
+        -debug      -type none -default 0
+        --timeout=  -type integer
+        @values -min 1 -max -1
+        cmdname -type string
+        cmdarg -type any -multiple 1 -optional 1
+    }]
+    proc runerr {args} {
+        set argd [punk::args::parse $args withid ::shellrun::runerr]
+        lassign [dict values $argd] leaders opts values received
+
+        if {[dict exists $received "-nonewline"]} {
+            set nonewline 1
+        } else {
+            set nonewline 0
+        }
+
+        #set_last_run_display [list]
+        variable runout
+        variable runerr
+        set runout ""
+        set runerr ""
+
+        #set splitargs [get_run_opts $args]
+        #set runopts [dict get $splitargs runopts]
+        #set cmdargs [dict get $splitargs cmdargs]
+
+        set callopts [dict create]
+        if {[dict exists $received "-tcl"]} {
+            dict set callopts -tclscript 1 
+        }
+        if {[dict exists $received "-debug"]} {
+            dict set callopts -debug 1
+        }
+        if {[dict exists $received --timeout]} {
+            dict set callopts -timeout [dict get $opts --timeout] ;#convert to single dash
+        }
+        set cmdname     [dict get $values cmdname]
+        if {[dict exists $received cmdarg]} {
+            set cmdarglist  [dict get $values cmdarg]
+        } else {
+            set cmdarglist {}
+        }
+        set cmdargs [concat $cmdname $cmdarglist]
+
+        if {[dict exists $received "-tcl"]} {
+            append callopts " -tclscript 1"
+        }
+        if {[dict exists $received "-echo"]} {
+            set stderr_stackid [shellfilter::stack::add stderr tee_to_var -action float-locked -settings {-varname ::shellrun::runerr}]
+            set stdout_stackid [shellfilter::stack::add stdout tee_to_var -action float-locked -settings {-varname ::shellrun::runout}]
+        } else {
+            set stderr_stackid [shellfilter::stack::add stderr var -action float-locked -settings {-varname ::shellrun::runerr}]
+            set stdout_stackid [shellfilter::stack::add stdout var -action float-locked -settings {-varname ::shellrun::runout}]
+        }
+
+
+        set exitinfo [shellfilter::run $cmdargs {*}$callopts -teehandle punksh -inbuffering none -outbuffering none -stdinhandler ::repl::repl_handler]
+        shellfilter::stack::remove stderr $stderr_stackid
+        shellfilter::stack::remove stdout $stdout_stackid
+
+
+        flush stderr
+        flush stdout
+
+        #we raise an error because an error during calling is different to collecting stderr from a command, and the caller should be able to wrap in a catch
+        # to determine something other than just a nonzero exit code or output  on stderr.
+        if {[dict exists $exitinfo error]} {
+            if {[dict exists $received "-tcl"]} {
+
+            } else {
+                #todo - check errorInfo makes sense.. return -code? tailcall?
+                error [dict get $exitinfo error]
+            }
+        }
+
+        set chunklist [list]
+
+        set n [a]
+        set c ""
+        if {[dict exists $exitinfo exitcode]} {
+            set code [dict get $exitinfo exitcode]
+            if {$code == 0} {
+                set c [a+ green]
+            } else {
+                set c [a+ white bold]
+            }
+            lappend chunklist [list "info" "$c$exitinfo$n"]
+        } elseif {[dict exists $exitinfo error]} {
+            # -tcl (with error)
+            set c [a+ yellow bold]
+            lappend chunklist [list "info" "error [dict get $exitinfo error]"]
+            lappend chunklist [list "info" "errorCode [dict get $exitinfo errorCode]"]
+            lappend chunklist [list "info" "errorInfo [list [dict get $exitinfo errorInfo]]"]
+        } else {
+            # -tcl (without error)
+            set c [a+ Green white bold]
+            #lappend chunklist [list "info" "$c$exitinfo$n"]
+            lappend chunklist [list "info" [punk::ansi::ansiwrap_raw $c "\x1b\[m" "" $exitinfo]]
+        }
+
+
+        lappend chunklist [list "info" "[a+ white bold]stdout[a]"]
+        set chunk ""
+        if {[string length $::shellrun::runout]} {
+            if {$nonewline} {
+                set o [string trimright $::shellrun::runout \r\n]
+            } else {
+                set o $::shellrun::runout
+            }
+            append chunk "[a+ white normal]$o[a]\n"  ;#this newline is the display output separator - always there whether data has trailing newline or not.
+        }
+        lappend chunklist [list stdout $chunk]
+
+
+        #set c_stderr [punk::config]
+        set chunk "[a+ red bold]stderr[a]"
+        lappend chunklist [list "info" $chunk]
+
+        set chunk ""
+        if {[string length $::shellrun::runerr]} {
+            if {$nonewline} {
+                set e [string trimright $::shellrun::runerr \r\n]
+            } else {
+                set e $::shellrun::runerr
+            }
+            append chunk "$e"
+        }
+        lappend chunklist [list resulterr $chunk]
+
+
+        #set_last_run_display $chunklist
+        tsv::lappend repl runchunks-[tsv::get repl runid] {*}$chunklist
+
+        if {$nonewline} {
+            return [string trimright $::shellrun::runerr \r\n]
+        }
+        return $::shellrun::runerr
+    }
+
+
+    proc runx {args} {
+        #set_last_run_display [list]
+        variable runout
+        variable runerr
+        set runout ""
+        set runerr ""
+
+        set splitargs [get_run_opts $args]
+        set runopts [dict get $splitargs runopts]
+        set cmdargs [dict get $splitargs cmdargs]
+
+        if {"-nonewline" in $runopts} {
+            set nonewline 1
+        } else {
+            set nonewline 0
+        }
+
+        #shellfilter::stack::remove stdout $::repl::id_outstack
+
+        if {"-echo" in $runopts} {
+            #float to ensure repl transform doesn't interfere with the output data
+            set stderr_stackid [shellfilter::stack::add stderr tee_to_var -action float -settings {-varname ::shellrun::runerr}]
+            set stdout_stackid [shellfilter::stack::add stdout tee_to_var -action float -settings {-varname ::shellrun::runout}]
+        } else {
+            #set stderr_stackid [shellfilter::stack::add stderr var -action sink-locked -settings {-varname ::shellrun::runerr}]
+            #set stdout_stackid [shellfilter::stack::add stdout var -action sink-locked -settings {-varname ::shellrun::runout}]
+
+            #float above the repl's tee_to_var to deliberately block it.
+            #a var transform is naturally a junction point because there is no flow-through..
+            # - but mark it with -junction 1 just to be explicit
+            set stderr_stackid [shellfilter::stack::add stderr var -action float-locked -junction 1 -settings {-varname ::shellrun::runerr}]
+            set stdout_stackid [shellfilter::stack::add stdout var -action float-locked -junction 1 -settings {-varname ::shellrun::runout}]
+        }
+
+        set callopts ""
+        if {"-tcl" in $runopts} {
+            append callopts " -tclscript 1"
+        }
+        #set exitinfo [shellfilter::run $cmdargs -teehandle punksh -inbuffering none -outbuffering none -stdinhandler ::repl::repl_handler]
+        set exitinfo [shellfilter::run $cmdargs {*}$callopts -teehandle punksh -inbuffering none -outbuffering none]
+
+        shellfilter::stack::remove stdout $stdout_stackid
+        shellfilter::stack::remove stderr $stderr_stackid
+
+
+        flush stderr
+        flush stdout
+
+        if {[dict exists $exitinfo error]} {
+            if {"-tcl" in $runopts} {
+
+            } else {
+                #todo - check errorInfo makes sense.. return -code? tailcall?
+                error [dict get $exitinfo error]
+            }
+        }
+
+        #set x [shellfilter::stack::add stdout var -action sink-locked -settings {-varname ::repl::runxoutput}]
+
+        set chunk ""
+        if {[string length $::shellrun::runout]} {
+            if {$nonewline} {
+                set o [string trimright $::shellrun::runout \r\n]
+            } else {
+                set o $::shellrun::runout
+            }
+            set chunk $o
+        }
+        set chunklist [list]
+        lappend chunklist [list "info" " "]
+        lappend chunklist [list "result" stdout] ;#key 'stdout' forms part of the resulting dictionary output
+        lappend chunklist [list "info" "[a+ white bold]stdout[a]"]
+        lappend chunklist [list result $chunk]   ;#value corresponding to 'stdout' key in resulting dict
+
+
+        lappend chunklist [list "info" " "]
+        set chunk "[a+ red bold]stderr[a]"
+        lappend chunklist [list "result" $chunk]
+        lappend chunklist [list "info" stderr]
+
+        set chunk ""
+        if {[string length $::shellrun::runerr]} {
+            if {$nonewline} {
+                set e [string trimright $::shellrun::runerr \r\n]
+            } else {
+                set e $::shellrun::runerr
+            }
+            set chunk $e
+        }
+        #stderr is part of the result
+        lappend chunklist [list "resulterr" $chunk]
+
+
+
+        set n [a]
+        set c ""
+        if {[dict exists $exitinfo exitcode]} {
+            set code [dict get $exitinfo exitcode]
+            if {$code == 0} {
+                set c [a+ green]
+            } else {
+                set c [a+ yellow bold]
+            }
+            lappend chunklist [list "info" " "]
+            lappend chunklist [list "result" exitcode]
+            lappend chunklist [list "info" "exitcode $code"]
+            lappend chunklist [list "result" "$c$code$n"]
+            set exitdict [list exitcode $code]
+        } elseif {[dict exists $exitinfo result]} {
+            # presumably from a -tcl call
+            set val [dict get $exitinfo result]
+            lappend chunklist [list "info" " "]
+            lappend chunklist [list "result" result]
+            lappend chunklist [list "info"   result]
+            lappend chunklist [list "result" $val]
+            set exitdict [list result $val]
+        } elseif {[dict exists $exitinfo error]} {
+            # -tcl call with error
+            #set exitdict [dict create]
+            lappend chunklist [list "info" " "]
+            lappend chunklist [list "result" error]
+            lappend chunklist [list "info"   error]
+            lappend chunklist [list "result" [dict get $exitinfo error]]
+
+            lappend chunklist [list "info" " "]
+            lappend chunklist [list "result" errorCode]
+            lappend chunklist [list "info"   errorCode]
+            lappend chunklist [list "result" [dict get $exitinfo errorCode]]
+
+            lappend chunklist [list "info" " "]
+            lappend chunklist [list "result" errorInfo]
+            lappend chunklist [list "info"   errorInfo]
+            lappend chunklist [list "result" [dict get $exitinfo errorInfo]]
+
+            set exitdict $exitinfo
+        } else {
+            #review - if no exitcode or result. then what is it?
+            lappend chunklist [list "info" exitinfo]
+            set c [a+ yellow bold]
+            lappend chunklist [list result "$c$exitinfo$n"]
+            set exitdict [list exitinfo $exitinfo]
+        }
+
+        #set_last_run_display $chunklist
+        tsv::lappend repl runchunks-[tsv::get repl runid] {*}$chunklist
+
+        #set ::repl::result_print 0
+        #return [lindex [list [list stdout $::runout stderr $::runerr {*}$exitinfo] [shellfilter::stack::remove stdout $x][puts -nonewline stdout $pretty][set ::repl::output ""]] 0]
+
+
+        if {$nonewline} {
+            return [list {*}$exitdict stdout [string trimright $::shellrun::runout \r\n] stderr [string trimright $::shellrun::runerr \r\n]]
+        }
+        #always return exitinfo $code at beginning of dict (so that punk unknown can interpret the exit code as a unix-style bool if double evaluated)
+        return [list {*}$exitdict stdout $::shellrun::runout stderr $::shellrun::runerr]
+    }
+
+    #an experiment
+    #
+    #run as raw string instead of tcl-list - no variable subst  etc
+    #
+    #dummy repl_runraw that repl will intercept
+    proc repl_runraw {args} {
+        error "runraw: only available in repl as direct call - not from script"
+    }
+    #we can only call runraw with a single (presumably braced) string if we want to use it from both repl and tcl scripts  (why? todo with unbalanced quotes/braces?)
+    proc runraw {commandline} {
+        #runraw fails as intended - because we can't bypass exec/open interference quoting :/
+        #set_last_run_display [list]
+        variable runout
+        variable runerr
+        set runout ""
+        set runerr ""
+
+        #return [shellfilter::run [lrange $args 1 end] -teehandle punksh -inbuffering none -outbuffering none -stdinhandler ::repl::repl_handler]
+        puts stdout ">>runraw got: $commandline"
+
+        #run always echoes anyway.. as we aren't diverting stdout/stderr off for capturing
+        #for consistency with other runxxx commands - we'll just consume it. (review)
+
+        set reallyraw 1
+        if {$reallyraw} {
+            set wordparts [regexp -inline -all {\S+} $commandline]
+            set runwords $wordparts
+        } else {
+            #shell style args parsing not suitable for windows where we can't assume matched quotes etc.
+            package require string::token::shell
+            set parts [string token shell -indices -- $commandline]
+            puts stdout ">>shellparts: $parts"
+            set runwords [list]
+            foreach p $parts {
+                set ptype [lindex $p 0]
+                set pval [lindex $p 3]
+                if {$ptype eq "PLAIN"} {
+                    lappend runwords [lindex $p 3]
+                } elseif {$ptype eq "D:QUOTED"} {
+                    set v {"}
+                    append v $pval
+                    append v {"}
+                    lappend runwords $v
+                } elseif {$ptype eq "S:QUOTED"} {
+                    set v {'}
+                    append v $pval
+                    append v {'}
+                    lappend runwords $v
+                }
+            }
+        }
+
+        puts stdout ">>runraw runwords: $runwords"
+        set runwords [lrange $runwords 1 end]
+
+        puts stdout ">>runraw runwords: $runwords"
+        #set args [lrange $args 1 end]
+        #set runwords [lrange $wordparts 1 end]
+
+        set known_runopts [list "-echo" "-e" "-terminal" "-t"]
+        set aliases [list "-e" "-echo" "-echo" "-echo" "-t" "-terminal" "-terminal" "-terminal"] ;#include map to self
+        set runopts [list]
+        set cmdwords [list]
+        set idx_first_cmdarg [lsearch -not $runwords "-*"]
+        set runopts [lrange $runwords 0 $idx_first_cmdarg-1]
+        set cmdwords [lrange $runwords $idx_first_cmdarg end]
+
+        foreach o $runopts {
+            if {$o ni $known_runopts} {
+                error "runraw: Unknown runoption $o"
+            }
+        }
+        set runopts [lmap o $runopts {dict get $aliases $o}]
+
+        set cmd_as_string [join $cmdwords " "]
+        puts stdout ">>cmd_as_string: $cmd_as_string"
+
+        if {"-terminal" in $runopts} {
+            #fake terminal using 'script' command.
+            #not ideal: smushes stdout & stderr together amongst other problems
+            set tcmd [shellfilter::get_scriptrun_from_cmdlist_dquote_if_not $cmdwords]
+            puts stdout ">>tcmd: $tcmd"
+            set exitinfo [shellfilter::run $tcmd -teehandle punksh -inbuffering line -outbuffering none ]
+            set exitinfo "exitcode not-implemented"
+        } else {
+            set exitinfo [shellfilter::run $cmdwords -teehandle punksh -inbuffering line -outbuffering none ]
+        }
+
+        if {[dict exists $exitinfo error]} {
+            #todo - check errorInfo makes sense.. return -code? tailcall?
+            error [dict get $exitinfo error]
+        }
+        set code [dict get $exitinfo exitcode]
+        if {$code == 0} {
+            set c [a+ green]
+        } else {
+            set c [a+ white bold]
+        }
+        puts stderr $c
+        return $exitinfo
+    }
+
+    proc sh_run {args} {
+        set splitargs [get_run_opts $args]
+        set runopts [dict get $splitargs runopts]
+        set cmdargs [dict get $splitargs cmdargs]
+        #e.g sh -c "ls -l *"
+        #we pass cmdargs to sh -c as a list, not individually
+        tailcall shellrun::run {*}$runopts sh -c $cmdargs
+    }
+    proc sh_runout {args} {
+        set splitargs [get_run_opts $args]
+        set runopts [dict get $splitargs runopts]
+        set cmdargs [dict get $splitargs cmdargs]
+        tailcall shellrun::runout {*}$runopts sh -c $cmdargs
+    }
+    proc sh_runerr {args} {
+        set splitargs [get_run_opts $args]
+        set runopts [dict get $splitargs runopts]
+        set cmdargs [dict get $splitargs cmdargs]
+        tailcall shellrun::runerr {*}$runopts sh -c $cmdargs
+    }
+    proc sh_runx {args} {
+        set splitargs [get_run_opts $args]
+        set runopts [dict get $splitargs runopts]
+        set cmdargs [dict get $splitargs cmdargs]
+        tailcall shellrun::runx {*}$runopts sh -c $cmdargs
+    }
+}
+
+namespace eval shellrun {
+    interp alias {} run     {} shellrun::run
+    interp alias {} sh_run  {} shellrun::sh_run
+    interp alias {} runout  {} shellrun::runout
+    interp alias {} sh_runout  {} shellrun::sh_runout
+    interp alias {} runerr  {} shellrun::runerr
+    interp alias {} sh_runerr  {} shellrun::sh_runerr
+    interp alias {} runx    {} shellrun::runx
+    interp alias {} sh_runx  {} shellrun::sh_runx
+
+    interp alias {} runc    {} shellrun::runconsole
+    interp alias {} runraw {} shellrun::runraw
+
+
+    #the shortened versions deliberately don't get pretty output from the repl
+    interp alias {} r       {} shellrun::run
+    interp alias {} ro      {} shellrun::runout
+    interp alias {} re      {} shellrun::runerr
+    interp alias {} rx      {} shellrun::runx
+
+
+}
+
+namespace eval shellrun {
+    proc test_cffi {} {
+        package require test_cffi
+        cffi::Wrapper create ::shellrun::kernel32 [file join $env(windir) system32 Kernel32.dll]
+        ::shellrun::kernel32 stdcall CreateProcessA
+        #todo - stuff.
+        return ::shellrun::kernel32
+    }
+
+}
+namespace eval ::punk::args::register {
+    #use fully qualified so 8.6 doesn't find existing var in global namespace
+    lappend ::punk::args::register::NAMESPACES ::shellrun
+}
+
+
+package provide shellrun [namespace eval shellrun {
+    variable version
+    set version 0.1.2
+}]

src/project_layouts/custom/_project/punk.project-0.1/src/bootsupport/modules/zzzload-0.1.0.tm

@@ -20,6 +20,7 @@
 
 package require Thread
 
+#EXPERIMENTAL.
 
 # ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++
 namespace eval zzzload {
@@ -63,6 +64,8 @@ namespace eval zzzload {
         }
         if {$loader_tid eq ""} {
             set loader_tid [thread::create -joinable -preserved]
+            #todo - set tcl::tm::list and ::auto_path in the loader thread to match the main thread.
+            #(startup process may have modified these paths)
         }
         if {![tsv::exists zzzload_pkg $pkgname]} {
             #puts stderr "zzzload pkg_require $pkgname"
@@ -85,7 +88,7 @@ namespace eval zzzload {
         }
     }
     proc pkg_wait {pkgname} {
-        if {[set ver [package provide twapi]] ne ""} {
+        if {[set ver [package provide $pkgname]] ne ""} {
             return $ver
         }
 
@@ -116,18 +119,6 @@ namespace eval zzzload {
 }
 
 
-
-
-
-
-
-
-
-
-
-
-
-
 # ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++
 ##  Ready
 package provide zzzload [namespace eval zzzload {

src/project_layouts/custom/_project/punk.project-0.1/src/make.tcl

@@ -2093,12 +2093,16 @@ if {$::punkboot::command in {project packages modules}} {
 
         #install .tm *and other files*
         puts stdout "VENDORMODULES$which: copying from $sourcefolder/$vf to $target_module_folder (if source file changed)"
-        set resultdict [punkcheck::install $sourcefolder/$vf  $target_module_folder\
+        set resultdict [punkcheck::install $sourcefolder/$vf  $target_module_folder {*}{
-            -installer make.tcl\
+            -installer make.tcl
-            -overwrite installedsourcechanged-targets\
+            -overwrite installedsourcechanged-targets
-            -antiglob_paths {README.md AGENTS.md **/AGENTS.md include_modules.config}\
+            -progresschannel stdout
-            -progresschannel stdout\
+            -exclude-filetails {AGENTS.md include_modules.config}
-            ]
+            -antiglob_paths {README.md}
+        }]
+        # -antiglob_paths {README.md AGENTS.md **/AGENTS.md include_modules.config}
+        #-exclude-filetails {AGENTS.md include_modules.config}
+        #-antiglob_paths {README.md}
         puts stdout [punkcheck::summarize_install_resultdict $resultdict]
     }
     if {![llength $vendormodulefolders]} {

src/project_layouts/custom/_project/punk.shell-0.1/src/bootsupport/modules/fauxlink-0.1.1.tm

@@ -205,7 +205,7 @@ namespace eval fauxlink {
     #    %2F     "/"
     #    %2f     "/"
     #    %7f     (del)
-    #we exlude the forward slash because we already have + for that - and multiple ways to specify it obscure intention.
+    #we exclude the forward slash because we already have + for that - and multiple ways to specify it obscure intention.
     #
     set decode_map [dict merge $decode_map [dict create\
         %09     \t\

src/project_layouts/custom/_project/punk.shell-0.1/src/bootsupport/modules/funcl-0.1.tm

@@ -1,3 +1,6 @@
+
+#experimental.
+
 package provide funcl [namespace eval funcl {
     variable version
     set version 0.1
@@ -235,7 +238,7 @@ namespace eval funcl {
         }
         set comp [list] ;#composition list
         set end [lindex $args end]
-        if {[lindex $end 0] in {_fn _call}]} {
+        if {[lindex $end 0] in {_fn _call}} {
             #is_funcl
             set endfunc [lindex $args end]
         } else {

src/project_layouts/custom/_project/punk.shell-0.1/src/bootsupport/modules/oolib-0.1.3.tm

@@ -0,0 +1,200 @@
+#JMN - api should be kept in sync with package patternlib where possible
+#
+
+namespace eval oolib {
+    oo::class create collection {
+        variable o_data ;#dict
+        #variable o_alias
+        constructor {} {
+            set o_data [dict create]
+        }
+        method info {} {
+            return [dict info $o_data]
+        }
+        method count {} {
+            return [dict size $o_data]
+        }
+        method isEmpty {} {
+            expr {[dict size $o_data] == 0}
+        }
+        method names {{globOrIdx {}}} {
+            if {[llength $globOrIdx]} {
+                if {[string is integer -strict $globOrIdx]} {
+                    set idx $globOrIdx
+                    if {$idx < 0} {
+                        set idx "end-[expr {abs($idx + 1)}]"
+                    }
+                    if {[catch {lindex [dict keys $o_data] $idx} result]} {
+                        error "[self object] no such index : '$idx'"
+                    } else {
+                        return $result
+                    }
+                } else {
+                    #glob
+                    return [lsearch -glob -all -inline [dict keys $o_data] $globOrIdx]
+                }
+            } else {
+                return [dict keys $o_data]
+            }
+        }
+        #like names but without globbing
+        method keys {} {
+            dict keys $o_data
+        }
+        method key {{posn 0}} {
+            if {$posn < 0} {
+                set posn "end-[expr {abs($posn + 1)}]"
+            }
+            if {[catch {lindex [dict keys $o_data] $posn} result]} {
+                error "[self object] no such index : '$posn'"
+            } else {
+                return $result
+            }
+        }
+        method hasKey {key} {
+            dict exists $o_data $key
+        }
+        method get {} {
+            return $o_data
+        }
+        method items {} {
+            return [dict values $o_data]
+        }
+        method item {key} {
+            if {[string is integer -strict $key]} {
+                if {$key >= 0} {
+                    set valposn [expr {(2*$key) +1}]
+                    return [lindex $o_data $valposn]
+                } else {
+                    set key "end-[expr {abs($key + 1)}]"
+                    return [lindex $o_data $key]
+                    #return [lindex [dict keys $o_data] $key]
+                }
+            }
+            if {[dict exists $o_data $key]} {
+                return [dict get $o_data $key]
+            }
+        }
+        #inverse lookup
+        method itemKeys {value} {
+            set value_indices [lsearch -all [dict values $o_data] $value]
+            set keylist [list]
+            foreach i $value_indices {
+                set idx [expr {(($i + 1) *2) -2}]
+                lappend keylist [lindex $o_data $idx]
+            }
+            return $keylist
+        }
+        method search {value args} {
+            set matches [lsearch {*}$args [dict values $o_data] $value]
+            if {"-inline" in $args} {
+                return $matches
+            } else {
+                set keylist [list]
+                foreach i $matches {
+                    set idx [expr {(($i + 1) *2) -2}]
+                    lappend keylist [lindex $o_data $idx]
+                }
+                return $keylist
+            }
+        }
+        #review - see patternlib. Is the intention for aliases to be configurable independent of whether the target exists?
+        #review - what is the point of alias anyway? - why slow down other operations when a variable can hold a keyname perfectly well?
+        #method alias {newAlias existingKeyOrAlias} {
+        #    if {[string is integer -strict $newAlias]} {
+        #        error "[self object] collection key alias cannot be integer"
+        #    }
+        #    if {[string length $existingKeyOrAlias]} {
+        #        set o_alias($newAlias) $existingKeyOrAlias
+        #    } else {
+        #        unset o_alias($newAlias)
+        #    }
+        #}
+        #method aliases {{key ""}} {
+        #    if {[string length $key]} {
+        #        set result [list]
+        #        foreach {n v} [array get o_alias] {
+        #            if {$v eq $key} {
+        #                lappend result $n $v
+        #            }
+        #        }
+        #        return $result
+        #    } else {
+        #        return [array get o_alias]
+        #    }
+        #}
+        ##if the supplied index is an alias, return the underlying key; else return the index supplied.
+        #method realKey {idx} {
+        #    if {[catch {set o_alias($idx)} key]} {
+        #        return $idx
+        #    } else {
+        #        return $key
+        #    }
+        #}
+        method add {value key} {
+            if {[string is integer -strict $key]} {
+                error "[self object] collection key must not be an integer. Use another structure if integer keys required"
+            }
+            if {[dict exists $o_data $key]} {
+                error "[self object] col_processors object error: key '$key' already exists in collection"
+            }
+            dict set o_data $key $value
+            return [expr {[dict size $o_data] - 1}] ;#return index of item
+        }
+        method remove {idx {endRange ""}} {
+            if {[string length $endRange]} {
+                error "[self object] collection error: ranged removal not yet implemented.. remove one item at a time"
+            }
+            if {[string is integer -strict $idx]} {
+                if {$idx < 0} {
+                    set idx "end-[expr {abs($idx+1)}]"
+                }
+                set key [lindex [dict keys $o_data] $idx]
+                set posn $idx
+            } else {
+                set key $idx
+                set posn [lsearch -exact [dict keys $o_data] $key]
+                if {$posn < 0} {
+                    error "[self object] no such index: '$idx' in this collection"
+                }
+            }
+            dict unset o_data $key
+            return
+        }
+        method clear {} {
+            set o_data [dict create]
+            return
+        }
+        method reverse_the_collection {} {
+            #named slightly obtusely because reversing the data when there may be references held is a potential source of bugs
+            #the name reverse_the_collection should make it clear that the object is being modified in place as opposed to simply 'reverse' which may imply a view/copy.
+            #todo - consider implementing a get_reverse which provides an interface to the same collection without affecting original references, yet both allowing delete/edit operations.
+            set dictnew [dict create]
+            foreach k [lreverse [dict keys $o_data]] {
+                dict set dictnew $k [dict get $o_data $k]
+            }
+            set o_data $dictnew
+            return
+        }
+        #review - cmd as list vs cmd as script?
+        method map {cmd} {
+            set seed [list]
+            dict for {k v} $o_data {
+                lappend seed [uplevel #0 [list {*}$cmd $v]]
+            }
+            return $seed
+        }
+        method objectmap {cmd} {
+            set seed [list]
+            dict for {k v} $o_data {
+                lappend seed [uplevel #0 [list $v {*}$cmd]]
+            }
+            return $seed
+        }
+    }
+}
+
+package provide oolib [namespace eval oolib {
+    variable version
+    set version 0.1.3
+}]

src/project_layouts/custom/_project/punk.shell-0.1/src/bootsupport/modules/punk/aliascore-0.1.0.tm

@@ -123,6 +123,7 @@ tcl::namespace::eval punk::aliascore {
         ansistrip     ::punk::ansi::ansistrip
         stripansi     ::punk::ansi::ansistrip
         ansiwrap      ::punk::ansi::ansiwrap
+        ansisplit     ::punk::ansi::ta::split_codes_single
         grepstr       ::punk::ansi::grepstr
         untabify      ::punk::ansi::untabify
         colour        ::punk::console::colour

src/project_layouts/custom/_project/punk.shell-0.1/src/bootsupport/modules/punk/ansi/sauce-0.1.0.tm

@@ -39,32 +39,35 @@ tcl::namespace::eval punk::ansi::sauce {
 
     proc from_file {fname} {
         if {[file size $fname] < 128} {
-            return
+            return [dict create posn -1]
         }
         set fd [open $fname r]
         chan conf $fd -translation binary
         chan seek $fd -128 end
+        set sauce_block_posn [expr {[chan tell $fd] -1}] ;#entire sauce block including ctrl-z and any comments - initial value assuming no comments
+        #If we treat the ctrl-z (\x1a) as part of the sauce - actual start of entire sauce info is 1 before sauce_header_posn,
+        #or further back if there are comments.
         set srec [read $fd]
         set srec_len 128 ;#This is the normal length of a SAUCE record - we may need to set it shorter if truncation detected
         if {[catch {set sdict [to_dict $srec]}]} {
             #review - have seen truncated SAUCE records < 128 bytes
             #we could search for SAUCE00 in the tail and see what records can be parsed?
             #specifically  publicdomain roysac images sometimes only 99 Bytes of sauce - suspect remaining were null \x0 padded and trimmed
-            set sauceposn [string first SAUCE00 $srec]
+            set saucestart [string first SAUCE00 $srec]
-            if {$sauceposn <= 0} {
+            if {$saucestart <= 0} {
                 close $fd
-                return
+                return [dict create posn -1]
             }
             #emit something to give user an indication something isn't right
             puts stderr "punk::ansi::sauce::from_file WARNING SAUCE record seems to be truncated - padding rhs with nulls and trying again.."
             #SAUCE00 is not at the beginning
             #pad the tail with nulls and try again
-            set srec [string range $srec $sauceposn end]
+            set srec [string range $srec $saucestart end]
             set srec_len [string length $srec]
             set srec ${srec}[string repeat \x0 [expr {128 - [string length $srec]}]]
             if {[catch {set sdict [to_dict $srec]}]} {
                 close $fd
-                return
+                return [dict create posn -1]
             }
             dict set sdict warning "SAUCE truncation to $srec_len bytes detected"
         }
@@ -73,6 +76,7 @@ tcl::namespace::eval punk::ansi::sauce {
             #Use srec_len instead of 128 - in case we had truncated source record which we padded and were able to parse
             set offset [expr {-1 *($srec_len + ($clines * 64) + 5)}]
             chan seek $fd $offset end
+            set sauce_block_posn [expr {[chan tell $fd] -1}] ;#entire sauce block including ctrl-z and any comments
             set tag [chan read $fd 5]
             if {$tag eq "COMNT"} {
                 #'character' data - shouldn't be null terminated c-style string - but can be
@@ -95,6 +99,7 @@ tcl::namespace::eval punk::ansi::sauce {
                 dict set sdict commentlines $commentlines
             }
         }
+        dict set sdict posn $sauce_block_posn
         close $fd
         return $sdict
     }
@@ -213,7 +218,9 @@ tcl::namespace::eval punk::ansi::sauce {
 
 
 
-
+    #---------------------------------------------------------------------------------------------------------------------------------------------
+    # This data comes from the sauce spec.
+    #---------------------------------------------------------------------------------------------------------------------------------------------
     #todo - fontName - which can also specify e.g code page 437
     ## Font name [1]	Font size	Resolution [2]	Aspect ratio [3]        Vertical stretch [6]	Description
     ##                                              Display [4]	Pixel [5]
@@ -221,7 +228,14 @@ tcl::namespace::eval punk::ansi::sauce {
     set fontnames [dict create]
 
     ## IBM VGA	9×16 [7]	720×400	4:3	20:27 (1:1.35)	35%	Standard hardware font on VGA cards for 80×25 text mode (code page 437)
-    dict set fontnames "IBM VGA" [list fontsize "9x16" resolution "720x400" aspect_ratio_display "4:3" aspect_ratio_pixel "20:27 (1:1.35)" vertical_stretch "35%" description "Standard hardware font on VGA cards for 80×25 text mode (code page 437)"]
+    dict set fontnames "IBM VGA" [list {*}{
+        fontsize                "9x16"
+        resolution              "720x400"
+        aspect_ratio_display    "4:3"
+        aspect_ratio_pixel      "20:27 (1:1.35)"
+        vertical_stretch        "35%"
+        description             "Standard hardware font on VGA cards for 80×25 text mode (code page 437)"
+    }]
     ## IBM VGA ### [8]	9×16 [7]	720×400	4:3	20:27 (1:1.35)	35%	Software installed code page font for VGA 80×25 text mode
     # - where ### is placeholder for 437,720,737 etc
 
@@ -247,6 +261,7 @@ tcl::namespace::eval punk::ansi::sauce {
     ## C64 PETSCII unshifted	8×8 [10]	320×200	4:3	5:6 (1:1.2)	20%	Original Commodore PETSCII font (PET, VIC-20, C64, CBM-II, Plus/4, C16, C116 and C128) in the unshifted mode. Unshifted mode (graphics) only has uppercase letters and additional graphic characters. This is the normal boot font.
     ## C64 PETSCII shifted	8×8 [10]	320×200	4:3	5:6 (1:1.2)	20%	Original PETSCII font in shifted mode. Shifted mode (text) has both uppercase and lowercase letters. This mode is actuated by pressing Shift+Commodore key.
     ## Atari ATASCII	8×8 [11]	320×192	4:3	4:5 (1:1.25)	25%	Original ATASCII font (Atari 400, 800, XL, XE)
+    #---------------------------------------------------------------------------------------------------------------------------------------------
 
 
     #expect a 128 Byte sauce record
@@ -256,6 +271,7 @@ tcl::namespace::eval punk::ansi::sauce {
         variable datatypes
         variable filetypes
         variable encodings
+        set warnings [list]
         if {[string length $saucerecord] != 128} {
             error "punk::ansi::sauce::to_dict: Unable to interpret data as a SAUCE record - length != 128"
         }
@@ -321,6 +337,8 @@ tcl::namespace::eval punk::ansi::sauce {
                 dict set sdict filetype_name ""
             }
         } else {
+            #how can a byte fail to scan with cu? is this even reachable?
+            puts stderr "punk::ansi::sauce::to_dict filetype byte failed to scan - setting filetype and filetype_name to empty string byte: [ansistring VIEW -lf 1 [string range $saucerecord 95 95]]"
             dict set sdict filetype ""
             dict set sdict filetype_name ""
         }
@@ -417,25 +435,40 @@ tcl::namespace::eval punk::ansi::sauce {
             5 {
                 #binarytext
                 #filetype is supposed to represent half the characterwidth (only widths with multiples of 2 can be specified)
-                #HOWEVER - in the wild we may see width/height specified in tinfo1/tinfo2 with some other value in filetype (eg 1)
+                #HOWEVER - in the wild we may see width/height specified in tinfo1/tinfo2 with some apparently unrelated value in filetype (eg 0 or 1) that doesn't match the intended image dimensions.
-                #If both tinfo1 and tinfo2 are non zero - we will use them, even though it's not in spec.
+                #If both tinfo1 and tinfo2 are non zero - we *could* use them, even though it's not in spec.
-                set t1 [dict get $sdict tinfo1]
+                #An example file (us-used1.bin) has filetype 0 and tinfo1/tinfo2 640/350
-                if {$t1 eq ""} {
+                #It's possible tinfo1/tinfo2 represent pixel dimensions for a 'standard' 8x16 font, but this image is 160 columns wide, so we would expect tinfo1 to be 1280.
-                    set t1 0
+                #The sauce spec seems to indicate we should ignore tinfo1/tinfo2 for binarytext and only use filetype to determine width.
-                }
+                #the default for binarytext is 160 columns.
-                set t2 [dict get $sdict tinfo2]
+
-                if {$t2 eq ""} {
+                #filetype 1 is theoretically possible, representing 2 columns
-                    set t2 0
+                #in practice we see this value for binarytext images that are definitely not intended to be 2 columns wide. Why?
+                #is there some assumption that that images are at least a certain width, and filetype has been repurposed to indicate something else?
+                #The spec would seem to rule out images of a single column due to filetype being half the character width but a value of 0.5 isn't supported.
+                #It specifically mentions that only even widths up to 510 can be specified. ($filetype * 2 where filetype is 1-255?)
+
+
+                #proper mechanism to specify columns for binarytext is the datatype field.
+                set cols [expr {2*[dict get $sdict filetype]}]
+                if {$cols == 0} {
+                    lappend warnings "binarytext filetype value of [dict get $sdict filetype] - using binarytext default cols of 160"
+                     #default for binarytext is 160 columns
+                    set cols 160
                 }
-                if {$t1 != 0 && $t2 != 0} {
+                if {$cols == 2 && [dict get $sdict tinfo1] != 0 && [dict get $sdict tinfo2] != 0} {
                     #not to spec - but we will assume these have values for a reason..
-                    puts stderr "punk::ansi::sauce::to_dict using tinfo1/tinfo2 data for columns/rows (non-compliant SAUCE data)"
+                    #---------------------------------------------------------------------------------------------------------------------
-                    dict set sdict columns [expr {2 * $t1}]
+                    #The sample file src/testansi/formatsamples/image/binaryText/test.bin has a filetype 1 and tinfo1 40 and tinfo2 25.
-                    dict set sdict rows $t2
+                    #(similarly ppe-ansi.bin has tinfo1 80 and tinfo2 26)
+                    #They seem to use the 1 in filetype to indicate that the tinfo1/tinfo2 values should be used.
+                    #(The 80 cols wide test.bin binaryText image matches the xbin sample file src/testansi/formatsamples/image/xbin/test.xb which is a more fully specified format using a header)
+                    #---------------------------------------------------------------------------------------------------------------------
+                    lappend warnings "binarytext filetype of 1 with non-zero tinfo1/tinfo2 - using tinfo1/tinfo2 data for columns/rows (possibly non-conforming SAUCE data - matching observed data in the wild)"
+                    set cols [expr {2 * [dict get $sdict tinfo1]}]
+                    dict set sdict columns $cols
+                    dict set sdict rows [dict get $sdict tinfo2]
                 } else {
-                    #proper mechanism to specify columns for binarytext is the datatype field.
-
-                    set cols [expr {2*[dict get $sdict filetype]}]
                     dict set sdict columns $cols
                     #rows must be calculated from file size
                     #rows = (filesize - sauceinfosize)/ filetype * 2 * 2
@@ -447,11 +480,13 @@ tcl::namespace::eval punk::ansi::sauce {
 
             }
             6 {
-                #xbin - only filtype is 0
+                #xbin - only filetype is 0
                 #https://web.archive.org/web/20120204063040/http://www.acid.org/info/xbin/x_spec.htm
                 dict set sdict columns [dict get $sdict tinfo1]
                 dict set sdict rows    [dict get $sdict tinfo2]
                 dict set sdict fontname [dict get $sdict tinfos]
+                #Values from sauce record are probably only informational, because xbin has an 11-byte header with width,height,fontsize and flags.
+                #presumably the header-info should take precedence over all sauce data (? review)
             }
         }
         if {[dict exists $sdict fontname]} {
@@ -474,6 +509,9 @@ tcl::namespace::eval punk::ansi::sauce {
                 }
             }
         }
+        if {[llength $warnings]} {
+            dict set sdict warnings $warnings
+        }
         return $sdict
     }
 

src/project_layouts/custom/_project/punk.shell-0.1/src/bootsupport/modules/punk/args-0.2.1.tm

@@ -9086,7 +9086,7 @@ tcl::namespace::eval punk::args {
                     }
                     if {!([tcl::string::first " " $a]>=0 || [tcl::string::first \t $a]>=0 || [tcl::string::last \n $a]>=0)} {
                         if {$OPT_ANY} {
-                            #exlude argument with whitespace from being a possible option e.g dict
+                            #exclude argument with whitespace from being a possible option e.g dict
                             #todo - passthrough of unrecognised --longopt=xxx without looking for following flag-value
                             set eposn [string first = $a]
                             if {$eposn > 2 && [string match --* $a]} {

src/project_layouts/custom/_project/punk.shell-0.1/src/bootsupport/modules/punk/char-0.1.0.tm

@@ -3033,14 +3033,18 @@ tcl::namespace::eval punk::char {
     #This still leaves a whole class of clusters.. korean etc unhandled :/
     #todo - tk::startOfCluster / tk::endOfCluster - try to get it brought into Tcl
     #https://core.tcl-lang.org/tcl/tktview/a4c7eeaf63
-    proc grapheme_split {text} {
+    proc grapheme_split {text {return list}} {
         #we should treat \r\n as a single grapheme cluster (as tk::endOfCluster does)
         set components [list]
         set csplits [combiner_split $text]
         foreach {pt combiners} [lrange $csplits 0 end-1] {
             set clist [split $pt ""]
-            lappend components {*}[lrange $clist 0 end-1]
+            #review
-            lappend components [tcl::string::cat [lindex $clist end] $combiners]
+            #lset clist end [tcl::string::cat [lindex $clist end] $combiners]
+            ledit clist end end [tcl::string::cat [lindex $clist end] $combiners]
+            lappend components {*}$clist
+            #lappend components {*}[lrange $clist 0 end-1]
+            #lappend components [tcl::string::cat [lindex $clist end] $combiners]
         }
         #last csplit never has a combiner (_perlish_split style) - and may be empty - in which case we don't append it as a grapheme
         if {[lindex $csplits end] ne ""} {
@@ -3066,7 +3070,6 @@ tcl::namespace::eval punk::char {
 
         #review \uFE0F variation selector 16 - forces emoji presentation for preceding char
 
-        if 1 {
         #This is a basic implementation that does not check that all combinations are valid.
         set graphemes [list]
         set current_cluster ""
@@ -3085,13 +3088,13 @@ tcl::namespace::eval punk::char {
                 lappend graphemes "\r\n"
                 incr i ;#skip the \n as we've already processed it as part of the cluster
                 set current_cluster ""
-                    grapheme_split::reset_base
+                set cluster_base 0; set cluster_base_RI 0 ;#grapheme_split::reset_base
                 set current_cluster_is_extensible 0
             } elseif {$component eq "\u200d"} {
                 if {$current_cluster eq ""} {
                     #ZWJ at start of string - treat as separate grapheme cluster - but isn't a valid base for further combining with more ZWJs or modifiers
                     set current_cluster $component
-                        grapheme_split::reset_base
+                    set cluster_base 0; set cluster_base_RI 0 ;#grapheme_split::reset_base
                     set current_cluster_is_extensible 0
                 } else {
                     if {$cluster_base} {
@@ -3103,7 +3106,7 @@ tcl::namespace::eval punk::char {
                             append current_cluster $component
                             if {$cluster_base_RI} {
                                 #regional indicators can only combine in pairs, and don't combine with modifiers or ZWJs to form longer clusters - so not extensible.
-                                    grapheme_split::reset_base
+                                set cluster_base 0; set cluster_base_RI 0 ;#grapheme_split::reset_base
                                 set current_cluster_is_extensible 0
                                 #we can keep adding ZWJs or modifiers though
                             } else {
@@ -3122,13 +3125,13 @@ tcl::namespace::eval punk::char {
                 if {$current_cluster eq ""} {
                     #modifier at start of string -  not a valid base for further combining with more modifiers or ZWJs - but we continue appending modifiers to the current cluster.
                     set current_cluster $component
-                        grapheme_split::reset_base
+                    set cluster_base 0; set cluster_base_RI 0 ;#grapheme_split::reset_base
                 } else {
                     if {$cluster_base} {
                         if {$current_cluster_is_extensible} {
                             append current_cluster $component
                             #invalidate the base!
-                                grapheme_split::reset_base
+                            set cluster_base 0; set cluster_base_RI 0 ;#grapheme_split::reset_base
                         } else {
                             append current_cluster $component
                         }
@@ -3155,7 +3158,7 @@ tcl::namespace::eval punk::char {
                         } else {
                             lappend graphemes $current_cluster
                             set current_cluster $component
-                                grapheme_split::reset_base
+                            set cluster_base 0; set cluster_base_RI 0 ;#grapheme_split::reset_base
                         }
                         set current_cluster_is_extensible 0
                     } elseif {$cluster_base_RI} {
@@ -3165,7 +3168,7 @@ tcl::namespace::eval punk::char {
 
                             #invalidate the base - we can't combine more than 2 RIs in a cluster, and they don't combine with modifiers or ZWJs to form longer clusters.
                             #we can however add more ZWJs or modifiers to the cluster - but they don't make it extensible for combining with more RIs
-                                grapheme_split::reset_base
+                            set cluster_base 0; set cluster_base_RI 0 ;#grapheme_split::reset_base
                         } else {
                             #something else while RI cluster is open - end the current cluster and start a new one with the current char.
                             lappend graphemes $current_cluster
@@ -3182,11 +3185,11 @@ tcl::namespace::eval punk::char {
         if {$current_cluster ne ""} {
             lappend graphemes $current_cluster
         }
+        if {$return eq "list"} {
+             return $graphemes
         } else {
-            set graphemes $components
+             return [dict create list $graphemes last_extensible $current_cluster_is_extensible base $cluster_base RI_base $cluster_base_RI]
         }
-
-        return $graphemes
     }
     namespace eval grapheme_split {
         proc about {} {

src/project_layouts/custom/_project/punk.shell-0.1/src/bootsupport/modules/punk/console-0.1.1.tm

@@ -71,11 +71,6 @@ package require punk::args
 
 
 
-#if {"windows" eq $::tcl_platform(platform)} {
-#    #package require zzzload
-#    #zzzload::pkg_require twapi
-#}
-
 #see https://learn.microsoft.com/en-us/windows/console/classic-vs-vt
 #https://learn.microsoft.com/en-us/windows/console/creating-a-pseudoconsole-session
 

src/project_layouts/custom/_project/punk.shell-0.1/src/bootsupport/modules/punk/du-0.1.0.tm

@@ -2529,21 +2529,30 @@ namespace eval punk::du {
                     #jmn disable twapi
                     #tailcall du_dirlisting_generic $folderpath {*}$args
 
-                    package require zzzload
+                    #package require zzzload
-                    set loadstate [zzzload::pkg_require twapi]
+                    #set loadstate [zzzload::pkg_require twapi]
-
+
-                    if {$loadstate ni [list loading failed]} {
+                    #if {$loadstate ni [list loading failed]} {
-                        #either already loaded by zzload or ordinary package require
+                    #    #either already loaded by zzload or ordinary package require
-                        package require twapi ;#should be fast once twapi dll loaded in zzzload thread
+                    #    package require twapi ;#should be fast once twapi dll loaded in zzzload thread
+                    #    set ::punk::du::has_twapi 1
+                    #    punk::du::active::set_active_function du_dirlisting du_dirlisting_twapi
+                    #    tailcall du_dirlisting_twapi $folderpath {*}$args
+                    #} else {
+                    #    if {$loadstate eq "failed"} {
+                    #        puts stderr "punk::du defaulting to du_dirlisting_generic because twapi load failed"
+                    #        punk::du::active::set_active_function du_dirlisting du_dirlisting_generic
+                    #    }
+                    #    tailcall du_dirlisting_generic $folderpath {*}$args
+                    #}
+                    if {[catch {package require twapi} errM]} {
+                        puts stderr "punk::du defaulting to du_dirlisting_generic because twapi load failed: $errM"
+                        punk::du::active::set_active_function du_dirlisting du_dirlisting_generic
+                        tailcall du_dirlisting_generic $folderpath {*}$args
+                    } else {
                         set ::punk::du::has_twapi 1
                         punk::du::active::set_active_function du_dirlisting du_dirlisting_twapi
                         tailcall du_dirlisting_twapi $folderpath {*}$args
-                    } else {
-                        if {$loadstate eq "failed"} {
-                            puts stderr "punk::du defaulting to du_dirlisting_generic because twapi load failed"
-                            punk::du::active::set_active_function du_dirlisting du_dirlisting_generic
-                        }
-                        tailcall du_dirlisting_generic $folderpath {*}$args
                     }
                 }
                 default {

src/project_layouts/custom/_project/punk.shell-0.1/src/bootsupport/modules/punk/lib-0.1.6.tm

@@ -138,25 +138,10 @@ tcl::namespace::eval punk::lib::check {
         if {"windows" ne $::tcl_platform(platform)} {
             set bug 0
         } else {
-            if {![catch {file tempdir} tmpdir]} {
+            set tmpdir [punk::lib::tempdir_newfolder] ;#uses 'file tempdir' on tcl9+ or fallback to env vars or current directory on older versions
-                #tcl 9+ has 'file tempdir'
-                set testfile [file join $tmpdir "bugtest"]
-            } else {
-                #fallback for older tcl versions - use env TEMP/TMP or current directory
-                set tmpdir ""
-                foreach e {TEMP TMP} {
-                    if {[info exists ::env($e)] && [file isdirectory ::env($e)]} {
-                        set tmpdir ::env($e)
-                        break
-                    }
-                }
-                if {$tmpdir eq ""} {
-                    #no env vars - fallback to current directory
-                    set tmpdir [pwd]
-                }
             set testfile [file join $tmpdir "bugtest"]
-            }
 
+            try {
                 set fd [open $testfile w]
                 puts $fd test
                 close $fd
@@ -170,6 +155,14 @@ tcl::namespace::eval punk::lib::check {
                         break
                     }
                 }
+            } finally {
+                if {[file exists $testfile]} {
+                    file delete $testfile
+                }
+                if {[file exists $tmpdir]} {
+                    file delete -force $tmpdir
+                }
+            }
         }
         #possibly related anomaly is that a returned result with case that doesn't match that of the file in the underlying filesystem can't be normalized
         # to the correct case without doing some sort of string manipulation on the result such as 'string range $f 0 end' to affect the internal representation.
@@ -679,7 +672,207 @@ namespace eval punk::lib {
         }
     }
 
+    namespace eval argdoc {
+        variable PUNKARGS
+        lappend PUNKARGS [list {
+            @id     -id     ::punk::lib::tempdir
+            @cmd    -name     punk::lib::tempdir\
+                -summary\
+                "Determine an appropriate temp dir for the process we are running under."\
+                -help\
+                "On windows:
+                  If the process is running under the system account use the modern windows location: %SystemRoot%\SystemTemp
+                  Detection of the system account relies on either twapi, or a combination of the whoami command and the 
+                  registry package.
+                  Proceeds with same fallback logic as for other platforms if we fail to detect the system account or its temp location.
+
+
+                For other platforms we use the environment variables TMPDIR/TEMP/TMP or fallback to /tmp or /var/tmp if those
+                env vars aren't set or aren't writable directories.
+
+                Final fallback attempt is the current working directory.
+                Result is normalized so resulting path will have forward slashes on all platforms.
+
+                Alternatives: see the tcllib fileutil::tempdir function.
+                "
+            @values -min 0 -max 0
+        }]
+    }
+    proc tempdir {} {
+        set trydirs [list]
+        if {"windows" eq $::tcl_platform(platform)} {
+            #review.
+            #consider also checking for whether running under various service accounts
+
+            if {![catch {package require twapi}]} {
+                set tok [twapi::open_process_token] ;#first call is a little pricy.
+                set sid [twapi::get_token_user $tok]
+                if {$sid eq "S-1-5-18"} {
+                    #system account - use system account temp location
+                    set sysroot [twapi::get_shell_folder csidl_windows] ;#first call is a little pricy.
+                    lappend trydirs [file join $sysroot "SystemTemp"]
+                }
+                #if not system account - use env vars as first choice.
+            } else {
+                #twapi not available - try to use builtin windows whoami to detect if we're running under system account - but this is less reliable than using twapi to check the process token - so we warn about it.
+                set whoami_exe [auto_execok whoami]
+                #test that system32 is somewhere in the whoami path as a basic attempt to avoid non-windows whoami commands that might be present in the path
+                set whoami_exe_parts [file split $whoami_exe]
+                if {"system32" in [string tolower $whoami_exe_parts]} {
+                    set whoamiresult [string trim [exec {*}$whoami_exe /USER /FO LIST] \n\r]
+                    set whoamiresult [string map {\r\n \n} $whoamiresult]
+                    set whoamiresult_lines [split $whoamiresult \n]
+                    set sid ""
+                    foreach line $whoamiresult_lines {
+                        if {[string match "SID:*" $line]} {
+                            set sid [lindex $line 1]
+                            break
+                        }
+                    }
+                    set has_registry [expr {![catch {package require registry}]}]
+                    if {$sid eq "S-1-5-18"} {
+                        #system account - use system account temp location
+                        set sysroot ""
+                        if {$has_registry} {
+                            #registry path is case-insensitive.
+                            catch {
+                                set sysroot [registry get {HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Windows NT\CurrentVersion} SystemRoot]
+                            }
+                        } else {
+                            if {[info exists ::env(SystemRoot)]} {
+                                set sysroot [set ::env(SystemRoot)]
+                            }
+                        }
+                        if {$sysroot ne ""} {
+                            lappend trydirs [file join $sysroot "SystemTemp"]
+                        }
+                    }
+                    #if not system account - use env vars as first choice.
+                }
+            }
+        }
+
+        foreach t {TMPDIR TEMP TMP} {
+            #TMPDIR is the posix standard as first choice for temp dir env var.
+            if {[info exists ::env($t)]} {
+                lappend trydirs $::env($t)
+            }
+        }
+
+        if {"windows" ne $::tcl_platform(platform)} {
+            #suitable for macos,linux and freebsd at least.
+            lappend trydirs [file join / tmp] [file join / var tmp]
+            #/usr/tmp is probably not a common location for a temp dir on modern unix-based systems.
+        }
+
+        foreach d $trydirs {
+            if {[file isdirectory $d] && [file writable $d]} {
+                return [file normalize $d]
+            }
+        }
+
+        #only even call 'pwd' as a last resort (mildly slow on first call).
+        set cwd [pwd]
+        if {[file isdirectory $cwd] && [file writable $cwd]} {
+            return $cwd
+        }
+
+        return -code error "punk::lib::tempdir unable to determine a suitable temp directory - tried: $trydirs"
+    }
+
+    namespace eval argdoc {
+        variable PUNKARGS
+        lappend PUNKARGS [list {
+            @id     -id     ::punk::lib::tempdir_newfolder
+            @cmd    -name     punk::lib::tempdir_newfolder\
+                -summary\
+                "Create unique folder within temp dir (or cwd as last resort)"\
+                -help\
+                "Creates a new unique folder within the temp dir determined by punk::lib::tempdir.
+                The folder is created before returning its full path and will be empty.
+                The folder is named with a tcl_ prefix followed by a random string.
+
+                See also: 'file tempdir' in tcl 9+ or fileutil::maketempdir from tcllib"
+            @opts
+                -dir -type string -default "" -help\
+                    "Base directory to create the temp folder in - defaults to the result of punk::lib::tempdir"
+                -prefix -type string -default tcl -help\
+                    "Prefix for the temp folder name
+                    An underscore is automatically appended to the prefix in the generated folder name.
+                    If prefix is the empty string - then the generated folder name will still be autoprefixed
+                    with tcl_  (consistent with tcl9 'file tempdir')"
+            @values -min 0 -max 0
+        }]
+    }
+    proc tempdir_newfolder {args} {
+        set argd [punk::args::parse $args withid ::punk::lib::tempdir_newfolder]
+        set opt_dir [dict get $argd opts -dir]
+        set opt_prefix [dict get $argd opts -prefix]
+        puts "opt_prefix: $opt_prefix"
+        if {[llength [file split $opt_prefix]] > 1} {
+            error "punk::lib::tempdir_newfolder -prefix option should not contain any path separators"
+        }
+        if {$opt_prefix eq ""} {
+            #don't allow empty prefix - for consistency with tcl9 'file tempdir' - which would still prefix with 'tcl_' even if prefix is empty string.
+            set opt_prefix "tcl"
+        }
+
+        if {$opt_dir ne ""} {
+            if {[file isdirectory $opt_dir] && [file writable $opt_dir]} {
+                set tmpbase [file normalize $opt_dir]
+            } else {
+                error "punk::lib::tempdir_newfolder -dir option '$opt_dir' is not a writable directory"
+            }
+        } else {
+            set tmpbase [punk::lib::tempdir] ;#will raise an error if no writable temp dir or cwd found.
+        }
+        #assert: tmpbase has no trailing slash - unless it is a root dir (e.g / on unix, or C:/ on windows)
+        #assert: tmpbase is normalized with forward slashes on all platforms.
 
+        set tcl9_template_base [string trimright $tmpbase "/"] ;#set it to a form that can join along with a forward slash to prefix for tcl 9 'file tempdir' style template.
+        #tcl9 'file tempdir' separates the prefix in the template from the random string with an underscore.
+
+        #now form template by always joining with a slash (even if opt_prefix is empty)
+        #(avoiding file join and file normalize to ensure template is properly formed)
+        #whether or not opt_prefix ends with an underscore - another will be added. (by file tempdir, or by our own code if tcl9 'file tempdir' isn't available)
+        #assert: opt_prefix is not empty string (will have defaulted to 'tcl') and does not contain any path separators.
+        set tcl9_template "$tcl9_template_base/$opt_prefix"
+
+
+        #tcl 9+ has 'file tempdir'
+        #we don't support the same template as 'file tempdir'
+        if {[catch {file tempdir $tcl9_template} tmpdir]} {
+
+            set prefix tcl_ ;#todo - accept option: -prefix
+
+            set chars abcddefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789
+            set nrand_chars 8
+            set maxtries 100
+            for {set i 0} {$i < $maxtries} {incr i} {
+                set dirname ${opt_prefix}_  ;#always add underscore for consistency with tcl9 'file tempdir'.
+                for {set j 0} {$j < $nrand_chars} {incr j} {
+                    append dirname [string index $chars [expr {int(rand()*62)}]]
+                }
+                set path [file join $tmpbase $dirname]
+                if {[file exists $path]} {
+                    continue
+                }
+                if {[catch {
+                        file mkdir $path
+                        if {"windows" ne $::tcl_platform(platform)} {
+                            file attributes $path -permissions 0o700
+                        }
+                    }]} {
+                    continue
+                }
+                return $path
+            }
+            return -code error "punk::lib::tempdir_newfolder unable to create unique tempdir in $tmpbase after $maxtries attempts - too many collisions - aborting"
+        }
+        #tcl 9 'file tempdir' return.
+        #normalize because on early tcl 9 versions on windows it can return with mixed forward and backward slashes when a template is supplied with forward slashes.
+        return [file normalize $tmpdir]
+    }
 
     # == == == == == == == == == == == == == == == == == == == == == == == == == == == == == == ==
     # Maintenance - This is the primary source for tm_version... functions
@@ -814,6 +1007,89 @@ namespace eval punk::lib {
             error "tm_version_required_canonical should have already returned a canonicalised versionspec - or produced an error with reason before this point"
         }
     }
+    proc tm_version_magic {} {
+        #maintenance instruction:
+        #  This specific 12 character nnnnnn.0a1.0 version number is used in text replacement when installing modules.
+        #  It should not be modified without also considering the impact on existing projects based on punkshell that may have used this version number in their own code or documentation as a placeholder for tm version handling.
+        #  A copy of this is also present in punk::lib.
+        #  Essentially this should probably never be changed - and instead deprecated if we want to stop using it and replaced with a different mechanism/placeholder.
+
+        #tm versioning is not quite the same as semantic versioning - but similarly we should see extremely high major version numbers rarely if ever in real use,
+        #even over decades of development.
+        set magicbase 999999  ;#deliberately large so given load-preference when testing!
+        #we split the literal to avoid the literal appearing here - reduce risk of accidentally converting to a release version
+        return ${magicbase}.0a1.0
+    }
+
+    namespace eval argdoc {
+        lappend PUNKARGS [list {
+            @id     -id     ::punk::lib::tm_split_name
+            @cmd    -name     punk::lib::tm_split_name\
+                -summary\
+                "Split a versioned module name into name and version parts, dropping trailing .tcl/.tm if any."\
+                -help\
+                "Splits a versioned module name (as present in a filename or namespaced name) into name and version parts,
+                Ignores any trailing .tm or .tcl file extension.
+
+                If the fullmodulename given is a namespaced name - then the returned modulename will also be namespaced,
+                but with any leading :: removed.
+
+                Returns a two element list - with the first element being the modulename and the second element being the version.
+
+                Tcl module version numbers are understood with leading zeros in each dotted part, but leading zeros are not canonical.
+
+                This split does not canonicalise the version number.
+                If the last dash-separated segment of the name doesn't look like a valid version number
+                - then it is treated as part of the modulename and an empty version string is returned.
+                e.g
+                  mymod-1.2.3.tm -> mymod 1.2.3
+                  mymod-1aa2.3.tm -> mymod-1aa2.3 {}
+                  (repeated a is not a valid version segment - so the entire '1aa2.3' is treated as part of the modulename)
+
+                see also: tm_version_canonical
+                "
+            @values -min 1 -max 1
+            fullmodulename -type string -help\
+                "The full module name to split - as present in a filename or namespaced name. E.g:
+                mymod-1.2.3
+                mymod-1.2.3.tm
+                mymod-1.2.3.tcl
+                /some/where/mymod-123.0a4.0.tm
+                mymod
+                mymod.tm
+                mymod.tcl
+                ns1::ns2::mymod-1.2.3
+                ::ns1::ns2::mymod"
+        }]
+    }
+    proc tm_split_name {fullmodulename} {
+        #split modulename (as present in a filename or namespaced name) into name/version ignoring leading namespace path
+        #ignore trailing .tm .TM if present
+        #if version doesn't pass validation - treat it as part of the modulename and return empty version string without error
+        #Up to caller to validate.
+        set lastpart [namespace tail $fullmodulename]
+        set lastpart [file tail $lastpart] ;# should be ok to use file tail now that we've ensured no namespace components
+        if {[string tolower [file extension $fullmodulename]] in {.tcl .tm}} {
+            set fileparts [split [file rootname $lastpart] -]
+        } else {
+            set fileparts [split $lastpart -]
+        }
+        if {[tm_version_isvalid [lindex $fileparts end]]} {
+            set versionsegment [lindex $fileparts end]
+            set namesegment [join [lrange $fileparts 0 end-1] -];#re-stitch
+        } else {
+            set namesegment [join $fileparts -]
+            set versionsegment ""
+        }
+        set base [string trimleft [namespace qualifiers $fullmodulename] :]
+        if {$base ne ""} {
+            set modulename "${base}::$namesegment"
+        } else {
+            set modulename $namesegment
+        }
+        return [list $modulename $versionsegment]
+    }
+
     # end tm_version... functions
     # == == == == == == == == == == == == == == == == == == == == == == == == == == == == == == ==
 
@@ -4210,6 +4486,9 @@ namespace eval punk::lib {
                                                 if {[string index $key 0] ne "%"} {
                                                     set key %$key
                                                 }
+                                                #puts "---key:'$key'"
+                                                set key [string map {; \\;} $key] ;#review
+                                                #puts "---key:'$key'"
                                                 #pipeline - use punk patterns.
                                                 % thisval.= $key= $thisval
                                             }

src/project_layouts/custom/_project/punk.shell-0.1/src/bootsupport/modules/punk/mix/cli-0.3.1.tm

@@ -499,7 +499,7 @@ namespace eval punk::mix::cli {
             # -- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- ---
             set opt_punkcheck_eventobj                   [dict get $opts -punkcheck_eventobj]
 
-            set magicversion [punk::mix::util::magic_tm_version] ;#deliberately large so given load-preference when testing
+            set magicversion [punk::mix::util::tm_version_magic] ;#deliberately large so given load-preference when testing
             set module_list [list]
 
             if {[file tail [file dirname $srcdir]] ne "src"} {

src/project_layouts/custom/_project/punk.shell-0.1/src/bootsupport/modules/punk/mix/commandset/loadedlib-0.1.0.tm

@@ -54,13 +54,18 @@ namespace eval punk::mix::commandset::loadedlib {
         if {$opt_refresh} {
             catch {package require frobznodule666} ;#ensure pkg system has loaded/searched for everything REVIEW - this doesn't result in full scans
             foreach tm_path [tcl::tm::list] {
+                #review - todo - adjust punk::path::subfolders to take arguments to do some filtering itself rather than recurse down unnecessary branches.
                 set paths_below [punk::path::subfolders -recursive $tm_path]
                 foreach folder $paths_below {
+                    if {[string match */_build/* $folder]} {continue}
                     set tail [file tail $folder]
-                    if {[string match #modpod-* $tail] || [string match #tarjar-* $tail]} {
+                    if {[string match #tarjar-* $tail]} {
+                        continue
+                    }
+                    if {[string match #modpod-* $tail]} {
+                        #manually do a 'package ifneeded' fore each module found here.
                         continue
                     }
-                    if {[string match */_build/* $folder]} {continue}
                     set relpath [string tolower [punk::path::relative $tm_path $folder]]
                     set modpath [string map {/ ::} $relpath]
                     catch {package require ${modpath}::flobrudder99}

src/project_layouts/custom/_project/punk.shell-0.1/src/bootsupport/modules/punk/mix/commandset/module-0.1.0.tm

@@ -362,7 +362,7 @@ namespace eval punk::mix::commandset::module {
         file mkdir $modulefolder
 
         set moduletail [namespace tail $modulename]
-        set magicversion [punk::mix::util::magic_tm_version] ;#deliberately large so given load-preference when testing
+        set magicversion [punk::mix::util::tm_version_magic] ;#deliberately large so given load-preference when testing
 
 
 

src/project_layouts/custom/_project/punk.shell-0.1/src/bootsupport/modules/punk/mix/commandset/project-0.1.0.tm

@@ -380,25 +380,25 @@ namespace eval punk::mix::commandset::project {
 
         puts stdout ">>> about to call punkcheck::install $layout_path $projectdir"
         set resultdict [dict create]
-        set antipaths [list\
+        set antipaths [list         {*}{
-            src/doc/*\
+            src/doc/*
-            src/doc/include/*\
+            src/doc/include/*
-            src/PROJECT_LAYOUTS_*\
+            src/PROJECT_LAYOUTS_*
-        ]
+        }]
 
-        #set antiglob_dir [list\
+        #set exclude_dirsegments [list {*}{
-        #    _ignore_*\
+        #    _ignore_*
-        #]
+        #}]
-        set antiglob_dir [list\
+        set exclude_dirsegments [list {*}{
-        ]
+        }]
 
-        #default antiglob_dir_core will stop .fossil* from being updated - which is generally desirable as these are likely to be customized
+        #default -exclude-dirsegments_core will stop .fossil* from being updated - which is generally desirable as these are likely to be customized
         if {$opt_force} {
             puts stdout "copying layout files - with force applied - overwrite all-targets"
-            set resultdict [punkcheck::install $layout_path $projectdir -installer project.new -createempty 1  -overwrite ALL-TARGETS -antiglob_paths $antipaths -antiglob_dir $antiglob_dir]
+            set resultdict [punkcheck::install $layout_path $projectdir -installer project.new -createempty 1  -overwrite ALL-TARGETS -antiglob_paths $antipaths -exclude-dirsegments $exclude_dirsegments]
         } else {
             puts stdout "copying layout files - (if source file changed)"
-            set resultdict [punkcheck::install $layout_path $projectdir -installer project.new -createempty 1 -overwrite installedsourcechanged-targets -antiglob_paths $antipaths -antiglob_dir $antiglob_dir]
+            set resultdict [punkcheck::install $layout_path $projectdir -installer project.new -createempty 1 -overwrite installedsourcechanged-targets -antiglob_paths $antipaths -exclude-dirsegments $exclude_dirsegments]
         }
         puts stdout [punkcheck::summarize_install_resultdict $resultdict]
 
@@ -412,11 +412,11 @@ namespace eval punk::mix::commandset::project {
 
         #target folders .fossil-custom and .fossil-settings may not exist. use -createdir 1 to ensure existence.
         #In this case we need to override the default dir antiglob - as .fossil-xxx folders need to be installed from template if missing, or if target is uncustomized.
-        ## default_antiglob_dir_core  [list "#*" "_aside" ".git" ".fossil*"]
+        ## default_exclude_dirsegments_core  [list "#*" "_aside" ".git" ".fossil*"]
-        set override_antiglob_dir_core [list #* _aside .git]
+        set override_exclude_dirsegments_core [list #* _aside .git]
         if {[file exists $layout_path/.fossil-custom]} {
             puts stdout "copying layout src/.fossil-custom files (if target missing or uncustomised)"
-            set resultdict [punkcheck::install $layout_path/.fossil-custom $projectdir/.fossil-custom -createdir 1 -createempty 1 -punkcheck_folder $projectdir -installer project.new -antiglob_dir_core $override_antiglob_dir_core -overwrite SYNCED-TARGETS]
+            set resultdict [punkcheck::install $layout_path/.fossil-custom $projectdir/.fossil-custom -createdir 1 -createempty 1 -punkcheck_folder $projectdir -installer project.new -exclude-dirsegments_core $override_exclude_dirsegments_core -overwrite SYNCED-TARGETS]
             puts stdout [punkcheck::summarize_install_resultdict $resultdict]
         } else {
             puts stdout "no .fossil-custom in source template - update not required"
@@ -424,7 +424,7 @@ namespace eval punk::mix::commandset::project {
 
         if {[file exists $layout_path/.fossil-settings]} {
             puts stdout "copying layout src/.fossil-settings files (if target missing or uncustomised)"
-            set resultdict [punkcheck::install $layout_path/.fossil-settings $projectdir/.fossil-settings -createdir 1 -createempty 1 -punkcheck_folder $projectdir -installer project.new -antiglob_dir_core $override_antiglob_dir_core -overwrite SYNCED-TARGETS]
+            set resultdict [punkcheck::install $layout_path/.fossil-settings $projectdir/.fossil-settings -createdir 1 -createempty 1 -punkcheck_folder $projectdir -installer project.new -exclude-dirsegments_core $override_exclude_dirsegments_core -overwrite SYNCED-TARGETS]
             puts stdout [punkcheck::summarize_install_resultdict $resultdict]
         } else {
             puts stdout "no .fossil-settings in source template - update not required"
@@ -470,8 +470,8 @@ namespace eval punk::mix::commandset::project {
         if {[file exists $projectdir/src/modules]} {
             foreach m $opt_modules {
                 #check if mod-ver.tm file or #modpod-mod-ver folder exist
-                set tmfile $projectdir/src/modules/$m-[punk::mix::util::magic_tm_version].tm
+                set tmfile $projectdir/src/modules/$m-[punk::mix::util::tm_version_magic].tm
-                set podfile $projectdir/src/modules/#modpod-$m-[punk::mix::util::magic_tm_version]/$m-[punk::mix::util::magic_tm_version].tm
+                set podfile $projectdir/src/modules/#modpod-$m-[punk::mix::util::tm_version_magic]/$m-[punk::mix::util::tm_version_magic].tm
 
                 set has_tm  [file exists $tmfile]
                 set has_pod [file exists $podfile]

src/project_layouts/custom/_project/punk.shell-0.1/src/bootsupport/modules/punk/mix/util-0.1.0.tm

@@ -367,7 +367,16 @@ namespace eval punk::mix::util {
     }
     #todo - semver conversion/validation for other systems?
 
-    proc magic_tm_version {} {
+    proc tm_version_magic {} {
+        #maintenance instruction:
+        #  This specific 12 character nnnnnn.0a1.0 version number is used in text replacement when installing modules.
+        #  It should not be modified without also considering the impact on existing projects based on punkshell that may have used this version number in their own code or documentation as a placeholder for tm version handling.
+        #  A copy of this is also present in punk::lib to aid in dependency management.
+        #  These 2 copies should be kept in sync.
+        #  Essentially this should probably never be changed - and instead deprecated if we want to stop using it and replaced with a different mechanism/placeholder.
+
+        #tm versioning is not quite the same as semantic versioning - but similarly we should see extremely high major version numbers rarely if ever in real use,
+        #even over decades of development.
         set magicbase 999999  ;#deliberately large so given load-preference when testing!
         #we split the literal to avoid the literal appearing here - reduce risk of accidentally converting to a release version
         return ${magicbase}.0a1.0

src/project_layouts/custom/_project/punk.shell-0.1/src/bootsupport/modules/punk/mod-0.1.1.tm

@@ -0,0 +1,158 @@
+#punkapps app manager
+# deck cli
+
+namespace eval punk::mod::cli {
+    namespace export help list run
+    namespace ensemble create
+
+   # namespace ensemble configure [namespace current] -unknown punk::mod::cli::_unknown
+    if 0 {
+    proc _unknown {ns args} {
+        puts stderr "punk::mod::cli::_unknown '$ns' '$args'"
+        puts stderr "punk::mod::cli::help $args"
+        puts stderr "arglen:[llength $args]"
+        punk::mod::cli::help {*}$args
+    }
+    }
+
+    #cli must have _init method - usually used to load commandsets lazily
+    #
+    variable initialised 0
+    proc _init {args} {
+        variable initialised
+        if {$initialised} {
+            return
+        }
+        #...
+        set initialised 1
+    }
+
+    proc help {args} {
+        set basehelp [punk::mix::base help {*}$args]
+        #namespace export
+        return $basehelp
+    }
+    proc getraw {appname} {
+        set app_folders [punk::config::configure running apps]
+        #todo search each app folder
+        set bases [::list]
+        set versions [::list]
+        set mains [::list]
+        set appinfo [::list bases {} mains {} versions {}]
+
+        foreach containerfolder $app_folders {
+            lappend bases $containerfolder
+            if {[file exists $containerfolder]} {
+                if {[file exists $containerfolder/$appname/main.tcl]} {
+                    #exact match - only return info for the exact one specified
+                    set namematches $appname
+                    set parts [split $appname -]
+                } else {
+                    set namematches [glob -nocomplain -dir $containerfolder -type d -tail ${appname}-*]
+                    set namematches [lsort $namematches] ;#todo - -ascii? -dictionary? natsort?
+                }
+                foreach nm $namematches {
+                    set mainfile $containerfolder/$nm/main.tcl
+                    set parts [split $nm -]
+                    if {[llength $parts] == 1} {
+                        set ver ""
+                    } else {
+                        set ver [lindex $parts end]
+                    }
+                    if {$ver ni $versions} {
+                        lappend versions $ver
+                        lappend mains $ver $mainfile
+                    } else {
+                        puts stderr "punk::apps::app version '$ver' of app '$appname' already encountered at $mainfile. (will use earliest encountered in running-config apps and ignore others of same version)"
+                    }
+                }
+            } else {
+                puts stderr "punk::apps::app missing apps_folder:'$containerfolder' Ensure apps_folder is set in punk::config"
+            }
+        }
+        dict set appinfo versions $versions
+        #todo - natsort!
+        set sorted_versions [lsort $versions]
+        set latest [lindex $sorted_versions 0]
+        if {$latest eq "" && [llength $sorted_versions] > 1} {
+            set latest [lindex $sorted_versions 1]
+        }
+        dict set appinfo latest $latest
+
+        dict set appinfo bases $bases
+        dict set appinfo mains $mains
+        return $appinfo
+    }
+
+    proc list {{glob *}} {
+        set apps_folder [punk::config::configure running apps]
+        if {[file exists $apps_folder]} {
+            if {[file exists $apps_folder/$glob]} {
+                #tailcall source $apps_folder/$glob/main.tcl
+                return $glob
+            }
+            set apps [glob -nocomplain -dir $apps_folder -type d -tail $glob]
+            if {[llength $apps] == 0} {
+                if {[string first * $glob] <0 && [string first ? $glob] <0} {
+                    #no glob chars supplied - only launch if exact match for name part
+                    set namematches [glob -nocomplain -dir $apps_folder -type d -tail ${glob}-*]
+                    set namematches [lsort $namematches] ;#todo - -ascii? -dictionary? natsort?
+                    if {[llength $namematches] > 0} {
+                        set latest [lindex $namematches end]
+                        lassign $latest nm ver
+                        #tailcall source $apps_folder/$latest/main.tcl
+                    }
+                }
+            }
+            return $apps
+        }
+    }
+
+    #todo - way to launch as separate process
+    # solo-opts only before appname - args following appname are passed to the app
+    proc run {args} {
+        set nameposn [lsearch -not $args -*]
+        if {$nameposn < 0} {
+            error "punkapp::run unable to determine application name"
+        }
+        set appname [lindex $args $nameposn]
+        set controlargs [lrange $args 0 $nameposn-1]
+        set appargs [lrange $args $nameposn+1 end]
+
+        set appinfo [punk::mod::cli::getraw $appname]
+        if {[llength [dict get $appinfo versions]]} {
+            set ver [dict get $appinfo latest]
+            puts stdout "info: $appinfo"
+            set ::argc [llength $appargs]
+            set ::argv $appargs
+            source [dict get $appinfo mains $ver]
+            if {"-hideconsole" in $controlargs} {
+                puts stderr "attempting console hide"
+                #todo - something better - a callback when window mapped?
+                after 500 {::punkapp::hide_console}
+            }
+            return $appinfo
+        } else {
+            error "punk::mod::cli unable to run '$appname'. main.tcl not found in [dict get $appinfo bases]"
+        }
+    }
+}
+
+namespace eval punk::mod::cli {
+    proc _cli {args} {
+        #don't use tailcall - base uses info level to determine caller
+        ::punk::mix::base::_cli {*}$args
+    }
+    variable default_command help
+    package require punk::mix::base
+    package require punk::overlay
+    punk::overlay::custom_from_base [namespace current] ::punk::mix::base
+}
+
+package provide punk::mod [namespace eval punk::mod {
+    variable version
+    set version 0.1.1
+}]
+
+
+

src/project_layouts/custom/_project/punk.shell-0.1/src/bootsupport/modules/punk/nav/fs-0.1.0.tm

@@ -847,7 +847,7 @@ tcl::namespace::eval punk::nav::fs {
 
               Regardless of whether -nonportable is supplied or not, some characters are not
               suitable for windows or most other platforms and will be rejected with an error.
-              An example of this is the null character (\0)."
+              An example of this is the null character (\\0)."
         @values -min 1 -max -1 -type string
         path -type string -multiple 1 -help\
             "Path(s) to create. Can be absolute or relative.

src/project_layouts/custom/_project/punk.shell-0.1/src/bootsupport/modules/punk/ns-0.1.0.tm

@@ -721,6 +721,22 @@ tcl::namespace::eval punk::ns {
     }
 
 
+    punk::args::define {
+        @id     -id ::punk::ns::nstree_list
+        @cmd    -name punk::ns::nstree_list\
+            -summary\
+            ""\
+            -help\
+            ""
+        @leaders
+            location -type path -optional 0
+        @opts
+            -subnslist -type list -default {} -help\
+            ""
+            -allbelow -type boolean -default 1 -help\
+            ""
+        @values -min 0 -max 0
+    }
     #important: add tests for tricky cases - e.g punk::m**::util  vs punk::m*::util vs punk::m*::**::util  - these should all be able to return different results depending on namespace structure.
     #e.g punk::m**::util will return punk::mix::util  but punk::m*::**::util will not because punk::mix::util is too short to match. Both will return deeper matches such as: punk::mix::commandset::repo::util
     proc nstree_list {location args} {
@@ -775,13 +791,8 @@ tcl::namespace::eval punk::ns {
         #set parent [nsprefix $ns_absolute]
         #set tail [nstail $ns_absolute]
 
-        #jjj
         #set allchildren [lsort [nseval $base [list ::namespace children]]]
-        #set allchildren [lsort [tcl::namespace::eval $base [list ::namespace children]]]
-        set allchildren [lsort [nseval $base [list ::namespace children]]]
 
-        #puts "->base:$base tailparts:$tailparts allchildren: $allchildren"
-        #puts "->base:$base tailparts:$tailparts childcount: [llength $allchildren]"
 
         #** only significant when it is the trailing part of a segment eg ::**::xxx  ::a**::xxx
         if {[llength $tailparts]} {
@@ -790,6 +801,7 @@ tcl::namespace::eval punk::ns {
                 set nslist [nstree_list $base -subnslist {} -allbelow 1]
             } elseif {[regexp {[*]{2}$} $nextglob]} {
                 set nslist [list]
+                set allchildren [lsort [nseval $base [list ::namespace children]]]
                 lappend nsmatches {*}[lsearch -all -inline -glob $allchildren [nsjoin ${base} $nextglob]]
                 foreach ch $nsmatches {
                     lappend nslist $ch
@@ -799,6 +811,7 @@ tcl::namespace::eval punk::ns {
             } else {
                 #lsearch with -glob ok even if nextglob has no globchars (no discernable speed diff, and earlier parts may have globchars anyway)
                 set nslist [list]
+                set allchildren [lsort [nseval $base [list ::namespace children]]]
                 lappend nsmatches {*}[lsearch -all -inline -glob $allchildren [nsjoin ${base} $nextglob]]
                 if {[llength $tailparts] >1 || $allbelow} {
                     foreach ch $nsmatches {
@@ -812,6 +825,7 @@ tcl::namespace::eval punk::ns {
             }
         } else {
             #puts "nstree_list: no tailparts base:$base"
+            set allchildren [lsort [nseval $base [list ::namespace children]]]
             if {$allbelow} {
                 set nsmatches $allchildren
                 set nslist [list]
@@ -2134,8 +2148,8 @@ y" {return quirkykeyscript}
         tcl::dict::set tinfo($target) procoffset 0
         tcl::dict::set tinfo($target) level [expr {[::tcl::info::level]+1}]
         tcl::dict::set tinfo($target) subcmds 0
-        puts "enter: $target -- $args"
+        puts stderr "enter: $target -- $args"
-        puts "frame-2: [::tcl::info::frame -2]"
+        #puts stderr "frame-2: [::tcl::info::frame -2]"
 
         set _cmdtrace_disabled false
     }
@@ -2481,7 +2495,7 @@ y" {return quirkykeyscript}
                         set line $traceline
                         dict set linedict $target eval_base $traceline
                         dict set linedict $target eval_offset 1
-                        puts "  step type: proc traceline:$traceline ** $args"
+                        puts "  step type: proc traceline:$traceline ** $args\x1b\[m"
                         #puts "** $callinfo"
                         if {[dict exists $callinfo cmd]} {
                             #set cmd [string trim [dict get $callinfo cmd]] ;#raw 'unexpanded' script from the stack frame
@@ -2504,8 +2518,8 @@ y" {return quirkykeyscript}
                         set eval_base   [dict get $linedict $target eval_base]
                         set eval_offset [dict get $linedict $target eval_offset]
                         set line [expr {$eval_base + ($eval_offset-1) + ($traceline-1)}]
-                        puts "stack-- $callinfo"
+                        #puts "stack-- $callinfo"
-                        puts "  step type: eval traceline: $traceline -- "
+                        puts stderr "  step type: eval traceline: $traceline -- "
                         if {[dict exists $callinfo cmd]} {
                             #set cmd [string trim [dict get $callinfo cmd]]
                             set cmdlist [lindex $args 0]
@@ -2627,6 +2641,8 @@ y" {return quirkykeyscript}
         }]
     }
     proc cmdtrace {args} {
+        #review - displaying argument values has to be done carefully. Small values are ok, but large lists or dicts can be overwhelming.
+        #Potentially we could apply some heuristics to truncate or summarise them.
         package require dictn ;#convenience to allow dictn::incr d {key subkey}
         variable tinfo
         array unset tinfo
@@ -2676,7 +2692,7 @@ y" {return quirkykeyscript}
         #if the target command has a leading colon (e.g expr alternative :) we can't put a trace directly on a fully qualified name with a triple colon such as :::
         #we will need to evaluate in the namespace
         foreach {tgt_cmd ns nscmd} $resolved_targets {
-            puts "tracing target: $tgt_cmd whilst running: $origin $arglist"
+            puts stderr "tracing target: $tgt_cmd whilst running: $origin $arglist"
 
             #::tcl::namespace::eval $ns [list ::trace add execution $nscmd enter       [list ::punk::ns::_cmdtrace_enter       ::punk::ns::linedict $tgt_cmd]]
             #::tcl::namespace::eval $ns [list ::trace add execution $nscmd enterstep   [list ::punk::ns::_cmdtrace_enterstep   ::punk::ns::linedict $tgt_cmd]]

src/project_layouts/custom/_project/punk.shell-0.1/src/bootsupport/modules/punk/overlay-0.1.1.tm

@@ -0,0 +1,192 @@
+
+
+package require punk::mix::util
+package require punk::args
+
+tcl::namespace::eval ::punk::overlay {
+    #based *loosely* on: wiki.tcl-lang.org/page/ensemble+extend
+    # extend an ensemble-like routine with the routines in some namespace
+    #
+    # e.g custom_from_base ::punk::mix::cli ::punk::mix::base
+    #
+    proc custom_from_base {routine base} {
+        if {![tcl::string::match ::* $routine]} {
+            set resolved [uplevel 1 [list ::tcl::namespace::which $routine]]
+            if {$resolved eq {}} {
+                error [list {no such routine} $routine]
+            }
+            set routine $resolved
+        }
+        set routinens [tcl::namespace::qualifiers $routine]
+        if {$routinens eq {::}} {
+            set routinens {}
+        }
+        set routinetail [tcl::namespace::tail $routine]
+
+        if {![tcl::string::match ::* $base]} {
+            set base [uplevel 1 [
+                list [tcl::namespace::which namespace] current]]::$base
+        }
+
+        if {![tcl::namespace::exists $base]} {
+            error [list {no such namespace} $base]
+        }
+
+        set base [tcl::namespace::eval $base [
+            list [tcl::namespace::which namespace] current]]
+
+
+        #while 1 {
+        #    set renamed ${routinens}::${routinetail}_[info cmdcount]
+        #    if {[namespace which $renamed] eq {}} break
+        #}
+
+        tcl::namespace::eval $routine [
+            ::list tcl::namespace::ensemble configure  $routine -unknown [
+                ::list ::apply {{base ensemble subcommand args} {
+                    ::list ${base}::_redirected $ensemble $subcommand 
+                }} $base
+            ]
+        ]
+
+        punk::mix::util::namespace_import_pattern_to_namespace_noclobber ::punk::mix::util::* ${routine}::util
+        #namespace eval ${routine}::util {
+            #::namespace import ::punk::mix::util::*
+        #}
+        punk::mix::util::namespace_import_pattern_to_namespace_noclobber ${base}::lib::* ${routine}::lib
+        #namespace eval ${routine}::lib [string map [list <base> $base] {
+        #    ::namespace import <base>::lib::*
+        #}]
+
+        tcl::namespace::eval ${routine}::lib [tcl::string::map [list <base> $base <routine> $routine] {
+            if {[tcl::namespace::exists <base>::lib]} {
+                ::set current_paths [tcl::namespace::path]
+                if {"<routine>" ni $current_paths} {
+                    ::lappend current_paths <routine>
+                }
+                tcl::namespace::path $current_paths
+            }
+        }]
+
+        tcl::namespace::eval $routine {
+            ::set exportlist [::list]
+            ::foreach cmd [tcl::info::commands [tcl::namespace::current]::*] {
+                ::set c [tcl::namespace::tail $cmd]
+                if {![tcl::string::match _* $c]} {
+                    ::lappend exportlist $c
+                }
+            }
+            tcl::namespace::export {*}$exportlist
+        }
+
+        return $routine
+    }
+    punk::args::define {
+        @id     -id     ::punk::overlay::import_commandset
+        @cmd    -name     punk::overlay::import_commandset\
+            -summary\
+            "Import commands into caller's namespace with optional prefix and separator."\
+            -help\
+            "Import commands that have been exported by another namespace into the caller's
+            namespace. Usually a prefix and optionally a separator should be used.
+            This is part of the punk::mix CLI commandset infrastructure - design in flux.
+            Todo - .toml configuration files for defining CLI configurations."
+        @values
+        prefix          -type string
+        separator       -type string -help\
+            "A string, usually punctuation, to separate the prefix and the command name
+              of the final imported command. The value \"::\" is disallowed in this context."
+        cmdnamespace    -type string -help\
+            "Namespace from which to import commands. Commands are those that have been exported."
+    }
+    #load *exported* commands from cmdnamespace into caller's namespace - prefixing each command with $prefix
+    #Note: commandset may be imported by different CLIs with different bases *at the same time*
+    #so we don't make commands from the cli or its base available automatically (will generally require fully-qualified commands to use code from cli/base) 
+    #we do load punk::mix::util::*  into the util subnamespace even though the commandset might not be loaded in a cli using punk::mix::base  i.e punk::mix::util is a common dependency for CLIs.
+    #commandsets designed to be used with a specific cli/base may choose to do their own import e.g with util::namespace_import_pattern_to_namespace_noclobber and/or set namespace path if they 
+    #want the convenience of using lib:xxx with commands coming from those packages.
+    #This won't stop the commandset being used with other cli/bases unless the import is done by looking up the callers namespace.
+    #The basic principle is that the commandset is loaded into the caller(s) with a prefix 
+    #- but commandsets should explicitly package require if they have any backwards dependencies on cli/base (which they may or may not be loaded into)
+    proc import_commandset {prefix separator cmdnamespace} {
+        set bad_seps [list "::"]
+        if {$separator in $bad_seps} {
+            error "import_commandset invalid separator '$separator'"
+        }
+        if {$prefix in $bad_seps} {
+            error "import_commandset invalid prefix '$prefix'"
+        }
+        if {"$prefix$separator" in $bad_seps} {
+            error "import_commandset invalid prefix/separator combination '$prefix$separator'"
+        }
+        if {"[string index $prefix end][string index $separator 0]" in $bad_seps} {
+            error "import_commandset invalid prefix/separator combination '$prefix$separator'"
+        }
+        #review - do we allow prefixes/separators such as a::b?
+
+        #namespace may or may not be a package
+        # allow with or without leading ::
+        if {[tcl::string::range $cmdnamespace 0 1] eq "::"} {
+            set cmdpackage [tcl::string::range $cmdnamespace 2 end]
+        } else {
+            set cmdpackage $cmdnamespace
+            set cmdnamespace ::$cmdnamespace
+        }
+
+        if {![tcl::namespace::exists $cmdnamespace]} {
+            #only do package require if the namespace not already present
+            catch {package require $cmdpackage} pkg_load_info 
+            #recheck
+            if {![tcl::namespace::exists $cmdnamespace]} {
+                set prov [package provide $cmdpackage]
+                if {[tcl::string::length $prov]} {
+                    set provinfo "(package $cmdpackage is present with version $prov)"
+                } else {
+                    set provinfo "(package $cmdpackage not present)"
+                }
+                error "punk::overlay::import_commandset supplied namespace '$cmdnamespace' doesn't exist. $provinfo  Pkg_load_result: $pkg_load_info  Usage: import_commandset prefix separator namespace"
+            }
+        }
+     
+        punk::mix::util::namespace_import_pattern_to_namespace_noclobber ::punk::mix::util::* ${cmdnamespace}::util
+
+        #let child namespace 'lib' resolve parent namespace and thus util::xxx
+        tcl::namespace::eval ${cmdnamespace}::lib [tcl::string::map [list <cmdns> $cmdnamespace] {
+            ::set nspaths [tcl::namespace::path]
+            if {"<cmdns>" ni $nspaths} {
+                ::lappend nspaths <cmdns>
+            }
+            tcl::namespace::path $nspaths
+        }]
+
+        set imported_commands [list]
+        set imported_tails [list]
+        set nscaller [uplevel 1 [list tcl::namespace::current]]
+        if {[catch {
+            #review - noclobber?
+            tcl::namespace::eval ${nscaller}::temp_import [list tcl::namespace::import ${cmdnamespace}::*]
+            foreach cmd [tcl::info::commands ${nscaller}::temp_import::*] {
+                set cmdtail [tcl::namespace::tail $cmd]
+                if {$cmdtail eq "_default"} {
+                    set import_as ${nscaller}::${prefix}
+                } else {
+                    set import_as ${nscaller}::${prefix}${separator}${cmdtail}
+                }
+                rename $cmd  $import_as
+                lappend imported_commands $import_as
+                lappend imported_tails [namespace tail $import_as]
+            }
+            #make imported commands exported so they are available to the ensemble
+            tcl::namespace::eval ${nscaller} [list namespace export {*}$imported_tails]
+        } errM]} {
+            puts stderr "Error loading commandset $prefix $separator $cmdnamespace"
+            puts stderr "err: $errM"
+        }
+        return $imported_commands
+    }
+}
+
+package provide punk::overlay [tcl::namespace::eval punk::overlay {
+    variable version
+    set version 0.1.1
+}]

src/project_layouts/custom/_project/punk.shell-0.1/src/bootsupport/modules/punk/path-0.1.0.tm

@@ -565,10 +565,45 @@ namespace eval punk::path {
                 end]]
     }
 
+
+    ## for comparison
+    #proc nsglob_as_re {glob} {
+    #    #any segment that is not just * must match exactly one segment in the path
+    #    set pats [list]
+    #    foreach seg [nsparts_cached $glob] {
+    #        switch -exact -- $seg {
+    #            "" {
+    #                lappend pats ""
+    #            }
+    #            * {
+    #                #review - ::g*t  will not find ::got:it   (won't match single inner colon) - this should be fixed
+    #                #lappend pats {[^:]*}
+    #                #negative lookahead
+    #                #any number of chars not followed by ::, followed by any number of non :
+    #                lappend pats {(?:.(?!::))*[^:]*}
+    #            }
+    #            ** {
+    #                lappend pats {.*}
+    #            }
+    #            default {
+    #                set seg [string map {. [.]} $seg]
+    #                if {[regexp {[*?]} $seg]} {
+    #                    #set pat [string map [list ** {.*} * {[^:]*} ? {[^:]}] $seg]
+    #                    set pat [string map [list ** {.*} * {(?:.(?!::))*[^:]*} ? {[^:]}] $seg]
+    #                    lappend pats "$pat"
+    #                } else {
+    #                    lappend pats "$seg"
+    #                }
+    #            }
+    #        }
+    #    }
+    #    return  "^[join $pats ::]\$"
+    #}
     proc pathglob_as_re {pathglob} {
         #*** !doctools
         #[call [fun pathglob_as_re] [arg pathglob]]
         #[para] Returns a regular expression for matching a path to a glob pattern which can contain glob chars *|? in any segment of the path structure
+        #[para] Does not support square bracket globs or character classes.
         #[para] ** matches any number of subdirectories.
         #[para] e.g /etc/**/*.txt will match any .txt files at any depth below /etc  (except directly within /etc itself)
         #[para] e.g /etc/**.txt will match any .txt files at any depth below /etc
@@ -589,7 +624,7 @@ namespace eval punk::path {
                 *   {lappend pats {[^/]*}}
                 **  {lappend pats {.*}}
                 default {
-                    set seg [string map [list {^ {\^} $ {\$} [} {\[} ( {\(} \{ \\\{ \\ {\\}] $seg] ;#treat regex characters in the input as literals
+                    set seg [string map [list ^ {\^} $ {\$} \[ {\[} \] {\]} ( {\(} ) {\)} \{ \\\{ \\ {\\}] $seg] ;#treat regex characters (or tcl glob square bracket chars) in the input as literals
                     #set seg [string map [list  . {[.]}] $seg]
                     set seg [string map {. [.]} $seg]
                     if {[regexp {[*?]} $seg]} {
@@ -603,6 +638,52 @@ namespace eval punk::path {
         }
         return  "^[join $pats /]\$"
     }
+
+    punk::args::define {
+        @id     -id     ::punk::path::globmatchpath
+        @cmd    -name     punk::path::globmatchpath\
+            -summary\
+            "Match path to *|**|? glob patterns"\
+            -help\
+            "Return a boolean indicating whether the path matches the specialised glob pattern.
+            A pattern such as /usr/*/bin will match any path that has /usr as the first segment and bin as the third segment,
+            with any single segment in between.
+            A pattern such as /usr/**/bin will match any path that has /usr as the first segment and bin as the last segment,
+            with 1 or more segments in between (so it will not match /usr/bin).
+            A pattern such as /usr/** will match any path that has /usr as the first segment, with 1 or more segments
+            following (so it will not match /usr itself).
+            A pattern such as **/*.txt will match any path that ends with .txt, with 1 or more leading segments 
+            (so it will not match test.txt or .txt).
+            A pattern such as ** will match any path.
+            The glob characters * and ? are the only special characters in the pathglob syntax.
+            - they are treated as glob characters regardless of where they appear in the pathglob string.
+            Note that this is different from other Tcl glob contexts where square brackets can be used.
+            The pathglob syntax treats other characters, including square brackets as literals.
+            For example, the pattern /usr/te?t will match /usr/test and /usr/text but not /usr/texxt, and the pattern /usr/te*t
+            will match /usr/test, /usr/teat, and /usr/teeeet but not /usr/te/t.
+            The pathglob syntax does not support escaping of glob characters - any glob characters in the pathglob are treated
+            as glob characters. For example, the pattern /usr/* will match any path that has /usr as the first segment and any
+            single segment as the second segment, but there is no way to specify a pattern that matches any path that has /usr
+            as the first segment and a literal * as the second segment.
+            Caller must ensure that file separator is forward slash. (e.g use file normalize on windows)
+
+            options:
+                -nocase 0|1  (default 0 - case sensitive)
+             If -nocase is not supplied - default to case sensitive *except for driveletter*
+             ie - the driveletter alone in paths such as c:/etc will still be case insensitive. (ie c:/ETC/* will match C:/ETC/blah but not C:/etc/blah)
+             Explicitly specifying -nocase 0 will require the entire case to match including the driveletter.
+            "
+        @leaders
+            pathglob -type string -help "glob pattern to match path against. See [fun pathglob_as_re] for syntax of glob patterns"
+            path -type string -help "path to match against glob pattern"
+        @opts
+            -nocase -type boolean -default 0 -help\
+                "case insensitive matching (default false - case sensitive)
+                - except for driveletter on windows which is always case insensitive
+                unless -nocase 0 is explicitly specified"
+        @values -min 0 -max 0 
+    }
+    # -id
     proc globmatchpath {pathglob path args} {
         #*** !doctools
         #[call [fun globmatchpath] [arg pathglob] [arg path] [opt {option value...}]]
@@ -659,294 +740,576 @@ namespace eval punk::path {
         return $ismatch
     }
     punk::args::define {
-        @id     -id     ::punk::path::subfolders
+        @id     -id     ::punk::path::subfolders1
-        @cmd    -name     punk::path::subfolders\
+        @cmd    -name     punk::path::subfolders1\
             -summary\
-            "Listing of directories within supplied path."\
+            "Listing of directories below supplied path."\
             -help\
             "List of folders below path.
             The resulting list is unsorted."
         @opts
         -recursive -type none -help\
             ""
+        -exclude-paths -type list -default {} -help\
+            "list of path patterns to exclude from results.
+            May include * and ** path segments e.g /usr/**
+            A single /*/ will match any single segment in the path, and a single /**/ will match any number of segments in the path.
+
+            e.g to exclude any path with _aside as a segment in the middle: -exclude-paths **/_aside/**
+            i.e this would exclude /usr/_aside/etc and /usr/x/_aside/etc but not /usr/x/_aside or _aside/etc
+
+            To exclude all paths with _aside as a segment anywhere: -exclude-paths { **/_aside/** **/_aside _aside/**}
+            "
         #todo -depth
         @values -min 0 -max 1
         path -type directory -optional 1 -help\
-            "Path of folder. If not supplied current directory is used."
+            "Path of folder. If not supplied current directory is used.
-    }
+            This may be a relative or absolute path. Relative paths are treated as relative to current directory.
-    proc subfolders {args} {
+            When using relative paths - the result will also be relative paths with the same relative prefix.
-        set argd [punk::args::parse $args withid ::punk::path::subfolders]
+            (e.g if path is ../test - the results will be ../test/subfolder1 ../test/subfolder2 etc)
+            Patterns in -exclude-paths are matched against the resulting paths
+            (so should be written to match the same relative prefix if path is relative)"
+    }
+
+    proc subfolders1 {args} {
+        #NOTE - this algorithm based on omit_only_patterns and prune_base_patterns was suggested by a 2026 AI model - it is apparent to this programmer that it is inadequate for the purpose.
+        #e.g consider subfolders1 -recursion -exclude {**/vfs/** **/src/**}
+        #This can still return something like c:/repo/etc/src/vfs - which should be excluded by the pattern **/src/**
+        #todo - review and fix properly.
+        set argd [punk::args::parse $args withid ::punk::path::subfolders1]
         lassign [dict values $argd] leaders opts values received
         set do_recursion    [dict exists $received -recursive]
+        set exclude_paths   [dict get $opts -exclude-paths]
+        if {"**" in $exclude_paths} {
+            #if ** is in exclude_paths - then we can skip all glob matching and just return empty list
+            #This is likely user error - so we'll be loud about it for now but will still return empty list rather than erroring.
+            #If user code is building exclude_paths dynamically - they can check for this case themselves and avoid the call to subfolders1 to suppress this message.
+            puts stderr "punk::path::subfolders1 Warning - exclude_paths contains '**' - all paths will be excluded"
+            return [list]
+        }
         if {[dict exists $received path]} {
             set path [dict get $values path]
         } else {
             set path [pwd]
         }
-        set folders [glob -nocomplain -directory $path -types d *]
+
+        set all_subfolders [glob -nocomplain -directory $path -types d *]
+
+
+        #example of expected exclude_paths pattern behaviour when recursion is enabled:
+        # **/dirname    -> omit /x/y/dirname, but still visit /x/y/dirname/*
+
+        # **/dirname/*   -> include /x/y/dirname and /x/y/dirname/a/b but omit directories that are a single level below /x/y/dirname such as /x/y/dirname/a
+
+        #c:/** - would exclude all subfolders below c: but not c: itself
+
+        # **/test/** - would exclude any path with test as a segment and all its subfolders
+        #- but not paths with test as a segment that is the final segment
+
+        set folders [list]
+        set recurse_subdirs [list]
+
+        foreach f $all_subfolders {
+            set include_in_results 1
+            set allow_recurse 1
+            foreach pat $exclude_paths {
+                set pat_parts [file split $pat] ;#note file split c:/test gives {c:/ test}  but file split **/test gives {** test}
+                #also note that file split on windows treats forward slashes and backslashes the same.
+                #by using file split, we gain some flexibility in syntax of paths and patterns,
+                #but lose the ability to use backslashes as escapes to allow literal glob characters in path segments.
+                #This is almost always a non-issue on windows since * and ? are not valid in path segments there, and is rarely an issue on unix even though
+                # * and ? are technically valid in path segments, but it is inadvisable there anyway for compatibility with shells etc.
+                if {[llength $pat_parts] >= 2 && [lindex $pat_parts end] eq "**"} {
+                    set base_pat [file join {*}[lrange $pat_parts 0 end-1]]
+                    if {[globmatchpath $pat $f]} {
+                        set include_in_results 0
+                        set allow_recurse 0
+                    } elseif {[globmatchpath $base_pat $f]} {
+                        set allow_recurse 0
+                    }
+                } elseif {[globmatchpath $pat $f]} {
+                    set include_in_results 0
+                }
+                if {!$include_in_results && !$allow_recurse} {
+                    break
+                }
+            }
+            if {$include_in_results} {
+                lappend folders $f
+            }
+            if {$allow_recurse} {
+                lappend recurse_subdirs $f
+            }
+        }
         if {$do_recursion} {
-            foreach subdir $folders {
+            foreach subdir $recurse_subdirs {
-                lappend folders {*}[subfolders -recursive $subdir]
+                lappend folders {*}[subfolders1 -exclude-paths $exclude_paths -recursive $subdir]
             }
         }
         return $folders
     }
 
-    #todo - treefolders with similar search caps as treefilenames
+    namespace eval subfolder_priv {
-
+        proc classify_exclude_pattern {pat} {
-    punk::args::define {
+            set parts [file split $pat]
-        @id     -id ::punk::path::treefilenames
+            if {[llength $parts] >= 2 && [lindex $parts end] eq "**"} {
-        -directory       -type directory  -help\
+                set boundary_pat [file join {*}[lrange $parts 0 end-1]]
-            "folder in which to begin recursive scan for files."
+                return [dict create \
-        -call-depth-internal    -default 0 -type integer
+                    pattern $pat \
-        -sort -type any -default natural -choices {none ascii dictionary natural}
+                    kind subtree \
-        -antiglob_paths         -default {} -help\
+                    boundary_pat $boundary_pat \
-            "list of path patterns to exclude
+                    descend_pat $pat]
-              may include * and ** path segments e.g
+            }
-              /usr/**       (exlude subfolders based at /usr but not
+            if {[llength $parts] >= 2 && [lindex $parts end] eq "*"} {
-                             files within /usr itself)
+                return [dict create \
-              **/_aside     (exlude  files where _aside is last segment)
+                    pattern $pat \
-              **/_aside/*   (exclude folders one below an _aside folder)
+                    kind child_only \
-              **/_aside/**  (exclude all folders with _aside as a segment)"
+                    match_pat $pat]
-        -antiglob_files         -default {}
+            }
-        @values -min 0 -max -1 -optional 1 -type string
+            return [dict create \
-        tailglobs -default * -multiple 1   -help\
+                pattern $pat \
-            "Patterns to match against filename portion (last segment) of each file path
+                kind exact \
-              within the directory tree being searched."
+                match_pat $pat]
         }
 
-    #todo - implement treefiles which acts like dirfiles but allows path globbing in the same way as punk::ns::ns/
+        proc compile_exclude_rules {exclude_paths} {
-    #then review if treefiles can replace dirfiles or if both should exist (dirfiles can have literal glob chars in path segments - but that is a rare usecase)
+            set rules [list]
-    proc treefilenames {args} {
+            foreach pat $exclude_paths {
-        #*** !doctools
+                lappend rules [classify_exclude_pattern $pat]
-        #[call [fun treefilenames]  [opt {option value...}] [opt {globpattern...}]]
+            }
-        #[para]basic (glob based) list of filenames matching each pattern in tailglobs - recursive
+            return $rules
-        #[para] options:
+        }
-        #[para] [opt -dir] <path>
-        #[para]     defaults to [lb]pwd[rb] - base path for tree to search
-        #[para] [opt -antiglob_paths] <list>
-        #[para]     list of path patterns to exclude - may include * and ** path segments e.g /usr/**
-        #[para]no natsorting - so order is dependent on filesystem
 
-        set argd [punk::args::parse $args withid ::punk::path::treefilenames]
+        proc match_rule_at_node {rule path} {
-        lassign [dict values $argd] leaders opts values received
+            set kind [dict get $rule kind]
-        set tailglobs [dict get $values tailglobs]
+            switch -- $kind {
-        # -- --- --- --- --- --- ---
+                exact - child_only {
-        set opt_sort           [dict get $opts -sort]
+                    if {[::punk::path::globmatchpath [dict get $rule match_pat] $path]} {
-        set opt_antiglob_paths [dict get $opts -antiglob_paths]
+                        return [dict create include_current 0 recurse_below 1 child_rules [list $rule]]
-        set opt_antiglob_files [dict get $opts -antiglob_files]
+                    }
-        set CALLDEPTH [dict get $opts -call-depth-internal]
+                    return [dict create include_current 1 recurse_below 1 child_rules [list $rule]]
-        # -- --- --- --- --- --- ---
+                }
-        # -- --- --- --- --- --- ---
+                subtree {
+                    set descend_pat  [dict get $rule descend_pat]
+                    set boundary_pat [dict get $rule boundary_pat]
+                    if {[::punk::path::globmatchpath $descend_pat $path]} {
+                        return [dict create include_current 0 recurse_below 0 child_rules [list]]
+                    }
+                    if {[::punk::path::globmatchpath $boundary_pat $path]} {
+                        return [dict create include_current 1 recurse_below 0 child_rules [list]]
+                    }
+                    return [dict create include_current 1 recurse_below 1 child_rules [list $rule]]
+                }
+                default {
+                    error "Unknown exclude rule kind '$kind'"
+                }
+            }
+        }
 
-        set files [list]
+        proc walk_subfolders {path rules do_recursion} {
-        if {$CALLDEPTH == 0} {
+            set all_subfolders [glob -nocomplain -directory $path -types d *]
-            if {$opt_sort eq "natural"} {
+            set folders [list]
-                package require natsort
+            foreach f $all_subfolders {
+                set include_current 1
+                set recurse_below $do_recursion
+                set child_rules [list]
+                foreach rule $rules {
+                    set outcome [match_rule_at_node $rule $f]
+                    if {![dict get $outcome include_current]} {
+                        set include_current 0
                     }
-            #set opts [dict merge $opts [list -directory $opt_dir]]
+                    if {![dict get $outcome recurse_below]} {
-            if {![dict exists $received -directory]} {
+                        set recurse_below 0
-                set opt_dir [pwd]
-            } else {
-                set opt_dir [dict get $opts -directory]
                     }
-            if {![file isdirectory $opt_dir]} {
+                    if {$do_recursion} {
+                        lappend child_rules {*}[dict get $outcome child_rules]
+                    }
+                    if {!$include_current && !$recurse_below} {
+                        break
+                    }
+                }
+                if {$include_current} {
+                    lappend folders $f
+                }
+                if {$do_recursion && $recurse_below} {
+                    lappend folders {*}[walk_subfolders $f $child_rules $do_recursion]
+                }
+            }
+            return $folders
+        }
+    }
+
+    punk::args::define {
+        @id     -id     ::punk::path::subfolders
+        @cmd    -name     punk::path::subfolders\
+            -summary\
+            "Listing of directories below supplied path."\
+            -help\
+            "List of folders below path.
+            The resulting list is unsorted.
+            "
+        @opts
+        -recursive -type none -help\
+            ""
+        -exclude-paths -type list -default {} -help\
+            "list of path patterns to exclude from results.
+            May include * and ** path segments e.g /usr/**
+            A single /*/ will match any single segment in the path, and a single /**/ will match any number of segments in the path.
+
+            e.g to exclude any path with _aside as a segment in the middle: -exclude-paths **/_aside/**
+            i.e this would exclude /usr/_aside/etc and /usr/x/_aside/etc but not /usr/x/_aside or _aside/etc
+
+            To exclude all paths with _aside as a segment anywhere: -exclude-paths { **/_aside/** **/_aside ./_aside/**}
+            "
+        #todo -depth
+        @values -min 0 -max 1
+        path -type directory -optional 1 -help\
+            "Path of base folder. If not supplied current directory is used.
+            This may be a relative or absolute path. Relative paths are treated as relative to current directory.
+            When using relative paths - the result will also be relative paths with the same relative prefix.
+            (e.g if path is ../test - the results will be ../test/subfolder1 ../test/subfolder2 etc)
+            Patterns in -exclude-paths are matched against the resulting paths
+            (so should be written to match the same relative prefix if path is relative)"
+    }
+
+    proc subfolders {args} {
+        set argd [punk::args::parse $args withid ::punk::path::subfolders]
+        lassign [dict values $argd] leaders opts values received
+        set do_recursion    [dict exists $received -recursive]
+        set exclude_paths   [dict get $opts -exclude-paths]
+        if {"**" in $exclude_paths} {
+            puts stderr "punk::path::subfolders Warning - exclude_paths contains '**' - all paths will be excluded"
             return [list]
         }
+        if {[dict exists $received path]} {
+            set path [dict get $values path]
         } else {
-            #assume/require to exist in any recursive call
+            set path [pwd]
-            set opt_dir [dict get $opts -directory]
+        }
+        set compiled_rules [subfolder_priv::compile_exclude_rules $exclude_paths]
+        return [subfolder_priv::walk_subfolders $path $compiled_rules $do_recursion]
     }
 
-        #comment out to compare timings with treefilenames_zipfs
+    namespace eval treefile_priv {
-        if {[string match //zipfs:/* $opt_dir]} {
+        proc _pattern_prefix_viable_parts {pattern_parts path_parts} {
-            return [treefilenames_zipfs {*}$args]
+            if {![llength $path_parts]} {
+                return 1
+            }
+            if {![llength $pattern_parts]} {
+                return 0
             }
 
-        set skip 0
+            set pattern_head [lindex $pattern_parts 0]
-        foreach anti $opt_antiglob_paths {
+            set path_head [lindex $path_parts 0]
-            if {[globmatchpath $anti $opt_dir]} {
+
-                set skip 1
+            if {$pattern_head eq "**"} {
+                if {[_pattern_prefix_viable_parts [lrange $pattern_parts 1 end] $path_parts]} {
+                    return 1
+                }
+                return [_pattern_prefix_viable_parts $pattern_parts [lrange $path_parts 1 end]]
+            }
+
+            if {[::punk::path::globmatchpath $pattern_head $path_head]} {
+                return [_pattern_prefix_viable_parts [lrange $pattern_parts 1 end] [lrange $path_parts 1 end]]
+            }
+            return 0
+        }
+
+        proc pattern_prefix_viable {pattern path} {
+            return [_pattern_prefix_viable_parts [file split $pattern] [file split $path]]
+        }
+
+        proc pattern_boundary {pattern} {
+            set parts [file split $pattern]
+            if {[llength $parts] >= 2 && [lindex $parts end] eq "**"} {
+                return [file join {*}[lrange $parts 0 end-1]]
+            }
+            return ""
+        }
+
+        proc directory_state {glob_paths path inherited_allbelow} {
+            if {$inherited_allbelow} {
+                return [dict create include_files 1 recurse_below 1 next_allbelow 1]
+            }
+
+            set include_files 0
+            set recurse_below 0
+            set next_allbelow 0
+
+            foreach gp $glob_paths {
+                if {[::punk::path::globmatchpath $gp $path]} {
+                    set include_files 1
+                    set recurse_below 1
+                    set next_allbelow 1
                     break
                 }
+
+                set boundary [pattern_boundary $gp]
+                if {$boundary ne "" && [::punk::path::globmatchpath $boundary $path]} {
+                    set recurse_below 1
+                    set next_allbelow 1
+                    continue
+                }
+
+                if {[pattern_prefix_viable $gp $path]} {
+                    set recurse_below 1
                 }
-        if {$skip} {
-            return [list]
             }
 
-        #todo - account for vfs where matched path could appear to be a directory but is mounted so could be a desired match?
+            return [dict create                     {*}{
-        if {[catch {glob -nocomplain -dir $opt_dir -type f {*}$tailglobs} matches]} {
+                    } include_files $include_files  {*}{
-            #we can get for example a permissions error
+                    } recurse_below $recurse_below  {*}{
-            puts stderr "treefilenames error while listing files in dir $opt_dir\n $matches"
+                    } next_allbelow $next_allbelow  {*}{
-            set dirfiles [list]
-        } else {
-            set retained [list]
-            if {[llength $opt_antiglob_files]} {
-                foreach m $matches {
-                    set skip 0
-                    set ftail [file tail $m]
-                    foreach anti $opt_antiglob_files {
-                        if {[string match $anti $ftail]} {
-                            set skip 1; break
                     }
+                ]
         }
-                    if {!$skip} {
+
-                        lappend retained $m
+        proc child_path_state {glob_paths child_path inherited_allbelow} {
+            if {$inherited_allbelow} {
+                return 1
             }
+            foreach gp $glob_paths {
+                if {[pattern_prefix_viable $gp $child_path]} {
+                    return 1
                 }
-            } else {
-                set retained $matches
             }
-            switch -- $opt_sort {
+            return 0
+        }
+
+        proc _sort_paths {paths sortmode} {
+            switch -- $sortmode {
                 ascii {
-                    set dirfiles [lsort $retained]
+                    return [lsort $paths]
                 }
                 dictionary {
-                    set dirfiles [lsort -dictionary $retained]
+                    return [lsort -dictionary $paths]
                 }
                 natural {
-                    set dirfiles [natsort::sort $retained]
+                    return [natsort::sort $paths]
                 }
                 default {
-                    set dirfiles $retained
+                    return $paths
                 }
             }
         }
 
-        lappend files  {*}$dirfiles
+        proc _path_matches_any {patterns path} {
-        if {[catch {glob -nocomplain -dir $opt_dir -type d *} dirdirs]} {
+            foreach pattern $patterns {
-            puts stderr "treefilenames error while listing subdirs in dir $opt_dir\n $dirdirs"
+                if {[::punk::path::globmatchpath $pattern $path]} {
-            set dirdirs [list]
+                    return 1
                 }
-        set okdirs [list]
+            }
-        foreach dir $dirdirs {
+            return 0
+        }
+
+        proc _tailbase_relative {tailbase path} {
+            if {$tailbase eq ""} {
+                return $path
+            }
+            return [::punk::path::relative $tailbase $path]
+        }
+
+        proc _tailbase_match_path {tailbase path} {
+            set match_path [_tailbase_relative $tailbase $path]
+            if {$match_path eq "."} {
+                return ""
+            }
+            return $match_path
+        }
+
+        proc _tailbase_relative_list {tailbase paths} {
+            if {$tailbase eq ""} {
+                return $paths
+            }
+            set relative_paths [list]
+            foreach path $paths {
+                lappend relative_paths [_tailbase_relative $tailbase $path]
+            }
+            return $relative_paths
+        }
+
+        proc _retain_files {matches exclude_files sortmode} {
+            set retained [list]
+            foreach match $matches {
                 set skip 0
-            foreach anti $opt_antiglob_paths {
+                set file_tail [file tail $match]
-                if {[globmatchpath $anti $dir]} {
+                foreach anti $exclude_files {
+                    if {[string match $anti $file_tail]} {
                         set skip 1
                         break
                     }
                 }
                 if {!$skip} {
-                lappend okdirs $dir
+                    lappend retained $match
                 }
             }
-        if {[llength $okdirs]} {
+            return [_sort_paths $retained $sortmode]
-            switch -- $opt_sort {
-                ascii {
-                    set finaldirs [lsort $okdirs]
         }
-                dictionary {
+
-                    set finaldirs [lsort -dictionary $okdirs]
+        proc _state_from_argd {argd} {
+            set opts        [dict get $argd opts]
+            set values      [dict get $argd values]
+            set received    [dict get $argd received]
+
+            if {[dict exists $received -directory]} {
+                set directory [dict get $opts -directory]
+            } else {
+                set directory [pwd]
             }
-                natural {
+
-                    set finaldirs [natsort::sort $okdirs]
+            set glob_paths [dict get $opts -include-paths]
+            if {"*" in $glob_paths} {
+                set glob_paths {*}
             }
-                default {
+
-                    set finaldirs $okdirs
+            set sortmode [dict get $opts -sort]
+            if {$sortmode eq "natural"} {
+                package require natsort
             }
+
+            return [dict create {*}{
+                    depth           0
+                    subvector       {}
+                    allbelow        0
+                }   sort            $sortmode                             {*}{
+                }   directory       $directory                            {*}{
+                }   tailbase        [dict get $opts -tailbase]            {*}{
+                }   exclude_paths   [dict get $opts -exclude-paths]       {*}{
+                }   exclude_files   [dict get $opts -exclude-files]       {*}{
+                }   glob_paths      $glob_paths                           {*}{
+                }   tailglobs       [dict get $values tailglobs]          {*}{
+                }
+            ]
+        }
+
+        proc walk_treefilenames {state} {
+            set opt_dir             [dict get $state directory]
+            set opt_tailbase        [dict get $state tailbase]
+            set depth               [dict get $state depth]
+            set subvector           [dict get $state subvector]
+            set callallbelow        [dict get $state allbelow]
+            set opt_sort            [dict get $state sort]
+            set opt_exclude_paths   [dict get $state exclude_paths]
+            set opt_exclude_files   [dict get $state exclude_files]
+            set opt_glob_paths      [dict get $state glob_paths]
+            set tailglobs           [dict get $state tailglobs]
+
+            if {![file isdirectory $opt_dir]} {
+                return [list]
             }
-            foreach dir $finaldirs {
+            if {[string match //zipfs:/* $opt_dir]} {
-                set nextopts [dict merge $opts [list -directory $dir -call-depth-internal [incr CALLDEPTH]]]
+                return [walk_treefilenames_zipfs $state]
-                lappend files {*}[treefilenames {*}$nextopts {*}$tailglobs]
             }
+            set opt_dir_match [_tailbase_match_path $opt_tailbase $opt_dir]
+            if {[_path_matches_any $opt_exclude_paths $opt_dir_match]} {
+                return [list]
             }
+
+            set files [list]
+            set dir_state [directory_state $opt_glob_paths $opt_dir_match $callallbelow]
+            if {[dict get $dir_state include_files]} {
+                if {[catch {glob -nocomplain -dir $opt_dir -type f {*}$tailglobs} matches]} {
+                    puts stderr "treefilenames error while listing files in dir $opt_dir\n $matches"
+                    set dirfiles [list]
+                } else {
+                    set dirfiles [_retain_files $matches $opt_exclude_files $opt_sort]
+                }
+                lappend files {*}[_tailbase_relative_list $opt_tailbase $dirfiles]
+            }
+
+            if {![dict get $dir_state recurse_below]} {
                 return $files
             }
-    proc treefilenames_zipfs {args} {
-        #seems to be 2 or 3 times faster than treefilenames for //zipfs:/ paths  - REVIEW
-        # is sort order the same?
-        set argd [punk::args::parse $args withid ::punk::path::treefilenames]
-        lassign [dict values $argd] leaders opts values received
-        set tailglobs [dict get $values tailglobs]
-        # -- --- --- --- --- --- ---
-        set opt_antiglob_paths [dict get $opts -antiglob_paths]
-        set opt_antiglob_files [dict get $opts -antiglob_files]
-        set opt_sort           [dict get $opts -sort]
-        set CALLDEPTH [dict get $opts -call-depth-internal]
-        # -- --- --- --- --- --- ---
-        # -- --- --- --- --- --- ---
 
-        set files [list]
+            if {[catch {glob -nocomplain -dir $opt_dir -type d *} dirdirs]} {
-        if {$CALLDEPTH == 0} {
+                puts stderr "treefilenames error while listing subdirs in dir $opt_dir\n $dirdirs"
-            if {$opt_sort eq "natural"} {
+                set dirdirs [list]
-                package require natsort
             }
-            #set opts [dict merge $opts [list -directory $opt_dir]]
+            set okdirs [list]
-            if {![dict exists $received -directory]} {
+            foreach dir $dirdirs {
-                set opt_dir [pwd]
+                if {![_path_matches_any $opt_exclude_paths [_tailbase_match_path $opt_tailbase $dir]]} {
-            } else {
+                    lappend okdirs $dir
-                set opt_dir [dict get $opts -directory]
                 }
-            if {![file isdirectory $opt_dir]} {
-                return [list]
             }
+
+            if {$opt_glob_paths eq "*"} {
+                set matchdirs $okdirs
             } else {
-            #assume/require to exist in any recursive call
+                set matchdirs [list]
-            set opt_dir [dict get $opts -directory]
+                foreach dir $okdirs {
+                    if {$callallbelow || [child_path_state $opt_glob_paths [_tailbase_match_path $opt_tailbase $dir] $callallbelow]} {
+                        lappend matchdirs $dir
+                    }
                 }
-        if {![string match [zipfs root]* $opt_dir]} {
-            error "treefilenames_zipfs can only be used on paths beginning with [zipfs root] on this systems"
             }
-        set dir [string trimright $opt_dir "/"] ;#e.g normalize //zipfs:/x/ to //zipfs:/x
-        set dirlen [string length $dir]
 
-        set skip 0
+            set finaldirs [_sort_paths $matchdirs $opt_sort]
-        foreach anti $opt_antiglob_paths {
+            set childallbelow [expr {$callallbelow || [dict get $dir_state next_allbelow]}]
-            if {[globmatchpath $anti $dir]} {
+            set nextsubvector [list {*}$subvector [file tail $opt_dir]]
-                set skip 1
+            foreach dir $finaldirs {
-                break
+                set child_state [dict merge $state [dict create {*}{} \
+                    directory   $dir \
+                    depth       [expr {$depth + 1}] \
+                    subvector   $nextsubvector \
+                    allbelow    $childallbelow]]
+                lappend files {*}[walk_treefilenames $child_state]
+            }
+            return $files
         }
+
+        proc walk_treefilenames_zipfs {state} {
+            set opt_dir             [dict get $state directory]
+            set opt_tailbase        [dict get $state tailbase]
+            set opt_exclude_paths   [dict get $state exclude_paths]
+            set opt_exclude_files   [dict get $state exclude_files]
+            set opt_glob_paths      [dict get $state glob_paths]
+            set opt_sort            [dict get $state sort]
+            set tailglobs           [dict get $state tailglobs]
+
+            if {![string match [zipfs root]* $opt_dir]} {
+                error "treefilenames_zipfs can only be used on paths beginning with [zipfs root] on this systems"
             }
-        if {$skip} {
+            set dir [string trimright $opt_dir "/"]
+            if {[_path_matches_any $opt_exclude_paths [_tailbase_match_path $opt_tailbase $dir]]} {
                 return [list]
             }
+            set dirlen [string length $dir]
             set subpaths [zipfs list $dir/*]
             set dirlist [list]
             set skipdirs [list]
             set filelist [list]
-        #process in the order they came - sorting large list more expensive?? review
             foreach sub $subpaths {
-            set tail [string range $sub $dirlen+1 end] ;#dirlen is without trailing slash
+                set tail [string range $sub $dirlen+1 end]
                 set tailparts [file split $tail]
                 set accum ""
                 set skipdir 0
-            foreach tp [lrange $tailparts 0 end-1] {
+                foreach tailpart [lrange $tailparts 0 end-1] {
-                append accum "/$tp"
+                    append accum "/$tailpart"
                     set superpath "${dir}${accum}"
                     if {$superpath in $skipdirs} {
-                    #subpart already in skipdirs
                         set skipdir 1
                         break
                     }
                     if {$superpath ni $dirlist} {
-                    set skip2 0
+                        if {[_path_matches_any $opt_exclude_paths [_tailbase_match_path $opt_tailbase $superpath]]} {
-                    foreach anti $opt_antiglob_paths {
-                        if {[globmatchpath $anti $superpath]} {
-                            set skip2 1
                             lappend skipdirs $superpath
-                            break
-                        }
-                    }
-                    if {!$skip2} {
-                        lappend dirlist $superpath
-                    } else {
                             set skipdir 1
                             break
+                        } else {
+                            lappend dirlist $superpath
                         }
                     }
                 }
                 if {!$skipdir} {
-                #process final part of path
                     append accum "/[lindex $tailparts end]"
                     set finalpart "${dir}${accum}"
                     if {$finalpart ni $dirlist} {
                         if {[file type $finalpart] eq "file"} {
-                        set ftail [lindex $tailparts end]
+                            set file_tail [lindex $tailparts end]
                             set match 0
                             if {"*" ni $tailglobs} {
-                            foreach tg $tailglobs {
+                                foreach tailglob $tailglobs {
-                                if {[string match $tg $ftail]} {
+                                    if {[string match $tailglob $file_tail]} {
                                         set match 1
                                         break
                                     }
@@ -955,31 +1318,29 @@ namespace eval punk::path {
                                 set match 1
                             }
                             if {$match} {
-                            if {[llength $opt_antiglob_files]} {
+                                if {$opt_glob_paths ne "*"} {
+                                    set file_dir_match [_tailbase_match_path $opt_tailbase [file dirname $finalpart]]
+                                    set file_dir_state [directory_state $opt_glob_paths $file_dir_match 0]
+                                    set match [dict get $file_dir_state include_files]
+                                }
+                            }
+                            if {$match} {
                                 set skipfile 0
-                                foreach anti $opt_antiglob_files {
+                                foreach anti $opt_exclude_files {
-                                    if {[string match $anti $ftail]} {
+                                    if {[string match $anti $file_tail]} {
-                                        set skipfile 1; break
+                                        set skipfile 1
+                                        break
                                     }
                                 }
                                 if {!$skipfile} {
-                                    lappend filelist $finalpart
+                                    lappend filelist [_tailbase_relative $opt_tailbase $finalpart]
-                                }
-                            } else {
-                                lappend filelist $finalpart
                                 }
                             }
                         } else {
                             if {$finalpart ni $dirlist} {
-                            set skip2 0
+                                if {[_path_matches_any $opt_exclude_paths [_tailbase_match_path $opt_tailbase $finalpart]]} {
-                            foreach anti $opt_antiglob_paths {
-                                if {[globmatchpath $anti $finalpart]} {
-                                    set skip2 1
                                     lappend skipdirs $finalpart
-                                    break
+                                } else {
-                                }
-                            }
-                            if {!$skip2} {
                                     lappend dirlist $finalpart
                                 }
                             }
@@ -987,21 +1348,81 @@ namespace eval punk::path {
                     }
                 }
             }
-        switch -- $opt_sort {
+            return [_sort_paths $filelist $opt_sort]
-            ascii {
-                set finalfilelist [lsort $filelist]
         }
-            dictionary {
-                set finalfilelist [lsort -dictionary $filelist]
     }
-            natural {
+
-                set finalfilelist [natsort::sort $filelist]
+    #todo - treefolders with similar search caps as treefilenames
+
+    punk::args::define {
+        @id     -id     ::punk::path::treefilenames
+        @cmd    -name     punk::path::treefilenames\
+            -summary\
+            "List of filenames below supplied path."\
+            -help\
+            "List of filenames below path.
+            The resulting list is unsorted.
+
+            The path globbing syntax supports *, ** and ? as glob characters in any segment of the path, with the following semantics:
+            * matches any single segment in the path
+            ** matches 1 or more segments in the path (so /usr/**/bin will match /usr/x/bin and user/x/y/bin but not /usr/bin )
+            ? matches any single character in a single segment of the path (so /usr/te?t will match /usr/test and /usr/text but not /usr/texxt)
+            "
+        -directory       -type directory  -help\
+            "folder in which to begin recursive scan for files."
+        -tailbase         -type string    -default "" -help\
+            "if supplied, only the relative path compared to the tailbase will be returned for each file.
+            So if tailbase is /usr and a file is found at /usr/x/y/file.txt, the returned path for that file would be x/y/file.txt.
+            If tailbase is not supplied, the full path to each file will be returned.
+
+            If tailbase is supplied, it should be a prefix of the directory supplied (or the directory itself)
+            The patterns in -exclude-paths should be written to match the returned paths (i.e with the tailbase prefix removed) if -tailbase is supplied.
+            If the tailbase is not a prefix of the directory supplied, the resulting paths may have /../ components in them to account for the difference,
+            but the behaviour is not well defined in this case and it is recommended to ensure tailbase is a prefix of the directory supplied if using -tailbase.
+
+            see: punk::path::relative to compute relative paths
+            "
+        -sort -type any         -default natural -choices {none ascii dictionary natural}
+        -exclude-paths         -default {} -help\
+            "list of path patterns to exclude
+              may include * and ** path segments e.g
+              /usr/**       (exclude subfolders based at /usr but not
+                             files within /usr itself)
+              **/_aside     (exclude  files where _aside is last segment)
+              **/_aside/*   (exclude folders one below an _aside folder)
+              **/_aside/**  (exclude files in all folders with _aside as a segment)"
+        -exclude-files         -default {}
+        -include-paths     -default {**} -help\
+            "list of path patterns to include
+              may include * and ** path segments e.g
+              /usr/**       (include files in subfolders based at /usr but not
+                             files within /usr itself)
+              **/_aside     (include  files where _aside is last segment in the folder)
+              **/_aside/*   (include files in folders one below an _aside folder)
+              **/_aside/**  (include all files in folders with _aside as a segment)"
+        @values -min 0 -max -1 -optional 1 -type string
+        tailglobs -default * -multiple 1   -help\
+            "Patterns to match against filename portion (last segment) of each file path
+              within the directory tree being searched."
     }
-            default {
+
-                set finalfilelist $filelist
+    #todo - implement treefiles which acts like dirfiles but allows path globbing in the same way as punk::ns::ns/
+    #then review if treefiles can replace dirfiles or if both should exist (dirfiles can have literal glob chars in path segments - but that is a rare usecase)
+    proc treefilenames {args} {
+        set argd [punk::args::parse $args withid ::punk::path::treefilenames]
+        set state [treefile_priv::_state_from_argd $argd]
+        return [treefile_priv::walk_treefilenames $state]
     }
+    punk::args::set_idalias ::punk::path::treefilenames_zipfs ::punk::path::treefilenames
+    proc treefilenames_zipfs {args} {
+        #seems to be 2 or 3 times faster than treefilenames for //zipfs:/ paths  - REVIEW
+        # is sort order the same?
+        set argd [punk::args::parse $args withid ::punk::path::treefilenames]
+        set state [treefile_priv::_state_from_argd $argd]
+        if {![file isdirectory [dict get $state directory]]} {
+            return [list]
         }
-        return $finalfilelist
+        return [treefile_priv::walk_treefilenames_zipfs $state]
     }
 
     #maint warning - also in punkcheck

src/project_layouts/custom/_project/punk.shell-0.1/src/bootsupport/modules/punk/pipe-1.0.tm

@@ -169,8 +169,8 @@ tcl::namespace::eval punk::pipe::lib {
         #This stops us matching {/@**@x x}  vs {/@**@x  x}
         #---
 
-        set rhs [tcl::string::map {: <c> ? <q> * <star> [ <lb> ] <rb> \\ <bsl> {"} <dq> " " <sp>} $rhs] 
+        set rhs [tcl::string::map {: <c> ; <sc> ? <q> * <star> [ <lb> ] <rb> \\ <bsl> {"} <dq> " " <sp>} $rhs] 
-        #review - we don't expect other command-incompatible chars such as colon?
+        #review - we don't expect other command-incompatible chars?
         return $rhs
     }
 
@@ -187,6 +187,7 @@ tcl::namespace::eval punk::pipe::lib {
 
     #exclude quoted whitespace
     proc arg_is_script_shaped {arg} {
+        set arg [string map {\\; "<escaped_semicolon>"} $arg]
         if {[tcl::string::first \n $arg] >= 0} {
             return 1
         } elseif {[tcl::string::first ";" $arg] >= 0} {