Browse Source

ansi fixes and tests, ansi xbin and png

master
Julian Noble 5 days ago
parent
commit
162201bf66
  1. 828
      src/modules/opunk/str-999999.0a1.0.tm
  2. 1150
      src/modules/overtype-999999.0a1.0.tm
  3. 54
      src/modules/punk-999999.0a1.0.tm
  4. 1
      src/modules/punk/aliascore-999999.0a1.0.tm
  5. 965
      src/modules/punk/ansi-999999.0a1.0.tm
  6. 67
      src/modules/punk/ansi/sauce-999999.0a1.0.tm
  7. 205
      src/modules/punk/char-999999.0a1.0.tm
  8. 3
      src/modules/punk/lib-999999.0a1.0.tm
  9. 22
      src/modules/punk/ns-999999.0a1.0.tm
  10. 290
      src/modules/punk/path-999999.0a1.0.tm
  11. 5
      src/modules/punk/pipe-999999.0a1.0.tm
  12. 17
      src/modules/punk/repo-999999.0a1.0.tm
  13. 755
      src/modules/test/#modpod-overtype-999999.0a1.0/overtype-1.7.4_testsuites/overtype/renderline.test
  14. 77
      src/modules/test/runtestmodules.tcl
  15. 66
      src/tests/all.tcl
  16. 39
      src/tests/modules/opunk/str/tests/all.tcl
  17. 84
      src/tests/modules/opunk/str/tests/str.test
  18. 38
      src/tests/modules/punk/path/tests/all.tcl
  19. 33
      src/tests/modules/punk/path/tests/path.test
  20. 828
      src/vfs/_vfscommon.vfs/modules/opunk/str-0.1.0.tm
  21. 1150
      src/vfs/_vfscommon.vfs/modules/overtype-1.7.4.tm
  22. 54
      src/vfs/_vfscommon.vfs/modules/punk-0.1.1.tm
  23. 1
      src/vfs/_vfscommon.vfs/modules/punk/aliascore-0.1.0.tm
  24. 965
      src/vfs/_vfscommon.vfs/modules/punk/ansi-0.1.1.tm
  25. 67
      src/vfs/_vfscommon.vfs/modules/punk/ansi/sauce-0.1.0.tm
  26. 205
      src/vfs/_vfscommon.vfs/modules/punk/char-0.1.0.tm
  27. 3
      src/vfs/_vfscommon.vfs/modules/punk/lib-0.1.6.tm
  28. 9
      src/vfs/_vfscommon.vfs/modules/punk/mix/commandset/loadedlib-0.1.0.tm
  29. 22
      src/vfs/_vfscommon.vfs/modules/punk/ns-0.1.0.tm
  30. 290
      src/vfs/_vfscommon.vfs/modules/punk/path-0.1.0.tm
  31. 5
      src/vfs/_vfscommon.vfs/modules/punk/pipe-1.0.tm
  32. 17
      src/vfs/_vfscommon.vfs/modules/punk/repo-0.1.1.tm
  33. BIN
      src/vfs/_vfscommon.vfs/modules/test/overtype-1.7.4.tm
  34. 77
      src/vfs/_vfscommon.vfs/modules/test/runtestmodules.tcl

828
src/modules/opunk/str-999999.0a1.0.tm

File diff suppressed because it is too large Load Diff

1150
src/modules/overtype-999999.0a1.0.tm

File diff suppressed because it is too large Load Diff

54
src/modules/punk-999999.0a1.0.tm

@ -2466,6 +2466,7 @@ namespace eval punk {
set splitchars "<splitchars>" set splitchars "<splitchars>"
set assigned [split $leveldata $splitchars] set assigned [split $leveldata $splitchars]
}] }]
puts "---split script: $script"
set level_script_complete 1 set level_script_complete 1
#todo %splitat- %splitn- ?? #todo %splitat- %splitn- ??
@ -4205,7 +4206,7 @@ namespace eval punk {
#avoid use of regexp match on each element - or we will unnecessarily force string reps on lists #avoid use of regexp match on each element - or we will unnecessarily force string reps on lists
#same with lsearch with a string pattern - #same with lsearch with a string pattern -
#wouldn't matter for small lists - but we need to be able to handle large ones efficiently without unneccessary string reps #wouldn't matter for small lists - but we need to be able to handle large ones efficiently without unneccessary string reps
set script [string map [list <scopep> $scopepattern <rhs> $equalsrhs] { set script [string map [list <scopep> [list $scopepattern] <rhs> $equalsrhs] {
#script built by punk::match_assign #script built by punk::match_assign
if {[llength $args]} { if {[llength $args]} {
#scan for existence of any pipe operator (|*> or <*|) only - we don't need position #scan for existence of any pipe operator (|*> or <*|) only - we don't need position
@ -4214,11 +4215,12 @@ namespace eval punk {
# x= <| # x= <|
# x= |> # x= |>
#both leave x empty. To assign a pipelike value to x we would have to do: x= <| |> (equiv: set x |>) #both leave x empty. To assign a pipelike value to x we would have to do: x= <| |> (equiv: set x |>)
set scopep <scopep>
foreach a $args { foreach a $args {
if {![catch {llength $a} sublen]} { if {![catch {llength $a} sublen]} {
#don't enforce sublen == 1. Legal to have whitespace including newlines {| x >} #don't enforce sublen == 1. Legal to have whitespace including newlines {| x >}
if {[string match |*> $a] || [string match <*| $a]} { if {[string match |*> $a] || [string match <*| $a]} {
tailcall punk::pipeline = "<scopep>" "<rhs>" {*}$args tailcall punk::pipeline = $scopep "<rhs>" {*}$args
} }
} }
} }
@ -4594,6 +4596,10 @@ namespace eval punk {
#debug.punk.pipe.rep {[rep_listname fulltail]} 6 #debug.punk.pipe.rep {[rep_listname fulltail]} 6
#review
set equalsrhs [string map [list {;} {\;}] $equalsrhs]
#--------------------------------------------------------------------- #---------------------------------------------------------------------
# test if we have an initial x.=y.= or x.= y.= # test if we have an initial x.=y.= or x.= y.=
@ -4643,26 +4649,31 @@ namespace eval punk {
#var1 will contain ETC (from entire pipeline), var2 will contain etc (from associated segment) #var1 will contain ETC (from entire pipeline), var2 will contain etc (from associated segment)
# #
if {([set nexteposn [string last = $next1]] >= 0) && (![punk::pipe::lib::arg_is_script_shaped $next1]) } {
set nexttail [lrange $args 1 end] if {([set nexteposn [string last = $next1]] >= 0)} {
#*SUB* pipeline recursion. set next1 [string map [list {;} {\;}] $next1] ;#review
#puts "======> recurse based on next1:$next1 " #do we really need to test for script_shaped if last char is = ?
if {[string index $next1 $nexteposn-1] eq {.}} { if {![punk::pipe::lib::arg_is_script_shaped $next1]} {
#var1.= var2.= ... set nexttail [lrange $args 1 end]
#non pipelined call to self - return result #*SUB* pipeline recursion.
#puts "======> recurse based on next1:$next1 "
if {[string index $next1 $nexteposn-1] eq {.}} {
#var1.= var2.= ...
#non pipelined call to self - return result
set results [uplevel 1 [list $next1 {*}$nexttail]]
#debug.punk.pipe.rep {==> rep recursive results: [rep $results]} 5
#debug.punk.pipe {>>> results: $results} 1
return [_handle_bind_result [_multi_bind_result $initial_returnvarspec $results]]
}
#puts "======> recurse assign based on next1:$next1 "
#if {[regexp {^([^ \t\r\n=\{]*)=(.*)} $next1 _ nextreturnvarspec nextrhs]} {
#}
#non pipelined call to plain = assignment - return result
set results [uplevel 1 [list $next1 {*}$nexttail]] set results [uplevel 1 [list $next1 {*}$nexttail]]
#debug.punk.pipe.rep {==> rep recursive results: [rep $results]} 5
#debug.punk.pipe {>>> results: $results} 1 #debug.punk.pipe {>>> results: $results} 1
return [_handle_bind_result [_multi_bind_result $initial_returnvarspec $results]] set d [_multi_bind_result $initial_returnvarspec $results]
return [_handle_bind_result $d]
} }
#puts "======> recurse assign based on next1:$next1 "
#if {[regexp {^([^ \t\r\n=\{]*)=(.*)} $next1 _ nextreturnvarspec nextrhs]} {
#}
#non pipelined call to plain = assignment - return result
set results [uplevel 1 [list $next1 {*}$nexttail]]
#debug.punk.pipe {>>> results: $results} 1
set d [_multi_bind_result $initial_returnvarspec $results]
return [_handle_bind_result $d]
} }
} }
@ -5981,6 +5992,9 @@ namespace eval punk {
tailcall {*}[list ::punk::pipeline = "" "" {*}$arglist] tailcall {*}[list ::punk::pipeline = "" "" {*}$arglist]
} }
#review
set assign [string map {; \\;} $assign]
set is_script [punk::pipe::lib::arg_is_script_shaped $assign] set is_script [punk::pipe::lib::arg_is_script_shaped $assign]
if {!$is_script && [string index $assign end] eq "="} { if {!$is_script && [string index $assign end] eq "="} {
@ -5999,7 +6013,7 @@ namespace eval punk {
if {$is_script} { if {$is_script} {
set cmdlist [list ::punk::pipeline "script" "" "" {*}$args] set cmdlist [list ::punk::pipeline "script" "" "" {*}$args]
} else { } else {
set cmdlist [list ::punk::pipeline ".=" "" "" {*}$args] set cmdlist [list ::punk::pipeline ".=" "" "" $assign {*}$arglist]
} }
} }
tailcall {*}$cmdlist tailcall {*}$cmdlist

1
src/modules/punk/aliascore-999999.0a1.0.tm

@ -123,6 +123,7 @@ tcl::namespace::eval punk::aliascore {
ansistrip ::punk::ansi::ansistrip ansistrip ::punk::ansi::ansistrip
stripansi ::punk::ansi::ansistrip stripansi ::punk::ansi::ansistrip
ansiwrap ::punk::ansi::ansiwrap ansiwrap ::punk::ansi::ansiwrap
ansisplit ::punk::ansi::ta::split_codes_single
grepstr ::punk::ansi::grepstr grepstr ::punk::ansi::grepstr
untabify ::punk::ansi::untabify untabify ::punk::ansi::untabify
colour ::punk::console::colour colour ::punk::console::colour

965
src/modules/punk/ansi-999999.0a1.0.tm

File diff suppressed because it is too large Load Diff

67
src/modules/punk/ansi/sauce-999999.0a1.0.tm

@ -218,7 +218,9 @@ tcl::namespace::eval punk::ansi::sauce {
#---------------------------------------------------------------------------------------------------------------------------------------------
# This data comes from the sauce spec.
#---------------------------------------------------------------------------------------------------------------------------------------------
#todo - fontName - which can also specify e.g code page 437 #todo - fontName - which can also specify e.g code page 437
## Font name [1] Font size Resolution [2] Aspect ratio [3] Vertical stretch [6] Description ## Font name [1] Font size Resolution [2] Aspect ratio [3] Vertical stretch [6] Description
## Display [4] Pixel [5] ## Display [4] Pixel [5]
@ -226,7 +228,14 @@ tcl::namespace::eval punk::ansi::sauce {
set fontnames [dict create] set fontnames [dict create]
## IBM VGA 9×16 [7] 720×400 4:3 20:27 (1:1.35) 35% Standard hardware font on VGA cards for 80×25 text mode (code page 437) ## IBM VGA 9×16 [7] 720×400 4:3 20:27 (1:1.35) 35% Standard hardware font on VGA cards for 80×25 text mode (code page 437)
dict set fontnames "IBM VGA" [list fontsize "9x16" resolution "720x400" aspect_ratio_display "4:3" aspect_ratio_pixel "20:27 (1:1.35)" vertical_stretch "35%" description "Standard hardware font on VGA cards for 80×25 text mode (code page 437)"] dict set fontnames "IBM VGA" [list {*}{
fontsize "9x16"
resolution "720x400"
aspect_ratio_display "4:3"
aspect_ratio_pixel "20:27 (1:1.35)"
vertical_stretch "35%"
description "Standard hardware font on VGA cards for 80×25 text mode (code page 437)"
}]
## IBM VGA ### [8] 9×16 [7] 720×400 4:3 20:27 (1:1.35) 35% Software installed code page font for VGA 80×25 text mode ## IBM VGA ### [8] 9×16 [7] 720×400 4:3 20:27 (1:1.35) 35% Software installed code page font for VGA 80×25 text mode
# - where ### is placeholder for 437,720,737 etc # - where ### is placeholder for 437,720,737 etc
@ -252,6 +261,7 @@ tcl::namespace::eval punk::ansi::sauce {
## C64 PETSCII unshifted 8×8 [10] 320×200 4:3 5:6 (1:1.2) 20% Original Commodore PETSCII font (PET, VIC-20, C64, CBM-II, Plus/4, C16, C116 and C128) in the unshifted mode. Unshifted mode (graphics) only has uppercase letters and additional graphic characters. This is the normal boot font. ## C64 PETSCII unshifted 8×8 [10] 320×200 4:3 5:6 (1:1.2) 20% Original Commodore PETSCII font (PET, VIC-20, C64, CBM-II, Plus/4, C16, C116 and C128) in the unshifted mode. Unshifted mode (graphics) only has uppercase letters and additional graphic characters. This is the normal boot font.
## C64 PETSCII shifted 8×8 [10] 320×200 4:3 5:6 (1:1.2) 20% Original PETSCII font in shifted mode. Shifted mode (text) has both uppercase and lowercase letters. This mode is actuated by pressing Shift+Commodore key. ## C64 PETSCII shifted 8×8 [10] 320×200 4:3 5:6 (1:1.2) 20% Original PETSCII font in shifted mode. Shifted mode (text) has both uppercase and lowercase letters. This mode is actuated by pressing Shift+Commodore key.
## Atari ATASCII 8×8 [11] 320×192 4:3 4:5 (1:1.25) 25% Original ATASCII font (Atari 400, 800, XL, XE) ## Atari ATASCII 8×8 [11] 320×192 4:3 4:5 (1:1.25) 25% Original ATASCII font (Atari 400, 800, XL, XE)
#---------------------------------------------------------------------------------------------------------------------------------------------
#expect a 128 Byte sauce record #expect a 128 Byte sauce record
@ -261,6 +271,7 @@ tcl::namespace::eval punk::ansi::sauce {
variable datatypes variable datatypes
variable filetypes variable filetypes
variable encodings variable encodings
set warnings [list]
if {[string length $saucerecord] != 128} { if {[string length $saucerecord] != 128} {
error "punk::ansi::sauce::to_dict: Unable to interpret data as a SAUCE record - length != 128" error "punk::ansi::sauce::to_dict: Unable to interpret data as a SAUCE record - length != 128"
} }
@ -326,6 +337,8 @@ tcl::namespace::eval punk::ansi::sauce {
dict set sdict filetype_name "" dict set sdict filetype_name ""
} }
} else { } else {
#how can a byte fail to scan with cu? is this even reachable?
puts stderr "punk::ansi::sauce::to_dict filetype byte failed to scan - setting filetype and filetype_name to empty string byte: [ansistring VIEW -lf 1 [string range $saucerecord 95 95]]"
dict set sdict filetype "" dict set sdict filetype ""
dict set sdict filetype_name "" dict set sdict filetype_name ""
} }
@ -422,25 +435,40 @@ tcl::namespace::eval punk::ansi::sauce {
5 { 5 {
#binarytext #binarytext
#filetype is supposed to represent half the characterwidth (only widths with multiples of 2 can be specified) #filetype is supposed to represent half the characterwidth (only widths with multiples of 2 can be specified)
#HOWEVER - in the wild we may see width/height specified in tinfo1/tinfo2 with some other value in filetype (eg 1) #HOWEVER - in the wild we may see width/height specified in tinfo1/tinfo2 with some apparently unrelated value in filetype (eg 0 or 1) that doesn't match the intended image dimensions.
#If both tinfo1 and tinfo2 are non zero - we will use them, even though it's not in spec. #If both tinfo1 and tinfo2 are non zero - we *could* use them, even though it's not in spec.
set t1 [dict get $sdict tinfo1] #An example file (us-used1.bin) has filetype 0 and tinfo1/tinfo2 640/350
if {$t1 eq ""} { #It's possible tinfo1/tinfo2 represent pixel dimensions for a 'standard' 8x16 font, but this image is 160 columns wide, so we would expect tinfo1 to be 1280.
set t1 0 #The sauce spec seems to indicate we should ignore tinfo1/tinfo2 for binarytext and only use filetype to determine width.
} #the default for binarytext is 160 columns.
set t2 [dict get $sdict tinfo2]
if {$t2 eq ""} { #filetype 1 is theoretically possible, representing 2 columns
set t2 0 #in practice we see this value for binarytext images that are definitely not intended to be 2 columns wide. Why?
#is there some assumption that that images are at least a certain width, and filetype has been repurposed to indicate something else?
#The spec would seem to rule out images of a single column due to filetype being half the character width but a value of 0.5 isn't supported.
#It specifically mentions that only even widths up to 510 can be specified. ($filetype * 2 where filetype is 1-255?)
#proper mechanism to specify columns for binarytext is the datatype field.
set cols [expr {2*[dict get $sdict filetype]}]
if {$cols == 0} {
lappend warnings "binarytext filetype value of [dict get $sdict filetype] - using binarytext default cols of 160"
#default for binarytext is 160 columns
set cols 160
} }
if {$t1 != 0 && $t2 != 0} { if {$cols == 2 && [dict get $sdict tinfo1] != 0 && [dict get $sdict tinfo2] != 0} {
#not to spec - but we will assume these have values for a reason.. #not to spec - but we will assume these have values for a reason..
puts stderr "punk::ansi::sauce::to_dict using tinfo1/tinfo2 data for columns/rows (non-compliant SAUCE data)" #---------------------------------------------------------------------------------------------------------------------
dict set sdict columns [expr {2 * $t1}] #The sample file src/testansi/formatsamples/image/binaryText/test.bin has a filetype 1 and tinfo1 40 and tinfo2 25.
dict set sdict rows $t2 #(similarly ppe-ansi.bin has tinfo1 80 and tinfo2 26)
#They seem to use the 1 in filetype to indicate that the tinfo1/tinfo2 values should be used.
#(The 80 cols wide test.bin binaryText image matches the xbin sample file src/testansi/formatsamples/image/xbin/test.xb which is a more fully specified format using a header)
#---------------------------------------------------------------------------------------------------------------------
lappend warnings "binarytext filetype of 1 with non-zero tinfo1/tinfo2 - using tinfo1/tinfo2 data for columns/rows (possibly non-conforming SAUCE data - matching observed data in the wild)"
set cols [expr {2 * [dict get $sdict tinfo1]}]
dict set sdict columns $cols
dict set sdict rows [dict get $sdict tinfo2]
} else { } else {
#proper mechanism to specify columns for binarytext is the datatype field.
set cols [expr {2*[dict get $sdict filetype]}]
dict set sdict columns $cols dict set sdict columns $cols
#rows must be calculated from file size #rows must be calculated from file size
#rows = (filesize - sauceinfosize)/ filetype * 2 * 2 #rows = (filesize - sauceinfosize)/ filetype * 2 * 2
@ -481,6 +509,9 @@ tcl::namespace::eval punk::ansi::sauce {
} }
} }
} }
if {[llength $warnings]} {
dict set sdict warnings $warnings
}
return $sdict return $sdict
} }

205
src/modules/punk/char-999999.0a1.0.tm

@ -3039,8 +3039,10 @@ tcl::namespace::eval punk::char {
set csplits [combiner_split $text] set csplits [combiner_split $text]
foreach {pt combiners} [lrange $csplits 0 end-1] { foreach {pt combiners} [lrange $csplits 0 end-1] {
set clist [split $pt ""] set clist [split $pt ""]
lappend components {*}[lrange $clist 0 end-1] lset clist end [tcl::string::cat [lindex $clist end] $combiners]
lappend components [tcl::string::cat [lindex $clist end] $combiners] lappend components {*}$clist
#lappend components {*}[lrange $clist 0 end-1]
#lappend components [tcl::string::cat [lindex $clist end] $combiners]
} }
#last csplit never has a combiner (_perlish_split style) - and may be empty - in which case we don't append it as a grapheme #last csplit never has a combiner (_perlish_split style) - and may be empty - in which case we don't append it as a grapheme
if {[lindex $csplits end] ne ""} { if {[lindex $csplits end] ne ""} {
@ -3066,126 +3068,121 @@ tcl::namespace::eval punk::char {
#review \uFE0F variation selector 16 - forces emoji presentation for preceding char #review \uFE0F variation selector 16 - forces emoji presentation for preceding char
if 1 { #This is a basic implementation that does not check that all combinations are valid.
#This is a basic implementation that does not check that all combinations are valid. set graphemes [list]
set graphemes [list] set current_cluster ""
set current_cluster ""
set cluster_base 0 ;#is the current cluster based on a char that can be combined with modifiers/ZWJs (e.g emoji or other cluster-based char)
set cluster_base 0 ;#is the current cluster based on a char that can be combined with modifiers/ZWJs (e.g emoji or other cluster-based char) # or is it based on a char that can't be combined with modifiers/ZWJs (e.g ascii letter)
# or is it based on a char that can't be combined with modifiers/ZWJs (e.g ascii letter) set cluster_base_RI 0 ;#is the current cluster based on a regional indicator char - which can only combine in pairs, and don't combine with modifiers or ZWJs to form longer clusters - so not extensible.
set cluster_base_RI 0 ;#is the current cluster based on a regional indicator char - which can only combine in pairs, and don't combine with modifiers or ZWJs to form longer clusters - so not extensible.
set current_cluster_is_extensible 0
set current_cluster_is_extensible 0 for {set i 0} {$i < [llength $components] } {incr i} {
for {set i 0} {$i < [llength $components] } {incr i} { set component [lindex $components $i]
set component [lindex $components $i] if {$component eq "\r" && [lindex $components $i+1] eq "\n"} {
if {$component eq "\r" && [lindex $components $i+1] eq "\n"} { if {$current_cluster ne ""} {
if {$current_cluster ne ""} { lappend graphemes $current_cluster
lappend graphemes $current_cluster }
} lappend graphemes "\r\n"
lappend graphemes "\r\n" incr i ;#skip the \n as we've already processed it as part of the cluster
incr i ;#skip the \n as we've already processed it as part of the cluster set current_cluster ""
set current_cluster "" set cluster_base 0; set cluster_base_RI 0 ;#grapheme_split::reset_base
grapheme_split::reset_base set current_cluster_is_extensible 0
} elseif {$component eq "\u200d"} {
if {$current_cluster eq ""} {
#ZWJ at start of string - treat as separate grapheme cluster - but isn't a valid base for further combining with more ZWJs or modifiers
set current_cluster $component
set cluster_base 0; set cluster_base_RI 0 ;#grapheme_split::reset_base
set current_cluster_is_extensible 0 set current_cluster_is_extensible 0
} elseif {$component eq "\u200d"} { } else {
if {$current_cluster eq ""} { if {$cluster_base} {
#ZWJ at start of string - treat as separate grapheme cluster - but isn't a valid base for further combining with more ZWJs or modifiers if {$current_cluster_is_extensible} {
set current_cluster $component #a double (or longer) ZWJ sequence in a row is part of the last cluster - but not extensible anymore.
grapheme_split::reset_base append current_cluster $component
set current_cluster_is_extensible 0 set current_is_cluster_extensible 0
} else {
if {$cluster_base} {
if {$current_cluster_is_extensible} {
#a double (or longer) ZWJ sequence in a row is part of the last cluster - but not extensible anymore.
append current_cluster $component
set current_is_cluster_extensible 0
} else {
append current_cluster $component
if {$cluster_base_RI} {
#regional indicators can only combine in pairs, and don't combine with modifiers or ZWJs to form longer clusters - so not extensible.
grapheme_split::reset_base
set current_cluster_is_extensible 0
#we can keep adding ZWJs or modifiers though
} else {
set current_cluster_is_extensible 1
}
}
} else { } else {
#ZWJ after non-cluster-based char - non extensible but we continue appending ZWJs to the current cluster.
append current_cluster $component append current_cluster $component
set current_cluster_is_extensible 0 if {$cluster_base_RI} {
} #regional indicators can only combine in pairs, and don't combine with modifiers or ZWJs to form longer clusters - so not extensible.
set cluster_base 0; set cluster_base_RI 0 ;#grapheme_split::reset_base
} set current_cluster_is_extensible 0
} elseif {[regexp {[\U1f3fb-\U1f3ff]} $component]} { #we can keep adding ZWJs or modifiers though
#emoji modifier - join with previous component
if {$current_cluster eq ""} {
#modifier at start of string - not a valid base for further combining with more modifiers or ZWJs - but we continue appending modifiers to the current cluster.
set current_cluster $component
grapheme_split::reset_base
} else {
if {$cluster_base} {
if {$current_cluster_is_extensible} {
append current_cluster $component
#invalidate the base!
grapheme_split::reset_base
} else { } else {
append current_cluster $component set current_cluster_is_extensible 1
} }
}
} else {
#ZWJ after non-cluster-based char - non extensible but we continue appending ZWJs to the current cluster.
append current_cluster $component
set current_cluster_is_extensible 0
}
}
} elseif {[regexp {[\U1f3fb-\U1f3ff]} $component]} {
#emoji modifier - join with previous component
if {$current_cluster eq ""} {
#modifier at start of string - not a valid base for further combining with more modifiers or ZWJs - but we continue appending modifiers to the current cluster.
set current_cluster $component
set cluster_base 0; set cluster_base_RI 0 ;#grapheme_split::reset_base
} else {
if {$cluster_base} {
if {$current_cluster_is_extensible} {
append current_cluster $component
#invalidate the base!
set cluster_base 0; set cluster_base_RI 0 ;#grapheme_split::reset_base
} else { } else {
#modifier after non-cluster-based char - non extensible but we continue appending modifiers to the current cluster.
append current_cluster $component append current_cluster $component
} }
#review } else {
# \u1f33e\u1f3fe\u200d\u2f3fe\u200d\u1f33e is 2 clusters #modifier after non-cluster-based char - non extensible but we continue appending modifiers to the current cluster.
#This is because after first zwj, we applied a modifier - not a valid base. append current_cluster $component
} }
set current_cluster_is_extensible 0 #review
# \u1f33e\u1f3fe\u200d\u2f3fe\u200d\u1f33e is 2 clusters
#This is because after first zwj, we applied a modifier - not a valid base.
}
set current_cluster_is_extensible 0
} else {
if {$current_cluster eq ""} {
grapheme_split::start_cluster $component
} else { } else {
if {$current_cluster eq ""} { #have existing cluster data
grapheme_split::start_cluster $component if {$current_cluster_is_extensible} {
} else { #assert - if current_cluster_is_extensible then cluster_base should currently be true.
#have existing cluster data #if the current char is a base - we can append to existing cluster, but if it's not a base, then we start a new cluster even if we had seen a ZWJ before.
if {$current_cluster_is_extensible} { if {[regexp {[\U1f600-\U1f64f\U1f300-\U1f5ff\U1f900-\U1f9ff\U1fa70-\U1faff\U1f680-\U1f6ff\U2700-\U27bf\U2600-\u26ff]} $component]} {
#assert - if current_cluster_is_extensible then cluster_base should currently be true. append current_cluster $component
#if the current char is a base - we can append to existing cluster, but if it's not a base, then we start a new cluster even if we had seen a ZWJ before. set cluster_base 1
if {[regexp {[\U1f600-\U1f64f\U1f300-\U1f5ff\U1f900-\U1f9ff\U1fa70-\U1faff\U1f680-\U1f6ff\U2700-\U27bf\U2600-\u26ff]} $component]} {
append current_cluster $component
set cluster_base 1
} else {
lappend graphemes $current_cluster
set current_cluster $component
grapheme_split::reset_base
}
set current_cluster_is_extensible 0
} elseif {$cluster_base_RI} {
#regional indicators can only combine in pairs, and don't combine with modifiers or ZWJs to form longer clusters - so not extensible.
if {[regexp {[\U1f1e6-\U1f1ff]} $component]} {
append current_cluster $component
#invalidate the base - we can't combine more than 2 RIs in a cluster, and they don't combine with modifiers or ZWJs to form longer clusters.
#we can however add more ZWJs or modifiers to the cluster - but they don't make it extensible for combining with more RIs
grapheme_split::reset_base
} else {
#something else while RI cluster is open - end the current cluster and start a new one with the current char.
lappend graphemes $current_cluster
grapheme_split::start_cluster $component
}
set current_cluster_is_extensible 0
} else { } else {
lappend graphemes $current_cluster
set current_cluster $component
set cluster_base 0; set cluster_base_RI 0 ;#grapheme_split::reset_base
}
set current_cluster_is_extensible 0
} elseif {$cluster_base_RI} {
#regional indicators can only combine in pairs, and don't combine with modifiers or ZWJs to form longer clusters - so not extensible.
if {[regexp {[\U1f1e6-\U1f1ff]} $component]} {
append current_cluster $component
#invalidate the base - we can't combine more than 2 RIs in a cluster, and they don't combine with modifiers or ZWJs to form longer clusters.
#we can however add more ZWJs or modifiers to the cluster - but they don't make it extensible for combining with more RIs
set cluster_base 0; set cluster_base_RI 0 ;#grapheme_split::reset_base
} else {
#something else while RI cluster is open - end the current cluster and start a new one with the current char.
lappend graphemes $current_cluster lappend graphemes $current_cluster
grapheme_split::start_cluster $component grapheme_split::start_cluster $component
} }
set current_cluster_is_extensible 0
} else {
lappend graphemes $current_cluster
grapheme_split::start_cluster $component
} }
} }
} }
if {$current_cluster ne ""} {
lappend graphemes $current_cluster
}
} else {
set graphemes $components
} }
if {$current_cluster ne ""} {
lappend graphemes $current_cluster
}
return $graphemes return $graphemes
} }
namespace eval grapheme_split { namespace eval grapheme_split {

3
src/modules/punk/lib-999999.0a1.0.tm

@ -4210,6 +4210,9 @@ namespace eval punk::lib {
if {[string index $key 0] ne "%"} { if {[string index $key 0] ne "%"} {
set key %$key set key %$key
} }
#puts "---key:'$key'"
set key [string map {; \\;} $key] ;#review
#puts "---key:'$key'"
#pipeline - use punk patterns. #pipeline - use punk patterns.
% thisval.= $key= $thisval % thisval.= $key= $thisval
} }

22
src/modules/punk/ns-999999.0a1.0.tm

@ -775,13 +775,8 @@ tcl::namespace::eval punk::ns {
#set parent [nsprefix $ns_absolute] #set parent [nsprefix $ns_absolute]
#set tail [nstail $ns_absolute] #set tail [nstail $ns_absolute]
#jjj
#set allchildren [lsort [nseval $base [list ::namespace children]]] #set allchildren [lsort [nseval $base [list ::namespace children]]]
#set allchildren [lsort [tcl::namespace::eval $base [list ::namespace children]]]
set allchildren [lsort [nseval $base [list ::namespace children]]]
#puts "->base:$base tailparts:$tailparts allchildren: $allchildren"
#puts "->base:$base tailparts:$tailparts childcount: [llength $allchildren]"
#** only significant when it is the trailing part of a segment eg ::**::xxx ::a**::xxx #** only significant when it is the trailing part of a segment eg ::**::xxx ::a**::xxx
if {[llength $tailparts]} { if {[llength $tailparts]} {
@ -790,6 +785,7 @@ tcl::namespace::eval punk::ns {
set nslist [nstree_list $base -subnslist {} -allbelow 1] set nslist [nstree_list $base -subnslist {} -allbelow 1]
} elseif {[regexp {[*]{2}$} $nextglob]} { } elseif {[regexp {[*]{2}$} $nextglob]} {
set nslist [list] set nslist [list]
set allchildren [lsort [nseval $base [list ::namespace children]]]
lappend nsmatches {*}[lsearch -all -inline -glob $allchildren [nsjoin ${base} $nextglob]] lappend nsmatches {*}[lsearch -all -inline -glob $allchildren [nsjoin ${base} $nextglob]]
foreach ch $nsmatches { foreach ch $nsmatches {
lappend nslist $ch lappend nslist $ch
@ -799,6 +795,7 @@ tcl::namespace::eval punk::ns {
} else { } else {
#lsearch with -glob ok even if nextglob has no globchars (no discernable speed diff, and earlier parts may have globchars anyway) #lsearch with -glob ok even if nextglob has no globchars (no discernable speed diff, and earlier parts may have globchars anyway)
set nslist [list] set nslist [list]
set allchildren [lsort [nseval $base [list ::namespace children]]]
lappend nsmatches {*}[lsearch -all -inline -glob $allchildren [nsjoin ${base} $nextglob]] lappend nsmatches {*}[lsearch -all -inline -glob $allchildren [nsjoin ${base} $nextglob]]
if {[llength $tailparts] >1 || $allbelow} { if {[llength $tailparts] >1 || $allbelow} {
foreach ch $nsmatches { foreach ch $nsmatches {
@ -812,6 +809,7 @@ tcl::namespace::eval punk::ns {
} }
} else { } else {
#puts "nstree_list: no tailparts base:$base" #puts "nstree_list: no tailparts base:$base"
set allchildren [lsort [nseval $base [list ::namespace children]]]
if {$allbelow} { if {$allbelow} {
set nsmatches $allchildren set nsmatches $allchildren
set nslist [list] set nslist [list]
@ -2134,8 +2132,8 @@ y" {return quirkykeyscript}
tcl::dict::set tinfo($target) procoffset 0 tcl::dict::set tinfo($target) procoffset 0
tcl::dict::set tinfo($target) level [expr {[::tcl::info::level]+1}] tcl::dict::set tinfo($target) level [expr {[::tcl::info::level]+1}]
tcl::dict::set tinfo($target) subcmds 0 tcl::dict::set tinfo($target) subcmds 0
puts "enter: $target -- $args" puts stderr "enter: $target -- $args"
puts "frame-2: [::tcl::info::frame -2]" #puts stderr "frame-2: [::tcl::info::frame -2]"
set _cmdtrace_disabled false set _cmdtrace_disabled false
} }
@ -2481,7 +2479,7 @@ y" {return quirkykeyscript}
set line $traceline set line $traceline
dict set linedict $target eval_base $traceline dict set linedict $target eval_base $traceline
dict set linedict $target eval_offset 1 dict set linedict $target eval_offset 1
puts " step type: proc traceline:$traceline ** $args" puts " step type: proc traceline:$traceline ** $args\x1b\[m"
#puts "** $callinfo" #puts "** $callinfo"
if {[dict exists $callinfo cmd]} { if {[dict exists $callinfo cmd]} {
#set cmd [string trim [dict get $callinfo cmd]] ;#raw 'unexpanded' script from the stack frame #set cmd [string trim [dict get $callinfo cmd]] ;#raw 'unexpanded' script from the stack frame
@ -2504,8 +2502,8 @@ y" {return quirkykeyscript}
set eval_base [dict get $linedict $target eval_base] set eval_base [dict get $linedict $target eval_base]
set eval_offset [dict get $linedict $target eval_offset] set eval_offset [dict get $linedict $target eval_offset]
set line [expr {$eval_base + ($eval_offset-1) + ($traceline-1)}] set line [expr {$eval_base + ($eval_offset-1) + ($traceline-1)}]
puts "stack-- $callinfo" #puts "stack-- $callinfo"
puts " step type: eval traceline: $traceline -- " puts stderr " step type: eval traceline: $traceline -- "
if {[dict exists $callinfo cmd]} { if {[dict exists $callinfo cmd]} {
#set cmd [string trim [dict get $callinfo cmd]] #set cmd [string trim [dict get $callinfo cmd]]
set cmdlist [lindex $args 0] set cmdlist [lindex $args 0]
@ -2627,6 +2625,8 @@ y" {return quirkykeyscript}
}] }]
} }
proc cmdtrace {args} { proc cmdtrace {args} {
#review - displaying argument values has to be done carefully. Small values are ok, but large lists or dicts can be overwhelming.
#Potentially we could apply some heuristics to truncate or summarise them.
package require dictn ;#convenience to allow dictn::incr d {key subkey} package require dictn ;#convenience to allow dictn::incr d {key subkey}
variable tinfo variable tinfo
array unset tinfo array unset tinfo
@ -2676,7 +2676,7 @@ y" {return quirkykeyscript}
#if the target command has a leading colon (e.g expr alternative :) we can't put a trace directly on a fully qualified name with a triple colon such as ::: #if the target command has a leading colon (e.g expr alternative :) we can't put a trace directly on a fully qualified name with a triple colon such as :::
#we will need to evaluate in the namespace #we will need to evaluate in the namespace
foreach {tgt_cmd ns nscmd} $resolved_targets { foreach {tgt_cmd ns nscmd} $resolved_targets {
puts "tracing target: $tgt_cmd whilst running: $origin $arglist" puts stderr "tracing target: $tgt_cmd whilst running: $origin $arglist"
#::tcl::namespace::eval $ns [list ::trace add execution $nscmd enter [list ::punk::ns::_cmdtrace_enter ::punk::ns::linedict $tgt_cmd]] #::tcl::namespace::eval $ns [list ::trace add execution $nscmd enter [list ::punk::ns::_cmdtrace_enter ::punk::ns::linedict $tgt_cmd]]
#::tcl::namespace::eval $ns [list ::trace add execution $nscmd enterstep [list ::punk::ns::_cmdtrace_enterstep ::punk::ns::linedict $tgt_cmd]] #::tcl::namespace::eval $ns [list ::trace add execution $nscmd enterstep [list ::punk::ns::_cmdtrace_enterstep ::punk::ns::linedict $tgt_cmd]]

290
src/modules/punk/path-999999.0a1.0.tm

@ -565,10 +565,45 @@ namespace eval punk::path {
end]] end]]
} }
## for comparison
#proc nsglob_as_re {glob} {
# #any segment that is not just * must match exactly one segment in the path
# set pats [list]
# foreach seg [nsparts_cached $glob] {
# switch -exact -- $seg {
# "" {
# lappend pats ""
# }
# * {
# #review - ::g*t will not find ::got:it (won't match single inner colon) - this should be fixed
# #lappend pats {[^:]*}
# #negative lookahead
# #any number of chars not followed by ::, followed by any number of non :
# lappend pats {(?:.(?!::))*[^:]*}
# }
# ** {
# lappend pats {.*}
# }
# default {
# set seg [string map {. [.]} $seg]
# if {[regexp {[*?]} $seg]} {
# #set pat [string map [list ** {.*} * {[^:]*} ? {[^:]}] $seg]
# set pat [string map [list ** {.*} * {(?:.(?!::))*[^:]*} ? {[^:]}] $seg]
# lappend pats "$pat"
# } else {
# lappend pats "$seg"
# }
# }
# }
# }
# return "^[join $pats ::]\$"
#}
proc pathglob_as_re {pathglob} { proc pathglob_as_re {pathglob} {
#*** !doctools #*** !doctools
#[call [fun pathglob_as_re] [arg pathglob]] #[call [fun pathglob_as_re] [arg pathglob]]
#[para] Returns a regular expression for matching a path to a glob pattern which can contain glob chars *|? in any segment of the path structure #[para] Returns a regular expression for matching a path to a glob pattern which can contain glob chars *|? in any segment of the path structure
#[para] Does not support square bracket globs or character classes.
#[para] ** matches any number of subdirectories. #[para] ** matches any number of subdirectories.
#[para] e.g /etc/**/*.txt will match any .txt files at any depth below /etc (except directly within /etc itself) #[para] e.g /etc/**/*.txt will match any .txt files at any depth below /etc (except directly within /etc itself)
#[para] e.g /etc/**.txt will match any .txt files at any depth below /etc #[para] e.g /etc/**.txt will match any .txt files at any depth below /etc
@ -589,7 +624,7 @@ namespace eval punk::path {
* {lappend pats {[^/]*}} * {lappend pats {[^/]*}}
** {lappend pats {.*}} ** {lappend pats {.*}}
default { default {
set seg [string map [list {^ {\^} $ {\$} [} {\[} ( {\(} \{ \\\{ \\ {\\}] $seg] ;#treat regex characters in the input as literals set seg [string map [list ^ {\^} $ {\$} \[ {\[} \] {\]} ( {\(} ) {\)} \{ \\\{ \\ {\\}] $seg] ;#treat regex characters (or tcl glob square bracket chars) in the input as literals
#set seg [string map [list . {[.]}] $seg] #set seg [string map [list . {[.]}] $seg]
set seg [string map {. [.]} $seg] set seg [string map {. [.]} $seg]
if {[regexp {[*?]} $seg]} { if {[regexp {[*?]} $seg]} {
@ -603,6 +638,52 @@ namespace eval punk::path {
} }
return "^[join $pats /]\$" return "^[join $pats /]\$"
} }
punk::args::define {
@id -id ::punk::path::globmatchpath
@cmd -name punk::path::globmatchpath\
-summary\
"Match path to *|**|? glob patterns"\
-help\
"Return a boolean indicating whether the path matches the specialised glob pattern.
A pattern such as /usr/*/bin will match any path that has /usr as the first segment and bin as the third segment,
with any single segment in between.
A pattern such as /usr/**/bin will match any path that has /usr as the first segment and bin as the last segment,
with 1 or more segments in between (so it will not match /usr/bin).
A pattern such as /usr/** will match any path that has /usr as the first segment, with 1 or more segments
following (so it will not match /usr itself).
A pattern such as **/*.txt will match any path that ends with .txt, with 1 or more leading segments
(so it will not match test.txt or .txt).
A pattern such as ** will match any path.
The glob characters * and ? are the only special characters in the pathglob syntax.
- they are treated as glob characters regardless of where they appear in the pathglob string.
Note that this is different from other Tcl glob contexts where square brackets can be used.
The pathglob syntax treats other characters, including square brackets as literals.
For example, the pattern /usr/te?t will match /usr/test and /usr/text but not /usr/texxt, and the pattern /usr/te*t
will match /usr/test, /usr/teat, and /usr/teeeet but not /usr/te/t.
The pathglob syntax does not support escaping of glob characters - any glob characters in the pathglob are treated
as glob characters. For example, the pattern /usr/* will match any path that has /usr as the first segment and any
single segment as the second segment, but there is no way to specify a pattern that matches any path that has /usr
as the first segment and a literal * as the second segment.
Caller must ensure that file separator is forward slash. (e.g use file normalize on windows)
options:
-nocase 0|1 (default 0 - case sensitive)
If -nocase is not supplied - default to case sensitive *except for driveletter*
ie - the driveletter alone in paths such as c:/etc will still be case insensitive. (ie c:/ETC/* will match C:/ETC/blah but not C:/etc/blah)
Explicitly specifying -nocase 0 will require the entire case to match including the driveletter.
"
@leaders
pathglob -type string -help "glob pattern to match path against. See [fun pathglob_as_re] for syntax of glob patterns"
path -type string -help "path to match against glob pattern"
@opts
-nocase -type boolean -default 0 -help\
"case insensitive matching (default false - case sensitive)
- except for driveletter on windows which is always case insensitive
unless -nocase 0 is explicitly specified"
@values -min 0 -max 0
}
# -id
proc globmatchpath {pathglob path args} { proc globmatchpath {pathglob path args} {
#*** !doctools #*** !doctools
#[call [fun globmatchpath] [arg pathglob] [arg path] [opt {option value...}]] #[call [fun globmatchpath] [arg pathglob] [arg path] [opt {option value...}]]
@ -669,46 +750,182 @@ namespace eval punk::path {
@opts @opts
-recursive -type none -help\ -recursive -type none -help\
"" ""
-exclude-paths -type list -default {} -help\
"list of path patterns to exclude from results.
May include * and ** path segments e.g /usr/**
A single /*/ will match any single segment in the path, and a single /**/ will match any number of segments in the path.
e.g to exclude any path with _aside as a segment in the middle: -exclude-paths **/_aside/**
i.e this would exclude /usr/_aside/etc and /usr/x/_aside/etc but not /usr/x/_aside or _aside/etc
To exclude all paths with _aside as a segment anywhere: -exclude-paths { **/_aside/** **/_aside _aside/**}
"
#todo -depth #todo -depth
@values -min 0 -max 1 @values -min 0 -max 1
path -type directory -optional 1 -help\ path -type directory -optional 1 -help\
"Path of folder. If not supplied current directory is used." "Path of folder. If not supplied current directory is used.
This may be a relative or absolute path. Relative paths are treated as relative to current directory.
When using relative paths - the result will also be relative paths with the same relative prefix.
(e.g if path is ../test - the results will be ../test/subfolder1 ../test/subfolder2 etc)
Patterns in -exclude-paths are matched against the resulting paths
(so should be written to match the same relative prefix if path is relative)"
} }
proc subfolders {args} { proc subfolders {args} {
#NOTE - this algorithm based on omit_only_patterns and prune_base_patterns was suggested by a 2026 AI model - it is apparent to this programmer that it is inadequate for the purpose.
#e.g consider subfolders -recursion -exclude {**/vfs/** **/src/**}
#This can still return something like c:/repo/etc/src/vfs - which should be excluded by the pattern **/src/**
#todo - review and fix properly.
set argd [punk::args::parse $args withid ::punk::path::subfolders] set argd [punk::args::parse $args withid ::punk::path::subfolders]
lassign [dict values $argd] leaders opts values received lassign [dict values $argd] leaders opts values received
set do_recursion [dict exists $received -recursive] set do_recursion [dict exists $received -recursive]
set exclude_paths [dict get $opts -exclude-paths]
if {"**" in $exclude_paths} {
#if ** is in exclude_paths - then we can skip all glob matching and just return empty list
#This is likely user error - so we'll be loud about it for now but will still return empty list rather than erroring.
#If user code is building exclude_paths dynamically - they can check for this case themselves and avoid the call to subfolders to suppress this message.
puts stderr "punk::path::subfolders Warning - exclude_paths contains '**' - all paths will be excluded"
return [list]
}
if {[dict exists $received path]} { if {[dict exists $received path]} {
set path [dict get $values path] set path [dict get $values path]
} else { } else {
set path [pwd] set path [pwd]
} }
set folders [glob -nocomplain -directory $path -types d *]
set all_subfolders [glob -nocomplain -directory $path -types d *]
#example of expected exclude_paths pattern behaviour when recursion is enabled:
# **/dirname -> omit /x/y/dirname, but still visit /x/y/dirname/*
# **/dirname/* -> include /x/y/dirname and /x/y/dirname/a/b but omit directories that are a single level below /x/y/dirname such as /x/y/dirname/a
#c:/** - would exclude all subfolders below c: but not c: itself
# **/test/** - would exclude any path with test as a segment and all its subfolders
#- but not paths with test as a segment that is the final segment
set omit_only_patterns [list]
set prune_base_patterns [list]
foreach pat $exclude_paths {
set pat_parts [file split $pat] ;#note file split c:/test gives {c:/ test} but file split **/test gives {** test}
#also note that file split on windows treats forward slashes and backslashes the same.
#by using file split, we gain some flexibility in syntax of paths and patterns,
#but lose the ability to use backslashes as escapes to allow literal glob characters in path segments.
#This is almost always a non-issue on windows since * and ? are not valid in path segments there, and is rarely an issue on unix even though
# * and ? are technically valid in path segments, but it is inadvisable there anyway for compatibility with shells etc.
if {[llength $pat_parts] >= 2 && [lindex $pat_parts end] eq "**"} {
#** at end of pattern - e.g /dir/etc/**
#Convert ".../" to base "...", and prune descendants of that base.
lappend prune_base_patterns [file join {*}[lrange $pat_parts 0 end-1]]
} else {
lappend omit_only_patterns $pat
}
}
set folders [list]
set recurse_subdirs [list]
foreach f $all_subfolders {
set include_in_results 1
set allow_recurse 1
foreach pat $omit_only_patterns {
if {[globmatchpath $pat $f]} {
set include_in_results 0
break
}
}
if {$allow_recurse && [llength $prune_base_patterns]} {
foreach base_pat $prune_base_patterns {
#prune both the matched base node and its decendants.
if {[globmatchpath $base_pat $f]} {
set allow_recurse 0
break
}
if {[globmatchpath "${base_pat}/**" $f]} {
set include_in_results 0
set allow_recurse 0
break
}
}
}
if {$include_in_results} {
lappend folders $f
}
if {$allow_recurse} {
lappend recurse_subdirs $f
}
}
if {$do_recursion} { if {$do_recursion} {
foreach subdir $folders { foreach subdir $recurse_subdirs {
lappend folders {*}[subfolders -recursive $subdir] lappend folders {*}[subfolders -exclude-paths $exclude_paths -recursive $subdir]
} }
} }
#if {[llength $exclude_paths]} {
# set folders [list]
# foreach f $all_subfolders {
# set skip 0
# foreach pat $exclude_paths {
# #review - this is slightly too simplistic.
# # for exclusion pattern **/dirname
# # this will exclude any path with dirname as final segment - but it will also exclude any path with dirname as a segment anywhere in the path - which is not intended.
# #puts stderr "Checking exclude pat '$pat' against '$f'"
# if {[globmatchpath $pat $f]} {
# set skip 1
# break
# }
# }
# if {!$skip} {
# lappend folders $f
# }
# }
#} else {
# set folders $all_subfolders
#}
#if {$do_recursion} {
# foreach subdir $folders {
# lappend folders {*}[subfolders -exclude-paths $exclude_paths -recursive $subdir]
# }
#}
return $folders return $folders
} }
#todo - treefolders with similar search caps as treefilenames #todo - treefolders with similar search caps as treefilenames
punk::args::define { punk::args::define {
@id -id ::punk::path::treefilenames @id -id ::punk::path::treefilenames
@cmd -name punk::path::treefilenames\
-summary\
"List of filenames below supplied path."\
-help\
"List of filenames below path.
The resulting list is unsorted."
-directory -type directory -help\ -directory -type directory -help\
"folder in which to begin recursive scan for files." "folder in which to begin recursive scan for files."
-call-depth-internal -default 0 -type integer -call-depth-internal -default 0 -type integer -help "internal use only - caller should not specify - used to track depth of recursive calls for internal logic"
-sort -type any -default natural -choices {none ascii dictionary natural} -call-subvector -default {} -type list -help "internal use only - caller should not specify - used to track path vector of recursive calls for internal logic"
-call-allbelow -default 1 -type boolean -help "internal use only - caller should not specify - used to track whether we are in a subtree below a match for glob_paths (which means we can skip glob matching and antiglob_paths checks and just include all files below here)"
-sort -type any -default natural -choices {none ascii dictionary natural}
-antiglob_paths -default {} -help\ -antiglob_paths -default {} -help\
"list of path patterns to exclude "list of path patterns to exclude
may include * and ** path segments e.g may include * and ** path segments e.g
/usr/** (exlude subfolders based at /usr but not /usr/** (exclude subfolders based at /usr but not
files within /usr itself) files within /usr itself)
**/_aside (exlude files where _aside is last segment) **/_aside (exclude files where _aside is last segment)
**/_aside/* (exclude folders one below an _aside folder) **/_aside/* (exclude folders one below an _aside folder)
**/_aside/** (exclude all folders with _aside as a segment)" **/_aside/** (exclude all folders with _aside as a segment)"
-antiglob_files -default {} -antiglob_files -default {}
-glob_paths -default {*} -help\
"list of path patterns to include
may include * and ** path segments e.g
/usr/** (include subfolders based at /usr but not
files within /usr itself)
**/_aside (include files where _aside is last segment)
**/_aside/* (include folders one below an _aside folder)
**/_aside/** (include all folders with _aside as a segment)"
@values -min 0 -max -1 -optional 1 -type string @values -min 0 -max -1 -optional 1 -type string
tailglobs -default * -multiple 1 -help\ tailglobs -default * -multiple 1 -help\
"Patterns to match against filename portion (last segment) of each file path "Patterns to match against filename portion (last segment) of each file path
@ -732,12 +949,20 @@ namespace eval punk::path {
lassign [dict values $argd] leaders opts values received lassign [dict values $argd] leaders opts values received
set tailglobs [dict get $values tailglobs] set tailglobs [dict get $values tailglobs]
# -- --- --- --- --- --- --- # -- --- --- --- --- --- ---
set opt_sort [dict get $opts -sort] set opt_sort [dict get $opts -sort]
set opt_antiglob_paths [dict get $opts -antiglob_paths] set opt_antiglob_paths [dict get $opts -antiglob_paths]
set opt_antiglob_files [dict get $opts -antiglob_files] set opt_glob_paths [dict get $opts -glob_paths]
set CALLDEPTH [dict get $opts -call-depth-internal] set opt_antiglob_files [dict get $opts -antiglob_files]
set CALLDEPTH [dict get $opts -call-depth-internal]
set callsubvector [dict get $opts -call-subvector]
set callallbelow [dict get $opts -call-allbelow] ;#whether to return matches longer than the matched glob-path
# -- --- --- --- --- --- --- # -- --- --- --- --- --- ---
# -- --- --- --- --- --- --- # -- --- --- --- --- --- ---
if {"*" in $opt_glob_paths} {
#if we have a * in the default glob_paths - then any other entries are redundant.
set opt_glob_paths {*}
}
set files [list] set files [list]
if {$CALLDEPTH == 0} { if {$CALLDEPTH == 0} {
@ -745,14 +970,17 @@ namespace eval punk::path {
package require natsort package require natsort
} }
#set opts [dict merge $opts [list -directory $opt_dir]] #set opts [dict merge $opts [list -directory $opt_dir]]
if {![dict exists $received -directory]} { if {[dict exists $received -directory]} {
set opt_dir [pwd]
} else {
set opt_dir [dict get $opts -directory] set opt_dir [dict get $opts -directory]
} else {
set opt_dir [pwd]
} }
if {![file isdirectory $opt_dir]} { if {![file isdirectory $opt_dir]} {
return [list] return [list]
} }
} else { } else {
#assume/require to exist in any recursive call #assume/require to exist in any recursive call
set opt_dir [dict get $opts -directory] set opt_dir [dict get $opts -directory]
@ -831,19 +1059,35 @@ namespace eval punk::path {
lappend okdirs $dir lappend okdirs $dir
} }
} }
if {[llength $okdirs]} { if {$opt_glob_paths eq {*}} {
set matchdirs $okdirs
} else {
#** only significant when it is the trailing part of a segment eg /**/xxx /a**/xxx
set matchdirs [list]
foreach dir $okdirs {
foreach gp $opt_glob_paths {
if {[globmatchpath $gp $dir] || [globmatchpath "$gp/**" $dir]} {
lappend matchdirs $dir
}
}
}
}
if {[llength $matchdirs]} {
switch -- $opt_sort { switch -- $opt_sort {
ascii { ascii {
set finaldirs [lsort $okdirs] set finaldirs [lsort $matchdirs]
} }
dictionary { dictionary {
set finaldirs [lsort -dictionary $okdirs] set finaldirs [lsort -dictionary $matchdirs]
} }
natural { natural {
set finaldirs [natsort::sort $okdirs] set finaldirs [natsort::sort $matchdirs]
} }
default { default {
set finaldirs $okdirs set finaldirs $matchdirs
} }
} }
foreach dir $finaldirs { foreach dir $finaldirs {

5
src/modules/punk/pipe-999999.0a1.0.tm

@ -169,8 +169,8 @@ tcl::namespace::eval punk::pipe::lib {
#This stops us matching {/@**@x x} vs {/@**@x x} #This stops us matching {/@**@x x} vs {/@**@x x}
#--- #---
set rhs [tcl::string::map {: <c> ? <q> * <star> [ <lb> ] <rb> \\ <bsl> {"} <dq> " " <sp>} $rhs] set rhs [tcl::string::map {: <c> ; <sc> ? <q> * <star> [ <lb> ] <rb> \\ <bsl> {"} <dq> " " <sp>} $rhs]
#review - we don't expect other command-incompatible chars such as colon? #review - we don't expect other command-incompatible chars?
return $rhs return $rhs
} }
@ -187,6 +187,7 @@ tcl::namespace::eval punk::pipe::lib {
#exclude quoted whitespace #exclude quoted whitespace
proc arg_is_script_shaped {arg} { proc arg_is_script_shaped {arg} {
set arg [string map {\\; "<escaped_semicolon>"} $arg]
if {[tcl::string::first \n $arg] >= 0} { if {[tcl::string::first \n $arg] >= 0} {
return 1 return 1
} elseif {[tcl::string::first ";" $arg] >= 0} { } elseif {[tcl::string::first ";" $arg] >= 0} {

17
src/modules/punk/repo-999999.0a1.0.tm

@ -1817,17 +1817,13 @@ namespace eval punk::repo {
error "unimplemented" error "unimplemented"
} }
#file normalize is expensive so this is too #file normalize can be a little expensive so this is too
proc norm {path {platform env}} { proc norm {path {platform env}} {
#kettle::path::norm
#see also wiki
#full path normalization
set platform [string tolower $platform]
if {$platform eq "env"} {
set platform $::tcl_platform(platform)
}
#set platform [string tolower $platform]
#if {$platform eq "env"} {
# set platform $::tcl_platform(platform)
#}
#No - don't do this sort of path translation here - leave as option for specific utils only such as ./ #No - don't do this sort of path translation here - leave as option for specific utils only such as ./
#Windows volume-relative syntax with specific volume specified is somewhat broken in Tcl - but leading slash volume-relative does work #Windows volume-relative syntax with specific volume specified is somewhat broken in Tcl - but leading slash volume-relative does work
#We shouldn't break it totally just because accessing WSL/mingw paths is slightly more useful #We shouldn't break it totally just because accessing WSL/mingw paths is slightly more useful
@ -1835,6 +1831,9 @@ namespace eval punk::repo {
#return [file dirname [file normalize [punk::unixywindows::towinpath $path]/__]] #return [file dirname [file normalize [punk::unixywindows::towinpath $path]/__]]
#} #}
#kettle::path::norm
#see also wiki
#full path normalization
return [file dirname [file normalize $path/__]] return [file dirname [file normalize $path/__]]
} }

755
src/modules/test/#modpod-overtype-999999.0a1.0/overtype-1.7.4_testsuites/overtype/renderline.test

@ -1,4 +1,5 @@
package require tcltest package require tcltest
package require overtype
namespace eval ::testspace { namespace eval ::testspace {
namespace import ::tcltest::* namespace import ::tcltest::*
@ -6,6 +7,32 @@ namespace eval ::testspace {
set result "" set result ""
} }
# Temporarily replaces punk::console::get_tabstops so tab-related tests are deterministic.
proc with_tabstops {tabstops body} {
variable __tabstops
set __tabstops $tabstops
set had_original [expr {[llength [info commands ::punk::console::get_tabstops]] > 0}]
if {$had_original} {
rename ::punk::console::get_tabstops ::testspace::__orig_get_tabstops
}
namespace eval ::punk::console {}
proc ::punk::console::get_tabstops {{inoutchannels {stdin stdout}}} {
return [set ::testspace::__tabstops]
}
set code [catch {uplevel 1 $body} out opts]
rename ::punk::console::get_tabstops {}
if {$had_original} {
rename ::testspace::__orig_get_tabstops ::punk::console::get_tabstops
}
return -options $opts $out
}
proc vis {s} {
string map [list "\u0000" "<NUL>" "\x1b" "<ESC>" "\n" "<LF>" "\r" "<CR>" "\t" "<TAB>"] $s
}
test renderline_basic_noansi {basic renderline calls with no ansi in underlay or overlay}\ test renderline_basic_noansi {basic renderline calls with no ansi in underlay or overlay}\
-setup $common -body { -setup $common -body {
set undertext "abcdefghij" set undertext "abcdefghij"
@ -26,6 +53,734 @@ namespace eval ::testspace {
ABCDEfghij ABCDEabcde ABCDEfghij ABCDEabcde
] ]
test renderline_empty_overlay_passthrough {empty overlay returns undertext unchanged} \
-setup $common -body {
overtype::renderline abcdef ""
}\
-cleanup {
}\
-result abcdef
test renderline_startcolumn_overtype_plain {startcolumn with overtype mode replaces at offset} \
-setup $common -body {
overtype::renderline -insert_mode 0 -startcolumn 3 abcdef XY
}\
-cleanup {
}\
-result abXYef
test renderline_error_newline_undertext {undertext cannot contain newline} \
-setup $common -body {
overtype::renderline "ab\ncd" XX
}\
-cleanup {
}\
-returnCodes error \
-match glob \
-result "*not allowed to contain newlines in undertext*"
test renderline_error_unknown_option {unknown option should error} \
-setup $common -body {
overtype::renderline -bogus 1 abc XX
}\
-cleanup {
}\
-returnCodes error \
-match glob \
-result "*unknown option*"
test renderline_error_cursor_row_non_integer {cursor_row must be integer if specified} \
-setup $common -body {
overtype::renderline -cursor_row x abc XX
}\
-cleanup {
}\
-returnCodes error \
-match glob \
-result "*-cursor_row must be empty for unspecified/unknown or a non-zero positive integer*"
test renderline_error_cursor_row_non_positive {cursor_row must be positive if specified} \
-setup $common -body {
overtype::renderline -cursor_row 0 abc XX
}\
-cleanup {
}\
-returnCodes error \
-match glob \
-result "*-cursor_row must be empty for unspecified/unknown or a non-zero positive integer*"
test renderline_info_basic_contract {info mode returns expected key shape for simple case} \
-setup $common -body {
set d [overtype::renderline -info 1 -insert_mode 0 abcdef XX]
expr {
[dict exists $d result]
&& [dict exists $d instruction]
&& [dict exists $d cursor_column]
&& [dict exists $d overflow_right]
&& [dict exists $d unapplied]
&& [dict get $d result] eq "XXcdef"
}
}\
-cleanup {
}\
-result 1
test renderline_basic_sgr_overlay_no_forced_reset {sgr overlay is preserved and no unconditional reset is appended} \
-setup $common -body {
set out [overtype::renderline aaaa "\x1b\[31mB"]
expr {
[string first "\x1b\[31m" $out] >= 0
&& [string first B $out] >= 0
&& ![string match "*\x1b\[0m" $out]
}
}\
-cleanup {
}\
-result 1
test renderline_transparent_default_space {transparent 1 makes spaces in overlay pass through underlay} \
-setup $common -body {
overtype::renderline -insert_mode 0 -transparent 1 abcde " X "
}\
-cleanup {
}\
-result aXcde
test renderline_transparent_custom_regex {custom transparent regexp is honored} \
-setup $common -body {
overtype::renderline -insert_mode 0 -transparent {[#]} abcde #Y#
}\
-cleanup {
}\
-result aYcde
test renderline_expand_right_off_tracks_unapplied {non-expanding insert leaves unapplied/overflow info} \
-setup $common -body {
set d [overtype::renderline -info 1 -insert_mode 1 -expand_right 0 abc WXYZ]
expr {
[string length [dict get $d result]] == 3
&& ([string length [dict get $d unapplied]] > 0 || [string length [dict get $d overflow_right]] > 0)
}
}\
-cleanup {
}\
-result 1
test renderline_expand_right_on_grows_result {expanding insert allows output growth on same shape} \
-setup $common -body {
set d [overtype::renderline -info 1 -insert_mode 1 -expand_right 1 abc WXYZ]
expr {
[string length [dict get $d result]] >= 4
&& [dict get $d unapplied] eq ""
}
}\
-cleanup {
}\
-result 1
test renderline_control_cr_repositions_to_start {carriage return moves cursor back to start column} \
-setup $common -body {
overtype::renderline -insert_mode 0 abcde "A\rZ"
}\
-cleanup {
}\
-result Zbcde
test renderline_control_bs_moves_back_one {backspace rewinds one column before next char} \
-setup $common -body {
overtype::renderline -insert_mode 0 abcde "AB\bZ"
}\
-cleanup {
}\
-result AZcde
test renderline_control_lf_sets_instruction_and_unapplied {linefeed reports lf_mid and keeps tail unapplied} \
-setup $common -body {
set d [overtype::renderline -info 1 -insert_mode 0 abcde "A\nZ"]
list [dict get $d instruction] [dict get $d unapplied]
}\
-cleanup {
}\
-result [list lf_mid Z]
test renderline_widechar_transparency_default {without transparency wide first-half can be replaced directly} \
-setup $common -body {
overtype::renderline -insert_mode 0 "A\uFF5EB" " X"
}\
-cleanup {
}\
-result " XB"
test renderline_widechar_transparency_enabled {transparency preserves first half and marks exposed second half} \
-setup $common -body {
overtype::renderline -insert_mode 0 -transparent 1 "A\uFF5EB" " X"
}\
-cleanup {
}\
-result "A\uFFFDXB"
test renderline_expand_right_off_overflow_column {overflow_right_column is tracked in non-expanding insert mode} \
-setup $common -body {
set d [overtype::renderline -info 1 -insert_mode 1 -expand_right 0 abc WXYZ]
dict get $d overflow_right_column
}\
-cleanup {
}\
-result 4
test renderline_tab_uses_stubbed_tabstops {tab advances to deterministic stop when tabstops are stubbed} \
-setup $common -body {
with_tabstops {1 9 17 25} {
set d [overtype::renderline -info 1 -insert_mode 0 abcdefghij "A\tZ"]
list [dict get $d result] [dict get $d cursor_column] [dict get $d instruction] [dict get $d unapplied]
}
}\
-cleanup {
}\
-result [list AbcdefghZj 10 "" ""]
test renderline_cursor_save_restore_esc7_esc8 {ESC7 and ESC8 restore cursor and leave trailing data unapplied} \
-setup $common -body {
set d [overtype::renderline -info 1 -insert_mode 0 abcde "AB\x1b7CD\x1b8Z"]
list [dict get $d result] [dict get $d instruction] [dict get $d unapplied] [dict get $d cursor_saved_position]
}\
-cleanup {
}\
-result [list ABCDe restore_cursor Z [dict create row 1 column 3]]
test renderline_cursor_save_restore_csi_s_u {CSI s and CSI u restore cursor and leave trailing data unapplied} \
-setup $common -body {
set over "AB\x1b\[sCD\x1b\[uZ"
set d [overtype::renderline -info 1 -insert_mode 0 abcde $over]
list [dict get $d result] [dict get $d instruction] [dict get $d unapplied] [dict get $d cursor_saved_position]
}\
-cleanup {
}\
-result [list ABCDe restore_cursor Z [dict create row 1 column 3]]
test renderline_csi_forward_basic {CSI nC moves cursor forward before rendering next grapheme} \
-setup $common -body {
set over "A\x1b\[3CZ"
set d [overtype::renderline -info 1 -insert_mode 0 abcdef $over]
list [dict get $d result] [dict get $d cursor_column] [dict get $d instruction] [dict get $d unapplied]
}\
-cleanup {
}\
-result [list AbcdZf 6 "" ""]
test renderline_csi_back_basic {CSI nD moves cursor backward before rendering next grapheme} \
-setup $common -body {
set over "ABCD\x1b\[2DZ"
set d [overtype::renderline -info 1 -insert_mode 0 abcdef $over]
list [dict get $d result] [dict get $d cursor_column] [dict get $d instruction] [dict get $d unapplied]
}\
-cleanup {
}\
-result [list ABZDef 4 "" ""]
test renderline_csi_forward_wrapmoveforward_instruction {large CSI nC can trigger wrapmoveforward with unapplied tail} \
-setup $common -body {
set over "A\x1b\[20CZ"
set d [overtype::renderline -info 1 -insert_mode 0 abcdef $over]
list [dict get $d result] [dict get $d instruction] [dict get $d cursor_column] [dict get $d unapplied]
}\
-cleanup {
}\
-result [list Abcdef wrapmoveforward 22 Z]
test renderline_csi_back_wrapmovebackward_instruction {CSI nD from start can trigger wrapmovebackward with unapplied tail} \
-setup $common -body {
set over "A\x1b\[2DZ"
set d [overtype::renderline -info 1 -insert_mode 0 abcdef $over]
list [dict get $d result] [dict get $d instruction] [dict get $d cursor_column] [dict get $d unapplied]
}\
-cleanup {
}\
-result [list Abcdef wrapmovebackward 0 Z]
test renderline_mode_irm_on_from0 {CSI 4h enables insert mode state} \
-setup $common -body {
set over "\x1b\[4hX"
set d [overtype::renderline -info 1 -insert_mode 0 -autowrap_mode 1 abcdef $over]
list [dict get $d insert_mode] [dict get $d result] [dict get $d instruction] [dict get $d unapplied]
}\
-cleanup {
}\
-result [list 1 Xabcde "" ""]
test renderline_mode_irm_off_from1 {CSI 4l disables insert mode state} \
-setup $common -body {
set over "\x1b\[4lX"
set d [overtype::renderline -info 1 -insert_mode 1 -autowrap_mode 1 abcdef $over]
list [dict get $d insert_mode] [dict get $d result] [dict get $d instruction] [dict get $d unapplied]
}\
-cleanup {
}\
-result [list 0 Xbcdef "" ""]
test renderline_mode_awm_off_from1 {CSI ?7l disables autowrap state} \
-setup $common -body {
set over "\x1b\[?7lX"
set d [overtype::renderline -info 1 -insert_mode 0 -autowrap_mode 1 abcdef $over]
list [dict get $d autowrap_mode] [dict get $d result] [dict get $d instruction] [dict get $d unapplied]
}\
-cleanup {
}\
-result [list 0 Xbcdef "" ""]
test renderline_mode_awm_on_from0 {CSI ?7h enables autowrap state} \
-setup $common -body {
set over "\x1b\[?7hX"
set d [overtype::renderline -info 1 -insert_mode 0 -autowrap_mode 0 abcdef $over]
list [dict get $d autowrap_mode] [dict get $d result] [dict get $d instruction] [dict get $d unapplied]
}\
-cleanup {
}\
-result [list 1 Xbcdef "" ""]
test renderline_mode_crm_on_from0 {CSI 3h enables CRM mode state} \
-setup $common -body {
set over "\x1b\[3hX"
set d [overtype::renderline -info 1 -insert_mode 0 -autowrap_mode 1 -crm_mode 0 abcdef $over]
expr {
[dict get $d crm_mode] == 1
&& [dict get $d instruction] eq ""
&& [dict get $d unapplied] eq ""
&& [string length [dict get $d result]] >= 1
}
}\
-cleanup {
}\
-result 1
test renderline_mode_crm_off_from1 {CSI 3l disables CRM mode state} \
-setup $common -body {
set over "\x1b\[3lX"
set d [overtype::renderline -info 1 -insert_mode 0 -autowrap_mode 1 -crm_mode 1 abcdef $over]
list [dict get $d crm_mode] [dict get $d result] [dict get $d instruction] [dict get $d unapplied]
}\
-cleanup {
}\
-result [list 0 Xbcdef "" ""]
#todo
#test renderline_erase_line_sequences_currently_noop {CSI K variants keep current rendered content in this implementation} \
# -setup $common -body {
# set k0 "AB\x1b\[0K"
# set k1 "AB\x1b\[1K"
# set k2 "AB\x1b\[2K"
# set d0 [overtype::renderline -info 1 -insert_mode 0 abcdef $k0]
# set d1 [overtype::renderline -info 1 -insert_mode 0 abcdef $k1]
# set d2 [overtype::renderline -info 1 -insert_mode 0 abcdef $k2]
# list \
# [dict get $d0 result] [dict get $d0 instruction] [dict get $d0 cursor_column] \
# [dict get $d1 result] [dict get $d1 instruction] [dict get $d1 cursor_column] \
# [dict get $d2 result] [dict get $d2 instruction] [dict get $d2 cursor_column]
# }\
# -cleanup {
# }\
# -result [list ABcdef "" 3 ABcdef "" 3 ABcdef "" 3]
#test renderline_erase_display_0_and_1_currently_noop {CSI J0 and J1 keep current rendered content in this implementation} \
# -setup $common -body {
# set j0 "AB\x1b\[0J"
# set j1 "AB\x1b\[1J"
# set d0 [overtype::renderline -info 1 -insert_mode 0 abcdef $j0]
# set d1 [overtype::renderline -info 1 -insert_mode 0 abcdef $j1]
# list \
# [dict get $d0 result] [dict get $d0 instruction] [dict get $d0 cursor_column] \
# [dict get $d1 result] [dict get $d1 instruction] [dict get $d1 cursor_column]
# }\
# -cleanup {
# }\
# -result [list ABcdef "" 3 ABcdef "" 3]
test renderline_erase_display_2_clears_and_moves {CSI J2 clears line and returns clear_and_move instruction} \
-setup $common -body {
set j2 "AB\x1b\[2J"
set d [overtype::renderline -info 1 -insert_mode 0 abcdef $j2]
list [dict get $d result] [dict get $d instruction] [dict get $d cursor_column] [dict get $d unapplied]
}\
-cleanup {
}\
-result [list " " clear_and_move 1 ""]
test renderline_csi_G_absolute_column {CSI G sets absolute column and renders next grapheme there} \
-setup $common -body {
set over "A\x1b\[5GZ"
set d [overtype::renderline -info 1 -insert_mode 0 abcdef $over]
list [dict get $d result] [dict get $d cursor_column] [dict get $d instruction] [dict get $d unapplied]
}\
-cleanup {
}\
-result [list AbcdZf 6 "" ""]
test renderline_csi_tick_absolute_column_alias {CSI backtick behaves as HPA alias like CSI G} \
-setup $common -body {
set over "A\x1b\[5`Z"
set d [overtype::renderline -info 1 -insert_mode 0 abcdef $over]
list [dict get $d result] [dict get $d cursor_column] [dict get $d instruction] [dict get $d unapplied]
}\
-cleanup {
}\
-result [list AbcdZf 6 "" ""]
test renderline_csi_G_zero_column {CSI 0G moves cursor to column zero in this implementation} \
-setup $common -body {
set over "A\x1b\[0GZ"
set d [overtype::renderline -info 1 -insert_mode 0 abcdef $over]
list [dict get $d result] [dict get $d cursor_column] [dict get $d instruction] [dict get $d unapplied]
}\
-cleanup {
}\
-result [list Abcdef 0 "" ""]
test renderline_csi_H_same_row_applies {CSI H on current row repositions and applies overlay} \
-setup $common -body {
set over "A\x1b\[1;2HZ"
set d [overtype::renderline -info 1 -insert_mode 0 -cursor_row 1 abcdef $over]
list [dict get $d result] [dict get $d cursor_column] [dict get $d cursor_row] [dict get $d instruction] [dict get $d unapplied]
}\
-cleanup {
}\
-result [list AZcdef 3 1 "" ""]
test renderline_csi_H_other_row_returns_move {CSI H to another row reports move and leaves tail unapplied} \
-setup $common -body {
set over "A\x1b\[2;1HZ"
set d [overtype::renderline -info 1 -insert_mode 0 -cursor_row 1 abcdef $over]
list [dict get $d result] [dict get $d cursor_column] [dict get $d cursor_row] [dict get $d instruction] [dict get $d unapplied]
}\
-cleanup {
}\
-result [list Abcdef 1 2 move Z]
test renderline_csi_up_reports_up_instruction {CSI A moves to a previous row and returns up instruction} \
-setup $common -body {
set over "A\x1b\[1AZ"
set d [overtype::renderline -info 1 -insert_mode 0 -cursor_row 2 abcdef $over]
list [dict get $d result] [dict get $d cursor_column] [dict get $d cursor_row] [dict get $d instruction] [dict get $d unapplied]
}\
-cleanup {
}\
-result [list Abcdef 2 1 up Z]
test renderline_csi_up_multiple_clamps_to_first_row {CSI multiple A updates row and leaves remaining text unapplied} \
-setup $common -body {
set over "A\x1b\[2AZ"
set d [overtype::renderline -info 1 -insert_mode 0 -cursor_row 3 abcdef $over]
list [dict get $d result] [dict get $d cursor_column] [dict get $d cursor_row] [dict get $d instruction] [dict get $d unapplied]
}\
-cleanup {
}\
-result [list Abcdef 2 1 up Z]
test renderline_csi_down_reports_down_instruction {CSI B moves to a later row and returns down instruction} \
-setup $common -body {
set over "A\x1b\[1BZ"
set d [overtype::renderline -info 1 -insert_mode 0 -cursor_row 1 abcdef $over]
list [dict get $d result] [dict get $d cursor_column] [dict get $d cursor_row] [dict get $d instruction] [dict get $d unapplied]
}\
-cleanup {
}\
-result [list Abcdef 2 2 down Z]
test renderline_csi_down_multiple_updates_target_row {CSI multiple B updates row and leaves remaining text unapplied} \
-setup $common -body {
set over "A\x1b\[2BZ"
set d [overtype::renderline -info 1 -insert_mode 0 -cursor_row 1 abcdef $over]
list [dict get $d result] [dict get $d cursor_column] [dict get $d cursor_row] [dict get $d instruction] [dict get $d unapplied]
}\
-cleanup {
}\
-result [list Abcdef 2 3 down Z]
test renderline_csi_next_line_moves_to_column_one {CSI E moves to next row column one and returns move instruction} \
-setup $common -body {
set over "A\x1b\[1EZ"
set d [overtype::renderline -info 1 -insert_mode 0 -cursor_row 1 abcdef $over]
list [dict get $d result] [dict get $d cursor_column] [dict get $d cursor_row] [dict get $d instruction] [dict get $d unapplied]
}\
-cleanup {
}\
-result [list Abcdef 1 2 move Z]
test renderline_csi_prev_line_moves_to_column_one {CSI F moves to previous row column one and returns move instruction} \
-setup $common -body {
set over "A\x1b\[1FZ"
set d [overtype::renderline -info 1 -insert_mode 0 -cursor_row 2 abcdef $over]
list [dict get $d result] [dict get $d cursor_column] [dict get $d cursor_row] [dict get $d instruction] [dict get $d unapplied]
}\
-cleanup {
}\
-result [list Abcdef 1 1 move Z]
test renderline_info_cursor_saved_attributes_esc7 {ESC7 save captures active SGR and ESC8 restore leaves tail unapplied} \
-setup $common -body {
set over "\x1b\[31mA\x1b7\x1b\[32mB\x1b8Z"
set d [overtype::renderline -info 1 -insert_mode 0 abcdef $over]
list \
[vis [dict get $d result]] \
[dict get $d instruction] \
[dict get $d cursor_saved_position] \
[vis [dict get $d cursor_saved_attributes]] \
[vis [dict get $d unapplied]]
}\
-cleanup {
}\
-result [list {<ESC>[31mA<ESC>[32mB<ESC>[0mcdef} restore_cursor [dict create row 1 column 2] {<ESC>[31m} Z]
test renderline_info_cursor_saved_attributes_csi_s_u {CSI s save captures active SGR and CSI u restore leaves tail unapplied} \
-setup $common -body {
set over "\x1b\[31mA\x1b\[s\x1b\[32mB\x1b\[uZ"
set d [overtype::renderline -info 1 -insert_mode 0 abcdef $over]
list \
[vis [dict get $d result]] \
[dict get $d instruction] \
[dict get $d cursor_saved_position] \
[vis [dict get $d cursor_saved_attributes]] \
[vis [dict get $d unapplied]]
}\
-cleanup {
}\
-result [list {<ESC>[31mA<ESC>[32mB<ESC>[0mcdef} restore_cursor [dict create row 1 column 2] {<ESC>[31m} Z]
test renderline_info_replay_codes_underlay_tracks_open_underlay_sgr {open underlay SGR is reported in replay_codes and replay_codes_underlay} \
-setup $common -body {
set under "\x1b\[34mabcdef"
set d [overtype::renderline -info 1 -insert_mode 0 $under X]
list \
[vis [dict get $d result]] \
[vis [dict get $d replay_codes]] \
[vis [dict get $d replay_codes_underlay]] \
[vis [dict get $d replay_codes_overlay]]
}\
-cleanup {
}\
-result [list {X<ESC>[34mbcdef} {<ESC>[34m} {<ESC>[34m} {}]
test renderline_info_replay_codes_overlay_tracks_overlay_sgr {overlay SGR is reported in replay_codes_overlay} \
-setup $common -body {
set over "\x1b\[31mX"
set d [overtype::renderline -info 1 -insert_mode 0 abcdef $over]
list \
[vis [dict get $d result]] \
[vis [dict get $d replay_codes]] \
[vis [dict get $d replay_codes_underlay]] \
[vis [dict get $d replay_codes_overlay]]
}\
-cleanup {
}\
-result [list {<ESC>[31mX<ESC>[0mbcdef} {} {} {<ESC>[31m}]
test renderline_info_replay_codes_preserve_underlay_reset_boundary {underlay reset becomes replay_codes_underlay while overlay keeps its own replay} \
-setup $common -body {
set under "\x1b\[34mabcdef\x1b\[0m"
set over "\x1b\[31mX"
set d [overtype::renderline -info 1 -insert_mode 0 $under $over]
list \
[vis [dict get $d result]] \
[vis [dict get $d replay_codes]] \
[vis [dict get $d replay_codes_underlay]] \
[vis [dict get $d replay_codes_overlay]]
}\
-cleanup {
}\
-result [list {<ESC>[31mX<ESC>[0;34mbcdef<ESC>[0m} {<ESC>[0m} {<ESC>[0m} {<ESC>[31m}]
test renderline_exposed_custom_overtype_default {custom exposed markers are unused when overlay fully replaces wide char second half without transparency} \
-setup $common -body {
overtype::renderline -insert_mode 0 -exposed1 L -exposed2 R "A\uFF5EB" " X"
}\
-cleanup {
}\
-result { XB}
test renderline_exposed_custom_transparent_uses_exposed1 {transparent split over wide char uses custom exposed1 marker} \
-setup $common -body {
overtype::renderline -insert_mode 0 -transparent 1 -exposed1 L -exposed2 R "A\uFF5EB" " X"
}\
-cleanup {
}\
-result ALXB
test renderline_exposed_custom_startcolumn_uses_exposed2 {starting on wide char second half uses custom exposed2 marker} \
-setup $common -body {
overtype::renderline -insert_mode 0 -startcolumn 2 -exposed1 L -exposed2 R "A\uFF5EB" X
}\
-cleanup {
}\
-result AXRB
test renderline_exposed_custom_insert_mode_preserves_shifted_wide_char {insert mode keeps shifted wide char rather than exposing second half marker} \
-setup $common -body {
overtype::renderline -insert_mode 1 -exposed1 L -exposed2 R "A\uFF5EB" X
}\
-cleanup {
}\
-result "XA\uFF5E"
test renderline_exposed_custom_info_contract {info mode reports stable fields for custom exposed transparency case} \
-setup $common -body {
set d [overtype::renderline -info 1 -insert_mode 0 -transparent 1 -exposed1 L -exposed2 R "A\uFF5EB" " X"]
list [dict get $d result] [dict get $d instruction] [dict get $d cursor_column] [dict get $d overflow_right] [dict get $d unapplied]
}\
-cleanup {
}\
-result [list ALXB "" 4 "" ""]
test renderline_del_deletes_at_cursor {literal DEL deletes at current cursor position} \
-setup $common -body {
overtype::renderline -insert_mode 0 abcdef "AB\x7f"
}\
-cleanup {
}\
-result ABdef
test renderline_del_after_backspace_deletes_previous_column {backspace followed by DEL removes the backed-up character position} \
-setup $common -body {
overtype::renderline -insert_mode 0 abcdef "AB\b\x7f"
}\
-cleanup {
}\
-result Acdef
test renderline_del_info_contract {DEL reports stable info fields} \
-setup $common -body {
set d [overtype::renderline -info 1 -insert_mode 0 abcdef "AB\x7f"]
list [dict get $d result] [dict get $d cursor_column] [dict get $d instruction] [dict get $d unapplied] [dict get $d overflow_right]
}\
-cleanup {
}\
-result [list ABdef 3 "" "" ""]
test renderline_ech_one_erases_single_character_to_space {CSI X with count 1 replaces one character with a space} \
-setup $common -body {
overtype::renderline -insert_mode 0 abcdef "AB\x1b\[1X"
}\
-cleanup {
}\
-result {AB def}
test renderline_ech_two_erases_two_characters_to_spaces {CSI X with count 2 replaces two characters with spaces} \
-setup $common -body {
overtype::renderline -insert_mode 0 abcdef "AB\x1b\[2X"
}\
-cleanup {
}\
-result {AB ef}
test renderline_ech_info_contract {ECH reports stable info fields} \
-setup $common -body {
set d [overtype::renderline -info 1 -insert_mode 0 abcdef "AB\x1b\[1X"]
list [dict get $d result] [dict get $d cursor_column] [dict get $d instruction] [dict get $d unapplied] [dict get $d overflow_right]
}\
-cleanup {
}\
-result [list {AB def} 3 "" "" ""]
test renderline_cp437_off_preserves_control_byte {without cp437 flag, low control byte is preserved in the rendered output} \
-setup $common -body {
set c [format "%c" 1]
overtype::renderline -insert_mode 0 abcdef "A${c}Z"
}\
-cleanup {
}\
-result "A\x01Zef"
test renderline_cp437_on_maps_control_to_glyph {cp437 flag maps low control bytes to printable cp437 glyphs} \
-setup $common -body {
set c [format "%c" 1]
overtype::renderline -cp437 1 -insert_mode 0 abcdef "A${c}Z"
}\
-cleanup {
}\
-result "A\u263aZdef"
test renderline_gx_overlay_passthrough {overlay gx0 on/off sequences are preserved in rendered output} \
-setup $common -body {
set gx "\x1b(0x\x1b(By"
set d [overtype::renderline -info 1 -insert_mode 0 abcdef $gx]
list [vis [dict get $d result]] [vis [dict get $d replay_codes]] [vis [dict get $d replay_codes_underlay]] [vis [dict get $d replay_codes_overlay]]
}\
-cleanup {
}\
-result [list {<ESC>(0x<ESC>(Bycdef} {} {} {}]
test renderline_gx_underlay_replay_tracks_gx_close {gx underlay with plain overlay returns gx close in replay_codes} \
-setup $common -body {
set under "\x1b(0abcdef\x1b(B"
set d [overtype::renderline -info 1 -insert_mode 0 $under X]
list [vis [dict get $d result]] [vis [dict get $d replay_codes]] [vis [dict get $d replay_codes_underlay]] [vis [dict get $d replay_codes_overlay]]
}\
-cleanup {
}\
-result [list {X<ESC>(0bcdef<ESC>(B} {<ESC>(B} {} {}]
test renderline_dch_one_deletes_and_pads_right_edge {CSI P deletes one character at cursor and pads the right edge with an empty cell} \
-setup $common -body {
set d [overtype::renderline -info 1 -insert_mode 0 abcdef "AB\x1b\[1P"]
list \
[vis [dict get $d result]] \
[dict get $d cursor_column] \
[dict get $d instruction] \
[vis [dict get $d unapplied]] \
[vis [dict get $d overflow_right]]
}\
-cleanup {
}\
-result [list {ABdef } 3 "" "" ""]
test renderline_dch_two_deletes_and_pads_right_edge {CSI P with count 2 deletes two characters and pads two empty cells on the right} \
-setup $common -body {
set d [overtype::renderline -info 1 -insert_mode 0 abcdef "AB\x1b\[2P"]
list \
[vis [dict get $d result]] \
[dict get $d cursor_column] \
[dict get $d instruction] \
[vis [dict get $d unapplied]] \
[vis [dict get $d overflow_right]]
}\
-cleanup {
}\
-result [list {ABef } 3 "" "" ""]
test renderline_dch_defaults_count_to_one {CSI P with empty parameter behaves like count 1} \
-setup $common -body {
set d [overtype::renderline -info 1 -insert_mode 0 abcdef "AB\x1b\[P"]
list [vis [dict get $d result]] [dict get $d cursor_column]
}\
-cleanup {
}\
-result [list {ABdef } 3]
test renderline_ich_one_inserts_blank_and_shifts_right {CSI @ with count 1 inserts one blank at the cursor position} \
-setup $common -body {
set d [overtype::renderline -info 1 -insert_mode 0 abcdef "AB\x1b\[1@"]
list [dict get $d result] [dict get $d cursor_column] [dict get $d instruction] [dict get $d unapplied] [dict get $d overflow_right]
}\
-cleanup {
}\
-result [list {AB cde} 3 "" "" ""]
test renderline_ich_two_inserts_two_blanks_and_shifts_right {CSI @ with count 2 inserts two blanks and shifts the rest of the line right} \
-setup $common -body {
set d [overtype::renderline -info 1 -insert_mode 0 abcdef "AB\x1b\[2@"]
list [dict get $d result] [dict get $d cursor_column] [dict get $d instruction] [dict get $d unapplied] [dict get $d overflow_right]
}\
-cleanup {
}\
-result [list {AB cd} 3 "" "" ""]
test renderline_ich_clamps_insert_count_to_line_width {CSI @ clamps very large insert counts to the current line width} \
-setup $common -body {
set d [overtype::renderline -info 1 -insert_mode 0 abcdef "AB\x1b\[10@"]
list [dict get $d result] [dict get $d cursor_column] [dict get $d instruction] [dict get $d unapplied] [dict get $d overflow_right]
}\
-cleanup {
}\
-result [list {AB } 3 "" "" ""]
#todo - test #todo - test
#P% overtype::left -transparent 1 [textblock::block 10 2 -] " [a+ underline yellow].\n [a+ underline yellow]yyy" #P% overtype::left -transparent 1 [textblock::block 10 2 -] " [a+ underline yellow].\n [a+ underline yellow]yyy"
#- --.------- #- --.-------

77
src/modules/test/runtestmodules.tcl

@ -3,6 +3,65 @@
#(plain tclsh may stall - todo - review reasons for this and whether shellfilter can be modified to support ordinary tclsh) #(plain tclsh may stall - todo - review reasons for this and whether shellfilter can be modified to support ordinary tclsh)
#A known working copy of a punk shell executable should be placed on the path and the shebang line updated to reflect this #A known working copy of a punk shell executable should be placed on the path and the shebang line updated to reflect this
#------------------------------------
lassign [split [info tclversion] .] tcl_major tcl_minor
set script_dir [file dirname [file normalize [info script]]]
set modules_posn [string first /modules/ $script_dir]
if {$modules_posn < 0} {
puts stderr "Error: script dir $script_dir does not contain /modules/"
#exit 2 ;#don't call exit. If run in a single proc it can cause the hole test suite exit before summary can be printed.
return -code error "Error: script dir $script_dir does not contain /modules/"
}
set modules_base [string range $script_dir 0 $modules_posn-1]
if {[file tail $modules_base] eq "src"} {
set project_root [file dirname $modules_base]
} else {
set project_root $modules_base
}
puts stderr "runtestmodules.tcl project_root: $project_root"
#use the unbuilt modules/libraries under development rather than the installed versions.
#The unbuilt modules should have a higher version number (such as the magic version number 999999.0a1.0) than any installed versions to ensure they are preferred.
tcl::tm::add [file normalize $project_root/src/modules]
tcl::tm::add [file normalize $project_root/src/modules_tcl$tcl_major]
tcl::tm::add [file normalize $project_root/src/vendormodules]
tcl::tm::add [file normalize $project_root/src/vendormodules_tcl$tcl_major]
# add 'package ifneeded' definitions for unbuilt #modpod modules.
#first gather subdirectories of modules that contain #modpod-*-999999.0a1.0 in their name - these should be the unbuilt versions of zip based modules.
#set subfolders [punk::path::subfolders -recursive [file normalize $project_root/src/modules] -match */#modpod-*-999999.0a1.0]
#'punk::path::subfolders' currently only supports negative matching with -exclude, so we have to filter for the positive match ourselves.
set subfolders [punk::path::subfolders -recursive [file normalize $project_root/src/modules] -exclude {**/_build/** **/_build}]
foreach sub $subfolders {
#In most cases we could use string match - but the * within modpod-*-999999.0a1.0 could match a forward slash which could then match some other file under a #modpod- folder structure,
#so we use globmatchpath which treats * as matching any characters except path separators.
if {[globmatchpath "**/#modpod-*-999999.0a1.0" $sub]} {
set modname [file tail $sub]
set modname [string range $modname 8 end-12] ;#strip off #modpod- and -999999.0a1.0
set modpath [file join $sub "$modname-999999.0a1.0.tm"]
#!!!!
#todo - calculate fully qualified module name based on path relative to the modules folder we added to the tcl::tm path.
if {[file exists $modpath]} {
puts stderr "runtestmodules.tcl adding package ifneeded for modpod module $modname at path $modpath"
package ifneeded $modname 999999.0a1.0 [list source $modpath]
} else {
puts stderr "runtestmodules.tcl warning: expected mod.tcl not found for modpod module $modname at path $modpath"
}
}
}
set libdir [file normalize $project_root/src/lib]
set libvdir [file normalize $project_root/src/lib/tcl$tcl_major]
set libvldir [file normalize $project_root/src/vendorlib]
set libvlvdir [file normalize $project_root/src/vendorlib_tcl$tcl_major]
foreach d [list $libdir $libvdir $libvldir $libvlvdir] {
if {$d ni $::auto_path} {
lappend ::auto_path $d
}
}
#------------------------------------
puts stderr "runtestmodules.tcl ::auto_path: $::auto_path"
puts stderr "runtestmodules.tcl tcl::tm::list: [tcl::tm::list]"
package require punk package require punk
package require punk::args package require punk::args
@ -122,7 +181,7 @@ foreach pkg $punktestpkgs {
foreach ln [split $chunk \n] { foreach ln [split $chunk \n] {
incr i incr i
if {[string match "Tests ended at*" $ln]} { if {[string match "Tests ended at*" $ln]} {
puts stdout "<stdout> [punk::ansi::ansistring VIEW -lf 2 -cr 1 "$pkg $ln"]" puts stdout "<stdout><$pkg> $ln"
} elseif {[string match "*:*Total*Passed*Skipped*Failed*" $ln]} { } elseif {[string match "*:*Total*Passed*Skipped*Failed*" $ln]} {
set fields [lrange $ln 1 end] set fields [lrange $ln 1 end]
dict for {K v} $fields { dict for {K v} $fields {
@ -136,16 +195,26 @@ foreach pkg $punktestpkgs {
} }
} }
} }
puts stdout "<stdout>$pkg $ln" puts stdout "<stdout><$pkg> $ln"
} elseif {[string match "*Sourced * Test Files*" $ln]} {
puts stdout "<stdout><$pkg> $ln"
} else { } else {
puts stdout "<stdout> $ln" if {[string trim $ln] ne ""} {
puts stdout "<stdout> $ln"
} else {
puts -nonewline stdout "\n"
}
#puts stdout "$i" #puts stdout "$i"
} }
} }
flush stdout flush stdout
} }
stderr { stderr {
puts stderr "<stderr> [punk::ansi::ansistring VIEW -lf 2 -cr 1 $chunk]" #puts stderr "<stderr> [punk::ansi::ansistring VIEW -lf 2 -cr 1 $chunk]"
set chunkview [punk::ansi::ansistring VIEW -lf 2 -cr 1 $chunk]
foreach ln [split $chunkview \n] {
puts stderr "<stderr> $ln"
}
flush stderr flush stderr
} }
default { default {

66
src/tests/all.tcl

@ -0,0 +1,66 @@
#!tclsh
#This script uses shellfilter::run calls under the hood
lassign [split [info tclversion] .] tcl_major tcl_minor
set script_dir [file dirname [info script]]
#------------------------------------
#use the unbuilt modules/libraries under development rather than the installed versions.
set original_tmlist [tcl::tm::list]
tcl::tm::remove {*}$original_tmlist
tcl::tm::add [file normalize $script_dir/../modules] ;#ie <projectroot>/src/modules
tcl::tm::add [file normalize $script_dir/../modules_tcl$tcl_major]
tcl::tm::add {*}[lreverse $original_tmlist]
set libdir [file normalize $script_dir/../lib]
set libvdir [file normalize $script_dir/../lib/tcl$tcl_major]
if {$libdir ni $::auto_path} {
lappend ::auto_path $libdir
}
if {$libvdir ni $::auto_path} {
lappend ::auto_path $libvdir
}
#------------------------------------
package require tcltest
package require punk
package require punk::args
punk::args::define {
@id -id (script)::runtestmodules
@cmd -name runtestmodules -help\
"Run test:: modules that support the packagetest api
(have RUN command)"
-tcltestoptions -type dict -default "" -help\
"pairs of flags/values that will be passed to tcltest::configure before running the tests.
For example, to run only tests with names matching *foo* and *bar* you could use:
-tcltestoptions {-file {*foo* *bar*}}
"
@values -min 0 -max -1
glob -type string -multiple 1 -optional 1 -help\
" names or glob patterns of test files to run."
}
set argd [punk::args::parse $::argv withid (script)::runtestmodules]
lassign [dict values $argd] leaders opts values received
set tcltestoptions [dict get $opts -tcltestoptions]
if {![dict exists $received glob]} {
set file_globs [list *.test]
} else {
set file_globs [dict get $values glob]
}
set ::argv $tcltestoptions
set ::argc [llength $tcltestoptions]
#set ::argv {}
#set ::argc 0
tcltest::configure -verbose "body pass skip error usec"
tcltest::configure -testdir $script_dir
tcltest::configure -file $file_globs
#review - single process has less isolation - but works better in this case.
#(some tclsh shells can hang when running with -singleproc false - needs investigation)
#tclte::configure -singleproc true
tcltest::configure -singleproc true
dict for {k v} $tcltestoptions {
tcltest::configure $k $v
}
tcltest::runAllTests

39
src/tests/modules/opunk/str/tests/all.tcl

@ -0,0 +1,39 @@
if {[llength $::argv]} {
puts stderr "$script_dir ::argv $::argv"
}
#------------------------------------
lassign [split [info tclversion] .] tcl_major tcl_minor
set script_dir [file dirname [file normalize [info script]]]
set src_tests_posn [string first /src/tests/ $script_dir]
if {$src_tests_posn < 0} {
puts stderr "Error: script dir $script_dir does not contain /src/tests/"
#exit 2 ;#don't call exit. If run in a single proc it can cause the hole test suite exit before summary can be printed.
return -code error "Error: script dir $script_dir does not contain /src/tests/"
}
set project_root [string range $script_dir 0 $src_tests_posn-1]
#use the unbuilt modules/libraries under development rather than the installed versions.
#The unbuilt modules should have a higher version number (such as the magic version number 999999.0a1.0) than any installed versions to ensure they are preferred.
tcl::tm::add [file normalize $project_root/src/modules]
tcl::tm::add [file normalize $project_root/src/modules_tcl$tcl_major]
set libdir [file normalize $project_root/src/lib]
set libvdir [file normalize $project_root/src/lib/tcl$tcl_major]
if {$libdir ni $::auto_path} {
lappend ::auto_path $libdir
}
if {$libvdir ni $::auto_path} {
lappend ::auto_path $libvdir
}
#------------------------------------
package require tcltest
puts "----------------- [tcl::tm::list]"
#tcltest::configure -debug 1
tcltest::configure -singleproc true
tcltest::configure {*}$::argv
tcltest::configure -testdir $script_dir
tcltest::configure -asidefromdir * ;#only the toplevel all.tcl should recurse.
tcltest::runAllTests

84
src/tests/modules/opunk/str/tests/str.test

@ -0,0 +1,84 @@
package require tcltest
tcltest::configure {*}$::argv
#------------------------------------
lassign [split [info tclversion] .] tcl_major tcl_minor
set script_dir [file dirname [file normalize [info script]]]
set src_tests_posn [string first /src/tests/ $script_dir]
if {$src_tests_posn < 0} {
puts stderr "Error: script dir $script_dir does not contain /src/tests/"
#exit 2 ;#don't call exit. If run in a single proc it can cause the hole test suite exit before summary can be printed.
return -code error "Error: script dir $script_dir does not contain /src/tests/"
}
set project_root [string range $script_dir 0 $src_tests_posn-1]
#use the unbuilt modules/libraries under development rather than the installed versions.
#The unbuilt modules should have a higher version number (such as the magic version number 999999.0a1.0) than any installed versions to ensure they are preferred.
tcl::tm::add [file normalize $project_root/src/modules]
tcl::tm::add [file normalize $project_root/src/modules_tcl$tcl_major]
set libdir [file normalize $project_root/src/lib]
set libvdir [file normalize $project_root/src/lib/tcl$tcl_major]
if {$libdir ni $::auto_path} {
lappend ::auto_path $libdir
}
if {$libvdir ni $::auto_path} {
lappend ::auto_path $libvdir
}
#------------------------------------
puts stdout "==================== $::argv"
puts stdout "==================== [tcl::tm::list]"
package require overtype
package require opunk::str
namespace eval ::testspace {
namespace import ::tcltest::*
variable common {
set result ""
}
# Temporarily replaces punk::console::get_tabstops so tab-related tests are deterministic.
proc with_tabstops {tabstops body} {
variable __tabstops
set __tabstops $tabstops
set had_original [expr {[llength [info commands ::punk::console::get_tabstops]] > 0}]
if {$had_original} {
rename ::punk::console::get_tabstops ::testspace::__orig_get_tabstops
}
namespace eval ::punk::console {}
proc ::punk::console::get_tabstops {{inoutchannels {stdin stdout}}} {
return [set ::testspace::__tabstops]
}
set code [catch {uplevel 1 $body} out opts]
rename ::punk::console::get_tabstops {}
if {$had_original} {
rename ::testspace::__orig_get_tabstops ::punk::console::get_tabstops
}
return -options $opts $out
}
proc vis {s} {
string map [list "\u0000" "<NUL>" "\x1b" "<ESC>" "\n" "<LF>" "\r" "<CR>" "\t" "<TAB>"] $s
}
test opunkstr_basic {basic string object creation and properties}\
-setup $common -body {
set text "abcde"
set o [opunk::Str new $text]
set result [list \
[string equal $text [opunk::Str::get $o]] \
[opunk::Str::count $o] \
[expr {[opunk::Str::has_ansi $o] && true}] \
]
}\
-cleanup {
}\
-result [list {*}{
1 5 0
}]
}
tcltest::cleanupTests ;#needed to produce test summary.

38
src/tests/modules/punk/path/tests/all.tcl

@ -0,0 +1,38 @@
lassign [split [info tclversion] .] tcl_major tcl_minor
set script_dir [file dirname [file normalize [info script]]]
if {[llength $::argv]} {
puts stderr "$script_dir ::argv $::argv"
}
set src_tests_posn [string first /src/tests/ $script_dir]
if {$src_tests_posn < 0} {
puts "Error: script dir $script_dir does not contain /src/tests/"
exit 2
}
set project_root [string range $script_dir 0 $src_tests_posn-1]
#------------------------------------
#use the unbuilt modules/libraries under development rather than the installed versions.
#The unbuilt modules should have a higher version number (such as the magic version number 999999.0a1.0) than any installed versions to ensure they are preferred.
tcl::tm::add [file normalize $project_root/src/modules]
tcl::tm::add [file normalize $project_root/src/modules_tcl$tcl_major]
set libdir [file normalize $project_root/src/lib]
set libvdir [file normalize $project_root/src/lib/tcl$tcl_major]
if {$libdir ni $::auto_path} {
lappend ::auto_path $libdir
}
if {$libvdir ni $::auto_path} {
lappend ::auto_path $libvdir
}
#------------------------------------
package require tcltest
#tcltest::configure -debug 1
tcltest::configure -singleproc true
tcltest::configure {*}$::argv
tcltest::configure -testdir $script_dir
tcltest::configure -asidefromdir * ;#only the toplevel all.tcl should recurse.
tcltest::runAllTests

33
src/tests/modules/punk/path/tests/path.test

@ -0,0 +1,33 @@
package require tcltest
tcltest::configure {*}$::argv
package require overtype
package require punk::path
namespace eval ::testspace {
namespace import ::tcltest::*
variable common {
set result ""
}
test globmatchpath_basic {Test single star between slashes pathglob argument will match exactly a single level}\
-setup $common -body {
set result [list {*}{
} [punk::path::globmatchpath /etc/*/*.doc /etc/A/test.doc] {*}{
} [punk::path::globmatchpath /etc/*/*.doc /etc/A/B/test.doc] {*}{
} [punk::path::globmatchpath /etc/*/*.doc /etc/test.doc]
]
}\
-cleanup {
}\
-result [list {*}{
1 0 0
}]
}
tcltest::cleanupTests ;#needed to produce test summary.

828
src/vfs/_vfscommon.vfs/modules/opunk/str-0.1.0.tm

File diff suppressed because it is too large Load Diff

1150
src/vfs/_vfscommon.vfs/modules/overtype-1.7.4.tm

File diff suppressed because it is too large Load Diff

54
src/vfs/_vfscommon.vfs/modules/punk-0.1.1.tm

@ -2466,6 +2466,7 @@ namespace eval punk {
set splitchars "<splitchars>" set splitchars "<splitchars>"
set assigned [split $leveldata $splitchars] set assigned [split $leveldata $splitchars]
}] }]
puts "---split script: $script"
set level_script_complete 1 set level_script_complete 1
#todo %splitat- %splitn- ?? #todo %splitat- %splitn- ??
@ -4205,7 +4206,7 @@ namespace eval punk {
#avoid use of regexp match on each element - or we will unnecessarily force string reps on lists #avoid use of regexp match on each element - or we will unnecessarily force string reps on lists
#same with lsearch with a string pattern - #same with lsearch with a string pattern -
#wouldn't matter for small lists - but we need to be able to handle large ones efficiently without unneccessary string reps #wouldn't matter for small lists - but we need to be able to handle large ones efficiently without unneccessary string reps
set script [string map [list <scopep> $scopepattern <rhs> $equalsrhs] { set script [string map [list <scopep> [list $scopepattern] <rhs> $equalsrhs] {
#script built by punk::match_assign #script built by punk::match_assign
if {[llength $args]} { if {[llength $args]} {
#scan for existence of any pipe operator (|*> or <*|) only - we don't need position #scan for existence of any pipe operator (|*> or <*|) only - we don't need position
@ -4214,11 +4215,12 @@ namespace eval punk {
# x= <| # x= <|
# x= |> # x= |>
#both leave x empty. To assign a pipelike value to x we would have to do: x= <| |> (equiv: set x |>) #both leave x empty. To assign a pipelike value to x we would have to do: x= <| |> (equiv: set x |>)
set scopep <scopep>
foreach a $args { foreach a $args {
if {![catch {llength $a} sublen]} { if {![catch {llength $a} sublen]} {
#don't enforce sublen == 1. Legal to have whitespace including newlines {| x >} #don't enforce sublen == 1. Legal to have whitespace including newlines {| x >}
if {[string match |*> $a] || [string match <*| $a]} { if {[string match |*> $a] || [string match <*| $a]} {
tailcall punk::pipeline = "<scopep>" "<rhs>" {*}$args tailcall punk::pipeline = $scopep "<rhs>" {*}$args
} }
} }
} }
@ -4594,6 +4596,10 @@ namespace eval punk {
#debug.punk.pipe.rep {[rep_listname fulltail]} 6 #debug.punk.pipe.rep {[rep_listname fulltail]} 6
#review
set equalsrhs [string map [list {;} {\;}] $equalsrhs]
#--------------------------------------------------------------------- #---------------------------------------------------------------------
# test if we have an initial x.=y.= or x.= y.= # test if we have an initial x.=y.= or x.= y.=
@ -4643,26 +4649,31 @@ namespace eval punk {
#var1 will contain ETC (from entire pipeline), var2 will contain etc (from associated segment) #var1 will contain ETC (from entire pipeline), var2 will contain etc (from associated segment)
# #
if {([set nexteposn [string last = $next1]] >= 0) && (![punk::pipe::lib::arg_is_script_shaped $next1]) } {
set nexttail [lrange $args 1 end] if {([set nexteposn [string last = $next1]] >= 0)} {
#*SUB* pipeline recursion. set next1 [string map [list {;} {\;}] $next1] ;#review
#puts "======> recurse based on next1:$next1 " #do we really need to test for script_shaped if last char is = ?
if {[string index $next1 $nexteposn-1] eq {.}} { if {![punk::pipe::lib::arg_is_script_shaped $next1]} {
#var1.= var2.= ... set nexttail [lrange $args 1 end]
#non pipelined call to self - return result #*SUB* pipeline recursion.
#puts "======> recurse based on next1:$next1 "
if {[string index $next1 $nexteposn-1] eq {.}} {
#var1.= var2.= ...
#non pipelined call to self - return result
set results [uplevel 1 [list $next1 {*}$nexttail]]
#debug.punk.pipe.rep {==> rep recursive results: [rep $results]} 5
#debug.punk.pipe {>>> results: $results} 1
return [_handle_bind_result [_multi_bind_result $initial_returnvarspec $results]]
}
#puts "======> recurse assign based on next1:$next1 "
#if {[regexp {^([^ \t\r\n=\{]*)=(.*)} $next1 _ nextreturnvarspec nextrhs]} {
#}
#non pipelined call to plain = assignment - return result
set results [uplevel 1 [list $next1 {*}$nexttail]] set results [uplevel 1 [list $next1 {*}$nexttail]]
#debug.punk.pipe.rep {==> rep recursive results: [rep $results]} 5
#debug.punk.pipe {>>> results: $results} 1 #debug.punk.pipe {>>> results: $results} 1
return [_handle_bind_result [_multi_bind_result $initial_returnvarspec $results]] set d [_multi_bind_result $initial_returnvarspec $results]
return [_handle_bind_result $d]
} }
#puts "======> recurse assign based on next1:$next1 "
#if {[regexp {^([^ \t\r\n=\{]*)=(.*)} $next1 _ nextreturnvarspec nextrhs]} {
#}
#non pipelined call to plain = assignment - return result
set results [uplevel 1 [list $next1 {*}$nexttail]]
#debug.punk.pipe {>>> results: $results} 1
set d [_multi_bind_result $initial_returnvarspec $results]
return [_handle_bind_result $d]
} }
} }
@ -5981,6 +5992,9 @@ namespace eval punk {
tailcall {*}[list ::punk::pipeline = "" "" {*}$arglist] tailcall {*}[list ::punk::pipeline = "" "" {*}$arglist]
} }
#review
set assign [string map {; \\;} $assign]
set is_script [punk::pipe::lib::arg_is_script_shaped $assign] set is_script [punk::pipe::lib::arg_is_script_shaped $assign]
if {!$is_script && [string index $assign end] eq "="} { if {!$is_script && [string index $assign end] eq "="} {
@ -5999,7 +6013,7 @@ namespace eval punk {
if {$is_script} { if {$is_script} {
set cmdlist [list ::punk::pipeline "script" "" "" {*}$args] set cmdlist [list ::punk::pipeline "script" "" "" {*}$args]
} else { } else {
set cmdlist [list ::punk::pipeline ".=" "" "" {*}$args] set cmdlist [list ::punk::pipeline ".=" "" "" $assign {*}$arglist]
} }
} }
tailcall {*}$cmdlist tailcall {*}$cmdlist

1
src/vfs/_vfscommon.vfs/modules/punk/aliascore-0.1.0.tm

@ -123,6 +123,7 @@ tcl::namespace::eval punk::aliascore {
ansistrip ::punk::ansi::ansistrip ansistrip ::punk::ansi::ansistrip
stripansi ::punk::ansi::ansistrip stripansi ::punk::ansi::ansistrip
ansiwrap ::punk::ansi::ansiwrap ansiwrap ::punk::ansi::ansiwrap
ansisplit ::punk::ansi::ta::split_codes_single
grepstr ::punk::ansi::grepstr grepstr ::punk::ansi::grepstr
untabify ::punk::ansi::untabify untabify ::punk::ansi::untabify
colour ::punk::console::colour colour ::punk::console::colour

965
src/vfs/_vfscommon.vfs/modules/punk/ansi-0.1.1.tm

File diff suppressed because it is too large Load Diff

67
src/vfs/_vfscommon.vfs/modules/punk/ansi/sauce-0.1.0.tm

@ -218,7 +218,9 @@ tcl::namespace::eval punk::ansi::sauce {
#---------------------------------------------------------------------------------------------------------------------------------------------
# This data comes from the sauce spec.
#---------------------------------------------------------------------------------------------------------------------------------------------
#todo - fontName - which can also specify e.g code page 437 #todo - fontName - which can also specify e.g code page 437
## Font name [1] Font size Resolution [2] Aspect ratio [3] Vertical stretch [6] Description ## Font name [1] Font size Resolution [2] Aspect ratio [3] Vertical stretch [6] Description
## Display [4] Pixel [5] ## Display [4] Pixel [5]
@ -226,7 +228,14 @@ tcl::namespace::eval punk::ansi::sauce {
set fontnames [dict create] set fontnames [dict create]
## IBM VGA 9×16 [7] 720×400 4:3 20:27 (1:1.35) 35% Standard hardware font on VGA cards for 80×25 text mode (code page 437) ## IBM VGA 9×16 [7] 720×400 4:3 20:27 (1:1.35) 35% Standard hardware font on VGA cards for 80×25 text mode (code page 437)
dict set fontnames "IBM VGA" [list fontsize "9x16" resolution "720x400" aspect_ratio_display "4:3" aspect_ratio_pixel "20:27 (1:1.35)" vertical_stretch "35%" description "Standard hardware font on VGA cards for 80×25 text mode (code page 437)"] dict set fontnames "IBM VGA" [list {*}{
fontsize "9x16"
resolution "720x400"
aspect_ratio_display "4:3"
aspect_ratio_pixel "20:27 (1:1.35)"
vertical_stretch "35%"
description "Standard hardware font on VGA cards for 80×25 text mode (code page 437)"
}]
## IBM VGA ### [8] 9×16 [7] 720×400 4:3 20:27 (1:1.35) 35% Software installed code page font for VGA 80×25 text mode ## IBM VGA ### [8] 9×16 [7] 720×400 4:3 20:27 (1:1.35) 35% Software installed code page font for VGA 80×25 text mode
# - where ### is placeholder for 437,720,737 etc # - where ### is placeholder for 437,720,737 etc
@ -252,6 +261,7 @@ tcl::namespace::eval punk::ansi::sauce {
## C64 PETSCII unshifted 8×8 [10] 320×200 4:3 5:6 (1:1.2) 20% Original Commodore PETSCII font (PET, VIC-20, C64, CBM-II, Plus/4, C16, C116 and C128) in the unshifted mode. Unshifted mode (graphics) only has uppercase letters and additional graphic characters. This is the normal boot font. ## C64 PETSCII unshifted 8×8 [10] 320×200 4:3 5:6 (1:1.2) 20% Original Commodore PETSCII font (PET, VIC-20, C64, CBM-II, Plus/4, C16, C116 and C128) in the unshifted mode. Unshifted mode (graphics) only has uppercase letters and additional graphic characters. This is the normal boot font.
## C64 PETSCII shifted 8×8 [10] 320×200 4:3 5:6 (1:1.2) 20% Original PETSCII font in shifted mode. Shifted mode (text) has both uppercase and lowercase letters. This mode is actuated by pressing Shift+Commodore key. ## C64 PETSCII shifted 8×8 [10] 320×200 4:3 5:6 (1:1.2) 20% Original PETSCII font in shifted mode. Shifted mode (text) has both uppercase and lowercase letters. This mode is actuated by pressing Shift+Commodore key.
## Atari ATASCII 8×8 [11] 320×192 4:3 4:5 (1:1.25) 25% Original ATASCII font (Atari 400, 800, XL, XE) ## Atari ATASCII 8×8 [11] 320×192 4:3 4:5 (1:1.25) 25% Original ATASCII font (Atari 400, 800, XL, XE)
#---------------------------------------------------------------------------------------------------------------------------------------------
#expect a 128 Byte sauce record #expect a 128 Byte sauce record
@ -261,6 +271,7 @@ tcl::namespace::eval punk::ansi::sauce {
variable datatypes variable datatypes
variable filetypes variable filetypes
variable encodings variable encodings
set warnings [list]
if {[string length $saucerecord] != 128} { if {[string length $saucerecord] != 128} {
error "punk::ansi::sauce::to_dict: Unable to interpret data as a SAUCE record - length != 128" error "punk::ansi::sauce::to_dict: Unable to interpret data as a SAUCE record - length != 128"
} }
@ -326,6 +337,8 @@ tcl::namespace::eval punk::ansi::sauce {
dict set sdict filetype_name "" dict set sdict filetype_name ""
} }
} else { } else {
#how can a byte fail to scan with cu? is this even reachable?
puts stderr "punk::ansi::sauce::to_dict filetype byte failed to scan - setting filetype and filetype_name to empty string byte: [ansistring VIEW -lf 1 [string range $saucerecord 95 95]]"
dict set sdict filetype "" dict set sdict filetype ""
dict set sdict filetype_name "" dict set sdict filetype_name ""
} }
@ -422,25 +435,40 @@ tcl::namespace::eval punk::ansi::sauce {
5 { 5 {
#binarytext #binarytext
#filetype is supposed to represent half the characterwidth (only widths with multiples of 2 can be specified) #filetype is supposed to represent half the characterwidth (only widths with multiples of 2 can be specified)
#HOWEVER - in the wild we may see width/height specified in tinfo1/tinfo2 with some other value in filetype (eg 1) #HOWEVER - in the wild we may see width/height specified in tinfo1/tinfo2 with some apparently unrelated value in filetype (eg 0 or 1) that doesn't match the intended image dimensions.
#If both tinfo1 and tinfo2 are non zero - we will use them, even though it's not in spec. #If both tinfo1 and tinfo2 are non zero - we *could* use them, even though it's not in spec.
set t1 [dict get $sdict tinfo1] #An example file (us-used1.bin) has filetype 0 and tinfo1/tinfo2 640/350
if {$t1 eq ""} { #It's possible tinfo1/tinfo2 represent pixel dimensions for a 'standard' 8x16 font, but this image is 160 columns wide, so we would expect tinfo1 to be 1280.
set t1 0 #The sauce spec seems to indicate we should ignore tinfo1/tinfo2 for binarytext and only use filetype to determine width.
} #the default for binarytext is 160 columns.
set t2 [dict get $sdict tinfo2]
if {$t2 eq ""} { #filetype 1 is theoretically possible, representing 2 columns
set t2 0 #in practice we see this value for binarytext images that are definitely not intended to be 2 columns wide. Why?
#is there some assumption that that images are at least a certain width, and filetype has been repurposed to indicate something else?
#The spec would seem to rule out images of a single column due to filetype being half the character width but a value of 0.5 isn't supported.
#It specifically mentions that only even widths up to 510 can be specified. ($filetype * 2 where filetype is 1-255?)
#proper mechanism to specify columns for binarytext is the datatype field.
set cols [expr {2*[dict get $sdict filetype]}]
if {$cols == 0} {
lappend warnings "binarytext filetype value of [dict get $sdict filetype] - using binarytext default cols of 160"
#default for binarytext is 160 columns
set cols 160
} }
if {$t1 != 0 && $t2 != 0} { if {$cols == 2 && [dict get $sdict tinfo1] != 0 && [dict get $sdict tinfo2] != 0} {
#not to spec - but we will assume these have values for a reason.. #not to spec - but we will assume these have values for a reason..
puts stderr "punk::ansi::sauce::to_dict using tinfo1/tinfo2 data for columns/rows (non-compliant SAUCE data)" #---------------------------------------------------------------------------------------------------------------------
dict set sdict columns [expr {2 * $t1}] #The sample file src/testansi/formatsamples/image/binaryText/test.bin has a filetype 1 and tinfo1 40 and tinfo2 25.
dict set sdict rows $t2 #(similarly ppe-ansi.bin has tinfo1 80 and tinfo2 26)
#They seem to use the 1 in filetype to indicate that the tinfo1/tinfo2 values should be used.
#(The 80 cols wide test.bin binaryText image matches the xbin sample file src/testansi/formatsamples/image/xbin/test.xb which is a more fully specified format using a header)
#---------------------------------------------------------------------------------------------------------------------
lappend warnings "binarytext filetype of 1 with non-zero tinfo1/tinfo2 - using tinfo1/tinfo2 data for columns/rows (possibly non-conforming SAUCE data - matching observed data in the wild)"
set cols [expr {2 * [dict get $sdict tinfo1]}]
dict set sdict columns $cols
dict set sdict rows [dict get $sdict tinfo2]
} else { } else {
#proper mechanism to specify columns for binarytext is the datatype field.
set cols [expr {2*[dict get $sdict filetype]}]
dict set sdict columns $cols dict set sdict columns $cols
#rows must be calculated from file size #rows must be calculated from file size
#rows = (filesize - sauceinfosize)/ filetype * 2 * 2 #rows = (filesize - sauceinfosize)/ filetype * 2 * 2
@ -481,6 +509,9 @@ tcl::namespace::eval punk::ansi::sauce {
} }
} }
} }
if {[llength $warnings]} {
dict set sdict warnings $warnings
}
return $sdict return $sdict
} }

205
src/vfs/_vfscommon.vfs/modules/punk/char-0.1.0.tm

@ -3039,8 +3039,10 @@ tcl::namespace::eval punk::char {
set csplits [combiner_split $text] set csplits [combiner_split $text]
foreach {pt combiners} [lrange $csplits 0 end-1] { foreach {pt combiners} [lrange $csplits 0 end-1] {
set clist [split $pt ""] set clist [split $pt ""]
lappend components {*}[lrange $clist 0 end-1] lset clist end [tcl::string::cat [lindex $clist end] $combiners]
lappend components [tcl::string::cat [lindex $clist end] $combiners] lappend components {*}$clist
#lappend components {*}[lrange $clist 0 end-1]
#lappend components [tcl::string::cat [lindex $clist end] $combiners]
} }
#last csplit never has a combiner (_perlish_split style) - and may be empty - in which case we don't append it as a grapheme #last csplit never has a combiner (_perlish_split style) - and may be empty - in which case we don't append it as a grapheme
if {[lindex $csplits end] ne ""} { if {[lindex $csplits end] ne ""} {
@ -3066,126 +3068,121 @@ tcl::namespace::eval punk::char {
#review \uFE0F variation selector 16 - forces emoji presentation for preceding char #review \uFE0F variation selector 16 - forces emoji presentation for preceding char
if 1 { #This is a basic implementation that does not check that all combinations are valid.
#This is a basic implementation that does not check that all combinations are valid. set graphemes [list]
set graphemes [list] set current_cluster ""
set current_cluster ""
set cluster_base 0 ;#is the current cluster based on a char that can be combined with modifiers/ZWJs (e.g emoji or other cluster-based char)
set cluster_base 0 ;#is the current cluster based on a char that can be combined with modifiers/ZWJs (e.g emoji or other cluster-based char) # or is it based on a char that can't be combined with modifiers/ZWJs (e.g ascii letter)
# or is it based on a char that can't be combined with modifiers/ZWJs (e.g ascii letter) set cluster_base_RI 0 ;#is the current cluster based on a regional indicator char - which can only combine in pairs, and don't combine with modifiers or ZWJs to form longer clusters - so not extensible.
set cluster_base_RI 0 ;#is the current cluster based on a regional indicator char - which can only combine in pairs, and don't combine with modifiers or ZWJs to form longer clusters - so not extensible.
set current_cluster_is_extensible 0
set current_cluster_is_extensible 0 for {set i 0} {$i < [llength $components] } {incr i} {
for {set i 0} {$i < [llength $components] } {incr i} { set component [lindex $components $i]
set component [lindex $components $i] if {$component eq "\r" && [lindex $components $i+1] eq "\n"} {
if {$component eq "\r" && [lindex $components $i+1] eq "\n"} { if {$current_cluster ne ""} {
if {$current_cluster ne ""} { lappend graphemes $current_cluster
lappend graphemes $current_cluster }
} lappend graphemes "\r\n"
lappend graphemes "\r\n" incr i ;#skip the \n as we've already processed it as part of the cluster
incr i ;#skip the \n as we've already processed it as part of the cluster set current_cluster ""
set current_cluster "" set cluster_base 0; set cluster_base_RI 0 ;#grapheme_split::reset_base
grapheme_split::reset_base set current_cluster_is_extensible 0
} elseif {$component eq "\u200d"} {
if {$current_cluster eq ""} {
#ZWJ at start of string - treat as separate grapheme cluster - but isn't a valid base for further combining with more ZWJs or modifiers
set current_cluster $component
set cluster_base 0; set cluster_base_RI 0 ;#grapheme_split::reset_base
set current_cluster_is_extensible 0 set current_cluster_is_extensible 0
} elseif {$component eq "\u200d"} { } else {
if {$current_cluster eq ""} { if {$cluster_base} {
#ZWJ at start of string - treat as separate grapheme cluster - but isn't a valid base for further combining with more ZWJs or modifiers if {$current_cluster_is_extensible} {
set current_cluster $component #a double (or longer) ZWJ sequence in a row is part of the last cluster - but not extensible anymore.
grapheme_split::reset_base append current_cluster $component
set current_cluster_is_extensible 0 set current_is_cluster_extensible 0
} else {
if {$cluster_base} {
if {$current_cluster_is_extensible} {
#a double (or longer) ZWJ sequence in a row is part of the last cluster - but not extensible anymore.
append current_cluster $component
set current_is_cluster_extensible 0
} else {
append current_cluster $component
if {$cluster_base_RI} {
#regional indicators can only combine in pairs, and don't combine with modifiers or ZWJs to form longer clusters - so not extensible.
grapheme_split::reset_base
set current_cluster_is_extensible 0
#we can keep adding ZWJs or modifiers though
} else {
set current_cluster_is_extensible 1
}
}
} else { } else {
#ZWJ after non-cluster-based char - non extensible but we continue appending ZWJs to the current cluster.
append current_cluster $component append current_cluster $component
set current_cluster_is_extensible 0 if {$cluster_base_RI} {
} #regional indicators can only combine in pairs, and don't combine with modifiers or ZWJs to form longer clusters - so not extensible.
set cluster_base 0; set cluster_base_RI 0 ;#grapheme_split::reset_base
} set current_cluster_is_extensible 0
} elseif {[regexp {[\U1f3fb-\U1f3ff]} $component]} { #we can keep adding ZWJs or modifiers though
#emoji modifier - join with previous component
if {$current_cluster eq ""} {
#modifier at start of string - not a valid base for further combining with more modifiers or ZWJs - but we continue appending modifiers to the current cluster.
set current_cluster $component
grapheme_split::reset_base
} else {
if {$cluster_base} {
if {$current_cluster_is_extensible} {
append current_cluster $component
#invalidate the base!
grapheme_split::reset_base
} else { } else {
append current_cluster $component set current_cluster_is_extensible 1
} }
}
} else {
#ZWJ after non-cluster-based char - non extensible but we continue appending ZWJs to the current cluster.
append current_cluster $component
set current_cluster_is_extensible 0
}
}
} elseif {[regexp {[\U1f3fb-\U1f3ff]} $component]} {
#emoji modifier - join with previous component
if {$current_cluster eq ""} {
#modifier at start of string - not a valid base for further combining with more modifiers or ZWJs - but we continue appending modifiers to the current cluster.
set current_cluster $component
set cluster_base 0; set cluster_base_RI 0 ;#grapheme_split::reset_base
} else {
if {$cluster_base} {
if {$current_cluster_is_extensible} {
append current_cluster $component
#invalidate the base!
set cluster_base 0; set cluster_base_RI 0 ;#grapheme_split::reset_base
} else { } else {
#modifier after non-cluster-based char - non extensible but we continue appending modifiers to the current cluster.
append current_cluster $component append current_cluster $component
} }
#review } else {
# \u1f33e\u1f3fe\u200d\u2f3fe\u200d\u1f33e is 2 clusters #modifier after non-cluster-based char - non extensible but we continue appending modifiers to the current cluster.
#This is because after first zwj, we applied a modifier - not a valid base. append current_cluster $component
} }
set current_cluster_is_extensible 0 #review
# \u1f33e\u1f3fe\u200d\u2f3fe\u200d\u1f33e is 2 clusters
#This is because after first zwj, we applied a modifier - not a valid base.
}
set current_cluster_is_extensible 0
} else {
if {$current_cluster eq ""} {
grapheme_split::start_cluster $component
} else { } else {
if {$current_cluster eq ""} { #have existing cluster data
grapheme_split::start_cluster $component if {$current_cluster_is_extensible} {
} else { #assert - if current_cluster_is_extensible then cluster_base should currently be true.
#have existing cluster data #if the current char is a base - we can append to existing cluster, but if it's not a base, then we start a new cluster even if we had seen a ZWJ before.
if {$current_cluster_is_extensible} { if {[regexp {[\U1f600-\U1f64f\U1f300-\U1f5ff\U1f900-\U1f9ff\U1fa70-\U1faff\U1f680-\U1f6ff\U2700-\U27bf\U2600-\u26ff]} $component]} {
#assert - if current_cluster_is_extensible then cluster_base should currently be true. append current_cluster $component
#if the current char is a base - we can append to existing cluster, but if it's not a base, then we start a new cluster even if we had seen a ZWJ before. set cluster_base 1
if {[regexp {[\U1f600-\U1f64f\U1f300-\U1f5ff\U1f900-\U1f9ff\U1fa70-\U1faff\U1f680-\U1f6ff\U2700-\U27bf\U2600-\u26ff]} $component]} {
append current_cluster $component
set cluster_base 1
} else {
lappend graphemes $current_cluster
set current_cluster $component
grapheme_split::reset_base
}
set current_cluster_is_extensible 0
} elseif {$cluster_base_RI} {
#regional indicators can only combine in pairs, and don't combine with modifiers or ZWJs to form longer clusters - so not extensible.
if {[regexp {[\U1f1e6-\U1f1ff]} $component]} {
append current_cluster $component
#invalidate the base - we can't combine more than 2 RIs in a cluster, and they don't combine with modifiers or ZWJs to form longer clusters.
#we can however add more ZWJs or modifiers to the cluster - but they don't make it extensible for combining with more RIs
grapheme_split::reset_base
} else {
#something else while RI cluster is open - end the current cluster and start a new one with the current char.
lappend graphemes $current_cluster
grapheme_split::start_cluster $component
}
set current_cluster_is_extensible 0
} else { } else {
lappend graphemes $current_cluster
set current_cluster $component
set cluster_base 0; set cluster_base_RI 0 ;#grapheme_split::reset_base
}
set current_cluster_is_extensible 0
} elseif {$cluster_base_RI} {
#regional indicators can only combine in pairs, and don't combine with modifiers or ZWJs to form longer clusters - so not extensible.
if {[regexp {[\U1f1e6-\U1f1ff]} $component]} {
append current_cluster $component
#invalidate the base - we can't combine more than 2 RIs in a cluster, and they don't combine with modifiers or ZWJs to form longer clusters.
#we can however add more ZWJs or modifiers to the cluster - but they don't make it extensible for combining with more RIs
set cluster_base 0; set cluster_base_RI 0 ;#grapheme_split::reset_base
} else {
#something else while RI cluster is open - end the current cluster and start a new one with the current char.
lappend graphemes $current_cluster lappend graphemes $current_cluster
grapheme_split::start_cluster $component grapheme_split::start_cluster $component
} }
set current_cluster_is_extensible 0
} else {
lappend graphemes $current_cluster
grapheme_split::start_cluster $component
} }
} }
} }
if {$current_cluster ne ""} {
lappend graphemes $current_cluster
}
} else {
set graphemes $components
} }
if {$current_cluster ne ""} {
lappend graphemes $current_cluster
}
return $graphemes return $graphemes
} }
namespace eval grapheme_split { namespace eval grapheme_split {

3
src/vfs/_vfscommon.vfs/modules/punk/lib-0.1.6.tm

@ -4210,6 +4210,9 @@ namespace eval punk::lib {
if {[string index $key 0] ne "%"} { if {[string index $key 0] ne "%"} {
set key %$key set key %$key
} }
#puts "---key:'$key'"
set key [string map {; \\;} $key] ;#review
#puts "---key:'$key'"
#pipeline - use punk patterns. #pipeline - use punk patterns.
% thisval.= $key= $thisval % thisval.= $key= $thisval
} }

9
src/vfs/_vfscommon.vfs/modules/punk/mix/commandset/loadedlib-0.1.0.tm

@ -54,13 +54,18 @@ namespace eval punk::mix::commandset::loadedlib {
if {$opt_refresh} { if {$opt_refresh} {
catch {package require frobznodule666} ;#ensure pkg system has loaded/searched for everything REVIEW - this doesn't result in full scans catch {package require frobznodule666} ;#ensure pkg system has loaded/searched for everything REVIEW - this doesn't result in full scans
foreach tm_path [tcl::tm::list] { foreach tm_path [tcl::tm::list] {
#review - todo - adjust punk::path::subfolders to take arguments to do some filtering itself rather than recurse down unnecessary branches.
set paths_below [punk::path::subfolders -recursive $tm_path] set paths_below [punk::path::subfolders -recursive $tm_path]
foreach folder $paths_below { foreach folder $paths_below {
if {[string match */_build/* $folder]} {continue}
set tail [file tail $folder] set tail [file tail $folder]
if {[string match #modpod-* $tail] || [string match #tarjar-* $tail]} { if {[string match #tarjar-* $tail]} {
continue
}
if {[string match #modpod-* $tail]} {
#manually do a 'package ifneeded' fore each module found here.
continue continue
} }
if {[string match */_build/* $folder]} {continue}
set relpath [string tolower [punk::path::relative $tm_path $folder]] set relpath [string tolower [punk::path::relative $tm_path $folder]]
set modpath [string map {/ ::} $relpath] set modpath [string map {/ ::} $relpath]
catch {package require ${modpath}::flobrudder99} catch {package require ${modpath}::flobrudder99}

22
src/vfs/_vfscommon.vfs/modules/punk/ns-0.1.0.tm

@ -775,13 +775,8 @@ tcl::namespace::eval punk::ns {
#set parent [nsprefix $ns_absolute] #set parent [nsprefix $ns_absolute]
#set tail [nstail $ns_absolute] #set tail [nstail $ns_absolute]
#jjj
#set allchildren [lsort [nseval $base [list ::namespace children]]] #set allchildren [lsort [nseval $base [list ::namespace children]]]
#set allchildren [lsort [tcl::namespace::eval $base [list ::namespace children]]]
set allchildren [lsort [nseval $base [list ::namespace children]]]
#puts "->base:$base tailparts:$tailparts allchildren: $allchildren"
#puts "->base:$base tailparts:$tailparts childcount: [llength $allchildren]"
#** only significant when it is the trailing part of a segment eg ::**::xxx ::a**::xxx #** only significant when it is the trailing part of a segment eg ::**::xxx ::a**::xxx
if {[llength $tailparts]} { if {[llength $tailparts]} {
@ -790,6 +785,7 @@ tcl::namespace::eval punk::ns {
set nslist [nstree_list $base -subnslist {} -allbelow 1] set nslist [nstree_list $base -subnslist {} -allbelow 1]
} elseif {[regexp {[*]{2}$} $nextglob]} { } elseif {[regexp {[*]{2}$} $nextglob]} {
set nslist [list] set nslist [list]
set allchildren [lsort [nseval $base [list ::namespace children]]]
lappend nsmatches {*}[lsearch -all -inline -glob $allchildren [nsjoin ${base} $nextglob]] lappend nsmatches {*}[lsearch -all -inline -glob $allchildren [nsjoin ${base} $nextglob]]
foreach ch $nsmatches { foreach ch $nsmatches {
lappend nslist $ch lappend nslist $ch
@ -799,6 +795,7 @@ tcl::namespace::eval punk::ns {
} else { } else {
#lsearch with -glob ok even if nextglob has no globchars (no discernable speed diff, and earlier parts may have globchars anyway) #lsearch with -glob ok even if nextglob has no globchars (no discernable speed diff, and earlier parts may have globchars anyway)
set nslist [list] set nslist [list]
set allchildren [lsort [nseval $base [list ::namespace children]]]
lappend nsmatches {*}[lsearch -all -inline -glob $allchildren [nsjoin ${base} $nextglob]] lappend nsmatches {*}[lsearch -all -inline -glob $allchildren [nsjoin ${base} $nextglob]]
if {[llength $tailparts] >1 || $allbelow} { if {[llength $tailparts] >1 || $allbelow} {
foreach ch $nsmatches { foreach ch $nsmatches {
@ -812,6 +809,7 @@ tcl::namespace::eval punk::ns {
} }
} else { } else {
#puts "nstree_list: no tailparts base:$base" #puts "nstree_list: no tailparts base:$base"
set allchildren [lsort [nseval $base [list ::namespace children]]]
if {$allbelow} { if {$allbelow} {
set nsmatches $allchildren set nsmatches $allchildren
set nslist [list] set nslist [list]
@ -2134,8 +2132,8 @@ y" {return quirkykeyscript}
tcl::dict::set tinfo($target) procoffset 0 tcl::dict::set tinfo($target) procoffset 0
tcl::dict::set tinfo($target) level [expr {[::tcl::info::level]+1}] tcl::dict::set tinfo($target) level [expr {[::tcl::info::level]+1}]
tcl::dict::set tinfo($target) subcmds 0 tcl::dict::set tinfo($target) subcmds 0
puts "enter: $target -- $args" puts stderr "enter: $target -- $args"
puts "frame-2: [::tcl::info::frame -2]" #puts stderr "frame-2: [::tcl::info::frame -2]"
set _cmdtrace_disabled false set _cmdtrace_disabled false
} }
@ -2481,7 +2479,7 @@ y" {return quirkykeyscript}
set line $traceline set line $traceline
dict set linedict $target eval_base $traceline dict set linedict $target eval_base $traceline
dict set linedict $target eval_offset 1 dict set linedict $target eval_offset 1
puts " step type: proc traceline:$traceline ** $args" puts " step type: proc traceline:$traceline ** $args\x1b\[m"
#puts "** $callinfo" #puts "** $callinfo"
if {[dict exists $callinfo cmd]} { if {[dict exists $callinfo cmd]} {
#set cmd [string trim [dict get $callinfo cmd]] ;#raw 'unexpanded' script from the stack frame #set cmd [string trim [dict get $callinfo cmd]] ;#raw 'unexpanded' script from the stack frame
@ -2504,8 +2502,8 @@ y" {return quirkykeyscript}
set eval_base [dict get $linedict $target eval_base] set eval_base [dict get $linedict $target eval_base]
set eval_offset [dict get $linedict $target eval_offset] set eval_offset [dict get $linedict $target eval_offset]
set line [expr {$eval_base + ($eval_offset-1) + ($traceline-1)}] set line [expr {$eval_base + ($eval_offset-1) + ($traceline-1)}]
puts "stack-- $callinfo" #puts "stack-- $callinfo"
puts " step type: eval traceline: $traceline -- " puts stderr " step type: eval traceline: $traceline -- "
if {[dict exists $callinfo cmd]} { if {[dict exists $callinfo cmd]} {
#set cmd [string trim [dict get $callinfo cmd]] #set cmd [string trim [dict get $callinfo cmd]]
set cmdlist [lindex $args 0] set cmdlist [lindex $args 0]
@ -2627,6 +2625,8 @@ y" {return quirkykeyscript}
}] }]
} }
proc cmdtrace {args} { proc cmdtrace {args} {
#review - displaying argument values has to be done carefully. Small values are ok, but large lists or dicts can be overwhelming.
#Potentially we could apply some heuristics to truncate or summarise them.
package require dictn ;#convenience to allow dictn::incr d {key subkey} package require dictn ;#convenience to allow dictn::incr d {key subkey}
variable tinfo variable tinfo
array unset tinfo array unset tinfo
@ -2676,7 +2676,7 @@ y" {return quirkykeyscript}
#if the target command has a leading colon (e.g expr alternative :) we can't put a trace directly on a fully qualified name with a triple colon such as ::: #if the target command has a leading colon (e.g expr alternative :) we can't put a trace directly on a fully qualified name with a triple colon such as :::
#we will need to evaluate in the namespace #we will need to evaluate in the namespace
foreach {tgt_cmd ns nscmd} $resolved_targets { foreach {tgt_cmd ns nscmd} $resolved_targets {
puts "tracing target: $tgt_cmd whilst running: $origin $arglist" puts stderr "tracing target: $tgt_cmd whilst running: $origin $arglist"
#::tcl::namespace::eval $ns [list ::trace add execution $nscmd enter [list ::punk::ns::_cmdtrace_enter ::punk::ns::linedict $tgt_cmd]] #::tcl::namespace::eval $ns [list ::trace add execution $nscmd enter [list ::punk::ns::_cmdtrace_enter ::punk::ns::linedict $tgt_cmd]]
#::tcl::namespace::eval $ns [list ::trace add execution $nscmd enterstep [list ::punk::ns::_cmdtrace_enterstep ::punk::ns::linedict $tgt_cmd]] #::tcl::namespace::eval $ns [list ::trace add execution $nscmd enterstep [list ::punk::ns::_cmdtrace_enterstep ::punk::ns::linedict $tgt_cmd]]

290
src/vfs/_vfscommon.vfs/modules/punk/path-0.1.0.tm

@ -565,10 +565,45 @@ namespace eval punk::path {
end]] end]]
} }
## for comparison
#proc nsglob_as_re {glob} {
# #any segment that is not just * must match exactly one segment in the path
# set pats [list]
# foreach seg [nsparts_cached $glob] {
# switch -exact -- $seg {
# "" {
# lappend pats ""
# }
# * {
# #review - ::g*t will not find ::got:it (won't match single inner colon) - this should be fixed
# #lappend pats {[^:]*}
# #negative lookahead
# #any number of chars not followed by ::, followed by any number of non :
# lappend pats {(?:.(?!::))*[^:]*}
# }
# ** {
# lappend pats {.*}
# }
# default {
# set seg [string map {. [.]} $seg]
# if {[regexp {[*?]} $seg]} {
# #set pat [string map [list ** {.*} * {[^:]*} ? {[^:]}] $seg]
# set pat [string map [list ** {.*} * {(?:.(?!::))*[^:]*} ? {[^:]}] $seg]
# lappend pats "$pat"
# } else {
# lappend pats "$seg"
# }
# }
# }
# }
# return "^[join $pats ::]\$"
#}
proc pathglob_as_re {pathglob} { proc pathglob_as_re {pathglob} {
#*** !doctools #*** !doctools
#[call [fun pathglob_as_re] [arg pathglob]] #[call [fun pathglob_as_re] [arg pathglob]]
#[para] Returns a regular expression for matching a path to a glob pattern which can contain glob chars *|? in any segment of the path structure #[para] Returns a regular expression for matching a path to a glob pattern which can contain glob chars *|? in any segment of the path structure
#[para] Does not support square bracket globs or character classes.
#[para] ** matches any number of subdirectories. #[para] ** matches any number of subdirectories.
#[para] e.g /etc/**/*.txt will match any .txt files at any depth below /etc (except directly within /etc itself) #[para] e.g /etc/**/*.txt will match any .txt files at any depth below /etc (except directly within /etc itself)
#[para] e.g /etc/**.txt will match any .txt files at any depth below /etc #[para] e.g /etc/**.txt will match any .txt files at any depth below /etc
@ -589,7 +624,7 @@ namespace eval punk::path {
* {lappend pats {[^/]*}} * {lappend pats {[^/]*}}
** {lappend pats {.*}} ** {lappend pats {.*}}
default { default {
set seg [string map [list {^ {\^} $ {\$} [} {\[} ( {\(} \{ \\\{ \\ {\\}] $seg] ;#treat regex characters in the input as literals set seg [string map [list ^ {\^} $ {\$} \[ {\[} \] {\]} ( {\(} ) {\)} \{ \\\{ \\ {\\}] $seg] ;#treat regex characters (or tcl glob square bracket chars) in the input as literals
#set seg [string map [list . {[.]}] $seg] #set seg [string map [list . {[.]}] $seg]
set seg [string map {. [.]} $seg] set seg [string map {. [.]} $seg]
if {[regexp {[*?]} $seg]} { if {[regexp {[*?]} $seg]} {
@ -603,6 +638,52 @@ namespace eval punk::path {
} }
return "^[join $pats /]\$" return "^[join $pats /]\$"
} }
punk::args::define {
@id -id ::punk::path::globmatchpath
@cmd -name punk::path::globmatchpath\
-summary\
"Match path to *|**|? glob patterns"\
-help\
"Return a boolean indicating whether the path matches the specialised glob pattern.
A pattern such as /usr/*/bin will match any path that has /usr as the first segment and bin as the third segment,
with any single segment in between.
A pattern such as /usr/**/bin will match any path that has /usr as the first segment and bin as the last segment,
with 1 or more segments in between (so it will not match /usr/bin).
A pattern such as /usr/** will match any path that has /usr as the first segment, with 1 or more segments
following (so it will not match /usr itself).
A pattern such as **/*.txt will match any path that ends with .txt, with 1 or more leading segments
(so it will not match test.txt or .txt).
A pattern such as ** will match any path.
The glob characters * and ? are the only special characters in the pathglob syntax.
- they are treated as glob characters regardless of where they appear in the pathglob string.
Note that this is different from other Tcl glob contexts where square brackets can be used.
The pathglob syntax treats other characters, including square brackets as literals.
For example, the pattern /usr/te?t will match /usr/test and /usr/text but not /usr/texxt, and the pattern /usr/te*t
will match /usr/test, /usr/teat, and /usr/teeeet but not /usr/te/t.
The pathglob syntax does not support escaping of glob characters - any glob characters in the pathglob are treated
as glob characters. For example, the pattern /usr/* will match any path that has /usr as the first segment and any
single segment as the second segment, but there is no way to specify a pattern that matches any path that has /usr
as the first segment and a literal * as the second segment.
Caller must ensure that file separator is forward slash. (e.g use file normalize on windows)
options:
-nocase 0|1 (default 0 - case sensitive)
If -nocase is not supplied - default to case sensitive *except for driveletter*
ie - the driveletter alone in paths such as c:/etc will still be case insensitive. (ie c:/ETC/* will match C:/ETC/blah but not C:/etc/blah)
Explicitly specifying -nocase 0 will require the entire case to match including the driveletter.
"
@leaders
pathglob -type string -help "glob pattern to match path against. See [fun pathglob_as_re] for syntax of glob patterns"
path -type string -help "path to match against glob pattern"
@opts
-nocase -type boolean -default 0 -help\
"case insensitive matching (default false - case sensitive)
- except for driveletter on windows which is always case insensitive
unless -nocase 0 is explicitly specified"
@values -min 0 -max 0
}
# -id
proc globmatchpath {pathglob path args} { proc globmatchpath {pathglob path args} {
#*** !doctools #*** !doctools
#[call [fun globmatchpath] [arg pathglob] [arg path] [opt {option value...}]] #[call [fun globmatchpath] [arg pathglob] [arg path] [opt {option value...}]]
@ -669,46 +750,182 @@ namespace eval punk::path {
@opts @opts
-recursive -type none -help\ -recursive -type none -help\
"" ""
-exclude-paths -type list -default {} -help\
"list of path patterns to exclude from results.
May include * and ** path segments e.g /usr/**
A single /*/ will match any single segment in the path, and a single /**/ will match any number of segments in the path.
e.g to exclude any path with _aside as a segment in the middle: -exclude-paths **/_aside/**
i.e this would exclude /usr/_aside/etc and /usr/x/_aside/etc but not /usr/x/_aside or _aside/etc
To exclude all paths with _aside as a segment anywhere: -exclude-paths { **/_aside/** **/_aside _aside/**}
"
#todo -depth #todo -depth
@values -min 0 -max 1 @values -min 0 -max 1
path -type directory -optional 1 -help\ path -type directory -optional 1 -help\
"Path of folder. If not supplied current directory is used." "Path of folder. If not supplied current directory is used.
This may be a relative or absolute path. Relative paths are treated as relative to current directory.
When using relative paths - the result will also be relative paths with the same relative prefix.
(e.g if path is ../test - the results will be ../test/subfolder1 ../test/subfolder2 etc)
Patterns in -exclude-paths are matched against the resulting paths
(so should be written to match the same relative prefix if path is relative)"
} }
proc subfolders {args} { proc subfolders {args} {
#NOTE - this algorithm based on omit_only_patterns and prune_base_patterns was suggested by a 2026 AI model - it is apparent to this programmer that it is inadequate for the purpose.
#e.g consider subfolders -recursion -exclude {**/vfs/** **/src/**}
#This can still return something like c:/repo/etc/src/vfs - which should be excluded by the pattern **/src/**
#todo - review and fix properly.
set argd [punk::args::parse $args withid ::punk::path::subfolders] set argd [punk::args::parse $args withid ::punk::path::subfolders]
lassign [dict values $argd] leaders opts values received lassign [dict values $argd] leaders opts values received
set do_recursion [dict exists $received -recursive] set do_recursion [dict exists $received -recursive]
set exclude_paths [dict get $opts -exclude-paths]
if {"**" in $exclude_paths} {
#if ** is in exclude_paths - then we can skip all glob matching and just return empty list
#This is likely user error - so we'll be loud about it for now but will still return empty list rather than erroring.
#If user code is building exclude_paths dynamically - they can check for this case themselves and avoid the call to subfolders to suppress this message.
puts stderr "punk::path::subfolders Warning - exclude_paths contains '**' - all paths will be excluded"
return [list]
}
if {[dict exists $received path]} { if {[dict exists $received path]} {
set path [dict get $values path] set path [dict get $values path]
} else { } else {
set path [pwd] set path [pwd]
} }
set folders [glob -nocomplain -directory $path -types d *]
set all_subfolders [glob -nocomplain -directory $path -types d *]
#example of expected exclude_paths pattern behaviour when recursion is enabled:
# **/dirname -> omit /x/y/dirname, but still visit /x/y/dirname/*
# **/dirname/* -> include /x/y/dirname and /x/y/dirname/a/b but omit directories that are a single level below /x/y/dirname such as /x/y/dirname/a
#c:/** - would exclude all subfolders below c: but not c: itself
# **/test/** - would exclude any path with test as a segment and all its subfolders
#- but not paths with test as a segment that is the final segment
set omit_only_patterns [list]
set prune_base_patterns [list]
foreach pat $exclude_paths {
set pat_parts [file split $pat] ;#note file split c:/test gives {c:/ test} but file split **/test gives {** test}
#also note that file split on windows treats forward slashes and backslashes the same.
#by using file split, we gain some flexibility in syntax of paths and patterns,
#but lose the ability to use backslashes as escapes to allow literal glob characters in path segments.
#This is almost always a non-issue on windows since * and ? are not valid in path segments there, and is rarely an issue on unix even though
# * and ? are technically valid in path segments, but it is inadvisable there anyway for compatibility with shells etc.
if {[llength $pat_parts] >= 2 && [lindex $pat_parts end] eq "**"} {
#** at end of pattern - e.g /dir/etc/**
#Convert ".../" to base "...", and prune descendants of that base.
lappend prune_base_patterns [file join {*}[lrange $pat_parts 0 end-1]]
} else {
lappend omit_only_patterns $pat
}
}
set folders [list]
set recurse_subdirs [list]
foreach f $all_subfolders {
set include_in_results 1
set allow_recurse 1
foreach pat $omit_only_patterns {
if {[globmatchpath $pat $f]} {
set include_in_results 0
break
}
}
if {$allow_recurse && [llength $prune_base_patterns]} {
foreach base_pat $prune_base_patterns {
#prune both the matched base node and its decendants.
if {[globmatchpath $base_pat $f]} {
set allow_recurse 0
break
}
if {[globmatchpath "${base_pat}/**" $f]} {
set include_in_results 0
set allow_recurse 0
break
}
}
}
if {$include_in_results} {
lappend folders $f
}
if {$allow_recurse} {
lappend recurse_subdirs $f
}
}
if {$do_recursion} { if {$do_recursion} {
foreach subdir $folders { foreach subdir $recurse_subdirs {
lappend folders {*}[subfolders -recursive $subdir] lappend folders {*}[subfolders -exclude-paths $exclude_paths -recursive $subdir]
} }
} }
#if {[llength $exclude_paths]} {
# set folders [list]
# foreach f $all_subfolders {
# set skip 0
# foreach pat $exclude_paths {
# #review - this is slightly too simplistic.
# # for exclusion pattern **/dirname
# # this will exclude any path with dirname as final segment - but it will also exclude any path with dirname as a segment anywhere in the path - which is not intended.
# #puts stderr "Checking exclude pat '$pat' against '$f'"
# if {[globmatchpath $pat $f]} {
# set skip 1
# break
# }
# }
# if {!$skip} {
# lappend folders $f
# }
# }
#} else {
# set folders $all_subfolders
#}
#if {$do_recursion} {
# foreach subdir $folders {
# lappend folders {*}[subfolders -exclude-paths $exclude_paths -recursive $subdir]
# }
#}
return $folders return $folders
} }
#todo - treefolders with similar search caps as treefilenames #todo - treefolders with similar search caps as treefilenames
punk::args::define { punk::args::define {
@id -id ::punk::path::treefilenames @id -id ::punk::path::treefilenames
@cmd -name punk::path::treefilenames\
-summary\
"List of filenames below supplied path."\
-help\
"List of filenames below path.
The resulting list is unsorted."
-directory -type directory -help\ -directory -type directory -help\
"folder in which to begin recursive scan for files." "folder in which to begin recursive scan for files."
-call-depth-internal -default 0 -type integer -call-depth-internal -default 0 -type integer -help "internal use only - caller should not specify - used to track depth of recursive calls for internal logic"
-sort -type any -default natural -choices {none ascii dictionary natural} -call-subvector -default {} -type list -help "internal use only - caller should not specify - used to track path vector of recursive calls for internal logic"
-call-allbelow -default 1 -type boolean -help "internal use only - caller should not specify - used to track whether we are in a subtree below a match for glob_paths (which means we can skip glob matching and antiglob_paths checks and just include all files below here)"
-sort -type any -default natural -choices {none ascii dictionary natural}
-antiglob_paths -default {} -help\ -antiglob_paths -default {} -help\
"list of path patterns to exclude "list of path patterns to exclude
may include * and ** path segments e.g may include * and ** path segments e.g
/usr/** (exlude subfolders based at /usr but not /usr/** (exclude subfolders based at /usr but not
files within /usr itself) files within /usr itself)
**/_aside (exlude files where _aside is last segment) **/_aside (exclude files where _aside is last segment)
**/_aside/* (exclude folders one below an _aside folder) **/_aside/* (exclude folders one below an _aside folder)
**/_aside/** (exclude all folders with _aside as a segment)" **/_aside/** (exclude all folders with _aside as a segment)"
-antiglob_files -default {} -antiglob_files -default {}
-glob_paths -default {*} -help\
"list of path patterns to include
may include * and ** path segments e.g
/usr/** (include subfolders based at /usr but not
files within /usr itself)
**/_aside (include files where _aside is last segment)
**/_aside/* (include folders one below an _aside folder)
**/_aside/** (include all folders with _aside as a segment)"
@values -min 0 -max -1 -optional 1 -type string @values -min 0 -max -1 -optional 1 -type string
tailglobs -default * -multiple 1 -help\ tailglobs -default * -multiple 1 -help\
"Patterns to match against filename portion (last segment) of each file path "Patterns to match against filename portion (last segment) of each file path
@ -732,12 +949,20 @@ namespace eval punk::path {
lassign [dict values $argd] leaders opts values received lassign [dict values $argd] leaders opts values received
set tailglobs [dict get $values tailglobs] set tailglobs [dict get $values tailglobs]
# -- --- --- --- --- --- --- # -- --- --- --- --- --- ---
set opt_sort [dict get $opts -sort] set opt_sort [dict get $opts -sort]
set opt_antiglob_paths [dict get $opts -antiglob_paths] set opt_antiglob_paths [dict get $opts -antiglob_paths]
set opt_antiglob_files [dict get $opts -antiglob_files] set opt_glob_paths [dict get $opts -glob_paths]
set CALLDEPTH [dict get $opts -call-depth-internal] set opt_antiglob_files [dict get $opts -antiglob_files]
set CALLDEPTH [dict get $opts -call-depth-internal]
set callsubvector [dict get $opts -call-subvector]
set callallbelow [dict get $opts -call-allbelow] ;#whether to return matches longer than the matched glob-path
# -- --- --- --- --- --- --- # -- --- --- --- --- --- ---
# -- --- --- --- --- --- --- # -- --- --- --- --- --- ---
if {"*" in $opt_glob_paths} {
#if we have a * in the default glob_paths - then any other entries are redundant.
set opt_glob_paths {*}
}
set files [list] set files [list]
if {$CALLDEPTH == 0} { if {$CALLDEPTH == 0} {
@ -745,14 +970,17 @@ namespace eval punk::path {
package require natsort package require natsort
} }
#set opts [dict merge $opts [list -directory $opt_dir]] #set opts [dict merge $opts [list -directory $opt_dir]]
if {![dict exists $received -directory]} { if {[dict exists $received -directory]} {
set opt_dir [pwd]
} else {
set opt_dir [dict get $opts -directory] set opt_dir [dict get $opts -directory]
} else {
set opt_dir [pwd]
} }
if {![file isdirectory $opt_dir]} { if {![file isdirectory $opt_dir]} {
return [list] return [list]
} }
} else { } else {
#assume/require to exist in any recursive call #assume/require to exist in any recursive call
set opt_dir [dict get $opts -directory] set opt_dir [dict get $opts -directory]
@ -831,19 +1059,35 @@ namespace eval punk::path {
lappend okdirs $dir lappend okdirs $dir
} }
} }
if {[llength $okdirs]} { if {$opt_glob_paths eq {*}} {
set matchdirs $okdirs
} else {
#** only significant when it is the trailing part of a segment eg /**/xxx /a**/xxx
set matchdirs [list]
foreach dir $okdirs {
foreach gp $opt_glob_paths {
if {[globmatchpath $gp $dir] || [globmatchpath "$gp/**" $dir]} {
lappend matchdirs $dir
}
}
}
}
if {[llength $matchdirs]} {
switch -- $opt_sort { switch -- $opt_sort {
ascii { ascii {
set finaldirs [lsort $okdirs] set finaldirs [lsort $matchdirs]
} }
dictionary { dictionary {
set finaldirs [lsort -dictionary $okdirs] set finaldirs [lsort -dictionary $matchdirs]
} }
natural { natural {
set finaldirs [natsort::sort $okdirs] set finaldirs [natsort::sort $matchdirs]
} }
default { default {
set finaldirs $okdirs set finaldirs $matchdirs
} }
} }
foreach dir $finaldirs { foreach dir $finaldirs {

5
src/vfs/_vfscommon.vfs/modules/punk/pipe-1.0.tm

@ -169,8 +169,8 @@ tcl::namespace::eval punk::pipe::lib {
#This stops us matching {/@**@x x} vs {/@**@x x} #This stops us matching {/@**@x x} vs {/@**@x x}
#--- #---
set rhs [tcl::string::map {: <c> ? <q> * <star> [ <lb> ] <rb> \\ <bsl> {"} <dq> " " <sp>} $rhs] set rhs [tcl::string::map {: <c> ; <sc> ? <q> * <star> [ <lb> ] <rb> \\ <bsl> {"} <dq> " " <sp>} $rhs]
#review - we don't expect other command-incompatible chars such as colon? #review - we don't expect other command-incompatible chars?
return $rhs return $rhs
} }
@ -187,6 +187,7 @@ tcl::namespace::eval punk::pipe::lib {
#exclude quoted whitespace #exclude quoted whitespace
proc arg_is_script_shaped {arg} { proc arg_is_script_shaped {arg} {
set arg [string map {\\; "<escaped_semicolon>"} $arg]
if {[tcl::string::first \n $arg] >= 0} { if {[tcl::string::first \n $arg] >= 0} {
return 1 return 1
} elseif {[tcl::string::first ";" $arg] >= 0} { } elseif {[tcl::string::first ";" $arg] >= 0} {

17
src/vfs/_vfscommon.vfs/modules/punk/repo-0.1.1.tm

@ -1817,17 +1817,13 @@ namespace eval punk::repo {
error "unimplemented" error "unimplemented"
} }
#file normalize is expensive so this is too #file normalize can be a little expensive so this is too
proc norm {path {platform env}} { proc norm {path {platform env}} {
#kettle::path::norm
#see also wiki
#full path normalization
set platform [string tolower $platform]
if {$platform eq "env"} {
set platform $::tcl_platform(platform)
}
#set platform [string tolower $platform]
#if {$platform eq "env"} {
# set platform $::tcl_platform(platform)
#}
#No - don't do this sort of path translation here - leave as option for specific utils only such as ./ #No - don't do this sort of path translation here - leave as option for specific utils only such as ./
#Windows volume-relative syntax with specific volume specified is somewhat broken in Tcl - but leading slash volume-relative does work #Windows volume-relative syntax with specific volume specified is somewhat broken in Tcl - but leading slash volume-relative does work
#We shouldn't break it totally just because accessing WSL/mingw paths is slightly more useful #We shouldn't break it totally just because accessing WSL/mingw paths is slightly more useful
@ -1835,6 +1831,9 @@ namespace eval punk::repo {
#return [file dirname [file normalize [punk::unixywindows::towinpath $path]/__]] #return [file dirname [file normalize [punk::unixywindows::towinpath $path]/__]]
#} #}
#kettle::path::norm
#see also wiki
#full path normalization
return [file dirname [file normalize $path/__]] return [file dirname [file normalize $path/__]]
} }

BIN
src/vfs/_vfscommon.vfs/modules/test/overtype-1.7.4.tm

Binary file not shown.

77
src/vfs/_vfscommon.vfs/modules/test/runtestmodules.tcl

@ -3,6 +3,65 @@
#(plain tclsh may stall - todo - review reasons for this and whether shellfilter can be modified to support ordinary tclsh) #(plain tclsh may stall - todo - review reasons for this and whether shellfilter can be modified to support ordinary tclsh)
#A known working copy of a punk shell executable should be placed on the path and the shebang line updated to reflect this #A known working copy of a punk shell executable should be placed on the path and the shebang line updated to reflect this
#------------------------------------
lassign [split [info tclversion] .] tcl_major tcl_minor
set script_dir [file dirname [file normalize [info script]]]
set modules_posn [string first /modules/ $script_dir]
if {$modules_posn < 0} {
puts stderr "Error: script dir $script_dir does not contain /modules/"
#exit 2 ;#don't call exit. If run in a single proc it can cause the hole test suite exit before summary can be printed.
return -code error "Error: script dir $script_dir does not contain /modules/"
}
set modules_base [string range $script_dir 0 $modules_posn-1]
if {[file tail $modules_base] eq "src"} {
set project_root [file dirname $modules_base]
} else {
set project_root $modules_base
}
puts stderr "runtestmodules.tcl project_root: $project_root"
#use the unbuilt modules/libraries under development rather than the installed versions.
#The unbuilt modules should have a higher version number (such as the magic version number 999999.0a1.0) than any installed versions to ensure they are preferred.
tcl::tm::add [file normalize $project_root/src/modules]
tcl::tm::add [file normalize $project_root/src/modules_tcl$tcl_major]
tcl::tm::add [file normalize $project_root/src/vendormodules]
tcl::tm::add [file normalize $project_root/src/vendormodules_tcl$tcl_major]
# add 'package ifneeded' definitions for unbuilt #modpod modules.
#first gather subdirectories of modules that contain #modpod-*-999999.0a1.0 in their name - these should be the unbuilt versions of zip based modules.
#set subfolders [punk::path::subfolders -recursive [file normalize $project_root/src/modules] -match */#modpod-*-999999.0a1.0]
#'punk::path::subfolders' currently only supports negative matching with -exclude, so we have to filter for the positive match ourselves.
set subfolders [punk::path::subfolders -recursive [file normalize $project_root/src/modules] -exclude {**/_build/** **/_build}]
foreach sub $subfolders {
#In most cases we could use string match - but the * within modpod-*-999999.0a1.0 could match a forward slash which could then match some other file under a #modpod- folder structure,
#so we use globmatchpath which treats * as matching any characters except path separators.
if {[globmatchpath "**/#modpod-*-999999.0a1.0" $sub]} {
set modname [file tail $sub]
set modname [string range $modname 8 end-12] ;#strip off #modpod- and -999999.0a1.0
set modpath [file join $sub "$modname-999999.0a1.0.tm"]
#!!!!
#todo - calculate fully qualified module name based on path relative to the modules folder we added to the tcl::tm path.
if {[file exists $modpath]} {
puts stderr "runtestmodules.tcl adding package ifneeded for modpod module $modname at path $modpath"
package ifneeded $modname 999999.0a1.0 [list source $modpath]
} else {
puts stderr "runtestmodules.tcl warning: expected mod.tcl not found for modpod module $modname at path $modpath"
}
}
}
set libdir [file normalize $project_root/src/lib]
set libvdir [file normalize $project_root/src/lib/tcl$tcl_major]
set libvldir [file normalize $project_root/src/vendorlib]
set libvlvdir [file normalize $project_root/src/vendorlib_tcl$tcl_major]
foreach d [list $libdir $libvdir $libvldir $libvlvdir] {
if {$d ni $::auto_path} {
lappend ::auto_path $d
}
}
#------------------------------------
puts stderr "runtestmodules.tcl ::auto_path: $::auto_path"
puts stderr "runtestmodules.tcl tcl::tm::list: [tcl::tm::list]"
package require punk package require punk
package require punk::args package require punk::args
@ -122,7 +181,7 @@ foreach pkg $punktestpkgs {
foreach ln [split $chunk \n] { foreach ln [split $chunk \n] {
incr i incr i
if {[string match "Tests ended at*" $ln]} { if {[string match "Tests ended at*" $ln]} {
puts stdout "<stdout> [punk::ansi::ansistring VIEW -lf 2 -cr 1 "$pkg $ln"]" puts stdout "<stdout><$pkg> $ln"
} elseif {[string match "*:*Total*Passed*Skipped*Failed*" $ln]} { } elseif {[string match "*:*Total*Passed*Skipped*Failed*" $ln]} {
set fields [lrange $ln 1 end] set fields [lrange $ln 1 end]
dict for {K v} $fields { dict for {K v} $fields {
@ -136,16 +195,26 @@ foreach pkg $punktestpkgs {
} }
} }
} }
puts stdout "<stdout>$pkg $ln" puts stdout "<stdout><$pkg> $ln"
} elseif {[string match "*Sourced * Test Files*" $ln]} {
puts stdout "<stdout><$pkg> $ln"
} else { } else {
puts stdout "<stdout> $ln" if {[string trim $ln] ne ""} {
puts stdout "<stdout> $ln"
} else {
puts -nonewline stdout "\n"
}
#puts stdout "$i" #puts stdout "$i"
} }
} }
flush stdout flush stdout
} }
stderr { stderr {
puts stderr "<stderr> [punk::ansi::ansistring VIEW -lf 2 -cr 1 $chunk]" #puts stderr "<stderr> [punk::ansi::ansistring VIEW -lf 2 -cr 1 $chunk]"
set chunkview [punk::ansi::ansistring VIEW -lf 2 -cr 1 $chunk]
foreach ln [split $chunkview \n] {
puts stderr "<stderr> $ln"
}
flush stderr flush stderr
} }
default { default {

Loading…
Cancel
Save