make.tcl fixes, reload libs with accelerators - progress output

src/bootsupport/modules/punk/mix/base-0.1.tm

@@ -433,7 +433,7 @@ namespace eval punk::mix::base {
         set default_hash sha1 ;#but fall back to md5 if either sha1 is unavailable or unaccelerated (pure tcl sha1 is way slower than pure tcl md5 - can take minutes on even moderate sized source files)
         if {![catch {package require sha1}]} {
             set impls [::sha1::Implementations]
-            if {[llength $impls] == 1} {
+            if {[llength $impls] == 1 && [string tolower [lindex $impls 0]] eq "tcl"} {
                 set default_hash md5
             }
         } else {

src/bootsupport/modules/punkcheck-0.1.0.tm

@@ -340,7 +340,9 @@ namespace eval punkcheck {
                         set ts_now [clock microseconds]
                         set metadata_us [expr {$ts_now - $ts_start}]
 
-                        dict set installing_record -metadata_us $metadata_us
+                        #??
+                        #JJJ
+                        #dict set installing_record -metadata_us $metadata_us
                         dict set installing_record -ts_start_transfer $ts_now
 
                         lset fileinfo_body end $installing_record
@@ -473,6 +475,7 @@ namespace eval punkcheck {
                         set path_cksum_caches [list]
                     }
                     set o_fileset_record [punkcheck::installfile_add_source_and_fetch_metadata $punkcheck_folder $rel_source_path $o_fileset_record $path_cksum_caches]
+                    #JJJ - update -metadata_us here?
 
                 }
                 method targetset_last_complete {} {
@@ -1618,13 +1621,23 @@ namespace eval punkcheck {
                     continue
                 }
                 #puts stdout "    checking file : $current_source_dir/$m"
-                set thismatchdir [file dirname $m]
+                #give some output - but not too deep
-                if {$last_processed_dir ne $thismatchdir} {
+                #set thismatchdir [file dirname [file dirname [file join $srcdir $relative_source_dir]]] 
-                    puts stdout "\n    checking files in [file join $current_source_dir $thismatchdir]"
+                #if {$last_processed_dir ne $thismatchdir} {
-                    set last_processed_dir $thismatchdir
+                #    puts stdout "\n    checking files in $thismatchdir"
+                #    set last_processed_dir $thismatchdir
+                #} else {
+                #    puts -nonewline stdout .
+                #}
+                if {$last_processed_dir ne $CALLDEPTH} {
+                    puts -nonewline stdout \n[string repeat " " $CALLDEPTH][file tail $relative_source_dir]
+                    puts -nonewline stdout \n[string repeat " " $CALLDEPTH].
+                    flush stdout
+                    set last_processed_dir $CALLDEPTH
                 } else {
                     puts -nonewline stdout .
                 }
+
                 set ts_start [clock microseconds]
                 set seconds [expr {$ts_start / 1000000}]
                 set ts_start_iso [clock format $seconds -format "%Y-%m-%dT%H:%M:%S"]
@@ -1670,7 +1683,19 @@ namespace eval punkcheck {
                 set ts2 [clock milliseconds]
                 set diff [expr {$ts2 - $ts1}]
                 if {$diff > 100} {
-                    puts stderr "punkcheck: performance warning: fetch_metdata for $m took $diff ms"
+                    set errprefix ">>> punkcheck:"
+                    puts stderr "$errprefix performance warning: fetch_metadata for $m took $diff ms."
+                    set lb [lindex [dict get $filerec body] end]
+                    #puts stderr "$errprefix filerec last body record:$lb"
+                    set records [dict get $lb body]
+                    set lr [lindex $records end]
+                    set alg [dict get $lr -cksum_all_opts -cksum_algorithm]
+                    if {$alg eq "sha1"} {
+                        puts "$errprefix cksum_algorithm: sha1 (accelerators: [::sha1::Implementations])"
+                        puts "$errprefix sha1 from: [package ifneeded sha1 [package present sha1]]"
+                    } else {
+                        puts "$errprefix cksum_algorithm: $alg"
+                    }
                 }
 
 
@@ -1806,6 +1831,8 @@ namespace eval punkcheck {
 
             if {$max_depth != -1 && $CALLDEPTH >= $max_depth} {
                 #don't process any more subdirs
+                #sometimes deliberately called with max_depth 1 - so don't warn here. review
+                #puts stderr "punkcheck::install warning - reached max_depth $max_depth"
                 set subdirs [list]
             } else {
                 set subdirs [glob -nocomplain -dir $current_source_dir -type d -tail *]
@@ -1895,7 +1922,7 @@ namespace eval punkcheck {
                     if {[llength $files_copied] || [llength $files_skipped]} {
                         #puts stdout ">>>>>>>>>>>>>>>>>>>"
                         set saveresult [punkcheck::save_records_to_file $punkcheck_records $punkcheck_file]
-                        puts stdout "punkcheck::install [dict get $saveresult recordcount] records saved as [dict get $saveresult linecount] lines to $punkcheck_file copied: [llength $files_copied] skipped: [llength $files_skipped]"
+                        puts stdout "\npunkcheck::install [dict get $saveresult recordcount] records saved as [dict get $saveresult linecount] lines to $punkcheck_file copied: [llength $files_copied] skipped: [llength $files_skipped]"
                         #puts stdout ">>>>>>>>>>>>>>>>>>>"
                     } else {
                         #todo - write db INSTALLER record if -debug true

src/make.tcl

@@ -22,7 +22,7 @@ namespace eval ::punkboot {
     variable pkg_requirements [list]; variable pkg_missing [list];variable pkg_loaded [list]
     variable non_help_flags [list -k]
     variable help_flags     [list -help --help /? -h]
-    variable known_commands [list project modules vfs info check shell vendorupdate bootsupport vfscommonupdate]
+    variable known_commands [list project modules vfs info check shell vendorupdate bootsupport vfscommonupdate ]
 }
 
 
@@ -242,12 +242,12 @@ if {[file tail $startdir] eq "src"} {
     }
 }
 # -------------------------------------------------------------------------------------
+set original_tm_list [tcl::tm::list]
+set original_auto_path $::auto_path
 
 set package_paths_modified 0
 if {$bootsupport_paths_exist || $sourcesupport_paths_exist} {
-    set original_tm_list [tcl::tm::list]
     tcl::tm::remove {*}$original_tm_list
-    set original_auto_path $::auto_path
 
     #very basic test there is something there..
     set support_contents_exist 0
@@ -1256,6 +1256,10 @@ if {$::punkboot::command eq "check"} {
 # - package path restore original module paths and auto_path entries to take effect in addition to bootsupport paths
 # - Order such that bootsupport entries are always higher priority (if same version number - prefer bootsupport)
 # - This must be done between the two "check" command sections
+
+#   Ideally we would be running make.tcl purely from bootsupport packages - but binary packages such as Thread are required,
+#   and without accelerators the performance is abysmal (e.g minutes vs seconds for common sha1 operations)
+
 if {$package_paths_modified} {
     set tm_list_boot [tcl::tm::list]
     tcl::tm::remove {*}$tm_list_boot
@@ -1271,6 +1275,25 @@ if {$package_paths_modified} {
     lappend ::auto_path {*}$original_auto_path
 }
 # -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- --
+
+
+# -- -- -- -- -- -- -- -- -- -- -- -- -- -- --
+#The problem with forcing our main packages to load only from bootsupport/sourcesupport is that if they require packages that use accelerators
+# and no acceleration is available in bootsupport/sourcesupport - then they will stay unaccelerated even if the os packages provide it
+# Also - some packages such as struct::set don't seem to handle reloading after forget (at least - not without destroying the command first)
+#Packages that provide acceleration don't use a consistent API for testing acceleration e.g md5, sha1, struct::set in tcllib all differ in
+#whether they provide functions such as Loaded, Implementations, SwitchTo
+set acceleratable [list sha1 md5]
+lappend acceleratable {*}[lsearch -all -inline [package names] struct::*]
+foreach p $acceleratable {
+    package forget $p
+    if {[string match struct::* $p]} {
+        catch {rename $p ""}
+    }
+}
+# -- -- -- -- -- -- -- -- -- -- -- -- -- -- --
+
+
 #2nd part of "check"
 if {$::punkboot::command eq "check"} {
     set sep [string repeat - 75]
@@ -2103,6 +2126,7 @@ puts "---------------------"
 #how can we do this for runtimes from other platforms? 
 #method1 try to mount as zip and kit  - depends on current runtime to have mkzip - just because there is zip data doesn't mean the kit can mount it
 #method2 analyze executable to determine if its for another platform - then ask user and save answers in a config file.?
+#mthod3 qemu?
 set runtime_caps [dict create]
 foreach runtime [dict keys $runtime_vfs_map] {
     set capscript {
@@ -2355,7 +2379,7 @@ dict set path_cksum_cache {*}[punk::mix::base::lib::get_relativecksum_from_base
 # punkcheck allows us to not rely purely on timestamps (which may be unreliable)  
 #
 foreach vfstail $vfs_tails {
-    
+
     set vfsname [file rootname $vfstail]
     puts stdout "  ------------------------------------"
     puts stdout "  checking vfs $sourcefolder/vfs/$vfstail for configured runtimes"
@@ -2372,21 +2396,25 @@ foreach vfstail $vfs_tails {
         #set runtimes [dict get $vfs_runtime_map $vfstail] 
         #runtimes in vfs_runtime_map_vfs dict are unsuffixed (.exe stripped or was not present)
         set applist [dict get $vfs_runtime_map $vfstail]
-        foreach rt_app $applist {
-            lappend runtimes [lindex $rt_app 0]
-        }
         if {"windows" eq $::tcl_platform(platform)} {
-            set runtimes_raw $runtimes
+            foreach rt_app $applist {
-            set runtimes [list]
+                set rt [lindex $rt_app 0]
-            foreach rt $runtimes_raw {
+                if {$rt ne "-" && [file exists $rtfolder/$rt.exe]} {
-                if {![string match *.exe $rt] && $rt ne "-"} {
+                    lappend runtimes $rt.exe
-                    set rt $rt.exe
+                }
+            }
+        } else {
+            foreach rt_app $applist {
+                set rt [lindex $rt_app 0]
+                if {[file exists $rtfolder/$rt]} {
+                    lappend runtimes $rt
                 }
-                lappend runtimes $rt
             }
         }
     } else {
         #only match this vfs to a correspondingly named runtime if there was no explicit entry for that runtime
+        #review - should this only be done if there are NO explicit entries? how does a user stop builds of unneeded exes that match without renaming files/folders?
+        #but conversely, adding an extra entry shouldn't stop default builds that used to run..
         set matchrt [file rootname [file tail $vfstail]] ;#e.g project.vfs -> project
         if {![dict exists $runtime_vfs_map $matchrt]} {
             if {"windows" eq $::tcl_platform(platform)} {
@@ -2394,7 +2422,9 @@ foreach vfstail $vfs_tails {
                     lappend runtimes $matchrt.exe
                 }
             } else {
-                lappend runtimes $matchrt
+                if {[file exists $rtfolder/$matchrt]} {
+                    lappend runtimes $matchrt
+                }
             }
         }
     }

src/modules/punk/mix/base-0.1.tm

@@ -433,7 +433,7 @@ namespace eval punk::mix::base {
         set default_hash sha1 ;#but fall back to md5 if either sha1 is unavailable or unaccelerated (pure tcl sha1 is way slower than pure tcl md5 - can take minutes on even moderate sized source files)
         if {![catch {package require sha1}]} {
             set impls [::sha1::Implementations]
-            if {[llength $impls] == 1} {
+            if {[llength $impls] == 1 && [string tolower [lindex $impls 0]] eq "tcl"} {
                 set default_hash md5
             }
         } else {

src/modules/punkcheck-0.1.0.tm

@@ -340,7 +340,9 @@ namespace eval punkcheck {
                         set ts_now [clock microseconds]
                         set metadata_us [expr {$ts_now - $ts_start}]
 
-                        dict set installing_record -metadata_us $metadata_us
+                        #??
+                        #JJJ
+                        #dict set installing_record -metadata_us $metadata_us
                         dict set installing_record -ts_start_transfer $ts_now
 
                         lset fileinfo_body end $installing_record
@@ -473,6 +475,7 @@ namespace eval punkcheck {
                         set path_cksum_caches [list]
                     }
                     set o_fileset_record [punkcheck::installfile_add_source_and_fetch_metadata $punkcheck_folder $rel_source_path $o_fileset_record $path_cksum_caches]
+                    #JJJ - update -metadata_us here?
 
                 }
                 method targetset_last_complete {} {
@@ -1618,13 +1621,23 @@ namespace eval punkcheck {
                     continue
                 }
                 #puts stdout "    checking file : $current_source_dir/$m"
-                set thismatchdir [file dirname $m]
+                #give some output - but not too deep
-                if {$last_processed_dir ne $thismatchdir} {
+                #set thismatchdir [file dirname [file dirname [file join $srcdir $relative_source_dir]]] 
-                    puts stdout "\n    checking files in [file join $current_source_dir $thismatchdir]"
+                #if {$last_processed_dir ne $thismatchdir} {
-                    set last_processed_dir $thismatchdir
+                #    puts stdout "\n    checking files in $thismatchdir"
+                #    set last_processed_dir $thismatchdir
+                #} else {
+                #    puts -nonewline stdout .
+                #}
+                if {$last_processed_dir ne $CALLDEPTH} {
+                    puts -nonewline stdout \n[string repeat " " $CALLDEPTH][file tail $relative_source_dir]
+                    puts -nonewline stdout \n[string repeat " " $CALLDEPTH].
+                    flush stdout
+                    set last_processed_dir $CALLDEPTH
                 } else {
                     puts -nonewline stdout .
                 }
+
                 set ts_start [clock microseconds]
                 set seconds [expr {$ts_start / 1000000}]
                 set ts_start_iso [clock format $seconds -format "%Y-%m-%dT%H:%M:%S"]
@@ -1670,7 +1683,19 @@ namespace eval punkcheck {
                 set ts2 [clock milliseconds]
                 set diff [expr {$ts2 - $ts1}]
                 if {$diff > 100} {
-                    puts stderr "punkcheck: performance warning: fetch_metdata for $m took $diff ms"
+                    set errprefix ">>> punkcheck:"
+                    puts stderr "\n$errprefix performance warning: fetch_metadata for $m took $diff ms."
+                    set lb [lindex [dict get $filerec body] end]
+                    #puts stderr "$errprefix filerec last body record:$lb"
+                    set records [dict get $lb body]
+                    set lr [lindex $records end]
+                    set alg [dict get $lr -cksum_all_opts -cksum_algorithm]
+                    if {$alg eq "sha1"} {
+                        puts stderr "$errprefix cksum_algorithm: sha1 (accelerators: [::sha1::Implementations])"
+                        puts stderr "$errprefix sha1 from: [package ifneeded sha1 [package present sha1]]"
+                    } else {
+                        puts stderr "$errprefix cksum_algorithm: $alg"
+                    }
                 }
 
 
@@ -1806,6 +1831,8 @@ namespace eval punkcheck {
 
             if {$max_depth != -1 && $CALLDEPTH >= $max_depth} {
                 #don't process any more subdirs
+                #sometimes deliberately called with max_depth 1 - so don't warn here. review
+                #puts stderr "punkcheck::install warning - reached max_depth $max_depth"
                 set subdirs [list]
             } else {
                 set subdirs [glob -nocomplain -dir $current_source_dir -type d -tail *]
@@ -1895,7 +1922,7 @@ namespace eval punkcheck {
                     if {[llength $files_copied] || [llength $files_skipped]} {
                         #puts stdout ">>>>>>>>>>>>>>>>>>>"
                         set saveresult [punkcheck::save_records_to_file $punkcheck_records $punkcheck_file]
-                        puts stdout "punkcheck::install [dict get $saveresult recordcount] records saved as [dict get $saveresult linecount] lines to $punkcheck_file copied: [llength $files_copied] skipped: [llength $files_skipped]"
+                        puts stdout "\npunkcheck::install [dict get $saveresult recordcount] records saved as [dict get $saveresult linecount] lines to $punkcheck_file copied: [llength $files_copied] skipped: [llength $files_skipped]"
                         #puts stdout ">>>>>>>>>>>>>>>>>>>"
                     } else {
                         #todo - write db INSTALLER record if -debug true

src/project_layouts/custom/_project/punk.basic/src/make.tcl

@@ -22,7 +22,7 @@ namespace eval ::punkboot {
     variable pkg_requirements [list]; variable pkg_missing [list];variable pkg_loaded [list]
     variable non_help_flags [list -k]
     variable help_flags     [list -help --help /? -h]
-    variable known_commands [list project modules vfs info check shell vendorupdate bootsupport vfscommonupdate]
+    variable known_commands [list project modules vfs info check shell vendorupdate bootsupport vfscommonupdate ]
 }
 
 
@@ -242,12 +242,12 @@ if {[file tail $startdir] eq "src"} {
     }
 }
 # -------------------------------------------------------------------------------------
+set original_tm_list [tcl::tm::list]
+set original_auto_path $::auto_path
 
 set package_paths_modified 0
 if {$bootsupport_paths_exist || $sourcesupport_paths_exist} {
-    set original_tm_list [tcl::tm::list]
     tcl::tm::remove {*}$original_tm_list
-    set original_auto_path $::auto_path
 
     #very basic test there is something there..
     set support_contents_exist 0
@@ -1256,6 +1256,10 @@ if {$::punkboot::command eq "check"} {
 # - package path restore original module paths and auto_path entries to take effect in addition to bootsupport paths
 # - Order such that bootsupport entries are always higher priority (if same version number - prefer bootsupport)
 # - This must be done between the two "check" command sections
+
+#   Ideally we would be running make.tcl purely from bootsupport packages - but binary packages such as Thread are required,
+#   and without accelerators the performance is abysmal (e.g minutes vs seconds for common sha1 operations)
+
 if {$package_paths_modified} {
     set tm_list_boot [tcl::tm::list]
     tcl::tm::remove {*}$tm_list_boot
@@ -1271,6 +1275,25 @@ if {$package_paths_modified} {
     lappend ::auto_path {*}$original_auto_path
 }
 # -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- --
+
+
+# -- -- -- -- -- -- -- -- -- -- -- -- -- -- --
+#The problem with forcing our main packages to load only from bootsupport/sourcesupport is that if they require packages that use accelerators
+# and no acceleration is available in bootsupport/sourcesupport - then they will stay unaccelerated even if the os packages provide it
+# Also - some packages such as struct::set don't seem to handle reloading after forget (at least - not without destroying the command first)
+#Packages that provide acceleration don't use a consistent API for testing acceleration e.g md5, sha1, struct::set in tcllib all differ in
+#whether they provide functions such as Loaded, Implementations, SwitchTo
+set acceleratable [list sha1 md5]
+lappend acceleratable {*}[lsearch -all -inline [package names] struct::*]
+foreach p $acceleratable {
+    package forget $p
+    if {[string match struct::* $p]} {
+        catch {rename $p ""}
+    }
+}
+# -- -- -- -- -- -- -- -- -- -- -- -- -- -- --
+
+
 #2nd part of "check"
 if {$::punkboot::command eq "check"} {
     set sep [string repeat - 75]
@@ -2103,6 +2126,7 @@ puts "---------------------"
 #how can we do this for runtimes from other platforms? 
 #method1 try to mount as zip and kit  - depends on current runtime to have mkzip - just because there is zip data doesn't mean the kit can mount it
 #method2 analyze executable to determine if its for another platform - then ask user and save answers in a config file.?
+#mthod3 qemu?
 set runtime_caps [dict create]
 foreach runtime [dict keys $runtime_vfs_map] {
     set capscript {
@@ -2355,7 +2379,7 @@ dict set path_cksum_cache {*}[punk::mix::base::lib::get_relativecksum_from_base
 # punkcheck allows us to not rely purely on timestamps (which may be unreliable)  
 #
 foreach vfstail $vfs_tails {
-    
+
     set vfsname [file rootname $vfstail]
     puts stdout "  ------------------------------------"
     puts stdout "  checking vfs $sourcefolder/vfs/$vfstail for configured runtimes"
@@ -2372,21 +2396,25 @@ foreach vfstail $vfs_tails {
         #set runtimes [dict get $vfs_runtime_map $vfstail] 
         #runtimes in vfs_runtime_map_vfs dict are unsuffixed (.exe stripped or was not present)
         set applist [dict get $vfs_runtime_map $vfstail]
-        foreach rt_app $applist {
-            lappend runtimes [lindex $rt_app 0]
-        }
         if {"windows" eq $::tcl_platform(platform)} {
-            set runtimes_raw $runtimes
+            foreach rt_app $applist {
-            set runtimes [list]
+                set rt [lindex $rt_app 0]
-            foreach rt $runtimes_raw {
+                if {$rt ne "-" && [file exists $rtfolder/$rt.exe]} {
-                if {![string match *.exe $rt] && $rt ne "-"} {
+                    lappend runtimes $rt.exe
-                    set rt $rt.exe
+                }
+            }
+        } else {
+            foreach rt_app $applist {
+                set rt [lindex $rt_app 0]
+                if {[file exists $rtfolder/$rt]} {
+                    lappend runtimes $rt
                 }
-                lappend runtimes $rt
             }
         }
     } else {
         #only match this vfs to a correspondingly named runtime if there was no explicit entry for that runtime
+        #review - should this only be done if there are NO explicit entries? how does a user stop builds of unneeded exes that match without renaming files/folders?
+        #but conversely, adding an extra entry shouldn't stop default builds that used to run..
         set matchrt [file rootname [file tail $vfstail]] ;#e.g project.vfs -> project
         if {![dict exists $runtime_vfs_map $matchrt]} {
             if {"windows" eq $::tcl_platform(platform)} {
@@ -2394,7 +2422,9 @@ foreach vfstail $vfs_tails {
                     lappend runtimes $matchrt.exe
                 }
             } else {
-                lappend runtimes $matchrt
+                if {[file exists $rtfolder/$matchrt]} {
+                    lappend runtimes $matchrt
+                }
             }
         }
     }

src/project_layouts/custom/_project/punk.project-0.1/src/bootsupport/modules/punk/ansi-0.1.1.tm

@@ -6439,6 +6439,8 @@ tcl::namespace::eval punk::ansi::class {
             set o_gx0states     [list]
             set o_splitindex    [list]
 
+            #sha1 takes *much* longer to compute than md5 if tcllibc not available - otherwise it is generally faster
+            #we should fall back to md5 if no acceleration available. check for command sha1::sha1c ?
             set o_cksum_command [list sha1::sha1 -hex]
 
 

src/project_layouts/custom/_project/punk.project-0.1/src/bootsupport/modules/punk/args-0.2.tm

@@ -876,7 +876,7 @@ tcl::namespace::eval punk::args {
 
         set cache_key $args
         #ideally we would use a fast hash algorithm to produce a short key with low collision probability.
-        #something like md5 would be ok (this is non cryptographic) - but md5 uses open and isn't usable by default in a safe interp. (sha1 often faster on modern cpus)
+        #something like md5 would be ok (this is non cryptographic) - but md5 uses open and isn't usable by default in a safe interp. (sha1 often faster on modern cpus but terribly slow without an accelerator)
         #review - check if there is a built-into-tcl way to do this quickly
         #for now we will just key using the whole string
         #performance seems ok - memory usage probably not ideal

src/project_layouts/custom/_project/punk.project-0.1/src/bootsupport/modules/punk/libunknown-0.1.tm

@@ -847,23 +847,24 @@ tcl::namespace::eval punk::libunknown {
         #keys are in reverse order due to tclPkgUnknown processing order
         set ordered_searchpaths [lreverse [dict keys $dict_added]];# ordered as in auto_path
 
+        #JN todo - look at tm epochs
         dict for {pkg versiond} $refresh_dict {
             set versions [dict keys $versiond]
-            puts stderr "---->pkg:$pkg versions: $versions"
+            #puts stderr "---->pkg:$pkg versions: $versions"
             foreach searchpath $ordered_searchpaths {
                 set addedinfo [dict get $dict_added $searchpath]
                 set vidx -1
                 foreach v $versions {
                     incr vidx
                     if {[dict exists $addedinfo $pkg $v]} {
-                        ledit versions $vidx $vidx
+                        ledit versions $vidx $vidx ;incr vidx -1 ;#maintain vidx as index into current state of $versions - not original state the foreach operates across.
                         set iscript [dict get $addedinfo $pkg $v scr]
                         #todo - find the iscript in the '$epoch pkg epochs <e> added paths' lists and determine os vs dev vs internal
                         #(scanning for path directly in the ifneeded script for pkgs is potentially error prone)
                         #for .tm ifneeded scripts - the syntax is simple enough to determine directly (and ifneeded scr not stored for those anyway)
                         set justaddedscript [package ifneeded $pkg $v]
                         if {$justaddedscript ne $iscript} {
-                                puts "---->refreshing $pkg $v  - reverting to already stored from path:$searchpath versions: $versions"
+                            #puts "---->refreshing $pkg $v  - reverting to already stored from path:$searchpath versions: $versions"
                             package ifneeded $pkg $v $iscript
                             #dict set pkgvdone $pkg $v 1
                         }

src/project_layouts/custom/_project/punk.project-0.1/src/bootsupport/modules/punk/mix/base-0.1.tm

@@ -430,7 +430,16 @@ namespace eval punk::mix::base {
 
         #not just used by cksum_path. used by caller (e.g fill_relativecksums_from_base_and_relativepathdict via cksum_filter_opts) to determine what opt names passed through
         variable cksum_default_opts
-        set cksum_default_opts [dict create -cksum_content 1 -cksum_meta auto -cksum_acls 0 -cksum_usetar auto -cksum_algorithm sha1]
+        set default_hash sha1 ;#but fall back to md5 if either sha1 is unavailable or unaccelerated (pure tcl sha1 is way slower than pure tcl md5 - can take minutes on even moderate sized source files)
+        if {![catch {package require sha1}]} {
+            set impls [::sha1::Implementations]
+            if {[llength $impls] == 1 && [string tolower [lindex $impls 0]] eq "tcl"} {
+                set default_hash md5
+            }
+        } else {
+            set default_hash md5
+        }
+        set cksum_default_opts [dict create -cksum_content 1 -cksum_meta auto -cksum_acls 0 -cksum_usetar auto -cksum_algorithm $default_hash]
         proc cksum_default_opts {} {
             variable cksum_default_opts
             return $cksum_default_opts
@@ -438,9 +447,9 @@ namespace eval punk::mix::base {
 
         #crc::cksum is extremely slow in tcllib as at 2023 e.g 20x slower (no c implementation?)
         # - try builtin zlib crc instead?
-        #sha1 is performant - and this is not being used in a cryptographic or adversarial context - so performance and practical unlikelihood of accidental collisions should be the main consideration.
+        #sha1 is performant (when accelerator present) - and this is not being used in a cryptographic or adversarial context - so performance and practical unlikelihood of accidental collisions should be the main consideration.
         #adler32 is fastest for some larger files of a few MB but slower on small files (possibly due to Tcl-based file load?)
-        #sha1 as at 2023 seems a reasonable default
+        #sha1 as at 2023 seems a reasonable default - (but only if accelerator present)
         proc cksum_algorithms {} {
             variable sha3_implementation
             #sha2 is an alias for sha256

src/project_layouts/custom/_project/punk.project-0.1/src/bootsupport/modules/punkcheck-0.1.0.tm

@@ -340,7 +340,9 @@ namespace eval punkcheck {
                         set ts_now [clock microseconds]
                         set metadata_us [expr {$ts_now - $ts_start}]
 
-                        dict set installing_record -metadata_us $metadata_us
+                        #??
+                        #JJJ
+                        #dict set installing_record -metadata_us $metadata_us
                         dict set installing_record -ts_start_transfer $ts_now
 
                         lset fileinfo_body end $installing_record
@@ -473,6 +475,7 @@ namespace eval punkcheck {
                         set path_cksum_caches [list]
                     }
                     set o_fileset_record [punkcheck::installfile_add_source_and_fetch_metadata $punkcheck_folder $rel_source_path $o_fileset_record $path_cksum_caches]
+                    #JJJ - update -metadata_us here?
 
                 }
                 method targetset_last_complete {} {
@@ -1599,6 +1602,7 @@ namespace eval punkcheck {
 
 
             #puts stdout     "Current target dir: $current_target_dir"
+            set last_processed_dir ""
             foreach m $match_list {
                 set new_tgt_cksum_info [list]
                 set relative_target_path [file join $relative_target_dir $m]
@@ -1617,6 +1621,23 @@ namespace eval punkcheck {
                     continue
                 }
                 #puts stdout "    checking file : $current_source_dir/$m"
+                #give some output - but not too deep
+                #set thismatchdir [file dirname [file dirname [file join $srcdir $relative_source_dir]]] 
+                #if {$last_processed_dir ne $thismatchdir} {
+                #    puts stdout "\n    checking files in $thismatchdir"
+                #    set last_processed_dir $thismatchdir
+                #} else {
+                #    puts -nonewline stdout .
+                #}
+                if {$last_processed_dir ne $CALLDEPTH} {
+                    puts -nonewline stdout \n[string repeat " " $CALLDEPTH][file tail $relative_source_dir]
+                    puts -nonewline stdout \n[string repeat " " $CALLDEPTH].
+                    flush stdout
+                    set last_processed_dir $CALLDEPTH
+                } else {
+                    puts -nonewline stdout .
+                }
+
                 set ts_start [clock microseconds]
                 set seconds [expr {$ts_start / 1000000}]
                 set ts_start_iso [clock format $seconds -format "%Y-%m-%dT%H:%M:%S"]
@@ -1657,7 +1678,25 @@ namespace eval punkcheck {
                 #Note this isn't a recordlist function - so it doesn't purely operate on the records
                 #this hits the filesystem for the sourcepath - gets checksums/timestamps depending on config.
                 #It doesn't save to .punkcheck (the only punkcheck::installfile_ method which doesn't)
+                set ts1 [clock milliseconds]
                 set filerec [punkcheck::installfile_add_source_and_fetch_metadata $punkcheck_folder $relative_source_path $filerec]
+                set ts2 [clock milliseconds]
+                set diff [expr {$ts2 - $ts1}]
+                if {$diff > 100} {
+                    set errprefix ">>> punkcheck:"
+                    puts stderr "$errprefix performance warning: fetch_metadata for $m took $diff ms."
+                    set lb [lindex [dict get $filerec body] end]
+                    #puts stderr "$errprefix filerec last body record:$lb"
+                    set records [dict get $lb body]
+                    set lr [lindex $records end]
+                    set alg [dict get $lr -cksum_all_opts -cksum_algorithm]
+                    if {$alg eq "sha1"} {
+                        puts "$errprefix cksum_algorithm: sha1 (accelerators: [::sha1::Implementations])"
+                        puts "$errprefix sha1 from: [package ifneeded sha1 [package present sha1]]"
+                    } else {
+                        puts "$errprefix cksum_algorithm: $alg"
+                    }
+                }
 
 
 
@@ -1681,6 +1720,7 @@ namespace eval punkcheck {
                     lappend files_copied $current_source_dir/$m
                 } else {
                     if {![file exists $current_target_dir/$m]} {
+                        puts stderr "punkcheck: first copy to $current_target_dir/$m "
                         file mkdir $current_target_dir
                         file copy $current_source_dir/$m $current_target_dir
                         set new_tgt_cksum_info [punk::mix::base::lib::cksum_path $current_target_dir/$m]
@@ -1691,9 +1731,15 @@ namespace eval punkcheck {
                             installedsourcechanged-targets {
                                 if {[llength $changed]} {
                                     #An unrecorded installation is considered a source change (from unknown/unrecorded source to recorded)
+                                    puts -nonewline stderr "punkcheck: about to: file copy -force $current_source_dir/$m $current_target_dir"
+                                    set ts1 [clock milliseconds]
                                     file mkdir $current_target_dir
                                     file copy -force $current_source_dir/$m $current_target_dir
+                                    set ts2 [clock milliseconds]
+                                    puts -nonewline stderr " (copy time [expr {$ts2 - $ts1}] ms)"
                                     set new_tgt_cksum_info [punk::mix::base::lib::cksum_path $current_target_dir/$m]
+                                    set ts3 [clock milliseconds]
+                                    puts stderr " (cksum time [expr {$ts2 - $ts1}] ms)"
                                     lappend files_copied $current_source_dir/$m
                                 } else {
                                     set is_skip 1
@@ -1785,6 +1831,8 @@ namespace eval punkcheck {
 
             if {$max_depth != -1 && $CALLDEPTH >= $max_depth} {
                 #don't process any more subdirs
+                #sometimes deliberately called with max_depth 1 - so don't warn here. review
+                #puts stderr "punkcheck::install warning - reached max_depth $max_depth"
                 set subdirs [list]
             } else {
                 set subdirs [glob -nocomplain -dir $current_source_dir -type d -tail *]
@@ -1874,7 +1922,7 @@ namespace eval punkcheck {
                     if {[llength $files_copied] || [llength $files_skipped]} {
                         #puts stdout ">>>>>>>>>>>>>>>>>>>"
                         set saveresult [punkcheck::save_records_to_file $punkcheck_records $punkcheck_file]
-                        puts stdout "punkcheck::install [dict get $saveresult recordcount] records saved as [dict get $saveresult linecount] lines to $punkcheck_file copied: [llength $files_copied] skipped: [llength $files_skipped]"
+                        puts stdout "\npunkcheck::install [dict get $saveresult recordcount] records saved as [dict get $saveresult linecount] lines to $punkcheck_file copied: [llength $files_copied] skipped: [llength $files_skipped]"
                         #puts stdout ">>>>>>>>>>>>>>>>>>>"
                     } else {
                         #todo - write db INSTALLER record if -debug true

src/project_layouts/custom/_project/punk.project-0.1/src/bootsupport/modules/textblock-0.1.3.tm

@@ -95,6 +95,7 @@ tcl::namespace::eval textblock {
     #NOTE sha1, although computationally more intensive, tends to be faster than md5 on modern cpus
     #(more likely to be optimised for modern cpu features?)
     #(This speed improvement may not apply for short strings)
+    #This is probably only true if tcllibc is available - pure-tcl sha1 is excruciatingly slow... 
 
     variable use_hash ;#framecache
     set use_hash none ;#slightly faster but uglier layout for viewing frame_cache display

src/project_layouts/custom/_project/punk.project-0.1/src/make.tcl

@@ -22,7 +22,7 @@ namespace eval ::punkboot {
     variable pkg_requirements [list]; variable pkg_missing [list];variable pkg_loaded [list]
     variable non_help_flags [list -k]
     variable help_flags     [list -help --help /? -h]
-    variable known_commands [list project modules vfs info check shell vendorupdate bootsupport vfscommonupdate]
+    variable known_commands [list project modules vfs info check shell vendorupdate bootsupport vfscommonupdate ]
 }
 
 
@@ -242,12 +242,12 @@ if {[file tail $startdir] eq "src"} {
     }
 }
 # -------------------------------------------------------------------------------------
+set original_tm_list [tcl::tm::list]
+set original_auto_path $::auto_path
 
 set package_paths_modified 0
 if {$bootsupport_paths_exist || $sourcesupport_paths_exist} {
-    set original_tm_list [tcl::tm::list]
     tcl::tm::remove {*}$original_tm_list
-    set original_auto_path $::auto_path
 
     #very basic test there is something there..
     set support_contents_exist 0
@@ -1256,6 +1256,10 @@ if {$::punkboot::command eq "check"} {
 # - package path restore original module paths and auto_path entries to take effect in addition to bootsupport paths
 # - Order such that bootsupport entries are always higher priority (if same version number - prefer bootsupport)
 # - This must be done between the two "check" command sections
+
+#   Ideally we would be running make.tcl purely from bootsupport packages - but binary packages such as Thread are required,
+#   and without accelerators the performance is abysmal (e.g minutes vs seconds for common sha1 operations)
+
 if {$package_paths_modified} {
     set tm_list_boot [tcl::tm::list]
     tcl::tm::remove {*}$tm_list_boot
@@ -1271,6 +1275,25 @@ if {$package_paths_modified} {
     lappend ::auto_path {*}$original_auto_path
 }
 # -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- --
+
+
+# -- -- -- -- -- -- -- -- -- -- -- -- -- -- --
+#The problem with forcing our main packages to load only from bootsupport/sourcesupport is that if they require packages that use accelerators
+# and no acceleration is available in bootsupport/sourcesupport - then they will stay unaccelerated even if the os packages provide it
+# Also - some packages such as struct::set don't seem to handle reloading after forget (at least - not without destroying the command first)
+#Packages that provide acceleration don't use a consistent API for testing acceleration e.g md5, sha1, struct::set in tcllib all differ in
+#whether they provide functions such as Loaded, Implementations, SwitchTo
+set acceleratable [list sha1 md5]
+lappend acceleratable {*}[lsearch -all -inline [package names] struct::*]
+foreach p $acceleratable {
+    package forget $p
+    if {[string match struct::* $p]} {
+        catch {rename $p ""}
+    }
+}
+# -- -- -- -- -- -- -- -- -- -- -- -- -- -- --
+
+
 #2nd part of "check"
 if {$::punkboot::command eq "check"} {
     set sep [string repeat - 75]
@@ -2103,6 +2126,7 @@ puts "---------------------"
 #how can we do this for runtimes from other platforms? 
 #method1 try to mount as zip and kit  - depends on current runtime to have mkzip - just because there is zip data doesn't mean the kit can mount it
 #method2 analyze executable to determine if its for another platform - then ask user and save answers in a config file.?
+#mthod3 qemu?
 set runtime_caps [dict create]
 foreach runtime [dict keys $runtime_vfs_map] {
     set capscript {
@@ -2355,7 +2379,7 @@ dict set path_cksum_cache {*}[punk::mix::base::lib::get_relativecksum_from_base
 # punkcheck allows us to not rely purely on timestamps (which may be unreliable)  
 #
 foreach vfstail $vfs_tails {
-    
+
     set vfsname [file rootname $vfstail]
     puts stdout "  ------------------------------------"
     puts stdout "  checking vfs $sourcefolder/vfs/$vfstail for configured runtimes"
@@ -2372,21 +2396,25 @@ foreach vfstail $vfs_tails {
         #set runtimes [dict get $vfs_runtime_map $vfstail] 
         #runtimes in vfs_runtime_map_vfs dict are unsuffixed (.exe stripped or was not present)
         set applist [dict get $vfs_runtime_map $vfstail]
-        foreach rt_app $applist {
-            lappend runtimes [lindex $rt_app 0]
-        }
         if {"windows" eq $::tcl_platform(platform)} {
-            set runtimes_raw $runtimes
+            foreach rt_app $applist {
-            set runtimes [list]
+                set rt [lindex $rt_app 0]
-            foreach rt $runtimes_raw {
+                if {$rt ne "-" && [file exists $rtfolder/$rt.exe]} {
-                if {![string match *.exe $rt] && $rt ne "-"} {
+                    lappend runtimes $rt.exe
-                    set rt $rt.exe
+                }
+            }
+        } else {
+            foreach rt_app $applist {
+                set rt [lindex $rt_app 0]
+                if {[file exists $rtfolder/$rt]} {
+                    lappend runtimes $rt
                 }
-                lappend runtimes $rt
             }
         }
     } else {
         #only match this vfs to a correspondingly named runtime if there was no explicit entry for that runtime
+        #review - should this only be done if there are NO explicit entries? how does a user stop builds of unneeded exes that match without renaming files/folders?
+        #but conversely, adding an extra entry shouldn't stop default builds that used to run..
         set matchrt [file rootname [file tail $vfstail]] ;#e.g project.vfs -> project
         if {![dict exists $runtime_vfs_map $matchrt]} {
             if {"windows" eq $::tcl_platform(platform)} {
@@ -2394,7 +2422,9 @@ foreach vfstail $vfs_tails {
                     lappend runtimes $matchrt.exe
                 }
             } else {
-                lappend runtimes $matchrt
+                if {[file exists $rtfolder/$matchrt]} {
+                    lappend runtimes $matchrt
+                }
             }
         }
     }

src/project_layouts/custom/_project/punk.shell-0.1/src/bootsupport/modules/punk/ansi-0.1.1.tm

@@ -6439,6 +6439,8 @@ tcl::namespace::eval punk::ansi::class {
             set o_gx0states     [list]
             set o_splitindex    [list]
 
+            #sha1 takes *much* longer to compute than md5 if tcllibc not available - otherwise it is generally faster
+            #we should fall back to md5 if no acceleration available. check for command sha1::sha1c ?
             set o_cksum_command [list sha1::sha1 -hex]
 
 

src/project_layouts/custom/_project/punk.shell-0.1/src/bootsupport/modules/punk/args-0.2.tm

@@ -876,7 +876,7 @@ tcl::namespace::eval punk::args {
 
         set cache_key $args
         #ideally we would use a fast hash algorithm to produce a short key with low collision probability.
-        #something like md5 would be ok (this is non cryptographic) - but md5 uses open and isn't usable by default in a safe interp. (sha1 often faster on modern cpus)
+        #something like md5 would be ok (this is non cryptographic) - but md5 uses open and isn't usable by default in a safe interp. (sha1 often faster on modern cpus but terribly slow without an accelerator)
         #review - check if there is a built-into-tcl way to do this quickly
         #for now we will just key using the whole string
         #performance seems ok - memory usage probably not ideal

src/project_layouts/custom/_project/punk.shell-0.1/src/bootsupport/modules/punk/libunknown-0.1.tm

@@ -847,23 +847,24 @@ tcl::namespace::eval punk::libunknown {
         #keys are in reverse order due to tclPkgUnknown processing order
         set ordered_searchpaths [lreverse [dict keys $dict_added]];# ordered as in auto_path
 
+        #JN todo - look at tm epochs
         dict for {pkg versiond} $refresh_dict {
             set versions [dict keys $versiond]
-            puts stderr "---->pkg:$pkg versions: $versions"
+            #puts stderr "---->pkg:$pkg versions: $versions"
             foreach searchpath $ordered_searchpaths {
                 set addedinfo [dict get $dict_added $searchpath]
                 set vidx -1
                 foreach v $versions {
                     incr vidx
                     if {[dict exists $addedinfo $pkg $v]} {
-                        ledit versions $vidx $vidx
+                        ledit versions $vidx $vidx ;incr vidx -1 ;#maintain vidx as index into current state of $versions - not original state the foreach operates across.
                         set iscript [dict get $addedinfo $pkg $v scr]
                         #todo - find the iscript in the '$epoch pkg epochs <e> added paths' lists and determine os vs dev vs internal
                         #(scanning for path directly in the ifneeded script for pkgs is potentially error prone)
                         #for .tm ifneeded scripts - the syntax is simple enough to determine directly (and ifneeded scr not stored for those anyway)
                         set justaddedscript [package ifneeded $pkg $v]
                         if {$justaddedscript ne $iscript} {
-                                puts "---->refreshing $pkg $v  - reverting to already stored from path:$searchpath versions: $versions"
+                            #puts "---->refreshing $pkg $v  - reverting to already stored from path:$searchpath versions: $versions"
                             package ifneeded $pkg $v $iscript
                             #dict set pkgvdone $pkg $v 1
                         }

src/project_layouts/custom/_project/punk.shell-0.1/src/bootsupport/modules/punk/mix/base-0.1.tm

@@ -430,7 +430,16 @@ namespace eval punk::mix::base {
 
         #not just used by cksum_path. used by caller (e.g fill_relativecksums_from_base_and_relativepathdict via cksum_filter_opts) to determine what opt names passed through
         variable cksum_default_opts
-        set cksum_default_opts [dict create -cksum_content 1 -cksum_meta auto -cksum_acls 0 -cksum_usetar auto -cksum_algorithm sha1]
+        set default_hash sha1 ;#but fall back to md5 if either sha1 is unavailable or unaccelerated (pure tcl sha1 is way slower than pure tcl md5 - can take minutes on even moderate sized source files)
+        if {![catch {package require sha1}]} {
+            set impls [::sha1::Implementations]
+            if {[llength $impls] == 1 && [string tolower [lindex $impls 0]] eq "tcl"} {
+                set default_hash md5
+            }
+        } else {
+            set default_hash md5
+        }
+        set cksum_default_opts [dict create -cksum_content 1 -cksum_meta auto -cksum_acls 0 -cksum_usetar auto -cksum_algorithm $default_hash]
         proc cksum_default_opts {} {
             variable cksum_default_opts
             return $cksum_default_opts
@@ -438,9 +447,9 @@ namespace eval punk::mix::base {
 
         #crc::cksum is extremely slow in tcllib as at 2023 e.g 20x slower (no c implementation?)
         # - try builtin zlib crc instead?
-        #sha1 is performant - and this is not being used in a cryptographic or adversarial context - so performance and practical unlikelihood of accidental collisions should be the main consideration.
+        #sha1 is performant (when accelerator present) - and this is not being used in a cryptographic or adversarial context - so performance and practical unlikelihood of accidental collisions should be the main consideration.
         #adler32 is fastest for some larger files of a few MB but slower on small files (possibly due to Tcl-based file load?)
-        #sha1 as at 2023 seems a reasonable default
+        #sha1 as at 2023 seems a reasonable default - (but only if accelerator present)
         proc cksum_algorithms {} {
             variable sha3_implementation
             #sha2 is an alias for sha256

src/project_layouts/custom/_project/punk.shell-0.1/src/bootsupport/modules/punkcheck-0.1.0.tm

@@ -340,7 +340,9 @@ namespace eval punkcheck {
                         set ts_now [clock microseconds]
                         set metadata_us [expr {$ts_now - $ts_start}]
 
-                        dict set installing_record -metadata_us $metadata_us
+                        #??
+                        #JJJ
+                        #dict set installing_record -metadata_us $metadata_us
                         dict set installing_record -ts_start_transfer $ts_now
 
                         lset fileinfo_body end $installing_record
@@ -473,6 +475,7 @@ namespace eval punkcheck {
                         set path_cksum_caches [list]
                     }
                     set o_fileset_record [punkcheck::installfile_add_source_and_fetch_metadata $punkcheck_folder $rel_source_path $o_fileset_record $path_cksum_caches]
+                    #JJJ - update -metadata_us here?
 
                 }
                 method targetset_last_complete {} {
@@ -1599,6 +1602,7 @@ namespace eval punkcheck {
 
 
             #puts stdout     "Current target dir: $current_target_dir"
+            set last_processed_dir ""
             foreach m $match_list {
                 set new_tgt_cksum_info [list]
                 set relative_target_path [file join $relative_target_dir $m]
@@ -1617,6 +1621,23 @@ namespace eval punkcheck {
                     continue
                 }
                 #puts stdout "    checking file : $current_source_dir/$m"
+                #give some output - but not too deep
+                #set thismatchdir [file dirname [file dirname [file join $srcdir $relative_source_dir]]] 
+                #if {$last_processed_dir ne $thismatchdir} {
+                #    puts stdout "\n    checking files in $thismatchdir"
+                #    set last_processed_dir $thismatchdir
+                #} else {
+                #    puts -nonewline stdout .
+                #}
+                if {$last_processed_dir ne $CALLDEPTH} {
+                    puts -nonewline stdout \n[string repeat " " $CALLDEPTH][file tail $relative_source_dir]
+                    puts -nonewline stdout \n[string repeat " " $CALLDEPTH].
+                    flush stdout
+                    set last_processed_dir $CALLDEPTH
+                } else {
+                    puts -nonewline stdout .
+                }
+
                 set ts_start [clock microseconds]
                 set seconds [expr {$ts_start / 1000000}]
                 set ts_start_iso [clock format $seconds -format "%Y-%m-%dT%H:%M:%S"]
@@ -1657,7 +1678,25 @@ namespace eval punkcheck {
                 #Note this isn't a recordlist function - so it doesn't purely operate on the records
                 #this hits the filesystem for the sourcepath - gets checksums/timestamps depending on config.
                 #It doesn't save to .punkcheck (the only punkcheck::installfile_ method which doesn't)
+                set ts1 [clock milliseconds]
                 set filerec [punkcheck::installfile_add_source_and_fetch_metadata $punkcheck_folder $relative_source_path $filerec]
+                set ts2 [clock milliseconds]
+                set diff [expr {$ts2 - $ts1}]
+                if {$diff > 100} {
+                    set errprefix ">>> punkcheck:"
+                    puts stderr "$errprefix performance warning: fetch_metadata for $m took $diff ms."
+                    set lb [lindex [dict get $filerec body] end]
+                    #puts stderr "$errprefix filerec last body record:$lb"
+                    set records [dict get $lb body]
+                    set lr [lindex $records end]
+                    set alg [dict get $lr -cksum_all_opts -cksum_algorithm]
+                    if {$alg eq "sha1"} {
+                        puts "$errprefix cksum_algorithm: sha1 (accelerators: [::sha1::Implementations])"
+                        puts "$errprefix sha1 from: [package ifneeded sha1 [package present sha1]]"
+                    } else {
+                        puts "$errprefix cksum_algorithm: $alg"
+                    }
+                }
 
 
 
@@ -1681,6 +1720,7 @@ namespace eval punkcheck {
                     lappend files_copied $current_source_dir/$m
                 } else {
                     if {![file exists $current_target_dir/$m]} {
+                        puts stderr "punkcheck: first copy to $current_target_dir/$m "
                         file mkdir $current_target_dir
                         file copy $current_source_dir/$m $current_target_dir
                         set new_tgt_cksum_info [punk::mix::base::lib::cksum_path $current_target_dir/$m]
@@ -1691,9 +1731,15 @@ namespace eval punkcheck {
                             installedsourcechanged-targets {
                                 if {[llength $changed]} {
                                     #An unrecorded installation is considered a source change (from unknown/unrecorded source to recorded)
+                                    puts -nonewline stderr "punkcheck: about to: file copy -force $current_source_dir/$m $current_target_dir"
+                                    set ts1 [clock milliseconds]
                                     file mkdir $current_target_dir
                                     file copy -force $current_source_dir/$m $current_target_dir
+                                    set ts2 [clock milliseconds]
+                                    puts -nonewline stderr " (copy time [expr {$ts2 - $ts1}] ms)"
                                     set new_tgt_cksum_info [punk::mix::base::lib::cksum_path $current_target_dir/$m]
+                                    set ts3 [clock milliseconds]
+                                    puts stderr " (cksum time [expr {$ts2 - $ts1}] ms)"
                                     lappend files_copied $current_source_dir/$m
                                 } else {
                                     set is_skip 1
@@ -1785,6 +1831,8 @@ namespace eval punkcheck {
 
             if {$max_depth != -1 && $CALLDEPTH >= $max_depth} {
                 #don't process any more subdirs
+                #sometimes deliberately called with max_depth 1 - so don't warn here. review
+                #puts stderr "punkcheck::install warning - reached max_depth $max_depth"
                 set subdirs [list]
             } else {
                 set subdirs [glob -nocomplain -dir $current_source_dir -type d -tail *]
@@ -1874,7 +1922,7 @@ namespace eval punkcheck {
                     if {[llength $files_copied] || [llength $files_skipped]} {
                         #puts stdout ">>>>>>>>>>>>>>>>>>>"
                         set saveresult [punkcheck::save_records_to_file $punkcheck_records $punkcheck_file]
-                        puts stdout "punkcheck::install [dict get $saveresult recordcount] records saved as [dict get $saveresult linecount] lines to $punkcheck_file copied: [llength $files_copied] skipped: [llength $files_skipped]"
+                        puts stdout "\npunkcheck::install [dict get $saveresult recordcount] records saved as [dict get $saveresult linecount] lines to $punkcheck_file copied: [llength $files_copied] skipped: [llength $files_skipped]"
                         #puts stdout ">>>>>>>>>>>>>>>>>>>"
                     } else {
                         #todo - write db INSTALLER record if -debug true

src/project_layouts/custom/_project/punk.shell-0.1/src/bootsupport/modules/textblock-0.1.3.tm

@@ -95,6 +95,7 @@ tcl::namespace::eval textblock {
     #NOTE sha1, although computationally more intensive, tends to be faster than md5 on modern cpus
     #(more likely to be optimised for modern cpu features?)
     #(This speed improvement may not apply for short strings)
+    #This is probably only true if tcllibc is available - pure-tcl sha1 is excruciatingly slow... 
 
     variable use_hash ;#framecache
     set use_hash none ;#slightly faster but uglier layout for viewing frame_cache display

src/project_layouts/custom/_project/punk.shell-0.1/src/make.tcl

@@ -22,7 +22,7 @@ namespace eval ::punkboot {
     variable pkg_requirements [list]; variable pkg_missing [list];variable pkg_loaded [list]
     variable non_help_flags [list -k]
     variable help_flags     [list -help --help /? -h]
-    variable known_commands [list project modules vfs info check shell vendorupdate bootsupport vfscommonupdate]
+    variable known_commands [list project modules vfs info check shell vendorupdate bootsupport vfscommonupdate ]
 }
 
 
@@ -242,12 +242,12 @@ if {[file tail $startdir] eq "src"} {
     }
 }
 # -------------------------------------------------------------------------------------
+set original_tm_list [tcl::tm::list]
+set original_auto_path $::auto_path
 
 set package_paths_modified 0
 if {$bootsupport_paths_exist || $sourcesupport_paths_exist} {
-    set original_tm_list [tcl::tm::list]
     tcl::tm::remove {*}$original_tm_list
-    set original_auto_path $::auto_path
 
     #very basic test there is something there..
     set support_contents_exist 0
@@ -1256,6 +1256,10 @@ if {$::punkboot::command eq "check"} {
 # - package path restore original module paths and auto_path entries to take effect in addition to bootsupport paths
 # - Order such that bootsupport entries are always higher priority (if same version number - prefer bootsupport)
 # - This must be done between the two "check" command sections
+
+#   Ideally we would be running make.tcl purely from bootsupport packages - but binary packages such as Thread are required,
+#   and without accelerators the performance is abysmal (e.g minutes vs seconds for common sha1 operations)
+
 if {$package_paths_modified} {
     set tm_list_boot [tcl::tm::list]
     tcl::tm::remove {*}$tm_list_boot
@@ -1271,6 +1275,25 @@ if {$package_paths_modified} {
     lappend ::auto_path {*}$original_auto_path
 }
 # -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- --
+
+
+# -- -- -- -- -- -- -- -- -- -- -- -- -- -- --
+#The problem with forcing our main packages to load only from bootsupport/sourcesupport is that if they require packages that use accelerators
+# and no acceleration is available in bootsupport/sourcesupport - then they will stay unaccelerated even if the os packages provide it
+# Also - some packages such as struct::set don't seem to handle reloading after forget (at least - not without destroying the command first)
+#Packages that provide acceleration don't use a consistent API for testing acceleration e.g md5, sha1, struct::set in tcllib all differ in
+#whether they provide functions such as Loaded, Implementations, SwitchTo
+set acceleratable [list sha1 md5]
+lappend acceleratable {*}[lsearch -all -inline [package names] struct::*]
+foreach p $acceleratable {
+    package forget $p
+    if {[string match struct::* $p]} {
+        catch {rename $p ""}
+    }
+}
+# -- -- -- -- -- -- -- -- -- -- -- -- -- -- --
+
+
 #2nd part of "check"
 if {$::punkboot::command eq "check"} {
     set sep [string repeat - 75]
@@ -2103,6 +2126,7 @@ puts "---------------------"
 #how can we do this for runtimes from other platforms? 
 #method1 try to mount as zip and kit  - depends on current runtime to have mkzip - just because there is zip data doesn't mean the kit can mount it
 #method2 analyze executable to determine if its for another platform - then ask user and save answers in a config file.?
+#mthod3 qemu?
 set runtime_caps [dict create]
 foreach runtime [dict keys $runtime_vfs_map] {
     set capscript {
@@ -2355,7 +2379,7 @@ dict set path_cksum_cache {*}[punk::mix::base::lib::get_relativecksum_from_base
 # punkcheck allows us to not rely purely on timestamps (which may be unreliable)  
 #
 foreach vfstail $vfs_tails {
-    
+
     set vfsname [file rootname $vfstail]
     puts stdout "  ------------------------------------"
     puts stdout "  checking vfs $sourcefolder/vfs/$vfstail for configured runtimes"
@@ -2372,21 +2396,25 @@ foreach vfstail $vfs_tails {
         #set runtimes [dict get $vfs_runtime_map $vfstail] 
         #runtimes in vfs_runtime_map_vfs dict are unsuffixed (.exe stripped or was not present)
         set applist [dict get $vfs_runtime_map $vfstail]
-        foreach rt_app $applist {
-            lappend runtimes [lindex $rt_app 0]
-        }
         if {"windows" eq $::tcl_platform(platform)} {
-            set runtimes_raw $runtimes
+            foreach rt_app $applist {
-            set runtimes [list]
+                set rt [lindex $rt_app 0]
-            foreach rt $runtimes_raw {
+                if {$rt ne "-" && [file exists $rtfolder/$rt.exe]} {
-                if {![string match *.exe $rt] && $rt ne "-"} {
+                    lappend runtimes $rt.exe
-                    set rt $rt.exe
+                }
+            }
+        } else {
+            foreach rt_app $applist {
+                set rt [lindex $rt_app 0]
+                if {[file exists $rtfolder/$rt]} {
+                    lappend runtimes $rt
                 }
-                lappend runtimes $rt
             }
         }
     } else {
         #only match this vfs to a correspondingly named runtime if there was no explicit entry for that runtime
+        #review - should this only be done if there are NO explicit entries? how does a user stop builds of unneeded exes that match without renaming files/folders?
+        #but conversely, adding an extra entry shouldn't stop default builds that used to run..
         set matchrt [file rootname [file tail $vfstail]] ;#e.g project.vfs -> project
         if {![dict exists $runtime_vfs_map $matchrt]} {
             if {"windows" eq $::tcl_platform(platform)} {
@@ -2394,7 +2422,9 @@ foreach vfstail $vfs_tails {
                     lappend runtimes $matchrt.exe
                 }
             } else {
-                lappend runtimes $matchrt
+                if {[file exists $rtfolder/$matchrt]} {
+                    lappend runtimes $matchrt
+                }
             }
         }
     }

src/vfs/_vfscommon.vfs/modules/punk/ansi-0.1.1.tm

@@ -6439,6 +6439,8 @@ tcl::namespace::eval punk::ansi::class {
             set o_gx0states     [list]
             set o_splitindex    [list]
 
+            #sha1 takes *much* longer to compute than md5 if tcllibc not available - otherwise it is generally faster
+            #we should fall back to md5 if no acceleration available. check for command sha1::sha1c ?
             set o_cksum_command [list sha1::sha1 -hex]
 
 

src/vfs/_vfscommon.vfs/modules/punk/args-0.2.tm

@@ -876,7 +876,7 @@ tcl::namespace::eval punk::args {
 
         set cache_key $args
         #ideally we would use a fast hash algorithm to produce a short key with low collision probability.
-        #something like md5 would be ok (this is non cryptographic) - but md5 uses open and isn't usable by default in a safe interp. (sha1 often faster on modern cpus)
+        #something like md5 would be ok (this is non cryptographic) - but md5 uses open and isn't usable by default in a safe interp. (sha1 often faster on modern cpus but terribly slow without an accelerator)
         #review - check if there is a built-into-tcl way to do this quickly
         #for now we will just key using the whole string
         #performance seems ok - memory usage probably not ideal

src/vfs/_vfscommon.vfs/modules/punk/libunknown-0.1.tm

@@ -847,22 +847,24 @@ tcl::namespace::eval punk::libunknown {
         #keys are in reverse order due to tclPkgUnknown processing order
         set ordered_searchpaths [lreverse [dict keys $dict_added]];# ordered as in auto_path
 
+        #JN todo - look at tm epochs
         dict for {pkg versiond} $refresh_dict {
             set versions [dict keys $versiond]
+            #puts stderr "---->pkg:$pkg versions: $versions"
             foreach searchpath $ordered_searchpaths {
                 set addedinfo [dict get $dict_added $searchpath]
                 set vidx -1
                 foreach v $versions {
                     incr vidx
                     if {[dict exists $addedinfo $pkg $v]} {
-                        ledit versions $vidx $vidx
+                        ledit versions $vidx $vidx ;incr vidx -1 ;#maintain vidx as index into current state of $versions - not original state the foreach operates across.
                         set iscript [dict get $addedinfo $pkg $v scr]
                         #todo - find the iscript in the '$epoch pkg epochs <e> added paths' lists and determine os vs dev vs internal
                         #(scanning for path directly in the ifneeded script for pkgs is potentially error prone)
                         #for .tm ifneeded scripts - the syntax is simple enough to determine directly (and ifneeded scr not stored for those anyway)
                         set justaddedscript [package ifneeded $pkg $v]
                         if {$justaddedscript ne $iscript} {
-                                puts "---->refreshing $pkg $v  - reverting to already stored from path:$searchpath versions: $versions"
+                            #puts "---->refreshing $pkg $v  - reverting to already stored from path:$searchpath versions: $versions"
                             package ifneeded $pkg $v $iscript
                             #dict set pkgvdone $pkg $v 1
                         }

src/vfs/_vfscommon.vfs/modules/punk/mix/base-0.1.tm

@@ -430,7 +430,16 @@ namespace eval punk::mix::base {
 
         #not just used by cksum_path. used by caller (e.g fill_relativecksums_from_base_and_relativepathdict via cksum_filter_opts) to determine what opt names passed through
         variable cksum_default_opts
-        set cksum_default_opts [dict create -cksum_content 1 -cksum_meta auto -cksum_acls 0 -cksum_usetar auto -cksum_algorithm sha1]
+        set default_hash sha1 ;#but fall back to md5 if either sha1 is unavailable or unaccelerated (pure tcl sha1 is way slower than pure tcl md5 - can take minutes on even moderate sized source files)
+        if {![catch {package require sha1}]} {
+            set impls [::sha1::Implementations]
+            if {[llength $impls] == 1 && [string tolower [lindex $impls 0]] eq "tcl"} {
+                set default_hash md5
+            }
+        } else {
+            set default_hash md5
+        }
+        set cksum_default_opts [dict create -cksum_content 1 -cksum_meta auto -cksum_acls 0 -cksum_usetar auto -cksum_algorithm $default_hash]
         proc cksum_default_opts {} {
             variable cksum_default_opts
             return $cksum_default_opts
@@ -438,9 +447,9 @@ namespace eval punk::mix::base {
 
         #crc::cksum is extremely slow in tcllib as at 2023 e.g 20x slower (no c implementation?)
         # - try builtin zlib crc instead?
-        #sha1 is performant - and this is not being used in a cryptographic or adversarial context - so performance and practical unlikelihood of accidental collisions should be the main consideration.
+        #sha1 is performant (when accelerator present) - and this is not being used in a cryptographic or adversarial context - so performance and practical unlikelihood of accidental collisions should be the main consideration.
         #adler32 is fastest for some larger files of a few MB but slower on small files (possibly due to Tcl-based file load?)
-        #sha1 as at 2023 seems a reasonable default
+        #sha1 as at 2023 seems a reasonable default - (but only if accelerator present)
         proc cksum_algorithms {} {
             variable sha3_implementation
             #sha2 is an alias for sha256

src/vfs/_vfscommon.vfs/modules/punkcheck-0.1.0.tm

@@ -340,7 +340,9 @@ namespace eval punkcheck {
                         set ts_now [clock microseconds]
                         set metadata_us [expr {$ts_now - $ts_start}]
 
-                        dict set installing_record -metadata_us $metadata_us
+                        #??
+                        #JJJ
+                        #dict set installing_record -metadata_us $metadata_us
                         dict set installing_record -ts_start_transfer $ts_now
 
                         lset fileinfo_body end $installing_record
@@ -473,6 +475,7 @@ namespace eval punkcheck {
                         set path_cksum_caches [list]
                     }
                     set o_fileset_record [punkcheck::installfile_add_source_and_fetch_metadata $punkcheck_folder $rel_source_path $o_fileset_record $path_cksum_caches]
+                    #JJJ - update -metadata_us here?
 
                 }
                 method targetset_last_complete {} {
@@ -1599,6 +1602,7 @@ namespace eval punkcheck {
 
 
             #puts stdout     "Current target dir: $current_target_dir"
+            set last_processed_dir ""
             foreach m $match_list {
                 set new_tgt_cksum_info [list]
                 set relative_target_path [file join $relative_target_dir $m]
@@ -1617,6 +1621,13 @@ namespace eval punkcheck {
                     continue
                 }
                 #puts stdout "    checking file : $current_source_dir/$m"
+                set thismatchdir [file dirname $m]
+                if {$last_processed_dir ne $thismatchdir} {
+                    puts stdout "\n    checking files in [file join $current_source_dir $thismatchdir]"
+                    set last_processed_dir $thismatchdir
+                } else {
+                    puts -nonewline stdout .
+                }
                 set ts_start [clock microseconds]
                 set seconds [expr {$ts_start / 1000000}]
                 set ts_start_iso [clock format $seconds -format "%Y-%m-%dT%H:%M:%S"]
@@ -1657,7 +1668,25 @@ namespace eval punkcheck {
                 #Note this isn't a recordlist function - so it doesn't purely operate on the records
                 #this hits the filesystem for the sourcepath - gets checksums/timestamps depending on config.
                 #It doesn't save to .punkcheck (the only punkcheck::installfile_ method which doesn't)
+                set ts1 [clock milliseconds]
                 set filerec [punkcheck::installfile_add_source_and_fetch_metadata $punkcheck_folder $relative_source_path $filerec]
+                set ts2 [clock milliseconds]
+                set diff [expr {$ts2 - $ts1}]
+                if {$diff > 100} {
+                    set errprefix ">>> punkcheck:"
+                    puts stderr "$errprefix performance warning: fetch_metadata for $m took $diff ms."
+                    set lb [lindex [dict get $filerec body] end]
+                    #puts stderr "$errprefix filerec last body record:$lb"
+                    set records [dict get $lb body]
+                    set lr [lindex $records end]
+                    set alg [dict get $lr -cksum_all_opts -cksum_algorithm]
+                    if {$alg eq "sha1"} {
+                        puts "$errprefix cksum_algorithm: sha1 (accelerators: [::sha1::Implementations])"
+                        puts "$errprefix sha1 from: [package ifneeded sha1 [package present sha1]]"
+                    } else {
+                        puts "$errprefix cksum_algorithm: $alg"
+                    }
+                }
 
 
 
@@ -1681,6 +1710,7 @@ namespace eval punkcheck {
                     lappend files_copied $current_source_dir/$m
                 } else {
                     if {![file exists $current_target_dir/$m]} {
+                        puts stderr "punkcheck: first copy to $current_target_dir/$m "
                         file mkdir $current_target_dir
                         file copy $current_source_dir/$m $current_target_dir
                         set new_tgt_cksum_info [punk::mix::base::lib::cksum_path $current_target_dir/$m]
@@ -1691,9 +1721,15 @@ namespace eval punkcheck {
                             installedsourcechanged-targets {
                                 if {[llength $changed]} {
                                     #An unrecorded installation is considered a source change (from unknown/unrecorded source to recorded)
+                                    puts -nonewline stderr "punkcheck: about to: file copy -force $current_source_dir/$m $current_target_dir"
+                                    set ts1 [clock milliseconds]
                                     file mkdir $current_target_dir
                                     file copy -force $current_source_dir/$m $current_target_dir
+                                    set ts2 [clock milliseconds]
+                                    puts -nonewline stderr " (copy time [expr {$ts2 - $ts1}] ms)"
                                     set new_tgt_cksum_info [punk::mix::base::lib::cksum_path $current_target_dir/$m]
+                                    set ts3 [clock milliseconds]
+                                    puts stderr " (cksum time [expr {$ts2 - $ts1}] ms)"
                                     lappend files_copied $current_source_dir/$m
                                 } else {
                                     set is_skip 1

src/vfs/_vfscommon.vfs/modules/textblock-0.1.3.tm

@@ -95,6 +95,7 @@ tcl::namespace::eval textblock {
     #NOTE sha1, although computationally more intensive, tends to be faster than md5 on modern cpus
     #(more likely to be optimised for modern cpu features?)
     #(This speed improvement may not apply for short strings)
+    #This is probably only true if tcllibc is available - pure-tcl sha1 is excruciatingly slow... 
 
     variable use_hash ;#framecache
     set use_hash none ;#slightly faster but uglier layout for viewing frame_cache display