From f9c5d4ac60030c59504bebe06264933aff7d9b0a Mon Sep 17 00:00:00 2001 From: Julian Noble Date: Thu, 30 Jul 2026 18:23:49 +1000 Subject: [PATCH] G-138 increment 1: library-artifacts emission step, both suites, e2e verified New tools/library_artifacts.tcl (suite_tcl90 + identical suite_tcl86 copy): emits the punkbin LIB tier from verified suite builds - installed-shape package folders as immutable --r.zip artifacts (lib/allplatforms for pure-tcl tcllib, lib/ for the tcllibc critcl accelerators) with an embedded schema-v2 class=library punkbin-artifact.toml at the package root (embed-then-hash: sha1/size/built stay sidecar-only), toml sidecars and per-tier punkbin-format sha1sums.txt. Provenance: [provenance] class=suite-built (the G-123 schema-v2 lineage), tcllib/critcl checkout uuids, critcl version from the package's own teapot.txt, driving_tcl_patchlevel, toolchain/optimize. Determinism: punkzip assembly (sorted members) over a staging copy with file/dir mtimes synced from the install tree, the embedded record's mtime pinned to the package's pkgIndex.tcl - re-emission over an unchanged install verified BYTE-IDENTICAL in both suites. The dlls inside are not bit-reproducible across zig rebuilds, so -Dlibrev bumps only on deliberate publish (default 1 - nothing published yet). Wiring: 'library-artifacts' zig step in build905.zig (runs under the plain family kit - proof-doubling per kit-family-artifacts) and build86.zig (runs under the installed static shell; brings the G-117 identity options to that recipe), in the default step lists (suite.tcl + bootstrap steps parity). punkzip staged as an upstream git source like critcl (sources.config record + lazy build.zig.zon pin, commit 0882f0373e = v2.3.1, generation-neutral) and compiled host-native by the recipes (ReleaseSafe - real deflate work). Verified 2026-07-30: suite_tcl90 emitted tcllib2.0-tcl9-r1.zip (871 members) + tcllibc-tcl9-r1.zip; suite_tcl86 emitted the -tcl8- pair (different bytes, same shape - the generation tag is identity, not decoration). Materialized load tests: packages extracted from the zips resolve and load under the plain family kit (tcl9) and the static 8.6 shell, critcl accelerator engaged from the extracted tcllibc, sources proven via package ifneeded paths (the 8.6 first run tripped the recorded tm-path shadowing trap - hermetic re-run clean). md5c dispositioned: NOT published - md5x.tcl prefers tcllibc, which bundles the md5 accelerator. Evidence and decisions in the goal detail Progress section; suite READMEs document the step; punkbin layout-docs update is the remaining acceptance item. Assisted-by: harness=claude; primary-model=claude-fable-5; api-location=anthropic.com --- goals/G-138-punkbin-library-artifact-tier.md | 57 +++ src/buildsuites/suite_tcl86/README.md | 18 +- src/buildsuites/suite_tcl86/build.zig.zon | 8 + src/buildsuites/suite_tcl86/build86.zig | 97 ++++- src/buildsuites/suite_tcl86/sources.config | 6 + src/buildsuites/suite_tcl86/suite.tcl | 15 +- .../suite_tcl86/tools/library_artifacts.tcl | 399 ++++++++++++++++++ src/buildsuites/suite_tcl90/README.md | 47 ++- src/buildsuites/suite_tcl90/build.zig.zon | 7 + src/buildsuites/suite_tcl90/build905.zig | 69 ++- src/buildsuites/suite_tcl90/sources.config | 6 + src/buildsuites/suite_tcl90/suite.tcl | 17 +- .../suite_tcl90/tools/library_artifacts.tcl | 399 ++++++++++++++++++ 13 files changed, 1127 insertions(+), 18 deletions(-) create mode 100644 src/buildsuites/suite_tcl86/tools/library_artifacts.tcl create mode 100644 src/buildsuites/suite_tcl90/tools/library_artifacts.tcl diff --git a/goals/G-138-punkbin-library-artifact-tier.md b/goals/G-138-punkbin-library-artifact-tier.md index e97735ef..d790582f 100644 --- a/goals/G-138-punkbin-library-artifact-tier.md +++ b/goals/G-138-punkbin-library-artifact-tier.md @@ -95,3 +95,60 @@ via an untracked bin/packages/ tier (G-139). no overlaps beyond the drafting survey. Remaining unlinked pairs (G-130/G-101/G-105/G-116 et al) share only the buildsuites tree, no contract-level relationship. +- md5c DISPOSITION (2026-07-30): NOT published as its own zip. tcllib's md5 + accelerator resolution prefers tcllibc (tcllib modules/md5/md5x.tcl + KnownImplementations probe: `package require tcllibc` FIRST, md5c the + fallback), tcllibc bundles the md5 accelerator (every suite build's + pkg_smoke `-accel md5 tcllibc` asserts ::md5::accel(critcl); the + materialized-tier load tests below show accel=1 with no md5c present), so a + standalone md5c adds nothing the published tcllibc does not provide. The + vendored trees' nested md5c copies retire with those trees under G-139. +- Naming decision (settled in the work, 2026-07-30): artifact names carry the + Tcl GENERATION tag - `--r.zip` + (tcllib2.0-tcl9-r1.zip, tcllibc-tcl8-r1.zip). Both generations install + identically-named folders (tcllib2.0, tcllibc), and the tcl8/tcl9 builds + are genuinely different bytes (verified: same member count, different + sha1s), so the tag is identity, not decoration. Recorded in the punkbin + layout docs. + +## Progress + +- 2026-07-30 emission landed, BOTH generations demonstrated end-to-end: + - tools/library_artifacts.tcl (suite_tcl90 + byte-identical suite_tcl86 + copy): mtime-synced staging copy, embedded schema-2 class=library record + (embed-then-hash; record mtime pinned to the package's pkgIndex.tcl), + punkzip assembly, porcelain re-verification of the embedded member, + sidecar + per-tier sha1sums emission. Records carry [provenance] + class="suite-built", tcllib/critcl checkout uuids, critcl version from + the package's own teapot.txt, driving_tcl_patchlevel. + - build905.zig/build86.zig 'library-artifacts' steps (in the default step + lists, suite.tcl + bootstrap parity): -Dlibrev revision option (default + 1 - nothing published yet), punkzip compiled host-native from the staged + upstream checkout (sources.config git record + build.zig.zon pin, commit + 0882f0373e = v2.3.1, generation-neutral). suite_tcl90 runs the emission + under the plain family kit (proof-doubling); suite_tcl86 under the + installed static shell. + - suite_tcl90 demo: tcllib2.0-tcl9-r1.zip (lib/allplatforms, 871 members, + sha1 aabda9db...) + tcllibc-tcl9-r1.zip (lib/win32-x86_64, sha1 + f9300c02...); dependency chain re-verified tcllib/tcllibc smokes + plain + family check on every emission. + - suite_tcl86 demo: tcllib2.0-tcl8-r1.zip + tcllibc-tcl8-r1.zip (same + layout, sha1s 5f385fab.../40bf00da...). + - Determinism verified BOTH suites: immediate re-emission reproduced + byte-identical zips (identical sha1s across independent runs). punkzip + field study: zip members carry file AND dir mtimes - hence the tool's + mtime-sync + record-mtime pinning design. + - Materialized-load verification: zips extracted with punkzip, then + md5/sha1/tcllibc package-required FROM the extracted trees (auto_path + pinned to the tier; attached/machine copies excluded) under the plain + family kit (tcl9) and the installed static 8.6 shell (tcl8) - all loads + from the tier confirmed via package ifneeded paths, critcl accelerator + engaged (accel=1) both generations. 8.6 test note: the first run tripped + the recorded tcl86 tm-path shadowing trap (machine module tree via + default tm paths) - hermetic re-run (tm paths cleared) proved the tier. + - zig-0.16 'failed command'-on-stderr-success cosmetic trap reconfirmed on + critcl re-runs (Run cache keys include the inherited PATH, so a different + shell session re-runs critcl; the step succeeds, zig still prints the + command under a 'failed command:' banner when stderr had warnings). +- Remaining for acceptance: punkbin layout docs record the lib tier (punkbin + repo AGENTS.md/README + build_sha1sums.tcl lib handling). diff --git a/src/buildsuites/suite_tcl86/README.md b/src/buildsuites/suite_tcl86/README.md index dea23621..457e6399 100644 --- a/src/buildsuites/suite_tcl86/README.md +++ b/src/buildsuites/suite_tcl86/README.md @@ -65,6 +65,21 @@ ACTIVE - see `goals/G-100-suite-tcl86-tk-tcllib.md` for the progress ledger. `winMsgbox.test` are excluded by default because their message-injection automation blocks waiting on a human under a batch shell - re-include with `-Dtestnotfiles-tk=""`). +- punkbin lib-tier library artifacts (G-138; `library-artifacts` step, default): + the tcl8 GENERATION of the suite's verified library packages as immutable + `-r` zips under `out/library/punkbin/lib/` - + `lib/allplatforms/tcllib2.0-tcl8-r1.zip` (platform-neutral) and + `lib/win32-x86_64/tcllibc-tcl8-r1.zip` (per-platform binary) - with embedded + schema-v2 `class = "library"` records, toml sidecars and per-tier sha1sums. + Forked from suite_tcl90's step (see its README "Library artifacts (G-138)" + for the full mechanism: generation-tagged names, embed-then-hash, punkzip + determinism - byte-identical re-emission verified here too, publish + discipline for the non-bit-reproducible dlls, md5c disposition). Runs under + the INSTALLED STATIC SHELL (this suite has no kit family): sha1/tcllibc + resolve from `/lib`, so emission still proves the built shell + executes the very packages being published. Revision `-Dlibrev=N` + (suite.tcl `-zigargs {-Dlibrev=N}`; default 1). Publication to the real + punkbin repo remains a deliberate user step. ## Sources (both flows carry the same identity) @@ -211,7 +226,8 @@ deliberately untouched here (out of this goal's scope). - FOSSIL DEV FLOW: `tclsh suite.tcl build|test|clean` (or via the make.tcl surface: `tclsh src/make.tcl buildsuite build suite_tcl86 ?driver-args?`). - Default build steps: `install install-libraries smoke tklib tcllib tcllibc`. + Default build steps: `install install-libraries smoke tklib tcllib tcllibc + library-artifacts`. - NO-TCLSH FLOW: `zig build --build-file build86.zig stage|bootstrap` from THIS tracked directory (materializes the `build.zig.zon` pins for any source that has no checkout in the stage, then drives the staged recipe). `stage` verified diff --git a/src/buildsuites/suite_tcl86/build.zig.zon b/src/buildsuites/suite_tcl86/build.zig.zon index aab23274..b429da6a 100644 --- a/src/buildsuites/suite_tcl86/build.zig.zon +++ b/src/buildsuites/suite_tcl86/build.zig.zon @@ -73,6 +73,14 @@ .hash = "N-V-__8AAF8-bABDlxChmtnyalKgofVfN3kzcZdmZ7pjEmE0", .lazy = true, }, + //punkzip v2.3.1 (commit 0882f0373e; no upstream tags) - generation-neutral + //like tklib/tcllib/critcl above: the SAME pin suite_tcl90 carries. + //Deterministic zip assembler for library-artifacts (G-138). + .punkzip = .{ + .url = "https://www.gitea1.intx.com.au/jn/punkzip/archive/0882f0373eb2c983142e713c814b91aa82cfcea7.tar.gz", + .hash = "punkzip-2.3.1-PjFoDUNlgwCyYfz8k0-SNGxaXsh53YWTXv5f9JRRXQK8", + .lazy = true, + }, }, .paths = .{""}, } diff --git a/src/buildsuites/suite_tcl86/build86.zig b/src/buildsuites/suite_tcl86/build86.zig index 07bf240f..c34370d6 100644 --- a/src/buildsuites/suite_tcl86/build86.zig +++ b/src/buildsuites/suite_tcl86/build86.zig @@ -851,6 +851,99 @@ pub fn build(b: *std.Build) !void { accel_bench.step.dependOn(&tcllibc_smoke.step); tcllibc_step.dependOn(&accel_bench.step); + //-- library-artifacts: punkbin LIB-TIER emission (G-138) - the tcl8 + //GENERATION of the library packages this suite builds and verifies + //(tcllib pure-tcl -> lib/allplatforms, tcllibc critcl accelerators -> + //lib/win32-x86_64) as immutable -tcl8-r zips with embedded schema-v2 + //class=library records, toml sidecars and per-tier sha1sums. Forked from + //suite_tcl90/build905.zig's step (the recorded duplication posture); key + //differences: runs under the INSTALLED STATIC SHELL (this suite has no + //kit family - the smoke-run pattern above; sha1/tcllibc resolve from + ///lib, so emission still proves the built shell executes the + //very packages being published), and the generation tag is tcl8 (folder + //names collide across generations: both install tcllib2.0 + tcllibc). + //punkzip: staged upstream checkout (sources.config / build.zig.zon pin), + //compiled for the build host; ReleaseSafe deliberately (real deflate + //work). Revisions bump ONLY on deliberate publish (-Dlibrev; the dlls + //are not bit-reproducible across zig rebuilds). Publication to the real + //punkbin repo stays a deliberate user step. + const library_artifacts_step = b.step("library-artifacts", "emit punkbin lib-tier artifacts (-tcl8-r.zip + toml + sha1sums) from the verified tcllib/tcllibc installs -> /library/punkbin/lib"); + const librev = b.option(u32, "librev", "assembly revision N for the -r library artifact names (library-artifacts; default 1 - nothing published yet)") orelse 1; + const punkzip_exe = b.addExecutable(.{ + .name = "punkzip", + .root_module = b.createModule(.{ + .root_source_file = b.path("../punkzip/src/punkzip.zig"), + .target = b.graph.host, + .optimize = .ReleaseSafe, + }), + }); + //G-117 schema identity options (introduced here by the library step - + //this suite emits no runtime-family artifacts): origin = canonical + //artifact repo the artifacts are BUILT FOR; packager = declared + //identity, resolution chain -Dpackager > env PUNKBIN_PACKAGER > + //building checkout's git identity > "unrecorded". + const originurl = b.option([]const u8, "originurl", "canonical artifact repo url the library artifacts are built FOR ('origin' field; default punkbin)") orelse "https://www.gitea1.intx.com.au/jn/punkbin"; + const projecturl = b.option([]const u8, "projecturl", "project url recorded in the artifact records ('project_url' field; default unrecorded)") orelse "unrecorded"; + const packager = blk: { + if (b.option([]const u8, "packager", "declared packager identity for the artifact records (-Dpackager > env PUNKBIN_PACKAGER > git identity > unrecorded)")) |p| break :blk p; + if (b.graph.environ_map.get("PUNKBIN_PACKAGER")) |p| { + if (p.len != 0) break :blk p; + } + //out_code is only meaningful on some runAllowFail paths - judge + //success by usable trimmed output, not the code + var gc: u8 = undefined; + const gname: ?[]const u8 = b.runAllowFail(&.{ "git", "config", "user.name" }, &gc, .ignore) catch null; + if (gname != null) { + const nm = std.mem.trim(u8, gname.?, " \t\r\n"); + if (nm.len != 0) { + const gmail: ?[]const u8 = b.runAllowFail(&.{ "git", "config", "user.email" }, &gc, .ignore) catch null; + if (gmail != null) { + const em = std.mem.trim(u8, gmail.?, " \t\r\n"); + if (em.len != 0) break :blk b.fmt("{s} <{s}>", .{ nm, em }); + } + break :blk nm; + } + } + break :blk "unrecorded"; + }; + //source checkout provenance: tcllib manifest.uuid (suite.tcl's fossil + //staging materializes it); critcl commit via best-effort git probe - + //bootstrap-flow tarballs carry no .git, 'unrecorded' then. + const uuid_tcllib = common.manifestUuid(b, "../tcllib"); + const critcl_uuid = blk: { + var gc: u8 = undefined; + const out: ?[]const u8 = b.runAllowFail(&.{ "git", "-C", b.pathFromRoot("../critcl"), "rev-parse", "HEAD" }, &gc, .ignore) catch null; + if (out != null) { + const u = std.mem.trim(u8, out.?, " \t\r\n"); + if (u.len == 40) break :blk b.dupe(u); + } + break :blk "unrecorded"; + }; + const libart_outdir = common.replaceAll(b, b.pathJoin(&.{ b.install_path, "library", "punkbin" }), "\\", "/"); + const tcllib_installed = common.replaceAll(b, b.pathJoin(&.{ b.install_path, "lib", b.fmt("tcllib{s}", .{tcllib_ver}) }), "\\", "/"); + const tcllibc_installed = common.replaceAll(b, b.pathJoin(&.{ b.install_path, "lib", "tcllibc" }), "\\", "/"); + const lib_emit = b.addSystemCommand(&.{installed_static}); + common.scrubTclEnv(lib_emit); + lib_emit.has_side_effects = true; + lib_emit.setName("library_artifacts emission"); + lib_emit.addFileArg(b.path("tools/library_artifacts.tcl")); + lib_emit.addArgs(&.{ "-outdir", libart_outdir, "-rev", b.fmt("{d}", .{librev}), "-generation", "tcl8", "-target", "win32-x86_64" }); + lib_emit.addArgs(&.{ "-tclpatch", tcl_h_patchlevel, "-suite", "suite_tcl86", "-zig", builtin.zig_version_string, "-optimize", @tagName(optimize) }); + lib_emit.addArgs(&.{ "-origin", originurl, "-packager", packager, "-project", "punkshell", "-projecturl", projecturl }); + lib_emit.addArgs(&.{ "-provenance", b.fmt("tcllib {s} critcl {s}", .{ uuid_tcllib, critcl_uuid }) }); + lib_emit.addArg("-punkzip"); + lib_emit.addArtifactArg(punkzip_exe); + lib_emit.addArgs(&.{ "-testreports", testreports_dir }); + //5-tuples: folder tier license testlibs {path} - tcllibc's [tests] + //evidence is the tcllib suite run (it IS tcllib's accelerators; the + //accel smoke above proved the bundled md5 accelerator loads) + lib_emit.addArgs(&.{ "-packages", b.fmt("tcllib{s} allplatforms BSD tcllib {{{s}}} tcllibc win32-x86_64 BSD tcllib {{{s}}}", .{ tcllib_ver, tcllib_installed, tcllibc_installed }) }); + //verified inputs only: the pure-tcl + accelerator smokes (accel_bench + //rides tcllibc_step, not this dependency chain) + lib_emit.step.dependOn(&tcllib_smoke.step); + lib_emit.step.dependOn(&tcllibc_smoke.step); + library_artifacts_step.dependOn(&lib_emit.step); + //-- test-gate: the 8.6 core testsuite under the installed static shell, //judged on PARSED totals (all.tcl exit codes lie) vs the tracked //dispositioned baseline. First census: run with -Dtestmode=record (the @@ -1010,6 +1103,8 @@ fn bootstrapMode(b: *std.Build) !void { .{ "tklib", "tklib" }, .{ "tcllib", "tcllib" }, .{ "critcl", "critcl" }, + //G-138 library-artifacts + .{ "punkzip", "punkzip" }, }; var deps_pending = false; for (source_dirs) |sd| { @@ -1067,7 +1162,7 @@ fn bootstrapMode(b: *std.Build) !void { //object caches must never be shared across zig versions; forward-slash paths //throughout: the prefix flows into -D macros as C string literals) //default mirrors suite.tcl's build default (keep the two in step) - const steps_opt = b.option([]const []const u8, "steps", "steps for the staged build (repeat the flag; default: install install-libraries smoke tklib tcllib tcllibc)") orelse @as([]const []const u8, &.{ "install", "install-libraries", "smoke", "tklib", "tcllib", "tcllibc" }); + const steps_opt = b.option([]const []const u8, "steps", "steps for the staged build (repeat the flag; default: install install-libraries smoke tklib tcllib tcllibc library-artifacts)") orelse @as([]const []const u8, &.{ "install", "install-libraries", "smoke", "tklib", "tcllib", "tcllibc", "library-artifacts" }); var cachever: []const u8 = builtin.zig_version_string; cachever = common.replaceAll(b, cachever, "+", "_"); cachever = common.replaceAll(b, cachever, "/", "_"); diff --git a/src/buildsuites/suite_tcl86/sources.config b/src/buildsuites/suite_tcl86/sources.config index 59d805a4..e60ba4d7 100644 --- a/src/buildsuites/suite_tcl86/sources.config +++ b/src/buildsuites/suite_tcl86/sources.config @@ -43,6 +43,7 @@ description Zig-built Tcl 8.6 windows runtime (core-8-6-branch): tclsh86ts/tclsh product tclsh86ts.exe static shell + tclsh86t.exe/tcl86t.dll dynamic pair with installed lib/tcl8.6 tree + tm modules (no zipfs in 8.6 - no self-contained szip runtime; kit containment is G-101 territory) product thread 2.8.13 (thread2813t.dll) and tclvfs 1.4.2 (vfs142t.dll) + registry/dde package dlls, all loadable in both shells product Tk 8.6.17 (tk86t.dll + lib/tk8.6 script library) with tklib 0.9 and tcllib 2.0 installed by their own installers, plus tcllibc critcl accelerators built with zig cc +product punkbin lib-tier library artifacts (G-138): tcllib/tcllibc -tcl8-r zips with embedded schema-v2 records - out/library/punkbin/lib product core + thread + tclvfs testsuite gates vs tracked expected_test_failures baselines (parsed all.tcl totals), tcllib/tklib record tiers and an opt-in test-tk doc README.md zigpin bin/tools/zig-x86_64-windows-0.16.0/zig.exe @@ -58,3 +59,8 @@ source tklib fossil https://core.tcl-lang.org/tklib trunk tklib source tcllib fossil https://core.tcl-lang.org/tcllib trunk tcllib source critcl git https://github.com/andreas-kupries/critcl 3.3.1 critcl + +# G-138: punkzip (punkshell-maintained hwzip fork) - deterministic zip assembly +# for the library-artifacts step. Generation-neutral (same pin as suite_tcl90); +# pinned by commit (v2.3.1; the repo carries no tags). Fetched lazily like critcl. +source punkzip git https://www.gitea1.intx.com.au/jn/punkzip 0882f0373eb2c983142e713c814b91aa82cfcea7 punkzip diff --git a/src/buildsuites/suite_tcl86/suite.tcl b/src/buildsuites/suite_tcl86/suite.tcl index ad001bb7..c8c5d758 100644 --- a/src/buildsuites/suite_tcl86/suite.tcl +++ b/src/buildsuites/suite_tcl86/suite.tcl @@ -24,10 +24,13 @@ # -seedfossils read-only seed collection for missing clones (default ~/.fossils; "" disables) # -all <0|1> with 'clean': reserved (the clone store lives outside the stage) # -steps zig build steps. 'build' default: {install install-libraries -# smoke tklib tcllib tcllibc} (runtimes + companions + Tk + -# script library + smokes, then the G-100 dependency set: -# tklib/tcllib installed by their own installers and the -# critcl-built tcllibc accelerators). +# smoke tklib tcllib tcllibc library-artifacts} (runtimes + +# companions + Tk + script library + smokes, then the G-100 +# dependency set: tklib/tcllib installed by their own +# installers and the critcl-built tcllibc accelerators, then +# the G-138 punkbin lib-tier library artifacts - tcl8 +# generation tcllib/tcllibc zips -> out/library/punkbin/lib; +# revision via -zigargs {-Dlibrev=N}). # 'test' default: {test-gate} (the 8.6 core suite gate vs the # tracked expected_test_failures baseline; first census via # -zigargs {-Dtestmode=record}). Also: test-thread, test-tclvfs, @@ -124,7 +127,7 @@ array set opt { -optimize ReleaseFast -tclbranch {} -refresh 0 - -steps {install install-libraries smoke tklib tcllib tcllibc} + -steps {install install-libraries smoke tklib tcllib tcllibc library-artifacts} -testargs {} -zigargs {} -repofolder {} @@ -313,7 +316,7 @@ foreach item {build86.zig build_common.zig build_zlib86 build_tclthread86 build_ } #-- sources --------------------------------------------------------------- -foreach required {tcl tclthread tclvfs tk tklib tcllib critcl} { +foreach required {tcl tclthread tclvfs tk tklib tcllib critcl punkzip} { if {![dict exists $sources $required]} { fail "sources.config: missing required source record '$required'" } diff --git a/src/buildsuites/suite_tcl86/tools/library_artifacts.tcl b/src/buildsuites/suite_tcl86/tools/library_artifacts.tcl new file mode 100644 index 00000000..46dd26da --- /dev/null +++ b/src/buildsuites/suite_tcl86/tools/library_artifacts.tcl @@ -0,0 +1,399 @@ +#library_artifacts.tcl (G-138, schema v2 lineage per G-123): emit punkbin +#LIB-TIER artifact zips + per-artifact toml metadata + per-tier sha1sums for the +#library packages a suite builds and verifies (tcllib pure-tcl, tcllibc critcl +#accelerators - the mechanism is package-generic: any installed-shape package +#folder can ride it). +# +#Layout emitted under -outdir (a punkbin repository mirror rooted at the lib tier): +# lib/allplatforms/--r.zip platform-neutral packages +# lib//--r.zip binary packages per platform +# beside each zip: .toml sidecar; per-tier sha1sums.txt +# (punkbin format ' *') +# +#Artifact naming: --r.zip - the generation +#tag (tcl8/tcl9) keeps the same package name distinct across Tcl generations +#(folder names collide: both generations install tcllib2.0 and tcllibc), and the +#-r assembly revision gives punkbin-immutable names (revision from the +#invocation; bump ONLY on deliberate publish - the dlls inside are NOT +#bit-reproducible across zig rebuilds, so a rebuild-and-republish under the same +#revision would violate punkbin immutability). +# +#Embedded record (embed-then-hash, the G-117 ordering rule): the record is +#written INTO the staged package copy at /punkbin-artifact.toml (package +#root, beside pkgIndex.tcl - inert to package loading) BEFORE the zip is +#assembled and hashed; the finished-zip facts (sha1, size, built) live only in +#the sidecar toml + sha1sums.txt, which remain the integrity authority. Single +#emitting actor here (unlike the runtime family's zig-staging/tcl-emission +#split), so embedded and sidecar copies are identical by construction; the +#porcelain listing of the finished zip re-verifies the embedded member. +# +#Determinism contract: punkzip 'build' walks with SORTED member order, and zip +#members carry file AND directory mtimes - so the staging copy syncs every +#file/dir mtime from the installed source tree (Tcl 'file copy' preserves file +#mtimes on windows but not unix; explicit sync makes it platform-independent), +#and the embedded record's own mtime is pinned to the package's pkgIndex.tcl so +#re-emission over an unchanged install tree reproduces byte-identical zips. +#(zig install steps rewrite files only on content change, so an unchanged +#package keeps its mtimes across no-op rebuilds.) +# +#Deliberately run UNDER A SUITE-BUILT SHELL (suite_tcl90: the plain family kit; +#suite_tcl86: the installed static shell): the sha1 digests come from the +#suite-built tcllib/tcllibc, so every emission run doubles as a proof that the +#built shell executes real tooling from the very packages being published. +# +#args: -outdir -rev -generation -target +# -tclpatch +# -suite -zig -optimize +# -origin -packager -project -projecturl +# -provenance { ...} source checkout uuids. The 'critcl' pair +# is included in a package's record only +# when its teapot.txt declares a critcl +# build (accelerator packages); other pairs +# are recorded for every package. +# -punkzip punkzip executable (deterministic zip) +# -testreports G-107 evidence summaries; optional +# -packages { ...} +# 5-tuples: installed folder name, lib-tier subfolder +# (allplatforms or a platform), license summary, comma-joined +# summary library names for the sidecar [tests] section ('' for +# none), absolute path of the installed package folder. + +proc fail {msg} {puts stderr "library_artifacts FAIL: $msg"; flush stderr; exit 1} +proc note {msg} {puts stdout "library_artifacts: $msg"; flush stdout} + +array set opt { + -outdir {} -rev 1 -generation {} -target {} -tclpatch {} + -suite {} -zig {} -optimize {} + -origin {} -packager {} -project {} -projecturl {} + -provenance {} -punkzip {} -testreports {} -packages {} +} +foreach {k v} $argv { + if {![info exists opt($k)]} {fail "unknown option '$k'"} + set opt($k) $v +} +foreach req {-outdir -generation -target -tclpatch -suite -zig -optimize -origin -packager -project -projecturl -punkzip -packages} { + if {$opt($req) eq ""} {fail "missing required option $req"} +} +if {![string is integer -strict $opt(-rev)] || $opt(-rev) < 1} {fail "-rev must be a positive integer"} +if {[llength $opt(-packages)] % 5 != 0} {fail "-packages must be a list of 5-tuples: folder tier license testlibs path"} +set punkzip [file normalize $opt(-punkzip)] +if {![file exists $punkzip]} {fail "punkzip executable not found: $punkzip"} + +if {[catch {package require sha1} sha1ver]} { + fail "package require sha1 failed under [info nameofexecutable] - the driving suite-built shell must resolve tcllib: $sha1ver" +} + +proc toml_str {s} { + #basic toml string: escape backslash and double-quote (values here are names, + #versions, uuids, iso dates - no control chars expected) + return "\"[string map {\\ \\\\ \" \\\"} $s]\"" +} + +proc zip_split {name} { + #{root ext} splitting only a .zip suffix - dotted package versions make + #[file rootname] wrong (tcllib2.0-tcl9-r1 would truncate at the version dot) + if {[string match -nocase "*.zip" $name]} { + return [list [string range $name 0 end-4] .zip] + } + return [list $name ""] +} + +proc runcap {args} { + #run a helper tool capturing combined output; fail with that output on error + if {[catch {exec {*}$args 2>@1} out]} { + fail "command failed: $args\n$out" + } + return $out +} + +proc copy_tree_synced {src dst} { + #recursive copy with every file AND directory mtime synced from the source: + #zip members carry both, so staged mtimes must be deterministic. Directories + #are synced bottom-up (writing children updates a directory's own mtime). + file mkdir $dst + set names {} + foreach pat {* .*} { + foreach n [glob -nocomplain -directory $src -tails $pat] { + if {$n in {. ..}} {continue} + if {$n ni $names} {lappend names $n} + } + } + foreach n [lsort $names] { + set s [file join $src $n] + set d [file join $dst $n] + if {[file isdirectory $s]} { + copy_tree_synced $s $d + } else { + file copy $s $d + file mtime $d [file mtime $s] + } + } + file mtime $dst [file mtime $src] +} + +proc teapot_facts {pkgdir} { + #best-effort facts from a critcl-generated teapot.txt at the package root: + #dict keys 'version' (first 'Package ' line) and 'critcl' + #(first 'Meta generated::by {critcl } ...' line). Empty dict when absent. + set facts [dict create] + set tp [file join $pkgdir teapot.txt] + if {![file exists $tp]} {return $facts} + set f [open $tp r] + set data [read $f] + close $f + foreach line [split $data \n] { + set line [string trim $line] + if {![dict exists $facts version] && [regexp {^Package\s+\S+\s+(\S+)} $line -> v]} { + dict set facts version $v + } + if {![dict exists $facts critcl] && [regexp {generated::by\s+\{critcl\s+([^\}]+)\}} $line -> cv]} { + dict set facts critcl [string trim $cv] + } + } + return $facts +} + +proc summary_test_lines {reportsdir testlibs} { + #G-107 evidence summaries -> '[tests]' toml lines, filtered to the summary + #'library' names in testlibs (result lines only; the full line-record + #summaries stay the canonical evidence artifacts) + set tlines {} + if {$reportsdir eq "" || ![file isdirectory $reportsdir] || ![llength $testlibs]} { + return $tlines + } + foreach sf [lsort [glob -nocomplain -directory $reportsdir *.summary]] { + set rec [dict create] + set f [open $sf r] + foreach line [split [read $f] \n] { + set line [string trim $line] + if {$line eq "" || [string index $line 0] eq "#"} continue + if {[catch {llength $line} n] || $n < 2} continue + dict set rec [lindex $line 0] [lrange $line 1 end] + } + close $f + if {![dict exists $rec library] || ![dict exists $rec result]} continue + set lib [dict get $rec library] + if {$lib ni $testlibs} continue + set parts [list "result=[dict get $rec result]"] + foreach fkey {mode total passed skipped failed} { + if {[dict exists $rec $fkey]} {lappend parts "$fkey=[dict get $rec $fkey]"} + } + lappend tlines "$lib = [toml_str [join $parts { }]]" + } + return $tlines +} + +proc compose_record {ident finished tests} { + #one composer for both copies (single emitting actor - identical by + #construction): 'finished' empty -> the EMBEDDED record; 'finished' a dict + #(sha1 size built) -> the sidecar, which adds those facts. 'tests' lines go + #to the sidecar only. + set emb [expr {[dict size $finished] == 0}] + set m {} + if {$emb} { + lappend m "#punkshell library artifact metadata - EMBEDDED copy (schema v2, class \"library\")," + lappend m "#written at library-artifacts emission into the staged package copy BEFORE the" + lappend m "#zip is assembled and hashed (embed-then-hash). Finished-zip facts (sha1, size," + lappend m "#built) live only in the sidecar toml + sha1sums.txt, which remain the" + lappend m "#integrity authority; this copy makes a materialized package tree self-describing." + } else { + lappend m "#punkshell library artifact metadata (schema v2, class \"library\") - generated" + lappend m "#by library_artifacts.tcl. Sidecar copy: the record embedded in the zip at" + lappend m "#/punkbin-artifact.toml plus the finished-zip facts (sha1, size, built)." + lappend m "#The sidecar + sha1sums.txt remain the integrity authority." + } + lappend m "schema = 2" + lappend m "" + lappend m "\[artifact\]" + lappend m "name = [toml_str [dict get $ident name]]" + lappend m "class = \"library\"" + lappend m "package = [toml_str [dict get $ident package]]" + lappend m "package_version = [toml_str [dict get $ident package_version]]" + lappend m "#folder: the installed-shape package folder the zip carries at its root" + lappend m "folder = [toml_str [dict get $ident folder]]" + lappend m "revision = [dict get $ident revision]" + lappend m "target = [toml_str [dict get $ident target]]" + lappend m "#tcl_generation: the Tcl major-generation the package was built/installed under" + lappend m "#(and, for binary packages, compiled against) - part of the artifact name." + lappend m "tcl_generation = [toml_str [dict get $ident tcl_generation]]" + if {!$emb} { + lappend m "sha1 = [toml_str [dict get $finished sha1]]" + lappend m "size = [dict get $finished size]" + lappend m "built = [toml_str [dict get $finished built]]" + } + lappend m "#build_id: offline correlation key re-joining a renamed copy to its record;" + lappend m "#identical in the embedded and sidecar copies (a deterministic identity digest," + lappend m "#not an integrity key - the sidecar sha1 is the integrity fact)." + lappend m "build_id = [toml_str [dict get $ident build_id]]" + lappend m "#origin: canonical artifact repo this artifact was BUILT FOR - not necessarily" + lappend m "#where it is hosted; mirrors preserve it." + lappend m "origin = [toml_str [dict get $ident origin]]" + lappend m "#packager: declared identity, not proof - signing (minisign sidecars) is the" + lappend m "#verification layer." + lappend m "packager = [toml_str [dict get $ident packager]]" + lappend m "project = [toml_str [dict get $ident project]]" + lappend m "project_url = [toml_str [dict get $ident project_url]]" + lappend m "#license: summary for the distributed package; the package's own license text" + lappend m "#rides inside the zip (license.terms etc)." + lappend m "license = [toml_str [dict get $ident license]]" + lappend m "build_host_platform = [toml_str [dict get $ident build_host_platform]]" + lappend m "" + lappend m "\[library\]" + lappend m "#driving_tcl_patchlevel: patchlevel of the suite-built shell that ran the" + lappend m "#installer/critcl - the generation witness, not a runtime version requirement." + lappend m "driving_tcl_patchlevel = [toml_str [dict get $ident tclpatch]]" + lappend m "" + lappend m "\[provenance\]" + lappend m "#class: build-origin class (schema v2): suite-built | third-party | local." + lappend m "#NOTE for line-based consumers: '\[artifact\] class' above is the first 'class ='" + lappend m "#line in the record by construction - whole-text single-key scans see that one." + lappend m "class = \"suite-built\"" + lappend m "suite = [toml_str [dict get $ident suite]]" + lappend m "toolchain = [toml_str [dict get $ident toolchain]]" + lappend m "optimize = [toml_str [dict get $ident optimize]]" + if {[dict exists $ident critcl]} { + lappend m "#critcl: version declared by the package's own critcl-generated teapot.txt" + lappend m "critcl = [toml_str [dict get $ident critcl]]" + } + foreach {n uuid} [dict get $ident provpairs] { + lappend m "${n}_checkout = [toml_str $uuid]" + } + if {!$emb && [llength $tests]} { + lappend m "" + lappend m "\[tests\]" + lappend m {*}$tests + } + return [join $m \n] +} + +set outdir [file normalize $opt(-outdir)] +set stageroot [file join $outdir .stage] +file delete -force $stageroot +file mkdir $outdir +set built [clock format [clock seconds] -format %Y-%m-%dT%H:%M:%SZ -timezone :UTC] +set sha1lines [dict create] ;#tier -> list of sha1sums lines +set emitted {} + +foreach {folder tier license testlibs pkgpath} $opt(-packages) { + set pkgpath [file normalize $pkgpath] + if {![file isdirectory $pkgpath]} {fail "package folder not found: $pkgpath"} + if {![file exists [file join $pkgpath pkgIndex.tcl]]} { + fail "$folder: no pkgIndex.tcl at $pkgpath - not an installed-shape package folder" + } + if {[file exists [file join $pkgpath punkbin-artifact.toml]]} { + fail "$folder: source tree already carries punkbin-artifact.toml ($pkgpath) - stale staging or a pre-recorded source; refusing to overwrite provenance" + } + if {[file tail $pkgpath] ne $folder} { + fail "$folder: installed folder name mismatch (path tail '[file tail $pkgpath]')" + } + + set artifact "$folder-$opt(-generation)-r$opt(-rev).zip" + lassign [zip_split $artifact] aroot _ + + #package name/version: trailing dotted version in the folder name (tcllib2.0), + #else the folder name as-is with the version from a critcl teapot.txt when + #present (tcllibc installs to an unversioned folder), else 'unrecorded'. + set tfacts [teapot_facts $pkgpath] + if {[regexp {^(.*[^0-9.])([0-9][0-9.]*)$} $folder -> pname pver]} { + } else { + set pname $folder + if {[dict exists $tfacts version]} { + set pver [dict get $tfacts version] + } else { + set pver "unrecorded" + } + } + + #provenance pairs for THIS package: the 'critcl' pair rides only with + #critcl-built packages (teapot.txt witness); other pairs are general. + set provpairs {} + foreach {n uuid} $opt(-provenance) { + if {$n eq "critcl" && ![dict exists $tfacts critcl]} {continue} + lappend provpairs $n $uuid + } + + #deterministic build_id (uuid-shaped sha1 digest of stable identity inputs - + #same input family as the runtime records: name, toolchain/optimize, driving + #patchlevel, per-member source checkout uuids) + set idseed "punkbin-artifact|$artifact|zig $opt(-zig)|$opt(-optimize)|tcl$opt(-tclpatch)|$provpairs" + set ihex [sha1::sha1 -hex $idseed] + set build_id "[string range $ihex 0 7]-[string range $ihex 8 11]-[string range $ihex 12 15]-[string range $ihex 16 19]-[string range $ihex 20 31]" + + set ident [dict create \ + name $artifact package $pname package_version $pver folder $folder \ + revision $opt(-rev) target $tier tcl_generation $opt(-generation) \ + build_id $build_id origin $opt(-origin) packager $opt(-packager) \ + project $opt(-project) project_url $opt(-projecturl) license $license \ + build_host_platform $opt(-target) tclpatch $opt(-tclpatch) \ + suite $opt(-suite) toolchain "zig $opt(-zig)" optimize $opt(-optimize) \ + provpairs $provpairs] + if {[dict exists $tfacts critcl]} { + dict set ident critcl [dict get $tfacts critcl] + } + + #stage: mtime-synced copy + embedded record, record mtime pinned to the + #package's pkgIndex.tcl (a stable per-install anchor), folder mtime re-synced + #after the write (creating the record file bumps it) + set astage [file join $stageroot $aroot] + file delete -force $astage + file mkdir $astage + copy_tree_synced $pkgpath [file join $astage $folder] + set recordtext [compose_record $ident [dict create] {}] + set recfile [file join $astage $folder punkbin-artifact.toml] + set f [open $recfile w] + fconfigure $f -translation lf + puts $f $recordtext + close $f + file mtime $recfile [file mtime [file join $pkgpath pkgIndex.tcl]] + file mtime [file join $astage $folder] [file mtime $pkgpath] + + #assemble (punkzip: recursive walk, sorted members) and re-verify the + #embedded member from the finished zip's porcelain listing + set tierdir [file join $outdir lib $tier] + file mkdir $tierdir + set dest [file join $tierdir $artifact] + file delete -force $dest + set savedpwd [pwd] + cd $astage + runcap $punkzip build $dest $folder + cd $savedpwd + set porcelain [runcap $punkzip list -porcelain $dest] + set expbytes [expr {[string length $recordtext] + 1}] ;#trailing newline (all-ASCII record) + set members 0 + set recline "" + foreach line [split $porcelain \n] { + if {[string match "m *" $line]} { + incr members + if {[string match "* $folder/punkbin-artifact.toml" $line]} {set recline $line} + } + } + if {$recline eq ""} {fail "$artifact: embedded record member $folder/punkbin-artifact.toml missing from finished zip"} + #porcelain v1 member line: m ... + if {[lindex $recline 3] != $expbytes} { + fail "$artifact: embedded record member size [lindex $recline 3] != composed record size $expbytes" + } + + set sha1 [sha1::sha1 -hex -file $dest] + set size [file size $dest] + dict lappend sha1lines $tier "$sha1 *$artifact" + + set tests [summary_test_lines $opt(-testreports) [split $testlibs ,]] + set mf [file join $tierdir "$aroot.toml"] + set f [open $mf w] + fconfigure $f -translation lf + puts $f [compose_record $ident [dict create sha1 $sha1 size $size built $built] $tests] + close $f + lappend emitted "$folder -> lib/$tier/$artifact" + note "emitted lib/$tier/$artifact (sha1 $sha1, [expr {$size/1024}] KB, $members members) + [file tail $mf]" +} + +dict for {tier lines} $sha1lines { + set f [open [file join $outdir lib $tier sha1sums.txt] w] + fconfigure $f -translation lf + puts $f [join $lines \n] + close $f + note "lib/$tier/sha1sums.txt written ([llength $lines] artifacts)" +} +file delete -force $stageroot +puts "library_artifacts OK: [join $emitted {; }] -> $outdir" +exit 0 diff --git a/src/buildsuites/suite_tcl90/README.md b/src/buildsuites/suite_tcl90/README.md index 2235c213..cd7fc26e 100644 --- a/src/buildsuites/suite_tcl90/README.md +++ b/src/buildsuites/suite_tcl90/README.md @@ -29,7 +29,7 @@ run from THIS directory with the pinned zig. Sources come from `build.zig.zon` - content-hashed per-checkin tarball pins ('refresh' = bump a pin; `zig fetch --save= ` edits the manifest). Steps for the nested staged build: `-Dsteps=...` (repeat the flag), default install install-libraries make-zipfs smoke -tklib tcllib tcllibc kit-family kit-family-artifacts. The test gate: +tklib tcllib tcllibc kit-family kit-family-artifacts library-artifacts. The test gate: `zig build bootstrap -Dsteps=test-gate` (or invoke the staged build file directly, as suite.tcl does). zig extracts fetched packages into `zig-pkg/` beside the manifest and `.zig-cache/` for the outer @@ -166,6 +166,48 @@ and referenced by `src/runtime/mapvfs.config` (punk -> punk9wintk905.vfs as `punk9_beta`, punk-bi -> punk9win_for_tkruntime.vfs as `punk9bi_beta`, per the *_beta trial convention). +## Library artifacts (G-138) + +The `library-artifacts` step (default; `tools/library_artifacts.tcl`) emits the +punkbin LIB tier into `out/library/punkbin/lib/`: the library packages this +suite builds and verifies, as immutable `-r` zips of the installed-shape +package folders - + +- `lib/allplatforms/tcllib2.0-tcl9-r1.zip` - platform-neutral (pure-tcl) +- `lib/win32-x86_64/tcllibc-tcl9-r1.zip` - per-platform binary (critcl + accelerators) + +plus per-artifact `.toml` sidecars and per-tier punkbin-format +`sha1sums.txt`. Names carry the GENERATION tag (`-tcl9-`; suite_tcl86 emits +`-tcl8-`): both generations install identically-named folders (tcllib2.0, +tcllibc), so the tag keeps the artifacts distinct server-side. Revision via +`-Dlibrev=N` (suite.tcl `-zigargs {-Dlibrev=N}`; default 1). + +Each zip embeds a schema-v2 `punkbin-artifact.toml` at the package root (beside +pkgIndex.tcl - inert to package loading; `[artifact] class = "library"`, +`[provenance] class = "suite-built"` with tcllib/critcl checkout uuids, critcl +version from the package's own teapot.txt, driving-shell patchlevel, toolchain) +written BEFORE zipping - embed-then-hash: the finished-zip facts (sha1, size, +built) live only in the sidecar + sha1sums.txt, the integrity authority. + +Zips are assembled with punkzip (staged upstream checkout - the `punkzip` +record in sources.config / build.zig.zon, compiled for the build host): +deterministic sorted member order, with staged file/dir mtimes synced from the +install tree and the embedded record's mtime pinned to the package's +pkgIndex.tcl - re-emission over an unchanged install reproduces BYTE-IDENTICAL +zips (verified). The dlls inside are NOT bit-reproducible across zig rebuilds +(recorded suite finding), hence the publish discipline: `-Dlibrev` bumps only +on deliberate publish, never per rebuild. + +The emission runs UNDER the plain family kit (depends on its self-containment +check plus the tcllib/tcllibc smokes) - the kit's attached tcllib/tcllibc hash +the very zips being published. md5c is deliberately NOT published: tcllib's md5 +accelerator resolution prefers tcllibc (modules/md5/md5x.tcl), which bundles +the md5 accelerator - a standalone md5c adds nothing (G-138 disposition). +Publication to the real punkbin repo remains a deliberate user step (copy +`out/library/punkbin/lib/*` into the punkbin repo's `lib/` tier + +`build_sha1sums.tcl` there). + ## Pinned zig The suite pin is **zig 0.16.0** (current official release; adopted 2026-07-20 - @@ -203,7 +245,8 @@ producing stale/duplicate artifacts at link time). and the suite-built-shell step scripts `test_gate.tcl` / `suite_smoke.tcl` / `pkg_smoke.tcl` (G-102) plus `family_check.tcl` / `family_artifacts.tcl` (G-103: kit-family self-containment verification; punkbin-layout artifact + - metadata emission). `test_gate.tcl` is the shared testsuite engine (G-107): + metadata emission) and `library_artifacts.tcl` (G-138: punkbin lib-tier + emission). `test_gate.tcl` is the shared testsuite engine (G-107): gate/record modes, driver selection, evidence summaries. - `expected_test_failures.txt` (tcl core) / `expected_test_failures_thread.txt` / `expected_test_failures_tclvfs.txt` - tracked dispositioned gate baselines. diff --git a/src/buildsuites/suite_tcl90/build.zig.zon b/src/buildsuites/suite_tcl90/build.zig.zon index 676473b7..994e60bd 100644 --- a/src/buildsuites/suite_tcl90/build.zig.zon +++ b/src/buildsuites/suite_tcl90/build.zig.zon @@ -63,6 +63,13 @@ .hash = "N-V-__8AAF8-bABDlxChmtnyalKgofVfN3kzcZdmZ7pjEmE0", .lazy = true, }, + //punkzip v2.3.1 (commit 0882f0373e; no upstream tags) - deterministic zip + //assembler for library-artifacts (G-138); punkshell-maintained hwzip fork + .punkzip = .{ + .url = "https://www.gitea1.intx.com.au/jn/punkzip/archive/0882f0373eb2c983142e713c814b91aa82cfcea7.tar.gz", + .hash = "punkzip-2.3.1-PjFoDUNlgwCyYfz8k0-SNGxaXsh53YWTXv5f9JRRXQK8", + .lazy = true, + }, }, .paths = .{""}, } diff --git a/src/buildsuites/suite_tcl90/build905.zig b/src/buildsuites/suite_tcl90/build905.zig index 78299c3b..78667210 100644 --- a/src/buildsuites/suite_tcl90/build905.zig +++ b/src/buildsuites/suite_tcl90/build905.zig @@ -2595,6 +2595,72 @@ pub fn build(b: *std.Build) !void { emit.addArgs(&.{ "-kits", b.fmt("plain {{{s}}} punk {{{s}}} punk-bi {{{s}}}", .{ family_installed[0], family_installed[1], family_installed[2] }) }); for (family_checks) |cs| emit.step.dependOn(cs); family_artifacts_step.dependOn(&emit.step); + + //-- library-artifacts: punkbin LIB-TIER emission (G-138) - the library + //packages this suite builds and verifies (tcllib pure-tcl -> + //lib/allplatforms, tcllibc critcl accelerators -> lib/win32-x86_64) + //as immutable -r zips with embedded schema-v2 class=library records + //(the G-123 lineage), toml sidecars and per-tier sha1sums. Zips are + //assembled with punkzip (sorted members; staged mtimes synced from the + //install tree) so re-emission over an unchanged install is + //byte-identical - but the dlls INSIDE are not bit-reproducible across + //zig rebuilds, so -Dlibrev bumps ONLY on deliberate publish (never per + //rebuild; default 1 - nothing published yet). Run UNDER THE PLAIN + //FAMILY KIT (kit-family-artifacts' proof-doubling rationale: the kit's + //attached tcllib/tcllibc hash the very zips being published). + //Publication to the real punkbin repo stays a deliberate user step. + const library_artifacts_step = b.step("library-artifacts", "emit punkbin lib-tier artifacts (-tcl9-r.zip + toml + sha1sums) from the verified tcllib/tcllibc installs -> /library/punkbin/lib"); + const librev = b.option(u32, "librev", "assembly revision N for the -r library artifact names (library-artifacts; default 1 - nothing published yet)") orelse 1; + //punkzip: staged upstream checkout (sources.config 'punkzip' record / + //build.zig.zon pin), compiled for the build host - a build tool, not a + //target artifact. ReleaseSafe deliberately: it does real deflate work + //over multi-MB package trees (Debug-mode compression is needlessly slow). + const punkzip_exe = b.addExecutable(.{ + .name = "punkzip", + .root_module = b.createModule(.{ + .root_source_file = b.path("../punkzip/src/punkzip.zig"), + .target = b.graph.host, + .optimize = .ReleaseSafe, + }), + }); + //critcl checkout commit: best-effort git probe (the packager-identity + //pattern above); bootstrap-flow tarballs carry no .git - 'unrecorded' + //then, matching the suite.tcl manifest.uuid note for pin-fetched trees. + const critcl_uuid = blk: { + var gc: u8 = undefined; + const out: ?[]const u8 = b.runAllowFail(&.{ "git", "-C", b.pathFromRoot("../critcl"), "rev-parse", "HEAD" }, &gc, .ignore) catch null; + if (out != null) { + const u = std.mem.trim(u8, out.?, " \t\r\n"); + if (u.len == 40) break :blk b.dupe(u); + } + break :blk "unrecorded"; + }; + const libart_outdir = common.replaceAll(b, b.pathJoin(&.{ b.install_path, "library", "punkbin" }), "\\", "/"); + const tcllib_installed = common.replaceAll(b, b.pathJoin(&.{ b.install_path, "lib", b.fmt("tcllib{s}", .{tcllib_ver}) }), "\\", "/"); + const tcllibc_installed = common.replaceAll(b, b.pathJoin(&.{ b.install_path, "lib", "tcllibc" }), "\\", "/"); + const lib_emit = b.addSystemCommand(&.{family_installed[0]}); + common.scrubTclEnv(lib_emit); + lib_emit.has_side_effects = true; + lib_emit.setName("library_artifacts emission"); + lib_emit.addFileArg(b.path("tools/library_artifacts.tcl")); + lib_emit.addArgs(&.{ "-outdir", libart_outdir, "-rev", b.fmt("{d}", .{librev}), "-generation", "tcl9", "-target", "win32-x86_64" }); + lib_emit.addArgs(&.{ "-tclpatch", tcl_h_patchlevel, "-suite", "suite_tcl90", "-zig", builtin.zig_version_string, "-optimize", @tagName(optimize) }); + lib_emit.addArgs(&.{ "-origin", originurl, "-packager", packager, "-project", "punkshell", "-projecturl", projecturl }); + lib_emit.addArgs(&.{ "-provenance", b.fmt("tcllib {s} critcl {s}", .{ uuid_tcllib, critcl_uuid }) }); + lib_emit.addArg("-punkzip"); + lib_emit.addArtifactArg(punkzip_exe); + lib_emit.addArgs(&.{ "-testreports", testreports_dir }); + //5-tuples: folder tier license testlibs {path} - tcllibc's [tests] + //evidence is the tcllib suite run (it IS tcllib's accelerators; the + //accel smoke at build proved the bundled md5 accelerator loads) + lib_emit.addArgs(&.{ "-packages", b.fmt("tcllib{s} allplatforms BSD tcllib {{{s}}} tcllibc win32-x86_64 BSD tcllib {{{s}}}", .{ tcllib_ver, tcllib_installed, tcllibc_installed }) }); + //verified inputs only: the pure-tcl + accelerator smokes, and the + //plain family kit's own self-containment check (the emission runs + //under that kit) + lib_emit.step.dependOn(&tcllib_smoke.step); + lib_emit.step.dependOn(&tcllibc_smoke.step); + lib_emit.step.dependOn(family_checks[0]); + library_artifacts_step.dependOn(&lib_emit.step); } // ================== @@ -2757,6 +2823,7 @@ fn bootstrapMode(b: *std.Build) !void { .{ "tklib", "tklib" }, .{ "tcllib", "tcllib" }, .{ "critcl", "critcl" }, + .{ "punkzip", "punkzip" }, }; var deps_pending = false; for (source_dirs) |sd| { @@ -2817,7 +2884,7 @@ fn bootstrapMode(b: *std.Build) !void { //suite.tcl performs after its fossil staging. Per-zig-version cache dir (object //caches must never be shared across zig versions - user-confirmed hazard), same //name suite.tcl derives so both flows share the staged cache. - const steps_opt = b.option([]const []const u8, "steps", "steps for the staged build (repeat the flag; default: install install-libraries make-zipfs smoke tklib tcllib tcllibc kit-family kit-family-artifacts - the full pipeline short of test-gate)") orelse @as([]const []const u8, &.{ "install", "install-libraries", "make-zipfs", "smoke", "tklib", "tcllib", "tcllibc", "kit-family", "kit-family-artifacts" }); + const steps_opt = b.option([]const []const u8, "steps", "steps for the staged build (repeat the flag; default: install install-libraries make-zipfs smoke tklib tcllib tcllibc kit-family kit-family-artifacts library-artifacts - the full pipeline short of test-gate)") orelse @as([]const []const u8, &.{ "install", "install-libraries", "make-zipfs", "smoke", "tklib", "tcllib", "tcllibc", "kit-family", "kit-family-artifacts", "library-artifacts" }); var cachever: []const u8 = builtin.zig_version_string; cachever = common.replaceAll(b, cachever, "+", "_"); cachever = common.replaceAll(b, cachever, "/", "_"); diff --git a/src/buildsuites/suite_tcl90/sources.config b/src/buildsuites/suite_tcl90/sources.config index 6c2e2476..c560c502 100644 --- a/src/buildsuites/suite_tcl90/sources.config +++ b/src/buildsuites/suite_tcl90/sources.config @@ -53,6 +53,7 @@ description Zig-built Tcl 9.0 windows runtime family: core + thread/tclvfs/Tk + tcllib/tcllibc, core-test gate, punk kit-family artifacts product tclsh9.0.x runtime family (plain / punk / punk-bi) with attached prefix-mirror image - out/family plus punkbin-layout -r artifacts with toml metadata product installed lib tree: thread, tclvfs, Tk, tklib, tcllib (+ tcllibc critcl accelerators) +product punkbin lib-tier library artifacts (G-138): tcllib/tcllibc -tcl9-r zips with embedded schema-v2 records - out/library/punkbin/lib product core testsuite gate vs the tracked expected_test_failures baseline (parsed all.tcl totals) doc README.md zigpin bin/tools/zig-x86_64-windows-0.16.0/zig.exe @@ -72,3 +73,8 @@ source tklib fossil https://core.tcl-lang.org/tklib trunk tklib source tcllib fossil https://core.tcl-lang.org/tcllib trunk tcllib source critcl git https://github.com/andreas-kupries/critcl 3.3.1 critcl + +# G-138: punkzip (punkshell-maintained hwzip fork) - deterministic zip assembly +# for the library-artifacts step. Pinned by commit (v2.3.1; the repo carries no +# tags). Fetched lazily like critcl - only when library artifacts are emitted. +source punkzip git https://www.gitea1.intx.com.au/jn/punkzip 0882f0373eb2c983142e713c814b91aa82cfcea7 punkzip diff --git a/src/buildsuites/suite_tcl90/suite.tcl b/src/buildsuites/suite_tcl90/suite.tcl index 46dd7971..7d450e1e 100644 --- a/src/buildsuites/suite_tcl90/suite.tcl +++ b/src/buildsuites/suite_tcl90/suite.tcl @@ -23,11 +23,14 @@ # -all <0|1> with 'clean': reserved (the clone store lives outside the stage) # -steps zig build steps. 'build' default: {install install-libraries # make-zipfs smoke tklib tcllib tcllibc kit-family -# kit-family-artifacts} - the full pipeline short of the test -# steps, ending in the G-103 runtime kit family (verified -# self-contained plain/punk/punk-bi shells -> out/family plus -# punkbin-layout -r artifacts + toml metadata; artifact -# revision via -zigargs {-Dfamilyrev=N}). +# kit-family-artifacts library-artifacts} - the full pipeline +# short of the test steps: the G-103 runtime kit family +# (verified self-contained plain/punk/punk-bi shells -> +# out/family plus punkbin-layout -r artifacts + toml +# metadata; artifact revision via -zigargs {-Dfamilyrev=N}) +# then the G-138 punkbin lib-tier library artifacts +# (tcllib/tcllibc zips -> out/library/punkbin/lib; revision +# via -zigargs {-Dlibrev=N}). # 'test' default: {test-gate} (the # core suite gate); other test steps (G-107 library runs): # test-libraries (thread+tclvfs gated, tcllib+tklib recorded), @@ -137,7 +140,7 @@ array set opt { -optimize ReleaseFast -tclbranch {} -refresh 0 - -steps {install install-libraries make-zipfs smoke tklib tcllib tcllibc kit-family kit-family-artifacts} + -steps {install install-libraries make-zipfs smoke tklib tcllib tcllibc kit-family kit-family-artifacts library-artifacts} -testargs {} -zigargs {} -repofolder {} @@ -334,7 +337,7 @@ foreach item {build905.zig build_common.zig build_libtommath build_zlib build_tc #(sources.config already loaded near the top - see load_sources_config there; #$sources holds the source records, $suiteconfig the full record set incl. the #G-104 self-description records) -foreach required {tcl tclthread tclvfs tk tklib tcllib critcl} { +foreach required {tcl tclthread tclvfs tk tklib tcllib critcl punkzip} { if {![dict exists $sources $required]} { fail "sources.config: missing required source record '$required'" } diff --git a/src/buildsuites/suite_tcl90/tools/library_artifacts.tcl b/src/buildsuites/suite_tcl90/tools/library_artifacts.tcl new file mode 100644 index 00000000..46dd26da --- /dev/null +++ b/src/buildsuites/suite_tcl90/tools/library_artifacts.tcl @@ -0,0 +1,399 @@ +#library_artifacts.tcl (G-138, schema v2 lineage per G-123): emit punkbin +#LIB-TIER artifact zips + per-artifact toml metadata + per-tier sha1sums for the +#library packages a suite builds and verifies (tcllib pure-tcl, tcllibc critcl +#accelerators - the mechanism is package-generic: any installed-shape package +#folder can ride it). +# +#Layout emitted under -outdir (a punkbin repository mirror rooted at the lib tier): +# lib/allplatforms/--r.zip platform-neutral packages +# lib//--r.zip binary packages per platform +# beside each zip: .toml sidecar; per-tier sha1sums.txt +# (punkbin format ' *') +# +#Artifact naming: --r.zip - the generation +#tag (tcl8/tcl9) keeps the same package name distinct across Tcl generations +#(folder names collide: both generations install tcllib2.0 and tcllibc), and the +#-r assembly revision gives punkbin-immutable names (revision from the +#invocation; bump ONLY on deliberate publish - the dlls inside are NOT +#bit-reproducible across zig rebuilds, so a rebuild-and-republish under the same +#revision would violate punkbin immutability). +# +#Embedded record (embed-then-hash, the G-117 ordering rule): the record is +#written INTO the staged package copy at /punkbin-artifact.toml (package +#root, beside pkgIndex.tcl - inert to package loading) BEFORE the zip is +#assembled and hashed; the finished-zip facts (sha1, size, built) live only in +#the sidecar toml + sha1sums.txt, which remain the integrity authority. Single +#emitting actor here (unlike the runtime family's zig-staging/tcl-emission +#split), so embedded and sidecar copies are identical by construction; the +#porcelain listing of the finished zip re-verifies the embedded member. +# +#Determinism contract: punkzip 'build' walks with SORTED member order, and zip +#members carry file AND directory mtimes - so the staging copy syncs every +#file/dir mtime from the installed source tree (Tcl 'file copy' preserves file +#mtimes on windows but not unix; explicit sync makes it platform-independent), +#and the embedded record's own mtime is pinned to the package's pkgIndex.tcl so +#re-emission over an unchanged install tree reproduces byte-identical zips. +#(zig install steps rewrite files only on content change, so an unchanged +#package keeps its mtimes across no-op rebuilds.) +# +#Deliberately run UNDER A SUITE-BUILT SHELL (suite_tcl90: the plain family kit; +#suite_tcl86: the installed static shell): the sha1 digests come from the +#suite-built tcllib/tcllibc, so every emission run doubles as a proof that the +#built shell executes real tooling from the very packages being published. +# +#args: -outdir -rev -generation -target +# -tclpatch +# -suite -zig -optimize +# -origin -packager -project -projecturl +# -provenance { ...} source checkout uuids. The 'critcl' pair +# is included in a package's record only +# when its teapot.txt declares a critcl +# build (accelerator packages); other pairs +# are recorded for every package. +# -punkzip punkzip executable (deterministic zip) +# -testreports G-107 evidence summaries; optional +# -packages { ...} +# 5-tuples: installed folder name, lib-tier subfolder +# (allplatforms or a platform), license summary, comma-joined +# summary library names for the sidecar [tests] section ('' for +# none), absolute path of the installed package folder. + +proc fail {msg} {puts stderr "library_artifacts FAIL: $msg"; flush stderr; exit 1} +proc note {msg} {puts stdout "library_artifacts: $msg"; flush stdout} + +array set opt { + -outdir {} -rev 1 -generation {} -target {} -tclpatch {} + -suite {} -zig {} -optimize {} + -origin {} -packager {} -project {} -projecturl {} + -provenance {} -punkzip {} -testreports {} -packages {} +} +foreach {k v} $argv { + if {![info exists opt($k)]} {fail "unknown option '$k'"} + set opt($k) $v +} +foreach req {-outdir -generation -target -tclpatch -suite -zig -optimize -origin -packager -project -projecturl -punkzip -packages} { + if {$opt($req) eq ""} {fail "missing required option $req"} +} +if {![string is integer -strict $opt(-rev)] || $opt(-rev) < 1} {fail "-rev must be a positive integer"} +if {[llength $opt(-packages)] % 5 != 0} {fail "-packages must be a list of 5-tuples: folder tier license testlibs path"} +set punkzip [file normalize $opt(-punkzip)] +if {![file exists $punkzip]} {fail "punkzip executable not found: $punkzip"} + +if {[catch {package require sha1} sha1ver]} { + fail "package require sha1 failed under [info nameofexecutable] - the driving suite-built shell must resolve tcllib: $sha1ver" +} + +proc toml_str {s} { + #basic toml string: escape backslash and double-quote (values here are names, + #versions, uuids, iso dates - no control chars expected) + return "\"[string map {\\ \\\\ \" \\\"} $s]\"" +} + +proc zip_split {name} { + #{root ext} splitting only a .zip suffix - dotted package versions make + #[file rootname] wrong (tcllib2.0-tcl9-r1 would truncate at the version dot) + if {[string match -nocase "*.zip" $name]} { + return [list [string range $name 0 end-4] .zip] + } + return [list $name ""] +} + +proc runcap {args} { + #run a helper tool capturing combined output; fail with that output on error + if {[catch {exec {*}$args 2>@1} out]} { + fail "command failed: $args\n$out" + } + return $out +} + +proc copy_tree_synced {src dst} { + #recursive copy with every file AND directory mtime synced from the source: + #zip members carry both, so staged mtimes must be deterministic. Directories + #are synced bottom-up (writing children updates a directory's own mtime). + file mkdir $dst + set names {} + foreach pat {* .*} { + foreach n [glob -nocomplain -directory $src -tails $pat] { + if {$n in {. ..}} {continue} + if {$n ni $names} {lappend names $n} + } + } + foreach n [lsort $names] { + set s [file join $src $n] + set d [file join $dst $n] + if {[file isdirectory $s]} { + copy_tree_synced $s $d + } else { + file copy $s $d + file mtime $d [file mtime $s] + } + } + file mtime $dst [file mtime $src] +} + +proc teapot_facts {pkgdir} { + #best-effort facts from a critcl-generated teapot.txt at the package root: + #dict keys 'version' (first 'Package ' line) and 'critcl' + #(first 'Meta generated::by {critcl } ...' line). Empty dict when absent. + set facts [dict create] + set tp [file join $pkgdir teapot.txt] + if {![file exists $tp]} {return $facts} + set f [open $tp r] + set data [read $f] + close $f + foreach line [split $data \n] { + set line [string trim $line] + if {![dict exists $facts version] && [regexp {^Package\s+\S+\s+(\S+)} $line -> v]} { + dict set facts version $v + } + if {![dict exists $facts critcl] && [regexp {generated::by\s+\{critcl\s+([^\}]+)\}} $line -> cv]} { + dict set facts critcl [string trim $cv] + } + } + return $facts +} + +proc summary_test_lines {reportsdir testlibs} { + #G-107 evidence summaries -> '[tests]' toml lines, filtered to the summary + #'library' names in testlibs (result lines only; the full line-record + #summaries stay the canonical evidence artifacts) + set tlines {} + if {$reportsdir eq "" || ![file isdirectory $reportsdir] || ![llength $testlibs]} { + return $tlines + } + foreach sf [lsort [glob -nocomplain -directory $reportsdir *.summary]] { + set rec [dict create] + set f [open $sf r] + foreach line [split [read $f] \n] { + set line [string trim $line] + if {$line eq "" || [string index $line 0] eq "#"} continue + if {[catch {llength $line} n] || $n < 2} continue + dict set rec [lindex $line 0] [lrange $line 1 end] + } + close $f + if {![dict exists $rec library] || ![dict exists $rec result]} continue + set lib [dict get $rec library] + if {$lib ni $testlibs} continue + set parts [list "result=[dict get $rec result]"] + foreach fkey {mode total passed skipped failed} { + if {[dict exists $rec $fkey]} {lappend parts "$fkey=[dict get $rec $fkey]"} + } + lappend tlines "$lib = [toml_str [join $parts { }]]" + } + return $tlines +} + +proc compose_record {ident finished tests} { + #one composer for both copies (single emitting actor - identical by + #construction): 'finished' empty -> the EMBEDDED record; 'finished' a dict + #(sha1 size built) -> the sidecar, which adds those facts. 'tests' lines go + #to the sidecar only. + set emb [expr {[dict size $finished] == 0}] + set m {} + if {$emb} { + lappend m "#punkshell library artifact metadata - EMBEDDED copy (schema v2, class \"library\")," + lappend m "#written at library-artifacts emission into the staged package copy BEFORE the" + lappend m "#zip is assembled and hashed (embed-then-hash). Finished-zip facts (sha1, size," + lappend m "#built) live only in the sidecar toml + sha1sums.txt, which remain the" + lappend m "#integrity authority; this copy makes a materialized package tree self-describing." + } else { + lappend m "#punkshell library artifact metadata (schema v2, class \"library\") - generated" + lappend m "#by library_artifacts.tcl. Sidecar copy: the record embedded in the zip at" + lappend m "#/punkbin-artifact.toml plus the finished-zip facts (sha1, size, built)." + lappend m "#The sidecar + sha1sums.txt remain the integrity authority." + } + lappend m "schema = 2" + lappend m "" + lappend m "\[artifact\]" + lappend m "name = [toml_str [dict get $ident name]]" + lappend m "class = \"library\"" + lappend m "package = [toml_str [dict get $ident package]]" + lappend m "package_version = [toml_str [dict get $ident package_version]]" + lappend m "#folder: the installed-shape package folder the zip carries at its root" + lappend m "folder = [toml_str [dict get $ident folder]]" + lappend m "revision = [dict get $ident revision]" + lappend m "target = [toml_str [dict get $ident target]]" + lappend m "#tcl_generation: the Tcl major-generation the package was built/installed under" + lappend m "#(and, for binary packages, compiled against) - part of the artifact name." + lappend m "tcl_generation = [toml_str [dict get $ident tcl_generation]]" + if {!$emb} { + lappend m "sha1 = [toml_str [dict get $finished sha1]]" + lappend m "size = [dict get $finished size]" + lappend m "built = [toml_str [dict get $finished built]]" + } + lappend m "#build_id: offline correlation key re-joining a renamed copy to its record;" + lappend m "#identical in the embedded and sidecar copies (a deterministic identity digest," + lappend m "#not an integrity key - the sidecar sha1 is the integrity fact)." + lappend m "build_id = [toml_str [dict get $ident build_id]]" + lappend m "#origin: canonical artifact repo this artifact was BUILT FOR - not necessarily" + lappend m "#where it is hosted; mirrors preserve it." + lappend m "origin = [toml_str [dict get $ident origin]]" + lappend m "#packager: declared identity, not proof - signing (minisign sidecars) is the" + lappend m "#verification layer." + lappend m "packager = [toml_str [dict get $ident packager]]" + lappend m "project = [toml_str [dict get $ident project]]" + lappend m "project_url = [toml_str [dict get $ident project_url]]" + lappend m "#license: summary for the distributed package; the package's own license text" + lappend m "#rides inside the zip (license.terms etc)." + lappend m "license = [toml_str [dict get $ident license]]" + lappend m "build_host_platform = [toml_str [dict get $ident build_host_platform]]" + lappend m "" + lappend m "\[library\]" + lappend m "#driving_tcl_patchlevel: patchlevel of the suite-built shell that ran the" + lappend m "#installer/critcl - the generation witness, not a runtime version requirement." + lappend m "driving_tcl_patchlevel = [toml_str [dict get $ident tclpatch]]" + lappend m "" + lappend m "\[provenance\]" + lappend m "#class: build-origin class (schema v2): suite-built | third-party | local." + lappend m "#NOTE for line-based consumers: '\[artifact\] class' above is the first 'class ='" + lappend m "#line in the record by construction - whole-text single-key scans see that one." + lappend m "class = \"suite-built\"" + lappend m "suite = [toml_str [dict get $ident suite]]" + lappend m "toolchain = [toml_str [dict get $ident toolchain]]" + lappend m "optimize = [toml_str [dict get $ident optimize]]" + if {[dict exists $ident critcl]} { + lappend m "#critcl: version declared by the package's own critcl-generated teapot.txt" + lappend m "critcl = [toml_str [dict get $ident critcl]]" + } + foreach {n uuid} [dict get $ident provpairs] { + lappend m "${n}_checkout = [toml_str $uuid]" + } + if {!$emb && [llength $tests]} { + lappend m "" + lappend m "\[tests\]" + lappend m {*}$tests + } + return [join $m \n] +} + +set outdir [file normalize $opt(-outdir)] +set stageroot [file join $outdir .stage] +file delete -force $stageroot +file mkdir $outdir +set built [clock format [clock seconds] -format %Y-%m-%dT%H:%M:%SZ -timezone :UTC] +set sha1lines [dict create] ;#tier -> list of sha1sums lines +set emitted {} + +foreach {folder tier license testlibs pkgpath} $opt(-packages) { + set pkgpath [file normalize $pkgpath] + if {![file isdirectory $pkgpath]} {fail "package folder not found: $pkgpath"} + if {![file exists [file join $pkgpath pkgIndex.tcl]]} { + fail "$folder: no pkgIndex.tcl at $pkgpath - not an installed-shape package folder" + } + if {[file exists [file join $pkgpath punkbin-artifact.toml]]} { + fail "$folder: source tree already carries punkbin-artifact.toml ($pkgpath) - stale staging or a pre-recorded source; refusing to overwrite provenance" + } + if {[file tail $pkgpath] ne $folder} { + fail "$folder: installed folder name mismatch (path tail '[file tail $pkgpath]')" + } + + set artifact "$folder-$opt(-generation)-r$opt(-rev).zip" + lassign [zip_split $artifact] aroot _ + + #package name/version: trailing dotted version in the folder name (tcllib2.0), + #else the folder name as-is with the version from a critcl teapot.txt when + #present (tcllibc installs to an unversioned folder), else 'unrecorded'. + set tfacts [teapot_facts $pkgpath] + if {[regexp {^(.*[^0-9.])([0-9][0-9.]*)$} $folder -> pname pver]} { + } else { + set pname $folder + if {[dict exists $tfacts version]} { + set pver [dict get $tfacts version] + } else { + set pver "unrecorded" + } + } + + #provenance pairs for THIS package: the 'critcl' pair rides only with + #critcl-built packages (teapot.txt witness); other pairs are general. + set provpairs {} + foreach {n uuid} $opt(-provenance) { + if {$n eq "critcl" && ![dict exists $tfacts critcl]} {continue} + lappend provpairs $n $uuid + } + + #deterministic build_id (uuid-shaped sha1 digest of stable identity inputs - + #same input family as the runtime records: name, toolchain/optimize, driving + #patchlevel, per-member source checkout uuids) + set idseed "punkbin-artifact|$artifact|zig $opt(-zig)|$opt(-optimize)|tcl$opt(-tclpatch)|$provpairs" + set ihex [sha1::sha1 -hex $idseed] + set build_id "[string range $ihex 0 7]-[string range $ihex 8 11]-[string range $ihex 12 15]-[string range $ihex 16 19]-[string range $ihex 20 31]" + + set ident [dict create \ + name $artifact package $pname package_version $pver folder $folder \ + revision $opt(-rev) target $tier tcl_generation $opt(-generation) \ + build_id $build_id origin $opt(-origin) packager $opt(-packager) \ + project $opt(-project) project_url $opt(-projecturl) license $license \ + build_host_platform $opt(-target) tclpatch $opt(-tclpatch) \ + suite $opt(-suite) toolchain "zig $opt(-zig)" optimize $opt(-optimize) \ + provpairs $provpairs] + if {[dict exists $tfacts critcl]} { + dict set ident critcl [dict get $tfacts critcl] + } + + #stage: mtime-synced copy + embedded record, record mtime pinned to the + #package's pkgIndex.tcl (a stable per-install anchor), folder mtime re-synced + #after the write (creating the record file bumps it) + set astage [file join $stageroot $aroot] + file delete -force $astage + file mkdir $astage + copy_tree_synced $pkgpath [file join $astage $folder] + set recordtext [compose_record $ident [dict create] {}] + set recfile [file join $astage $folder punkbin-artifact.toml] + set f [open $recfile w] + fconfigure $f -translation lf + puts $f $recordtext + close $f + file mtime $recfile [file mtime [file join $pkgpath pkgIndex.tcl]] + file mtime [file join $astage $folder] [file mtime $pkgpath] + + #assemble (punkzip: recursive walk, sorted members) and re-verify the + #embedded member from the finished zip's porcelain listing + set tierdir [file join $outdir lib $tier] + file mkdir $tierdir + set dest [file join $tierdir $artifact] + file delete -force $dest + set savedpwd [pwd] + cd $astage + runcap $punkzip build $dest $folder + cd $savedpwd + set porcelain [runcap $punkzip list -porcelain $dest] + set expbytes [expr {[string length $recordtext] + 1}] ;#trailing newline (all-ASCII record) + set members 0 + set recline "" + foreach line [split $porcelain \n] { + if {[string match "m *" $line]} { + incr members + if {[string match "* $folder/punkbin-artifact.toml" $line]} {set recline $line} + } + } + if {$recline eq ""} {fail "$artifact: embedded record member $folder/punkbin-artifact.toml missing from finished zip"} + #porcelain v1 member line: m ... + if {[lindex $recline 3] != $expbytes} { + fail "$artifact: embedded record member size [lindex $recline 3] != composed record size $expbytes" + } + + set sha1 [sha1::sha1 -hex -file $dest] + set size [file size $dest] + dict lappend sha1lines $tier "$sha1 *$artifact" + + set tests [summary_test_lines $opt(-testreports) [split $testlibs ,]] + set mf [file join $tierdir "$aroot.toml"] + set f [open $mf w] + fconfigure $f -translation lf + puts $f [compose_record $ident [dict create sha1 $sha1 size $size built $built] $tests] + close $f + lappend emitted "$folder -> lib/$tier/$artifact" + note "emitted lib/$tier/$artifact (sha1 $sha1, [expr {$size/1024}] KB, $members members) + [file tail $mf]" +} + +dict for {tier lines} $sha1lines { + set f [open [file join $outdir lib $tier sha1sums.txt] w] + fconfigure $f -translation lf + puts $f [join $lines \n] + close $f + note "lib/$tier/sha1sums.txt written ([llength $lines] artifacts)" +} +file delete -force $stageroot +puts "library_artifacts OK: [join $emitted {; }] -> $outdir" +exit 0