diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ab1b400c2e..5158b39117 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -356,7 +356,7 @@ jobs: - name: untar build run: tar xzvf coatjava.tar.gz - name: hipo2npz - run: ./coatjava/bin/hipo2npz rec.hipo rec.npz RUN::config,REC::Event,REC::Particle + run: ./coatjava/bin/hipo2npz rec.hipo rec.npz - name: hipo2npz-dump run: ./coatjava/bin/hipo2npz-dump rec.npz 1 diff --git a/bin/hipo2npz-diff b/bin/hipo2npz-diff new file mode 100755 index 0000000000..95f606cdee --- /dev/null +++ b/bin/hipo2npz-diff @@ -0,0 +1,132 @@ +#!/usr/bin/env python3 + +###################################### +# author: generated by Claude Sonnet 5 +###################################### + +""" +hipo2npz-diff - Compare two NumPy .npz files and report differences. + +Usage: + hipo2npz-diff file1.npz file2.npz + hipo2npz-diff file1.npz file2.npz --rtol 1e-5 --atol 1e-8 + hipo2npz-diff file1.npz file2.npz --exact + hipo2npz-diff file1.npz file2.npz --verbose + +Exit codes: + 0 - files are equivalent (given tolerance) + 1 - differences found + 2 - error (bad file, etc.) +""" + +import argparse +import sys + +import numpy as np + + +def diff_npz(path_a, path_b, rtol, atol, exact, verbose): + try: + a = np.load(path_a, allow_pickle=True) + b = np.load(path_b, allow_pickle=True) + except Exception as e: + print(f"Error loading files: {e}", file=sys.stderr) + sys.exit(2) + + keys_a = set(a.files) + keys_b = set(b.files) + + only_in_a = sorted(keys_a - keys_b) + only_in_b = sorted(keys_b - keys_a) + common = sorted(keys_a & keys_b) + + has_diff = False + + if only_in_a: + has_diff = True + print(f"Keys only in {path_a}:") + for k in only_in_a: + print(f" - {k}") + + if only_in_b: + has_diff = True + print(f"Keys only in {path_b}:") + for k in only_in_b: + print(f" - {k}") + + for key in common: + arr_a, arr_b = a[key], b[key] + + if arr_a.shape != arr_b.shape: + has_diff = True + print(f"[{key}] shape mismatch: {arr_a.shape} vs {arr_b.shape}") + continue + + if arr_a.dtype != arr_b.dtype and verbose: + print(f"[{key}] dtype differs: {arr_a.dtype} vs {arr_b.dtype}") + + is_float = np.issubdtype(arr_a.dtype, np.floating) + + try: + if exact: + # equal_nan=True: NaNs in the same position count as matching, + # since NaN != NaN by IEEE rules but that's rarely what you want here + equal = np.array_equal(arr_a, arr_b, equal_nan=is_float) + else: + equal = np.allclose(arr_a, arr_b, rtol=rtol, atol=atol, equal_nan=True) + except TypeError: + # Non-numeric / object arrays: fall back to plain equality (no equal_nan support) + equal = np.array_equal(arr_a, arr_b) + + if not equal: + has_diff = True + print(f"[{key}] values differ", end="") + try: + fa = arr_a.astype(np.float64) + fb = arr_b.astype(np.float64) + diff = np.abs(fa - fb) + + # Positions where exactly one side is NaN (a "real" mismatch, not just + # matching NaNs) vs. positions with a finite numeric difference + nan_mismatch = np.isnan(fa) != np.isnan(fb) + finite_mask = ~np.isnan(fa) & ~np.isnan(fb) + + n_diff = np.count_nonzero( + finite_mask & (diff > (atol + rtol * np.abs(fb))) + ) + n_nan_mismatch = np.count_nonzero(nan_mismatch) + + max_diff = np.max(diff[finite_mask]) if finite_mask.any() else 0.0 + print( + f" (max abs diff = {max_diff:.6g}, " + f"{n_diff}/{arr_a.size} elements differ, " + f"{n_nan_mismatch} NaN-mismatch positions)" + ) + except (TypeError, ValueError): + print() + elif verbose: + print(f"[{key}] OK (identical within tolerance)") + + if not has_diff: + print(f"No differences found between {path_a} and {path_b}" + + ("" if exact else f" (rtol={rtol}, atol={atol})")) + + return has_diff + + +def main(): + parser = argparse.ArgumentParser(description="Diff two .npz files.") + parser.add_argument("file_a", help="First .npz file") + parser.add_argument("file_b", help="Second .npz file") + parser.add_argument("--rtol", type=float, default=1e-5, help="Relative tolerance for float comparison (default: 1e-5)") + parser.add_argument("--atol", type=float, default=1e-8, help="Absolute tolerance for float comparison (default: 1e-8)") + parser.add_argument("--exact", action="store_true", help="Require exact equality instead of tolerance-based comparison") + parser.add_argument("--verbose", action="store_true", help="Print status for matching keys too") + args = parser.parse_args() + + has_diff = diff_npz(args.file_a, args.file_b, args.rtol, args.atol, args.exact, args.verbose) + sys.exit(1 if has_diff else 0) + + +if __name__ == "__main__": + main() diff --git a/common-tools/clas-io/src/main/java/org/jlab/io/hipo/Hipo2Npz.java b/common-tools/clas-io/src/main/java/org/jlab/io/hipo/Hipo2Npz.java index ba026c0943..04ed2b77a4 100644 --- a/common-tools/clas-io/src/main/java/org/jlab/io/hipo/Hipo2Npz.java +++ b/common-tools/clas-io/src/main/java/org/jlab/io/hipo/Hipo2Npz.java @@ -1,16 +1,20 @@ package org.jlab.io.hipo; +import java.io.BufferedInputStream; import java.io.BufferedOutputStream; import java.io.ByteArrayOutputStream; +import java.io.Closeable; import java.io.File; +import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; +import java.io.UncheckedIOException; import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.nio.charset.StandardCharsets; import java.nio.file.Files; +import java.nio.file.Path; import java.util.ArrayList; -import java.util.Arrays; import java.util.Comparator; import java.util.HashMap; import java.util.LinkedHashMap; @@ -18,7 +22,6 @@ import java.util.List; import java.util.Map; import java.util.Set; -import java.util.zip.CRC32; import java.util.zip.Deflater; import java.util.zip.ZipEntry; import java.util.zip.ZipOutputStream; @@ -82,7 +85,7 @@ public static void main(String[] args) throws Exception { } Converter converter = new Converter(options.selectedBanks, schemaTypes); - converter.convert(options.input, options.output); + converter.convert(options.input, options.output, options.numEvents, options.firstEvent); } // ------------------------------------------------------------------------ @@ -93,12 +96,16 @@ private static final class CliOptions { final File input; final File output; final File schemaDir; + final long numEvents; + final long firstEvent; final Set selectedBanks; // null means all banks - private CliOptions(File input, File output, File schemaDir, Set selectedBanks) { - this.input = input; - this.output = output; - this.schemaDir = schemaDir; + private CliOptions(File input, File output, File schemaDir, long numEvents, long firstEvent, Set selectedBanks) { + this.input = input; + this.output = output; + this.schemaDir = schemaDir; + this.numEvents = numEvents; + this.firstEvent = firstEvent; this.selectedBanks = selectedBanks; } @@ -107,10 +114,11 @@ static CliOptions parse(String[] args) throws Exception { printUsageAndExit(); } - File input = new File(args[0]); - File output = new File(args[1]); - - File schemaDir = null; + File input = new File(args[0]); + File output = new File(args[1]); + File schemaDir = null; + long numEvents = 0; + long firstEvent = 0; Set selectedBanks = new LinkedHashSet<>(); boolean selectAll = true; @@ -138,6 +146,22 @@ static CliOptions parse(String[] args) throws Exception { continue; } + if ("--num-events".equals(arg)) { + if (i + 1 >= args.length) { + throw new IllegalArgumentException("--num-events requires a number"); + } + numEvents = Long.parseLong(args[++i]); + continue; + } + + if ("--first-event".equals(arg)) { + if (i + 1 >= args.length) { + throw new IllegalArgumentException("--first-event requires a number"); + } + firstEvent = Long.parseLong(args[++i]); + continue; + } + if ("*".equals(arg)) { selectedBanks.clear(); selectAll = true; @@ -159,11 +183,12 @@ static CliOptions parse(String[] args) throws Exception { if (!schemaDir.isDirectory()) { throw new IllegalArgumentException("Schema directory not found: " + schemaDir.getAbsolutePath()); } + if (!input.exists()) { throw new IllegalArgumentException("Input HIPO file not found: " + input.getAbsolutePath()); } - return new CliOptions(input, output, schemaDir, selectAll ? null : selectedBanks); + return new CliOptions(input, output, schemaDir, numEvents, firstEvent, selectAll ? null : selectedBanks); } private static Set readBankNames(File bankFile) throws IOException { @@ -192,6 +217,9 @@ private static void printUsageAndExit() { System.err.println(" --bank-file FILE a file with one bank name per line, '#' comments allowed;"); System.err.println(" both comma list and `--bank-file` may be used together"); System.err.println(" --schema-dir DIR use a custom schema directory"); + System.err.println(" default: the one included with this coatjava installation"); + System.err.println(" --num-events NUM process this many events (default: all)"); + System.err.println(" --first-event NUM start from this event (default: 0)"); System.err.println(""); System.err.println("EXAMPLES:"); System.err.println("* hipo2npz input.hipo output.npz"); @@ -206,17 +234,19 @@ private static void printUsageAndExit() { // ------------------------------------------------------------------------ private enum ColumnType { - BYTE(" banks = new LinkedHashMap<>(); private final Set selectedBanks; // null means all banks private final Map schemaTypes; // BANK/COLUMN -> type + private Path tmpDir; + private Thread cleanupHook; Converter(Set selectedBanks, Map schemaTypes) { this.selectedBanks = selectedBanks; - this.schemaTypes = schemaTypes; + this.schemaTypes = schemaTypes; } - void convert(File input, File output) throws Exception { - if (selectedBanks == null) { - System.out.println("Including all banks"); - } else { - System.out.println("Including selected banks only"); - } + void convert(File input, File output, long numEvents, long firstEvent) throws Exception { + System.out.println(selectedBanks==null ? "Including all banks" : "Including selected banks only"); + + File tmpDirFile = new File(output.getPath() + ".tmp"); + tmpDir = createRunDir(tmpDirFile); + cleanupHook = new Thread(() -> deleteRecursively(tmpDir)); + Runtime.getRuntime().addShutdownHook(cleanupHook); HipoDataSource reader = new HipoDataSource(); reader.open(input); - long nEvents = 0; - while (reader.hasEvent()) { - DataEvent event = reader.getNextEvent(); - nEvents++; - ingestEvent(event); - - if ((nEvents % 10000) == 0) { - System.out.printf("Processed %,d events%n", nEvents); + long nevRead = 0; + long nevProc = 0; + try { + try { + while (reader.hasEvent()) { + DataEvent event = reader.getNextEvent(); + nevRead++; + if (nevRead <= firstEvent) continue; + ingestEvent(event); + nevProc++; + if ((nevProc % 10000) == 0) System.out.printf("Processed %,d events%n", nevProc); + if (numEvents > 0 && nevProc >= numEvents) break; + } + } finally { + reader.close(); + } + System.out.println("Writing NPZ file..."); + writeNpz(output); + } finally { + closeAllQuietly(); + deleteRecursively(tmpDir); + try { + Runtime.getRuntime().removeShutdownHook(cleanupHook); + } catch (IllegalStateException ignored) { + // JVM is already shutting down — the hook itself will run deleteRecursively } } - reader.close(); - writeNpz(output); - System.out.printf("Wrote %s with %,d events and %,d banks%n", - output.getAbsolutePath(), nEvents, banks.size()); + System.out.printf("Wrote %s with %,d events and %,d banks%n", output.getAbsolutePath(), nevProc, banks.size()); } private boolean keepBank(String bankName) { return selectedBanks == null || selectedBanks.contains(bankName); } - private void ingestEvent(DataEvent event) { + private void ingestEvent(DataEvent event) throws IOException { String[] bankNames = event.getBankList(); if (bankNames == null) { return; @@ -302,7 +349,7 @@ private void ingestEvent(DataEvent event) { int rows = bank.rows(); presentRows.put(bankName, rows); - BankStore store = banks.computeIfAbsent(bankName, BankStore::new); + BankStore store = banks.computeIfAbsent(bankName, name -> new BankStore(name, tmpDir.toFile())); ensureColumns(store, bank); String[] cols = bank.getColumnList(); @@ -325,7 +372,7 @@ private void ingestEvent(DataEvent event) { } } - private void ensureColumns(BankStore store, DataBank bank) { + private void ensureColumns(BankStore store, DataBank bank) throws IOException { String[] cols = bank.getColumnList(); if (cols == null) { return; @@ -336,7 +383,7 @@ private void ensureColumns(BankStore store, DataBank bank) { continue; } ColumnType type = discoverColumnType(bank, col); - store.columns.put(col, new ColumnStore(store.bankName, col, type)); + store.columns.put(col, new ColumnStore(col, type, tmpDir.toFile())); } } @@ -449,35 +496,86 @@ private void writeNpz(File output) throws IOException { zos.setLevel(Deflater.BEST_SPEED); for (BankStore bank : banks.values()) { + bank.close(); // flush + close the rowsPerEvent/offsets temp-file streams + String bankBase = sanitize(bank.bankName); - addEntry(zos, bankBase + "__rows_per_event.npy", - Npy.writeIntArray(bank.rowsPerEvent.toArray(), " p.toFile().delete()); + } catch (IOException | UncheckedIOException ignored) { + // best-effort; nothing more we can do here + } + } } // ------------------------------------------------------------------------ @@ -582,71 +680,94 @@ static Map loadSchemaTypes(Set schemaFiles) throws IOE } // ------------------------------------------------------------------------ - // Stores + // BankStore // ------------------------------------------------------------------------ - private static final class BankStore { + private static final class BankStore implements Closeable { final String bankName; final Map columns = new LinkedHashMap<>(); - final IntList rowsPerEvent = new IntList(); - final LongList offsets = new LongList(); + final File rowsPerEventFile; + final File offsetsFile; + private final BufferedOutputStream rowsPerEventOut; + private final BufferedOutputStream offsetsOut; + private final ByteBuffer scratch4 = ByteBuffer.allocate(4).order(ByteOrder.LITTLE_ENDIAN); + private final ByteBuffer scratch8 = ByteBuffer.allocate(8).order(ByteOrder.LITTLE_ENDIAN); + long nEvents = 0; long totalRows = 0; - BankStore(String bankName) { + BankStore(String bankName, File tmpDir) { this.bankName = bankName; - this.offsets.add(0L); + try { + // create rows per event temp file + this.rowsPerEventFile = File.createTempFile("hipo2npz_rpe_", ".bin", tmpDir); + this.rowsPerEventOut = new BufferedOutputStream(new FileOutputStream(rowsPerEventFile), 1 << 16); + // create offsets file + this.offsetsFile = File.createTempFile("hipo2npz_off_", ".bin", tmpDir); + this.offsetsOut = new BufferedOutputStream(new FileOutputStream(offsetsFile), 1 << 16); + writeOffset(0L); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + void appendEventRows(int rows) throws IOException { + scratch4.clear(); + scratch4.putInt(rows); + rowsPerEventOut.write(scratch4.array(), 0, 4); + nEvents++; + totalRows += rows; + writeOffset(totalRows); } - void appendEventRows(int rows) { - rowsPerEvent.add(rows); - totalRows += rows; - offsets.add(totalRows); + private void writeOffset(long value) throws IOException { + scratch8.clear(); + scratch8.putLong(value); + offsetsOut.write(scratch8.array(), 0, 8); + } + + @Override + public void close() throws IOException { + rowsPerEventOut.close(); + offsetsOut.close(); } } - private static final class ColumnStore { - final String bankName; + // ------------------------------------------------------------------------ + // ColumnStore + // ------------------------------------------------------------------------ + + private static final class ColumnStore implements Closeable { final String columnName; final ColumnType type; - final ByteList bytes; - final ShortList shorts; - final IntList ints; - final LongList longs; - final FloatList floats; - final DoubleList doubles; - - ColumnStore(String bankName, String columnName, ColumnType type) { - this.bankName = bankName; + final File tempFile; + private final BufferedOutputStream out; + private final ByteBuffer scratch = ByteBuffer.allocate(8).order(ByteOrder.LITTLE_ENDIAN); + long count = 0; + + ColumnStore(String columnName, ColumnType type, File tmpDir) throws IOException { this.columnName = columnName; this.type = type; - this.bytes = type == ColumnType.BYTE ? new ByteList() : null; - this.shorts = type == ColumnType.SHORT ? new ShortList() : null; - this.ints = type == ColumnType.INT ? new IntList() : null; - this.longs = type == ColumnType.LONG ? new LongList() : null; - this.floats = type == ColumnType.FLOAT ? new FloatList() : null; - this.doubles = type == ColumnType.DOUBLE ? new DoubleList() : null; + this.tempFile = File.createTempFile("hipo2npz_col_", ".bin", tmpDir); + this.out = new BufferedOutputStream(new FileOutputStream(tempFile), 1 << 16); } - void append(DataBank bank, String col, int row) { + void append(DataBank bank, String col, int row) throws IOException { + scratch.clear(); switch (type) { - case BYTE -> bytes.add(bank.getByte(col, row)); - case SHORT -> shorts.add(bank.getShort(col, row)); - case INT -> ints.add(bank.getInt(col, row)); - case LONG -> longs.add(bank.getLong(col, row)); - case FLOAT -> floats.add(bank.getFloat(col, row)); - case DOUBLE -> doubles.add(bank.getDouble(col, row)); - } - } - - byte[] toNpyBytes() throws IOException { - return switch (type) { - case BYTE -> Npy.writeByteArray(bytes.toArray(), type.npyDescr); - case SHORT -> Npy.writeShortArray(shorts.toArray(), type.npyDescr); - case INT -> Npy.writeIntArray(ints.toArray(), type.npyDescr); - case LONG -> Npy.writeLongArray(longs.toArray(), type.npyDescr); - case FLOAT -> Npy.writeFloatArray(floats.toArray(), type.npyDescr); - case DOUBLE -> Npy.writeDoubleArray(doubles.toArray(), type.npyDescr); - }; + case BYTE -> scratch.put(bank.getByte(col, row)); + case SHORT -> scratch.putShort(bank.getShort(col, row)); + case INT -> scratch.putInt(bank.getInt(col, row)); + case LONG -> scratch.putLong(bank.getLong(col, row)); + case FLOAT -> scratch.putFloat(bank.getFloat(col, row)); + case DOUBLE -> scratch.putDouble(bank.getDouble(col, row)); + } + out.write(scratch.array(), 0, type.byteWidth); + count++; + } + + @Override + public void close() throws IOException { + out.close(); } } @@ -657,59 +778,12 @@ byte[] toNpyBytes() throws IOException { private static final class Npy { private static final byte[] MAGIC = {(byte) 0x93, 'N', 'U', 'M', 'P', 'Y'}; - static byte[] writeByteArray(byte[] values, String descr) throws IOException { - ByteArrayOutputStream out = new ByteArrayOutputStream(); - writeHeader(out, descr, values.length); - out.write(values); - return out.toByteArray(); - } - - static byte[] writeShortArray(short[] values, String descr) throws IOException { - ByteArrayOutputStream out = new ByteArrayOutputStream(); - writeHeader(out, descr, values.length); - ByteBuffer bb = ByteBuffer.allocate(values.length * 2).order(ByteOrder.LITTLE_ENDIAN); - for (short v : values) bb.putShort(v); - out.write(bb.array()); - return out.toByteArray(); - } - - static byte[] writeIntArray(int[] values, String descr) throws IOException { - ByteArrayOutputStream out = new ByteArrayOutputStream(); - writeHeader(out, descr, values.length); - ByteBuffer bb = ByteBuffer.allocate(values.length * 4).order(ByteOrder.LITTLE_ENDIAN); - for (int v : values) bb.putInt(v); - out.write(bb.array()); - return out.toByteArray(); - } - - static byte[] writeLongArray(long[] values, String descr) throws IOException { - ByteArrayOutputStream out = new ByteArrayOutputStream(); - writeHeader(out, descr, values.length); - ByteBuffer bb = ByteBuffer.allocate(values.length * 8).order(ByteOrder.LITTLE_ENDIAN); - for (long v : values) bb.putLong(v); - out.write(bb.array()); - return out.toByteArray(); - } - - static byte[] writeFloatArray(float[] values, String descr) throws IOException { + /** + * Builds just the NPY header bytes for an array of the given dtype and length. + * The actual array data is streamed separately from a temp file. + */ + static byte[] buildHeader(String descr, long length) throws IOException { ByteArrayOutputStream out = new ByteArrayOutputStream(); - writeHeader(out, descr, values.length); - ByteBuffer bb = ByteBuffer.allocate(values.length * 4).order(ByteOrder.LITTLE_ENDIAN); - for (float v : values) bb.putFloat(v); - out.write(bb.array()); - return out.toByteArray(); - } - - static byte[] writeDoubleArray(double[] values, String descr) throws IOException { - ByteArrayOutputStream out = new ByteArrayOutputStream(); - writeHeader(out, descr, values.length); - ByteBuffer bb = ByteBuffer.allocate(values.length * 8).order(ByteOrder.LITTLE_ENDIAN); - for (double v : values) bb.putDouble(v); - out.write(bb.array()); - return out.toByteArray(); - } - - private static void writeHeader(ByteArrayOutputStream out, String descr, int length) throws IOException { out.write(MAGIC); out.write(1); out.write(0); @@ -728,58 +802,8 @@ private static void writeHeader(ByteArrayOutputStream out, String descr, int len hlen.putShort((short) fullHeaderBytes.length); out.write(hlen.array()); out.write(fullHeaderBytes); - } - } - - // ------------------------------------------------------------------------ - // Primitive dynamic arrays - // ------------------------------------------------------------------------ - private static final class ByteList { - private byte[] data = new byte[1024]; - private int size = 0; - void add(byte v) { ensure(size + 1); data[size++] = v; } - byte[] toArray() { return Arrays.copyOf(data, size); } - private void ensure(int n) { if (n > data.length) data = Arrays.copyOf(data, Math.max(n, data.length * 2)); } - } - - private static final class ShortList { - private short[] data = new short[1024]; - private int size = 0; - void add(short v) { ensure(size + 1); data[size++] = v; } - short[] toArray() { return Arrays.copyOf(data, size); } - private void ensure(int n) { if (n > data.length) data = Arrays.copyOf(data, Math.max(n, data.length * 2)); } - } - - private static final class IntList { - private int[] data = new int[1024]; - private int size = 0; - void add(int v) { ensure(size + 1); data[size++] = v; } - int[] toArray() { return Arrays.copyOf(data, size); } - private void ensure(int n) { if (n > data.length) data = Arrays.copyOf(data, Math.max(n, data.length * 2)); } - } - - private static final class LongList { - private long[] data = new long[1024]; - private int size = 0; - void add(long v) { ensure(size + 1); data[size++] = v; } - long[] toArray() { return Arrays.copyOf(data, size); } - private void ensure(int n) { if (n > data.length) data = Arrays.copyOf(data, Math.max(n, data.length * 2)); } - } - - private static final class FloatList { - private float[] data = new float[1024]; - private int size = 0; - void add(float v) { ensure(size + 1); data[size++] = v; } - float[] toArray() { return Arrays.copyOf(data, size); } - private void ensure(int n) { if (n > data.length) data = Arrays.copyOf(data, Math.max(n, data.length * 2)); } - } - - private static final class DoubleList { - private double[] data = new double[1024]; - private int size = 0; - void add(double v) { ensure(size + 1); data[size++] = v; } - double[] toArray() { return Arrays.copyOf(data, size); } - private void ensure(int n) { if (n > data.length) data = Arrays.copyOf(data, Math.max(n, data.length * 2)); } + return out.toByteArray(); + } } } diff --git a/etc/data/nnet b/etc/data/nnet index 577aac477b..73aa4341c7 160000 --- a/etc/data/nnet +++ b/etc/data/nnet @@ -1 +1 @@ -Subproject commit 577aac477bd87dacafb4bbf18f4787c7e59d1bae +Subproject commit 73aa4341c739175950def77252a018549d1411b0