From efef34db53bc97d542f49f8c04e49136fd640c98 Mon Sep 17 00:00:00 2001 From: Christopher Dilks Date: Mon, 3 Aug 2026 15:53:51 -0400 Subject: [PATCH 1/6] perf: `hipo2npz` memory efficiency --- .../main/java/org/jlab/io/hipo/Hipo2Npz.java | 340 +++++++++--------- 1 file changed, 163 insertions(+), 177 deletions(-) diff --git a/common-tools/clas-io/src/main/java/org/jlab/io/hipo/Hipo2Npz.java b/common-tools/clas-io/src/main/java/org/jlab/io/hipo/Hipo2Npz.java index ba026c0943..4a5f4b3fca 100644 --- a/common-tools/clas-io/src/main/java/org/jlab/io/hipo/Hipo2Npz.java +++ b/common-tools/clas-io/src/main/java/org/jlab/io/hipo/Hipo2Npz.java @@ -1,16 +1,19 @@ package org.jlab.io.hipo; +import java.io.BufferedInputStream; import java.io.BufferedOutputStream; import java.io.ByteArrayOutputStream; +import java.io.Closeable; import java.io.File; +import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; +import java.io.UncheckedIOException; import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.util.ArrayList; -import java.util.Arrays; import java.util.Comparator; import java.util.HashMap; import java.util.LinkedHashMap; @@ -18,7 +21,6 @@ import java.util.List; import java.util.Map; import java.util.Set; -import java.util.zip.CRC32; import java.util.zip.Deflater; import java.util.zip.ZipEntry; import java.util.zip.ZipOutputStream; @@ -206,17 +208,19 @@ private static void printUsageAndExit() { // ------------------------------------------------------------------------ private enum ColumnType { - BYTE(" loadSchemaTypes(Set schemaFiles) throws IOE } // ------------------------------------------------------------------------ - // Stores + // BankStore // ------------------------------------------------------------------------ - private static final class BankStore { + private static final class BankStore implements Closeable { final String bankName; final Map columns = new LinkedHashMap<>(); - final IntList rowsPerEvent = new IntList(); - final LongList offsets = new LongList(); + final File rowsPerEventFile; + final File offsetsFile; + private final BufferedOutputStream rowsPerEventOut; + private final BufferedOutputStream offsetsOut; + private final ByteBuffer scratch4 = ByteBuffer.allocate(4).order(ByteOrder.LITTLE_ENDIAN); + private final ByteBuffer scratch8 = ByteBuffer.allocate(8).order(ByteOrder.LITTLE_ENDIAN); + long nEvents = 0; long totalRows = 0; BankStore(String bankName) { this.bankName = bankName; - this.offsets.add(0L); + try { + // create rows per event temp file + this.rowsPerEventFile = File.createTempFile("hipo2npz_rpe_", ".bin"); + this.rowsPerEventFile.deleteOnExit(); + this.rowsPerEventOut = new BufferedOutputStream(new FileOutputStream(rowsPerEventFile), 1 << 16); + // create offsets file + this.offsetsFile = File.createTempFile("hipo2npz_off_", ".bin"); + this.offsetsFile.deleteOnExit(); + this.offsetsOut = new BufferedOutputStream(new FileOutputStream(offsetsFile), 1 << 16); + writeOffset(0L); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + void appendEventRows(int rows) throws IOException { + scratch4.clear(); + scratch4.putInt(rows); + rowsPerEventOut.write(scratch4.array(), 0, 4); + nEvents++; + totalRows += rows; + writeOffset(totalRows); } - void appendEventRows(int rows) { - rowsPerEvent.add(rows); - totalRows += rows; - offsets.add(totalRows); + private void writeOffset(long value) throws IOException { + scratch8.clear(); + scratch8.putLong(value); + offsetsOut.write(scratch8.array(), 0, 8); + } + + @Override + public void close() throws IOException { + rowsPerEventOut.close(); + offsetsOut.close(); } } - private static final class ColumnStore { + // ------------------------------------------------------------------------ + // ColumnStore + // ------------------------------------------------------------------------ + + private static final class ColumnStore implements Closeable { final String bankName; final String columnName; final ColumnType type; - final ByteList bytes; - final ShortList shorts; - final IntList ints; - final LongList longs; - final FloatList floats; - final DoubleList doubles; - - ColumnStore(String bankName, String columnName, ColumnType type) { + final File tempFile; + private final BufferedOutputStream out; + private final ByteBuffer scratch = ByteBuffer.allocate(8).order(ByteOrder.LITTLE_ENDIAN); + long count = 0; + + ColumnStore(String bankName, String columnName, ColumnType type) throws IOException { this.bankName = bankName; this.columnName = columnName; this.type = type; - this.bytes = type == ColumnType.BYTE ? new ByteList() : null; - this.shorts = type == ColumnType.SHORT ? new ShortList() : null; - this.ints = type == ColumnType.INT ? new IntList() : null; - this.longs = type == ColumnType.LONG ? new LongList() : null; - this.floats = type == ColumnType.FLOAT ? new FloatList() : null; - this.doubles = type == ColumnType.DOUBLE ? new DoubleList() : null; + this.tempFile = File.createTempFile("hipo2npz_col_", ".bin"); + this.tempFile.deleteOnExit(); + this.out = new BufferedOutputStream(new FileOutputStream(tempFile), 1 << 16); } - void append(DataBank bank, String col, int row) { + void append(DataBank bank, String col, int row) throws IOException { switch (type) { - case BYTE -> bytes.add(bank.getByte(col, row)); - case SHORT -> shorts.add(bank.getShort(col, row)); - case INT -> ints.add(bank.getInt(col, row)); - case LONG -> longs.add(bank.getLong(col, row)); - case FLOAT -> floats.add(bank.getFloat(col, row)); - case DOUBLE -> doubles.add(bank.getDouble(col, row)); - } - } - - byte[] toNpyBytes() throws IOException { - return switch (type) { - case BYTE -> Npy.writeByteArray(bytes.toArray(), type.npyDescr); - case SHORT -> Npy.writeShortArray(shorts.toArray(), type.npyDescr); - case INT -> Npy.writeIntArray(ints.toArray(), type.npyDescr); - case LONG -> Npy.writeLongArray(longs.toArray(), type.npyDescr); - case FLOAT -> Npy.writeFloatArray(floats.toArray(), type.npyDescr); - case DOUBLE -> Npy.writeDoubleArray(doubles.toArray(), type.npyDescr); - }; + case BYTE -> out.write(bank.getByte(col, row)); + case SHORT -> { + scratch.clear(); + scratch.putShort(bank.getShort(col, row)); + out.write(scratch.array(), 0, 2); + } + case INT -> { + scratch.clear(); + scratch.putInt(bank.getInt(col, row)); + out.write(scratch.array(), 0, 4); + } + case LONG -> { + scratch.clear(); + scratch.putLong(bank.getLong(col, row)); + out.write(scratch.array(), 0, 8); + } + case FLOAT -> { + scratch.clear(); + scratch.putFloat(bank.getFloat(col, row)); + out.write(scratch.array(), 0, 4); + } + case DOUBLE -> { + scratch.clear(); + scratch.putDouble(bank.getDouble(col, row)); + out.write(scratch.array(), 0, 8); + } + } + count++; + } + + @Override + public void close() throws IOException { + out.close(); } } @@ -657,59 +740,12 @@ byte[] toNpyBytes() throws IOException { private static final class Npy { private static final byte[] MAGIC = {(byte) 0x93, 'N', 'U', 'M', 'P', 'Y'}; - static byte[] writeByteArray(byte[] values, String descr) throws IOException { - ByteArrayOutputStream out = new ByteArrayOutputStream(); - writeHeader(out, descr, values.length); - out.write(values); - return out.toByteArray(); - } - - static byte[] writeShortArray(short[] values, String descr) throws IOException { - ByteArrayOutputStream out = new ByteArrayOutputStream(); - writeHeader(out, descr, values.length); - ByteBuffer bb = ByteBuffer.allocate(values.length * 2).order(ByteOrder.LITTLE_ENDIAN); - for (short v : values) bb.putShort(v); - out.write(bb.array()); - return out.toByteArray(); - } - - static byte[] writeIntArray(int[] values, String descr) throws IOException { - ByteArrayOutputStream out = new ByteArrayOutputStream(); - writeHeader(out, descr, values.length); - ByteBuffer bb = ByteBuffer.allocate(values.length * 4).order(ByteOrder.LITTLE_ENDIAN); - for (int v : values) bb.putInt(v); - out.write(bb.array()); - return out.toByteArray(); - } - - static byte[] writeLongArray(long[] values, String descr) throws IOException { + /** + * Builds just the NPY header bytes for an array of the given dtype and length. + * The actual array data is streamed separately from a temp file. + */ + static byte[] buildHeader(String descr, long length) throws IOException { ByteArrayOutputStream out = new ByteArrayOutputStream(); - writeHeader(out, descr, values.length); - ByteBuffer bb = ByteBuffer.allocate(values.length * 8).order(ByteOrder.LITTLE_ENDIAN); - for (long v : values) bb.putLong(v); - out.write(bb.array()); - return out.toByteArray(); - } - - static byte[] writeFloatArray(float[] values, String descr) throws IOException { - ByteArrayOutputStream out = new ByteArrayOutputStream(); - writeHeader(out, descr, values.length); - ByteBuffer bb = ByteBuffer.allocate(values.length * 4).order(ByteOrder.LITTLE_ENDIAN); - for (float v : values) bb.putFloat(v); - out.write(bb.array()); - return out.toByteArray(); - } - - static byte[] writeDoubleArray(double[] values, String descr) throws IOException { - ByteArrayOutputStream out = new ByteArrayOutputStream(); - writeHeader(out, descr, values.length); - ByteBuffer bb = ByteBuffer.allocate(values.length * 8).order(ByteOrder.LITTLE_ENDIAN); - for (double v : values) bb.putDouble(v); - out.write(bb.array()); - return out.toByteArray(); - } - - private static void writeHeader(ByteArrayOutputStream out, String descr, int length) throws IOException { out.write(MAGIC); out.write(1); out.write(0); @@ -728,58 +764,8 @@ private static void writeHeader(ByteArrayOutputStream out, String descr, int len hlen.putShort((short) fullHeaderBytes.length); out.write(hlen.array()); out.write(fullHeaderBytes); - } - } - - // ------------------------------------------------------------------------ - // Primitive dynamic arrays - // ------------------------------------------------------------------------ - private static final class ByteList { - private byte[] data = new byte[1024]; - private int size = 0; - void add(byte v) { ensure(size + 1); data[size++] = v; } - byte[] toArray() { return Arrays.copyOf(data, size); } - private void ensure(int n) { if (n > data.length) data = Arrays.copyOf(data, Math.max(n, data.length * 2)); } - } - - private static final class ShortList { - private short[] data = new short[1024]; - private int size = 0; - void add(short v) { ensure(size + 1); data[size++] = v; } - short[] toArray() { return Arrays.copyOf(data, size); } - private void ensure(int n) { if (n > data.length) data = Arrays.copyOf(data, Math.max(n, data.length * 2)); } - } - - private static final class IntList { - private int[] data = new int[1024]; - private int size = 0; - void add(int v) { ensure(size + 1); data[size++] = v; } - int[] toArray() { return Arrays.copyOf(data, size); } - private void ensure(int n) { if (n > data.length) data = Arrays.copyOf(data, Math.max(n, data.length * 2)); } - } - - private static final class LongList { - private long[] data = new long[1024]; - private int size = 0; - void add(long v) { ensure(size + 1); data[size++] = v; } - long[] toArray() { return Arrays.copyOf(data, size); } - private void ensure(int n) { if (n > data.length) data = Arrays.copyOf(data, Math.max(n, data.length * 2)); } - } - - private static final class FloatList { - private float[] data = new float[1024]; - private int size = 0; - void add(float v) { ensure(size + 1); data[size++] = v; } - float[] toArray() { return Arrays.copyOf(data, size); } - private void ensure(int n) { if (n > data.length) data = Arrays.copyOf(data, Math.max(n, data.length * 2)); } - } - - private static final class DoubleList { - private double[] data = new double[1024]; - private int size = 0; - void add(double v) { ensure(size + 1); data[size++] = v; } - double[] toArray() { return Arrays.copyOf(data, size); } - private void ensure(int n) { if (n > data.length) data = Arrays.copyOf(data, Math.max(n, data.length * 2)); } + return out.toByteArray(); + } } } From a97dd0620d6b21e6f25781b53ac14426c0444645 Mon Sep 17 00:00:00 2001 From: Christopher Dilks Date: Mon, 3 Aug 2026 17:31:46 -0400 Subject: [PATCH 2/6] feat: `--tmp-dir` option and some cleanup --- .../main/java/org/jlab/io/hipo/Hipo2Npz.java | 106 +++++++++--------- 1 file changed, 50 insertions(+), 56 deletions(-) diff --git a/common-tools/clas-io/src/main/java/org/jlab/io/hipo/Hipo2Npz.java b/common-tools/clas-io/src/main/java/org/jlab/io/hipo/Hipo2Npz.java index 4a5f4b3fca..ab0ac11728 100644 --- a/common-tools/clas-io/src/main/java/org/jlab/io/hipo/Hipo2Npz.java +++ b/common-tools/clas-io/src/main/java/org/jlab/io/hipo/Hipo2Npz.java @@ -83,7 +83,7 @@ public static void main(String[] args) throws Exception { } } - Converter converter = new Converter(options.selectedBanks, schemaTypes); + Converter converter = new Converter(options.selectedBanks, schemaTypes, options.tmpDir); converter.convert(options.input, options.output); } @@ -95,12 +95,14 @@ private static final class CliOptions { final File input; final File output; final File schemaDir; + final File tmpDir; final Set selectedBanks; // null means all banks - private CliOptions(File input, File output, File schemaDir, Set selectedBanks) { - this.input = input; - this.output = output; - this.schemaDir = schemaDir; + private CliOptions(File input, File output, File schemaDir, File tmpDir, Set selectedBanks) { + this.input = input; + this.output = output; + this.schemaDir = schemaDir; + this.tmpDir = tmpDir; this.selectedBanks = selectedBanks; } @@ -113,6 +115,7 @@ static CliOptions parse(String[] args) throws Exception { File output = new File(args[1]); File schemaDir = null; + File tmpDir = null; Set selectedBanks = new LinkedHashSet<>(); boolean selectAll = true; @@ -130,6 +133,14 @@ static CliOptions parse(String[] args) throws Exception { continue; } + if ("--tmp-dir".equals(arg)) { + if (i + 1 >= args.length) { + throw new IllegalArgumentException("--tmp-dir requires a directory path"); + } + tmpDir = new File(args[++i]); + continue; + } + if ("--bank-file".equals(arg)) { if (i + 1 >= args.length) { throw new IllegalArgumentException("--bank-file requires a file path"); @@ -161,11 +172,21 @@ static CliOptions parse(String[] args) throws Exception { if (!schemaDir.isDirectory()) { throw new IllegalArgumentException("Schema directory not found: " + schemaDir.getAbsolutePath()); } + + if (tmpDir != null) { + if (!tmpDir.isDirectory()) { + throw new IllegalArgumentException("Temp directory not found: " + tmpDir.getAbsolutePath()); + } + if (!tmpDir.canWrite()) { + throw new IllegalArgumentException("Temp directory not writable: " + tmpDir.getAbsolutePath()); + } + } + if (!input.exists()) { throw new IllegalArgumentException("Input HIPO file not found: " + input.getAbsolutePath()); } - return new CliOptions(input, output, schemaDir, selectAll ? null : selectedBanks); + return new CliOptions(input, output, schemaDir, tmpDir, selectAll ? null : selectedBanks); } private static Set readBankNames(File bankFile) throws IOException { @@ -194,6 +215,7 @@ private static void printUsageAndExit() { System.err.println(" --bank-file FILE a file with one bank name per line, '#' comments allowed;"); System.err.println(" both comma list and `--bank-file` may be used together"); System.err.println(" --schema-dir DIR use a custom schema directory"); + System.err.println(" --tmp-dir DIR use a custom directory for temporary files"); System.err.println(""); System.err.println("EXAMPLES:"); System.err.println("* hipo2npz input.hipo output.npz"); @@ -245,18 +267,16 @@ private static final class Converter { private final Map banks = new LinkedHashMap<>(); private final Set selectedBanks; // null means all banks private final Map schemaTypes; // BANK/COLUMN -> type + private final File tmpDir; - Converter(Set selectedBanks, Map schemaTypes) { + Converter(Set selectedBanks, Map schemaTypes, File tmpDir) { this.selectedBanks = selectedBanks; - this.schemaTypes = schemaTypes; + this.schemaTypes = schemaTypes; + this.tmpDir = tmpDir; } void convert(File input, File output) throws Exception { - if (selectedBanks == null) { - System.out.println("Including all banks"); - } else { - System.out.println("Including selected banks only"); - } + System.out.println(selectedBanks==null ? "Including all banks" : "Including selected banks only"); HipoDataSource reader = new HipoDataSource(); reader.open(input); @@ -267,23 +287,17 @@ void convert(File input, File output) throws Exception { DataEvent event = reader.getNextEvent(); nEvents++; ingestEvent(event); - if ((nEvents % 10000) == 0) { System.out.printf("Processed %,d events%n", nEvents); } } - } finally { reader.close(); - } - - try { writeNpz(output); } finally { cleanupTempFiles(); } - System.out.printf("Wrote %s with %,d events and %,d banks%n", - output.getAbsolutePath(), nEvents, banks.size()); + System.out.printf("Wrote %s with %,d events and %,d banks%n", output.getAbsolutePath(), nEvents, banks.size()); } private boolean keepBank(String bankName) { @@ -314,7 +328,7 @@ private void ingestEvent(DataEvent event) throws IOException { int rows = bank.rows(); presentRows.put(bankName, rows); - BankStore store = banks.computeIfAbsent(bankName, BankStore::new); + BankStore store = banks.computeIfAbsent(bankName, name -> new BankStore(name, tmpDir)); ensureColumns(store, bank); String[] cols = bank.getColumnList(); @@ -348,7 +362,7 @@ private void ensureColumns(BankStore store, DataBank bank) throws IOException { continue; } ColumnType type = discoverColumnType(bank, col); - store.columns.put(col, new ColumnStore(store.bankName, col, type)); + store.columns.put(col, new ColumnStore(col, type, tmpDir)); } } @@ -634,15 +648,15 @@ private static final class BankStore implements Closeable { long nEvents = 0; long totalRows = 0; - BankStore(String bankName) { + BankStore(String bankName, File tmpDir) { this.bankName = bankName; try { // create rows per event temp file - this.rowsPerEventFile = File.createTempFile("hipo2npz_rpe_", ".bin"); + this.rowsPerEventFile = File.createTempFile("hipo2npz_rpe_", ".bin", tmpDir); this.rowsPerEventFile.deleteOnExit(); this.rowsPerEventOut = new BufferedOutputStream(new FileOutputStream(rowsPerEventFile), 1 << 16); // create offsets file - this.offsetsFile = File.createTempFile("hipo2npz_off_", ".bin"); + this.offsetsFile = File.createTempFile("hipo2npz_off_", ".bin", tmpDir); this.offsetsFile.deleteOnExit(); this.offsetsOut = new BufferedOutputStream(new FileOutputStream(offsetsFile), 1 << 16); writeOffset(0L); @@ -678,7 +692,6 @@ public void close() throws IOException { // ------------------------------------------------------------------------ private static final class ColumnStore implements Closeable { - final String bankName; final String columnName; final ColumnType type; final File tempFile; @@ -686,44 +699,25 @@ private static final class ColumnStore implements Closeable { private final ByteBuffer scratch = ByteBuffer.allocate(8).order(ByteOrder.LITTLE_ENDIAN); long count = 0; - ColumnStore(String bankName, String columnName, ColumnType type) throws IOException { - this.bankName = bankName; + ColumnStore(String columnName, ColumnType type, File tmpDir) throws IOException { this.columnName = columnName; this.type = type; - this.tempFile = File.createTempFile("hipo2npz_col_", ".bin"); + this.tempFile = File.createTempFile("hipo2npz_col_", ".bin", tmpDir); this.tempFile.deleteOnExit(); this.out = new BufferedOutputStream(new FileOutputStream(tempFile), 1 << 16); } void append(DataBank bank, String col, int row) throws IOException { + scratch.clear(); switch (type) { - case BYTE -> out.write(bank.getByte(col, row)); - case SHORT -> { - scratch.clear(); - scratch.putShort(bank.getShort(col, row)); - out.write(scratch.array(), 0, 2); - } - case INT -> { - scratch.clear(); - scratch.putInt(bank.getInt(col, row)); - out.write(scratch.array(), 0, 4); - } - case LONG -> { - scratch.clear(); - scratch.putLong(bank.getLong(col, row)); - out.write(scratch.array(), 0, 8); - } - case FLOAT -> { - scratch.clear(); - scratch.putFloat(bank.getFloat(col, row)); - out.write(scratch.array(), 0, 4); - } - case DOUBLE -> { - scratch.clear(); - scratch.putDouble(bank.getDouble(col, row)); - out.write(scratch.array(), 0, 8); - } - } + case BYTE -> scratch.put(bank.getByte(col, row)); + case SHORT -> scratch.putShort(bank.getShort(col, row)); + case INT -> scratch.putInt(bank.getInt(col, row)); + case LONG -> scratch.putLong(bank.getLong(col, row)); + case FLOAT -> scratch.putFloat(bank.getFloat(col, row)); + case DOUBLE -> scratch.putDouble(bank.getDouble(col, row)); + } + out.write(scratch.array(), 0, type.byteWidth); count++; } From 57b81c89e4674821bda8dc817002b96e923c2e4e Mon Sep 17 00:00:00 2001 From: Christopher Dilks Date: Mon, 3 Aug 2026 18:21:25 -0400 Subject: [PATCH 3/6] fix: create local `.tmp` dir instead of using `/tmp` --- .../main/java/org/jlab/io/hipo/Hipo2Npz.java | 112 ++++++++++-------- 1 file changed, 65 insertions(+), 47 deletions(-) diff --git a/common-tools/clas-io/src/main/java/org/jlab/io/hipo/Hipo2Npz.java b/common-tools/clas-io/src/main/java/org/jlab/io/hipo/Hipo2Npz.java index ab0ac11728..dfa0a60e9a 100644 --- a/common-tools/clas-io/src/main/java/org/jlab/io/hipo/Hipo2Npz.java +++ b/common-tools/clas-io/src/main/java/org/jlab/io/hipo/Hipo2Npz.java @@ -13,6 +13,7 @@ import java.nio.ByteOrder; import java.nio.charset.StandardCharsets; import java.nio.file.Files; +import java.nio.file.Path; import java.util.ArrayList; import java.util.Comparator; import java.util.HashMap; @@ -83,7 +84,7 @@ public static void main(String[] args) throws Exception { } } - Converter converter = new Converter(options.selectedBanks, schemaTypes, options.tmpDir); + Converter converter = new Converter(options.selectedBanks, schemaTypes); converter.convert(options.input, options.output); } @@ -95,14 +96,12 @@ private static final class CliOptions { final File input; final File output; final File schemaDir; - final File tmpDir; final Set selectedBanks; // null means all banks - private CliOptions(File input, File output, File schemaDir, File tmpDir, Set selectedBanks) { + private CliOptions(File input, File output, File schemaDir, Set selectedBanks) { this.input = input; this.output = output; this.schemaDir = schemaDir; - this.tmpDir = tmpDir; this.selectedBanks = selectedBanks; } @@ -115,7 +114,6 @@ static CliOptions parse(String[] args) throws Exception { File output = new File(args[1]); File schemaDir = null; - File tmpDir = null; Set selectedBanks = new LinkedHashSet<>(); boolean selectAll = true; @@ -133,14 +131,6 @@ static CliOptions parse(String[] args) throws Exception { continue; } - if ("--tmp-dir".equals(arg)) { - if (i + 1 >= args.length) { - throw new IllegalArgumentException("--tmp-dir requires a directory path"); - } - tmpDir = new File(args[++i]); - continue; - } - if ("--bank-file".equals(arg)) { if (i + 1 >= args.length) { throw new IllegalArgumentException("--bank-file requires a file path"); @@ -173,20 +163,11 @@ static CliOptions parse(String[] args) throws Exception { throw new IllegalArgumentException("Schema directory not found: " + schemaDir.getAbsolutePath()); } - if (tmpDir != null) { - if (!tmpDir.isDirectory()) { - throw new IllegalArgumentException("Temp directory not found: " + tmpDir.getAbsolutePath()); - } - if (!tmpDir.canWrite()) { - throw new IllegalArgumentException("Temp directory not writable: " + tmpDir.getAbsolutePath()); - } - } - if (!input.exists()) { throw new IllegalArgumentException("Input HIPO file not found: " + input.getAbsolutePath()); } - return new CliOptions(input, output, schemaDir, tmpDir, selectAll ? null : selectedBanks); + return new CliOptions(input, output, schemaDir, selectAll ? null : selectedBanks); } private static Set readBankNames(File bankFile) throws IOException { @@ -215,7 +196,7 @@ private static void printUsageAndExit() { System.err.println(" --bank-file FILE a file with one bank name per line, '#' comments allowed;"); System.err.println(" both comma list and `--bank-file` may be used together"); System.err.println(" --schema-dir DIR use a custom schema directory"); - System.err.println(" --tmp-dir DIR use a custom directory for temporary files"); + System.err.println(" default: the one included with this coatjava installation"); System.err.println(""); System.err.println("EXAMPLES:"); System.err.println("* hipo2npz input.hipo output.npz"); @@ -267,34 +248,48 @@ private static final class Converter { private final Map banks = new LinkedHashMap<>(); private final Set selectedBanks; // null means all banks private final Map schemaTypes; // BANK/COLUMN -> type - private final File tmpDir; + private Path tmpDir; + private Thread cleanupHook; - Converter(Set selectedBanks, Map schemaTypes, File tmpDir) { + Converter(Set selectedBanks, Map schemaTypes) { this.selectedBanks = selectedBanks; this.schemaTypes = schemaTypes; - this.tmpDir = tmpDir; } void convert(File input, File output) throws Exception { System.out.println(selectedBanks==null ? "Including all banks" : "Including selected banks only"); + File tmpDirFile = new File(output.getPath() + ".tmp"); + tmpDir = createRunDir(tmpDirFile); + cleanupHook = new Thread(() -> deleteRecursively(tmpDir)); + Runtime.getRuntime().addShutdownHook(cleanupHook); + HipoDataSource reader = new HipoDataSource(); reader.open(input); long nEvents = 0; try { - while (reader.hasEvent()) { - DataEvent event = reader.getNextEvent(); - nEvents++; - ingestEvent(event); - if ((nEvents % 10000) == 0) { - System.out.printf("Processed %,d events%n", nEvents); + try { + while (reader.hasEvent()) { + DataEvent event = reader.getNextEvent(); + nEvents++; + ingestEvent(event); + if ((nEvents % 10000) == 0) { + System.out.printf("Processed %,d events%n", nEvents); + } } + } finally { + reader.close(); } - reader.close(); writeNpz(output); } finally { - cleanupTempFiles(); + closeAllQuietly(); + deleteRecursively(tmpDir); + try { + Runtime.getRuntime().removeShutdownHook(cleanupHook); + } catch (IllegalStateException ignored) { + // JVM is already shutting down — the hook itself will run deleteRecursively + } } System.out.printf("Wrote %s with %,d events and %,d banks%n", output.getAbsolutePath(), nEvents, banks.size()); @@ -328,7 +323,7 @@ private void ingestEvent(DataEvent event) throws IOException { int rows = bank.rows(); presentRows.put(bankName, rows); - BankStore store = banks.computeIfAbsent(bankName, name -> new BankStore(name, tmpDir)); + BankStore store = banks.computeIfAbsent(bankName, name -> new BankStore(name, tmpDir.toFile())); ensureColumns(store, bank); String[] cols = bank.getColumnList(); @@ -362,7 +357,7 @@ private void ensureColumns(BankStore store, DataBank bank) throws IOException { continue; } ColumnType type = discoverColumnType(bank, col); - store.columns.put(col, new ColumnStore(col, type, tmpDir)); + store.columns.put(col, new ColumnStore(col, type, tmpDir.toFile())); } } @@ -510,25 +505,51 @@ private void streamEntry(ZipOutputStream zos, String name, String descr, long co zos.closeEntry(); } - private void cleanupTempFiles() { + /** + * Safety net for any stores that weren't already closed by writeNpz (e.g. because an + * earlier bank threw partway through). Closing an already-closed stream is a no-op. + */ + private void closeAllQuietly() { for (BankStore bank : banks.values()) { - deleteQuietly(bank.rowsPerEventFile); - deleteQuietly(bank.offsetsFile); + closeQuietly(bank); for (ColumnStore col : bank.columns.values()) { - deleteQuietly(col.tempFile); + closeQuietly(col); } } } - private void deleteQuietly(File f) { - if (f != null) { - f.delete(); + private void closeQuietly(Closeable c) { + if (c == null) { + return; + } + try { + c.close(); + } catch (IOException ignored) { } } private String sanitize(String s) { return s.replace("::", "__").replace('/', '_').replace(' ', '_'); } + + private static Path createRunDir(File dir) throws IOException { + if (dir.exists()) { + throw new RuntimeException("tmp directory still exists, possibly from a failed previous run: " + dir.getAbsolutePath()); + } + return Files.createDirectory(dir.toPath()); + } + + private static void deleteRecursively(Path dir) { + if (dir == null || !Files.exists(dir)) { + return; + } + try (var files = Files.walk(dir)) { + files.sorted(Comparator.reverseOrder()) + .forEach(p -> p.toFile().delete()); + } catch (IOException | UncheckedIOException ignored) { + // best-effort; nothing more we can do here + } + } } // ------------------------------------------------------------------------ @@ -653,11 +674,9 @@ private static final class BankStore implements Closeable { try { // create rows per event temp file this.rowsPerEventFile = File.createTempFile("hipo2npz_rpe_", ".bin", tmpDir); - this.rowsPerEventFile.deleteOnExit(); this.rowsPerEventOut = new BufferedOutputStream(new FileOutputStream(rowsPerEventFile), 1 << 16); // create offsets file this.offsetsFile = File.createTempFile("hipo2npz_off_", ".bin", tmpDir); - this.offsetsFile.deleteOnExit(); this.offsetsOut = new BufferedOutputStream(new FileOutputStream(offsetsFile), 1 << 16); writeOffset(0L); } catch (IOException e) { @@ -703,7 +722,6 @@ private static final class ColumnStore implements Closeable { this.columnName = columnName; this.type = type; this.tempFile = File.createTempFile("hipo2npz_col_", ".bin", tmpDir); - this.tempFile.deleteOnExit(); this.out = new BufferedOutputStream(new FileOutputStream(tempFile), 1 << 16); } From 2cde18b74bb8c5b9c965cae98677aab9e909e654 Mon Sep 17 00:00:00 2001 From: Christopher Dilks Date: Tue, 4 Aug 2026 17:28:28 -0400 Subject: [PATCH 4/6] feat: process range of events --- .../main/java/org/jlab/io/hipo/Hipo2Npz.java | 54 ++++++++++++++----- 1 file changed, 40 insertions(+), 14 deletions(-) diff --git a/common-tools/clas-io/src/main/java/org/jlab/io/hipo/Hipo2Npz.java b/common-tools/clas-io/src/main/java/org/jlab/io/hipo/Hipo2Npz.java index dfa0a60e9a..04ed2b77a4 100644 --- a/common-tools/clas-io/src/main/java/org/jlab/io/hipo/Hipo2Npz.java +++ b/common-tools/clas-io/src/main/java/org/jlab/io/hipo/Hipo2Npz.java @@ -85,7 +85,7 @@ public static void main(String[] args) throws Exception { } Converter converter = new Converter(options.selectedBanks, schemaTypes); - converter.convert(options.input, options.output); + converter.convert(options.input, options.output, options.numEvents, options.firstEvent); } // ------------------------------------------------------------------------ @@ -96,12 +96,16 @@ private static final class CliOptions { final File input; final File output; final File schemaDir; + final long numEvents; + final long firstEvent; final Set selectedBanks; // null means all banks - private CliOptions(File input, File output, File schemaDir, Set selectedBanks) { + private CliOptions(File input, File output, File schemaDir, long numEvents, long firstEvent, Set selectedBanks) { this.input = input; this.output = output; this.schemaDir = schemaDir; + this.numEvents = numEvents; + this.firstEvent = firstEvent; this.selectedBanks = selectedBanks; } @@ -110,10 +114,11 @@ static CliOptions parse(String[] args) throws Exception { printUsageAndExit(); } - File input = new File(args[0]); - File output = new File(args[1]); - - File schemaDir = null; + File input = new File(args[0]); + File output = new File(args[1]); + File schemaDir = null; + long numEvents = 0; + long firstEvent = 0; Set selectedBanks = new LinkedHashSet<>(); boolean selectAll = true; @@ -141,6 +146,22 @@ static CliOptions parse(String[] args) throws Exception { continue; } + if ("--num-events".equals(arg)) { + if (i + 1 >= args.length) { + throw new IllegalArgumentException("--num-events requires a number"); + } + numEvents = Long.parseLong(args[++i]); + continue; + } + + if ("--first-event".equals(arg)) { + if (i + 1 >= args.length) { + throw new IllegalArgumentException("--first-event requires a number"); + } + firstEvent = Long.parseLong(args[++i]); + continue; + } + if ("*".equals(arg)) { selectedBanks.clear(); selectAll = true; @@ -167,7 +188,7 @@ static CliOptions parse(String[] args) throws Exception { throw new IllegalArgumentException("Input HIPO file not found: " + input.getAbsolutePath()); } - return new CliOptions(input, output, schemaDir, selectAll ? null : selectedBanks); + return new CliOptions(input, output, schemaDir, numEvents, firstEvent, selectAll ? null : selectedBanks); } private static Set readBankNames(File bankFile) throws IOException { @@ -197,6 +218,8 @@ private static void printUsageAndExit() { System.err.println(" both comma list and `--bank-file` may be used together"); System.err.println(" --schema-dir DIR use a custom schema directory"); System.err.println(" default: the one included with this coatjava installation"); + System.err.println(" --num-events NUM process this many events (default: all)"); + System.err.println(" --first-event NUM start from this event (default: 0)"); System.err.println(""); System.err.println("EXAMPLES:"); System.err.println("* hipo2npz input.hipo output.npz"); @@ -256,7 +279,7 @@ private static final class Converter { this.schemaTypes = schemaTypes; } - void convert(File input, File output) throws Exception { + void convert(File input, File output, long numEvents, long firstEvent) throws Exception { System.out.println(selectedBanks==null ? "Including all banks" : "Including selected banks only"); File tmpDirFile = new File(output.getPath() + ".tmp"); @@ -267,20 +290,23 @@ void convert(File input, File output) throws Exception { HipoDataSource reader = new HipoDataSource(); reader.open(input); - long nEvents = 0; + long nevRead = 0; + long nevProc = 0; try { try { while (reader.hasEvent()) { DataEvent event = reader.getNextEvent(); - nEvents++; + nevRead++; + if (nevRead <= firstEvent) continue; ingestEvent(event); - if ((nEvents % 10000) == 0) { - System.out.printf("Processed %,d events%n", nEvents); - } + nevProc++; + if ((nevProc % 10000) == 0) System.out.printf("Processed %,d events%n", nevProc); + if (numEvents > 0 && nevProc >= numEvents) break; } } finally { reader.close(); } + System.out.println("Writing NPZ file..."); writeNpz(output); } finally { closeAllQuietly(); @@ -292,7 +318,7 @@ void convert(File input, File output) throws Exception { } } - System.out.printf("Wrote %s with %,d events and %,d banks%n", output.getAbsolutePath(), nEvents, banks.size()); + System.out.printf("Wrote %s with %,d events and %,d banks%n", output.getAbsolutePath(), nevProc, banks.size()); } private boolean keepBank(String bankName) { From 99098e2bd76e0911ce68e49d594dd5deb95559b4 Mon Sep 17 00:00:00 2001 From: Christopher Dilks Date: Tue, 4 Aug 2026 18:13:53 -0400 Subject: [PATCH 5/6] feat: NPZ diff tool Co-Authored-By: Claude --- bin/hipo2npz-diff | 132 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 132 insertions(+) create mode 100755 bin/hipo2npz-diff diff --git a/bin/hipo2npz-diff b/bin/hipo2npz-diff new file mode 100755 index 0000000000..95f606cdee --- /dev/null +++ b/bin/hipo2npz-diff @@ -0,0 +1,132 @@ +#!/usr/bin/env python3 + +###################################### +# author: generated by Claude Sonnet 5 +###################################### + +""" +hipo2npz-diff - Compare two NumPy .npz files and report differences. + +Usage: + hipo2npz-diff file1.npz file2.npz + hipo2npz-diff file1.npz file2.npz --rtol 1e-5 --atol 1e-8 + hipo2npz-diff file1.npz file2.npz --exact + hipo2npz-diff file1.npz file2.npz --verbose + +Exit codes: + 0 - files are equivalent (given tolerance) + 1 - differences found + 2 - error (bad file, etc.) +""" + +import argparse +import sys + +import numpy as np + + +def diff_npz(path_a, path_b, rtol, atol, exact, verbose): + try: + a = np.load(path_a, allow_pickle=True) + b = np.load(path_b, allow_pickle=True) + except Exception as e: + print(f"Error loading files: {e}", file=sys.stderr) + sys.exit(2) + + keys_a = set(a.files) + keys_b = set(b.files) + + only_in_a = sorted(keys_a - keys_b) + only_in_b = sorted(keys_b - keys_a) + common = sorted(keys_a & keys_b) + + has_diff = False + + if only_in_a: + has_diff = True + print(f"Keys only in {path_a}:") + for k in only_in_a: + print(f" - {k}") + + if only_in_b: + has_diff = True + print(f"Keys only in {path_b}:") + for k in only_in_b: + print(f" - {k}") + + for key in common: + arr_a, arr_b = a[key], b[key] + + if arr_a.shape != arr_b.shape: + has_diff = True + print(f"[{key}] shape mismatch: {arr_a.shape} vs {arr_b.shape}") + continue + + if arr_a.dtype != arr_b.dtype and verbose: + print(f"[{key}] dtype differs: {arr_a.dtype} vs {arr_b.dtype}") + + is_float = np.issubdtype(arr_a.dtype, np.floating) + + try: + if exact: + # equal_nan=True: NaNs in the same position count as matching, + # since NaN != NaN by IEEE rules but that's rarely what you want here + equal = np.array_equal(arr_a, arr_b, equal_nan=is_float) + else: + equal = np.allclose(arr_a, arr_b, rtol=rtol, atol=atol, equal_nan=True) + except TypeError: + # Non-numeric / object arrays: fall back to plain equality (no equal_nan support) + equal = np.array_equal(arr_a, arr_b) + + if not equal: + has_diff = True + print(f"[{key}] values differ", end="") + try: + fa = arr_a.astype(np.float64) + fb = arr_b.astype(np.float64) + diff = np.abs(fa - fb) + + # Positions where exactly one side is NaN (a "real" mismatch, not just + # matching NaNs) vs. positions with a finite numeric difference + nan_mismatch = np.isnan(fa) != np.isnan(fb) + finite_mask = ~np.isnan(fa) & ~np.isnan(fb) + + n_diff = np.count_nonzero( + finite_mask & (diff > (atol + rtol * np.abs(fb))) + ) + n_nan_mismatch = np.count_nonzero(nan_mismatch) + + max_diff = np.max(diff[finite_mask]) if finite_mask.any() else 0.0 + print( + f" (max abs diff = {max_diff:.6g}, " + f"{n_diff}/{arr_a.size} elements differ, " + f"{n_nan_mismatch} NaN-mismatch positions)" + ) + except (TypeError, ValueError): + print() + elif verbose: + print(f"[{key}] OK (identical within tolerance)") + + if not has_diff: + print(f"No differences found between {path_a} and {path_b}" + + ("" if exact else f" (rtol={rtol}, atol={atol})")) + + return has_diff + + +def main(): + parser = argparse.ArgumentParser(description="Diff two .npz files.") + parser.add_argument("file_a", help="First .npz file") + parser.add_argument("file_b", help="Second .npz file") + parser.add_argument("--rtol", type=float, default=1e-5, help="Relative tolerance for float comparison (default: 1e-5)") + parser.add_argument("--atol", type=float, default=1e-8, help="Absolute tolerance for float comparison (default: 1e-8)") + parser.add_argument("--exact", action="store_true", help="Require exact equality instead of tolerance-based comparison") + parser.add_argument("--verbose", action="store_true", help="Print status for matching keys too") + args = parser.parse_args() + + has_diff = diff_npz(args.file_a, args.file_b, args.rtol, args.atol, args.exact, args.verbose) + sys.exit(1 if has_diff else 0) + + +if __name__ == "__main__": + main() From 3742b29406b382099041a106ee431b3e28c6a7eb Mon Sep 17 00:00:00 2001 From: Christopher Dilks Date: Tue, 4 Aug 2026 18:26:01 -0400 Subject: [PATCH 6/6] ci: convert it all --- .github/workflows/ci.yml | 2 +- etc/data/nnet | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ab1b400c2e..5158b39117 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -356,7 +356,7 @@ jobs: - name: untar build run: tar xzvf coatjava.tar.gz - name: hipo2npz - run: ./coatjava/bin/hipo2npz rec.hipo rec.npz RUN::config,REC::Event,REC::Particle + run: ./coatjava/bin/hipo2npz rec.hipo rec.npz - name: hipo2npz-dump run: ./coatjava/bin/hipo2npz-dump rec.npz 1 diff --git a/etc/data/nnet b/etc/data/nnet index 577aac477b..73aa4341c7 160000 --- a/etc/data/nnet +++ b/etc/data/nnet @@ -1 +1 @@ -Subproject commit 577aac477bd87dacafb4bbf18f4787c7e59d1bae +Subproject commit 73aa4341c739175950def77252a018549d1411b0