From 1b95e138ffc73daa6194a739068b6a720603e778 Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Sun, 9 Aug 2026 23:49:31 +0200 Subject: [PATCH 1/3] fix: separate persistent eval storage from subroutine lexicals Keep source-filtered subroutine locals fresh instead of recovering them from BEGIN/eval storage, and preserve fixed-length input record separator semantics when $/ is dynamically localized. This fixes Class::Std and Getopt::Param teardown behavior as well as DateTime::TimeZone::Tzfile fixed-length header reads. Generated with [Codex](https://openai.com/codex/) Co-Authored-By: Codex --- .../backend/bytecode/BytecodeCompiler.java | 4 +- .../backend/bytecode/CompileAssignment.java | 8 +-- .../perlonjava/backend/jvm/EmitVariable.java | 2 +- .../frontend/parser/OperatorParser.java | 5 ++ .../frontend/parser/SpecialBlockParser.java | 12 ++++ .../frontend/parser/SubroutineParser.java | 61 ++++++++++++++++++- .../runtime/operators/Readline.java | 31 +++++++++- .../runtime/runtimetypes/RuntimeCode.java | 12 ++++ .../unit/eval_begin_lexical_storage.t | 27 ++++++++ .../input_record_separator_fixed_length.t | 23 +++++++ ...source_filter_subroutine_lexical_storage.t | 26 ++++++++ 11 files changed, 201 insertions(+), 10 deletions(-) create mode 100644 src/test/resources/unit/eval_begin_lexical_storage.t create mode 100644 src/test/resources/unit/input_record_separator_fixed_length.t create mode 100644 src/test/resources/unit/source_filter_subroutine_lexical_storage.t diff --git a/src/main/java/org/perlonjava/backend/bytecode/BytecodeCompiler.java b/src/main/java/org/perlonjava/backend/bytecode/BytecodeCompiler.java index 8a02957d9..0f4dafbd0 100644 --- a/src/main/java/org/perlonjava/backend/bytecode/BytecodeCompiler.java +++ b/src/main/java/org/perlonjava/backend/bytecode/BytecodeCompiler.java @@ -2899,7 +2899,7 @@ void compileVariableDeclaration(OperatorNode node, String op) { boolean isDeclaredReference = node.annotations != null && Boolean.TRUE.equals(node.annotations.get("isDeclaredReference")); - Integer beginId = RuntimeCode.evalBeginIds.get(sigilOp); + Integer beginId = RuntimeCode.persistentDeclarationIds.get(sigilOp); if (beginId != null) { // BEGIN-captured variable: use RETRIEVE_BEGIN_* (destructive removal from global storage) int persistId = beginId; @@ -3316,7 +3316,7 @@ void compileVariableDeclaration(OperatorNode node, String op) { continue; } - Integer beginId2 = RuntimeCode.evalBeginIds.get(sigilOp); + Integer beginId2 = RuntimeCode.persistentDeclarationIds.get(sigilOp); if (beginId2 != null || op.equals("state")) { int persistId = beginId2 != null ? beginId2 : sigilOp.id; int reg = allocateRegister(); diff --git a/src/main/java/org/perlonjava/backend/bytecode/CompileAssignment.java b/src/main/java/org/perlonjava/backend/bytecode/CompileAssignment.java index 55f3bae70..5655de2c5 100644 --- a/src/main/java/org/perlonjava/backend/bytecode/CompileAssignment.java +++ b/src/main/java/org/perlonjava/backend/bytecode/CompileAssignment.java @@ -430,7 +430,7 @@ public static void compileAssignmentOperator(BytecodeCompiler bytecodeCompiler, if (sigilOp.operator.equals("$") && sigilOp.operand instanceof IdentifierNode) { String varName = "$" + ((IdentifierNode) sigilOp.operand).name; - Integer beginIdObj = RuntimeCode.evalBeginIds.get(sigilOp); + Integer beginIdObj = RuntimeCode.persistentDeclarationIds.get(sigilOp); if (beginIdObj != null) { int beginId = beginIdObj; int nameIdx = bytecodeCompiler.addToStringPool(varName); @@ -532,7 +532,7 @@ public static void compileAssignmentOperator(BytecodeCompiler bytecodeCompiler, // Handle my @array = ... String varName = "@" + ((IdentifierNode) sigilOp.operand).name; - Integer beginIdArr = RuntimeCode.evalBeginIds.get(sigilOp); + Integer beginIdArr = RuntimeCode.persistentDeclarationIds.get(sigilOp); if (beginIdArr != null) { int beginId = beginIdArr; int nameIdx = bytecodeCompiler.addToStringPool(varName); @@ -607,7 +607,7 @@ public static void compileAssignmentOperator(BytecodeCompiler bytecodeCompiler, // Handle my %hash = ... String varName = "%" + ((IdentifierNode) sigilOp.operand).name; - Integer beginIdHash = RuntimeCode.evalBeginIds.get(sigilOp); + Integer beginIdHash = RuntimeCode.persistentDeclarationIds.get(sigilOp); if (beginIdHash != null) { int beginId = beginIdHash; int nameIdx = bytecodeCompiler.addToStringPool(varName); @@ -737,7 +737,7 @@ public static void compileAssignmentOperator(BytecodeCompiler bytecodeCompiler, String varName = sigil + ((IdentifierNode) sigilOp.operand).name; int varReg; - Integer beginIdList = RuntimeCode.evalBeginIds.get(sigilOp); + Integer beginIdList = RuntimeCode.persistentDeclarationIds.get(sigilOp); if (beginIdList != null) { int beginId = beginIdList; int nameIdx = bytecodeCompiler.addToStringPool(varName); diff --git a/src/main/java/org/perlonjava/backend/jvm/EmitVariable.java b/src/main/java/org/perlonjava/backend/jvm/EmitVariable.java index 950b9682e..4984a99db 100644 --- a/src/main/java/org/perlonjava/backend/jvm/EmitVariable.java +++ b/src/main/java/org/perlonjava/backend/jvm/EmitVariable.java @@ -1487,7 +1487,7 @@ static void handleMyOperator(EmitterVisitor emitterVisitor, OperatorNode node) { String className = EmitterMethodCreator.getVariableClassName(sigil); if (operator.equals("my")) { - Integer beginId = RuntimeCode.evalBeginIds.get(sigilNode); + Integer beginId = RuntimeCode.persistentDeclarationIds.get(sigilNode); if (beginId == null) { ctx.mv.visitTypeInsn(Opcodes.NEW, className); ctx.mv.visitInsn(Opcodes.DUP); diff --git a/src/main/java/org/perlonjava/frontend/parser/OperatorParser.java b/src/main/java/org/perlonjava/frontend/parser/OperatorParser.java index 24379c9f0..c335d19dd 100644 --- a/src/main/java/org/perlonjava/frontend/parser/OperatorParser.java +++ b/src/main/java/org/perlonjava/frontend/parser/OperatorParser.java @@ -384,6 +384,11 @@ && isGlobalOnlyVariable(name)) { } } + if ((operator.equals("my") || operator.equals("state")) + && ctx.symbolTable.isInSubroutineBody()) { + RuntimeCode.subroutineLocalDeclarationNodes.add(node); + RuntimeCode.persistentDeclarationIds.remove(node); + } int varIndex = ctx.symbolTable.addVariable(var, operator, node); // Note: the isDeclaredReference flag is stored in node.annotations // and will be used during code generation diff --git a/src/main/java/org/perlonjava/frontend/parser/SpecialBlockParser.java b/src/main/java/org/perlonjava/frontend/parser/SpecialBlockParser.java index 1f602fc94..81729135c 100644 --- a/src/main/java/org/perlonjava/frontend/parser/SpecialBlockParser.java +++ b/src/main/java/org/perlonjava/frontend/parser/SpecialBlockParser.java @@ -273,6 +273,18 @@ static RuntimeList runSpecialBlock(Parser parser, String blockPhase, Node block, int beginId = RuntimeCode.evalBeginIds.computeIfAbsent( ast, k -> EmitterMethodCreator.classCounter++); + // Source filters may pre-populate the shared symbol table + // with declarations that occur later in the rewritten + // source (including locals inside future subroutines). + // A BEGIN block can only capture lexicals declared before + // the block, so do not make those future declarations + // process-persistent merely because they are visible in + // this compiler data structure. + if (ast != null + && ast.tokenIndex < block.getIndex() + && !RuntimeCode.subroutineLocalDeclarationNodes.contains(ast)) { + RuntimeCode.persistentDeclarationIds.putIfAbsent(ast, beginId); + } packageName = PersistentVariable.beginPackage(beginId); // Emit: package BEGIN_PKG nodes.add( diff --git a/src/main/java/org/perlonjava/frontend/parser/SubroutineParser.java b/src/main/java/org/perlonjava/frontend/parser/SubroutineParser.java index 57121dbc6..521d6abc9 100644 --- a/src/main/java/org/perlonjava/frontend/parser/SubroutineParser.java +++ b/src/main/java/org/perlonjava/frontend/parser/SubroutineParser.java @@ -959,9 +959,28 @@ public static Node parseSubroutineDefinition( int definitionFeatureFlags = parser.ctx.symbolTable.featureFlagsStack.peek(); int definitionStrictOptions = parser.ctx.symbolTable.strictOptionsStack.peek(); + // Remember which declaration nodes existed before the body was parsed. + // The symbol table remains available to lazy compilation and therefore + // also contains the body's own lexicals afterwards. Source-filtered + // subs can lose the explicit `my` wrapper from the collected AST, so + // declaration scanning alone cannot reliably distinguish those locals + // from lexicals captured from the enclosing compile-time scope. + Set enclosingLexicalDeclarations = + Collections.newSetFromMap(new IdentityHashMap<>()); + for (SymbolTable.SymbolEntry entry + : parser.ctx.symbolTable.getAllVisibleVariables().values()) { + if (entry.ast() != null) { + enclosingLexicalDeclarations.add(entry.ast()); + } + } + try { // Parse the block of the subroutine, which contains the actual code. + int subroutineBodyStartTokenIndex = parser.tokenIndex; BlockNode block = ParseBlock.parseBlock(parser); + block.setAnnotation("enclosingLexicalDeclarations", enclosingLexicalDeclarations); + block.setAnnotation("subroutineBodyStartTokenIndex", subroutineBodyStartTokenIndex); + block.setAnnotation("subroutineBodyEndTokenIndex", parser.tokenIndex); if (futureAsyncAwaitSub) { block.setAnnotation("futureAsyncAwaitSub", true); FutureAsyncAwaitParser.markFutureClass(block); @@ -1419,9 +1438,9 @@ public static ListNode handleNamedSubWithFilter(Parser parser, String subName, S // This prevents hitting JVM's 255 constructor argument limit for named subs // in modules like Perl::Tidy that have 200+ lexicals in scope. Set usedVars = null; + Set declaredVarSet = new LinkedHashSet<>(); { Set usedVarSet = new HashSet<>(); - Set declaredVarSet = new LinkedHashSet<>(); VariableCollectorVisitor collector = new VariableCollectorVisitor(usedVarSet, declaredVarSet); block.accept(collector); @@ -1431,6 +1450,22 @@ public static ListNode handleNamedSubWithFilter(Parser parser, String subName, S } } + // Classify body-owned declarations before selective capture skips + // them. A local declaration is normally not a free-variable use, so + // waiting until the capture loop would miss exactly the entries that + // must override an earlier tentative BEGIN classification. + Integer bodyStart = (Integer) block.getAnnotation("subroutineBodyStartTokenIndex"); + Integer bodyEnd = (Integer) block.getAnnotation("subroutineBodyEndTokenIndex"); + for (SymbolTable.SymbolEntry entry : outerVars.values()) { + OperatorNode ast = entry.ast(); + boolean declaredInBody = ast != null && bodyStart != null && bodyEnd != null + && ast.tokenIndex >= bodyStart && ast.tokenIndex <= bodyEnd; + if (ast != null && (declaredVarSet.contains(entry.name()) || declaredInBody)) { + RuntimeCode.subroutineLocalDeclarationNodes.add(ast); + RuntimeCode.persistentDeclarationIds.remove(ast); + } + } + ArrayList classList = new ArrayList<>(); ArrayList paramList = new ArrayList<>(); ArrayList capturedNames = new ArrayList<>(); @@ -1485,6 +1520,30 @@ public static ListNode handleNamedSubWithFilter(Parser parser, String subName, S ast, k -> EmitterMethodCreator.classCounter++); } + // The parser's symbol table can still contain lexicals + // declared while parsing this body. They may remain in the + // lazy compiler snapshot, but their declaration must create + // a fresh cell on every invocation. Only declaration nodes + // that existed before parsing the body are enclosing + // lexicals recovered from compile-time storage. + @SuppressWarnings("unchecked") + Set enclosingDeclarations = + (Set) block.getAnnotation("enclosingLexicalDeclarations"); + boolean declaredInBody = ast != null && bodyStart != null && bodyEnd != null + && ast.tokenIndex >= bodyStart && ast.tokenIndex <= bodyEnd; + boolean isSubroutineLocal = declaredVarSet.contains(entry.name()) + || declaredInBody; + if (isSubroutineLocal && ast != null) { + RuntimeCode.subroutineLocalDeclarationNodes.add(ast); + // A BEGIN block encountered while parsing this body may + // already have tentatively classified the shared symbol + // table entry as persistent. The completed body gives + // us the authoritative lexical ownership information. + RuntimeCode.persistentDeclarationIds.remove(ast); + } else if (enclosingDeclarations != null + && enclosingDeclarations.contains(ast)) { + RuntimeCode.persistentDeclarationIds.putIfAbsent(ast, beginId); + } variableName = NameNormalizer.normalizeVariableName( entry.name().substring(1), PersistentVariable.beginPackage(beginId)); diff --git a/src/main/java/org/perlonjava/runtime/operators/Readline.java b/src/main/java/org/perlonjava/runtime/operators/Readline.java index 639ca37ce..856c92860 100644 --- a/src/main/java/org/perlonjava/runtime/operators/Readline.java +++ b/src/main/java/org/perlonjava/runtime/operators/Readline.java @@ -111,9 +111,11 @@ public static RuntimeScalar readline(RuntimeIO runtimeIO) { return readParagraphMode(runtimeIO); } - if (rs != null && rs.isRecordLengthMode()) { + int recordLength = rs != null && rs.isRecordLengthMode() + ? rs.getRecordLength() + : recordLengthFromLocalizedSeparator(rsScalar); + if (recordLength > 0) { // Handle record length mode when $/ = \N - int recordLength = rs.getRecordLength(); return readFixedLength(runtimeIO, recordLength); } @@ -135,6 +137,31 @@ public static RuntimeScalar readline(RuntimeIO runtimeIO) { } } + /** + * A dynamically localized special variable is represented by the saved + * global binding rather than by the original InputRecordSeparator + * subclass. Preserve Perl's $/ = \N mode across that representation + * change. + */ + private static int recordLengthFromLocalizedSeparator(RuntimeScalar separator) { + if (separator.type != RuntimeScalarType.REFERENCE + || !(separator.value instanceof RuntimeScalar referenced)) { + return -1; + } + if (referenced.type == RuntimeScalarType.INTEGER) { + return referenced.getInt(); + } + if (referenced.type == RuntimeScalarType.STRING + || referenced.type == RuntimeScalarType.BYTE_STRING) { + try { + return Integer.parseInt(referenced.toString()); + } catch (NumberFormatException ignored) { + return -1; + } + } + return -1; + } + private static RuntimeScalar readParagraphMode(RuntimeIO runtimeIO) { boolean isByteMode = runtimeIO.isByteMode(); StringBuilder paragraph = new StringBuilder(); diff --git a/src/main/java/org/perlonjava/runtime/runtimetypes/RuntimeCode.java b/src/main/java/org/perlonjava/runtime/runtimetypes/RuntimeCode.java index e64595c90..f3b3f900d 100644 --- a/src/main/java/org/perlonjava/runtime/runtimetypes/RuntimeCode.java +++ b/src/main/java/org/perlonjava/runtime/runtimetypes/RuntimeCode.java @@ -55,6 +55,18 @@ public class RuntimeCode extends RuntimeBase implements RuntimeScalarReference { public static final MethodHandles.Lookup lookup = MethodHandles.lookup(); public static final IdentityHashMap evalBeginIds = new IdentityHashMap<>(); + /** + * Declaration nodes whose lexical storage must be recovered from a + * compile-time BEGIN capture. Kept separate from {@link #evalBeginIds}: + * runtime eval aliases also need a stable package id, but must not turn a + * subroutine-local {@code my} into process-persistent storage. + */ + public static final IdentityHashMap persistentDeclarationIds = + new IdentityHashMap<>(); + /** Declaration nodes owned by subroutine bodies, even when a source filter + * leaves them visible in the parser's shared symbol table. */ + public static final Set subroutineLocalDeclarationNodes = + Collections.newSetFromMap(new IdentityHashMap<>()); /** * Flag to control whether eval STRING should use the interpreter backend. diff --git a/src/test/resources/unit/eval_begin_lexical_storage.t b/src/test/resources/unit/eval_begin_lexical_storage.t new file mode 100644 index 000000000..d40dc1890 --- /dev/null +++ b/src/test/resources/unit/eval_begin_lexical_storage.t @@ -0,0 +1,27 @@ +use strict; +use warnings; +use Scalar::Util qw(refaddr); +use Test::More tests => 6; + +my %defaults = (value => 1); + +sub make_inside_out_object { + my ($class) = @_; + my $object = bless \my($anonymous_scalar), $class; + my $value = eval '$defaults{value}'; + die $@ if $@; + return ($object, $value); +} + +my ($first, $first_value) = make_inside_out_object('First'); +my ($second, $second_value) = make_inside_out_object('Second'); + +is $first_value, 1, 'runtime eval sees the enclosing file lexical'; +is $second_value, 1, 'runtime eval keeps working on later calls'; +isnt refaddr($first), refaddr($second), 'referenced my scalar is fresh on each call'; +is ref($first), 'First', 'first object keeps its original class'; +is ref($second), 'Second', 'second object receives its requested class'; + +$defaults{value} = 2; +my (undef, $updated_value) = make_inside_out_object('Third'); +is $updated_value, 2, 'runtime eval observes updates without persistent local cells'; diff --git a/src/test/resources/unit/input_record_separator_fixed_length.t b/src/test/resources/unit/input_record_separator_fixed_length.t new file mode 100644 index 000000000..83eb519d7 --- /dev/null +++ b/src/test/resources/unit/input_record_separator_fixed_length.t @@ -0,0 +1,23 @@ +use strict; +use warnings; +use Test::More tests => 8; + +my $data = "abcdef"; +open my $fh, '<', \$data or die $!; + +{ + local $/ = \1; + is scalar($fh->getline), 'a', 'localized fixed-length separator reads one character'; + is tell($fh), 1, 'fixed-length getline advances by one character'; +} + +{ + local $/ = \2; + is scalar($fh->getline), 'bc', 'localized fixed-length separator accepts larger records'; + is tell($fh), 3, 'larger fixed-length getline advances by the record size'; + is scalar($fh->getline), 'de', 'fixed-length mode remains active for repeated reads'; + is scalar($fh->getline), 'f', 'final short record is returned'; + is scalar($fh->getline), undef, 'read after the final record returns undef'; +} + +is $/, "\n", 'localized input record separator is restored'; diff --git a/src/test/resources/unit/source_filter_subroutine_lexical_storage.t b/src/test/resources/unit/source_filter_subroutine_lexical_storage.t new file mode 100644 index 000000000..db68c0f01 --- /dev/null +++ b/src/test/resources/unit/source_filter_subroutine_lexical_storage.t @@ -0,0 +1,26 @@ +use strict; +use warnings; +use Test::More tests => 5; +use Scalar::Util qw(refaddr); +use Filter::Simple; + +# A no-op filter is enough to exercise the rewritten-source compiler path. +FILTER { }; + +sub make_filtered_inside_out_object { + my ($class) = @_; + return bless \my($anonymous_scalar), $class; +} + +# Source-filter preprocessing used to leave the subroutine's lexical in the +# shared symbol table, where later BEGIN blocks classified it as persistent. +BEGIN { my $compile_time_only = 1 } + +my $first = make_filtered_inside_out_object('FilteredFirst'); +my $second = make_filtered_inside_out_object('FilteredSecond'); + +isnt refaddr($first), refaddr($second), 'filtered sub creates a fresh scalar referent'; +is ref($first), 'FilteredFirst', 'first filtered object keeps its class'; +is ref($second), 'FilteredSecond', 'second filtered object gets its class'; +isnt "$first", "$second", 'filtered objects remain distinct'; +pass 'later BEGIN block does not make a filtered sub lexical persistent'; From 436f32cc7c53079428f854db884a7cb45fb9ebb6 Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Mon, 10 Aug 2026 10:24:55 +0200 Subject: [PATCH 2/3] fix: limit BEGIN capture to referenced lexicals Restore the established evalBeginIds storage path for ordinary closure captures. The separate persistent declaration map caused file-scope lexicals captured by helper subs to arrive empty, aborting dozens of Perl core tests. Instead, use the variable collector to capture only lexicals referenced by each BEGIN block. This prevents stale source-filter symbol entries from making Class::Std subroutine locals persistent while preserving legitimate closure and generated eval captures. Generated with [Codex](https://openai.com/codex/) Co-Authored-By: Codex --- .../backend/bytecode/BytecodeCompiler.java | 4 +- .../backend/bytecode/CompileAssignment.java | 8 +-- .../perlonjava/backend/jvm/EmitVariable.java | 2 +- .../frontend/parser/OperatorParser.java | 5 -- .../frontend/parser/SpecialBlockParser.java | 22 +++---- .../frontend/parser/SubroutineParser.java | 61 +------------------ .../runtime/runtimetypes/RuntimeCode.java | 12 ---- 7 files changed, 18 insertions(+), 96 deletions(-) diff --git a/src/main/java/org/perlonjava/backend/bytecode/BytecodeCompiler.java b/src/main/java/org/perlonjava/backend/bytecode/BytecodeCompiler.java index 0f4dafbd0..8a02957d9 100644 --- a/src/main/java/org/perlonjava/backend/bytecode/BytecodeCompiler.java +++ b/src/main/java/org/perlonjava/backend/bytecode/BytecodeCompiler.java @@ -2899,7 +2899,7 @@ void compileVariableDeclaration(OperatorNode node, String op) { boolean isDeclaredReference = node.annotations != null && Boolean.TRUE.equals(node.annotations.get("isDeclaredReference")); - Integer beginId = RuntimeCode.persistentDeclarationIds.get(sigilOp); + Integer beginId = RuntimeCode.evalBeginIds.get(sigilOp); if (beginId != null) { // BEGIN-captured variable: use RETRIEVE_BEGIN_* (destructive removal from global storage) int persistId = beginId; @@ -3316,7 +3316,7 @@ void compileVariableDeclaration(OperatorNode node, String op) { continue; } - Integer beginId2 = RuntimeCode.persistentDeclarationIds.get(sigilOp); + Integer beginId2 = RuntimeCode.evalBeginIds.get(sigilOp); if (beginId2 != null || op.equals("state")) { int persistId = beginId2 != null ? beginId2 : sigilOp.id; int reg = allocateRegister(); diff --git a/src/main/java/org/perlonjava/backend/bytecode/CompileAssignment.java b/src/main/java/org/perlonjava/backend/bytecode/CompileAssignment.java index 5655de2c5..55f3bae70 100644 --- a/src/main/java/org/perlonjava/backend/bytecode/CompileAssignment.java +++ b/src/main/java/org/perlonjava/backend/bytecode/CompileAssignment.java @@ -430,7 +430,7 @@ public static void compileAssignmentOperator(BytecodeCompiler bytecodeCompiler, if (sigilOp.operator.equals("$") && sigilOp.operand instanceof IdentifierNode) { String varName = "$" + ((IdentifierNode) sigilOp.operand).name; - Integer beginIdObj = RuntimeCode.persistentDeclarationIds.get(sigilOp); + Integer beginIdObj = RuntimeCode.evalBeginIds.get(sigilOp); if (beginIdObj != null) { int beginId = beginIdObj; int nameIdx = bytecodeCompiler.addToStringPool(varName); @@ -532,7 +532,7 @@ public static void compileAssignmentOperator(BytecodeCompiler bytecodeCompiler, // Handle my @array = ... String varName = "@" + ((IdentifierNode) sigilOp.operand).name; - Integer beginIdArr = RuntimeCode.persistentDeclarationIds.get(sigilOp); + Integer beginIdArr = RuntimeCode.evalBeginIds.get(sigilOp); if (beginIdArr != null) { int beginId = beginIdArr; int nameIdx = bytecodeCompiler.addToStringPool(varName); @@ -607,7 +607,7 @@ public static void compileAssignmentOperator(BytecodeCompiler bytecodeCompiler, // Handle my %hash = ... String varName = "%" + ((IdentifierNode) sigilOp.operand).name; - Integer beginIdHash = RuntimeCode.persistentDeclarationIds.get(sigilOp); + Integer beginIdHash = RuntimeCode.evalBeginIds.get(sigilOp); if (beginIdHash != null) { int beginId = beginIdHash; int nameIdx = bytecodeCompiler.addToStringPool(varName); @@ -737,7 +737,7 @@ public static void compileAssignmentOperator(BytecodeCompiler bytecodeCompiler, String varName = sigil + ((IdentifierNode) sigilOp.operand).name; int varReg; - Integer beginIdList = RuntimeCode.persistentDeclarationIds.get(sigilOp); + Integer beginIdList = RuntimeCode.evalBeginIds.get(sigilOp); if (beginIdList != null) { int beginId = beginIdList; int nameIdx = bytecodeCompiler.addToStringPool(varName); diff --git a/src/main/java/org/perlonjava/backend/jvm/EmitVariable.java b/src/main/java/org/perlonjava/backend/jvm/EmitVariable.java index 4984a99db..950b9682e 100644 --- a/src/main/java/org/perlonjava/backend/jvm/EmitVariable.java +++ b/src/main/java/org/perlonjava/backend/jvm/EmitVariable.java @@ -1487,7 +1487,7 @@ static void handleMyOperator(EmitterVisitor emitterVisitor, OperatorNode node) { String className = EmitterMethodCreator.getVariableClassName(sigil); if (operator.equals("my")) { - Integer beginId = RuntimeCode.persistentDeclarationIds.get(sigilNode); + Integer beginId = RuntimeCode.evalBeginIds.get(sigilNode); if (beginId == null) { ctx.mv.visitTypeInsn(Opcodes.NEW, className); ctx.mv.visitInsn(Opcodes.DUP); diff --git a/src/main/java/org/perlonjava/frontend/parser/OperatorParser.java b/src/main/java/org/perlonjava/frontend/parser/OperatorParser.java index c335d19dd..24379c9f0 100644 --- a/src/main/java/org/perlonjava/frontend/parser/OperatorParser.java +++ b/src/main/java/org/perlonjava/frontend/parser/OperatorParser.java @@ -384,11 +384,6 @@ && isGlobalOnlyVariable(name)) { } } - if ((operator.equals("my") || operator.equals("state")) - && ctx.symbolTable.isInSubroutineBody()) { - RuntimeCode.subroutineLocalDeclarationNodes.add(node); - RuntimeCode.persistentDeclarationIds.remove(node); - } int varIndex = ctx.symbolTable.addVariable(var, operator, node); // Note: the isDeclaredReference flag is stored in node.annotations // and will be used during code generation diff --git a/src/main/java/org/perlonjava/frontend/parser/SpecialBlockParser.java b/src/main/java/org/perlonjava/frontend/parser/SpecialBlockParser.java index 81729135c..d35b53e6e 100644 --- a/src/main/java/org/perlonjava/frontend/parser/SpecialBlockParser.java +++ b/src/main/java/org/perlonjava/frontend/parser/SpecialBlockParser.java @@ -2,6 +2,7 @@ import org.perlonjava.app.cli.CompilerOptions; import org.perlonjava.app.scriptengine.PerlLanguageProvider; +import org.perlonjava.backend.bytecode.VariableCollectorVisitor; import org.perlonjava.backend.jvm.EmitterMethodCreator; import org.perlonjava.frontend.astnode.*; import org.perlonjava.frontend.lexer.LexerTokenType; @@ -14,8 +15,10 @@ import java.util.ArrayDeque; import java.util.BitSet; import java.util.Deque; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.Stack; import static org.perlonjava.runtime.runtimetypes.GlobalContext.GLOBAL_PHASE; @@ -246,6 +249,10 @@ static RuntimeList runSpecialBlock(Parser parser, String blockPhase, Node block, // Declare capture variables Map outerVars = parser.ctx.symbolTable.getAllVisibleVariables(); + Set usedVars = new HashSet<>(); + VariableCollectorVisitor collector = new VariableCollectorVisitor(usedVars); + block.accept(collector); + boolean captureAllVisibleVariables = collector.hasEvalString(); for (SymbolTable.SymbolEntry entry : outerVars.values()) { if (!entry.name().equals("@_") && !entry.decl().isEmpty()) { // Skip lexical subs (entries starting with &) - they are stored as hidden variables @@ -257,6 +264,9 @@ static RuntimeList runSpecialBlock(Parser parser, String blockPhase, Node block, || "$@%*".indexOf(entry.name().charAt(0)) < 0) { continue; } + if (!captureAllVisibleVariables && !usedVars.contains(entry.name())) { + continue; + } String packageName; boolean isFromOuterScope = false; @@ -273,18 +283,6 @@ static RuntimeList runSpecialBlock(Parser parser, String blockPhase, Node block, int beginId = RuntimeCode.evalBeginIds.computeIfAbsent( ast, k -> EmitterMethodCreator.classCounter++); - // Source filters may pre-populate the shared symbol table - // with declarations that occur later in the rewritten - // source (including locals inside future subroutines). - // A BEGIN block can only capture lexicals declared before - // the block, so do not make those future declarations - // process-persistent merely because they are visible in - // this compiler data structure. - if (ast != null - && ast.tokenIndex < block.getIndex() - && !RuntimeCode.subroutineLocalDeclarationNodes.contains(ast)) { - RuntimeCode.persistentDeclarationIds.putIfAbsent(ast, beginId); - } packageName = PersistentVariable.beginPackage(beginId); // Emit: package BEGIN_PKG nodes.add( diff --git a/src/main/java/org/perlonjava/frontend/parser/SubroutineParser.java b/src/main/java/org/perlonjava/frontend/parser/SubroutineParser.java index 521d6abc9..57121dbc6 100644 --- a/src/main/java/org/perlonjava/frontend/parser/SubroutineParser.java +++ b/src/main/java/org/perlonjava/frontend/parser/SubroutineParser.java @@ -959,28 +959,9 @@ public static Node parseSubroutineDefinition( int definitionFeatureFlags = parser.ctx.symbolTable.featureFlagsStack.peek(); int definitionStrictOptions = parser.ctx.symbolTable.strictOptionsStack.peek(); - // Remember which declaration nodes existed before the body was parsed. - // The symbol table remains available to lazy compilation and therefore - // also contains the body's own lexicals afterwards. Source-filtered - // subs can lose the explicit `my` wrapper from the collected AST, so - // declaration scanning alone cannot reliably distinguish those locals - // from lexicals captured from the enclosing compile-time scope. - Set enclosingLexicalDeclarations = - Collections.newSetFromMap(new IdentityHashMap<>()); - for (SymbolTable.SymbolEntry entry - : parser.ctx.symbolTable.getAllVisibleVariables().values()) { - if (entry.ast() != null) { - enclosingLexicalDeclarations.add(entry.ast()); - } - } - try { // Parse the block of the subroutine, which contains the actual code. - int subroutineBodyStartTokenIndex = parser.tokenIndex; BlockNode block = ParseBlock.parseBlock(parser); - block.setAnnotation("enclosingLexicalDeclarations", enclosingLexicalDeclarations); - block.setAnnotation("subroutineBodyStartTokenIndex", subroutineBodyStartTokenIndex); - block.setAnnotation("subroutineBodyEndTokenIndex", parser.tokenIndex); if (futureAsyncAwaitSub) { block.setAnnotation("futureAsyncAwaitSub", true); FutureAsyncAwaitParser.markFutureClass(block); @@ -1438,9 +1419,9 @@ public static ListNode handleNamedSubWithFilter(Parser parser, String subName, S // This prevents hitting JVM's 255 constructor argument limit for named subs // in modules like Perl::Tidy that have 200+ lexicals in scope. Set usedVars = null; - Set declaredVarSet = new LinkedHashSet<>(); { Set usedVarSet = new HashSet<>(); + Set declaredVarSet = new LinkedHashSet<>(); VariableCollectorVisitor collector = new VariableCollectorVisitor(usedVarSet, declaredVarSet); block.accept(collector); @@ -1450,22 +1431,6 @@ public static ListNode handleNamedSubWithFilter(Parser parser, String subName, S } } - // Classify body-owned declarations before selective capture skips - // them. A local declaration is normally not a free-variable use, so - // waiting until the capture loop would miss exactly the entries that - // must override an earlier tentative BEGIN classification. - Integer bodyStart = (Integer) block.getAnnotation("subroutineBodyStartTokenIndex"); - Integer bodyEnd = (Integer) block.getAnnotation("subroutineBodyEndTokenIndex"); - for (SymbolTable.SymbolEntry entry : outerVars.values()) { - OperatorNode ast = entry.ast(); - boolean declaredInBody = ast != null && bodyStart != null && bodyEnd != null - && ast.tokenIndex >= bodyStart && ast.tokenIndex <= bodyEnd; - if (ast != null && (declaredVarSet.contains(entry.name()) || declaredInBody)) { - RuntimeCode.subroutineLocalDeclarationNodes.add(ast); - RuntimeCode.persistentDeclarationIds.remove(ast); - } - } - ArrayList classList = new ArrayList<>(); ArrayList paramList = new ArrayList<>(); ArrayList capturedNames = new ArrayList<>(); @@ -1520,30 +1485,6 @@ public static ListNode handleNamedSubWithFilter(Parser parser, String subName, S ast, k -> EmitterMethodCreator.classCounter++); } - // The parser's symbol table can still contain lexicals - // declared while parsing this body. They may remain in the - // lazy compiler snapshot, but their declaration must create - // a fresh cell on every invocation. Only declaration nodes - // that existed before parsing the body are enclosing - // lexicals recovered from compile-time storage. - @SuppressWarnings("unchecked") - Set enclosingDeclarations = - (Set) block.getAnnotation("enclosingLexicalDeclarations"); - boolean declaredInBody = ast != null && bodyStart != null && bodyEnd != null - && ast.tokenIndex >= bodyStart && ast.tokenIndex <= bodyEnd; - boolean isSubroutineLocal = declaredVarSet.contains(entry.name()) - || declaredInBody; - if (isSubroutineLocal && ast != null) { - RuntimeCode.subroutineLocalDeclarationNodes.add(ast); - // A BEGIN block encountered while parsing this body may - // already have tentatively classified the shared symbol - // table entry as persistent. The completed body gives - // us the authoritative lexical ownership information. - RuntimeCode.persistentDeclarationIds.remove(ast); - } else if (enclosingDeclarations != null - && enclosingDeclarations.contains(ast)) { - RuntimeCode.persistentDeclarationIds.putIfAbsent(ast, beginId); - } variableName = NameNormalizer.normalizeVariableName( entry.name().substring(1), PersistentVariable.beginPackage(beginId)); diff --git a/src/main/java/org/perlonjava/runtime/runtimetypes/RuntimeCode.java b/src/main/java/org/perlonjava/runtime/runtimetypes/RuntimeCode.java index f3b3f900d..e64595c90 100644 --- a/src/main/java/org/perlonjava/runtime/runtimetypes/RuntimeCode.java +++ b/src/main/java/org/perlonjava/runtime/runtimetypes/RuntimeCode.java @@ -55,18 +55,6 @@ public class RuntimeCode extends RuntimeBase implements RuntimeScalarReference { public static final MethodHandles.Lookup lookup = MethodHandles.lookup(); public static final IdentityHashMap evalBeginIds = new IdentityHashMap<>(); - /** - * Declaration nodes whose lexical storage must be recovered from a - * compile-time BEGIN capture. Kept separate from {@link #evalBeginIds}: - * runtime eval aliases also need a stable package id, but must not turn a - * subroutine-local {@code my} into process-persistent storage. - */ - public static final IdentityHashMap persistentDeclarationIds = - new IdentityHashMap<>(); - /** Declaration nodes owned by subroutine bodies, even when a source filter - * leaves them visible in the parser's shared symbol table. */ - public static final Set subroutineLocalDeclarationNodes = - Collections.newSetFromMap(new IdentityHashMap<>()); /** * Flag to control whether eval STRING should use the interpreter backend. From b32860610bdb99900bc58fe185e3ed411762ce02 Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Mon, 10 Aug 2026 12:09:05 +0200 Subject: [PATCH 3/3] fix: prevent croak core test timeout regressions Give the subprocess-heavy Perl core croak test a 600-second minimum allowance while retaining larger caller-supplied timeouts. Keep ordinary tests on the configured base timeout and document the distinction in the runner output and help. Generated with [Codex](https://developers.openai.com/codex/) Co-Authored-By: Codex --- dev/tools/perl_test_runner.pl | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/dev/tools/perl_test_runner.pl b/dev/tools/perl_test_runner.pl index 31f4b544f..1449a93f2 100755 --- a/dev/tools/perl_test_runner.pl +++ b/dev/tools/perl_test_runner.pl @@ -86,7 +86,7 @@ my $total_files = @test_files; print "Found $total_files test files\n"; -print "Running tests with $jperl_path (${jobs} parallel jobs, ${timeout}s timeout)\n"; +print "Running tests with $jperl_path (${jobs} parallel jobs, ${timeout}s base timeout)\n"; print "-" x 60, "\n"; # Run tests in parallel @@ -233,6 +233,13 @@ sub process_test_result { sub run_single_test { my ($test_file) = @_; + # lib/croak.t launches a fresh jperl process for each of its 300+ cases. + # Under a full parallel run it competes with the other workers and has + # repeatedly finished within a second of the normal per-file deadline. + # Give this subprocess-heavy test a stable minimum wall-clock allowance + # while preserving any larger timeout requested by the caller. + my $test_timeout = timeout_for_test($test_file); + # Temporarily disable fatal unimplemented errors # so we can run tests that mix implemented and unimplemented features local $ENV{JPERL_UNIMPLEMENTED} = $test_file =~ m{ @@ -347,10 +354,10 @@ sub run_single_test { my $kill_after = 10; # seconds between SIGTERM and SIGKILL if (!$is_windows) { if (system('which timeout >/dev/null 2>&1') == 0) { - $timeout_cmd = "timeout -k ${kill_after}s ${timeout}s "; + $timeout_cmd = "timeout -k ${kill_after}s ${test_timeout}s "; } elsif (system('which gtimeout >/dev/null 2>&1') == 0) { # macOS with coreutils - $timeout_cmd = "gtimeout -k ${kill_after}s ${timeout}s "; + $timeout_cmd = "gtimeout -k ${kill_after}s ${test_timeout}s "; } } @@ -369,7 +376,7 @@ sub run_single_test { # Fallback to alarm-based timeout eval { local $SIG{ALRM} = sub { die "timeout\n" }; - alarm($timeout); + alarm($test_timeout); $output = `$abs_jperl $test_name 2>&1`; $exit_code = $? >> 8; alarm(0); @@ -408,6 +415,14 @@ sub run_single_test { return $result; } +sub timeout_for_test { + my ($test_file) = @_; + + return 600 if $test_file =~ m{(?:^|/)perl5_t/t/lib/croak\.t$} + && $timeout < 600; + return $timeout; +} + sub start_test_job { my ($test_queue, $children, $total_files, $completed) = @_; @@ -708,7 +723,8 @@ sub print_usage { Options: --jperl PATH Path to jperl executable (default: ./jperl) - --timeout SEC Timeout per test in seconds (default: 3) + --timeout SEC Base timeout per test in seconds (default: 300; selected + subprocess-heavy tests have a documented minimum) --jobs|-j NUM Number of parallel jobs (default: 4) --output FILE Save detailed results to JSON file --help Show this help message