From 8ade347957c5a7574a5c756ab8c3fd78e946e136 Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Sun, 2 Aug 2026 20:32:25 +0800 Subject: [PATCH 1/3] feat: remote manifest v2 --- .../apache/fluss/config/ConfigOptions.java | 27 + .../org/apache/fluss/metrics/MetricNames.java | 4 + .../fluss/remote/RemoteLogManifest.java | 296 +++++++++- .../remote/RemoteLogManifestJsonSerde.java | 197 +++++-- .../RemoteLogManifestReplacementPlanner.java | 249 ++++++++ .../remote/RemoteLogManifestV2Migration.java | 121 ++++ .../apache/fluss/remote/RemoteLogSegment.java | 28 +- .../remote/RemoteLogSegmentReference.java | 98 ++++ .../remote/UnreferencedRemoteLogSegment.java | 111 ++++ .../RemoteLogManifestJsonSerdeTest.java | 96 ++- .../fluss/remote/RemoteLogManifestV2Test.java | 335 +++++++++++ .../fluss/remote/RemoteLogSegmentTest.java | 70 +++ .../action/orphan/OrphanFilesCleanITCase.java | 18 +- fluss-rpc/src/main/proto/FlussApi.proto | 11 + .../CoordinatorEventProcessor.java | 144 ++++- .../coordinator/CoordinatorService.java | 18 +- .../watcher/TabletServerChangeWatcher.java | 3 +- .../entity/CommitRemoteLogManifestData.java | 186 +++++- .../entity/RemoteLogManifestCommitResult.java | 45 ++ .../RemoteLogManifestExpectedHandleState.java | 43 ++ .../fluss/server/kv/RemoteLogFetcher.java | 58 +- .../log/remote/DefaultRemoteLogStorage.java | 78 +++ .../server/log/remote/LogTieringTask.java | 551 ++++++++++++++++-- .../server/log/remote/RemoteLogManager.java | 86 ++- .../remote/RemoteLogManifestCommitter.java | 166 ++++++ .../remote/RemoteLogManifestUpdatePlan.java | 127 ++++ .../server/log/remote/RemoteLogStorage.java | 17 + .../log/remote/RemoteLogStorageObject.java | 46 ++ .../server/log/remote/RemoteLogTablet.java | 220 +++++-- .../fluss/server/metadata/ServerInfo.java | 30 +- .../metrics/group/TableMetricGroup.java | 7 + .../fluss/server/replica/ReplicaManager.java | 3 +- .../fluss/server/tablet/TabletServer.java | 5 +- .../server/utils/ServerRpcMessageUtils.java | 96 ++- .../fluss/server/zk/ZooKeeperClient.java | 68 +++ .../zk/data/RemoteLogManifestHandle.java | 91 ++- .../RemoteLogManifestHandleJsonSerde.java | 54 +- .../zk/data/TabletServerRegistration.java | 36 +- .../TabletServerRegistrationJsonSerde.java | 27 +- .../VersionedRemoteLogManifestHandle.java | 39 ++ .../CoordinatorEventProcessorTest.java | 78 ++- .../coordinator/TestCoordinatorGateway.java | 39 ++ .../fluss/server/kv/RemoteLogFetcherTest.java | 104 ++++ .../log/remote/RemoteLogManagerTest.java | 80 +++ .../RemoteLogManifestCommitterTest.java | 310 ++++++++++ .../remote/RemoteLogManifestV2WriterTest.java | 331 +++++++++++ .../log/remote/RemoteLogTabletTest.java | 91 +++ .../log/remote/TestingRemoteLogStorage.java | 27 + .../fluss/server/tablet/TabletServerTest.java | 5 + .../utils/ServerRpcMessageUtilsTest.java | 67 +++ .../fluss/server/zk/ZooKeeperClientTest.java | 50 ++ .../RemoteLogManifestHandleJsonSerdeTest.java | 36 +- ...TabletServerRegistrationJsonSerdeTest.java | 48 +- 53 files changed, 4914 insertions(+), 257 deletions(-) create mode 100644 fluss-common/src/main/java/org/apache/fluss/remote/RemoteLogManifestReplacementPlanner.java create mode 100644 fluss-common/src/main/java/org/apache/fluss/remote/RemoteLogManifestV2Migration.java create mode 100644 fluss-common/src/main/java/org/apache/fluss/remote/RemoteLogSegmentReference.java create mode 100644 fluss-common/src/main/java/org/apache/fluss/remote/UnreferencedRemoteLogSegment.java create mode 100644 fluss-common/src/test/java/org/apache/fluss/remote/RemoteLogManifestV2Test.java create mode 100644 fluss-common/src/test/java/org/apache/fluss/remote/RemoteLogSegmentTest.java create mode 100644 fluss-server/src/main/java/org/apache/fluss/server/entity/RemoteLogManifestCommitResult.java create mode 100644 fluss-server/src/main/java/org/apache/fluss/server/entity/RemoteLogManifestExpectedHandleState.java create mode 100644 fluss-server/src/main/java/org/apache/fluss/server/log/remote/RemoteLogManifestCommitter.java create mode 100644 fluss-server/src/main/java/org/apache/fluss/server/log/remote/RemoteLogManifestUpdatePlan.java create mode 100644 fluss-server/src/main/java/org/apache/fluss/server/log/remote/RemoteLogStorageObject.java create mode 100644 fluss-server/src/main/java/org/apache/fluss/server/zk/data/VersionedRemoteLogManifestHandle.java create mode 100644 fluss-server/src/test/java/org/apache/fluss/server/log/remote/RemoteLogManifestCommitterTest.java create mode 100644 fluss-server/src/test/java/org/apache/fluss/server/log/remote/RemoteLogManifestV2WriterTest.java create mode 100644 fluss-server/src/test/java/org/apache/fluss/server/utils/ServerRpcMessageUtilsTest.java diff --git a/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java b/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java index 2fd74f7c64..b6d9bd9246 100644 --- a/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java +++ b/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java @@ -1091,6 +1091,33 @@ public class ConfigOptions { + "prevent overwhelming the remote storage when there is a large " + "backlog of segments to upload."); + public static final ConfigOption REMOTE_LOG_MANIFEST_V2_WRITER_ENABLED = + key("remote.log.manifest-v2-writer-enabled") + .booleanType() + .defaultValue(false) + .withDescription( + "Enables CAS publishing of Manifest V2 snapshots and overlap-aware " + + "remote segment replacement. Keep disabled until every " + + "remote-log reader supports manifest version dispatch."); + + public static final ConfigOption REMOTE_LOG_MANIFEST_V2_GC_ENABLED = + key("remote.log.manifest-v2-gc-enabled") + .booleanType() + .defaultValue(false) + .withDescription( + "Enables grace-period deletion of Manifest V2 unreferenced segments " + + "and orphan remote objects. This option only takes effect " + + "when the Manifest V2 writer is enabled."); + + public static final ConfigOption REMOTE_LOG_MANIFEST_V2_GC_GRACE_PERIOD = + key("remote.log.manifest-v2-gc-grace-period") + .durationType() + .defaultValue(Duration.ofHours(1)) + .withDescription( + "Minimum time that an unreferenced segment, superseded manifest, or " + + "orphan object is retained before Manifest V2 garbage " + + "collection can delete it."); + public static final ConfigOption REMOTE_LOG_INDEX_FILE_CACHE_SIZE = key("remote.log.index-file-cache-size") .memoryType() diff --git a/fluss-common/src/main/java/org/apache/fluss/metrics/MetricNames.java b/fluss-common/src/main/java/org/apache/fluss/metrics/MetricNames.java index 505c92d159..d7b6866e6f 100644 --- a/fluss-common/src/main/java/org/apache/fluss/metrics/MetricNames.java +++ b/fluss-common/src/main/java/org/apache/fluss/metrics/MetricNames.java @@ -127,6 +127,10 @@ public class MetricNames { public static final String REMOTE_LOG_COPY_ERROR_RATE = "remoteLogCopyErrorPerSecond"; public static final String REMOTE_LOG_DELETE_REQUESTS_RATE = "remoteLogDeleteRequestsPerSecond"; public static final String REMOTE_LOG_DELETE_ERROR_RATE = "remoteLogDeleteErrorPerSecond"; + public static final String REMOTE_UNREFERENCED_BYTES = "remoteUnreferencedBytes"; + public static final String REMOTE_UNREFERENCED_SEGMENT_COUNT = "remoteUnreferencedSegmentCount"; + public static final String REMOTE_ORPHAN_MANIFEST_COUNT = "remoteOrphanManifestCount"; + public static final String REMOTE_GC_FAILURE_COUNT = "remoteGcFailureCount"; public static final String TOTAL_LOOKUP_REQUESTS_RATE = "totalLookupRequestsPerSecond"; public static final String FAILED_LOOKUP_REQUESTS_RATE = "failedLookupRequestsPerSecond"; diff --git a/fluss-common/src/main/java/org/apache/fluss/remote/RemoteLogManifest.java b/fluss-common/src/main/java/org/apache/fluss/remote/RemoteLogManifest.java index bc856e361d..ea20c226f4 100644 --- a/fluss-common/src/main/java/org/apache/fluss/remote/RemoteLogManifest.java +++ b/fluss-common/src/main/java/org/apache/fluss/remote/RemoteLogManifest.java @@ -20,9 +20,12 @@ import org.apache.fluss.metadata.PhysicalTablePath; import org.apache.fluss.metadata.TableBucket; +import javax.annotation.Nullable; + import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; +import java.util.HashSet; import java.util.List; import java.util.Objects; import java.util.Set; @@ -30,37 +33,240 @@ import java.util.stream.Collectors; /** - * A remote log manifest is an immutable list of current {@link RemoteLogSegment} which can - * represent a snapshot of a remote log tablet. + * A remote log manifest is an immutable list of current {@link RemoteLogSegment} which represents a + * snapshot of a remote log tablet. + * + *

Every segment uses a half-open physical offset range {@code [startOffset, endOffset)}. */ -public class RemoteLogManifest { +public final class RemoteLogManifest { + public static final int VERSION_1 = 1; + public static final int VERSION_2 = 2; + + private final int version; + private final long generation; private final PhysicalTablePath physicalTablePath; private final TableBucket tableBucket; private final List remoteLogSegmentList; + private final @Nullable Long persistedRemoteLogStartOffset; + private final List unreferencedRemoteLogSegments; + private final List activeReferences; + private final long logicalRemoteLogStartOffset; + private final long logicalRemoteLogEndOffset; public RemoteLogManifest( PhysicalTablePath physicalTablePath, TableBucket tableBucket, List remoteLogSegmentList) { + this( + VERSION_1, + 0L, + physicalTablePath, + tableBucket, + remoteLogSegmentList, + null, + Collections.emptyList()); + } + + private RemoteLogManifest( + int version, + long generation, + PhysicalTablePath physicalTablePath, + TableBucket tableBucket, + List remoteLogSegmentList, + @Nullable Long remoteLogStartOffset, + List unreferencedRemoteLogSegments) { + this.version = version; + this.generation = generation; this.physicalTablePath = physicalTablePath; this.tableBucket = tableBucket; - this.remoteLogSegmentList = Collections.unmodifiableList(remoteLogSegmentList); + this.remoteLogSegmentList = + Collections.unmodifiableList(new ArrayList<>(remoteLogSegmentList)); + this.persistedRemoteLogStartOffset = remoteLogStartOffset; + this.unreferencedRemoteLogSegments = + Collections.unmodifiableList(new ArrayList<>(unreferencedRemoteLogSegments)); + + validateSegmentIdentities(); + if (version == VERSION_1) { + if (generation != 0L + || remoteLogStartOffset != null + || !unreferencedRemoteLogSegments.isEmpty()) { + throw new IllegalArgumentException("V1 manifest contains V2-only metadata"); + } + } else if (version == VERSION_2) { + validateV2(); + } else { + throw new IllegalArgumentException( + "Unsupported remote log manifest version: " + version); + } + this.activeReferences = + Collections.unmodifiableList(buildActiveReferences(remoteLogStartOffset)); + this.logicalRemoteLogStartOffset = calculateLogicalStartOffset(activeReferences); + this.logicalRemoteLogEndOffset = calculateLogicalEndOffset(activeReferences); + } + + public static RemoteLogManifest createV2( + long generation, + PhysicalTablePath physicalTablePath, + TableBucket tableBucket, + List remoteLogSegmentList, + @Nullable Long remoteLogStartOffset, + List unreferencedRemoteLogSegments) { + return new RemoteLogManifest( + VERSION_2, + generation, + physicalTablePath, + tableBucket, + remoteLogSegmentList, + remoteLogStartOffset, + unreferencedRemoteLogSegments); + } - // sanity check + private void validateSegmentIdentities() { for (RemoteLogSegment remoteLogSegment : remoteLogSegmentList) { - if (!remoteLogSegment.physicalTablePath().equals(physicalTablePath)) { + validateSegmentIdentity(remoteLogSegment); + } + for (UnreferencedRemoteLogSegment unreferencedSegment : unreferencedRemoteLogSegments) { + validateSegmentIdentity(unreferencedSegment.remoteLogSegment()); + } + } + + private void validateSegmentIdentity(RemoteLogSegment remoteLogSegment) { + if (!remoteLogSegment.physicalTablePath().equals(physicalTablePath)) { + throw new IllegalArgumentException( + "RemoteLogSegment's tablePath should be the same as the tablePath of RemoteLogManifestSnapshot"); + } + if (!remoteLogSegment.tableBucket().equals(tableBucket)) { + throw new IllegalArgumentException( + "RemoteLogSegment's tableBucket should be the same as the tableBucket of RemoteLogManifestSnapshot"); + } + } + + private void validateV2() { + if (generation <= 0L) { + throw new IllegalArgumentException( + "V2 manifest generation must be greater than 0: " + generation); + } + if (remoteLogSegmentList.isEmpty()) { + if (persistedRemoteLogStartOffset != null) { + throw new IllegalArgumentException( + "Empty V2 manifest must not define remote log start offset"); + } + } else { + if (persistedRemoteLogStartOffset == null) { throw new IllegalArgumentException( - "RemoteLogSegment's tablePath should be the same as the tablePath of RemoteLogManifestSnapshot"); + "Non-empty V2 manifest must define remote log start offset"); } - if (!remoteLogSegment.tableBucket().equals(tableBucket)) { + RemoteLogSegment firstSegment = remoteLogSegmentList.get(0); + long firstLogicalEndOffset = + remoteLogSegmentList.size() == 1 + ? firstSegment.remoteLogEndOffset() + : remoteLogSegmentList.get(1).remoteLogStartOffset(); + if (persistedRemoteLogStartOffset < firstSegment.remoteLogStartOffset() + || persistedRemoteLogStartOffset >= firstLogicalEndOffset) { throw new IllegalArgumentException( - "RemoteLogSegment's tableBucket should be the same as the tableBucket of RemoteLogManifestSnapshot"); + "Remote log start offset " + + persistedRemoteLogStartOffset + + " is outside the first logical range [" + + firstSegment.remoteLogStartOffset() + + ", " + + firstLogicalEndOffset + + ")"); + } + } + + Set activeSegmentIds = new HashSet<>(); + RemoteLogSegment previousSegment = null; + for (RemoteLogSegment segment : remoteLogSegmentList) { + if (!activeSegmentIds.add(segment.remoteLogSegmentId())) { + throw new IllegalArgumentException( + "Duplicate active remote log segment id: " + segment.remoteLogSegmentId()); + } + if (previousSegment != null) { + if (previousSegment.remoteLogStartOffset() >= segment.remoteLogStartOffset()) { + throw new IllegalArgumentException( + "Active segment start offsets must be strictly increasing"); + } + if (previousSegment.remoteLogEndOffset() >= segment.remoteLogEndOffset()) { + throw new IllegalArgumentException( + "Active segment end offsets must be strictly increasing"); + } + if (previousSegment.remoteLogEndOffset() < segment.remoteLogStartOffset()) { + throw new IllegalArgumentException("Active remote log segments contain a gap"); + } + } + previousSegment = segment; + } + + Set unreferencedSegmentIds = new HashSet<>(); + for (UnreferencedRemoteLogSegment unreferencedSegment : unreferencedRemoteLogSegments) { + UUID segmentId = unreferencedSegment.remoteLogSegment().remoteLogSegmentId(); + if (activeSegmentIds.contains(segmentId)) { + throw new IllegalArgumentException( + "Remote log segment is both active and unreferenced: " + segmentId); + } + if (!unreferencedSegmentIds.add(segmentId)) { + throw new IllegalArgumentException( + "Duplicate unreferenced remote log segment id: " + segmentId); } } } + private List buildActiveReferences( + @Nullable Long remoteLogStartOffset) { + if (version == VERSION_1) { + return buildV1ActiveReferences(); + } + return buildV2ActiveReferences(remoteLogStartOffset); + } + + private List buildV1ActiveReferences() { + List references = new ArrayList<>(remoteLogSegmentList.size()); + for (RemoteLogSegment segment : remoteLogSegmentList) { + references.add( + new RemoteLogSegmentReference( + segment, segment.remoteLogStartOffset(), segment.remoteLogEndOffset())); + } + return references; + } + + private List buildV2ActiveReferences( + @Nullable Long remoteLogStartOffset) { + List references = new ArrayList<>(remoteLogSegmentList.size()); + for (int i = 0; i < remoteLogSegmentList.size(); i++) { + RemoteLogSegment segment = remoteLogSegmentList.get(i); + long logicalStartOffset = + i == 0 ? remoteLogStartOffset.longValue() : segment.remoteLogStartOffset(); + long logicalEndOffset = + i + 1 < remoteLogSegmentList.size() + ? remoteLogSegmentList.get(i + 1).remoteLogStartOffset() + : segment.remoteLogEndOffset(); + references.add( + new RemoteLogSegmentReference(segment, logicalStartOffset, logicalEndOffset)); + } + return references; + } + + private static long calculateLogicalStartOffset(List references) { + long startOffset = Long.MAX_VALUE; + for (RemoteLogSegmentReference reference : references) { + startOffset = Math.min(startOffset, reference.logicalStartOffset()); + } + return startOffset; + } + + private static long calculateLogicalEndOffset(List references) { + long endOffset = -1L; + for (RemoteLogSegmentReference reference : references) { + endOffset = Math.max(endOffset, reference.logicalEndOffset()); + } + return endOffset; + } + public RemoteLogManifest trimAndMerge( List deletedSegments, List addedSegments) { + if (version != VERSION_1) { + throw new IllegalStateException("trimAndMerge only supports V1 manifests"); + } Set deletedIds = deletedSegments.stream() .map(RemoteLogSegment::remoteLogSegmentId) @@ -76,24 +282,14 @@ public RemoteLogManifest trimAndMerge( return new RemoteLogManifest(physicalTablePath, tableBucket, newSegments); } + /** Returns the inclusive start offset, or {@link Long#MAX_VALUE} when the manifest is empty. */ public long getRemoteLogStartOffset() { - long startOffset = Long.MAX_VALUE; - for (RemoteLogSegment remoteLogSegment : remoteLogSegmentList) { - if (remoteLogSegment.remoteLogStartOffset() < startOffset) { - startOffset = remoteLogSegment.remoteLogStartOffset(); - } - } - return startOffset; + return logicalRemoteLogStartOffset; } + /** Returns the exclusive end offset, or {@code -1} when the manifest is empty. */ public long getRemoteLogEndOffset() { - long endOffset = -1; - for (RemoteLogSegment remoteLogSegment : remoteLogSegmentList) { - if (endOffset == -1 || remoteLogSegment.remoteLogEndOffset() > endOffset) { - endOffset = remoteLogSegment.remoteLogEndOffset(); - } - } - return endOffset; + return logicalRemoteLogEndOffset; } public long getRemoteLogSize() { @@ -124,6 +320,29 @@ public List getRemoteLogSegmentList() { return remoteLogSegmentList; } + public int getVersion() { + return version; + } + + public long getGeneration() { + return generation; + } + + /** Returns the V2 start field persisted in JSON, or null when it is not present. */ + @Nullable + public Long getPersistedRemoteLogStartOffset() { + return persistedRemoteLogStartOffset; + } + + public List getUnreferencedRemoteLogSegments() { + return unreferencedRemoteLogSegments; + } + + /** Returns the immutable logical view normalized once during Manifest construction. */ + public List getRemoteLogSegmentReferences() { + return activeReferences; + } + @Override public boolean equals(Object o) { if (this == o) { @@ -133,16 +352,41 @@ public boolean equals(Object o) { return false; } RemoteLogManifest that = (RemoteLogManifest) o; - return Objects.equals(remoteLogSegmentList, that.remoteLogSegmentList); + return version == that.version + && generation == that.generation + && Objects.equals(physicalTablePath, that.physicalTablePath) + && Objects.equals(tableBucket, that.tableBucket) + && Objects.equals(remoteLogSegmentList, that.remoteLogSegmentList) + && Objects.equals(persistedRemoteLogStartOffset, that.persistedRemoteLogStartOffset) + && Objects.equals( + unreferencedRemoteLogSegments, that.unreferencedRemoteLogSegments); } @Override public int hashCode() { - return Objects.hash(remoteLogSegmentList); + return Objects.hash( + version, + generation, + physicalTablePath, + tableBucket, + remoteLogSegmentList, + persistedRemoteLogStartOffset, + unreferencedRemoteLogSegments); } @Override public String toString() { - return "RemoteLogManifestSnapshot{" + "remoteLogSegmentList=" + remoteLogSegmentList + '}'; + return "RemoteLogManifest{" + + "version=" + + version + + ", generation=" + + generation + + ", remoteLogSegmentList=" + + remoteLogSegmentList + + ", persistedRemoteLogStartOffset=" + + persistedRemoteLogStartOffset + + ", unreferencedRemoteLogSegments=" + + unreferencedRemoteLogSegments + + '}'; } } diff --git a/fluss-common/src/main/java/org/apache/fluss/remote/RemoteLogManifestJsonSerde.java b/fluss-common/src/main/java/org/apache/fluss/remote/RemoteLogManifestJsonSerde.java index c90a85ea02..b2665e9d46 100644 --- a/fluss-common/src/main/java/org/apache/fluss/remote/RemoteLogManifestJsonSerde.java +++ b/fluss-common/src/main/java/org/apache/fluss/remote/RemoteLogManifestJsonSerde.java @@ -27,7 +27,6 @@ import java.io.IOException; import java.util.ArrayList; -import java.util.Iterator; import java.util.List; import java.util.UUID; @@ -43,22 +42,28 @@ public class RemoteLogManifestJsonSerde private static final String TABLE_ID_FIELD = "table_id"; private static final String PARTITION_ID_FIELD = "partition_id"; private static final String BUCKET_ID_FIELD = "bucket_id"; - private static final String MANIFEST_ENTRIES_FILED = "remote_log_segments"; + private static final String MANIFEST_ENTRIES_FIELD = "remote_log_segments"; + private static final String GENERATION_FIELD = "generation"; + private static final String REMOTE_LOG_START_OFFSET_FIELD = "remote_log_start_offset"; + private static final String UNREFERENCED_SEGMENTS_FIELD = "unreferenced_segments"; + private static final String SEGMENT_FIELD = "segment"; + private static final String UNREFERENCED_AT_MS_FIELD = "unreferenced_at_ms"; + private static final String REASON_FIELD = "reason"; + private static final String REPLACEMENT_SEGMENT_ID_FIELD = "replacement_segment_id"; private static final String REMOTE_LOG_SEGMENT_ID_FIELD = "segment_id"; private static final String START_OFFSET_FIELD = "start_offset"; private static final String END_OFFSET_FIELD = "end_offset"; private static final String MAX_TIMESTAMP_FIELD = "max_timestamp"; private static final String SEGMENT_SIZE_IN_BYTES_FIELD = "size_in_bytes"; - private static final int SNAPSHOT_VERSION = 1; @Override public void serialize(RemoteLogManifest manifest, JsonGenerator generator) throws IOException { generator.writeStartObject(); + generator.writeNumberField(VERSION_KEY, manifest.getVersion()); + if (manifest.getVersion() == RemoteLogManifest.VERSION_2) { + generator.writeNumberField(GENERATION_FIELD, manifest.getGeneration()); + } - // serialize data version. - generator.writeNumberField(VERSION_KEY, SNAPSHOT_VERSION); - - // serialize metadata PhysicalTablePath physicalTablePath = manifest.getPhysicalTablePath(); generator.writeStringField(DATABASE_NAME_FIELD, physicalTablePath.getDatabaseName()); generator.writeStringField(TABLE_NAME_FIELD, physicalTablePath.getTableName()); @@ -73,27 +78,55 @@ public void serialize(RemoteLogManifest manifest, JsonGenerator generator) throw } generator.writeNumberField(BUCKET_ID_FIELD, tb.getBucket()); - // serialize writer id entries. - generator.writeArrayFieldStart(MANIFEST_ENTRIES_FILED); - for (RemoteLogSegment remoteLogSegment : manifest.getRemoteLogSegmentList()) { - generator.writeStartObject(); - generator.writeStringField( - REMOTE_LOG_SEGMENT_ID_FIELD, remoteLogSegment.remoteLogSegmentId().toString()); - generator.writeNumberField(START_OFFSET_FIELD, remoteLogSegment.remoteLogStartOffset()); - generator.writeNumberField(END_OFFSET_FIELD, remoteLogSegment.remoteLogEndOffset()); - generator.writeNumberField(MAX_TIMESTAMP_FIELD, remoteLogSegment.maxTimestamp()); + if (manifest.getVersion() == RemoteLogManifest.VERSION_2 + && manifest.getPersistedRemoteLogStartOffset() != null) { generator.writeNumberField( - SEGMENT_SIZE_IN_BYTES_FIELD, remoteLogSegment.segmentSizeInBytes()); - generator.writeEndObject(); + REMOTE_LOG_START_OFFSET_FIELD, manifest.getPersistedRemoteLogStartOffset()); + } + + generator.writeArrayFieldStart(MANIFEST_ENTRIES_FIELD); + for (RemoteLogSegment remoteLogSegment : manifest.getRemoteLogSegmentList()) { + serializeSegment(remoteLogSegment, generator); } generator.writeEndArray(); + + if (manifest.getVersion() == RemoteLogManifest.VERSION_2) { + generator.writeArrayFieldStart(UNREFERENCED_SEGMENTS_FIELD); + for (UnreferencedRemoteLogSegment unreferencedSegment : + manifest.getUnreferencedRemoteLogSegments()) { + generator.writeStartObject(); + generator.writeFieldName(SEGMENT_FIELD); + serializeSegment(unreferencedSegment.remoteLogSegment(), generator); + generator.writeNumberField( + UNREFERENCED_AT_MS_FIELD, unreferencedSegment.unreferencedAtMs()); + generator.writeStringField(REASON_FIELD, unreferencedSegment.reason().name()); + if (unreferencedSegment.replacementSegmentId() != null) { + generator.writeStringField( + REPLACEMENT_SEGMENT_ID_FIELD, + unreferencedSegment.replacementSegmentId().toString()); + } + generator.writeEndObject(); + } + generator.writeEndArray(); + } generator.writeEndObject(); } @Override public RemoteLogManifest deserialize(JsonNode node) { - String databaseName = node.get(DATABASE_NAME_FIELD).asText(); - String tableName = node.get(TABLE_NAME_FIELD).asText(); + int version = required(node, VERSION_KEY).asInt(); + if (version != RemoteLogManifest.VERSION_1 && version != RemoteLogManifest.VERSION_2) { + throw new IllegalArgumentException( + "Unsupported remote log manifest version: " + version); + } + if (version == RemoteLogManifest.VERSION_1) { + rejectField(node, GENERATION_FIELD, version); + rejectField(node, REMOTE_LOG_START_OFFSET_FIELD, version); + rejectField(node, UNREFERENCED_SEGMENTS_FIELD, version); + } + + String databaseName = required(node, DATABASE_NAME_FIELD).asText(); + String tableName = required(node, TABLE_NAME_FIELD).asText(); JsonNode partitionNameNode = node.get(PARTITION_NAME_FIELD); PhysicalTablePath physicalTablePath; if (partitionNameNode == null) { @@ -103,34 +136,110 @@ public RemoteLogManifest deserialize(JsonNode node) { PhysicalTablePath.of(databaseName, tableName, partitionNameNode.asText()); } - long tableId = node.get(TABLE_ID_FIELD).asLong(); + long tableId = required(node, TABLE_ID_FIELD).asLong(); JsonNode partitionIdNode = node.get(PARTITION_ID_FIELD); Long partitionId = partitionIdNode == null ? null : partitionIdNode.asLong(); - int bucketId = node.get(BUCKET_ID_FIELD).asInt(); + int bucketId = required(node, BUCKET_ID_FIELD).asInt(); TableBucket tableBucket = new TableBucket(tableId, partitionId, bucketId); - Iterator entriesJson = node.get(MANIFEST_ENTRIES_FILED).elements(); - List snapshotEntries = new ArrayList<>(); - while (entriesJson.hasNext()) { - JsonNode entryJson = entriesJson.next(); - String remoteLogSegmentId = entryJson.get(REMOTE_LOG_SEGMENT_ID_FIELD).asText(); - long startOffset = entryJson.get(START_OFFSET_FIELD).asLong(); - long endOffset = entryJson.get(END_OFFSET_FIELD).asLong(); - long maxTimestamp = entryJson.get(MAX_TIMESTAMP_FIELD).asLong(); - int segmentSizeInBytes = entryJson.get(SEGMENT_SIZE_IN_BYTES_FIELD).asInt(); - snapshotEntries.add( - RemoteLogSegment.Builder.builder() - .physicalTablePath(physicalTablePath) - .tableBucket(tableBucket) - .remoteLogSegmentId(UUID.fromString(remoteLogSegmentId)) - .remoteLogStartOffset(startOffset) - .remoteLogEndOffset(endOffset) - .maxTimestamp(maxTimestamp) - .segmentSizeInBytes(segmentSizeInBytes) - .build()); - } - - return new RemoteLogManifest(physicalTablePath, tableBucket, snapshotEntries); + JsonNode entriesNode = required(node, MANIFEST_ENTRIES_FIELD); + if (!entriesNode.isArray()) { + throw new IllegalArgumentException(MANIFEST_ENTRIES_FIELD + " must be an array"); + } + List activeSegments = new ArrayList<>(); + for (JsonNode entryJson : entriesNode) { + activeSegments.add(parseSegment(entryJson, physicalTablePath, tableBucket)); + } + + if (version == RemoteLogManifest.VERSION_1) { + return new RemoteLogManifest(physicalTablePath, tableBucket, activeSegments); + } + + long generation = required(node, GENERATION_FIELD).asLong(); + JsonNode remoteStartNode = node.get(REMOTE_LOG_START_OFFSET_FIELD); + Long remoteStartOffset = remoteStartNode == null ? null : remoteStartNode.asLong(); + if (!activeSegments.isEmpty() && remoteStartOffset == null) { + throw new IllegalArgumentException( + "Non-empty V2 manifest is missing " + REMOTE_LOG_START_OFFSET_FIELD); + } + if (activeSegments.isEmpty() && remoteStartOffset != null) { + throw new IllegalArgumentException( + "Empty V2 manifest must not contain " + REMOTE_LOG_START_OFFSET_FIELD); + } + + JsonNode unreferencedNode = required(node, UNREFERENCED_SEGMENTS_FIELD); + if (!unreferencedNode.isArray()) { + throw new IllegalArgumentException(UNREFERENCED_SEGMENTS_FIELD + " must be an array"); + } + List unreferencedSegments = new ArrayList<>(); + for (JsonNode entryJson : unreferencedNode) { + RemoteLogSegment segment = + parseSegment( + required(entryJson, SEGMENT_FIELD), physicalTablePath, tableBucket); + long unreferencedAtMs = required(entryJson, UNREFERENCED_AT_MS_FIELD).asLong(); + UnreferencedRemoteLogSegment.Reason reason; + try { + reason = + UnreferencedRemoteLogSegment.Reason.valueOf( + required(entryJson, REASON_FIELD).asText()); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException("Unsupported unreferenced segment reason", e); + } + JsonNode replacementIdNode = entryJson.get(REPLACEMENT_SEGMENT_ID_FIELD); + UUID replacementSegmentId = + replacementIdNode == null ? null : UUID.fromString(replacementIdNode.asText()); + unreferencedSegments.add( + new UnreferencedRemoteLogSegment( + segment, unreferencedAtMs, reason, replacementSegmentId)); + } + return RemoteLogManifest.createV2( + generation, + physicalTablePath, + tableBucket, + activeSegments, + remoteStartOffset, + unreferencedSegments); + } + + private static void serializeSegment(RemoteLogSegment segment, JsonGenerator generator) + throws IOException { + generator.writeStartObject(); + generator.writeStringField( + REMOTE_LOG_SEGMENT_ID_FIELD, segment.remoteLogSegmentId().toString()); + generator.writeNumberField(START_OFFSET_FIELD, segment.remoteLogStartOffset()); + generator.writeNumberField(END_OFFSET_FIELD, segment.remoteLogEndOffset()); + generator.writeNumberField(MAX_TIMESTAMP_FIELD, segment.maxTimestamp()); + generator.writeNumberField(SEGMENT_SIZE_IN_BYTES_FIELD, segment.segmentSizeInBytes()); + generator.writeEndObject(); + } + + private static RemoteLogSegment parseSegment( + JsonNode node, PhysicalTablePath physicalTablePath, TableBucket tableBucket) { + return RemoteLogSegment.Builder.builder() + .physicalTablePath(physicalTablePath) + .tableBucket(tableBucket) + .remoteLogSegmentId( + UUID.fromString(required(node, REMOTE_LOG_SEGMENT_ID_FIELD).asText())) + .remoteLogStartOffset(required(node, START_OFFSET_FIELD).asLong()) + .remoteLogEndOffset(required(node, END_OFFSET_FIELD).asLong()) + .maxTimestamp(required(node, MAX_TIMESTAMP_FIELD).asLong()) + .segmentSizeInBytes(required(node, SEGMENT_SIZE_IN_BYTES_FIELD).asInt()) + .build(); + } + + private static JsonNode required(JsonNode node, String fieldName) { + JsonNode field = node.get(fieldName); + if (field == null || field.isNull()) { + throw new IllegalArgumentException("Missing required field: " + fieldName); + } + return field; + } + + private static void rejectField(JsonNode node, String fieldName, int version) { + if (node.get(fieldName) != null) { + throw new IllegalArgumentException( + "Field " + fieldName + " is not valid for manifest version " + version); + } } public static RemoteLogManifest fromJson(byte[] json) { diff --git a/fluss-common/src/main/java/org/apache/fluss/remote/RemoteLogManifestReplacementPlanner.java b/fluss-common/src/main/java/org/apache/fluss/remote/RemoteLogManifestReplacementPlanner.java new file mode 100644 index 0000000000..70a0415daf --- /dev/null +++ b/fluss-common/src/main/java/org/apache/fluss/remote/RemoteLogManifestReplacementPlanner.java @@ -0,0 +1,249 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.remote; + +import org.apache.fluss.annotation.Internal; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.UUID; + +import static org.apache.fluss.utils.Preconditions.checkArgument; +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** + * Builds an immutable V2 manifest update for one completely copied candidate segment. + * + *

Replacement correctness requires every candidate start offset used as a logical boundary to be + * a record-batch boundary in every overlapping segment. Clean ISR replication preserves the encoded + * record batches, and local log rolling only creates segment boundaries between complete batches. + * The planner validates offset ranges but does not read remote log objects to revalidate their + * batch layout. + */ +@Internal +public final class RemoteLogManifestReplacementPlanner { + + /** Classification of a candidate physical segment against the current logical remote range. */ + public enum PlanType { + INITIAL_COPY, + ALREADY_COVERED, + GAP, + APPEND, + REPLACE_AND_CLIP, + REPLACE_AND_CLIP_START + } + + /** The result of classifying and, when needed, applying one candidate segment. */ + public static final class Result { + private final PlanType planType; + private final RemoteLogManifest resultManifest; + private final List segmentsToUnreference; + + private Result( + PlanType planType, + RemoteLogManifest resultManifest, + List segmentsToUnreference) { + this.planType = checkNotNull(planType); + this.resultManifest = checkNotNull(resultManifest); + this.segmentsToUnreference = + Collections.unmodifiableList(new ArrayList<>(segmentsToUnreference)); + } + + public PlanType planType() { + return planType; + } + + public RemoteLogManifest resultManifest() { + return resultManifest; + } + + public List segmentsToUnreference() { + return segmentsToUnreference; + } + + public boolean requiresManifestCommit() { + return planType != PlanType.ALREADY_COVERED && planType != PlanType.GAP; + } + } + + public static Result plan( + RemoteLogManifest manifest, RemoteLogSegment candidate, long unreferencedAtMs) { + checkArgument( + manifest.getVersion() == RemoteLogManifest.VERSION_2, + "Replacement planning requires a V2 manifest"); + checkArgument( + candidate.physicalTablePath().equals(manifest.getPhysicalTablePath()), + "Candidate physical table path does not match manifest"); + checkArgument( + candidate.tableBucket().equals(manifest.getTableBucket()), + "Candidate table bucket does not match manifest"); + ensureNewSegmentId(manifest, candidate.remoteLogSegmentId()); + + if (manifest.getRemoteLogSegmentList().isEmpty()) { + RemoteLogManifest resultManifest = + RemoteLogManifest.createV2( + manifest.getGeneration() + 1, + manifest.getPhysicalTablePath(), + manifest.getTableBucket(), + Collections.singletonList(candidate), + candidate.remoteLogStartOffset(), + manifest.getUnreferencedRemoteLogSegments()); + return new Result(PlanType.INITIAL_COPY, resultManifest, Collections.emptyList()); + } + + long remoteStartOffset = manifest.getRemoteLogStartOffset(); + long remoteEndOffset = manifest.getRemoteLogEndOffset(); + long candidateStartOffset = candidate.remoteLogStartOffset(); + long candidateEndOffset = candidate.remoteLogEndOffset(); + + if (candidateEndOffset <= remoteEndOffset) { + return new Result(PlanType.ALREADY_COVERED, manifest, Collections.emptyList()); + } + if (candidateStartOffset > remoteEndOffset) { + return new Result(PlanType.GAP, manifest, Collections.emptyList()); + } + + PlanType planType; + if (candidateStartOffset == remoteEndOffset) { + planType = PlanType.APPEND; + } else if (candidateStartOffset < remoteStartOffset) { + planType = PlanType.REPLACE_AND_CLIP_START; + } else { + planType = PlanType.REPLACE_AND_CLIP; + } + + long insertionOffset = Math.max(candidateStartOffset, remoteStartOffset); + List activeSegments = new ArrayList<>(); + List segmentsToUnreference = new ArrayList<>(); + for (RemoteLogSegmentReference reference : manifest.getRemoteLogSegmentReferences()) { + if (reference.logicalStartOffset() < insertionOffset) { + activeSegments.add(reference.remoteLogSegment()); + } else { + segmentsToUnreference.add(reference.remoteLogSegment()); + } + } + + if (insertionOffset > remoteStartOffset) { + checkArgument(!activeSegments.isEmpty(), "Replacement is missing a prefix segment"); + RemoteLogSegment prefix = activeSegments.get(activeSegments.size() - 1); + checkArgument( + prefix.remoteLogEndOffset() >= insertionOffset, + "Replacement creates a logical gap before offset %s", + insertionOffset); + } + activeSegments.add(candidate); + + List unreferencedSegments = + new ArrayList<>(manifest.getUnreferencedRemoteLogSegments()); + for (RemoteLogSegment segment : segmentsToUnreference) { + unreferencedSegments.add( + new UnreferencedRemoteLogSegment( + segment, + unreferencedAtMs, + UnreferencedRemoteLogSegment.Reason.REPLACED, + candidate.remoteLogSegmentId())); + } + + RemoteLogManifest resultManifest = + RemoteLogManifest.createV2( + manifest.getGeneration() + 1, + manifest.getPhysicalTablePath(), + manifest.getTableBucket(), + activeSegments, + remoteStartOffset, + unreferencedSegments); + return new Result(planType, resultManifest, segmentsToUnreference); + } + + /** + * Moves a continuous expired logical prefix to unreferenced metadata without deleting objects. + * + *

The last active reference is retained because the current V2 handle format represents only + * non-empty ranges. This is conservative: expiration may be delayed, but readable data is never + * removed early. + */ + public static RemoteLogManifest expireContinuousPrefix( + RemoteLogManifest manifest, + long currentTimeMs, + long ttlMs, + Long lakeLogEndOffset, + long unreferencedAtMs) { + checkArgument( + manifest.getVersion() == RemoteLogManifest.VERSION_2, + "Expiration planning requires a V2 manifest"); + if (ttlMs <= 0L || manifest.getRemoteLogSegmentList().size() <= 1) { + return manifest; + } + + List references = manifest.getRemoteLogSegmentReferences(); + int expireCount = 0; + while (expireCount < references.size() - 1) { + RemoteLogSegment segment = references.get(expireCount).remoteLogSegment(); + boolean expired = currentTimeMs - segment.maxTimestamp() > ttlMs; + boolean tieredToLake = + lakeLogEndOffset == null + || references.get(expireCount).logicalEndOffset() <= lakeLogEndOffset; + if (!expired || !tieredToLake) { + break; + } + expireCount++; + } + if (expireCount == 0) { + return manifest; + } + + List activeSegments = + new ArrayList<>( + manifest.getRemoteLogSegmentList().subList(expireCount, references.size())); + List unreferencedSegments = + new ArrayList<>(manifest.getUnreferencedRemoteLogSegments()); + for (int index = 0; index < expireCount; index++) { + unreferencedSegments.add( + new UnreferencedRemoteLogSegment( + references.get(index).remoteLogSegment(), + unreferencedAtMs, + UnreferencedRemoteLogSegment.Reason.EXPIRED, + null)); + } + return RemoteLogManifest.createV2( + manifest.getGeneration(), + manifest.getPhysicalTablePath(), + manifest.getTableBucket(), + activeSegments, + references.get(expireCount).logicalStartOffset(), + unreferencedSegments); + } + + private static void ensureNewSegmentId(RemoteLogManifest manifest, UUID candidateId) { + for (RemoteLogSegment segment : manifest.getRemoteLogSegmentList()) { + checkArgument( + !segment.remoteLogSegmentId().equals(candidateId), + "Candidate segment id already exists in active segments: %s", + candidateId); + } + for (UnreferencedRemoteLogSegment segment : manifest.getUnreferencedRemoteLogSegments()) { + checkArgument( + !segment.remoteLogSegment().remoteLogSegmentId().equals(candidateId), + "Candidate segment id already exists in unreferenced segments: %s", + candidateId); + } + } + + private RemoteLogManifestReplacementPlanner() {} +} diff --git a/fluss-common/src/main/java/org/apache/fluss/remote/RemoteLogManifestV2Migration.java b/fluss-common/src/main/java/org/apache/fluss/remote/RemoteLogManifestV2Migration.java new file mode 100644 index 0000000000..d28a6c38ab --- /dev/null +++ b/fluss-common/src/main/java/org/apache/fluss/remote/RemoteLogManifestV2Migration.java @@ -0,0 +1,121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.remote; + +import org.apache.fluss.annotation.Internal; + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.UUID; + +import static org.apache.fluss.utils.Preconditions.checkArgument; + +/** Converts an authoritative V1 manifest into a canonical V2 manifest. */ +@Internal +public final class RemoteLogManifestV2Migration { + + public static RemoteLogManifest migrate( + RemoteLogManifest v1Manifest, long newGeneration, long unreferencedAtMs) { + checkArgument( + v1Manifest.getVersion() == RemoteLogManifest.VERSION_1, + "Only V1 manifests can be migrated"); + + List sortedSegments = + new ArrayList<>(v1Manifest.getRemoteLogSegmentList()); + sortedSegments.sort( + Comparator.comparingLong(RemoteLogSegment::remoteLogStartOffset) + .thenComparingLong(RemoteLogSegment::remoteLogEndOffset)); + + List sameStartWinners = new ArrayList<>(); + List unreferencedSegments = new ArrayList<>(); + int index = 0; + while (index < sortedSegments.size()) { + int groupEnd = index + 1; + long startOffset = sortedSegments.get(index).remoteLogStartOffset(); + while (groupEnd < sortedSegments.size() + && sortedSegments.get(groupEnd).remoteLogStartOffset() == startOffset) { + groupEnd++; + } + + RemoteLogSegment winner = sortedSegments.get(groupEnd - 1); + if (groupEnd - index > 1 + && sortedSegments.get(groupEnd - 2).remoteLogEndOffset() + == winner.remoteLogEndOffset()) { + throw new IllegalArgumentException( + "Cannot deterministically migrate V1 segments with identical start and end " + + "offsets at " + + startOffset); + } + sameStartWinners.add(winner); + for (int i = index; i < groupEnd - 1; i++) { + unreferencedSegments.add( + replaced( + sortedSegments.get(i), + winner.remoteLogSegmentId(), + unreferencedAtMs)); + } + index = groupEnd; + } + + List activeSegments = new ArrayList<>(); + for (RemoteLogSegment segment : sameStartWinners) { + if (activeSegments.isEmpty()) { + activeSegments.add(segment); + continue; + } + + RemoteLogSegment previous = activeSegments.get(activeSegments.size() - 1); + if (segment.remoteLogStartOffset() > previous.remoteLogEndOffset()) { + throw new IllegalArgumentException( + "Cannot migrate V1 manifest with gap [" + + previous.remoteLogEndOffset() + + ", " + + segment.remoteLogStartOffset() + + ")"); + } + if (segment.remoteLogEndOffset() <= previous.remoteLogEndOffset()) { + unreferencedSegments.add( + replaced(segment, previous.remoteLogSegmentId(), unreferencedAtMs)); + } else { + activeSegments.add(segment); + } + } + + Long remoteStartOffset = + activeSegments.isEmpty() ? null : activeSegments.get(0).remoteLogStartOffset(); + return RemoteLogManifest.createV2( + newGeneration, + v1Manifest.getPhysicalTablePath(), + v1Manifest.getTableBucket(), + activeSegments, + remoteStartOffset, + unreferencedSegments); + } + + private static UnreferencedRemoteLogSegment replaced( + RemoteLogSegment segment, UUID replacementSegmentId, long unreferencedAtMs) { + return new UnreferencedRemoteLogSegment( + segment, + unreferencedAtMs, + UnreferencedRemoteLogSegment.Reason.REPLACED, + replacementSegmentId); + } + + private RemoteLogManifestV2Migration() {} +} diff --git a/fluss-common/src/main/java/org/apache/fluss/remote/RemoteLogSegment.java b/fluss-common/src/main/java/org/apache/fluss/remote/RemoteLogSegment.java index 39480e4d10..6f6fcbf309 100644 --- a/fluss-common/src/main/java/org/apache/fluss/remote/RemoteLogSegment.java +++ b/fluss-common/src/main/java/org/apache/fluss/remote/RemoteLogSegment.java @@ -27,8 +27,10 @@ import static org.apache.fluss.utils.Preconditions.checkNotNull; /** - * It describes the metadata about table bucket's remote log segment in the remote storage. This is - * uniquely represent by remoteLogSegmentId. + * Describes the metadata for a table bucket's physical remote log segment. + * + *

The segment contains offsets in the half-open range {@code [remoteLogStartOffset, + * remoteLogEndOffset)} and is uniquely identified by {@code remoteLogSegmentId}. */ @Internal public class RemoteLogSegment { @@ -39,10 +41,10 @@ public class RemoteLogSegment { /** Universally unique remote log segment id. */ private final UUID remoteLogSegmentId; - /** Remote log start offset of this segment. */ + /** Inclusive physical start offset of this segment. */ private final long remoteLogStartOffset; - /** Remote log end offset of this segment. */ + /** Exclusive physical end offset of this segment. */ private final long remoteLogEndOffset; /** Max timestamp of this segment. */ @@ -70,11 +72,13 @@ private RemoteLogSegment( } this.remoteLogStartOffset = remoteLogStartOffset; - if (remoteLogEndOffset < remoteLogStartOffset) { + if (remoteLogEndOffset <= remoteLogStartOffset) { throw new IllegalArgumentException( "Unexpected remote log end offset: " + remoteLogEndOffset - + ". EndOffset for a remote segment must be greater than remote log start offset"); + + ". The exclusive end offset for a remote segment must be greater " + + "than its start offset: " + + remoteLogStartOffset); } this.remoteLogEndOffset = remoteLogEndOffset; this.maxTimestamp = maxTimestamp; @@ -94,14 +98,14 @@ public UUID remoteLogSegmentId() { } /** - * @return Remote log start offset of this segment (inclusive). + * @return inclusive physical start offset of this segment */ public long remoteLogStartOffset() { return remoteLogStartOffset; } /** - * @return Remote log end offset of this segment (inclusive). + * @return exclusive physical end offset of this segment */ public long remoteLogEndOffset() { return remoteLogEndOffset; @@ -184,13 +188,13 @@ public Builder remoteLogSegmentId(UUID remoteLogSegmentId) { return this; } - public Builder remoteLogStartOffset(long startOffset) { - this.remoteLogStartOffset = startOffset; + public Builder remoteLogStartOffset(long startOffsetInclusive) { + this.remoteLogStartOffset = startOffsetInclusive; return this; } - public Builder remoteLogEndOffset(long endOffset) { - this.remoteLogEndOffset = endOffset; + public Builder remoteLogEndOffset(long endOffsetExclusive) { + this.remoteLogEndOffset = endOffsetExclusive; return this; } diff --git a/fluss-common/src/main/java/org/apache/fluss/remote/RemoteLogSegmentReference.java b/fluss-common/src/main/java/org/apache/fluss/remote/RemoteLogSegmentReference.java new file mode 100644 index 0000000000..035cac4de5 --- /dev/null +++ b/fluss-common/src/main/java/org/apache/fluss/remote/RemoteLogSegmentReference.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.remote; + +import org.apache.fluss.annotation.Internal; + +import java.util.Objects; + +import static org.apache.fluss.utils.Preconditions.checkArgument; +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** A runtime logical reference to a physical {@link RemoteLogSegment}. */ +@Internal +public final class RemoteLogSegmentReference { + private final RemoteLogSegment remoteLogSegment; + private final long logicalStartOffset; + private final long logicalEndOffset; + + public RemoteLogSegmentReference( + RemoteLogSegment remoteLogSegment, long logicalStartOffset, long logicalEndOffset) { + this.remoteLogSegment = checkNotNull(remoteLogSegment); + checkArgument( + remoteLogSegment.remoteLogStartOffset() <= logicalStartOffset, + "Logical start offset %s is before physical start offset %s", + logicalStartOffset, + remoteLogSegment.remoteLogStartOffset()); + checkArgument( + logicalStartOffset < logicalEndOffset, + "Logical range [%s, %s) must not be empty", + logicalStartOffset, + logicalEndOffset); + checkArgument( + logicalEndOffset <= remoteLogSegment.remoteLogEndOffset(), + "Logical end offset %s is after physical end offset %s", + logicalEndOffset, + remoteLogSegment.remoteLogEndOffset()); + this.logicalStartOffset = logicalStartOffset; + this.logicalEndOffset = logicalEndOffset; + } + + public RemoteLogSegment remoteLogSegment() { + return remoteLogSegment; + } + + public long logicalStartOffset() { + return logicalStartOffset; + } + + public long logicalEndOffset() { + return logicalEndOffset; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + RemoteLogSegmentReference that = (RemoteLogSegmentReference) o; + return logicalStartOffset == that.logicalStartOffset + && logicalEndOffset == that.logicalEndOffset + && Objects.equals(remoteLogSegment, that.remoteLogSegment); + } + + @Override + public int hashCode() { + return Objects.hash(remoteLogSegment, logicalStartOffset, logicalEndOffset); + } + + @Override + public String toString() { + return "RemoteLogSegmentReference{" + + "remoteLogSegment=" + + remoteLogSegment + + ", logicalStartOffset=" + + logicalStartOffset + + ", logicalEndOffset=" + + logicalEndOffset + + '}'; + } +} diff --git a/fluss-common/src/main/java/org/apache/fluss/remote/UnreferencedRemoteLogSegment.java b/fluss-common/src/main/java/org/apache/fluss/remote/UnreferencedRemoteLogSegment.java new file mode 100644 index 0000000000..21fddbc71f --- /dev/null +++ b/fluss-common/src/main/java/org/apache/fluss/remote/UnreferencedRemoteLogSegment.java @@ -0,0 +1,111 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.remote; + +import org.apache.fluss.annotation.Internal; + +import javax.annotation.Nullable; + +import java.util.Objects; +import java.util.UUID; + +import static org.apache.fluss.utils.Preconditions.checkArgument; +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** Metadata retained for a remote segment that no longer participates in logical reads. */ +@Internal +public final class UnreferencedRemoteLogSegment { + + /** The reason why a physical remote segment stopped participating in logical reads. */ + public enum Reason { + REPLACED, + EXPIRED, + DELETE_STARTED + } + + private final RemoteLogSegment remoteLogSegment; + private final long unreferencedAtMs; + private final Reason reason; + private final @Nullable UUID replacementSegmentId; + + public UnreferencedRemoteLogSegment( + RemoteLogSegment remoteLogSegment, + long unreferencedAtMs, + Reason reason, + @Nullable UUID replacementSegmentId) { + this.remoteLogSegment = checkNotNull(remoteLogSegment); + checkArgument( + unreferencedAtMs >= 0, + "Unreferenced timestamp must not be negative: %s", + unreferencedAtMs); + this.unreferencedAtMs = unreferencedAtMs; + this.reason = checkNotNull(reason); + this.replacementSegmentId = replacementSegmentId; + } + + public RemoteLogSegment remoteLogSegment() { + return remoteLogSegment; + } + + public long unreferencedAtMs() { + return unreferencedAtMs; + } + + public Reason reason() { + return reason; + } + + @Nullable + public UUID replacementSegmentId() { + return replacementSegmentId; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + UnreferencedRemoteLogSegment that = (UnreferencedRemoteLogSegment) o; + return unreferencedAtMs == that.unreferencedAtMs + && Objects.equals(remoteLogSegment, that.remoteLogSegment) + && reason == that.reason + && Objects.equals(replacementSegmentId, that.replacementSegmentId); + } + + @Override + public int hashCode() { + return Objects.hash(remoteLogSegment, unreferencedAtMs, reason, replacementSegmentId); + } + + @Override + public String toString() { + return "UnreferencedRemoteLogSegment{" + + "remoteLogSegment=" + + remoteLogSegment + + ", unreferencedAtMs=" + + unreferencedAtMs + + ", reason=" + + reason + + ", replacementSegmentId=" + + replacementSegmentId + + '}'; + } +} diff --git a/fluss-common/src/test/java/org/apache/fluss/remote/RemoteLogManifestJsonSerdeTest.java b/fluss-common/src/test/java/org/apache/fluss/remote/RemoteLogManifestJsonSerdeTest.java index e095132158..ca82eeafa2 100644 --- a/fluss-common/src/test/java/org/apache/fluss/remote/RemoteLogManifestJsonSerdeTest.java +++ b/fluss-common/src/test/java/org/apache/fluss/remote/RemoteLogManifestJsonSerdeTest.java @@ -22,9 +22,15 @@ import org.apache.fluss.metadata.TablePath; import org.apache.fluss.utils.json.JsonSerdeTestBase; +import org.junit.jupiter.api.Test; + +import java.nio.charset.StandardCharsets; import java.util.Arrays; +import java.util.Collections; import java.util.UUID; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + /** Tests of {@link RemoteLogManifestJsonSerde}. */ class RemoteLogManifestJsonSerdeTest extends JsonSerdeTestBase { private static final PhysicalTablePath TABLE_PATH1 = @@ -46,7 +52,7 @@ class RemoteLogManifestJsonSerdeTest extends JsonSerdeTestBase deserialize("{\"database\":\"db\"}")) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("Missing required field: version"); + assertThatThrownBy(() -> deserialize("{\"version\":3}")) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("Unsupported remote log manifest version: 3"); + } + + @Test + void testRejectVersionFieldMismatchAndMissingV2Fields() { + String identityFields = + "\"database\":\"db\",\"table\":\"mytable\",\"table_id\":1001,\"bucket_id\":1"; + assertThatThrownBy( + () -> + deserialize( + "{\"version\":1,\"generation\":1," + + identityFields + + ",\"remote_log_segments\":[]}")) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("generation is not valid for manifest version 1"); + assertThatThrownBy( + () -> + deserialize( + "{\"version\":2,\"generation\":1," + + identityFields + + ",\"remote_log_segments\":[{\"segment_id\":\"aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa\",\"start_offset\":0,\"end_offset\":10,\"max_timestamp\":10,\"size_in_bytes\":10}],\"unreferenced_segments\":[]}")) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("missing remote_log_start_offset"); + } + + private static RemoteLogManifest deserialize(String json) { + return RemoteLogManifestJsonSerde.fromJson(json.getBytes(StandardCharsets.UTF_8)); + } + + private static RemoteLogSegment remoteLogSegment( + String id, + PhysicalTablePath physicalTablePath, + TableBucket tableBucket, + long startOffset, + long endOffset) { + return RemoteLogSegment.Builder.builder() + .physicalTablePath(physicalTablePath) + .tableBucket(tableBucket) + .remoteLogSegmentId(UUID.fromString(id)) + .remoteLogStartOffset(startOffset) + .remoteLogEndOffset(endOffset) + .maxTimestamp(endOffset) + .segmentSizeInBytes((int) (endOffset - startOffset)) + .build(); } } diff --git a/fluss-common/src/test/java/org/apache/fluss/remote/RemoteLogManifestV2Test.java b/fluss-common/src/test/java/org/apache/fluss/remote/RemoteLogManifestV2Test.java new file mode 100644 index 0000000000..c11bdea33c --- /dev/null +++ b/fluss-common/src/test/java/org/apache/fluss/remote/RemoteLogManifestV2Test.java @@ -0,0 +1,335 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.remote; + +import org.apache.fluss.metadata.PhysicalTablePath; +import org.apache.fluss.metadata.TableBucket; +import org.apache.fluss.metadata.TablePath; + +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.UUID; + +import static org.apache.fluss.remote.RemoteLogManifestReplacementPlanner.PlanType.ALREADY_COVERED; +import static org.apache.fluss.remote.RemoteLogManifestReplacementPlanner.PlanType.APPEND; +import static org.apache.fluss.remote.RemoteLogManifestReplacementPlanner.PlanType.GAP; +import static org.apache.fluss.remote.RemoteLogManifestReplacementPlanner.PlanType.INITIAL_COPY; +import static org.apache.fluss.remote.RemoteLogManifestReplacementPlanner.PlanType.REPLACE_AND_CLIP; +import static org.apache.fluss.remote.RemoteLogManifestReplacementPlanner.PlanType.REPLACE_AND_CLIP_START; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Tests for the Manifest V2 model, migration, and replacement planner. */ +class RemoteLogManifestV2Test { + private static final PhysicalTablePath PHYSICAL_TABLE_PATH = + PhysicalTablePath.of(TablePath.of("database", "table")); + private static final TableBucket TABLE_BUCKET = new TableBucket(1L, 0); + private static final long UNREFERENCED_AT_MS = 1000L; + + @Test + void testDeriveLogicalReferences() { + RemoteLogSegment segmentA = segment(0, 10); + RemoteLogSegment segmentB = segment(5, 20); + RemoteLogManifest manifest = v2(8L, 0L, segmentA, segmentB); + + assertThat(manifest.getRemoteLogStartOffset()).isEqualTo(0L); + assertThat(manifest.getRemoteLogEndOffset()).isEqualTo(20L); + assertThat(manifest.getRemoteLogSegmentReferences()) + .containsExactly( + new RemoteLogSegmentReference(segmentA, 0L, 5L), + new RemoteLogSegmentReference(segmentB, 5L, 20L)); + assertThat(manifest.getRemoteLogSegmentReferences()) + .isSameAs(manifest.getRemoteLogSegmentReferences()); + } + + @Test + void testCachedLogicalViewUsesDefensiveSegmentCopy() { + RemoteLogSegment segment = segment(0, 10); + List inputSegments = new ArrayList<>(); + inputSegments.add(segment); + RemoteLogManifest manifest = + new RemoteLogManifest(PHYSICAL_TABLE_PATH, TABLE_BUCKET, inputSegments); + List references = manifest.getRemoteLogSegmentReferences(); + + inputSegments.clear(); + + assertThat(manifest.getRemoteLogSegmentList()).containsExactly(segment); + assertThat(manifest.getRemoteLogSegmentReferences()).isSameAs(references); + assertThat(references).containsExactly(new RemoteLogSegmentReference(segment, 0L, 10L)); + assertThat(manifest.getRemoteLogStartOffset()).isEqualTo(0L); + assertThat(manifest.getRemoteLogEndOffset()).isEqualTo(10L); + } + + @Test + void testV1CachedBoundsPreserveMinMaxCompatibility() { + RemoteLogManifest manifest = + new RemoteLogManifest( + PHYSICAL_TABLE_PATH, + TABLE_BUCKET, + Arrays.asList(segment(10, 30), segment(0, 20))); + + assertThat(manifest.getRemoteLogStartOffset()).isEqualTo(0L); + assertThat(manifest.getRemoteLogEndOffset()).isEqualTo(30L); + } + + @Test + void testDeriveStartClippedReference() { + RemoteLogSegment segment = segment(50, 250); + RemoteLogManifest manifest = v2(2L, 100L, segment); + + assertThat(manifest.getRemoteLogSegmentReferences()) + .containsExactly(new RemoteLogSegmentReference(segment, 100L, 250L)); + } + + @Test + void testRejectInvalidV2Manifests() { + RemoteLogSegment segmentA = segment(0, 10); + RemoteLogSegment segmentWithGap = segment(11, 20); + RemoteLogSegment segmentWithCoveredEnd = segment(5, 8); + + assertThatThrownBy( + () -> + RemoteLogManifest.createV2( + 1L, + PHYSICAL_TABLE_PATH, + TABLE_BUCKET, + Collections.singletonList(segmentA), + null, + Collections.emptyList())) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("must define remote log start offset"); + assertThatThrownBy(() -> v2(1L, 0L, segmentA, segmentWithGap)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("gap"); + assertThatThrownBy(() -> v2(1L, 0L, segmentA, segmentWithCoveredEnd)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("end offsets must be strictly increasing"); + + UnreferencedRemoteLogSegment alsoUnreferenced = + new UnreferencedRemoteLogSegment( + segmentA, + UNREFERENCED_AT_MS, + UnreferencedRemoteLogSegment.Reason.REPLACED, + null); + assertThatThrownBy( + () -> + RemoteLogManifest.createV2( + 1L, + PHYSICAL_TABLE_PATH, + TABLE_BUCKET, + Collections.singletonList(segmentA), + 0L, + Collections.singletonList(alsoUnreferenced))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("both active and unreferenced"); + } + + @Test + void testMigrateV1ManifestWithSameStartAndCoveredSegments() { + RemoteLogSegment shortSameStart = segment(0, 10); + RemoteLogSegment winner = segment(0, 20); + RemoteLogSegment fullyCovered = segment(5, 15); + RemoteLogSegment extending = segment(10, 30); + RemoteLogManifest v1 = + new RemoteLogManifest( + PHYSICAL_TABLE_PATH, + TABLE_BUCKET, + Arrays.asList(fullyCovered, extending, shortSameStart, winner)); + + RemoteLogManifest migrated = + RemoteLogManifestV2Migration.migrate(v1, 1L, UNREFERENCED_AT_MS); + + assertThat(migrated.getVersion()).isEqualTo(RemoteLogManifest.VERSION_2); + assertThat(migrated.getGeneration()).isEqualTo(1L); + assertThat(migrated.getRemoteLogSegmentList()).containsExactly(winner, extending); + assertThat(migrated.getUnreferencedRemoteLogSegments()) + .extracting(UnreferencedRemoteLogSegment::remoteLogSegment) + .containsExactlyInAnyOrder(shortSameStart, fullyCovered); + } + + @Test + void testRejectAmbiguousOrGappedV1Migration() { + RemoteLogSegment sameRangeA = segment(0, 10); + RemoteLogSegment sameRangeB = segment(0, 10); + RemoteLogManifest ambiguous = + new RemoteLogManifest( + PHYSICAL_TABLE_PATH, TABLE_BUCKET, Arrays.asList(sameRangeA, sameRangeB)); + + assertThatThrownBy( + () -> + RemoteLogManifestV2Migration.migrate( + ambiguous, 1L, UNREFERENCED_AT_MS)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("Cannot deterministically migrate"); + + RemoteLogManifest gapped = + new RemoteLogManifest( + PHYSICAL_TABLE_PATH, + TABLE_BUCKET, + Arrays.asList(segment(0, 10), segment(11, 20))); + assertThatThrownBy( + () -> RemoteLogManifestV2Migration.migrate(gapped, 1L, UNREFERENCED_AT_MS)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("gap"); + } + + @Test + void testPlanInitialAppendAlreadyCoveredAndGap() { + RemoteLogManifest empty = + RemoteLogManifest.createV2( + 1L, + PHYSICAL_TABLE_PATH, + TABLE_BUCKET, + Collections.emptyList(), + null, + Collections.emptyList()); + RemoteLogSegment first = segment(0, 10); + RemoteLogManifestReplacementPlanner.Result initial = plan(empty, first); + assertThat(initial.planType()).isEqualTo(INITIAL_COPY); + assertThat(initial.resultManifest().getGeneration()).isEqualTo(2L); + + RemoteLogSegment append = segment(10, 20); + RemoteLogManifestReplacementPlanner.Result appended = + plan(initial.resultManifest(), append); + assertThat(appended.planType()).isEqualTo(APPEND); + assertThat(appended.resultManifest().getRemoteLogSegmentList()) + .containsExactly(first, append); + + RemoteLogManifestReplacementPlanner.Result covered = + plan(appended.resultManifest(), segment(5, 15)); + assertThat(covered.planType()).isEqualTo(ALREADY_COVERED); + assertThat(covered.requiresManifestCommit()).isFalse(); + + RemoteLogManifestReplacementPlanner.Result gap = + plan(appended.resultManifest(), segment(21, 30)); + assertThat(gap.planType()).isEqualTo(GAP); + assertThat(gap.requiresManifestCommit()).isFalse(); + } + + @Test + void testPlanPartialAndMultiSegmentReplacement() { + RemoteLogSegment segmentA = segment(0, 10); + RemoteLogSegment segmentC = segment(10, 15); + RemoteLogManifest manifest = v2(1L, 0L, segmentA, segmentC); + RemoteLogSegment replacement = segment(5, 20); + + RemoteLogManifestReplacementPlanner.Result result = plan(manifest, replacement); + + assertThat(result.planType()).isEqualTo(REPLACE_AND_CLIP); + assertThat(result.resultManifest().getRemoteLogSegmentList()) + .containsExactly(segmentA, replacement); + assertThat(result.resultManifest().getRemoteLogSegmentReferences()) + .containsExactly( + new RemoteLogSegmentReference(segmentA, 0L, 5L), + new RemoteLogSegmentReference(replacement, 5L, 20L)); + assertThat(result.segmentsToUnreference()).containsExactly(segmentC); + } + + @Test + void testPlanSameStartAndStartClippedReplacement() { + RemoteLogSegment old = segment(0, 10); + RemoteLogSegment sameStart = segment(0, 20); + RemoteLogManifestReplacementPlanner.Result sameStartResult = + plan(v2(1L, 0L, old), sameStart); + + assertThat(sameStartResult.planType()).isEqualTo(REPLACE_AND_CLIP); + assertThat(sameStartResult.resultManifest().getRemoteLogSegmentList()) + .containsExactly(sameStart); + assertThat(sameStartResult.segmentsToUnreference()).containsExactly(old); + + RemoteLogSegment startClipped = segment(50, 250); + RemoteLogManifestReplacementPlanner.Result startClippedResult = + plan(v2(1L, 100L, segment(100, 200)), startClipped); + assertThat(startClippedResult.planType()).isEqualTo(REPLACE_AND_CLIP_START); + assertThat(startClippedResult.resultManifest().getRemoteLogStartOffset()).isEqualTo(100L); + assertThat(startClippedResult.resultManifest().getRemoteLogSegmentReferences()) + .containsExactly(new RemoteLogSegmentReference(startClipped, 100L, 250L)); + } + + @Test + void testExpireOnlyContinuousLogicalPrefixWithoutDeletingObjects() { + RemoteLogSegment first = segment(0, 10); + RemoteLogSegment overlapping = segment(5, 20); + RemoteLogSegment newest = segment(20, 30); + RemoteLogManifest manifest = v2(3L, 0L, first, overlapping, newest); + + RemoteLogManifest expired = + RemoteLogManifestReplacementPlanner.expireContinuousPrefix( + manifest, 100L, 85L, null, UNREFERENCED_AT_MS); + + assertThat(expired.getGeneration()).isEqualTo(3L); + assertThat(expired.getRemoteLogStartOffset()).isEqualTo(5L); + assertThat(expired.getRemoteLogSegmentList()).containsExactly(overlapping, newest); + assertThat(expired.getUnreferencedRemoteLogSegments()) + .extracting(UnreferencedRemoteLogSegment::remoteLogSegment) + .containsExactly(first); + assertThat(expired.getUnreferencedRemoteLogSegments()) + .extracting(UnreferencedRemoteLogSegment::reason) + .containsExactly(UnreferencedRemoteLogSegment.Reason.EXPIRED); + } + + @Test + void testExpirationStopsAtFirstIneligibleReferenceAndRetainsLastReference() { + RemoteLogSegment first = segment(0, 10); + RemoteLogSegment second = segment(10, 20); + RemoteLogManifest twoSegments = v2(1L, 0L, first, second); + + assertThat( + RemoteLogManifestReplacementPlanner.expireContinuousPrefix( + twoSegments, 100L, 0L, 5L, UNREFERENCED_AT_MS)) + .isSameAs(twoSegments); + + RemoteLogManifest onlyOne = v2(1L, 0L, first); + assertThat( + RemoteLogManifestReplacementPlanner.expireContinuousPrefix( + onlyOne, 100L, 0L, null, UNREFERENCED_AT_MS)) + .isSameAs(onlyOne); + } + + private RemoteLogManifestReplacementPlanner.Result plan( + RemoteLogManifest manifest, RemoteLogSegment segment) { + return RemoteLogManifestReplacementPlanner.plan(manifest, segment, UNREFERENCED_AT_MS); + } + + private RemoteLogManifest v2( + long generation, Long remoteStartOffset, RemoteLogSegment... segments) { + return RemoteLogManifest.createV2( + generation, + PHYSICAL_TABLE_PATH, + TABLE_BUCKET, + Arrays.asList(segments), + remoteStartOffset, + Collections.emptyList()); + } + + private RemoteLogSegment segment(long startOffset, long endOffset) { + return RemoteLogSegment.Builder.builder() + .physicalTablePath(PHYSICAL_TABLE_PATH) + .tableBucket(TABLE_BUCKET) + .remoteLogSegmentId(UUID.randomUUID()) + .remoteLogStartOffset(startOffset) + .remoteLogEndOffset(endOffset) + .maxTimestamp(endOffset) + .segmentSizeInBytes(1) + .build(); + } +} diff --git a/fluss-common/src/test/java/org/apache/fluss/remote/RemoteLogSegmentTest.java b/fluss-common/src/test/java/org/apache/fluss/remote/RemoteLogSegmentTest.java new file mode 100644 index 0000000000..917a98a66f --- /dev/null +++ b/fluss-common/src/test/java/org/apache/fluss/remote/RemoteLogSegmentTest.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.remote; + +import org.apache.fluss.metadata.PhysicalTablePath; +import org.apache.fluss.metadata.TableBucket; +import org.apache.fluss.metadata.TablePath; + +import org.junit.jupiter.api.Test; + +import java.util.UUID; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Tests for {@link RemoteLogSegment}. */ +class RemoteLogSegmentTest { + private static final PhysicalTablePath PHYSICAL_TABLE_PATH = + PhysicalTablePath.of(TablePath.of("database", "table")); + private static final TableBucket TABLE_BUCKET = new TableBucket(1L, 0); + + @Test + void testHalfOpenOffsetRange() { + RemoteLogSegment segment = createRemoteLogSegment(10L, 20L); + + assertThat(segment.remoteLogStartOffset()).isEqualTo(10L); + assertThat(segment.remoteLogEndOffset()).isEqualTo(20L); + } + + @Test + void testRejectEmptyOffsetRange() { + assertThatThrownBy(() -> createRemoteLogSegment(10L, 10L)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("exclusive end offset"); + } + + @Test + void testRejectReversedOffsetRange() { + assertThatThrownBy(() -> createRemoteLogSegment(20L, 10L)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("exclusive end offset"); + } + + private RemoteLogSegment createRemoteLogSegment(long startOffset, long endOffset) { + return RemoteLogSegment.Builder.builder() + .physicalTablePath(PHYSICAL_TABLE_PATH) + .tableBucket(TABLE_BUCKET) + .remoteLogSegmentId(UUID.randomUUID()) + .remoteLogStartOffset(startOffset) + .remoteLogEndOffset(endOffset) + .maxTimestamp(1L) + .segmentSizeInBytes(1) + .build(); + } +} diff --git a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/OrphanFilesCleanITCase.java b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/OrphanFilesCleanITCase.java index 6128495b2f..4d9df2b6f1 100644 --- a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/OrphanFilesCleanITCase.java +++ b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/action/orphan/OrphanFilesCleanITCase.java @@ -194,16 +194,16 @@ void mixedOrphanAndActiveFilesInSameBucket() throws Exception { + "\"remote_log_segments\":[" + "{\"segment_id\":\"" + activeId1 - + "\",\"start_offset\":0,\"end_offset\":99," + + "\",\"start_offset\":0,\"end_offset\":100," + "\"max_timestamp\":0,\"size_in_bytes\":1}," + "{\"segment_id\":\"" + activeId2 - + "\",\"start_offset\":100,\"end_offset\":199," + + "\",\"start_offset\":100,\"end_offset\":200," + "\"max_timestamp\":0,\"size_in_bytes\":1}" + "]}"; Files.write(manifest, manifestContent.getBytes(StandardCharsets.UTF_8)); makeOld(manifest); - upsertManifest(tableBucket, manifestPath, 199L); + upsertManifest(tableBucket, manifestPath, 200L); Path activeFile1 = writeSegmentFile(remoteLogTabletDir, activeId1, 0L); Path activeFile2 = writeSegmentFile(remoteLogTabletDir, activeId2, 100L); @@ -299,8 +299,8 @@ private Path seedActiveBucketManifest(TablePath tablePath) throws Exception { .toString()); String activeSegmentId = UUID.randomUUID().toString(); Path activeSegment = - seedManifestAndSegment(remoteLogTabletDir, manifestPath, activeSegmentId, 0L, 0L); - upsertManifest(tableBucket, manifestPath, 0L); + seedManifestAndSegment(remoteLogTabletDir, manifestPath, activeSegmentId, 0L, 1L); + upsertManifest(tableBucket, manifestPath, 1L); return activeSegment; } @@ -512,8 +512,8 @@ void multipleRoundsConvergeAfterManifestUpsert() throws Exception { .resolve("metadata/p0.manifest") .toUri() .toString()); - Path oldSegment = seedManifestAndSegment(remoteLogTabletDir, manifest0, segmentId, 0L, 0L); - upsertManifest(tableBucket, manifest0, 0L); + Path oldSegment = seedManifestAndSegment(remoteLogTabletDir, manifest0, segmentId, 0L, 1L); + upsertManifest(tableBucket, manifest0, 1L); runCleanerForDatabase(false, dbName); @@ -526,8 +526,8 @@ void multipleRoundsConvergeAfterManifestUpsert() throws Exception { .toUri() .toString()); Path newSegment = - seedManifestAndSegment(remoteLogTabletDir, manifest1, segmentId, 100L, 100L); - upsertManifest(tableBucket, manifest1, 100L); + seedManifestAndSegment(remoteLogTabletDir, manifest1, segmentId, 100L, 101L); + upsertManifest(tableBucket, manifest1, 101L); runCleanerForDatabase(false, dbName); diff --git a/fluss-rpc/src/main/proto/FlussApi.proto b/fluss-rpc/src/main/proto/FlussApi.proto index 2e6104b4cf..371d8e58ef 100644 --- a/fluss-rpc/src/main/proto/FlussApi.proto +++ b/fluss-rpc/src/main/proto/FlussApi.proto @@ -565,10 +565,21 @@ message CommitRemoteLogManifestRequest { required int64 remote_log_end_offset = 6; required int32 coordinator_epoch = 7; required int32 bucket_leader_epoch = 8; + // Manifest V2 CAS fields. All are omitted by the legacy V1 commit path. + optional int32 manifest_format_version = 9; + // 0 = ABSENT, 1 = PRESENT. + optional int32 expected_handle_state = 10; + optional string expected_manifest_path = 11; + optional int64 expected_manifest_generation = 12; + optional int32 expected_zk_version = 13; + optional int64 new_manifest_generation = 14; } message CommitRemoteLogManifestResponse { required bool commitSuccess = 1; + // 0 = COMMITTED, 1 = CONFLICT, 2 = FENCED, 3 = INVALID_MANIFEST. + // Omitted by legacy coordinators. + optional int32 commit_result = 2; } // notify remote log offsets request and response diff --git a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorEventProcessor.java b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorEventProcessor.java index c5b90f5904..ca0499b9d6 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorEventProcessor.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorEventProcessor.java @@ -105,6 +105,8 @@ import org.apache.fluss.server.entity.CommitRemoteLogManifestData; import org.apache.fluss.server.entity.DeleteReplicaResultForBucket; import org.apache.fluss.server.entity.NotifyLeaderAndIsrResultForBucket; +import org.apache.fluss.server.entity.RemoteLogManifestCommitResult; +import org.apache.fluss.server.entity.RemoteLogManifestExpectedHandleState; import org.apache.fluss.server.kv.snapshot.CompletedSnapshot; import org.apache.fluss.server.kv.snapshot.CompletedSnapshotStore; import org.apache.fluss.server.metadata.CoordinatorMetadataCache; @@ -120,6 +122,7 @@ import org.apache.fluss.server.zk.data.ServerTags; import org.apache.fluss.server.zk.data.TableAssignment; import org.apache.fluss.server.zk.data.TabletServerRegistration; +import org.apache.fluss.server.zk.data.VersionedRemoteLogManifestHandle; import org.apache.fluss.server.zk.data.ZkData.PartitionIdsZNode; import org.apache.fluss.server.zk.data.ZkData.TableIdsZNode; import org.apache.fluss.server.zk.data.lake.LakeTableHelper; @@ -194,6 +197,7 @@ public class CoordinatorEventProcessor implements EventProcessor { private final RebalanceManager rebalanceManager; private final Scheduler scheduler; private final long offlineLeaderRetryDelayMs; + private final boolean manifestV2WriterEnabled; private final CompletedSnapshotStoreManager completedSnapshotStoreManager; private final LakeTableHelper lakeTableHelper; private ScheduledFuture offlineLeaderRetryTask; @@ -275,6 +279,8 @@ public CoordinatorEventProcessor( this, zooKeeperClient, coordinatorEventManager, SystemClock.getInstance()); this.offlineLeaderRetryDelayMs = conf.get(ConfigOptions.COORDINATOR_OFFLINE_LEADER_RETRY_DELAY).toMillis(); + this.manifestV2WriterEnabled = + conf.getBoolean(ConfigOptions.REMOTE_LOG_MANIFEST_V2_WRITER_ENABLED); if (offlineLeaderRetryDelayMs <= 0) { throw new IllegalArgumentException( String.format( @@ -389,13 +395,23 @@ private void initCoordinatorContext() throws Exception { zooKeeperClient.getTabletServers(currentServers); for (int server : currentServers) { TabletServerRegistration registration = tabletServerRegistrations.get(server); + if (manifestV2WriterEnabled + && !registration.supportsCapability( + TabletServerRegistration.REMOTE_MANIFEST_VERSION_DISPATCH_CAPABILITY)) { + LOG.error( + "Excluding tablet server {} because Manifest V2 writer is enabled but " + + "the server lacks version-dispatch capability.", + server); + continue; + } ServerInfo serverInfo = new ServerInfo( server, registration.getRack(), registration.getEndpoints(), ServerType.TABLET_SERVER, - registration.getResource()); + registration.getResource(), + registration.getCapabilities()); // Get internal listener endpoint to send request to tablet server. Endpoint internalEndpoint = serverInfo.endpoint(internalListenerName); if (internalEndpoint == null) { @@ -1291,6 +1307,15 @@ private void processNewTabletServer(NewTabletServerEvent newTabletServerEvent) { // when we finish the logic of tablet server ServerInfo serverInfo = newTabletServerEvent.getServerInfo(); int tabletServerId = serverInfo.id(); + if (manifestV2WriterEnabled + && !serverInfo.supportsCapability( + TabletServerRegistration.REMOTE_MANIFEST_VERSION_DISPATCH_CAPABILITY)) { + LOG.error( + "Rejecting tablet server {} while Manifest V2 writer is enabled because it " + + "lacks version-dispatch capability.", + tabletServerId); + return; + } if (coordinatorContext.getLiveTabletServers().containsKey(serverInfo.id())) { // if the dead server is already in live servers, return directly // it may happen during coordinator server initiation, the watcher watch a new tablet @@ -2142,23 +2167,84 @@ private CommitRemoteLogManifestResponse tryProcessCommitRemoteLogManifest( CommitRemoteLogManifestData manifestData = event.getCommitRemoteLogManifestData(); CommitRemoteLogManifestResponse response = new CommitRemoteLogManifestResponse(); TableBucket tb = event.getTableBucket(); - try { - validateFencedEvent(event); - // do commit remote log manifest snapshot path to zk. - zooKeeperClient.upsertRemoteLogManifestHandle( - tb, - new RemoteLogManifestHandle( + if (manifestData.isV2CasCommit()) { + if (!manifestV2WriterEnabled || !allAssignedReplicasSupportManifestV2(tb)) { + LOG.error( + "Rejected Manifest V2 commit for {} because the writer gate is disabled " + + "or an assigned replica lacks version-dispatch capability.", + tb); + return response.setCommitSuccess(false) + .setCommitResult(RemoteLogManifestCommitResult.INVALID_MANIFEST.code()); + } + try { + validateFencedEvent(event); + } catch (RuntimeException e) { + LOG.warn("Rejected fenced remote log manifest commit for {}.", tb, e); + return response.setCommitSuccess(false) + .setCommitResult(RemoteLogManifestCommitResult.FENCED.code()); + } + + RemoteLogManifestHandle newHandle = + RemoteLogManifestHandle.v2( manifestData.getRemoteLogManifestPath(), - manifestData.getRemoteLogEndOffset())); - } catch (Exception e) { - LOG.error( - "Error when commit remote log manifest, the leader need to revert the commit.", - e); - response.setCommitSuccess(false); - return response; + manifestData.getNewManifestGeneration(), + manifestData.getRemoteLogStartOffset(), + manifestData.getRemoteLogEndOffset()); + boolean committed; + try { + if (manifestData.getExpectedHandleState() + == RemoteLogManifestExpectedHandleState.ABSENT) { + committed = + zooKeeperClient.createRemoteLogManifestHandleIfAbsent(tb, newHandle); + } else { + committed = + zooKeeperClient.compareAndSetRemoteLogManifestHandle( + tb, + manifestData.getExpectedManifestPath(), + manifestData.getExpectedManifestGeneration(), + manifestData.getExpectedZkVersion(), + newHandle); + } + } catch (Exception e) { + // An I/O/transport failure is UNKNOWN. Complete exceptionally so the writer must + // reconcile against the authoritative handle before retrying or cleaning up. + throw new RuntimeException( + "Unable to determine remote log manifest CAS result for " + tb, e); + } + if (!committed) { + return response.setCommitSuccess(false) + .setCommitResult(RemoteLogManifestCommitResult.CONFLICT.code()); + } + response.setCommitSuccess(true) + .setCommitResult(RemoteLogManifestCommitResult.COMMITTED.code()); + } else { + try { + validateFencedEvent(event); + Optional currentHandle = + zooKeeperClient.getVersionedRemoteLogManifestHandle(tb); + if (currentHandle.isPresent() + && currentHandle.get().handle().getVersion() + == RemoteLogManifestHandle.VERSION_2) { + LOG.error( + "Rejected legacy remote log writer for {} because Manifest V2 is " + + "already authoritative.", + tb); + return response.setCommitSuccess(false); + } + zooKeeperClient.upsertRemoteLogManifestHandle( + tb, + new RemoteLogManifestHandle( + manifestData.getRemoteLogManifestPath(), + manifestData.getRemoteLogEndOffset())); + } catch (Exception e) { + LOG.error( + "Error when commit remote log manifest, the leader need to revert the commit.", + e); + response.setCommitSuccess(false); + return response; + } + response.setCommitSuccess(true); } - - response.setCommitSuccess(true); // send notify remote log offsets request to all replicas. coordinatorRequestBatch.newBatch(); coordinatorContext @@ -2177,6 +2263,32 @@ private CommitRemoteLogManifestResponse tryProcessCommitRemoteLogManifest( return response; } + private boolean allAssignedReplicasSupportManifestV2(TableBucket tableBucket) { + List assignedReplicas = coordinatorContext.getAssignment(tableBucket); + if (assignedReplicas.isEmpty()) { + return false; + } + try { + int[] replicaIds = assignedReplicas.stream().mapToInt(Integer::intValue).toArray(); + Map registrations = + zooKeeperClient.getTabletServers(replicaIds); + for (Integer replicaId : assignedReplicas) { + TabletServerRegistration registration = registrations.get(replicaId); + if (registration == null + || !registration.supportsCapability( + TabletServerRegistration + .REMOTE_MANIFEST_VERSION_DISPATCH_CAPABILITY)) { + return false; + } + } + return true; + } catch (Exception e) { + LOG.error( + "Failed to verify Manifest V2 capability for replicas of {}.", tableBucket, e); + return false; + } + } + private void processAccessContext(AccessContextEvent event) { try { T result = event.getAccessFunction().apply(coordinatorContext); diff --git a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorService.java b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorService.java index ef6a735677..f40952e864 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorService.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorService.java @@ -164,8 +164,10 @@ import org.apache.fluss.server.coordinator.rebalance.goal.Goal; import org.apache.fluss.server.coordinator.remote.RemoteDirDynamicLoader; import org.apache.fluss.server.entity.CommitKvSnapshotData; +import org.apache.fluss.server.entity.CommitRemoteLogManifestData; import org.apache.fluss.server.entity.DatabasePropertyChanges; import org.apache.fluss.server.entity.LakeTieringTableInfo; +import org.apache.fluss.server.entity.RemoteLogManifestCommitResult; import org.apache.fluss.server.entity.TablePropertyChanges; import org.apache.fluss.server.kv.snapshot.CompletedSnapshot; import org.apache.fluss.server.kv.snapshot.CompletedSnapshotJsonSerde; @@ -1014,11 +1016,17 @@ public CompletableFuture commitKvSnapshot( public CompletableFuture commitRemoteLogManifest( CommitRemoteLogManifestRequest request) { CompletableFuture response = new CompletableFuture<>(); - eventManagerSupplier - .get() - .put( - new CommitRemoteLogManifestEvent( - getCommitRemoteLogManifestData(request), response)); + CommitRemoteLogManifestData commitData; + try { + commitData = getCommitRemoteLogManifestData(request); + } catch (IllegalArgumentException e) { + return CompletableFuture.completedFuture( + new CommitRemoteLogManifestResponse() + .setCommitSuccess(false) + .setCommitResult( + RemoteLogManifestCommitResult.INVALID_MANIFEST.code())); + } + eventManagerSupplier.get().put(new CommitRemoteLogManifestEvent(commitData, response)); return response; } diff --git a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/event/watcher/TabletServerChangeWatcher.java b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/event/watcher/TabletServerChangeWatcher.java index 6fe5e39e5d..544a70c53f 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/event/watcher/TabletServerChangeWatcher.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/event/watcher/TabletServerChangeWatcher.java @@ -87,7 +87,8 @@ public void event(Type type, ChildData oldData, ChildData newData) { registration.getRack(), registration.getEndpoints(), ServerType.TABLET_SERVER, - registration.getResource()))); + registration.getResource(), + registration.getCapabilities()))); } break; } diff --git a/fluss-server/src/main/java/org/apache/fluss/server/entity/CommitRemoteLogManifestData.java b/fluss-server/src/main/java/org/apache/fluss/server/entity/CommitRemoteLogManifestData.java index 9b7e946f9d..9918853be6 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/entity/CommitRemoteLogManifestData.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/entity/CommitRemoteLogManifestData.java @@ -21,8 +21,12 @@ import org.apache.fluss.metadata.TableBucket; import org.apache.fluss.rpc.messages.CommitRemoteLogManifestRequest; +import javax.annotation.Nullable; + import java.util.Objects; +import static org.apache.fluss.utils.Preconditions.checkArgument; + /** The data for request {@link CommitRemoteLogManifestRequest}. */ public class CommitRemoteLogManifestData { @@ -44,6 +48,13 @@ public class CommitRemoteLogManifestData { /** The leader epoch of the bucket when the snapshot is triggered. */ private final int bucketLeaderEpoch; + private final @Nullable Integer manifestFormatVersion; + private final @Nullable RemoteLogManifestExpectedHandleState expectedHandleState; + private final @Nullable FsPath expectedManifestPath; + private final @Nullable Long expectedManifestGeneration; + private final @Nullable Integer expectedZkVersion; + private final @Nullable Long newManifestGeneration; + public CommitRemoteLogManifestData( TableBucket tableBucket, FsPath remoteLogManifestPath, @@ -51,12 +62,131 @@ public CommitRemoteLogManifestData( long remoteLogEndOffset, int coordinatorEpoch, int bucketLeaderEpoch) { + this( + tableBucket, + remoteLogManifestPath, + remoteLogStartOffset, + remoteLogEndOffset, + coordinatorEpoch, + bucketLeaderEpoch, + null, + null, + null, + null, + null, + null); + } + + private CommitRemoteLogManifestData( + TableBucket tableBucket, + FsPath remoteLogManifestPath, + long remoteLogStartOffset, + long remoteLogEndOffset, + int coordinatorEpoch, + int bucketLeaderEpoch, + @Nullable Integer manifestFormatVersion, + @Nullable RemoteLogManifestExpectedHandleState expectedHandleState, + @Nullable FsPath expectedManifestPath, + @Nullable Long expectedManifestGeneration, + @Nullable Integer expectedZkVersion, + @Nullable Long newManifestGeneration) { + if (manifestFormatVersion == null) { + checkArgument( + expectedHandleState == null + && expectedManifestPath == null + && expectedManifestGeneration == null + && expectedZkVersion == null + && newManifestGeneration == null, + "Legacy manifest commit must not contain V2 CAS fields"); + } else { + checkArgument(manifestFormatVersion == 2, "Only Manifest V2 supports CAS publish"); + checkArgument(expectedHandleState != null, "Expected handle state is required"); + checkArgument( + newManifestGeneration != null && newManifestGeneration > 0L, + "New manifest generation must be positive"); + checkArgument( + remoteLogStartOffset < remoteLogEndOffset, + "V2 remote log offsets must form a non-empty half-open range"); + if (expectedHandleState == RemoteLogManifestExpectedHandleState.ABSENT) { + checkArgument( + expectedManifestPath == null + && expectedManifestGeneration == null + && expectedZkVersion == null, + "ABSENT commit must not contain expected handle fields"); + checkArgument( + newManifestGeneration == 1L, "Initial Manifest V2 generation must be 1"); + } else { + checkArgument( + expectedManifestPath != null + && expectedManifestGeneration != null + && expectedZkVersion != null, + "PRESENT commit requires expected path, generation, and ZK version"); + checkArgument( + newManifestGeneration == expectedManifestGeneration + 1L, + "New generation must equal expected generation + 1"); + } + } this.tableBucket = tableBucket; this.remoteLogManifestPath = remoteLogManifestPath; this.remoteLogStartOffset = remoteLogStartOffset; this.remoteLogEndOffset = remoteLogEndOffset; this.coordinatorEpoch = coordinatorEpoch; this.bucketLeaderEpoch = bucketLeaderEpoch; + this.manifestFormatVersion = manifestFormatVersion; + this.expectedHandleState = expectedHandleState; + this.expectedManifestPath = expectedManifestPath; + this.expectedManifestGeneration = expectedManifestGeneration; + this.expectedZkVersion = expectedZkVersion; + this.newManifestGeneration = newManifestGeneration; + } + + public static CommitRemoteLogManifestData v2Absent( + TableBucket tableBucket, + FsPath remoteLogManifestPath, + long remoteLogStartOffset, + long remoteLogEndOffset, + long newManifestGeneration, + int coordinatorEpoch, + int bucketLeaderEpoch) { + return new CommitRemoteLogManifestData( + tableBucket, + remoteLogManifestPath, + remoteLogStartOffset, + remoteLogEndOffset, + coordinatorEpoch, + bucketLeaderEpoch, + 2, + RemoteLogManifestExpectedHandleState.ABSENT, + null, + null, + null, + newManifestGeneration); + } + + public static CommitRemoteLogManifestData v2Present( + TableBucket tableBucket, + FsPath remoteLogManifestPath, + long remoteLogStartOffset, + long remoteLogEndOffset, + long newManifestGeneration, + FsPath expectedManifestPath, + long expectedManifestGeneration, + int expectedZkVersion, + int coordinatorEpoch, + int bucketLeaderEpoch) { + return new CommitRemoteLogManifestData( + tableBucket, + remoteLogManifestPath, + remoteLogStartOffset, + remoteLogEndOffset, + coordinatorEpoch, + bucketLeaderEpoch, + 2, + RemoteLogManifestExpectedHandleState.PRESENT, + expectedManifestPath, + expectedManifestGeneration, + expectedZkVersion, + newManifestGeneration); } public TableBucket getTableBucket() { @@ -83,6 +213,34 @@ public int getBucketLeaderEpoch() { return bucketLeaderEpoch; } + public boolean isV2CasCommit() { + return manifestFormatVersion != null; + } + + public @Nullable Integer getManifestFormatVersion() { + return manifestFormatVersion; + } + + public @Nullable RemoteLogManifestExpectedHandleState getExpectedHandleState() { + return expectedHandleState; + } + + public @Nullable FsPath getExpectedManifestPath() { + return expectedManifestPath; + } + + public @Nullable Long getExpectedManifestGeneration() { + return expectedManifestGeneration; + } + + public @Nullable Integer getExpectedZkVersion() { + return expectedZkVersion; + } + + public @Nullable Long getNewManifestGeneration() { + return newManifestGeneration; + } + @Override public String toString() { return "CommitRemoteLogManifestData{" @@ -96,6 +254,18 @@ public String toString() { + coordinatorEpoch + ", bucketLeaderEpoch=" + bucketLeaderEpoch + + ", manifestFormatVersion=" + + manifestFormatVersion + + ", expectedHandleState=" + + expectedHandleState + + ", expectedManifestPath=" + + expectedManifestPath + + ", expectedManifestGeneration=" + + expectedManifestGeneration + + ", expectedZkVersion=" + + expectedZkVersion + + ", newManifestGeneration=" + + newManifestGeneration + '}'; } @@ -112,7 +282,13 @@ public boolean equals(Object o) { && Objects.equals(remoteLogManifestPath, that.remoteLogManifestPath) && remoteLogEndOffset == that.remoteLogEndOffset && coordinatorEpoch == that.coordinatorEpoch - && bucketLeaderEpoch == that.bucketLeaderEpoch; + && bucketLeaderEpoch == that.bucketLeaderEpoch + && Objects.equals(manifestFormatVersion, that.manifestFormatVersion) + && expectedHandleState == that.expectedHandleState + && Objects.equals(expectedManifestPath, that.expectedManifestPath) + && Objects.equals(expectedManifestGeneration, that.expectedManifestGeneration) + && Objects.equals(expectedZkVersion, that.expectedZkVersion) + && Objects.equals(newManifestGeneration, that.newManifestGeneration); } @Override @@ -122,6 +298,12 @@ public int hashCode() { remoteLogManifestPath, remoteLogEndOffset, coordinatorEpoch, - bucketLeaderEpoch); + bucketLeaderEpoch, + manifestFormatVersion, + expectedHandleState, + expectedManifestPath, + expectedManifestGeneration, + expectedZkVersion, + newManifestGeneration); } } diff --git a/fluss-server/src/main/java/org/apache/fluss/server/entity/RemoteLogManifestCommitResult.java b/fluss-server/src/main/java/org/apache/fluss/server/entity/RemoteLogManifestCommitResult.java new file mode 100644 index 0000000000..071b893398 --- /dev/null +++ b/fluss-server/src/main/java/org/apache/fluss/server/entity/RemoteLogManifestCommitResult.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.server.entity; + +/** Result of publishing a remote log manifest handle. */ +public enum RemoteLogManifestCommitResult { + COMMITTED(0), + CONFLICT(1), + FENCED(2), + INVALID_MANIFEST(3); + + private final int code; + + RemoteLogManifestCommitResult(int code) { + this.code = code; + } + + public int code() { + return code; + } + + public static RemoteLogManifestCommitResult fromCode(int code) { + for (RemoteLogManifestCommitResult result : values()) { + if (result.code == code) { + return result; + } + } + throw new IllegalArgumentException("Unknown remote log manifest commit result: " + code); + } +} diff --git a/fluss-server/src/main/java/org/apache/fluss/server/entity/RemoteLogManifestExpectedHandleState.java b/fluss-server/src/main/java/org/apache/fluss/server/entity/RemoteLogManifestExpectedHandleState.java new file mode 100644 index 0000000000..a624eb64ce --- /dev/null +++ b/fluss-server/src/main/java/org/apache/fluss/server/entity/RemoteLogManifestExpectedHandleState.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.server.entity; + +/** Expected existence state of the authoritative remote log manifest handle. */ +public enum RemoteLogManifestExpectedHandleState { + ABSENT(0), + PRESENT(1); + + private final int code; + + RemoteLogManifestExpectedHandleState(int code) { + this.code = code; + } + + public int code() { + return code; + } + + public static RemoteLogManifestExpectedHandleState fromCode(int code) { + for (RemoteLogManifestExpectedHandleState state : values()) { + if (state.code == code) { + return state; + } + } + throw new IllegalArgumentException("Unknown expected handle state: " + code); + } +} diff --git a/fluss-server/src/main/java/org/apache/fluss/server/kv/RemoteLogFetcher.java b/fluss-server/src/main/java/org/apache/fluss/server/kv/RemoteLogFetcher.java index b5943d8383..cf33096fb1 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/kv/RemoteLogFetcher.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/kv/RemoteLogFetcher.java @@ -23,6 +23,7 @@ import org.apache.fluss.record.FileLogRecords; import org.apache.fluss.record.LogRecordBatch; import org.apache.fluss.remote.RemoteLogSegment; +import org.apache.fluss.remote.RemoteLogSegmentReference; import org.apache.fluss.server.log.remote.RemoteLogManager; import org.apache.fluss.server.log.remote.RemoteLogStorage; import org.apache.fluss.utils.ExponentialBackoff; @@ -161,9 +162,9 @@ public Iterable fetch(long startOffset, long localLogStartOffset prev.close(); } - List segments = - remoteLogManager.relevantRemoteLogSegments(tableBucket, startOffset); - if (segments.isEmpty()) { + List references = + remoteLogManager.relevantRemoteLogSegmentReferences(tableBucket, startOffset); + if (references.isEmpty()) { throw new RemoteStorageException( String.format( "No remote log segments found for table bucket %s at offset %d", @@ -173,7 +174,7 @@ public Iterable fetch(long startOffset, long localLogStartOffset LOG.info( "Found {} remote log segments for table bucket {} from offset {} to localLogStartOffset {} " + "(prefetchNum={}, downloadThreads={})", - segments.size(), + references.size(), tableBucket, startOffset, localLogStartOffset, @@ -181,7 +182,7 @@ public Iterable fetch(long startOffset, long localLogStartOffset ((java.util.concurrent.ThreadPoolExecutor) downloadExecutor).getCorePoolSize()); RemoteLogBatchIterator iterator = - new RemoteLogBatchIterator(segments, startOffset, localLogStartOffset); + new RemoteLogBatchIterator(references, startOffset, localLogStartOffset); this.activeIterator = iterator; // Kick off the initial prefetch window so downloads start before the first advance(). iterator.fillPrefetchWindow(); @@ -363,7 +364,7 @@ private File downloadSegmentWithRetry(RemoteLogSegment segment) throws IOExcepti * within [startOffset, localLogStartOffset). */ private class RemoteLogBatchIterator implements Iterator { - private final List segments; + private final List references; private final long localLogStartOffset; /** Tracks the current read offset, advancing as batches are consumed. */ @@ -372,6 +373,11 @@ private class RemoteLogBatchIterator implements Iterator { private int currentSegmentIndex = 0; private FileLogRecords currentFileLogRecords; private Iterator currentBatchIterator; + /** + * Exclusive logical boundary of the current physical segment. The boundary is required to + * align with a record-batch boundary; crossing it fails closed in {@link #advance()}. + */ + private long currentReferenceLogicalEndOffset; /** * Ring of in-flight prefetch futures, indexed by {@code seq % prefetchNum}. A non-null slot @@ -408,8 +414,10 @@ private class RemoteLogBatchIterator implements Iterator { private boolean closed = false; RemoteLogBatchIterator( - List segments, long startOffset, long localLogStartOffset) { - this.segments = segments; + List references, + long startOffset, + long localLogStartOffset) { + this.references = references; this.currentOffset = startOffset; this.localLogStartOffset = localLogStartOffset; @SuppressWarnings({"unchecked", "rawtypes"}) @@ -475,6 +483,19 @@ private void advance() { if (batch.nextLogOffset() <= currentOffset) { continue; } + if (batch.baseLogOffset() >= currentReferenceLogicalEndOffset) { + closeCurrentFileLogRecords(); + continue; + } + if (batch.nextLogOffset() > currentReferenceLogicalEndOffset) { + throw new IllegalStateException( + "Remote log batch [" + + batch.baseLogOffset() + + ", " + + batch.nextLogOffset() + + ") crosses logical reference end " + + currentReferenceLogicalEndOffset); + } // stop if we've reached localLogStartOffset if (batch.baseLogOffset() >= localLogStartOffset) { finished = true; @@ -491,21 +512,23 @@ private void advance() { closeCurrentFileLogRecords(); // move to next segment - if (currentSegmentIndex >= segments.size()) { + if (currentSegmentIndex >= references.size()) { finished = true; return; } - RemoteLogSegment segment = segments.get(currentSegmentIndex++); - // remoteLogEndOffset() is inclusive — last offset in segment - if (segment.remoteLogEndOffset() < currentOffset) { + RemoteLogSegmentReference reference = references.get(currentSegmentIndex++); + RemoteLogSegment segment = reference.remoteLogSegment(); + if (reference.logicalEndOffset() <= currentOffset) { continue; } // skip segments that start at or after localLogStartOffset - if (segment.remoteLogStartOffset() >= localLogStartOffset) { + if (reference.logicalStartOffset() >= localLogStartOffset) { finished = true; return; } + currentOffset = Math.max(currentOffset, reference.logicalStartOffset()); + currentReferenceLogicalEndOffset = reference.logicalEndOffset(); try { File localFile = fetchSegmentFile(segment); @@ -617,16 +640,17 @@ private void fillPrefetchWindow() { return; } - while (nextPrefetchIndex < segments.size()) { - RemoteLogSegment segment = segments.get(nextPrefetchIndex); + while (nextPrefetchIndex < references.size()) { + RemoteLogSegmentReference reference = references.get(nextPrefetchIndex); + RemoteLogSegment segment = reference.remoteLogSegment(); // segment entirely before currentOffset — skip and advance index. - if (segment.remoteLogEndOffset() < currentOffset) { + if (reference.logicalEndOffset() <= currentOffset) { nextPrefetchIndex++; continue; } // segment starts at or beyond localLogStartOffset — nothing more to prefetch. - if (segment.remoteLogStartOffset() >= localLogStartOffset) { + if (reference.logicalStartOffset() >= localLogStartOffset) { return; } diff --git a/fluss-server/src/main/java/org/apache/fluss/server/log/remote/DefaultRemoteLogStorage.java b/fluss-server/src/main/java/org/apache/fluss/server/log/remote/DefaultRemoteLogStorage.java index 56e8e24091..9e2c9e0b45 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/log/remote/DefaultRemoteLogStorage.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/log/remote/DefaultRemoteLogStorage.java @@ -21,6 +21,7 @@ import org.apache.fluss.config.Configuration; import org.apache.fluss.exception.RemoteStorageException; import org.apache.fluss.fs.FSDataInputStream; +import org.apache.fluss.fs.FileStatus; import org.apache.fluss.fs.FileSystem; import org.apache.fluss.fs.FsPath; import org.apache.fluss.metadata.PhysicalTablePath; @@ -54,6 +55,7 @@ import java.util.concurrent.ExecutorService; import static org.apache.fluss.utils.FlussPaths.INDEX_FILE_SUFFIX; +import static org.apache.fluss.utils.FlussPaths.REMOTE_LOG_METADATA_DIR_NAME; import static org.apache.fluss.utils.FlussPaths.TIME_INDEX_FILE_SUFFIX; import static org.apache.fluss.utils.FlussPaths.WRITER_SNAPSHOT_FILE_SUFFIX; import static org.apache.fluss.utils.FlussPaths.remoteLogIndexFile; @@ -236,6 +238,82 @@ public void deleteRemoteLogManifestSnapshot(FsPath remoteLogManifestPath) } } + @Override + public List listRemoteLogSegmentObjects( + PhysicalTablePath physicalTablePath, TableBucket tableBucket) + throws RemoteStorageException { + FsPath tabletDir = + FlussPaths.remoteLogTabletDir(remoteLogDir, physicalTablePath, tableBucket); + try { + if (!fileSystem.exists(tabletDir)) { + return new ArrayList<>(); + } + List objects = new ArrayList<>(); + for (FileStatus status : fileSystem.listStatus(tabletDir)) { + if (!status.isDir() + || REMOTE_LOG_METADATA_DIR_NAME.equals(status.getPath().getName())) { + continue; + } + try { + UUID.fromString(status.getPath().getName()); + objects.add( + new RemoteLogStorageObject( + status.getPath(), status.getModificationTime())); + } catch (IllegalArgumentException ignored) { + LOG.warn( + "Ignoring unexpected remote log bucket directory {}", status.getPath()); + } + } + return objects; + } catch (IOException e) { + throw new RemoteStorageException( + "Failed to list remote log segment objects under " + tabletDir, e); + } + } + + @Override + public List listRemoteLogManifestSnapshots( + PhysicalTablePath physicalTablePath, TableBucket tableBucket) + throws RemoteStorageException { + FsPath metadataDir = + new FsPath( + FlussPaths.remoteLogTabletDir(remoteLogDir, physicalTablePath, tableBucket), + REMOTE_LOG_METADATA_DIR_NAME); + try { + if (!fileSystem.exists(metadataDir)) { + return new ArrayList<>(); + } + List objects = new ArrayList<>(); + for (FileStatus status : fileSystem.listStatus(metadataDir)) { + if (!status.isDir() && status.getPath().getName().endsWith(".manifest")) { + objects.add( + new RemoteLogStorageObject( + status.getPath(), status.getModificationTime())); + } + } + return objects; + } catch (IOException e) { + throw new RemoteStorageException( + "Failed to list remote log manifest snapshots under " + metadataDir, e); + } + } + + @Override + public void deleteRemoteLogSegmentObject( + PhysicalTablePath physicalTablePath, TableBucket tableBucket, UUID segmentId) + throws RemoteStorageException { + FsPath segmentDirectory = + FlussPaths.remoteLogSegmentDir( + FlussPaths.remoteLogTabletDir(remoteLogDir, physicalTablePath, tableBucket), + segmentId); + try { + fileSystem.delete(segmentDirectory, true); + } catch (IOException e) { + throw new RemoteStorageException( + "Failed to delete orphan remote log segment object " + segmentDirectory, e); + } + } + /** Write log manifest snapshot to remote file system. */ @Override public FsPath writeRemoteLogManifestSnapshot(RemoteLogManifest manifest) diff --git a/fluss-server/src/main/java/org/apache/fluss/server/log/remote/LogTieringTask.java b/fluss-server/src/main/java/org/apache/fluss/server/log/remote/LogTieringTask.java index c84aff7188..82098fdc1a 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/log/remote/LogTieringTask.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/log/remote/LogTieringTask.java @@ -23,25 +23,37 @@ import org.apache.fluss.metadata.PhysicalTablePath; import org.apache.fluss.metadata.TableBucket; import org.apache.fluss.remote.RemoteLogManifest; +import org.apache.fluss.remote.RemoteLogManifestReplacementPlanner; +import org.apache.fluss.remote.RemoteLogManifestReplacementPlanner.PlanType; +import org.apache.fluss.remote.RemoteLogManifestV2Migration; import org.apache.fluss.remote.RemoteLogSegment; +import org.apache.fluss.remote.UnreferencedRemoteLogSegment; import org.apache.fluss.rpc.gateway.CoordinatorGateway; import org.apache.fluss.rpc.messages.CommitRemoteLogManifestRequest; import org.apache.fluss.server.entity.CommitRemoteLogManifestData; +import org.apache.fluss.server.entity.RemoteLogManifestCommitResult; import org.apache.fluss.server.log.LogSegment; import org.apache.fluss.server.log.LogTablet; import org.apache.fluss.server.metrics.group.TableMetricGroup; import org.apache.fluss.server.replica.Replica; +import org.apache.fluss.server.zk.ZooKeeperClient; +import org.apache.fluss.server.zk.data.VersionedRemoteLogManifestHandle; import org.apache.fluss.utils.clock.Clock; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.File; +import java.io.IOException; import java.nio.file.Path; import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; import java.util.List; import java.util.Objects; +import java.util.Optional; import java.util.OptionalLong; +import java.util.Set; import java.util.UUID; import java.util.stream.Collectors; @@ -60,12 +72,18 @@ public class LogTieringTask implements Runnable { private final RemoteLogStorage remoteLogStorage; private final RemoteLogIndexCache remoteLogIndexCache; private final CoordinatorGateway coordinatorGateway; + private final RemoteLogManifestCommitter remoteLogManifestCommitter; + private final ZooKeeperClient zooKeeperClient; private final Clock clock; private final int maxUploadSegmentsPerTask; + private final boolean manifestV2WriterEnabled; + private final boolean manifestV2GcEnabled; + private final long manifestV2GcGracePeriodMs; // The copied offset is empty initially for a new leader LogTieringTask, and needs to // be fetched inside the task's run() method. - private volatile Long copiedOffset = null; + /** Exclusive offset from which the next remote copy should resume. */ + private volatile Long nextCopyOffset = null; private volatile boolean cancelled = false; @@ -75,8 +93,12 @@ public LogTieringTask( RemoteLogStorage remoteLogStorage, RemoteLogIndexCache remoteLogIndexCache, CoordinatorGateway coordinatorGateway, + ZooKeeperClient zooKeeperClient, Clock clock, - int maxUploadSegmentsPerTask) { + int maxUploadSegmentsPerTask, + boolean manifestV2WriterEnabled, + boolean manifestV2GcEnabled, + long manifestV2GcGracePeriodMs) { this.replica = replica; this.remoteLog = remoteLog; this.physicalTablePath = replica.getPhysicalTablePath(); @@ -84,8 +106,14 @@ public LogTieringTask( this.remoteLogStorage = remoteLogStorage; this.remoteLogIndexCache = remoteLogIndexCache; this.coordinatorGateway = coordinatorGateway; + this.zooKeeperClient = zooKeeperClient; + this.remoteLogManifestCommitter = + new RemoteLogManifestCommitter(coordinatorGateway, zooKeeperClient); this.clock = clock; this.maxUploadSegmentsPerTask = maxUploadSegmentsPerTask; + this.manifestV2WriterEnabled = manifestV2WriterEnabled; + this.manifestV2GcEnabled = manifestV2GcEnabled; + this.manifestV2GcGracePeriodMs = manifestV2GcGracePeriodMs; } @Override @@ -130,7 +158,11 @@ private void runOnce() throws InterruptedException { try { LogTablet logTablet = replica.getLogTablet(); TableMetricGroup metricGroup = replica.tableMetrics(); - maybeUpdateCopiedOffset(logTablet); + maybeInitializeNextCopyOffset(logTablet); + if (manifestV2WriterEnabled) { + runOnceV2(logTablet, metricGroup, true); + return; + } // Get these candidate log segments to copy and these expired remote log segments to // clean up. @@ -170,10 +202,8 @@ private void runOnce() throws InterruptedException { .collect(Collectors.toList())); } - if (endOffset > 0) { - // endOffset is the next segment base offset, so we need to decrement it - // by 1 to get the last copied segment's highest offset. - copiedOffset = endOffset - 1; + if (endOffset >= 0) { + nextCopyOffset = endOffset; } } else { LOG.error( @@ -201,6 +231,437 @@ private void runOnce() throws InterruptedException { } } + private void runOnceV2( + LogTablet logTablet, TableMetricGroup metricGroup, boolean allowConflictReplan) + throws Exception { + RemoteLogManifest baseManifest = remoteLog.currentManifest(); + VersionedRemoteLogManifestHandle baseHandle = remoteLog.currentHandle(); + if (baseHandle != null) { + RemoteLogManager.validateManifestHandle(baseHandle.handle(), baseManifest); + } else if (!baseManifest.getRemoteLogSegmentList().isEmpty()) { + throw new IllegalStateException("Remote manifest has no authoritative handle"); + } + + long now = clock.milliseconds(); + if (manifestV2GcEnabled) { + sweepOrphanObjects(baseManifest, baseHandle, now, metricGroup); + } + long targetGeneration = + baseHandle == null + ? 1L + : baseHandle.handle().getManifestGeneration().orElse(0L) + 1L; + RemoteLogManifest normalizedBase = + normalizeBaseManifest(baseManifest, targetGeneration, now); + List candidates = candidateToCopyLogSegments(logTablet); + List candidatePlans = planCandidates(normalizedBase, candidates, now); + + RemoteLogManifest resultManifest = normalizedBase; + List copiedSegments = new ArrayList<>(); + List appliedPlanTypes = new ArrayList<>(); + long successfulNextCopyOffset = nextCopyOffset; + for (CandidatePlan candidatePlan : candidatePlans) { + if (candidatePlan.planType == PlanType.GAP) { + LOG.warn( + "Stopping Manifest V2 planning at remote gap for bucket {} and segment {}", + tableBucket, + candidatePlan.remoteLogSegment); + break; + } + if (candidatePlan.planType == PlanType.ALREADY_COVERED) { + appliedPlanTypes.add(candidatePlan.planType); + successfulNextCopyOffset = candidatePlan.enrichedLogSegment.nextSegmentOffset; + continue; + } + if (!copyPlannedSegment(logTablet, candidatePlan, metricGroup)) { + break; + } + copiedSegments.add(candidatePlan.remoteLogSegment); + RemoteLogManifestReplacementPlanner.Result replayed = + RemoteLogManifestReplacementPlanner.plan( + resultManifest, candidatePlan.remoteLogSegment, now); + resultManifest = withGeneration(replayed.resultManifest(), targetGeneration); + appliedPlanTypes.add(candidatePlan.planType); + successfulNextCopyOffset = candidatePlan.enrichedLogSegment.nextSegmentOffset; + } + + RemoteLogManifest beforeExpiration = resultManifest; + resultManifest = + RemoteLogManifestReplacementPlanner.expireContinuousPrefix( + resultManifest, + now, + replica.getLogTTLMs(), + logTablet.isDataLakeEnabled() ? logTablet.getLakeLogEndOffset() : null, + now); + boolean expirationChanged = resultManifest != beforeExpiration; + RemoteLogManifest beforeGarbageCollection = resultManifest; + if (manifestV2GcEnabled) { + resultManifest = collectUnreferencedSegments(resultManifest, now, metricGroup); + } + boolean garbageCollectionChanged = resultManifest != beforeGarbageCollection; + if (copiedSegments.isEmpty() && !expirationChanged && !garbageCollectionChanged) { + nextCopyOffset = successfulNextCopyOffset; + return; + } + + RemoteLogManifestUpdatePlan updatePlan = + new RemoteLogManifestUpdatePlan( + baseHandle, + resultManifest, + copiedSegments, + newlyUnreferencedSegments(baseManifest, resultManifest), + appliedPlanTypes, + successfulNextCopyOffset); + FsPath manifestPath; + try { + manifestPath = remoteLogStorage.writeRemoteLogManifestSnapshot(resultManifest); + } catch (Exception e) { + deleteRemoteLogSegmentFiles(copiedSegments, metricGroup); + throw e; + } + + CommitRemoteLogManifestData commitData = + updatePlan.toCommitData( + manifestPath, replica.getCoordinatorEpoch(), replica.getBucketEpoch()); + RemoteLogManifestCommitResult commitResult; + try { + commitResult = + remoteLogManifestCommitter.commitAndApply( + commitData, resultManifest, remoteLog); + } catch (Exception unknownResult) { + // The result is UNKNOWN or was superseded before local apply. Objects must remain + // untouched until a later authoritative reconciliation proves they are unreferenced. + throw unknownResult; + } + + if (commitResult == RemoteLogManifestCommitResult.COMMITTED) { + applyPublishedOffsets(logTablet, resultManifest); + nextCopyOffset = updatePlan.nextCopyOffset(); + return; + } + + cleanupRejectedPlan(updatePlan, manifestPath, metricGroup, logTablet); + if (commitResult == RemoteLogManifestCommitResult.CONFLICT && allowConflictReplan) { + nextCopyOffset = findNextCopyOffset(logTablet); + runOnceV2(logTablet, metricGroup, false); + } + } + + private RemoteLogManifest collectUnreferencedSegments( + RemoteLogManifest manifest, long now, TableMetricGroup metricGroup) { + List retained = new ArrayList<>(); + List deletedSegmentIds = new ArrayList<>(); + for (UnreferencedRemoteLogSegment unreferenced : + manifest.getUnreferencedRemoteLogSegments()) { + if (!gracePeriodElapsed(unreferenced.unreferencedAtMs(), now)) { + retained.add(unreferenced); + continue; + } + try { + remoteLogStorage.deleteLogSegmentFiles(unreferenced.remoteLogSegment()); + metricGroup.remoteLogDeleteRequests().inc(); + deletedSegmentIds.add(unreferenced.remoteLogSegment().remoteLogSegmentId()); + } catch (Exception e) { + retained.add(unreferenced); + metricGroup.remoteLogDeleteErrors().inc(); + metricGroup.remoteGcFailures().inc(); + LOG.warn( + "Manifest V2 GC will retry deleting unreferenced segment {} for {}", + unreferenced.remoteLogSegment().remoteLogSegmentId(), + tableBucket, + e); + } + } + if (retained.size() == manifest.getUnreferencedRemoteLogSegments().size()) { + return manifest; + } + remoteLogIndexCache.removeAll(deletedSegmentIds); + return RemoteLogManifest.createV2( + manifest.getGeneration(), + manifest.getPhysicalTablePath(), + manifest.getTableBucket(), + manifest.getRemoteLogSegmentList(), + manifest.getRemoteLogSegmentList().isEmpty() + ? null + : manifest.getRemoteLogStartOffset(), + retained); + } + + private void sweepOrphanObjects( + RemoteLogManifest baseManifest, + VersionedRemoteLogManifestHandle baseHandle, + long now, + TableMetricGroup metricGroup) { + try { + if (!authoritativeHandleUnchanged(baseHandle)) { + LOG.info("Skipping orphan sweep for {} because its handle changed", tableBucket); + return; + } + Set referencedSegmentIds = new HashSet<>(); + baseManifest + .getRemoteLogSegmentList() + .forEach(segment -> referencedSegmentIds.add(segment.remoteLogSegmentId())); + baseManifest + .getUnreferencedRemoteLogSegments() + .forEach( + segment -> + referencedSegmentIds.add( + segment.remoteLogSegment().remoteLogSegmentId())); + + for (RemoteLogStorageObject object : + remoteLogStorage.listRemoteLogSegmentObjects(physicalTablePath, tableBucket)) { + UUID segmentId = UUID.fromString(object.path().getName()); + if (!referencedSegmentIds.contains(segmentId) + && orphanGracePeriodElapsed(object.modificationTimeMs(), now) + && authoritativeHandleUnchanged(baseHandle)) { + try { + remoteLogStorage.deleteRemoteLogSegmentObject( + physicalTablePath, tableBucket, segmentId); + metricGroup.remoteLogDeleteRequests().inc(); + } catch (Exception e) { + metricGroup.remoteGcFailures().inc(); + LOG.warn( + "Failed to delete orphan segment object {} for {}", + object.path(), + tableBucket, + e); + } + } + } + + FsPath authoritativeManifestPath = + baseHandle == null ? null : baseHandle.handle().getRemoteLogManifestPath(); + List manifestSnapshots = + remoteLogStorage.listRemoteLogManifestSnapshots(physicalTablePath, tableBucket); + long orphanManifestCount = 0L; + for (RemoteLogStorageObject object : manifestSnapshots) { + if (sameStoragePath(object.path(), authoritativeManifestPath)) { + continue; + } + boolean deleted = false; + if (orphanGracePeriodElapsed(object.modificationTimeMs(), now) + && authoritativeHandleUnchanged(baseHandle)) { + try { + remoteLogStorage.deleteRemoteLogManifestSnapshot(object.path()); + deleted = true; + } catch (Exception e) { + metricGroup.remoteGcFailures().inc(); + LOG.warn( + "Failed to delete orphan manifest snapshot {} for {}", + object.path(), + tableBucket, + e); + } + } + if (!deleted) { + orphanManifestCount++; + } + } + remoteLog.updateOrphanManifestCount(orphanManifestCount); + } catch (Exception e) { + metricGroup.remoteGcFailures().inc(); + LOG.warn("Failed to sweep orphan remote objects for {}", tableBucket, e); + } + } + + private boolean authoritativeHandleUnchanged(VersionedRemoteLogManifestHandle expected) + throws Exception { + Optional current = + zooKeeperClient.getVersionedRemoteLogManifestHandle(tableBucket); + if (expected == null) { + return !current.isPresent(); + } + if (!current.isPresent()) { + return false; + } + VersionedRemoteLogManifestHandle actual = current.get(); + return actual.zkVersion() == expected.zkVersion() + && actual.handle() + .getRemoteLogManifestPath() + .equals(expected.handle().getRemoteLogManifestPath()) + && actual.handle().getManifestGeneration().orElse(0L) + == expected.handle().getManifestGeneration().orElse(0L); + } + + private boolean gracePeriodElapsed(long timestampMs, long now) { + return timestampMs >= 0L + && timestampMs != Long.MAX_VALUE + && timestampMs <= now + && now - timestampMs >= manifestV2GcGracePeriodMs; + } + + private boolean orphanGracePeriodElapsed(long modificationTimeMs, long now) { + // A zero or unavailable timestamp is not trustworthy enough for destructive orphan GC. + return modificationTimeMs > 0L && gracePeriodElapsed(modificationTimeMs, now); + } + + private static boolean sameStoragePath(FsPath first, FsPath second) { + return first != null && second != null && first.getPath().equals(second.getPath()); + } + + private List planCandidates( + RemoteLogManifest normalizedBase, + List candidates, + long unreferencedAtMs) + throws IOException { + List plans = new ArrayList<>(); + RemoteLogManifest workingManifest = normalizedBase; + for (EnrichedLogSegment candidate : candidates) { + RemoteLogSegment remoteLogSegment = createRemoteLogSegment(candidate); + RemoteLogManifestReplacementPlanner.Result result = + RemoteLogManifestReplacementPlanner.plan( + workingManifest, remoteLogSegment, unreferencedAtMs); + plans.add(new CandidatePlan(candidate, remoteLogSegment, result.planType())); + if (result.planType() == PlanType.GAP) { + break; + } + if (result.requiresManifestCommit()) { + workingManifest = + withGeneration(result.resultManifest(), normalizedBase.getGeneration()); + } + } + return plans; + } + + private RemoteLogManifest normalizeBaseManifest( + RemoteLogManifest baseManifest, long targetGeneration, long unreferencedAtMs) { + if (baseManifest.getVersion() == RemoteLogManifest.VERSION_1) { + return RemoteLogManifestV2Migration.migrate( + baseManifest, targetGeneration, unreferencedAtMs); + } + return RemoteLogManifest.createV2( + targetGeneration, + baseManifest.getPhysicalTablePath(), + baseManifest.getTableBucket(), + baseManifest.getRemoteLogSegmentList(), + baseManifest.getRemoteLogSegmentList().isEmpty() + ? null + : baseManifest.getRemoteLogStartOffset(), + baseManifest.getUnreferencedRemoteLogSegments()); + } + + private static RemoteLogManifest withGeneration(RemoteLogManifest manifest, long generation) { + return RemoteLogManifest.createV2( + generation, + manifest.getPhysicalTablePath(), + manifest.getTableBucket(), + manifest.getRemoteLogSegmentList(), + manifest.getRemoteLogSegmentList().isEmpty() + ? null + : manifest.getRemoteLogStartOffset(), + manifest.getUnreferencedRemoteLogSegments()); + } + + private static List newlyUnreferencedSegments( + RemoteLogManifest baseManifest, RemoteLogManifest resultManifest) { + Set baseUnreferencedIds = new HashSet<>(); + for (UnreferencedRemoteLogSegment entry : baseManifest.getUnreferencedRemoteLogSegments()) { + baseUnreferencedIds.add(entry.remoteLogSegment().remoteLogSegmentId()); + } + List result = new ArrayList<>(); + for (UnreferencedRemoteLogSegment entry : + resultManifest.getUnreferencedRemoteLogSegments()) { + if (!baseUnreferencedIds.contains(entry.remoteLogSegment().remoteLogSegmentId())) { + result.add(entry.remoteLogSegment()); + } + } + return result; + } + + private RemoteLogSegment createRemoteLogSegment(EnrichedLogSegment enrichedSegment) + throws IOException { + LogSegment segment = enrichedSegment.logSegment; + return RemoteLogSegment.Builder.builder() + .physicalTablePath(physicalTablePath) + .tableBucket(tableBucket) + .remoteLogSegmentId(UUID.randomUUID()) + .remoteLogStartOffset(segment.getBaseOffset()) + .remoteLogEndOffset(enrichedSegment.nextSegmentOffset) + .maxTimestamp(segment.maxTimestampSoFar()) + .segmentSizeInBytes(segment.getFileLogRecords().sizeInBytes()) + .build(); + } + + private boolean copyPlannedSegment( + LogTablet logTablet, CandidatePlan candidatePlan, TableMetricGroup metricGroup) + throws Exception { + LogSegment segment = candidatePlan.enrichedLogSegment.logSegment; + File writerIdSnapshotFile = + logTablet + .writerStateManager() + .fetchSnapshot(candidatePlan.enrichedLogSegment.nextSegmentOffset) + .orElse(null); + LogSegmentFiles logSegmentFiles = + new LogSegmentFiles( + segment.getFileLogRecords().file().toPath(), + toPathIfExists(segment.offsetIndex().file()), + toPathIfExists(segment.timeIndex().file()), + writerIdSnapshotFile == null ? null : writerIdSnapshotFile.toPath()); + try { + remoteLogStorage.copyLogSegmentFiles(candidatePlan.remoteLogSegment, logSegmentFiles); + } catch (RemoteStorageException e) { + metricGroup.remoteLogCopyErrors().inc(); + LOG.warn( + "Failed to copy planned Manifest V2 segment {} for bucket {}", + candidatePlan.remoteLogSegment, + tableBucket, + e); + return false; + } + metricGroup.remoteLogCopyRequests().inc(); + metricGroup.remoteLogCopyBytes().inc(candidatePlan.remoteLogSegment.segmentSizeInBytes()); + return true; + } + + private void cleanupRejectedPlan( + RemoteLogManifestUpdatePlan updatePlan, + FsPath manifestPath, + TableMetricGroup metricGroup, + LogTablet logTablet) + throws Exception { + Optional authoritativeHandleOpt = + zooKeeperClient.getVersionedRemoteLogManifestHandle(tableBucket); + Set authoritativeSegmentIds = new HashSet<>(); + if (authoritativeHandleOpt.isPresent()) { + VersionedRemoteLogManifestHandle authoritativeHandle = authoritativeHandleOpt.get(); + RemoteLogManifest authoritativeManifest = + remoteLogStorage.readRemoteLogManifestSnapshot( + authoritativeHandle.handle().getRemoteLogManifestPath()); + RemoteLogManager.validateManifestHandle( + authoritativeHandle.handle(), authoritativeManifest); + authoritativeManifest + .getRemoteLogSegmentList() + .forEach(segment -> authoritativeSegmentIds.add(segment.remoteLogSegmentId())); + authoritativeManifest + .getUnreferencedRemoteLogSegments() + .forEach( + segment -> + authoritativeSegmentIds.add( + segment.remoteLogSegment().remoteLogSegmentId())); + remoteLog.replaceManifest(authoritativeManifest, authoritativeHandle); + applyPublishedOffsets(logTablet, authoritativeManifest); + } else { + remoteLog.replaceManifest( + new RemoteLogManifest(physicalTablePath, tableBucket, Collections.emptyList()), + null); + } + + for (RemoteLogSegment copiedSegment : updatePlan.segmentsToCopy()) { + if (authoritativeSegmentIds.contains(copiedSegment.remoteLogSegmentId())) { + throw new IllegalStateException( + "Rejected plan segment is referenced by the authoritative manifest: " + + copiedSegment.remoteLogSegmentId()); + } + } + deleteRemoteLogSegmentFiles(updatePlan.segmentsToCopy(), metricGroup); + remoteLogStorage.deleteRemoteLogManifestSnapshot(manifestPath); + } + + private void applyPublishedOffsets(LogTablet logTablet, RemoteLogManifest manifest) { + logTablet.updateRemoteLogStartOffset(manifest.getRemoteLogStartOffset()); + logTablet.updateRemoteLogEndOffset(manifest.getRemoteLogEndOffset()); + logTablet.updateRemoteLogSize(manifest.getRemoteLogSize()); + } + private List candidateToCopyLogSegments(LogTablet log) { List candidateLogSegments = new ArrayList<>(); // Get highWatermark. @@ -210,32 +671,32 @@ private List candidateToCopyLogSegments(LogTablet log) { "The highWatermark for bucket {} is {}, which should not be negative", tableBucket, highWatermark); - } else if (highWatermark > 0 && copiedOffset < highWatermark) { - // local-log-start-offset can be ahead of the copied-offset, when enabling the + } else if (highWatermark > 0 && nextCopyOffset < highWatermark) { + // local-log-start-offset can be ahead of the next-copy-offset, when enabling the // remote log for the first time - long fromOffset = Math.max(copiedOffset + 1, log.localLogStartOffset()); + long fromOffset = Math.max(nextCopyOffset, log.localLogStartOffset()); candidateLogSegments = candidateLogSegments(log, fromOffset, highWatermark); LOG.debug( - "Candidate log segments for bucket {}: logLocalStartOffset: {}, copiedOffset: {}, " + "Candidate log segments for bucket {}: logLocalStartOffset: {}, nextCopyOffset: {}, " + "fromOffset: {}, highWatermark: {} and candidateLogSegments: {}", tableBucket, log.localLogStartOffset(), - copiedOffset, + nextCopyOffset, fromOffset, highWatermark, candidateLogSegments); if (candidateLogSegments.isEmpty()) { LOG.debug( - "no segments found to be copied for bucket {} which copied-offset: {} and active segment's base-offset: {}", + "no segments found to be copied for bucket {} which next-copy-offset: {} and active segment's base-offset: {}", tableBucket, - copiedOffset, + nextCopyOffset, log.activeLogSegment().getBaseOffset()); } } else { LOG.debug( - "Skipping copying segments for bucket {} to remote, current read-offset:{}, and highWatermark:{}", + "Skipping copying segments for bucket {} to remote, next-copy-offset:{}, and highWatermark:{}", tableBucket, - copiedOffset, + nextCopyOffset, highWatermark); } @@ -373,7 +834,8 @@ public boolean tryToCommitRemoteLogManifest( // and this should be moved into Replica under read lock of // leaderIsrUpdateLock, see FLUSS-56282058 replica.getCoordinatorEpoch(), - replica.getBucketEpoch())); + replica.getBucketEpoch()), + newRemoteLogManifest); if (!success) { // the commit failed, it means the commit snapshot is invalid or register zk // failed, we will revert this commit and delete the remote log manifest @@ -387,8 +849,11 @@ public boolean tryToCommitRemoteLogManifest( return false; } else { // commit succeed. - // TODO: commit with version to avoid the manifest has been updated - remoteLogTablet.addAndDeleteLogSegments(newAddedSegments, expiredSegments); + if (newRemoteLogManifest.getVersion() == RemoteLogManifest.VERSION_1) { + // The legacy V1 writer mutates the in-memory indexes incrementally. V2 + // commitAndApply has already installed the complete immutable snapshot. + remoteLogTablet.addAndDeleteLogSegments(newAddedSegments, expiredSegments); + } LogTablet logTablet = replica.getLogTablet(); logTablet.updateRemoteLogStartOffset(newRemoteLogStartOffset); // make the local log cleaner clean log segments that are committed to remote. @@ -416,7 +881,12 @@ public boolean tryToCommitRemoteLogManifest( return false; } - private boolean commitRemoteLogManifest(CommitRemoteLogManifestData data) throws Exception { + private boolean commitRemoteLogManifest( + CommitRemoteLogManifestData data, RemoteLogManifest manifest) throws Exception { + if (data.isV2CasCommit()) { + return remoteLogManifestCommitter.commitAndApply(data, manifest, remoteLog) + == RemoteLogManifestCommitResult.COMMITTED; + } CommitRemoteLogManifestRequest request = makeCommitRemoteLogManifestRequest(data); return coordinatorGateway.commitRemoteLogManifest(request).get().isCommitSuccess(); } @@ -425,19 +895,19 @@ private Path toPathIfExists(File file) { return file.exists() ? file.toPath() : null; } - private void maybeUpdateCopiedOffset(LogTablet logTablet) { - if (copiedOffset == null) { - copiedOffset = findRemoteLogEndOffset(logTablet); + private void maybeInitializeNextCopyOffset(LogTablet logTablet) { + if (nextCopyOffset == null) { + nextCopyOffset = findNextCopyOffset(logTablet); LOG.info( - "Found the copied remote log end offset: {} for bucket {} after becoming leader", - copiedOffset, + "Found the next remote copy offset: {} for bucket {} after becoming leader", + nextCopyOffset, tableBucket); } } - private long findRemoteLogEndOffset(LogTablet logTablet) { + private long findNextCopyOffset(LogTablet logTablet) { OptionalLong remoteLogEndOffsetOpt = remoteLog.getRemoteLogEndOffset(); - long newRemoteLogEndOffset; + long nextOffset; if (remoteLogEndOffsetOpt.isPresent()) { long remoteLogEndOffset = remoteLogEndOffsetOpt.getAsLong(); long localEndOffset = logTablet.localLogEndOffset(); @@ -449,15 +919,15 @@ private long findRemoteLogEndOffset(LogTablet logTablet) { tableBucket, localEndOffset, remoteLogEndOffset); - newRemoteLogEndOffset = localEndOffset; + nextOffset = localEndOffset; } else { - newRemoteLogEndOffset = remoteLogEndOffset; + nextOffset = remoteLogEndOffset; } } else { - newRemoteLogEndOffset = -1L; + nextOffset = 0L; } - return newRemoteLogEndOffset; + return nextOffset; } /** @@ -466,8 +936,8 @@ private long findRemoteLogEndOffset(LogTablet logTablet) { * *

1. Segment is not the active segment. * - *

2. Segment end-offset is less than the highWatermark as remote storage should contain only - * committed/acked records. + *

2. The segment's exclusive end offset is not greater than the high watermark, as remote + * storage should contain only committed/acked records. * *

The number of returned segments is capped at {@code maxUploadSegmentsPerTask} to prevent * overwhelming the remote storage when there is a large backlog. @@ -526,6 +996,21 @@ public String toString() { return this.getClass() + "[" + tableBucket + "]"; } + private static final class CandidatePlan { + private final EnrichedLogSegment enrichedLogSegment; + private final RemoteLogSegment remoteLogSegment; + private final PlanType planType; + + private CandidatePlan( + EnrichedLogSegment enrichedLogSegment, + RemoteLogSegment remoteLogSegment, + PlanType planType) { + this.enrichedLogSegment = enrichedLogSegment; + this.remoteLogSegment = remoteLogSegment; + this.planType = planType; + } + } + private static class EnrichedLogSegment { private final LogSegment logSegment; private final long nextSegmentOffset; diff --git a/fluss-server/src/main/java/org/apache/fluss/server/log/remote/RemoteLogManager.java b/fluss-server/src/main/java/org/apache/fluss/server/log/remote/RemoteLogManager.java index 4f27c2872a..6ff5238f3c 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/log/remote/RemoteLogManager.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/log/remote/RemoteLogManager.java @@ -27,6 +27,7 @@ import org.apache.fluss.metadata.TableBucket; import org.apache.fluss.remote.RemoteLogManifest; import org.apache.fluss.remote.RemoteLogSegment; +import org.apache.fluss.remote.RemoteLogSegmentReference; import org.apache.fluss.rpc.gateway.CoordinatorGateway; import org.apache.fluss.server.log.LogManager; import org.apache.fluss.server.log.LogTablet; @@ -34,6 +35,7 @@ import org.apache.fluss.server.storage.LocalDiskManager; import org.apache.fluss.server.zk.ZooKeeperClient; import org.apache.fluss.server.zk.data.RemoteLogManifestHandle; +import org.apache.fluss.server.zk.data.VersionedRemoteLogManifestHandle; import org.apache.fluss.utils.IOUtils; import org.apache.fluss.utils.clock.Clock; import org.apache.fluss.utils.concurrent.ExecutorThreadFactory; @@ -75,6 +77,9 @@ public class RemoteLogManager implements Closeable { private final long taskInterval; private final int maxUploadSegmentsPerTask; + private final boolean manifestV2WriterEnabled; + private final boolean manifestV2GcEnabled; + private final long manifestV2GcGracePeriodMs; private final Map remoteLogIndexCachesByDir; private final RemoteLogStorage remoteLogStorage; private final CoordinatorGateway coordinatorGateway; @@ -132,6 +137,18 @@ public RemoteLogManager( this.taskInterval = conf.get(ConfigOptions.REMOTE_LOG_TASK_INTERVAL_DURATION).toMillis(); this.maxUploadSegmentsPerTask = conf.getInt(ConfigOptions.REMOTE_LOG_TASK_MAX_UPLOAD_SEGMENTS); + this.manifestV2WriterEnabled = + conf.getBoolean(ConfigOptions.REMOTE_LOG_MANIFEST_V2_WRITER_ENABLED); + this.manifestV2GcEnabled = conf.getBoolean(ConfigOptions.REMOTE_LOG_MANIFEST_V2_GC_ENABLED); + this.manifestV2GcGracePeriodMs = + conf.get(ConfigOptions.REMOTE_LOG_MANIFEST_V2_GC_GRACE_PERIOD).toMillis(); + if (manifestV2GcEnabled && !manifestV2WriterEnabled) { + throw new IllegalArgumentException( + "Manifest V2 GC requires remote.log.manifest-v2-writer-enabled=true"); + } + if (manifestV2GcGracePeriodMs < 0L) { + throw new IllegalArgumentException("Manifest V2 GC grace period must not be negative"); + } this.rlManagerScheduledThreadPool = scheduledExecutor; this.clock = clock; } @@ -154,15 +171,17 @@ public void registerReplica(Replica replica) throws Exception { LogTablet log = replica.getLogTablet(); RemoteLogTablet remoteLog = new RemoteLogTablet(physicalTablePath, tableBucket, replica.getLogTTLMs()); - Optional remoteLogManifestHandleOpt = - zkClient.getRemoteLogManifestHandle(tableBucket); + Optional remoteLogManifestHandleOpt = + zkClient.getVersionedRemoteLogManifestHandle(tableBucket); if (remoteLogManifestHandleOpt.isPresent()) { // If there is remote log manifest handle in remote, we will download // the manifest snapshot from remote storage and write to cache. + VersionedRemoteLogManifestHandle versionedHandle = remoteLogManifestHandleOpt.get(); RemoteLogManifest manifest = remoteLogStorage.readRemoteLogManifestSnapshot( - remoteLogManifestHandleOpt.get().getRemoteLogManifestPath()); - remoteLog.loadRemoteLogManifest(manifest); + versionedHandle.handle().getRemoteLogManifestPath()); + validateManifestHandle(versionedHandle.handle(), manifest); + remoteLog.loadRemoteLogManifest(manifest, versionedHandle); } remoteLog.getRemoteLogEndOffset().ifPresent(log::updateRemoteLogEndOffset); log.updateRemoteLogStartOffset(remoteLog.getRemoteLogStartOffset()); @@ -258,13 +277,22 @@ public long lookupOffsetForTimestamp(TableBucket tableBucket, long timestamp) { return -1L; } - RemoteLogSegment segment = remoteLogTablet.findSegmentByTimestamp(timestamp); - if (segment == null) { - return -1L; - } else { - return remoteLogIndexCacheForBucket(tableBucket) - .lookupOffsetForTimestamp(segment, timestamp); + for (RemoteLogSegmentReference reference : + remoteLogTablet.allRemoteLogSegmentReferences()) { + RemoteLogSegment segment = reference.remoteLogSegment(); + // A clipped segment's physical max timestamp is only a conservative candidate: it + // may come from the hidden suffix. Validate the index result against logical bounds. + if (segment.maxTimestamp() < timestamp) { + continue; + } + long offset = + remoteLogIndexCacheForBucket(tableBucket) + .lookupOffsetForTimestamp(segment, timestamp); + if (offset >= 0L && offset < reference.logicalEndOffset()) { + return Math.max(offset, reference.logicalStartOffset()); + } } + return -1L; } /** @@ -276,10 +304,42 @@ public List relevantRemoteLogSegments(TableBucket tableBucket, return remoteLogTablet(tableBucket).relevantRemoteLogSegments(offset); } + /** Returns the physical segment prefix to include in a FetchLog v0 response. */ + public List relevantRemoteLogSegmentsForFetchV0( + TableBucket tableBucket, long offset) { + return remoteLogTablet(tableBucket).relevantRemoteLogSegmentsForFetchV0(offset); + } + + /** Returns active logical remote log references needed to read from the given offset. */ + public List relevantRemoteLogSegmentReferences( + TableBucket tableBucket, long offset) { + return remoteLogTablet(tableBucket).relevantRemoteLogSegmentReferences(offset); + } + private boolean remoteDisabled() { return taskInterval <= 0L; } + @VisibleForTesting + static void validateManifestHandle(RemoteLogManifestHandle handle, RemoteLogManifest manifest) { + if (handle.getVersion() == RemoteLogManifestHandle.VERSION_1) { + if (manifest.getVersion() != RemoteLogManifest.VERSION_1 + || handle.getRemoteLogEndOffset() != manifest.getRemoteLogEndOffset()) { + throw new IllegalStateException( + "V1 remote log manifest handle does not match manifest content"); + } + return; + } + if (manifest.getVersion() != RemoteLogManifest.VERSION_2 + || handle.getManifestGeneration().getAsLong() != manifest.getGeneration() + || handle.getRemoteLogStartOffset().getAsLong() + != manifest.getRemoteLogStartOffset() + || handle.getRemoteLogEndOffset() != manifest.getRemoteLogEndOffset()) { + throw new IllegalStateException( + "V2 remote log manifest handle hints do not match manifest content"); + } + } + /** * Delete the remote log. This method will delete all the files in remote for this table bucket * and will delete the log segment manifests cache in local for leader replica. @@ -317,8 +377,12 @@ private void doHandleLeaderReplica( remoteLogStorage, remoteLogIndexCache(replica.getLogTablet().getDataDir()), coordinatorGateway, + zkClient, clock, - maxUploadSegmentsPerTask); + maxUploadSegmentsPerTask, + manifestV2WriterEnabled, + manifestV2GcEnabled, + manifestV2GcGracePeriodMs); LOG.info( "Created a new remote log task for table-bucket{}: {} and getting scheduled", tableBucket, diff --git a/fluss-server/src/main/java/org/apache/fluss/server/log/remote/RemoteLogManifestCommitter.java b/fluss-server/src/main/java/org/apache/fluss/server/log/remote/RemoteLogManifestCommitter.java new file mode 100644 index 0000000000..3dbc01f4ad --- /dev/null +++ b/fluss-server/src/main/java/org/apache/fluss/server/log/remote/RemoteLogManifestCommitter.java @@ -0,0 +1,166 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.server.log.remote; + +import org.apache.fluss.remote.RemoteLogManifest; +import org.apache.fluss.rpc.gateway.CoordinatorGateway; +import org.apache.fluss.rpc.messages.CommitRemoteLogManifestResponse; +import org.apache.fluss.server.entity.CommitRemoteLogManifestData; +import org.apache.fluss.server.entity.RemoteLogManifestCommitResult; +import org.apache.fluss.server.entity.RemoteLogManifestExpectedHandleState; +import org.apache.fluss.server.zk.ZooKeeperClient; +import org.apache.fluss.server.zk.data.RemoteLogManifestHandle; +import org.apache.fluss.server.zk.data.VersionedRemoteLogManifestHandle; + +import java.util.Optional; + +import static org.apache.fluss.server.utils.ServerRpcMessageUtils.makeCommitRemoteLogManifestRequest; +import static org.apache.fluss.utils.Preconditions.checkArgument; + +/** Publishes Manifest V2 handles and reconciles transport-unknown commit outcomes. */ +public final class RemoteLogManifestCommitter { + private static final int MAX_COMMIT_ATTEMPTS = 10; + + private final CoordinatorGateway coordinatorGateway; + private final ZooKeeperClient zooKeeperClient; + + public RemoteLogManifestCommitter( + CoordinatorGateway coordinatorGateway, ZooKeeperClient zooKeeperClient) { + this.coordinatorGateway = coordinatorGateway; + this.zooKeeperClient = zooKeeperClient; + } + + public RemoteLogManifestCommitResult commit(CommitRemoteLogManifestData data) throws Exception { + checkArgument(data.isV2CasCommit(), "CAS committer only accepts Manifest V2 commits"); + Exception lastTransportError = null; + for (int attempt = 0; attempt < MAX_COMMIT_ATTEMPTS; attempt++) { + try { + CommitRemoteLogManifestResponse response = + coordinatorGateway + .commitRemoteLogManifest(makeCommitRemoteLogManifestRequest(data)) + .get(); + if (!response.hasCommitResult()) { + throw new IllegalStateException( + "Coordinator did not return a V2 manifest commit result"); + } + return RemoteLogManifestCommitResult.fromCode(response.getCommitResult()); + } catch (Exception transportError) { + lastTransportError = transportError; + ReconciliationResult reconciliationResult = reconcile(data); + if (reconciliationResult == ReconciliationResult.COMMITTED) { + return RemoteLogManifestCommitResult.COMMITTED; + } + if (reconciliationResult == ReconciliationResult.CONFLICT) { + return RemoteLogManifestCommitResult.CONFLICT; + } + // The authoritative state is still the expected base (or remains absent), so + // retry exactly the same idempotent CAS/create request. + } + } + throw new IllegalStateException( + "Remote log manifest commit remains unknown after reconciliation retries", + lastTransportError); + } + + /** Commits a V2 snapshot and atomically installs it only after its handle is authoritative. */ + public RemoteLogManifestCommitResult commitAndApply( + CommitRemoteLogManifestData data, + RemoteLogManifest manifest, + RemoteLogTablet remoteLogTablet) + throws Exception { + validateManifest(data, manifest); + RemoteLogManifestCommitResult result = commit(data); + if (result != RemoteLogManifestCommitResult.COMMITTED) { + return result; + } + + Optional authoritativeOpt = + zooKeeperClient.getVersionedRemoteLogManifestHandle(data.getTableBucket()); + if (!authoritativeOpt.isPresent() + || !isPublishedResult(authoritativeOpt.get().handle(), data)) { + throw new IllegalStateException( + "Committed remote log manifest is no longer authoritative before local apply"); + } + remoteLogTablet.replaceManifest(manifest, authoritativeOpt.get()); + return RemoteLogManifestCommitResult.COMMITTED; + } + + private static void validateManifest( + CommitRemoteLogManifestData data, RemoteLogManifest manifest) { + checkArgument( + manifest.getVersion() == RemoteLogManifest.VERSION_2, + "CAS commit requires a Manifest V2 snapshot"); + checkArgument( + manifest.getTableBucket().equals(data.getTableBucket()), + "Manifest table bucket does not match commit request"); + checkArgument( + manifest.getGeneration() == data.getNewManifestGeneration(), + "Manifest generation does not match commit request"); + checkArgument( + manifest.getRemoteLogStartOffset() == data.getRemoteLogStartOffset(), + "Manifest start offset does not match commit request"); + checkArgument( + manifest.getRemoteLogEndOffset() == data.getRemoteLogEndOffset(), + "Manifest end offset does not match commit request"); + } + + private ReconciliationResult reconcile(CommitRemoteLogManifestData data) throws Exception { + Optional currentOpt = + zooKeeperClient.getVersionedRemoteLogManifestHandle(data.getTableBucket()); + if (currentOpt.isPresent() && isPublishedResult(currentOpt.get().handle(), data)) { + return ReconciliationResult.COMMITTED; + } + + if (data.getExpectedHandleState() == RemoteLogManifestExpectedHandleState.ABSENT) { + return currentOpt.isPresent() + ? ReconciliationResult.CONFLICT + : ReconciliationResult.RETRY; + } + if (!currentOpt.isPresent()) { + return ReconciliationResult.CONFLICT; + } + + VersionedRemoteLogManifestHandle current = currentOpt.get(); + long currentGeneration = current.handle().getManifestGeneration().orElse(0L); + if (current.zkVersion() == data.getExpectedZkVersion() + && current.handle() + .getRemoteLogManifestPath() + .equals(data.getExpectedManifestPath()) + && currentGeneration == data.getExpectedManifestGeneration()) { + return ReconciliationResult.RETRY; + } + return ReconciliationResult.CONFLICT; + } + + private static boolean isPublishedResult( + RemoteLogManifestHandle current, CommitRemoteLogManifestData data) { + return current.getVersion() == RemoteLogManifestHandle.VERSION_2 + && current.getRemoteLogManifestPath().equals(data.getRemoteLogManifestPath()) + && current.getManifestGeneration().isPresent() + && current.getManifestGeneration().getAsLong() == data.getNewManifestGeneration() + && current.getRemoteLogStartOffset().isPresent() + && current.getRemoteLogStartOffset().getAsLong() == data.getRemoteLogStartOffset() + && current.getRemoteLogEndOffset() == data.getRemoteLogEndOffset(); + } + + private enum ReconciliationResult { + COMMITTED, + CONFLICT, + RETRY + } +} diff --git a/fluss-server/src/main/java/org/apache/fluss/server/log/remote/RemoteLogManifestUpdatePlan.java b/fluss-server/src/main/java/org/apache/fluss/server/log/remote/RemoteLogManifestUpdatePlan.java new file mode 100644 index 0000000000..f77c812173 --- /dev/null +++ b/fluss-server/src/main/java/org/apache/fluss/server/log/remote/RemoteLogManifestUpdatePlan.java @@ -0,0 +1,127 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.server.log.remote; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.fs.FsPath; +import org.apache.fluss.remote.RemoteLogManifest; +import org.apache.fluss.remote.RemoteLogManifestReplacementPlanner.PlanType; +import org.apache.fluss.remote.RemoteLogSegment; +import org.apache.fluss.server.entity.CommitRemoteLogManifestData; +import org.apache.fluss.server.zk.data.VersionedRemoteLogManifestHandle; + +import javax.annotation.Nullable; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import static org.apache.fluss.utils.Preconditions.checkArgument; +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** Immutable overlap-aware Manifest V2 update and its authoritative CAS base. */ +@Internal +public final class RemoteLogManifestUpdatePlan { + private final @Nullable VersionedRemoteLogManifestHandle baseHandle; + private final RemoteLogManifest resultManifest; + private final List segmentsToCopy; + private final List segmentsToUnreference; + private final List planTypes; + private final long nextCopyOffset; + + public RemoteLogManifestUpdatePlan( + @Nullable VersionedRemoteLogManifestHandle baseHandle, + RemoteLogManifest resultManifest, + List segmentsToCopy, + List segmentsToUnreference, + List planTypes, + long nextCopyOffset) { + this.resultManifest = checkNotNull(resultManifest); + checkArgument( + resultManifest.getVersion() == RemoteLogManifest.VERSION_2, + "Update plan requires Manifest V2"); + long expectedGeneration = + baseHandle == null + ? 1L + : baseHandle.handle().getManifestGeneration().orElse(0L) + 1L; + checkArgument( + resultManifest.getGeneration() == expectedGeneration, + "Result generation %s does not follow authoritative base generation %s", + resultManifest.getGeneration(), + expectedGeneration - 1L); + this.baseHandle = baseHandle; + this.segmentsToCopy = immutableCopy(segmentsToCopy); + this.segmentsToUnreference = immutableCopy(segmentsToUnreference); + this.planTypes = Collections.unmodifiableList(new ArrayList<>(checkNotNull(planTypes))); + this.nextCopyOffset = nextCopyOffset; + } + + public @Nullable VersionedRemoteLogManifestHandle baseHandle() { + return baseHandle; + } + + public RemoteLogManifest resultManifest() { + return resultManifest; + } + + public List segmentsToCopy() { + return segmentsToCopy; + } + + public List segmentsToUnreference() { + return segmentsToUnreference; + } + + public List planTypes() { + return planTypes; + } + + public long nextCopyOffset() { + return nextCopyOffset; + } + + /** Builds the commit request after the immutable result snapshot has been written. */ + public CommitRemoteLogManifestData toCommitData( + FsPath manifestPath, int coordinatorEpoch, int bucketLeaderEpoch) { + if (baseHandle == null) { + return CommitRemoteLogManifestData.v2Absent( + resultManifest.getTableBucket(), + manifestPath, + resultManifest.getRemoteLogStartOffset(), + resultManifest.getRemoteLogEndOffset(), + resultManifest.getGeneration(), + coordinatorEpoch, + bucketLeaderEpoch); + } + return CommitRemoteLogManifestData.v2Present( + resultManifest.getTableBucket(), + manifestPath, + resultManifest.getRemoteLogStartOffset(), + resultManifest.getRemoteLogEndOffset(), + resultManifest.getGeneration(), + baseHandle.handle().getRemoteLogManifestPath(), + baseHandle.handle().getManifestGeneration().orElse(0L), + baseHandle.zkVersion(), + coordinatorEpoch, + bucketLeaderEpoch); + } + + private static List immutableCopy(List segments) { + return Collections.unmodifiableList(new ArrayList<>(checkNotNull(segments))); + } +} diff --git a/fluss-server/src/main/java/org/apache/fluss/server/log/remote/RemoteLogStorage.java b/fluss-server/src/main/java/org/apache/fluss/server/log/remote/RemoteLogStorage.java index 6e1de16cf3..19b6aeb024 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/log/remote/RemoteLogStorage.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/log/remote/RemoteLogStorage.java @@ -27,6 +27,8 @@ import java.io.Closeable; import java.io.InputStream; +import java.util.List; +import java.util.UUID; import static org.apache.fluss.utils.FlussPaths.INDEX_FILE_SUFFIX; import static org.apache.fluss.utils.FlussPaths.TIME_INDEX_FILE_SUFFIX; @@ -151,6 +153,21 @@ RemoteLogManifest readRemoteLogManifestSnapshot(FsPath remoteLogManifestPath) void deleteRemoteLogManifestSnapshot(FsPath remoteLogManifestPath) throws RemoteStorageException; + /** Lists UUID segment directories for a single bucket for orphan collection. */ + List listRemoteLogSegmentObjects( + PhysicalTablePath physicalTablePath, TableBucket tableBucket) + throws RemoteStorageException; + + /** Lists manifest snapshots for a single bucket for orphan collection. */ + List listRemoteLogManifestSnapshots( + PhysicalTablePath physicalTablePath, TableBucket tableBucket) + throws RemoteStorageException; + + /** Recursively deletes one bucket-scoped UUID segment directory found by the orphan sweeper. */ + void deleteRemoteLogSegmentObject( + PhysicalTablePath physicalTablePath, TableBucket tableBucket, UUID segmentId) + throws RemoteStorageException; + /** * Writes the remote log manifest to remote manifest file path. * diff --git a/fluss-server/src/main/java/org/apache/fluss/server/log/remote/RemoteLogStorageObject.java b/fluss-server/src/main/java/org/apache/fluss/server/log/remote/RemoteLogStorageObject.java new file mode 100644 index 0000000000..f58230a2b7 --- /dev/null +++ b/fluss-server/src/main/java/org/apache/fluss/server/log/remote/RemoteLogStorageObject.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.server.log.remote; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.fs.FsPath; + +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** A bucket-scoped remote storage object discovered by the orphan sweeper. */ +@Internal +public final class RemoteLogStorageObject { + private final FsPath path; + private final long modificationTimeMs; + + /** Creates a discovered object with its storage modification timestamp. */ + public RemoteLogStorageObject(FsPath path, long modificationTimeMs) { + this.path = checkNotNull(path); + this.modificationTimeMs = modificationTimeMs; + } + + /** Returns the full remote path. */ + public FsPath path() { + return path; + } + + /** Returns the storage modification time, or {@link Long#MAX_VALUE} when unavailable. */ + public long modificationTimeMs() { + return modificationTimeMs; + } +} diff --git a/fluss-server/src/main/java/org/apache/fluss/server/log/remote/RemoteLogTablet.java b/fluss-server/src/main/java/org/apache/fluss/server/log/remote/RemoteLogTablet.java index 9f0ae6e949..4d822b5520 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/log/remote/RemoteLogTablet.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/log/remote/RemoteLogTablet.java @@ -24,13 +24,14 @@ import org.apache.fluss.metrics.groups.MetricGroup; import org.apache.fluss.remote.RemoteLogManifest; import org.apache.fluss.remote.RemoteLogSegment; +import org.apache.fluss.remote.RemoteLogSegmentReference; import org.apache.fluss.server.metrics.group.BucketMetricGroup; +import org.apache.fluss.server.zk.data.VersionedRemoteLogManifestHandle; import javax.annotation.Nullable; import javax.annotation.concurrent.ThreadSafe; import java.util.ArrayList; -import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -44,6 +45,7 @@ import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; +import static org.apache.fluss.utils.Preconditions.checkArgument; import static org.apache.fluss.utils.concurrent.LockUtils.inReadLock; import static org.apache.fluss.utils.concurrent.LockUtils.inWriteLock; @@ -63,11 +65,8 @@ public class RemoteLogTablet { */ private final Map idToRemoteLogSegment = new HashMap<>(); - /** - * It contains segment remote log start offset to segment ids mapping which the segment did not - * delete in remote storage. - */ - private final NavigableMap offsetToRemoteLogSegmentId = new TreeMap<>(); + /** It contains logical start offset to active remote log segment reference mappings. */ + private final NavigableMap offsetToReference = new TreeMap<>(); /** * It contains max timestamp to segment ids mapping which the segment did not delete in remote @@ -84,20 +83,20 @@ public class RemoteLogTablet { /** The registered metrics for remote log. */ private volatile MetricGroup remoteLogMetrics; + private volatile long orphanManifestCount; + private volatile RemoteLogManifest currentManifest; + private volatile @Nullable VersionedRemoteLogManifestHandle currentHandle; + private volatile long remoteSizeInBytes; private volatile int numRemoteLogSegments; - /** - * It represents the remote log start offset of the segments that have copied to remote storage. - */ + /** The inclusive start offset of the segments copied to remote storage. */ private volatile long remoteLogStartOffset; - /** - * It represents the remote log end offset of the segments that have copied to remote storage. - */ + /** The exclusive end offset of the segments copied to remote storage. */ private volatile long remoteLogEndOffset; private volatile boolean closed = false; @@ -132,6 +131,14 @@ public void registerMetrics(BucketMetricGroup bucketMetricGroup) { }); metricGroup.gauge(MetricNames.LOG_END_OFFSET, () -> remoteLogEndOffset); metricGroup.gauge(MetricNames.REMOTE_LOG_SIZE, this::getRemoteSizeInBytes); + metricGroup.gauge( + MetricNames.REMOTE_UNREFERENCED_BYTES, + this::getUnreferencedSizeInBytes); + metricGroup.gauge( + MetricNames.REMOTE_UNREFERENCED_SEGMENT_COUNT, + this::getUnreferencedSegmentCount); + metricGroup.gauge( + MetricNames.REMOTE_ORPHAN_MANIFEST_COUNT, () -> orphanManifestCount); remoteLogMetrics = metricGroup; }); } @@ -140,6 +147,27 @@ public long getRemoteSizeInBytes() { return remoteSizeInBytes; } + /** Returns bytes retained by persisted unreferenced segment metadata. */ + public long getUnreferencedSizeInBytes() { + return inReadLock( + lock, + () -> + currentManifest.getUnreferencedRemoteLogSegments().stream() + .mapToLong( + segment -> segment.remoteLogSegment().segmentSizeInBytes()) + .sum()); + } + + /** Returns the number of persisted unreferenced segments waiting for GC. */ + public int getUnreferencedSegmentCount() { + return inReadLock(lock, () -> currentManifest.getUnreferencedRemoteLogSegments().size()); + } + + /** Updates the number of non-authoritative manifest snapshots seen by the latest sweep. */ + public void updateOrphanManifestCount(long orphanManifestCount) { + this.orphanManifestCount = orphanManifestCount; + } + public void unregisterMetrics() { inWriteLock( lock, @@ -156,12 +184,19 @@ public List allRemoteLogSegments() { return inReadLock(lock, () -> currentManifest.getRemoteLogSegmentList()); } + /** Get all active remote log references in logical offset order. */ + public List allRemoteLogSegmentReferences() { + return inReadLock( + lock, + () -> Collections.unmodifiableList(new ArrayList<>(offsetToReference.values()))); + } + /** * Returns the expired segments based on the given time and lake log end offset. * - *

Only segments that have been tiered to lake (i.e., remoteLogEndOffset <= lakeLogEndOffset) - * can be safely deleted. This ensures that we don't delete segments that haven't been tiered to - * lake yet. + *

Both end offsets are exclusive. Only segments that have been fully tiered to lake (i.e., + * {@code remoteLogEndOffset <= lakeLogEndOffset}) can be safely deleted. This ensures that we + * don't delete segments that haven't been tiered to lake yet. * * @param currentTimeMs the current time in milliseconds * @param lakeLogEndOffset the log end offset that has been synced to lake, null if data lake is @@ -238,22 +273,69 @@ public List expiredRemoteLogSegments( * segment whose remote log start offset smaller than this offset (floor key). */ public List relevantRemoteLogSegments(long offset) { + List segments = new ArrayList<>(); + for (RemoteLogSegmentReference reference : relevantRemoteLogSegmentReferences(offset)) { + segments.add(reference.remoteLogSegment()); + } + return segments; + } + + /** + * Returns the maximal physically contiguous segment prefix safe to include in a FetchLog v0 + * response. + * + *

V1 preserves the legacy behavior and returns the complete relevant tail. For V2, adjacent + * non-overlapping segments can be consumed by an unchanged client in one response. The prefix + * stops before the first physical overlap because that segment would be downloaded and then + * discarded as stale after the preceding segment advances the client offset. The next request + * resolves the advanced offset against the logical reference index. + */ + public List relevantRemoteLogSegmentsForFetchV0(long offset) { return inReadLock( lock, () -> { - Long floorKey = offsetToRemoteLogSegmentId.floorKey(offset); - Collection segmentIds = - offsetToRemoteLogSegmentId - .tailMap(floorKey == null ? 0L : floorKey, true) - .values(); - List remoteLogSegmentList = new ArrayList<>(); - for (UUID id : segmentIds) { - RemoteLogSegment remoteLogSegment = idToRemoteLogSegment.get(id); - if (offset < remoteLogSegment.remoteLogEndOffset()) { - remoteLogSegmentList.add(remoteLogSegment); + List references = + relevantRemoteLogSegmentReferences(offset); + List segments = new ArrayList<>(); + if (currentManifest.getVersion() == RemoteLogManifest.VERSION_1) { + for (RemoteLogSegmentReference reference : references) { + segments.add(reference.remoteLogSegment()); + } + return segments; + } + + long previousPhysicalEndOffset = -1L; + for (RemoteLogSegmentReference reference : references) { + RemoteLogSegment segment = reference.remoteLogSegment(); + if (!segments.isEmpty() + && segment.remoteLogStartOffset() != previousPhysicalEndOffset) { + break; } + segments.add(segment); + previousPhysicalEndOffset = segment.remoteLogEndOffset(); } - return remoteLogSegmentList; + return segments; + }); + } + + /** Returns all active logical references needed for a sequential read from the given offset. */ + public List relevantRemoteLogSegmentReferences(long offset) { + return inReadLock( + lock, + () -> { + if (offset < remoteLogStartOffset || offset >= remoteLogEndOffset) { + return Collections.emptyList(); + } + Map.Entry floorEntry = + offsetToReference.floorEntry(offset); + if (floorEntry == null || offset >= floorEntry.getValue().logicalEndOffset()) { + throw new IllegalStateException( + "Remote manifest contains no logical reference for offset " + + offset); + } + return Collections.unmodifiableList( + new ArrayList<>( + offsetToReference.tailMap(floorEntry.getKey(), true).values())); }); } @@ -275,13 +357,67 @@ public RemoteLogManifest currentManifest() { return inReadLock(lock, () -> currentManifest); } + /** Returns the authoritative handle snapshot used to load the current manifest, if any. */ + public @Nullable VersionedRemoteLogManifestHandle currentHandle() { + return inReadLock(lock, () -> currentHandle); + } + public void loadRemoteLogManifest(RemoteLogManifest manifestSnapshot) { + replaceManifest(manifestSnapshot); + } + + /** Loads a manifest together with the authoritative handle snapshot that selected it. */ + public void loadRemoteLogManifest( + RemoteLogManifest manifestSnapshot, VersionedRemoteLogManifestHandle handle) { + replaceManifest(manifestSnapshot, handle); + } + + /** Atomically replaces all active indexes with the supplied immutable manifest snapshot. */ + public void replaceManifest(RemoteLogManifest manifest) { + replaceManifest(manifest, currentHandle); + } + + /** Atomically replaces all active indexes and their authoritative handle snapshot. */ + public void replaceManifest( + RemoteLogManifest manifest, @Nullable VersionedRemoteLogManifestHandle handle) { + checkArgument( + manifest.getPhysicalTablePath().equals(physicalTablePath), + "Manifest physical table path does not match remote log tablet"); + checkArgument( + manifest.getTableBucket().equals(tableBucket), + "Manifest table bucket does not match remote log tablet"); inWriteLock( lock, () -> { - reset(); - addAndDeleteLogSegments( - manifestSnapshot.getRemoteLogSegmentList(), Collections.emptyList()); + Map newIdToSegment = new HashMap<>(); + NavigableMap newOffsetToReference = + new TreeMap<>(); + NavigableMap> newTimestampToSegmentIds = new TreeMap<>(); + long newRemoteSizeInBytes = 0L; + + for (RemoteLogSegmentReference reference : + manifest.getRemoteLogSegmentReferences()) { + RemoteLogSegment segment = reference.remoteLogSegment(); + newIdToSegment.put(segment.remoteLogSegmentId(), segment); + newOffsetToReference.put(reference.logicalStartOffset(), reference); + newTimestampToSegmentIds + .computeIfAbsent(segment.maxTimestamp(), ignored -> new HashSet<>()) + .add(segment.remoteLogSegmentId()); + newRemoteSizeInBytes += segment.segmentSizeInBytes(); + } + + idToRemoteLogSegment.clear(); + idToRemoteLogSegment.putAll(newIdToSegment); + offsetToReference.clear(); + offsetToReference.putAll(newOffsetToReference); + timestampToRemoteLogSegmentId.clear(); + timestampToRemoteLogSegmentId.putAll(newTimestampToSegmentIds); + remoteSizeInBytes = newRemoteSizeInBytes; + numRemoteLogSegments = newIdToSegment.size(); + remoteLogStartOffset = manifest.getRemoteLogStartOffset(); + remoteLogEndOffset = manifest.getRemoteLogEndOffset(); + currentManifest = manifest; + currentHandle = handle; }); } @@ -302,8 +438,12 @@ public void addAndDeleteLogSegments( // TODO maybe need to check the leader epoch. idToRemoteLogSegment.put(remoteLogSegmentId, remoteLogSegment); - offsetToRemoteLogSegmentId.put( - remoteLogSegment.remoteLogStartOffset(), remoteLogSegmentId); + offsetToReference.put( + remoteLogSegment.remoteLogStartOffset(), + new RemoteLogSegmentReference( + remoteLogSegment, + remoteLogSegment.remoteLogStartOffset(), + remoteLogSegment.remoteLogEndOffset())); timestampToRemoteLogSegmentId .computeIfAbsent( remoteLogSegment.maxTimestamp(), k -> new HashSet<>()) @@ -325,7 +465,7 @@ public void addAndDeleteLogSegments( RemoteLogSegment removeSegment = idToRemoteLogSegment.remove(remoteLogSegmentId); - offsetToRemoteLogSegmentId.remove(remoteLogSegment.remoteLogStartOffset()); + offsetToReference.remove(remoteLogSegment.remoteLogStartOffset()); // remove k,v mapping if the set is empty. timestampToRemoteLogSegmentId.compute( @@ -351,14 +491,18 @@ public void addAndDeleteLogSegments( // reset to default values if no segments exist after expiration. reset(); } else { - remoteLogStartOffset = offsetToRemoteLogSegmentId.firstKey(); + remoteLogStartOffset = offsetToReference.firstKey(); } + List activeSegments = + new ArrayList<>(idToRemoteLogSegment.values()); + activeSegments.sort( + (left, right) -> + Long.compare( + left.remoteLogStartOffset(), + right.remoteLogStartOffset())); currentManifest = - new RemoteLogManifest( - physicalTablePath, - tableBucket, - new ArrayList<>(idToRemoteLogSegment.values())); + new RemoteLogManifest(physicalTablePath, tableBucket, activeSegments); }); } @@ -368,7 +512,7 @@ private boolean logExpireEnable() { private void reset() { idToRemoteLogSegment.clear(); - offsetToRemoteLogSegmentId.clear(); + offsetToReference.clear(); timestampToRemoteLogSegmentId.clear(); remoteSizeInBytes = 0L; numRemoteLogSegments = 0; diff --git a/fluss-server/src/main/java/org/apache/fluss/server/metadata/ServerInfo.java b/fluss-server/src/main/java/org/apache/fluss/server/metadata/ServerInfo.java index 9e6cdcc26c..3ef4f4a75a 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/metadata/ServerInfo.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/metadata/ServerInfo.java @@ -28,8 +28,13 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Set; +import java.util.TreeSet; import java.util.stream.Collectors; +import static java.util.Collections.emptySet; +import static java.util.Collections.unmodifiableSet; + /** * ServerInfo is used to save the endpoint metadata in controller and synchronize for each server. */ @@ -45,6 +50,7 @@ public class ServerInfo { private final Map endpointMap; private final ServerType serverType; private final TabletServerResource resource; + private final Set capabilities; public ServerInfo( Integer id, @Nullable String rack, List endpoints, ServerType serverType) { @@ -57,6 +63,17 @@ public ServerInfo( List endpoints, ServerType serverType, TabletServerResource resource) { + this(id, rack, endpoints, serverType, resource, emptySet()); + } + + /** Creates server metadata with the capabilities advertised during registration. */ + public ServerInfo( + Integer id, + @Nullable String rack, + List endpoints, + ServerType serverType, + TabletServerResource resource, + Set capabilities) { this.id = id; this.rack = rack; this.endpointMap = @@ -64,6 +81,7 @@ public ServerInfo( .collect(Collectors.toMap(Endpoint::getListenerName, endpoint -> endpoint)); this.serverType = serverType; this.resource = resource; + this.capabilities = unmodifiableSet(new TreeSet<>(capabilities)); } public Integer id() { @@ -112,6 +130,11 @@ public TabletServerResource resource() { return resource; } + /** Returns whether the server advertised the requested protocol capability. */ + public boolean supportsCapability(String capability) { + return capabilities.contains(capability); + } + @Override public boolean equals(Object o) { if (o == null || getClass() != o.getClass()) { @@ -121,12 +144,13 @@ public boolean equals(Object o) { return Objects.equals(id, that.id) && Objects.equals(rack, that.rack) && Objects.equals(endpointMap, that.endpointMap) - && Objects.equals(resource, that.resource); + && Objects.equals(resource, that.resource) + && Objects.equals(capabilities, that.capabilities); } @Override public int hashCode() { - return Objects.hash(id, rack, endpointMap, resource); + return Objects.hash(id, rack, endpointMap, resource, capabilities); } @Override @@ -142,6 +166,8 @@ public String toString() { + serverType + ", resource=" + resource + + ", capabilities=" + + capabilities + '}'; } } diff --git a/fluss-server/src/main/java/org/apache/fluss/server/metrics/group/TableMetricGroup.java b/fluss-server/src/main/java/org/apache/fluss/server/metrics/group/TableMetricGroup.java index 68387b9419..7dc3f25c15 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/metrics/group/TableMetricGroup.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/metrics/group/TableMetricGroup.java @@ -155,6 +155,10 @@ public Counter remoteLogDeleteErrors() { return logMetrics.remoteLogDeleteErrors; } + public Counter remoteGcFailures() { + return logMetrics.remoteGcFailures; + } + public void incKvMessageIn(long n) { if (kvMetrics == null) { NoOpCounter.INSTANCE.inc(n); @@ -480,6 +484,7 @@ private static class LogMetricGroup extends TabletMetricGroup { private final Counter remoteLogCopyErrors; private final Counter remoteLogDeleteRequests; private final Counter remoteLogDeleteErrors; + private final Counter remoteGcFailures; private LogMetricGroup(TableMetricGroup tableMetricGroup, TabletType groupType) { super(tableMetricGroup, groupType); @@ -518,6 +523,8 @@ private LogMetricGroup(TableMetricGroup tableMetricGroup, TabletType groupType) new MeterView(remoteLogDeleteRequests)); remoteLogDeleteErrors = new ThreadSafeSimpleCounter(); meter(MetricNames.REMOTE_LOG_DELETE_ERROR_RATE, new MeterView(remoteLogDeleteErrors)); + remoteGcFailures = new ThreadSafeSimpleCounter(); + counter(MetricNames.REMOTE_GC_FAILURE_COUNT, remoteGcFailures); } @Override diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java index 9448463579..e70668c9f3 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java @@ -1678,7 +1678,8 @@ private boolean canFetchFromRemoteLog(Replica replica, long fetchOffset) { private @Nullable RemoteLogFetchInfo fetchLogFromRemote(Replica replica, long fetchOffset) { List remoteLogSegmentList = - remoteLogManager.relevantRemoteLogSegments(replica.getTableBucket(), fetchOffset); + remoteLogManager.relevantRemoteLogSegmentsForFetchV0( + replica.getTableBucket(), fetchOffset); if (!remoteLogSegmentList.isEmpty()) { int firstStartPos = remoteLogManager.lookupPositionForOffset( diff --git a/fluss-server/src/main/java/org/apache/fluss/server/tablet/TabletServer.java b/fluss-server/src/main/java/org/apache/fluss/server/tablet/TabletServer.java index 3d1ad4ef70..a0d97f6f81 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/tablet/TabletServer.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/tablet/TabletServer.java @@ -73,6 +73,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.List; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutorService; @@ -83,6 +84,7 @@ import static org.apache.fluss.config.ConfigOptions.BACKGROUND_THREADS; import static org.apache.fluss.config.FlussConfigUtils.validateTabletConfigs; import static org.apache.fluss.server.utils.ServerRpcMessageUtils.toTableBucket; +import static org.apache.fluss.server.zk.data.TabletServerRegistration.REMOTE_MANIFEST_VERSION_DISPATCH_CAPABILITY; /** * Tablet server implementation. The tablet server is responsible to manage the log tablet and kv @@ -388,7 +390,8 @@ private void registerTabletServer() throws Exception { rack, Endpoint.loadAdvertisedEndpoints(bindEndpoints, conf), startTime, - tabletServerResource); + tabletServerResource, + Collections.singleton(REMOTE_MANIFEST_VERSION_DISPATCH_CAPABILITY)); while (true) { try { diff --git a/fluss-server/src/main/java/org/apache/fluss/server/utils/ServerRpcMessageUtils.java b/fluss-server/src/main/java/org/apache/fluss/server/utils/ServerRpcMessageUtils.java index e96be16c17..5c7202ebfa 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/utils/ServerRpcMessageUtils.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/utils/ServerRpcMessageUtils.java @@ -187,6 +187,7 @@ import org.apache.fluss.server.entity.NotifyLeaderAndIsrData; import org.apache.fluss.server.entity.NotifyLeaderAndIsrResultForBucket; import org.apache.fluss.server.entity.NotifyRemoteLogOffsetsData; +import org.apache.fluss.server.entity.RemoteLogManifestExpectedHandleState; import org.apache.fluss.server.entity.StopReplicaData; import org.apache.fluss.server.entity.StopReplicaResultForBucket; import org.apache.fluss.server.kv.snapshot.CompletedSnapshot; @@ -1024,20 +1025,11 @@ public static FetchLogResponse makeFetchLogResponse( .setLogStartOffset(0L); if (bucketResult.fetchFromRemote()) { - // set remote log fetch info. RemoteLogFetchInfo rlfInfo = bucketResult.remoteLogFetchInfo(); checkNotNull(rlfInfo, "Remote log fetch info is null."); List remoteLogSegmentList = new ArrayList<>(); for (RemoteLogSegment logSegment : rlfInfo.remoteLogSegmentList()) { - PbRemoteLogSegment pbRemoteLogSegment = - new PbRemoteLogSegment() - .setRemoteLogStartOffset(logSegment.remoteLogStartOffset()) - .setRemoteLogSegmentId( - logSegment.remoteLogSegmentId().toString()) - .setRemoteLogEndOffset(logSegment.remoteLogEndOffset()) - .setSegmentSizeInBytes(logSegment.segmentSizeInBytes()) - .setMaxTimestamp(logSegment.maxTimestamp()); - remoteLogSegmentList.add(pbRemoteLogSegment); + remoteLogSegmentList.add(toPbRemoteLogSegment(logSegment)); } fetchLogRespForBucket .setRemoteLogFetchInfo() @@ -1100,6 +1092,15 @@ public static FetchLogResponse makeFetchLogResponse( return fetchLogResponse; } + private static PbRemoteLogSegment toPbRemoteLogSegment(RemoteLogSegment logSegment) { + return new PbRemoteLogSegment() + .setRemoteLogStartOffset(logSegment.remoteLogStartOffset()) + .setRemoteLogSegmentId(logSegment.remoteLogSegmentId().toString()) + .setRemoteLogEndOffset(logSegment.remoteLogEndOffset()) + .setSegmentSizeInBytes(logSegment.segmentSizeInBytes()) + .setMaxTimestamp(logSegment.maxTimestamp()); + } + public static Map getPutKvData(PutKvRequest putKvRequest) { long tableId = putKvRequest.getTableId(); Map produceEntryData = new HashMap<>(); @@ -1630,14 +1631,68 @@ public static GetFileSystemSecurityTokenResponse toGetFileSystemSecurityTokenRes public static CommitRemoteLogManifestData getCommitRemoteLogManifestData( CommitRemoteLogManifestRequest request) { - return new CommitRemoteLogManifestData( + TableBucket tableBucket = new TableBucket( request.getTableId(), request.hasPartitionId() ? request.getPartitionId() : null, - request.getBucketId()), - new FsPath(request.getRemoteLogManifestPath()), + request.getBucketId()); + FsPath manifestPath = new FsPath(request.getRemoteLogManifestPath()); + if (!request.hasManifestFormatVersion()) { + if (request.hasExpectedHandleState() + || request.hasExpectedManifestPath() + || request.hasExpectedManifestGeneration() + || request.hasExpectedZkVersion() + || request.hasNewManifestGeneration()) { + throw new IllegalArgumentException("Legacy manifest commit contains V2 CAS fields"); + } + return new CommitRemoteLogManifestData( + tableBucket, + manifestPath, + request.getRemoteLogStartOffset(), + request.getRemoteLogEndOffset(), + request.getCoordinatorEpoch(), + request.getBucketLeaderEpoch()); + } + + if (request.getManifestFormatVersion() != 2 + || !request.hasExpectedHandleState() + || !request.hasNewManifestGeneration()) { + throw new IllegalArgumentException( + "Manifest V2 commit requires format version, expected state, and generation"); + } + RemoteLogManifestExpectedHandleState expectedState = + RemoteLogManifestExpectedHandleState.fromCode(request.getExpectedHandleState()); + if (expectedState == RemoteLogManifestExpectedHandleState.ABSENT) { + if (request.hasExpectedManifestPath() + || request.hasExpectedManifestGeneration() + || request.hasExpectedZkVersion()) { + throw new IllegalArgumentException( + "ABSENT manifest commit contains PRESENT-only fields"); + } + return CommitRemoteLogManifestData.v2Absent( + tableBucket, + manifestPath, + request.getRemoteLogStartOffset(), + request.getRemoteLogEndOffset(), + request.getNewManifestGeneration(), + request.getCoordinatorEpoch(), + request.getBucketLeaderEpoch()); + } + if (!request.hasExpectedManifestPath() + || !request.hasExpectedManifestGeneration() + || !request.hasExpectedZkVersion()) { + throw new IllegalArgumentException( + "PRESENT manifest commit requires expected path, generation, and ZK version"); + } + return CommitRemoteLogManifestData.v2Present( + tableBucket, + manifestPath, request.getRemoteLogStartOffset(), request.getRemoteLogEndOffset(), + request.getNewManifestGeneration(), + new FsPath(request.getExpectedManifestPath()), + request.getExpectedManifestGeneration(), + request.getExpectedZkVersion(), request.getCoordinatorEpoch(), request.getBucketLeaderEpoch()); } @@ -1657,6 +1712,21 @@ public static CommitRemoteLogManifestRequest makeCommitRemoteLogManifestRequest( .setRemoteLogEndOffset(commitRemoteLogManifestData.getRemoteLogEndOffset()) .setCoordinatorEpoch(commitRemoteLogManifestData.getCoordinatorEpoch()) .setBucketLeaderEpoch(commitRemoteLogManifestData.getBucketLeaderEpoch()); + if (commitRemoteLogManifestData.isV2CasCommit()) { + request.setManifestFormatVersion(commitRemoteLogManifestData.getManifestFormatVersion()) + .setExpectedHandleState( + commitRemoteLogManifestData.getExpectedHandleState().code()) + .setNewManifestGeneration( + commitRemoteLogManifestData.getNewManifestGeneration()); + if (commitRemoteLogManifestData.getExpectedHandleState() + == RemoteLogManifestExpectedHandleState.PRESENT) { + request.setExpectedManifestPath( + commitRemoteLogManifestData.getExpectedManifestPath().toString()) + .setExpectedManifestGeneration( + commitRemoteLogManifestData.getExpectedManifestGeneration()) + .setExpectedZkVersion(commitRemoteLogManifestData.getExpectedZkVersion()); + } + } return request; } diff --git a/fluss-server/src/main/java/org/apache/fluss/server/zk/ZooKeeperClient.java b/fluss-server/src/main/java/org/apache/fluss/server/zk/ZooKeeperClient.java index e9eab00e00..80f5b2554f 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/zk/ZooKeeperClient.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/zk/ZooKeeperClient.java @@ -24,6 +24,7 @@ import org.apache.fluss.config.ConfigurationUtils; import org.apache.fluss.config.FlussConfigUtils; import org.apache.fluss.exception.CoordinatorEpochFencedException; +import org.apache.fluss.fs.FsPath; import org.apache.fluss.metadata.DatabaseSummary; import org.apache.fluss.metadata.PhysicalTablePath; import org.apache.fluss.metadata.ResolvedPartitionSpec; @@ -58,6 +59,7 @@ import org.apache.fluss.server.zk.data.TableAssignment; import org.apache.fluss.server.zk.data.TableRegistration; import org.apache.fluss.server.zk.data.TabletServerRegistration; +import org.apache.fluss.server.zk.data.VersionedRemoteLogManifestHandle; import org.apache.fluss.server.zk.data.ZkData; import org.apache.fluss.server.zk.data.ZkData.AclChangeNotificationNode; import org.apache.fluss.server.zk.data.ZkData.BucketIdsZNode; @@ -1351,6 +1353,72 @@ public Optional getRemoteLogManifestHandle(TableBucket return getOrEmpty(path).map(BucketRemoteLogsZNode::decode); } + /** Returns the authoritative remote log handle together with its ZK data version. */ + public Optional getVersionedRemoteLogManifestHandle( + TableBucket tableBucket) throws Exception { + String path = BucketRemoteLogsZNode.path(tableBucket); + Stat stat = new Stat(); + try { + byte[] data = zkClient.getData().storingStatIn(stat).forPath(path); + return Optional.of( + new VersionedRemoteLogManifestHandle( + BucketRemoteLogsZNode.decode(data), stat.getVersion())); + } catch (KeeperException.NoNodeException e) { + return Optional.empty(); + } + } + + /** Creates the first V2 remote log handle only if no authoritative handle exists. */ + public boolean createRemoteLogManifestHandleIfAbsent( + TableBucket tableBucket, RemoteLogManifestHandle newHandle) throws Exception { + String path = BucketRemoteLogsZNode.path(tableBucket); + try { + zkClient.create() + .creatingParentsIfNeeded() + .forPath(path, BucketRemoteLogsZNode.encode(newHandle)); + return true; + } catch (KeeperException.NodeExistsException e) { + return false; + } + } + + /** + * Replaces an existing authoritative handle only when path, virtual generation, and ZK version + * all match the caller's base snapshot. + */ + public boolean compareAndSetRemoteLogManifestHandle( + TableBucket tableBucket, + FsPath expectedManifestPath, + long expectedManifestGeneration, + int expectedZkVersion, + RemoteLogManifestHandle newHandle) + throws Exception { + Optional currentOpt = + getVersionedRemoteLogManifestHandle(tableBucket); + if (!currentOpt.isPresent()) { + return false; + } + VersionedRemoteLogManifestHandle current = currentOpt.get(); + RemoteLogManifestHandle currentHandle = current.handle(); + long currentGeneration = currentHandle.getManifestGeneration().orElse(0L); + if (current.zkVersion() != expectedZkVersion + || !currentHandle.getRemoteLogManifestPath().equals(expectedManifestPath) + || currentGeneration != expectedManifestGeneration) { + return false; + } + + try { + zkClient.setData() + .withVersion(expectedZkVersion) + .forPath( + BucketRemoteLogsZNode.path(tableBucket), + BucketRemoteLogsZNode.encode(newHandle)); + return true; + } catch (KeeperException.BadVersionException | KeeperException.NoNodeException e) { + return false; + } + } + /** * Lists all remote log manifest handles for buckets of the given table (or partition when * {@code partitionId != null}). Reads the {@code BucketRemoteLogsZNode} subtree under either diff --git a/fluss-server/src/main/java/org/apache/fluss/server/zk/data/RemoteLogManifestHandle.java b/fluss-server/src/main/java/org/apache/fluss/server/zk/data/RemoteLogManifestHandle.java index a3a6e2cee4..2b8c2e948a 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/zk/data/RemoteLogManifestHandle.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/zk/data/RemoteLogManifestHandle.java @@ -19,20 +19,74 @@ import org.apache.fluss.fs.FsPath; +import javax.annotation.Nullable; + +import java.util.Objects; +import java.util.OptionalLong; + +import static org.apache.fluss.utils.Preconditions.checkArgument; + /** * The remote log manifest handle of a table bucket stored in {@link ZkData.BucketRemoteLogsZNode}. * * @see RemoteLogManifestHandleJsonSerde for json serialization and deserialization. */ public class RemoteLogManifestHandle { + public static final int VERSION_1 = 1; + public static final int VERSION_2 = 2; + + private final int version; private final FsPath remoteLogManifestPath; + private final @Nullable Long manifestGeneration; + private final @Nullable Long remoteLogStartOffset; private final long remoteLogEndOffset; public RemoteLogManifestHandle(FsPath remoteLogManifestPath, long remoteLogEndOffset) { + this(VERSION_1, remoteLogManifestPath, null, null, remoteLogEndOffset); + } + + private RemoteLogManifestHandle( + int version, + FsPath remoteLogManifestPath, + @Nullable Long manifestGeneration, + @Nullable Long remoteLogStartOffset, + long remoteLogEndOffset) { + checkArgument( + version == VERSION_1 || version == VERSION_2, + "Unsupported remote log manifest handle version: %s", + version); + if (version == VERSION_1) { + checkArgument( + manifestGeneration == null && remoteLogStartOffset == null, + "V1 remote log manifest handle must not contain V2 hints"); + } else { + checkArgument( + manifestGeneration != null && manifestGeneration > 0L, + "V2 manifest generation must be positive"); + checkArgument( + remoteLogStartOffset != null && remoteLogStartOffset < remoteLogEndOffset, + "V2 remote log offsets must form a non-empty half-open range"); + } + this.version = version; this.remoteLogManifestPath = remoteLogManifestPath; + this.manifestGeneration = manifestGeneration; + this.remoteLogStartOffset = remoteLogStartOffset; this.remoteLogEndOffset = remoteLogEndOffset; } + public static RemoteLogManifestHandle v2( + FsPath remoteLogManifestPath, + long manifestGeneration, + long remoteLogStartOffset, + long remoteLogEndOffset) { + return new RemoteLogManifestHandle( + VERSION_2, + remoteLogManifestPath, + manifestGeneration, + remoteLogStartOffset, + remoteLogEndOffset); + } + public static FsPath fromRemoteLogManifestPath(String remoteLogManifestPath) { return new FsPath(remoteLogManifestPath); } @@ -41,6 +95,22 @@ public FsPath getRemoteLogManifestPath() { return remoteLogManifestPath; } + public int getVersion() { + return version; + } + + public OptionalLong getManifestGeneration() { + return manifestGeneration == null + ? OptionalLong.empty() + : OptionalLong.of(manifestGeneration); + } + + public OptionalLong getRemoteLogStartOffset() { + return remoteLogStartOffset == null + ? OptionalLong.empty() + : OptionalLong.of(remoteLogStartOffset); + } + public long getRemoteLogEndOffset() { return remoteLogEndOffset; } @@ -54,15 +124,34 @@ public boolean equals(Object o) { return false; } RemoteLogManifestHandle that = (RemoteLogManifestHandle) o; - return remoteLogManifestPath.equals(that.remoteLogManifestPath) + return version == that.version + && remoteLogManifestPath.equals(that.remoteLogManifestPath) + && Objects.equals(manifestGeneration, that.manifestGeneration) + && Objects.equals(remoteLogStartOffset, that.remoteLogStartOffset) && remoteLogEndOffset == that.remoteLogEndOffset; } + @Override + public int hashCode() { + return Objects.hash( + version, + remoteLogManifestPath, + manifestGeneration, + remoteLogStartOffset, + remoteLogEndOffset); + } + @Override public String toString() { return "RemoteLogManifestHandle{" + "remoteLogManifestPath=" + remoteLogManifestPath + + ", version=" + + version + + ", manifestGeneration=" + + manifestGeneration + + ", remoteLogStartOffset=" + + remoteLogStartOffset + ", remoteLogEndOffset=" + remoteLogEndOffset + '}'; diff --git a/fluss-server/src/main/java/org/apache/fluss/server/zk/data/RemoteLogManifestHandleJsonSerde.java b/fluss-server/src/main/java/org/apache/fluss/server/zk/data/RemoteLogManifestHandleJsonSerde.java index 62cea381a7..270692b48c 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/zk/data/RemoteLogManifestHandleJsonSerde.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/zk/data/RemoteLogManifestHandleJsonSerde.java @@ -38,8 +38,9 @@ public class RemoteLogManifestHandleJsonSerde private static final String VERSION_KEY = "version"; private static final String REMOTE_LOG_MANIFEST_PATH = "remote_log_manifest_path"; + private static final String MANIFEST_GENERATION = "manifest_generation"; + private static final String REMOTE_LOG_START_OFFSET = "remote_log_start_offset"; private static final String REMOTE_LOG_END_OFFSET = "remote_log_end_offset"; - private static final int VERSION = 1; @Override public void serialize(RemoteLogManifestHandle remoteLogManifestHandle, JsonGenerator generator) @@ -47,21 +48,62 @@ public void serialize(RemoteLogManifestHandle remoteLogManifestHandle, JsonGener generator.writeStartObject(); // serialize data version. - generator.writeNumberField(VERSION_KEY, VERSION); + generator.writeNumberField(VERSION_KEY, remoteLogManifestHandle.getVersion()); generator.writeStringField( REMOTE_LOG_MANIFEST_PATH, remoteLogManifestHandle.getRemoteLogManifestPath().toString()); generator.writeNumberField( REMOTE_LOG_END_OFFSET, remoteLogManifestHandle.getRemoteLogEndOffset()); + if (remoteLogManifestHandle.getVersion() == RemoteLogManifestHandle.VERSION_2) { + generator.writeNumberField( + MANIFEST_GENERATION, + remoteLogManifestHandle.getManifestGeneration().getAsLong()); + generator.writeNumberField( + REMOTE_LOG_START_OFFSET, + remoteLogManifestHandle.getRemoteLogStartOffset().getAsLong()); + } generator.writeEndObject(); } @Override public RemoteLogManifestHandle deserialize(JsonNode node) { - String remoteLogManifestPath = node.get(REMOTE_LOG_MANIFEST_PATH).asText(); - long remoteLogEndOffset = node.get(REMOTE_LOG_END_OFFSET).asLong(); - return new RemoteLogManifestHandle( - fromRemoteLogManifestPath(remoteLogManifestPath), remoteLogEndOffset); + int version = required(node, VERSION_KEY).asInt(); + String remoteLogManifestPath = required(node, REMOTE_LOG_MANIFEST_PATH).asText(); + long remoteLogEndOffset = required(node, REMOTE_LOG_END_OFFSET).asLong(); + if (version == RemoteLogManifestHandle.VERSION_1) { + rejectField(node, MANIFEST_GENERATION, version); + rejectField(node, REMOTE_LOG_START_OFFSET, version); + return new RemoteLogManifestHandle( + fromRemoteLogManifestPath(remoteLogManifestPath), remoteLogEndOffset); + } + if (version == RemoteLogManifestHandle.VERSION_2) { + return RemoteLogManifestHandle.v2( + fromRemoteLogManifestPath(remoteLogManifestPath), + required(node, MANIFEST_GENERATION).asLong(), + required(node, REMOTE_LOG_START_OFFSET).asLong(), + remoteLogEndOffset); + } + throw new IllegalArgumentException( + "Unsupported remote log manifest handle version: " + version); + } + + private static JsonNode required(JsonNode node, String field) { + JsonNode value = node.get(field); + if (value == null || value.isNull()) { + throw new IllegalArgumentException( + "Missing required remote log manifest handle field: " + field); + } + return value; + } + + private static void rejectField(JsonNode node, String field, int version) { + if (node.has(field)) { + throw new IllegalArgumentException( + "Remote log manifest handle field " + + field + + " is not valid for version " + + version); + } } } diff --git a/fluss-server/src/main/java/org/apache/fluss/server/zk/data/TabletServerRegistration.java b/fluss-server/src/main/java/org/apache/fluss/server/zk/data/TabletServerRegistration.java index 5ea57e846d..463a2a01b9 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/zk/data/TabletServerRegistration.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/zk/data/TabletServerRegistration.java @@ -24,6 +24,11 @@ import java.util.List; import java.util.Objects; +import java.util.Set; +import java.util.TreeSet; + +import static java.util.Collections.emptySet; +import static java.util.Collections.unmodifiableSet; /** * The register information of tablet server stored in {@link ZkData.ServerIdZNode}. @@ -31,14 +36,18 @@ * @see TabletServerRegistrationJsonSerde for json serialization and deserialization. */ public class TabletServerRegistration { + public static final String REMOTE_MANIFEST_VERSION_DISPATCH_CAPABILITY = + "remote-manifest-version-dispatch"; + private final @Nullable String rack; private final List endpoints; private final long registerTimestamp; private final TabletServerResource resource; + private final Set capabilities; public TabletServerRegistration( @Nullable String rack, List endpoints, long registerTimestamp) { - this(rack, endpoints, registerTimestamp, TabletServerResource.unknown()); + this(rack, endpoints, registerTimestamp, TabletServerResource.unknown(), emptySet()); } public TabletServerRegistration( @@ -46,10 +55,20 @@ public TabletServerRegistration( List endpoints, long registerTimestamp, TabletServerResource resource) { + this(rack, endpoints, registerTimestamp, resource, emptySet()); + } + + public TabletServerRegistration( + @Nullable String rack, + List endpoints, + long registerTimestamp, + TabletServerResource resource, + Set capabilities) { this.rack = rack; this.endpoints = endpoints; this.registerTimestamp = registerTimestamp; this.resource = resource; + this.capabilities = unmodifiableSet(new TreeSet<>(capabilities)); } public List getEndpoints() { @@ -68,6 +87,14 @@ public TabletServerResource getResource() { return resource; } + public Set getCapabilities() { + return capabilities; + } + + public boolean supportsCapability(String capability) { + return capabilities.contains(capability); + } + @Override public boolean equals(Object o) { if (this == o) { @@ -80,12 +107,13 @@ public boolean equals(Object o) { return registerTimestamp == that.registerTimestamp && Objects.equals(endpoints, that.endpoints) && Objects.equals(rack, that.rack) - && Objects.equals(resource, that.resource); + && Objects.equals(resource, that.resource) + && Objects.equals(capabilities, that.capabilities); } @Override public int hashCode() { - return Objects.hash(endpoints, registerTimestamp, rack, resource); + return Objects.hash(endpoints, registerTimestamp, rack, resource, capabilities); } @Override @@ -99,6 +127,8 @@ public String toString() { + rack + "', resource=" + resource + + ", capabilities=" + + capabilities + '}'; } } diff --git a/fluss-server/src/main/java/org/apache/fluss/server/zk/data/TabletServerRegistrationJsonSerde.java b/fluss-server/src/main/java/org/apache/fluss/server/zk/data/TabletServerRegistrationJsonSerde.java index 389d84d103..e35beff4a5 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/zk/data/TabletServerRegistrationJsonSerde.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/zk/data/TabletServerRegistrationJsonSerde.java @@ -27,8 +27,11 @@ import java.io.IOException; import java.util.Collections; +import java.util.HashSet; import java.util.List; +import java.util.Set; +import static java.util.Collections.emptySet; import static org.apache.fluss.config.ConfigOptions.DEFAULT_LISTENER_NAME; /** Json serializer and deserializer for {@link TabletServerRegistration}. */ @@ -41,7 +44,9 @@ public class TabletServerRegistrationJsonSerde new TabletServerRegistrationJsonSerde(); private static final String VERSION_KEY = "version"; - private static final int VERSION = 4; + private static final int VERSION = 5; + private static final int RESOURCE_VERSION = 4; + private static final int CAPABILITIES_VERSION = 5; @Deprecated private static final String HOST = "host"; @Deprecated private static final String PORT = "port"; @@ -50,6 +55,7 @@ public class TabletServerRegistrationJsonSerde private static final String RACK = "rack"; private static final String CPU_CORES = "cpu_cores"; private static final String MEMORY_BYTES = "memory_bytes"; + private static final String CAPABILITIES = "capabilities"; @Override public void serialize( @@ -71,6 +77,11 @@ public void serialize( if (resource.getMemoryBytes() != null) { generator.writeNumberField(MEMORY_BYTES, resource.getMemoryBytes()); } + generator.writeArrayFieldStart(CAPABILITIES); + for (String capability : tabletServerRegistration.getCapabilities()) { + generator.writeString(capability); + } + generator.writeEndArray(); generator.writeEndObject(); } @@ -80,6 +91,7 @@ public TabletServerRegistration deserialize(JsonNode node) { List endpoints; String rack = null; TabletServerResource resource = TabletServerResource.unknown(); + Set capabilities = emptySet(); if (version == 1) { String host = node.get(HOST).asText(); int port = node.get(PORT).asInt(); @@ -91,14 +103,23 @@ public TabletServerRegistration deserialize(JsonNode node) { if (node.has(RACK)) { rack = node.get(RACK).asText(); } - if (version >= VERSION) { + if (version >= RESOURCE_VERSION) { Double cpuCores = node.has(CPU_CORES) ? node.get(CPU_CORES).asDouble() : null; Long memoryBytes = node.has(MEMORY_BYTES) ? node.get(MEMORY_BYTES).asLong() : null; resource = new TabletServerResource(cpuCores, memoryBytes); } + if (version >= CAPABILITIES_VERSION) { + capabilities = new HashSet<>(); + if (node.has(CAPABILITIES)) { + for (JsonNode capability : node.get(CAPABILITIES)) { + capabilities.add(capability.asText()); + } + } + } } long registerTimestamp = node.get(REGISTER_TIMESTAMP).asLong(); - return new TabletServerRegistration(rack, endpoints, registerTimestamp, resource); + return new TabletServerRegistration( + rack, endpoints, registerTimestamp, resource, capabilities); } } diff --git a/fluss-server/src/main/java/org/apache/fluss/server/zk/data/VersionedRemoteLogManifestHandle.java b/fluss-server/src/main/java/org/apache/fluss/server/zk/data/VersionedRemoteLogManifestHandle.java new file mode 100644 index 0000000000..7aee150309 --- /dev/null +++ b/fluss-server/src/main/java/org/apache/fluss/server/zk/data/VersionedRemoteLogManifestHandle.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.server.zk.data; + +import java.util.Objects; + +/** Authoritative remote log manifest handle together with its ZooKeeper data version. */ +public final class VersionedRemoteLogManifestHandle { + private final RemoteLogManifestHandle handle; + private final int zkVersion; + + public VersionedRemoteLogManifestHandle(RemoteLogManifestHandle handle, int zkVersion) { + this.handle = Objects.requireNonNull(handle, "handle"); + this.zkVersion = zkVersion; + } + + public RemoteLogManifestHandle handle() { + return handle; + } + + public int zkVersion() { + return zkVersion; + } +} diff --git a/fluss-server/src/test/java/org/apache/fluss/server/coordinator/CoordinatorEventProcessorTest.java b/fluss-server/src/test/java/org/apache/fluss/server/coordinator/CoordinatorEventProcessorTest.java index 5a5a68b3c8..95313c3783 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/coordinator/CoordinatorEventProcessorTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/coordinator/CoordinatorEventProcessorTest.java @@ -64,6 +64,7 @@ import org.apache.fluss.server.entity.CommitKvSnapshotData; import org.apache.fluss.server.entity.CommitRemoteLogManifestData; import org.apache.fluss.server.entity.NotifyLeaderAndIsrResultForBucket; +import org.apache.fluss.server.entity.RemoteLogManifestCommitResult; import org.apache.fluss.server.entity.TablePropertyChanges; import org.apache.fluss.server.kv.snapshot.CompletedSnapshot; import org.apache.fluss.server.kv.snapshot.ZooKeeperCompletedSnapshotHandleStore; @@ -72,6 +73,7 @@ import org.apache.fluss.server.metadata.CoordinatorMetadataCache; import org.apache.fluss.server.metadata.ServerInfo; import org.apache.fluss.server.metadata.TableMetadata; +import org.apache.fluss.server.metadata.TabletServerResource; import org.apache.fluss.server.metrics.group.TestingMetricGroups; import org.apache.fluss.server.tablet.TestTabletServerGateway; import org.apache.fluss.server.zk.NOPErrorHandler; @@ -82,8 +84,10 @@ import org.apache.fluss.server.zk.data.CoordinatorAddress; import org.apache.fluss.server.zk.data.LeaderAndIsr; import org.apache.fluss.server.zk.data.PartitionAssignment; +import org.apache.fluss.server.zk.data.RemoteLogManifestHandle; import org.apache.fluss.server.zk.data.TableAssignment; import org.apache.fluss.server.zk.data.TabletServerRegistration; +import org.apache.fluss.server.zk.data.VersionedRemoteLogManifestHandle; import org.apache.fluss.server.zk.data.ZkData; import org.apache.fluss.server.zk.data.ZkData.PartitionIdsZNode; import org.apache.fluss.server.zk.data.ZkData.TableIdsZNode; @@ -135,6 +139,7 @@ import static org.apache.fluss.server.utils.ServerRpcMessageUtils.getAdjustIsrResponseData; import static org.apache.fluss.server.utils.ServerRpcMessageUtils.getUpdateMetadataRequestData; import static org.apache.fluss.server.utils.TableAssignmentUtils.generateAssignment; +import static org.apache.fluss.server.zk.data.TabletServerRegistration.REMOTE_MANIFEST_VERSION_DISPATCH_CAPABILITY; import static org.apache.fluss.testutils.common.CommonTestUtils.retry; import static org.apache.fluss.testutils.common.CommonTestUtils.waitValue; import static org.assertj.core.api.Assertions.assertThat; @@ -177,6 +182,7 @@ class CoordinatorEventProcessorTest { private KvSnapshotLeaseManager kvSnapshotLeaseManager; private Scheduler scheduler; private String remoteDataDir; + private boolean manifestV2WriterEnabledForTest; @BeforeAll static void baseBeforeAll() throws Exception { @@ -204,7 +210,9 @@ static void baseBeforeAll() throws Exception { "rack" + i, Collections.singletonList( new Endpoint("host" + i, 1000, DEFAULT_LISTENER_NAME)), - System.currentTimeMillis())); + System.currentTimeMillis(), + TabletServerResource.unknown(), + Collections.singleton(REMOTE_MANIFEST_VERSION_DISPATCH_CAPABILITY))); } } @@ -1145,6 +1153,10 @@ void testStartupResumesDropPartitionThroughCleanupManager() throws Exception { @Test void testNotifyOffsetsWithShrinkISR(@TempDir Path tempDir) throws Exception { + eventProcessor.shutdown(); + manifestV2WriterEnabledForTest = true; + eventProcessor = buildCoordinatorEventProcessor(); + eventProcessor.startup(); initCoordinatorChannel(Collections.singleton(ApiKeys.UPDATE_METADATA)); TablePath t1 = TablePath.of(defaultDatabase, "test_notify_with_shrink_isr"); final long t1Id = @@ -1218,6 +1230,67 @@ void testNotifyOffsetsWithShrinkISR(@TempDir Path tempDir) throws Exception { () -> verifyReceiveRequestExceptFor( 3, leader, NotifyKvSnapshotOffsetRequest.class)); + + VersionedRemoteLogManifestHandle legacyHandle = + zookeeperClient.getVersionedRemoteLogManifestHandle(tableBucket).get(); + CommitRemoteLogManifestData v2Commit = + CommitRemoteLogManifestData.v2Present( + tableBucket, + new FsPath(tempDir.resolve("v2.manifest").toString()), + 0L, + 10L, + 1L, + legacyHandle.handle().getRemoteLogManifestPath(), + 0L, + legacyHandle.zkVersion(), + coordinatorEpoch, + bucketLeaderEpoch); + CompletableFuture v2Response = new CompletableFuture<>(); + coordinatorEventManager.put(new CommitRemoteLogManifestEvent(v2Commit, v2Response)); + assertThat(v2Response.get().getCommitResult()) + .isEqualTo(RemoteLogManifestCommitResult.COMMITTED.code()); + + CompletableFuture legacyOverwriteResponse = + new CompletableFuture<>(); + coordinatorEventManager.put( + new CommitRemoteLogManifestEvent( + new CommitRemoteLogManifestData( + tableBucket, + new FsPath(tempDir.resolve("legacy-overwrite.manifest").toString()), + 0L, + 20L, + coordinatorEpoch, + bucketLeaderEpoch), + legacyOverwriteResponse)); + assertThat(legacyOverwriteResponse.get().isCommitSuccess()).isFalse(); + assertThat(zookeeperClient.getRemoteLogManifestHandle(tableBucket).get().getVersion()) + .isEqualTo(RemoteLogManifestHandle.VERSION_2); + + CompletableFuture conflictResponse = + new CompletableFuture<>(); + coordinatorEventManager.put(new CommitRemoteLogManifestEvent(v2Commit, conflictResponse)); + assertThat(conflictResponse.get().getCommitResult()) + .isEqualTo(RemoteLogManifestCommitResult.CONFLICT.code()); + + VersionedRemoteLogManifestHandle currentHandle = + zookeeperClient.getVersionedRemoteLogManifestHandle(tableBucket).get(); + CommitRemoteLogManifestData fencedCommit = + CommitRemoteLogManifestData.v2Present( + tableBucket, + new FsPath(tempDir.resolve("v3.manifest").toString()), + 0L, + 20L, + 2L, + currentHandle.handle().getRemoteLogManifestPath(), + 1L, + currentHandle.zkVersion(), + coordinatorEpoch, + bucketLeaderEpoch - 1); + CompletableFuture fencedResponse = + new CompletableFuture<>(); + coordinatorEventManager.put(new CommitRemoteLogManifestEvent(fencedCommit, fencedResponse)); + assertThat(fencedResponse.get().getCommitResult()) + .isEqualTo(RemoteLogManifestCommitResult.FENCED.code()); } @Test @@ -2088,6 +2161,9 @@ private void verifyIsr(TableBucket tb, int expectedLeader, List expecte private CoordinatorEventProcessor buildCoordinatorEventProcessor() { Configuration conf = new Configuration(); conf.set(ConfigOptions.REMOTE_DATA_DIR, remoteDataDir); + conf.set( + ConfigOptions.REMOTE_LOG_MANIFEST_V2_WRITER_ENABLED, + manifestV2WriterEnabledForTest); conf.set(ConfigOptions.COORDINATOR_OFFLINE_LEADER_RETRY_DELAY, Duration.ofDays(1)); return new CoordinatorEventProcessor( zookeeperClient, diff --git a/fluss-server/src/test/java/org/apache/fluss/server/coordinator/TestCoordinatorGateway.java b/fluss-server/src/test/java/org/apache/fluss/server/coordinator/TestCoordinatorGateway.java index 7f3bc32e8c..d88171044d 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/coordinator/TestCoordinatorGateway.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/coordinator/TestCoordinatorGateway.java @@ -121,6 +121,8 @@ import org.apache.fluss.rpc.protocol.ApiError; import org.apache.fluss.server.entity.AdjustIsrResultForBucket; import org.apache.fluss.server.entity.CommitRemoteLogManifestData; +import org.apache.fluss.server.entity.RemoteLogManifestCommitResult; +import org.apache.fluss.server.entity.RemoteLogManifestExpectedHandleState; import org.apache.fluss.server.zk.ZooKeeperClient; import org.apache.fluss.server.zk.data.LeaderAndIsr; import org.apache.fluss.server.zk.data.RemoteLogManifestHandle; @@ -147,6 +149,7 @@ public class TestCoordinatorGateway implements CoordinatorGateway { private final @Nullable ZooKeeperClient zkClient; public final AtomicBoolean commitRemoteLogManifestFail = new AtomicBoolean(false); + public final AtomicBoolean commitRemoteLogManifestConflictOnce = new AtomicBoolean(false); public final Map currentLeaderEpoch = new HashMap<>(); private Set shutdownTabletServers; private boolean networkIssueEnable = false; @@ -369,7 +372,43 @@ public CompletableFuture commitRemoteLogManifes CommitRemoteLogManifestData commitRemoteLogManifestData = getCommitRemoteLogManifestData(request); CommitRemoteLogManifestResponse response = new CommitRemoteLogManifestResponse(); + if (commitRemoteLogManifestData.isV2CasCommit() + && commitRemoteLogManifestConflictOnce.getAndSet(false)) { + return CompletableFuture.completedFuture( + response.setCommitSuccess(false) + .setCommitResult(RemoteLogManifestCommitResult.CONFLICT.code())); + } try { + if (commitRemoteLogManifestData.isV2CasCommit()) { + RemoteLogManifestHandle newHandle = + RemoteLogManifestHandle.v2( + commitRemoteLogManifestData.getRemoteLogManifestPath(), + commitRemoteLogManifestData.getNewManifestGeneration(), + commitRemoteLogManifestData.getRemoteLogStartOffset(), + commitRemoteLogManifestData.getRemoteLogEndOffset()); + boolean committed; + if (commitRemoteLogManifestData.getExpectedHandleState() + == RemoteLogManifestExpectedHandleState.ABSENT) { + committed = + zkClient.createRemoteLogManifestHandleIfAbsent( + commitRemoteLogManifestData.getTableBucket(), newHandle); + } else { + committed = + zkClient.compareAndSetRemoteLogManifestHandle( + commitRemoteLogManifestData.getTableBucket(), + commitRemoteLogManifestData.getExpectedManifestPath(), + commitRemoteLogManifestData.getExpectedManifestGeneration(), + commitRemoteLogManifestData.getExpectedZkVersion(), + newHandle); + } + return CompletableFuture.completedFuture( + response.setCommitSuccess(committed) + .setCommitResult( + (committed + ? RemoteLogManifestCommitResult.COMMITTED + : RemoteLogManifestCommitResult.CONFLICT) + .code())); + } zkClient.upsertRemoteLogManifestHandle( commitRemoteLogManifestData.getTableBucket(), new RemoteLogManifestHandle( diff --git a/fluss-server/src/test/java/org/apache/fluss/server/kv/RemoteLogFetcherTest.java b/fluss-server/src/test/java/org/apache/fluss/server/kv/RemoteLogFetcherTest.java index d8bd5e9d96..5f08bec283 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/kv/RemoteLogFetcherTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/kv/RemoteLogFetcherTest.java @@ -20,8 +20,13 @@ import org.apache.fluss.exception.RemoteStorageException; import org.apache.fluss.metadata.TableBucket; import org.apache.fluss.record.LogRecordBatch; +import org.apache.fluss.record.MemoryLogRecords; +import org.apache.fluss.remote.RemoteLogManifest; import org.apache.fluss.remote.RemoteLogSegment; +import org.apache.fluss.remote.RemoteLogSegmentReference; +import org.apache.fluss.server.log.LogSegment; import org.apache.fluss.server.log.LogTablet; +import org.apache.fluss.server.log.remote.LogSegmentFiles; import org.apache.fluss.server.log.remote.RemoteLogTestBase; import org.apache.fluss.server.replica.Replica; @@ -32,8 +37,11 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; import java.util.Iterator; import java.util.List; +import java.util.Optional; import java.util.UUID; import java.util.concurrent.CancellationException; import java.util.concurrent.CountDownLatch; @@ -42,7 +50,9 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; +import static org.apache.fluss.record.TestData.DATA1; import static org.apache.fluss.record.TestData.DATA1_TABLE_ID; +import static org.apache.fluss.testutils.DataTestUtils.genMemoryLogRecordsWithWriterId; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -107,6 +117,74 @@ void testBasicFetch() throws Exception { } } + @Test + void testFetchOverlappingV2SegmentsFromReplicasWithDifferentBoundaries() throws Exception { + TableBucket targetBucket = new TableBucket(DATA1_TABLE_ID, 0); + TableBucket sourceBucket = new TableBucket(DATA1_TABLE_ID, 1); + makeLogTableAsLeader(targetBucket, false); + makeLogTableAsLeader(sourceBucket, false); + LogTablet oldLeaderLog = replicaManager.getReplicaOrException(targetBucket).getLogTablet(); + LogTablet newLeaderLog = replicaManager.getReplicaOrException(sourceBucket).getLogTablet(); + + // Both logs append the exact same record batches. They only roll at different batch + // boundaries: the old leader rolls at 20, while the future leader rolls at 10 and 30. + for (int offset = 0; offset < 30; offset++) { + MemoryLogRecords records = + genMemoryLogRecordsWithWriterId( + Collections.singletonList(DATA1.get(offset % DATA1.size())), + offset + 1L, + 0, + offset); + oldLeaderLog.appendAsLeader(records); + newLeaderLog.appendAsFollower(records); + if (offset == 9 || offset == 29) { + newLeaderLog.roll(Optional.empty()); + } + if (offset == 19) { + oldLeaderLog.roll(Optional.empty()); + } + } + oldLeaderLog.updateHighWatermark(oldLeaderLog.localLogEndOffset()); + newLeaderLog.updateHighWatermark(newLeaderLog.localLogEndOffset()); + + RemoteLogSegment oldLeaderSegment = + copyLogSegmentToRemote(oldLeaderLog, remoteLogStorage, 0); + RemoteLogSegment newLeaderSegment = + copySegmentToRemoteForBucket(newLeaderLog, 1, targetBucket); + assertThat(oldLeaderSegment.remoteLogStartOffset()).isZero(); + assertThat(oldLeaderSegment.remoteLogEndOffset()).isEqualTo(20L); + assertThat(newLeaderSegment.remoteLogStartOffset()).isEqualTo(10L); + assertThat(newLeaderSegment.remoteLogEndOffset()).isEqualTo(30L); + + RemoteLogManifest manifest = + RemoteLogManifest.createV2( + 1L, + oldLeaderLog.getPhysicalTablePath(), + targetBucket, + Arrays.asList(oldLeaderSegment, newLeaderSegment), + 0L, + Collections.emptyList()); + remoteLogManager.remoteLogTablet(targetBucket).replaceManifest(manifest); + assertThat(manifest.getRemoteLogSegmentReferences()) + .containsExactly( + new RemoteLogSegmentReference(oldLeaderSegment, 0L, 10L), + new RemoteLogSegmentReference(newLeaderSegment, 10L, 30L)); + + List fetchedBatches = new ArrayList<>(); + try (RemoteLogFetcher fetcher = newFetcher(targetBucket, oldLeaderLog.getLogDir())) { + for (LogRecordBatch batch : fetcher.fetch(0L, 30L)) { + fetchedBatches.add(batch); + } + } + + assertThat(fetchedBatches).hasSize(30); + for (int offset = 0; offset < fetchedBatches.size(); offset++) { + LogRecordBatch batch = fetchedBatches.get(offset); + assertThat(batch.baseLogOffset()).isEqualTo(offset); + assertThat(batch.nextLogOffset()).isEqualTo(offset + 1L); + } + } + @Test void testFetchWithStartOffsetInMiddleOfSegment() throws Exception { TableBucket tb = new TableBucket(DATA1_TABLE_ID, 0); @@ -853,6 +931,32 @@ void testCloseInterruptsOngoingDownload() throws Exception { } } + private RemoteLogSegment copySegmentToRemoteForBucket( + LogTablet sourceLog, int segmentIndex, TableBucket targetBucket) throws Exception { + List segments = sourceLog.getSegments(); + LogSegment sourceSegment = segments.get(segmentIndex); + long nextOffset = segments.get(segmentIndex + 1).getBaseOffset(); + File writerSnapshot = sourceLog.writerStateManager().fetchSnapshot(nextOffset).orElse(null); + LogSegmentFiles files = + new LogSegmentFiles( + sourceSegment.getFileLogRecords().file().toPath(), + sourceSegment.offsetIndex().file().toPath(), + sourceSegment.timeIndex().file().toPath(), + writerSnapshot == null ? null : writerSnapshot.toPath()); + RemoteLogSegment remoteSegment = + RemoteLogSegment.Builder.builder() + .remoteLogSegmentId(UUID.randomUUID()) + .remoteLogStartOffset(sourceSegment.getBaseOffset()) + .remoteLogEndOffset(nextOffset) + .maxTimestamp(sourceSegment.maxTimestampSoFar()) + .segmentSizeInBytes(sourceSegment.getFileLogRecords().sizeInBytes()) + .tableBucket(targetBucket) + .physicalTablePath(sourceLog.getPhysicalTablePath()) + .build(); + remoteLogStorage.copyLogSegmentFiles(remoteSegment, files); + return remoteSegment; + } + private static void waitUntilLogFileCountAtLeast(Path dir, int expectedCount) throws Exception { long deadline = System.currentTimeMillis() + 5_000L; while (System.currentTimeMillis() < deadline) { diff --git a/fluss-server/src/test/java/org/apache/fluss/server/log/remote/RemoteLogManagerTest.java b/fluss-server/src/test/java/org/apache/fluss/server/log/remote/RemoteLogManagerTest.java index 96e8ae704f..79e4d0a7ae 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/log/remote/RemoteLogManagerTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/log/remote/RemoteLogManagerTest.java @@ -22,6 +22,8 @@ import org.apache.fluss.fs.FsPath; import org.apache.fluss.metadata.TableBucket; import org.apache.fluss.remote.RemoteLogFetchInfo; +import org.apache.fluss.remote.RemoteLogManifest; +import org.apache.fluss.remote.RemoteLogManifestV2Migration; import org.apache.fluss.remote.RemoteLogSegment; import org.apache.fluss.rpc.entity.FetchLogResultForBucket; import org.apache.fluss.rpc.protocol.ApiError; @@ -37,6 +39,7 @@ import org.apache.fluss.server.replica.ReplicaManager; import org.apache.fluss.server.testutils.ServerTestTags; import org.apache.fluss.server.zk.data.LeaderAndIsr; +import org.apache.fluss.server.zk.data.RemoteLogManifestHandle; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Tag; @@ -107,6 +110,52 @@ void testBecomeLeaderWithoutRemoteLogManifest(boolean partitionTable) throws Exc .isGreaterThan(10); } + @Test + void testValidateManifestHandleHints() throws Exception { + TableBucket tableBucket = makeTableBucket(false); + makeLogTableAsLeader(tableBucket, false); + LogTablet logTablet = replicaManager.getReplicaOrException(tableBucket).getLogTablet(); + addMultiSegmentsToLogTablet(logTablet, 1); + RemoteLogSegment segment = createRemoteLogSegmentList(logTablet).get(0); + RemoteLogManifest manifest = + RemoteLogManifest.createV2( + 2L, + logTablet.getPhysicalTablePath(), + tableBucket, + Collections.singletonList(segment), + segment.remoteLogStartOffset(), + Collections.emptyList()); + FsPath manifestPath = new FsPath("file:///remote/m2"); + + RemoteLogManager.validateManifestHandle( + RemoteLogManifestHandle.v2( + manifestPath, + 2L, + manifest.getRemoteLogStartOffset(), + manifest.getRemoteLogEndOffset()), + manifest); + + assertThatThrownBy( + () -> + RemoteLogManager.validateManifestHandle( + RemoteLogManifestHandle.v2( + manifestPath, + 3L, + manifest.getRemoteLogStartOffset(), + manifest.getRemoteLogEndOffset()), + manifest)) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("hints do not match"); + assertThatThrownBy( + () -> + RemoteLogManager.validateManifestHandle( + new RemoteLogManifestHandle( + manifestPath, manifest.getRemoteLogEndOffset()), + manifest)) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("V1 remote log manifest handle"); + } + @ParameterizedTest @ValueSource(booleans = {true, false}) void testBecomeLeaderWithRemoteLogLogManifest(boolean partitionTable) throws Exception { @@ -355,6 +404,37 @@ void testFetchRecordsFromRemote(boolean partitionTable) throws Exception { assertThat(remoteLogFetchInfo).isNotNull(); assertThat(remoteLogFetchInfo.remoteLogSegmentList().size()).isEqualTo(4); + // A V2 manifest with physically adjacent segments preserves the legacy v0 batching + // behavior and returns the complete contiguous, non-overlapping tail. + RemoteLogTablet remoteLogTablet = remoteLogManager.remoteLogTablet(tb); + remoteLogTablet.replaceManifest( + RemoteLogManifestV2Migration.migrate( + remoteLogTablet.currentManifest(), 1L, System.currentTimeMillis())); + future = new CompletableFuture<>(); + replicaManager.fetchLogRecords( + new FetchParams(-1, Integer.MAX_VALUE), + Collections.singletonMap(tb, new FetchReqInfo(tb.getTableId(), 0L, 1024 * 1024)), + null, + future::complete); + RemoteLogFetchInfo legacyV2FetchInfo = future.get().get(tb).remoteLogFetchInfo(); + assertThat(legacyV2FetchInfo).isNotNull(); + assertThat(legacyV2FetchInfo.remoteLogSegmentList()).hasSize(4); + assertThat(legacyV2FetchInfo.remoteLogSegmentList().get(0).remoteLogStartOffset()) + .isEqualTo(0L); + + // A request starting at a later segment returns the remaining contiguous tail. + future = new CompletableFuture<>(); + replicaManager.fetchLogRecords( + new FetchParams(-1, Integer.MAX_VALUE), + Collections.singletonMap(tb, new FetchReqInfo(tb.getTableId(), 10L, 1024 * 1024)), + null, + future::complete); + legacyV2FetchInfo = future.get().get(tb).remoteLogFetchInfo(); + assertThat(legacyV2FetchInfo.remoteLogSegmentList()).hasSize(3); + assertThat(legacyV2FetchInfo.remoteLogSegmentList().get(0).remoteLogStartOffset()) + .isEqualTo(10L); + assertThat(legacyV2FetchInfo.firstStartPos()).isZero(); + // 2. then, fetch records from active segments, result rhe local records in server. future = new CompletableFuture<>(); replicaManager.fetchLogRecords( diff --git a/fluss-server/src/test/java/org/apache/fluss/server/log/remote/RemoteLogManifestCommitterTest.java b/fluss-server/src/test/java/org/apache/fluss/server/log/remote/RemoteLogManifestCommitterTest.java new file mode 100644 index 0000000000..f75999dc00 --- /dev/null +++ b/fluss-server/src/test/java/org/apache/fluss/server/log/remote/RemoteLogManifestCommitterTest.java @@ -0,0 +1,310 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.server.log.remote; + +import org.apache.fluss.fs.FsPath; +import org.apache.fluss.metadata.TableBucket; +import org.apache.fluss.remote.RemoteLogManifest; +import org.apache.fluss.remote.RemoteLogSegment; +import org.apache.fluss.rpc.gateway.CoordinatorGateway; +import org.apache.fluss.rpc.messages.CommitRemoteLogManifestRequest; +import org.apache.fluss.rpc.messages.CommitRemoteLogManifestResponse; +import org.apache.fluss.server.entity.CommitRemoteLogManifestData; +import org.apache.fluss.server.entity.RemoteLogManifestCommitResult; +import org.apache.fluss.server.zk.NOPErrorHandler; +import org.apache.fluss.server.zk.ZooKeeperClient; +import org.apache.fluss.server.zk.ZooKeeperExtension; +import org.apache.fluss.server.zk.data.RemoteLogManifestHandle; +import org.apache.fluss.server.zk.data.VersionedRemoteLogManifestHandle; +import org.apache.fluss.testutils.common.AllCallbackWrapper; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +import java.util.Collections; +import java.util.UUID; +import java.util.concurrent.CompletableFuture; + +import static org.apache.fluss.record.TestData.DATA1_PHYSICAL_TABLE_PATH; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** Tests transport-unknown reconciliation for {@link RemoteLogManifestCommitter}. */ +class RemoteLogManifestCommitterTest { + @RegisterExtension + public static final AllCallbackWrapper ZOO_KEEPER_EXTENSION_WRAPPER = + new AllCallbackWrapper<>(new ZooKeeperExtension()); + + private static ZooKeeperClient zooKeeperClient; + + @BeforeAll + static void beforeAll() { + zooKeeperClient = + ZOO_KEEPER_EXTENSION_WRAPPER + .getCustomExtension() + .getZooKeeperClient(NOPErrorHandler.INSTANCE); + } + + @AfterEach + void afterEach() { + ZOO_KEEPER_EXTENSION_WRAPPER.getCustomExtension().cleanupRoot(); + } + + @Test + void testReconcileLostAbsentCreateResponse() throws Exception { + TableBucket tableBucket = new TableBucket(1L, 0); + CommitRemoteLogManifestData data = absentData(tableBucket, "m1"); + CoordinatorGateway gateway = mock(CoordinatorGateway.class); + when(gateway.commitRemoteLogManifest(any(CommitRemoteLogManifestRequest.class))) + .thenAnswer( + ignored -> { + zooKeeperClient.createRemoteLogManifestHandleIfAbsent( + tableBucket, handleFor(data)); + return failedFuture(); + }); + + RemoteLogManifestCommitResult result = + new RemoteLogManifestCommitter(gateway, zooKeeperClient).commit(data); + + assertThat(result).isEqualTo(RemoteLogManifestCommitResult.COMMITTED); + verify(gateway).commitRemoteLogManifest(any(CommitRemoteLogManifestRequest.class)); + } + + @Test + void testReconcileAbsentCreateConflict() throws Exception { + TableBucket tableBucket = new TableBucket(2L, 0); + CommitRemoteLogManifestData data = absentData(tableBucket, "ours"); + zooKeeperClient.createRemoteLogManifestHandleIfAbsent( + tableBucket, + RemoteLogManifestHandle.v2(new FsPath("file:///remote/other"), 1L, 0L, 10L)); + CoordinatorGateway gateway = mock(CoordinatorGateway.class); + when(gateway.commitRemoteLogManifest(any(CommitRemoteLogManifestRequest.class))) + .thenReturn(failedFuture()); + + assertThat(new RemoteLogManifestCommitter(gateway, zooKeeperClient).commit(data)) + .isEqualTo(RemoteLogManifestCommitResult.CONFLICT); + } + + @Test + void testRetryWhenHandleRemainsAbsent() throws Exception { + TableBucket tableBucket = new TableBucket(3L, 0); + CommitRemoteLogManifestData data = absentData(tableBucket, "m1"); + CoordinatorGateway gateway = mock(CoordinatorGateway.class); + when(gateway.commitRemoteLogManifest(any(CommitRemoteLogManifestRequest.class))) + .thenReturn( + failedFuture(), + CompletableFuture.completedFuture( + new CommitRemoteLogManifestResponse() + .setCommitSuccess(true) + .setCommitResult( + RemoteLogManifestCommitResult.COMMITTED.code()))); + + assertThat(new RemoteLogManifestCommitter(gateway, zooKeeperClient).commit(data)) + .isEqualTo(RemoteLogManifestCommitResult.COMMITTED); + verify(gateway, times(2)) + .commitRemoteLogManifest(any(CommitRemoteLogManifestRequest.class)); + } + + @Test + void testLegacyBooleanResponseIsNotAcceptedAsV2Commit() throws Exception { + TableBucket tableBucket = new TableBucket(8L, 0); + CommitRemoteLogManifestData data = absentData(tableBucket, "m1"); + CoordinatorGateway gateway = mock(CoordinatorGateway.class); + when(gateway.commitRemoteLogManifest(any(CommitRemoteLogManifestRequest.class))) + .thenReturn( + CompletableFuture.completedFuture( + new CommitRemoteLogManifestResponse().setCommitSuccess(true))); + + assertThatThrownBy( + () -> new RemoteLogManifestCommitter(gateway, zooKeeperClient).commit(data)) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("remains unknown"); + verify(gateway, times(10)) + .commitRemoteLogManifest(any(CommitRemoteLogManifestRequest.class)); + } + + @Test + void testReconcileLostPresentCasResponse() throws Exception { + TableBucket tableBucket = new TableBucket(4L, 0); + FsPath basePath = new FsPath("file:///remote/v1"); + zooKeeperClient.createRemoteLogManifestHandleIfAbsent( + tableBucket, new RemoteLogManifestHandle(basePath, 10L)); + VersionedRemoteLogManifestHandle base = + zooKeeperClient.getVersionedRemoteLogManifestHandle(tableBucket).get(); + CommitRemoteLogManifestData data = + CommitRemoteLogManifestData.v2Present( + tableBucket, + new FsPath("file:///remote/v2"), + 0L, + 20L, + 1L, + basePath, + 0L, + base.zkVersion(), + 1, + 1); + CoordinatorGateway gateway = mock(CoordinatorGateway.class); + when(gateway.commitRemoteLogManifest(any(CommitRemoteLogManifestRequest.class))) + .thenAnswer( + ignored -> { + zooKeeperClient.compareAndSetRemoteLogManifestHandle( + tableBucket, basePath, 0L, base.zkVersion(), handleFor(data)); + return failedFuture(); + }); + + assertThat(new RemoteLogManifestCommitter(gateway, zooKeeperClient).commit(data)) + .isEqualTo(RemoteLogManifestCommitResult.COMMITTED); + } + + @Test + void testRetryWhenPresentBaseRemainsAuthoritative() throws Exception { + TableBucket tableBucket = new TableBucket(5L, 0); + FsPath basePath = new FsPath("file:///remote/v1"); + zooKeeperClient.createRemoteLogManifestHandleIfAbsent( + tableBucket, new RemoteLogManifestHandle(basePath, 10L)); + VersionedRemoteLogManifestHandle base = + zooKeeperClient.getVersionedRemoteLogManifestHandle(tableBucket).get(); + CommitRemoteLogManifestData data = presentData(tableBucket, base, "v2"); + CoordinatorGateway gateway = mock(CoordinatorGateway.class); + when(gateway.commitRemoteLogManifest(any(CommitRemoteLogManifestRequest.class))) + .thenReturn( + failedFuture(), + CompletableFuture.completedFuture( + new CommitRemoteLogManifestResponse() + .setCommitSuccess(true) + .setCommitResult( + RemoteLogManifestCommitResult.COMMITTED.code()))); + + assertThat(new RemoteLogManifestCommitter(gateway, zooKeeperClient).commit(data)) + .isEqualTo(RemoteLogManifestCommitResult.COMMITTED); + verify(gateway, times(2)) + .commitRemoteLogManifest(any(CommitRemoteLogManifestRequest.class)); + } + + @Test + void testReconcilePresentConflict() throws Exception { + TableBucket tableBucket = new TableBucket(6L, 0); + FsPath basePath = new FsPath("file:///remote/v1"); + zooKeeperClient.createRemoteLogManifestHandleIfAbsent( + tableBucket, new RemoteLogManifestHandle(basePath, 10L)); + VersionedRemoteLogManifestHandle base = + zooKeeperClient.getVersionedRemoteLogManifestHandle(tableBucket).get(); + CommitRemoteLogManifestData data = presentData(tableBucket, base, "ours"); + zooKeeperClient.compareAndSetRemoteLogManifestHandle( + tableBucket, + basePath, + 0L, + base.zkVersion(), + RemoteLogManifestHandle.v2(new FsPath("file:///remote/other"), 1L, 0L, 20L)); + CoordinatorGateway gateway = mock(CoordinatorGateway.class); + when(gateway.commitRemoteLogManifest(any(CommitRemoteLogManifestRequest.class))) + .thenReturn(failedFuture()); + + assertThat(new RemoteLogManifestCommitter(gateway, zooKeeperClient).commit(data)) + .isEqualTo(RemoteLogManifestCommitResult.CONFLICT); + } + + @Test + void testCommitAndApplyFullManifestReplacement() throws Exception { + TableBucket tableBucket = new TableBucket(7L, 0); + CommitRemoteLogManifestData data = absentData(tableBucket, "m1"); + RemoteLogSegment segment = + RemoteLogSegment.Builder.builder() + .remoteLogSegmentId(UUID.randomUUID()) + .remoteLogStartOffset(0L) + .remoteLogEndOffset(10L) + .maxTimestamp(1L) + .segmentSizeInBytes(100) + .tableBucket(tableBucket) + .physicalTablePath(DATA1_PHYSICAL_TABLE_PATH) + .build(); + RemoteLogManifest manifest = + RemoteLogManifest.createV2( + 1L, + DATA1_PHYSICAL_TABLE_PATH, + tableBucket, + Collections.singletonList(segment), + 0L, + Collections.emptyList()); + RemoteLogTablet tablet = new RemoteLogTablet(DATA1_PHYSICAL_TABLE_PATH, tableBucket, -1L); + CoordinatorGateway gateway = mock(CoordinatorGateway.class); + when(gateway.commitRemoteLogManifest(any(CommitRemoteLogManifestRequest.class))) + .thenAnswer( + ignored -> { + zooKeeperClient.createRemoteLogManifestHandleIfAbsent( + tableBucket, handleFor(data)); + return CompletableFuture.completedFuture( + new CommitRemoteLogManifestResponse() + .setCommitSuccess(true) + .setCommitResult( + RemoteLogManifestCommitResult.COMMITTED + .code())); + }); + + RemoteLogManifestCommitResult result = + new RemoteLogManifestCommitter(gateway, zooKeeperClient) + .commitAndApply(data, manifest, tablet); + + assertThat(result).isEqualTo(RemoteLogManifestCommitResult.COMMITTED); + assertThat(tablet.currentManifest()).isSameAs(manifest); + assertThat(tablet.currentHandle()).isNotNull(); + assertThat(tablet.currentHandle().handle()).isEqualTo(handleFor(data)); + } + + private static CommitRemoteLogManifestData absentData( + TableBucket tableBucket, String manifestName) { + return CommitRemoteLogManifestData.v2Absent( + tableBucket, new FsPath("file:///remote/" + manifestName), 0L, 10L, 1L, 1, 1); + } + + private static CommitRemoteLogManifestData presentData( + TableBucket tableBucket, VersionedRemoteLogManifestHandle base, String manifestName) { + return CommitRemoteLogManifestData.v2Present( + tableBucket, + new FsPath("file:///remote/" + manifestName), + 0L, + 20L, + 1L, + base.handle().getRemoteLogManifestPath(), + base.handle().getManifestGeneration().orElse(0L), + base.zkVersion(), + 1, + 1); + } + + private static RemoteLogManifestHandle handleFor(CommitRemoteLogManifestData data) { + return RemoteLogManifestHandle.v2( + data.getRemoteLogManifestPath(), + data.getNewManifestGeneration(), + data.getRemoteLogStartOffset(), + data.getRemoteLogEndOffset()); + } + + private static CompletableFuture failedFuture() { + CompletableFuture future = new CompletableFuture<>(); + future.completeExceptionally(new RuntimeException("response lost")); + return future; + } +} diff --git a/fluss-server/src/test/java/org/apache/fluss/server/log/remote/RemoteLogManifestV2WriterTest.java b/fluss-server/src/test/java/org/apache/fluss/server/log/remote/RemoteLogManifestV2WriterTest.java new file mode 100644 index 0000000000..7171fb6bd8 --- /dev/null +++ b/fluss-server/src/test/java/org/apache/fluss/server/log/remote/RemoteLogManifestV2WriterTest.java @@ -0,0 +1,331 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.server.log.remote; + +import org.apache.fluss.config.ConfigOptions; +import org.apache.fluss.config.Configuration; +import org.apache.fluss.config.MemorySize; +import org.apache.fluss.fs.FsPath; +import org.apache.fluss.metadata.TableBucket; +import org.apache.fluss.remote.RemoteLogManifest; +import org.apache.fluss.remote.RemoteLogSegment; +import org.apache.fluss.remote.RemoteLogSegmentReference; +import org.apache.fluss.server.log.LogTablet; +import org.apache.fluss.server.zk.data.RemoteLogManifestHandle; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.time.Duration; +import java.util.Collections; +import java.util.List; +import java.util.UUID; +import java.util.stream.Collectors; + +import static org.apache.fluss.record.TestData.DATA1_PHYSICAL_TABLE_PATH; +import static org.apache.fluss.record.TestData.DATA1_TABLE_ID; +import static org.assertj.core.api.Assertions.assertThat; + +/** Integration tests for the feature-gated Manifest V2 writer. */ +class RemoteLogManifestV2WriterTest extends RemoteLogTestBase { + + @Override + public Configuration getServerConf() { + Configuration conf = new Configuration(); + conf.set(ConfigOptions.LOG_INDEX_INTERVAL_SIZE, MemorySize.parse("1b")); + conf.set(ConfigOptions.REMOTE_LOG_INDEX_FILE_CACHE_SIZE, MemorySize.parse("1mb")); + conf.set(ConfigOptions.REMOTE_FS_WRITE_BUFFER_SIZE, MemorySize.parse("10b")); + conf.setInt(ConfigOptions.REMOTE_LOG_TASK_MAX_UPLOAD_SEGMENTS, Integer.MAX_VALUE); + conf.setBoolean(ConfigOptions.REMOTE_LOG_MANIFEST_V2_WRITER_ENABLED, true); + conf.setBoolean(ConfigOptions.REMOTE_LOG_MANIFEST_V2_GC_ENABLED, true); + conf.set(ConfigOptions.REMOTE_LOG_MANIFEST_V2_GC_GRACE_PERIOD, Duration.ofHours(1)); + return conf; + } + + @BeforeEach + public void setup() throws Exception { + super.setup(); + } + + @Test + void testInitialCopyPublishesOneGenerationForMultipleSegments() throws Exception { + TableBucket tableBucket = new TableBucket(DATA1_TABLE_ID, 0); + makeLogTableAsLeader(tableBucket, false); + addMultiSegmentsToLogTablet( + replicaManager.getReplicaOrException(tableBucket).getLogTablet(), 5); + + remoteLogTaskScheduler.triggerPeriodicScheduledTasks(); + + RemoteLogTablet remoteLogTablet = remoteLogManager.remoteLogTablet(tableBucket); + assertThat(remoteLogTablet.currentManifest().getVersion()) + .isEqualTo(RemoteLogManifest.VERSION_2); + assertThat(remoteLogTablet.currentManifest().getGeneration()).isEqualTo(1L); + assertThat(remoteLogTablet.allRemoteLogSegments()).hasSize(4); + assertThat(remoteLogTablet.currentHandle()).isNotNull(); + assertThat(remoteLogTablet.currentHandle().handle().getVersion()) + .isEqualTo(RemoteLogManifestHandle.VERSION_2); + } + + @Test + void testOverlappingLocalSegmentReplacesRemoteSuffix() throws Exception { + TableBucket tableBucket = new TableBucket(DATA1_TABLE_ID, 0); + RemoteLogSegment remoteSegment = + RemoteLogSegment.Builder.builder() + .physicalTablePath(DATA1_PHYSICAL_TABLE_PATH) + .tableBucket(tableBucket) + .remoteLogSegmentId(UUID.randomUUID()) + .remoteLogStartOffset(0L) + .remoteLogEndOffset(15L) + .maxTimestamp(manualClock.milliseconds()) + .segmentSizeInBytes(1) + .build(); + RemoteLogManifest baseManifest = + new RemoteLogManifest( + DATA1_PHYSICAL_TABLE_PATH, + tableBucket, + Collections.singletonList(remoteSegment)); + FsPath basePath = remoteLogStorage.writeRemoteLogManifestSnapshot(baseManifest); + zkClient.createRemoteLogManifestHandleIfAbsent( + tableBucket, new RemoteLogManifestHandle(basePath, 15L)); + + makeLogTableAsLeader(tableBucket, false); + LogTablet logTablet = replicaManager.getReplicaOrException(tableBucket).getLogTablet(); + addMultiSegmentsToLogTablet(logTablet, 3); + + remoteLogTaskScheduler.triggerPeriodicScheduledTasks(); + + RemoteLogManifest result = remoteLogManager.remoteLogTablet(tableBucket).currentManifest(); + assertThat(result.getGeneration()).isEqualTo(1L); + assertThat(result.getRemoteLogSegmentList()).hasSize(2); + RemoteLogSegment replacement = result.getRemoteLogSegmentList().get(1); + assertThat(replacement.remoteLogStartOffset()).isEqualTo(10L); + assertThat(replacement.remoteLogEndOffset()).isEqualTo(20L); + assertThat(result.getRemoteLogSegmentReferences()) + .containsExactly( + new RemoteLogSegmentReference(remoteSegment, 0L, 10L), + new RemoteLogSegmentReference(replacement, 10L, 20L)); + } + + @Test + void testPartialCopyCommitsOnlySuccessfulPrefix() throws Exception { + TableBucket tableBucket = new TableBucket(DATA1_TABLE_ID, 0); + makeLogTableAsLeader(tableBucket, false); + addMultiSegmentsToLogTablet( + replicaManager.getReplicaOrException(tableBucket).getLogTablet(), 5); + remoteLogStorage.copySegmentFailAfterNCopies.set(2); + + remoteLogTaskScheduler.triggerPeriodicScheduledTasks(); + + RemoteLogManifest first = remoteLogManager.remoteLogTablet(tableBucket).currentManifest(); + assertThat(first.getGeneration()).isEqualTo(1L); + assertThat(first.getRemoteLogSegmentList()).hasSize(2); + assertThat(first.getRemoteLogEndOffset()).isEqualTo(20L); + + remoteLogStorage.copySegmentFailAfterNCopies.set(-1); + remoteLogTaskScheduler.triggerPeriodicScheduledTasks(); + + RemoteLogManifest second = remoteLogManager.remoteLogTablet(tableBucket).currentManifest(); + assertThat(second.getGeneration()).isEqualTo(2L); + assertThat(second.getRemoteLogSegmentList()).hasSize(4); + assertThat(second.getRemoteLogEndOffset()).isEqualTo(40L); + } + + @Test + void testCasConflictCleansRejectedObjectsAndReplans() throws Exception { + TableBucket tableBucket = new TableBucket(DATA1_TABLE_ID, 0); + makeLogTableAsLeader(tableBucket, false); + addMultiSegmentsToLogTablet( + replicaManager.getReplicaOrException(tableBucket).getLogTablet(), 3); + testCoordinatorGateway.commitRemoteLogManifestConflictOnce.set(true); + + remoteLogTaskScheduler.triggerPeriodicScheduledTasks(); + + RemoteLogManifest result = remoteLogManager.remoteLogTablet(tableBucket).currentManifest(); + assertThat(result.getGeneration()).isEqualTo(1L); + assertThat(result.getRemoteLogSegmentList()).hasSize(2); + assertThat(listRemoteLogFiles(tableBucket)) + .isEqualTo( + result.getRemoteLogSegmentList().stream() + .map(segment -> segment.remoteLogSegmentId().toString()) + .collect(Collectors.toSet())); + } + + @Test + void testTtlMovesPrefixToUnreferencedWithoutPhysicalDeletion() throws Exception { + TableBucket tableBucket = new TableBucket(DATA1_TABLE_ID, 0); + makeLogTableAsLeader(tableBucket, false); + LogTablet logTablet = replicaManager.getReplicaOrException(tableBucket).getLogTablet(); + addMultiSegmentsToLogTablet(logTablet, 5, false); + remoteLogTaskScheduler.triggerPeriodicScheduledTasks(); + List firstGenerationSegments = + remoteLogManager + .remoteLogTablet(tableBucket) + .currentManifest() + .getRemoteLogSegmentList(); + + manualClock.advanceTime(Duration.ofDays(8)); + addMultiSegmentsToLogTablet(logTablet, 2, false); + remoteLogTaskScheduler.triggerPeriodicScheduledTasks(); + + RemoteLogManifest result = remoteLogManager.remoteLogTablet(tableBucket).currentManifest(); + assertThat(result.getUnreferencedRemoteLogSegments()).isNotEmpty(); + assertThat(result.getUnreferencedRemoteLogSegments()) + .extracting(entry -> entry.remoteLogSegment().remoteLogSegmentId()) + .containsAll( + firstGenerationSegments.stream() + .map(RemoteLogSegment::remoteLogSegmentId) + .collect(Collectors.toSet())); + assertThat(listRemoteLogFiles(tableBucket)) + .containsAll( + result.getUnreferencedRemoteLogSegments().stream() + .map( + entry -> + entry.remoteLogSegment() + .remoteLogSegmentId() + .toString()) + .collect(Collectors.toSet())); + } + + @Test + void testGcDeletesAfterGraceAndThenRemovesMetadata() throws Exception { + TableBucket tableBucket = new TableBucket(DATA1_TABLE_ID, 0); + makeLogTableAsLeader(tableBucket, false); + LogTablet logTablet = replicaManager.getReplicaOrException(tableBucket).getLogTablet(); + addMultiSegmentsToLogTablet(logTablet, 5, false); + remoteLogTaskScheduler.triggerPeriodicScheduledTasks(); + + manualClock.advanceTime(Duration.ofDays(8)); + addMultiSegmentsToLogTablet(logTablet, 2, false); + remoteLogTaskScheduler.triggerPeriodicScheduledTasks(); + RemoteLogManifest beforeGc = + remoteLogManager.remoteLogTablet(tableBucket).currentManifest(); + assertThat(beforeGc.getUnreferencedRemoteLogSegments()).isNotEmpty(); + assertThat(remoteLogManager.remoteLogTablet(tableBucket).getUnreferencedSegmentCount()) + .isEqualTo(beforeGc.getUnreferencedRemoteLogSegments().size()); + assertThat(remoteLogManager.remoteLogTablet(tableBucket).getUnreferencedSizeInBytes()) + .isPositive(); + List garbageIds = + beforeGc.getUnreferencedRemoteLogSegments().stream() + .map(entry -> entry.remoteLogSegment().remoteLogSegmentId().toString()) + .collect(Collectors.toList()); + + manualClock.advanceTime(Duration.ofHours(1)); + remoteLogTaskScheduler.triggerPeriodicScheduledTasks(); + + RemoteLogManifest afterGc = remoteLogManager.remoteLogTablet(tableBucket).currentManifest(); + assertThat(afterGc.getGeneration()).isEqualTo(beforeGc.getGeneration() + 1L); + assertThat(afterGc.getUnreferencedRemoteLogSegments()).isEmpty(); + assertThat(remoteLogManager.remoteLogTablet(tableBucket).getUnreferencedSizeInBytes()) + .isZero(); + assertThat(listRemoteLogFiles(tableBucket)).doesNotContainAnyElementsOf(garbageIds); + } + + @Test + void testGcDeleteFailureRetainsMetadataAndRetries() throws Exception { + TableBucket tableBucket = new TableBucket(DATA1_TABLE_ID, 0); + makeLogTableAsLeader(tableBucket, false); + LogTablet logTablet = replicaManager.getReplicaOrException(tableBucket).getLogTablet(); + addMultiSegmentsToLogTablet(logTablet, 5, false); + remoteLogTaskScheduler.triggerPeriodicScheduledTasks(); + + manualClock.advanceTime(Duration.ofDays(8)); + addMultiSegmentsToLogTablet(logTablet, 2, false); + remoteLogTaskScheduler.triggerPeriodicScheduledTasks(); + RemoteLogManifest beforeGc = + remoteLogManager.remoteLogTablet(tableBucket).currentManifest(); + int unreferencedCount = beforeGc.getUnreferencedRemoteLogSegments().size(); + + manualClock.advanceTime(Duration.ofHours(1)); + remoteLogStorage.deleteSegmentFailFirstN.set(unreferencedCount); + remoteLogTaskScheduler.triggerPeriodicScheduledTasks(); + assertThat( + remoteLogManager + .remoteLogTablet(tableBucket) + .currentManifest() + .getUnreferencedRemoteLogSegments()) + .hasSize(unreferencedCount); + assertThat( + replicaManager + .getReplicaOrException(tableBucket) + .tableMetrics() + .remoteGcFailures() + .getCount()) + .isEqualTo(unreferencedCount); + + remoteLogTaskScheduler.triggerPeriodicScheduledTasks(); + assertThat( + remoteLogManager + .remoteLogTablet(tableBucket) + .currentManifest() + .getUnreferencedRemoteLogSegments()) + .isEmpty(); + } + + @Test + void testOrphanSweeperCleansRejectedCopyAfterGrace() throws Exception { + TableBucket tableBucket = new TableBucket(DATA1_TABLE_ID, 0); + makeLogTableAsLeader(tableBucket, false); + addMultiSegmentsToLogTablet( + replicaManager.getReplicaOrException(tableBucket).getLogTablet(), 3); + testCoordinatorGateway.commitRemoteLogManifestConflictOnce.set(true); + remoteLogStorage.deleteSegmentFailFirstN.set(2); + + remoteLogTaskScheduler.triggerPeriodicScheduledTasks(); + RemoteLogManifest authoritative = + remoteLogManager.remoteLogTablet(tableBucket).currentManifest(); + assertThat(listRemoteLogFiles(tableBucket).size()) + .isGreaterThan(authoritative.getRemoteLogSegmentList().size()); + + manualClock.advanceTime(Duration.ofHours(1).plusSeconds(1)); + remoteLogStorage.deleteSegmentObjectFailFirstN.set(1); + remoteLogTaskScheduler.triggerPeriodicScheduledTasks(); + assertThat(listRemoteLogFiles(tableBucket).size()) + .isGreaterThan(authoritative.getRemoteLogSegmentList().size()); + + remoteLogTaskScheduler.triggerPeriodicScheduledTasks(); + assertThat(listRemoteLogFiles(tableBucket)) + .isEqualTo( + authoritative.getRemoteLogSegmentList().stream() + .map(segment -> segment.remoteLogSegmentId().toString()) + .collect(Collectors.toSet())); + } + + @Test + void testOrphanManifestSweeperPreservesAuthoritativeSnapshot() throws Exception { + TableBucket tableBucket = new TableBucket(DATA1_TABLE_ID, 0); + makeLogTableAsLeader(tableBucket, false); + addMultiSegmentsToLogTablet( + replicaManager.getReplicaOrException(tableBucket).getLogTablet(), 3); + remoteLogTaskScheduler.triggerPeriodicScheduledTasks(); + RemoteLogTablet remoteLogTablet = remoteLogManager.remoteLogTablet(tableBucket); + FsPath authoritativePath = + remoteLogTablet.currentHandle().handle().getRemoteLogManifestPath(); + FsPath orphanPath = + remoteLogStorage.writeRemoteLogManifestSnapshot(remoteLogTablet.currentManifest()); + + manualClock.advanceTime(Duration.ofHours(1).plusSeconds(1)); + remoteLogTaskScheduler.triggerPeriodicScheduledTasks(); + + assertThat( + remoteLogStorage.listRemoteLogManifestSnapshots( + DATA1_PHYSICAL_TABLE_PATH, tableBucket)) + .extracting(object -> object.path().getPath()) + .contains(authoritativePath.getPath()) + .doesNotContain(orphanPath.getPath()); + } +} diff --git a/fluss-server/src/test/java/org/apache/fluss/server/log/remote/RemoteLogTabletTest.java b/fluss-server/src/test/java/org/apache/fluss/server/log/remote/RemoteLogTabletTest.java index 0d571e1ea5..ba2a0594b5 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/log/remote/RemoteLogTabletTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/log/remote/RemoteLogTabletTest.java @@ -17,10 +17,16 @@ package org.apache.fluss.server.log.remote; +import org.apache.fluss.fs.FsPath; +import org.apache.fluss.remote.RemoteLogManifest; import org.apache.fluss.remote.RemoteLogSegment; +import org.apache.fluss.remote.RemoteLogSegmentReference; import org.apache.fluss.server.log.LogTablet; +import org.apache.fluss.server.zk.data.RemoteLogManifestHandle; +import org.apache.fluss.server.zk.data.VersionedRemoteLogManifestHandle; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; @@ -181,6 +187,91 @@ void testFindRemoteLogSegmentByTimestamp(boolean partitionTable) throws Exceptio assertThat(remoteLogTablet.findSegmentByTimestamp(51L)).isNull(); } + @Test + void testReplaceV2ManifestAndLookupLogicalReferences() throws Exception { + LogTablet logTablet = makeLogTabletAndAddSegments(false); + RemoteLogTablet remoteLogTablet = buildRemoteLogTablet(logTablet); + RemoteLogSegment segmentA = createLogSegmentWithMaxTimestamp(logTablet, 10, 0, 10); + RemoteLogSegment segmentB = createLogSegmentWithMaxTimestamp(logTablet, 20, 5, 20); + RemoteLogManifest manifest = + RemoteLogManifest.createV2( + 1L, + logTablet.getPhysicalTablePath(), + logTablet.getTableBucket(), + Arrays.asList(segmentA, segmentB), + 0L, + Collections.emptyList()); + + remoteLogTablet.replaceManifest(manifest); + + assertThat(remoteLogTablet.currentManifest()).isSameAs(manifest); + assertThat(remoteLogTablet.getRemoteLogStartOffset()).isEqualTo(0L); + assertThat(remoteLogTablet.getRemoteLogEndOffset()).hasValue(20L); + assertThat(remoteLogTablet.relevantRemoteLogSegmentReferences(2L)) + .containsExactly( + new RemoteLogSegmentReference(segmentA, 0L, 5L), + new RemoteLogSegmentReference(segmentB, 5L, 20L)); + assertThat(remoteLogTablet.relevantRemoteLogSegmentReferences(7L)) + .containsExactly(new RemoteLogSegmentReference(segmentB, 5L, 20L)); + assertThat(remoteLogTablet.relevantRemoteLogSegmentsForFetchV0(2L)) + .containsExactly(segmentA); + assertThat(remoteLogTablet.relevantRemoteLogSegmentsForFetchV0(7L)) + .containsExactly(segmentB); + assertThat(remoteLogTablet.relevantRemoteLogSegmentReferences(20L)).isEmpty(); + } + + @Test + void testFetchV0ReturnsContiguousV2PrefixBeforeOverlap() throws Exception { + LogTablet logTablet = makeLogTabletAndAddSegments(false); + RemoteLogTablet remoteLogTablet = buildRemoteLogTablet(logTablet); + RemoteLogSegment segmentA = createLogSegmentWithMaxTimestamp(logTablet, 5, 0, 5); + RemoteLogSegment segmentB = createLogSegmentWithMaxTimestamp(logTablet, 10, 5, 10); + RemoteLogSegment segmentC = createLogSegmentWithMaxTimestamp(logTablet, 20, 8, 20); + RemoteLogManifest manifest = + RemoteLogManifest.createV2( + 1L, + logTablet.getPhysicalTablePath(), + logTablet.getTableBucket(), + Arrays.asList(segmentA, segmentB, segmentC), + 0L, + Collections.emptyList()); + + remoteLogTablet.replaceManifest(manifest); + + assertThat(remoteLogTablet.relevantRemoteLogSegmentsForFetchV0(0L)) + .containsExactly(segmentA, segmentB); + assertThat(remoteLogTablet.relevantRemoteLogSegmentsForFetchV0(5L)) + .containsExactly(segmentB); + assertThat(remoteLogTablet.relevantRemoteLogSegmentsForFetchV0(10L)) + .containsExactly(segmentC); + } + + @Test + void testReplaceManifestAndAuthoritativeHandleAtomically() throws Exception { + LogTablet logTablet = makeLogTabletAndAddSegments(false); + RemoteLogTablet remoteLogTablet = buildRemoteLogTablet(logTablet); + RemoteLogSegment segment = createLogSegmentWithMaxTimestamp(logTablet, 10, 0, 10); + RemoteLogManifest manifest = + RemoteLogManifest.createV2( + 2L, + logTablet.getPhysicalTablePath(), + logTablet.getTableBucket(), + Collections.singletonList(segment), + 0L, + Collections.emptyList()); + VersionedRemoteLogManifestHandle handle = + new VersionedRemoteLogManifestHandle( + RemoteLogManifestHandle.v2(new FsPath("file:///remote/m2"), 2L, 0L, 10L), + 7); + + remoteLogTablet.replaceManifest(manifest, handle); + + assertThat(remoteLogTablet.currentManifest()).isSameAs(manifest); + assertThat(remoteLogTablet.currentHandle()).isSameAs(handle); + assertThat(remoteLogTablet.allRemoteLogSegmentReferences()) + .containsExactly(new RemoteLogSegmentReference(segment, 0L, 10L)); + } + RemoteLogSegment createLogSegmentWithMaxTimestamp( LogTablet logTablet, long timestamp, diff --git a/fluss-server/src/test/java/org/apache/fluss/server/log/remote/TestingRemoteLogStorage.java b/fluss-server/src/test/java/org/apache/fluss/server/log/remote/TestingRemoteLogStorage.java index eeba26a54c..f22d2733df 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/log/remote/TestingRemoteLogStorage.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/log/remote/TestingRemoteLogStorage.java @@ -20,6 +20,8 @@ import org.apache.fluss.config.Configuration; import org.apache.fluss.exception.RemoteStorageException; import org.apache.fluss.fs.FsPath; +import org.apache.fluss.metadata.PhysicalTablePath; +import org.apache.fluss.metadata.TableBucket; import org.apache.fluss.remote.RemoteLogManifest; import org.apache.fluss.remote.RemoteLogSegment; @@ -48,6 +50,12 @@ public class TestingRemoteLogStorage extends DefaultRemoteLogStorage { */ public final AtomicInteger copySegmentFailAfterNCopies = new AtomicInteger(-1); + /** Number of upcoming metadata-driven segment deletions that should fail. */ + public final AtomicInteger deleteSegmentFailFirstN = new AtomicInteger(0); + + /** Number of upcoming orphan-directory deletions that should fail. */ + public final AtomicInteger deleteSegmentObjectFailFirstN = new AtomicInteger(0); + private final AtomicInteger copySegmentCount = new AtomicInteger(0); /** @@ -125,6 +133,25 @@ public FsPath writeRemoteLogManifestSnapshot(RemoteLogManifest manifest) return super.writeRemoteLogManifestSnapshot(manifest); } + @Override + public void deleteLogSegmentFiles(RemoteLogSegment remoteLogSegment) + throws RemoteStorageException { + if (deleteSegmentFailFirstN.getAndUpdate(n -> n > 0 ? n - 1 : 0) > 0) { + throw new RemoteStorageException("Simulated segment delete failure"); + } + super.deleteLogSegmentFiles(remoteLogSegment); + } + + @Override + public void deleteRemoteLogSegmentObject( + PhysicalTablePath physicalTablePath, TableBucket tableBucket, UUID segmentId) + throws RemoteStorageException { + if (deleteSegmentObjectFailFirstN.getAndUpdate(n -> n > 0 ? n - 1 : 0) > 0) { + throw new RemoteStorageException("Simulated orphan segment delete failure"); + } + super.deleteRemoteLogSegmentObject(physicalTablePath, tableBucket, segmentId); + } + @Override public InputStream fetchLogData(RemoteLogSegment remoteLogSegment) throws RemoteStorageException { diff --git a/fluss-server/src/test/java/org/apache/fluss/server/tablet/TabletServerTest.java b/fluss-server/src/test/java/org/apache/fluss/server/tablet/TabletServerTest.java index 52257735ef..5698caee1f 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/tablet/TabletServerTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/tablet/TabletServerTest.java @@ -30,6 +30,7 @@ import java.io.File; import java.util.Optional; +import static org.apache.fluss.server.zk.data.TabletServerRegistration.REMOTE_MANIFEST_VERSION_DISPATCH_CAPABILITY; import static org.assertj.core.api.Assertions.assertThat; /** Test for {@link TabletServer}. */ @@ -86,6 +87,10 @@ protected void checkAfterStartServer() throws Exception { TabletServerRegistration tabletServerRegistration = optionalTabletServerRegistration.get(); assertThat(tabletServerRegistration.getRack()).isEqualTo(RACK); + assertThat( + tabletServerRegistration.supportsCapability( + REMOTE_MANIFEST_VERSION_DISPATCH_CAPABILITY)) + .isTrue(); verifyEndpoint( tabletServerRegistration.getEndpoints(), server.getRpcServer().getBindEndpoints()); } diff --git a/fluss-server/src/test/java/org/apache/fluss/server/utils/ServerRpcMessageUtilsTest.java b/fluss-server/src/test/java/org/apache/fluss/server/utils/ServerRpcMessageUtilsTest.java new file mode 100644 index 0000000000..a1cfea1b4b --- /dev/null +++ b/fluss-server/src/test/java/org/apache/fluss/server/utils/ServerRpcMessageUtilsTest.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.server.utils; + +import org.apache.fluss.fs.FsPath; +import org.apache.fluss.metadata.TableBucket; +import org.apache.fluss.rpc.messages.CommitRemoteLogManifestRequest; +import org.apache.fluss.server.entity.CommitRemoteLogManifestData; + +import org.junit.jupiter.api.Test; + +import static org.apache.fluss.server.utils.ServerRpcMessageUtils.getCommitRemoteLogManifestData; +import static org.apache.fluss.server.utils.ServerRpcMessageUtils.makeCommitRemoteLogManifestRequest; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Tests for server RPC message conversions. */ +class ServerRpcMessageUtilsTest { + + @Test + void testRemoteLogManifestCasRequestRoundTrip() { + TableBucket tableBucket = new TableBucket(1L, 0); + CommitRemoteLogManifestData absent = + CommitRemoteLogManifestData.v2Absent( + tableBucket, new FsPath("file:///remote/m1"), 0L, 10L, 1L, 3, 4); + CommitRemoteLogManifestRequest absentRequest = makeCommitRemoteLogManifestRequest(absent); + assertThat(getCommitRemoteLogManifestData(absentRequest)).isEqualTo(absent); + assertThat(absentRequest.hasExpectedManifestPath()).isFalse(); + assertThat(absentRequest.hasExpectedManifestGeneration()).isFalse(); + assertThat(absentRequest.hasExpectedZkVersion()).isFalse(); + + CommitRemoteLogManifestData present = + CommitRemoteLogManifestData.v2Present( + tableBucket, + new FsPath("file:///remote/m2"), + 0L, + 20L, + 2L, + new FsPath("file:///remote/m1"), + 1L, + 7, + 3, + 4); + assertThat(getCommitRemoteLogManifestData(makeCommitRemoteLogManifestRequest(present))) + .isEqualTo(present); + + assertThatThrownBy( + () -> getCommitRemoteLogManifestData(absentRequest.setExpectedZkVersion(1))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("ABSENT"); + } +} diff --git a/fluss-server/src/test/java/org/apache/fluss/server/zk/ZooKeeperClientTest.java b/fluss-server/src/test/java/org/apache/fluss/server/zk/ZooKeeperClientTest.java index ea753c885a..702673ebd7 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/zk/ZooKeeperClientTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/zk/ZooKeeperClientTest.java @@ -38,10 +38,12 @@ import org.apache.fluss.server.zk.data.PartitionAssignment; import org.apache.fluss.server.zk.data.PartitionRegistration; import org.apache.fluss.server.zk.data.RebalanceTask; +import org.apache.fluss.server.zk.data.RemoteLogManifestHandle; import org.apache.fluss.server.zk.data.ServerTags; import org.apache.fluss.server.zk.data.TableAssignment; import org.apache.fluss.server.zk.data.TableRegistration; import org.apache.fluss.server.zk.data.TabletServerRegistration; +import org.apache.fluss.server.zk.data.VersionedRemoteLogManifestHandle; import org.apache.fluss.server.zk.data.lease.KvSnapshotLeaseMetadata; import org.apache.fluss.shaded.curator5.org.apache.curator.CuratorZookeeperClient; import org.apache.fluss.shaded.curator5.org.apache.curator.framework.CuratorFramework; @@ -104,6 +106,54 @@ void beforeEach() throws Exception { zkEpoch = zookeeperClient.fenceBecomeCoordinatorLeader("tmp"); } + @Test + void testRemoteLogManifestHandleCas() throws Exception { + TableBucket tableBucket = new TableBucket(1001L, 0); + FsPath v1Path = new FsPath("file:///remote/v1.manifest"); + FsPath v2Path = new FsPath("file:///remote/v2.manifest"); + FsPath v3Path = new FsPath("file:///remote/v3.manifest"); + RemoteLogManifestHandle v1 = new RemoteLogManifestHandle(v1Path, 10L); + RemoteLogManifestHandle v2 = RemoteLogManifestHandle.v2(v2Path, 1L, 0L, 20L); + RemoteLogManifestHandle v3 = RemoteLogManifestHandle.v2(v3Path, 2L, 0L, 30L); + + assertThat(zookeeperClient.getVersionedRemoteLogManifestHandle(tableBucket)).isEmpty(); + assertThat(zookeeperClient.createRemoteLogManifestHandleIfAbsent(tableBucket, v1)).isTrue(); + assertThat(zookeeperClient.createRemoteLogManifestHandleIfAbsent(tableBucket, v2)) + .isFalse(); + + VersionedRemoteLogManifestHandle initial = + zookeeperClient.getVersionedRemoteLogManifestHandle(tableBucket).get(); + assertThat(initial.handle()).isEqualTo(v1); + assertThat(initial.zkVersion()).isZero(); + + assertThat( + zookeeperClient.compareAndSetRemoteLogManifestHandle( + tableBucket, + new FsPath("file:///remote/not-authoritative.manifest"), + 0L, + initial.zkVersion(), + v2)) + .isFalse(); + assertThat( + zookeeperClient.compareAndSetRemoteLogManifestHandle( + tableBucket, v1Path, 0L, initial.zkVersion() + 1, v2)) + .isFalse(); + assertThat( + zookeeperClient.compareAndSetRemoteLogManifestHandle( + tableBucket, v1Path, 0L, initial.zkVersion(), v2)) + .isTrue(); + + VersionedRemoteLogManifestHandle migrated = + zookeeperClient.getVersionedRemoteLogManifestHandle(tableBucket).get(); + assertThat(migrated.handle()).isEqualTo(v2); + assertThat(migrated.zkVersion()).isEqualTo(1); + assertThat( + zookeeperClient.compareAndSetRemoteLogManifestHandle( + tableBucket, v2Path, 1L, migrated.zkVersion(), v3)) + .isTrue(); + assertThat(zookeeperClient.getRemoteLogManifestHandle(tableBucket)).contains(v3); + } + @AfterEach void afterEach() { ZOO_KEEPER_EXTENSION_WRAPPER.getCustomExtension().cleanupRoot(); diff --git a/fluss-server/src/test/java/org/apache/fluss/server/zk/data/RemoteLogManifestHandleJsonSerdeTest.java b/fluss-server/src/test/java/org/apache/fluss/server/zk/data/RemoteLogManifestHandleJsonSerdeTest.java index b7fb7cf5ea..9ad23d2d3c 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/zk/data/RemoteLogManifestHandleJsonSerdeTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/zk/data/RemoteLogManifestHandleJsonSerdeTest.java @@ -19,6 +19,13 @@ import org.apache.fluss.fs.FsPath; import org.apache.fluss.utils.json.JsonSerdeTestBase; +import org.apache.fluss.utils.json.JsonSerdeUtils; + +import org.junit.jupiter.api.Test; + +import java.nio.charset.StandardCharsets; + +import static org.assertj.core.api.Assertions.assertThatThrownBy; /** Test for {@link org.apache.fluss.server.zk.data.RemoteLogManifestHandleJsonSerde}. */ public class RemoteLogManifestHandleJsonSerdeTest @@ -34,6 +41,8 @@ protected RemoteLogManifestHandle[] createObjects() { new FsPath( "oss://test/log/testDb/testTable_150001/0/847532e6-1fec-4d7a-9b17-ce28223a6e72.manifest"), 100L), + RemoteLogManifestHandle.v2( + new FsPath("oss://test/log/db/table/0/v2.manifest"), 8L, 5L, 20L), }; } @@ -41,7 +50,32 @@ protected RemoteLogManifestHandle[] createObjects() { protected String[] expectedJsons() { return new String[] { "{\"version\":1,\"remote_log_manifest_path\":\"oss://test/log/testDb/testTable_150001/0/" - + "847532e6-1fec-4d7a-9b17-ce28223a6e72.manifest\",\"remote_log_end_offset\":100}" + + "847532e6-1fec-4d7a-9b17-ce28223a6e72.manifest\",\"remote_log_end_offset\":100}", + "{\"version\":2,\"remote_log_manifest_path\":\"oss://test/log/db/table/0/v2.manifest\"," + + "\"remote_log_end_offset\":20,\"manifest_generation\":8," + + "\"remote_log_start_offset\":5}" }; } + + @Test + void testStrictVersionDispatch() { + assertInvalid("{\"remote_log_manifest_path\":\"x\",\"remote_log_end_offset\":10}"); + assertInvalid( + "{\"version\":1,\"remote_log_manifest_path\":\"x\",\"remote_log_end_offset\":10," + + "\"manifest_generation\":1}"); + assertInvalid( + "{\"version\":2,\"remote_log_manifest_path\":\"x\",\"remote_log_end_offset\":10," + + "\"manifest_generation\":1}"); + assertInvalid( + "{\"version\":3,\"remote_log_manifest_path\":\"x\",\"remote_log_end_offset\":10}"); + } + + private static void assertInvalid(String json) { + assertThatThrownBy( + () -> + JsonSerdeUtils.readValue( + json.getBytes(StandardCharsets.UTF_8), + RemoteLogManifestHandleJsonSerde.INSTANCE)) + .isInstanceOf(IllegalArgumentException.class); + } } diff --git a/fluss-server/src/test/java/org/apache/fluss/server/zk/data/TabletServerRegistrationJsonSerdeTest.java b/fluss-server/src/test/java/org/apache/fluss/server/zk/data/TabletServerRegistrationJsonSerdeTest.java index 01359be76c..4d2569dc05 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/zk/data/TabletServerRegistrationJsonSerdeTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/zk/data/TabletServerRegistrationJsonSerdeTest.java @@ -27,6 +27,10 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; +import java.util.Collections; + +import static org.apache.fluss.server.zk.data.TabletServerRegistration.REMOTE_MANIFEST_VERSION_DISPATCH_CAPABILITY; +import static org.assertj.core.api.Assertions.assertThat; /** Test for {@link TabletServerRegistrationJsonSerde}. */ public class TabletServerRegistrationJsonSerdeTest @@ -57,17 +61,29 @@ protected TabletServerRegistration[] createObjects() { "CLIENT://localhost:2345,FLUSS://127.0.0.1:2346"), 10000, new TabletServerResource(8.0, 1024L)); + TabletServerRegistration tabletServerRegistration4 = + new TabletServerRegistration( + "cn-hangzhou-server10", + Endpoint.fromListenersString( + "CLIENT://localhost:2345,FLUSS://127.0.0.1:2346"), + 10000, + new TabletServerResource(8.0, 1024L), + Collections.singleton(REMOTE_MANIFEST_VERSION_DISPATCH_CAPABILITY)); return new TabletServerRegistration[] { - tabletServerRegistration1, tabletServerRegistration2, tabletServerRegistration3 + tabletServerRegistration1, + tabletServerRegistration2, + tabletServerRegistration3, + tabletServerRegistration4 }; } @Override protected String[] expectedJsons() { return new String[] { - "{\"version\":4,\"listeners\":\"CLIENT://localhost:2345,FLUSS://127.0.0.1:2346\",\"register_timestamp\":10000}", - "{\"version\":4,\"listeners\":\"CLIENT://localhost:2345,FLUSS://127.0.0.1:2346\",\"register_timestamp\":10000,\"rack\":\"cn-hangzhou-server10\"}", - "{\"version\":4,\"listeners\":\"CLIENT://localhost:2345,FLUSS://127.0.0.1:2346\",\"register_timestamp\":10000,\"rack\":\"cn-hangzhou-server10\",\"cpu_cores\":8.0,\"memory_bytes\":1024}" + "{\"version\":5,\"listeners\":\"CLIENT://localhost:2345,FLUSS://127.0.0.1:2346\",\"register_timestamp\":10000,\"capabilities\":[]}", + "{\"version\":5,\"listeners\":\"CLIENT://localhost:2345,FLUSS://127.0.0.1:2346\",\"register_timestamp\":10000,\"rack\":\"cn-hangzhou-server10\",\"capabilities\":[]}", + "{\"version\":5,\"listeners\":\"CLIENT://localhost:2345,FLUSS://127.0.0.1:2346\",\"register_timestamp\":10000,\"rack\":\"cn-hangzhou-server10\",\"cpu_cores\":8.0,\"memory_bytes\":1024,\"capabilities\":[]}", + "{\"version\":5,\"listeners\":\"CLIENT://localhost:2345,FLUSS://127.0.0.1:2346\",\"register_timestamp\":10000,\"rack\":\"cn-hangzhou-server10\",\"cpu_cores\":8.0,\"memory_bytes\":1024,\"capabilities\":[\"remote-manifest-version-dispatch\"]}" }; } @@ -85,7 +101,7 @@ void testCompatibility() throws IOException { TabletServerRegistration expectedTabletServerRegistration = new TabletServerRegistration( null, Endpoint.fromListenersString("FLUSS://localhost:1001"), 10000); - assertEquals(tabletServerRegistration, expectedTabletServerRegistration); + assertThat(tabletServerRegistration).isEqualTo(expectedTabletServerRegistration); // compatibility with version 2 JsonNode jsonInVersion2 = @@ -102,7 +118,7 @@ void testCompatibility() throws IOException { Endpoint.fromListenersString( "CLIENT://localhost:2345,FLUSS://127.0.0.1:2346"), 10000); - assertEquals(tabletServerRegistration, expectedTabletServerRegistration); + assertThat(tabletServerRegistration).isEqualTo(expectedTabletServerRegistration); // compatibility with version 3 JsonNode jsonInVersion3 = @@ -119,6 +135,24 @@ void testCompatibility() throws IOException { Endpoint.fromListenersString( "CLIENT://localhost:2345,FLUSS://127.0.0.1:2346"), 10000); - assertEquals(tabletServerRegistration, expectedTabletServerRegistration); + assertThat(tabletServerRegistration).isEqualTo(expectedTabletServerRegistration); + + // compatibility with version 4: a legacy registration has no capabilities. + JsonNode jsonInVersion4 = + new ObjectMapper() + .readTree( + ("{\"version\":4,\"listeners\":\"CLIENT://localhost:2345,FLUSS://127.0.0.1:2346\"," + + "\"register_timestamp\":10000,\"rack\":\"cn-hangzhou-server10\"," + + "\"cpu_cores\":8.0,\"memory_bytes\":1024}") + .getBytes(StandardCharsets.UTF_8)); + tabletServerRegistration = + TabletServerRegistrationJsonSerde.INSTANCE.deserialize(jsonInVersion4); + assertThat(tabletServerRegistration.getResource()) + .isEqualTo(new TabletServerResource(8.0, 1024L)); + assertThat(tabletServerRegistration.getCapabilities()).isEmpty(); + assertThat( + tabletServerRegistration.supportsCapability( + REMOTE_MANIFEST_VERSION_DISPATCH_CAPABILITY)) + .isFalse(); } } From 41121801d577a6244bf40c37f28d1ac9460efcbc Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Sun, 2 Aug 2026 20:32:25 +0800 Subject: [PATCH 2/3] feat: remote manifest v2 --- .../table/scanner/log/CompletedFetch.java | 66 +++++++++-- .../client/table/scanner/log/LogFetcher.java | 53 ++++++++- .../scanner/log/RemoteCompletedFetch.java | 27 ++++- .../table/scanner/log/RemotePendingFetch.java | 27 +++++ .../scanner/log/RemoteCompletedFetchTest.java | 111 +++++++++++++++++- .../scanner/log/RemoteLogDownloaderTest.java | 2 +- .../fluss/remote/RemoteLogFetchInfoV2.java | 61 ++++++++++ .../rpc/entity/FetchLogResultForBucket.java | 31 ++++- .../apache/fluss/rpc/protocol/ApiKeys.java | 4 +- .../fluss/rpc/util/CommonRpcMessageUtils.java | 67 ++++++++--- fluss-rpc/src/main/proto/FlussApi.proto | 14 +++ .../apache/fluss/server/log/FetchParams.java | 33 +++++- .../fluss/server/log/FetchParamsBuilder.java | 9 +- .../fluss/server/replica/ReplicaManager.java | 72 ++++++++---- .../replica/fetcher/ReplicaFetcherThread.java | 30 +++-- .../fluss/server/tablet/TabletService.java | 6 +- .../server/utils/ServerRpcMessageUtils.java | 52 ++++++-- .../log/remote/RemoteLogManagerTest.java | 42 +++++++ .../utils/ServerRpcMessageUtilsTest.java | 60 ++++++++++ 19 files changed, 679 insertions(+), 88 deletions(-) create mode 100644 fluss-common/src/main/java/org/apache/fluss/remote/RemoteLogFetchInfoV2.java diff --git a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/CompletedFetch.java b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/CompletedFetch.java index de9cb1d3f6..ae677d3529 100644 --- a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/CompletedFetch.java +++ b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/CompletedFetch.java @@ -63,6 +63,7 @@ public abstract class CompletedFetch { final long highWatermark; private final long fetchOffset; private final long filteredEndOffset; + private final long logicalEndOffset; private final boolean isCheckCrcs; private final Iterator batches; @@ -92,6 +93,34 @@ public CompletedFetch( boolean isCheckCrcs, long fetchOffset, long filteredEndOffset) { + this( + tableBucket, + tablePath, + error, + sizeInBytes, + highWatermark, + batches, + readContext, + logScannerStatus, + isCheckCrcs, + fetchOffset, + filteredEndOffset, + Long.MAX_VALUE); + } + + public CompletedFetch( + TableBucket tableBucket, + TablePath tablePath, + ApiError error, + int sizeInBytes, + long highWatermark, + Iterator batches, + LogRecordReadContext readContext, + LogScannerStatus logScannerStatus, + boolean isCheckCrcs, + long fetchOffset, + long filteredEndOffset, + long logicalEndOffset) { this.tableBucket = tableBucket; this.tablePath = tablePath; this.error = error; @@ -110,6 +139,13 @@ public CompletedFetch( fetchOffset, tableBucket); this.filteredEndOffset = filteredEndOffset; + checkArgument( + logicalEndOffset > fetchOffset, + "logicalEndOffset (%s) must be greater than fetchOffset (%s) for bucket %s.", + logicalEndOffset, + fetchOffset, + tableBucket); + this.logicalEndOffset = logicalEndOffset; this.nextFetchOffset = fetchOffset; } @@ -370,15 +406,29 @@ private LogRecord nextFetchedRecord() throws Exception { private LogRecordBatch nextFetchedBatch() { maybeCloseRecordStream(); - if (!batches.hasNext()) { - finishFetchedBatches(); - return null; + while (batches.hasNext()) { + LogRecordBatch nextBatch = batches.next(); + if (nextBatch.baseLogOffset() >= logicalEndOffset) { + finishFetchedBatches(); + return null; + } + if (nextBatch.nextLogOffset() > logicalEndOffset) { + throw new FetchException( + "Remote logical end offset " + + logicalEndOffset + + " is not a batch boundary for bucket " + + tableBucket); + } + if (nextBatch.nextLogOffset() <= nextFetchOffset) { + currentBatch = nextBatch; + continue; + } + currentBatch = nextBatch; + maybeEnsureValid(currentBatch); + return currentBatch; } - - currentBatch = batches.next(); - // TODO get last epoch. - maybeEnsureValid(currentBatch); - return currentBatch; + finishFetchedBatches(); + return null; } private void finishFetchedBatches() { diff --git a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/LogFetcher.java b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/LogFetcher.java index a178ab2061..c9bd5f9bb1 100644 --- a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/LogFetcher.java +++ b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/LogFetcher.java @@ -41,7 +41,9 @@ import org.apache.fluss.record.LogRecords; import org.apache.fluss.record.MemoryLogRecords; import org.apache.fluss.remote.RemoteLogFetchInfo; +import org.apache.fluss.remote.RemoteLogFetchInfoV2; import org.apache.fluss.remote.RemoteLogSegment; +import org.apache.fluss.remote.RemoteLogSegmentReference; import org.apache.fluss.rpc.entity.FetchLogResultForBucket; import org.apache.fluss.rpc.gateway.TabletServerGateway; import org.apache.fluss.rpc.messages.FetchLogRequest; @@ -459,11 +461,19 @@ private synchronized void handleFetchLogResponse( tb); } else { if (fetchResultForBucket.fetchFromRemote()) { - pendRemoteFetches( - trc, - fetchResultForBucket.remoteLogFetchInfo(), - fetchOffset, - fetchResultForBucket.getHighWatermark()); + if (fetchResultForBucket.remoteLogFetchInfoV2() != null) { + pendRemoteFetchesV2( + trc, + fetchResultForBucket.remoteLogFetchInfoV2(), + fetchOffset, + fetchResultForBucket.getHighWatermark()); + } else { + pendRemoteFetches( + trc, + fetchResultForBucket.remoteLogFetchInfo(), + fetchOffset, + fetchResultForBucket.getHighWatermark()); + } } else { LogRecords logRecords = fetchResultForBucket.recordsOrEmpty(); boolean hasRecords = !MemoryLogRecords.EMPTY.equals(logRecords); @@ -561,6 +571,39 @@ private void pendRemoteFetches( } } + private void pendRemoteFetchesV2( + TableReadContext tableReadContext, + RemoteLogFetchInfoV2 remoteLogFetchInfo, + long firstFetchOffset, + long highWatermark) { + FsPath remoteLogTabletDir = new FsPath(remoteLogFetchInfo.remoteLogTabletDir()); + List references = remoteLogFetchInfo.activeReferences(); + for (int i = 0; i < references.size(); i++) { + RemoteLogSegmentReference reference = references.get(i); + RemoteLogSegment segment = reference.remoteLogSegment(); + long fetchOffset = + i == 0 + ? Math.max(firstFetchOffset, reference.logicalStartOffset()) + : reference.logicalStartOffset(); + RemoteLogDownloadFuture downloadFuture = + remoteLogDownloader.requestRemoteLog(remoteLogTabletDir, segment); + RemotePendingFetch pendingFetch = + new RemotePendingFetch( + segment, + downloadFuture, + tableReadContext.tablePath, + 0, + fetchOffset, + reference.logicalEndOffset(), + highWatermark, + tableReadContext.remoteReadContext, + logScannerStatus, + isCheckCrcs); + logFetchBuffer.pend(pendingFetch); + downloadFuture.onComplete(() -> logFetchBuffer.tryComplete(segment.tableBucket())); + } + } + @VisibleForTesting Map prepareFetchLogRequests(List fetchableBuckets) { // Outer key: leaderId, inner key: tableId -> list of bucket requests diff --git a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/RemoteCompletedFetch.java b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/RemoteCompletedFetch.java index 9d7d1cbc03..977dbf381c 100644 --- a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/RemoteCompletedFetch.java +++ b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/RemoteCompletedFetch.java @@ -48,6 +48,30 @@ class RemoteCompletedFetch extends CompletedFetch { boolean isCheckCrc, long fetchOffset, Runnable recycleCallback) { + this( + tableBucket, + tablePath, + fileLogRecords, + highWatermark, + readContext, + logScannerStatus, + isCheckCrc, + fetchOffset, + Long.MAX_VALUE, + recycleCallback); + } + + RemoteCompletedFetch( + TableBucket tableBucket, + TablePath tablePath, + FileLogRecords fileLogRecords, + long highWatermark, + LogRecordReadContext readContext, + LogScannerStatus logScannerStatus, + boolean isCheckCrc, + long fetchOffset, + long logicalEndOffset, + Runnable recycleCallback) { super( tableBucket, tablePath, @@ -59,7 +83,8 @@ class RemoteCompletedFetch extends CompletedFetch { logScannerStatus, isCheckCrc, fetchOffset, - CompletedFetch.NO_FILTERED_END_OFFSET); + CompletedFetch.NO_FILTERED_END_OFFSET, + logicalEndOffset); this.fileLogRecords = fileLogRecords; this.recycleCallback = recycleCallback; } diff --git a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/RemotePendingFetch.java b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/RemotePendingFetch.java index 5bb6d35681..c2d14302d1 100644 --- a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/RemotePendingFetch.java +++ b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/RemotePendingFetch.java @@ -35,6 +35,7 @@ class RemotePendingFetch implements PendingFetch { private final int posInLogSegment; private final long fetchOffset; + private final long logicalEndOffset; private final long highWatermark; private final LogRecordReadContext readContext; private final LogScannerStatus logScannerStatus; @@ -50,11 +51,36 @@ class RemotePendingFetch implements PendingFetch { LogRecordReadContext readContext, LogScannerStatus logScannerStatus, boolean isCheckCrc) { + this( + remoteLogSegment, + downloadFuture, + tablePath, + posInLogSegment, + fetchOffset, + remoteLogSegment.remoteLogEndOffset(), + highWatermark, + readContext, + logScannerStatus, + isCheckCrc); + } + + RemotePendingFetch( + RemoteLogSegment remoteLogSegment, + RemoteLogDownloadFuture downloadFuture, + TablePath tablePath, + int posInLogSegment, + long fetchOffset, + long logicalEndOffset, + long highWatermark, + LogRecordReadContext readContext, + LogScannerStatus logScannerStatus, + boolean isCheckCrc) { this.remoteLogSegment = remoteLogSegment; this.downloadFuture = downloadFuture; this.tablePath = tablePath; this.posInLogSegment = posInLogSegment; this.fetchOffset = fetchOffset; + this.logicalEndOffset = logicalEndOffset; this.highWatermark = highWatermark; this.readContext = readContext; this.logScannerStatus = logScannerStatus; @@ -83,6 +109,7 @@ public CompletedFetch toCompletedFetch() { logScannerStatus, isCheckCrc, fetchOffset, + logicalEndOffset, downloadFuture.getRecycleCallback()); } diff --git a/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/RemoteCompletedFetchTest.java b/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/RemoteCompletedFetchTest.java index c01ea7e2fc..59fd2405e4 100644 --- a/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/RemoteCompletedFetchTest.java +++ b/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/RemoteCompletedFetchTest.java @@ -18,6 +18,7 @@ package org.apache.fluss.client.table.scanner.log; import org.apache.fluss.client.table.scanner.ScanRecord; +import org.apache.fluss.exception.FetchException; import org.apache.fluss.fs.FsPath; import org.apache.fluss.metadata.LogFormat; import org.apache.fluss.metadata.PhysicalTablePath; @@ -55,6 +56,7 @@ import java.util.UUID; import java.util.concurrent.atomic.AtomicBoolean; +import static org.apache.fluss.record.LogRecordBatch.CURRENT_LOG_MAGIC_VALUE; import static org.apache.fluss.record.TestData.DATA2; import static org.apache.fluss.record.TestData.DATA2_PHYSICAL_TABLE_PATH; import static org.apache.fluss.record.TestData.DATA2_ROW_TYPE; @@ -64,9 +66,11 @@ import static org.apache.fluss.record.TestData.DATA2_TABLE_PATH; import static org.apache.fluss.record.TestData.DEFAULT_REMOTE_DATA_DIR; import static org.apache.fluss.record.TestData.DEFAULT_SCHEMA_ID; +import static org.apache.fluss.testutils.DataTestUtils.createRecordsWithoutBaseLogOffset; import static org.apache.fluss.testutils.DataTestUtils.genLogFile; import static org.apache.fluss.utils.FlussPaths.remoteLogSegmentDir; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; /** Test for {@link org.apache.fluss.client.table.scanner.log.RemoteCompletedFetch}. */ class RemoteCompletedFetchTest { @@ -197,6 +201,86 @@ void testNoRecordsInFetch() throws Exception { assertThat(scanRecords.size()).isEqualTo(0); } + @Test + void testClipPhysicalPrefixAtLogicalStart() throws Exception { + TableBucket tableBucket = new TableBucket(DATA2_TABLE_ID, 0); + FileLogRecords fileLogRecords = + createFileLogRecords( + tableBucket, DATA2_PHYSICAL_TABLE_PATH, DATA2, LogFormat.ARROW); + RemoteCompletedFetch completedFetch = + new RemoteCompletedFetch( + tableBucket, + DATA2_TABLE_PATH, + fileLogRecords, + 10L, + remoteReadContext, + logScannerStatus, + true, + 5L, + 10L, + () -> {}); + + List scanRecords = completedFetch.fetchRecords(10); + + assertThat(scanRecords).hasSize(5); + assertThat(scanRecords) + .extracting(ScanRecord::logOffset) + .containsExactly(5L, 6L, 7L, 8L, 9L); + } + + @Test + void testClipPhysicalSuffixAtLogicalEnd() throws Exception { + TableBucket tableBucket = new TableBucket(DATA2_TABLE_ID, 0); + FileLogRecords fileLogRecords = + createTwoBatchFileLogRecords( + tableBucket, DATA2_PHYSICAL_TABLE_PATH, DATA2, LogFormat.ARROW); + RemoteCompletedFetch completedFetch = + new RemoteCompletedFetch( + tableBucket, + DATA2_TABLE_PATH, + fileLogRecords, + 10L, + remoteReadContext, + logScannerStatus, + true, + 0L, + 5L, + () -> {}); + + List scanRecords = completedFetch.fetchRecords(10); + + assertThat(scanRecords).hasSize(5); + assertThat(scanRecords) + .extracting(ScanRecord::logOffset) + .containsExactly(0L, 1L, 2L, 3L, 4L); + } + + @Test + void testRejectLogicalEndInsideBatch() throws Exception { + TableBucket tableBucket = new TableBucket(DATA2_TABLE_ID, 0); + FileLogRecords fileLogRecords = + createTwoBatchFileLogRecords( + tableBucket, DATA2_PHYSICAL_TABLE_PATH, DATA2, LogFormat.ARROW); + RemoteCompletedFetch completedFetch = + new RemoteCompletedFetch( + tableBucket, + DATA2_TABLE_PATH, + fileLogRecords, + 10L, + remoteReadContext, + logScannerStatus, + true, + 0L, + 6L, + () -> {}); + + assertThat(completedFetch.fetchRecords(10)).hasSize(5); + assertThatThrownBy(() -> completedFetch.fetchRecords(10)) + .isInstanceOf(FetchException.class) + .rootCause() + .hasMessageContaining("not a batch boundary"); + } + @ParameterizedTest @ValueSource(strings = {"INDEXED", "ARROW"}) void testProjection(String format) throws Exception { @@ -282,7 +366,7 @@ private FileLogRecords createFileLogRecords( .physicalTablePath(physicalTablePath) .remoteLogSegmentId(segmentId) .remoteLogStartOffset(0L) - .remoteLogEndOffset(9L) + .remoteLogEndOffset(10L) .segmentSizeInBytes(Integer.MAX_VALUE) .build(); File logFile = @@ -291,6 +375,31 @@ private FileLogRecords createFileLogRecords( return FileLogRecords.open(logFile, false); } + private FileLogRecords createTwoBatchFileLogRecords( + TableBucket tableBucket, + PhysicalTablePath physicalTablePath, + List objects, + LogFormat logFormat) + throws Exception { + FileLogRecords firstBatch = + createFileLogRecords( + tableBucket, physicalTablePath, objects.subList(0, 5), logFormat); + File logFile = firstBatch.file(); + firstBatch.close(); + FileLogRecords twoBatches = FileLogRecords.open(logFile, false, 1024 * 1024, false); + twoBatches.append( + createRecordsWithoutBaseLogOffset( + DATA2_ROW_TYPE, + DEFAULT_SCHEMA_ID, + 5L, + System.currentTimeMillis(), + CURRENT_LOG_MAGIC_VALUE, + objects.subList(5, 10), + logFormat)); + twoBatches.flush(); + return twoBatches; + } + private RemoteCompletedFetch makeCompletedFetch( TableBucket tableBucket, FileLogRecords fileLogRecords, diff --git a/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/RemoteLogDownloaderTest.java b/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/RemoteLogDownloaderTest.java index 77f6dc998e..db27b1d5ad 100644 --- a/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/RemoteLogDownloaderTest.java +++ b/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/RemoteLogDownloaderTest.java @@ -549,7 +549,7 @@ private static List buildRemoteLogSegmentList( .physicalTablePath(physicalTablePath) .remoteLogSegmentId(segmentId) .remoteLogStartOffset(baseOffset) - .remoteLogEndOffset(baseOffset + 9) + .remoteLogEndOffset(baseOffset + 10) .maxTimestamp(maxTimestamp) .segmentSizeInBytes(Integer.MAX_VALUE) .build(); diff --git a/fluss-common/src/main/java/org/apache/fluss/remote/RemoteLogFetchInfoV2.java b/fluss-common/src/main/java/org/apache/fluss/remote/RemoteLogFetchInfoV2.java new file mode 100644 index 0000000000..f4a9c180a0 --- /dev/null +++ b/fluss-common/src/main/java/org/apache/fluss/remote/RemoteLogFetchInfoV2.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.remote; + +import org.apache.fluss.annotation.Internal; + +import javax.annotation.Nullable; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import static org.apache.fluss.utils.Preconditions.checkArgument; +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** Remote log fetch information containing authoritative logical segment references. */ +@Internal +public final class RemoteLogFetchInfoV2 { + private final String remoteLogTabletDir; + private final @Nullable String partitionName; + private final List activeReferences; + + public RemoteLogFetchInfoV2( + String remoteLogTabletDir, + @Nullable String partitionName, + List activeReferences) { + this.remoteLogTabletDir = checkNotNull(remoteLogTabletDir); + this.partitionName = partitionName; + checkArgument( + !activeReferences.isEmpty(), "Active remote log references must not be empty"); + this.activeReferences = Collections.unmodifiableList(new ArrayList<>(activeReferences)); + } + + public String remoteLogTabletDir() { + return remoteLogTabletDir; + } + + @Nullable + public String partitionName() { + return partitionName; + } + + public List activeReferences() { + return activeReferences; + } +} diff --git a/fluss-rpc/src/main/java/org/apache/fluss/rpc/entity/FetchLogResultForBucket.java b/fluss-rpc/src/main/java/org/apache/fluss/rpc/entity/FetchLogResultForBucket.java index e239d8e052..75293d2567 100644 --- a/fluss-rpc/src/main/java/org/apache/fluss/rpc/entity/FetchLogResultForBucket.java +++ b/fluss-rpc/src/main/java/org/apache/fluss/rpc/entity/FetchLogResultForBucket.java @@ -22,6 +22,7 @@ import org.apache.fluss.record.LogRecords; import org.apache.fluss.record.MemoryLogRecords; import org.apache.fluss.remote.RemoteLogFetchInfo; +import org.apache.fluss.remote.RemoteLogFetchInfoV2; import org.apache.fluss.rpc.messages.FetchLogRequest; import org.apache.fluss.rpc.protocol.ApiError; @@ -33,6 +34,7 @@ @Internal public class FetchLogResultForBucket extends ResultForBucket { private final @Nullable RemoteLogFetchInfo remoteLogFetchInfo; + private final @Nullable RemoteLogFetchInfoV2 remoteLogFetchInfoV2; private final @Nullable LogRecords records; private final long highWatermark; private final long filteredEndOffset; @@ -42,6 +44,7 @@ public FetchLogResultForBucket( this( tableBucket, null, + null, checkNotNull(records, "records can not be null"), highWatermark, -1L, @@ -56,6 +59,7 @@ public FetchLogResultForBucket( this( tableBucket, null, + null, checkNotNull(records, "records can not be null"), highWatermark, filteredEndOffset, @@ -63,7 +67,7 @@ public FetchLogResultForBucket( } public FetchLogResultForBucket(TableBucket tableBucket, ApiError error) { - this(tableBucket, null, null, -1L, -1L, error); + this(tableBucket, null, null, null, -1L, -1L, error); } public FetchLogResultForBucket( @@ -72,6 +76,21 @@ public FetchLogResultForBucket( tableBucket, checkNotNull(remoteLogFetchInfo, "remote log fetch info can not be null"), null, + null, + highWatermark, + -1L, + ApiError.NONE); + } + + public FetchLogResultForBucket( + TableBucket tableBucket, + RemoteLogFetchInfoV2 remoteLogFetchInfoV2, + long highWatermark) { + this( + tableBucket, + null, + checkNotNull(remoteLogFetchInfoV2, "remote log fetch info V2 can not be null"), + null, highWatermark, -1L, ApiError.NONE); @@ -84,18 +103,20 @@ public FetchLogResultForBucket( */ public FetchLogResultForBucket( TableBucket tableBucket, long highWatermark, long filteredEndOffset) { - this(tableBucket, null, null, highWatermark, filteredEndOffset, ApiError.NONE); + this(tableBucket, null, null, null, highWatermark, filteredEndOffset, ApiError.NONE); } private FetchLogResultForBucket( TableBucket tableBucket, @Nullable RemoteLogFetchInfo remoteLogFetchInfo, + @Nullable RemoteLogFetchInfoV2 remoteLogFetchInfoV2, @Nullable LogRecords records, long highWatermark, long filteredEndOffset, ApiError error) { super(tableBucket, error); this.remoteLogFetchInfo = remoteLogFetchInfo; + this.remoteLogFetchInfoV2 = remoteLogFetchInfoV2; this.records = records; this.highWatermark = highWatermark; this.filteredEndOffset = filteredEndOffset; @@ -109,7 +130,7 @@ private FetchLogResultForBucket( * @return {@code true} if the log is fetched from remote. */ public boolean fetchFromRemote() { - return remoteLogFetchInfo != null; + return remoteLogFetchInfo != null || remoteLogFetchInfoV2 != null; } public @Nullable LogRecords records() { @@ -128,6 +149,10 @@ public LogRecords recordsOrEmpty() { return remoteLogFetchInfo; } + public @Nullable RemoteLogFetchInfoV2 remoteLogFetchInfoV2() { + return remoteLogFetchInfoV2; + } + public long getHighWatermark() { return highWatermark; } diff --git a/fluss-rpc/src/main/java/org/apache/fluss/rpc/protocol/ApiKeys.java b/fluss-rpc/src/main/java/org/apache/fluss/rpc/protocol/ApiKeys.java index e9f18b3d67..4573866532 100644 --- a/fluss-rpc/src/main/java/org/apache/fluss/rpc/protocol/ApiKeys.java +++ b/fluss-rpc/src/main/java/org/apache/fluss/rpc/protocol/ApiKeys.java @@ -43,7 +43,9 @@ public enum ApiKeys { GET_METADATA(1012, 0, 0, PUBLIC), UPDATE_METADATA(1013, 0, 0, PRIVATE), PRODUCE_LOG(1014, 0, 0, PUBLIC), - FETCH_LOG(1015, 0, 0, PUBLIC), + // Version 0: Returns physical-only remote log segments. + // Version 1: Returns remote log segment references with authoritative logical bounds. + FETCH_LOG(1015, 0, 1, PUBLIC), // Version 0: Uses lake's encoder for primary key encoding (legacy behavior). // Version 1: Uses CompactedKeyEncoder for primary key encoding when bucket key differs from diff --git a/fluss-rpc/src/main/java/org/apache/fluss/rpc/util/CommonRpcMessageUtils.java b/fluss-rpc/src/main/java/org/apache/fluss/rpc/util/CommonRpcMessageUtils.java index 2cf783a118..d4a5db5426 100644 --- a/fluss-rpc/src/main/java/org/apache/fluss/rpc/util/CommonRpcMessageUtils.java +++ b/fluss-rpc/src/main/java/org/apache/fluss/rpc/util/CommonRpcMessageUtils.java @@ -24,7 +24,9 @@ import org.apache.fluss.record.LogRecords; import org.apache.fluss.record.MemoryLogRecords; import org.apache.fluss.remote.RemoteLogFetchInfo; +import org.apache.fluss.remote.RemoteLogFetchInfoV2; import org.apache.fluss.remote.RemoteLogSegment; +import org.apache.fluss.remote.RemoteLogSegmentReference; import org.apache.fluss.rpc.entity.FetchLogResultForBucket; import org.apache.fluss.rpc.messages.PbAclFilter; import org.apache.fluss.rpc.messages.PbAclInfo; @@ -32,7 +34,9 @@ import org.apache.fluss.rpc.messages.PbKeyValue; import org.apache.fluss.rpc.messages.PbPartitionSpec; import org.apache.fluss.rpc.messages.PbRemoteLogFetchInfo; +import org.apache.fluss.rpc.messages.PbRemoteLogFetchInfoV2; import org.apache.fluss.rpc.messages.PbRemoteLogSegment; +import org.apache.fluss.rpc.messages.PbRemoteLogSegmentReference; import org.apache.fluss.rpc.protocol.ApiError; import org.apache.fluss.security.acl.AccessControlEntry; import org.apache.fluss.security.acl.AccessControlEntryFilter; @@ -161,31 +165,40 @@ public static FetchLogResultForBucket getFetchLogResultForBucket( fetchLogResultForBucket = new FetchLogResultForBucket(tb, ApiError.fromErrorMessage(respForBucket)); } else { - if (respForBucket.hasRemoteLogFetchInfo()) { + if (respForBucket.hasRemoteLogFetchInfo() && respForBucket.hasRemoteLogFetchInfoV2()) { + throw new IllegalArgumentException( + "FetchLog response contains both v0 and v1 remote fetch info"); + } + if (respForBucket.hasRemoteLogFetchInfoV2()) { + PbRemoteLogFetchInfoV2 pbRlfInfo = respForBucket.getRemoteLogFetchInfoV2(); + String partitionName = + pbRlfInfo.hasPartitionName() ? pbRlfInfo.getPartitionName() : null; + PhysicalTablePath physicalTablePath = PhysicalTablePath.of(tp, partitionName); + List references = new ArrayList<>(); + for (PbRemoteLogSegmentReference pbReference : + pbRlfInfo.getActiveReferencesList()) { + references.add( + new RemoteLogSegmentReference( + fromPbRemoteLogSegment( + pbReference.getSegment(), physicalTablePath, tb), + pbReference.getLogicalStartOffset(), + pbReference.getLogicalEndOffset())); + } + RemoteLogFetchInfoV2 rlFetchInfo = + new RemoteLogFetchInfoV2( + pbRlfInfo.getRemoteLogTabletDir(), partitionName, references); + fetchLogResultForBucket = + new FetchLogResultForBucket( + tb, rlFetchInfo, respForBucket.getHighWatermark()); + } else if (respForBucket.hasRemoteLogFetchInfo()) { PbRemoteLogFetchInfo pbRlfInfo = respForBucket.getRemoteLogFetchInfo(); String partitionName = pbRlfInfo.hasPartitionName() ? pbRlfInfo.getPartitionName() : null; PhysicalTablePath physicalTablePath = PhysicalTablePath.of(tp, partitionName); List remoteLogSegmentList = new ArrayList<>(); for (PbRemoteLogSegment pbRemoteLogSegment : pbRlfInfo.getRemoteLogSegmentsList()) { - long maxTimestamp = - pbRemoteLogSegment.hasMaxTimestamp() - ? pbRemoteLogSegment.getMaxTimestamp() - : -1; - RemoteLogSegment remoteLogSegment = - RemoteLogSegment.Builder.builder() - .tableBucket(tb) - .physicalTablePath(physicalTablePath) - .remoteLogSegmentId( - UUID.fromString( - pbRemoteLogSegment.getRemoteLogSegmentId())) - .remoteLogEndOffset(pbRemoteLogSegment.getRemoteLogEndOffset()) - .remoteLogStartOffset( - pbRemoteLogSegment.getRemoteLogStartOffset()) - .segmentSizeInBytes(pbRemoteLogSegment.getSegmentSizeInBytes()) - .maxTimestamp(maxTimestamp) - .build(); - remoteLogSegmentList.add(remoteLogSegment); + remoteLogSegmentList.add( + fromPbRemoteLogSegment(pbRemoteLogSegment, physicalTablePath, tb)); } RemoteLogFetchInfo rlFetchInfo = new RemoteLogFetchInfo( @@ -221,6 +234,22 @@ public static FetchLogResultForBucket getFetchLogResultForBucket( return fetchLogResultForBucket; } + private static RemoteLogSegment fromPbRemoteLogSegment( + PbRemoteLogSegment segment, + PhysicalTablePath physicalTablePath, + TableBucket tableBucket) { + long maxTimestamp = segment.hasMaxTimestamp() ? segment.getMaxTimestamp() : -1; + return RemoteLogSegment.Builder.builder() + .tableBucket(tableBucket) + .physicalTablePath(physicalTablePath) + .remoteLogSegmentId(UUID.fromString(segment.getRemoteLogSegmentId())) + .remoteLogEndOffset(segment.getRemoteLogEndOffset()) + .remoteLogStartOffset(segment.getRemoteLogStartOffset()) + .segmentSizeInBytes(segment.getSegmentSizeInBytes()) + .maxTimestamp(maxTimestamp) + .build(); + } + public static ByteBuffer toByteBuffer(ByteBuf buf) { if (buf.isDirect()) { return buf.nioBuffer(); diff --git a/fluss-rpc/src/main/proto/FlussApi.proto b/fluss-rpc/src/main/proto/FlussApi.proto index 371d8e58ef..afb25a7ced 100644 --- a/fluss-rpc/src/main/proto/FlussApi.proto +++ b/fluss-rpc/src/main/proto/FlussApi.proto @@ -936,6 +936,8 @@ message PbFetchLogRespForBucket { // filter. When set (>= 0), the records field MUST be empty — the two fields are mutually // exclusive. The client should use this offset as the starting offset for the next fetch. optional int64 filtered_end_offset = 9; + // FetchLog v1 only. Mutually exclusive with remote_log_fetch_info. + optional PbRemoteLogFetchInfoV2 remote_log_fetch_info_v2 = 10; } message PbPutKvReqForBucket { @@ -1120,6 +1122,18 @@ message PbRemoteLogSegment { optional int64 max_timestamp = 5; } +message PbRemoteLogSegmentReference { + required PbRemoteLogSegment segment = 1; + required int64 logical_start_offset = 2; + required int64 logical_end_offset = 3; +} + +message PbRemoteLogFetchInfoV2 { + required string remote_log_tablet_dir = 1; + optional string partition_name = 2; + repeated PbRemoteLogSegmentReference active_references = 3; +} + message PbPartitionInfo { required int64 partition_id = 1; required PbPartitionSpec partition_spec = 2; diff --git a/fluss-server/src/main/java/org/apache/fluss/server/log/FetchParams.java b/fluss-server/src/main/java/org/apache/fluss/server/log/FetchParams.java index 76a30a1ddc..c4eb074869 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/log/FetchParams.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/log/FetchParams.java @@ -71,6 +71,7 @@ public final class FetchParams { private final int minFetchBytes; private final long maxWaitMs; private final FetchLogReadPreference readPreference; + private final short apiVersion; // TODO: add more params like epoch etc. public FetchParams(int replicaId, int maxFetchBytes) { @@ -104,6 +105,27 @@ public FetchParams( long maxWaitMs, @Nullable Map tableFilterInfoMap, FetchLogReadPreference readPreference) { + this( + replicaId, + fetchOnlyLeader, + maxFetchBytes, + minFetchBytes, + maxWaitMs, + tableFilterInfoMap, + readPreference, + (short) 0); + } + + @VisibleForTesting + public FetchParams( + int replicaId, + boolean fetchOnlyLeader, + int maxFetchBytes, + int minFetchBytes, + long maxWaitMs, + @Nullable Map tableFilterInfoMap, + FetchLogReadPreference readPreference, + short apiVersion) { this.replicaId = replicaId; this.fetchOnlyLeader = fetchOnlyLeader; this.maxFetchBytes = maxFetchBytes; @@ -114,6 +136,7 @@ public FetchParams( this.maxWaitMs = maxWaitMs; this.tableFilterInfoMap = tableFilterInfoMap; this.readPreference = readPreference; + this.apiVersion = apiVersion; } @VisibleForTesting @@ -231,6 +254,10 @@ public boolean fetchOnlyLeader() { return isFromFollower() || fetchOnlyLeader; } + public short apiVersion() { + return apiVersion; + } + public long fetchOffset() { return fetchOffset; } @@ -248,6 +275,7 @@ public boolean equals(Object o) { && maxFetchBytes == that.maxFetchBytes && minFetchBytes == that.minFetchBytes && maxWaitMs == that.maxWaitMs + && apiVersion == that.apiVersion && readPreference == that.readPreference && Objects.equals(tableFilterInfoMap, that.tableFilterInfoMap); } @@ -260,7 +288,8 @@ public int hashCode() { minFetchBytes, maxWaitMs, tableFilterInfoMap, - readPreference); + readPreference, + apiVersion); } @Override @@ -278,6 +307,8 @@ public String toString() { + tableFilterInfoMap + ", readPreference=" + readPreference + + ", apiVersion=" + + apiVersion + ')'; } } diff --git a/fluss-server/src/main/java/org/apache/fluss/server/log/FetchParamsBuilder.java b/fluss-server/src/main/java/org/apache/fluss/server/log/FetchParamsBuilder.java index b8de03cf81..b44ab0a491 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/log/FetchParamsBuilder.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/log/FetchParamsBuilder.java @@ -36,6 +36,7 @@ public final class FetchParamsBuilder { private int minFetchBytes; private long maxWaitMs; private FetchLogReadPreference readPreference = FetchLogReadPreference.LOCAL_FIRST; + private short apiVersion; public FetchParamsBuilder(int replicaId, int maxFetchBytes) { this.replicaId = replicaId; @@ -70,6 +71,11 @@ public FetchParamsBuilder withReadPreference(FetchLogReadPreference readPreferen return this; } + public FetchParamsBuilder withApiVersion(short apiVersion) { + this.apiVersion = apiVersion; + return this; + } + public FetchParams build() { return new FetchParams( replicaId, @@ -78,6 +84,7 @@ public FetchParams build() { minFetchBytes, maxWaitMs, tableFilterInfoMap, - readPreference); + readPreference, + apiVersion); } } diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java index e70668c9f3..e560625bf6 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java @@ -49,7 +49,9 @@ import org.apache.fluss.record.MemoryLogRecords; import org.apache.fluss.record.ProjectionPushdownCache; import org.apache.fluss.remote.RemoteLogFetchInfo; +import org.apache.fluss.remote.RemoteLogFetchInfoV2; import org.apache.fluss.remote.RemoteLogSegment; +import org.apache.fluss.remote.RemoteLogSegmentReference; import org.apache.fluss.rpc.RpcClient; import org.apache.fluss.rpc.entity.FetchLogResultForBucket; import org.apache.fluss.rpc.entity.LimitScanResultForBucket; @@ -1519,7 +1521,7 @@ public Map readFromLog( // info. if (fetchParams.isRemoteFirstClientFetch()) { FetchLogResultForBucket remoteFirstFetchResult = - tryFetchRemoteFirst(replica, fetchOffset); + tryFetchRemoteFirst(replica, fetchOffset, fetchParams.apiVersion()); if (remoteFirstFetchResult != null) { logReadResult.put( tb, @@ -1577,7 +1579,9 @@ public Map readFromLog( FetchLogResultForBucket result; if (replica != null && e instanceof LogOffsetOutOfRangeException) { - result = handleFetchOutOfRangeException(replica, fetchOffset, e); + result = + handleFetchOutOfRangeException( + replica, fetchOffset, fetchParams.apiVersion(), e); } else { result = new FetchLogResultForBucket(tb, ApiError.fromThrowable(e)); } @@ -1589,7 +1593,7 @@ public Map readFromLog( } private @Nullable FetchLogResultForBucket tryFetchRemoteFirst( - Replica replica, long fetchOffset) { + Replica replica, long fetchOffset, short apiVersion) { if (!replica.isLeader()) { throw new NotLeaderOrFollowerException( String.format( @@ -1607,11 +1611,10 @@ public Map readFromLog( } try { - RemoteLogFetchInfo remoteLogFetchInfo = - fetchLogFromRemote(replica, normalizedFetchOffset); - if (remoteLogFetchInfo != null) { - return new FetchLogResultForBucket( - tb, remoteLogFetchInfo, replica.getLogHighWatermark()); + FetchLogResultForBucket remoteFetchResult = + fetchLogFromRemote(replica, normalizedFetchOffset, apiVersion); + if (remoteFetchResult != null) { + return remoteFetchResult; } // Remote log is expected to cover the offset, but segments/manifest may not be ready // yet. @@ -1629,7 +1632,7 @@ public Map readFromLog( } private FetchLogResultForBucket handleFetchOutOfRangeException( - Replica replica, long fetchOffset, Exception e) { + Replica replica, long fetchOffset, short apiVersion, Exception e) { TableBucket tb = replica.getTableBucket(); if (fetchOffset == FetchParams.FETCH_FROM_EARLIEST_OFFSET) { fetchOffset = replica.getLogStartOffset(); @@ -1648,10 +1651,10 @@ private FetchLogResultForBucket handleFetchOutOfRangeException( // For follower, it can update its local metadata to adjust the next fetch offset. else if (canFetchFromRemoteLog(replica, fetchOffset)) { try { - RemoteLogFetchInfo remoteLogFetchInfo = fetchLogFromRemote(replica, fetchOffset); - if (remoteLogFetchInfo != null) { - return new FetchLogResultForBucket( - tb, remoteLogFetchInfo, replica.getLogHighWatermark()); + FetchLogResultForBucket remoteFetchResult = + fetchLogFromRemote(replica, fetchOffset, apiVersion); + if (remoteFetchResult != null) { + return remoteFetchResult; } } catch (Exception ex) { return new FetchLogResultForBucket(tb, ApiError.fromThrowable(ex)); @@ -1676,7 +1679,29 @@ private boolean canFetchFromRemoteLog(Replica replica, long fetchOffset) { return replica.getLogTablet().canFetchFromRemoteLog(fetchOffset); } - private @Nullable RemoteLogFetchInfo fetchLogFromRemote(Replica replica, long fetchOffset) { + private @Nullable FetchLogResultForBucket fetchLogFromRemote( + Replica replica, long fetchOffset, short apiVersion) { + PhysicalTablePath physicalTablePath = replica.getPhysicalTablePath(); + FsPath remoteLogTabletDir = + FlussPaths.remoteLogTabletDir( + remoteLogManager.remoteLogDir(), + physicalTablePath, + replica.getTableBucket()); + if (apiVersion >= 1) { + List references = + remoteLogManager.relevantRemoteLogSegmentReferences( + replica.getTableBucket(), fetchOffset); + if (references.isEmpty()) { + return null; + } + RemoteLogFetchInfoV2 remoteLogFetchInfo = + new RemoteLogFetchInfoV2( + remoteLogTabletDir.toString(), + physicalTablePath.getPartitionName(), + references); + return new FetchLogResultForBucket( + replica.getTableBucket(), remoteLogFetchInfo, replica.getLogHighWatermark()); + } List remoteLogSegmentList = remoteLogManager.relevantRemoteLogSegmentsForFetchV0( replica.getTableBucket(), fetchOffset); @@ -1684,17 +1709,14 @@ private boolean canFetchFromRemoteLog(Replica replica, long fetchOffset) { int firstStartPos = remoteLogManager.lookupPositionForOffset( remoteLogSegmentList.get(0), fetchOffset); - PhysicalTablePath physicalTablePath = replica.getPhysicalTablePath(); - FsPath remoteLogTabletDir = - FlussPaths.remoteLogTabletDir( - remoteLogManager.remoteLogDir(), - physicalTablePath, - replica.getTableBucket()); - return new RemoteLogFetchInfo( - remoteLogTabletDir.toString(), - physicalTablePath.getPartitionName(), - remoteLogSegmentList, - firstStartPos); + RemoteLogFetchInfo remoteLogFetchInfo = + new RemoteLogFetchInfo( + remoteLogTabletDir.toString(), + physicalTablePath.getPartitionName(), + remoteLogSegmentList, + firstStartPos); + return new FetchLogResultForBucket( + replica.getTableBucket(), remoteLogFetchInfo, replica.getLogHighWatermark()); } else { return null; } diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/fetcher/ReplicaFetcherThread.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/fetcher/ReplicaFetcherThread.java index cc430a19f9..3606750779 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/replica/fetcher/ReplicaFetcherThread.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/fetcher/ReplicaFetcherThread.java @@ -28,7 +28,9 @@ import org.apache.fluss.metadata.TablePath; import org.apache.fluss.record.MemoryLogRecords; import org.apache.fluss.remote.RemoteLogFetchInfo; +import org.apache.fluss.remote.RemoteLogFetchInfoV2; import org.apache.fluss.remote.RemoteLogSegment; +import org.apache.fluss.remote.RemoteLogSegmentReference; import org.apache.fluss.rpc.entity.FetchLogResultForBucket; import org.apache.fluss.rpc.messages.FetchLogRequest; import org.apache.fluss.server.log.LogAppendInfo; @@ -68,7 +70,7 @@ import java.util.concurrent.locks.ReentrantLock; import java.util.stream.Collectors; -import static org.apache.fluss.utils.Preconditions.checkNotNull; +import static org.apache.fluss.utils.Preconditions.checkArgument; import static org.apache.fluss.utils.concurrent.LockUtils.inLock; /* This file is based on source code of Apache Kafka Project (https://kafka.apache.org/), licensed by the Apache @@ -603,7 +605,10 @@ private LogAppendInfo processFetchResultFromLocalStorage( private long processFetchResultFromRemoteStorage( TableBucket tb, FetchLogResultForBucket replicaData) { RemoteLogFetchInfo rlFetchInfo = replicaData.remoteLogFetchInfo(); - checkNotNull(rlFetchInfo, "RemoteLogFetchInfo is null"); + RemoteLogFetchInfoV2 rlFetchInfoV2 = replicaData.remoteLogFetchInfoV2(); + checkArgument( + rlFetchInfo != null || rlFetchInfoV2 != null, + "Both RemoteLogFetchInfo versions are null"); Replica replica = replicaManager.getReplicaOrException(tb); RemoteLogManager rlm = replicaManager.getRemoteLogManager(); @@ -611,13 +616,24 @@ private long processFetchResultFromRemoteStorage( // cache. Trace by https://github.com/apache/fluss/issues/673 // update next fetch offset and writer id snapshot in local. - RemoteLogSegment remoteLogSegmentWithMaxStartOffset = - rlFetchInfo - .remoteLogSegmentList() - .get(rlFetchInfo.remoteLogSegmentList().size() - 1); + RemoteLogSegment remoteLogSegmentWithMaxStartOffset; + long nextFetchOffset; + if (rlFetchInfoV2 != null) { + RemoteLogSegmentReference lastReference = + rlFetchInfoV2 + .activeReferences() + .get(rlFetchInfoV2.activeReferences().size() - 1); + remoteLogSegmentWithMaxStartOffset = lastReference.remoteLogSegment(); + nextFetchOffset = lastReference.logicalEndOffset(); + } else { + remoteLogSegmentWithMaxStartOffset = + rlFetchInfo + .remoteLogSegmentList() + .get(rlFetchInfo.remoteLogSegmentList().size() - 1); + nextFetchOffset = remoteLogSegmentWithMaxStartOffset.remoteLogEndOffset(); + } // build writer snapshots until remoteLogSegment.endOffset() and start segment from // until remoteLogSegment.endOffset(). - long nextFetchOffset = remoteLogSegmentWithMaxStartOffset.remoteLogEndOffset(); try { // Truncate the existing local log before restoring the writer id snapshots. diff --git a/fluss-server/src/main/java/org/apache/fluss/server/tablet/TabletService.java b/fluss-server/src/main/java/org/apache/fluss/server/tablet/TabletService.java index 0c2058915d..e2c0ee52c1 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/tablet/TabletService.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/tablet/TabletService.java @@ -235,7 +235,7 @@ public CompletableFuture fetchLog(FetchLogRequest request) { } CompletableFuture response = new CompletableFuture<>(); - FetchParams fetchParams = getFetchParams(request); + FetchParams fetchParams = getFetchParams(request, currentSession().getApiVersion()); replicaManager.fetchLogRecords( fetchParams, interesting, @@ -246,7 +246,7 @@ public CompletableFuture fetchLog(FetchLogRequest request) { return response; } - private static FetchParams getFetchParams(FetchLogRequest request) { + private static FetchParams getFetchParams(FetchLogRequest request, short apiVersion) { FetchParams fetchParams; Map tableFilterInfoMap = getTableFilterInfoMap(request); FetchLogReadPreference readPreference = @@ -263,12 +263,14 @@ private static FetchParams getFetchParams(FetchLogRequest request) { : DEFAULT_MAX_WAIT_MS_WHEN_MIN_BYTES_ENABLE) .withTableFilterInfoMap(tableFilterInfoMap) .withReadPreference(readPreference) + .withApiVersion(apiVersion) .build(); } else { fetchParams = new FetchParamsBuilder(request.getFollowerServerId(), request.getMaxBytes()) .withTableFilterInfoMap(tableFilterInfoMap) .withReadPreference(readPreference) + .withApiVersion(apiVersion) .build(); } diff --git a/fluss-server/src/main/java/org/apache/fluss/server/utils/ServerRpcMessageUtils.java b/fluss-server/src/main/java/org/apache/fluss/server/utils/ServerRpcMessageUtils.java index 5c7202ebfa..0f53c34fd1 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/utils/ServerRpcMessageUtils.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/utils/ServerRpcMessageUtils.java @@ -54,7 +54,9 @@ import org.apache.fluss.record.LogRecords; import org.apache.fluss.record.MemoryLogRecords; import org.apache.fluss.remote.RemoteLogFetchInfo; +import org.apache.fluss.remote.RemoteLogFetchInfoV2; import org.apache.fluss.remote.RemoteLogSegment; +import org.apache.fluss.remote.RemoteLogSegmentReference; import org.apache.fluss.rpc.entity.FetchLogResultForBucket; import org.apache.fluss.rpc.entity.LimitScanResultForBucket; import org.apache.fluss.rpc.entity.ListOffsetsResultForBucket; @@ -148,6 +150,7 @@ import org.apache.fluss.rpc.messages.PbRebalanceProgressForBucket; import org.apache.fluss.rpc.messages.PbRemoteLogManifestEntry; import org.apache.fluss.rpc.messages.PbRemoteLogSegment; +import org.apache.fluss.rpc.messages.PbRemoteLogSegmentReference; import org.apache.fluss.rpc.messages.PbRemotePathAndLocalFile; import org.apache.fluss.rpc.messages.PbRenameColumn; import org.apache.fluss.rpc.messages.PbServerNode; @@ -1025,21 +1028,44 @@ public static FetchLogResponse makeFetchLogResponse( .setLogStartOffset(0L); if (bucketResult.fetchFromRemote()) { - RemoteLogFetchInfo rlfInfo = bucketResult.remoteLogFetchInfo(); - checkNotNull(rlfInfo, "Remote log fetch info is null."); - List remoteLogSegmentList = new ArrayList<>(); - for (RemoteLogSegment logSegment : rlfInfo.remoteLogSegmentList()) { - remoteLogSegmentList.add(toPbRemoteLogSegment(logSegment)); - } - fetchLogRespForBucket - .setRemoteLogFetchInfo() - .setRemoteLogTabletDir(rlfInfo.remoteLogTabletDir()) - .addAllRemoteLogSegments(remoteLogSegmentList) - .setFirstStartPos(rlfInfo.firstStartPos()); - if (rlfInfo.partitionName() != null) { + RemoteLogFetchInfoV2 rlfInfoV2 = bucketResult.remoteLogFetchInfoV2(); + if (rlfInfoV2 != null) { + List references = new ArrayList<>(); + for (RemoteLogSegmentReference reference : rlfInfoV2.activeReferences()) { + references.add( + new PbRemoteLogSegmentReference() + .setSegment( + toPbRemoteLogSegment( + reference.remoteLogSegment())) + .setLogicalStartOffset(reference.logicalStartOffset()) + .setLogicalEndOffset(reference.logicalEndOffset())); + } + fetchLogRespForBucket + .setRemoteLogFetchInfoV2() + .setRemoteLogTabletDir(rlfInfoV2.remoteLogTabletDir()) + .addAllActiveReferences(references); + if (rlfInfoV2.partitionName() != null) { + fetchLogRespForBucket + .setRemoteLogFetchInfoV2() + .setPartitionName(rlfInfoV2.partitionName()); + } + } else { + RemoteLogFetchInfo rlfInfo = bucketResult.remoteLogFetchInfo(); + checkNotNull(rlfInfo, "Remote log fetch info is null."); + List remoteLogSegmentList = new ArrayList<>(); + for (RemoteLogSegment logSegment : rlfInfo.remoteLogSegmentList()) { + remoteLogSegmentList.add(toPbRemoteLogSegment(logSegment)); + } fetchLogRespForBucket .setRemoteLogFetchInfo() - .setPartitionName(rlfInfo.partitionName()); + .setRemoteLogTabletDir(rlfInfo.remoteLogTabletDir()) + .addAllRemoteLogSegments(remoteLogSegmentList) + .setFirstStartPos(rlfInfo.firstStartPos()); + if (rlfInfo.partitionName() != null) { + fetchLogRespForBucket + .setRemoteLogFetchInfo() + .setPartitionName(rlfInfo.partitionName()); + } } } else { // set records diff --git a/fluss-server/src/test/java/org/apache/fluss/server/log/remote/RemoteLogManagerTest.java b/fluss-server/src/test/java/org/apache/fluss/server/log/remote/RemoteLogManagerTest.java index 79e4d0a7ae..7d0c3e6580 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/log/remote/RemoteLogManagerTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/log/remote/RemoteLogManagerTest.java @@ -22,6 +22,7 @@ import org.apache.fluss.fs.FsPath; import org.apache.fluss.metadata.TableBucket; import org.apache.fluss.remote.RemoteLogFetchInfo; +import org.apache.fluss.remote.RemoteLogFetchInfoV2; import org.apache.fluss.remote.RemoteLogManifest; import org.apache.fluss.remote.RemoteLogManifestV2Migration; import org.apache.fluss.remote.RemoteLogSegment; @@ -404,6 +405,28 @@ void testFetchRecordsFromRemote(boolean partitionTable) throws Exception { assertThat(remoteLogFetchInfo).isNotNull(); assertThat(remoteLogFetchInfo.remoteLogSegmentList().size()).isEqualTo(4); + // A v1 reader receives logical references even while the persisted manifest is V1. + future = new CompletableFuture<>(); + replicaManager.fetchLogRecords( + new FetchParams( + -1, + true, + Integer.MAX_VALUE, + -1, + -1, + null, + FetchLogReadPreference.LOCAL_FIRST, + (short) 1), + Collections.singletonMap(tb, new FetchReqInfo(tb.getTableId(), 0L, 1024 * 1024)), + null, + future::complete); + RemoteLogFetchInfoV2 remoteLogFetchInfoV2 = future.get().get(tb).remoteLogFetchInfoV2(); + assertThat(remoteLogFetchInfoV2).isNotNull(); + assertThat(remoteLogFetchInfoV2.activeReferences()).hasSize(4); + assertThat(remoteLogFetchInfoV2.activeReferences().get(0).logicalStartOffset()).isZero(); + assertThat(remoteLogFetchInfoV2.activeReferences().get(0).logicalEndOffset()) + .isEqualTo(10L); + // A V2 manifest with physically adjacent segments preserves the legacy v0 batching // behavior and returns the complete contiguous, non-overlapping tail. RemoteLogTablet remoteLogTablet = remoteLogManager.remoteLogTablet(tb); @@ -422,6 +445,25 @@ void testFetchRecordsFromRemote(boolean partitionTable) throws Exception { assertThat(legacyV2FetchInfo.remoteLogSegmentList().get(0).remoteLogStartOffset()) .isEqualTo(0L); + // FetchLog v1 exposes the authoritative logical bounds from the V2 manifest. + future = new CompletableFuture<>(); + replicaManager.fetchLogRecords( + new FetchParams( + -1, + true, + Integer.MAX_VALUE, + -1, + -1, + null, + FetchLogReadPreference.LOCAL_FIRST, + (short) 1), + Collections.singletonMap(tb, new FetchReqInfo(tb.getTableId(), 0L, 1024 * 1024)), + null, + future::complete); + remoteLogFetchInfoV2 = future.get().get(tb).remoteLogFetchInfoV2(); + assertThat(remoteLogFetchInfoV2).isNotNull(); + assertThat(remoteLogFetchInfoV2.activeReferences()).hasSize(4); + // A request starting at a later segment returns the remaining contiguous tail. future = new CompletableFuture<>(); replicaManager.fetchLogRecords( diff --git a/fluss-server/src/test/java/org/apache/fluss/server/utils/ServerRpcMessageUtilsTest.java b/fluss-server/src/test/java/org/apache/fluss/server/utils/ServerRpcMessageUtilsTest.java index a1cfea1b4b..1c8829f50c 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/utils/ServerRpcMessageUtilsTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/utils/ServerRpcMessageUtilsTest.java @@ -18,14 +18,28 @@ package org.apache.fluss.server.utils; import org.apache.fluss.fs.FsPath; +import org.apache.fluss.metadata.PhysicalTablePath; import org.apache.fluss.metadata.TableBucket; +import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.remote.RemoteLogFetchInfoV2; +import org.apache.fluss.remote.RemoteLogSegment; +import org.apache.fluss.remote.RemoteLogSegmentReference; +import org.apache.fluss.rpc.entity.FetchLogResultForBucket; import org.apache.fluss.rpc.messages.CommitRemoteLogManifestRequest; +import org.apache.fluss.rpc.messages.FetchLogResponse; +import org.apache.fluss.rpc.messages.PbFetchLogRespForBucket; +import org.apache.fluss.rpc.messages.PbRemoteLogFetchInfo; import org.apache.fluss.server.entity.CommitRemoteLogManifestData; import org.junit.jupiter.api.Test; +import java.util.Collections; +import java.util.UUID; + +import static org.apache.fluss.rpc.util.CommonRpcMessageUtils.getFetchLogResultForBucket; import static org.apache.fluss.server.utils.ServerRpcMessageUtils.getCommitRemoteLogManifestData; import static org.apache.fluss.server.utils.ServerRpcMessageUtils.makeCommitRemoteLogManifestRequest; +import static org.apache.fluss.server.utils.ServerRpcMessageUtils.makeFetchLogResponse; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -64,4 +78,50 @@ void testRemoteLogManifestCasRequestRoundTrip() { .isInstanceOf(IllegalArgumentException.class) .hasMessageContaining("ABSENT"); } + + @Test + void testRemoteLogFetchInfoV2RoundTrip() { + TablePath tablePath = TablePath.of("db", "table"); + PhysicalTablePath physicalTablePath = PhysicalTablePath.of(tablePath); + TableBucket tableBucket = new TableBucket(1L, 0); + RemoteLogSegment segment = + RemoteLogSegment.Builder.builder() + .physicalTablePath(physicalTablePath) + .tableBucket(tableBucket) + .remoteLogSegmentId(UUID.randomUUID()) + .remoteLogStartOffset(0L) + .remoteLogEndOffset(20L) + .maxTimestamp(1L) + .segmentSizeInBytes(100) + .build(); + RemoteLogSegmentReference reference = new RemoteLogSegmentReference(segment, 5L, 20L); + FetchLogResultForBucket result = + new FetchLogResultForBucket( + tableBucket, + new RemoteLogFetchInfoV2( + "file:///remote/tablet", + null, + Collections.singletonList(reference)), + 20L); + + FetchLogResponse response = + makeFetchLogResponse(Collections.singletonMap(tableBucket, result)); + PbFetchLogRespForBucket pbBucket = response.getTablesRespAt(0).getBucketsRespAt(0); + + assertThat(pbBucket.hasRemoteLogFetchInfo()).isFalse(); + assertThat(pbBucket.hasRemoteLogFetchInfoV2()).isTrue(); + + FetchLogResultForBucket decoded = + getFetchLogResultForBucket(tableBucket, tablePath, pbBucket); + assertThat(decoded.remoteLogFetchInfo()).isNull(); + assertThat(decoded.remoteLogFetchInfoV2().activeReferences()).containsExactly(reference); + + PbFetchLogRespForBucket ambiguousBucket = + pbBucket.setRemoteLogFetchInfo( + new PbRemoteLogFetchInfo().setRemoteLogTabletDir("file:///legacy")); + assertThatThrownBy( + () -> getFetchLogResultForBucket(tableBucket, tablePath, ambiguousBucket)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("both v0 and v1"); + } } From d716b39ef9c3ab177bef9c298de6326a607e3e7b Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Sun, 2 Aug 2026 22:23:13 +0800 Subject: [PATCH 3/3] feat: fix rust test --- .../src/client/table/log_fetch_buffer.rs | 45 ++++++++++- .../fluss/src/client/table/remote_log.rs | 58 ++++++++++++++- .../crates/fluss/src/client/table/scanner.rs | 74 +++++++++++++++++-- fluss-rust/crates/fluss/src/rpc/api_key.rs | 12 ++- 4 files changed, 180 insertions(+), 9 deletions(-) diff --git a/fluss-rust/crates/fluss/src/client/table/log_fetch_buffer.rs b/fluss-rust/crates/fluss/src/client/table/log_fetch_buffer.rs index b057612d6e..efc6b210c7 100644 --- a/fluss-rust/crates/fluss/src/client/table/log_fetch_buffer.rs +++ b/fluss-rust/crates/fluss/src/client/table/log_fetch_buffer.rs @@ -365,6 +365,7 @@ pub struct DefaultCompletedFetch { resolver: Arc, is_remote: bool, next_fetch_offset: i64, + logical_end_offset: i64, high_watermark: i64, size_in_bytes: usize, consumed: bool, @@ -405,6 +406,7 @@ impl DefaultCompletedFetch { resolver, is_remote, next_fetch_offset: fetch_offset, + logical_end_offset: i64::MAX, high_watermark, size_in_bytes, consumed: false, @@ -420,6 +422,11 @@ impl DefaultCompletedFetch { } } + fn with_logical_end(mut self, logical_end_offset: i64) -> Self { + self.logical_end_offset = logical_end_offset; + self + } + pub(crate) fn from_error( table_bucket: TableBucket, error: Error, @@ -435,6 +442,7 @@ impl DefaultCompletedFetch { resolver, is_remote: false, next_fetch_offset: fetch_offset, + logical_end_offset: i64::MAX, high_watermark: -1, size_in_bytes: 0, consumed: false, @@ -466,6 +474,7 @@ impl DefaultCompletedFetch { resolver, is_remote: false, next_fetch_offset: fetch_offset, + logical_end_offset: i64::MAX, high_watermark: -1, size_in_bytes: 0, consumed: false, @@ -508,6 +517,21 @@ impl DefaultCompletedFetch { .pending_record_batch .as_ref() .expect("pending batch must be set before schema resolution"); + if batch.base_log_offset() >= self.logical_end_offset { + self.drain(); + return Ok(FetchStep::End); + } + if batch.next_log_offset() > self.logical_end_offset { + return Err(Error::UnexpectedError { + message: format!( + "Remote log batch [{}, {}) crosses logical end offset {}", + batch.base_log_offset(), + batch.next_log_offset(), + self.logical_end_offset + ), + source: None, + }); + } let Some(read_context) = self.resolve_context_for_batch(batch) else { return Ok(FetchStep::SchemaRequired(batch.schema_id())); }; @@ -598,6 +622,21 @@ impl DefaultCompletedFetch { .pending_record_batch .as_ref() .expect("pending batch must be set before schema resolution"); + if log_batch.base_log_offset() >= self.logical_end_offset { + self.drain(); + return Ok(FetchStep::End); + } + if log_batch.next_log_offset() > self.logical_end_offset { + return Err(Error::UnexpectedError { + message: format!( + "Remote log batch [{}, {}) crosses logical end offset {}", + log_batch.base_log_offset(), + log_batch.next_log_offset(), + self.logical_end_offset + ), + source: None, + }); + } let Some(read_context) = self.resolve_context_for_batch(log_batch) else { return Ok(FetchStep::SchemaRequired(log_batch.schema_id())); }; @@ -892,6 +931,7 @@ pub struct RemotePendingFetch { pos_in_log_segment: i32, fetch_offset: i64, high_watermark: i64, + logical_end_offset: i64, resolver: Arc, } @@ -902,6 +942,7 @@ impl RemotePendingFetch { pos_in_log_segment: i32, fetch_offset: i64, high_watermark: i64, + logical_end_offset: i64, resolver: Arc, ) -> Self { Self { @@ -910,6 +951,7 @@ impl RemotePendingFetch { pos_in_log_segment, fetch_offset, high_watermark, + logical_end_offset, resolver, } } @@ -966,7 +1008,8 @@ impl PendingFetch for RemotePendingFetch { true, // is_remote self.fetch_offset, self.high_watermark, - ); + ) + .with_logical_end(self.logical_end_offset); // Wrap it with RemoteCompletedFetch to hold the permit // Permit manages the prefetch slot (releases semaphore and notifies coordinator) when dropped; diff --git a/fluss-rust/crates/fluss/src/client/table/remote_log.rs b/fluss-rust/crates/fluss/src/client/table/remote_log.rs index c843d1a2d7..32d6054ce9 100644 --- a/fluss-rust/crates/fluss/src/client/table/remote_log.rs +++ b/fluss-rust/crates/fluss/src/client/table/remote_log.rs @@ -19,7 +19,7 @@ use crate::error::{Error, Result}; use crate::io::{FileIO, Storage}; use crate::metadata::TableBucket; use crate::metrics::ScannerMetrics; -use crate::proto::{PbRemoteLogFetchInfo, PbRemoteLogSegment}; +use crate::proto::{PbRemoteLogFetchInfo, PbRemoteLogFetchInfoV2, PbRemoteLogSegment}; use futures::TryStreamExt; use parking_lot::Mutex; use std::{ @@ -139,6 +139,9 @@ pub struct RemoteLogFetchInfo { pub partition_name: Option, pub remote_log_segments: Vec, pub first_start_pos: i32, + /// Logical half-open ranges corresponding one-to-one with `remote_log_segments`. + /// Absent for FetchLog v0, whose byte position already selects the first offset. + pub logical_ranges: Option>, } impl RemoteLogFetchInfo { @@ -154,8 +157,61 @@ impl RemoteLogFetchInfo { partition_name: info.partition_name.clone(), remote_log_segments: segments, first_start_pos: info.first_start_pos.unwrap_or(0), + logical_ranges: None, } } + + pub fn from_proto_v2(info: &PbRemoteLogFetchInfoV2, table_bucket: TableBucket) -> Result { + let mut segments = Vec::with_capacity(info.active_references.len()); + let mut logical_ranges = Vec::with_capacity(info.active_references.len()); + let mut previous_logical_end = None; + for reference in &info.active_references { + let segment = &reference.segment; + if reference.logical_start_offset >= reference.logical_end_offset + || reference.logical_start_offset < segment.remote_log_start_offset + || reference.logical_end_offset > segment.remote_log_end_offset + { + return Err(Error::UnexpectedError { + message: format!( + "Invalid remote log logical range [{}, {}) for physical range [{}, {})", + reference.logical_start_offset, + reference.logical_end_offset, + segment.remote_log_start_offset, + segment.remote_log_end_offset + ), + source: None, + }); + } + if let Some(previous_end) = previous_logical_end + && reference.logical_start_offset != previous_end + { + return Err(Error::UnexpectedError { + message: format!( + "Remote log logical references are not contiguous: previous end {}, next start {}", + previous_end, reference.logical_start_offset + ), + source: None, + }); + } + segments.push(RemoteLogSegment::from_proto(segment, table_bucket.clone())); + logical_ranges.push((reference.logical_start_offset, reference.logical_end_offset)); + previous_logical_end = Some(reference.logical_end_offset); + } + if segments.is_empty() { + return Err(Error::UnexpectedError { + message: "Remote log fetch info v2 contains no active references".to_string(), + source: None, + }); + } + + Ok(Self { + remote_log_tablet_dir: info.remote_log_tablet_dir.clone(), + partition_name: info.partition_name.clone(), + remote_log_segments: segments, + first_start_pos: 0, + logical_ranges: Some(logical_ranges), + }) + } } /// RAII guard for prefetch permit that notifies coordinator on drop diff --git a/fluss-rust/crates/fluss/src/client/table/scanner.rs b/fluss-rust/crates/fluss/src/client/table/scanner.rs index 6e4d046e7c..7696793577 100644 --- a/fluss-rust/crates/fluss/src/client/table/scanner.rs +++ b/fluss-rust/crates/fluss/src/client/table/scanner.rs @@ -1621,8 +1621,53 @@ impl LogFetcher { continue; } - // Check if this is a remote log fetch - if let Some(ref remote_log_fetch_info) = fetch_log_for_bucket.remote_log_fetch_info + // Check if this is a remote log fetch. V0 and V1 response fields are mutually + // exclusive so that a client never has to guess which logical view is active. + if fetch_log_for_bucket.remote_log_fetch_info.is_some() + && fetch_log_for_bucket.remote_log_fetch_info_v2.is_some() + { + log_fetch_buffer.add(Box::new(DefaultCompletedFetch::from_error( + table_bucket.clone(), + Error::UnexpectedError { + message: + "FetchLog response contains both remote log v0 and v2 metadata" + .to_string(), + source: None, + }, + fetch_offset, + Arc::clone(&resolver), + ))); + } else if let Some(ref remote_log_fetch_info) = + fetch_log_for_bucket.remote_log_fetch_info_v2 + { + // Remote fs props are already set by the background SecurityTokenManager. + match RemoteLogFetchInfo::from_proto_v2( + remote_log_fetch_info, + table_bucket.clone(), + ) { + Ok(remote_fetch_info) => { + let high_watermark = fetch_log_for_bucket.high_watermark.unwrap_or(-1); + Self::pending_remote_fetches( + remote_log_downloader.clone(), + log_fetch_buffer.clone(), + Arc::clone(&resolver), + &table_bucket, + remote_fetch_info, + fetch_offset, + high_watermark, + ); + } + Err(error) => { + log_fetch_buffer.add(Box::new(DefaultCompletedFetch::from_error( + table_bucket.clone(), + error, + fetch_offset, + Arc::clone(&resolver), + ))); + } + } + } else if let Some(ref remote_log_fetch_info) = + fetch_log_for_bucket.remote_log_fetch_info { // Remote fs props are already set by the background SecurityTokenManager let remote_fetch_info = @@ -1673,10 +1718,23 @@ impl LogFetcher { let mut pos_in_log_segment = remote_fetch_info.first_start_pos; let mut current_fetch_offset = fetch_offset; for (i, segment) in remote_fetch_info.remote_log_segments.iter().enumerate() { - if i > 0 { - pos_in_log_segment = 0; - current_fetch_offset = segment.start_offset; - } + let logical_end_offset = + if let Some(logical_ranges) = remote_fetch_info.logical_ranges.as_ref() { + let (logical_start_offset, logical_end_offset) = logical_ranges[i]; + pos_in_log_segment = 0; + current_fetch_offset = if i == 0 { + fetch_offset.max(logical_start_offset) + } else { + logical_start_offset + }; + logical_end_offset + } else { + if i > 0 { + pos_in_log_segment = 0; + current_fetch_offset = segment.start_offset; + } + segment.end_offset + }; // todo: // 1: control the max threads to download remote segment @@ -1699,6 +1757,7 @@ impl LogFetcher { pos_in_log_segment, current_fetch_offset, high_watermark, + logical_end_offset, Arc::clone(&resolver), ); // Add to pending fetches in buffer (similar to Java's logFetchBuffer.pend) @@ -2602,6 +2661,7 @@ mod tests { high_watermark: None, log_start_offset: None, remote_log_fetch_info: None, + remote_log_fetch_info_v2: None, records: None, filtered_end_offset: None, }], @@ -2660,6 +2720,7 @@ mod tests { high_watermark: None, log_start_offset: None, remote_log_fetch_info: None, + remote_log_fetch_info_v2: None, records: None, filtered_end_offset: None, }], @@ -3006,6 +3067,7 @@ mod tests { high_watermark: Some(7), log_start_offset: Some(0), remote_log_fetch_info: None, + remote_log_fetch_info_v2: None, records: None, filtered_end_offset: None, }], diff --git a/fluss-rust/crates/fluss/src/rpc/api_key.rs b/fluss-rust/crates/fluss/src/rpc/api_key.rs index cab034d216..eb150ce0ff 100644 --- a/fluss-rust/crates/fluss/src/rpc/api_key.rs +++ b/fluss-rust/crates/fluss/src/rpc/api_key.rs @@ -94,7 +94,6 @@ impl ApiKey { | ApiKey::GetTableSchema | ApiKey::MetaData | ApiKey::ProduceLog - | ApiKey::FetchLog | ApiKey::ListOffsets | ApiKey::GetLatestKvSnapshots | ApiKey::GetKvSnapshotMetadata @@ -129,6 +128,9 @@ impl ApiKey { | ApiKey::GetClusterHealth | ApiKey::ListRemoteLogManifests | ApiKey::ListKvSnapshots => Some(ApiVersionRange::new(ApiVersion(0), ApiVersion(0))), + // FetchLog v1 adds logical remote-log segment references. The request body is + // unchanged, so clients can negotiate v1 without changing request construction. + ApiKey::FetchLog => Some(ApiVersionRange::new(ApiVersion(0), ApiVersion(1))), // PutKv / Lookup / PrefixLookup support v0 (legacy key encoding) // and v1 (Paimon BinaryRow key encoding for kv_format_version=2 // non-default bucket keys). The Rust client encodes both. @@ -333,4 +335,12 @@ mod tests { let mapped: i16 = unknown.into(); assert_eq!(mapped, 9999); } + + #[test] + fn fetch_log_supports_logical_reference_version() { + assert_eq!( + ApiKey::FetchLog.supported_versions(), + Some(ApiVersionRange::new(ApiVersion(0), ApiVersion(1))) + ); + } }