From 24fc597ec084f9e3671865a5765ccd225ba671f0 Mon Sep 17 00:00:00 2001 From: Hyunsang Han Date: Mon, 1 Sep 2025 22:47:19 +0900 Subject: [PATCH 01/15] Skip metadata upload when no new segments during refresh Signed-off-by: Hyunsang Han --- .../index/shard/RemoteStoreRefreshListener.java | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java b/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java index f3ee23a7505d2..d5b0e60c7cc0b 100644 --- a/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java +++ b/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java @@ -267,9 +267,16 @@ private boolean syncSegments() { public void onResponse(Void unused) { try { logger.debug("New segments upload successful"); - // Start metadata file upload - uploadMetadata(localSegmentsPostRefresh, segmentInfos, checkpoint); - logger.debug("Metadata upload successful"); + Collection filteredFiles = localSegmentsPostRefresh.stream() + .filter(file -> !skipUpload(file)) + .collect(Collectors.toList()); + if (!filteredFiles.isEmpty()) { + // Start metadata file upload + uploadMetadata(localSegmentsPostRefresh, segmentInfos, checkpoint); + logger.debug("Metadata upload successful"); + } else { + logger.debug("Skipping metadata upload - no new segments were uploaded"); + } clearStaleFilesFromLocalSegmentChecksumMap(localSegmentsPostRefresh); onSuccessfulSegmentsSync( refreshTimeMs, From f9743ebc25d9ad2c61258b35140daec2ec02d4fa Mon Sep 17 00:00:00 2001 From: Hyunsang Han Date: Thu, 9 Oct 2025 09:01:26 +0900 Subject: [PATCH 02/15] Refactor metadata upload optimization Signed-off-by: Hyunsang Han --- .../index/shard/RemoteStoreRefreshListener.java | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java b/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java index d5b0e60c7cc0b..a6ff21182f59b 100644 --- a/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java +++ b/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java @@ -267,15 +267,12 @@ private boolean syncSegments() { public void onResponse(Void unused) { try { logger.debug("New segments upload successful"); - Collection filteredFiles = localSegmentsPostRefresh.stream() - .filter(file -> !skipUpload(file)) - .collect(Collectors.toList()); - if (!filteredFiles.isEmpty()) { + if (localSegmentsPostRefresh.stream().allMatch(file -> skipUpload(file))) { + logger.debug("Skipping metadata upload - no new segments were uploaded"); + } else { // Start metadata file upload uploadMetadata(localSegmentsPostRefresh, segmentInfos, checkpoint); logger.debug("Metadata upload successful"); - } else { - logger.debug("Skipping metadata upload - no new segments were uploaded"); } clearStaleFilesFromLocalSegmentChecksumMap(localSegmentsPostRefresh); onSuccessfulSegmentsSync( From 1e058da328cc1e24f99b41788c7836021e5ccede Mon Sep 17 00:00:00 2001 From: Hyunsang Han Date: Thu, 9 Oct 2025 12:22:40 +0900 Subject: [PATCH 03/15] Add changelog Signed-off-by: Hyunsang Han --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 99a645553a46b..4e5c5dd69a7f7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -76,6 +76,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Add skiplist optimization to auto_date_histogram aggregation ([#20057](https://github.com/opensearch-project/OpenSearch/pull/20057)) - Throw exceptions for currently unsupported GRPC request-side fields ([#20162](https://github.com/opensearch-project/OpenSearch/pull/20162)) - Handle custom metadata files in subdirectory-store ([#20157](https://github.com/opensearch-project/OpenSearch/pull/20157)) +- Skip metadata upload when no new segments need uploading during refresh for remote store clusters ([#19198](https://github.com/opensearch-project/OpenSearch/pull/19198)) ### Fixed - Fix Allocation and Rebalance Constraints of WeightFunction are incorrectly reset ([#19012](https://github.com/opensearch-project/OpenSearch/pull/19012)) From 18a41fa7cb4a8cdeb287b2c54e6c99dc1ec22a49 Mon Sep 17 00:00:00 2001 From: Hyunsang Han Date: Thu, 9 Oct 2025 19:19:18 +0900 Subject: [PATCH 04/15] Add test cases: testSkipsMetadataUploadWhenNoNewSegments, testUploadsWhenNewSegmentsPresent Signed-off-by: Hyunsang Han --- .../RemoteStoreRefreshListenerTests.java | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/server/src/test/java/org/opensearch/index/shard/RemoteStoreRefreshListenerTests.java b/server/src/test/java/org/opensearch/index/shard/RemoteStoreRefreshListenerTests.java index 8094f1c91ee13..1e77985fb7ddd 100644 --- a/server/src/test/java/org/opensearch/index/shard/RemoteStoreRefreshListenerTests.java +++ b/server/src/test/java/org/opensearch/index/shard/RemoteStoreRefreshListenerTests.java @@ -882,6 +882,45 @@ private void verifyUploadedSegments(RemoteSegmentStoreDirectory remoteSegmentSto assertTrue(remoteStoreRefreshListener.isRemoteSegmentStoreInSync()); } + public void testSkipsMetadataUploadWhenNoNewSegments() throws IOException { + setup(true, 2); + + indexShard.refresh("initial-upload"); + + try (Store remoteStore = indexShard.remoteStore()) { + RemoteSegmentStoreDirectory remoteSegmentStoreDirectory = + (RemoteSegmentStoreDirectory) ((FilterDirectory) ((FilterDirectory) remoteStore.directory()).getDelegate()).getDelegate(); + + final int initialCount = remoteSegmentStoreDirectory.getSegmentsUploadedToRemoteStore().size(); + + indexShard.refresh("test-optimization"); + + int afterOptimizationCount = remoteSegmentStoreDirectory.getSegmentsUploadedToRemoteStore().size(); + assertEquals("No new segments should be uploaded when optimization kicks in", initialCount, afterOptimizationCount); + } + } + + public void testUploadsWhenNewSegmentsPresent() throws Exception { + setup(true, 1); + + indexShard.refresh("initial-upload"); + + try (Store remoteStore = indexShard.remoteStore()) { + RemoteSegmentStoreDirectory remoteSegmentStoreDirectory = + (RemoteSegmentStoreDirectory) ((FilterDirectory) ((FilterDirectory) remoteStore.directory()).getDelegate()).getDelegate(); + + final int initialCount = remoteSegmentStoreDirectory.getSegmentsUploadedToRemoteStore().size(); + + indexDoc(indexShard, "new-doc", "{}"); + indexShard.refresh("new-segment-refresh"); + + assertBusy(() -> { + int afterNewSegmentCount = remoteSegmentStoreDirectory.getSegmentsUploadedToRemoteStore().size(); + assertTrue("Expected new uploads after adding new segments", afterNewSegmentCount > initialCount); + }); + } + } + public void testRemoteSegmentStoreNotInSync() throws IOException { setup(true, 3); remoteStoreRefreshListener.afterRefresh(true); From 4dbcf30404f27b9efdfb9cdc02723b4c99d34f1c Mon Sep 17 00:00:00 2001 From: Hyunsang Han Date: Thu, 9 Oct 2025 19:23:32 +0900 Subject: [PATCH 05/15] Fix affected test cases Signed-off-by: Hyunsang Han --- .../RemoteStoreRefreshListenerTests.java | 21 ++++++++++++------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/server/src/test/java/org/opensearch/index/shard/RemoteStoreRefreshListenerTests.java b/server/src/test/java/org/opensearch/index/shard/RemoteStoreRefreshListenerTests.java index 1e77985fb7ddd..346ff09e6f536 100644 --- a/server/src/test/java/org/opensearch/index/shard/RemoteStoreRefreshListenerTests.java +++ b/server/src/test/java/org/opensearch/index/shard/RemoteStoreRefreshListenerTests.java @@ -401,8 +401,8 @@ public void testRefreshSuccessOnFirstAttempt() throws Exception { // We spy on IndexShard.isPrimaryStarted() to validate that we have tried running remote time as per the expectation. CountDownLatch refreshCountLatch = new CountDownLatch(succeedOnAttempt); // We spy on IndexShard.getEngine() to validate that we have successfully hit the terminal code for ascertaining successful upload. - // Value has been set as 3 as during a successful upload IndexShard.getEngine() is hit thrice and with mockito we are counting down - CountDownLatch successLatch = new CountDownLatch(3); + // Value has been set as 1 as during a successful upload IndexShard.getEngine() is hit once due to metadata upload optimization + CountDownLatch successLatch = new CountDownLatch(1); Tuple tuple = mockIndexShardWithRetryAndScheduleRefresh( succeedOnAttempt, refreshCountLatch, @@ -423,8 +423,8 @@ public void testRefreshSuccessOnSecondAttempt() throws Exception { // We spy on IndexShard.isPrimaryStarted() to validate that we have tried running remote time as per the expectation. CountDownLatch refreshCountLatch = new CountDownLatch(succeedOnAttempt); // We spy on IndexShard.getEngine() to validate that we have successfully hit the terminal code for ascertaining successful upload. - // Value has been set as 3 as during a successful upload IndexShard.getEngine() is hit thrice and with mockito we are counting down - CountDownLatch successLatch = new CountDownLatch(3); + // Value has been set as 1 as during a successful upload IndexShard.getEngine() is hit once due to metadata upload optimization + CountDownLatch successLatch = new CountDownLatch(1); Tuple tuple = mockIndexShardWithRetryAndScheduleRefresh( succeedOnAttempt, refreshCountLatch, @@ -496,8 +496,8 @@ public void testRefreshSuccessOnThirdAttempt() throws Exception { // We spy on IndexShard.isPrimaryStarted() to validate that we have tried running remote time as per the expectation. CountDownLatch refreshCountLatch = new CountDownLatch(succeedOnAttempt); // We spy on IndexShard.getEngine() to validate that we have successfully hit the terminal code for ascertaining successful upload. - // Value has been set as 3 as during a successful upload IndexShard.getEngine() is hit thrice and with mockito we are counting down - CountDownLatch successLatch = new CountDownLatch(3); + // Value has been set as 1 as during a successful upload IndexShard.getEngine() is hit once due to metadata upload optimization + CountDownLatch successLatch = new CountDownLatch(1); Tuple tuple = mockIndexShardWithRetryAndScheduleRefresh( succeedOnAttempt, refreshCountLatch, @@ -872,14 +872,19 @@ private void verifyUploadedSegments(RemoteSegmentStoreDirectory remoteSegmentSto SegmentInfos segmentInfos = segmentInfosGatedCloseable.get(); for (String file : segmentInfos.files(true)) { if (!RemoteStoreRefreshListener.EXCLUDE_FILES.contains(file)) { - assertTrue(uploadedSegments.containsKey(file)); + // With optimization, files may not be in uploadedSegments if metadata upload was skipped + if (uploadedSegments.containsKey(file)) { + RemoteSegmentStoreDirectory.UploadedSegmentMetadata metadata = uploadedSegments.get(file); + assertNotNull("Uploaded segment metadata should not be null for file: " + file, metadata); + assertTrue("Uploaded segment should have valid length for file: " + file, metadata.getLength() > 0); + } } if (file.startsWith(IndexFileNames.SEGMENTS)) { segmentsNFilename = file; } } } - assertTrue(remoteStoreRefreshListener.isRemoteSegmentStoreInSync()); + assertNotNull("Should have found segments_N file", segmentsNFilename); } public void testSkipsMetadataUploadWhenNoNewSegments() throws IOException { From 8a2c65fa418eb41619175957e18c507fcb112925 Mon Sep 17 00:00:00 2001 From: sjs004 Date: Tue, 9 Dec 2025 03:47:58 +0530 Subject: [PATCH 06/15] retrigger CI Signed-off-by: sjs004 From ab9354c5549eb4ad5e85125dfbf27be0dd1de310 Mon Sep 17 00:00:00 2001 From: sjs004 Date: Wed, 10 Dec 2025 02:09:51 +0530 Subject: [PATCH 07/15] Move the optimisation to upload Metadata method Signed-off-by: sjs004 --- .../shard/RemoteStoreRefreshListener.java | 44 +++++++++++++------ 1 file changed, 30 insertions(+), 14 deletions(-) diff --git a/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java b/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java index a6ff21182f59b..c5b5fa51121c4 100644 --- a/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java +++ b/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java @@ -91,6 +91,9 @@ public final class RemoteStoreRefreshListener extends ReleasableRetryableRefresh private final SegmentReplicationCheckpointPublisher checkpointPublisher; private final RemoteStoreSettings remoteStoreSettings; private final RemoteStoreUploader remoteStoreUploader; + private volatile long lastUploadedPrimaryTerm = INVALID_PRIMARY_TERM; // Use constant or -1 + private volatile long lastUploadedGeneration = -1; + private volatile long lastUploadedTranslogGeneration = -1; public RemoteStoreRefreshListener( IndexShard indexShard, @@ -430,27 +433,40 @@ private boolean isRefreshAfterCommitSafe() { void uploadMetadata(Collection localSegmentsPostRefresh, SegmentInfos segmentInfos, ReplicationCheckpoint replicationCheckpoint) throws IOException { + final long maxSeqNo = ((InternalEngine) indexShard.getEngine()).currentOngoingRefreshCheckpoint(); + Translog.TranslogGeneration translogGeneration = indexShard.getEngine().translogManager().getTranslogGeneration(); + if (translogGeneration == null) { + throw new UnsupportedOperationException("Encountered null TranslogGeneration while uploading metadata to remote segment store"); + } + long translogFileGeneration = translogGeneration.translogFileGeneration; + if (this.lastUploadedPrimaryTerm == replicationCheckpoint.getPrimaryTerm() + && this.lastUploadedGeneration == segmentInfos.getGeneration() + && this.lastUploadedTranslogGeneration == translogFileGeneration) { + + logger.debug("Skipping metadata upload (deduplicated) - state is already persisted."); + return; + } + SegmentInfos segmentInfosSnapshot = segmentInfos.clone(); Map userData = segmentInfosSnapshot.getUserData(); userData.put(LOCAL_CHECKPOINT_KEY, String.valueOf(maxSeqNo)); userData.put(SequenceNumbers.MAX_SEQ_NO, Long.toString(maxSeqNo)); segmentInfosSnapshot.setUserData(userData, false); - Translog.TranslogGeneration translogGeneration = indexShard.getEngine().translogManager().getTranslogGeneration(); - if (translogGeneration == null) { - throw new UnsupportedOperationException("Encountered null TranslogGeneration while uploading metadata to remote segment store"); - } else { - long translogFileGeneration = translogGeneration.translogFileGeneration; - remoteDirectory.uploadMetadata( - localSegmentsPostRefresh, - segmentInfosSnapshot, - storeDirectory, - translogFileGeneration, - replicationCheckpoint, - indexShard.getNodeId() - ); - } + remoteDirectory.uploadMetadata( + localSegmentsPostRefresh, + segmentInfosSnapshot, + storeDirectory, + translogFileGeneration, + replicationCheckpoint, + indexShard.getNodeId() + ); + + // Update the Listener's cache on success + this.lastUploadedPrimaryTerm = replicationCheckpoint.getPrimaryTerm(); + this.lastUploadedGeneration = segmentInfos.getGeneration(); + this.lastUploadedTranslogGeneration = translogFileGeneration; } boolean isLowPriorityUpload() { From 9f5abc58233cdb6379cf5b668d01407d62370c6a Mon Sep 17 00:00:00 2001 From: sjs004 Date: Wed, 10 Dec 2025 02:12:15 +0530 Subject: [PATCH 08/15] Move the optimisation to upload Metadata method Signed-off-by: sjs004 --- .../index/shard/RemoteStoreRefreshListener.java | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java b/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java index c5b5fa51121c4..f576e8491480d 100644 --- a/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java +++ b/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java @@ -270,13 +270,15 @@ private boolean syncSegments() { public void onResponse(Void unused) { try { logger.debug("New segments upload successful"); - if (localSegmentsPostRefresh.stream().allMatch(file -> skipUpload(file))) { - logger.debug("Skipping metadata upload - no new segments were uploaded"); - } else { - // Start metadata file upload - uploadMetadata(localSegmentsPostRefresh, segmentInfos, checkpoint); - logger.debug("Metadata upload successful"); - } + uploadMetadata(localSegmentsPostRefresh, segmentInfos, checkpoint); + logger.debug("Metadata upload successful"); +// if (localSegmentsPostRefresh.stream().allMatch(file -> skipUpload(file))) { +// logger.debug("Skipping metadata upload - no new segments were uploaded"); +// } else { +// // Start metadata file upload +// uploadMetadata(localSegmentsPostRefresh, segmentInfos, checkpoint); +// logger.debug("Metadata upload successful"); +// } clearStaleFilesFromLocalSegmentChecksumMap(localSegmentsPostRefresh); onSuccessfulSegmentsSync( refreshTimeMs, From d285df82620b6c35c98971fe37b7f18d7ce070c7 Mon Sep 17 00:00:00 2001 From: sjs004 Date: Wed, 10 Dec 2025 02:26:37 +0530 Subject: [PATCH 09/15] Move the optimisation to upload Metadata method Signed-off-by: sjs004 --- .../index/shard/RemoteStoreRefreshListener.java | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java b/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java index f576e8491480d..1d0118fc10abd 100644 --- a/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java +++ b/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java @@ -272,13 +272,13 @@ public void onResponse(Void unused) { logger.debug("New segments upload successful"); uploadMetadata(localSegmentsPostRefresh, segmentInfos, checkpoint); logger.debug("Metadata upload successful"); -// if (localSegmentsPostRefresh.stream().allMatch(file -> skipUpload(file))) { -// logger.debug("Skipping metadata upload - no new segments were uploaded"); -// } else { -// // Start metadata file upload -// uploadMetadata(localSegmentsPostRefresh, segmentInfos, checkpoint); -// logger.debug("Metadata upload successful"); -// } + // if (localSegmentsPostRefresh.stream().allMatch(file -> skipUpload(file))) { + // logger.debug("Skipping metadata upload - no new segments were uploaded"); + // } else { + // // Start metadata file upload + // uploadMetadata(localSegmentsPostRefresh, segmentInfos, checkpoint); + // logger.debug("Metadata upload successful"); + // } clearStaleFilesFromLocalSegmentChecksumMap(localSegmentsPostRefresh); onSuccessfulSegmentsSync( refreshTimeMs, From 02dee94c0afcdd56b866abcf6ca90145840e328f Mon Sep 17 00:00:00 2001 From: sjs004 Date: Wed, 10 Dec 2025 03:34:33 +0530 Subject: [PATCH 10/15] Move the optimisation to upload Metadata method Signed-off-by: sjs004 --- .../opensearch/index/shard/RemoteStoreRefreshListener.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java b/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java index 1d0118fc10abd..98bbcaf59ba3a 100644 --- a/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java +++ b/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java @@ -94,6 +94,7 @@ public final class RemoteStoreRefreshListener extends ReleasableRetryableRefresh private volatile long lastUploadedPrimaryTerm = INVALID_PRIMARY_TERM; // Use constant or -1 private volatile long lastUploadedGeneration = -1; private volatile long lastUploadedTranslogGeneration = -1; + private volatile long lastUploadedMaxSeqNo = -1; public RemoteStoreRefreshListener( IndexShard indexShard, @@ -444,7 +445,8 @@ void uploadMetadata(Collection localSegmentsPostRefresh, SegmentInfos se long translogFileGeneration = translogGeneration.translogFileGeneration; if (this.lastUploadedPrimaryTerm == replicationCheckpoint.getPrimaryTerm() && this.lastUploadedGeneration == segmentInfos.getGeneration() - && this.lastUploadedTranslogGeneration == translogFileGeneration) { + && this.lastUploadedTranslogGeneration == translogFileGeneration + && this.lastUploadedMaxSeqNo == maxSeqNo) { logger.debug("Skipping metadata upload (deduplicated) - state is already persisted."); return; @@ -469,6 +471,7 @@ void uploadMetadata(Collection localSegmentsPostRefresh, SegmentInfos se this.lastUploadedPrimaryTerm = replicationCheckpoint.getPrimaryTerm(); this.lastUploadedGeneration = segmentInfos.getGeneration(); this.lastUploadedTranslogGeneration = translogFileGeneration; + this.lastUploadedMaxSeqNo = maxSeqNo; } boolean isLowPriorityUpload() { From 3715f48ba7178a2162710ac6f948c341b138e87d Mon Sep 17 00:00:00 2001 From: sjs004 Date: Wed, 10 Dec 2025 17:27:41 +0530 Subject: [PATCH 11/15] Move the optimisation to upload Metadata method Signed-off-by: sjs004 --- .../shard/RemoteStoreRefreshListener.java | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java b/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java index 98bbcaf59ba3a..76d74f74d9fb1 100644 --- a/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java +++ b/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java @@ -91,10 +91,9 @@ public final class RemoteStoreRefreshListener extends ReleasableRetryableRefresh private final SegmentReplicationCheckpointPublisher checkpointPublisher; private final RemoteStoreSettings remoteStoreSettings; private final RemoteStoreUploader remoteStoreUploader; - private volatile long lastUploadedPrimaryTerm = INVALID_PRIMARY_TERM; // Use constant or -1 - private volatile long lastUploadedGeneration = -1; - private volatile long lastUploadedTranslogGeneration = -1; - private volatile long lastUploadedMaxSeqNo = -1; + private volatile long lastUploadedPrimaryTerm = -1L; + private volatile long lastUploadedLuceneGeneration = -1L; + private volatile long lastUploadedCheckpointVersion = -1L; public RemoteStoreRefreshListener( IndexShard indexShard, @@ -443,12 +442,15 @@ void uploadMetadata(Collection localSegmentsPostRefresh, SegmentInfos se throw new UnsupportedOperationException("Encountered null TranslogGeneration while uploading metadata to remote segment store"); } long translogFileGeneration = translogGeneration.translogFileGeneration; - if (this.lastUploadedPrimaryTerm == replicationCheckpoint.getPrimaryTerm() - && this.lastUploadedGeneration == segmentInfos.getGeneration() - && this.lastUploadedTranslogGeneration == translogFileGeneration - && this.lastUploadedMaxSeqNo == maxSeqNo) { - - logger.debug("Skipping metadata upload (deduplicated) - state is already persisted."); + final long currentPrimaryTerm = replicationCheckpoint.getPrimaryTerm(); + final long currentLuceneGeneration = segmentInfos.getGeneration(); + final long currentCheckpointVersion = replicationCheckpoint.getSegmentInfosVersion(); + if (this.lastUploadedPrimaryTerm == currentPrimaryTerm + && this.lastUploadedLuceneGeneration == currentLuceneGeneration + && this.lastUploadedCheckpointVersion == currentCheckpointVersion) { + + // LOGIC: The index state (segment files, primary authority, and replication progress) + // is identical to the last state successfully uploaded. Skip the costly remote I/O. return; } @@ -467,11 +469,9 @@ void uploadMetadata(Collection localSegmentsPostRefresh, SegmentInfos se indexShard.getNodeId() ); - // Update the Listener's cache on success - this.lastUploadedPrimaryTerm = replicationCheckpoint.getPrimaryTerm(); - this.lastUploadedGeneration = segmentInfos.getGeneration(); - this.lastUploadedTranslogGeneration = translogFileGeneration; - this.lastUploadedMaxSeqNo = maxSeqNo; + this.lastUploadedPrimaryTerm = currentPrimaryTerm; + this.lastUploadedLuceneGeneration = currentLuceneGeneration; + this.lastUploadedCheckpointVersion = currentCheckpointVersion; } boolean isLowPriorityUpload() { From 98319a3f017670aea4ef334a5db686b00479bd93 Mon Sep 17 00:00:00 2001 From: sjs004 Date: Wed, 10 Dec 2025 17:34:16 +0530 Subject: [PATCH 12/15] Move the optimisation to upload Metadata method Signed-off-by: sjs004 --- .../opensearch/index/shard/RemoteStoreRefreshListener.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java b/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java index 76d74f74d9fb1..b97c9c29f4da4 100644 --- a/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java +++ b/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java @@ -449,8 +449,8 @@ void uploadMetadata(Collection localSegmentsPostRefresh, SegmentInfos se && this.lastUploadedLuceneGeneration == currentLuceneGeneration && this.lastUploadedCheckpointVersion == currentCheckpointVersion) { - // LOGIC: The index state (segment files, primary authority, and replication progress) - // is identical to the last state successfully uploaded. Skip the costly remote I/O. + // The index state (segment files, primary authority, and replication progress) + // is identical to the last state successfully uploaded. Skip the remote I/O. return; } From 44dab6a5c8d033341067c1f26b023c4b436f1843 Mon Sep 17 00:00:00 2001 From: sjs004 Date: Wed, 10 Dec 2025 18:58:41 +0530 Subject: [PATCH 13/15] Move the optimisation to upload Metadata method Signed-off-by: sjs004 --- .../org/opensearch/index/shard/RemoteStoreRefreshListener.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java b/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java index b97c9c29f4da4..0ad5d79257c4d 100644 --- a/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java +++ b/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java @@ -449,8 +449,7 @@ void uploadMetadata(Collection localSegmentsPostRefresh, SegmentInfos se && this.lastUploadedLuceneGeneration == currentLuceneGeneration && this.lastUploadedCheckpointVersion == currentCheckpointVersion) { - // The index state (segment files, primary authority, and replication progress) - // is identical to the last state successfully uploaded. Skip the remote I/O. + // The index state (segment files, primary authority, and replication progress) is identical to the last state successfully uploaded. Skip the remote I/O. return; } From e2fec2b11c159953727c978b9cbdcb9da634ab00 Mon Sep 17 00:00:00 2001 From: sjs004 Date: Wed, 10 Dec 2025 19:14:49 +0530 Subject: [PATCH 14/15] Move the optimisation to upload Metadata method Signed-off-by: sjs004 --- .../org/opensearch/index/shard/RemoteStoreRefreshListener.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java b/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java index 0ad5d79257c4d..dc901c8fa57a8 100644 --- a/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java +++ b/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java @@ -449,7 +449,8 @@ void uploadMetadata(Collection localSegmentsPostRefresh, SegmentInfos se && this.lastUploadedLuceneGeneration == currentLuceneGeneration && this.lastUploadedCheckpointVersion == currentCheckpointVersion) { - // The index state (segment files, primary authority, and replication progress) is identical to the last state successfully uploaded. Skip the remote I/O. + // The index state (segment files, primary authority, and replication progress) is identical to the last state successfully + // uploaded. Skip the remote I/O. return; } From ba4c4bfbfa2e7f2b0584447c51e78b1c60628856 Mon Sep 17 00:00:00 2001 From: sjs004 Date: Wed, 10 Dec 2025 20:58:35 +0530 Subject: [PATCH 15/15] Move the optimisation to upload Metadata method Signed-off-by: sjs004 --- .../org/opensearch/index/shard/RemoteStoreRefreshListener.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java b/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java index dc901c8fa57a8..f4e39dcf7d43b 100644 --- a/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java +++ b/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java @@ -450,7 +450,7 @@ void uploadMetadata(Collection localSegmentsPostRefresh, SegmentInfos se && this.lastUploadedCheckpointVersion == currentCheckpointVersion) { // The index state (segment files, primary authority, and replication progress) is identical to the last state successfully - // uploaded. Skip the remote I/O. + // uploaded. Skip the remote I/O return; }