From 42809619de359406436e4f1a23ac95f61903171a Mon Sep 17 00:00:00 2001 From: browndav-msft Date: Mon, 18 May 2026 14:08:04 -0400 Subject: [PATCH 1/5] update service versions (#49174) * add 103 version * add 104 version --- .../com/azure/storage/blob/BlobServiceVersion.java | 14 ++++++++++++-- .../azure-storage-common/ci.system.properties | 4 ++-- .../storage/common/implementation/Constants.java | 4 ++-- .../file/datalake/DataLakeServiceVersion.java | 14 ++++++++++++-- .../implementation/util/TransformUtils.java | 6 ++++++ .../storage/file/share/ShareServiceVersion.java | 14 ++++++++++++-- .../azure/storage/queue/QueueServiceVersion.java | 14 ++++++++++++-- 7 files changed, 58 insertions(+), 12 deletions(-) diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/BlobServiceVersion.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/BlobServiceVersion.java index 75fb74a59a5d..d7a60722a3ec 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/BlobServiceVersion.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/BlobServiceVersion.java @@ -162,7 +162,17 @@ public enum BlobServiceVersion implements ServiceVersion { /** * Service version {@code 2026-06-06}. */ - V2026_06_06("2026-06-06"); + V2026_06_06("2026-06-06"), + + /** + * Service version {@code 2026-10-06}. + */ + V2026_10_06("2026-10-06"), + + /** + * Service version {@code 2026-12-06}. + */ + V2026_12_06("2026-12-06"); private final String version; @@ -184,6 +194,6 @@ public String getVersion() { * @return the latest {@link BlobServiceVersion} */ public static BlobServiceVersion getLatest() { - return V2026_06_06; + return V2026_12_06; } } diff --git a/sdk/storage/azure-storage-common/ci.system.properties b/sdk/storage/azure-storage-common/ci.system.properties index 1d1c46cd13b4..dc3baf90585b 100644 --- a/sdk/storage/azure-storage-common/ci.system.properties +++ b/sdk/storage/azure-storage-common/ci.system.properties @@ -1,2 +1,2 @@ -AZURE_LIVE_TEST_SERVICE_VERSION=V2026_06_06 -AZURE_STORAGE_SAS_SERVICE_VERSION=2026-06-06 +AZURE_LIVE_TEST_SERVICE_VERSION=V2026_12_06 +AZURE_STORAGE_SAS_SERVICE_VERSION=2026-12-06 diff --git a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/Constants.java b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/Constants.java index e3b88b661134..8d1d66b59126 100644 --- a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/Constants.java +++ b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/Constants.java @@ -88,7 +88,7 @@ public final class Constants { public static final String PROPERTY_AZURE_STORAGE_SAS_SERVICE_VERSION = "AZURE_STORAGE_SAS_SERVICE_VERSION"; public static final String SAS_SERVICE_VERSION - = Configuration.getGlobalConfiguration().get(PROPERTY_AZURE_STORAGE_SAS_SERVICE_VERSION, "2026-06-06"); + = Configuration.getGlobalConfiguration().get(PROPERTY_AZURE_STORAGE_SAS_SERVICE_VERSION, "2026-12-06"); public static final String ADJUSTED_BLOB_LENGTH_KEY = "adjustedBlobLength"; @@ -220,7 +220,7 @@ public static final class HeaderConstants { * @deprecated For SAS Service Version use {@link Constants#SAS_SERVICE_VERSION}. */ @Deprecated - public static final String TARGET_STORAGE_VERSION = "2026-06-06"; + public static final String TARGET_STORAGE_VERSION = "2026-12-06"; /** * Error code returned from the service. diff --git a/sdk/storage/azure-storage-file-datalake/src/main/java/com/azure/storage/file/datalake/DataLakeServiceVersion.java b/sdk/storage/azure-storage-file-datalake/src/main/java/com/azure/storage/file/datalake/DataLakeServiceVersion.java index 3e3ec7595cee..70b574b062aa 100644 --- a/sdk/storage/azure-storage-file-datalake/src/main/java/com/azure/storage/file/datalake/DataLakeServiceVersion.java +++ b/sdk/storage/azure-storage-file-datalake/src/main/java/com/azure/storage/file/datalake/DataLakeServiceVersion.java @@ -162,7 +162,17 @@ public enum DataLakeServiceVersion implements ServiceVersion { /** * Service version {@code 2026-06-06}. */ - V2026_06_06("2026-06-06"); + V2026_06_06("2026-06-06"), + + /** + * Service version {@code 2026-10-06}. + */ + V2026_10_06("2026-10-06"), + + /** + * Service version {@code 2026-12-06}. + */ + V2026_12_06("2026-12-06"); private final String version; @@ -184,6 +194,6 @@ public String getVersion() { * @return the latest {@link DataLakeServiceVersion} */ public static DataLakeServiceVersion getLatest() { - return V2026_06_06; + return V2026_12_06; } } diff --git a/sdk/storage/azure-storage-file-datalake/src/main/java/com/azure/storage/file/datalake/implementation/util/TransformUtils.java b/sdk/storage/azure-storage-file-datalake/src/main/java/com/azure/storage/file/datalake/implementation/util/TransformUtils.java index fe07d636f95c..c53079d81b0d 100644 --- a/sdk/storage/azure-storage-file-datalake/src/main/java/com/azure/storage/file/datalake/implementation/util/TransformUtils.java +++ b/sdk/storage/azure-storage-file-datalake/src/main/java/com/azure/storage/file/datalake/implementation/util/TransformUtils.java @@ -107,6 +107,12 @@ public static BlobServiceVersion toBlobServiceVersion(DataLakeServiceVersion ver case V2026_06_06: return BlobServiceVersion.V2026_06_06; + case V2026_10_06: + return BlobServiceVersion.V2026_10_06; + + case V2026_12_06: + return BlobServiceVersion.V2026_12_06; + default: return null; } diff --git a/sdk/storage/azure-storage-file-share/src/main/java/com/azure/storage/file/share/ShareServiceVersion.java b/sdk/storage/azure-storage-file-share/src/main/java/com/azure/storage/file/share/ShareServiceVersion.java index 20d1e6de7cea..5c5aa77fd24c 100644 --- a/sdk/storage/azure-storage-file-share/src/main/java/com/azure/storage/file/share/ShareServiceVersion.java +++ b/sdk/storage/azure-storage-file-share/src/main/java/com/azure/storage/file/share/ShareServiceVersion.java @@ -162,7 +162,17 @@ public enum ShareServiceVersion implements ServiceVersion { /** * Service version {@code 2026-06-06}. */ - V2026_06_06("2026-06-06"); + V2026_06_06("2026-06-06"), + + /** + * Service version {@code 2026-10-06}. + */ + V2026_10_06("2026-10-06"), + + /** + * Service version {@code 2026-12-06}. + */ + V2026_12_06("2026-12-06"); private final String version; @@ -184,6 +194,6 @@ public String getVersion() { * @return the latest {@link ShareServiceVersion} */ public static ShareServiceVersion getLatest() { - return V2026_06_06; + return V2026_12_06; } } diff --git a/sdk/storage/azure-storage-queue/src/main/java/com/azure/storage/queue/QueueServiceVersion.java b/sdk/storage/azure-storage-queue/src/main/java/com/azure/storage/queue/QueueServiceVersion.java index efd0cded7d73..246f3f4fd797 100644 --- a/sdk/storage/azure-storage-queue/src/main/java/com/azure/storage/queue/QueueServiceVersion.java +++ b/sdk/storage/azure-storage-queue/src/main/java/com/azure/storage/queue/QueueServiceVersion.java @@ -162,7 +162,17 @@ public enum QueueServiceVersion implements ServiceVersion { /** * Service version {@code 2026-06-06}. */ - V2026_06_06("2026-06-06"); + V2026_06_06("2026-06-06"), + + /** + * Service version {@code 2026-10-06}. + */ + V2026_10_06("2026-10-06"), + + /** + * Service version {@code 2026-12-06}. + */ + V2026_12_06("2026-12-06"); private final String version; @@ -184,6 +194,6 @@ public String getVersion() { * @return the latest {@link QueueServiceVersion} */ public static QueueServiceVersion getLatest() { - return V2026_06_06; + return V2026_12_06; } } From ce83bc7bf08b0cf118be593991f264fc751ef548 Mon Sep 17 00:00:00 2001 From: browndav-msft Date: Tue, 26 May 2026 20:36:15 -0400 Subject: [PATCH 2/5] Storage - STG104 Add Blob Access Tier to Get Blob Response (#49219) * generate swagger changes * addBlobDownloadHeaders wrapper support, add tests * add recordings * added missing file updates to BlobApiTests * remove added getProperties tests, add check tierChangeTime * remove unused imports * rewrite tests, add recording, add custom check for null for access-tier-inferred --- sdk/storage/azure-storage-blob/assets.json | 2 +- .../models/BlobsDownloadHeaders.java | 143 ++++++++++++++++++ .../blob/models/BlobDownloadHeaders.java | 91 +++++++++++ .../com/azure/storage/blob/BlobApiTests.java | 30 ++++ .../azure/storage/blob/BlobAsyncApiTests.java | 27 ++++ .../azure-storage-blob/swagger/README.md | 2 +- 6 files changed, 293 insertions(+), 2 deletions(-) diff --git a/sdk/storage/azure-storage-blob/assets.json b/sdk/storage/azure-storage-blob/assets.json index 8cad139f33ff..7f5e5b611d0c 100644 --- a/sdk/storage/azure-storage-blob/assets.json +++ b/sdk/storage/azure-storage-blob/assets.json @@ -2,5 +2,5 @@ "AssetsRepo": "Azure/azure-sdk-assets", "AssetsRepoPrefixPath": "java", "TagPrefix": "java/storage/azure-storage-blob", - "Tag": "java/storage/azure-storage-blob_47f4243e59" + "Tag": "java/storage/azure-storage-blob_1da9542ee2" } diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/models/BlobsDownloadHeaders.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/models/BlobsDownloadHeaders.java index 1d409a5a4cdd..7dd424fe666c 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/models/BlobsDownloadHeaders.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/models/BlobsDownloadHeaders.java @@ -295,6 +295,30 @@ public final class BlobsDownloadHeaders { @Generated private Long xMsStructuredContentLength; + /* + * The x-ms-access-tier property. + */ + @Generated + private String xMsAccessTier; + + /* + * The x-ms-access-tier-inferred property. + */ + @Generated + private Boolean xMsAccessTierInferred; + + /* + * The x-ms-access-tier-change-time property. + */ + @Generated + private DateTimeRfc1123 xMsAccessTierChangeTime; + + /* + * The x-ms-smart-access-tier property. + */ + @Generated + private String xMsSmartAccessTier; + /* * The x-ms-content-crc64 property. */ @@ -367,6 +391,16 @@ public final class BlobsDownloadHeaders { private static final HttpHeaderName X_MS_STRUCTURED_CONTENT_LENGTH = HttpHeaderName.fromString("x-ms-structured-content-length"); + private static final HttpHeaderName X_MS_ACCESS_TIER = HttpHeaderName.fromString("x-ms-access-tier"); + + private static final HttpHeaderName X_MS_ACCESS_TIER_INFERRED + = HttpHeaderName.fromString("x-ms-access-tier-inferred"); + + private static final HttpHeaderName X_MS_ACCESS_TIER_CHANGE_TIME + = HttpHeaderName.fromString("x-ms-access-tier-change-time"); + + private static final HttpHeaderName X_MS_SMART_ACCESS_TIER = HttpHeaderName.fromString("x-ms-smart-access-tier"); + private static final HttpHeaderName X_MS_CONTENT_CRC64 = HttpHeaderName.fromString("x-ms-content-crc64"); // HttpHeaders containing the raw property values. @@ -529,6 +563,20 @@ public BlobsDownloadHeaders(HttpHeaders rawHeaders) { } else { this.xMsStructuredContentLength = null; } + this.xMsAccessTier = rawHeaders.getValue(X_MS_ACCESS_TIER); + String xMsAccessTierInferred = rawHeaders.getValue(X_MS_ACCESS_TIER_INFERRED); + if (xMsAccessTierInferred != null) { + this.xMsAccessTierInferred = Boolean.parseBoolean(xMsAccessTierInferred); + } else { + this.xMsAccessTierInferred = null; + } + String xMsAccessTierChangeTime = rawHeaders.getValue(X_MS_ACCESS_TIER_CHANGE_TIME); + if (xMsAccessTierChangeTime != null) { + this.xMsAccessTierChangeTime = new DateTimeRfc1123(xMsAccessTierChangeTime); + } else { + this.xMsAccessTierChangeTime = null; + } + this.xMsSmartAccessTier = rawHeaders.getValue(X_MS_SMART_ACCESS_TIER); String xMsContentCrc64 = rawHeaders.getValue(X_MS_CONTENT_CRC64); if (xMsContentCrc64 != null) { this.xMsContentCrc64 = Base64.getDecoder().decode(xMsContentCrc64); @@ -1584,6 +1632,101 @@ public BlobsDownloadHeaders setXMsStructuredContentLength(Long xMsStructuredCont return this; } + /** + * Get the xMsAccessTier property: The x-ms-access-tier property. + * + * @return the xMsAccessTier value. + */ + @Generated + public String getXMsAccessTier() { + return this.xMsAccessTier; + } + + /** + * Set the xMsAccessTier property: The x-ms-access-tier property. + * + * @param xMsAccessTier the xMsAccessTier value to set. + * @return the BlobsDownloadHeaders object itself. + */ + @Generated + public BlobsDownloadHeaders setXMsAccessTier(String xMsAccessTier) { + this.xMsAccessTier = xMsAccessTier; + return this; + } + + /** + * Get the xMsAccessTierInferred property: The x-ms-access-tier-inferred property. + * + * @return the xMsAccessTierInferred value. + */ + @Generated + public Boolean isXMsAccessTierInferred() { + return this.xMsAccessTierInferred; + } + + /** + * Set the xMsAccessTierInferred property: The x-ms-access-tier-inferred property. + * + * @param xMsAccessTierInferred the xMsAccessTierInferred value to set. + * @return the BlobsDownloadHeaders object itself. + */ + @Generated + public BlobsDownloadHeaders setXMsAccessTierInferred(Boolean xMsAccessTierInferred) { + this.xMsAccessTierInferred = xMsAccessTierInferred; + return this; + } + + /** + * Get the xMsAccessTierChangeTime property: The x-ms-access-tier-change-time property. + * + * @return the xMsAccessTierChangeTime value. + */ + @Generated + public OffsetDateTime getXMsAccessTierChangeTime() { + if (this.xMsAccessTierChangeTime == null) { + return null; + } + return this.xMsAccessTierChangeTime.getDateTime(); + } + + /** + * Set the xMsAccessTierChangeTime property: The x-ms-access-tier-change-time property. + * + * @param xMsAccessTierChangeTime the xMsAccessTierChangeTime value to set. + * @return the BlobsDownloadHeaders object itself. + */ + @Generated + public BlobsDownloadHeaders setXMsAccessTierChangeTime(OffsetDateTime xMsAccessTierChangeTime) { + if (xMsAccessTierChangeTime == null) { + this.xMsAccessTierChangeTime = null; + } else { + this.xMsAccessTierChangeTime = new DateTimeRfc1123(xMsAccessTierChangeTime); + } + return this; + } + + /** + * Get the xMsSmartAccessTier property: The x-ms-smart-access-tier property. + * + * @return the xMsSmartAccessTier value. + */ + @Generated + public String getXMsSmartAccessTier() { + return this.xMsSmartAccessTier; + } + + /** + * Set the xMsSmartAccessTier property: The x-ms-smart-access-tier property. + * + * @param xMsSmartAccessTier the xMsSmartAccessTier value to set. + * @return the BlobsDownloadHeaders object itself. + */ + @Generated + public BlobsDownloadHeaders setXMsSmartAccessTier(String xMsSmartAccessTier) { + this.xMsSmartAccessTier = xMsSmartAccessTier; + return this; + } + /** * Get the xMsContentCrc64 property: The x-ms-content-crc64 property. * diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/models/BlobDownloadHeaders.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/models/BlobDownloadHeaders.java index dfd93a535fc6..6a372f5b2c74 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/models/BlobDownloadHeaders.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/models/BlobDownloadHeaders.java @@ -825,6 +825,97 @@ public BlobDownloadHeaders setEncryptionScope(String encryptionScope) { return this; } + /** + * Gets the access tier of the blob. + * + * @return the access tier of the blob. This is only set for Page blobs on a premium storage account or for Block + * blobs on blob storage or general purpose V2 account. + */ + public AccessTier getAccessTier() { + String accessTier = internalHeaders.getXMsAccessTier(); + return accessTier == null ? null : AccessTier.fromString(accessTier); + } + + /** + * Sets the access tier of the blob. + * + * @param accessTier the access tier of the blob. + * @return the BlobDownloadHeaders object itself. + */ + public BlobDownloadHeaders setAccessTier(AccessTier accessTier) { + internalHeaders.setXMsAccessTier(accessTier == null ? null : accessTier.toString()); + return this; + } + + /** + * Gets the status of the tier being inferred for the blob. + * + * @return the status of the tier being inferred for the blob. This is only set for Page blobs on a premium storage + * account or for Block blobs on blob storage or general purpose V2 account. + */ + public Boolean isAccessTierInferred() { + return Boolean.TRUE.equals(internalHeaders.isXMsAccessTierInferred()); + } + + /** + * Sets the status of the tier being inferred for the blob. + * + * @param accessTierInferred the status of the tier being inferred for the blob. + * @return the BlobDownloadHeaders object itself. + */ + public BlobDownloadHeaders setAccessTierInferred(Boolean accessTierInferred) { + internalHeaders.setXMsAccessTierInferred(accessTierInferred); + return this; + } + + /** + * Gets the time when the access tier for the blob was last changed. + * + * @return the time when the access tier for the blob was last changed. + */ + public OffsetDateTime getAccessTierChangeTime() { + return internalHeaders.getXMsAccessTierChangeTime(); + } + + /** + * Sets the time when the access tier for the blob was last changed. + * + * @param accessTierChangeTime the time when the access tier for the blob was last changed. + * @return the BlobDownloadHeaders object itself. + */ + public BlobDownloadHeaders setAccessTierChangeTime(OffsetDateTime accessTierChangeTime) { + internalHeaders.setXMsAccessTierChangeTime(accessTierChangeTime); + return this; + } + + /** + * Gets the underlying access tier of the blob when its access tier is {@link AccessTier#SMART}. + *

+ * This value is only populated when {@link #getAccessTier()} returns {@link AccessTier#SMART}. In that case, it + * represents the concrete access tier (for example {@link AccessTier#HOT} or {@link AccessTier#COOL}) that the + * service has selected for the blob. For all other access tiers, this property is {@code null} and should be + * ignored. + * + * @return the underlying access tier chosen by the service when the blob's access tier is {@link AccessTier#SMART}, + * or {@code null} if the blob is not using the smart access tier. + */ + public AccessTier getSmartAccessTier() { + String smartAccessTier = internalHeaders.getXMsSmartAccessTier(); + return smartAccessTier == null ? null : AccessTier.fromString(smartAccessTier); + } + + /** + * Sets the underlying access tier of the blob when its access tier is {@link AccessTier#SMART}. + * + * @param smartAccessTier the underlying access tier chosen by the service when the blob's access tier is + * {@link AccessTier#SMART}. + * @return the BlobDownloadHeaders object itself. + */ + public BlobDownloadHeaders setSmartAccessTier(AccessTier smartAccessTier) { + internalHeaders.setXMsSmartAccessTier(smartAccessTier == null ? null : smartAccessTier.toString()); + return this; + } + /** * Get the blobContentMD5 property: If the blob has a MD5 hash, and if request contains range header (Range or * x-ms-range), this response header is returned with the value of the whole blob's MD5 value. This value may or may diff --git a/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobApiTests.java b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobApiTests.java index 50a9eb63ef21..71c474ba295c 100644 --- a/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobApiTests.java +++ b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobApiTests.java @@ -543,6 +543,36 @@ public void downloadAllNullBinaryData() { // headers.getLastAccessedTime() /* TODO (gapra): re-enable when last access time enabled. */ } + @RequiredServiceVersion(clazz = BlobServiceVersion.class, min = "2026-10-06") + @Test + public void downloadSmartAccessTierHeaders() { + ByteArrayOutputStream stream = new ByteArrayOutputStream(); + bc.setAccessTier(AccessTier.SMART); + + BlobDownloadResponse response = bc.downloadStreamWithResponse(stream, null, null, null, false, null, null); + ByteBuffer body = ByteBuffer.wrap(stream.toByteArray()); + + assertEquals(DATA.getDefaultData(), body); + assertSmartAccessTierHeaders(response.getDeserializedHeaders()); + } + + @RequiredServiceVersion(clazz = BlobServiceVersion.class, min = "2026-10-06") + @Test + public void downloadContentSmartAccessTierHeaders() { + bc.setAccessTier(AccessTier.SMART); + BlobDownloadContentResponse response = bc.downloadContentWithResponse(null, null, null, null); + + TestUtils.assertArraysEqual(DATA.getDefaultBytes(), response.getValue().toBytes()); + assertSmartAccessTierHeaders(response.getDeserializedHeaders()); + } + + private static void assertSmartAccessTierHeaders(BlobDownloadHeaders headers) { + assertEquals(AccessTier.SMART, headers.getAccessTier()); + assertNotNull(headers.getSmartAccessTier()); + assertFalse(headers.isAccessTierInferred()); + assertNotEquals(OffsetDateTime.now(), headers.getAccessTierChangeTime()); + } + @Test public void downloadEmptyFile() { AppendBlobClient bc = cc.getBlobClient("emptyAppendBlob").getAppendBlobClient(); diff --git a/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobAsyncApiTests.java b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobAsyncApiTests.java index 049e4254e92a..ea01df338d18 100644 --- a/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobAsyncApiTests.java +++ b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobAsyncApiTests.java @@ -382,6 +382,33 @@ public void downloadAllNullBinaryData() { .verifyComplete(); } + @RequiredServiceVersion(clazz = BlobServiceVersion.class, min = "2026-10-06") + @Test + public void downloadSmartAccessTierHeaders() { + Mono response = bc.setAccessTier(AccessTier.SMART) + .then(bc.downloadStreamWithResponse(null, null, null, false)) + .flatMap(r -> { + assertSmartAccessTierHeaders(r.getDeserializedHeaders()); + return FluxUtil.collectBytesInByteBufferStream(r.getValue()); + }) + .flatMap(r -> { + TestUtils.assertArraysEqual(DATA.getDefaultBytes(), r); + return bc.downloadContentWithResponse(null, null); + }); + + StepVerifier.create(response).assertNext(r -> { + assertSmartAccessTierHeaders(r.getDeserializedHeaders()); + TestUtils.assertArraysEqual(DATA.getDefaultBytes(), r.getValue().toBytes()); + }).verifyComplete(); + } + + private static void assertSmartAccessTierHeaders(BlobDownloadHeaders headers) { + assertEquals(AccessTier.SMART, headers.getAccessTier()); + assertNotNull(headers.getSmartAccessTier()); + assertFalse(headers.isAccessTierInferred()); + assertNotEquals(OffsetDateTime.now(), headers.getAccessTierChangeTime()); + } + @Test public void downloadEmptyFile() { AppendBlobAsyncClient bc = ccAsync.getBlobAsyncClient("emptyAppendBlob").getAppendBlobAsyncClient(); diff --git a/sdk/storage/azure-storage-blob/swagger/README.md b/sdk/storage/azure-storage-blob/swagger/README.md index 292d2f7c231d..98afe0c616dc 100644 --- a/sdk/storage/azure-storage-blob/swagger/README.md +++ b/sdk/storage/azure-storage-blob/swagger/README.md @@ -16,7 +16,7 @@ autorest ### Code generation settings ``` yaml use: '@autorest/java@4.1.63' -input-file: https://raw.githubusercontent.com/Azure/azure-rest-api-specs/15d7f54a5389d5906ffb4e56bb2f38fe5525c0d3/specification/storage/data-plane/Microsoft.BlobStorage/stable/2026-06-06/blob.json +input-file: https://raw.githubusercontent.com/seanmcc-msft/azure-rest-api-specs/eb29a830edf5db50758e7d044160c7f18077f7f7/specification/storage/data-plane/Microsoft.BlobStorage/stable/2026-10-06/blob.json java: true output-folder: ../ namespace: com.azure.storage.blob From b3eaac335f72c61e90d32e38fbffd2d4498a80cf Mon Sep 17 00:00:00 2001 From: gunjansingh-msft Date: Tue, 23 Jun 2026 12:37:55 +0530 Subject: [PATCH 3/5] Storage - STG104 Container Change Feed (#49307) * container feed changes * code cleanup * code cleanup * code cleanup * addressing review comments --------- Co-authored-by: Local Merge --- .../InternalBlobChangefeedEventData.java | 51 +- .../models/BlobChangefeedEventData.java | 29 ++ .../models/BlobChangefeedEventType.java | 25 + .../storage/blob/changefeed/ChunkTests.java | 4 + .../changefeed/MockedChangefeedResources.java | 3 +- ...obChangefeedEventDeserializationTests.java | 463 ++++++++++++++++++ .../src/test/resources/EventSchemaV6.json | 84 ++++ .../src/test/resources/EventSchemaV7.json | 85 ++++ .../src/test/resources/EventSchemaV8.json | 86 ++++ 9 files changed, 824 insertions(+), 6 deletions(-) create mode 100644 sdk/storage/azure-storage-blob-changefeed/src/test/java/com/azure/storage/blob/changefeed/implementation/models/BlobChangefeedEventDeserializationTests.java create mode 100644 sdk/storage/azure-storage-blob-changefeed/src/test/resources/EventSchemaV6.json create mode 100644 sdk/storage/azure-storage-blob-changefeed/src/test/resources/EventSchemaV7.json create mode 100644 sdk/storage/azure-storage-blob-changefeed/src/test/resources/EventSchemaV8.json diff --git a/sdk/storage/azure-storage-blob-changefeed/src/main/java/com/azure/storage/blob/changefeed/implementation/models/InternalBlobChangefeedEventData.java b/sdk/storage/azure-storage-blob-changefeed/src/main/java/com/azure/storage/blob/changefeed/implementation/models/InternalBlobChangefeedEventData.java index b29c1cf0bea2..2e92d87a1a6d 100644 --- a/sdk/storage/azure-storage-blob-changefeed/src/main/java/com/azure/storage/blob/changefeed/implementation/models/InternalBlobChangefeedEventData.java +++ b/sdk/storage/azure-storage-blob-changefeed/src/main/java/com/azure/storage/blob/changefeed/implementation/models/InternalBlobChangefeedEventData.java @@ -8,6 +8,7 @@ import com.azure.storage.internal.avro.implementation.AvroConstants; import com.azure.storage.internal.avro.implementation.schema.AvroSchema; +import java.time.OffsetDateTime; import java.util.Map; import java.util.Objects; @@ -29,6 +30,9 @@ public class InternalBlobChangefeedEventData implements BlobChangefeedEventData private final String blobUrl; private final boolean recursive; private final String sequencer; + private final OffsetDateTime creationTime; + private final OffsetDateTime lastAccessTime; + private final String restoredContainerVersion; /** * Constructs a {@link InternalBlobChangefeedEventData}. @@ -46,10 +50,14 @@ public class InternalBlobChangefeedEventData implements BlobChangefeedEventData * @param blobUrl The blob url. * @param recursive Whether this operation was recursive. * @param sequencer The sequencer. + * @param creationTime The blob creation time. Schema V6. + * @param lastAccessTime The last access time. Schema V7. + * @param restoredContainerVersion The restored container version. Schema V8. */ public InternalBlobChangefeedEventData(String api, String clientRequestId, String requestId, String eTag, String contentType, Long contentLength, BlobType blobType, Long contentOffset, String destinationUrl, - String sourceUrl, String blobUrl, boolean recursive, String sequencer) { + String sourceUrl, String blobUrl, boolean recursive, String sequencer, OffsetDateTime creationTime, + OffsetDateTime lastAccessTime, String restoredContainerVersion) { this.api = api; this.clientRequestId = clientRequestId; this.requestId = requestId; @@ -63,6 +71,9 @@ public InternalBlobChangefeedEventData(String api, String clientRequestId, Strin this.blobUrl = blobUrl; this.recursive = recursive; this.sequencer = sequencer; + this.creationTime = creationTime; + this.lastAccessTime = lastAccessTime; + this.restoredContainerVersion = restoredContainerVersion; } static InternalBlobChangefeedEventData fromRecord(Object d) { @@ -86,6 +97,9 @@ static InternalBlobChangefeedEventData fromRecord(Object d) { Object blobUrl = data.get("url"); Object recursive = data.get("recursive"); Object sequencer = data.get("sequencer"); + Object createTime = data.get("createTime"); + Object lastAccessTime = data.get("lastAccessTime"); + Object restoredContainerVersion = data.get("restoredContainerVersion"); return new InternalBlobChangefeedEventData(ChangefeedTypeValidator.nullOr("api", api, String.class), ChangefeedTypeValidator.nullOr("clientRequestId", clientRequestId, String.class), @@ -101,7 +115,14 @@ static InternalBlobChangefeedEventData fromRecord(Object d) { ChangefeedTypeValidator.nullOr("sourceUrl", sourceUrl, String.class), ChangefeedTypeValidator.nullOr("url", blobUrl, String.class), Boolean.TRUE.equals(ChangefeedTypeValidator.nullOr("recursive", recursive, Boolean.class)), - ChangefeedTypeValidator.nullOr("sequencer", sequencer, String.class)); + ChangefeedTypeValidator.nullOr("sequencer", sequencer, String.class), + ChangefeedTypeValidator.isNull(createTime) + ? null + : OffsetDateTime.parse(ChangefeedTypeValidator.nullOr("createTime", createTime, String.class)), + ChangefeedTypeValidator.isNull(lastAccessTime) + ? null + : OffsetDateTime.parse(ChangefeedTypeValidator.nullOr("lastAccessTime", lastAccessTime, String.class)), + ChangefeedTypeValidator.nullOr("restoredContainerVersion", restoredContainerVersion, String.class)); } @Override @@ -169,6 +190,21 @@ public String getSequencer() { return sequencer; } + @Override + public OffsetDateTime getCreationTime() { + return creationTime; + } + + @Override + public OffsetDateTime getLastAccessTime() { + return lastAccessTime; + } + + @Override + public String getRestoredContainerVersion() { + return restoredContainerVersion; + } + @Override public boolean equals(Object o) { if (this == o) { @@ -190,14 +226,17 @@ && getBlobType() == that.getBlobType() && Objects.equals(getSourceUrl(), that.getSourceUrl()) && Objects.equals(getBlobUrl(), that.getBlobUrl()) && Objects.equals(isRecursive(), that.isRecursive()) - && Objects.equals(getSequencer(), that.getSequencer()); + && Objects.equals(getSequencer(), that.getSequencer()) + && Objects.equals(getCreationTime(), that.getCreationTime()) + && Objects.equals(getLastAccessTime(), that.getLastAccessTime()) + && Objects.equals(getRestoredContainerVersion(), that.getRestoredContainerVersion()); } @Override public int hashCode() { return Objects.hash(getApi(), getClientRequestId(), getRequestId(), getETag(), getContentType(), getContentLength(), getBlobType(), getContentOffset(), getDestinationUrl(), getSourceUrl(), getBlobUrl(), - isRecursive(), getSequencer()); + isRecursive(), getSequencer(), getCreationTime(), getLastAccessTime(), getRestoredContainerVersion()); } @Override @@ -206,6 +245,8 @@ public String toString() { + ", requestId='" + requestId + '\'' + ", eTag='" + eTag + '\'' + ", contentType='" + contentType + '\'' + ", contentLength=" + contentLength + ", blobType=" + blobType + ", contentOffset=" + contentOffset + ", destinationUrl='" + destinationUrl + '\'' + ", sourceUrl='" + sourceUrl + '\'' + ", blobUrl='" - + blobUrl + '\'' + ", recursive=" + recursive + ", sequencer='" + sequencer + '\'' + '}'; + + blobUrl + '\'' + ", recursive=" + recursive + ", sequencer='" + sequencer + '\'' + ", creationTime=" + + creationTime + ", lastAccessTime=" + lastAccessTime + ", restoredContainerVersion='" + + restoredContainerVersion + '\'' + '}'; } } diff --git a/sdk/storage/azure-storage-blob-changefeed/src/main/java/com/azure/storage/blob/changefeed/models/BlobChangefeedEventData.java b/sdk/storage/azure-storage-blob-changefeed/src/main/java/com/azure/storage/blob/changefeed/models/BlobChangefeedEventData.java index dc07dfb90e72..b9f917cb8838 100644 --- a/sdk/storage/azure-storage-blob-changefeed/src/main/java/com/azure/storage/blob/changefeed/models/BlobChangefeedEventData.java +++ b/sdk/storage/azure-storage-blob-changefeed/src/main/java/com/azure/storage/blob/changefeed/models/BlobChangefeedEventData.java @@ -5,6 +5,8 @@ import com.azure.storage.blob.models.BlobType; +import java.time.OffsetDateTime; + /** * This class contains properties of a BlobChangefeedEventData. */ @@ -101,4 +103,31 @@ public interface BlobChangefeedEventData { */ String getSequencer(); + /** + * Gets the blob creation time. Present in schema V6 and later for AppendBlob data-updated events. + * + * @return The creation time, or null if not present. + */ + default OffsetDateTime getCreationTime() { + return null; + } + + /** + * Gets the last access time of the blob. Present in schema V7 and later. + * + * @return The last access time, or null if not present. + */ + default OffsetDateTime getLastAccessTime() { + return null; + } + + /** + * Gets the restored container version. Present in schema V8 and later for RestoreContainer events. + * + * @return The restored container version, or null if not present. + */ + default String getRestoredContainerVersion() { + return null; + } + } diff --git a/sdk/storage/azure-storage-blob-changefeed/src/main/java/com/azure/storage/blob/changefeed/models/BlobChangefeedEventType.java b/sdk/storage/azure-storage-blob-changefeed/src/main/java/com/azure/storage/blob/changefeed/models/BlobChangefeedEventType.java index 358b0cac36ea..38834b1a0920 100644 --- a/sdk/storage/azure-storage-blob-changefeed/src/main/java/com/azure/storage/blob/changefeed/models/BlobChangefeedEventType.java +++ b/sdk/storage/azure-storage-blob-changefeed/src/main/java/com/azure/storage/blob/changefeed/models/BlobChangefeedEventType.java @@ -22,6 +22,31 @@ public final class BlobChangefeedEventType extends ExpandableStringEnum getMockChangefeedEventDataRecord(BlobChangefe cfEventData.put("url", data.getBlobUrl()); cfEventData.put("sequencer", data.getSequencer()); cfEventData.put("recursive", data.isRecursive()); + cfEventData.put("createTime", data.getCreationTime() != null ? data.getCreationTime().toString() : null); + cfEventData.put("lastAccessTime", + data.getLastAccessTime() != null ? data.getLastAccessTime().toString() : null); + cfEventData.put("restoredContainerVersion", data.getRestoredContainerVersion()); return cfEventData; } } diff --git a/sdk/storage/azure-storage-blob-changefeed/src/test/java/com/azure/storage/blob/changefeed/MockedChangefeedResources.java b/sdk/storage/azure-storage-blob-changefeed/src/test/java/com/azure/storage/blob/changefeed/MockedChangefeedResources.java index 39480424de9a..c6de3d53e8c9 100644 --- a/sdk/storage/azure-storage-blob-changefeed/src/test/java/com/azure/storage/blob/changefeed/MockedChangefeedResources.java +++ b/sdk/storage/azure-storage-blob-changefeed/src/test/java/com/azure/storage/blob/changefeed/MockedChangefeedResources.java @@ -44,7 +44,8 @@ static BlobChangefeedEvent getMockBlobChangefeedEvent(int index) { static BlobChangefeedEventData getMockBlobChangefeedEventData() { return new InternalBlobChangefeedEventData("PutBlob", "clientRequestId", "requestId", "etag", "application/octet-stream", 100L, BlobType.BLOCK_BLOB, 0L, "destinationUrl", "sourceUrl", "", false, - "sequencer"); + "sequencer", OffsetDateTime.of(2020, 4, 4, 6, 30, 0, 0, ZoneOffset.UTC), + OffsetDateTime.of(2020, 4, 6, 6, 30, 0, 0, ZoneOffset.UTC), "restoredContainerVersion"); } private MockedChangefeedResources() { diff --git a/sdk/storage/azure-storage-blob-changefeed/src/test/java/com/azure/storage/blob/changefeed/implementation/models/BlobChangefeedEventDeserializationTests.java b/sdk/storage/azure-storage-blob-changefeed/src/test/java/com/azure/storage/blob/changefeed/implementation/models/BlobChangefeedEventDeserializationTests.java new file mode 100644 index 000000000000..a48d32043550 --- /dev/null +++ b/sdk/storage/azure-storage-blob-changefeed/src/test/java/com/azure/storage/blob/changefeed/implementation/models/BlobChangefeedEventDeserializationTests.java @@ -0,0 +1,463 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.blob.changefeed.implementation.models; + +import com.azure.json.JsonProviders; +import com.azure.json.JsonReader; +import com.azure.json.JsonToken; +import com.azure.storage.blob.changefeed.models.BlobChangefeedEventType; +import com.azure.storage.blob.models.BlobType; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.io.InputStream; +import java.time.OffsetDateTime; +import java.time.ZoneOffset; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; + +/** + * Tests deserialization of BlobChangefeedEvent and BlobChangefeedEventData for schema versions V6, V7, and V8. + */ +public class BlobChangefeedEventDeserializationTests { + + // Values from EventSchemaV6.json / EventSchemaV7.json / EventSchemaV8.json + private static final long CONTENT_OFFSET = 256L; + private static final String CREATE_TIME = "2022-02-17T13:11:52.5901564Z"; + private static final String LAST_ACCESS_TIME = "2022-02-17T13:11:53.5901564Z"; + private static final String RESTORED_CONTAINER_VERSION = "0000000000000002"; + private static final String CLIENT_REQUEST_ID = "clientRequestId"; + private static final String REQUEST_ID = "requestId"; + private static final String ETAG = "0x8D9F2171BE32588"; + private static final String CONTENT_TYPE = "application/octet-stream"; + private static final long CONTENT_LENGTH = 128L; + private static final String SEQUENCER = "00000000000000010000000000000002000000000000001d"; + private static final String DESTINATION_URL = "destinationUrl"; + private static final String SOURCE_URL = "sourceUrl"; + private static final String BLOB_URL = "https://www.myurl.com"; + + // ======================== Schema V6 ======================== + + @Test + public void schemaV6AppendBlobDataUpdatedEventTypeDeserializes() { + assertEquals(BlobChangefeedEventType.APPEND_BLOB_DATA_UPDATED, + BlobChangefeedEventType.fromString("AppendBlobDataUpdated")); + } + + @Test + public void schemaV6ContentOffsetDeserializes() { + InternalBlobChangefeedEventData data = InternalBlobChangefeedEventData.fromRecord(buildDataRecord(r -> { + r.put("api", "PutBlob"); + r.put("contentOffset", CONTENT_OFFSET); + })); + assertEquals(CONTENT_OFFSET, data.getContentOffset()); + } + + @Test + public void schemaV6CreationTimeDeserializes() { + InternalBlobChangefeedEventData data = InternalBlobChangefeedEventData.fromRecord(buildDataRecord(r -> { + r.put("api", "PutBlob"); + r.put("createTime", CREATE_TIME); + })); + assertEquals(OffsetDateTime.parse(CREATE_TIME), data.getCreationTime()); + } + + @Test + public void schemaV6ContentOffsetAndCreationTimeNullWhenAbsent() { + InternalBlobChangefeedEventData data = InternalBlobChangefeedEventData.fromRecord(buildDataRecord(r -> { + })); + assertNull(data.getContentOffset()); + assertNull(data.getCreationTime()); + } + + @Test + public void schemaV6FullEventDeserializes() { + Map eventMap = buildEventRecord(r -> { + r.put("eventType", "BlobCreated"); + r.put("data", buildDataRecord(d -> { + d.put("clientRequestId", CLIENT_REQUEST_ID); + d.put("requestId", REQUEST_ID); + d.put("etag", ETAG); + d.put("contentType", CONTENT_TYPE); + d.put("contentLength", CONTENT_LENGTH); + d.put("blobType", "BlockBlob"); + d.put("contentOffset", CONTENT_OFFSET); + d.put("destinationUrl", DESTINATION_URL); + d.put("sourceUrl", SOURCE_URL); + d.put("url", BLOB_URL); + d.put("recursive", false); + d.put("sequencer", SEQUENCER); + d.put("createTime", CREATE_TIME); + })); + }); + + InternalBlobChangefeedEvent event = InternalBlobChangefeedEvent.fromRecord(eventMap); + + assertEquals("topic", event.getTopic()); + assertEquals("subject", event.getSubject()); + assertEquals(BlobChangefeedEventType.BLOB_CREATED, event.getEventType()); + assertEquals(OffsetDateTime.of(2022, 2, 17, 13, 12, 11, 0, ZoneOffset.UTC), event.getEventTime()); + assertEquals("62616073-8020-0000-00ff-233467060cc0", event.getId()); + assertEquals(1L, event.getDataVersion()); + assertEquals("1", event.getMetadataVersion()); + assertEquals("PutBlob", event.getData().getApi()); + assertEquals(CLIENT_REQUEST_ID, event.getData().getClientRequestId()); + assertEquals(REQUEST_ID, event.getData().getRequestId()); + assertEquals(ETAG, event.getData().getETag()); + assertEquals(CONTENT_TYPE, event.getData().getContentType()); + assertEquals(CONTENT_LENGTH, event.getData().getContentLength()); + assertEquals(BlobType.BLOCK_BLOB, event.getData().getBlobType()); + assertEquals(CONTENT_OFFSET, event.getData().getContentOffset()); + assertEquals(DESTINATION_URL, event.getData().getDestinationUrl()); + assertEquals(SOURCE_URL, event.getData().getSourceUrl()); + assertEquals(BLOB_URL, event.getData().getBlobUrl()); + assertFalse(event.getData().isRecursive()); + assertEquals(SEQUENCER, event.getData().getSequencer()); + assertEquals(OffsetDateTime.parse(CREATE_TIME), event.getData().getCreationTime()); + assertNull(event.getData().getLastAccessTime()); + assertNull(event.getData().getRestoredContainerVersion()); + } + + // ======================== Schema V7 ======================== + + @Test + public void schemaV7BlobLastAccessTimeUpdatedEventTypeDeserializes() { + assertEquals(BlobChangefeedEventType.BLOB_LAST_ACCESS_TIME_UPDATED, + BlobChangefeedEventType.fromString("BlobLastAccessTimeUpdated")); + } + + @Test + public void schemaV7LastAccessTimeDeserializes() { + InternalBlobChangefeedEventData data = InternalBlobChangefeedEventData.fromRecord(buildDataRecord(r -> { + r.put("api", "PutBlob"); + r.put("lastAccessTime", LAST_ACCESS_TIME); + })); + assertEquals(OffsetDateTime.parse(LAST_ACCESS_TIME), data.getLastAccessTime()); + } + + @Test + public void schemaV7LastAccessTimeNullWhenAbsent() { + InternalBlobChangefeedEventData data = InternalBlobChangefeedEventData.fromRecord(buildDataRecord(r -> { + })); + assertNull(data.getLastAccessTime()); + } + + @Test + public void schemaV7FullEventDeserializes() { + Map eventMap = buildEventRecord(r -> { + r.put("eventType", "BlobCreated"); + r.put("data", buildDataRecord(d -> { + d.put("clientRequestId", CLIENT_REQUEST_ID); + d.put("requestId", REQUEST_ID); + d.put("etag", ETAG); + d.put("contentType", CONTENT_TYPE); + d.put("contentLength", CONTENT_LENGTH); + d.put("blobType", "BlockBlob"); + d.put("contentOffset", CONTENT_OFFSET); + d.put("destinationUrl", DESTINATION_URL); + d.put("sourceUrl", SOURCE_URL); + d.put("url", BLOB_URL); + d.put("recursive", false); + d.put("sequencer", SEQUENCER); + d.put("createTime", CREATE_TIME); + d.put("lastAccessTime", LAST_ACCESS_TIME); + })); + }); + + InternalBlobChangefeedEvent event = InternalBlobChangefeedEvent.fromRecord(eventMap); + + assertEquals("topic", event.getTopic()); + assertEquals("subject", event.getSubject()); + assertEquals(BlobChangefeedEventType.BLOB_CREATED, event.getEventType()); + assertEquals(OffsetDateTime.of(2022, 2, 17, 13, 12, 11, 0, ZoneOffset.UTC), event.getEventTime()); + assertEquals("62616073-8020-0000-00ff-233467060cc0", event.getId()); + assertEquals(1L, event.getDataVersion()); + assertEquals("1", event.getMetadataVersion()); + assertEquals("PutBlob", event.getData().getApi()); + assertEquals(CLIENT_REQUEST_ID, event.getData().getClientRequestId()); + assertEquals(REQUEST_ID, event.getData().getRequestId()); + assertEquals(ETAG, event.getData().getETag()); + assertEquals(CONTENT_TYPE, event.getData().getContentType()); + assertEquals(CONTENT_LENGTH, event.getData().getContentLength()); + assertEquals(BlobType.BLOCK_BLOB, event.getData().getBlobType()); + assertEquals(CONTENT_OFFSET, event.getData().getContentOffset()); + assertEquals(DESTINATION_URL, event.getData().getDestinationUrl()); + assertEquals(SOURCE_URL, event.getData().getSourceUrl()); + assertEquals(BLOB_URL, event.getData().getBlobUrl()); + assertFalse(event.getData().isRecursive()); + assertEquals(SEQUENCER, event.getData().getSequencer()); + assertEquals(OffsetDateTime.parse(CREATE_TIME), event.getData().getCreationTime()); + assertEquals(OffsetDateTime.parse(LAST_ACCESS_TIME), event.getData().getLastAccessTime()); + assertNull(event.getData().getRestoredContainerVersion()); + } + + // ======================== Schema V8 / Container Change Feed ======================== + + @Test + public void schemaV8ContainerCreatedEventTypeDeserializes() { + assertEquals(BlobChangefeedEventType.CONTAINER_CREATED, BlobChangefeedEventType.fromString("ContainerCreated")); + } + + @Test + public void schemaV8ContainerDeletedEventTypeDeserializes() { + assertEquals(BlobChangefeedEventType.CONTAINER_DELETED, BlobChangefeedEventType.fromString("ContainerDeleted")); + } + + @Test + public void schemaV8ContainerPropertiesUpdatedEventTypeDeserializes() { + assertEquals(BlobChangefeedEventType.CONTAINER_PROPERTIES_UPDATED, + BlobChangefeedEventType.fromString("ContainerPropertiesUpdated")); + } + + @Test + public void schemaV8RestoredContainerVersionDeserializes() { + InternalBlobChangefeedEventData data = InternalBlobChangefeedEventData.fromRecord(buildDataRecord(r -> { + r.put("api", "PutBlob"); + r.put("restoredContainerVersion", RESTORED_CONTAINER_VERSION); + })); + assertEquals(RESTORED_CONTAINER_VERSION, data.getRestoredContainerVersion()); + } + + @Test + public void schemaV8RestoredContainerVersionNullWhenAbsent() { + InternalBlobChangefeedEventData data = InternalBlobChangefeedEventData.fromRecord(buildDataRecord(r -> { + })); + assertNull(data.getRestoredContainerVersion()); + } + + @Test + public void schemaV8FullEventDeserializes() { + Map eventMap = buildEventRecord(r -> { + r.put("eventType", "BlobCreated"); + r.put("data", buildDataRecord(d -> { + d.put("clientRequestId", CLIENT_REQUEST_ID); + d.put("requestId", REQUEST_ID); + d.put("etag", ETAG); + d.put("contentType", CONTENT_TYPE); + d.put("contentLength", CONTENT_LENGTH); + d.put("blobType", "BlockBlob"); + d.put("contentOffset", CONTENT_OFFSET); + d.put("destinationUrl", DESTINATION_URL); + d.put("sourceUrl", SOURCE_URL); + d.put("url", BLOB_URL); + d.put("recursive", false); + d.put("sequencer", SEQUENCER); + d.put("createTime", CREATE_TIME); + d.put("lastAccessTime", LAST_ACCESS_TIME); + d.put("restoredContainerVersion", RESTORED_CONTAINER_VERSION); + })); + }); + + InternalBlobChangefeedEvent event = InternalBlobChangefeedEvent.fromRecord(eventMap); + + assertEquals("topic", event.getTopic()); + assertEquals("subject", event.getSubject()); + assertEquals(BlobChangefeedEventType.BLOB_CREATED, event.getEventType()); + assertEquals(OffsetDateTime.of(2022, 2, 17, 13, 12, 11, 0, ZoneOffset.UTC), event.getEventTime()); + assertEquals("62616073-8020-0000-00ff-233467060cc0", event.getId()); + assertEquals(1L, event.getDataVersion()); + assertEquals("1", event.getMetadataVersion()); + assertEquals("PutBlob", event.getData().getApi()); + assertEquals(CLIENT_REQUEST_ID, event.getData().getClientRequestId()); + assertEquals(REQUEST_ID, event.getData().getRequestId()); + assertEquals(ETAG, event.getData().getETag()); + assertEquals(CONTENT_TYPE, event.getData().getContentType()); + assertEquals(CONTENT_LENGTH, event.getData().getContentLength()); + assertEquals(BlobType.BLOCK_BLOB, event.getData().getBlobType()); + assertEquals(CONTENT_OFFSET, event.getData().getContentOffset()); + assertEquals(DESTINATION_URL, event.getData().getDestinationUrl()); + assertEquals(SOURCE_URL, event.getData().getSourceUrl()); + assertEquals(BLOB_URL, event.getData().getBlobUrl()); + assertFalse(event.getData().isRecursive()); + assertEquals(SEQUENCER, event.getData().getSequencer()); + assertEquals(OffsetDateTime.parse(CREATE_TIME), event.getData().getCreationTime()); + assertEquals(OffsetDateTime.parse(LAST_ACCESS_TIME), event.getData().getLastAccessTime()); + assertEquals(RESTORED_CONTAINER_VERSION, event.getData().getRestoredContainerVersion()); + } + + // ======================== JSON File Loading ======================== + + @Test + public void schemaV6JsonFileDeserializes() throws IOException { + Map eventMap = loadJsonAsAvroMap("EventSchemaV6.json"); + InternalBlobChangefeedEvent event = InternalBlobChangefeedEvent.fromRecord(eventMap); + assertEquals(CONTENT_OFFSET, event.getData().getContentOffset()); + assertEquals(OffsetDateTime.parse(CREATE_TIME), event.getData().getCreationTime()); + assertNull(event.getData().getLastAccessTime()); + assertNull(event.getData().getRestoredContainerVersion()); + } + + @Test + public void schemaV7JsonFileDeserializes() throws IOException { + Map eventMap = loadJsonAsAvroMap("EventSchemaV7.json"); + InternalBlobChangefeedEvent event = InternalBlobChangefeedEvent.fromRecord(eventMap); + assertEquals(CONTENT_OFFSET, event.getData().getContentOffset()); + assertEquals(OffsetDateTime.parse(CREATE_TIME), event.getData().getCreationTime()); + assertEquals(OffsetDateTime.parse(LAST_ACCESS_TIME), event.getData().getLastAccessTime()); + assertNull(event.getData().getRestoredContainerVersion()); + } + + @Test + public void schemaV8JsonFileDeserializes() throws IOException { + Map eventMap = loadJsonAsAvroMap("EventSchemaV8.json"); + InternalBlobChangefeedEvent event = InternalBlobChangefeedEvent.fromRecord(eventMap); + assertEquals(CONTENT_OFFSET, event.getData().getContentOffset()); + assertEquals(OffsetDateTime.parse(CREATE_TIME), event.getData().getCreationTime()); + assertEquals(OffsetDateTime.parse(LAST_ACCESS_TIME), event.getData().getLastAccessTime()); + assertEquals(RESTORED_CONTAINER_VERSION, event.getData().getRestoredContainerVersion()); + } + + // ======================== Regression Tests ======================== + + @Test + public void olderSchemaPayloadDeserializesWithoutNewFields() { + Map eventMap = buildEventRecord(r -> { + r.put("eventType", "BlobCreated"); + r.put("data", buildDataRecord(d -> { + d.put("api", "PutBlob"); + d.put("etag", "0x8D9F2171BE32588"); + d.put("contentType", "application/octet-stream"); + d.put("contentLength", 128L); + d.put("blobType", "BlockBlob"); + d.put("url", "https://www.myurl.com"); + })); + }); + + InternalBlobChangefeedEvent event = InternalBlobChangefeedEvent.fromRecord(eventMap); + + assertEquals(BlobChangefeedEventType.BLOB_CREATED, event.getEventType()); + assertEquals("PutBlob", event.getData().getApi()); + assertNull(event.getData().getContentOffset()); + assertNull(event.getData().getCreationTime()); + assertNull(event.getData().getLastAccessTime()); + assertNull(event.getData().getRestoredContainerVersion()); + } + + @Test + public void existingBlobEventsUnaffected() { + Map eventMap = buildEventRecord(r -> { + r.put("eventType", "BlobDeleted"); + r.put("data", buildDataRecord(d -> { + d.put("api", "DeleteBlob"); + d.put("sequencer", "00000000000000010000000000000002000000000000001d"); + })); + }); + + InternalBlobChangefeedEvent event = InternalBlobChangefeedEvent.fromRecord(eventMap); + + assertEquals(BlobChangefeedEventType.BLOB_DELETED, event.getEventType()); + assertEquals("DeleteBlob", event.getData().getApi()); + assertNull(event.getData().getCreationTime()); + assertNull(event.getData().getLastAccessTime()); + assertNull(event.getData().getRestoredContainerVersion()); + } + + @Test + public void unknownOptionalFieldsDoNotFailDeserialization() { + Map dataRecord = buildDataRecord(r -> { + r.put("unknownFutureField", "someValue"); + r.put("anotherUnknownField", 42L); + }); + + InternalBlobChangefeedEventData data = InternalBlobChangefeedEventData.fromRecord(dataRecord); + assertEquals("PutBlob", data.getApi()); + } + + @Test + public void dataVersionFieldUnaffected() { + Map eventMap = buildEventRecord(r -> r.put("dataVersion", 8L)); + + InternalBlobChangefeedEvent event = InternalBlobChangefeedEvent.fromRecord(eventMap); + assertEquals(8L, event.getDataVersion()); + } + + // ======================== Helpers ======================== + + @FunctionalInterface + private interface MapCustomizer { + void customize(Map map); + } + + @SuppressWarnings("unchecked") + private static Map loadJsonAsAvroMap(String resourceName) throws IOException { + try (InputStream is = Objects.requireNonNull( + BlobChangefeedEventDeserializationTests.class.getClassLoader().getResourceAsStream(resourceName), + "Test resource not found: " + resourceName); JsonReader reader = JsonProviders.createReader(is)) { + reader.nextToken(); + Map map = readJsonObject(reader); + map.put("$record", "BlobChangeEvent"); + Map data = (Map) map.get("data"); + if (data != null) { + data.put("$record", "BlobChangeEventData"); + } + return map; + } + } + + private static Map readJsonObject(JsonReader reader) throws IOException { + Map map = new HashMap<>(); + while (reader.nextToken() != JsonToken.END_OBJECT) { + String fieldName = reader.getFieldName(); + reader.nextToken(); + map.put(fieldName, readJsonValue(reader)); + } + return map; + } + + private static Object readJsonValue(JsonReader reader) throws IOException { + switch (reader.currentToken()) { + case NULL: + return null; + + case STRING: + return reader.getString(); + + case NUMBER: + return reader.getLong(); + + case BOOLEAN: + return reader.getBoolean(); + + case START_OBJECT: + return readJsonObject(reader); + + case START_ARRAY: + reader.skipChildren(); + return null; + + default: + return null; + } + } + + private static Map buildEventRecord(MapCustomizer customizer) { + Map record = new HashMap<>(); + record.put("$record", "BlobChangeEvent"); + record.put("schemaVersion", 1); + record.put("topic", "topic"); + record.put("subject", "subject"); + record.put("eventType", "BlobCreated"); + record.put("eventTime", OffsetDateTime.of(2022, 2, 17, 13, 12, 11, 0, ZoneOffset.UTC).toString()); + record.put("id", "62616073-8020-0000-00ff-233467060cc0"); + record.put("dataVersion", 1L); + record.put("metadataVersion", "1"); + record.put("data", buildDataRecord(d -> { + })); + customizer.customize(record); + return record; + } + + private static Map buildDataRecord(MapCustomizer customizer) { + Map record = new HashMap<>(); + record.put("$record", "BlobChangeEventData"); + record.put("api", "PutBlob"); + customizer.customize(record); + return record; + } +} diff --git a/sdk/storage/azure-storage-blob-changefeed/src/test/resources/EventSchemaV6.json b/sdk/storage/azure-storage-blob-changefeed/src/test/resources/EventSchemaV6.json new file mode 100644 index 000000000000..d6f2b6e18018 --- /dev/null +++ b/sdk/storage/azure-storage-blob-changefeed/src/test/resources/EventSchemaV6.json @@ -0,0 +1,84 @@ +{ + "schemaVersion": 6, + "topic": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/haambaga/providers/Microsoft.Storage/storageAccounts/HAAMBAGA-DEV", + "subject": "/blobServices/default/containers/apitestcontainerver/blobs/20220217_131202494_Blob_oaG6iu7ImEB1cX8M", + "eventType": "BlobCreated", + "eventTime": "2022-02-17T13:12:11.5746587Z", + "id": "62616073-8020-0000-00ff-233467060cc0", + "data": { + "api": "PutBlob", + "clientRequestId": "00000000-0000-0000-0000-000000000000", + "requestId": "62616073-8020-0000-00ff-233467000000", + "etag": "0x8D9F2171BE32588", + "contentType": "application/octet-stream", + "contentLength": 128, + "blobType": "BlockBlob", + "blobVersion": "2022-02-17T16:11:52.5901564Z", + "containerVersion": "0000000000000001", + "blobTier": "Archive", + "url": "https://www.myurl.com", + "sequencer": "00000000000000010000000000000002000000000000001d", + "previousInfo": { + "SoftDeleteSnapshot": "2022-02-17T13:12:11.5726507Z", + "WasBlobSoftDeleted": "true", + "BlobVersion": "2024-02-17T16:11:52.0781797Z", + "LastVersion" : "2022-02-17T16:11:52.0781797Z", + "PreviousTier": "Hot" + }, + "snapshot" : "2022-02-17T16:09:16.7261278Z", + "blobPropertiesUpdated" : { + "ContentLanguage" : { + "current" : "pl-Pl", + "previous" : "nl-NL" + }, + "CacheControl" : { + "current" : "max-age=100", + "previous" : "max-age=99" + }, + "ContentEncoding" : { + "current" : "gzip, identity", + "previous" : "gzip" + }, + "ContentMD5" : { + "current" : "Q2h1Y2sgSW51ZwDIAXR5IQ==", + "previous" : "Q2h1Y2sgSW=" + }, + "ContentDisposition" : { + "current" : "attachment", + "previous" : "" + }, + "ContentType" : { + "current" : "application/json", + "previous" : "application/octet-stream" + } + }, + "asyncOperationInfo": { + "DestinationTier": "Hot", + "WasAsyncOperation": "true", + "CopyId": "copyId" + }, + "blobTagsUpdated": { + "previous": { + "Tag1": "Value1_3", + "Tag2": "Value2_3" + }, + "current": { + "Tag1": "Value1_4", + "Tag2": "Value2_4" + } + }, + "restorePointMarker": { + "rpi": "00000000-0000-0000-0000-000000000000", + "rpp": "00000000-0000-0000-0000-000000000000", + "rpl": "test-restore-label", + "rpt": "2022-02-17T13:56:09.3559772Z" + }, + "contentOffset": 256, + "createTime": "2022-02-17T13:11:52.5901564Z", + "storageDiagnostics": { + "bid": "9d726db1-8006-0000-00ff-233467000000", + "seq": "(2,18446744073709551615,29,29)", + "sid": "00000000-0000-0000-0000-000000000000" + } + } +} diff --git a/sdk/storage/azure-storage-blob-changefeed/src/test/resources/EventSchemaV7.json b/sdk/storage/azure-storage-blob-changefeed/src/test/resources/EventSchemaV7.json new file mode 100644 index 000000000000..22f1f3554586 --- /dev/null +++ b/sdk/storage/azure-storage-blob-changefeed/src/test/resources/EventSchemaV7.json @@ -0,0 +1,85 @@ +{ + "schemaVersion": 7, + "topic": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/haambaga/providers/Microsoft.Storage/storageAccounts/HAAMBAGA-DEV", + "subject": "/blobServices/default/containers/apitestcontainerver/blobs/20220217_131202494_Blob_oaG6iu7ImEB1cX8M", + "eventType": "BlobCreated", + "eventTime": "2022-02-17T13:12:11.5746587Z", + "id": "62616073-8020-0000-00ff-233467060cc0", + "data": { + "api": "PutBlob", + "clientRequestId": "00000000-0000-0000-0000-000000000000", + "requestId": "62616073-8020-0000-00ff-233467000000", + "etag": "0x8D9F2171BE32588", + "contentType": "application/octet-stream", + "contentLength": 128, + "blobType": "BlockBlob", + "blobVersion": "2022-02-17T16:11:52.5901564Z", + "containerVersion": "0000000000000001", + "blobTier": "Archive", + "url": "https://www.myurl.com", + "sequencer": "00000000000000010000000000000002000000000000001d", + "previousInfo": { + "SoftDeleteSnapshot": "2022-02-17T13:12:11.5726507Z", + "WasBlobSoftDeleted": "true", + "BlobVersion": "2024-02-17T16:11:52.0781797Z", + "LastVersion" : "2022-02-17T16:11:52.0781797Z", + "PreviousTier": "Hot" + }, + "snapshot" : "2022-02-17T16:09:16.7261278Z", + "blobPropertiesUpdated" : { + "ContentLanguage" : { + "current" : "pl-Pl", + "previous" : "nl-NL" + }, + "CacheControl" : { + "current" : "max-age=100", + "previous" : "max-age=99" + }, + "ContentEncoding" : { + "current" : "gzip, identity", + "previous" : "gzip" + }, + "ContentMD5" : { + "current" : "Q2h1Y2sgSW51ZwDIAXR5IQ==", + "previous" : "Q2h1Y2sgSW=" + }, + "ContentDisposition" : { + "current" : "attachment", + "previous" : "" + }, + "ContentType" : { + "current" : "application/json", + "previous" : "application/octet-stream" + } + }, + "asyncOperationInfo": { + "DestinationTier": "Hot", + "WasAsyncOperation": "true", + "CopyId": "copyId" + }, + "blobTagsUpdated": { + "previous": { + "Tag1": "Value1_3", + "Tag2": "Value2_3" + }, + "current": { + "Tag1": "Value1_4", + "Tag2": "Value2_4" + } + }, + "restorePointMarker": { + "rpi": "00000000-0000-0000-0000-000000000000", + "rpp": "00000000-0000-0000-0000-000000000000", + "rpl": "test-restore-label", + "rpt": "2022-02-17T13:56:09.3559772Z" + }, + "contentOffset": 256, + "createTime": "2022-02-17T13:11:52.5901564Z", + "lastAccessTime": "2022-02-17T13:11:53.5901564Z", + "storageDiagnostics": { + "bid": "9d726db1-8006-0000-00ff-233467000000", + "seq": "(2,18446744073709551615,29,29)", + "sid": "00000000-0000-0000-0000-000000000000" + } + } +} diff --git a/sdk/storage/azure-storage-blob-changefeed/src/test/resources/EventSchemaV8.json b/sdk/storage/azure-storage-blob-changefeed/src/test/resources/EventSchemaV8.json new file mode 100644 index 000000000000..167d1478ec31 --- /dev/null +++ b/sdk/storage/azure-storage-blob-changefeed/src/test/resources/EventSchemaV8.json @@ -0,0 +1,86 @@ +{ + "schemaVersion": 8, + "topic": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/haambaga/providers/Microsoft.Storage/storageAccounts/HAAMBAGA-DEV", + "subject": "/blobServices/default/containers/apitestcontainerver/blobs/20220217_131202494_Blob_oaG6iu7ImEB1cX8M", + "eventType": "BlobCreated", + "eventTime": "2022-02-17T13:12:11.5746587Z", + "id": "62616073-8020-0000-00ff-233467060cc0", + "data": { + "api": "PutBlob", + "clientRequestId": "00000000-0000-0000-0000-000000000000", + "requestId": "62616073-8020-0000-00ff-233467000000", + "etag": "0x8D9F2171BE32588", + "contentType": "application/octet-stream", + "contentLength": 128, + "blobType": "BlockBlob", + "blobVersion": "2022-02-17T16:11:52.5901564Z", + "containerVersion": "0000000000000001", + "blobTier": "Archive", + "url": "https://www.myurl.com", + "sequencer": "00000000000000010000000000000002000000000000001d", + "previousInfo": { + "SoftDeleteSnapshot": "2022-02-17T13:12:11.5726507Z", + "WasBlobSoftDeleted": "true", + "BlobVersion": "2024-02-17T16:11:52.0781797Z", + "LastVersion" : "2022-02-17T16:11:52.0781797Z", + "PreviousTier": "Hot" + }, + "snapshot" : "2022-02-17T16:09:16.7261278Z", + "blobPropertiesUpdated" : { + "ContentLanguage" : { + "current" : "pl-Pl", + "previous" : "nl-NL" + }, + "CacheControl" : { + "current" : "max-age=100", + "previous" : "max-age=99" + }, + "ContentEncoding" : { + "current" : "gzip, identity", + "previous" : "gzip" + }, + "ContentMD5" : { + "current" : "Q2h1Y2sgSW51ZwDIAXR5IQ==", + "previous" : "Q2h1Y2sgSW=" + }, + "ContentDisposition" : { + "current" : "attachment", + "previous" : "" + }, + "ContentType" : { + "current" : "application/json", + "previous" : "application/octet-stream" + } + }, + "asyncOperationInfo": { + "DestinationTier": "Hot", + "WasAsyncOperation": "true", + "CopyId": "copyId" + }, + "blobTagsUpdated": { + "previous": { + "Tag1": "Value1_3", + "Tag2": "Value2_3" + }, + "current": { + "Tag1": "Value1_4", + "Tag2": "Value2_4" + } + }, + "restorePointMarker": { + "rpi": "00000000-0000-0000-0000-000000000000", + "rpp": "00000000-0000-0000-0000-000000000000", + "rpl": "test-restore-label", + "rpt": "2022-02-17T13:56:09.3559772Z" + }, + "restoredContainerVersion": "0000000000000002", + "contentOffset": 256, + "createTime": "2022-02-17T13:11:52.5901564Z", + "lastAccessTime": "2022-02-17T13:11:53.5901564Z", + "storageDiagnostics": { + "bid": "9d726db1-8006-0000-00ff-233467000000", + "seq": "(2,18446744073709551615,29,29)", + "sid": "00000000-0000-0000-0000-000000000000" + } + } +} From 71762ab587c68d120233dec13beccce251300696 Mon Sep 17 00:00:00 2001 From: gunjansingh-msft Date: Wed, 24 Jun 2026 01:07:30 +0530 Subject: [PATCH 4/5] Storage - Content Validation STG104 (#49610) * Crc polynomial (#43752) * Message encoder (#43803) * wip * basic message encoder logic working * removing print statements and making slightly more readable * fixing several bugs * adding more tests * adding comments for building purposes and to branch off for service testing * wip * wip * working encode that takes in bytebuffer * removing redundant 'this' and making small readability changes * fixing flag bug, adding more comments, adjusting incorrect test case * adding structuredmessage package in implemenation and addressing other comments * refactoring * adding more tests, adjusting test encoder, adding more validation * re-ordering empty buffer return * adding correct comments * addressing comments and fixing ci issues * trying to resolve spotbugs issue * formatting * Extracting non-decoder changes from the decoder PR (#45546) * creating class * replacing encoder constants with new class constants * adding fromValue to StructuredMessageFlags * updating module info to contain more packages * style * undoing module info change * Storage Content Validation - Encoder Performance Improvements (#47531) * adjusting encoder logic * adjusting tests to work with encoder * addressing copilot comments * adding more documentation * Storage - Content Validation Public Interface (#48074) * public facing interface wip * adjusting output stream constructors * removing redundant apis * fixing pageblob api * bloboutputstream adjustment * wip * wip * importing things correctly * formatting * throwing error with client logger * fixing revapi visibility increased error * overload adjustments * adding final to options bags and hopefully resolving revapi error * more edits * more edits * more edits * spotless * spotless * more edits * removing javadoc again * Storage - Content Validation Message Encoder Perf Updates (#47158) * all perf adjustments * addressing copilot comments * Consolidate content validation types under contentvalidation package (#48272) * Storage - Content Validation Encoder Pipeline Implementation (#48354) * adding perf to message encoder tests * changing package name * changing package name * moving checksum algorithm to new package * wip * moving storageschecksumalgorithm out of impl so it can actually be used * removing unused file * bruh * small cleanups * implementation for other APIs * test updates * undoing silly agent changes * async tests * perf tests for all APIs * new path for crc64s without accessible array * md5 compatibility tests * progress reporter tests * changing upload file test to stream file * consolidating conflicting transactional checksum logic * addressing copilot comments * changing policy to compute crc64 header in a non-blocking manner * fixing crc64 policy * addressing copilot comments * deleting perf tests * more tests * addressing more copilot comments * adding recordings and marking multi part tests as live only due to the random block ids * removing unused tests * updating assets * addressing github comments * addressing comments * renaming missed setters * idk * mode resolver cleanup * adjusting mode resolver tests * un-deprecating compute md5 * addressing API view comments * addressing comments * swapping test behavior for new md5 compatibility util * wip * adding validation to prevent progress listener being used with content val * adding comment about flux.generate in encoder class * adding recordings * removing unused imports * Storage - Content Validation Decoder Implementation (#47016) * adding the StructuredMessageDecoder * adding the pipeline policy changes * smart retry changes * fixing smart retry impl * smart retry changes * smart retry changes * smart retry changes * smart retry changes * adding content validation tests * code refactoring * fixing errors i introduced :( * addressing review comments * code refactoring based on latest review comments * code refactoring based on latest review comments * simplifying retry mechanism * removing dead code * addressing Kyle's review comments * addressing latest review comments * refactoring based on latest review comments * refactoring based on latest review comments * refactoring based on latest review comments * refactoring based on latest review comments from kyle * expanding test coverage * removing unused imports * recordings * adding documentation to decoder classes * small fixes and failure path tests * addressing context comment * analyze error * removing close override from decodedresponse * line removal --------- Co-authored-by: Isabelle * Storage Content Validation - Defer MD5 + CRC64 Val Conflict to Service (#48999) * impl and tests * cleanup * updating assets * strengthening sync assertations * Storage Content Validation - adding failure tests for content validation decoder (#49073) * adding failure tests for content validation decoder * adding Decoder Random Byte Failure Case Test * rearrange code to make it a little more human readable * fix linting --------- Co-authored-by: browndav * Storage Content Validation - Audit DecodedResponse (#49147) * Audit DecodedResponse: tighten override surface, pin UTF-8, add unit tests * Reduce DecodedResponse to minimal override surface Strip overrides and tests that aren't required for transparent HttpResponse behavior, leaving exactly the compiler-required abstract methods plus one discretionary override that fixes a proven base-class bug. DecodedResponse.java: - Remove close() override. Current consumers fully drain the body Flux, so Reactor's onComplete signals release the underlying transport; the explicit close-forwarding was defensive against a try-with-resources / status-only pattern that no caller in the codebase actually uses. - Collapse no-arg getBodyAsString() to delegate to the charset overload, pinning UTF-8 in one place instead of duplicating the lambda. - Final override surface (8 methods): the 7 abstract methods on HttpResponse in azure-core 1.57.1 (compiler-required) plus getBodyAsBinaryData, which is concrete in the base but seeds BinaryData with the wire Content-Length header, making BinaryData.getLength() return the encoded size (frames + CRC trailers) instead of the decoded payload size. DecodedResponseTests.java: - Remove closeDelegatesToWrappedResponse and closeDoesNotSubscribeToDecodedBody (no override left to validate). - Drop the close-counting assertion from the writeBodyTo test and rename it to inheritedWriteBodyToWritesDecodedBytes; it now uses MockHttpResponse directly. - Strengthen getBodyAsBinaryDataReportsDecodedSizeNotContentLength: assert data.getLength() equals the decoded size (post-consumption). Without the override this returns 71 instead of 7, empirically reproduced. - Drop now-unused trackingResponse helper, AtomicInteger import. Tests: 10 of 10 passing. Each test maps 1:1 to one override or one inherited integration; no test is redundant. * some code cleanup * Address Copilot PR #49147 review comments DecodedResponse.java (comment r3220562536): - Restore HttpResponse#getBodyAsString() base contract by switching from unconditional UTF-8 to CoreUtils.bomAwareToString(bytes, contentType), matching BufferedHttpResponse's idiom. The previous UTF-8 pin silently dropped BOM detection and Content-Type charset honoring, which callers viewing this as a generic HttpResponse expect. DecodedResponseTests.java (comment r3220562553): - Add inheritedGetBodyAsBinaryDataReturnsDecodedBytes: exercises the inherited HttpResponse#getBodyAsBinaryData() and asserts the bytes match the decoded payload. Uses a divergent Content-Length header to make the wire vs decoded distinction explicit and guard against header-forwarding regressions. - Add getBodyAsStringHonorsCharsetFromContentTypeHeader: proves the bom-aware fix actually honors a charset declared in Content-Type (would fail if the previous UTF-8 pinning were still in place). - Add getBodyAsStringDetectsUtf8BomAndStripsIt: pins the BOM-detection arm of CoreUtils.bomAwareToString. - Update getBodyAsStringDefaultsToUtf8WhenNoCharsetSpecified docstring to describe the new bom-aware fallback semantics rather than the old unconditional UTF-8 behavior. Tests: 12 of 12 passing. * some code cleanup * analyze error --------- Co-authored-by: Isabelle * Storage Content Validation - Assert Request and Response Headers for Decoder Tests (#49075) * add checks for response headers * fix linting * refactor async, change messageDownloadRequestHeaders signatures * fix methods with wrong method names * refactor sync * remove unnecessary helper in BlobTestBase * reuse hasStructuredMessageDownloadRequestHeaders with List * remove unused imports * apply copilot suggestions * fix linting * inline local variables * remove try-finally, replace with @TempDir * add checks for response headers * add recording for downloadStreamWithResponseContentValidationRange * readd recording * rerecord recordings * Storage Content Validation - Verify Download Progress Listener Support (#49157) * tests and blob client changes * addressing copilot comments * updating assets * Storage Content Validation - Fuzzy Upload Tests (#49031) * tests and assets * addressing copilot comments * making things actually fuzzy * cleanup * cleanup * addressing copilot comments * recordings * resolving conflicts * updating assets * Storage Content Validation - adding fuzzy tests for content validation decoder (#49058) * adding fuzzy tests for content validation decoder * adding recordings * adding few fuzzy tests * randomizing fuzzy test payloads * randomizing fuzzy test payloads * fixing code review comments * fix spotless formatting violation in BlobContentValidationAsyncDownloadTests Co-Authored-By: Claude Sonnet 4.6 --------- Co-authored-by: Local Merge Co-authored-by: Claude Sonnet 4.6 * Storage - Content Validation Decoder Dynamic Segment Size Tests and Large GET Behavior Notes (#49172) * tests for dynamic segment size encoding and decoding * cleanup * cleanup * Apply suggestions from code review Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> * style * cant change pr name while ci is running i guess. no-op change * cleaning up test file --------- Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> * Storage - Decoder Perf Improvements (#49205) * refactoring * removing unused code * reverting unecessary changes * reverting unecessary changes * reverting unecessary changes * not pre-allocating entire byte array of segment size * renaming variable * adding back unecessary removals * renaming variable * addressing copilot comments * Storage - Content validation content-length override (#49226) * content length override changes * content length override changes * removing stuff * addressing review comments * addressing review comments * addressing review comments * resolving merge conflicts * Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> * fixing spotbugs * fixing spellchecks --------- Co-authored-by: Local Merge Co-authored-by: Isabelle <141270045+ibrandes@users.noreply.github.com> Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> * Storage - Content Validation Decoder Stress Tests (#49255) * adding stress tests for content validation decoder * fix TelemetryHelper, add tru to registerOberservers() * Storage - Fix Flaky Stress Tests (#48359) * removed enableDeterministic * change .delete() to .deleteIfExists() * remove Sinks.EmitFailureHandler.FAIL_FAST from CrcInputStream - read functions had FAIL_FAST which would throw an error when the stream had reached then end and we wanted to read from the stream again. So we removed from both reads. - refactor code so that the exit criteria is a tthe beginning - refactor the emitContentInfo for dry * prevent crashes on reattempted close on stream - changed emitValue to tryEmitValue - remove Sinks.EmitFailureHandler.FAIL_FAST so that multiple closes does not cause an error to be thrown * fix telemetry so that it doesnt swallow errors * roll back two deps because they were causing failures in the containers - opentelemetry-runtime-telemetry-java8 from 2.24.0-alpha -> 2.15.0-alpha - opentelemetry-logback-appender-1.0 from 2.24.0-alpha -> 2.15.0-alpha * rollback azure-client-sdk-parent linting extensions from 1.0.0-beta.2 t0 beta.1 * revert linting extensions to beta2 * remove comments with old code * add logging for errors * remove catches for double close issue and okay status * recursively delete files then delete the directory * change to sync deletes, refactor for easier reading * restructing share clean up so super calls only once * incorporate copilot suggestions * incorporate copilot suggestions * incorporate copilot suggestions * incorporate copilot suggestions * fix all deletes to make sync and wrap in try-catch * fix tests so that super.globalCleanupAsync() is only called once * change telemetry to loggin only returns final state instead of failed retries when ultimately successful * undo versio downgrade for linting-extensions * Fixing spacing in error messages Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> * refactor datalake delete all so that it is easier to read * refactor runAsync in ShareScenarioBase so retry failures dont show as failures upon success --------- Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * changes to cvdownload content, lazyload versus eagerloading * update to TelemetryHelper based on previous cherry picked stress tests fixes * Fix storage stress fault injector certificate trust Export the fault injector certificate as PEM and wait for it before importing it into the Java truststore so storage stress tests fail fast instead of hitting PKIX errors. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * bump opentelemetry version * add cv tests to App.class * add script to delete resource groups * update scenarios matrix with tests for cv * fixing stress tests * stageblocksmall scenario adjustment * wip * avoid depending on raw short baseName being available globally * dep pinning * reversion * adding back * wip * storageseekablebytechannel change * bare minimum changes * removing perf core change * removing redundant test file and non-blob package changes * removing redundant test file and non-blob package changes * renaming files and hardcoding crc64 option * removing unused test * removing unused imports --------- Co-authored-by: browndav Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Co-authored-by: Local Merge Co-authored-by: Isabelle * Storage - Content Validation Encoder Stress Tests (#48979) * Add encoder stress tests * enabling fault injector * restoring commitblocklist * renaming files and hardcoding crc64 option * adding scenarios back to app.java --------- Co-authored-by: Rabab Ibrahim Co-authored-by: Isabelle <141270045+ibrandes@users.noreply.github.com> Co-authored-by: Isabelle Co-authored-by: browndav Co-authored-by: Local Merge Co-authored-by: Claude Sonnet 4.6 Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../scenarios-matrix.yaml | 384 +++ .../com/azure/storage/blob/stress/App.java | 15 + .../AppendBlobOutputStreamWithCRC64.java | 106 + .../blob/stress/AppendBlockWithCRC64.java | 78 + .../BlockBlobOutputStreamWithCRC64.java | 77 + .../blob/stress/BlockBlobUploadWithCRC64.java | 71 + .../blob/stress/DownloadContentWithCRC64.java | 70 + .../blob/stress/DownloadStreamWithCRC64.java | 67 + .../blob/stress/DownloadToFileWithCRC64.java | 105 + .../blob/stress/OpenInputStreamWithCRC64.java | 71 + .../OpenSeekableByteChannelReadWithCRC64.java | 73 + .../stress/PageBlobOutputStreamWithCRC64.java | 107 + .../SeekableByteChannelWriteWithCRC64.java | 84 + .../blob/stress/StageBlockWithCRC64.java | 91 + .../blob/stress/UploadFromFileWithCRC64.java | 140 + .../blob/stress/UploadPagesWithCRC64.java | 84 + .../storage/blob/stress/UploadWithCRC64.java | 71 + sdk/storage/azure-storage-blob/assets.json | 2 +- .../azure/storage/blob/BlobAsyncClient.java | 60 +- .../implementation/util/BuilderHelper.java | 5 + .../options/AppendBlobAppendBlockOptions.java | 152 + .../AppendBlobOutputStreamOptions.java | 64 + .../options/BlobDownloadContentOptions.java | 129 + .../options/BlobDownloadStreamOptions.java | 131 + .../options/BlobDownloadToFileOptions.java | 25 + .../blob/options/BlobInputStreamOptions.java | 24 + .../options/BlobParallelUploadOptions.java | 26 + .../BlobSeekableByteChannelReadOptions.java | 25 + .../options/BlobUploadFromFileOptions.java | 25 + .../options/BlockBlobOutputStreamOptions.java | 25 + ...ckBlobSeekableByteChannelWriteOptions.java | 24 + .../options/BlockBlobSimpleUploadOptions.java | 26 + .../options/BlockBlobStageBlockOptions.java | 25 + .../options/PageBlobOutputStreamOptions.java | 83 + .../options/PageBlobUploadPagesOptions.java | 149 + .../specialized/AppendBlobAsyncClient.java | 44 +- .../blob/specialized/AppendBlobClient.java | 54 +- .../blob/specialized/BlobAsyncClientBase.java | 77 +- .../blob/specialized/BlobClientBase.java | 92 +- .../blob/specialized/BlobOutputStream.java | 46 +- .../specialized/BlockBlobAsyncClient.java | 79 +- .../blob/specialized/BlockBlobClient.java | 19 +- .../blob/specialized/PageBlobAsyncClient.java | 48 +- .../blob/specialized/PageBlobClient.java | 53 +- ...ableByteChannelBlockBlobWriteBehavior.java | 21 + ...obContentValidationAsyncDownloadTests.java | 681 +++++ ...BlobContentValidationAsyncUploadTests.java | 1073 +++++++ .../BlobContentValidationDownloadTests.java | 658 ++++ .../BlobContentValidationUploadTests.java | 1406 +++++++++ .../com/azure/storage/blob/BlobTestBase.java | 505 ++++ .../BlobSeekableByteChannelTests.java | 2 +- .../blob/specialized/BlockBlobApiTests.java | 2 +- .../specialized/BlockBlobAsyncApiTests.java | 2 +- .../common/ContentValidationAlgorithm.java | 31 + .../common/implementation/Constants.java | 14 + .../common/implementation/UploadUtils.java | 4 +- .../ContentValidationModeResolver.java | 168 ++ .../StorageCrc64Calculator.java | 2660 +++++++++++++++++ .../StructuredMessageConstants.java | 57 + .../StructuredMessageDecoder.java | 536 ++++ .../StructuredMessageEncoder.java | 275 ++ .../StructuredMessageFlags.java | 71 + .../contentvalidation/package-info.java | 7 + .../common/policy/DecodedResponse.java | 86 + ...StorageContentValidationDecoderPolicy.java | 207 ++ .../StorageContentValidationPolicy.java | 175 ++ .../src/main/java/module-info.java | 3 + .../policy/MockDownloadHttpResponse.java | 24 +- .../policy/MockPartialResponsePolicy.java | 128 +- .../ContentValidationModeResolverTests.java | 236 ++ .../MessageEncoderTests.java | 330 ++ .../StorageCrc64CalculatorTests.java | 222 ++ .../StructuredMessageDecoderTests.java | 631 ++++ .../StructuredMessageFlagsTests.java | 39 + .../common/policy/DecodedResponseTests.java | 241 ++ ...geContentValidationDecoderPolicyTests.java | 183 ++ 76 files changed, 13714 insertions(+), 170 deletions(-) create mode 100644 sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/AppendBlobOutputStreamWithCRC64.java create mode 100644 sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/AppendBlockWithCRC64.java create mode 100644 sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/BlockBlobOutputStreamWithCRC64.java create mode 100644 sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/BlockBlobUploadWithCRC64.java create mode 100644 sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/DownloadContentWithCRC64.java create mode 100644 sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/DownloadStreamWithCRC64.java create mode 100644 sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/DownloadToFileWithCRC64.java create mode 100644 sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/OpenInputStreamWithCRC64.java create mode 100644 sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/OpenSeekableByteChannelReadWithCRC64.java create mode 100644 sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/PageBlobOutputStreamWithCRC64.java create mode 100644 sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/SeekableByteChannelWriteWithCRC64.java create mode 100644 sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/StageBlockWithCRC64.java create mode 100644 sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/UploadFromFileWithCRC64.java create mode 100644 sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/UploadPagesWithCRC64.java create mode 100644 sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/UploadWithCRC64.java create mode 100644 sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/AppendBlobAppendBlockOptions.java create mode 100644 sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/AppendBlobOutputStreamOptions.java create mode 100644 sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/BlobDownloadContentOptions.java create mode 100644 sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/BlobDownloadStreamOptions.java create mode 100644 sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/PageBlobOutputStreamOptions.java create mode 100644 sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/PageBlobUploadPagesOptions.java create mode 100644 sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobContentValidationAsyncDownloadTests.java create mode 100644 sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobContentValidationAsyncUploadTests.java create mode 100644 sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobContentValidationDownloadTests.java create mode 100644 sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobContentValidationUploadTests.java create mode 100644 sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/ContentValidationAlgorithm.java create mode 100644 sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/ContentValidationModeResolver.java create mode 100644 sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/StorageCrc64Calculator.java create mode 100644 sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageConstants.java create mode 100644 sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageDecoder.java create mode 100644 sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageEncoder.java create mode 100644 sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageFlags.java create mode 100644 sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/package-info.java create mode 100644 sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/DecodedResponse.java create mode 100644 sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/StorageContentValidationDecoderPolicy.java create mode 100644 sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/StorageContentValidationPolicy.java create mode 100644 sdk/storage/azure-storage-common/src/test/java/com/azure/storage/common/implementation/contentvalidation/ContentValidationModeResolverTests.java create mode 100644 sdk/storage/azure-storage-common/src/test/java/com/azure/storage/common/implementation/contentvalidation/MessageEncoderTests.java create mode 100644 sdk/storage/azure-storage-common/src/test/java/com/azure/storage/common/implementation/contentvalidation/StorageCrc64CalculatorTests.java create mode 100644 sdk/storage/azure-storage-common/src/test/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageDecoderTests.java create mode 100644 sdk/storage/azure-storage-common/src/test/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageFlagsTests.java create mode 100644 sdk/storage/azure-storage-common/src/test/java/com/azure/storage/common/policy/DecodedResponseTests.java create mode 100644 sdk/storage/azure-storage-common/src/test/java/com/azure/storage/common/policy/StorageContentValidationDecoderPolicyTests.java diff --git a/sdk/storage/azure-storage-blob-stress/scenarios-matrix.yaml b/sdk/storage/azure-storage-blob-stress/scenarios-matrix.yaml index 3a8920a67b35..4587e20a1bd6 100644 --- a/sdk/storage/azure-storage-blob-stress/scenarios-matrix.yaml +++ b/sdk/storage/azure-storage-blob-stress/scenarios-matrix.yaml @@ -108,6 +108,150 @@ matrix: durationMin: 60 imageBuildDir: "../../.." + # content validation downloads using BlobDownloadStreamOptions with CRC64 validation + crc64downloadstreamsm: + testScenario: downloadstreamwithcrc64 + sync: true + sizeBytes: 1024 + downloadFaults: true + durationMin: 25 + imageBuildDir: "../../.." + + # content validation downloads using BlobDownloadStreamOptions with CRC64 validation and async client + crc64downloadstreamasyncsm: + testScenario: downloadstreamwithcrc64 + sync: false + sizeBytes: 1024 + downloadFaults: true + durationMin: 25 + imageBuildDir: "../../.." + + # content validation downloads using BlobDownloadStreamOptions with CRC64 validation and large payload + crc64downloadstreamlg: + testScenario: downloadstreamwithcrc64 + sync: true + sizeBytes: "52428800" + downloadFaults: true + durationMin: 60 + imageBuildDir: "../../.." + + # content validation downloads using BlobDownloadStreamOptions with CRC64 validation, async client, and large payload + crc64downloadstreamasynclg: + testScenario: downloadstreamwithcrc64 + sync: false + sizeBytes: "52428800" + downloadFaults: true + durationMin: 60 + imageBuildDir: "../../.." + + # content validation downloads using BlobDownloadContentOptions with CRC64 validation + crc64downloadcontentsm: + testScenario: downloadcontentwithcrc64 + sync: true + sizeBytes: 1024 + downloadFaults: true + durationMin: 25 + imageBuildDir: "../../.." + + # content validation downloads using BlobDownloadContentOptions with CRC64 validation and async client + crc64downloadcontentasyncsm: + testScenario: downloadcontentwithcrc64 + sync: false + sizeBytes: 1024 + downloadFaults: true + durationMin: 25 + imageBuildDir: "../../.." + + # content validation downloads using BlobDownloadContentOptions with CRC64 validation and large payload + crc64downloadcontentlg: + testScenario: downloadcontentwithcrc64 + sync: true + sizeBytes: "52428800" + downloadFaults: true + durationMin: 60 + imageBuildDir: "../../.." + + # content validation downloads using BlobDownloadContentOptions with CRC64 validation, async client, and large payload + crc64downloadcontentasynclg: + testScenario: downloadcontentwithcrc64 + sync: false + sizeBytes: "52428800" + downloadFaults: true + durationMin: 60 + imageBuildDir: "../../.." + + # content validation downloads using BlobDownloadToFileOptions with CRC64 validation + crc64downloadfilesm: + testScenario: downloadtofilewithcrc64 + sync: true + sizeBytes: 1024 + downloadFaults: true + durationMin: 25 + imageBuildDir: "../../.." + + # content validation downloads using BlobDownloadToFileOptions with CRC64 validation and async client + crc64downloadfileasyncsm: + testScenario: downloadtofilewithcrc64 + sync: false + sizeBytes: 1024 + downloadFaults: true + durationMin: 25 + imageBuildDir: "../../.." + + # content validation downloads using BlobDownloadToFileOptions with CRC64 validation and multi-block payload + crc64downloadfilemd: + testScenario: downloadtofilewithcrc64 + sync: true + sizeBytes: "16777216" + downloadFaults: true + durationMin: 60 + imageBuildDir: "../../.." + + # content validation downloads using BlobDownloadToFileOptions with CRC64 validation, async client, and multi-block payload + crc64downloadfileasyncmd: + testScenario: downloadtofilewithcrc64 + sync: false + sizeBytes: "16777216" + downloadFaults: true + durationMin: 60 + imageBuildDir: "../../.." + + # content validation downloads using BlobInputStreamOptions with CRC64 validation + crc64inputstreamsm: + testScenario: openinputstreamwithcrc64 + sync: true + sizeBytes: 1024 + downloadFaults: true + durationMin: 25 + imageBuildDir: "../../.." + + # content validation downloads using BlobInputStreamOptions with CRC64 validation and large payload + crc64inputstreamlg: + testScenario: openinputstreamwithcrc64 + sync: true + sizeBytes: "52428800" + downloadFaults: true + durationMin: 60 + imageBuildDir: "../../.." + + # content validation downloads using BlobSeekableByteChannelReadOptions with CRC64 validation + crc64bytechannelreadsm: + testScenario: openseekablebytechannelreadwithcrc64 + sync: true + sizeBytes: 1024 + downloadFaults: true + durationMin: 25 + imageBuildDir: "../../.." + + # content validation downloads using BlobSeekableByteChannelReadOptions with CRC64 validation and large payload + crc64bytechannelreadlg: + testScenario: openseekablebytechannelreadwithcrc64 + sync: true + sizeBytes: "52428800" + downloadFaults: true + durationMin: 60 + imageBuildDir: "../../.." + # this test uploads 1KB (1024 bytes) to append blob, no chunking appendblocksmall: testScenario: appendblock @@ -323,3 +467,243 @@ matrix: uploadFaults: true durationMin: 60 imageBuildDir: "../../.." + + crc64appendblock-sm: + testScenario: appendblockwithcrc64 + sync: true + sizeBytes: 1024 + uploadFaults: true + durationMin: 25 + imageBuildDir: "../../.." + + crc64appendblock-lg: + testScenario: appendblockwithcrc64 + sync: true + sizeBytes: "26214400" + uploadFaults: true + durationMin: 30 + imageBuildDir: "../../.." + + crc64appendblockasync-sm: + testScenario: appendblockwithcrc64 + sync: false + sizeBytes: 1024 + uploadFaults: true + durationMin: 25 + imageBuildDir: "../../.." + + crc64appendblockasync-lg: + testScenario: appendblockwithcrc64 + sync: false + sizeBytes: "26214400" + uploadFaults: true + durationMin: 30 + imageBuildDir: "../../.." + + crc64appendoutputstream-sm: + testScenario: appendbloboutputstreamwithcrc64 + sync: true + sizeBytes: 1024 + uploadFaults: true + durationMin: 25 + imageBuildDir: "../../.." + + crc64appendoutputstream-lg: + testScenario: appendbloboutputstreamwithcrc64 + sync: true + sizeBytes: "10240" + uploadFaults: true + durationMin: 60 + imageBuildDir: "../../.." + + crc64blockblobupload-sm: + testScenario: blockblobuploadwithcrc64 + sync: true + sizeBytes: 1024 + uploadFaults: true + durationMin: 25 + imageBuildDir: "../../.." + + crc64blockblobupload-lg: + testScenario: blockblobuploadwithcrc64 + sync: true + sizeBytes: "26214400" + uploadFaults: true + durationMin: 60 + imageBuildDir: "../../.." + + crc64blockoutputstream-sm: + testScenario: blockbloboutputstreamwithcrc64 + sync: true + sizeBytes: 1024 + uploadFaults: true + durationMin: 25 + imageBuildDir: "../../.." + + crc64blockoutputstream-lg: + testScenario: blockbloboutputstreamwithcrc64 + sync: true + sizeBytes: "26214400" + uploadFaults: true + durationMin: 60 + imageBuildDir: "../../.." + + crc64bytechannelwrite-sm: + testScenario: seekablebytechannelwritewithcrc64 + sync: true + sizeBytes: 1024 + uploadFaults: true + durationMin: 10 + imageBuildDir: "../../.." + + crc64bytechannelwrite-lg: + testScenario: seekablebytechannelwritewithcrc64 + sync: true + sizeBytes: "52428800" + uploadFaults: true + durationMin: 60 + imageBuildDir: "../../.." + + crc64stageblock-sm: + testScenario: stageblockwithcrc64 + sync: true + sizeBytes: 1024 + uploadFaults: true + durationMin: 25 + imageBuildDir: "../../.." + + crc64stageblock-lg: + testScenario: stageblockwithcrc64 + sync: true + sizeBytes: "26214400" + uploadFaults: true + durationMin: 60 + imageBuildDir: "../../.." + + crc64stageblockasync-sm: + testScenario: stageblockwithcrc64 + sync: false + sizeBytes: 1024 + uploadFaults: true + durationMin: 25 + imageBuildDir: "../../.." + + crc64stageblockasync-lg: + testScenario: stageblockwithcrc64 + sync: false + sizeBytes: "26214400" + uploadFaults: true + durationMin: 60 + imageBuildDir: "../../.." + + crc64pageoutputstream-sm: + testScenario: pagebloboutputstreamwithcrc64 + sync: true + sizeBytes: 1024 + uploadFaults: true + durationMin: 25 + imageBuildDir: "../../.." + + crc64pageoutputstream-lg: + testScenario: pagebloboutputstreamwithcrc64 + sync: true + sizeBytes: "10240" + uploadFaults: true + durationMin: 60 + imageBuildDir: "../../.." + + crc64uploadpages-sm: + testScenario: uploadpageswithcrc64 + sync: true + sizeBytes: 1024 + uploadFaults: true + durationMin: 25 + imageBuildDir: "../../.." + + crc64uploadpages-lg: + testScenario: uploadpageswithcrc64 + sync: true + sizeBytes: "4194304" + uploadFaults: true + durationMin: 60 + imageBuildDir: "../../.." + + crc64uploadpagesasync-sm: + testScenario: uploadpageswithcrc64 + sync: false + sizeBytes: 1024 + uploadFaults: true + durationMin: 25 + imageBuildDir: "../../.." + + crc64uploadpagesasync-lg: + testScenario: uploadpageswithcrc64 + sync: false + sizeBytes: "4194304" + uploadFaults: true + durationMin: 60 + imageBuildDir: "../../.." + + crc64upload-sm: + testScenario: uploadwithcrc64 + sync: true + sizeBytes: 1024 + uploadFaults: true + durationMin: 25 + imageBuildDir: "../../.." + + crc64upload-lg: + testScenario: uploadwithcrc64 + sync: true + sizeBytes: "52428800" + uploadFaults: true + durationMin: 60 + imageBuildDir: "../../.." + + crc64uploadasync-sm: + testScenario: uploadwithcrc64 + sync: false + sizeBytes: 1024 + uploadFaults: true + durationMin: 25 + imageBuildDir: "../../.." + + crc64uploadasync-lg: + testScenario: uploadwithcrc64 + sync: false + sizeBytes: "52428800" + uploadFaults: true + durationMin: 60 + imageBuildDir: "../../.." + + crc64uploadfromfile-sm: + testScenario: uploadfromfilewithcrc64 + sync: true + sizeBytes: 1024 + uploadFaults: true + durationMin: 25 + imageBuildDir: "../../.." + + crc64uploadfromfile-lg: + testScenario: uploadfromfilewithcrc64 + sync: true + sizeBytes: "52428800" + uploadFaults: true + durationMin: 60 + imageBuildDir: "../../.." + + crc64uploadfromfileasync-sm: + testScenario: uploadfromfilewithcrc64 + sync: false + sizeBytes: 1024 + uploadFaults: true + durationMin: 25 + imageBuildDir: "../../.." + + crc64uploadfromfileasync-lg: + testScenario: uploadfromfilewithcrc64 + sync: false + sizeBytes: "52428800" + uploadFaults: true + durationMin: 60 + imageBuildDir: "../../.." diff --git a/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/App.java b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/App.java index e38bd16791ca..f146945db423 100644 --- a/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/App.java +++ b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/App.java @@ -15,6 +15,21 @@ public static void main(String[] args) { BlockBlobOutputStream.class, BlockBlobUpload.class, CommitBlockList.class, + DownloadContentWithCRC64.class, + DownloadStreamWithCRC64.class, + DownloadToFileWithCRC64.class, + OpenInputStreamWithCRC64.class, + OpenSeekableByteChannelReadWithCRC64.class, + AppendBlobOutputStreamWithCRC64.class, + AppendBlockWithCRC64.class, + BlockBlobOutputStreamWithCRC64.class, + BlockBlobUploadWithCRC64.class, + PageBlobOutputStreamWithCRC64.class, + StageBlockWithCRC64.class, + SeekableByteChannelWriteWithCRC64.class, + UploadWithCRC64.class, + UploadFromFileWithCRC64.class, + UploadPagesWithCRC64.class, DownloadToFile.class, DownloadStream.class, DownloadContent.class, diff --git a/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/AppendBlobOutputStreamWithCRC64.java b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/AppendBlobOutputStreamWithCRC64.java new file mode 100644 index 000000000000..7ea8a3841f32 --- /dev/null +++ b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/AppendBlobOutputStreamWithCRC64.java @@ -0,0 +1,106 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.blob.stress; + +import com.azure.storage.common.ContentValidationAlgorithm; +import com.azure.storage.stress.StorageStressOptions; +import com.azure.core.util.Context; +import com.azure.storage.common.ContentValidationAlgorithm; +import com.azure.storage.stress.StorageStressOptions; +import com.azure.core.util.logging.ClientLogger; +import com.azure.storage.common.ContentValidationAlgorithm; +import com.azure.storage.stress.StorageStressOptions; +import com.azure.storage.blob.BlobAsyncClient; +import com.azure.storage.common.ContentValidationAlgorithm; +import com.azure.storage.stress.StorageStressOptions; +import com.azure.storage.blob.BlobClient; +import com.azure.storage.common.ContentValidationAlgorithm; +import com.azure.storage.stress.StorageStressOptions; +import com.azure.storage.blob.options.AppendBlobOutputStreamOptions; +import com.azure.storage.common.ContentValidationAlgorithm; +import com.azure.storage.stress.StorageStressOptions; +import com.azure.storage.blob.specialized.AppendBlobClient; +import com.azure.storage.common.ContentValidationAlgorithm; +import com.azure.storage.stress.StorageStressOptions; +import com.azure.storage.blob.specialized.BlobOutputStream; +import com.azure.storage.common.ContentValidationAlgorithm; +import com.azure.storage.stress.StorageStressOptions; +import com.azure.storage.blob.stress.utils.OriginalContent; +import com.azure.storage.common.ContentValidationAlgorithm; +import com.azure.storage.stress.StorageStressOptions; +import com.azure.storage.stress.CrcInputStream; +import reactor.core.publisher.Mono; + +import java.io.IOException; + +import static com.azure.core.util.FluxUtil.monoError; + +/** + * Append blob output stream with CRC64 enabled (sync only). + */ +public class AppendBlobOutputStreamWithCRC64 extends BlobScenarioBase { + private static final ClientLogger LOGGER = new ClientLogger(AppendBlobOutputStreamWithCRC64.class); + private final OriginalContent originalContent = new OriginalContent(); + private final BlobClient syncClient; + private final BlobAsyncClient asyncNoFaultClient; + /** Separate blob used to upload reference content for {@link OriginalContent} checksum (block blob). */ + private final BlobAsyncClient tempSetupBlobClient; + + public AppendBlobOutputStreamWithCRC64(StorageStressOptions options) { + super(options); + String blobName = generateBlobName(); + String tempBlobName = generateBlobName(); + + this.asyncNoFaultClient = getAsyncContainerClientNoFault().getBlobAsyncClient(blobName); + this.syncClient = getSyncContainerClient().getBlobClient(blobName); + this.tempSetupBlobClient = getAsyncContainerClientNoFault().getBlobAsyncClient(tempBlobName); + } + + @Override + protected void runInternal(Context span) throws IOException { + AppendBlobClient appendBlobClient = syncClient.getAppendBlobClient(); + // Reset the append blob at the start of each iteration. The boolean overload + // getBlobOutputStream(true) does this implicitly via create(true); the options overload + // does not, so we replicate that behavior here. Without this reset, fault-injection + // sequences that commit a block server-side but drop the response leave the cached + // appendPosition stale, causing subsequent retries to fail with 412 AppendPositionConditionNotMet, + // which combined with non-retriable Crc64Mismatch on truncated-body faults collapses the pass rate. + appendBlobClient.create(true); + + AppendBlobOutputStreamOptions streamOptions = new AppendBlobOutputStreamOptions() + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + + try (CrcInputStream inputStream = new CrcInputStream(originalContent.getBlobContentHead(), options.getSize()); + BlobOutputStream outputStream = appendBlobClient.getBlobOutputStream(streamOptions)) { + byte[] buffer = new byte[4096]; + int bytesRead; + + while ((bytesRead = inputStream.read(buffer)) != -1) { + outputStream.write(buffer, 0, bytesRead); + } + + outputStream.close(); + originalContent.checkMatch(inputStream.getContentInfo(), span).block(); + } + } + + @Override + protected Mono runInternalAsync(Context span) { + return monoError(LOGGER, new RuntimeException("getBlobOutputStream() does not exist on the async client")); + } + + @Override + public Mono setupAsync() { + return super.setupAsync() + .then(asyncNoFaultClient.getAppendBlobAsyncClient().create()) + .then(originalContent.setupBlob(tempSetupBlobClient, options.getSize())); + } + + @Override + public Mono cleanupAsync() { + return asyncNoFaultClient.deleteIfExists() + .then(tempSetupBlobClient.deleteIfExists()) + .then(super.cleanupAsync()); + } +} diff --git a/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/AppendBlockWithCRC64.java b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/AppendBlockWithCRC64.java new file mode 100644 index 000000000000..360d19995080 --- /dev/null +++ b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/AppendBlockWithCRC64.java @@ -0,0 +1,78 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.blob.stress; + +import com.azure.core.util.Context; +import com.azure.storage.blob.BlobAsyncClient; +import com.azure.storage.blob.BlobClient; +import com.azure.storage.blob.options.AppendBlobAppendBlockOptions; +import com.azure.storage.blob.specialized.AppendBlobAsyncClient; +import com.azure.storage.blob.specialized.AppendBlobClient; +import com.azure.storage.blob.stress.utils.OriginalContent; +import com.azure.storage.common.ContentValidationAlgorithm; +import com.azure.storage.stress.CrcInputStream; +import com.azure.storage.stress.StorageStressOptions; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; + +import java.nio.ByteBuffer; + +/** + * Append block with CRC64 enabled. + */ +public class AppendBlockWithCRC64 extends BlobScenarioBase { + private final OriginalContent originalContent = new OriginalContent(); + private final BlobClient syncClient; + private final BlobAsyncClient asyncClient; + private final BlobAsyncClient asyncNoFaultClient; + private final BlobAsyncClient tempSetupBlobClient; + + public AppendBlockWithCRC64(StorageStressOptions options) { + super(options); + String blobName = generateBlobName(); + String tempBlobName = generateBlobName(); + + this.asyncNoFaultClient = getAsyncContainerClientNoFault().getBlobAsyncClient(blobName); + this.syncClient = getSyncContainerClient().getBlobClient(blobName); + this.asyncClient = getAsyncContainerClient().getBlobAsyncClient(blobName); + this.tempSetupBlobClient = getAsyncContainerClientNoFault().getBlobAsyncClient(tempBlobName); + } + + @Override + protected void runInternal(Context span) { + try (CrcInputStream inputStream = new CrcInputStream(originalContent.getBlobContentHead(), options.getSize())) { + AppendBlobClient appendBlobClient = syncClient.getAppendBlobClient(); + appendBlobClient.appendBlockWithResponse( + new AppendBlobAppendBlockOptions(inputStream, options.getSize()) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64), + null, span); + originalContent.checkMatch(inputStream.getContentInfo(), span).block(); + } + } + + @Override + protected Mono runInternalAsync(Context span) { + AppendBlobAsyncClient appendBlobAsyncClient = asyncClient.getAppendBlobAsyncClient(); + Flux byteBufferFlux = new CrcInputStream(originalContent.getBlobContentHead(), options.getSize()) + .convertStreamToByteBuffer(); + return appendBlobAsyncClient.appendBlockWithResponse( + new AppendBlobAppendBlockOptions(byteBufferFlux, options.getSize()) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64)) + .then(originalContent.checkMatch(byteBufferFlux, span)); + } + + @Override + public Mono setupAsync() { + return super.setupAsync().then(asyncNoFaultClient.getAppendBlobAsyncClient().create()) + .then(originalContent.setupBlob(tempSetupBlobClient, options.getSize())); + } + + @Override + public Mono cleanupAsync() { + return asyncNoFaultClient.getAppendBlobAsyncClient().deleteIfExists() + .onErrorResume(e -> Mono.empty()) + .then(tempSetupBlobClient.deleteIfExists()) + .then(super.cleanupAsync()); + } +} diff --git a/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/BlockBlobOutputStreamWithCRC64.java b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/BlockBlobOutputStreamWithCRC64.java new file mode 100644 index 000000000000..1873ac740d72 --- /dev/null +++ b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/BlockBlobOutputStreamWithCRC64.java @@ -0,0 +1,77 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.blob.stress; + +import com.azure.core.util.Context; +import com.azure.core.util.logging.ClientLogger; +import com.azure.storage.blob.BlobAsyncClient; +import com.azure.storage.blob.BlobClient; +import com.azure.storage.blob.models.ParallelTransferOptions; +import com.azure.storage.blob.options.BlockBlobOutputStreamOptions; +import com.azure.storage.blob.specialized.BlobOutputStream; +import com.azure.storage.blob.specialized.BlockBlobClient; +import com.azure.storage.blob.stress.utils.OriginalContent; +import com.azure.storage.common.ContentValidationAlgorithm; +import com.azure.storage.stress.CrcInputStream; +import com.azure.storage.stress.StorageStressOptions; +import reactor.core.publisher.Mono; + +import java.io.IOException; + +import static com.azure.core.util.FluxUtil.monoError; + +/** + * Block blob output stream with CRC64 enabled (sync only). + */ +public class BlockBlobOutputStreamWithCRC64 extends BlobScenarioBase { + private static final ClientLogger LOGGER = new ClientLogger(BlockBlobOutputStreamWithCRC64.class); + private final OriginalContent originalContent = new OriginalContent(); + private final BlobClient syncClient; + private final BlobAsyncClient asyncNoFaultClient; + private final ParallelTransferOptions parallelTransferOptions; + + public BlockBlobOutputStreamWithCRC64(StorageStressOptions options) { + super(options); + String blobName = generateBlobName(); + this.asyncNoFaultClient = getAsyncContainerClientNoFault().getBlobAsyncClient(blobName); + this.syncClient = getSyncContainerClient().getBlobClient(blobName); + this.parallelTransferOptions = new ParallelTransferOptions().setMaxConcurrency(options.getMaxConcurrency()); + } + + @Override + protected void runInternal(Context span) throws IOException { + BlockBlobClient blockBlobClient = syncClient.getBlockBlobClient(); + BlockBlobOutputStreamOptions streamOptions = new BlockBlobOutputStreamOptions() + .setParallelTransferOptions(parallelTransferOptions) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + + try (CrcInputStream inputStream = new CrcInputStream(originalContent.getBlobContentHead(), options.getSize()); + BlobOutputStream outputStream = blockBlobClient.getBlobOutputStream(streamOptions, span)) { + byte[] buffer = new byte[4096]; + int bytesRead; + + while ((bytesRead = inputStream.read(buffer)) != -1) { + outputStream.write(buffer, 0, bytesRead); + } + + outputStream.close(); + originalContent.checkMatch(inputStream.getContentInfo(), span).block(); + } + } + + @Override + protected Mono runInternalAsync(Context span) { + return monoError(LOGGER, new RuntimeException("getBlobOutputStream() does not exist on the async client")); + } + + @Override + public Mono setupAsync() { + return super.setupAsync().then(originalContent.setupBlob(asyncNoFaultClient, options.getSize())); + } + + @Override + public Mono cleanupAsync() { + return asyncNoFaultClient.deleteIfExists().then(super.cleanupAsync()); + } +} diff --git a/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/BlockBlobUploadWithCRC64.java b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/BlockBlobUploadWithCRC64.java new file mode 100644 index 000000000000..b88b9b5c808d --- /dev/null +++ b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/BlockBlobUploadWithCRC64.java @@ -0,0 +1,71 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.blob.stress; + +import com.azure.core.util.Context; +import com.azure.storage.blob.BlobAsyncClient; +import com.azure.storage.blob.BlobClient; +import com.azure.storage.blob.options.BlockBlobSimpleUploadOptions; +import com.azure.storage.blob.specialized.BlockBlobAsyncClient; +import com.azure.storage.blob.specialized.BlockBlobClient; +import com.azure.storage.blob.stress.utils.OriginalContent; +import com.azure.storage.common.ContentValidationAlgorithm; +import com.azure.storage.stress.CrcInputStream; +import com.azure.storage.stress.StorageStressOptions; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; + +import java.nio.ByteBuffer; + +/** + * Single-shot block blob upload with request content validation with CRC64 enabled. + */ +public class BlockBlobUploadWithCRC64 extends BlobScenarioBase { + private final OriginalContent originalContent = new OriginalContent(); + private final BlobClient syncClient; + private final BlobAsyncClient asyncClient; + private final BlobAsyncClient asyncNoFaultClient; + + public BlockBlobUploadWithCRC64(StorageStressOptions options) { + super(options); + String blobName = generateBlobName(); + this.asyncNoFaultClient = getAsyncContainerClientNoFault().getBlobAsyncClient(blobName); + this.syncClient = getSyncContainerClient().getBlobClient(blobName); + this.asyncClient = getAsyncContainerClient().getBlobAsyncClient(blobName); + } + + @Override + protected void runInternal(Context span) { + BlockBlobClient blockBlobClient = syncClient.getBlockBlobClient(); + try (CrcInputStream inputStream = new CrcInputStream(originalContent.getBlobContentHead(), options.getSize())) { + blockBlobClient.uploadWithResponse( + new BlockBlobSimpleUploadOptions(inputStream, options.getSize()) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64), + null, span); + originalContent.checkMatch(inputStream.getContentInfo(), span).block(); + } + } + + @Override + protected Mono runInternalAsync(Context span) { + Flux byteBufferFlux = new CrcInputStream(originalContent.getBlobContentHead(), options.getSize()) + .convertStreamToByteBuffer(); + BlockBlobAsyncClient blockBlobAsyncClient = asyncClient.getBlockBlobAsyncClient(); + return blockBlobAsyncClient.uploadWithResponse( + new BlockBlobSimpleUploadOptions(byteBufferFlux, options.getSize()) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64)) + .then(originalContent.checkMatch(byteBufferFlux, span)); + } + + @Override + public Mono setupAsync() { + return super.setupAsync().then(originalContent.setupBlob(asyncNoFaultClient, options.getSize())); + } + + @Override + public Mono cleanupAsync() { + return asyncNoFaultClient.deleteIfExists() + .then(super.cleanupAsync()); + } +} diff --git a/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/DownloadContentWithCRC64.java b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/DownloadContentWithCRC64.java new file mode 100644 index 000000000000..6c92e1563c60 --- /dev/null +++ b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/DownloadContentWithCRC64.java @@ -0,0 +1,70 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.blob.stress; + +import com.azure.core.http.HttpHeaderName; +import com.azure.core.util.BinaryData; +import com.azure.core.util.Context; +import com.azure.storage.blob.BlobAsyncClient; +import com.azure.storage.blob.BlobClient; +import com.azure.storage.blob.options.BlobDownloadContentOptions; +import com.azure.storage.blob.options.BlobDownloadStreamOptions; +import com.azure.storage.blob.stress.utils.OriginalContent; +import com.azure.storage.common.ContentValidationAlgorithm; +import com.azure.storage.stress.StorageStressOptions; +import reactor.core.publisher.Mono; + +/** + * Download content with CRC64 Algorithm enabled. + * Verifies the correctness of the download response content via CRC. + */ +public class DownloadContentWithCRC64 extends BlobScenarioBase { + private final OriginalContent originalContent = new OriginalContent(); + private final BlobClient syncClient; + private final BlobAsyncClient asyncClient; + private final BlobAsyncClient asyncNoFaultClient; + + public DownloadContentWithCRC64(StorageStressOptions options) { + super(options); + String blobName = generateBlobName(); + this.asyncNoFaultClient = getAsyncContainerClientNoFault().getBlobAsyncClient(blobName); + this.syncClient = getSyncContainerClient().getBlobClient(blobName); + this.asyncClient = getAsyncContainerClient().getBlobAsyncClient(blobName); + } + + @Override + protected void runInternal(Context span) { + originalContent.checkMatch( + syncClient.downloadContentWithResponse( + new BlobDownloadContentOptions() + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64), + null, span).getValue(), + span).block(); + } + + @Override + protected Mono runInternalAsync(Context span) { + // TODO return downloadContent once it stops buffering. + return asyncClient.downloadStreamWithResponse( + new BlobDownloadStreamOptions() + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64)) + .flatMap(response -> { + long contentLength = Long.valueOf(response.getHeaders().getValue(HttpHeaderName.CONTENT_LENGTH)); + return BinaryData.fromFlux(response.getValue(), contentLength, false); + }) + .flatMap(bd -> originalContent.checkMatch(bd, span)); + } + + @Override + public Mono setupAsync() { + return super.setupAsync() + .then(originalContent.setupBlob(asyncNoFaultClient, options.getSize())); + } + + @Override + public Mono cleanupAsync() { + return asyncNoFaultClient.deleteIfExists() + .then(super.cleanupAsync()); + } +} diff --git a/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/DownloadStreamWithCRC64.java b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/DownloadStreamWithCRC64.java new file mode 100644 index 000000000000..64cccb2c7dd4 --- /dev/null +++ b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/DownloadStreamWithCRC64.java @@ -0,0 +1,67 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.blob.stress; + +import com.azure.core.util.Context; +import com.azure.storage.blob.BlobAsyncClient; +import com.azure.storage.blob.BlobClient; +import com.azure.storage.blob.options.BlobDownloadStreamOptions; +import com.azure.storage.blob.stress.utils.OriginalContent; +import com.azure.storage.common.ContentValidationAlgorithm; +import com.azure.storage.stress.CrcOutputStream; +import com.azure.storage.stress.StorageStressOptions; +import reactor.core.publisher.Mono; + +import java.io.IOException; + +/** + * Streaming blob download with CRC64 Algorithm enabled. + * Verifies the correctness of the download response content via CRC. + */ +public class DownloadStreamWithCRC64 extends BlobScenarioBase { + private final OriginalContent originalContent = new OriginalContent(); + private final BlobClient syncClient; + private final BlobAsyncClient asyncClient; + private final BlobAsyncClient asyncNoFaultClient; + + public DownloadStreamWithCRC64(StorageStressOptions options) { + super(options); + String blobName = generateBlobName(); + this.asyncNoFaultClient = getAsyncContainerClientNoFault().getBlobAsyncClient(blobName); + this.syncClient = getSyncContainerClient().getBlobClient(blobName); + this.asyncClient = getAsyncContainerClient().getBlobAsyncClient(blobName); + } + + @Override + protected void runInternal(Context span) throws IOException { + try (CrcOutputStream outputStream = new CrcOutputStream()) { + syncClient.downloadStreamWithResponse(outputStream, + new BlobDownloadStreamOptions() + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64), + null, span); + outputStream.close(); + originalContent.checkMatch(outputStream.getContentInfo(), span).block(); + } + } + + @Override + protected Mono runInternalAsync(Context span) { + return asyncClient.downloadStreamWithResponse( + new BlobDownloadStreamOptions() + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64)) + .flatMap(response -> originalContent.checkMatch(response.getValue(), span)); + } + + @Override + public Mono setupAsync() { + return super.setupAsync() + .then(originalContent.setupBlob(asyncNoFaultClient, options.getSize())); + } + + @Override + public Mono cleanupAsync() { + return asyncNoFaultClient.deleteIfExists() + .then(super.cleanupAsync()); + } +} diff --git a/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/DownloadToFileWithCRC64.java b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/DownloadToFileWithCRC64.java new file mode 100644 index 000000000000..ceb2faa8b153 --- /dev/null +++ b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/DownloadToFileWithCRC64.java @@ -0,0 +1,105 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.blob.stress; + +import com.azure.core.util.BinaryData; +import com.azure.core.util.Context; +import com.azure.core.util.logging.ClientLogger; +import com.azure.storage.blob.BlobAsyncClient; +import com.azure.storage.blob.BlobClient; +import com.azure.storage.blob.options.BlobDownloadToFileOptions; +import com.azure.storage.blob.stress.utils.OriginalContent; +import com.azure.storage.common.ContentValidationAlgorithm; +import com.azure.storage.common.ParallelTransferOptions; +import com.azure.storage.stress.StorageStressOptions; +import reactor.core.publisher.Mono; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; +import java.util.UUID; + +/** + * Download to file with CRC64 Algorithm enabled. + * Verifies the correctness of the download response content via CRC. + */ +public class DownloadToFileWithCRC64 extends BlobScenarioBase { + private static final ClientLogger LOGGER = new ClientLogger(DownloadToFileWithCRC64.class); + private final Path directoryPath; + private final OriginalContent originalContent = new OriginalContent(); + private final BlobClient syncClient; + private final BlobAsyncClient asyncClient; + private final BlobAsyncClient asyncNoFaultClient; + private final ParallelTransferOptions parallelTransferOptions; + + public DownloadToFileWithCRC64(StorageStressOptions options) { + super(options); + this.directoryPath = getTempPath("test"); + String blobName = generateBlobName(); + this.asyncNoFaultClient = getAsyncContainerClientNoFault().getBlobAsyncClient(blobName); + this.syncClient = getSyncContainerClient().getBlobClient(blobName); + this.asyncClient = getAsyncContainerClient().getBlobAsyncClient(blobName); + this.parallelTransferOptions = new ParallelTransferOptions() + .setMaxConcurrency(options.getMaxConcurrency()); + } + + @Override + protected void runInternal(Context span) { + Path downloadPath = directoryPath.resolve(UUID.randomUUID() + ".txt"); + BlobDownloadToFileOptions blobOptions = new BlobDownloadToFileOptions(downloadPath.toString()) + .setParallelTransferOptions(parallelTransferOptions) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + + try { + syncClient.downloadToFileWithResponse(blobOptions, Duration.ofSeconds(options.getDuration()), span); + originalContent.checkMatch(BinaryData.fromFile(downloadPath), span).block(); + } finally { + deleteFile(downloadPath); + } + } + + @Override + protected Mono runInternalAsync(Context span) { + return Mono.using( + () -> directoryPath.resolve(UUID.randomUUID() + ".txt"), + path -> asyncClient.downloadToFileWithResponse( + new BlobDownloadToFileOptions(path.toString()) + .setParallelTransferOptions(parallelTransferOptions) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64)) + .flatMap(ignored -> originalContent.checkMatch(BinaryData.fromFile(path), span)), + DownloadToFileWithCRC64::deleteFile); + } + + @Override + public Mono setupAsync() { + return super.setupAsync() + .then(originalContent.setupBlob(asyncNoFaultClient, options.getSize())); + } + + @Override + public Mono cleanupAsync() { + return asyncNoFaultClient.deleteIfExists() + .then(super.cleanupAsync()); + } + + private Path getTempPath(String prefix) { + try { + return Files.createTempDirectory(prefix); + } catch (IOException e) { + throw LOGGER.logExceptionAsError(new UncheckedIOException(e)); + } + } + + private static void deleteFile(Path path) { + try { + Files.deleteIfExists(path); + } catch (Throwable e) { + LOGGER.atError() + .addKeyValue("path", path) + .log("failed to delete file", e); + } + } +} diff --git a/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/OpenInputStreamWithCRC64.java b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/OpenInputStreamWithCRC64.java new file mode 100644 index 000000000000..eaae1570560b --- /dev/null +++ b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/OpenInputStreamWithCRC64.java @@ -0,0 +1,71 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.blob.stress; + +import com.azure.core.util.Context; +import com.azure.core.util.logging.ClientLogger; +import com.azure.storage.blob.BlobAsyncClient; +import com.azure.storage.blob.BlobClient; +import com.azure.storage.blob.options.BlobInputStreamOptions; +import com.azure.storage.blob.stress.utils.OriginalContent; +import com.azure.storage.common.ContentValidationAlgorithm; +import com.azure.storage.stress.CrcInputStream; +import com.azure.storage.stress.StorageStressOptions; +import reactor.core.publisher.Mono; + +import java.io.IOException; +import java.io.InputStream; + +import static com.azure.core.util.FluxUtil.monoError; + +/** + * Open input stream with CRC64 Algorithm enabled (sync only). + * Verifies the correctness of the download response content via CRC. + */ +public class OpenInputStreamWithCRC64 extends BlobScenarioBase { + private static final ClientLogger LOGGER = new ClientLogger(OpenInputStreamWithCRC64.class); + private final OriginalContent originalContent = new OriginalContent(); + private final BlobClient syncClient; + private final BlobAsyncClient asyncNoFaultClient; + + public OpenInputStreamWithCRC64(StorageStressOptions options) { + super(options); + String blobName = generateBlobName(); + this.syncClient = getSyncContainerClient().getBlobClient(blobName); + this.asyncNoFaultClient = getAsyncContainerClientNoFault().getBlobAsyncClient(blobName); + } + + @Override + protected void runInternal(Context span) throws IOException { + try (InputStream stream = syncClient.openInputStream( + new BlobInputStreamOptions() + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64), + span)) { + try (CrcInputStream crcStream = new CrcInputStream(stream)) { + byte[] buffer = new byte[8192]; + while (crcStream.read(buffer) != -1) { + // do nothing + } + originalContent.checkMatch(crcStream.getContentInfo(), span).block(); + } + } + } + + @Override + protected Mono runInternalAsync(Context span) { + return monoError(LOGGER, new RuntimeException("openInputStream() does not exist on the async client")); + } + + @Override + public Mono setupAsync() { + return super.setupAsync() + .then(originalContent.setupBlob(asyncNoFaultClient, options.getSize())); + } + + @Override + public Mono cleanupAsync() { + return asyncNoFaultClient.deleteIfExists() + .then(super.cleanupAsync()); + } +} diff --git a/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/OpenSeekableByteChannelReadWithCRC64.java b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/OpenSeekableByteChannelReadWithCRC64.java new file mode 100644 index 000000000000..e6376238fe92 --- /dev/null +++ b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/OpenSeekableByteChannelReadWithCRC64.java @@ -0,0 +1,73 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.blob.stress; + +import com.azure.core.util.Context; +import com.azure.core.util.logging.ClientLogger; +import com.azure.storage.blob.BlobAsyncClient; +import com.azure.storage.blob.BlobClient; +import com.azure.storage.blob.models.BlobSeekableByteChannelReadResult; +import com.azure.storage.blob.options.BlobSeekableByteChannelReadOptions; +import com.azure.storage.blob.stress.utils.OriginalContent; +import com.azure.storage.common.ContentValidationAlgorithm; +import com.azure.storage.stress.CrcInputStream; +import com.azure.storage.stress.StorageStressOptions; +import reactor.core.publisher.Mono; + +import java.io.IOException; +import java.nio.channels.Channels; + +import static com.azure.core.util.FluxUtil.monoError; + +/** + * Seekable byte channel read with CRC64 Algorithm enabled (sync only). + * Verifies the correctness of the download response content via CRC. + */ +public class OpenSeekableByteChannelReadWithCRC64 + extends BlobScenarioBase { + private static final ClientLogger LOGGER = new ClientLogger(OpenSeekableByteChannelReadWithCRC64.class); + private final OriginalContent originalContent = new OriginalContent(); + private final BlobClient syncClient; + private final BlobAsyncClient asyncNoFaultClient; + + public OpenSeekableByteChannelReadWithCRC64(StorageStressOptions options) { + super(options); + String blobName = generateBlobName(); + this.asyncNoFaultClient = getAsyncContainerClientNoFault().getBlobAsyncClient(blobName); + this.syncClient = getSyncContainerClient().getBlobClient(blobName); + } + + @Override + protected void runInternal(Context span) throws IOException { + BlobSeekableByteChannelReadResult result = syncClient.openSeekableByteChannelRead( + new BlobSeekableByteChannelReadOptions() + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64), + span); + try (CrcInputStream crcStream = new CrcInputStream(Channels.newInputStream(result.getChannel()))) { + byte[] buffer = new byte[8192]; + while (crcStream.read(buffer) != -1) { + // do nothing + } + originalContent.checkMatch(crcStream.getContentInfo(), span).block(); + } + } + + @Override + protected Mono runInternalAsync(Context span) { + return monoError(LOGGER, + new RuntimeException("openSeekableByteChannelRead() does not exist on the async client")); + } + + @Override + public Mono setupAsync() { + return super.setupAsync() + .then(originalContent.setupBlob(asyncNoFaultClient, options.getSize())); + } + + @Override + public Mono cleanupAsync() { + return asyncNoFaultClient.deleteIfExists() + .then(super.cleanupAsync()); + } +} diff --git a/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/PageBlobOutputStreamWithCRC64.java b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/PageBlobOutputStreamWithCRC64.java new file mode 100644 index 000000000000..f4dda3295b18 --- /dev/null +++ b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/PageBlobOutputStreamWithCRC64.java @@ -0,0 +1,107 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.blob.stress; + +import com.azure.core.util.Context; +import com.azure.core.util.logging.ClientLogger; +import com.azure.storage.blob.BlobAsyncClient; +import com.azure.storage.blob.BlobClient; +import com.azure.storage.blob.models.PageRange; +import com.azure.storage.blob.options.PageBlobOutputStreamOptions; +import com.azure.storage.blob.specialized.BlobOutputStream; +import com.azure.storage.blob.specialized.PageBlobAsyncClient; +import com.azure.storage.blob.specialized.PageBlobClient; +import com.azure.storage.blob.stress.utils.OriginalContent; +import com.azure.storage.common.ContentValidationAlgorithm; +import com.azure.storage.stress.CrcInputStream; +import com.azure.storage.stress.StorageStressOptions; +import reactor.core.publisher.Mono; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; + +import static com.azure.core.util.FluxUtil.monoError; + +/** + * Page blob output stream with CRC64 enabled (sync only). + */ +public class PageBlobOutputStreamWithCRC64 extends PageBlobScenarioBase { + private static final ClientLogger LOGGER = new ClientLogger(PageBlobOutputStreamWithCRC64.class); + private final OriginalContent originalContent = new OriginalContent(); + private final BlobClient syncClient; + private final BlobAsyncClient asyncNoFaultClient; + /** Page blob used only to seed {@link OriginalContent} (same pattern as {@link PageBlobOutputStream}). */ + private final PageBlobAsyncClient tempSetupPageBlobClient; + + public PageBlobOutputStreamWithCRC64(StorageStressOptions options) { + super(options); + String blobName = generateBlobName(); + String tempBlobName = generateBlobName(); + + this.syncClient = getSyncContainerClient().getBlobClient(blobName); + this.asyncNoFaultClient = getAsyncContainerClientNoFault().getBlobAsyncClient(blobName); + BlobAsyncClient tempSetupBlobClient = getAsyncContainerClientNoFault().getBlobAsyncClient(tempBlobName); + this.tempSetupPageBlobClient = tempSetupBlobClient.getPageBlobAsyncClient(); + } + + @Override + protected void runInternal(Context span) throws IOException { + PageBlobClient pageBlobClient = syncClient.getPageBlobClient(); + PageBlobOutputStreamOptions streamOptions = new PageBlobOutputStreamOptions( + new PageRange().setStart(0).setEnd(options.getSize() - 1)) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + + try (CrcInputStream inputStream = new CrcInputStream(originalContent.getBlobContentHead(), options.getSize()); + BlobOutputStream outputStream = pageBlobClient.getBlobOutputStream(streamOptions)) { + ByteArrayOutputStream bufferStream = new ByteArrayOutputStream(); + byte[] buffer = new byte[512]; + int bytesRead; + + while ((bytesRead = inputStream.read(buffer)) != -1) { + // Always accumulate into bufferStream to avoid dropping or reordering bytes + bufferStream.write(buffer, 0, bytesRead); + // Flush all full 512-byte pages from the accumulator + if (bufferStream.size() >= buffer.length) { + byte[] toWrite = bufferStream.toByteArray(); + int length = toWrite.length - (toWrite.length % buffer.length); + if (length > 0) { + outputStream.write(toWrite, 0, length); + bufferStream.reset(); + // Keep any remaining partial page bytes in the accumulator + bufferStream.write(toWrite, length, toWrite.length - length); + } + } + } + // For page blobs, total content size must be a multiple of 512 bytes. + // Any remaining bytes here indicate misalignment and would result in silent truncation. + if (bufferStream.size() != 0) { + throw new IOException("Remaining bytes in buffer that do not align to 512-byte page size."); + } + + outputStream.close(); + originalContent.checkMatch(inputStream.getContentInfo(), span).block(); + } + } + + @Override + protected Mono runInternalAsync(Context span) { + return monoError(LOGGER, new RuntimeException("getBlobOutputStream() does not exist on the async client")); + } + + @Override + public Mono setupAsync() { + return super.setupAsync() + .then(asyncNoFaultClient.getPageBlobAsyncClient().create(options.getSize())) + .then(tempSetupPageBlobClient.create(options.getSize())) + .then(originalContent.setupPageBlob(tempSetupPageBlobClient, options.getSize())); + } + + @Override + public Mono cleanupAsync() { + return asyncNoFaultClient.getPageBlobAsyncClient().deleteIfExists() + .onErrorResume(e -> Mono.empty()) + .then(tempSetupPageBlobClient.deleteIfExists()) + .then(super.cleanupAsync()); + } +} diff --git a/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/SeekableByteChannelWriteWithCRC64.java b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/SeekableByteChannelWriteWithCRC64.java new file mode 100644 index 000000000000..5f345bbfd5f1 --- /dev/null +++ b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/SeekableByteChannelWriteWithCRC64.java @@ -0,0 +1,84 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.blob.stress; + +import com.azure.core.util.Context; +import com.azure.core.util.logging.ClientLogger; +import com.azure.storage.blob.BlobAsyncClient; +import com.azure.storage.blob.BlobClient; +import com.azure.storage.blob.options.BlockBlobSeekableByteChannelWriteOptions; +import com.azure.storage.blob.specialized.BlockBlobClient; +import com.azure.storage.blob.stress.utils.OriginalContent; +import com.azure.storage.common.ContentValidationAlgorithm; +import com.azure.storage.common.implementation.StorageSeekableByteChannel; +import com.azure.storage.stress.CrcInputStream; +import com.azure.storage.stress.StorageStressOptions; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; + +import java.io.IOException; +import java.nio.ByteBuffer; + +import static com.azure.core.util.FluxUtil.monoError; +import static com.azure.storage.blob.options.BlockBlobSeekableByteChannelWriteOptions.WriteMode.OVERWRITE; + +/** + * Block-blob seekable byte channel write with CRC64 enabled (sync only). + */ +public class SeekableByteChannelWriteWithCRC64 extends BlobScenarioBase { + private static final ClientLogger LOGGER = new ClientLogger(SeekableByteChannelWriteWithCRC64.class); + private final OriginalContent originalContent = new OriginalContent(); + private final BlobClient syncClient; + private final BlobAsyncClient asyncNoFaultClient; + + public SeekableByteChannelWriteWithCRC64(StorageStressOptions options) { + super(options); + String blobName = generateBlobName(); + this.syncClient = getSyncContainerClient().getBlobClient(blobName); + this.asyncNoFaultClient = getAsyncContainerClientNoFault().getBlobAsyncClient(blobName); + } + + @Override + protected void runInternal(Context span) throws IOException { + BlockBlobClient blockBlobClient = syncClient.getBlockBlobClient(); + BlockBlobSeekableByteChannelWriteOptions writeOptions = new BlockBlobSeekableByteChannelWriteOptions(OVERWRITE) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + + try (CrcInputStream crcInput = new CrcInputStream(originalContent.getBlobContentHead(), options.getSize())) { + Flux byteBufferFlux = crcInput.convertStreamToByteBuffer(); + try (StorageSeekableByteChannel channel = (StorageSeekableByteChannel) blockBlobClient.openSeekableByteChannelWrite( + writeOptions)) { + Mono writeOperation = byteBufferFlux + .doOnNext(buffer -> { + try { + while (buffer.hasRemaining()) { + channel.write(buffer); + } + } catch (IOException e) { + throw LOGGER.logExceptionAsError(new RuntimeException(e)); + } + }).then(); + writeOperation.block(); + channel.getWriteBehavior().commit(options.getSize()); + } + originalContent.checkMatch(byteBufferFlux, span).block(); + } + } + + @Override + protected Mono runInternalAsync(Context span) { + return monoError(LOGGER, new RuntimeException( + "openSeekableByteChannelWrite() does not exist on the async client")); + } + + @Override + public Mono setupAsync() { + return super.setupAsync().then(originalContent.setupBlob(asyncNoFaultClient, options.getSize())); + } + + @Override + public Mono cleanupAsync() { + return asyncNoFaultClient.deleteIfExists().then(super.cleanupAsync()); + } +} diff --git a/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/StageBlockWithCRC64.java b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/StageBlockWithCRC64.java new file mode 100644 index 000000000000..c06bbe6a44b6 --- /dev/null +++ b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/StageBlockWithCRC64.java @@ -0,0 +1,91 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.blob.stress; + +import com.azure.core.util.BinaryData; +import com.azure.core.util.Context; +import com.azure.core.util.CoreUtils; +import com.azure.storage.blob.BlobAsyncClient; +import com.azure.storage.blob.BlobClient; +import com.azure.storage.blob.options.BlockBlobCommitBlockListOptions; +import com.azure.storage.blob.options.BlockBlobStageBlockOptions; +import com.azure.storage.blob.specialized.BlockBlobAsyncClient; +import com.azure.storage.blob.specialized.BlockBlobClient; +import com.azure.storage.blob.stress.utils.OriginalContent; +import com.azure.storage.common.ContentValidationAlgorithm; +import com.azure.storage.stress.CrcInputStream; +import com.azure.storage.stress.StorageStressOptions; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; + +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.Base64; +import java.util.Collections; + +/** + * Stage block with CRC64 enabled on the faulted client, then commit via the non-faulted client. + */ +public class StageBlockWithCRC64 extends BlobScenarioBase { + private final OriginalContent originalContent = new OriginalContent(); + private final BlobClient syncClient; + private final BlobAsyncClient asyncClient; + private final BlobClient syncNoFaultClient; + private final BlobAsyncClient asyncNoFaultClient; + + public StageBlockWithCRC64(StorageStressOptions options) { + super(options); + String blobName = generateBlobName(); + this.syncNoFaultClient = getSyncContainerClientNoFault().getBlobClient(blobName); + this.asyncNoFaultClient = getAsyncContainerClientNoFault().getBlobAsyncClient(blobName); + this.syncClient = getSyncContainerClient().getBlobClient(blobName); + this.asyncClient = getAsyncContainerClient().getBlobAsyncClient(blobName); + } + + @Override + protected void runInternal(Context span) { + BlockBlobClient blockBlobClient = syncClient.getBlockBlobClient(); + BlockBlobClient blockBlobClientNoFault = syncNoFaultClient.getBlockBlobClient(); + String blockId = Base64.getEncoder().encodeToString(CoreUtils.randomUuid().toString() + .getBytes(StandardCharsets.UTF_8)); + try (CrcInputStream inputStream = new CrcInputStream(originalContent.getBlobContentHead(), options.getSize())) { + BinaryData data = BinaryData.fromStream(inputStream, options.getSize()); + blockBlobClient.stageBlockWithResponse( + new BlockBlobStageBlockOptions(blockId, data) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64), + null, span); + blockBlobClientNoFault.commitBlockListWithResponse( + new BlockBlobCommitBlockListOptions(Collections.singletonList(blockId)), null, span); + originalContent.checkMatch(inputStream.getContentInfo(), span).block(); + } + } + + @Override + protected Mono runInternalAsync(Context span) { + BlockBlobAsyncClient blockBlobAsyncClient = asyncClient.getBlockBlobAsyncClient(); + BlockBlobAsyncClient blockBlobAsyncClientNoFault = asyncNoFaultClient.getBlockBlobAsyncClient(); + String blockId = Base64.getEncoder().encodeToString(CoreUtils.randomUuid().toString() + .getBytes(StandardCharsets.UTF_8)); + Flux byteBufferFlux = new CrcInputStream(originalContent.getBlobContentHead(), options.getSize()) + .convertStreamToByteBuffer(); + return BinaryData.fromFlux(byteBufferFlux, options.getSize(), false) + .flatMap(binaryData -> blockBlobAsyncClient.stageBlockWithResponse( + new BlockBlobStageBlockOptions(blockId, binaryData) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64))) + .then(blockBlobAsyncClientNoFault.commitBlockListWithResponse( + new BlockBlobCommitBlockListOptions(Collections.singletonList(blockId)))) + .then(originalContent.checkMatch(byteBufferFlux, span)); + } + + @Override + public Mono setupAsync() { + return super.setupAsync().then(originalContent.setupBlob(asyncNoFaultClient, options.getSize())); + } + + @Override + public Mono cleanupAsync() { + return asyncNoFaultClient.deleteIfExists() + .then(super.cleanupAsync()); + } +} diff --git a/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/UploadFromFileWithCRC64.java b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/UploadFromFileWithCRC64.java new file mode 100644 index 000000000000..b9594326e6ad --- /dev/null +++ b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/UploadFromFileWithCRC64.java @@ -0,0 +1,140 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.blob.stress; + +import com.azure.core.util.BinaryData; +import com.azure.core.util.Context; +import com.azure.core.util.CoreUtils; +import com.azure.core.util.logging.ClientLogger; +import com.azure.storage.blob.BlobAsyncClient; +import com.azure.storage.blob.BlobClient; +import com.azure.storage.blob.models.ParallelTransferOptions; +import com.azure.storage.blob.options.BlobDownloadToFileOptions; +import com.azure.storage.blob.options.BlobUploadFromFileOptions; +import com.azure.storage.blob.stress.utils.OriginalContent; +import com.azure.storage.common.ContentValidationAlgorithm; +import com.azure.storage.stress.CrcInputStream; +import com.azure.storage.stress.StorageStressOptions; +import reactor.core.publisher.Mono; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.util.UUID; + +/** + * Upload from file with CRC64 enabled. + */ +public class UploadFromFileWithCRC64 extends BlobScenarioBase { + private static final ClientLogger LOGGER = new ClientLogger(UploadFromFileWithCRC64.class); + private final OriginalContent originalContent = new OriginalContent(); + private final BlobClient syncClient; + private final BlobAsyncClient asyncClient; + private final BlobClient syncNoFaultClient; + private final BlobAsyncClient asyncNoFaultClient; + private final ParallelTransferOptions parallelTransferOptions; + + public UploadFromFileWithCRC64(StorageStressOptions options) { + super(options); + String blobName = generateBlobName(); + this.asyncNoFaultClient = getAsyncContainerClientNoFault().getBlobAsyncClient(blobName); + this.syncNoFaultClient = getSyncContainerClientNoFault().getBlobClient(blobName); + this.syncClient = getSyncContainerClient().getBlobClient(blobName); + this.asyncClient = getAsyncContainerClient().getBlobAsyncClient(blobName); + this.parallelTransferOptions = new ParallelTransferOptions() + .setMaxConcurrency(options.getMaxConcurrency()) + .setMaxSingleUploadSizeLong(4 * 1024 * 1024L); + } + + @Override + protected void runInternal(Context span) { + Path downloadPath = getTempPath("test"); + Path uploadFilePath = null; + try (CrcInputStream inputStream = new CrcInputStream(originalContent.getBlobContentHead(), options.getSize())) { + uploadFilePath = generateFile(inputStream); + downloadPath = downloadPath.resolve(CoreUtils.randomUuid() + ".txt"); + syncClient.uploadFromFileWithResponse(new BlobUploadFromFileOptions(uploadFilePath.toString()) + .setParallelTransferOptions(parallelTransferOptions) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64), + null, span); + syncNoFaultClient.downloadToFileWithResponse( + new BlobDownloadToFileOptions(downloadPath.toString()), null, span); + originalContent.checkMatch(BinaryData.fromFile(downloadPath), span).block(); + } finally { + deleteFile(downloadPath); + deleteFile(uploadFilePath); + } + } + + @Override + protected Mono runInternalAsync(Context span) { + Path downloadPath = getTempPath("test"); + // This is written differently than the other runInternalAsync methods because uploadFromFile requires a file + // path, so we need to generate the temp file. + return Mono.using( + () -> new CrcInputStream(originalContent.getBlobContentHead(), options.getSize()), + inputStream -> uploadAndVerifyAsync(inputStream, downloadPath, span), + CrcInputStream::close); + } + + private Mono uploadAndVerifyAsync(CrcInputStream inputStream, Path downloadDir, Context span) { + Path uploadFilePath = generateFile(inputStream); + Path downloadFilePath = downloadDir.resolve(UUID.randomUUID() + ".txt"); + + return asyncClient.uploadFromFileWithResponse(new BlobUploadFromFileOptions(uploadFilePath.toString()) + .setParallelTransferOptions(parallelTransferOptions) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64)) + .flatMap(ignored -> asyncNoFaultClient.downloadToFileWithResponse( + new BlobDownloadToFileOptions(downloadFilePath.toString()))) + .flatMap(ignored -> originalContent.checkMatch(BinaryData.fromFile(downloadFilePath), span)) + .doFinally(signal -> { + deleteFile(uploadFilePath); + deleteFile(downloadFilePath); + }); + } + + @Override + public Mono setupAsync() { + return super.setupAsync().then(originalContent.setupBlob(asyncNoFaultClient, options.getSize())); + } + + @Override + public Mono cleanupAsync() { + return asyncNoFaultClient.deleteIfExists() + .then(super.cleanupAsync()); + } + + private Path getTempPath(String prefix) { + try { + return Files.createTempDirectory(prefix); + } catch (IOException e) { + throw LOGGER.logExceptionAsError(new UncheckedIOException(e)); + } + } + + private static void deleteFile(Path path) { + try { + Files.deleteIfExists(path); + } catch (Throwable e) { + LOGGER.atError() + .addKeyValue("path", path) + .log("failed to delete file", e); + } + } + + private static Path generateFile(InputStream inputStream) { + try { + File file = Files.createTempFile(CoreUtils.randomUuid().toString(), ".txt").toFile(); + file.deleteOnExit(); + Files.copy(inputStream, file.toPath(), StandardCopyOption.REPLACE_EXISTING); + return file.toPath(); + } catch (IOException e) { + throw LOGGER.logExceptionAsError(new UncheckedIOException(e)); + } + } +} diff --git a/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/UploadPagesWithCRC64.java b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/UploadPagesWithCRC64.java new file mode 100644 index 000000000000..3572871bea9f --- /dev/null +++ b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/UploadPagesWithCRC64.java @@ -0,0 +1,84 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.blob.stress; + +import com.azure.core.util.Context; +import com.azure.storage.blob.BlobAsyncClient; +import com.azure.storage.blob.BlobClient; +import com.azure.storage.blob.models.PageRange; +import com.azure.storage.blob.options.PageBlobUploadPagesOptions; +import com.azure.storage.blob.specialized.PageBlobAsyncClient; +import com.azure.storage.blob.specialized.PageBlobClient; +import com.azure.storage.blob.stress.utils.OriginalContent; +import com.azure.storage.common.ContentValidationAlgorithm; +import com.azure.storage.stress.CrcInputStream; +import com.azure.storage.stress.StorageStressOptions; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; + +import java.nio.ByteBuffer; + +/** + * Page blob upload pages with CRC64 enabled. + */ +public class UploadPagesWithCRC64 extends PageBlobScenarioBase { + private final OriginalContent originalContent = new OriginalContent(); + private final BlobClient syncClient; + private final BlobAsyncClient asyncClient; + private final BlobAsyncClient asyncNoFaultClient; + private final PageBlobAsyncClient tempSetupPageBlobClient; + + public UploadPagesWithCRC64(StorageStressOptions options) { + super(options); + String blobName = generateBlobName(); + String tempBlobName = generateBlobName(); + + this.asyncNoFaultClient = getAsyncContainerClientNoFault().getBlobAsyncClient(blobName); + this.syncClient = getSyncContainerClient().getBlobClient(blobName); + this.asyncClient = getAsyncContainerClient().getBlobAsyncClient(blobName); + BlobAsyncClient tempSetupBlobClient = getAsyncContainerClientNoFault().getBlobAsyncClient(tempBlobName); + this.tempSetupPageBlobClient = tempSetupBlobClient.getPageBlobAsyncClient(); + } + + @Override + protected void runInternal(Context span) { + try (CrcInputStream inputStream = new CrcInputStream(originalContent.getBlobContentHead(), options.getSize())) { + PageBlobClient pageBlobClient = syncClient.getPageBlobClient(); + PageRange range = new PageRange().setStart(0).setEnd(options.getSize() - 1); + pageBlobClient.uploadPagesWithResponse( + new PageBlobUploadPagesOptions(range, inputStream) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64), + null, span); + originalContent.checkMatch(inputStream.getContentInfo(), span).block(); + } + } + + @Override + protected Mono runInternalAsync(Context span) { + PageBlobAsyncClient pageBlobAsyncClient = asyncClient.getPageBlobAsyncClient(); + Flux byteBufferFlux = new CrcInputStream(originalContent.getBlobContentHead(), options.getSize()) + .convertStreamToByteBuffer(); + PageRange range = new PageRange().setStart(0).setEnd(options.getSize() - 1); + return pageBlobAsyncClient.uploadPagesWithResponse( + new PageBlobUploadPagesOptions(range, byteBufferFlux) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64)) + .then(originalContent.checkMatch(byteBufferFlux, span)); + } + + @Override + public Mono setupAsync() { + return super.setupAsync() + .then(asyncNoFaultClient.getPageBlobAsyncClient().create(options.getSize())) + .then(tempSetupPageBlobClient.create(options.getSize())) + .then(originalContent.setupPageBlob(tempSetupPageBlobClient, options.getSize())); + } + + @Override + public Mono cleanupAsync() { + return asyncNoFaultClient.getPageBlobAsyncClient().deleteIfExists() + .onErrorResume(e -> Mono.empty()) + .then(tempSetupPageBlobClient.deleteIfExists()) + .then(super.cleanupAsync()); + } +} diff --git a/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/UploadWithCRC64.java b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/UploadWithCRC64.java new file mode 100644 index 000000000000..55340ba93946 --- /dev/null +++ b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/UploadWithCRC64.java @@ -0,0 +1,71 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.blob.stress; + +import com.azure.core.util.Context; +import com.azure.storage.blob.BlobAsyncClient; +import com.azure.storage.blob.BlobClient; +import com.azure.storage.blob.models.ParallelTransferOptions; +import com.azure.storage.blob.options.BlobParallelUploadOptions; +import com.azure.storage.blob.stress.utils.OriginalContent; +import com.azure.storage.common.ContentValidationAlgorithm; +import com.azure.storage.stress.CrcInputStream; +import com.azure.storage.stress.StorageStressOptions; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; + +import java.nio.ByteBuffer; + +/** + * Parallel blob upload with CRC64 enabled. + */ +public class UploadWithCRC64 extends BlobScenarioBase { + private final OriginalContent originalContent = new OriginalContent(); + private final BlobClient syncClient; + private final BlobAsyncClient asyncClient; + private final BlobAsyncClient asyncNoFaultClient; + private final ParallelTransferOptions parallelTransferOptions; + + public UploadWithCRC64(StorageStressOptions options) { + super(options); + String blobName = generateBlobName(); + this.asyncNoFaultClient = getAsyncContainerClientNoFault().getBlobAsyncClient(blobName); + this.syncClient = getSyncContainerClient().getBlobClient(blobName); + this.asyncClient = getAsyncContainerClient().getBlobAsyncClient(blobName); + parallelTransferOptions = new ParallelTransferOptions() + .setMaxConcurrency(options.getMaxConcurrency()) + .setMaxSingleUploadSizeLong(4 * 1024 * 1024L); + } + + @Override + protected void runInternal(Context span) { + try (CrcInputStream inputStream = new CrcInputStream(originalContent.getBlobContentHead(), options.getSize())) { + syncClient.uploadWithResponse(new BlobParallelUploadOptions(inputStream) + .setParallelTransferOptions(parallelTransferOptions) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64), null, span); + originalContent.checkMatch(inputStream.getContentInfo(), span).block(); + } + } + + @Override + protected Mono runInternalAsync(Context span) { + Flux byteBufferFlux = new CrcInputStream(originalContent.getBlobContentHead(), options.getSize()) + .convertStreamToByteBuffer(); + return asyncClient.uploadWithResponse(new BlobParallelUploadOptions(byteBufferFlux) + .setParallelTransferOptions(parallelTransferOptions) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64)) + .then(originalContent.checkMatch(byteBufferFlux, span)); + } + + @Override + public Mono setupAsync() { + return super.setupAsync().then(originalContent.setupBlob(asyncNoFaultClient, options.getSize())); + } + + @Override + public Mono cleanupAsync() { + return asyncNoFaultClient.deleteIfExists() + .then(super.cleanupAsync()); + } +} diff --git a/sdk/storage/azure-storage-blob/assets.json b/sdk/storage/azure-storage-blob/assets.json index 3949125ccf26..85028bbf87f6 100644 --- a/sdk/storage/azure-storage-blob/assets.json +++ b/sdk/storage/azure-storage-blob/assets.json @@ -2,5 +2,5 @@ "AssetsRepo": "Azure/azure-sdk-assets", "AssetsRepoPrefixPath": "java", "TagPrefix": "java/storage/azure-storage-blob", - "Tag": "java/storage/azure-storage-blob_2eefea69c0" + "Tag": "java/storage/azure-storage-blob_afc0d343f3" } diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/BlobAsyncClient.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/BlobAsyncClient.java index 5c4b16b3b9e9..6f4a32f70dda 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/BlobAsyncClient.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/BlobAsyncClient.java @@ -37,10 +37,14 @@ import com.azure.storage.blob.specialized.BlockBlobClient; import com.azure.storage.blob.specialized.PageBlobAsyncClient; import com.azure.storage.blob.specialized.SpecializedBlobClientBuilder; +import com.azure.storage.common.ContentValidationAlgorithm; + import com.azure.storage.common.implementation.BufferStagingArea; import com.azure.storage.common.implementation.Constants; import com.azure.storage.common.implementation.StorageImplUtils; import com.azure.storage.common.implementation.UploadUtils; +import com.azure.storage.common.implementation.contentvalidation.ContentValidationModeResolver; + import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; @@ -695,16 +699,23 @@ public Mono> uploadWithResponse(BlobParallelUploadOption ? new BlobImmutabilityPolicy() : options.getImmutabilityPolicy(); final Boolean legalHold = options.isLegalHold(); + final ContentValidationAlgorithm contentValidationAlgorithm = options.getContentValidationAlgorithm(); + + ContentValidationModeResolver.validateTransactionalChecksumOptions(computeMd5, contentValidationAlgorithm); + ContentValidationModeResolver.validateProgressWithContentValidation( + parallelTransferOptions.getProgressListener(), contentValidationAlgorithm); BlockBlobAsyncClient blockBlobAsyncClient = getBlockBlobAsyncClient(); Function, Mono>> uploadInChunksFunction = (stream) -> uploadInChunks(blockBlobAsyncClient, stream, parallelTransferOptions, headers, metadata, - tags, tier, requestConditions, computeMd5, immutabilityPolicy, legalHold); + tags, tier, requestConditions, computeMd5, immutabilityPolicy, legalHold, + contentValidationAlgorithm); BiFunction, Long, Mono>> uploadFullBlobFunction = (stream, length) -> uploadFullBlob(blockBlobAsyncClient, stream, length, parallelTransferOptions, - headers, metadata, tags, tier, requestConditions, computeMd5, immutabilityPolicy, legalHold); + headers, metadata, tags, tier, requestConditions, computeMd5, immutabilityPolicy, legalHold, + contentValidationAlgorithm); Flux data = options.getDataFlux(); data = UploadUtils.extractByteBuffer(data, options.getOptionalLength(), @@ -721,7 +732,7 @@ private Mono> uploadFullBlob(BlockBlobAsyncClient blockB Flux data, long length, ParallelTransferOptions parallelTransferOptions, BlobHttpHeaders headers, Map metadata, Map tags, AccessTier tier, BlobRequestConditions requestConditions, boolean computeMd5, BlobImmutabilityPolicy immutabilityPolicy, - Boolean legalHold) { + Boolean legalHold, ContentValidationAlgorithm contentValidationAlgorithm) { /* * Note that there is no need to buffer here as the flux returned by the size gate in this case is created @@ -738,7 +749,8 @@ private Mono> uploadFullBlob(BlockBlobAsyncClient blockB .setImmutabilityPolicy(immutabilityPolicy) .setLegalHold(legalHold)) .flatMap(options -> { - Mono> responseMono = blockBlobAsyncClient.uploadWithResponse(options); + Mono> responseMono = ContentValidationModeResolver.addContentValidationMode( + blockBlobAsyncClient.uploadWithResponse(options), contentValidationAlgorithm, length, false); if (parallelTransferOptions.getProgressListener() != null) { ProgressReporter progressReporter = ProgressReporter.withProgressListener(parallelTransferOptions.getProgressListener()); @@ -753,7 +765,7 @@ private Mono> uploadInChunks(BlockBlobAsyncClient blockB Flux data, ParallelTransferOptions parallelTransferOptions, BlobHttpHeaders headers, Map metadata, Map tags, AccessTier tier, BlobRequestConditions requestConditions, boolean computeMd5, BlobImmutabilityPolicy immutabilityPolicy, - Boolean legalHold) { + Boolean legalHold, ContentValidationAlgorithm contentValidationAlgorithm) { // TODO: Sample/api reference ProgressListener progressListener = parallelTransferOptions.getProgressListener(); @@ -777,12 +789,12 @@ private Mono> uploadInChunks(BlockBlobAsyncClient blockB .concatWith(Flux.defer(stagingArea::flush)) .flatMapSequential(bufferAggregator -> { Flux chunkData = bufferAggregator.asFlux(); - String blockId = Base64.getEncoder().encodeToString(CoreUtils.randomUuid().toString().getBytes(UTF_8)); return UploadUtils.computeMd5(chunkData, computeMd5, LOGGER).flatMap(fluxMd5Wrapper -> { - Mono> responseMono - = blockBlobAsyncClient.stageBlockWithResponse(blockId, fluxMd5Wrapper.getData(), - bufferAggregator.length(), fluxMd5Wrapper.getMd5(), requestConditions.getLeaseId()); + Mono> responseMono = ContentValidationModeResolver.addContentValidationMode( + blockBlobAsyncClient.stageBlockWithResponse(blockId, fluxMd5Wrapper.getData(), + bufferAggregator.length(), fluxMd5Wrapper.getMd5(), requestConditions.getLeaseId()), + contentValidationAlgorithm, 0, true); if (progressReporter != null) { responseMono = responseMono.contextWrite(FluxUtil.toReactorContext(Contexts.empty() .setHttpRequestProgressReporter(progressReporter.createChild()) @@ -968,7 +980,12 @@ public Mono> uploadFromFileWithResponse(BlobUploadFromFi : options.getParallelTransferOptions().getBlockSizeLong(); final ParallelTransferOptions finalParallelTransferOptions = ModelHelper.populateAndApplyDefaults(options.getParallelTransferOptions()); + final ContentValidationAlgorithm contentValidationAlgorithm = options.getContentValidationAlgorithm(); + try { + ContentValidationModeResolver.validateProgressWithContentValidation( + finalParallelTransferOptions.getProgressListener(), contentValidationAlgorithm); + Path filePath = Paths.get(options.getFilePath()); BlockBlobAsyncClient blockBlobAsyncClient = getBlockBlobAsyncClient(); // This will retrieve file length but won't read file body. @@ -980,15 +997,17 @@ public Mono> uploadFromFileWithResponse(BlobUploadFromFi if (fileSize > finalParallelTransferOptions.getMaxSingleUploadSizeLong()) { return uploadFileChunks(fileSize, finalParallelTransferOptions, originalBlockSize, options.getHeaders(), options.getMetadata(), options.getTags(), options.getTier(), options.getRequestConditions(), - filePath, blockBlobAsyncClient); + filePath, blockBlobAsyncClient, contentValidationAlgorithm); } else { // Otherwise, we know it can be sent in a single request reducing network overhead. - Mono> responseMono = blockBlobAsyncClient - .uploadWithResponse(new BlockBlobSimpleUploadOptions(fullFileData).setHeaders(options.getHeaders()) - .setMetadata(options.getMetadata()) - .setTags(options.getTags()) - .setTier(options.getTier()) - .setRequestConditions(options.getRequestConditions())); + Mono> responseMono = ContentValidationModeResolver.addContentValidationMode( + blockBlobAsyncClient.uploadWithResponse( + new BlockBlobSimpleUploadOptions(fullFileData).setHeaders(options.getHeaders()) + .setMetadata(options.getMetadata()) + .setTags(options.getTags()) + .setTier(options.getTier()) + .setRequestConditions(options.getRequestConditions())), + contentValidationAlgorithm, fileSize, false); if (finalParallelTransferOptions.getProgressListener() != null) { ProgressReporter progressReporter = ProgressReporter.withProgressListener(finalParallelTransferOptions.getProgressListener()); @@ -1005,7 +1024,8 @@ public Mono> uploadFromFileWithResponse(BlobUploadFromFi private Mono> uploadFileChunks(long fileSize, ParallelTransferOptions parallelTransferOptions, Long originalBlockSize, BlobHttpHeaders headers, Map metadata, Map tags, AccessTier tier, - BlobRequestConditions requestConditions, Path filePath, BlockBlobAsyncClient client) { + BlobRequestConditions requestConditions, Path filePath, BlockBlobAsyncClient client, + ContentValidationAlgorithm contentValidationAlgorithm) { final BlobRequestConditions finalRequestConditions = (requestConditions == null) ? new BlobRequestConditions() : requestConditions; // parallelTransferOptions are finalized in the calling method. @@ -1023,8 +1043,10 @@ private Mono> uploadFileChunks(long fileSize, BinaryData data = BinaryData.fromFile(filePath, chunk.getOffset(), chunk.getCount(), DEFAULT_FILE_READ_CHUNK_SIZE); - Mono> responseMono = client.stageBlockWithResponse( - new BlockBlobStageBlockOptions(blockId, data).setLeaseId(finalRequestConditions.getLeaseId())); + Mono> responseMono = ContentValidationModeResolver.addContentValidationMode( + client.stageBlockWithResponse( + new BlockBlobStageBlockOptions(blockId, data).setLeaseId(finalRequestConditions.getLeaseId())), + contentValidationAlgorithm, 0, true); if (progressReporter != null) { responseMono = responseMono.contextWrite(FluxUtil.toReactorContext( Contexts.empty().setHttpRequestProgressReporter(progressReporter.createChild()).getContext())); diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/BuilderHelper.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/BuilderHelper.java index 0866d310981c..53fcb67447dc 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/BuilderHelper.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/BuilderHelper.java @@ -39,6 +39,8 @@ import com.azure.storage.common.policy.ResponseValidationPolicyBuilder; import com.azure.storage.common.policy.ScrubEtagPolicy; import com.azure.storage.common.policy.StorageBearerTokenChallengeAuthorizationPolicy; +import com.azure.storage.common.policy.StorageContentValidationDecoderPolicy; +import com.azure.storage.common.policy.StorageContentValidationPolicy; import com.azure.storage.common.policy.StorageSharedKeyCredentialPolicy; import java.net.MalformedURLException; @@ -115,6 +117,9 @@ public static HttpPipeline buildPipeline(StorageSharedKeyCredential storageShare } policies.add(new MetadataValidationPolicy()); + policies.add(new StorageContentValidationPolicy()); + policies.add(new StorageContentValidationDecoderPolicy()); + if (storageSharedKeyCredential != null) { policies.add(new StorageSharedKeyCredentialPolicy(storageSharedKeyCredential)); } diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/AppendBlobAppendBlockOptions.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/AppendBlobAppendBlockOptions.java new file mode 100644 index 000000000000..7c349b2a088c --- /dev/null +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/AppendBlobAppendBlockOptions.java @@ -0,0 +1,152 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.blob.options; + +import com.azure.core.annotation.Fluent; +import com.azure.core.util.CoreUtils; +import com.azure.storage.blob.models.AppendBlobRequestConditions; +import com.azure.storage.common.ContentValidationAlgorithm; +import com.azure.storage.common.implementation.StorageImplUtils; + +import reactor.core.publisher.Flux; + +import java.io.InputStream; +import java.nio.ByteBuffer; + +/** + * Extended options that may be passed when appending a block to an append blob. + */ +@Fluent +public final class AppendBlobAppendBlockOptions { + private final InputStream dataStream; + private final Flux dataFlux; + private final long length; + private byte[] contentMd5; + private AppendBlobRequestConditions requestConditions; + private ContentValidationAlgorithm contentValidationAlgorithm; + + /** + * Creates a new instance of {@link AppendBlobAppendBlockOptions} for use with the sync client. + * + * @param data The data to write to the blob. Must be markable for retries. + * @param length The exact length of the data. + * @throws NullPointerException If {@code data} is null. + * @throws IllegalArgumentException If {@code length} is negative. + */ + public AppendBlobAppendBlockOptions(InputStream data, long length) { + StorageImplUtils.assertNotNull("data", data); + if (length < 0) { + throw new IllegalArgumentException("'length' must be >= 0"); + } + this.dataStream = data; + this.dataFlux = null; + this.length = length; + } + + /** + * Creates a new instance of {@link AppendBlobAppendBlockOptions} for use with the async client. + * + * @param data The data to write to the blob. Must be replayable if retries are enabled. + * @param length The exact length of the data. + * @throws NullPointerException If {@code data} is null. + * @throws IllegalArgumentException If {@code length} is negative. + */ + public AppendBlobAppendBlockOptions(Flux data, long length) { + StorageImplUtils.assertNotNull("data", data); + if (length < 0) { + throw new IllegalArgumentException("'length' must be >= 0"); + } + this.dataStream = null; + this.dataFlux = data; + this.length = length; + } + + /** + * Gets the body as an InputStream. Null if constructed with {@link Flux}. + * + * @return The body stream, or null. + */ + public InputStream getDataStream() { + return dataStream; + } + + /** + * Gets the body as a Flux. Null if constructed with {@link InputStream}. + * + * @return The body flux, or null. + */ + public Flux getDataFlux() { + return dataFlux; + } + + /** + * Gets the exact length of the block data. + * + * @return The length in bytes. + */ + public long getLength() { + return length; + } + + /** + * Gets the MD5 hash of the block content. + * + * @return An MD5 hash of the content, or null. + */ + public byte[] getContentMd5() { + return CoreUtils.clone(contentMd5); + } + + /** + * Sets the MD5 hash of the block content for transactional verification. + * + * @param contentMd5 An MD5 hash of the block content. + * @return The updated options. + */ + public AppendBlobAppendBlockOptions setContentMd5(byte[] contentMd5) { + this.contentMd5 = CoreUtils.clone(contentMd5); + return this; + } + + /** + * Gets the {@link AppendBlobRequestConditions}. + * + * @return The request conditions. + */ + public AppendBlobRequestConditions getRequestConditions() { + return requestConditions; + } + + /** + * Sets the {@link AppendBlobRequestConditions}. + * + * @param requestConditions The request conditions. + * @return The updated options. + */ + public AppendBlobAppendBlockOptions setRequestConditions(AppendBlobRequestConditions requestConditions) { + this.requestConditions = requestConditions; + return this; + } + + /** + * Gets the algorithm to use for transfer content validation. See {@link ContentValidationAlgorithm} for more details. + * + * @return The transfer validation checksum algorithm. + */ + public ContentValidationAlgorithm getContentValidationAlgorithm() { + return contentValidationAlgorithm; + } + + /** + * Sets the algorithm to use for transfer content validation. See {@link ContentValidationAlgorithm} for more details. + * + * @param contentValidationAlgorithm The transfer validation checksum algorithm. + * @return The updated options. + */ + public AppendBlobAppendBlockOptions + setContentValidationAlgorithm(ContentValidationAlgorithm contentValidationAlgorithm) { + this.contentValidationAlgorithm = contentValidationAlgorithm; + return this; + } +} diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/AppendBlobOutputStreamOptions.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/AppendBlobOutputStreamOptions.java new file mode 100644 index 000000000000..df12b00c32d1 --- /dev/null +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/AppendBlobOutputStreamOptions.java @@ -0,0 +1,64 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.blob.options; + +import com.azure.core.annotation.Fluent; +import com.azure.storage.blob.models.AppendBlobRequestConditions; +import com.azure.storage.common.ContentValidationAlgorithm; + +/** + * Extended options that may be passed when opening an output stream to an append blob. + */ +@Fluent +public final class AppendBlobOutputStreamOptions { + private AppendBlobRequestConditions requestConditions; + private ContentValidationAlgorithm contentValidationAlgorithm; + + /** + * Creates a new instance of {@link AppendBlobOutputStreamOptions}. + */ + public AppendBlobOutputStreamOptions() { + } + + /** + * Gets the {@link AppendBlobRequestConditions}. + * + * @return The request conditions. + */ + public AppendBlobRequestConditions getRequestConditions() { + return requestConditions; + } + + /** + * Sets the {@link AppendBlobRequestConditions}. + * + * @param requestConditions The request conditions. + * @return The updated options. + */ + public AppendBlobOutputStreamOptions setRequestConditions(AppendBlobRequestConditions requestConditions) { + this.requestConditions = requestConditions; + return this; + } + + /** + * Gets the algorithm to use for transfer content validation. See {@link ContentValidationAlgorithm} for more details. + * + * @return The transfer validation checksum algorithm. + */ + public ContentValidationAlgorithm getContentValidationAlgorithm() { + return contentValidationAlgorithm; + } + + /** + * Sets the algorithm to use for transfer content validation. See {@link ContentValidationAlgorithm} for more details. + * + * @param contentValidationAlgorithm The transfer validation checksum algorithm. + * @return The updated options. + */ + public AppendBlobOutputStreamOptions + setContentValidationAlgorithm(ContentValidationAlgorithm contentValidationAlgorithm) { + this.contentValidationAlgorithm = contentValidationAlgorithm; + return this; + } +} diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/BlobDownloadContentOptions.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/BlobDownloadContentOptions.java new file mode 100644 index 000000000000..df63b74238fc --- /dev/null +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/BlobDownloadContentOptions.java @@ -0,0 +1,129 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.blob.options; + +import com.azure.core.annotation.Fluent; +import com.azure.storage.blob.models.BlobRange; +import com.azure.storage.blob.models.BlobRequestConditions; +import com.azure.storage.blob.models.DownloadRetryOptions; +import com.azure.storage.common.ContentValidationAlgorithm; + +/** + * Extended options that may be passed when downloading blob content (full blob or range in memory). + */ +@Fluent +public final class BlobDownloadContentOptions { + private BlobRange range; + private DownloadRetryOptions downloadRetryOptions; + private BlobRequestConditions requestConditions; + private boolean retrieveContentRangeMd5; + private ContentValidationAlgorithm contentValidationAlgorithm; + + /** + * Creates a new instance of {@link BlobDownloadContentOptions}. + */ + public BlobDownloadContentOptions() { + } + + /** + * Gets the {@link BlobRange}. + * + * @return The blob range. + */ + public BlobRange getRange() { + return range; + } + + /** + * Sets the {@link BlobRange}. + * + * @param range The blob range. + * @return The updated options. + */ + public BlobDownloadContentOptions setRange(BlobRange range) { + this.range = range; + return this; + } + + /** + * Gets the {@link DownloadRetryOptions}. + * + * @return The download retry options. + */ + public DownloadRetryOptions getDownloadRetryOptions() { + return downloadRetryOptions; + } + + /** + * Sets the {@link DownloadRetryOptions}. + * + * @param downloadRetryOptions The download retry options. + * @return The updated options. + */ + public BlobDownloadContentOptions setDownloadRetryOptions(DownloadRetryOptions downloadRetryOptions) { + this.downloadRetryOptions = downloadRetryOptions; + return this; + } + + /** + * Gets the {@link BlobRequestConditions}. + * + * @return The request conditions. + */ + public BlobRequestConditions getRequestConditions() { + return requestConditions; + } + + /** + * Sets the {@link BlobRequestConditions}. + * + * @param requestConditions The request conditions. + * @return The updated options. + */ + public BlobDownloadContentOptions setRequestConditions(BlobRequestConditions requestConditions) { + this.requestConditions = requestConditions; + return this; + } + + /** + * Gets whether the content MD5 for the specified blob range should be returned. + * + * @return Whether to retrieve content range MD5. + */ + public boolean isRetrieveContentRangeMd5() { + return retrieveContentRangeMd5; + } + + /** + * Sets whether the content MD5 for the specified blob range should be returned. + * + * @param retrieveContentRangeMd5 Whether to retrieve content range MD5. + * @return The updated options. + */ + public BlobDownloadContentOptions setRetrieveContentRangeMd5(boolean retrieveContentRangeMd5) { + this.retrieveContentRangeMd5 = retrieveContentRangeMd5; + return this; + } + + /** + * Gets the algorithm to use for transfer content validation. See {@link ContentValidationAlgorithm} for more details. + * + * @return The transfer validation checksum algorithm. + */ + public ContentValidationAlgorithm getContentValidationAlgorithm() { + return contentValidationAlgorithm; + } + + /** + * Sets the algorithm to use for transfer content validation. See {@link ContentValidationAlgorithm} for more details. + * + * @param contentValidationAlgorithm The transfer validation checksum algorithm. + * @return The updated options. + */ + public BlobDownloadContentOptions + setContentValidationAlgorithm(ContentValidationAlgorithm contentValidationAlgorithm) { + this.contentValidationAlgorithm = contentValidationAlgorithm; + return this; + } +} diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/BlobDownloadStreamOptions.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/BlobDownloadStreamOptions.java new file mode 100644 index 000000000000..c8bf93b7ae03 --- /dev/null +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/BlobDownloadStreamOptions.java @@ -0,0 +1,131 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.blob.options; + +import com.azure.core.annotation.Fluent; +import com.azure.storage.blob.models.BlobRange; +import com.azure.storage.blob.models.BlobRequestConditions; +import com.azure.storage.blob.models.DownloadRetryOptions; +import com.azure.storage.common.ContentValidationAlgorithm; + +/** + * Extended options that may be passed when downloading a blob range to an output stream. + */ +@Fluent +public final class BlobDownloadStreamOptions { + private BlobRange range; + private DownloadRetryOptions downloadRetryOptions; + private BlobRequestConditions requestConditions; + private boolean retrieveContentRangeMd5; + private ContentValidationAlgorithm contentValidationAlgorithm; + + /** + * Creates a new instance of {@link BlobDownloadStreamOptions}. + */ + public BlobDownloadStreamOptions() { + } + + /** + * Gets the {@link BlobRange}. + * + * @return The blob range. + */ + public BlobRange getRange() { + return range; + } + + /** + * Sets the {@link BlobRange}. + * + * @param range The blob range. + * @return The updated options. + */ + public BlobDownloadStreamOptions setRange(BlobRange range) { + this.range = range; + return this; + } + + /** + * Gets the {@link DownloadRetryOptions}. + * + * @return The download retry options. + */ + public DownloadRetryOptions getDownloadRetryOptions() { + return downloadRetryOptions; + } + + /** + * Sets the {@link DownloadRetryOptions}. + * + * @param downloadRetryOptions The download retry options. + * @return The updated options. + */ + public BlobDownloadStreamOptions setDownloadRetryOptions(DownloadRetryOptions downloadRetryOptions) { + this.downloadRetryOptions = downloadRetryOptions; + return this; + } + + /** + * Gets the {@link BlobRequestConditions}. + * + * @return The request conditions. + */ + public BlobRequestConditions getRequestConditions() { + return requestConditions; + } + + /** + * Sets the {@link BlobRequestConditions}. + * + * @param requestConditions The request conditions. + * @return The updated options. + */ + public BlobDownloadStreamOptions setRequestConditions(BlobRequestConditions requestConditions) { + this.requestConditions = requestConditions; + return this; + } + + /** + * Gets whether the content MD5 for the specified blob range should be returned. + * + * @return Whether to retrieve content range MD5. + */ + public boolean isRetrieveContentRangeMd5() { + return retrieveContentRangeMd5; + } + + /** + * Sets whether the content MD5 for the specified blob range should be returned. + * + * @param retrieveContentRangeMd5 Whether to retrieve content range MD5. + * @return The updated options. + */ + public BlobDownloadStreamOptions setRetrieveContentRangeMd5(boolean retrieveContentRangeMd5) { + this.retrieveContentRangeMd5 = retrieveContentRangeMd5; + return this; + } + + /** + * Gets the algorithm to use for transfer content validation on the response. See {@link ContentValidationAlgorithm} + * for more details. + * + * @return The transfer validation checksum algorithm. + */ + public ContentValidationAlgorithm getContentValidationAlgorithm() { + return contentValidationAlgorithm; + } + + /** + * Sets the algorithm to use for transfer content validation on the response. See {@link ContentValidationAlgorithm} + * for more details. + * + * @param contentValidationAlgorithm The transfer validation checksum algorithm. + * @return The updated options. + */ + public BlobDownloadStreamOptions + setContentValidationAlgorithm(ContentValidationAlgorithm contentValidationAlgorithm) { + this.contentValidationAlgorithm = contentValidationAlgorithm; + return this; + } +} diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/BlobDownloadToFileOptions.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/BlobDownloadToFileOptions.java index 434844645f58..0214ca4b8613 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/BlobDownloadToFileOptions.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/BlobDownloadToFileOptions.java @@ -8,6 +8,7 @@ import com.azure.storage.blob.models.BlobRequestConditions; import com.azure.storage.blob.models.DownloadRetryOptions; import com.azure.storage.common.ParallelTransferOptions; +import com.azure.storage.common.ContentValidationAlgorithm; import com.azure.storage.common.implementation.StorageImplUtils; import java.nio.file.OpenOption; @@ -25,6 +26,7 @@ public class BlobDownloadToFileOptions { private BlobRequestConditions requestConditions; private boolean retrieveContentRangeMd5; private Set openOptions; + private ContentValidationAlgorithm contentValidationAlgorithm; /** * Constructs a {@link BlobDownloadToFileOptions}. @@ -165,4 +167,27 @@ public BlobDownloadToFileOptions setOpenOptions(Set openOptions) { this.openOptions = openOptions; return this; } + + /** + * Gets the algorithm to use for transfer content validation on the response. See {@link ContentValidationAlgorithm} + * for more details. + * + * @return The transfer validation checksum algorithm. + */ + public ContentValidationAlgorithm getContentValidationAlgorithm() { + return contentValidationAlgorithm; + } + + /** + * Sets the algorithm to use for transfer content validation on the response. See {@link ContentValidationAlgorithm} + * for more details. + * + * @param contentValidationAlgorithm The transfer validation checksum algorithm. + * @return The updated options. + */ + public BlobDownloadToFileOptions + setContentValidationAlgorithm(ContentValidationAlgorithm contentValidationAlgorithm) { + this.contentValidationAlgorithm = contentValidationAlgorithm; + return this; + } } diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/BlobInputStreamOptions.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/BlobInputStreamOptions.java index d98d744f3dd3..bb2e2f240215 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/BlobInputStreamOptions.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/BlobInputStreamOptions.java @@ -7,6 +7,7 @@ import com.azure.storage.blob.models.BlobRange; import com.azure.storage.blob.models.BlobRequestConditions; import com.azure.storage.blob.models.ConsistentReadControl; +import com.azure.storage.common.ContentValidationAlgorithm; /** * Extended options that may be passed when opening a blob input stream. @@ -17,6 +18,7 @@ public class BlobInputStreamOptions { private BlobRequestConditions requestConditions; private Integer blockSize; private ConsistentReadControl consistentReadControl; + private ContentValidationAlgorithm contentValidationAlgorithm; /** * Creates a new instance of {@link BlobInputStreamOptions}. @@ -111,4 +113,26 @@ public BlobInputStreamOptions setConsistentReadControl(ConsistentReadControl con this.consistentReadControl = consistentReadControl; return this; } + + /** + * Gets the algorithm to use for transfer content validation on the response. See {@link ContentValidationAlgorithm} + * for more details. + * + * @return The transfer validation checksum algorithm. + */ + public ContentValidationAlgorithm getContentValidationAlgorithm() { + return contentValidationAlgorithm; + } + + /** + * Sets the algorithm to use for transfer content validation on the response. See {@link ContentValidationAlgorithm} + * for more details. + * + * @param contentValidationAlgorithm The transfer validation checksum algorithm. + * @return The updated options. + */ + public BlobInputStreamOptions setContentValidationAlgorithm(ContentValidationAlgorithm contentValidationAlgorithm) { + this.contentValidationAlgorithm = contentValidationAlgorithm; + return this; + } } diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/BlobParallelUploadOptions.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/BlobParallelUploadOptions.java index 681ff994ae62..e4aac1781f61 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/BlobParallelUploadOptions.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/BlobParallelUploadOptions.java @@ -12,7 +12,9 @@ import com.azure.storage.blob.models.BlobImmutabilityPolicy; import com.azure.storage.blob.models.BlobRequestConditions; import com.azure.storage.blob.models.ParallelTransferOptions; +import com.azure.storage.common.ContentValidationAlgorithm; import com.azure.storage.common.implementation.StorageImplUtils; + import reactor.core.publisher.Flux; import java.io.InputStream; @@ -38,6 +40,7 @@ public class BlobParallelUploadOptions { private Duration timeout; private BlobImmutabilityPolicy immutabilityPolicy; private Boolean legalHold; + private ContentValidationAlgorithm contentValidationAlgorithm; /** * Constructs a new {@link BlobParallelUploadOptions}. @@ -366,4 +369,27 @@ public BlobParallelUploadOptions setLegalHold(Boolean legalHold) { this.legalHold = legalHold; return this; } + + /** + * Gets the algorithm to use for transfer content validation on the request. See {@link ContentValidationAlgorithm} + * for more details. + * + * @return The transfer validation checksum algorithm. + */ + public ContentValidationAlgorithm getContentValidationAlgorithm() { + return contentValidationAlgorithm; + } + + /** + * Sets the algorithm to use for transfer content validation on the request. See {@link ContentValidationAlgorithm} + * for more details. + * + * @param contentValidationAlgorithm The transfer validation checksum algorithm. + * @return The updated options. + */ + public BlobParallelUploadOptions + setContentValidationAlgorithm(ContentValidationAlgorithm contentValidationAlgorithm) { + this.contentValidationAlgorithm = contentValidationAlgorithm; + return this; + } } diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/BlobSeekableByteChannelReadOptions.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/BlobSeekableByteChannelReadOptions.java index ac92a1a022b3..91f61b9b509e 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/BlobSeekableByteChannelReadOptions.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/BlobSeekableByteChannelReadOptions.java @@ -6,6 +6,7 @@ import com.azure.core.annotation.Fluent; import com.azure.storage.blob.models.BlobRequestConditions; import com.azure.storage.blob.models.ConsistentReadControl; +import com.azure.storage.common.ContentValidationAlgorithm; import java.nio.channels.SeekableByteChannel; @@ -18,6 +19,7 @@ public final class BlobSeekableByteChannelReadOptions { private BlobRequestConditions requestConditions; private Integer readSizeInBytes; private ConsistentReadControl consistentReadControl; + private ContentValidationAlgorithm contentValidationAlgorithm; /** * Creates a new instance of {@link BlobSeekableByteChannelReadOptions}. @@ -108,4 +110,27 @@ public BlobSeekableByteChannelReadOptions setConsistentReadControl(ConsistentRea this.consistentReadControl = consistentReadControl; return this; } + + /** + * Gets the algorithm to use for transfer content validation on the response. See {@link ContentValidationAlgorithm} + * for more details. + * + * @return The transfer validation checksum algorithm. + */ + public ContentValidationAlgorithm getContentValidationAlgorithm() { + return contentValidationAlgorithm; + } + + /** + * Sets the algorithm to use for transfer content validation on the response. See {@link ContentValidationAlgorithm} + * for more details. + * + * @param contentValidationAlgorithm The transfer validation checksum algorithm. + * @return The updated options. + */ + public BlobSeekableByteChannelReadOptions + setContentValidationAlgorithm(ContentValidationAlgorithm contentValidationAlgorithm) { + this.contentValidationAlgorithm = contentValidationAlgorithm; + return this; + } } diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/BlobUploadFromFileOptions.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/BlobUploadFromFileOptions.java index 65ca9b0ab052..a9e283466af6 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/BlobUploadFromFileOptions.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/BlobUploadFromFileOptions.java @@ -8,6 +8,7 @@ import com.azure.storage.blob.models.BlobHttpHeaders; import com.azure.storage.blob.models.BlobRequestConditions; import com.azure.storage.blob.models.ParallelTransferOptions; +import com.azure.storage.common.ContentValidationAlgorithm; import com.azure.storage.common.implementation.StorageImplUtils; import java.util.Map; @@ -24,6 +25,7 @@ public class BlobUploadFromFileOptions { private Map tags; private AccessTier tier; private BlobRequestConditions requestConditions; + private ContentValidationAlgorithm contentValidationAlgorithm; /** * Constructs a {@link BlobUploadFromFileOptions}. @@ -164,4 +166,27 @@ public BlobUploadFromFileOptions setRequestConditions(BlobRequestConditions requ this.requestConditions = requestConditions; return this; } + + /** + * Gets the algorithm to use for transfer content validation on the request. See {@link ContentValidationAlgorithm} + * for more details. + * + * @return The transfer validation checksum algorithm. + */ + public ContentValidationAlgorithm getContentValidationAlgorithm() { + return contentValidationAlgorithm; + } + + /** + * Sets the algorithm to use for transfer content validation on the request. See {@link ContentValidationAlgorithm} + * for more details. + * + * @param contentValidationAlgorithm The transfer validation checksum algorithm. + * @return The updated options. + */ + public BlobUploadFromFileOptions + setContentValidationAlgorithm(ContentValidationAlgorithm contentValidationAlgorithm) { + this.contentValidationAlgorithm = contentValidationAlgorithm; + return this; + } } diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/BlockBlobOutputStreamOptions.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/BlockBlobOutputStreamOptions.java index e6385dfe5207..d02b6fd11681 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/BlockBlobOutputStreamOptions.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/BlockBlobOutputStreamOptions.java @@ -7,6 +7,7 @@ import com.azure.storage.blob.models.BlobHttpHeaders; import com.azure.storage.blob.models.BlobRequestConditions; import com.azure.storage.blob.models.ParallelTransferOptions; +import com.azure.storage.common.ContentValidationAlgorithm; import java.util.Map; @@ -20,6 +21,7 @@ public class BlockBlobOutputStreamOptions { private Map tags; private AccessTier tier; private BlobRequestConditions requestConditions; + private ContentValidationAlgorithm contentValidationAlgorithm; /** * Creates a new instance of {@link BlockBlobOutputStreamOptions}. @@ -146,4 +148,27 @@ public BlockBlobOutputStreamOptions setRequestConditions(BlobRequestConditions r this.requestConditions = requestConditions; return this; } + + /** + * Gets the algorithm to use for transfer content validation on the request. See {@link ContentValidationAlgorithm} + * for more details. + * + * @return The transfer validation checksum algorithm. + */ + public ContentValidationAlgorithm getContentValidationAlgorithm() { + return contentValidationAlgorithm; + } + + /** + * Sets the algorithm to use for transfer content validation on the request. See {@link ContentValidationAlgorithm} + * for more details. + * + * @param contentValidationAlgorithm The transfer validation checksum algorithm. + * @return The updated options. + */ + public BlockBlobOutputStreamOptions + setContentValidationAlgorithm(ContentValidationAlgorithm contentValidationAlgorithm) { + this.contentValidationAlgorithm = contentValidationAlgorithm; + return this; + } } diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/BlockBlobSeekableByteChannelWriteOptions.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/BlockBlobSeekableByteChannelWriteOptions.java index 1234ea04c738..3ab7ff719ead 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/BlockBlobSeekableByteChannelWriteOptions.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/BlockBlobSeekableByteChannelWriteOptions.java @@ -6,6 +6,7 @@ import com.azure.storage.blob.models.AccessTier; import com.azure.storage.blob.models.BlobHttpHeaders; import com.azure.storage.blob.models.BlobRequestConditions; +import com.azure.storage.common.ContentValidationAlgorithm; import java.util.Collection; import java.util.Map; @@ -60,6 +61,7 @@ public static Collection values() { private Map tags; private AccessTier tier; private BlobRequestConditions conditions; + private ContentValidationAlgorithm contentValidationAlgorithm; /** * Options constructor. @@ -199,4 +201,26 @@ public BlockBlobSeekableByteChannelWriteOptions setRequestConditions(BlobRequest return this; } + /** + * Gets the algorithm to use for transfer content validation on the request. See {@link ContentValidationAlgorithm} + * for more details. + * + * @return The transfer validation checksum algorithm. + */ + public ContentValidationAlgorithm getContentValidationAlgorithm() { + return contentValidationAlgorithm; + } + + /** + * Sets the algorithm to use for transfer content validation on the request. See {@link ContentValidationAlgorithm} + * for more details. + * + * @param contentValidationAlgorithm The transfer validation checksum algorithm. + * @return The updated instance. + */ + public BlockBlobSeekableByteChannelWriteOptions + setContentValidationAlgorithm(ContentValidationAlgorithm contentValidationAlgorithm) { + this.contentValidationAlgorithm = contentValidationAlgorithm; + return this; + } } diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/BlockBlobSimpleUploadOptions.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/BlockBlobSimpleUploadOptions.java index bef1780c2f59..47212d6033d8 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/BlockBlobSimpleUploadOptions.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/BlockBlobSimpleUploadOptions.java @@ -9,7 +9,9 @@ import com.azure.storage.blob.models.BlobHttpHeaders; import com.azure.storage.blob.models.BlobImmutabilityPolicy; import com.azure.storage.blob.models.BlobRequestConditions; +import com.azure.storage.common.ContentValidationAlgorithm; import com.azure.storage.common.implementation.StorageImplUtils; + import reactor.core.publisher.Flux; import java.io.InputStream; @@ -32,6 +34,7 @@ public class BlockBlobSimpleUploadOptions { private BlobRequestConditions requestConditions; private BlobImmutabilityPolicy immutabilityPolicy; private Boolean legalHold; + private ContentValidationAlgorithm contentValidationAlgorithm; /** * Creates a new instance of {@link BlockBlobSimpleUploadOptions}. @@ -293,4 +296,27 @@ public BlockBlobSimpleUploadOptions setLegalHold(Boolean legalHold) { this.legalHold = legalHold; return this; } + + /** + * Gets the algorithm to use for transfer content validation on the request. See {@link ContentValidationAlgorithm} + * for more details. + * + * @return The transfer validation checksum algorithm. + */ + public ContentValidationAlgorithm getContentValidationAlgorithm() { + return contentValidationAlgorithm; + } + + /** + * Sets the algorithm to use for transfer content validation on the request. See {@link ContentValidationAlgorithm} + * for more details. + * + * @param contentValidationAlgorithm The transfer validation checksum algorithm. + * @return The updated options. + */ + public BlockBlobSimpleUploadOptions + setContentValidationAlgorithm(ContentValidationAlgorithm contentValidationAlgorithm) { + this.contentValidationAlgorithm = contentValidationAlgorithm; + return this; + } } diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/BlockBlobStageBlockOptions.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/BlockBlobStageBlockOptions.java index 37445b450a28..9c9868d57ed4 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/BlockBlobStageBlockOptions.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/BlockBlobStageBlockOptions.java @@ -6,6 +6,7 @@ import com.azure.core.annotation.Fluent; import com.azure.core.util.BinaryData; import com.azure.core.util.CoreUtils; +import com.azure.storage.common.ContentValidationAlgorithm; import com.azure.storage.common.implementation.StorageImplUtils; /** @@ -17,6 +18,7 @@ public final class BlockBlobStageBlockOptions { private final BinaryData data; private String leaseId; private byte[] contentMd5; + private ContentValidationAlgorithm contentValidationAlgorithm; /** * Creates a new instance of {@link BlockBlobStageBlockOptions}. @@ -97,4 +99,27 @@ public BlockBlobStageBlockOptions setContentMd5(byte[] contentMd5) { this.contentMd5 = CoreUtils.clone(contentMd5); return this; } + + /** + * Gets the algorithm to use for transfer content validation on the request. See {@link ContentValidationAlgorithm} + * for more details. + * + * @return The transfer validation checksum algorithm. + */ + public ContentValidationAlgorithm getContentValidationAlgorithm() { + return contentValidationAlgorithm; + } + + /** + * Sets the algorithm to use for transfer content validation on the request. See {@link ContentValidationAlgorithm} + * for more details. + * + * @param contentValidationAlgorithm The transfer validation checksum algorithm. + * @return The updated options. + */ + public BlockBlobStageBlockOptions + setContentValidationAlgorithm(ContentValidationAlgorithm contentValidationAlgorithm) { + this.contentValidationAlgorithm = contentValidationAlgorithm; + return this; + } } diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/PageBlobOutputStreamOptions.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/PageBlobOutputStreamOptions.java new file mode 100644 index 000000000000..5a9ce4b49af4 --- /dev/null +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/PageBlobOutputStreamOptions.java @@ -0,0 +1,83 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.blob.options; + +import com.azure.core.annotation.Fluent; +import com.azure.storage.blob.models.BlobRequestConditions; +import com.azure.storage.blob.models.PageRange; +import com.azure.storage.common.ContentValidationAlgorithm; +import com.azure.storage.common.implementation.StorageImplUtils; + +/** + * Extended options that may be passed when opening an output stream to a page blob. + */ +@Fluent +public final class PageBlobOutputStreamOptions { + private final PageRange pageRange; + private BlobRequestConditions requestConditions; + private ContentValidationAlgorithm contentValidationAlgorithm; + + /** + * Creates a new instance of {@link PageBlobOutputStreamOptions}. + * + * @param pageRange The {@link PageRange} for the write. Pages must be aligned with 512-byte boundaries. + * @throws NullPointerException If {@code pageRange} is null. + */ + public PageBlobOutputStreamOptions(PageRange pageRange) { + StorageImplUtils.assertNotNull("pageRange", pageRange); + this.pageRange = pageRange; + } + + /** + * Gets the page range. + * + * @return The page range. + */ + public PageRange getPageRange() { + return pageRange; + } + + /** + * Gets the {@link BlobRequestConditions}. + * + * @return The request conditions. + */ + public BlobRequestConditions getRequestConditions() { + return requestConditions; + } + + /** + * Sets the {@link BlobRequestConditions}. + * + * @param requestConditions The request conditions. + * @return The updated options. + */ + public PageBlobOutputStreamOptions setRequestConditions(BlobRequestConditions requestConditions) { + this.requestConditions = requestConditions; + return this; + } + + /** + * Gets the algorithm to use for transfer content validation on the request. See {@link ContentValidationAlgorithm} + * for more details. + * + * @return The transfer validation checksum algorithm. + */ + public ContentValidationAlgorithm getContentValidationAlgorithm() { + return contentValidationAlgorithm; + } + + /** + * Sets the algorithm to use for transfer content validation on the request. See {@link ContentValidationAlgorithm} + * for more details. + * + * @param contentValidationAlgorithm The transfer validation checksum algorithm. + * @return The updated options. + */ + public PageBlobOutputStreamOptions + setContentValidationAlgorithm(ContentValidationAlgorithm contentValidationAlgorithm) { + this.contentValidationAlgorithm = contentValidationAlgorithm; + return this; + } +} diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/PageBlobUploadPagesOptions.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/PageBlobUploadPagesOptions.java new file mode 100644 index 000000000000..335bef5f7a1b --- /dev/null +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/PageBlobUploadPagesOptions.java @@ -0,0 +1,149 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.blob.options; + +import com.azure.core.annotation.Fluent; +import com.azure.core.util.CoreUtils; +import com.azure.storage.blob.models.PageBlobRequestConditions; +import com.azure.storage.blob.models.PageRange; +import com.azure.storage.common.ContentValidationAlgorithm; +import com.azure.storage.common.implementation.StorageImplUtils; + +import reactor.core.publisher.Flux; + +import java.io.InputStream; +import java.nio.ByteBuffer; + +/** + * Extended options that may be passed when uploading pages to a page blob. + */ +@Fluent +public final class PageBlobUploadPagesOptions { + private final PageRange pageRange; + private final Flux dataFlux; + private final InputStream dataStream; + private byte[] contentMd5; + private PageBlobRequestConditions requestConditions; + private ContentValidationAlgorithm contentValidationAlgorithm; + + /** + * Creates a new instance of {@link PageBlobUploadPagesOptions}. + * + * @param pageRange The {@link PageRange} for the upload. Pages must be aligned with 512-byte boundaries. + * @param body The data to upload. Must be replayable if retries are enabled. + * @throws NullPointerException If {@code pageRange} or {@code body} is null. + */ + public PageBlobUploadPagesOptions(PageRange pageRange, Flux body) { + StorageImplUtils.assertNotNull("pageRange", pageRange); + StorageImplUtils.assertNotNull("body", body); + this.pageRange = pageRange; + this.dataFlux = body; + this.dataStream = null; + } + + /** + * Creates a new instance of {@link PageBlobUploadPagesOptions}. + * + * @param pageRange The {@link PageRange} for the upload. Pages must be aligned with 512-byte boundaries. + * @param body The data to upload. Must be markable for retries. + * @throws NullPointerException If {@code pageRange} or {@code body} is null. + */ + public PageBlobUploadPagesOptions(PageRange pageRange, InputStream body) { + StorageImplUtils.assertNotNull("pageRange", pageRange); + StorageImplUtils.assertNotNull("body", body); + this.pageRange = pageRange; + this.dataFlux = null; + this.dataStream = body; + } + + /** + * Gets the page range. + * + * @return The page range. + */ + public PageRange getPageRange() { + return pageRange; + } + + /** + * Gets the body as a Flux. Null if constructed with InputStream. + * + * @return The body flux, or null. + */ + public Flux getDataFlux() { + return dataFlux; + } + + /** + * Gets the body as an InputStream. Null if constructed with Flux. + * + * @return The body stream, or null. + */ + public InputStream getDataStream() { + return dataStream; + } + + /** + * Gets the MD5 hash of the page content. + * + * @return An MD5 hash of the content, or null. + */ + public byte[] getContentMd5() { + return CoreUtils.clone(contentMd5); + } + + /** + * Sets the MD5 hash of the page content for transactional verification. + * + * @param contentMd5 An MD5 hash of the page content. + * @return The updated options. + */ + public PageBlobUploadPagesOptions setContentMd5(byte[] contentMd5) { + this.contentMd5 = CoreUtils.clone(contentMd5); + return this; + } + + /** + * Gets the {@link PageBlobRequestConditions}. + * + * @return The request conditions. + */ + public PageBlobRequestConditions getRequestConditions() { + return requestConditions; + } + + /** + * Sets the {@link PageBlobRequestConditions}. + * + * @param requestConditions The request conditions. + * @return The updated options. + */ + public PageBlobUploadPagesOptions setRequestConditions(PageBlobRequestConditions requestConditions) { + this.requestConditions = requestConditions; + return this; + } + + /** + * Gets the algorithm to use for transfer content validation on the request. See {@link ContentValidationAlgorithm} + * for more details. + * + * @return The transfer validation checksum algorithm. + */ + public ContentValidationAlgorithm getContentValidationAlgorithm() { + return contentValidationAlgorithm; + } + + /** + * Sets the algorithm to use for transfer content validation on the request. See {@link ContentValidationAlgorithm} + * for more details. + * + * @param contentValidationAlgorithm The transfer validation checksum algorithm. + * @return The updated options. + */ + public PageBlobUploadPagesOptions + setContentValidationAlgorithm(ContentValidationAlgorithm contentValidationAlgorithm) { + this.contentValidationAlgorithm = contentValidationAlgorithm; + return this; + } +} diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/AppendBlobAsyncClient.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/AppendBlobAsyncClient.java index 42b7b3f76f1c..3a47a4be200a 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/AppendBlobAsyncClient.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/AppendBlobAsyncClient.java @@ -34,10 +34,14 @@ import com.azure.storage.blob.models.CpkInfo; import com.azure.storage.blob.models.CustomerProvidedKey; import com.azure.storage.blob.models.EncryptionAlgorithmType; +import com.azure.storage.blob.options.AppendBlobAppendBlockOptions; import com.azure.storage.blob.options.AppendBlobAppendBlockFromUrlOptions; import com.azure.storage.blob.options.AppendBlobCreateOptions; import com.azure.storage.blob.options.AppendBlobSealOptions; +import com.azure.storage.common.ContentValidationAlgorithm; import com.azure.storage.common.implementation.Constants; +import com.azure.storage.common.implementation.contentvalidation.ContentValidationModeResolver; + import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; @@ -451,9 +455,34 @@ public Mono appendBlock(Flux data, long length) { @ServiceMethod(returns = ReturnType.SINGLE) public Mono> appendBlockWithResponse(Flux data, long length, byte[] contentMd5, AppendBlobRequestConditions appendBlobRequestConditions) { + if (data == null) { + return monoError(LOGGER, new NullPointerException("'data' cannot be null.")); + } + return appendBlockWithResponse(new AppendBlobAppendBlockOptions(data, length).setContentMd5(contentMd5) + .setRequestConditions(appendBlobRequestConditions)); + } + + /** + * Commits a new block of data to the end of the existing append blob with options. + * + * @param options {@link AppendBlobAppendBlockOptions} containing the block data. + * @return A {@link Mono} containing {@link Response} whose value contains the append blob operation. + * @throws NullPointerException If {@code options} is null. + * @throws IllegalArgumentException If options were not constructed with Flux (async client). + */ + @ServiceMethod(returns = ReturnType.SINGLE) + public Mono> appendBlockWithResponse(AppendBlobAppendBlockOptions options) { try { - return withContext( - context -> appendBlockWithResponse(data, length, contentMd5, appendBlobRequestConditions, context)); + if (options == null) { + return monoError(LOGGER, new NullPointerException("'options' cannot be null.")); + } + if (options.getDataFlux() == null) { + return monoError(LOGGER, new IllegalArgumentException( + "AppendBlobAppendBlockOptions must be constructed with Flux for async client.")); + } + return withContext(context -> appendBlockWithResponseInternal(options.getDataFlux(), options.getLength(), + options.getContentMd5(), options.getRequestConditions(), options.getContentValidationAlgorithm(), + context)); } catch (RuntimeException ex) { return monoError(LOGGER, ex); } @@ -461,14 +490,21 @@ public Mono> appendBlockWithResponse(Flux d Mono> appendBlockWithResponse(Flux data, long length, byte[] contentMd5, AppendBlobRequestConditions appendBlobRequestConditions, Context context) { + // Prevents revapi visibility increased error + return appendBlockWithResponseInternal(data, length, contentMd5, appendBlobRequestConditions, null, context); + } + Mono> appendBlockWithResponseInternal(Flux data, long length, + byte[] contentMd5, AppendBlobRequestConditions appendBlobRequestConditions, + ContentValidationAlgorithm contentValidationAlgorithm, Context context) { if (data == null) { - return Mono.error(new NullPointerException("'data' cannot be null.")); + return monoError(LOGGER, new NullPointerException("'data' cannot be null.")); } appendBlobRequestConditions = appendBlobRequestConditions == null ? new AppendBlobRequestConditions() : appendBlobRequestConditions; - context = context == null ? Context.NONE : context; + context = ContentValidationModeResolver.addContentValidationMode(context, contentValidationAlgorithm, length, + false); return this.azureBlobStorage.getAppendBlobs() .appendBlockWithResponseAsync(containerName, blobName, length, data, null, contentMd5, null, diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/AppendBlobClient.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/AppendBlobClient.java index 0fd58f5ef575..320f54caa3a7 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/AppendBlobClient.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/AppendBlobClient.java @@ -6,7 +6,6 @@ import com.azure.core.annotation.ReturnType; import com.azure.core.annotation.ServiceClient; import com.azure.core.annotation.ServiceMethod; -import com.azure.core.exception.UnexpectedLengthException; import com.azure.core.http.HttpPipeline; import com.azure.core.http.HttpResponse; import com.azure.core.http.rest.Response; @@ -32,8 +31,10 @@ import com.azure.storage.blob.models.BlobStorageException; import com.azure.storage.blob.models.CpkInfo; import com.azure.storage.blob.models.CustomerProvidedKey; +import com.azure.storage.blob.options.AppendBlobAppendBlockOptions; import com.azure.storage.blob.options.AppendBlobAppendBlockFromUrlOptions; import com.azure.storage.blob.options.AppendBlobCreateOptions; +import com.azure.storage.blob.options.AppendBlobOutputStreamOptions; import com.azure.storage.blob.options.AppendBlobSealOptions; import com.azure.storage.common.Utility; import com.azure.storage.common.implementation.Constants; @@ -45,7 +46,6 @@ import java.nio.ByteBuffer; import java.time.Duration; import java.util.Map; -import java.util.Objects; import java.util.concurrent.Callable; import static com.azure.storage.common.implementation.StorageImplUtils.sendRequest; @@ -181,8 +181,9 @@ public AppendBlobClient getCustomerProvidedKeyClient(CustomerProvidedKey custome * @return A {@link BlobOutputStream} object used to write data to the blob. * @throws BlobStorageException If a storage service error occurred. */ + @ServiceMethod(returns = ReturnType.SINGLE) public BlobOutputStream getBlobOutputStream() { - return getBlobOutputStream(null); + return getBlobOutputStream((AppendBlobRequestConditions) null); } /** @@ -194,6 +195,7 @@ public BlobOutputStream getBlobOutputStream() { * @param overwrite Whether an existing blob should be deleted and recreated, should data exist on the blob. * @throws BlobStorageException If a storage service error occurred. */ + @ServiceMethod(returns = ReturnType.SINGLE) public BlobOutputStream getBlobOutputStream(boolean overwrite) { AppendBlobRequestConditions requestConditions = null; if (!overwrite) { @@ -214,10 +216,24 @@ public BlobOutputStream getBlobOutputStream(boolean overwrite) { * @return A {@link BlobOutputStream} object used to write data to the blob. * @throws BlobStorageException If a storage service error occurred. */ + @ServiceMethod(returns = ReturnType.SINGLE) public BlobOutputStream getBlobOutputStream(AppendBlobRequestConditions requestConditions) { return BlobOutputStream.appendBlobOutputStream(appendBlobAsyncClient, requestConditions); } + /** + * Creates and opens an output stream to write data to the append blob. + * + * @param options {@link AppendBlobOutputStreamOptions} + * @return A {@link BlobOutputStream} object used to write data to the blob. + */ + @ServiceMethod(returns = ReturnType.SINGLE) + public BlobOutputStream getBlobOutputStream(AppendBlobOutputStreamOptions options) { + options = options == null ? new AppendBlobOutputStreamOptions() : options; + return BlobOutputStream.appendBlobOutputStream(appendBlobAsyncClient, options.getRequestConditions(), + options.getContentValidationAlgorithm()); + } + /** * Creates a 0-length append blob. Call appendBlock to append data to an append blob. By default this method will * not overwrite an existing blob. @@ -499,21 +515,43 @@ public AppendBlobItem appendBlock(InputStream data, long length) { * @param timeout An optional timeout value beyond which a {@link RuntimeException} will be raised. * @param context Additional context that is passed through the Http pipeline during the service call. * @return A {@link Response} whose {@link Response#getValue() value} contains the append blob operation. - * @throws UnexpectedLengthException when the length of data does not match the input {@code length}. * @throws NullPointerException if the input data is null. */ @ServiceMethod(returns = ReturnType.SINGLE) public Response appendBlockWithResponse(InputStream data, long length, byte[] contentMd5, AppendBlobRequestConditions appendBlobRequestConditions, Duration timeout, Context context) { - Objects.requireNonNull(data, "'data' cannot be null."); + return appendBlockWithResponse(new AppendBlobAppendBlockOptions(data, length).setContentMd5(contentMd5) + .setRequestConditions(appendBlobRequestConditions), timeout, context); + } + + /** + * Commits a new block of data to the end of the existing append blob with options. + * + * @param options {@link AppendBlobAppendBlockOptions} containing the block data. + * @param timeout An optional timeout value. + * @param context Additional context. + * @return The information of the append blob operation. + * @throws NullPointerException If {@code options} is null. + * @throws IllegalArgumentException If {@code options} is not constructed with {@link InputStream}. + */ + @ServiceMethod(returns = ReturnType.SINGLE) + public Response appendBlockWithResponse(AppendBlobAppendBlockOptions options, Duration timeout, + Context context) { + StorageImplUtils.assertNotNull("options", options); + if (options.getDataStream() == null) { + throw LOGGER.logExceptionAsError(new IllegalArgumentException( + "AppendBlobAppendBlockOptions must be constructed with InputStream for sync client.")); + } Flux fbb; // service versions 2022-11-02 and above support uploading block bytes up to 100MB, all older service versions // support up to 4MB - fbb = Utility.convertStreamToByteBuffer(data, length, getMaxAppendBlockBytes(), true); + fbb = Utility.convertStreamToByteBuffer(options.getDataStream(), options.getLength(), getMaxAppendBlockBytes(), + true); - Mono> response = appendBlobAsyncClient.appendBlockWithResponse(fbb, length, contentMd5, - appendBlobRequestConditions, context); + Mono> response + = appendBlobAsyncClient.appendBlockWithResponseInternal(fbb, options.getLength(), options.getContentMd5(), + options.getRequestConditions(), options.getContentValidationAlgorithm(), context); return StorageImplUtils.blockWithOptionalTimeout(response, timeout); } diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobAsyncClientBase.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobAsyncClientBase.java index ef8ec9d2d4d8..c3392bfef4f9 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobAsyncClientBase.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobAsyncClientBase.java @@ -73,16 +73,21 @@ import com.azure.storage.blob.models.UserDelegationKey; import com.azure.storage.blob.options.BlobBeginCopyOptions; import com.azure.storage.blob.options.BlobCopyFromUrlOptions; +import com.azure.storage.blob.options.BlobDownloadContentOptions; +import com.azure.storage.blob.options.BlobDownloadStreamOptions; import com.azure.storage.blob.options.BlobDownloadToFileOptions; import com.azure.storage.blob.options.BlobGetTagsOptions; import com.azure.storage.blob.options.BlobQueryOptions; import com.azure.storage.blob.options.BlobSetAccessTierOptions; import com.azure.storage.blob.options.BlobSetTagsOptions; import com.azure.storage.blob.sas.BlobServiceSasSignatureValues; +import com.azure.storage.common.ContentValidationAlgorithm; import com.azure.storage.common.StorageSharedKeyCredential; import com.azure.storage.common.Utility; import com.azure.storage.common.implementation.SasImplUtils; import com.azure.storage.common.implementation.StorageImplUtils; +import com.azure.storage.common.implementation.contentvalidation.ContentValidationModeResolver; + import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; import reactor.core.publisher.SignalType; @@ -529,7 +534,7 @@ Mono> existsWithResponse(Context context) { return Mono.just(new SimpleResponse<>(response.getRequest(), response.getStatusCode(), response.getHeaders(), false)); } else { - return Mono.error(e); + return monoError(LOGGER, e); } }); } @@ -722,8 +727,8 @@ public PollerFlux beginCopy(BlobBeginCopyOptions options) { } }, (pollingContext, firstResponse) -> { if (firstResponse == null || firstResponse.getValue() == null) { - return Mono.error(LOGGER.logExceptionAsError( - new IllegalArgumentException("Cannot cancel a poll response that never started."))); + return monoError(LOGGER, + new IllegalArgumentException("Cannot cancel a poll response that never started.")); } final String copyIdentifier = firstResponse.getValue().getCopyId(); @@ -1166,9 +1171,25 @@ public Mono downloadWithResponse(BlobRange range, Dow @ServiceMethod(returns = ReturnType.SINGLE) public Mono downloadStreamWithResponse(BlobRange range, DownloadRetryOptions options, BlobRequestConditions requestConditions, boolean getRangeContentMd5) { + return downloadStreamWithResponse(new BlobDownloadStreamOptions().setRange(range) + .setDownloadRetryOptions(options) + .setRequestConditions(requestConditions) + .setRetrieveContentRangeMd5(getRangeContentMd5)); + } + + /** + * Reads a range of bytes from a blob with options. + * + * @param options {@link BlobDownloadStreamOptions} + * @return A reactive response containing the blob data. + */ + @ServiceMethod(returns = ReturnType.SINGLE) + public Mono downloadStreamWithResponse(BlobDownloadStreamOptions options) { try { - return withContext( - context -> downloadStreamWithResponse(range, options, requestConditions, getRangeContentMd5, context)); + BlobDownloadStreamOptions finalOptions = options == null ? new BlobDownloadStreamOptions() : options; + return withContext(context -> downloadStreamWithResponseInternal(finalOptions.getRange(), + finalOptions.getDownloadRetryOptions(), finalOptions.getRequestConditions(), + finalOptions.isRetrieveContentRangeMd5(), finalOptions.getContentValidationAlgorithm(), context)); } catch (RuntimeException ex) { return monoError(LOGGER, ex); } @@ -1205,11 +1226,26 @@ public Mono downloadStreamWithResponse(BlobRange rang @ServiceMethod(returns = ReturnType.SINGLE) public Mono downloadContentWithResponse(DownloadRetryOptions options, BlobRequestConditions requestConditions) { + return downloadContentWithResponse( + new BlobDownloadContentOptions().setDownloadRetryOptions(options).setRequestConditions(requestConditions)); + } + + /** + * Reads blob content (full blob or range) with options. + * + * @param options {@link BlobDownloadContentOptions} + * @return A reactive response containing the blob content. + */ + @ServiceMethod(returns = ReturnType.SINGLE) + public Mono downloadContentWithResponse(BlobDownloadContentOptions options) { try { - return withContext(context -> downloadStreamWithResponse(null, options, requestConditions, false, context) - .flatMap(r -> BinaryData.fromFlux(r.getValue()) - .map(data -> new BlobDownloadContentAsyncResponse(r.getRequest(), r.getStatusCode(), r.getHeaders(), - data, r.getDeserializedHeaders())))); + BlobDownloadContentOptions finalOptions = options == null ? new BlobDownloadContentOptions() : options; + return withContext(context -> downloadStreamWithResponseInternal(finalOptions.getRange(), + finalOptions.getDownloadRetryOptions(), finalOptions.getRequestConditions(), + finalOptions.isRetrieveContentRangeMd5(), finalOptions.getContentValidationAlgorithm(), context) + .flatMap(r -> BinaryData.fromFlux(r.getValue()) + .map(data -> new BlobDownloadContentAsyncResponse(r.getRequest(), r.getStatusCode(), + r.getHeaders(), data, r.getDeserializedHeaders())))); } catch (RuntimeException ex) { return monoError(LOGGER, ex); } @@ -1217,17 +1253,27 @@ public Mono downloadContentWithResponse(Downlo Mono downloadStreamWithResponse(BlobRange range, DownloadRetryOptions options, BlobRequestConditions requestConditions, boolean getRangeContentMd5, Context context) { + // Prevents revapi visibility increased error + return downloadStreamWithResponseInternal(range, options, requestConditions, getRangeContentMd5, null, context); + } + + Mono downloadStreamWithResponseInternal(BlobRange range, DownloadRetryOptions options, + BlobRequestConditions requestConditions, boolean getRangeContentMd5, + ContentValidationAlgorithm contentValidationAlgorithm, Context context) { BlobRange finalRange = range == null ? new BlobRange(0) : range; Boolean getMD5 = getRangeContentMd5 ? getRangeContentMd5 : null; BlobRequestConditions finalRequestConditions = requestConditions == null ? new BlobRequestConditions() : requestConditions; DownloadRetryOptions finalOptions = (options == null) ? new DownloadRetryOptions() : options; + context + = ContentValidationModeResolver.addStructuredMessageDecodingToContext(context, contentValidationAlgorithm); + // The first range should eagerly convert headers as they'll be used to create response types. Context firstRangeContext = context == null ? new Context("azure-eagerly-convert-headers", true) : context.addData("azure-eagerly-convert-headers", true); - + Context nextRangeContext = context; return downloadRange(finalRange, finalRequestConditions, finalRequestConditions.getIfMatch(), getMD5, firstRangeContext).map(response -> { BlobsDownloadHeaders blobsDownloadHeaders = new BlobsDownloadHeaders(response.getHeaders()); @@ -1272,7 +1318,7 @@ Mono downloadStreamWithResponse(BlobRange range, Down try { return downloadRange(new BlobRange(initialOffset + offset, newCount), finalRequestConditions, - eTag, getMD5, context); + eTag, getMD5, nextRangeContext); } catch (Exception e) { return Mono.error(e); } @@ -1504,7 +1550,8 @@ Mono> downloadToFileWithResponse(BlobDownloadToFileOpti AsynchronousFileChannel channel = downloadToFileResourceSupplier(options.getFilePath(), openOptions); return Mono.just(channel) .flatMap(c -> this.downloadToFileImpl(c, finalRange, finalParallelTransferOptions, - options.getDownloadRetryOptions(), finalConditions, options.isRetrieveContentRangeMd5(), context)) + options.getDownloadRetryOptions(), finalConditions, options.isRetrieveContentRangeMd5(), + options.getContentValidationAlgorithm(), context)) .doFinally(signalType -> this.downloadToFileCleanup(channel, options.getFilePath(), signalType)); } @@ -1519,7 +1566,7 @@ private AsynchronousFileChannel downloadToFileResourceSupplier(String filePath, private Mono> downloadToFileImpl(AsynchronousFileChannel file, BlobRange finalRange, com.azure.storage.common.ParallelTransferOptions finalParallelTransferOptions, DownloadRetryOptions downloadRetryOptions, BlobRequestConditions requestConditions, boolean rangeGetContentMd5, - Context context) { + ContentValidationAlgorithm contentValidationAlgorithm, Context context) { // See ProgressReporter for an explanation on why this lock is necessary and why we use AtomicLong. ProgressListener progressReceiver = finalParallelTransferOptions.getProgressListener(); ProgressReporter progressReporter @@ -1529,8 +1576,8 @@ private Mono> downloadToFileImpl(AsynchronousFileChanne * Downloads the first chunk and gets the size of the data and etag if not specified by the user. */ BiFunction> downloadFunc - = (range, conditions) -> this.downloadStreamWithResponse(range, downloadRetryOptions, conditions, - rangeGetContentMd5, context); + = (range, conditions) -> this.downloadStreamWithResponseInternal(range, downloadRetryOptions, conditions, + rangeGetContentMd5, contentValidationAlgorithm, context); return ChunkedDownloadUtils .downloadFirstChunk(finalRange, finalParallelTransferOptions, requestConditions, downloadFunc, true, diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobClientBase.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobClientBase.java index 9c44f4cc8e84..b35c654a1132 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobClientBase.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobClientBase.java @@ -76,6 +76,8 @@ import com.azure.storage.blob.models.UserDelegationKey; import com.azure.storage.blob.options.BlobBeginCopyOptions; import com.azure.storage.blob.options.BlobCopyFromUrlOptions; +import com.azure.storage.blob.options.BlobDownloadContentOptions; +import com.azure.storage.blob.options.BlobDownloadStreamOptions; import com.azure.storage.blob.options.BlobDownloadToFileOptions; import com.azure.storage.blob.options.BlobGetTagsOptions; import com.azure.storage.blob.options.BlobInputStreamOptions; @@ -84,6 +86,7 @@ import com.azure.storage.blob.options.BlobSetAccessTierOptions; import com.azure.storage.blob.options.BlobSetTagsOptions; import com.azure.storage.blob.sas.BlobServiceSasSignatureValues; +import com.azure.storage.common.ContentValidationAlgorithm; import com.azure.storage.common.StorageSharedKeyCredential; import com.azure.storage.common.Utility; import com.azure.storage.common.implementation.Constants; @@ -91,6 +94,7 @@ import com.azure.storage.common.implementation.SasImplUtils; import com.azure.storage.common.implementation.StorageImplUtils; import com.azure.storage.common.implementation.StorageSeekableByteChannel; + import reactor.core.publisher.Mono; import java.io.IOException; @@ -500,6 +504,7 @@ public BlobInputStream openInputStream(BlobInputStreamOptions options) { public BlobInputStream openInputStream(BlobInputStreamOptions options, Context context) { Context contextFinal = context == null ? Context.NONE : context; options = options == null ? new BlobInputStreamOptions() : options; + final ContentValidationAlgorithm contentValidationAlgorithm = options.getContentValidationAlgorithm(); ConsistentReadControl consistentReadControl = options.getConsistentReadControl() == null ? ConsistentReadControl.ETAG : options.getConsistentReadControl(); @@ -511,8 +516,9 @@ public BlobInputStream openInputStream(BlobInputStreamOptions options, Context c com.azure.storage.common.ParallelTransferOptions parallelTransferOptions = new com.azure.storage.common.ParallelTransferOptions().setBlockSizeLong((long) chunkSize); - BiFunction> downloadFunc = (chunkRange, - conditions) -> client.downloadStreamWithResponse(chunkRange, null, conditions, false, contextFinal); + BiFunction> downloadFunc + = (chunkRange, conditions) -> client.downloadStreamWithResponseInternal(chunkRange, null, conditions, false, + contentValidationAlgorithm, contextFinal); return ChunkedDownloadUtils .downloadFirstChunk(range, parallelTransferOptions, requestConditions, downloadFunc, true) .flatMap(tuple3 -> { @@ -588,8 +594,12 @@ public BlobSeekableByteChannelReadResult openSeekableByteChannelRead(BlobSeekabl BlobDownloadResponse response; try (ByteBufferBackedOutputStreamUtil dstStream = new ByteBufferBackedOutputStreamUtil(initialRange)) { response = this.downloadStreamWithResponse(dstStream, - new BlobRange(initialPosition, (long) initialRange.remaining()), null /*downloadRetryOptions*/, - options.getRequestConditions(), false, null, context); + new BlobDownloadStreamOptions() + .setRange(new BlobRange(initialPosition, (long) initialRange.remaining())) + .setRequestConditions(options.getRequestConditions()) + .setRetrieveContentRangeMd5(false) + .setContentValidationAlgorithm(options.getContentValidationAlgorithm()), + null, context); properties = ModelHelper.buildBlobPropertiesResponse(response).getValue(); } catch (IOException e) { throw LOGGER.logExceptionAsError(new UncheckedIOException(e)); @@ -1266,12 +1276,35 @@ public BlobDownloadResponse downloadWithResponse(OutputStream stream, BlobRange @ServiceMethod(returns = ReturnType.SINGLE) public BlobDownloadResponse downloadStreamWithResponse(OutputStream stream, BlobRange range, DownloadRetryOptions options, BlobRequestConditions requestConditions, boolean getRangeContentMd5, + Duration timeout, Context context) { + return downloadStreamWithResponse(stream, + new BlobDownloadStreamOptions().setRange(range) + .setDownloadRetryOptions(options) + .setRequestConditions(requestConditions) + .setRetrieveContentRangeMd5(getRangeContentMd5), + timeout, context); + } + + /** + * Downloads a range of bytes from a blob into an output stream with options. + * + * @param stream The output stream where the downloaded data will be written. + * @param options {@link BlobDownloadStreamOptions} + * @param timeout An optional timeout value. + * @param context Additional context. + * @return A response containing status code and HTTP headers. + */ + @ServiceMethod(returns = ReturnType.SINGLE) + public BlobDownloadResponse downloadStreamWithResponse(OutputStream stream, BlobDownloadStreamOptions options, Duration timeout, Context context) { StorageImplUtils.assertNotNull("stream", stream); - Mono download - = client.downloadStreamWithResponse(range, options, requestConditions, getRangeContentMd5, context) - .flatMap(response -> FluxUtil.writeToOutputStream(response.getValue(), stream) - .thenReturn(new BlobDownloadResponse(response))); + options = options == null ? new BlobDownloadStreamOptions() : options; + Mono download = client + .downloadStreamWithResponseInternal(options.getRange(), options.getDownloadRetryOptions(), + options.getRequestConditions(), options.isRetrieveContentRangeMd5(), + options.getContentValidationAlgorithm(), context) + .flatMap(response -> FluxUtil.writeToOutputStream(response.getValue(), stream) + .thenReturn(new BlobDownloadResponse(response))); return blockWithOptionalTimeout(download, timeout); } @@ -1310,14 +1343,9 @@ public BlobDownloadResponse downloadStreamWithResponse(OutputStream stream, Blob @ServiceMethod(returns = ReturnType.SINGLE) public BlobDownloadContentResponse downloadContentWithResponse(DownloadRetryOptions options, BlobRequestConditions requestConditions, Duration timeout, Context context) { - Mono download - = client.downloadStreamWithResponse(null, options, requestConditions, false, context) - .flatMap(r -> BinaryData.fromFlux(r.getValue()) - .map(data -> new BlobDownloadContentAsyncResponse(r.getRequest(), r.getStatusCode(), r.getHeaders(), - data, r.getDeserializedHeaders()))) - .map(BlobDownloadContentResponse::new); - - return blockWithOptionalTimeout(download, timeout); + return downloadContentWithResponse( + new BlobDownloadContentOptions().setDownloadRetryOptions(options).setRequestConditions(requestConditions), + timeout, context); } /** @@ -1358,12 +1386,32 @@ public BlobDownloadContentResponse downloadContentWithResponse(DownloadRetryOpti public BlobDownloadContentResponse downloadContentWithResponse(DownloadRetryOptions options, BlobRequestConditions requestConditions, BlobRange range, boolean getRangeContentMd5, Duration timeout, Context context) { - Mono download - = client.downloadStreamWithResponse(range, options, requestConditions, getRangeContentMd5, context) - .flatMap(r -> BinaryData.fromFlux(r.getValue()) - .map(data -> new BlobDownloadContentAsyncResponse(r.getRequest(), r.getStatusCode(), r.getHeaders(), - data, r.getDeserializedHeaders()))) - .map(BlobDownloadContentResponse::new); + return downloadContentWithResponse(new BlobDownloadContentOptions().setDownloadRetryOptions(options) + .setRequestConditions(requestConditions) + .setRange(range) + .setRetrieveContentRangeMd5(getRangeContentMd5), timeout, context); + } + + /** + * Downloads blob content (full blob or range) with options. + * + * @param options {@link BlobDownloadContentOptions} + * @param timeout An optional timeout value. + * @param context Additional context. + * @return A response containing status code and HTTP headers. + */ + @ServiceMethod(returns = ReturnType.SINGLE) + public BlobDownloadContentResponse downloadContentWithResponse(BlobDownloadContentOptions options, Duration timeout, + Context context) { + options = options == null ? new BlobDownloadContentOptions() : options; + Mono download = client + .downloadStreamWithResponseInternal(options.getRange(), options.getDownloadRetryOptions(), + options.getRequestConditions(), options.isRetrieveContentRangeMd5(), + options.getContentValidationAlgorithm(), context) + .flatMap(r -> BinaryData.fromFlux(r.getValue()) + .map(data -> new BlobDownloadContentAsyncResponse(r.getRequest(), r.getStatusCode(), r.getHeaders(), + data, r.getDeserializedHeaders()))) + .map(BlobDownloadContentResponse::new); return blockWithOptionalTimeout(download, timeout); } diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobOutputStream.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobOutputStream.java index c43277ff30de..0ba385c12ca1 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobOutputStream.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobOutputStream.java @@ -16,10 +16,13 @@ import com.azure.storage.blob.models.PageBlobRequestConditions; import com.azure.storage.blob.models.PageRange; import com.azure.storage.blob.models.ParallelTransferOptions; +import com.azure.storage.blob.options.AppendBlobAppendBlockOptions; import com.azure.storage.blob.options.BlobParallelUploadOptions; import com.azure.storage.blob.options.BlockBlobOutputStreamOptions; +import com.azure.storage.common.ContentValidationAlgorithm; import com.azure.storage.common.StorageOutputStream; import com.azure.storage.common.implementation.Constants; + import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; @@ -47,7 +50,13 @@ public abstract class BlobOutputStream extends StorageOutputStream { static BlobOutputStream appendBlobOutputStream(final AppendBlobAsyncClient client, final AppendBlobRequestConditions appendBlobRequestConditions) { - return new AppendBlobOutputStream(client, appendBlobRequestConditions); + return new AppendBlobOutputStream(client, appendBlobRequestConditions, null); + } + + static BlobOutputStream appendBlobOutputStream(final AppendBlobAsyncClient client, + final AppendBlobRequestConditions appendBlobRequestConditions, + final ContentValidationAlgorithm contentValidationAlgorithm) { + return new AppendBlobOutputStream(client, appendBlobRequestConditions, contentValidationAlgorithm); } /** @@ -104,12 +113,18 @@ public static BlobOutputStream blockBlobOutputStream(final BlobAsyncClient clien BlockBlobOutputStreamOptions options, Context context) { options = options == null ? new BlockBlobOutputStreamOptions() : options; return new BlockBlobOutputStream(client, options.getParallelTransferOptions(), options.getHeaders(), - options.getMetadata(), options.getTags(), options.getTier(), options.getRequestConditions(), context); + options.getMetadata(), options.getTags(), options.getTier(), options.getRequestConditions(), + options.getContentValidationAlgorithm(), context); } static BlobOutputStream pageBlobOutputStream(final PageBlobAsyncClient client, final PageRange pageRange, final BlobRequestConditions requestConditions) { - return new PageBlobOutputStream(client, pageRange, requestConditions); + return pageBlobOutputStream(client, pageRange, requestConditions, null); + } + + static BlobOutputStream pageBlobOutputStream(final PageBlobAsyncClient client, final PageRange pageRange, + final BlobRequestConditions requestConditions, final ContentValidationAlgorithm contentValidationAlgorithm) { + return new PageBlobOutputStream(client, pageRange, requestConditions, contentValidationAlgorithm); } abstract void commit(); @@ -157,9 +172,11 @@ private static final class AppendBlobOutputStream extends BlobOutputStream { private final AppendBlobRequestConditions appendBlobRequestConditions; private final AppendBlobAsyncClient client; + private final ContentValidationAlgorithm contentValidationAlgorithm; private AppendBlobOutputStream(final AppendBlobAsyncClient client, - final AppendBlobRequestConditions appendBlobRequestConditions) { + final AppendBlobRequestConditions appendBlobRequestConditions, + final ContentValidationAlgorithm contentValidationAlgorithm) { // service versions 2022-11-02 and above support uploading block bytes up to 100MB, all older service // versions support up to 4MB super(client.getServiceVersion().ordinal() < BlobServiceVersion.V2022_11_02.ordinal() @@ -170,6 +187,7 @@ private AppendBlobOutputStream(final AppendBlobAsyncClient client, this.appendBlobRequestConditions = (appendBlobRequestConditions == null) ? new AppendBlobRequestConditions() : appendBlobRequestConditions; + this.contentValidationAlgorithm = contentValidationAlgorithm; if (this.appendBlobRequestConditions.getAppendPosition() == null) { this.appendBlobRequestConditions.setAppendPosition(client.getProperties().block().getBlobSize()); @@ -178,7 +196,10 @@ private AppendBlobOutputStream(final AppendBlobAsyncClient client, private Mono appendBlock(Flux blockData, long writeLength) { long newAppendOffset = appendBlobRequestConditions.getAppendPosition() + writeLength; - return client.appendBlockWithResponse(blockData, writeLength, null, appendBlobRequestConditions) + AppendBlobAppendBlockOptions opts = new AppendBlobAppendBlockOptions(blockData, writeLength) + .setRequestConditions(appendBlobRequestConditions) + .setContentValidationAlgorithm(contentValidationAlgorithm); + return client.appendBlockWithResponse(opts) .doOnNext(ignored -> appendBlobRequestConditions.setAppendPosition(newAppendOffset)) .then() .onErrorResume(t -> t instanceof IOException || t instanceof BlobStorageException, e -> { @@ -223,7 +244,8 @@ private static final class BlockBlobOutputStream extends BlobOutputStream { private BlockBlobOutputStream(final BlobAsyncClient client, final ParallelTransferOptions parallelTransferOptions, final BlobHttpHeaders headers, final Map metadata, Map tags, final AccessTier tier, - final BlobRequestConditions requestConditions, Context context) { + final BlobRequestConditions requestConditions, final ContentValidationAlgorithm contentValidationAlgorithm, + Context context) { super(Integer.MAX_VALUE); // writeThreshold is effectively not used by BlockBlobOutputStream. // There is a bug in reactor core that does not handle converting Context.NONE to a reactor context. context = context == null || context.equals(Context.NONE) ? null : context; @@ -241,7 +263,8 @@ private BlockBlobOutputStream(final BlobAsyncClient client, .setMetadata(metadata) .setTags(tags) .setTier(tier) - .setRequestConditions(requestConditions)) + .setRequestConditions(requestConditions) + .setContentValidationAlgorithm(contentValidationAlgorithm)) // This allows the operation to continue while maintaining the error that occurred. .onErrorResume(e -> { if (e instanceof IOException) { @@ -319,12 +342,15 @@ private static final class PageBlobOutputStream extends BlobOutputStream { private final PageBlobAsyncClient client; private final PageBlobRequestConditions pageBlobRequestConditions; private final PageRange pageRange; + private final ContentValidationAlgorithm contentValidationAlgorithm; private PageBlobOutputStream(final PageBlobAsyncClient client, final PageRange pageRange, - final BlobRequestConditions blobRequestConditions) { + final BlobRequestConditions blobRequestConditions, + final ContentValidationAlgorithm contentValidationAlgorithm) { super(PageBlobClient.MAX_PUT_PAGES_BYTES); this.client = client; this.pageRange = pageRange; + this.contentValidationAlgorithm = contentValidationAlgorithm; if (blobRequestConditions != null) { this.pageBlobRequestConditions @@ -340,8 +366,8 @@ private PageBlobOutputStream(final PageBlobAsyncClient client, final PageRange p private Mono writePages(Flux pageData, int length, long offset) { return client - .uploadPagesWithResponse(new PageRange().setStart(offset).setEnd(offset + length - 1), pageData, null, - pageBlobRequestConditions) + .uploadPagesWithResponseInternal(new PageRange().setStart(offset).setEnd(offset + length - 1), pageData, + null, pageBlobRequestConditions, contentValidationAlgorithm, com.azure.core.util.Context.NONE) .then() .onErrorResume(BlobStorageException.class, e -> { this.lastError = new IOException(e); diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlockBlobAsyncClient.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlockBlobAsyncClient.java index f938e21b4793..7546ae561a87 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlockBlobAsyncClient.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlockBlobAsyncClient.java @@ -42,6 +42,8 @@ import com.azure.storage.common.Utility; import com.azure.storage.common.implementation.Constants; import com.azure.storage.common.implementation.StorageImplUtils; +import com.azure.storage.common.ContentValidationAlgorithm; +import com.azure.storage.common.implementation.contentvalidation.ContentValidationModeResolver; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; @@ -415,6 +417,9 @@ public Mono> uploadWithResponse(BlockBlobSimpleUploadOpt Mono> uploadWithResponse(BlockBlobSimpleUploadOptions options, Context context) { StorageImplUtils.assertNotNull("options", options); + + ContentValidationAlgorithm contentValidationAlgorithm = options.getContentValidationAlgorithm(); + Mono dataMono; BinaryData binaryData = options.getData(); if (binaryData == null) { @@ -428,25 +433,32 @@ Mono> uploadWithResponse(BlockBlobSimpleUploadOptions op } BlobRequestConditions requestConditions = options.getRequestConditions() == null ? new BlobRequestConditions() : options.getRequestConditions(); - Context finalContext = context == null ? Context.NONE : context; BlobImmutabilityPolicy immutabilityPolicy = options.getImmutabilityPolicy() == null ? new BlobImmutabilityPolicy() : options.getImmutabilityPolicy(); - return dataMono.flatMap(data -> this.azureBlobStorage.getBlockBlobs() - .uploadWithResponseAsync(containerName, blobName, options.getLength(), data, null, options.getContentMd5(), - options.getMetadata(), requestConditions.getLeaseId(), options.getTier(), - requestConditions.getIfModifiedSince(), requestConditions.getIfUnmodifiedSince(), - requestConditions.getIfMatch(), requestConditions.getIfNoneMatch(), - requestConditions.getTagsConditions(), null, ModelHelper.tagsToString(options.getTags()), - immutabilityPolicy.getExpiryTime(), immutabilityPolicy.getPolicyMode(), options.isLegalHold(), null, - null, null, options.getHeaders(), getCustomerProvidedKey(), encryptionScope, finalContext) - .map(rb -> { - BlockBlobsUploadHeaders hd = rb.getDeserializedHeaders(); - BlockBlobItem item = new BlockBlobItem(hd.getETag(), hd.getLastModified(), hd.getContentMD5(), - hd.isXMsRequestServerEncrypted(), hd.getXMsEncryptionKeySha256(), hd.getXMsEncryptionScope(), - hd.getXMsVersionId()); - return new SimpleResponse<>(rb, item); - })); + context = ContentValidationModeResolver.addContentValidationMode(context, contentValidationAlgorithm, + options.getLength(), false); + + Context finalContext = context; + + return dataMono.flatMap(data -> { + Mono> responseMono = this.azureBlobStorage.getBlockBlobs() + .uploadWithResponseAsync(containerName, blobName, options.getLength(), data, null, + options.getContentMd5(), options.getMetadata(), requestConditions.getLeaseId(), options.getTier(), + requestConditions.getIfModifiedSince(), requestConditions.getIfUnmodifiedSince(), + requestConditions.getIfMatch(), requestConditions.getIfNoneMatch(), + requestConditions.getTagsConditions(), null, ModelHelper.tagsToString(options.getTags()), + immutabilityPolicy.getExpiryTime(), immutabilityPolicy.getPolicyMode(), options.isLegalHold(), null, + null, null, options.getHeaders(), getCustomerProvidedKey(), encryptionScope, finalContext) + .map(rb -> { + BlockBlobsUploadHeaders hd = rb.getDeserializedHeaders(); + BlockBlobItem item = new BlockBlobItem(hd.getETag(), hd.getLastModified(), hd.getContentMD5(), + hd.isXMsRequestServerEncrypted(), hd.getXMsEncryptionKeySha256(), hd.getXMsEncryptionScope(), + hd.getXMsVersionId()); + return new SimpleResponse<>(rb, item); + }); + return responseMono; + }); } /** @@ -707,12 +719,10 @@ public Mono stageBlock(String base64BlockId, BinaryData data) { @ServiceMethod(returns = ReturnType.SINGLE) public Mono> stageBlockWithResponse(String base64BlockId, Flux data, long length, byte[] contentMd5, String leaseId) { - try { - return withContext( - context -> stageBlockWithResponse(base64BlockId, data, length, contentMd5, leaseId, context)); - } catch (RuntimeException ex) { - return monoError(LOGGER, ex); - } + return BinaryData.fromFlux(data, length, false) + .flatMap(binaryData -> stageBlockWithResponse( + new BlockBlobStageBlockOptions(base64BlockId, binaryData).setContentMd5(contentMd5) + .setLeaseId(leaseId))); } /** @@ -745,27 +755,24 @@ public Mono> stageBlockWithResponse(String base64BlockId, Flux> stageBlockWithResponse(BlockBlobStageBlockOptions options) { Objects.requireNonNull(options, "options must not be null"); try { - return withContext(context -> stageBlockWithResponse(options.getBase64BlockId(), options.getData(), - options.getContentMd5(), options.getLeaseId(), context)); + return withContext(context -> stageBlockWithResponseInternal(options, context)); } catch (RuntimeException ex) { return monoError(LOGGER, ex); } } - Mono> stageBlockWithResponse(String base64BlockId, Flux data, long length, - byte[] contentMd5, String leaseId, Context context) { - return BinaryData.fromFlux(data, length, false) - .flatMap(binaryData -> stageBlockWithResponse(base64BlockId, binaryData, contentMd5, leaseId, context)); - } + Mono> stageBlockWithResponseInternal(BlockBlobStageBlockOptions options, Context context) { + Objects.requireNonNull(options.getData(), "data must not be null"); + Objects.requireNonNull(options.getData().getLength(), "data must have defined length"); + + context = ContentValidationModeResolver.addContentValidationMode(context, + options.getContentValidationAlgorithm(), options.getData().getLength(), false); - Mono> stageBlockWithResponse(String base64BlockId, BinaryData data, byte[] contentMd5, - String leaseId, Context context) { - Objects.requireNonNull(data, "data must not be null"); - Objects.requireNonNull(data.getLength(), "data must have defined length"); - context = context == null ? Context.NONE : context; return this.azureBlobStorage.getBlockBlobs() - .stageBlockNoCustomHeadersWithResponseAsync(containerName, blobName, base64BlockId, data.getLength(), data, - contentMd5, null, null, leaseId, null, null, null, getCustomerProvidedKey(), encryptionScope, context); + .stageBlockNoCustomHeadersWithResponseAsync(containerName, blobName, options.getBase64BlockId(), + options.getData().getLength(), options.getData(), options.getContentMd5(), null, null, + options.getLeaseId(), null, null, null, getCustomerProvidedKey(), encryptionScope, context); + } /** diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlockBlobClient.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlockBlobClient.java index 8667e7afe28c..a9a68ada5e71 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlockBlobClient.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlockBlobClient.java @@ -38,17 +38,14 @@ import com.azure.storage.blob.options.BlockBlobSimpleUploadOptions; import com.azure.storage.blob.options.BlockBlobStageBlockFromUrlOptions; import com.azure.storage.blob.options.BlockBlobStageBlockOptions; -import com.azure.storage.common.Utility; import com.azure.storage.common.implementation.Constants; import com.azure.storage.common.implementation.StorageImplUtils; import com.azure.storage.common.implementation.StorageSeekableByteChannel; -import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; import java.io.InputStream; import java.io.UncheckedIOException; import java.net.URL; -import java.nio.ByteBuffer; import java.nio.channels.SeekableByteChannel; import java.time.Duration; import java.util.List; @@ -314,9 +311,7 @@ public SeekableByteChannel openSeekableByteChannelWrite(BlockBlobSeekableByteCha options.getBlockSizeInBytes() != null ? options.getBlockSizeInBytes().intValue() : BlobAsyncClient.BLOB_DEFAULT_UPLOAD_BLOCK_SIZE, - new StorageSeekableByteChannelBlockBlobWriteBehavior(this, options.getHeaders(), options.getMetadata(), - options.getTags(), options.getTier(), options.getRequestConditions(), internalMode, null), - startingPosition); + new StorageSeekableByteChannelBlockBlobWriteBehavior(this, options, internalMode, null), startingPosition); } private BlobClientBuilder prepareBuilder() { @@ -787,11 +782,11 @@ public void stageBlock(String base64BlockId, BinaryData data) { public Response stageBlockWithResponse(String base64BlockId, InputStream data, long length, byte[] contentMd5, String leaseId, Duration timeout, Context context) { StorageImplUtils.assertNotNull("data", data); - Flux fbb - = Utility.convertStreamToByteBuffer(data, length, BlobAsyncClient.BLOB_DEFAULT_UPLOAD_BLOCK_SIZE, true); - Mono> response - = client.stageBlockWithResponse(base64BlockId, fbb, length, contentMd5, leaseId, context); + Mono> response = client.stageBlockWithResponseInternal( + new BlockBlobStageBlockOptions(base64BlockId, BinaryData.fromStream(data, length)).setContentMd5(contentMd5) + .setLeaseId(leaseId), + context); return blockWithOptionalTimeout(response, timeout); } @@ -827,8 +822,8 @@ public Response stageBlockWithResponse(String base64BlockId, InputStream d public Response stageBlockWithResponse(BlockBlobStageBlockOptions options, Duration timeout, Context context) { Objects.requireNonNull(options, "options must not be null"); - Mono> response = client.stageBlockWithResponse(options.getBase64BlockId(), options.getData(), - options.getContentMd5(), options.getLeaseId(), context); + + Mono> response = client.stageBlockWithResponseInternal(options, context); return blockWithOptionalTimeout(response, timeout); } diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/PageBlobAsyncClient.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/PageBlobAsyncClient.java index 5943047cf49b..d46ea4e9fb0b 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/PageBlobAsyncClient.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/PageBlobAsyncClient.java @@ -56,8 +56,12 @@ import com.azure.storage.blob.options.PageBlobCopyIncrementalOptions; import com.azure.storage.blob.options.PageBlobCreateOptions; import com.azure.storage.blob.options.PageBlobUploadPagesFromUrlOptions; +import com.azure.storage.blob.options.PageBlobUploadPagesOptions; +import com.azure.storage.common.ContentValidationAlgorithm; import com.azure.storage.common.implementation.Constants; +import com.azure.storage.common.implementation.contentvalidation.ContentValidationModeResolver; import com.azure.storage.common.implementation.StorageImplUtils; + import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; @@ -488,18 +492,36 @@ public Mono uploadPages(PageRange pageRange, Flux body * operation will fail. * @param pageBlobRequestConditions {@link PageBlobRequestConditions} * @return A reactive response containing the information of the uploaded pages. - * - * @throws IllegalArgumentException If {@code pageRange} is {@code null} */ @ServiceMethod(returns = ReturnType.SINGLE) public Mono> uploadPagesWithResponse(PageRange pageRange, Flux body, byte[] contentMd5, PageBlobRequestConditions pageBlobRequestConditions) { if (body == null) { - return Mono.error(new NullPointerException("'body' cannot be null.")); + return monoError(LOGGER, new NullPointerException("'body' cannot be null.")); + } + return uploadPagesWithResponse(new PageBlobUploadPagesOptions(pageRange, body).setContentMd5(contentMd5) + .setRequestConditions(pageBlobRequestConditions)); + } + + /** + * Writes one or more pages to the page blob with options. + * + * @param options {@link PageBlobUploadPagesOptions} (must be constructed with {@link Flux} body for async). + * @return A reactive response containing the information of the uploaded pages. + */ + @ServiceMethod(returns = ReturnType.SINGLE) + public Mono> uploadPagesWithResponse(PageBlobUploadPagesOptions options) { + if (options == null) { + return monoError(LOGGER, new NullPointerException("'options' cannot be null.")); + } + if (options.getDataFlux() == null) { + return monoError(LOGGER, new IllegalArgumentException( + "PageBlobUploadPagesOptions must be constructed with Flux for async client.")); } try { - return withContext( - context -> uploadPagesWithResponse(pageRange, body, contentMd5, pageBlobRequestConditions, context)); + return withContext(context -> uploadPagesWithResponseInternal(options.getPageRange(), options.getDataFlux(), + options.getContentMd5(), options.getRequestConditions(), options.getContentValidationAlgorithm(), + context)); } catch (RuntimeException ex) { return monoError(LOGGER, ex); } @@ -507,6 +529,13 @@ public Mono> uploadPagesWithResponse(PageRange pageRange, Mono> uploadPagesWithResponse(PageRange pageRange, Flux body, byte[] contentMd5, PageBlobRequestConditions pageBlobRequestConditions, Context context) { + // Prevents revapi visibility increased error + return uploadPagesWithResponseInternal(pageRange, body, contentMd5, pageBlobRequestConditions, null, context); + } + + Mono> uploadPagesWithResponseInternal(PageRange pageRange, Flux body, + byte[] contentMd5, PageBlobRequestConditions pageBlobRequestConditions, + ContentValidationAlgorithm contentValidationAlgorithm, Context context) { pageBlobRequestConditions = pageBlobRequestConditions == null ? new PageBlobRequestConditions() : pageBlobRequestConditions; @@ -515,12 +544,15 @@ Mono> uploadPagesWithResponse(PageRange pageRange, Flux uploadPagesWithResponse(PageRange pageRange, InputStream body, byte[] contentMd5, PageBlobRequestConditions pageBlobRequestConditions, Duration timeout, Context context) { - Objects.requireNonNull(body, "'body' cannot be null."); - final long length = pageRange.getEnd() - pageRange.getStart() + 1; - Flux fbb = Utility.convertStreamToByteBuffer(body, length, PAGE_BYTES, true); + return uploadPagesWithResponse(new PageBlobUploadPagesOptions(pageRange, body).setContentMd5(contentMd5) + .setRequestConditions(pageBlobRequestConditions), timeout, context); + } - Mono> response = pageBlobAsyncClient.uploadPagesWithResponse(pageRange, fbb, contentMd5, - pageBlobRequestConditions, context); + /** + * Writes one or more pages to the page blob with options. + * + * @param options {@link PageBlobUploadPagesOptions} + * @param timeout An optional timeout value. + * @param context Additional context. + * @return The information of the uploaded pages. + * @throws NullPointerException If {@code options} is {@code null}. + * @throws IllegalArgumentException if options is not constructed with InputStream. + * @throws UnexpectedLengthException If the length of the data read from the provided stream does not match the + * expected length based on the specified page range. + */ + @ServiceMethod(returns = ReturnType.SINGLE) + public Response uploadPagesWithResponse(PageBlobUploadPagesOptions options, Duration timeout, + Context context) { + StorageImplUtils.assertNotNull("options", options); + if (options.getDataStream() == null) { + throw LOGGER.logExceptionAsError(new IllegalArgumentException( + "PageBlobUploadPagesOptions must be constructed with InputStream for sync client.")); + } + final long length = options.getPageRange().getEnd() - options.getPageRange().getStart() + 1; + Flux fbb = Utility.convertStreamToByteBuffer(options.getDataStream(), length, PAGE_BYTES, true); + Mono> response + = pageBlobAsyncClient.uploadPagesWithResponseInternal(options.getPageRange(), fbb, options.getContentMd5(), + options.getRequestConditions(), options.getContentValidationAlgorithm(), context); return StorageImplUtils.blockWithOptionalTimeout(response, timeout); } diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/StorageSeekableByteChannelBlockBlobWriteBehavior.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/StorageSeekableByteChannelBlockBlobWriteBehavior.java index 71eb080a48cc..d98a78072519 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/StorageSeekableByteChannelBlockBlobWriteBehavior.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/StorageSeekableByteChannelBlockBlobWriteBehavior.java @@ -10,7 +10,9 @@ import com.azure.storage.blob.models.BlobHttpHeaders; import com.azure.storage.blob.models.BlobRequestConditions; import com.azure.storage.blob.options.BlockBlobCommitBlockListOptions; +import com.azure.storage.blob.options.BlockBlobSeekableByteChannelWriteOptions; import com.azure.storage.blob.options.BlockBlobStageBlockOptions; +import com.azure.storage.common.ContentValidationAlgorithm; import com.azure.storage.common.implementation.StorageSeekableByteChannel; import java.io.IOException; @@ -43,6 +45,7 @@ enum WriteMode { private final WriteMode mode; private final List existingBlockIds; private final List newBlockIds = new ArrayList<>(); + private final ContentValidationAlgorithm contentValidationAlgorithm; StorageSeekableByteChannelBlockBlobWriteBehavior(BlockBlobClient client, BlobHttpHeaders headers, Map metadata, Map tags, AccessTier tier, BlobRequestConditions conditions, @@ -55,6 +58,21 @@ enum WriteMode { this.conditions = conditions; this.mode = Objects.requireNonNull(mode); this.existingBlockIds = existingBlockIds != null ? existingBlockIds : Collections.emptyList(); + this.contentValidationAlgorithm = null; + } + + StorageSeekableByteChannelBlockBlobWriteBehavior(BlockBlobClient client, + BlockBlobSeekableByteChannelWriteOptions options, WriteMode mode, List existingBlockIds) { + this.client = Objects.requireNonNull(client); + Objects.requireNonNull(options); + this.headers = options.getHeaders(); + this.metadata = options.getMetadata(); + this.tags = options.getTags(); + this.tier = options.getTier(); + this.conditions = options.getRequestConditions(); + this.mode = Objects.requireNonNull(mode); + this.existingBlockIds = existingBlockIds != null ? existingBlockIds : Collections.emptyList(); + this.contentValidationAlgorithm = options.getContentValidationAlgorithm(); } BlockBlobClient getClient() { @@ -100,6 +118,9 @@ public void write(ByteBuffer src, long destOffset) throws IOException { if (conditions != null) { options.setLeaseId(conditions.getLeaseId()); } + if (contentValidationAlgorithm != null) { + options.setContentValidationAlgorithm(contentValidationAlgorithm); + } client.stageBlockWithResponse(options, null, null); newBlockIds.add(blockId); diff --git a/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobContentValidationAsyncDownloadTests.java b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobContentValidationAsyncDownloadTests.java new file mode 100644 index 000000000000..5691f78d6c72 --- /dev/null +++ b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobContentValidationAsyncDownloadTests.java @@ -0,0 +1,681 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.blob; + +import com.azure.core.http.HttpHeaders; +import com.azure.core.http.policy.HttpPipelinePolicy; +import com.azure.core.test.utils.TestUtils; +import com.azure.core.util.BinaryData; +import com.azure.core.util.FluxUtil; +import com.azure.core.util.ProgressListener; +import com.azure.storage.blob.models.BlobRange; +import com.azure.storage.blob.models.DownloadRetryOptions; +import com.azure.storage.blob.options.BlobDownloadContentOptions; +import com.azure.storage.blob.options.BlobDownloadStreamOptions; +import com.azure.storage.blob.options.BlobDownloadToFileOptions; +import com.azure.storage.blob.options.BlobUploadFromFileOptions; +import com.azure.storage.common.ParallelTransferOptions; +import com.azure.storage.common.ContentValidationAlgorithm; +import com.azure.storage.common.implementation.Constants; +import com.azure.storage.common.implementation.contentvalidation.StorageCrc64Calculator; +import com.azure.storage.common.test.shared.extensions.LiveOnly; +import com.azure.storage.common.test.shared.policy.MockPartialResponsePolicy; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; +import org.junit.jupiter.params.provider.MethodSource; +import org.junit.jupiter.params.provider.ValueSource; +import reactor.core.publisher.Flux; +import reactor.test.StepVerifier; +import reactor.util.function.Tuples; + +import java.io.File; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.ByteBuffer; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.atomic.AtomicLong; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Async tests for structured message decoding during blob downloads using StorageContentValidationDecoderPolicy. + * These tests verify that the pipeline policy correctly decodes structured messages when content validation is enabled. + */ +public class BlobContentValidationAsyncDownloadTests extends BlobTestBase { + private static final int TEN_MB = 10 * Constants.MB; + private static final int BLOCK_SIZE = 4 * Constants.MB; + /** + * {@link BlobTestBase#fuzzyParallelDownloadLargeMultiPartCases()} starts at ~96 MiB; above this threshold fuzzy + * parallel download helpers use temp files + {@link BlobTestBase#compareFiles(File, File, long, long)} so the full + * payload never lives twice in heap. + */ + private static final int FUZZY_PARALLEL_DOWNLOAD_FILE_ROUND_TRIP_THRESHOLD_BYTES = 96 * Constants.MB; + + /** + * Live-only random payload band for the dedicated random-size parallel-download fuzzy test + * ({@link #fuzzyParallelDownloadLiveRandomRoundTrip(ContentValidationAlgorithm)}): each run draws a per-run + * payload size in {@code (256 MiB, 500 MiB]} (matches the encoder fuzzy upload range) so the structured-message + * decoder is exercised against payloads whose size varies per run in addition to the random byte contents. + */ + private static final long LIVE_RANDOM_PARALLEL_DOWNLOAD_PAYLOAD_MIN_BYTES_EXCLUSIVE = 256L * Constants.MB; + private static final long LIVE_RANDOM_PARALLEL_DOWNLOAD_PAYLOAD_MAX_BYTES_INCLUSIVE = 500L * Constants.MB; + + private final List createdFiles = new ArrayList<>(); + + private File createRandomFile(Path tempDir, int size) throws IOException { + File file = Files.createTempFile(tempDir, "blob-cv-source", ".bin").toFile(); + + if (size > Constants.MB) { + try (OutputStream outputStream = Files.newOutputStream(file.toPath())) { + byte[] data = getRandomByteArray(Constants.MB); + int mbChunks = size / Constants.MB; + int remaining = size % Constants.MB; + for (int i = 0; i < mbChunks; i++) { + outputStream.write(data); + } + if (remaining > 0) { + outputStream.write(data, 0, remaining); + } + } + } else { + Files.write(file.toPath(), getRandomByteArray(size)); + } + + return file; + } + + /** + * downloadStreamWithResponse with CRC64 content validation. + */ + @Test + public void downloadStreamWithResponseContentValidation() { + byte[] data = getRandomByteArray(TEN_MB); + List recordedRequestHeaders = new CopyOnWriteArrayList<>(); + BlobAsyncClient blobClient = createBlobAsyncClientWithRequestSniffer(recordedRequestHeaders); + blobClient.upload(BinaryData.fromBytes(data)).block(); + + BlobDownloadStreamOptions options + = new BlobDownloadStreamOptions().setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + + StepVerifier.create(blobClient.downloadStreamWithResponse(options).flatMap(r -> { + assertTrue(hasStructuredMessageDownloadResponseHeaders(r.getHeaders())); + return FluxUtil.collectBytesInByteBufferStream(r.getValue()); + })).assertNext(result -> TestUtils.assertArraysEqual(data, result)).verifyComplete(); + assertTrue(hasStructuredMessageDownloadRequestHeaders(recordedRequestHeaders, false)); + } + + /** + * downloadContentWithResponse with CRC64 content validation. + */ + @Test + public void downloadContentWithResponseContentValidation() { + byte[] data = getRandomByteArray(TEN_MB); + List recordedRequestHeaders = new CopyOnWriteArrayList<>(); + BlobAsyncClient blobClient = createBlobAsyncClientWithRequestSniffer(recordedRequestHeaders); + blobClient.upload(BinaryData.fromBytes(data)).block(); + + BlobDownloadContentOptions options + = new BlobDownloadContentOptions().setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + + StepVerifier.create(blobClient.downloadContentWithResponse(options)).assertNext(r -> { + assertTrue(hasStructuredMessageDownloadResponseHeaders(r.getHeaders())); + TestUtils.assertArraysEqual(data, r.getValue().toBytes()); + }).verifyComplete(); + assertTrue(hasStructuredMessageDownloadRequestHeaders(recordedRequestHeaders, false)); + } + + /** + * downloadToFileWithResponse with CRC64 content validation. + */ + @ParameterizedTest + @ValueSource( + ints = { + 0, // empty file + 20, // small file + 16 * 1024 * 1024, // medium file in several chunks + 8 * 1026 * 1024 + 10, // medium file not aligned to block + }) + public void downloadToFileWithResponseContentValidation(int fileSize, @TempDir Path tempDir) throws IOException { + File file = createRandomFile(tempDir, fileSize); + File outFile = tempDir.resolve("download.bin").toFile(); + List recordedRequestHeaders = new CopyOnWriteArrayList<>(); + BlobAsyncClient blobClient = createBlobAsyncClientWithRequestSniffer(recordedRequestHeaders); + blobClient.uploadFromFile(file.toPath().toString(), true).block(); + + Files.deleteIfExists(outFile.toPath()); + + ParallelTransferOptions parallelOptions = new ParallelTransferOptions().setBlockSizeLong((long) BLOCK_SIZE); + BlobDownloadToFileOptions options + = new BlobDownloadToFileOptions(outFile.toPath().toString()).setParallelTransferOptions(parallelOptions) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + + StepVerifier.create(blobClient.downloadToFileWithResponse(options)).assertNext(r -> { + assertTrue(hasStructuredMessageDownloadResponseHeaders(r.getHeaders())); + assertNotNull(r.getValue()); + }).verifyComplete(); + + assertTrue(compareFiles(file, outFile, 0, fileSize)); + assertTrue(hasStructuredMessageDownloadRequestHeaders(recordedRequestHeaders, false)); + } + + /** + * downloadToFileWithResponse with CRC64 content validation (parallel, multiple block sizes). + */ + @LiveOnly + @ParameterizedTest + @ValueSource( + ints = { + 50 * Constants.MB, //large file requiring multiple requests + 50 * Constants.MB + 22 // large file not on MB boundary + }) + public void downloadToFileLargeWithResponseContentValidation(int fileSize, @TempDir Path tempDir) + throws IOException { + File file = createRandomFile(tempDir, fileSize); + File outFile = tempDir.resolve("download.bin").toFile(); + List recordedRequestHeaders = new CopyOnWriteArrayList<>(); + BlobAsyncClient blobClient = createBlobAsyncClientWithRequestSniffer(recordedRequestHeaders); + blobClient.uploadFromFile(file.toPath().toString(), true).block(); + + Files.deleteIfExists(outFile.toPath()); + + ParallelTransferOptions parallelOptions = new ParallelTransferOptions().setBlockSizeLong((long) BLOCK_SIZE); + BlobDownloadToFileOptions options + = new BlobDownloadToFileOptions(outFile.toPath().toString()).setParallelTransferOptions(parallelOptions) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + + StepVerifier.create(blobClient.downloadToFileWithResponse(options)).assertNext(r -> { + assertTrue(hasStructuredMessageDownloadResponseHeaders(r.getHeaders())); + assertNotNull(r.getValue()); + }).verifyComplete(); + + assertTrue(compareFiles(file, outFile, 0, fileSize)); + assertTrue(hasStructuredMessageDownloadRequestHeaders(recordedRequestHeaders, false)); + } + + /** + * Default behavior: when no algorithm is specified, default is NONE (no validation). + */ + @Test + public void downloadStreamDefaultAlgorithmIsNone() { + byte[] data = getRandomByteArray(TEN_MB); + List recordedRequestHeaders = new CopyOnWriteArrayList<>(); + BlobAsyncClient blobClient = createBlobAsyncClientWithRequestSniffer(recordedRequestHeaders); + blobClient.upload(Flux.just(ByteBuffer.wrap(data)), null, true).block(); + + StepVerifier.create(blobClient.downloadStreamWithResponse(new BlobDownloadStreamOptions()) + .flatMap(r -> FluxUtil.collectBytesInByteBufferStream(r.getValue()))).assertNext(result -> { + assertNotNull(result); + assertEquals(data.length, result.length); + }).verifyComplete(); + assertFalse(hasStructuredMessageDownloadRequestHeaders(recordedRequestHeaders, false)); + } + + @Test + public void downloadStreamWithResponseContentValidationRange() { + byte[] data = getRandomByteArray(4 * Constants.KB); + List recordedRequestHeaders = new CopyOnWriteArrayList<>(); + BlobAsyncClient blobClient = createBlobAsyncClientWithRequestSniffer(recordedRequestHeaders); + blobClient.upload(BinaryData.fromBytes(data)).block(); + + BlobDownloadStreamOptions options = new BlobDownloadStreamOptions().setRange(new BlobRange(0, 512L)); + + StepVerifier.create(blobClient.downloadStreamWithResponse(options).flatMap(r -> { + assertFalse(hasStructuredMessageDownloadResponseHeaders(r.getHeaders())); + return FluxUtil.collectBytesInByteBufferStream(r.getValue()); + })).assertNext(result -> assertEquals(512, result.length)).verifyComplete(); + + assertFalse(hasStructuredMessageDownloadRequestHeaders(recordedRequestHeaders, false)); + } + + /** + * AUTO on downloadStream resolves to CRC64 behavior. + */ + @Test + public void downloadStreamWithAuto() { + byte[] data = getRandomByteArray(TEN_MB); + List recordedRequestHeaders = new CopyOnWriteArrayList<>(); + BlobAsyncClient blobClient = createBlobAsyncClientWithRequestSniffer(recordedRequestHeaders); + blobClient.upload(BinaryData.fromBytes(data)).block(); + + StepVerifier.create(blobClient + .downloadStreamWithResponse( + new BlobDownloadStreamOptions().setContentValidationAlgorithm(ContentValidationAlgorithm.AUTO)) + .flatMap(r -> { + assertTrue(hasStructuredMessageDownloadResponseHeaders(r.getHeaders())); + return FluxUtil.collectBytesInByteBufferStream(r.getValue()); + })).assertNext(result -> TestUtils.assertArraysEqual(data, result)).verifyComplete(); + assertTrue(hasStructuredMessageDownloadRequestHeaders(recordedRequestHeaders, false)); + } + + /** + * downloadContentWithResponse with NONE: no validation triggered. + */ + @Test + public void downloadContentWithNone() { + byte[] data = getRandomByteArray(TEN_MB); + List recordedRequestHeaders = new CopyOnWriteArrayList<>(); + BlobAsyncClient blobClient = createBlobAsyncClientWithRequestSniffer(recordedRequestHeaders); + blobClient.upload(Flux.just(ByteBuffer.wrap(data)), null, true).block(); + + StepVerifier + .create(blobClient.downloadContentWithResponse( + new BlobDownloadContentOptions().setContentValidationAlgorithm(ContentValidationAlgorithm.NONE))) + .assertNext(r -> TestUtils.assertArraysEqual(data, r.getValue().toBytes())) + .verifyComplete(); + assertFalse(hasStructuredMessageDownloadRequestHeaders(recordedRequestHeaders, false)); + } + + /** + * downloadContentWithResponse with AUTO resolves to CRC64 behavior. + */ + @Test + public void downloadContentWithAuto() { + byte[] data = getRandomByteArray(TEN_MB); + List recordedRequestHeaders = new CopyOnWriteArrayList<>(); + BlobAsyncClient blobClient = createBlobAsyncClientWithRequestSniffer(recordedRequestHeaders); + blobClient.upload(BinaryData.fromBytes(data)).block(); + + StepVerifier + .create(blobClient.downloadContentWithResponse( + new BlobDownloadContentOptions().setContentValidationAlgorithm(ContentValidationAlgorithm.AUTO))) + .assertNext(r -> { + assertTrue(hasStructuredMessageDownloadResponseHeaders(r.getHeaders())); + TestUtils.assertArraysEqual(data, r.getValue().toBytes()); + }) + .verifyComplete(); + assertTrue(hasStructuredMessageDownloadRequestHeaders(recordedRequestHeaders, false)); + } + + /** + * Interrupt with proper rewind to segment boundary; verifies retry range headers. + */ + @Test + public void interruptAndVerifyProperRewind() { + final int segmentSize = Constants.KB; + byte[] data = getRandomByteArray(2 * segmentSize); + List recordedRequestHeaders = new CopyOnWriteArrayList<>(); + HttpHeaders recordedResponseHeaders = new HttpHeaders(); + BlobAsyncClient blobClient = createBlobAsyncClientWithRequestSniffer(recordedRequestHeaders); + + int interruptPos = segmentSize + (2 * (segmentSize / 4)) + 10; + MockPartialResponsePolicy mockPolicy = new MockPartialResponsePolicy(1, interruptPos, blobClient.getBlobUrl()); + HttpPipelinePolicy sniffPolicy = getRequestAndResponseHeaderSniffer(blobClient.getBlobUrl(), + recordedRequestHeaders, recordedResponseHeaders); + + blobClient.upload(Flux.just(ByteBuffer.wrap(data)), null, true).block(); + + BlobAsyncClient downloadClient = getBlobAsyncClient(ENVIRONMENT.getPrimaryAccount().getCredential(), + blobClient.getBlobUrl(), sniffPolicy, mockPolicy); + + DownloadRetryOptions retryOptions = new DownloadRetryOptions().setMaxRetryRequests(5); + + StepVerifier.create(downloadClient + .downloadStreamWithResponse(new BlobDownloadStreamOptions().setDownloadRetryOptions(retryOptions) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64)) + .doFinally(signalType -> assertTrue(mockPolicy.getHits() > 0, "Mock interruption policy was not invoked")) + .flatMap(r -> { + assertTrue(hasStructuredMessageDownloadResponseHeaders(r.getHeaders())); + return FluxUtil.collectBytesInByteBufferStream(r.getValue()); + })).assertNext(result -> TestUtils.assertArraysEqual(data, result)).verifyComplete(); + + assertEquals(0, mockPolicy.getTriesRemaining(), "Expected the configured interruption to be consumed"); + assertTrue(mockPolicy.getRangeHeaders().size() >= 2, + "Expected at least the initial request and one retry with a range header"); + assertTrue(hasStructuredMessageDownloadResponseHeaders(recordedResponseHeaders)); + assertTrue(hasStructuredMessageDownloadRequestHeaders(recordedRequestHeaders, false)); + } + + /** + * Proper decode across retries (single and multiple interrupts). + */ + @ParameterizedTest + @ValueSource(booleans = { false, true }) + public void interruptAndVerifyProperDecode(boolean multipleInterrupts) { + final int segmentSize = 128 * Constants.KB; + final int dataSize = 4 * Constants.KB; + byte[] data = getRandomByteArray(dataSize); + List recordedRequestHeaders = new CopyOnWriteArrayList<>(); + HttpHeaders recordedResponseHeaders = new HttpHeaders(); + BlobAsyncClient blobClient = createBlobAsyncClientWithRequestSniffer(recordedRequestHeaders); + + int interruptPos = segmentSize + (3 * (8 * Constants.KB)) + 10; + MockPartialResponsePolicy mockPolicy + = new MockPartialResponsePolicy(multipleInterrupts ? 2 : 1, interruptPos, blobClient.getBlobUrl()); + HttpPipelinePolicy sniffPolicy = getRequestAndResponseHeaderSniffer(blobClient.getBlobUrl(), + recordedRequestHeaders, recordedResponseHeaders); + + blobClient.upload(Flux.just(ByteBuffer.wrap(data)), null, true).block(); + + BlobAsyncClient downloadClient = getBlobAsyncClient(ENVIRONMENT.getPrimaryAccount().getCredential(), + blobClient.getBlobUrl(), sniffPolicy, mockPolicy); + + DownloadRetryOptions retryOptions = new DownloadRetryOptions().setMaxRetryRequests(10); + + StepVerifier.create(downloadClient + .downloadStreamWithResponse(new BlobDownloadStreamOptions().setDownloadRetryOptions(retryOptions) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64)) + .flatMap(r -> { + assertTrue(hasStructuredMessageDownloadResponseHeaders(r.getHeaders())); + return FluxUtil.collectBytesInByteBufferStream(r.getValue()); + })).assertNext(result -> { + assertEquals(dataSize, result.length, "Decoded data should have exactly " + dataSize + " bytes"); + TestUtils.assertArraysEqual(data, result); + }).verifyComplete(); + assertTrue(hasStructuredMessageDownloadResponseHeaders(recordedResponseHeaders)); + assertTrue(hasStructuredMessageDownloadRequestHeaders(recordedRequestHeaders, false)); + } + + /** + * After consuming the response stream with CRC64 validation, decoded payload preserves the expected CRC64. + */ + @Test + public void structuredMessageVerifiesDecodedCrc64DownloadStreaming() { + byte[] data = getRandomByteArray(TEN_MB); + List recordedRequestHeaders = new CopyOnWriteArrayList<>(); + BlobAsyncClient blobClient = createBlobAsyncClientWithRequestSniffer(recordedRequestHeaders); + blobClient.upload(BinaryData.fromBytes(data)).block(); + + long expectedCrc = StorageCrc64Calculator.compute(data, 0); + + StepVerifier + .create(blobClient + .downloadStreamWithResponse( + new BlobDownloadStreamOptions().setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64)) + .flatMap(r -> FluxUtil.collectBytesInByteBufferStream(r.getValue()).map(bytes -> Tuples.of(r, bytes)))) + .assertNext(tuple -> { + TestUtils.assertArraysEqual(data, tuple.getT2()); + long actualCrc = StorageCrc64Calculator.compute(tuple.getT2(), 0); + assertEquals(expectedCrc, actualCrc); + }) + .verifyComplete(); + assertTrue(hasStructuredMessageDownloadRequestHeaders(recordedRequestHeaders, false)); + } + + /** + * Single interrupt with data intact: fault policy + decoder; structured message retry recovers. + */ + @Test + public void interruptWithDataIntact() { + final int segmentSize = Constants.KB; + byte[] data = getRandomByteArray(4 * segmentSize); + List recordedRequestHeaders = new CopyOnWriteArrayList<>(); + HttpHeaders recordedResponseHeaders = new HttpHeaders(); + BlobAsyncClient blobClient = createBlobAsyncClientWithRequestSniffer(recordedRequestHeaders); + + int interruptPos = segmentSize + (3 * 128) + 10; + MockPartialResponsePolicy mockPolicy = new MockPartialResponsePolicy(1, interruptPos, blobClient.getBlobUrl()); + HttpPipelinePolicy sniffPolicy = getRequestAndResponseHeaderSniffer(blobClient.getBlobUrl(), + recordedRequestHeaders, recordedResponseHeaders); + + blobClient.upload(Flux.just(ByteBuffer.wrap(data)), null, true).block(); + + BlobAsyncClient downloadClient = getBlobAsyncClient(ENVIRONMENT.getPrimaryAccount().getCredential(), + blobClient.getBlobUrl(), sniffPolicy, mockPolicy); + + DownloadRetryOptions retryOptions = new DownloadRetryOptions().setMaxRetryRequests(5); + + StepVerifier + .create(downloadClient + .downloadStreamWithResponse(new BlobDownloadStreamOptions().setDownloadRetryOptions(retryOptions) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64)) + .flatMap(r -> FluxUtil.collectBytesInByteBufferStream(r.getValue()))) + .assertNext(result -> TestUtils.assertArraysEqual(data, result)) + .verifyComplete(); + assertTrue(hasStructuredMessageDownloadRequestHeaders(recordedRequestHeaders, false)); + assertTrue(hasStructuredMessageDownloadResponseHeaders(recordedResponseHeaders)); + } + + /** + * Multiple interrupts with data intact: fault policy + decoder; structured message retry recovers. + */ + @Test + public void interruptMultipleTimesWithDataIntact() { + final int segmentSize = Constants.KB; + byte[] data = getRandomByteArray(4 * segmentSize); + List recordedRequestHeaders = new CopyOnWriteArrayList<>(); + HttpHeaders recordedResponseHeaders = new HttpHeaders(); + BlobAsyncClient blobClient = createBlobAsyncClientWithRequestSniffer(recordedRequestHeaders); + + int interruptPos = segmentSize + (3 * 128) + 10; + MockPartialResponsePolicy mockPolicy = new MockPartialResponsePolicy(3, interruptPos, blobClient.getBlobUrl()); + HttpPipelinePolicy sniffPolicy = getRequestAndResponseHeaderSniffer(blobClient.getBlobUrl(), + recordedRequestHeaders, recordedResponseHeaders); + + blobClient.upload(Flux.just(ByteBuffer.wrap(data)), null, true).block(); + + BlobAsyncClient downloadClient = getBlobAsyncClient(ENVIRONMENT.getPrimaryAccount().getCredential(), + blobClient.getBlobUrl(), sniffPolicy, mockPolicy); + + DownloadRetryOptions retryOptions = new DownloadRetryOptions().setMaxRetryRequests(10); + + StepVerifier + .create(downloadClient + .downloadStreamWithResponse(new BlobDownloadStreamOptions().setDownloadRetryOptions(retryOptions) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64)) + .flatMap(r -> FluxUtil.collectBytesInByteBufferStream(r.getValue()))) + .assertNext(result -> TestUtils.assertArraysEqual(data, result)) + .verifyComplete(); + assertTrue(hasStructuredMessageDownloadRequestHeaders(recordedRequestHeaders, false)); + assertTrue(hasStructuredMessageDownloadResponseHeaders(recordedResponseHeaders)); + } + + @Test + public void verifyProgressListenerIsCompatibleWithContentValidation(@TempDir Path tempDir) throws IOException { + byte[] data = getRandomByteArray(10 * Constants.MB); + + BlobAsyncClient client = ccAsync.getBlobAsyncClient(generateBlobName()); + + MockProgressListener mockListenerWithContentVal = new MockProgressListener(); + MockProgressListener mockListenerWithoutContentVal = new MockProgressListener(); + + ParallelTransferOptions parallelOptionsWithContentVal + = new ParallelTransferOptions().setProgressListener(mockListenerWithContentVal); + ParallelTransferOptions parallelOptionsWithoutContentVal + = new ParallelTransferOptions().setProgressListener(mockListenerWithoutContentVal); + + File fileWithContentVal = createRandomFile(tempDir, 10 * Constants.MB); + File outFileWithContentVal = tempDir.resolve("withcontentval.bin").toFile(); + File fileWithoutContentVal = createRandomFile(tempDir, 10 * Constants.MB); + File outFileWithoutContentVal = tempDir.resolve("withoutcontentval.bin").toFile(); + + Files.deleteIfExists(outFileWithContentVal.toPath()); + Files.deleteIfExists(outFileWithoutContentVal.toPath()); + + BlobDownloadToFileOptions optionsWithContentVal + = new BlobDownloadToFileOptions(outFileWithContentVal.getAbsolutePath()) + .setParallelTransferOptions(parallelOptionsWithContentVal) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + BlobDownloadToFileOptions optionsWithoutContentVal + = new BlobDownloadToFileOptions(outFileWithoutContentVal.getAbsolutePath()) + .setParallelTransferOptions(parallelOptionsWithoutContentVal); + + StepVerifier.create(client.upload(BinaryData.fromBytes(data)) + .then(client.downloadToFileWithResponse(optionsWithContentVal)) + .then(client.downloadToFileWithResponse(optionsWithoutContentVal))).assertNext(ignored -> { + long expectedBytes = data.length; + assertEquals(expectedBytes, mockListenerWithContentVal.getReportedByteCount()); + assertEquals(expectedBytes, mockListenerWithoutContentVal.getReportedByteCount()); + }).verifyComplete(); + } + + private static final class MockProgressListener implements ProgressListener { + private final AtomicLong reportedByteCount = new AtomicLong(0L); + + @Override + public void handleProgress(long bytesTransferred) { + this.reportedByteCount.updateAndGet(current -> Math.max(current, bytesTransferred)); + } + + long getReportedByteCount() { + return this.reportedByteCount.get(); + } + } + + // ---------- Fuzzy parallel download (deterministic grids) ---------- + + @ParameterizedTest + @MethodSource("com.azure.storage.blob.BlobTestBase#fuzzyParallelDownloadReplayableCases") + public void fuzzyParallelDownloadReplayableRoundTrip(int payloadBytes, long blockSizeBytes, int maxConcurrency) + throws IOException { + assertParallelDownloadFuzzyRoundTripAsync("replayable", payloadBytes, blockSizeBytes, maxConcurrency); + } + + @LiveOnly // payload > blockSize with tiny totals; many small range GETs not replayable under the proxy. + @ParameterizedTest + @MethodSource("com.azure.storage.blob.BlobTestBase#fuzzyParallelDownloadSmallMultiPartCases") + public void fuzzyParallelDownloadSmallMultiPartRoundTrip(int payloadBytes, long blockSizeBytes, int maxConcurrency) + throws IOException { + assertParallelDownloadFuzzyRoundTripAsync("smallMultiPart", payloadBytes, blockSizeBytes, maxConcurrency); + } + + @LiveOnly // sub-4 MiB chunked range GETs not replayable under the proxy. + @ParameterizedTest + @MethodSource("com.azure.storage.blob.BlobTestBase#fuzzyParallelDownloadSubFourMiBCases") + public void fuzzyParallelDownloadSubFourMiBRoundTrip(int payloadBytes, long blockSizeBytes, int maxConcurrency) + throws IOException { + assertParallelDownloadFuzzyRoundTripAsync("subFourMiB", payloadBytes, blockSizeBytes, maxConcurrency); + } + + @LiveOnly // 4 MiB boundary tuples that fan out into chunked range GETs. + @ParameterizedTest + @MethodSource("com.azure.storage.blob.BlobTestBase#fuzzyParallelDownloadFourMiBBoundaryCases") + public void fuzzyParallelDownloadFourMiBBoundaryRoundTrip(int payloadBytes, long blockSizeBytes, int maxConcurrency) + throws IOException { + assertParallelDownloadFuzzyRoundTripAsync("fourMiBBoundary", payloadBytes, blockSizeBytes, maxConcurrency); + } + + @LiveOnly // payload > blockSize for every tuple; chunked range GETs across many requests. + @ParameterizedTest + @MethodSource("com.azure.storage.blob.BlobTestBase#fuzzyParallelDownloadMediumMultiPartCases") + public void fuzzyParallelDownloadMediumMultiPartRoundTrip(int payloadBytes, long blockSizeBytes, int maxConcurrency) + throws IOException { + assertParallelDownloadFuzzyRoundTripAsync("mediumMultiPart", payloadBytes, blockSizeBytes, maxConcurrency); + } + + @LiveOnly // payload >> blockSize; ~96-320 MiB downloads. + @ParameterizedTest + @MethodSource("com.azure.storage.blob.BlobTestBase#fuzzyParallelDownloadLargeMultiPartCases") + public void fuzzyParallelDownloadLargeMultiPartRoundTrip(int payloadBytes, long blockSizeBytes, int maxConcurrency) + throws IOException { + assertParallelDownloadFuzzyRoundTripAsync("largeMultiPart", payloadBytes, blockSizeBytes, maxConcurrency); + } + + @LiveOnly // ~1 GiB single case; far too large for the test proxy. + @ParameterizedTest + @MethodSource("com.azure.storage.blob.BlobTestBase#fuzzyParallelDownloadOneGiBCases") + public void fuzzyParallelDownloadOneGiBRoundTrip(int payloadBytes, long blockSizeBytes, int maxConcurrency) + throws IOException { + assertParallelDownloadFuzzyRoundTripAsync("oneGiB", payloadBytes, blockSizeBytes, maxConcurrency); + } + + /** + * Live-only random-size parallel download fuzzy round-trip. Each run draws a per-run payload size in + * {@code (256 MiB, 500 MiB]} (matches the encoder fuzzy upload range) and exercises both CRC64 and AUTO + * content-validation algorithms so the structured-message decoder is tested against payloads whose total size + * varies per run in addition to the random byte contents that the deterministic grids already exercise. Kept + * separate from the parameterized {@link #fuzzyParallelDownloadLargeMultiPartRoundTrip(int, long, int)} so the + * deterministic per-grid round-trips and the randomized round-trip don't share work or cost. + */ + @LiveOnly + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void fuzzyParallelDownloadLiveRandomRoundTrip(ContentValidationAlgorithm algorithm) throws IOException { + int sizeBytes = (int) randomLongFromNamer(LIVE_RANDOM_PARALLEL_DOWNLOAD_PAYLOAD_MIN_BYTES_EXCLUSIVE + 1, + LIVE_RANDOM_PARALLEL_DOWNLOAD_PAYLOAD_MAX_BYTES_INCLUSIVE + 1); + assertParallelDownloadFuzzyRoundTripAsync("liveRandom", sizeBytes, 8L * Constants.MB, 8, algorithm); + } + + private void assertParallelDownloadFuzzyRoundTripAsync(String caseKind, int payloadBytes, long blockSizeBytes, + int maxConcurrency) throws IOException { + assertParallelDownloadFuzzyRoundTripAsync(caseKind, payloadBytes, blockSizeBytes, maxConcurrency, + ContentValidationAlgorithm.CRC64); + } + + private void assertParallelDownloadFuzzyRoundTripAsync(String caseKind, int payloadBytes, long blockSizeBytes, + int maxConcurrency, ContentValidationAlgorithm algorithm) throws IOException { + List recorded = new CopyOnWriteArrayList<>(); + BlobAsyncClient client = createBlobAsyncClientWithRequestSniffer(recorded); + + ParallelTransferOptions parallelOptions + = new ParallelTransferOptions().setBlockSizeLong(blockSizeBytes).setMaxConcurrency(maxConcurrency); + + String assertionMessage = "Fuzzy parallel download [" + caseKind + "] payloadBytes=" + payloadBytes + + ", blockSize=" + blockSizeBytes + ", maxConcurrency=" + maxConcurrency + ", algorithm=" + algorithm; + + if (payloadBytes >= FUZZY_PARALLEL_DOWNLOAD_FILE_ROUND_TRIP_THRESHOLD_BYTES) { + File sourceFile = getRandomFile(payloadBytes); + sourceFile.deleteOnExit(); + createdFiles.add(sourceFile); + File outFile = Files.createTempFile("blob-cv-fuzzy-parallel-dl-async", ".bin").toFile(); + outFile.deleteOnExit(); + createdFiles.add(outFile); + Files.deleteIfExists(outFile.toPath()); + + BlobUploadFromFileOptions uploadOptions + = new BlobUploadFromFileOptions(sourceFile.getAbsolutePath()).setParallelTransferOptions( + new com.azure.storage.blob.models.ParallelTransferOptions().setBlockSizeLong(blockSizeBytes) + .setMaxConcurrency(maxConcurrency)); + assertNotNull(client.uploadFromFileWithResponse(uploadOptions).block().getValue().getETag(), + assertionMessage); + + BlobDownloadToFileOptions downloadOptions + = new BlobDownloadToFileOptions(outFile.toPath().toString()).setParallelTransferOptions(parallelOptions) + .setContentValidationAlgorithm(algorithm); + + StepVerifier.create(client.downloadToFileWithResponse(downloadOptions)) + .assertNext(r -> assertNotNull(r.getValue(), assertionMessage)) + .verifyComplete(); + + assertTrue(compareFiles(sourceFile, outFile, 0, payloadBytes), assertionMessage); + } else { + byte[] randomData = getRandomByteArray(payloadBytes); + client.upload(BinaryData.fromBytes(randomData), true).block(); + + if (payloadBytes > blockSizeBytes) { + File outFile = Files.createTempFile("blob-cv-fuzzy-parallel-dl-async-mp", ".bin").toFile(); + outFile.deleteOnExit(); + createdFiles.add(outFile); + Files.deleteIfExists(outFile.toPath()); + + BlobDownloadToFileOptions downloadOptions = new BlobDownloadToFileOptions(outFile.toPath().toString()) + .setParallelTransferOptions(parallelOptions) + .setContentValidationAlgorithm(algorithm); + + StepVerifier.create(client.downloadToFileWithResponse(downloadOptions)) + .assertNext(r -> assertNotNull(r.getValue(), assertionMessage)) + .verifyComplete(); + + byte[] downloaded = Files.readAllBytes(outFile.toPath()); + assertArrayEquals(randomData, downloaded, assertionMessage); + } else { + BlobDownloadContentOptions downloadOptions + = new BlobDownloadContentOptions().setContentValidationAlgorithm(algorithm); + + StepVerifier.create(client.downloadContentWithResponse(downloadOptions)) + .assertNext(r -> assertArrayEquals(randomData, r.getValue().toBytes(), assertionMessage)) + .verifyComplete(); + + BlobDownloadStreamOptions streamOptions + = new BlobDownloadStreamOptions().setContentValidationAlgorithm(algorithm); + StepVerifier + .create(client.downloadStreamWithResponse(streamOptions) + .flatMap(r -> FluxUtil.collectBytesInByteBufferStream(r.getValue()))) + .assertNext(bytes -> assertArrayEquals(randomData, bytes, assertionMessage)) + .verifyComplete(); + } + } + assertTrue(hasStructuredMessageDownloadRequestHeaders(recorded, false), assertionMessage); + } + +} diff --git a/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobContentValidationAsyncUploadTests.java b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobContentValidationAsyncUploadTests.java new file mode 100644 index 000000000000..1ee2466e1fb6 --- /dev/null +++ b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobContentValidationAsyncUploadTests.java @@ -0,0 +1,1073 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.blob; + +import com.azure.core.http.HttpHeaders; +import com.azure.core.util.BinaryData; +import com.azure.core.util.FluxUtil; +import com.azure.storage.blob.models.BlobRequestConditions; +import com.azure.storage.blob.models.BlobStorageException; +import com.azure.storage.blob.models.PageRange; +import com.azure.storage.blob.models.ParallelTransferOptions; +import com.azure.storage.blob.options.AppendBlobAppendBlockOptions; +import com.azure.storage.blob.options.BlobParallelUploadOptions; +import com.azure.storage.blob.options.BlobUploadFromFileOptions; +import com.azure.storage.blob.options.BlockBlobSimpleUploadOptions; +import com.azure.storage.blob.options.BlockBlobStageBlockOptions; +import com.azure.storage.blob.options.PageBlobUploadPagesOptions; +import com.azure.storage.blob.specialized.AppendBlobAsyncClient; +import com.azure.storage.blob.specialized.BlockBlobAsyncClient; +import com.azure.storage.blob.specialized.PageBlobAsyncClient; +import com.azure.storage.blob.specialized.PageBlobClient; +import com.azure.storage.common.ContentValidationAlgorithm; +import com.azure.storage.common.implementation.contentvalidation.ContentValidationModeResolver; +import com.azure.storage.common.implementation.Constants; +import com.azure.storage.common.test.shared.extensions.LiveOnly; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; +import org.junit.jupiter.params.provider.MethodSource; +import reactor.core.publisher.Flux; +import reactor.test.StepVerifier; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.ByteBuffer; +import java.nio.channels.AsynchronousFileChannel; +import java.nio.file.Files; +import java.nio.file.StandardOpenOption; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.Base64; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Tests content validation (CRC64 / structured message) for upload operations using async clients. + * Upload types that have no async counterpart (e.g. OutputStream, SeekableByteChannel) are tested in + * {@link BlobContentValidationUploadTests}. + */ +public class BlobContentValidationAsyncUploadTests extends BlobTestBase { + private static final int TEN_MB = 10 * Constants.MB; + /* Single-part uploads with length < 4MB use CRC64 header; >= 4MB use structured message. */ + private static final int UNDER_4MB = 2 * Constants.MB; + + /** + * Live-only random payload band (256–500 MiB, inclusive upper bound via {@code randomLongFromNamer}+1) for + * {@code uploadWithResponse}, {@code uploadFromFileWithResponse}, and single-block {@code stageBlock}. + */ + private static final long LIVE_RANDOM_PARALLEL_PAYLOAD_MIN_BYTES_EXCLUSIVE = 256L * Constants.MB; + private static final long LIVE_RANDOM_PARALLEL_PAYLOAD_MAX_BYTES_INCLUSIVE = 500L * Constants.MB; + + /** + * Live-only random payload band for sequential append-block puts only ({@link + * #appendBlockLiveRandomRoundTripDataIntegrity()}): {@code Flux.concatMap} issues one append REST call per chunk in + * order (not parallel staging); use a smaller band than {@link #LIVE_RANDOM_PARALLEL_PAYLOAD_MIN_BYTES_EXCLUSIVE}. + */ + private static final long LIVE_RANDOM_SEQUENTIAL_APPEND_PAYLOAD_MIN_BYTES_EXCLUSIVE = 32L * Constants.MB; + private static final long LIVE_RANDOM_SEQUENTIAL_APPEND_PAYLOAD_MAX_BYTES_INCLUSIVE = 64L * Constants.MB; + + private static final String MD5_AND_CRC64_EXCLUSIVE_MESSAGE + = "Only one form of transactional content validation may be used."; + + // =========================================================================================== + // BlobAsyncClient.uploadWithResponse + // =========================================================================================== + + /** + * Single-part upload under 4MB: content validation uses CRC64 header only (no structured message). + */ + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void uploadWithCrc64Header(ContentValidationAlgorithm algorithm) { + List recorded = new CopyOnWriteArrayList<>(); + BlobAsyncClient client = createBlobAsyncClientWithRequestSniffer(recorded); + + byte[] randomData = getRandomByteArray(UNDER_4MB); + Flux data = Flux.just(ByteBuffer.wrap(randomData)); + + BlobParallelUploadOptions options = new BlobParallelUploadOptions(data) + .setParallelTransferOptions(new ParallelTransferOptions().setMaxSingleUploadSizeLong((long) UNDER_4MB)) + .setRequestConditions(new BlobRequestConditions()) + .setContentValidationAlgorithm(algorithm); + + StepVerifier.create(client.uploadWithResponse(options)).assertNext(response -> { + assertNotNull(response.getValue().getETag()); + assertTrue(hasOnlyCrc64Headers(recorded)); + }).verifyComplete(); + } + + /** + * Single-part upload >= 4MB: content validation uses structured message. + */ + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void uploadWithStructuredMessage(ContentValidationAlgorithm algorithm) { + List recorded = new CopyOnWriteArrayList<>(); + BlobAsyncClient client = createBlobAsyncClientWithRequestSniffer(recorded); + + byte[] randomData = getRandomByteArray(TEN_MB); + Flux data = Flux.just(ByteBuffer.wrap(randomData)); + + BlobParallelUploadOptions options = new BlobParallelUploadOptions(data) + .setParallelTransferOptions(new ParallelTransferOptions().setMaxSingleUploadSizeLong((long) TEN_MB)) + .setRequestConditions(new BlobRequestConditions()) + .setContentValidationAlgorithm(algorithm); + + StepVerifier.create(client.uploadWithResponse(options)).assertNext(response -> { + assertNotNull(response.getValue().getETag()); + assertTrue(hasOnlyStructuredMessageHeaders(recorded)); + }).verifyComplete(); + } + + /** + * Multipart (chunked) upload; content validation uses structured message on each stage block. + */ + @LiveOnly // Put Block URLs include random block IDs; not replayable with the test proxy. + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void uploadChunkedWithStructuredMessage(ContentValidationAlgorithm algorithm) { + List recorded = new CopyOnWriteArrayList<>(); + BlobAsyncClient client = createBlobAsyncClientWithRequestSniffer(recorded); + + byte[] randomData = getRandomByteArray(TEN_MB); + Flux data = Flux.just(ByteBuffer.wrap(randomData)); + long blockSize = 2 * (long) Constants.MB; + + BlobParallelUploadOptions options = new BlobParallelUploadOptions(data) + .setParallelTransferOptions( + new ParallelTransferOptions().setBlockSizeLong(blockSize).setMaxSingleUploadSizeLong(blockSize)) + .setRequestConditions(new BlobRequestConditions()) + .setContentValidationAlgorithm(algorithm); + + StepVerifier.create(client.uploadWithResponse(options)).assertNext(response -> { + assertNotNull(response.getValue().getETag()); + assertTrue(hasOnlyStructuredMessageHeaders(recorded)); + }).verifyComplete(); + } + + @Test + public void uploadWithoutContentValidation() { + List recorded = new CopyOnWriteArrayList<>(); + BlobAsyncClient client = createBlobAsyncClientWithRequestSniffer(recorded); + + byte[] randomData = getRandomByteArray(TEN_MB); + Flux data = Flux.just(ByteBuffer.wrap(randomData)); + + BlobParallelUploadOptions options = new BlobParallelUploadOptions(data) + .setParallelTransferOptions(new ParallelTransferOptions().setMaxSingleUploadSizeLong((long) TEN_MB)) + .setRequestConditions(new BlobRequestConditions()) + .setContentValidationAlgorithm(ContentValidationAlgorithm.NONE); + + StepVerifier.create(client.uploadWithResponse(options)).assertNext(response -> { + assertNotNull(response.getValue().getETag()); + assertTrue(hasNoContentValidationHeaders(recorded)); + }).verifyComplete(); + } + + /** + * Blob parallel upload rejects using both computeMd5 (SDK-computed MD5) and CRC64 (transfer validation checksum algorithm) at once. + */ + @Test + public void uploadWithComputeMd5AndCrc64Throws() { + BlobAsyncClient client = createBlobAsyncClientWithRequestSniffer(new CopyOnWriteArrayList<>()); + + byte[] randomData = getRandomByteArray(UNDER_4MB); + Flux data = Flux.just(ByteBuffer.wrap(randomData)); + + BlobParallelUploadOptions options = new BlobParallelUploadOptions(data) + .setParallelTransferOptions(new ParallelTransferOptions().setMaxSingleUploadSizeLong((long) UNDER_4MB)) + .setRequestConditions(new BlobRequestConditions()) + .setComputeMd5(true) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + + StepVerifier.create(client.uploadWithResponse(options)) + .verifyErrorMatches(throwable -> throwable instanceof IllegalArgumentException + && throwable.getMessage().contains(MD5_AND_CRC64_EXCLUSIVE_MESSAGE)); + } + + // =========================================================================================== + // BlockBlobAsyncClient.uploadWithResponse (BlockBlobSimpleUpload / Put Blob) tests + // =========================================================================================== + + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void blockBlobSimpleUploadWithCrc64Header(ContentValidationAlgorithm algorithm) { + List recorded = new CopyOnWriteArrayList<>(); + BlobAsyncClient blobClient = createBlobAsyncClientWithRequestSniffer(recorded); + BlockBlobAsyncClient client = blobClient.getBlockBlobAsyncClient(); + + byte[] randomData = getRandomByteArray(UNDER_4MB); + BinaryData data = BinaryData.fromBytes(randomData); + + BlockBlobSimpleUploadOptions options + = new BlockBlobSimpleUploadOptions(data).setContentValidationAlgorithm(algorithm); + + StepVerifier.create(client.uploadWithResponse(options)).assertNext(response -> { + assertNotNull(response.getValue().getETag()); + assertTrue(hasOnlyCrc64Headers(recorded)); + }).verifyComplete(); + } + + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void blockBlobSimpleUploadWithStructuredMessage(ContentValidationAlgorithm algorithm) { + List recorded = new CopyOnWriteArrayList<>(); + BlobAsyncClient blobClient = createBlobAsyncClientWithRequestSniffer(recorded); + BlockBlobAsyncClient client = blobClient.getBlockBlobAsyncClient(); + + byte[] randomData = getRandomByteArray(TEN_MB); + BinaryData data = BinaryData.fromBytes(randomData); + + BlockBlobSimpleUploadOptions options + = new BlockBlobSimpleUploadOptions(data).setContentValidationAlgorithm(algorithm); + + StepVerifier.create(client.uploadWithResponse(options)).assertNext(response -> { + assertNotNull(response.getValue().getETag()); + assertTrue(hasOnlyStructuredMessageHeaders(recorded)); + }).verifyComplete(); + } + + @Test + public void blockBlobSimpleUploadWithNoContentValidation() { + List recorded = new CopyOnWriteArrayList<>(); + BlobAsyncClient blobClient = createBlobAsyncClientWithRequestSniffer(recorded); + BlockBlobAsyncClient client = blobClient.getBlockBlobAsyncClient(); + + byte[] randomData = getRandomByteArray(TEN_MB); + BinaryData data = BinaryData.fromBytes(randomData); + + BlockBlobSimpleUploadOptions options + = new BlockBlobSimpleUploadOptions(data).setContentValidationAlgorithm(ContentValidationAlgorithm.NONE); + + StepVerifier.create(client.uploadWithResponse(options)).assertNext(response -> { + assertNotNull(response.getValue().getETag()); + assertTrue(hasNoContentValidationHeaders(recorded)); + }).verifyComplete(); + } + + // =========================================================================================== + // BlockBlobAsyncClient.stageBlockWithResponse (Put Block) tests + // =========================================================================================== + + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void stageBlockWithCrc64Header(ContentValidationAlgorithm algorithm) { + List recorded = new CopyOnWriteArrayList<>(); + BlobAsyncClient blobClient = createBlobAsyncClientWithRequestSniffer(recorded); + BlockBlobAsyncClient client = blobClient.getBlockBlobAsyncClient(); + + byte[] randomData = getRandomByteArray(UNDER_4MB); + BinaryData data = BinaryData.fromBytes(randomData); + + BlockBlobStageBlockOptions options + = new BlockBlobStageBlockOptions(getBlockID(), data).setContentValidationAlgorithm(algorithm); + + StepVerifier.create(client.stageBlockWithResponse(options)) + .assertNext(response -> assertTrue(hasOnlyCrc64Headers(recorded))) + .verifyComplete(); + } + + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void stageBlockWithStructuredMessage(ContentValidationAlgorithm algorithm) { + List recorded = new CopyOnWriteArrayList<>(); + BlobAsyncClient blobClient = createBlobAsyncClientWithRequestSniffer(recorded); + BlockBlobAsyncClient client = blobClient.getBlockBlobAsyncClient(); + + byte[] randomData = getRandomByteArray(TEN_MB); + BinaryData data = BinaryData.fromBytes(randomData); + + BlockBlobStageBlockOptions options + = new BlockBlobStageBlockOptions(getBlockID(), data).setContentValidationAlgorithm(algorithm); + + StepVerifier.create(client.stageBlockWithResponse(options)) + .assertNext(response -> assertTrue(hasOnlyStructuredMessageHeaders(recorded))) + .verifyComplete(); + } + + @Test + public void stageBlockWithNoContentValidation() { + List recorded = new CopyOnWriteArrayList<>(); + BlobAsyncClient blobClient = createBlobAsyncClientWithRequestSniffer(recorded); + BlockBlobAsyncClient client = blobClient.getBlockBlobAsyncClient(); + + byte[] randomData = getRandomByteArray(TEN_MB); + BinaryData data = BinaryData.fromBytes(randomData); + + BlockBlobStageBlockOptions options = new BlockBlobStageBlockOptions(getBlockID(), data) + .setContentValidationAlgorithm(ContentValidationAlgorithm.NONE); + + StepVerifier.create(client.stageBlockWithResponse(options)) + .assertNext(response -> assertTrue(hasNoContentValidationHeaders(recorded))) + .verifyComplete(); + } + + // =========================================================================================== + // AppendBlobAsyncClient.appendBlockWithResponse (Append Block) tests + // =========================================================================================== + + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void appendBlockWithCrc64Header(ContentValidationAlgorithm algorithm) { + List recorded = new CopyOnWriteArrayList<>(); + BlobAsyncClient blobClient = createBlobAsyncClientWithRequestSniffer(recorded); + AppendBlobAsyncClient client = blobClient.getAppendBlobAsyncClient(); + client.create().block(); + + byte[] randomData = getRandomByteArray(UNDER_4MB); + Flux data = Flux.just(ByteBuffer.wrap(randomData)); + + AppendBlobAppendBlockOptions options + = new AppendBlobAppendBlockOptions(data, UNDER_4MB).setContentValidationAlgorithm(algorithm); + + StepVerifier.create(client.appendBlockWithResponse(options)).assertNext(response -> { + assertNotNull(response.getValue().getETag()); + assertTrue(hasOnlyCrc64Headers(recorded)); + }).verifyComplete(); + } + + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void appendBlockWithStructuredMessage(ContentValidationAlgorithm algorithm) { + List recorded = new CopyOnWriteArrayList<>(); + BlobAsyncClient blobClient = createBlobAsyncClientWithRequestSniffer(recorded); + AppendBlobAsyncClient client = blobClient.getAppendBlobAsyncClient(); + client.create().block(); + + byte[] randomData = getRandomByteArray(TEN_MB); + Flux data = Flux.just(ByteBuffer.wrap(randomData)); + + AppendBlobAppendBlockOptions options + = new AppendBlobAppendBlockOptions(data, TEN_MB).setContentValidationAlgorithm(algorithm); + + StepVerifier.create(client.appendBlockWithResponse(options)).assertNext(response -> { + assertNotNull(response.getValue().getETag()); + assertTrue(hasOnlyStructuredMessageHeaders(recorded)); + }).verifyComplete(); + } + + @Test + public void appendBlockWithNoContentValidation() { + List recorded = new CopyOnWriteArrayList<>(); + BlobAsyncClient blobClient = createBlobAsyncClientWithRequestSniffer(recorded); + AppendBlobAsyncClient client = blobClient.getAppendBlobAsyncClient(); + client.create().block(); + + byte[] randomData = getRandomByteArray(TEN_MB); + Flux data = Flux.just(ByteBuffer.wrap(randomData)); + + AppendBlobAppendBlockOptions options = new AppendBlobAppendBlockOptions(data, TEN_MB) + .setContentValidationAlgorithm(ContentValidationAlgorithm.NONE); + + StepVerifier.create(client.appendBlockWithResponse(options)).assertNext(response -> { + assertNotNull(response.getValue().getETag()); + assertTrue(hasNoContentValidationHeaders(recorded)); + }).verifyComplete(); + } + + // =========================================================================================== + // PageBlobAsyncClient.uploadPagesWithResponse (Put Page) tests + // =========================================================================================== + + private static final int PAGE_BYTES = PageBlobClient.PAGE_BYTES; + private static final int UNDER_4MB_PAGE_ALIGNED = (UNDER_4MB / PAGE_BYTES) * PAGE_BYTES; + private static final int FOUR_MB_PAGE_ALIGNED = (4 * Constants.MB / PAGE_BYTES) * PAGE_BYTES; + + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void uploadPagesWithCrc64Header(ContentValidationAlgorithm algorithm) { + List recorded = new CopyOnWriteArrayList<>(); + BlobAsyncClient blobClient = createBlobAsyncClientWithRequestSniffer(recorded); + PageBlobAsyncClient client = blobClient.getPageBlobAsyncClient(); + client.create(UNDER_4MB_PAGE_ALIGNED).block(); + + byte[] randomData = getRandomByteArray(UNDER_4MB_PAGE_ALIGNED); + Flux data = Flux.just(ByteBuffer.wrap(randomData)); + + PageBlobUploadPagesOptions options + = new PageBlobUploadPagesOptions(new PageRange().setStart(0).setEnd(UNDER_4MB_PAGE_ALIGNED - 1), data) + .setContentValidationAlgorithm(algorithm); + + StepVerifier.create(client.uploadPagesWithResponse(options)).assertNext(response -> { + assertNotNull(response.getValue().getETag()); + assertTrue(hasOnlyCrc64Headers(recorded)); + }).verifyComplete(); + } + + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void uploadPagesWithStructuredMessage(ContentValidationAlgorithm algorithm) { + List recorded = new CopyOnWriteArrayList<>(); + BlobAsyncClient blobClient = createBlobAsyncClientWithRequestSniffer(recorded); + PageBlobAsyncClient client = blobClient.getPageBlobAsyncClient(); + client.create(FOUR_MB_PAGE_ALIGNED).block(); + + byte[] randomData = getRandomByteArray(FOUR_MB_PAGE_ALIGNED); + Flux data = Flux.just(ByteBuffer.wrap(randomData)); + + PageBlobUploadPagesOptions options + = new PageBlobUploadPagesOptions(new PageRange().setStart(0).setEnd(FOUR_MB_PAGE_ALIGNED - 1), data) + .setContentValidationAlgorithm(algorithm); + + StepVerifier.create(client.uploadPagesWithResponse(options)).assertNext(response -> { + assertNotNull(response.getValue().getETag()); + assertTrue(hasOnlyStructuredMessageHeaders(recorded)); + }).verifyComplete(); + } + + @Test + public void uploadPagesWithNoContentValidation() { + List recorded = new CopyOnWriteArrayList<>(); + BlobAsyncClient blobClient = createBlobAsyncClientWithRequestSniffer(recorded); + PageBlobAsyncClient client = blobClient.getPageBlobAsyncClient(); + client.create(FOUR_MB_PAGE_ALIGNED).block(); + + byte[] randomData = getRandomByteArray(FOUR_MB_PAGE_ALIGNED); + Flux data = Flux.just(ByteBuffer.wrap(randomData)); + + PageBlobUploadPagesOptions options + = new PageBlobUploadPagesOptions(new PageRange().setStart(0).setEnd(FOUR_MB_PAGE_ALIGNED - 1), data) + .setContentValidationAlgorithm(ContentValidationAlgorithm.NONE); + + StepVerifier.create(client.uploadPagesWithResponse(options)).assertNext(response -> { + assertNotNull(response.getValue().getETag()); + assertTrue(hasNoContentValidationHeaders(recorded)); + }).verifyComplete(); + } + + // =========================================================================================== + // BlobAsyncClient.uploadFromFileWithResponse tests + // =========================================================================================== + + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void uploadFromFileWithCrc64Header(ContentValidationAlgorithm algorithm) throws IOException { + List recorded = new CopyOnWriteArrayList<>(); + BlobAsyncClient client = createBlobAsyncClientWithRequestSniffer(recorded); + + File tempFile = getRandomFile(UNDER_4MB); + + BlobUploadFromFileOptions options = new BlobUploadFromFileOptions(tempFile.getAbsolutePath()) + .setParallelTransferOptions(new ParallelTransferOptions().setMaxSingleUploadSizeLong((long) UNDER_4MB)) + .setContentValidationAlgorithm(algorithm); + + StepVerifier.create(client.uploadFromFileWithResponse(options)).assertNext(response -> { + assertNotNull(response.getValue().getETag()); + assertTrue(hasOnlyCrc64Headers(recorded)); + }).verifyComplete(); + } + + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void uploadFromFileWithStructuredMessage(ContentValidationAlgorithm algorithm) throws IOException { + List recorded = new CopyOnWriteArrayList<>(); + BlobAsyncClient client = createBlobAsyncClientWithRequestSniffer(recorded); + + File tempFile = getRandomFile(TEN_MB); + + BlobUploadFromFileOptions options = new BlobUploadFromFileOptions(tempFile.getAbsolutePath()) + .setParallelTransferOptions(new ParallelTransferOptions().setMaxSingleUploadSizeLong((long) TEN_MB)) + .setContentValidationAlgorithm(algorithm); + + StepVerifier.create(client.uploadFromFileWithResponse(options)).assertNext(response -> { + assertNotNull(response.getValue().getETag()); + assertTrue(hasOnlyStructuredMessageHeaders(recorded)); + }).verifyComplete(); + } + + @LiveOnly // Put Block URLs include random block IDs; not replayable with the test proxy. + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void uploadFromFileChunkedWithStructuredMessage(ContentValidationAlgorithm algorithm) throws IOException { + List recorded = new CopyOnWriteArrayList<>(); + BlobAsyncClient client = createBlobAsyncClientWithRequestSniffer(recorded); + + File tempFile = getRandomFile(TEN_MB); + long blockSize = 2 * (long) Constants.MB; + + BlobUploadFromFileOptions options = new BlobUploadFromFileOptions(tempFile.getAbsolutePath()) + .setParallelTransferOptions( + new ParallelTransferOptions().setBlockSizeLong(blockSize).setMaxSingleUploadSizeLong(blockSize)) + .setContentValidationAlgorithm(algorithm); + + StepVerifier.create(client.uploadFromFileWithResponse(options)).assertNext(response -> { + assertNotNull(response.getValue().getETag()); + assertTrue(hasOnlyStructuredMessageHeaders(recorded)); + }).verifyComplete(); + } + + @Test + public void uploadFromFileWithNoContentValidation() throws IOException { + List recorded = new CopyOnWriteArrayList<>(); + BlobAsyncClient client = createBlobAsyncClientWithRequestSniffer(recorded); + + File tempFile = getRandomFile(TEN_MB); + + BlobUploadFromFileOptions options = new BlobUploadFromFileOptions(tempFile.getAbsolutePath()) + .setParallelTransferOptions(new ParallelTransferOptions().setMaxSingleUploadSizeLong((long) TEN_MB)) + .setContentValidationAlgorithm(ContentValidationAlgorithm.NONE); + + StepVerifier.create(client.uploadFromFileWithResponse(options)).assertNext(response -> { + assertNotNull(response.getValue().getETag()); + assertTrue(hasNoContentValidationHeaders(recorded)); + }).verifyComplete(); + } + + // =========================================================================================== + // Exact 4MB boundary tests + // + // The cutoff between CRC64 header and structured message is exactly 4MB. + // Uploads of exactly 4MB should use structured message (>= threshold), not CRC64 header. + // =========================================================================================== + + private static final int EXACTLY_4MB = 4 * Constants.MB; + + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void uploadAtExactly4MBUsesStructuredMessage(ContentValidationAlgorithm algorithm) { + List recorded = new CopyOnWriteArrayList<>(); + BlobAsyncClient client = createBlobAsyncClientWithRequestSniffer(recorded); + + byte[] randomData = getRandomByteArray(EXACTLY_4MB); + Flux data = Flux.just(ByteBuffer.wrap(randomData)); + + BlobParallelUploadOptions options = new BlobParallelUploadOptions(data) + .setParallelTransferOptions(new ParallelTransferOptions().setMaxSingleUploadSizeLong((long) EXACTLY_4MB)) + .setRequestConditions(new BlobRequestConditions()) + .setContentValidationAlgorithm(algorithm); + + StepVerifier.create(client.uploadWithResponse(options)).assertNext(response -> { + assertNotNull(response.getValue().getETag()); + assertTrue(hasOnlyStructuredMessageHeaders(recorded)); + }).verifyComplete(); + } + + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void blockBlobSimpleUploadAtExactly4MBUsesStructuredMessage(ContentValidationAlgorithm algorithm) { + List recorded = new CopyOnWriteArrayList<>(); + BlobAsyncClient blobClient = createBlobAsyncClientWithRequestSniffer(recorded); + BlockBlobAsyncClient client = blobClient.getBlockBlobAsyncClient(); + + byte[] randomData = getRandomByteArray(EXACTLY_4MB); + BinaryData data = BinaryData.fromBytes(randomData); + + BlockBlobSimpleUploadOptions options + = new BlockBlobSimpleUploadOptions(data).setContentValidationAlgorithm(algorithm); + + StepVerifier.create(client.uploadWithResponse(options)).assertNext(response -> { + assertNotNull(response.getValue().getETag()); + assertTrue(hasOnlyStructuredMessageHeaders(recorded)); + }).verifyComplete(); + } + + // =========================================================================================== + // Progress reporting (transfer validation must be NONE/null when a progress listener is set) + // =========================================================================================== + + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void uploadWithProgressAndNonNoneContentValidationThrows(ContentValidationAlgorithm algorithm) { + BlobAsyncClient client = createBlobAsyncClientWithRequestSniffer(new CopyOnWriteArrayList<>()); + + byte[] randomData = getRandomByteArray(TEN_MB); + Flux data = Flux.just(ByteBuffer.wrap(randomData)); + + BlobParallelUploadOptions options = new BlobParallelUploadOptions(data).setParallelTransferOptions( + new ParallelTransferOptions().setMaxSingleUploadSizeLong((long) TEN_MB).setProgressListener(l -> { + })).setRequestConditions(new BlobRequestConditions()).setContentValidationAlgorithm(algorithm); + + StepVerifier.create(client.uploadWithResponse(options)) + .expectErrorSatisfies(ex -> assertEquals( + ContentValidationModeResolver.PROGRESS_CONFLICTS_TRANSFER_CONTENT_VALIDATION_MESSAGE, ex.getMessage())) + .verify(); + } + + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void uploadFromFileWithProgressAndNonNoneContentValidationThrows(ContentValidationAlgorithm algorithm) + throws IOException { + BlobAsyncClient client = createBlobAsyncClientWithRequestSniffer(new CopyOnWriteArrayList<>()); + + File tempFile = getRandomFile(TEN_MB); + + BlobUploadFromFileOptions options + = new BlobUploadFromFileOptions(tempFile.getAbsolutePath()).setParallelTransferOptions( + new ParallelTransferOptions().setMaxSingleUploadSizeLong((long) TEN_MB).setProgressListener(l -> { + })).setContentValidationAlgorithm(algorithm); + + StepVerifier.create(client.uploadFromFileWithResponse(options)) + .expectErrorSatisfies(ex -> assertEquals( + ContentValidationModeResolver.PROGRESS_CONFLICTS_TRANSFER_CONTENT_VALIDATION_MESSAGE, ex.getMessage())) + .verify(); + } + + // =========================================================================================== + // Data integrity round-trip tests (upload with content validation, download, verify) + // + // Previous tests verify that the correct headers are sent. These tests verify end-to-end + // integrity: the data uploaded with CRC64/structured message can be downloaded and matches + // the original byte-for-byte. + // =========================================================================================== + + @Test + public void uploadWithCrc64RoundTripDataIntegrity() { + BlobAsyncClient client = createBlobAsyncClientWithRequestSniffer(new CopyOnWriteArrayList<>()); + + byte[] randomData = getRandomByteArray(UNDER_4MB); + Flux data = Flux.just(ByteBuffer.wrap(randomData)); + + BlobParallelUploadOptions options = new BlobParallelUploadOptions(data) + .setParallelTransferOptions(new ParallelTransferOptions().setMaxSingleUploadSizeLong((long) UNDER_4MB)) + .setRequestConditions(new BlobRequestConditions()) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + + StepVerifier.create(client.uploadWithResponse(options).then(client.downloadContent())) + .assertNext(downloaded -> assertArrayEquals(randomData, downloaded.toBytes())) + .verifyComplete(); + } + + @Test + public void uploadWithStructuredMessageRoundTripDataIntegrity() { + BlobAsyncClient client = createBlobAsyncClientWithRequestSniffer(new CopyOnWriteArrayList<>()); + + byte[] randomData = getRandomByteArray(TEN_MB); + Flux data = Flux.just(ByteBuffer.wrap(randomData)); + + BlobParallelUploadOptions options = new BlobParallelUploadOptions(data) + .setParallelTransferOptions(new ParallelTransferOptions().setMaxSingleUploadSizeLong((long) TEN_MB)) + .setRequestConditions(new BlobRequestConditions()) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + + StepVerifier.create(client.uploadWithResponse(options).then(client.downloadContent())) + .assertNext(downloaded -> assertArrayEquals(randomData, downloaded.toBytes())) + .verifyComplete(); + } + + @LiveOnly // Put Block URLs include random block IDs; not replayable with the test proxy. + @Test + public void uploadChunkedWithStructuredMessageRoundTripDataIntegrity() { + BlobAsyncClient client = createBlobAsyncClientWithRequestSniffer(new CopyOnWriteArrayList<>()); + + byte[] randomData = getRandomByteArray(TEN_MB); + Flux data = Flux.just(ByteBuffer.wrap(randomData)); + long blockSize = 2 * (long) Constants.MB; + + BlobParallelUploadOptions options = new BlobParallelUploadOptions(data) + .setParallelTransferOptions( + new ParallelTransferOptions().setBlockSizeLong(blockSize).setMaxSingleUploadSizeLong(blockSize)) + .setRequestConditions(new BlobRequestConditions()) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + + StepVerifier.create(client.uploadWithResponse(options).then(client.downloadContent())) + .assertNext(downloaded -> assertArrayEquals(randomData, downloaded.toBytes())) + .verifyComplete(); + } + + @Test + public void blockBlobSimpleUploadRoundTripDataIntegrity() { + BlobAsyncClient blobClient = createBlobAsyncClientWithRequestSniffer(new CopyOnWriteArrayList<>()); + BlockBlobAsyncClient client = blobClient.getBlockBlobAsyncClient(); + + byte[] randomData = getRandomByteArray(TEN_MB); + BinaryData data = BinaryData.fromBytes(randomData); + + BlockBlobSimpleUploadOptions options + = new BlockBlobSimpleUploadOptions(data).setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + + StepVerifier.create(client.uploadWithResponse(options).then(client.downloadContent())) + .assertNext(downloaded -> assertArrayEquals(randomData, downloaded.toBytes())) + .verifyComplete(); + } + + @Test + public void appendBlockRoundTripDataIntegrity() { + BlobAsyncClient blobClient = createBlobAsyncClientWithRequestSniffer(new CopyOnWriteArrayList<>()); + AppendBlobAsyncClient client = blobClient.getAppendBlobAsyncClient(); + client.create().block(); + + byte[] randomData = getRandomByteArray(TEN_MB); + Flux data = Flux.just(ByteBuffer.wrap(randomData)); + + AppendBlobAppendBlockOptions options = new AppendBlobAppendBlockOptions(data, TEN_MB) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + + StepVerifier.create(client.appendBlockWithResponse(options).then(client.downloadContent())) + .assertNext(downloaded -> assertArrayEquals(randomData, downloaded.toBytes())) + .verifyComplete(); + } + + @Test + public void uploadPagesRoundTripDataIntegrity() { + BlobAsyncClient blobClient = createBlobAsyncClientWithRequestSniffer(new CopyOnWriteArrayList<>()); + PageBlobAsyncClient client = blobClient.getPageBlobAsyncClient(); + client.create(FOUR_MB_PAGE_ALIGNED).block(); + + byte[] randomData = getRandomByteArray(FOUR_MB_PAGE_ALIGNED); + Flux data = Flux.just(ByteBuffer.wrap(randomData)); + + PageBlobUploadPagesOptions options + = new PageBlobUploadPagesOptions(new PageRange().setStart(0).setEnd(FOUR_MB_PAGE_ALIGNED - 1), data) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + + StepVerifier.create(client.uploadPagesWithResponse(options).then(client.downloadContent())) + .assertNext(downloaded -> assertArrayEquals(randomData, downloaded.toBytes())) + .verifyComplete(); + } + + // =========================================================================================== + // Live-only random payload bands. + // - 256–500 MiB: parallelUpload, uploadFromFile, stageBlock — parallel staging / giant block; default transfer + // options where applicable. + // - 32–64 MiB (sequential append blocks only): appendBlockLiveRandom… — one append REST call per chunk in order. + // =========================================================================================== + + @LiveOnly // This test is too large for the test proxy. + @Test + public void parallelUploadLiveRandomRoundTripDataIntegrity() throws Exception { + int chosenPayloadSizeBytes = (int) randomLongFromNamer(LIVE_RANDOM_PARALLEL_PAYLOAD_MIN_BYTES_EXCLUSIVE + 1, + LIVE_RANDOM_PARALLEL_PAYLOAD_MAX_BYTES_INCLUSIVE + 1); + try { + String prefix = "chosenPayloadSizeBytes=" + chosenPayloadSizeBytes + ". "; + BlobAsyncClient client = createBlobAsyncClientWithRequestSniffer(new CopyOnWriteArrayList<>()); + File sourceFile = getRandomFile(chosenPayloadSizeBytes); + File outFile = Files.createTempFile("blob-cv-live-par-dl-async", ".bin").toFile(); + outFile.deleteOnExit(); + + try { + try (InputStream data = new FileInputStream(sourceFile)) { + BlobParallelUploadOptions options + = new BlobParallelUploadOptions(data).setRequestConditions(new BlobRequestConditions()) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + client.uploadWithResponse(options).block(); + } + + client.downloadToFile(outFile.getPath(), true).block(); + assertTrue(compareFiles(sourceFile, outFile, 0, chosenPayloadSizeBytes), prefix); + } finally { + if (!sourceFile.delete()) { + sourceFile.deleteOnExit(); + } + if (!outFile.delete()) { + outFile.deleteOnExit(); + } + } + } catch (Exception e) { + throw new Exception("chosenPayloadSizeBytes=" + chosenPayloadSizeBytes + ". " + e.getMessage(), e); + } + } + + @LiveOnly // This test is too large for the test proxy. + @Test + public void stageBlockLiveRandomRoundTripDataIntegrity() throws Exception { + int chosenPayloadSizeBytes = (int) randomLongFromNamer(LIVE_RANDOM_PARALLEL_PAYLOAD_MIN_BYTES_EXCLUSIVE + 1, + LIVE_RANDOM_PARALLEL_PAYLOAD_MAX_BYTES_INCLUSIVE + 1); + try { + String prefix = "chosenPayloadSizeBytes=" + chosenPayloadSizeBytes + ". "; + BlobAsyncClient blobClient = createBlobAsyncClientWithRequestSniffer(new CopyOnWriteArrayList<>()); + BlockBlobAsyncClient client = blobClient.getBlockBlobAsyncClient(); + String blockId = getBlockID(); + + File sourceFile = getRandomFile(chosenPayloadSizeBytes); + File outFile = Files.createTempFile("blob-cv-live-stage-async-dl", ".bin").toFile(); + outFile.deleteOnExit(); + try { + BinaryData binaryData = BinaryData.fromFile(sourceFile.toPath()); + BlockBlobStageBlockOptions stageOptions = new BlockBlobStageBlockOptions(blockId, binaryData) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + client.stageBlockWithResponse(stageOptions).block(); + client.commitBlockList(Collections.singletonList(blockId)).block(); + blobClient.downloadToFile(outFile.getPath(), true).block(); + assertTrue(compareFiles(sourceFile, outFile, 0, chosenPayloadSizeBytes), prefix); + } finally { + if (!sourceFile.delete()) { + sourceFile.deleteOnExit(); + } + if (!outFile.delete()) { + outFile.deleteOnExit(); + } + } + } catch (Exception e) { + throw new Exception("chosenPayloadSizeBytes=" + chosenPayloadSizeBytes + ". " + e.getMessage(), e); + } + } + + @LiveOnly // This test is too large for the test proxy. + @Test + public void appendBlockLiveRandomRoundTripDataIntegrity() throws Exception { + int chosenPayloadSizeBytes + = (int) randomLongFromNamer(LIVE_RANDOM_SEQUENTIAL_APPEND_PAYLOAD_MIN_BYTES_EXCLUSIVE + 1, + LIVE_RANDOM_SEQUENTIAL_APPEND_PAYLOAD_MAX_BYTES_INCLUSIVE + 1); + try { + String prefix = "chosenPayloadSizeBytes=" + chosenPayloadSizeBytes + ". "; + BlobAsyncClient blobClient = createBlobAsyncClientWithRequestSniffer(new CopyOnWriteArrayList<>()); + AppendBlobAsyncClient client = blobClient.getAppendBlobAsyncClient(); + client.create().block(); + + int maxAppendBlockBytes = client.getMaxAppendBlockBytes(); + File sourceFile = getRandomFile(chosenPayloadSizeBytes); + File outFile = Files.createTempFile("blob-cv-live-append-async-dl", ".bin").toFile(); + outFile.deleteOnExit(); + + try { + try (AsynchronousFileChannel channel + = AsynchronousFileChannel.open(sourceFile.toPath(), StandardOpenOption.READ)) { + FluxUtil.readFile(channel, maxAppendBlockBytes, 0, chosenPayloadSizeBytes).concatMap(bb -> { + AppendBlobAppendBlockOptions appendOptions + = new AppendBlobAppendBlockOptions(Flux.just(bb), bb.remaining()) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + return client.appendBlockWithResponse(appendOptions); + }).then().block(); + } + + blobClient.downloadToFile(outFile.getPath(), true).block(); + assertTrue(compareFiles(sourceFile, outFile, 0, chosenPayloadSizeBytes), prefix); + } finally { + if (!sourceFile.delete()) { + sourceFile.deleteOnExit(); + } + if (!outFile.delete()) { + outFile.deleteOnExit(); + } + } + } catch (Exception e) { + throw new Exception("chosenPayloadSizeBytes=" + chosenPayloadSizeBytes + ". " + e.getMessage(), e); + } + } + + @LiveOnly // This test is too large for the test proxy. + @Test + public void uploadFromFileLiveRandomRoundTripDataIntegrity() throws Exception { + int chosenPayloadSizeBytes = (int) randomLongFromNamer(LIVE_RANDOM_PARALLEL_PAYLOAD_MIN_BYTES_EXCLUSIVE + 1, + LIVE_RANDOM_PARALLEL_PAYLOAD_MAX_BYTES_INCLUSIVE + 1); + try { + String prefix = "chosenPayloadSizeBytes=" + chosenPayloadSizeBytes + ". "; + BlobAsyncClient client = createBlobAsyncClientWithRequestSniffer(new CopyOnWriteArrayList<>()); + File sourceFile = getRandomFile(chosenPayloadSizeBytes); + File outFile = Files.createTempFile("blob-cv-live-uploadfromfile-async-dl", ".bin").toFile(); + outFile.deleteOnExit(); + try { + BlobUploadFromFileOptions options = new BlobUploadFromFileOptions(sourceFile.getAbsolutePath()) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + assertNotNull(client.uploadFromFileWithResponse(options).block().getValue().getETag(), + prefix + "Missing E-Tag on upload-from-file."); + client.downloadToFile(outFile.getPath(), true).block(); + assertTrue(compareFiles(sourceFile, outFile, 0, chosenPayloadSizeBytes), prefix); + } finally { + if (!sourceFile.delete()) { + sourceFile.deleteOnExit(); + } + if (!outFile.delete()) { + outFile.deleteOnExit(); + } + } + } catch (Exception e) { + throw new Exception("chosenPayloadSizeBytes=" + chosenPayloadSizeBytes + ". " + e.getMessage(), e); + } + } + + // ---------- Deterministic parallel upload (async) ---------- + + @ParameterizedTest + @MethodSource("com.azure.storage.blob.BlobTestBase#fuzzyParallelUploadPutBlobReplayableCases") + public void fuzzyParallelUploadPutBlobReplayableRoundTrip(int payloadBytes, long segmentBytes, int maxConcurrency) + throws IOException { + assertParallelUploadFuzzyRoundTripAsync("putBlobReplay", payloadBytes, segmentBytes, maxConcurrency); + } + + @LiveOnly // Staging-only cases: Put Block URLs include random IDs. + @ParameterizedTest + @MethodSource("com.azure.storage.blob.BlobTestBase#fuzzyParallelUploadSmallPayloadStagingCases") + public void fuzzyParallelUploadSmallPayloadRoundTripRequiresLiveStaging(int payloadBytes, long segmentBytes, + int maxConcurrency) throws IOException { + assertParallelUploadFuzzyRoundTripAsync("smallPayloadStaging", payloadBytes, segmentBytes, maxConcurrency); + } + + @LiveOnly // payload > segment for every tuple; always staging/Put Block. + @ParameterizedTest + @MethodSource("com.azure.storage.blob.BlobTestBase#fuzzyParallelUploadSub4MiBCases") + public void fuzzyParallelUploadSubFourMiBBlobRoundTrip(int payloadBytes, long segmentBytes, int maxConcurrency) + throws IOException { + assertParallelUploadFuzzyRoundTripAsync("subFourMiB", payloadBytes, segmentBytes, maxConcurrency); + } + + @LiveOnly // Staging-only cases. + @ParameterizedTest + @MethodSource("com.azure.storage.blob.BlobTestBase#fuzzyParallelUploadFourMiBBoundaryStagingCases") + public void fuzzyParallelUploadFourMiBBoundaryRoundTripRequiresLiveStaging(int payloadBytes, long segmentBytes, + int maxConcurrency) throws IOException { + assertParallelUploadFuzzyRoundTripAsync("fourMiBBoundaryStaging", payloadBytes, segmentBytes, maxConcurrency); + } + + @LiveOnly // Chunked uploads only. + @ParameterizedTest + @MethodSource("com.azure.storage.blob.BlobTestBase#fuzzyParallelUploadMediumMultiPartCases") + public void fuzzyParallelUploadMediumMultiPartRoundTrip(int payloadBytes, long segmentBytes, int maxConcurrency) + throws IOException { + assertParallelUploadFuzzyRoundTripAsync("mediumMultiPart", payloadBytes, segmentBytes, maxConcurrency); + } + + @LiveOnly // Large chunked uploads. + @ParameterizedTest + @MethodSource("com.azure.storage.blob.BlobTestBase#fuzzyParallelUploadLargeMultiPartCases") + public void fuzzyParallelUploadLargeMultiPartRoundTrip(int payloadBytes, long segmentBytes, int maxConcurrency) + throws IOException { + assertParallelUploadFuzzyRoundTripAsync("largeMultiPart", payloadBytes, segmentBytes, maxConcurrency); + } + + private void assertParallelUploadFuzzyRoundTripAsync(String caseKind, int payloadBytes, long segmentBytes, + int maxConcurrency) throws IOException { + BlobAsyncClient client = createBlobAsyncClientWithRequestSniffer(new CopyOnWriteArrayList<>()); + + ParallelTransferOptions parallelOptions = new ParallelTransferOptions().setBlockSizeLong(segmentBytes) + .setMaxSingleUploadSizeLong(segmentBytes) + .setMaxConcurrency(maxConcurrency); + + String assertionMessage = "Fuzzy parallel upload [" + caseKind + "] payloadBytes=" + payloadBytes + + ", segmentBytes=" + segmentBytes + ", maxConcurrency=" + maxConcurrency; + + // above this threshold the fuzzy parallel upload helpers stream from a temp source file + // to avoid materializing the full payload twice in heap. + if (payloadBytes >= 96 * Constants.MB) { + File sourceFile = getRandomFile(payloadBytes); + File outFile = Files.createTempFile("blob-cv-fuzzy-parallel-dl-async", ".bin").toFile(); + outFile.deleteOnExit(); + int readChunkSize = (int) Math.min(8L * Constants.MB, Math.max(64 * Constants.KB, segmentBytes)); + AsynchronousFileChannel channel + = AsynchronousFileChannel.open(sourceFile.toPath(), StandardOpenOption.READ); + try { + try { + Flux data = FluxUtil.readFile(channel, readChunkSize, 0, payloadBytes); + BlobParallelUploadOptions options + = new BlobParallelUploadOptions(data).setParallelTransferOptions(parallelOptions) + .setRequestConditions(new BlobRequestConditions()) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + client.uploadWithResponse(options).block(); + } finally { + channel.close(); + } + client.downloadToFile(outFile.getPath(), true).block(); + assertTrue(compareFiles(sourceFile, outFile, 0, payloadBytes), assertionMessage); + } finally { + if (!sourceFile.delete()) { + sourceFile.deleteOnExit(); + } + if (!outFile.delete()) { + outFile.deleteOnExit(); + } + } + } else { + byte[] randomData = getRandomByteArray(payloadBytes); + Flux data = Flux.just(ByteBuffer.wrap(randomData)); + BlobParallelUploadOptions options + = new BlobParallelUploadOptions(data).setParallelTransferOptions(parallelOptions) + .setRequestConditions(new BlobRequestConditions()) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + + StepVerifier.create(client.uploadWithResponse(options).then(client.downloadContent())) + .assertNext(downloaded -> assertArrayEquals(randomData, downloaded.toBytes(), assertionMessage)) + .verifyComplete(); + } + } + + // =========================================================================================== + // Customer Provided MD5 Byte[] with Content Validation Algorithm + // =========================================================================================== + + private static final byte[] DEFAULT_MD5 = createDefaultMd5(); + private static final String MESSAGE = "Both x-ms-content-crc64 header and Content-MD5 header are present."; + + private static byte[] createDefaultMd5() { + try { + return Base64.getEncoder().encode(MessageDigest.getInstance("MD5").digest(DATA.getDefaultBytes())); + } catch (NoSuchAlgorithmException ex) { + throw LOGGER.logExceptionAsError(new RuntimeException("MD5 algorithm unavailable.", ex)); + } + } + + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void blockBlobUploadWithCustomerProvidedMd5AndCrc64Header(ContentValidationAlgorithm algorithm) { + BlockBlobAsyncClient client + = createBlobAsyncClientWithRequestSniffer(new CopyOnWriteArrayList<>()).getBlockBlobAsyncClient(); + + BlockBlobSimpleUploadOptions options + = new BlockBlobSimpleUploadOptions(DATA.getDefaultBinaryData()).setContentValidationAlgorithm(algorithm) + .setContentMd5(DEFAULT_MD5); + + StepVerifier.create(client.uploadWithResponse(options)).verifyErrorSatisfies(ex -> { + BlobStorageException e = assertInstanceOf(BlobStorageException.class, ex); + assertEquals(400, e.getStatusCode()); + assertTrue(e.getMessage().contains(MESSAGE)); + }); + } + + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void stageBlockWithCustomerProvidedMd5AndCrc64Header(ContentValidationAlgorithm algorithm) { + BlockBlobAsyncClient client + = createBlobAsyncClientWithRequestSniffer(new CopyOnWriteArrayList<>()).getBlockBlobAsyncClient(); + + BlockBlobStageBlockOptions options = new BlockBlobStageBlockOptions(getBlockID(), DATA.getDefaultBinaryData()) + .setContentValidationAlgorithm(algorithm) + .setContentMd5(DEFAULT_MD5); + + StepVerifier.create(client.stageBlockWithResponse(options)).verifyErrorSatisfies(ex -> { + BlobStorageException e = assertInstanceOf(BlobStorageException.class, ex); + assertEquals(400, e.getStatusCode()); + assertTrue(e.getMessage().contains(MESSAGE)); + }); + } + + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void appendBlockWithCustomerProvidedMd5AndCrc64Header(ContentValidationAlgorithm algorithm) { + AppendBlobAsyncClient client + = createBlobAsyncClientWithRequestSniffer(new CopyOnWriteArrayList<>()).getAppendBlobAsyncClient(); + client.create().block(); + + byte[] randomData = DATA.getDefaultBytes(); + AppendBlobAppendBlockOptions options + = new AppendBlobAppendBlockOptions(Flux.just(ByteBuffer.wrap(randomData)), randomData.length) + .setContentValidationAlgorithm(algorithm) + .setContentMd5(DEFAULT_MD5); + + StepVerifier.create(client.appendBlockWithResponse(options)).verifyErrorSatisfies(ex -> { + BlobStorageException e = assertInstanceOf(BlobStorageException.class, ex); + assertEquals(400, e.getStatusCode()); + assertTrue(e.getMessage().contains(MESSAGE)); + }); + } + + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void uploadPagesWithCustomerProvidedMd5AndCrc64Header(ContentValidationAlgorithm algorithm) + throws NoSuchAlgorithmException { + PageBlobAsyncClient client + = createBlobAsyncClientWithRequestSniffer(new CopyOnWriteArrayList<>()).getPageBlobAsyncClient(); + client.create(UNDER_4MB_PAGE_ALIGNED).block(); + + byte[] randomData = getRandomByteArray(UNDER_4MB_PAGE_ALIGNED); + byte[] md5 = MessageDigest.getInstance("MD5").digest(randomData); + PageBlobUploadPagesOptions options + = new PageBlobUploadPagesOptions(new PageRange().setStart(0).setEnd(UNDER_4MB_PAGE_ALIGNED - 1), + Flux.just(ByteBuffer.wrap(randomData))).setContentValidationAlgorithm(algorithm).setContentMd5(md5); + + StepVerifier.create(client.uploadPagesWithResponse(options)).verifyErrorSatisfies(ex -> { + BlobStorageException e = assertInstanceOf(BlobStorageException.class, ex); + assertEquals(400, e.getStatusCode()); + assertTrue(e.getMessage().contains(MESSAGE)); + }); + } +} diff --git a/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobContentValidationDownloadTests.java b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobContentValidationDownloadTests.java new file mode 100644 index 000000000000..ef710cf69054 --- /dev/null +++ b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobContentValidationDownloadTests.java @@ -0,0 +1,658 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.blob; + +import com.azure.core.http.HttpHeaders; +import com.azure.core.http.rest.Response; +import com.azure.core.http.policy.HttpPipelinePolicy; +import com.azure.core.test.utils.TestUtils; +import com.azure.core.util.BinaryData; +import com.azure.core.util.Context; +import com.azure.core.util.ProgressListener; +import com.azure.storage.blob.models.BlobDownloadContentResponse; +import com.azure.storage.blob.models.BlobDownloadResponse; +import com.azure.storage.blob.models.BlobProperties; +import com.azure.storage.blob.models.BlobSeekableByteChannelReadResult; +import com.azure.storage.blob.models.BlobRange; +import com.azure.storage.blob.models.DownloadRetryOptions; +import com.azure.storage.blob.options.BlobDownloadContentOptions; +import com.azure.storage.blob.options.BlobDownloadStreamOptions; +import com.azure.storage.blob.options.BlobDownloadToFileOptions; +import com.azure.storage.blob.options.BlobInputStreamOptions; +import com.azure.storage.blob.options.BlobUploadFromFileOptions; +import com.azure.storage.blob.options.BlobSeekableByteChannelReadOptions; +import com.azure.storage.blob.specialized.BlobInputStream; +import com.azure.storage.common.ParallelTransferOptions; +import com.azure.storage.common.ContentValidationAlgorithm; +import com.azure.storage.common.implementation.Constants; +import com.azure.storage.common.test.shared.extensions.LiveOnly; +import com.azure.storage.common.test.shared.policy.MockPartialResponsePolicy; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.EnumSource; +import org.junit.jupiter.params.provider.MethodSource; +import org.junit.jupiter.params.provider.ValueSource; + +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.channels.SeekableByteChannel; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.atomic.AtomicLong; +import java.util.stream.Stream; + +import static com.azure.storage.blob.specialized.BlobSeekableByteChannelTests.copy; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Sync tests for structured message decoding during blob downloads using StorageContentValidationDecoderPolicy. + * These tests verify that the pipeline policy correctly decodes structured messages when content validation is enabled. + */ +public class BlobContentValidationDownloadTests extends BlobTestBase { + private static final int TEN_MB = 10 * Constants.MB; + private static final int BLOCK_SIZE = 4 * Constants.MB; + /** + * {@link BlobTestBase#fuzzyParallelDownloadLargeMultiPartCases()} starts at ~96 MiB; above this threshold fuzzy + * parallel download helpers use temp files + {@link BlobTestBase#compareFiles(File, File, long, long)} so the full + * payload never lives twice in heap. + */ + private static final int FUZZY_PARALLEL_DOWNLOAD_FILE_ROUND_TRIP_THRESHOLD_BYTES = 96 * Constants.MB; + + /** + * Live-only random payload band for the dedicated random-size parallel-download fuzzy test + * ({@link #fuzzyParallelDownloadLiveRandomRoundTrip(ContentValidationAlgorithm)}): each run draws a per-run + * payload size in {@code (256 MiB, 500 MiB]} (matches the encoder fuzzy upload range) so the structured-message + * decoder is exercised against payloads whose size varies per run in addition to the random byte contents. + */ + private static final long LIVE_RANDOM_PARALLEL_DOWNLOAD_PAYLOAD_MIN_BYTES_EXCLUSIVE = 256L * Constants.MB; + private static final long LIVE_RANDOM_PARALLEL_DOWNLOAD_PAYLOAD_MAX_BYTES_INCLUSIVE = 500L * Constants.MB; + + private final List createdFiles = new ArrayList<>(); + + private File createRandomFile(Path tempDir, int size) throws IOException { + File file = Files.createTempFile(tempDir, "blob-cv-source", ".bin").toFile(); + + if (size > Constants.MB) { + try (OutputStream outputStream = Files.newOutputStream(file.toPath())) { + byte[] data = getRandomByteArray(Constants.MB); + int mbChunks = size / Constants.MB; + int remaining = size % Constants.MB; + for (int i = 0; i < mbChunks; i++) { + outputStream.write(data); + } + if (remaining > 0) { + outputStream.write(data, 0, remaining); + } + } + } else { + Files.write(file.toPath(), getRandomByteArray(size)); + } + + return file; + } + + /** + * downloadStreamWithResponse with CRC64 content validation. + */ + @Test + public void downloadStreamWithResponseContentValidation() { + byte[] data = getRandomByteArray(TEN_MB); + List recordedRequestHeaders = new CopyOnWriteArrayList<>(); + BlobClient blobClient = createBlobClientWithRequestSniffer(recordedRequestHeaders); + blobClient.upload(BinaryData.fromBytes(data)); + + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + BlobDownloadResponse response = blobClient.downloadStreamWithResponse(outputStream, + new BlobDownloadStreamOptions().setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64), null, + Context.NONE); + + assertTrue(hasStructuredMessageDownloadResponseHeaders(response.getHeaders())); + TestUtils.assertArraysEqual(data, outputStream.toByteArray()); + assertTrue(hasStructuredMessageDownloadRequestHeaders(recordedRequestHeaders, false)); + } + + /** + * downloadContentWithResponse with CRC64 content validation. + */ + @Test + public void downloadContentWithResponseContentValidation() { + byte[] data = getRandomByteArray(TEN_MB); + List recordedRequestHeaders = new CopyOnWriteArrayList<>(); + BlobClient blobClient = createBlobClientWithRequestSniffer(recordedRequestHeaders); + blobClient.upload(BinaryData.fromBytes(data)); + + BlobDownloadContentResponse response = blobClient.downloadContentWithResponse( + new BlobDownloadContentOptions().setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64), null, + Context.NONE); + byte[] result = response.getValue().toBytes(); + + assertTrue(hasStructuredMessageDownloadResponseHeaders(response.getHeaders())); + TestUtils.assertArraysEqual(data, result); + assertTrue(hasStructuredMessageDownloadRequestHeaders(recordedRequestHeaders, false)); + } + + /** + * downloadToFileWithResponse with CRC64 content validation (parallel, multiple block sizes). + */ + @ParameterizedTest + @ValueSource( + ints = { + 0, // empty file + 20, // small file + 16 * 1024 * 1024, // medium file in several chunks + 8 * 1026 * 1024 + 10, // medium file not aligned to block + }) + public void downloadToFileWithResponseContentValidation(int fileSize, @TempDir Path tempDir) throws IOException { + File file = createRandomFile(tempDir, fileSize); + File outFile = tempDir.resolve("download.bin").toFile(); + List recordedRequestHeaders = new CopyOnWriteArrayList<>(); + BlobClient blobClient = createBlobClientWithRequestSniffer(recordedRequestHeaders); + blobClient.uploadFromFile(file.toPath().toString(), true); + + Files.deleteIfExists(outFile.toPath()); + + ParallelTransferOptions parallelOptions = new ParallelTransferOptions().setBlockSizeLong((long) BLOCK_SIZE); + BlobDownloadToFileOptions options + = new BlobDownloadToFileOptions(outFile.toPath().toString()).setParallelTransferOptions(parallelOptions) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + + Response response = blobClient.downloadToFileWithResponse(options, null, Context.NONE); + assertTrue(hasStructuredMessageDownloadResponseHeaders(response.getHeaders())); + assertNotNull(response.getValue()); + assertTrue(compareFiles(file, outFile, 0, fileSize)); + assertTrue(hasStructuredMessageDownloadRequestHeaders(recordedRequestHeaders, false)); + } + + /** + * downloadToFileWithResponse with CRC64 content validation (parallel, multiple block sizes). + */ + @LiveOnly + @ParameterizedTest + @ValueSource( + ints = { + 50 * Constants.MB, //large file requiring multiple requests + 50 * Constants.MB + 22 // large file not on MB boundary + }) + public void downloadToFileLargeWithResponseContentValidation(int fileSize, @TempDir Path tempDir) + throws IOException { + File file = createRandomFile(tempDir, fileSize); + File outFile = tempDir.resolve("download.bin").toFile(); + List recordedRequestHeaders = new CopyOnWriteArrayList<>(); + BlobClient blobClient = createBlobClientWithRequestSniffer(recordedRequestHeaders); + blobClient.uploadFromFile(file.toPath().toString(), true); + + Files.deleteIfExists(outFile.toPath()); + + ParallelTransferOptions parallelOptions = new ParallelTransferOptions().setBlockSizeLong((long) BLOCK_SIZE); + BlobDownloadToFileOptions options + = new BlobDownloadToFileOptions(outFile.toPath().toString()).setParallelTransferOptions(parallelOptions) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + + Response response = blobClient.downloadToFileWithResponse(options, null, Context.NONE); + assertTrue(hasStructuredMessageDownloadResponseHeaders(response.getHeaders())); + assertNotNull(response.getValue()); + assertTrue(compareFiles(file, outFile, 0, fileSize)); + assertTrue(hasStructuredMessageDownloadRequestHeaders(recordedRequestHeaders, false)); + } + + /** + * Default behavior: when no algorithm is specified, default is NONE (no validation). + */ + @Test + public void downloadStreamDefaultAlgorithmIsNone() { + byte[] data = getRandomByteArray(TEN_MB); + List recordedRequestHeaders = new CopyOnWriteArrayList<>(); + BlobClient blobClient = createBlobClientWithRequestSniffer(recordedRequestHeaders); + blobClient.upload(BinaryData.fromBytes(data)); + + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + blobClient.downloadStreamWithResponse(outputStream, new BlobDownloadStreamOptions(), null, Context.NONE); + + TestUtils.assertArraysEqual(data, outputStream.toByteArray()); + assertFalse(hasStructuredMessageDownloadRequestHeaders(recordedRequestHeaders, false)); + } + + /** + * AUTO on downloadStream resolves to CRC64 behavior. + */ + @Test + public void downloadStreamWithAuto() { + byte[] data = getRandomByteArray(TEN_MB); + List recordedRequestHeaders = new CopyOnWriteArrayList<>(); + BlobClient blobClient = createBlobClientWithRequestSniffer(recordedRequestHeaders); + blobClient.upload(BinaryData.fromBytes(data)); + + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + BlobDownloadStreamOptions options + = new BlobDownloadStreamOptions().setContentValidationAlgorithm(ContentValidationAlgorithm.AUTO); + BlobDownloadResponse response + = blobClient.downloadStreamWithResponse(outputStream, options, null, Context.NONE); + + assertTrue(hasStructuredMessageDownloadResponseHeaders(response.getHeaders())); + TestUtils.assertArraysEqual(data, outputStream.toByteArray()); + assertTrue(hasStructuredMessageDownloadRequestHeaders(recordedRequestHeaders, false)); + } + + /** + * downloadContentWithResponse with NONE: no validation triggered. + */ + @Test + public void downloadContentWithNone() { + byte[] data = getRandomByteArray(TEN_MB); + List recordedRequestHeaders = new CopyOnWriteArrayList<>(); + BlobClient blobClient = createBlobClientWithRequestSniffer(recordedRequestHeaders); + blobClient.upload(BinaryData.fromBytes(data)); + + byte[] result + = blobClient + .downloadContentWithResponse( + new BlobDownloadContentOptions().setContentValidationAlgorithm(ContentValidationAlgorithm.NONE), + null, Context.NONE) + .getValue() + .toBytes(); + + TestUtils.assertArraysEqual(data, result); + assertFalse(hasStructuredMessageDownloadRequestHeaders(recordedRequestHeaders, false)); + } + + /** + * downloadContentWithResponse with AUTO resolves to CRC64 behavior. + */ + @Test + public void downloadContentWithAuto() { + byte[] data = getRandomByteArray(TEN_MB); + List recordedRequestHeaders = new CopyOnWriteArrayList<>(); + BlobClient blobClient = createBlobClientWithRequestSniffer(recordedRequestHeaders); + blobClient.upload(BinaryData.fromBytes(data)); + + BlobDownloadContentResponse response = blobClient.downloadContentWithResponse( + new BlobDownloadContentOptions().setContentValidationAlgorithm(ContentValidationAlgorithm.AUTO), null, + Context.NONE); + byte[] result = response.getValue().toBytes(); + + assertTrue(hasStructuredMessageDownloadResponseHeaders(response.getHeaders())); + TestUtils.assertArraysEqual(data, result); + assertTrue(hasStructuredMessageDownloadRequestHeaders(recordedRequestHeaders, false)); + } + + /** + * Interrupt with proper rewind to segment boundary; verifies retry range headers. + */ + @Test + public void interruptAndVerifyProperRewind() { + final int segmentSize = Constants.KB; + byte[] data = getRandomByteArray(2 * segmentSize); + List recordedRequestHeaders = new CopyOnWriteArrayList<>(); + HttpHeaders recordedResponseHeaders = new HttpHeaders(); + BlobClient blobClient = createBlobClientWithRequestSniffer(recordedRequestHeaders); + + blobClient.upload(BinaryData.fromBytes(data)); + + int interruptPos = segmentSize + (2 * (segmentSize / 4)) + 10; + MockPartialResponsePolicy mockPolicy = new MockPartialResponsePolicy(1, interruptPos, blobClient.getBlobUrl()); + HttpPipelinePolicy sniffPolicy = getRequestAndResponseHeaderSniffer(blobClient.getBlobUrl(), + recordedRequestHeaders, recordedResponseHeaders); + + BlobClient downloadClient = getBlobClient(ENVIRONMENT.getPrimaryAccount().getCredential(), + blobClient.getBlobUrl(), sniffPolicy, mockPolicy); + DownloadRetryOptions retryOptions = new DownloadRetryOptions().setMaxRetryRequests(5); + + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + BlobDownloadStreamOptions options = new BlobDownloadStreamOptions().setDownloadRetryOptions(retryOptions) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + BlobDownloadResponse response + = downloadClient.downloadStreamWithResponse(outputStream, options, null, Context.NONE); + + assertTrue(hasStructuredMessageDownloadResponseHeaders(response.getHeaders())); + TestUtils.assertArraysEqual(data, outputStream.toByteArray()); + assertEquals(0, mockPolicy.getTriesRemaining(), "Expected the configured interruption to be consumed"); + assertTrue(mockPolicy.getRangeHeaders().size() >= 2, + "Expected at least the initial request and one retry with a range header"); + assertTrue(hasStructuredMessageDownloadResponseHeaders(recordedResponseHeaders)); + assertTrue(hasStructuredMessageDownloadRequestHeaders(recordedRequestHeaders, false)); + } + + /** + * Proper decode across retries (single and multiple interrupts). + */ + @ParameterizedTest + @ValueSource(booleans = { false, true }) + public void interruptAndVerifyProperDecode(boolean multipleInterrupts) { + final int segmentSize = 128 * Constants.KB; + final int dataSize = 4 * Constants.KB; + byte[] data = getRandomByteArray(dataSize); + List recordedRequestHeaders = new CopyOnWriteArrayList<>(); + HttpHeaders recordedResponseHeaders = new HttpHeaders(); + BlobClient blobClient = createBlobClientWithRequestSniffer(recordedRequestHeaders); + + blobClient.upload(BinaryData.fromBytes(data)); + + int interruptPos = segmentSize + (3 * (8 * Constants.KB)) + 10; + MockPartialResponsePolicy mockPolicy + = new MockPartialResponsePolicy(multipleInterrupts ? 2 : 1, interruptPos, blobClient.getBlobUrl()); + HttpPipelinePolicy sniffPolicy = getRequestAndResponseHeaderSniffer(blobClient.getBlobUrl(), + recordedRequestHeaders, recordedResponseHeaders); + + BlobClient downloadClient = getBlobClient(ENVIRONMENT.getPrimaryAccount().getCredential(), + blobClient.getBlobUrl(), sniffPolicy, mockPolicy); + DownloadRetryOptions retryOptions = new DownloadRetryOptions().setMaxRetryRequests(10); + + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + BlobDownloadStreamOptions options = new BlobDownloadStreamOptions().setDownloadRetryOptions(retryOptions) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + BlobDownloadResponse response + = downloadClient.downloadStreamWithResponse(outputStream, options, null, Context.NONE); + + assertTrue(hasStructuredMessageDownloadResponseHeaders(response.getHeaders())); + byte[] result = outputStream.toByteArray(); + assertEquals(dataSize, result.length, "Decoded data should have exactly " + dataSize + " bytes"); + TestUtils.assertArraysEqual(data, result); + assertTrue(hasStructuredMessageDownloadResponseHeaders(recordedResponseHeaders)); + assertTrue(hasStructuredMessageDownloadRequestHeaders(recordedRequestHeaders, false)); + } + + // Only run this test in live mode as BlobOutputStream dynamically assigns blocks + @LiveOnly + @Test + public void openInputStreamContentValidation() { + byte[] data = getRandomByteArray(TEN_MB); + List recordedRequestHeaders = new CopyOnWriteArrayList<>(); + BlobClient blobClient = createBlobClientWithRequestSniffer(recordedRequestHeaders); + blobClient.upload(BinaryData.fromBytes(data)); + + BlobInputStreamOptions options + = new BlobInputStreamOptions().setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + BlobInputStream inputStream = blobClient.openInputStream(options, Context.NONE); + + TestUtils.assertArraysEqual(data, convertInputStreamToByteArray(inputStream)); + assertTrue(hasStructuredMessageDownloadRequestHeaders(recordedRequestHeaders, false)); + } + + // Only run this test in live mode as BlobOutputStream dynamically assigns blocks + @LiveOnly + @Test + public void openInputStreamRangeContentValidation() { + byte[] data = getRandomByteArray(TEN_MB); + List recordedRequestHeaders = new CopyOnWriteArrayList<>(); + BlobClient blobClient = createBlobClientWithRequestSniffer(recordedRequestHeaders); + + int start = Constants.MB; + int count = 3 * Constants.MB + 257; + + blobClient.upload(BinaryData.fromBytes(data)); + + BlobInputStreamOptions options = new BlobInputStreamOptions().setRange(new BlobRange(start, (long) count)) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64) + .setBlockSize(Constants.MB); + BlobInputStream inputStream = blobClient.openInputStream(options, Context.NONE); + + byte[] downloadedRange = convertInputStreamToByteArray(inputStream); + assertEquals(count, downloadedRange.length); + TestUtils.assertArraysEqual(data, start, downloadedRange, 0, count); + assertTrue(hasStructuredMessageDownloadRequestHeaders(recordedRequestHeaders, false)); + } + + /** + * openSeekableByteChannelRead with CRC64 content validation. + */ + @ParameterizedTest + @MethodSource("channelReadDataSupplier") + public void openSeekableByteChannelReadContentValidation(Integer streamBufferSize, int copyBufferSize, + int dataLength) throws IOException { + byte[] data = getRandomByteArray(dataLength); + List recordedRequestHeaders = new CopyOnWriteArrayList<>(); + BlobClient blobClient = createBlobClientWithRequestSniffer(recordedRequestHeaders); + + blobClient.upload(BinaryData.fromBytes(data)); + + // when: "Channel initialized" + BlobSeekableByteChannelReadOptions options + = new BlobSeekableByteChannelReadOptions().setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64) + .setReadSizeInBytes(streamBufferSize); + BlobSeekableByteChannelReadResult result = blobClient.openSeekableByteChannelRead(options, Context.NONE); + SeekableByteChannel channel = result.getChannel(); + + // then: "Channel initialized to position zero" + assertEquals(0, channel.position()); + assertNotNull(result.getProperties()); + assertEquals(data.length, result.getProperties().getBlobSize()); + + // when: "read from channel" + ByteArrayOutputStream downloadedData = new ByteArrayOutputStream(); + int copied = copy(channel, downloadedData, copyBufferSize); + + // then: "channel position updated accordingly" + assertEquals(dataLength, copied); + assertEquals(dataLength, channel.position()); + + // and: "expected data downloaded" + TestUtils.assertArraysEqual(data, downloadedData.toByteArray()); + assertTrue(hasStructuredMessageDownloadRequestHeaders(recordedRequestHeaders, false)); + } + + @Test + public void verifyProgressListenerIsCompatibleWithContentValidation(@TempDir Path tempDir) throws IOException { + byte[] data = getRandomByteArray(10 * Constants.MB); + + BlobClient client = cc.getBlobClient(generateBlobName()); + client.upload(BinaryData.fromBytes(data)); + + MockProgressListener mockListenerWithContentVal = new MockProgressListener(); + MockProgressListener mockListenerWithoutContentVal = new MockProgressListener(); + + ParallelTransferOptions parallelOptionsWithContentVal + = new ParallelTransferOptions().setProgressListener(mockListenerWithContentVal); + ParallelTransferOptions parallelOptionsWithoutContentVal + = new ParallelTransferOptions().setProgressListener(mockListenerWithoutContentVal); + + File fileWithContentVal = createRandomFile(tempDir, 10 * Constants.MB); + File outFileWithContentVal = tempDir.resolve("withcontentval.bin").toFile(); + File fileWithoutContentVal = createRandomFile(tempDir, 10 * Constants.MB); + File outFileWithoutContentVal = tempDir.resolve("withoutcontentval.bin").toFile(); + + Files.deleteIfExists(outFileWithContentVal.toPath()); + Files.deleteIfExists(outFileWithoutContentVal.toPath()); + + BlobDownloadToFileOptions optionsWithContentVal + = new BlobDownloadToFileOptions(outFileWithContentVal.getAbsolutePath()) + .setParallelTransferOptions(parallelOptionsWithContentVal) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + BlobDownloadToFileOptions optionsWithoutContentVal + = new BlobDownloadToFileOptions(outFileWithoutContentVal.getAbsolutePath()) + .setParallelTransferOptions(parallelOptionsWithoutContentVal); + + client.downloadToFileWithResponse(optionsWithContentVal, null, Context.NONE); + client.downloadToFileWithResponse(optionsWithoutContentVal, null, Context.NONE); + + long expectedBytes = data.length; + assertEquals(expectedBytes, mockListenerWithContentVal.getReportedByteCount()); + assertEquals(expectedBytes, mockListenerWithoutContentVal.getReportedByteCount()); + } + + private static final class MockProgressListener implements ProgressListener { + private final AtomicLong reportedByteCount = new AtomicLong(0L); + + @Override + public void handleProgress(long bytesTransferred) { + this.reportedByteCount.updateAndGet(current -> Math.max(current, bytesTransferred)); + } + + long getReportedByteCount() { + return this.reportedByteCount.get(); + } + } + + // ---------- Fuzzy parallel download (deterministic grids) ---------- + + @ParameterizedTest + @MethodSource("com.azure.storage.blob.BlobTestBase#fuzzyParallelDownloadReplayableCases") + public void fuzzyParallelDownloadReplayableRoundTrip(int payloadBytes, long blockSizeBytes, int maxConcurrency) + throws IOException { + assertParallelDownloadFuzzyRoundTrip("replayable", payloadBytes, blockSizeBytes, maxConcurrency); + } + + @LiveOnly // payload > blockSize with tiny totals; many small range GETs not replayable under the proxy. + @ParameterizedTest + @MethodSource("com.azure.storage.blob.BlobTestBase#fuzzyParallelDownloadSmallMultiPartCases") + public void fuzzyParallelDownloadSmallMultiPartRoundTrip(int payloadBytes, long blockSizeBytes, int maxConcurrency) + throws IOException { + assertParallelDownloadFuzzyRoundTrip("smallMultiPart", payloadBytes, blockSizeBytes, maxConcurrency); + } + + @LiveOnly // sub-4 MiB chunked range GETs not replayable under the proxy. + @ParameterizedTest + @MethodSource("com.azure.storage.blob.BlobTestBase#fuzzyParallelDownloadSubFourMiBCases") + public void fuzzyParallelDownloadSubFourMiBRoundTrip(int payloadBytes, long blockSizeBytes, int maxConcurrency) + throws IOException { + assertParallelDownloadFuzzyRoundTrip("subFourMiB", payloadBytes, blockSizeBytes, maxConcurrency); + } + + @LiveOnly // 4 MiB boundary tuples that fan out into chunked range GETs. + @ParameterizedTest + @MethodSource("com.azure.storage.blob.BlobTestBase#fuzzyParallelDownloadFourMiBBoundaryCases") + public void fuzzyParallelDownloadFourMiBBoundaryRoundTrip(int payloadBytes, long blockSizeBytes, int maxConcurrency) + throws IOException { + assertParallelDownloadFuzzyRoundTrip("fourMiBBoundary", payloadBytes, blockSizeBytes, maxConcurrency); + } + + @LiveOnly // payload > blockSize for every tuple; chunked range GETs across many requests. + @ParameterizedTest + @MethodSource("com.azure.storage.blob.BlobTestBase#fuzzyParallelDownloadMediumMultiPartCases") + public void fuzzyParallelDownloadMediumMultiPartRoundTrip(int payloadBytes, long blockSizeBytes, int maxConcurrency) + throws IOException { + assertParallelDownloadFuzzyRoundTrip("mediumMultiPart", payloadBytes, blockSizeBytes, maxConcurrency); + } + + @LiveOnly // payload >> blockSize; ~96-320 MiB downloads. + @ParameterizedTest + @MethodSource("com.azure.storage.blob.BlobTestBase#fuzzyParallelDownloadLargeMultiPartCases") + public void fuzzyParallelDownloadLargeMultiPartRoundTrip(int payloadBytes, long blockSizeBytes, int maxConcurrency) + throws IOException { + assertParallelDownloadFuzzyRoundTrip("largeMultiPart", payloadBytes, blockSizeBytes, maxConcurrency); + } + + @LiveOnly // ~1 GiB single case; far too large for the test proxy. + @ParameterizedTest + @MethodSource("com.azure.storage.blob.BlobTestBase#fuzzyParallelDownloadOneGiBCases") + public void fuzzyParallelDownloadOneGiBRoundTrip(int payloadBytes, long blockSizeBytes, int maxConcurrency) + throws IOException { + assertParallelDownloadFuzzyRoundTrip("oneGiB", payloadBytes, blockSizeBytes, maxConcurrency); + } + + /** + * Live-only random-size parallel download fuzzy round-trip. Each run draws a per-run payload size in + * {@code (256 MiB, 500 MiB]} (matches the encoder fuzzy upload range) and exercises both CRC64 and AUTO + * content-validation algorithms so the structured-message decoder is tested against payloads whose total size + * varies per run in addition to the random byte contents that the deterministic grids already exercise. Kept + * separate from the parameterized {@link #fuzzyParallelDownloadLargeMultiPartRoundTrip(int, long, int)} so the + * deterministic per-grid round-trips and the randomized round-trip don't share work or cost. + */ + @LiveOnly + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void fuzzyParallelDownloadLiveRandomRoundTrip(ContentValidationAlgorithm algorithm) throws IOException { + int sizeBytes = (int) randomLongFromNamer(LIVE_RANDOM_PARALLEL_DOWNLOAD_PAYLOAD_MIN_BYTES_EXCLUSIVE + 1, + LIVE_RANDOM_PARALLEL_DOWNLOAD_PAYLOAD_MAX_BYTES_INCLUSIVE + 1); + assertParallelDownloadFuzzyRoundTrip("liveRandom", sizeBytes, 8L * Constants.MB, 8, algorithm); + } + + private void assertParallelDownloadFuzzyRoundTrip(String caseKind, int payloadBytes, long blockSizeBytes, + int maxConcurrency) throws IOException { + assertParallelDownloadFuzzyRoundTrip(caseKind, payloadBytes, blockSizeBytes, maxConcurrency, + ContentValidationAlgorithm.CRC64); + } + + private void assertParallelDownloadFuzzyRoundTrip(String caseKind, int payloadBytes, long blockSizeBytes, + int maxConcurrency, ContentValidationAlgorithm algorithm) throws IOException { + List recorded = new CopyOnWriteArrayList<>(); + BlobClient client = createBlobClientWithRequestSniffer(recorded); + + ParallelTransferOptions parallelOptions + = new ParallelTransferOptions().setBlockSizeLong(blockSizeBytes).setMaxConcurrency(maxConcurrency); + + String assertionMessage = "Fuzzy parallel download [" + caseKind + "] payloadBytes=" + payloadBytes + + ", blockSize=" + blockSizeBytes + ", maxConcurrency=" + maxConcurrency + ", algorithm=" + algorithm; + + if (payloadBytes >= FUZZY_PARALLEL_DOWNLOAD_FILE_ROUND_TRIP_THRESHOLD_BYTES) { + File sourceFile = getRandomFile(payloadBytes); + sourceFile.deleteOnExit(); + createdFiles.add(sourceFile); + File outFile = Files.createTempFile("blob-cv-fuzzy-parallel-dl", ".bin").toFile(); + outFile.deleteOnExit(); + createdFiles.add(outFile); + Files.deleteIfExists(outFile.toPath()); + + BlobUploadFromFileOptions uploadOptions + = new BlobUploadFromFileOptions(sourceFile.getAbsolutePath()).setParallelTransferOptions( + new com.azure.storage.blob.models.ParallelTransferOptions().setBlockSizeLong(blockSizeBytes) + .setMaxConcurrency(maxConcurrency)); + assertNotNull(client.uploadFromFileWithResponse(uploadOptions, null, Context.NONE).getValue().getETag(), + assertionMessage); + + BlobDownloadToFileOptions downloadOptions + = new BlobDownloadToFileOptions(outFile.toPath().toString()).setParallelTransferOptions(parallelOptions) + .setContentValidationAlgorithm(algorithm); + assertNotNull(client.downloadToFileWithResponse(downloadOptions, null, Context.NONE).getValue(), + assertionMessage); + + assertTrue(compareFiles(sourceFile, outFile, 0, payloadBytes), assertionMessage); + } else { + byte[] randomData = getRandomByteArray(payloadBytes); + client.upload(BinaryData.fromBytes(randomData), true); + + if (payloadBytes > blockSizeBytes) { + File outFile = Files.createTempFile("blob-cv-fuzzy-parallel-dl-mp", ".bin").toFile(); + outFile.deleteOnExit(); + createdFiles.add(outFile); + Files.deleteIfExists(outFile.toPath()); + + BlobDownloadToFileOptions downloadOptions = new BlobDownloadToFileOptions(outFile.toPath().toString()) + .setParallelTransferOptions(parallelOptions) + .setContentValidationAlgorithm(algorithm); + assertNotNull(client.downloadToFileWithResponse(downloadOptions, null, Context.NONE).getValue(), + assertionMessage); + + byte[] downloaded = readAllBytesFromFile(outFile); + assertArrayEquals(randomData, downloaded, assertionMessage); + } else { + BlobDownloadContentOptions downloadOptions + = new BlobDownloadContentOptions().setContentValidationAlgorithm(algorithm); + byte[] downloaded + = client.downloadContentWithResponse(downloadOptions, null, Context.NONE).getValue().toBytes(); + assertArrayEquals(randomData, downloaded, assertionMessage); + } + } + assertTrue(hasStructuredMessageDownloadRequestHeaders(recorded, false), assertionMessage); + } + + private static byte[] readAllBytesFromFile(File file) throws IOException { + try (InputStream is = Files.newInputStream(file.toPath())) { + byte[] buffer = new byte[(int) file.length()]; + int offset = 0; + int read; + while (offset < buffer.length && (read = is.read(buffer, offset, buffer.length - offset)) != -1) { + offset += read; + } + return buffer; + } + } + + static Stream channelReadDataSupplier() { + return Stream.of(Arguments.of(50, 40, Constants.KB), Arguments.of(Constants.KB + 50, 40, Constants.KB), + Arguments.of(null, Constants.MB, TEN_MB)); + } + +} diff --git a/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobContentValidationUploadTests.java b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobContentValidationUploadTests.java new file mode 100644 index 000000000000..f038d0dad772 --- /dev/null +++ b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobContentValidationUploadTests.java @@ -0,0 +1,1406 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.blob; + +import com.azure.core.http.HttpHeaders; +import com.azure.core.util.BinaryData; +import com.azure.core.util.Context; +import com.azure.storage.blob.models.BlobRequestConditions; +import com.azure.storage.blob.models.BlobStorageException; +import com.azure.storage.blob.models.PageRange; +import com.azure.storage.blob.models.ParallelTransferOptions; +import com.azure.storage.blob.options.AppendBlobAppendBlockOptions; +import com.azure.storage.blob.options.BlobParallelUploadOptions; +import com.azure.storage.blob.options.AppendBlobOutputStreamOptions; +import com.azure.storage.blob.options.BlobUploadFromFileOptions; +import com.azure.storage.blob.options.BlockBlobOutputStreamOptions; +import com.azure.storage.blob.options.BlockBlobSeekableByteChannelWriteOptions; +import com.azure.storage.blob.options.BlockBlobSimpleUploadOptions; +import com.azure.storage.blob.options.BlockBlobStageBlockOptions; +import com.azure.storage.blob.options.PageBlobOutputStreamOptions; +import com.azure.storage.blob.options.PageBlobUploadPagesOptions; +import com.azure.storage.blob.specialized.AppendBlobClient; +import com.azure.storage.blob.specialized.BlobOutputStream; +import com.azure.storage.blob.specialized.BlockBlobClient; +import com.azure.storage.blob.specialized.PageBlobClient; +import com.azure.storage.common.ContentValidationAlgorithm; +import com.azure.storage.common.implementation.contentvalidation.ContentValidationModeResolver; +import com.azure.storage.common.implementation.Constants; +import com.azure.storage.common.test.shared.extensions.LiveOnly; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; +import org.junit.jupiter.params.provider.MethodSource; + +import java.io.ByteArrayInputStream; +import java.io.EOFException; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.ByteBuffer; +import java.nio.channels.Channels; +import java.nio.file.Files; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.Base64; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Tests content validation (CRC64 / structured message) for upload operations using sync clients. + * Upload types that have no async counterpart (OutputStream, SeekableByteChannel) are tested only here. + * Async counterparts of the same operations are in {@link BlobContentValidationAsyncUploadTests}. + */ +public class BlobContentValidationUploadTests extends BlobTestBase { + private static final int TEN_MB = 10 * Constants.MB; + /* single-shot uploads with length < 4MB use CRC64 header; >= 4MB use structured message. */ + private static final int UNDER_4MB = 2 * Constants.MB; + + /** + * Live-only random payload band (256–500 MiB, inclusive upper bound via {@code randomLongFromNamer}+1) for + * {@code uploadWithResponse}, {@code uploadFromFileWithResponse}, and single-block {@code stageBlock}. + */ + private static final long LIVE_RANDOM_PARALLEL_PAYLOAD_MIN_BYTES_EXCLUSIVE = 256L * Constants.MB; + private static final long LIVE_RANDOM_PARALLEL_PAYLOAD_MAX_BYTES_INCLUSIVE = 500L * Constants.MB; + + /** + * Live-only random payload band for sequential append-block puts only. + * {@code Flux.concatMap} issues one append REST call per chunk in + * order (not parallel staging); use a smaller band than {@link #LIVE_RANDOM_PARALLEL_PAYLOAD_MIN_BYTES_EXCLUSIVE}. + */ + private static final long LIVE_RANDOM_SEQUENTIAL_APPEND_PAYLOAD_MIN_BYTES_EXCLUSIVE = 32L * Constants.MB; + private static final long LIVE_RANDOM_SEQUENTIAL_APPEND_PAYLOAD_MAX_BYTES_INCLUSIVE = 64L * Constants.MB; + + private static final String MD5_AND_CRC64_EXCLUSIVE_MESSAGE + = "Only one form of transactional content validation may be used."; + + // =========================================================================================== + // BlobClient.uploadWithResponse + // =========================================================================================== + + /** + * Single-shot upload under 4MB: content validation uses CRC64 header only (no structured message). + */ + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void uploadWithCrc64Header(ContentValidationAlgorithm algorithm) { + List recorded = new CopyOnWriteArrayList<>(); + BlobClient client = createBlobClientWithRequestSniffer(recorded); + + byte[] randomData = getRandomByteArray(UNDER_4MB); + InputStream data = new ByteArrayInputStream(randomData); + + BlobParallelUploadOptions options = new BlobParallelUploadOptions(data) + .setParallelTransferOptions(new ParallelTransferOptions().setMaxSingleUploadSizeLong((long) UNDER_4MB)) + .setRequestConditions(new BlobRequestConditions()) + .setContentValidationAlgorithm(algorithm); + + assertNotNull(client.uploadWithResponse(options, null, Context.NONE).getValue().getETag()); + assertTrue(hasOnlyCrc64Headers(recorded)); + } + + /** + * Single-shot upload >= 4MB: content validation uses structured message. + */ + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void uploadWithStructuredMessage(ContentValidationAlgorithm algorithm) { + List recorded = new CopyOnWriteArrayList<>(); + BlobClient client = createBlobClientWithRequestSniffer(recorded); + + byte[] randomData = getRandomByteArray(TEN_MB); + InputStream data = new ByteArrayInputStream(randomData); + + BlobParallelUploadOptions options = new BlobParallelUploadOptions(data) + .setParallelTransferOptions(new ParallelTransferOptions().setMaxSingleUploadSizeLong((long) TEN_MB)) + .setRequestConditions(new BlobRequestConditions()) + .setContentValidationAlgorithm(algorithm); + + assertNotNull(client.uploadWithResponse(options, null, Context.NONE).getValue().getETag()); + assertTrue(hasOnlyStructuredMessageHeaders(recorded)); + } + + /** + * Multi-shot (chunked) upload; content validation uses structured message on each stage block. + */ + @LiveOnly // Put Block URLs include random block IDs; not replayable with the test proxy. + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void uploadChunkedWithStructuredMessage(ContentValidationAlgorithm algorithm) { + List recorded = new CopyOnWriteArrayList<>(); + BlobClient client = createBlobClientWithRequestSniffer(recorded); + + byte[] randomData = getRandomByteArray(TEN_MB); + InputStream data = new ByteArrayInputStream(randomData); + long blockSize = 2 * (long) Constants.MB; + + BlobParallelUploadOptions options = new BlobParallelUploadOptions(data) + .setParallelTransferOptions( + new ParallelTransferOptions().setBlockSizeLong(blockSize).setMaxSingleUploadSizeLong(blockSize)) + .setRequestConditions(new BlobRequestConditions()) + .setContentValidationAlgorithm(algorithm); + + assertNotNull(client.uploadWithResponse(options, null, Context.NONE).getValue().getETag()); + assertTrue(hasOnlyStructuredMessageHeaders(recorded)); + } + + @Test + public void uploadWithoutContentValidation() { + List recorded = new CopyOnWriteArrayList<>(); + BlobClient client = createBlobClientWithRequestSniffer(recorded); + + byte[] randomData = getRandomByteArray(TEN_MB); + InputStream data = new ByteArrayInputStream(randomData); + + BlobParallelUploadOptions options = new BlobParallelUploadOptions(data) + .setParallelTransferOptions(new ParallelTransferOptions().setMaxSingleUploadSizeLong((long) TEN_MB)) + .setRequestConditions(new BlobRequestConditions()) + .setContentValidationAlgorithm(ContentValidationAlgorithm.NONE); + + assertNotNull(client.uploadWithResponse(options, null, Context.NONE).getValue().getETag()); + assertTrue(hasNoContentValidationHeaders(recorded)); + } + + /** + * Blob parallel upload rejects using both computeMd5 (SDK-computed MD5) and CRC64 (transfer validation checksum algorithm) at once. + */ + @Test + public void uploadWithComputeMd5AndCrc64Throws() { + BlobClient client = createBlobClientWithRequestSniffer(new CopyOnWriteArrayList<>()); + + byte[] randomData = getRandomByteArray(UNDER_4MB); + InputStream data = new ByteArrayInputStream(randomData); + + BlobParallelUploadOptions options = new BlobParallelUploadOptions(data) + .setParallelTransferOptions(new ParallelTransferOptions().setMaxSingleUploadSizeLong((long) UNDER_4MB)) + .setRequestConditions(new BlobRequestConditions()) + .setComputeMd5(true) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + + IllegalArgumentException ex = assertThrows(IllegalArgumentException.class, + () -> client.uploadWithResponse(options, null, Context.NONE)); + assertTrue(ex.getMessage().contains(MD5_AND_CRC64_EXCLUSIVE_MESSAGE)); + } + + // =========================================================================================== + // BlockBlobClient.uploadWithResponse (BlockBlobSimpleUpload / Put Blob) tests + // =========================================================================================== + + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void blockBlobSimpleUploadWithCrc64Header(ContentValidationAlgorithm algorithm) { + List recorded = new CopyOnWriteArrayList<>(); + BlobClient blobClient = createBlobClientWithRequestSniffer(recorded); + BlockBlobClient client = blobClient.getBlockBlobClient(); + + byte[] randomData = getRandomByteArray(UNDER_4MB); + BinaryData data = BinaryData.fromBytes(randomData); + + BlockBlobSimpleUploadOptions options + = new BlockBlobSimpleUploadOptions(data).setContentValidationAlgorithm(algorithm); + + assertNotNull(client.uploadWithResponse(options, null, Context.NONE).getValue().getETag()); + assertTrue(hasOnlyCrc64Headers(recorded)); + } + + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void blockBlobSimpleUploadWithStructuredMessage(ContentValidationAlgorithm algorithm) { + List recorded = new CopyOnWriteArrayList<>(); + BlobClient blobClient = createBlobClientWithRequestSniffer(recorded); + BlockBlobClient client = blobClient.getBlockBlobClient(); + + byte[] randomData = getRandomByteArray(TEN_MB); + BinaryData data = BinaryData.fromBytes(randomData); + + BlockBlobSimpleUploadOptions options + = new BlockBlobSimpleUploadOptions(data).setContentValidationAlgorithm(algorithm); + + assertNotNull(client.uploadWithResponse(options, null, Context.NONE).getValue().getETag()); + assertTrue(hasOnlyStructuredMessageHeaders(recorded)); + } + + @Test + public void blockBlobSimpleUploadWithNoContentValidation() { + List recorded = new CopyOnWriteArrayList<>(); + BlobClient blobClient = createBlobClientWithRequestSniffer(recorded); + BlockBlobClient client = blobClient.getBlockBlobClient(); + + byte[] randomData = getRandomByteArray(TEN_MB); + BinaryData data = BinaryData.fromBytes(randomData); + + BlockBlobSimpleUploadOptions options + = new BlockBlobSimpleUploadOptions(data).setContentValidationAlgorithm(ContentValidationAlgorithm.NONE); + + assertNotNull(client.uploadWithResponse(options, null, Context.NONE).getValue().getETag()); + assertTrue(hasNoContentValidationHeaders(recorded)); + } + + // =========================================================================================== + // BlockBlobClient.stageBlockWithResponse (Put Block) tests + // =========================================================================================== + + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void stageBlockWithCrc64Header(ContentValidationAlgorithm algorithm) { + List recorded = new CopyOnWriteArrayList<>(); + BlobClient blobClient = createBlobClientWithRequestSniffer(recorded); + BlockBlobClient client = blobClient.getBlockBlobClient(); + + byte[] randomData = getRandomByteArray(UNDER_4MB); + BinaryData data = BinaryData.fromBytes(randomData); + + BlockBlobStageBlockOptions options + = new BlockBlobStageBlockOptions(getBlockID(), data).setContentValidationAlgorithm(algorithm); + + client.stageBlockWithResponse(options, null, Context.NONE); + assertTrue(hasOnlyCrc64Headers(recorded)); + } + + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void stageBlockWithStructuredMessage(ContentValidationAlgorithm algorithm) { + List recorded = new CopyOnWriteArrayList<>(); + BlobClient blobClient = createBlobClientWithRequestSniffer(recorded); + BlockBlobClient client = blobClient.getBlockBlobClient(); + + byte[] randomData = getRandomByteArray(TEN_MB); + BinaryData data = BinaryData.fromBytes(randomData); + + BlockBlobStageBlockOptions options + = new BlockBlobStageBlockOptions(getBlockID(), data).setContentValidationAlgorithm(algorithm); + + client.stageBlockWithResponse(options, null, Context.NONE); + assertTrue(hasOnlyStructuredMessageHeaders(recorded)); + } + + @Test + public void stageBlockWithNoContentValidation() { + List recorded = new CopyOnWriteArrayList<>(); + BlobClient blobClient = createBlobClientWithRequestSniffer(recorded); + BlockBlobClient client = blobClient.getBlockBlobClient(); + + byte[] randomData = getRandomByteArray(TEN_MB); + BinaryData data = BinaryData.fromBytes(randomData); + + BlockBlobStageBlockOptions options = new BlockBlobStageBlockOptions(getBlockID(), data) + .setContentValidationAlgorithm(ContentValidationAlgorithm.NONE); + + client.stageBlockWithResponse(options, null, Context.NONE); + assertTrue(hasNoContentValidationHeaders(recorded)); + } + + // =========================================================================================== + // AppendBlobClient.appendBlockWithResponse (Append Block) tests + // =========================================================================================== + + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void appendBlockWithCrc64Header(ContentValidationAlgorithm algorithm) { + List recorded = new CopyOnWriteArrayList<>(); + BlobClient blobClient = createBlobClientWithRequestSniffer(recorded); + AppendBlobClient client = blobClient.getAppendBlobClient(); + client.create(); + + byte[] randomData = getRandomByteArray(UNDER_4MB); + InputStream data = new ByteArrayInputStream(randomData); + + AppendBlobAppendBlockOptions options + = new AppendBlobAppendBlockOptions(data, UNDER_4MB).setContentValidationAlgorithm(algorithm); + + assertNotNull(client.appendBlockWithResponse(options, null, Context.NONE).getValue().getETag()); + assertTrue(hasOnlyCrc64Headers(recorded)); + } + + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void appendBlockWithStructuredMessage(ContentValidationAlgorithm algorithm) { + List recorded = new CopyOnWriteArrayList<>(); + BlobClient blobClient = createBlobClientWithRequestSniffer(recorded); + AppendBlobClient client = blobClient.getAppendBlobClient(); + client.create(); + + byte[] randomData = getRandomByteArray(TEN_MB); + InputStream data = new ByteArrayInputStream(randomData); + + AppendBlobAppendBlockOptions options + = new AppendBlobAppendBlockOptions(data, TEN_MB).setContentValidationAlgorithm(algorithm); + + assertNotNull(client.appendBlockWithResponse(options, null, Context.NONE).getValue().getETag()); + assertTrue(hasOnlyStructuredMessageHeaders(recorded)); + } + + @Test + public void appendBlockWithNoContentValidation() { + List recorded = new CopyOnWriteArrayList<>(); + BlobClient blobClient = createBlobClientWithRequestSniffer(recorded); + AppendBlobClient client = blobClient.getAppendBlobClient(); + client.create(); + + byte[] randomData = getRandomByteArray(TEN_MB); + InputStream data = new ByteArrayInputStream(randomData); + + AppendBlobAppendBlockOptions options = new AppendBlobAppendBlockOptions(data, TEN_MB) + .setContentValidationAlgorithm(ContentValidationAlgorithm.NONE); + + assertNotNull(client.appendBlockWithResponse(options, null, Context.NONE).getValue().getETag()); + assertTrue(hasNoContentValidationHeaders(recorded)); + } + + // =========================================================================================== + // PageBlobClient.uploadPagesWithResponse (Put Page) tests + // =========================================================================================== + + private static final int PAGE_BYTES = PageBlobClient.PAGE_BYTES; + private static final int UNDER_4MB_PAGE_ALIGNED = (UNDER_4MB / PAGE_BYTES) * PAGE_BYTES; + private static final int FOUR_MB_PAGE_ALIGNED = (4 * Constants.MB / PAGE_BYTES) * PAGE_BYTES; + + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void uploadPagesWithCrc64Header(ContentValidationAlgorithm algorithm) { + List recorded = new CopyOnWriteArrayList<>(); + BlobClient blobClient = createBlobClientWithRequestSniffer(recorded); + PageBlobClient client = blobClient.getPageBlobClient(); + client.create(UNDER_4MB_PAGE_ALIGNED); + + byte[] randomData = getRandomByteArray(UNDER_4MB_PAGE_ALIGNED); + InputStream data = new ByteArrayInputStream(randomData); + + PageBlobUploadPagesOptions options + = new PageBlobUploadPagesOptions(new PageRange().setStart(0).setEnd(UNDER_4MB_PAGE_ALIGNED - 1), data) + .setContentValidationAlgorithm(algorithm); + + assertNotNull(client.uploadPagesWithResponse(options, null, Context.NONE).getValue().getETag()); + assertTrue(hasOnlyCrc64Headers(recorded)); + } + + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void uploadPagesWithStructuredMessage(ContentValidationAlgorithm algorithm) { + List recorded = new CopyOnWriteArrayList<>(); + BlobClient blobClient = createBlobClientWithRequestSniffer(recorded); + PageBlobClient client = blobClient.getPageBlobClient(); + client.create(FOUR_MB_PAGE_ALIGNED); + + byte[] randomData = getRandomByteArray(FOUR_MB_PAGE_ALIGNED); + InputStream data = new ByteArrayInputStream(randomData); + + PageBlobUploadPagesOptions options + = new PageBlobUploadPagesOptions(new PageRange().setStart(0).setEnd(FOUR_MB_PAGE_ALIGNED - 1), data) + .setContentValidationAlgorithm(algorithm); + + assertNotNull(client.uploadPagesWithResponse(options, null, Context.NONE).getValue().getETag()); + assertTrue(hasOnlyStructuredMessageHeaders(recorded)); + } + + @Test + public void uploadPagesWithNoContentValidation() { + List recorded = new CopyOnWriteArrayList<>(); + BlobClient blobClient = createBlobClientWithRequestSniffer(recorded); + PageBlobClient client = blobClient.getPageBlobClient(); + client.create(FOUR_MB_PAGE_ALIGNED); + + byte[] randomData = getRandomByteArray(FOUR_MB_PAGE_ALIGNED); + InputStream data = new ByteArrayInputStream(randomData); + + PageBlobUploadPagesOptions options + = new PageBlobUploadPagesOptions(new PageRange().setStart(0).setEnd(FOUR_MB_PAGE_ALIGNED - 1), data) + .setContentValidationAlgorithm(ContentValidationAlgorithm.NONE); + + assertNotNull(client.uploadPagesWithResponse(options, null, Context.NONE).getValue().getETag()); + assertTrue(hasNoContentValidationHeaders(recorded)); + } + + // =========================================================================================== + // BlobClient.uploadFromFileWithResponse tests + // =========================================================================================== + + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void uploadFromFileWithCrc64Header(ContentValidationAlgorithm algorithm) throws IOException { + List recorded = new CopyOnWriteArrayList<>(); + BlobClient client = createBlobClientWithRequestSniffer(recorded); + + File tempFile = getRandomFile(UNDER_4MB); + + BlobUploadFromFileOptions options = new BlobUploadFromFileOptions(tempFile.getAbsolutePath()) + .setParallelTransferOptions(new ParallelTransferOptions().setMaxSingleUploadSizeLong((long) UNDER_4MB)) + .setContentValidationAlgorithm(algorithm); + + assertNotNull(client.uploadFromFileWithResponse(options, null, Context.NONE).getValue().getETag()); + assertTrue(hasOnlyCrc64Headers(recorded)); + } + + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void uploadFromFileWithStructuredMessage(ContentValidationAlgorithm algorithm) throws IOException { + List recorded = new CopyOnWriteArrayList<>(); + BlobClient client = createBlobClientWithRequestSniffer(recorded); + + File tempFile = getRandomFile(TEN_MB); + + BlobUploadFromFileOptions options = new BlobUploadFromFileOptions(tempFile.getAbsolutePath()) + .setParallelTransferOptions(new ParallelTransferOptions().setMaxSingleUploadSizeLong((long) TEN_MB)) + .setContentValidationAlgorithm(algorithm); + + assertNotNull(client.uploadFromFileWithResponse(options, null, Context.NONE).getValue().getETag()); + assertTrue(hasOnlyStructuredMessageHeaders(recorded)); + } + + @LiveOnly // Put Block URLs include random block IDs; not replayable with the test proxy. + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void uploadFromFileChunkedWithStructuredMessage(ContentValidationAlgorithm algorithm) throws IOException { + List recorded = new CopyOnWriteArrayList<>(); + BlobClient client = createBlobClientWithRequestSniffer(recorded); + + File tempFile = getRandomFile(TEN_MB); + long blockSize = 2 * (long) Constants.MB; + + BlobUploadFromFileOptions options = new BlobUploadFromFileOptions(tempFile.getAbsolutePath()) + .setParallelTransferOptions( + new ParallelTransferOptions().setBlockSizeLong(blockSize).setMaxSingleUploadSizeLong(blockSize)) + .setContentValidationAlgorithm(algorithm); + + assertNotNull(client.uploadFromFileWithResponse(options, null, Context.NONE).getValue().getETag()); + assertTrue(hasOnlyStructuredMessageHeaders(recorded)); + } + + @Test + public void uploadFromFileWithNoContentValidation() throws IOException { + List recorded = new CopyOnWriteArrayList<>(); + BlobClient client = createBlobClientWithRequestSniffer(recorded); + + File tempFile = getRandomFile(TEN_MB); + + BlobUploadFromFileOptions options = new BlobUploadFromFileOptions(tempFile.getAbsolutePath()) + .setParallelTransferOptions(new ParallelTransferOptions().setMaxSingleUploadSizeLong((long) TEN_MB)) + .setContentValidationAlgorithm(ContentValidationAlgorithm.NONE); + + assertNotNull(client.uploadFromFileWithResponse(options, null, Context.NONE).getValue().getETag()); + assertTrue(hasNoContentValidationHeaders(recorded)); + } + + // =========================================================================================== + // Sync BlobOutputStream tests (getBlobOutputStream) + // =========================================================================================== + + // --- AppendBlobClient.getBlobOutputStream --- + + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void appendBlobOutputStreamWithCrc64Header(ContentValidationAlgorithm algorithm) throws Exception { + List recorded = new CopyOnWriteArrayList<>(); + BlobClient blobClient = createBlobClientWithRequestSniffer(recorded); + AppendBlobClient client = blobClient.getAppendBlobClient(); + client.create(); + + byte[] randomData = getRandomByteArray(UNDER_4MB); + + try (BlobOutputStream os = client + .getBlobOutputStream(new AppendBlobOutputStreamOptions().setContentValidationAlgorithm(algorithm))) { + os.write(randomData); + } + + assertTrue(hasOnlyCrc64Headers(recorded)); + } + + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void appendBlobOutputStreamWithStructuredMessage(ContentValidationAlgorithm algorithm) throws Exception { + List recorded = new CopyOnWriteArrayList<>(); + BlobClient blobClient = createBlobClientWithRequestSniffer(recorded); + AppendBlobClient client = blobClient.getAppendBlobClient(); + client.create(); + + byte[] randomData = getRandomByteArray(TEN_MB); + + try (BlobOutputStream os = client + .getBlobOutputStream(new AppendBlobOutputStreamOptions().setContentValidationAlgorithm(algorithm))) { + os.write(randomData); + } + + assertTrue(hasOnlyStructuredMessageHeaders(recorded)); + } + + @Test + public void appendBlobOutputStreamWithNoContentValidation() throws Exception { + List recorded = new CopyOnWriteArrayList<>(); + BlobClient blobClient = createBlobClientWithRequestSniffer(recorded); + AppendBlobClient client = blobClient.getAppendBlobClient(); + client.create(); + + byte[] randomData = getRandomByteArray(TEN_MB); + + try (BlobOutputStream os = client.getBlobOutputStream( + new AppendBlobOutputStreamOptions().setContentValidationAlgorithm(ContentValidationAlgorithm.NONE))) { + os.write(randomData); + } + + assertTrue(hasNoContentValidationHeaders(recorded)); + } + + // --- BlockBlobClient.getBlobOutputStream --- + + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void blockBlobOutputStreamWithCrc64Header(ContentValidationAlgorithm algorithm) throws Exception { + List recorded = new CopyOnWriteArrayList<>(); + BlobClient blobClient = createBlobClientWithRequestSniffer(recorded); + BlockBlobClient client = blobClient.getBlockBlobClient(); + + byte[] randomData = getRandomByteArray(UNDER_4MB); + + try (BlobOutputStream os = client.getBlobOutputStream(new BlockBlobOutputStreamOptions() + .setParallelTransferOptions(new ParallelTransferOptions().setMaxSingleUploadSizeLong((long) UNDER_4MB)) + .setContentValidationAlgorithm(algorithm))) { + os.write(randomData); + } + + assertTrue(hasOnlyCrc64Headers(recorded)); + } + + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void blockBlobOutputStreamWithStructuredMessage(ContentValidationAlgorithm algorithm) throws Exception { + List recorded = new CopyOnWriteArrayList<>(); + BlobClient blobClient = createBlobClientWithRequestSniffer(recorded); + BlockBlobClient client = blobClient.getBlockBlobClient(); + + byte[] randomData = getRandomByteArray(TEN_MB); + + try (BlobOutputStream os = client.getBlobOutputStream(new BlockBlobOutputStreamOptions() + .setParallelTransferOptions(new ParallelTransferOptions().setMaxSingleUploadSizeLong((long) TEN_MB)) + .setContentValidationAlgorithm(algorithm))) { + os.write(randomData); + } + + assertTrue(hasOnlyStructuredMessageHeaders(recorded)); + } + + @LiveOnly // Put Block URLs include random block IDs; not replayable with the test proxy. + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void blockBlobOutputStreamChunkedWithStructuredMessage(ContentValidationAlgorithm algorithm) + throws Exception { + List recorded = new CopyOnWriteArrayList<>(); + BlobClient blobClient = createBlobClientWithRequestSniffer(recorded); + BlockBlobClient client = blobClient.getBlockBlobClient(); + + byte[] randomData = getRandomByteArray(TEN_MB); + long blockSize = 2 * (long) Constants.MB; + + try (BlobOutputStream os = client.getBlobOutputStream(new BlockBlobOutputStreamOptions() + .setParallelTransferOptions( + new ParallelTransferOptions().setBlockSizeLong(blockSize).setMaxSingleUploadSizeLong(blockSize)) + .setContentValidationAlgorithm(algorithm))) { + os.write(randomData); + } + + assertTrue(hasOnlyStructuredMessageHeaders(recorded)); + } + + @Test + public void blockBlobOutputStreamWithNoContentValidation() throws Exception { + List recorded = new CopyOnWriteArrayList<>(); + BlobClient blobClient = createBlobClientWithRequestSniffer(recorded); + BlockBlobClient client = blobClient.getBlockBlobClient(); + + byte[] randomData = getRandomByteArray(TEN_MB); + + try (BlobOutputStream os = client.getBlobOutputStream(new BlockBlobOutputStreamOptions() + .setParallelTransferOptions(new ParallelTransferOptions().setMaxSingleUploadSizeLong((long) TEN_MB)) + .setContentValidationAlgorithm(ContentValidationAlgorithm.NONE))) { + os.write(randomData); + } + + assertTrue(hasNoContentValidationHeaders(recorded)); + } + + // --- PageBlobClient.getBlobOutputStream --- + + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void pageBlobOutputStreamWithCrc64Header(ContentValidationAlgorithm algorithm) throws Exception { + List recorded = new CopyOnWriteArrayList<>(); + BlobClient blobClient = createBlobClientWithRequestSniffer(recorded); + PageBlobClient client = blobClient.getPageBlobClient(); + client.create(UNDER_4MB_PAGE_ALIGNED); + + byte[] randomData = getRandomByteArray(UNDER_4MB_PAGE_ALIGNED); + + try (BlobOutputStream os = client.getBlobOutputStream( + new PageBlobOutputStreamOptions(new PageRange().setStart(0).setEnd(UNDER_4MB_PAGE_ALIGNED - 1)) + .setContentValidationAlgorithm(algorithm))) { + os.write(randomData); + } + + assertTrue(hasOnlyCrc64Headers(recorded)); + } + + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void pageBlobOutputStreamWithStructuredMessage(ContentValidationAlgorithm algorithm) throws Exception { + List recorded = new CopyOnWriteArrayList<>(); + BlobClient blobClient = createBlobClientWithRequestSniffer(recorded); + PageBlobClient client = blobClient.getPageBlobClient(); + client.create(FOUR_MB_PAGE_ALIGNED); + + byte[] randomData = getRandomByteArray(FOUR_MB_PAGE_ALIGNED); + + try (BlobOutputStream os = client.getBlobOutputStream( + new PageBlobOutputStreamOptions(new PageRange().setStart(0).setEnd(FOUR_MB_PAGE_ALIGNED - 1)) + .setContentValidationAlgorithm(algorithm))) { + os.write(randomData); + } + + assertTrue(hasOnlyStructuredMessageHeaders(recorded)); + } + + @Test + public void pageBlobOutputStreamWithNoContentValidation() throws Exception { + List recorded = new CopyOnWriteArrayList<>(); + BlobClient blobClient = createBlobClientWithRequestSniffer(recorded); + PageBlobClient client = blobClient.getPageBlobClient(); + client.create(FOUR_MB_PAGE_ALIGNED); + + byte[] randomData = getRandomByteArray(FOUR_MB_PAGE_ALIGNED); + + try (BlobOutputStream os = client.getBlobOutputStream( + new PageBlobOutputStreamOptions(new PageRange().setStart(0).setEnd(FOUR_MB_PAGE_ALIGNED - 1)) + .setContentValidationAlgorithm(ContentValidationAlgorithm.NONE))) { + os.write(randomData); + } + + assertTrue(hasNoContentValidationHeaders(recorded)); + } + + // =========================================================================================== + // BlockBlobClient.openSeekableByteChannelWrite tests + // =========================================================================================== + + @LiveOnly // Seekable channel staging uses Put Block with random block IDs; not replayable with the test proxy. + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void seekableByteChannelWriteWithCrc64Header(ContentValidationAlgorithm algorithm) throws Exception { + List recorded = new CopyOnWriteArrayList<>(); + BlobClient blobClient = createBlobClientWithRequestSniffer(recorded); + BlockBlobClient client = blobClient.getBlockBlobClient(); + + byte[] randomData = getRandomByteArray(UNDER_4MB); + + try (java.nio.channels.SeekableByteChannel channel = client.openSeekableByteChannelWrite( + new BlockBlobSeekableByteChannelWriteOptions(BlockBlobSeekableByteChannelWriteOptions.WriteMode.OVERWRITE) + .setContentValidationAlgorithm(algorithm))) { + channel.write(ByteBuffer.wrap(randomData)); + } + + assertTrue(hasOnlyCrc64Headers(recorded)); + } + + @LiveOnly // Put Block URLs include random block IDs; not replayable with the test proxy. + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void seekableByteChannelWriteWithStructuredMessage(ContentValidationAlgorithm algorithm) throws Exception { + List recorded = new CopyOnWriteArrayList<>(); + BlobClient blobClient = createBlobClientWithRequestSniffer(recorded); + BlockBlobClient client = blobClient.getBlockBlobClient(); + + byte[] randomData = getRandomByteArray(TEN_MB); + + try (java.nio.channels.SeekableByteChannel channel = client.openSeekableByteChannelWrite( + new BlockBlobSeekableByteChannelWriteOptions(BlockBlobSeekableByteChannelWriteOptions.WriteMode.OVERWRITE) + .setContentValidationAlgorithm(algorithm))) { + channel.write(ByteBuffer.wrap(randomData)); + } + + assertTrue(hasOnlyStructuredMessageHeaders(recorded)); + } + + @LiveOnly // Put Block URLs include random block IDs; not replayable with the test proxy. + @Test + public void seekableByteChannelWriteWithNoContentValidation() throws Exception { + List recorded = new CopyOnWriteArrayList<>(); + BlobClient blobClient = createBlobClientWithRequestSniffer(recorded); + BlockBlobClient client = blobClient.getBlockBlobClient(); + + byte[] randomData = getRandomByteArray(TEN_MB); + + try (java.nio.channels.SeekableByteChannel channel = client.openSeekableByteChannelWrite( + new BlockBlobSeekableByteChannelWriteOptions(BlockBlobSeekableByteChannelWriteOptions.WriteMode.OVERWRITE) + .setContentValidationAlgorithm(ContentValidationAlgorithm.NONE))) { + channel.write(ByteBuffer.wrap(randomData)); + } + + assertTrue(hasNoContentValidationHeaders(recorded)); + } + + // =========================================================================================== + // Exact 4MB boundary tests + // + // The cutoff between CRC64 header and structured message is exactly 4MB. + // Uploads of exactly 4MB should use structured message (>= threshold), not CRC64 header. + // =========================================================================================== + + private static final int EXACTLY_4MB = 4 * Constants.MB; + + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void uploadAtExactly4MBUsesStructuredMessage(ContentValidationAlgorithm algorithm) { + List recorded = new CopyOnWriteArrayList<>(); + BlobClient client = createBlobClientWithRequestSniffer(recorded); + + byte[] randomData = getRandomByteArray(EXACTLY_4MB); + InputStream data = new ByteArrayInputStream(randomData); + + BlobParallelUploadOptions options = new BlobParallelUploadOptions(data) + .setParallelTransferOptions(new ParallelTransferOptions().setMaxSingleUploadSizeLong((long) EXACTLY_4MB)) + .setRequestConditions(new BlobRequestConditions()) + .setContentValidationAlgorithm(algorithm); + + assertNotNull(client.uploadWithResponse(options, null, Context.NONE).getValue().getETag()); + assertTrue(hasOnlyStructuredMessageHeaders(recorded)); + } + + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void blockBlobSimpleUploadAtExactly4MBUsesStructuredMessage(ContentValidationAlgorithm algorithm) { + List recorded = new CopyOnWriteArrayList<>(); + BlobClient blobClient = createBlobClientWithRequestSniffer(recorded); + BlockBlobClient client = blobClient.getBlockBlobClient(); + + byte[] randomData = getRandomByteArray(EXACTLY_4MB); + BinaryData data = BinaryData.fromBytes(randomData); + + BlockBlobSimpleUploadOptions options + = new BlockBlobSimpleUploadOptions(data).setContentValidationAlgorithm(algorithm); + + assertNotNull(client.uploadWithResponse(options, null, Context.NONE).getValue().getETag()); + assertTrue(hasOnlyStructuredMessageHeaders(recorded)); + } + + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void stageBlockAtExactly4MBUsesStructuredMessage(ContentValidationAlgorithm algorithm) { + List recorded = new CopyOnWriteArrayList<>(); + BlobClient blobClient = createBlobClientWithRequestSniffer(recorded); + BlockBlobClient client = blobClient.getBlockBlobClient(); + + byte[] randomData = getRandomByteArray(EXACTLY_4MB); + BinaryData data = BinaryData.fromBytes(randomData); + + BlockBlobStageBlockOptions options + = new BlockBlobStageBlockOptions(getBlockID(), data).setContentValidationAlgorithm(algorithm); + + client.stageBlockWithResponse(options, null, Context.NONE); + assertTrue(hasOnlyStructuredMessageHeaders(recorded)); + } + + // =========================================================================================== + // Progress reporting (transfer validation must be NONE/null when a progress listener is set) + // =========================================================================================== + + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void uploadWithProgressAndNonNoneContentValidationThrows(ContentValidationAlgorithm algorithm) { + BlobClient client = createBlobClientWithRequestSniffer(new CopyOnWriteArrayList<>()); + + byte[] randomData = getRandomByteArray(TEN_MB); + InputStream data = new ByteArrayInputStream(randomData); + + BlobParallelUploadOptions options = new BlobParallelUploadOptions(data).setParallelTransferOptions( + new ParallelTransferOptions().setMaxSingleUploadSizeLong((long) TEN_MB).setProgressListener(l -> { + })).setRequestConditions(new BlobRequestConditions()).setContentValidationAlgorithm(algorithm); + + IllegalArgumentException ex = assertThrows(IllegalArgumentException.class, + () -> client.uploadWithResponse(options, null, Context.NONE)); + assertEquals(ContentValidationModeResolver.PROGRESS_CONFLICTS_TRANSFER_CONTENT_VALIDATION_MESSAGE, + ex.getMessage()); + } + + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void uploadFromFileWithProgressAndNonNoneContentValidationThrows(ContentValidationAlgorithm algorithm) + throws IOException { + BlobClient client = createBlobClientWithRequestSniffer(new CopyOnWriteArrayList<>()); + + File tempFile = getRandomFile(TEN_MB); + + BlobUploadFromFileOptions options + = new BlobUploadFromFileOptions(tempFile.getAbsolutePath()).setParallelTransferOptions( + new ParallelTransferOptions().setMaxSingleUploadSizeLong((long) TEN_MB).setProgressListener(l -> { + })).setContentValidationAlgorithm(algorithm); + + IllegalArgumentException ex = assertThrows(IllegalArgumentException.class, + () -> client.uploadFromFileWithResponse(options, null, Context.NONE)); + assertEquals(ContentValidationModeResolver.PROGRESS_CONFLICTS_TRANSFER_CONTENT_VALIDATION_MESSAGE, + ex.getMessage()); + } + + // =========================================================================================== + // Data integrity round-trip tests (upload with content validation, download, verify) + // + // Previous tests verify that the correct headers are sent. These tests verify end-to-end + // integrity: the data uploaded with CRC64/structured message can be downloaded and matches + // the original byte-for-byte. + // =========================================================================================== + + @Test + public void uploadWithCrc64RoundTripDataIntegrity() { + BlobClient client = createBlobClientWithRequestSniffer(new CopyOnWriteArrayList<>()); + + byte[] randomData = getRandomByteArray(UNDER_4MB); + InputStream data = new ByteArrayInputStream(randomData); + + BlobParallelUploadOptions options = new BlobParallelUploadOptions(data) + .setParallelTransferOptions(new ParallelTransferOptions().setMaxSingleUploadSizeLong((long) UNDER_4MB)) + .setRequestConditions(new BlobRequestConditions()) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + + client.uploadWithResponse(options, null, Context.NONE); + + byte[] downloaded = client.downloadContent().toBytes(); + assertArrayEquals(randomData, downloaded, "Downloaded data must match uploaded data (CRC64 header path)"); + } + + @Test + public void uploadWithStructuredMessageRoundTripDataIntegrity() { + BlobClient client = createBlobClientWithRequestSniffer(new CopyOnWriteArrayList<>()); + + byte[] randomData = getRandomByteArray(TEN_MB); + InputStream data = new ByteArrayInputStream(randomData); + + BlobParallelUploadOptions options = new BlobParallelUploadOptions(data) + .setParallelTransferOptions(new ParallelTransferOptions().setMaxSingleUploadSizeLong((long) TEN_MB)) + .setRequestConditions(new BlobRequestConditions()) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + + client.uploadWithResponse(options, null, Context.NONE); + + byte[] downloaded = client.downloadContent().toBytes(); + assertArrayEquals(randomData, downloaded, "Downloaded data must match uploaded data (structured message path)"); + } + + @LiveOnly // Put Block URLs include random block IDs; not replayable with the test proxy. + @Test + public void uploadChunkedWithStructuredMessageRoundTripDataIntegrity() { + BlobClient client = createBlobClientWithRequestSniffer(new CopyOnWriteArrayList<>()); + + byte[] randomData = getRandomByteArray(TEN_MB); + InputStream data = new ByteArrayInputStream(randomData); + long blockSize = 2 * (long) Constants.MB; + + BlobParallelUploadOptions options = new BlobParallelUploadOptions(data) + .setParallelTransferOptions( + new ParallelTransferOptions().setBlockSizeLong(blockSize).setMaxSingleUploadSizeLong(blockSize)) + .setRequestConditions(new BlobRequestConditions()) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + + client.uploadWithResponse(options, null, Context.NONE); + + byte[] downloaded = client.downloadContent().toBytes(); + assertArrayEquals(randomData, downloaded, + "Downloaded data must match uploaded data (chunked structured message path)"); + } + + @Test + public void blockBlobSimpleUploadRoundTripDataIntegrity() { + BlobClient blobClient = createBlobClientWithRequestSniffer(new CopyOnWriteArrayList<>()); + BlockBlobClient client = blobClient.getBlockBlobClient(); + + byte[] randomData = getRandomByteArray(TEN_MB); + BinaryData data = BinaryData.fromBytes(randomData); + + BlockBlobSimpleUploadOptions options + = new BlockBlobSimpleUploadOptions(data).setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + + client.uploadWithResponse(options, null, Context.NONE); + + byte[] downloaded = blobClient.downloadContent().toBytes(); + assertArrayEquals(randomData, downloaded, + "Downloaded data must match uploaded data (block blob simple upload)"); + } + + @Test + public void appendBlockRoundTripDataIntegrity() { + BlobClient blobClient = createBlobClientWithRequestSniffer(new CopyOnWriteArrayList<>()); + AppendBlobClient client = blobClient.getAppendBlobClient(); + client.create(); + + byte[] randomData = getRandomByteArray(TEN_MB); + InputStream data = new ByteArrayInputStream(randomData); + + AppendBlobAppendBlockOptions options = new AppendBlobAppendBlockOptions(data, TEN_MB) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + + client.appendBlockWithResponse(options, null, Context.NONE); + + byte[] downloaded = blobClient.downloadContent().toBytes(); + assertArrayEquals(randomData, downloaded, "Downloaded data must match uploaded data (append block)"); + } + + @Test + public void uploadPagesRoundTripDataIntegrity() { + BlobClient blobClient = createBlobClientWithRequestSniffer(new CopyOnWriteArrayList<>()); + PageBlobClient client = blobClient.getPageBlobClient(); + client.create(FOUR_MB_PAGE_ALIGNED); + + byte[] randomData = getRandomByteArray(FOUR_MB_PAGE_ALIGNED); + InputStream data = new ByteArrayInputStream(randomData); + + PageBlobUploadPagesOptions options + = new PageBlobUploadPagesOptions(new PageRange().setStart(0).setEnd(FOUR_MB_PAGE_ALIGNED - 1), data) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + + client.uploadPagesWithResponse(options, null, Context.NONE); + + byte[] downloaded = blobClient.downloadContent().toBytes(); + assertArrayEquals(randomData, downloaded, "Downloaded data must match uploaded data (page blob upload pages)"); + } + + @Test + public void uploadFromFileRoundTripDataIntegrity() throws IOException { + BlobClient client = createBlobClientWithRequestSniffer(new CopyOnWriteArrayList<>()); + + byte[] randomData = getRandomByteArray(TEN_MB); + File tempFile = File.createTempFile("blob-cv-roundtrip", ".bin"); + tempFile.deleteOnExit(); + try (java.io.FileOutputStream fos = new java.io.FileOutputStream(tempFile)) { + fos.write(randomData); + } + + BlobUploadFromFileOptions options = new BlobUploadFromFileOptions(tempFile.getAbsolutePath()) + .setParallelTransferOptions(new ParallelTransferOptions().setMaxSingleUploadSizeLong((long) TEN_MB)) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + + client.uploadFromFileWithResponse(options, null, Context.NONE); + + byte[] downloaded = client.downloadContent().toBytes(); + assertArrayEquals(randomData, downloaded, "Downloaded data must match uploaded file data"); + } + + // =========================================================================================== + // Live-only random payload bands. + // - 256–500 MiB: parallelUpload, uploadFromFile, stageBlock, block BlobOutputStream, SeekableByteChannel — + // parallel staging or single giant block / default transfer options as applicable. + // - 32–64 MiB (sequential append blocks only): appendBlobAppendBlocksLiveRandom… — one append REST call per + // chunk in order. + // =========================================================================================== + + @LiveOnly // This test is too large for the test proxy. + @Test + public void parallelUploadLiveRandomRoundTripDataIntegrity() throws Exception { + int chosenPayloadSizeBytes = (int) randomLongFromNamer(LIVE_RANDOM_PARALLEL_PAYLOAD_MIN_BYTES_EXCLUSIVE + 1, + LIVE_RANDOM_PARALLEL_PAYLOAD_MAX_BYTES_INCLUSIVE + 1); + try { + String prefix = "chosenPayloadSizeBytes=" + chosenPayloadSizeBytes + ". "; + BlobClient client = createBlobClientWithRequestSniffer(new CopyOnWriteArrayList<>()); + File sourceFile = getRandomFile(chosenPayloadSizeBytes); + File outFile = Files.createTempFile("blob-cv-live-par-dl", ".bin").toFile(); + outFile.deleteOnExit(); + try { + try (InputStream data = new FileInputStream(sourceFile)) { + BlobParallelUploadOptions options + = new BlobParallelUploadOptions(data).setRequestConditions(new BlobRequestConditions()) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + client.uploadWithResponse(options, null, Context.NONE); + } + client.downloadToFile(outFile.getPath(), true); + assertTrue(compareFiles(sourceFile, outFile, 0, chosenPayloadSizeBytes), prefix); + } finally { + if (!sourceFile.delete()) { + sourceFile.deleteOnExit(); + } + if (!outFile.delete()) { + outFile.deleteOnExit(); + } + } + } catch (Exception e) { + throw new Exception("chosenPayloadSizeBytes=" + chosenPayloadSizeBytes + ". " + e.getMessage(), e); + } + } + + @LiveOnly // This test is too large for the test proxy. + @Test + public void stageBlockLiveRandomRoundTripDataIntegrity() throws Exception { + int chosenPayloadSizeBytes = (int) randomLongFromNamer(LIVE_RANDOM_PARALLEL_PAYLOAD_MIN_BYTES_EXCLUSIVE + 1, + LIVE_RANDOM_PARALLEL_PAYLOAD_MAX_BYTES_INCLUSIVE + 1); + try { + String prefix = "chosenPayloadSizeBytes=" + chosenPayloadSizeBytes + ". "; + BlobClient blobClient = createBlobClientWithRequestSniffer(new CopyOnWriteArrayList<>()); + BlockBlobClient client = blobClient.getBlockBlobClient(); + String blockId = getBlockID(); + + File sourceFile = getRandomFile(chosenPayloadSizeBytes); + File outFile = Files.createTempFile("blob-cv-live-stage-dl", ".bin").toFile(); + outFile.deleteOnExit(); + try { + BinaryData binaryData = BinaryData.fromFile(sourceFile.toPath()); + BlockBlobStageBlockOptions stageOptions = new BlockBlobStageBlockOptions(blockId, binaryData) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + client.stageBlockWithResponse(stageOptions, null, Context.NONE); + client.commitBlockList(Collections.singletonList(blockId)); + blobClient.downloadToFile(outFile.getPath(), true); + + assertTrue(compareFiles(sourceFile, outFile, 0, chosenPayloadSizeBytes), prefix); + } finally { + if (!sourceFile.delete()) { + sourceFile.deleteOnExit(); + } + if (!outFile.delete()) { + outFile.deleteOnExit(); + } + } + } catch (Exception e) { + throw new Exception("chosenPayloadSizeBytes=" + chosenPayloadSizeBytes + ". " + e.getMessage(), e); + } + } + + @LiveOnly // This test is too large for the test proxy. + @Test + public void appendBlobAppendBlocksLiveRandomRoundTripDataIntegrity() throws Exception { + int chosenPayloadSizeBytes + = (int) randomLongFromNamer(LIVE_RANDOM_SEQUENTIAL_APPEND_PAYLOAD_MIN_BYTES_EXCLUSIVE + 1, + LIVE_RANDOM_SEQUENTIAL_APPEND_PAYLOAD_MAX_BYTES_INCLUSIVE + 1); + try { + String prefix = "chosenPayloadSizeBytes=" + chosenPayloadSizeBytes + ". "; + BlobClient blobClient = createBlobClientWithRequestSniffer(new CopyOnWriteArrayList<>()); + AppendBlobClient client = blobClient.getAppendBlobClient(); + File sourceFile = getRandomFile(chosenPayloadSizeBytes); + File outFile = Files.createTempFile("blob-cv-live-append-blocks-dl", ".bin").toFile(); + outFile.deleteOnExit(); + client.create(); + int maxAppendBlockBytes = client.getMaxAppendBlockBytes(); + try { + try (FileInputStream fis = new FileInputStream(sourceFile)) { + long remaining = chosenPayloadSizeBytes; + byte[] buf = new byte[maxAppendBlockBytes]; + while (remaining > 0) { + int chunk = (int) Math.min(maxAppendBlockBytes, remaining); + int totalRead = 0; + while (totalRead < chunk) { + int n = fis.read(buf, totalRead, chunk - totalRead); + if (n == -1) { + throw new EOFException( + prefix + "Unexpected EOF after " + totalRead + " bytes of chunk."); + } + totalRead += n; + } + ByteArrayInputStream chunkStream = new ByteArrayInputStream(buf, 0, chunk); + AppendBlobAppendBlockOptions appendOptions + = new AppendBlobAppendBlockOptions(chunkStream, chunk) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + client.appendBlockWithResponse(appendOptions, null, Context.NONE); + remaining -= chunk; + } + } + blobClient.downloadToFile(outFile.getPath(), true); + assertTrue(compareFiles(sourceFile, outFile, 0, chosenPayloadSizeBytes), prefix); + } finally { + if (!sourceFile.delete()) { + sourceFile.deleteOnExit(); + } + if (!outFile.delete()) { + outFile.deleteOnExit(); + } + } + } catch (Exception e) { + throw new Exception("chosenPayloadSizeBytes=" + chosenPayloadSizeBytes + ". " + e.getMessage(), e); + } + } + + @LiveOnly // This test is too large for the test proxy. + @Test + public void uploadFromFileLiveRandomRoundTripDataIntegrity() throws Exception { + int chosenPayloadSizeBytes = (int) randomLongFromNamer(LIVE_RANDOM_PARALLEL_PAYLOAD_MIN_BYTES_EXCLUSIVE + 1, + LIVE_RANDOM_PARALLEL_PAYLOAD_MAX_BYTES_INCLUSIVE + 1); + try { + String prefix = "chosenPayloadSizeBytes=" + chosenPayloadSizeBytes + ". "; + BlobClient client = createBlobClientWithRequestSniffer(new CopyOnWriteArrayList<>()); + File sourceFile = getRandomFile(chosenPayloadSizeBytes); + File outFile = Files.createTempFile("blob-cv-live-uploadfromfile-dl", ".bin").toFile(); + outFile.deleteOnExit(); + try { + BlobUploadFromFileOptions options = new BlobUploadFromFileOptions(sourceFile.getAbsolutePath()) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + client.uploadFromFileWithResponse(options, null, Context.NONE); + client.downloadToFile(outFile.getPath(), true); + assertTrue(compareFiles(sourceFile, outFile, 0, chosenPayloadSizeBytes), prefix); + } finally { + if (!sourceFile.delete()) { + sourceFile.deleteOnExit(); + } + if (!outFile.delete()) { + outFile.deleteOnExit(); + } + } + } catch (Exception e) { + throw new Exception("chosenPayloadSizeBytes=" + chosenPayloadSizeBytes + ". " + e.getMessage(), e); + } + } + + @LiveOnly // This test is too large for the test proxy. + @Test + public void blockBlobOutputStreamLiveRandomRoundTripDataIntegrity() throws Exception { + int chosenPayloadSizeBytes = (int) randomLongFromNamer(LIVE_RANDOM_PARALLEL_PAYLOAD_MIN_BYTES_EXCLUSIVE + 1, + LIVE_RANDOM_PARALLEL_PAYLOAD_MAX_BYTES_INCLUSIVE + 1); + try { + String prefix = "chosenPayloadSizeBytes=" + chosenPayloadSizeBytes + ". "; + BlobClient blobClient = createBlobClientWithRequestSniffer(new CopyOnWriteArrayList<>()); + BlockBlobClient client = blobClient.getBlockBlobClient(); + + // Explicit parallel-transfer tuning (8 MiB blocks × concurrency 8) on the stream ingest path. + ParallelTransferOptions parallelTransferOptions + = new ParallelTransferOptions().setBlockSizeLong(8L * Constants.MB) + .setMaxSingleUploadSizeLong(8L * Constants.MB) + .setMaxConcurrency(8); + + File sourceFile = getRandomFile(chosenPayloadSizeBytes); + File outFile = Files.createTempFile("blob-cv-live-block-os-dl", ".bin").toFile(); + outFile.deleteOnExit(); + try { + try (BlobOutputStream outputStream = client.getBlobOutputStream( + new BlockBlobOutputStreamOptions().setParallelTransferOptions(parallelTransferOptions) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64))) { + Files.copy(sourceFile.toPath(), outputStream); + } + + blobClient.downloadToFile(outFile.getPath(), true); + assertTrue(compareFiles(sourceFile, outFile, 0, chosenPayloadSizeBytes), prefix); + } finally { + if (!sourceFile.delete()) { + sourceFile.deleteOnExit(); + } + if (!outFile.delete()) { + outFile.deleteOnExit(); + } + } + } catch (Exception e) { + throw new Exception("chosenPayloadSizeBytes=" + chosenPayloadSizeBytes + ". " + e.getMessage(), e); + } + } + + @LiveOnly // This test is too large for the test proxy. + @Test + public void seekableByteChannelWriteLiveRandomRoundTripDataIntegrity() throws Exception { + int chosenPayloadSizeBytes = (int) randomLongFromNamer(LIVE_RANDOM_PARALLEL_PAYLOAD_MIN_BYTES_EXCLUSIVE + 1, + LIVE_RANDOM_PARALLEL_PAYLOAD_MAX_BYTES_INCLUSIVE + 1); + try { + String prefix = "chosenPayloadSizeBytes=" + chosenPayloadSizeBytes + ". "; + BlobClient blobClient = createBlobClientWithRequestSniffer(new CopyOnWriteArrayList<>()); + BlockBlobClient client = blobClient.getBlockBlobClient(); + File sourceFile = getRandomFile(chosenPayloadSizeBytes); + File outFile = Files.createTempFile("blob-cv-live-sbc-dl", ".bin").toFile(); + outFile.deleteOnExit(); + try { + try (java.nio.channels.SeekableByteChannel seekableByteChannel + = client.openSeekableByteChannelWrite(new BlockBlobSeekableByteChannelWriteOptions( + BlockBlobSeekableByteChannelWriteOptions.WriteMode.OVERWRITE) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64))) { + Files.copy(sourceFile.toPath(), Channels.newOutputStream(seekableByteChannel)); + } + + blobClient.downloadToFile(outFile.getPath(), true); + assertTrue(compareFiles(sourceFile, outFile, 0, chosenPayloadSizeBytes), prefix); + } finally { + if (!sourceFile.delete()) { + sourceFile.deleteOnExit(); + } + if (!outFile.delete()) { + outFile.deleteOnExit(); + } + } + } catch (Exception e) { + throw new Exception("chosenPayloadSizeBytes=" + chosenPayloadSizeBytes + ". " + e.getMessage(), e); + } + } + + // ---------- Deterministic parallel upload ---------- + + @ParameterizedTest + @MethodSource("com.azure.storage.blob.BlobTestBase#fuzzyParallelUploadPutBlobReplayableCases") + public void fuzzyParallelUploadPutBlobReplayableRoundTrip(int payloadBytes, long segmentBytes, int maxConcurrency) + throws IOException { + assertParallelUploadFuzzyRoundTrip("putBlobReplay", payloadBytes, segmentBytes, maxConcurrency); + } + + @LiveOnly // Staging-only cases: Put Block URLs include random IDs + @ParameterizedTest + @MethodSource("com.azure.storage.blob.BlobTestBase#fuzzyParallelUploadSmallPayloadStagingCases") + public void fuzzyParallelUploadSmallPayloadRoundTripRequiresLiveStaging(int payloadBytes, long segmentBytes, + int maxConcurrency) throws IOException { + assertParallelUploadFuzzyRoundTrip("smallPayloadStaging", payloadBytes, segmentBytes, maxConcurrency); + } + + @LiveOnly // payload > segment for every tuple; always staging/Put Block. + @ParameterizedTest + @MethodSource("com.azure.storage.blob.BlobTestBase#fuzzyParallelUploadSub4MiBCases") + public void fuzzyParallelUploadSubFourMiBBlobRoundTrip(int payloadBytes, long segmentBytes, int maxConcurrency) + throws IOException { + assertParallelUploadFuzzyRoundTrip("subFourMiB", payloadBytes, segmentBytes, maxConcurrency); + } + + @LiveOnly // Staging-only cases. + @ParameterizedTest + @MethodSource("com.azure.storage.blob.BlobTestBase#fuzzyParallelUploadFourMiBBoundaryStagingCases") + public void fuzzyParallelUploadFourMiBBoundaryRoundTripRequiresLiveStaging(int payloadBytes, long segmentBytes, + int maxConcurrency) throws IOException { + assertParallelUploadFuzzyRoundTrip("fourMiBBoundaryStaging", payloadBytes, segmentBytes, maxConcurrency); + } + + @LiveOnly // payload > segment throughout; chunked upload. + @ParameterizedTest + @MethodSource("com.azure.storage.blob.BlobTestBase#fuzzyParallelUploadMediumMultiPartCases") + public void fuzzyParallelUploadMediumMultiPartRoundTrip(int payloadBytes, long segmentBytes, int maxConcurrency) + throws IOException { + assertParallelUploadFuzzyRoundTrip("mediumMultiPart", payloadBytes, segmentBytes, maxConcurrency); + } + + @LiveOnly // payload >> segment throughout; chunked upload / large payloads. + @ParameterizedTest + @MethodSource("com.azure.storage.blob.BlobTestBase#fuzzyParallelUploadLargeMultiPartCases") + public void fuzzyParallelUploadLargeMultiPartRoundTrip(int payloadBytes, long segmentBytes, int maxConcurrency) + throws IOException { + assertParallelUploadFuzzyRoundTrip("largeMultiPart", payloadBytes, segmentBytes, maxConcurrency); + } + + private void assertParallelUploadFuzzyRoundTrip(String caseKind, int payloadBytes, long segmentBytes, + int maxConcurrency) throws IOException { + BlobClient client = createBlobClientWithRequestSniffer(new CopyOnWriteArrayList<>()); + + ParallelTransferOptions parallelOptions = new ParallelTransferOptions().setBlockSizeLong(segmentBytes) + .setMaxSingleUploadSizeLong(segmentBytes) + .setMaxConcurrency(maxConcurrency); + + String assertionMessage = "Fuzzy parallel upload [" + caseKind + "] payloadBytes=" + payloadBytes + + ", segmentBytes=" + segmentBytes + ", maxConcurrency=" + maxConcurrency; + + // above this threshold the fuzzy parallel upload helpers stream from a temp source file + // to avoid materializing the full payload twice in heap. + if (payloadBytes >= 96 * Constants.MB) { + File sourceFile = getRandomFile(payloadBytes); + File outFile = Files.createTempFile("blob-cv-fuzzy-parallel-dl", ".bin").toFile(); + outFile.deleteOnExit(); + try { + try (InputStream data = new FileInputStream(sourceFile)) { + BlobParallelUploadOptions options + = new BlobParallelUploadOptions(data).setParallelTransferOptions(parallelOptions) + .setRequestConditions(new BlobRequestConditions()) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + client.uploadWithResponse(options, null, Context.NONE); + } + client.downloadToFile(outFile.getPath(), true); + assertTrue(compareFiles(sourceFile, outFile, 0, payloadBytes), assertionMessage); + } finally { + if (!sourceFile.delete()) { + sourceFile.deleteOnExit(); + } + if (!outFile.delete()) { + outFile.deleteOnExit(); + } + } + } else { + byte[] randomData = getRandomByteArray(payloadBytes); + InputStream data = new ByteArrayInputStream(randomData); + BlobParallelUploadOptions options + = new BlobParallelUploadOptions(data).setParallelTransferOptions(parallelOptions) + .setRequestConditions(new BlobRequestConditions()) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + client.uploadWithResponse(options, null, Context.NONE); + byte[] downloaded = client.downloadContent().toBytes(); + assertArrayEquals(randomData, downloaded, assertionMessage); + } + } + + // =========================================================================================== + // Customer Provided MD5 Byte[] with Content Validation Algorithm + // =========================================================================================== + + private static final byte[] DEFAULT_MD5 = createDefaultMd5(); + private static final String MESSAGE = "Both x-ms-content-crc64 header and Content-MD5 header are present."; + + private static byte[] createDefaultMd5() { + try { + return Base64.getEncoder().encode(MessageDigest.getInstance("MD5").digest(DATA.getDefaultBytes())); + } catch (NoSuchAlgorithmException ex) { + throw LOGGER.logExceptionAsError(new RuntimeException("MD5 algorithm unavailable.", ex)); + } + } + + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void blockBlobUploadWithCustomerProvidedMd5AndCrc64Header(ContentValidationAlgorithm algorithm) { + BlockBlobClient client = createBlobClientWithRequestSniffer(new CopyOnWriteArrayList<>()).getBlockBlobClient(); + + BlockBlobSimpleUploadOptions options + = new BlockBlobSimpleUploadOptions(DATA.getDefaultBinaryData()).setContentValidationAlgorithm(algorithm) + .setContentMd5(DEFAULT_MD5); + + BlobStorageException e + = assertThrows(BlobStorageException.class, () -> client.uploadWithResponse(options, null, Context.NONE)); + assertEquals(400, e.getStatusCode()); + assertTrue(e.getMessage().contains(MESSAGE)); + } + + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void stageBlockWithCustomerProvidedMd5AndCrc64Header(ContentValidationAlgorithm algorithm) { + BlockBlobClient client = createBlobClientWithRequestSniffer(new CopyOnWriteArrayList<>()).getBlockBlobClient(); + + BlockBlobStageBlockOptions options = new BlockBlobStageBlockOptions(getBlockID(), DATA.getDefaultBinaryData()) + .setContentValidationAlgorithm(algorithm) + .setContentMd5(DEFAULT_MD5); + + BlobStorageException e = assertThrows(BlobStorageException.class, + () -> client.stageBlockWithResponse(options, null, Context.NONE)); + assertEquals(400, e.getStatusCode()); + assertTrue(e.getMessage().contains(MESSAGE)); + } + + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void appendBlockWithCustomerProvidedMd5AndCrc64Header(ContentValidationAlgorithm algorithm) { + AppendBlobClient client + = createBlobClientWithRequestSniffer(new CopyOnWriteArrayList<>()).getAppendBlobClient(); + client.create(); + + byte[] randomData = DATA.getDefaultBytes(); + AppendBlobAppendBlockOptions options + = new AppendBlobAppendBlockOptions(new ByteArrayInputStream(randomData), randomData.length) + .setContentValidationAlgorithm(algorithm) + .setContentMd5(DEFAULT_MD5); + + BlobStorageException e = assertThrows(BlobStorageException.class, + () -> client.appendBlockWithResponse(options, null, Context.NONE)); + assertEquals(400, e.getStatusCode()); + assertTrue(e.getMessage().contains(MESSAGE)); + } + + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void uploadPagesWithCustomerProvidedMd5AndCrc64Header(ContentValidationAlgorithm algorithm) + throws NoSuchAlgorithmException { + PageBlobClient client = createBlobClientWithRequestSniffer(new CopyOnWriteArrayList<>()).getPageBlobClient(); + client.create(UNDER_4MB_PAGE_ALIGNED); + + byte[] randomData = getRandomByteArray(UNDER_4MB_PAGE_ALIGNED); + byte[] md5 = MessageDigest.getInstance("MD5").digest(randomData); + PageBlobUploadPagesOptions options + = new PageBlobUploadPagesOptions(new PageRange().setStart(0).setEnd(UNDER_4MB_PAGE_ALIGNED - 1), + new ByteArrayInputStream(randomData)).setContentValidationAlgorithm(algorithm).setContentMd5(md5); + + BlobStorageException e = assertThrows(BlobStorageException.class, + () -> client.uploadPagesWithResponse(options, null, Context.NONE)); + assertEquals(400, e.getStatusCode()); + assertTrue(e.getMessage().contains(MESSAGE)); + } +} diff --git a/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobTestBase.java b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobTestBase.java index 99d925bff60c..71345472c393 100644 --- a/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobTestBase.java +++ b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobTestBase.java @@ -12,6 +12,9 @@ import com.azure.core.http.HttpMethod; import com.azure.core.http.HttpPipeline; import com.azure.core.http.HttpPipelineBuilder; +import com.azure.core.http.HttpPipelineCallContext; +import com.azure.core.http.HttpPipelineNextPolicy; +import com.azure.core.http.HttpPipelinePosition; import com.azure.core.http.HttpRequest; import com.azure.core.http.HttpResponse; import com.azure.core.http.policy.AddDatePolicy; @@ -61,6 +64,9 @@ import com.azure.storage.common.Utility; import com.azure.storage.common.implementation.Constants; import com.azure.storage.common.policy.RequestRetryOptions; +import com.azure.storage.common.implementation.contentvalidation.StructuredMessageConstants; +import com.azure.storage.common.implementation.contentvalidation.StructuredMessageEncoder; +import com.azure.storage.common.implementation.contentvalidation.StructuredMessageFlags; import com.azure.storage.common.test.shared.StorageCommonTestUtils; import com.azure.storage.common.test.shared.TestAccount; import com.azure.storage.common.test.shared.TestDataFactory; @@ -91,7 +97,10 @@ import java.util.Map; import java.util.Objects; import java.util.Queue; +import java.util.Random; +import java.util.UUID; import java.util.concurrent.Callable; +import java.util.function.Consumer; import java.util.function.Supplier; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -904,6 +913,37 @@ protected File getRandomFile(int size) throws IOException { return StorageCommonTestUtils.getRandomFile(size, testResourceNamer); } + /** + * Pseudorandom {@link Random} seeded from {@code testResourceNamer.randomUuid()} so values replay in playback + * mode (unlike {@link java.util.concurrent.ThreadLocalRandom}). + */ + protected Random newRandomFromNamer() { + long seed = UUID.fromString(testResourceNamer.randomUuid()).getMostSignificantBits() & Long.MAX_VALUE; + return new Random(seed); + } + + /** + * Pseudorandom int in {@code [origin, bound)} from the namer. Consumes one recorded UUID. + */ + protected int randomIntFromNamer(int origin, int bound) { + int span = bound - origin; + if (span <= 0) { + throw new IllegalArgumentException("bound must be greater than origin"); + } + return origin + newRandomFromNamer().nextInt(span); + } + + /** + * Pseudorandom long in {@code [origin, bound)} from the namer. Consumes one recorded UUID. + */ + protected long randomLongFromNamer(long origin, long bound) { + long span = bound - origin; + if (span <= 0) { + throw new IllegalArgumentException("bound must be greater than origin"); + } + return origin + Math.floorMod(newRandomFromNamer().nextLong(), span); + } + /*https://learn.microsoft.com/en-us/rest/api/storageservices/define-stored-access-policy#creating-or-modifying-a-stored-access-policy Second note, it can take up to 30 seconds to set/create an access policy and this was causing flakeyness in the live test pipeline */ @@ -1344,4 +1384,469 @@ public static HttpPipelinePolicy getAddHeadersAndQueryPolicy(Map return next.process(); }; } + + protected static boolean hasOnlyStructuredMessageHeaders(List recordedRequestHeaders) { + return hasStructuredMessageDownloadRequestHeaders(recordedRequestHeaders, true); + } + + protected static boolean hasStructuredMessageDownloadRequestHeaders(HttpHeaders recordedRequestHeaders) { + if (recordedRequestHeaders == null || recordedRequestHeaders.getSize() == 0) { + return false; + } + return hasStructuredMessageDownloadRequestHeaders(Collections.singletonList(recordedRequestHeaders), false); + } + + protected static boolean hasStructuredMessageDownloadResponseHeaders(HttpHeaders headers) { + return validateBasicHeaders(headers) + && StructuredMessageConstants.STRUCTURED_BODY_TYPE_VALUE + .equalsIgnoreCase(headers.getValue(Constants.HeaderConstants.STRUCTURED_BODY_TYPE_HEADER_NAME)) + && hasValidStructuredContentLengthHeader(headers); + } + + protected static HttpPipelinePolicy getRequestAndResponseHeaderSniffer(String targetUrlPrefix, + HttpHeaders recordedRequestHeaders, HttpHeaders recordedResponseHeaders) { + return getRequestAndResponseHeaderSniffer(targetUrlPrefix, headers -> { + synchronized (recordedRequestHeaders) { + recordedRequestHeaders.setAllHttpHeaders(headers); + } + }, recordedResponseHeaders); + } + + protected static HttpPipelinePolicy getRequestAndResponseHeaderSniffer(String targetUrlPrefix, + List recordedRequestHeaders, HttpHeaders recordedResponseHeaders) { + return getRequestAndResponseHeaderSniffer(targetUrlPrefix, recordedRequestHeaders::add, + recordedResponseHeaders); + } + + private static HttpPipelinePolicy getRequestAndResponseHeaderSniffer(String targetUrlPrefix, + Consumer requestRecorder, HttpHeaders recordedResponseHeaders) { + return new HttpPipelinePolicy() { + @Override + public HttpPipelinePosition getPipelinePosition() { + return HttpPipelinePosition.PER_RETRY; + } + + @Override + public Mono process(HttpPipelineCallContext context, HttpPipelineNextPolicy next) { + requestRecorder.accept(context.getHttpRequest().getHeaders()); + return next.process().map(response -> { + if (response.getRequest().getHttpMethod() == HttpMethod.GET + && response.getRequest().getUrl().toString().startsWith(targetUrlPrefix)) { + synchronized (recordedResponseHeaders) { + recordedResponseHeaders.setAllHttpHeaders(response.getHeaders()); + } + } + return response; + }); + } + }; + } + + protected static boolean hasStructuredMessageDownloadRequestHeaders(List recordedRequestHeaders, + boolean requireStructuredContentLength) { + if (recordedRequestHeaders == null || recordedRequestHeaders.isEmpty()) { + return false; + } + // Only consider requests where any of the structured-message or CRC64-related headers is present. + List headersWithContentValidation = recordedRequestHeaders.stream().filter(headers -> { + String bodyType = headers.getValue(Constants.HeaderConstants.STRUCTURED_BODY_TYPE_HEADER_NAME); + String contentLength = headers.getValue(Constants.HeaderConstants.STRUCTURED_CONTENT_LENGTH_HEADER_NAME); + String contentCrc64 = headers.getValue(Constants.HeaderConstants.CONTENT_CRC64_HEADER_NAME); + return bodyType != null || contentLength != null || contentCrc64 != null; + }).collect(Collectors.toList()); + // If no requests had any content-validation headers at all, we cannot claim structured-message was applied. + if (headersWithContentValidation.isEmpty()) { + return false; + } + // All requests that used any content-validation header must be consistent structured-message requests. + return headersWithContentValidation.stream().allMatch(headers -> { + String bodyType = headers.getValue(Constants.HeaderConstants.STRUCTURED_BODY_TYPE_HEADER_NAME); + String contentLength = headers.getValue(Constants.HeaderConstants.STRUCTURED_CONTENT_LENGTH_HEADER_NAME); + String contentCrc64 = headers.getValue(Constants.HeaderConstants.CONTENT_CRC64_HEADER_NAME); + if (!StructuredMessageConstants.STRUCTURED_BODY_TYPE_VALUE.equals(bodyType) || contentCrc64 != null) { + return false; + } + if (!requireStructuredContentLength) { + return true; + } + // Require non-blank content length that parses as non-negative long (same format as policy uses). + // Rejects empty string, whitespace, or non-numeric values so we never return true when + // structured message was not actually applied. + if (contentLength == null || contentLength.trim().isEmpty()) { + return false; + } + try { + return Long.parseLong(contentLength.trim()) >= 0; + } catch (NumberFormatException e) { + return false; + } + }); + } + + protected static boolean hasOnlyCrc64Headers(List recordedRequestHeaders) { + if (recordedRequestHeaders == null || recordedRequestHeaders.isEmpty()) { + return false; + } + // Only consider requests where any of the structured-message or CRC64-related headers is present. + List headersWithContentValidation = recordedRequestHeaders.stream().filter(headers -> { + String bodyType = headers.getValue(Constants.HeaderConstants.STRUCTURED_BODY_TYPE_HEADER_NAME); + String contentLength = headers.getValue(Constants.HeaderConstants.STRUCTURED_CONTENT_LENGTH_HEADER_NAME); + String contentCrc64 = headers.getValue(Constants.HeaderConstants.CONTENT_CRC64_HEADER_NAME); + return bodyType != null || contentLength != null || contentCrc64 != null; + }).collect(Collectors.toList()); + // If no requests had any content-validation headers at all, we cannot claim CRC64 was applied. + if (headersWithContentValidation.isEmpty()) { + return false; + } + // All requests that used any content-validation header must be consistent CRC64-only requests. + return headersWithContentValidation.stream().allMatch(headers -> { + String contentCrc64 = headers.getValue(Constants.HeaderConstants.CONTENT_CRC64_HEADER_NAME); + String bodyType = headers.getValue(Constants.HeaderConstants.STRUCTURED_BODY_TYPE_HEADER_NAME); + String contentLength = headers.getValue(Constants.HeaderConstants.STRUCTURED_CONTENT_LENGTH_HEADER_NAME); + // Require CRC64 header to be present and non-blank (policy sets Base64-encoded 8 bytes). + // Reject empty/whitespace so we never return true when CRC64 was not actually applied. + if (contentCrc64 == null || contentCrc64.trim().isEmpty() || bodyType != null || contentLength != null) { + return false; + } + return true; + }); + } + + protected static boolean hasNoContentValidationHeaders(List recordedRequestHeaders) { + if (recordedRequestHeaders == null || recordedRequestHeaders.isEmpty()) { + return false; + } + return recordedRequestHeaders.stream().allMatch(headers -> { + String bodyType = headers.getValue(Constants.HeaderConstants.STRUCTURED_BODY_TYPE_HEADER_NAME); + String contentLength = headers.getValue(Constants.HeaderConstants.STRUCTURED_CONTENT_LENGTH_HEADER_NAME); + String contentCrc64 = headers.getValue(Constants.HeaderConstants.CONTENT_CRC64_HEADER_NAME); + // All three must be absent (null). If any header is present, even with empty value, we return false. + return bodyType == null && contentLength == null && contentCrc64 == null; + }); + } + + /** + * Creates a BlobClient that records all outgoing request headers into the supplied list. + * Each test should use its own list so tests can run concurrently. + */ + protected BlobClient createBlobClientWithRequestSniffer(List recordedRequestHeaders) { + HttpPipelinePolicy sniffPolicy = (context, next) -> { + recordedRequestHeaders.add(context.getHttpRequest().getHeaders()); + return next.process(); + }; + BlobServiceClient serviceClient = getServiceClient(ENVIRONMENT.getPrimaryAccount().getCredential(), + ENVIRONMENT.getPrimaryAccount().getBlobEndpoint(), sniffPolicy); + return serviceClient.getBlobContainerClient(containerName).getBlobClient(generateBlobName()); + } + + /** + * Creates a BlobAsyncClient that records all outgoing request headers into the supplied list. + */ + protected BlobAsyncClient createBlobAsyncClientWithRequestSniffer(List recordedRequestHeaders) { + HttpPipelinePolicy sniffPolicy = (context, next) -> { + recordedRequestHeaders.add(context.getHttpRequest().getHeaders()); + return next.process(); + }; + BlobServiceAsyncClient serviceClient = getServiceAsyncClient(ENVIRONMENT.getPrimaryAccount().getCredential(), + ENVIRONMENT.getPrimaryAccount().getBlobEndpoint(), sniffPolicy); + return serviceClient.getBlobContainerAsyncClient(containerName).getBlobAsyncClient(generateBlobName()); + } + + protected static long expectedStructuredMessageEncodedLength(int unencodedContentBytes) { + return new StructuredMessageEncoder(unencodedContentBytes, + StructuredMessageConstants.V1_DEFAULT_SEGMENT_CONTENT_LENGTH, StructuredMessageFlags.STORAGE_CRC64) + .getEncodedMessageLength(); + } + + /** + * Sum of encoded lengths per block upload (each HTTP request carries its own structured message wrapper). + */ + protected static long expectedStructuredMessageEncodedLengthChunked(int totalUnencodedBytes, long blockSizeBytes) { + long sum = 0; + int remaining = totalUnencodedBytes; + while (remaining > 0) { + int chunk = (int) Math.min(remaining, blockSizeBytes); + sum += expectedStructuredMessageEncodedLength(chunk); + remaining -= chunk; + } + return sum; + } + + /** + * Every tuple keeps payloadBytes <= blockSizeBytes, so the parallel download path issues a single GET (no + * follow-on range requests for additional blocks), which replays under the test proxy. + *

+ * Sizes are deliberately non-power-of-two (e.g. 7 * KB + 3) and use mixed block ceilings (64 KiB through + * multi-MiB) to catch alignment and decoder edge cases at structural boundaries (message header, segment + * footer, message footer); the 4 MiB boundary row exercises the exact service-side default segment length. + */ + protected static Stream fuzzyParallelDownloadReplayableCases() { + return Stream.of(Arguments.of(1, 64L * Constants.KB, 1), + Arguments.of(7 * Constants.KB + 3, 64L * Constants.KB, 1), + Arguments.of(7 * Constants.KB + 3, 128L * Constants.KB, 4), + Arguments.of(41 * Constants.KB + 17, 256L * Constants.KB, 1), + Arguments.of(41 * Constants.KB + 17, 256L * Constants.KB, 8), + Arguments.of(199 * Constants.KB + 5, 512L * Constants.KB, 2), + Arguments.of(512 * Constants.KB - 31, 1L * Constants.MB, 8), + Arguments.of(896 * Constants.KB + 101, 1L * Constants.MB, 6), + Arguments.of(2 * Constants.MB - 1, 4L * Constants.MB, 4), + Arguments.of(2 * Constants.MB + 33, 4L * Constants.MB, 1), + Arguments.of(4 * Constants.MB - 1, 4L * Constants.MB, 2), + Arguments.of(4 * Constants.MB, 4L * Constants.MB, 1), + Arguments.of(4 * Constants.MB, 7L * Constants.MB + 919, 3)); + } + + /** + * Every tuple keeps payloadBytes <= segmentBytes, so the parallel upload path issues a single Put Blob (no + * Put Block IDs), which replays under the test proxy unlike staging-heavy cases. + *

+ * Sizes are deliberately non-power-of-two (e.g. 7 * KB + 3) and use mixed segment ceilings (64 KiB + * through multi-MiB) to catch alignment and buffer edge cases; rows include the exact 4 MiB service boundary + * and several concurrency values (1–8) to exercise parallel request fan-out without live-only staging. + */ + protected static Stream fuzzyParallelUploadPutBlobReplayableCases() { + return Stream.of(Arguments.of(7 * Constants.KB + 3, 64L * Constants.KB, 1), + Arguments.of(7 * Constants.KB + 3, 128L * Constants.KB, 4), + Arguments.of(41 * Constants.KB + 17, 256L * Constants.KB, 1), + Arguments.of(41 * Constants.KB + 17, 256L * Constants.KB, 8), + Arguments.of(199 * Constants.KB + 5, 512L * Constants.KB, 2), + Arguments.of(512 * Constants.KB - 31, 1L * Constants.MB, 8), + Arguments.of(896 * Constants.KB + 101, 1L * Constants.MB, 6), + Arguments.of(4 * Constants.MB, 4L * Constants.MB, 1), + Arguments.of(4 * Constants.MB, 7L * Constants.MB + 919, 3)); + } + + /** + * payloadBytes > blockSizeBytes but totals stay in the hundreds of KiB, so downloads issue many small ranged + * GETs even though the blob itself is small. Live-only because chunked range GETs across many tiny requests + * produce per-block proxy churn similar to the upload-side staging cases. + *

+ * One row pairs a ~200 KiB payload with a 64 KiB block size and moderate concurrency; the other uses a + * ~512 KiB payload with a 1 KiB block size to force many tiny range GETs (stress decoder framing and + * scheduling) without the cost of the large multi-part grids. + */ + protected static Stream fuzzyParallelDownloadSmallMultiPartCases() { + return Stream.of(Arguments.of(200 * Constants.KB, 64L * Constants.KB, 3), + Arguments.of(512 * Constants.KB - 31, 1L * Constants.KB, 1)); + } + + /** + * payloadBytes > segmentBytes, so uploads still go through Put Block staging even though totals are only + * hundreds of KiB—too small for the proxy when block IDs vary per run. + *

+ * One row pairs a ~200 KiB payload with a 64 KiB segment and moderate concurrency; the other uses a + * ~512 KiB payload with a 1 KiB segment to force many tiny blocks (stress scheduling and per-block CRC64 + * framing) without the cost of the large multi-part grids. + */ + protected static Stream fuzzyParallelUploadSmallPayloadStagingCases() { + return Stream.of(Arguments.of(200 * Constants.KB, 64L * Constants.KB, 3), + Arguments.of(512 * Constants.KB - 31, 1L * Constants.KB, 1)); + } + + /** + * payloadBytes > blockSizeBytes and payloadBytes <= 4 * Constants.MB - 1 (the ceiling field), so the blob + * stays strictly under the 4 MiB single-shot CRC64-header path while downloads remain chunked across + * multiple range GETs. + *

+ * Values mix MiB/KiB block sizes with offsets (e.g. + 19, - 903) so part counts and last-block lengths are + * not powers of two; the last rows hug ceiling with awkward divisors in blockSizeBytes to stress remainder + * handling near the sub-4 MiB limit. + */ + protected static Stream fuzzyParallelDownloadSubFourMiBCases() { + final int ceiling = (4 * Constants.MB) - 1; + return Stream.of(Arguments.of(1 * Constants.MB + 1, 1L * Constants.MB, 1), + Arguments.of(1 * Constants.MB + 1, 2L * Constants.KB, 8), + Arguments.of((5 * Constants.MB) / 4 + 19, 256L * Constants.KB, 4), + Arguments.of(2 * Constants.MB - 903, 1L * Constants.MB, 2), + Arguments.of(2 * Constants.MB + 33, 1L * Constants.KB, 1), + Arguments.of(2 * Constants.MB + 33, 1L * Constants.MB, 8), + Arguments.of((11 * Constants.MB) / 4 - 17, 512L * Constants.KB, 6), + Arguments.of(3 * Constants.MB - 777, 2L * Constants.MB, 8), + Arguments.of(3 * Constants.MB - 1, 1L * Constants.MB, 1), Arguments.of(ceiling - 511, 1L * Constants.MB, 4), + Arguments.of(ceiling - 511, 1L * Constants.MB + 511, 2), + Arguments.of(ceiling, (long) (ceiling / 7 + 17), 3), Arguments.of(ceiling, (long) (ceiling / 2 + 1), 8)); + } + + /** + * payloadBytes > segmentBytes and payloadBytes <= 4 * Constants.MB - 1 (the ceiling field), so the blob + * stays strictly under the 4 MiB transactional CRC64-header path while uploads remain + * chunked—live-only because of Put Block identity churn. + *

+ * Values mix MiB/KiB segment sizes with offsets (e.g. + 19, - 903) so part counts and last-block + * lengths are not powers of two; the last rows hug ceiling with awkward divisors in segmentBytes to + * stress remainder handling near the sub-4 MiB limit. + */ + protected static Stream fuzzyParallelUploadSub4MiBCases() { + final int ceiling = (4 * Constants.MB) - 1; + return Stream.of(Arguments.of(1 * Constants.MB + 1, 1L * Constants.MB, 1), + Arguments.of(1 * Constants.MB + 1, 2L * Constants.KB, 8), + Arguments.of((5 * Constants.MB) / 4 + 19, 256L * Constants.KB, 4), + Arguments.of(2 * Constants.MB - 903, 1L * Constants.MB, 2), + Arguments.of(2 * Constants.MB + 33, 1L * Constants.KB, 1), + Arguments.of(2 * Constants.MB + 33, 1L * Constants.MB, 8), + Arguments.of((11 * Constants.MB) / 4 - 17, 512L * Constants.KB, 6), + Arguments.of(3 * Constants.MB - 777, 2L * Constants.MB, 8), + Arguments.of(3 * Constants.MB - 1, 1L * Constants.MB, 1), Arguments.of(ceiling - 511, 1L * Constants.MB, 4), + Arguments.of(ceiling - 511, 1L * Constants.MB + 511, 2), + Arguments.of(ceiling, (long) (ceiling / 7 + 17), 3), Arguments.of(ceiling, (long) (ceiling / 2 + 1), 8)); + } + + /** + * Centers on 4 * Constants.MB - 1, exactly 4 * Constants.MB, and just above 4 MiB, with block sizes spanning + * KiB through multi-MiB - exercising the SDK/service boundary where single-shot vs chunked range GET and + * CRC64 header vs structured-message rules flip, while keeping deterministic single-GET coverage in the + * replayable supplier above. + *

+ * Includes near-boundary payloads (e.g. -8192, +31, +8191 from 4 MiB) so neither total size nor last segment + * length aligns to typical buffer multiples. + */ + protected static Stream fuzzyParallelDownloadFourMiBBoundaryCases() { + final int minus = (4 * Constants.MB) - 1; + final int plus = (4 * Constants.MB) + 1; + return Stream.of(Arguments.of(minus, 1L * Constants.MB, 1), Arguments.of(minus, 512L * Constants.KB, 6), + Arguments.of(minus, 2L * Constants.MB, 8), Arguments.of((4 * Constants.MB) - 8192, 1L * Constants.KB, 4), + Arguments.of(4 * Constants.MB, (long) (4 * Constants.MB / 2), 8), + Arguments.of(4 * Constants.MB, 256L * Constants.KB, 2), Arguments.of(plus, 1L * Constants.MB, 1), + Arguments.of(plus, 2L * Constants.MB, 8), Arguments.of(plus, 1L * Constants.KB, 7), + Arguments.of((4 * Constants.MB) + 31, 511L * Constants.KB + 409, 4), + Arguments.of((4 * Constants.MB) + 8191, 3L * Constants.MB - 413, 6)); + } + + /** + * Centers on 4 * Constants.MB - 1, exactly 4 * Constants.MB, and just above 4 MiB, with segment + * sizes spanning KiB through multi-MiB—exercising the SDK/service boundary where single-shot vs block staging and + * CRC64 header vs structured-message rules flip, while keeping deterministic Put Blob coverage in the replayable + * supplier above. + *

+ * Includes near-boundary payloads (e.g. -8192, +31, +8191 from 4 MiB) so neither total size nor last segment + * length aligns to typical buffer multiples. + */ + protected static Stream fuzzyParallelUploadFourMiBBoundaryStagingCases() { + final int minus = (4 * Constants.MB) - 1; + final int plus = (4 * Constants.MB) + 1; + return Stream.of(Arguments.of(minus, 1L * Constants.MB, 1), Arguments.of(minus, 512L * Constants.KB, 6), + Arguments.of(minus, 2L * Constants.MB, 8), Arguments.of((4 * Constants.MB) - 8192, 1L * Constants.KB, 4), + Arguments.of(4 * Constants.MB, (long) (4 * Constants.MB / 2), 8), + Arguments.of(4 * Constants.MB, 256L * Constants.KB, 2), Arguments.of(plus, 1L * Constants.MB, 1), + Arguments.of(plus, 2L * Constants.MB, 8), Arguments.of(plus, 1L * Constants.KB, 7), + Arguments.of((4 * Constants.MB) + 31, 511L * Constants.KB + 409, 4), + Arguments.of((4 * Constants.MB) + 8191, 3L * Constants.MB - 413, 6)); + } + + /** + * payloadBytes > blockSizeBytes, so downloads always go through multiple ranged GETs (parallel download + * fan-out) with totals roughly 6-80 MiB. Large enough to exercise the structured-message decoder over + * multiple HTTP responses, but cheaper than {@link #fuzzyParallelDownloadLargeMultiPartCases}. + *

+ * Block sizes step through common service limits (1-8 MiB, half-MiB tail values); concurrency 1-8 pairs + * with imbalanced payloads (e.g. 701, 333) to flush merge/retry edge cases. + */ + protected static Stream fuzzyParallelDownloadMediumMultiPartCases() { + return Stream.of(Arguments.of(6 * Constants.MB + 701, Constants.MB, 1), + Arguments.of(6 * Constants.MB + 701, 3L * Constants.MB + 271, 4), + Arguments.of(9 * Constants.MB + 333, 2L * Constants.MB, 1), + Arguments.of(9 * Constants.MB + 333, 3L * Constants.MB + 199, 8), + Arguments.of(12 * Constants.MB + 901, 4L * Constants.MB + 901, 2), + Arguments.of(14 * Constants.MB, 500L * Constants.KB + 13, 6), + Arguments.of(18 * Constants.MB - 4021, 5L * Constants.MB - 701, 3), + Arguments.of(24 * Constants.MB, 8L * Constants.MB, 8), + Arguments.of(28 * Constants.MB + 56789, 7L * Constants.MB + 13, 2), + Arguments.of(31 * Constants.MB, 1024L * Constants.KB + 17, 4), + Arguments.of(40 * Constants.MB + 12345, 7L * Constants.MB + 13, 3), + Arguments.of(48 * Constants.MB - 777, 5L * Constants.MB + 809L * Constants.KB, 6), + Arguments.of(56 * Constants.MB + 19, 9L * Constants.MB + 4096, 8), + Arguments.of(72 * Constants.MB, 4L * Constants.MB + 65536, 8), + Arguments.of(80 * Constants.MB + 321, 13L * Constants.MB - 3073, 1)); + } + + /** + * All rows keep payloadBytes > segmentBytes with totals roughly 6–80 MiB—large enough for meaningful parallel + * block fan-out and structured-message segments, but cheaper than {@link #fuzzyParallelUploadLargeMultiPartCases}. + *

+ * Block sizes step through common service limits (1–8 MiB, half-MiB tail values); concurrency 1–8 pairs with + * imbalanced payloads (e.g. 701, 333) to flush merge/retry edge cases. + */ + protected static Stream fuzzyParallelUploadMediumMultiPartCases() { + return Stream.of(Arguments.of(6 * Constants.MB + 701, Constants.MB, 1), + Arguments.of(6 * Constants.MB + 701, 3L * Constants.MB + 271, 4), + Arguments.of(9 * Constants.MB + 333, 2L * Constants.MB, 1), + Arguments.of(9 * Constants.MB + 333, 3L * Constants.MB + 199, 8), + Arguments.of(12 * Constants.MB + 901, 4L * Constants.MB + 901, 2), + Arguments.of(14 * Constants.MB, 500L * Constants.KB + 13, 6), + Arguments.of(18 * Constants.MB - 4021, 5L * Constants.MB - 701, 3), + Arguments.of(24 * Constants.MB, 8L * Constants.MB, 8), + Arguments.of(28 * Constants.MB + 56789, 7L * Constants.MB + 13, 2), + Arguments.of(31 * Constants.MB, 1024L * Constants.KB + 17, 4), + Arguments.of(40 * Constants.MB + 12345, 7L * Constants.MB + 13, 3), + Arguments.of(48 * Constants.MB - 777, 5L * Constants.MB + 809L * Constants.KB, 6), + Arguments.of(56 * Constants.MB + 19, 9L * Constants.MB + 4096, 8), + Arguments.of(72 * Constants.MB, 4L * Constants.MB + 65536, 8), + Arguments.of(80 * Constants.MB + 321, 13L * Constants.MB - 3073, 1)); + } + + /** + * Stresses high block counts and long-running parallel downloads (~96-320 MiB payloads) with service-realistic + * block sizes (8-61 MiB class) and heavy concurrency. + *

+ * The final rows use named near-256/288/320 MiB totals with irregular byte tails to keep total bytes and block + * remainders off common multiples while still bounding runtime for Live-only CI. + */ + protected static Stream fuzzyParallelDownloadLargeMultiPartCases() { + final int payload257MiBPlus = (int) (257L * Constants.MB + 18881); + final int payload288MiBPlus = (int) (288L * Constants.MB + 7777); + final int payload320MiBPlus = (int) (320L * Constants.MB + 1999); + return Stream.of(Arguments.of(96 * Constants.MB + 17, 8L * Constants.MB + 511, 2), + Arguments.of(112 * Constants.MB, 15L * Constants.MB + 4096, 8), + Arguments.of(128 * Constants.MB + 45673, 17L * Constants.MB - 11264 + 173, 4), + Arguments.of(160 * Constants.MB + 12345, 12L * Constants.MB + 8192, 8), + Arguments.of(192 * Constants.MB + 9876, 31L * Constants.MB - 513, 8), + Arguments.of(224 * Constants.MB, 23L * Constants.MB + 524288, 8), + Arguments.of(payload257MiBPlus, 61L * Constants.MB + 23L * Constants.KB, 6), + Arguments.of(payload288MiBPlus, 36L * Constants.MB + 513, 8), + Arguments.of(payload320MiBPlus, 16L * Constants.MB + 511, 8)); + } + + /** + * Stresses high block counts and long-running parallel uploads (~96–320 MiB payloads) with service-realistic segment + * sizes (8–61 MiB class) and heavy concurrency. + *

+ * The final rows use named near-256/288/320 MiB totals with irregular byte tails to keep total bytes and + * block remainders off common multiples while still bounding runtime for Live-only CI. + */ + protected static Stream fuzzyParallelUploadLargeMultiPartCases() { + final int payload257MiBPlus = (int) (257L * Constants.MB + 18881); + final int payload288MiBPlus = (int) (288L * Constants.MB + 7777); + final int payload320MiBPlus = (int) (320L * Constants.MB + 1999); + return Stream.of(Arguments.of(96 * Constants.MB + 17, 8L * Constants.MB + 511, 2), + Arguments.of(112 * Constants.MB, 15L * Constants.MB + 4096, 8), + Arguments.of(128 * Constants.MB + 45673, 17L * Constants.MB - 11264 + 173, 4), + Arguments.of(160 * Constants.MB + 12345, 12L * Constants.MB + 8192, 8), + Arguments.of(192 * Constants.MB + 9876, 31L * Constants.MB - 513, 8), + Arguments.of(224 * Constants.MB, 23L * Constants.MB + 524288, 8), + Arguments.of(payload257MiBPlus, 61L * Constants.MB + 23L * Constants.KB, 6), + Arguments.of(payload288MiBPlus, 36L * Constants.MB + 513, 8), + Arguments.of(payload320MiBPlus, 16L * Constants.MB + 511, 8)); + } + + /** + * Single ~1 GiB download with high concurrency and an awkward (non-aligned) tail to exercise the structured + * message decoder under a sustained, fan-out-heavy parallel download. Live-only and file-backed so payload + * never materializes twice in heap. + */ + protected static Stream fuzzyParallelDownloadOneGiBCases() { + return Stream.of(Arguments.of((int) (1L * Constants.GB + 1377), 16L * Constants.MB + 511, 8)); + } + + private static boolean hasValidStructuredContentLengthHeader(HttpHeaders headers) { + String structuredContentLength = headers.getValue("x-ms-structured-content-length"); + if (CoreUtils.isNullOrEmpty(structuredContentLength) + || CoreUtils.isNullOrEmpty(structuredContentLength.trim())) { + return false; + } + try { + return Long.parseLong(structuredContentLength.trim()) >= 0; + } catch (NumberFormatException ex) { + return false; + } + } } diff --git a/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/specialized/BlobSeekableByteChannelTests.java b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/specialized/BlobSeekableByteChannelTests.java index 4ff8054894a6..24a9ca7e781a 100644 --- a/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/specialized/BlobSeekableByteChannelTests.java +++ b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/specialized/BlobSeekableByteChannelTests.java @@ -106,7 +106,7 @@ static Stream channelReadDataSupplier() { * @param copySize Size of array to copy contents with. * @return Total number of bytes read from src. */ - private static int copy(SeekableByteChannel src, OutputStream dst, int copySize) throws IOException { + public static int copy(SeekableByteChannel src, OutputStream dst, int copySize) throws IOException { int read; int totalRead = 0; byte[] temp = new byte[copySize]; diff --git a/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/specialized/BlockBlobApiTests.java b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/specialized/BlockBlobApiTests.java index c2a903145440..ba9d985b308f 100644 --- a/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/specialized/BlockBlobApiTests.java +++ b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/specialized/BlockBlobApiTests.java @@ -210,7 +210,7 @@ public void stageBlockIllegalArguments(boolean getBlockId, InputStream stream, i private static Stream stageBlockIllegalArgumentsSupplier() { return Stream.of( - Arguments.of(false, DATA.getDefaultInputStream(), DATA.getDefaultDataSize(), BlobStorageException.class), + Arguments.of(false, DATA.getDefaultInputStream(), DATA.getDefaultDataSize(), NullPointerException.class), Arguments.of(true, null, DATA.getDefaultDataSize(), NullPointerException.class), Arguments.of(true, DATA.getDefaultInputStream(), DATA.getDefaultDataSize() + 1, UnexpectedLengthException.class), diff --git a/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/specialized/BlockBlobAsyncApiTests.java b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/specialized/BlockBlobAsyncApiTests.java index 664fe555846d..184f4eed38ed 100644 --- a/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/specialized/BlockBlobAsyncApiTests.java +++ b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/specialized/BlockBlobAsyncApiTests.java @@ -250,7 +250,7 @@ public void stageBlockIllegalArguments(boolean getBlockId, Flux stre private static Stream stageBlockIllegalArgumentsSupplier() { return Stream.of( - Arguments.of(false, DATA.getDefaultFlux(), DATA.getDefaultDataSize(), BlobStorageException.class), + Arguments.of(false, DATA.getDefaultFlux(), DATA.getDefaultDataSize(), NullPointerException.class), Arguments.of(true, null, DATA.getDefaultDataSize(), NullPointerException.class), Arguments.of(true, DATA.getDefaultFlux(), DATA.getDefaultDataSize() + 1, UnexpectedLengthException.class), Arguments.of(true, DATA.getDefaultFlux(), DATA.getDefaultDataSize() - 1, UnexpectedLengthException.class)); diff --git a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/ContentValidationAlgorithm.java b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/ContentValidationAlgorithm.java new file mode 100644 index 000000000000..0d1a7ea22b07 --- /dev/null +++ b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/ContentValidationAlgorithm.java @@ -0,0 +1,31 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.common; + +/** + * Algorithm for content validation on upload and download operations. When enabled, the SDK computes checksums and + * validates content integrity using the selected algorithm. Content validation is off by default. + *

+ * Supported in Azure Storage Blob, Data Lake, and File Share packages for methods that use APIs supporting + * transactional CRC64, or structured message format. + */ +public enum ContentValidationAlgorithm { + + /** + * No content validation. This is the default; no checksum is computed or validated. + */ + NONE, + + /** + * Allow the SDK to choose the validation algorithm. Currently resolves to CRC64 where supported. Different + * library versions may make different choices. The resolution may change in the future. Please set an + * explicit algorithm if you need a specific behavior. + */ + AUTO, + + /** + * Azure Storage custom 64-bit CRC. The SDK computes and validates CRC64 checksums for the transfer. + */ + CRC64 +} diff --git a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/Constants.java b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/Constants.java index 8d1d66b59126..687e80c71961 100644 --- a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/Constants.java +++ b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/Constants.java @@ -250,6 +250,20 @@ public static final class HeaderConstants { public static final String ETAG_WILDCARD = "*"; + public static final String CONTENT_CRC64 = "x-ms-content-crc64"; + + public static final HttpHeaderName CONTENT_CRC64_HEADER_NAME = HttpHeaderName.fromString(CONTENT_CRC64); + + public static final String STRUCTURED_BODY_TYPE = "x-ms-structured-body"; + + public static final HttpHeaderName STRUCTURED_BODY_TYPE_HEADER_NAME + = HttpHeaderName.fromString(STRUCTURED_BODY_TYPE); + + public static final String STRUCTURED_CONTENT_LENGTH = "x-ms-structured-content-length"; + + public static final HttpHeaderName STRUCTURED_CONTENT_LENGTH_HEADER_NAME + = HttpHeaderName.fromString(STRUCTURED_CONTENT_LENGTH); + /** * Metadata key ("hdi_isfolder") used to mark virtual directories in Azure Blob Storage. * diff --git a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/UploadUtils.java b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/UploadUtils.java index 32e10374b249..0b81cff41227 100644 --- a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/UploadUtils.java +++ b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/UploadUtils.java @@ -98,7 +98,9 @@ public static Flux chunkSource(Flux data, ParallelTransf } int numSplits = (int) Math.ceil(buffer.remaining() / (double) chunkSize); return Flux.range(0, numSplits).map(i -> { - ByteBuffer duplicate = buffer.duplicate().asReadOnlyBuffer(); + // While duplicate.asReadOnlyBuffer() is safer, it significantly slows down crc64 calculation because it forces a copy of the buffer. + // No downstream buffers should be modifying the buffer anyways. + ByteBuffer duplicate = buffer.duplicate(); duplicate.position(i * chunkSize); duplicate.limit(Math.min(duplicate.limit(), (i + 1) * chunkSize)); return duplicate; diff --git a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/ContentValidationModeResolver.java b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/ContentValidationModeResolver.java new file mode 100644 index 000000000000..5c3394117971 --- /dev/null +++ b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/ContentValidationModeResolver.java @@ -0,0 +1,168 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.common.implementation.contentvalidation; + +import static com.azure.storage.common.implementation.contentvalidation.StructuredMessageConstants.CONTENT_VALIDATION_MODE_KEY; +import static com.azure.storage.common.implementation.contentvalidation.StructuredMessageConstants.MAXIMUM_SINGLE_SHOT_UPLOAD_SIZE_TO_USE_CRC64_HEADER; +import static com.azure.storage.common.implementation.contentvalidation.StructuredMessageConstants.USE_CRC64_CHECKSUM_HEADER_CONTEXT; +import static com.azure.storage.common.implementation.contentvalidation.StructuredMessageConstants.USE_STRUCTURED_MESSAGE_CONTEXT; +import static com.azure.storage.common.implementation.contentvalidation.StructuredMessageConstants.STRUCTURED_MESSAGE_DECODING_CONTEXT_KEY; + +import com.azure.core.util.Context; +import com.azure.core.util.FluxUtil; +import com.azure.core.util.ProgressListener; +import com.azure.storage.common.ContentValidationAlgorithm; +import com.azure.storage.common.ParallelTransferOptions; +import reactor.core.publisher.Mono; + +/** + * Determines the content validation mode string to pass in the pipeline context for upload operations. + * Callers put the returned value under {@link StructuredMessageConstants#CONTENT_VALIDATION_MODE_KEY}. + *

+ * Single-shot: use CRC64 header when length < 4 MiB, otherwise structured message. + * Chunked (multi-shot): always use structured message. + */ +public final class ContentValidationModeResolver { + + private ContentValidationModeResolver() { + } + + public static final String CONFLICTING_TRANSACTIONAL_CONTENT_VALIDATION_MESSAGE + = "Individual MD5 option and checksum algorithm option bag are both used. Only one form of transactional content validation may be used."; + + /** + * Progress reporting counts bytes on the wire; transfer validation (CRC64/AUTO) may use structured messages, so the + * two cannot be combined. Use {@link ContentValidationAlgorithm#NONE} or null, or omit the progress listener. + */ + public static final String PROGRESS_CONFLICTS_TRANSFER_CONTENT_VALIDATION_MESSAGE + = "Progress reporting cannot be combined with ContentValidationAlgorithm.CRC64 or ContentValidationAlgorithm.AUTO. " + + "Set ContentValidationAlgorithm to NONE or null, or remove the progress listener."; + + /** + * Resolves content validation mode and adds it to the Azure {@link Context} when non-null. + * + * @param context The request context; {@code null} is treated as {@link Context#NONE}. + * @param algorithm The transfer validation checksum algorithm. + * @param contentLength The upload length in bytes. + * @param chunkedUpload Whether this request is part of a multi-shot upload. + * @return The context, with {@link StructuredMessageConstants#CONTENT_VALIDATION_MODE_KEY} set when applicable. + */ + public static Context addContentValidationMode(Context context, ContentValidationAlgorithm algorithm, + long contentLength, boolean chunkedUpload) { + Context baseContext = context == null ? Context.NONE : context; + String mode + = chunkedUpload ? getModeForChunkedUpload(algorithm) : getModeForSingleShotUpload(algorithm, contentLength); + return mode == null ? baseContext : baseContext.addData(CONTENT_VALIDATION_MODE_KEY, mode); + } + + /** + * Resolves content validation mode and propagates it on the Reactor context for {@code mono} when non-null. + * + * @param mono The reactive sequence to augment. + * @param algorithm The transfer validation checksum algorithm. + * @param contentLength The upload length in bytes. + * @param chunkedUpload Whether this request is part of a multi-shot upload. + * @param The type of the elements in the reactive sequence. + * @return {@code mono}, possibly augmented with Reactor context writes. + */ + public static Mono addContentValidationMode(Mono mono, ContentValidationAlgorithm algorithm, + long contentLength, boolean chunkedUpload) { + String mode + = chunkedUpload ? getModeForChunkedUpload(algorithm) : getModeForSingleShotUpload(algorithm, contentLength); + if (mode == null) { + return mono; + } + return mono.contextWrite(FluxUtil.toReactorContext(new Context(CONTENT_VALIDATION_MODE_KEY, mode))); + } + + /** + * Mode for a single-shot upload. Use CRC64 header when length is less than 4MB, otherwise structured + * message. + */ + private static String getModeForSingleShotUpload(ContentValidationAlgorithm algorithm, long length) { + if (isCrc64OrAuto(algorithm)) { + return length < MAXIMUM_SINGLE_SHOT_UPLOAD_SIZE_TO_USE_CRC64_HEADER + ? USE_CRC64_CHECKSUM_HEADER_CONTEXT + : USE_STRUCTURED_MESSAGE_CONTEXT; + } + return null; + } + + /** + * Mode for a chunked (multi-shot) upload. Always use structured message. + */ + private static String getModeForChunkedUpload(ContentValidationAlgorithm algorithm) { + if (isCrc64OrAuto(algorithm)) { + return USE_STRUCTURED_MESSAGE_CONTEXT; + } + return null; + } + + /** + * Validates transactional checksum options when MD5 may be SDK-computed. Throws if {@code computeMd5} and a + * non-none {@code contentValidationAlgorithm} are both active. + * + * @param computeMd5 Whether the SDK will compute transactional MD5. + * @param contentValidationAlgorithm Transfer validation checksum algorithm from options. + * @throws IllegalArgumentException if options conflict. + */ + public static void validateTransactionalChecksumOptions(boolean computeMd5, + ContentValidationAlgorithm contentValidationAlgorithm) { + if (computeMd5 && contentValidationAlgorithm != null) { + throw new IllegalArgumentException(CONFLICTING_TRANSACTIONAL_CONTENT_VALIDATION_MESSAGE); + } + } + + /** + * @return {@code true} when {@code contentValidationAlgorithm} enables CRC64 or AUTO transfer validation (not + * {@code null} and not {@link ContentValidationAlgorithm#NONE}). + */ + public static boolean isContentValidationAlgorithmPresent(ContentValidationAlgorithm contentValidationAlgorithm) { + return contentValidationAlgorithm != null && contentValidationAlgorithm != ContentValidationAlgorithm.NONE; + } + + /** + * @return {@code true} when {@code algorithm} is {@link ContentValidationAlgorithm#CRC64} or + * {@link ContentValidationAlgorithm#AUTO}. Upload and download structured-message validation use this rule. + */ + public static boolean isCrc64OrAuto(ContentValidationAlgorithm algorithm) { + return algorithm == ContentValidationAlgorithm.CRC64 || algorithm == ContentValidationAlgorithm.AUTO; + } + + /** + * When the transfer validation mode is {@link ContentValidationAlgorithm#CRC64} or + * {@link ContentValidationAlgorithm#AUTO}, adds + * {@link StructuredMessageConstants#STRUCTURED_MESSAGE_DECODING_CONTEXT_KEY} so the HTTP + * pipeline can decode/validate the structured message response. For {@code null} or + * {@link ContentValidationAlgorithm#NONE}, returns the context unchanged (no key added), matching "no + * structured-message validation" for that download. + * + * @param context The base {@link Context}; null is treated as {@link Context#NONE}. + * @param contentValidationAlgorithm The algorithm from download options, or null. + * @return The same context, or a copy with the decoding key set when applicable. + */ + public static Context addStructuredMessageDecodingToContext(Context context, + ContentValidationAlgorithm contentValidationAlgorithm) { + Context base = context == null ? Context.NONE : context; + if (!isCrc64OrAuto(contentValidationAlgorithm)) { + return base; + } + return base.addData(STRUCTURED_MESSAGE_DECODING_CONTEXT_KEY, true); + } + + /** + * Validates that progress reporting is not combined with CRC64/AUTO content validation. + * + * @param progressListener Progress listener from {@link ParallelTransferOptions} or equivalent; may be null. + * @param contentValidationAlgorithm Transfer validation algorithm from options. + * @throws IllegalArgumentException if {@code progressListener} is non-null and {@link #isContentValidationAlgorithmPresent} + * is true. + */ + public static void validateProgressWithContentValidation(ProgressListener progressListener, + ContentValidationAlgorithm contentValidationAlgorithm) { + if (progressListener != null && isContentValidationAlgorithmPresent(contentValidationAlgorithm)) { + throw new IllegalArgumentException(PROGRESS_CONFLICTS_TRANSFER_CONTENT_VALIDATION_MESSAGE); + } + } +} diff --git a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/StorageCrc64Calculator.java b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/StorageCrc64Calculator.java new file mode 100644 index 000000000000..ed21ef153823 --- /dev/null +++ b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/StorageCrc64Calculator.java @@ -0,0 +1,2660 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.common.implementation.contentvalidation; + +import java.nio.ByteBuffer; +import java.nio.ByteOrder; + +/** + * This class provides methods to compute and manipulate CRC64 checksums using the Azure Storage CRC64 polynomial. + * It includes methods for computing CRC64 checksums for byte arrays, updating CRC values using lookup tables, and + * concatenating CRC values. + *

+ * RESERVED FOR INTERNAL USE. + * + */ +public class StorageCrc64Calculator { + + private static final long POLY = 0x9A6C9329AC4BC9B5L; + + private static final long[] M_U1 = { + 0x0000000000000000L, + 0x7f6ef0c830358979L, + 0xfedde190606b12f2L, + 0x81b31158505e9b8bL, + 0xc962e5739841b68fL, + 0xb60c15bba8743ff6L, + 0x37bf04e3f82aa47dL, + 0x48d1f42bc81f2d04L, + 0xa61cecb46814fe75L, + 0xd9721c7c5821770cL, + 0x58c10d24087fec87L, + 0x27affdec384a65feL, + 0x6f7e09c7f05548faL, + 0x1010f90fc060c183L, + 0x91a3e857903e5a08L, + 0xeecd189fa00bd371L, + 0x78e0ff3b88be6f81L, + 0x078e0ff3b88be6f8L, + 0x863d1eabe8d57d73L, + 0xf953ee63d8e0f40aL, + 0xb1821a4810ffd90eL, + 0xceecea8020ca5077L, + 0x4f5ffbd87094cbfcL, + 0x30310b1040a14285L, + 0xdefc138fe0aa91f4L, + 0xa192e347d09f188dL, + 0x2021f21f80c18306L, + 0x5f4f02d7b0f40a7fL, + 0x179ef6fc78eb277bL, + 0x68f0063448deae02L, + 0xe943176c18803589L, + 0x962de7a428b5bcf0L, + 0xf1c1fe77117cdf02L, + 0x8eaf0ebf2149567bL, + 0x0f1c1fe77117cdf0L, + 0x7072ef2f41224489L, + 0x38a31b04893d698dL, + 0x47cdebccb908e0f4L, + 0xc67efa94e9567b7fL, + 0xb9100a5cd963f206L, + 0x57dd12c379682177L, + 0x28b3e20b495da80eL, + 0xa900f35319033385L, + 0xd66e039b2936bafcL, + 0x9ebff7b0e12997f8L, + 0xe1d10778d11c1e81L, + 0x606216208142850aL, + 0x1f0ce6e8b1770c73L, + 0x8921014c99c2b083L, + 0xf64ff184a9f739faL, + 0x77fce0dcf9a9a271L, + 0x08921014c99c2b08L, + 0x4043e43f0183060cL, + 0x3f2d14f731b68f75L, + 0xbe9e05af61e814feL, + 0xc1f0f56751dd9d87L, + 0x2f3dedf8f1d64ef6L, + 0x50531d30c1e3c78fL, + 0xd1e00c6891bd5c04L, + 0xae8efca0a188d57dL, + 0xe65f088b6997f879L, + 0x9931f84359a27100L, + 0x1882e91b09fcea8bL, + 0x67ec19d339c963f2L, + 0xd75adabd7a6e2d6fL, + 0xa8342a754a5ba416L, + 0x29873b2d1a053f9dL, + 0x56e9cbe52a30b6e4L, + 0x1e383fcee22f9be0L, + 0x6156cf06d21a1299L, + 0xe0e5de5e82448912L, + 0x9f8b2e96b271006bL, + 0x71463609127ad31aL, + 0x0e28c6c1224f5a63L, + 0x8f9bd7997211c1e8L, + 0xf0f5275142244891L, + 0xb824d37a8a3b6595L, + 0xc74a23b2ba0eececL, + 0x46f932eaea507767L, + 0x3997c222da65fe1eL, + 0xafba2586f2d042eeL, + 0xd0d4d54ec2e5cb97L, + 0x5167c41692bb501cL, + 0x2e0934dea28ed965L, + 0x66d8c0f56a91f461L, + 0x19b6303d5aa47d18L, + 0x980521650afae693L, + 0xe76bd1ad3acf6feaL, + 0x09a6c9329ac4bc9bL, + 0x76c839faaaf135e2L, + 0xf77b28a2faafae69L, + 0x8815d86aca9a2710L, + 0xc0c42c4102850a14L, + 0xbfaadc8932b0836dL, + 0x3e19cdd162ee18e6L, + 0x41773d1952db919fL, + 0x269b24ca6b12f26dL, + 0x59f5d4025b277b14L, + 0xd846c55a0b79e09fL, + 0xa72835923b4c69e6L, + 0xeff9c1b9f35344e2L, + 0x90973171c366cd9bL, + 0x1124202993385610L, + 0x6e4ad0e1a30ddf69L, + 0x8087c87e03060c18L, + 0xffe938b633338561L, + 0x7e5a29ee636d1eeaL, + 0x0134d92653589793L, + 0x49e52d0d9b47ba97L, + 0x368bddc5ab7233eeL, + 0xb738cc9dfb2ca865L, + 0xc8563c55cb19211cL, + 0x5e7bdbf1e3ac9decL, + 0x21152b39d3991495L, + 0xa0a63a6183c78f1eL, + 0xdfc8caa9b3f20667L, + 0x97193e827bed2b63L, + 0xe877ce4a4bd8a21aL, + 0x69c4df121b863991L, + 0x16aa2fda2bb3b0e8L, + 0xf86737458bb86399L, + 0x8709c78dbb8deae0L, + 0x06bad6d5ebd3716bL, + 0x79d4261ddbe6f812L, + 0x3105d23613f9d516L, + 0x4e6b22fe23cc5c6fL, + 0xcfd833a67392c7e4L, + 0xb0b6c36e43a74e9dL, + 0x9a6c9329ac4bc9b5L, + 0xe50263e19c7e40ccL, + 0x64b172b9cc20db47L, + 0x1bdf8271fc15523eL, + 0x530e765a340a7f3aL, + 0x2c608692043ff643L, + 0xadd397ca54616dc8L, + 0xd2bd67026454e4b1L, + 0x3c707f9dc45f37c0L, + 0x431e8f55f46abeb9L, + 0xc2ad9e0da4342532L, + 0xbdc36ec59401ac4bL, + 0xf5129aee5c1e814fL, + 0x8a7c6a266c2b0836L, + 0x0bcf7b7e3c7593bdL, + 0x74a18bb60c401ac4L, + 0xe28c6c1224f5a634L, + 0x9de29cda14c02f4dL, + 0x1c518d82449eb4c6L, + 0x633f7d4a74ab3dbfL, + 0x2bee8961bcb410bbL, + 0x548079a98c8199c2L, + 0xd53368f1dcdf0249L, + 0xaa5d9839ecea8b30L, + 0x449080a64ce15841L, + 0x3bfe706e7cd4d138L, + 0xba4d61362c8a4ab3L, + 0xc52391fe1cbfc3caL, + 0x8df265d5d4a0eeceL, + 0xf29c951de49567b7L, + 0x732f8445b4cbfc3cL, + 0x0c41748d84fe7545L, + 0x6bad6d5ebd3716b7L, + 0x14c39d968d029fceL, + 0x95708ccedd5c0445L, + 0xea1e7c06ed698d3cL, + 0xa2cf882d2576a038L, + 0xdda178e515432941L, + 0x5c1269bd451db2caL, + 0x237c997575283bb3L, + 0xcdb181ead523e8c2L, + 0xb2df7122e51661bbL, + 0x336c607ab548fa30L, + 0x4c0290b2857d7349L, + 0x04d364994d625e4dL, + 0x7bbd94517d57d734L, + 0xfa0e85092d094cbfL, + 0x856075c11d3cc5c6L, + 0x134d926535897936L, + 0x6c2362ad05bcf04fL, + 0xed9073f555e26bc4L, + 0x92fe833d65d7e2bdL, + 0xda2f7716adc8cfb9L, + 0xa54187de9dfd46c0L, + 0x24f29686cda3dd4bL, + 0x5b9c664efd965432L, + 0xb5517ed15d9d8743L, + 0xca3f8e196da80e3aL, + 0x4b8c9f413df695b1L, + 0x34e26f890dc31cc8L, + 0x7c339ba2c5dc31ccL, + 0x035d6b6af5e9b8b5L, + 0x82ee7a32a5b7233eL, + 0xfd808afa9582aa47L, + 0x4d364994d625e4daL, + 0x3258b95ce6106da3L, + 0xb3eba804b64ef628L, + 0xcc8558cc867b7f51L, + 0x8454ace74e645255L, + 0xfb3a5c2f7e51db2cL, + 0x7a894d772e0f40a7L, + 0x05e7bdbf1e3ac9deL, + 0xeb2aa520be311aafL, + 0x944455e88e0493d6L, + 0x15f744b0de5a085dL, + 0x6a99b478ee6f8124L, + 0x224840532670ac20L, + 0x5d26b09b16452559L, + 0xdc95a1c3461bbed2L, + 0xa3fb510b762e37abL, + 0x35d6b6af5e9b8b5bL, + 0x4ab846676eae0222L, + 0xcb0b573f3ef099a9L, + 0xb465a7f70ec510d0L, + 0xfcb453dcc6da3dd4L, + 0x83daa314f6efb4adL, + 0x0269b24ca6b12f26L, + 0x7d0742849684a65fL, + 0x93ca5a1b368f752eL, + 0xeca4aad306bafc57L, + 0x6d17bb8b56e467dcL, + 0x12794b4366d1eea5L, + 0x5aa8bf68aecec3a1L, + 0x25c64fa09efb4ad8L, + 0xa4755ef8cea5d153L, + 0xdb1bae30fe90582aL, + 0xbcf7b7e3c7593bd8L, + 0xc399472bf76cb2a1L, + 0x422a5673a732292aL, + 0x3d44a6bb9707a053L, + 0x759552905f188d57L, + 0x0afba2586f2d042eL, + 0x8b48b3003f739fa5L, + 0xf42643c80f4616dcL, + 0x1aeb5b57af4dc5adL, + 0x6585ab9f9f784cd4L, + 0xe436bac7cf26d75fL, + 0x9b584a0fff135e26L, + 0xd389be24370c7322L, + 0xace74eec0739fa5bL, + 0x2d545fb4576761d0L, + 0x523aaf7c6752e8a9L, + 0xc41748d84fe75459L, + 0xbb79b8107fd2dd20L, + 0x3acaa9482f8c46abL, + 0x45a459801fb9cfd2L, + 0x0d75adabd7a6e2d6L, + 0x721b5d63e7936bafL, + 0xf3a84c3bb7cdf024L, + 0x8cc6bcf387f8795dL, + 0x620ba46c27f3aa2cL, + 0x1d6554a417c62355L, + 0x9cd645fc4798b8deL, + 0xe3b8b53477ad31a7L, + 0xab69411fbfb21ca3L, + 0xd407b1d78f8795daL, + 0x55b4a08fdfd90e51L, + 0x2ada5047efec8728L, }; + + private static final long[] M_U32 = { + 0x0000000000000000L, + 0xb8c533c1177eb231L, + 0x455341d1766af709L, + 0xfd96721061144538L, + 0x8aa683a2ecd5ee12L, + 0x3263b063fbab5c23L, + 0xcff5c2739abf191bL, + 0x7730f1b28dc1ab2aL, + 0x21942116813c4f4fL, + 0x995112d79642fd7eL, + 0x64c760c7f756b846L, + 0xdc025306e0280a77L, + 0xab32a2b46de9a15dL, + 0x13f791757a97136cL, + 0xee61e3651b835654L, + 0x56a4d0a40cfde465L, + 0x4328422d02789e9eL, + 0xfbed71ec15062cafL, + 0x067b03fc74126997L, + 0xbebe303d636cdba6L, + 0xc98ec18feead708cL, + 0x714bf24ef9d3c2bdL, + 0x8cdd805e98c78785L, + 0x3418b39f8fb935b4L, + 0x62bc633b8344d1d1L, + 0xda7950fa943a63e0L, + 0x27ef22eaf52e26d8L, + 0x9f2a112be25094e9L, + 0xe81ae0996f913fc3L, + 0x50dfd35878ef8df2L, + 0xad49a14819fbc8caL, + 0x158c92890e857afbL, + 0x8650845a04f13d3cL, + 0x3e95b79b138f8f0dL, + 0xc303c58b729bca35L, + 0x7bc6f64a65e57804L, + 0x0cf607f8e824d32eL, + 0xb4333439ff5a611fL, + 0x49a546299e4e2427L, + 0xf16075e889309616L, + 0xa7c4a54c85cd7273L, + 0x1f01968d92b3c042L, + 0xe297e49df3a7857aL, + 0x5a52d75ce4d9374bL, + 0x2d6226ee69189c61L, + 0x95a7152f7e662e50L, + 0x6831673f1f726b68L, + 0xd0f454fe080cd959L, + 0xc578c6770689a3a2L, + 0x7dbdf5b611f71193L, + 0x802b87a670e354abL, + 0x38eeb467679de69aL, + 0x4fde45d5ea5c4db0L, + 0xf71b7614fd22ff81L, + 0x0a8d04049c36bab9L, + 0xb24837c58b480888L, + 0xe4ece76187b5ecedL, + 0x5c29d4a090cb5edcL, + 0xa1bfa6b0f1df1be4L, + 0x197a9571e6a1a9d5L, + 0x6e4a64c36b6002ffL, + 0xd68f57027c1eb0ceL, + 0x2b1925121d0af5f6L, + 0x93dc16d30a7447c7L, + 0x38782ee75175e913L, + 0x80bd1d26460b5b22L, + 0x7d2b6f36271f1e1aL, + 0xc5ee5cf73061ac2bL, + 0xb2dead45bda00701L, + 0x0a1b9e84aadeb530L, + 0xf78dec94cbcaf008L, + 0x4f48df55dcb44239L, + 0x19ec0ff1d049a65cL, + 0xa1293c30c737146dL, + 0x5cbf4e20a6235155L, + 0xe47a7de1b15de364L, + 0x934a8c533c9c484eL, + 0x2b8fbf922be2fa7fL, + 0xd619cd824af6bf47L, + 0x6edcfe435d880d76L, + 0x7b506cca530d778dL, + 0xc3955f0b4473c5bcL, + 0x3e032d1b25678084L, + 0x86c61eda321932b5L, + 0xf1f6ef68bfd8999fL, + 0x4933dca9a8a62baeL, + 0xb4a5aeb9c9b26e96L, + 0x0c609d78deccdca7L, + 0x5ac44ddcd23138c2L, + 0xe2017e1dc54f8af3L, + 0x1f970c0da45bcfcbL, + 0xa7523fccb3257dfaL, + 0xd062ce7e3ee4d6d0L, + 0x68a7fdbf299a64e1L, + 0x95318faf488e21d9L, + 0x2df4bc6e5ff093e8L, + 0xbe28aabd5584d42fL, + 0x06ed997c42fa661eL, + 0xfb7beb6c23ee2326L, + 0x43bed8ad34909117L, + 0x348e291fb9513a3dL, + 0x8c4b1adeae2f880cL, + 0x71dd68cecf3bcd34L, + 0xc9185b0fd8457f05L, + 0x9fbc8babd4b89b60L, + 0x2779b86ac3c62951L, + 0xdaefca7aa2d26c69L, + 0x622af9bbb5acde58L, + 0x151a0809386d7572L, + 0xaddf3bc82f13c743L, + 0x504949d84e07827bL, + 0xe88c7a195979304aL, + 0xfd00e89057fc4ab1L, + 0x45c5db514082f880L, + 0xb853a9412196bdb8L, + 0x00969a8036e80f89L, + 0x77a66b32bb29a4a3L, + 0xcf6358f3ac571692L, + 0x32f52ae3cd4353aaL, + 0x8a301922da3de19bL, + 0xdc94c986d6c005feL, + 0x6451fa47c1beb7cfL, + 0x99c78857a0aaf2f7L, + 0x2102bb96b7d440c6L, + 0x56324a243a15ebecL, + 0xeef779e52d6b59ddL, + 0x13610bf54c7f1ce5L, + 0xaba438345b01aed4L, + 0x70f05dcea2ebd226L, + 0xc8356e0fb5956017L, + 0x35a31c1fd481252fL, + 0x8d662fdec3ff971eL, + 0xfa56de6c4e3e3c34L, + 0x4293edad59408e05L, + 0xbf059fbd3854cb3dL, + 0x07c0ac7c2f2a790cL, + 0x51647cd823d79d69L, + 0xe9a14f1934a92f58L, + 0x14373d0955bd6a60L, + 0xacf20ec842c3d851L, + 0xdbc2ff7acf02737bL, + 0x6307ccbbd87cc14aL, + 0x9e91beabb9688472L, + 0x26548d6aae163643L, + 0x33d81fe3a0934cb8L, + 0x8b1d2c22b7edfe89L, + 0x768b5e32d6f9bbb1L, + 0xce4e6df3c1870980L, + 0xb97e9c414c46a2aaL, + 0x01bbaf805b38109bL, + 0xfc2ddd903a2c55a3L, + 0x44e8ee512d52e792L, + 0x124c3ef521af03f7L, + 0xaa890d3436d1b1c6L, + 0x571f7f2457c5f4feL, + 0xefda4ce540bb46cfL, + 0x98eabd57cd7aede5L, + 0x202f8e96da045fd4L, + 0xddb9fc86bb101aecL, + 0x657ccf47ac6ea8ddL, + 0xf6a0d994a61aef1aL, + 0x4e65ea55b1645d2bL, + 0xb3f39845d0701813L, + 0x0b36ab84c70eaa22L, + 0x7c065a364acf0108L, + 0xc4c369f75db1b339L, + 0x39551be73ca5f601L, + 0x819028262bdb4430L, + 0xd734f8822726a055L, + 0x6ff1cb4330581264L, + 0x9267b953514c575cL, + 0x2aa28a924632e56dL, + 0x5d927b20cbf34e47L, + 0xe55748e1dc8dfc76L, + 0x18c13af1bd99b94eL, + 0xa0040930aae70b7fL, + 0xb5889bb9a4627184L, + 0x0d4da878b31cc3b5L, + 0xf0dbda68d208868dL, + 0x481ee9a9c57634bcL, + 0x3f2e181b48b79f96L, + 0x87eb2bda5fc92da7L, + 0x7a7d59ca3edd689fL, + 0xc2b86a0b29a3daaeL, + 0x941cbaaf255e3ecbL, + 0x2cd9896e32208cfaL, + 0xd14ffb7e5334c9c2L, + 0x698ac8bf444a7bf3L, + 0x1eba390dc98bd0d9L, + 0xa67f0accdef562e8L, + 0x5be978dcbfe127d0L, + 0xe32c4b1da89f95e1L, + 0x48887329f39e3b35L, + 0xf04d40e8e4e08904L, + 0x0ddb32f885f4cc3cL, + 0xb51e0139928a7e0dL, + 0xc22ef08b1f4bd527L, + 0x7aebc34a08356716L, + 0x877db15a6921222eL, + 0x3fb8829b7e5f901fL, + 0x691c523f72a2747aL, + 0xd1d961fe65dcc64bL, + 0x2c4f13ee04c88373L, + 0x948a202f13b63142L, + 0xe3bad19d9e779a68L, + 0x5b7fe25c89092859L, + 0xa6e9904ce81d6d61L, + 0x1e2ca38dff63df50L, + 0x0ba03104f1e6a5abL, + 0xb36502c5e698179aL, + 0x4ef370d5878c52a2L, + 0xf636431490f2e093L, + 0x8106b2a61d334bb9L, + 0x39c381670a4df988L, + 0xc455f3776b59bcb0L, + 0x7c90c0b67c270e81L, + 0x2a34101270daeae4L, + 0x92f123d367a458d5L, + 0x6f6751c306b01dedL, + 0xd7a2620211ceafdcL, + 0xa09293b09c0f04f6L, + 0x1857a0718b71b6c7L, + 0xe5c1d261ea65f3ffL, + 0x5d04e1a0fd1b41ceL, + 0xced8f773f76f0609L, + 0x761dc4b2e011b438L, + 0x8b8bb6a28105f100L, + 0x334e8563967b4331L, + 0x447e74d11bbae81bL, + 0xfcbb47100cc45a2aL, + 0x012d35006dd01f12L, + 0xb9e806c17aaead23L, + 0xef4cd66576534946L, + 0x5789e5a4612dfb77L, + 0xaa1f97b40039be4fL, + 0x12daa47517470c7eL, + 0x65ea55c79a86a754L, + 0xdd2f66068df81565L, + 0x20b91416ecec505dL, + 0x987c27d7fb92e26cL, + 0x8df0b55ef5179897L, + 0x3535869fe2692aa6L, + 0xc8a3f48f837d6f9eL, + 0x7066c74e9403ddafL, + 0x075636fc19c27685L, + 0xbf93053d0ebcc4b4L, + 0x4205772d6fa8818cL, + 0xfac044ec78d633bdL, + 0xac649448742bd7d8L, + 0x14a1a789635565e9L, + 0xe937d599024120d1L, + 0x51f2e658153f92e0L, + 0x26c217ea98fe39caL, + 0x9e07242b8f808bfbL, + 0x6391563bee94cec3L, + 0xdb5465faf9ea7cf2L, + + 0x0000000000000000L, + 0xf6f734b768e04748L, + 0xd9374f3d89571dfbL, + 0x2fc07b8ae1b75ab3L, + 0x86b7b8284a39a89dL, + 0x70408c9f22d9efd5L, + 0x5f80f715c36eb566L, + 0xa977c3a2ab8ef22eL, + 0x39b65603cce4c251L, + 0xcf4162b4a4048519L, + 0xe081193e45b3dfaaL, + 0x16762d892d5398e2L, + 0xbf01ee2b86dd6accL, + 0x49f6da9cee3d2d84L, + 0x6636a1160f8a7737L, + 0x90c195a1676a307fL, + 0x736cac0799c984a2L, + 0x859b98b0f129c3eaL, + 0xaa5be33a109e9959L, + 0x5cacd78d787ede11L, + 0xf5db142fd3f02c3fL, + 0x032c2098bb106b77L, + 0x2cec5b125aa731c4L, + 0xda1b6fa53247768cL, + 0x4adafa04552d46f3L, + 0xbc2dceb33dcd01bbL, + 0x93edb539dc7a5b08L, + 0x651a818eb49a1c40L, + 0xcc6d422c1f14ee6eL, + 0x3a9a769b77f4a926L, + 0x155a0d119643f395L, + 0xe3ad39a6fea3b4ddL, + 0xe6d9580f33930944L, + 0x102e6cb85b734e0cL, + 0x3fee1732bac414bfL, + 0xc9192385d22453f7L, + 0x606ee02779aaa1d9L, + 0x9699d490114ae691L, + 0xb959af1af0fdbc22L, + 0x4fae9bad981dfb6aL, + 0xdf6f0e0cff77cb15L, + 0x29983abb97978c5dL, + 0x065841317620d6eeL, + 0xf0af75861ec091a6L, + 0x59d8b624b54e6388L, + 0xaf2f8293ddae24c0L, + 0x80eff9193c197e73L, + 0x7618cdae54f9393bL, + 0x95b5f408aa5a8de6L, + 0x6342c0bfc2bacaaeL, + 0x4c82bb35230d901dL, + 0xba758f824bedd755L, + 0x13024c20e063257bL, + 0xe5f5789788836233L, + 0xca35031d69343880L, + 0x3cc237aa01d47fc8L, + 0xac03a20b66be4fb7L, + 0x5af496bc0e5e08ffL, + 0x7534ed36efe9524cL, + 0x83c3d98187091504L, + 0x2ab41a232c87e72aL, + 0xdc432e944467a062L, + 0xf383551ea5d0fad1L, + 0x057461a9cd30bd99L, + 0xf96b964d3fb181e3L, + 0x0f9ca2fa5751c6abL, + 0x205cd970b6e69c18L, + 0xd6abedc7de06db50L, + 0x7fdc2e657588297eL, + 0x892b1ad21d686e36L, + 0xa6eb6158fcdf3485L, + 0x501c55ef943f73cdL, + 0xc0ddc04ef35543b2L, + 0x362af4f99bb504faL, + 0x19ea8f737a025e49L, + 0xef1dbbc412e21901L, + 0x466a7866b96ceb2fL, + 0xb09d4cd1d18cac67L, + 0x9f5d375b303bf6d4L, + 0x69aa03ec58dbb19cL, + 0x8a073a4aa6780541L, + 0x7cf00efdce984209L, + 0x533075772f2f18baL, + 0xa5c741c047cf5ff2L, + 0x0cb08262ec41addcL, + 0xfa47b6d584a1ea94L, + 0xd587cd5f6516b027L, + 0x2370f9e80df6f76fL, + 0xb3b16c496a9cc710L, + 0x454658fe027c8058L, + 0x6a862374e3cbdaebL, + 0x9c7117c38b2b9da3L, + 0x3506d46120a56f8dL, + 0xc3f1e0d6484528c5L, + 0xec319b5ca9f27276L, + 0x1ac6afebc112353eL, + 0x1fb2ce420c2288a7L, + 0xe945faf564c2cfefL, + 0xc685817f8575955cL, + 0x3072b5c8ed95d214L, + 0x9905766a461b203aL, + 0x6ff242dd2efb6772L, + 0x40323957cf4c3dc1L, + 0xb6c50de0a7ac7a89L, + 0x26049841c0c64af6L, + 0xd0f3acf6a8260dbeL, + 0xff33d77c4991570dL, + 0x09c4e3cb21711045L, + 0xa0b320698affe26bL, + 0x564414dee21fa523L, + 0x79846f5403a8ff90L, + 0x8f735be36b48b8d8L, + 0x6cde624595eb0c05L, + 0x9a2956f2fd0b4b4dL, + 0xb5e92d781cbc11feL, + 0x431e19cf745c56b6L, + 0xea69da6ddfd2a498L, + 0x1c9eeedab732e3d0L, + 0x335e95505685b963L, + 0xc5a9a1e73e65fe2bL, + 0x55683446590fce54L, + 0xa39f00f131ef891cL, + 0x8c5f7b7bd058d3afL, + 0x7aa84fccb8b894e7L, + 0xd3df8c6e133666c9L, + 0x2528b8d97bd62181L, + 0x0ae8c3539a617b32L, + 0xfc1ff7e4f2813c7aL, + 0xc60e0ac927f490adL, + 0x30f93e7e4f14d7e5L, + 0x1f3945f4aea38d56L, + 0xe9ce7143c643ca1eL, + 0x40b9b2e16dcd3830L, + 0xb64e8656052d7f78L, + 0x998efddce49a25cbL, + 0x6f79c96b8c7a6283L, + 0xffb85ccaeb1052fcL, + 0x094f687d83f015b4L, + 0x268f13f762474f07L, + 0xd07827400aa7084fL, + 0x790fe4e2a129fa61L, + 0x8ff8d055c9c9bd29L, + 0xa038abdf287ee79aL, + 0x56cf9f68409ea0d2L, + 0xb562a6cebe3d140fL, + 0x43959279d6dd5347L, + 0x6c55e9f3376a09f4L, + 0x9aa2dd445f8a4ebcL, + 0x33d51ee6f404bc92L, + 0xc5222a519ce4fbdaL, + 0xeae251db7d53a169L, + 0x1c15656c15b3e621L, + 0x8cd4f0cd72d9d65eL, + 0x7a23c47a1a399116L, + 0x55e3bff0fb8ecba5L, + 0xa3148b47936e8cedL, + 0x0a6348e538e07ec3L, + 0xfc947c525000398bL, + 0xd35407d8b1b76338L, + 0x25a3336fd9572470L, + 0x20d752c6146799e9L, + 0xd62066717c87dea1L, + 0xf9e01dfb9d308412L, + 0x0f17294cf5d0c35aL, + 0xa660eaee5e5e3174L, + 0x5097de5936be763cL, + 0x7f57a5d3d7092c8fL, + 0x89a09164bfe96bc7L, + 0x196104c5d8835bb8L, + 0xef963072b0631cf0L, + 0xc0564bf851d44643L, + 0x36a17f4f3934010bL, + 0x9fd6bced92baf325L, + 0x6921885afa5ab46dL, + 0x46e1f3d01bedeedeL, + 0xb016c767730da996L, + 0x53bbfec18dae1d4bL, + 0xa54cca76e54e5a03L, + 0x8a8cb1fc04f900b0L, + 0x7c7b854b6c1947f8L, + 0xd50c46e9c797b5d6L, + 0x23fb725eaf77f29eL, + 0x0c3b09d44ec0a82dL, + 0xfacc3d632620ef65L, + 0x6a0da8c2414adf1aL, + 0x9cfa9c7529aa9852L, + 0xb33ae7ffc81dc2e1L, + 0x45cdd348a0fd85a9L, + 0xecba10ea0b737787L, + 0x1a4d245d639330cfL, + 0x358d5fd782246a7cL, + 0xc37a6b60eac42d34L, + 0x3f659c841845114eL, + 0xc992a83370a55606L, + 0xe652d3b991120cb5L, + 0x10a5e70ef9f24bfdL, + 0xb9d224ac527cb9d3L, + 0x4f25101b3a9cfe9bL, + 0x60e56b91db2ba428L, + 0x96125f26b3cbe360L, + 0x06d3ca87d4a1d31fL, + 0xf024fe30bc419457L, + 0xdfe485ba5df6cee4L, + 0x2913b10d351689acL, + 0x806472af9e987b82L, + 0x76934618f6783ccaL, + 0x59533d9217cf6679L, + 0xafa409257f2f2131L, + 0x4c093083818c95ecL, + 0xbafe0434e96cd2a4L, + 0x953e7fbe08db8817L, + 0x63c94b09603bcf5fL, + 0xcabe88abcbb53d71L, + 0x3c49bc1ca3557a39L, + 0x1389c79642e2208aL, + 0xe57ef3212a0267c2L, + 0x75bf66804d6857bdL, + 0x83485237258810f5L, + 0xac8829bdc43f4a46L, + 0x5a7f1d0aacdf0d0eL, + 0xf308dea80751ff20L, + 0x05ffea1f6fb1b868L, + 0x2a3f91958e06e2dbL, + 0xdcc8a522e6e6a593L, + 0xd9bcc48b2bd6180aL, + 0x2f4bf03c43365f42L, + 0x008b8bb6a28105f1L, + 0xf67cbf01ca6142b9L, + 0x5f0b7ca361efb097L, + 0xa9fc4814090ff7dfL, + 0x863c339ee8b8ad6cL, + 0x70cb07298058ea24L, + 0xe00a9288e732da5bL, + 0x16fda63f8fd29d13L, + 0x393dddb56e65c7a0L, + 0xcfcae902068580e8L, + 0x66bd2aa0ad0b72c6L, + 0x904a1e17c5eb358eL, + 0xbf8a659d245c6f3dL, + 0x497d512a4cbc2875L, + 0xaad0688cb21f9ca8L, + 0x5c275c3bdaffdbe0L, + 0x73e727b13b488153L, + 0x8510130653a8c61bL, + 0x2c67d0a4f8263435L, + 0xda90e41390c6737dL, + 0xf5509f99717129ceL, + 0x03a7ab2e19916e86L, + 0x93663e8f7efb5ef9L, + 0x65910a38161b19b1L, + 0x4a5171b2f7ac4302L, + 0xbca645059f4c044aL, + 0x15d186a734c2f664L, + 0xe326b2105c22b12cL, + 0xcce6c99abd95eb9fL, + 0x3a11fd2dd575acd7L, + + 0x0000000000000000L, + 0x71b0c13da512335dL, + 0xe361827b4a2466baL, + 0x92d14346ef3655e7L, + 0xf21a22a5ccdf5e1fL, + 0x83aae39869cd6d42L, + 0x117ba0de86fb38a5L, + 0x60cb61e323e90bf8L, + 0xd0ed6318c1292f55L, + 0xa15da225643b1c08L, + 0x338ce1638b0d49efL, + 0x423c205e2e1f7ab2L, + 0x22f741bd0df6714aL, + 0x53478080a8e44217L, + 0xc196c3c647d217f0L, + 0xb02602fbe2c024adL, + 0x9503e062dac5cdc1L, + 0xe4b3215f7fd7fe9cL, + 0x7662621990e1ab7bL, + 0x07d2a32435f39826L, + 0x6719c2c7161a93deL, + 0x16a903fab308a083L, + 0x847840bc5c3ef564L, + 0xf5c88181f92cc639L, + 0x45ee837a1bece294L, + 0x345e4247befed1c9L, + 0xa68f010151c8842eL, + 0xd73fc03cf4dab773L, + 0xb7f4a1dfd733bc8bL, + 0xc64460e272218fd6L, + 0x549523a49d17da31L, + 0x2525e2993805e96cL, + 0x1edee696ed1c08e9L, + 0x6f6e27ab480e3bb4L, + 0xfdbf64eda7386e53L, + 0x8c0fa5d0022a5d0eL, + 0xecc4c43321c356f6L, + 0x9d74050e84d165abL, + 0x0fa546486be7304cL, + 0x7e158775cef50311L, + 0xce33858e2c3527bcL, + 0xbf8344b3892714e1L, + 0x2d5207f566114106L, + 0x5ce2c6c8c303725bL, + 0x3c29a72be0ea79a3L, + 0x4d99661645f84afeL, + 0xdf482550aace1f19L, + 0xaef8e46d0fdc2c44L, + 0x8bdd06f437d9c528L, + 0xfa6dc7c992cbf675L, + 0x68bc848f7dfda392L, + 0x190c45b2d8ef90cfL, + 0x79c72451fb069b37L, + 0x0877e56c5e14a86aL, + 0x9aa6a62ab122fd8dL, + 0xeb1667171430ced0L, + 0x5b3065ecf6f0ea7dL, + 0x2a80a4d153e2d920L, + 0xb851e797bcd48cc7L, + 0xc9e126aa19c6bf9aL, + 0xa92a47493a2fb462L, + 0xd89a86749f3d873fL, + 0x4a4bc532700bd2d8L, + 0x3bfb040fd519e185L, + 0x3dbdcd2dda3811d2L, + 0x4c0d0c107f2a228fL, + 0xdedc4f56901c7768L, + 0xaf6c8e6b350e4435L, + 0xcfa7ef8816e74fcdL, + 0xbe172eb5b3f57c90L, + 0x2cc66df35cc32977L, + 0x5d76accef9d11a2aL, + 0xed50ae351b113e87L, + 0x9ce06f08be030ddaL, + 0x0e312c4e5135583dL, + 0x7f81ed73f4276b60L, + 0x1f4a8c90d7ce6098L, + 0x6efa4dad72dc53c5L, + 0xfc2b0eeb9dea0622L, + 0x8d9bcfd638f8357fL, + 0xa8be2d4f00fddc13L, + 0xd90eec72a5efef4eL, + 0x4bdfaf344ad9baa9L, + 0x3a6f6e09efcb89f4L, + 0x5aa40feacc22820cL, + 0x2b14ced76930b151L, + 0xb9c58d918606e4b6L, + 0xc8754cac2314d7ebL, + 0x78534e57c1d4f346L, + 0x09e38f6a64c6c01bL, + 0x9b32cc2c8bf095fcL, + 0xea820d112ee2a6a1L, + 0x8a496cf20d0bad59L, + 0xfbf9adcfa8199e04L, + 0x6928ee89472fcbe3L, + 0x18982fb4e23df8beL, + 0x23632bbb3724193bL, + 0x52d3ea8692362a66L, + 0xc002a9c07d007f81L, + 0xb1b268fdd8124cdcL, + 0xd179091efbfb4724L, + 0xa0c9c8235ee97479L, + 0x32188b65b1df219eL, + 0x43a84a5814cd12c3L, + 0xf38e48a3f60d366eL, + 0x823e899e531f0533L, + 0x10efcad8bc2950d4L, + 0x615f0be5193b6389L, + 0x01946a063ad26871L, + 0x7024ab3b9fc05b2cL, + 0xe2f5e87d70f60ecbL, + 0x93452940d5e43d96L, + 0xb660cbd9ede1d4faL, + 0xc7d00ae448f3e7a7L, + 0x550149a2a7c5b240L, + 0x24b1889f02d7811dL, + 0x447ae97c213e8ae5L, + 0x35ca2841842cb9b8L, + 0xa71b6b076b1aec5fL, + 0xd6abaa3ace08df02L, + 0x668da8c12cc8fbafL, + 0x173d69fc89dac8f2L, + 0x85ec2aba66ec9d15L, + 0xf45ceb87c3feae48L, + 0x94978a64e017a5b0L, + 0xe5274b59450596edL, + 0x77f6081faa33c30aL, + 0x0646c9220f21f057L, + 0x7b7b9a5bb47023a4L, + 0x0acb5b66116210f9L, + 0x981a1820fe54451eL, + 0xe9aad91d5b467643L, + 0x8961b8fe78af7dbbL, + 0xf8d179c3ddbd4ee6L, + 0x6a003a85328b1b01L, + 0x1bb0fbb89799285cL, + 0xab96f94375590cf1L, + 0xda26387ed04b3facL, + 0x48f77b383f7d6a4bL, + 0x3947ba059a6f5916L, + 0x598cdbe6b98652eeL, + 0x283c1adb1c9461b3L, + 0xbaed599df3a23454L, + 0xcb5d98a056b00709L, + 0xee787a396eb5ee65L, + 0x9fc8bb04cba7dd38L, + 0x0d19f842249188dfL, + 0x7ca9397f8183bb82L, + 0x1c62589ca26ab07aL, + 0x6dd299a107788327L, + 0xff03dae7e84ed6c0L, + 0x8eb31bda4d5ce59dL, + 0x3e951921af9cc130L, + 0x4f25d81c0a8ef26dL, + 0xddf49b5ae5b8a78aL, + 0xac445a6740aa94d7L, + 0xcc8f3b8463439f2fL, + 0xbd3ffab9c651ac72L, + 0x2feeb9ff2967f995L, + 0x5e5e78c28c75cac8L, + 0x65a57ccd596c2b4dL, + 0x1415bdf0fc7e1810L, + 0x86c4feb613484df7L, + 0xf7743f8bb65a7eaaL, + 0x97bf5e6895b37552L, + 0xe60f9f5530a1460fL, + 0x74dedc13df9713e8L, + 0x056e1d2e7a8520b5L, + 0xb5481fd598450418L, + 0xc4f8dee83d573745L, + 0x56299daed26162a2L, + 0x27995c93777351ffL, + 0x47523d70549a5a07L, + 0x36e2fc4df188695aL, + 0xa433bf0b1ebe3cbdL, + 0xd5837e36bbac0fe0L, + 0xf0a69caf83a9e68cL, + 0x81165d9226bbd5d1L, + 0x13c71ed4c98d8036L, + 0x6277dfe96c9fb36bL, + 0x02bcbe0a4f76b893L, + 0x730c7f37ea648bceL, + 0xe1dd3c710552de29L, + 0x906dfd4ca040ed74L, + 0x204bffb74280c9d9L, + 0x51fb3e8ae792fa84L, + 0xc32a7dcc08a4af63L, + 0xb29abcf1adb69c3eL, + 0xd251dd128e5f97c6L, + 0xa3e11c2f2b4da49bL, + 0x31305f69c47bf17cL, + 0x40809e546169c221L, + 0x46c657766e483276L, + 0x3776964bcb5a012bL, + 0xa5a7d50d246c54ccL, + 0xd4171430817e6791L, + 0xb4dc75d3a2976c69L, + 0xc56cb4ee07855f34L, + 0x57bdf7a8e8b30ad3L, + 0x260d36954da1398eL, + 0x962b346eaf611d23L, + 0xe79bf5530a732e7eL, + 0x754ab615e5457b99L, + 0x04fa7728405748c4L, + 0x643116cb63be433cL, + 0x1581d7f6c6ac7061L, + 0x875094b0299a2586L, + 0xf6e0558d8c8816dbL, + 0xd3c5b714b48dffb7L, + 0xa2757629119fcceaL, + 0x30a4356ffea9990dL, + 0x4114f4525bbbaa50L, + 0x21df95b17852a1a8L, + 0x506f548cdd4092f5L, + 0xc2be17ca3276c712L, + 0xb30ed6f79764f44fL, + 0x0328d40c75a4d0e2L, + 0x72981531d0b6e3bfL, + 0xe04956773f80b658L, + 0x91f9974a9a928505L, + 0xf132f6a9b97b8efdL, + 0x808237941c69bda0L, + 0x125374d2f35fe847L, + 0x63e3b5ef564ddb1aL, + 0x5818b1e083543a9fL, + 0x29a870dd264609c2L, + 0xbb79339bc9705c25L, + 0xcac9f2a66c626f78L, + 0xaa0293454f8b6480L, + 0xdbb25278ea9957ddL, + 0x4963113e05af023aL, + 0x38d3d003a0bd3167L, + 0x88f5d2f8427d15caL, + 0xf94513c5e76f2697L, + 0x6b94508308597370L, + 0x1a2491bead4b402dL, + 0x7aeff05d8ea24bd5L, + 0x0b5f31602bb07888L, + 0x998e7226c4862d6fL, + 0xe83eb31b61941e32L, + 0xcd1b51825991f75eL, + 0xbcab90bffc83c403L, + 0x2e7ad3f913b591e4L, + 0x5fca12c4b6a7a2b9L, + 0x3f017327954ea941L, + 0x4eb1b21a305c9a1cL, + 0xdc60f15cdf6acffbL, + 0xadd030617a78fca6L, + 0x1df6329a98b8d80bL, + 0x6c46f3a73daaeb56L, + 0xfe97b0e1d29cbeb1L, + 0x8f2771dc778e8decL, + 0xefec103f54678614L, + 0x9e5cd102f175b549L, + 0x0c8d92441e43e0aeL, + 0x7d3d5379bb51d3f3L, + + 0x0000000000000000L, + 0xbfdb6c480f15915eL, + 0x4b6ffec346bcb1d7L, + 0xf4b4928b49a92089L, + 0x96dffd868d7963aeL, + 0x290491ce826cf2f0L, + 0xddb00345cbc5d279L, + 0x626b6f0dc4d04327L, + 0x1966dd5e42655437L, + 0xa6bdb1164d70c569L, + 0x5209239d04d9e5e0L, + 0xedd24fd50bcc74beL, + 0x8fb920d8cf1c3799L, + 0x30624c90c009a6c7L, + 0xc4d6de1b89a0864eL, + 0x7b0db25386b51710L, + 0x32cdbabc84caa86eL, + 0x8d16d6f48bdf3930L, + 0x79a2447fc27619b9L, + 0xc6792837cd6388e7L, + 0xa412473a09b3cbc0L, + 0x1bc92b7206a65a9eL, + 0xef7db9f94f0f7a17L, + 0x50a6d5b1401aeb49L, + 0x2bab67e2c6affc59L, + 0x94700baac9ba6d07L, + 0x60c4992180134d8eL, + 0xdf1ff5698f06dcd0L, + 0xbd749a644bd69ff7L, + 0x02aff62c44c30ea9L, + 0xf61b64a70d6a2e20L, + 0x49c008ef027fbf7eL, + 0x659b7579099550dcL, + 0xda4019310680c182L, + 0x2ef48bba4f29e10bL, + 0x912fe7f2403c7055L, + 0xf34488ff84ec3372L, + 0x4c9fe4b78bf9a22cL, + 0xb82b763cc25082a5L, + 0x07f01a74cd4513fbL, + 0x7cfda8274bf004ebL, + 0xc326c46f44e595b5L, + 0x379256e40d4cb53cL, + 0x88493aac02592462L, + 0xea2255a1c6896745L, + 0x55f939e9c99cf61bL, + 0xa14dab628035d692L, + 0x1e96c72a8f2047ccL, + 0x5756cfc58d5ff8b2L, + 0xe88da38d824a69ecL, + 0x1c393106cbe34965L, + 0xa3e25d4ec4f6d83bL, + 0xc189324300269b1cL, + 0x7e525e0b0f330a42L, + 0x8ae6cc80469a2acbL, + 0x353da0c8498fbb95L, + 0x4e30129bcf3aac85L, + 0xf1eb7ed3c02f3ddbL, + 0x055fec5889861d52L, + 0xba84801086938c0cL, + 0xd8efef1d4243cf2bL, + 0x673483554d565e75L, + 0x938011de04ff7efcL, + 0x2c5b7d960beaefa2L, + 0xcb36eaf2132aa1b8L, + 0x74ed86ba1c3f30e6L, + 0x805914315596106fL, + 0x3f8278795a838131L, + 0x5de917749e53c216L, + 0xe2327b3c91465348L, + 0x1686e9b7d8ef73c1L, + 0xa95d85ffd7fae29fL, + 0xd25037ac514ff58fL, + 0x6d8b5be45e5a64d1L, + 0x993fc96f17f34458L, + 0x26e4a52718e6d506L, + 0x448fca2adc369621L, + 0xfb54a662d323077fL, + 0x0fe034e99a8a27f6L, + 0xb03b58a1959fb6a8L, + 0xf9fb504e97e009d6L, + 0x46203c0698f59888L, + 0xb294ae8dd15cb801L, + 0x0d4fc2c5de49295fL, + 0x6f24adc81a996a78L, + 0xd0ffc180158cfb26L, + 0x244b530b5c25dbafL, + 0x9b903f4353304af1L, + 0xe09d8d10d5855de1L, + 0x5f46e158da90ccbfL, + 0xabf273d39339ec36L, + 0x14291f9b9c2c7d68L, + 0x7642709658fc3e4fL, + 0xc9991cde57e9af11L, + 0x3d2d8e551e408f98L, + 0x82f6e21d11551ec6L, + 0xaead9f8b1abff164L, + 0x1176f3c315aa603aL, + 0xe5c261485c0340b3L, + 0x5a190d005316d1edL, + 0x3872620d97c692caL, + 0x87a90e4598d30394L, + 0x731d9cced17a231dL, + 0xccc6f086de6fb243L, + 0xb7cb42d558daa553L, + 0x08102e9d57cf340dL, + 0xfca4bc161e661484L, + 0x437fd05e117385daL, + 0x2114bf53d5a3c6fdL, + 0x9ecfd31bdab657a3L, + 0x6a7b4190931f772aL, + 0xd5a02dd89c0ae674L, + 0x9c6025379e75590aL, + 0x23bb497f9160c854L, + 0xd70fdbf4d8c9e8ddL, + 0x68d4b7bcd7dc7983L, + 0x0abfd8b1130c3aa4L, + 0xb564b4f91c19abfaL, + 0x41d0267255b08b73L, + 0xfe0b4a3a5aa51a2dL, + 0x8506f869dc100d3dL, + 0x3add9421d3059c63L, + 0xce6906aa9aacbceaL, + 0x71b26ae295b92db4L, + 0x13d905ef51696e93L, + 0xac0269a75e7cffcdL, + 0x58b6fb2c17d5df44L, + 0xe76d976418c04e1aL, + 0xa2b4f3b77ec2d01bL, + 0x1d6f9fff71d74145L, + 0xe9db0d74387e61ccL, + 0x5600613c376bf092L, + 0x346b0e31f3bbb3b5L, + 0x8bb06279fcae22ebL, + 0x7f04f0f2b5070262L, + 0xc0df9cbaba12933cL, + 0xbbd22ee93ca7842cL, + 0x040942a133b21572L, + 0xf0bdd02a7a1b35fbL, + 0x4f66bc62750ea4a5L, + 0x2d0dd36fb1dee782L, + 0x92d6bf27becb76dcL, + 0x66622dacf7625655L, + 0xd9b941e4f877c70bL, + 0x9079490bfa087875L, + 0x2fa22543f51de92bL, + 0xdb16b7c8bcb4c9a2L, + 0x64cddb80b3a158fcL, + 0x06a6b48d77711bdbL, + 0xb97dd8c578648a85L, + 0x4dc94a4e31cdaa0cL, + 0xf21226063ed83b52L, + 0x891f9455b86d2c42L, + 0x36c4f81db778bd1cL, + 0xc2706a96fed19d95L, + 0x7dab06def1c40ccbL, + 0x1fc069d335144fecL, + 0xa01b059b3a01deb2L, + 0x54af971073a8fe3bL, + 0xeb74fb587cbd6f65L, + 0xc72f86ce775780c7L, + 0x78f4ea8678421199L, + 0x8c40780d31eb3110L, + 0x339b14453efea04eL, + 0x51f07b48fa2ee369L, + 0xee2b1700f53b7237L, + 0x1a9f858bbc9252beL, + 0xa544e9c3b387c3e0L, + 0xde495b903532d4f0L, + 0x619237d83a2745aeL, + 0x9526a553738e6527L, + 0x2afdc91b7c9bf479L, + 0x4896a616b84bb75eL, + 0xf74dca5eb75e2600L, + 0x03f958d5fef70689L, + 0xbc22349df1e297d7L, + 0xf5e23c72f39d28a9L, + 0x4a39503afc88b9f7L, + 0xbe8dc2b1b521997eL, + 0x0156aef9ba340820L, + 0x633dc1f47ee44b07L, + 0xdce6adbc71f1da59L, + 0x28523f373858fad0L, + 0x9789537f374d6b8eL, + 0xec84e12cb1f87c9eL, + 0x535f8d64beededc0L, + 0xa7eb1feff744cd49L, + 0x183073a7f8515c17L, + 0x7a5b1caa3c811f30L, + 0xc58070e233948e6eL, + 0x3134e2697a3daee7L, + 0x8eef8e2175283fb9L, + 0x698219456de871a3L, + 0xd659750d62fde0fdL, + 0x22ede7862b54c074L, + 0x9d368bce2441512aL, + 0xff5de4c3e091120dL, + 0x4086888bef848353L, + 0xb4321a00a62da3daL, + 0x0be97648a9383284L, + 0x70e4c41b2f8d2594L, + 0xcf3fa8532098b4caL, + 0x3b8b3ad869319443L, + 0x845056906624051dL, + 0xe63b399da2f4463aL, + 0x59e055d5ade1d764L, + 0xad54c75ee448f7edL, + 0x128fab16eb5d66b3L, + 0x5b4fa3f9e922d9cdL, + 0xe494cfb1e6374893L, + 0x10205d3aaf9e681aL, + 0xaffb3172a08bf944L, + 0xcd905e7f645bba63L, + 0x724b32376b4e2b3dL, + 0x86ffa0bc22e70bb4L, + 0x3924ccf42df29aeaL, + 0x42297ea7ab478dfaL, + 0xfdf212efa4521ca4L, + 0x09468064edfb3c2dL, + 0xb69dec2ce2eead73L, + 0xd4f68321263eee54L, + 0x6b2def69292b7f0aL, + 0x9f997de260825f83L, + 0x204211aa6f97ceddL, + 0x0c196c3c647d217fL, + 0xb3c200746b68b021L, + 0x477692ff22c190a8L, + 0xf8adfeb72dd401f6L, + 0x9ac691bae90442d1L, + 0x251dfdf2e611d38fL, + 0xd1a96f79afb8f306L, + 0x6e720331a0ad6258L, + 0x157fb16226187548L, + 0xaaa4dd2a290de416L, + 0x5e104fa160a4c49fL, + 0xe1cb23e96fb155c1L, + 0x83a04ce4ab6116e6L, + 0x3c7b20aca47487b8L, + 0xc8cfb227eddda731L, + 0x7714de6fe2c8366fL, + 0x3ed4d680e0b78911L, + 0x810fbac8efa2184fL, + 0x75bb2843a60b38c6L, + 0xca60440ba91ea998L, + 0xa80b2b066dceeabfL, + 0x17d0474e62db7be1L, + 0xe364d5c52b725b68L, + 0x5cbfb98d2467ca36L, + 0x27b20bdea2d2dd26L, + 0x98696796adc74c78L, + 0x6cddf51de46e6cf1L, + 0xd3069955eb7bfdafL, + 0xb16df6582fabbe88L, + 0x0eb69a1020be2fd6L, + 0xfa02089b69170f5fL, + 0x45d964d366029e01L, + + 0x0000000000000000L, + 0x3ea616bd2ae10d77L, + 0x7d4c2d7a55c21aeeL, + 0x43ea3bc77f231799L, + 0xfa985af4ab8435dcL, + 0xc43e4c49816538abL, + 0x87d4778efe462f32L, + 0xb9726133d4a72245L, + 0xc1e993ba0f9ff8d3L, + 0xff4f8507257ef5a4L, + 0xbca5bec05a5de23dL, + 0x8203a87d70bcef4aL, + 0x3b71c94ea41bcd0fL, + 0x05d7dff38efac078L, + 0x463de434f1d9d7e1L, + 0x789bf289db38da96L, + 0xb70a012747a862cdL, + 0x89ac179a6d496fbaL, + 0xca462c5d126a7823L, + 0xf4e03ae0388b7554L, + 0x4d925bd3ec2c5711L, + 0x73344d6ec6cd5a66L, + 0x30de76a9b9ee4dffL, + 0x0e786014930f4088L, + 0x76e3929d48379a1eL, + 0x4845842062d69769L, + 0x0bafbfe71df580f0L, + 0x3509a95a37148d87L, + 0x8c7bc869e3b3afc2L, + 0xb2ddded4c952a2b5L, + 0xf137e513b671b52cL, + 0xcf91f3ae9c90b85bL, + 0x5acd241dd7c756f1L, + 0x646b32a0fd265b86L, + 0x2781096782054c1fL, + 0x19271fdaa8e44168L, + 0xa0557ee97c43632dL, + 0x9ef3685456a26e5aL, + 0xdd195393298179c3L, + 0xe3bf452e036074b4L, + 0x9b24b7a7d858ae22L, + 0xa582a11af2b9a355L, + 0xe6689add8d9ab4ccL, + 0xd8ce8c60a77bb9bbL, + 0x61bced5373dc9bfeL, + 0x5f1afbee593d9689L, + 0x1cf0c029261e8110L, + 0x2256d6940cff8c67L, + 0xedc7253a906f343cL, + 0xd3613387ba8e394bL, + 0x908b0840c5ad2ed2L, + 0xae2d1efdef4c23a5L, + 0x175f7fce3beb01e0L, + 0x29f96973110a0c97L, + 0x6a1352b46e291b0eL, + 0x54b5440944c81679L, + 0x2c2eb6809ff0ccefL, + 0x1288a03db511c198L, + 0x51629bfaca32d601L, + 0x6fc48d47e0d3db76L, + 0xd6b6ec743474f933L, + 0xe810fac91e95f444L, + 0xabfac10e61b6e3ddL, + 0x955cd7b34b57eeaaL, + 0xb59a483baf8eade2L, + 0x8b3c5e86856fa095L, + 0xc8d66541fa4cb70cL, + 0xf67073fcd0adba7bL, + 0x4f0212cf040a983eL, + 0x71a404722eeb9549L, + 0x324e3fb551c882d0L, + 0x0ce829087b298fa7L, + 0x7473db81a0115531L, + 0x4ad5cd3c8af05846L, + 0x093ff6fbf5d34fdfL, + 0x3799e046df3242a8L, + 0x8eeb81750b9560edL, + 0xb04d97c821746d9aL, + 0xf3a7ac0f5e577a03L, + 0xcd01bab274b67774L, + 0x0290491ce826cf2fL, + 0x3c365fa1c2c7c258L, + 0x7fdc6466bde4d5c1L, + 0x417a72db9705d8b6L, + 0xf80813e843a2faf3L, + 0xc6ae05556943f784L, + 0x85443e921660e01dL, + 0xbbe2282f3c81ed6aL, + 0xc379daa6e7b937fcL, + 0xfddfcc1bcd583a8bL, + 0xbe35f7dcb27b2d12L, + 0x8093e161989a2065L, + 0x39e180524c3d0220L, + 0x074796ef66dc0f57L, + 0x44adad2819ff18ceL, + 0x7a0bbb95331e15b9L, + 0xef576c267849fb13L, + 0xd1f17a9b52a8f664L, + 0x921b415c2d8be1fdL, + 0xacbd57e1076aec8aL, + 0x15cf36d2d3cdcecfL, + 0x2b69206ff92cc3b8L, + 0x68831ba8860fd421L, + 0x56250d15aceed956L, + 0x2ebeff9c77d603c0L, + 0x1018e9215d370eb7L, + 0x53f2d2e62214192eL, + 0x6d54c45b08f51459L, + 0xd426a568dc52361cL, + 0xea80b3d5f6b33b6bL, + 0xa96a881289902cf2L, + 0x97cc9eafa3712185L, + 0x585d6d013fe199deL, + 0x66fb7bbc150094a9L, + 0x2511407b6a238330L, + 0x1bb756c640c28e47L, + 0xa2c537f59465ac02L, + 0x9c632148be84a175L, + 0xdf891a8fc1a7b6ecL, + 0xe12f0c32eb46bb9bL, + 0x99b4febb307e610dL, + 0xa712e8061a9f6c7aL, + 0xe4f8d3c165bc7be3L, + 0xda5ec57c4f5d7694L, + 0x632ca44f9bfa54d1L, + 0x5d8ab2f2b11b59a6L, + 0x1e608935ce384e3fL, + 0x20c69f88e4d94348L, + 0x5fedb624078ac8afL, + 0x614ba0992d6bc5d8L, + 0x22a19b5e5248d241L, + 0x1c078de378a9df36L, + 0xa575ecd0ac0efd73L, + 0x9bd3fa6d86eff004L, + 0xd839c1aaf9cce79dL, + 0xe69fd717d32deaeaL, + 0x9e04259e0815307cL, + 0xa0a2332322f43d0bL, + 0xe34808e45dd72a92L, + 0xddee1e59773627e5L, + 0x649c7f6aa39105a0L, + 0x5a3a69d7897008d7L, + 0x19d05210f6531f4eL, + 0x277644addcb21239L, + 0xe8e7b7034022aa62L, + 0xd641a1be6ac3a715L, + 0x95ab9a7915e0b08cL, + 0xab0d8cc43f01bdfbL, + 0x127fedf7eba69fbeL, + 0x2cd9fb4ac14792c9L, + 0x6f33c08dbe648550L, + 0x5195d63094858827L, + 0x290e24b94fbd52b1L, + 0x17a83204655c5fc6L, + 0x544209c31a7f485fL, + 0x6ae41f7e309e4528L, + 0xd3967e4de439676dL, + 0xed3068f0ced86a1aL, + 0xaeda5337b1fb7d83L, + 0x907c458a9b1a70f4L, + 0x05209239d04d9e5eL, + 0x3b868484faac9329L, + 0x786cbf43858f84b0L, + 0x46caa9feaf6e89c7L, + 0xffb8c8cd7bc9ab82L, + 0xc11ede705128a6f5L, + 0x82f4e5b72e0bb16cL, + 0xbc52f30a04eabc1bL, + 0xc4c90183dfd2668dL, + 0xfa6f173ef5336bfaL, + 0xb9852cf98a107c63L, + 0x87233a44a0f17114L, + 0x3e515b7774565351L, + 0x00f74dca5eb75e26L, + 0x431d760d219449bfL, + 0x7dbb60b00b7544c8L, + 0xb22a931e97e5fc93L, + 0x8c8c85a3bd04f1e4L, + 0xcf66be64c227e67dL, + 0xf1c0a8d9e8c6eb0aL, + 0x48b2c9ea3c61c94fL, + 0x7614df571680c438L, + 0x35fee49069a3d3a1L, + 0x0b58f22d4342ded6L, + 0x73c300a4987a0440L, + 0x4d651619b29b0937L, + 0x0e8f2ddecdb81eaeL, + 0x30293b63e75913d9L, + 0x895b5a5033fe319cL, + 0xb7fd4ced191f3cebL, + 0xf417772a663c2b72L, + 0xcab161974cdd2605L, + 0xea77fe1fa804654dL, + 0xd4d1e8a282e5683aL, + 0x973bd365fdc67fa3L, + 0xa99dc5d8d72772d4L, + 0x10efa4eb03805091L, + 0x2e49b25629615de6L, + 0x6da3899156424a7fL, + 0x53059f2c7ca34708L, + 0x2b9e6da5a79b9d9eL, + 0x15387b188d7a90e9L, + 0x56d240dff2598770L, + 0x68745662d8b88a07L, + 0xd10637510c1fa842L, + 0xefa021ec26fea535L, + 0xac4a1a2b59ddb2acL, + 0x92ec0c96733cbfdbL, + 0x5d7dff38efac0780L, + 0x63dbe985c54d0af7L, + 0x2031d242ba6e1d6eL, + 0x1e97c4ff908f1019L, + 0xa7e5a5cc4428325cL, + 0x9943b3716ec93f2bL, + 0xdaa988b611ea28b2L, + 0xe40f9e0b3b0b25c5L, + 0x9c946c82e033ff53L, + 0xa2327a3fcad2f224L, + 0xe1d841f8b5f1e5bdL, + 0xdf7e57459f10e8caL, + 0x660c36764bb7ca8fL, + 0x58aa20cb6156c7f8L, + 0x1b401b0c1e75d061L, + 0x25e60db13494dd16L, + 0xb0bada027fc333bcL, + 0x8e1cccbf55223ecbL, + 0xcdf6f7782a012952L, + 0xf350e1c500e02425L, + 0x4a2280f6d4470660L, + 0x7484964bfea60b17L, + 0x376ead8c81851c8eL, + 0x09c8bb31ab6411f9L, + 0x715349b8705ccb6fL, + 0x4ff55f055abdc618L, + 0x0c1f64c2259ed181L, + 0x32b9727f0f7fdcf6L, + 0x8bcb134cdbd8feb3L, + 0xb56d05f1f139f3c4L, + 0xf6873e368e1ae45dL, + 0xc821288ba4fbe92aL, + 0x07b0db25386b5171L, + 0x3916cd98128a5c06L, + 0x7afcf65f6da94b9fL, + 0x445ae0e2474846e8L, + 0xfd2881d193ef64adL, + 0xc38e976cb90e69daL, + 0x8064acabc62d7e43L, + 0xbec2ba16eccc7334L, + 0xc659489f37f4a9a2L, + 0xf8ff5e221d15a4d5L, + 0xbb1565e56236b34cL, + 0x85b3735848d7be3bL, + 0x3cc1126b9c709c7eL, + 0x026704d6b6919109L, + 0x418d3f11c9b28690L, + 0x7f2b29ace3538be7L, + + 0x0000000000000000L, + 0x169489cc969951e5L, + 0x2d2913992d32a3caL, + 0x3bbd9a55bbabf22fL, + 0x5a5227325a654794L, + 0x4cc6aefeccfc1671L, + 0x777b34ab7757e45eL, + 0x61efbd67e1ceb5bbL, + 0xb4a44e64b4ca8f28L, + 0xa230c7a82253decdL, + 0x998d5dfd99f82ce2L, + 0x8f19d4310f617d07L, + 0xeef66956eeafc8bcL, + 0xf862e09a78369959L, + 0xc3df7acfc39d6b76L, + 0xd54bf30355043a93L, + 0x5d91ba9a31028d3bL, + 0x4b053356a79bdcdeL, + 0x70b8a9031c302ef1L, + 0x662c20cf8aa97f14L, + 0x07c39da86b67caafL, + 0x11571464fdfe9b4aL, + 0x2aea8e3146556965L, + 0x3c7e07fdd0cc3880L, + 0xe935f4fe85c80213L, + 0xffa17d32135153f6L, + 0xc41ce767a8faa1d9L, + 0xd2886eab3e63f03cL, + 0xb367d3ccdfad4587L, + 0xa5f35a0049341462L, + 0x9e4ec055f29fe64dL, + 0x88da49996406b7a8L, + 0xbb23753462051a76L, + 0xadb7fcf8f49c4b93L, + 0x960a66ad4f37b9bcL, + 0x809eef61d9aee859L, + 0xe171520638605de2L, + 0xf7e5dbcaaef90c07L, + 0xcc58419f1552fe28L, + 0xdaccc85383cbafcdL, + 0x0f873b50d6cf955eL, + 0x1913b29c4056c4bbL, + 0x22ae28c9fbfd3694L, + 0x343aa1056d646771L, + 0x55d51c628caad2caL, + 0x434195ae1a33832fL, + 0x78fc0ffba1987100L, + 0x6e688637370120e5L, + 0xe6b2cfae5307974dL, + 0xf0264662c59ec6a8L, + 0xcb9bdc377e353487L, + 0xdd0f55fbe8ac6562L, + 0xbce0e89c0962d0d9L, + 0xaa7461509ffb813cL, + 0x91c9fb0524507313L, + 0x875d72c9b2c922f6L, + 0x521681cae7cd1865L, + 0x4482080671544980L, + 0x7f3f9253caffbbafL, + 0x69ab1b9f5c66ea4aL, + 0x0844a6f8bda85ff1L, + 0x1ed02f342b310e14L, + 0x256db561909afc3bL, + 0x33f93cad0603addeL, + 0x429fcc3b9c9da787L, + 0x540b45f70a04f662L, + 0x6fb6dfa2b1af044dL, + 0x7922566e273655a8L, + 0x18cdeb09c6f8e013L, + 0x0e5962c55061b1f6L, + 0x35e4f890ebca43d9L, + 0x2370715c7d53123cL, + 0xf63b825f285728afL, + 0xe0af0b93bece794aL, + 0xdb1291c605658b65L, + 0xcd86180a93fcda80L, + 0xac69a56d72326f3bL, + 0xbafd2ca1e4ab3edeL, + 0x8140b6f45f00ccf1L, + 0x97d43f38c9999d14L, + 0x1f0e76a1ad9f2abcL, + 0x099aff6d3b067b59L, + 0x3227653880ad8976L, + 0x24b3ecf41634d893L, + 0x455c5193f7fa6d28L, + 0x53c8d85f61633ccdL, + 0x6875420adac8cee2L, + 0x7ee1cbc64c519f07L, + 0xabaa38c51955a594L, + 0xbd3eb1098fccf471L, + 0x86832b5c3467065eL, + 0x9017a290a2fe57bbL, + 0xf1f81ff74330e200L, + 0xe76c963bd5a9b3e5L, + 0xdcd10c6e6e0241caL, + 0xca4585a2f89b102fL, + 0xf9bcb90ffe98bdf1L, + 0xef2830c36801ec14L, + 0xd495aa96d3aa1e3bL, + 0xc201235a45334fdeL, + 0xa3ee9e3da4fdfa65L, + 0xb57a17f13264ab80L, + 0x8ec78da489cf59afL, + 0x985304681f56084aL, + 0x4d18f76b4a5232d9L, + 0x5b8c7ea7dccb633cL, + 0x6031e4f267609113L, + 0x76a56d3ef1f9c0f6L, + 0x174ad0591037754dL, + 0x01de599586ae24a8L, + 0x3a63c3c03d05d687L, + 0x2cf74a0cab9c8762L, + 0xa42d0395cf9a30caL, + 0xb2b98a595903612fL, + 0x8904100ce2a89300L, + 0x9f9099c07431c2e5L, + 0xfe7f24a795ff775eL, + 0xe8ebad6b036626bbL, + 0xd356373eb8cdd494L, + 0xc5c2bef22e548571L, + 0x10894df17b50bfe2L, + 0x061dc43dedc9ee07L, + 0x3da05e6856621c28L, + 0x2b34d7a4c0fb4dcdL, + 0x4adb6ac32135f876L, + 0x5c4fe30fb7aca993L, + 0x67f2795a0c075bbcL, + 0x7166f0969a9e0a59L, + 0x853f9877393b4f0eL, + 0x93ab11bbafa21eebL, + 0xa8168bee1409ecc4L, + 0xbe8202228290bd21L, + 0xdf6dbf45635e089aL, + 0xc9f93689f5c7597fL, + 0xf244acdc4e6cab50L, + 0xe4d02510d8f5fab5L, + 0x319bd6138df1c026L, + 0x270f5fdf1b6891c3L, + 0x1cb2c58aa0c363ecL, + 0x0a264c46365a3209L, + 0x6bc9f121d79487b2L, + 0x7d5d78ed410dd657L, + 0x46e0e2b8faa62478L, + 0x50746b746c3f759dL, + 0xd8ae22ed0839c235L, + 0xce3aab219ea093d0L, + 0xf5873174250b61ffL, + 0xe313b8b8b392301aL, + 0x82fc05df525c85a1L, + 0x94688c13c4c5d444L, + 0xafd516467f6e266bL, + 0xb9419f8ae9f7778eL, + 0x6c0a6c89bcf34d1dL, + 0x7a9ee5452a6a1cf8L, + 0x41237f1091c1eed7L, + 0x57b7f6dc0758bf32L, + 0x36584bbbe6960a89L, + 0x20ccc277700f5b6cL, + 0x1b715822cba4a943L, + 0x0de5d1ee5d3df8a6L, + 0x3e1ced435b3e5578L, + 0x2888648fcda7049dL, + 0x1335feda760cf6b2L, + 0x05a17716e095a757L, + 0x644eca71015b12ecL, + 0x72da43bd97c24309L, + 0x4967d9e82c69b126L, + 0x5ff35024baf0e0c3L, + 0x8ab8a327eff4da50L, + 0x9c2c2aeb796d8bb5L, + 0xa791b0bec2c6799aL, + 0xb1053972545f287fL, + 0xd0ea8415b5919dc4L, + 0xc67e0dd92308cc21L, + 0xfdc3978c98a33e0eL, + 0xeb571e400e3a6febL, + 0x638d57d96a3cd843L, + 0x7519de15fca589a6L, + 0x4ea44440470e7b89L, + 0x5830cd8cd1972a6cL, + 0x39df70eb30599fd7L, + 0x2f4bf927a6c0ce32L, + 0x14f663721d6b3c1dL, + 0x0262eabe8bf26df8L, + 0xd72919bddef6576bL, + 0xc1bd9071486f068eL, + 0xfa000a24f3c4f4a1L, + 0xec9483e8655da544L, + 0x8d7b3e8f849310ffL, + 0x9befb743120a411aL, + 0xa0522d16a9a1b335L, + 0xb6c6a4da3f38e2d0L, + 0xc7a0544ca5a6e889L, + 0xd134dd80333fb96cL, + 0xea8947d588944b43L, + 0xfc1dce191e0d1aa6L, + 0x9df2737effc3af1dL, + 0x8b66fab2695afef8L, + 0xb0db60e7d2f10cd7L, + 0xa64fe92b44685d32L, + 0x73041a28116c67a1L, + 0x659093e487f53644L, + 0x5e2d09b13c5ec46bL, + 0x48b9807daac7958eL, + 0x29563d1a4b092035L, + 0x3fc2b4d6dd9071d0L, + 0x047f2e83663b83ffL, + 0x12eba74ff0a2d21aL, + 0x9a31eed694a465b2L, + 0x8ca5671a023d3457L, + 0xb718fd4fb996c678L, + 0xa18c74832f0f979dL, + 0xc063c9e4cec12226L, + 0xd6f74028585873c3L, + 0xed4ada7de3f381ecL, + 0xfbde53b1756ad009L, + 0x2e95a0b2206eea9aL, + 0x3801297eb6f7bb7fL, + 0x03bcb32b0d5c4950L, + 0x15283ae79bc518b5L, + 0x74c787807a0bad0eL, + 0x62530e4cec92fcebL, + 0x59ee941957390ec4L, + 0x4f7a1dd5c1a05f21L, + 0x7c832178c7a3f2ffL, + 0x6a17a8b4513aa31aL, + 0x51aa32e1ea915135L, + 0x473ebb2d7c0800d0L, + 0x26d1064a9dc6b56bL, + 0x30458f860b5fe48eL, + 0x0bf815d3b0f416a1L, + 0x1d6c9c1f266d4744L, + 0xc8276f1c73697dd7L, + 0xdeb3e6d0e5f02c32L, + 0xe50e7c855e5bde1dL, + 0xf39af549c8c28ff8L, + 0x9275482e290c3a43L, + 0x84e1c1e2bf956ba6L, + 0xbf5c5bb7043e9989L, + 0xa9c8d27b92a7c86cL, + 0x21129be2f6a17fc4L, + 0x3786122e60382e21L, + 0x0c3b887bdb93dc0eL, + 0x1aaf01b74d0a8debL, + 0x7b40bcd0acc43850L, + 0x6dd4351c3a5d69b5L, + 0x5669af4981f69b9aL, + 0x40fd2685176fca7fL, + 0x95b6d586426bf0ecL, + 0x83225c4ad4f2a109L, + 0xb89fc61f6f595326L, + 0xae0b4fd3f9c002c3L, + 0xcfe4f2b4180eb778L, + 0xd9707b788e97e69dL, + 0xe2cde12d353c14b2L, + 0xf45968e1a3a54557L, + + 0x0000000000000000L, + 0x0aed36d1a3bb9d7fL, + 0x15da6da347773afeL, + 0x1f375b72e4cca781L, + 0x2bb4db468eee75fcL, + 0x2159ed972d55e883L, + 0x3e6eb6e5c9994f02L, + 0x348380346a22d27dL, + 0x5769b68d1ddcebf8L, + 0x5d84805cbe677687L, + 0x42b3db2e5aabd106L, + 0x485eedfff9104c79L, + 0x7cdd6dcb93329e04L, + 0x76305b1a3089037bL, + 0x69070068d445a4faL, + 0x63ea36b977fe3985L, + 0xaed36d1a3bb9d7f0L, + 0xa43e5bcb98024a8fL, + 0xbb0900b97cceed0eL, + 0xb1e43668df757071L, + 0x8567b65cb557a20cL, + 0x8f8a808d16ec3f73L, + 0x90bddbfff22098f2L, + 0x9a50ed2e519b058dL, + 0xf9badb9726653c08L, + 0xf357ed4685dea177L, + 0xec60b634611206f6L, + 0xe68d80e5c2a99b89L, + 0xd20e00d1a88b49f4L, + 0xd8e336000b30d48bL, + 0xc7d46d72effc730aL, + 0xcd395ba34c47ee75L, + 0x697ffc672fe43c8bL, + 0x6392cab68c5fa1f4L, + 0x7ca591c468930675L, + 0x7648a715cb289b0aL, + 0x42cb2721a10a4977L, + 0x482611f002b1d408L, + 0x57114a82e67d7389L, + 0x5dfc7c5345c6eef6L, + 0x3e164aea3238d773L, + 0x34fb7c3b91834a0cL, + 0x2bcc2749754fed8dL, + 0x21211198d6f470f2L, + 0x15a291acbcd6a28fL, + 0x1f4fa77d1f6d3ff0L, + 0x0078fc0ffba19871L, + 0x0a95cade581a050eL, + 0xc7ac917d145deb7bL, + 0xcd41a7acb7e67604L, + 0xd276fcde532ad185L, + 0xd89bca0ff0914cfaL, + 0xec184a3b9ab39e87L, + 0xe6f57cea390803f8L, + 0xf9c22798ddc4a479L, + 0xf32f11497e7f3906L, + 0x90c527f009810083L, + 0x9a281121aa3a9dfcL, + 0x851f4a534ef63a7dL, + 0x8ff27c82ed4da702L, + 0xbb71fcb6876f757fL, + 0xb19cca6724d4e800L, + 0xaeab9115c0184f81L, + 0xa446a7c463a3d2feL, + 0xd2fff8ce5fc87916L, + 0xd812ce1ffc73e469L, + 0xc725956d18bf43e8L, + 0xcdc8a3bcbb04de97L, + 0xf94b2388d1260ceaL, + 0xf3a61559729d9195L, + 0xec914e2b96513614L, + 0xe67c78fa35eaab6bL, + 0x85964e43421492eeL, + 0x8f7b7892e1af0f91L, + 0x904c23e00563a810L, + 0x9aa11531a6d8356fL, + 0xae229505ccfae712L, + 0xa4cfa3d46f417a6dL, + 0xbbf8f8a68b8dddecL, + 0xb115ce7728364093L, + 0x7c2c95d46471aee6L, + 0x76c1a305c7ca3399L, + 0x69f6f87723069418L, + 0x631bcea680bd0967L, + 0x57984e92ea9fdb1aL, + 0x5d75784349244665L, + 0x42422331ade8e1e4L, + 0x48af15e00e537c9bL, + 0x2b45235979ad451eL, + 0x21a81588da16d861L, + 0x3e9f4efa3eda7fe0L, + 0x3472782b9d61e29fL, + 0x00f1f81ff74330e2L, + 0x0a1ccece54f8ad9dL, + 0x152b95bcb0340a1cL, + 0x1fc6a36d138f9763L, + 0xbb8004a9702c459dL, + 0xb16d3278d397d8e2L, + 0xae5a690a375b7f63L, + 0xa4b75fdb94e0e21cL, + 0x9034dfeffec23061L, + 0x9ad9e93e5d79ad1eL, + 0x85eeb24cb9b50a9fL, + 0x8f03849d1a0e97e0L, + 0xece9b2246df0ae65L, + 0xe60484f5ce4b331aL, + 0xf933df872a87949bL, + 0xf3dee956893c09e4L, + 0xc75d6962e31edb99L, + 0xcdb05fb340a546e6L, + 0xd28704c1a469e167L, + 0xd86a321007d27c18L, + 0x155369b34b95926dL, + 0x1fbe5f62e82e0f12L, + 0x008904100ce2a893L, + 0x0a6432c1af5935ecL, + 0x3ee7b2f5c57be791L, + 0x340a842466c07aeeL, + 0x2b3ddf56820cdd6fL, + 0x21d0e98721b74010L, + 0x423adf3e56497995L, + 0x48d7e9eff5f2e4eaL, + 0x57e0b29d113e436bL, + 0x5d0d844cb285de14L, + 0x698e0478d8a70c69L, + 0x636332a97b1c9116L, + 0x7c5469db9fd03697L, + 0x76b95f0a3c6babe8L, + 0x9126d7cfe7076147L, + 0x9bcbe11e44bcfc38L, + 0x84fcba6ca0705bb9L, + 0x8e118cbd03cbc6c6L, + 0xba920c8969e914bbL, + 0xb07f3a58ca5289c4L, + 0xaf48612a2e9e2e45L, + 0xa5a557fb8d25b33aL, + 0xc64f6142fadb8abfL, + 0xcca25793596017c0L, + 0xd3950ce1bdacb041L, + 0xd9783a301e172d3eL, + 0xedfbba047435ff43L, + 0xe7168cd5d78e623cL, + 0xf821d7a73342c5bdL, + 0xf2cce17690f958c2L, + 0x3ff5bad5dcbeb6b7L, + 0x35188c047f052bc8L, + 0x2a2fd7769bc98c49L, + 0x20c2e1a738721136L, + 0x144161935250c34bL, + 0x1eac5742f1eb5e34L, + 0x019b0c301527f9b5L, + 0x0b763ae1b69c64caL, + 0x689c0c58c1625d4fL, + 0x62713a8962d9c030L, + 0x7d4661fb861567b1L, + 0x77ab572a25aefaceL, + 0x4328d71e4f8c28b3L, + 0x49c5e1cfec37b5ccL, + 0x56f2babd08fb124dL, + 0x5c1f8c6cab408f32L, + 0xf8592ba8c8e35dccL, + 0xf2b41d796b58c0b3L, + 0xed83460b8f946732L, + 0xe76e70da2c2ffa4dL, + 0xd3edf0ee460d2830L, + 0xd900c63fe5b6b54fL, + 0xc6379d4d017a12ceL, + 0xccdaab9ca2c18fb1L, + 0xaf309d25d53fb634L, + 0xa5ddabf476842b4bL, + 0xbaeaf08692488ccaL, + 0xb007c65731f311b5L, + 0x848446635bd1c3c8L, + 0x8e6970b2f86a5eb7L, + 0x915e2bc01ca6f936L, + 0x9bb31d11bf1d6449L, + 0x568a46b2f35a8a3cL, + 0x5c67706350e11743L, + 0x43502b11b42db0c2L, + 0x49bd1dc017962dbdL, + 0x7d3e9df47db4ffc0L, + 0x77d3ab25de0f62bfL, + 0x68e4f0573ac3c53eL, + 0x6209c68699785841L, + 0x01e3f03fee8661c4L, + 0x0b0ec6ee4d3dfcbbL, + 0x14399d9ca9f15b3aL, + 0x1ed4ab4d0a4ac645L, + 0x2a572b7960681438L, + 0x20ba1da8c3d38947L, + 0x3f8d46da271f2ec6L, + 0x3560700b84a4b3b9L, + 0x43d92f01b8cf1851L, + 0x493419d01b74852eL, + 0x560342a2ffb822afL, + 0x5cee74735c03bfd0L, + 0x686df44736216dadL, + 0x6280c296959af0d2L, + 0x7db799e471565753L, + 0x775aaf35d2edca2cL, + 0x14b0998ca513f3a9L, + 0x1e5daf5d06a86ed6L, + 0x016af42fe264c957L, + 0x0b87c2fe41df5428L, + 0x3f0442ca2bfd8655L, + 0x35e9741b88461b2aL, + 0x2ade2f696c8abcabL, + 0x203319b8cf3121d4L, + 0xed0a421b8376cfa1L, + 0xe7e774ca20cd52deL, + 0xf8d02fb8c401f55fL, + 0xf23d196967ba6820L, + 0xc6be995d0d98ba5dL, + 0xcc53af8cae232722L, + 0xd364f4fe4aef80a3L, + 0xd989c22fe9541ddcL, + 0xba63f4969eaa2459L, + 0xb08ec2473d11b926L, + 0xafb99935d9dd1ea7L, + 0xa554afe47a6683d8L, + 0x91d72fd0104451a5L, + 0x9b3a1901b3ffccdaL, + 0x840d427357336b5bL, + 0x8ee074a2f488f624L, + 0x2aa6d366972b24daL, + 0x204be5b73490b9a5L, + 0x3f7cbec5d05c1e24L, + 0x3591881473e7835bL, + 0x0112082019c55126L, + 0x0bff3ef1ba7ecc59L, + 0x14c865835eb26bd8L, + 0x1e255352fd09f6a7L, + 0x7dcf65eb8af7cf22L, + 0x7722533a294c525dL, + 0x68150848cd80f5dcL, + 0x62f83e996e3b68a3L, + 0x567bbead0419badeL, + 0x5c96887ca7a227a1L, + 0x43a1d30e436e8020L, + 0x494ce5dfe0d51d5fL, + 0x8475be7cac92f32aL, + 0x8e9888ad0f296e55L, + 0x91afd3dfebe5c9d4L, + 0x9b42e50e485e54abL, + 0xafc1653a227c86d6L, + 0xa52c53eb81c71ba9L, + 0xba1b0899650bbc28L, + 0xb0f63e48c6b02157L, + 0xd31c08f1b14e18d2L, + 0xd9f13e2012f585adL, + 0xc6c66552f639222cL, + 0xcc2b53835582bf53L, + 0xf8a8d3b73fa06d2eL, + 0xf245e5669c1bf051L, + 0xed72be1478d757d0L, + 0xe79f88c5db6ccaafL, + 0x0000000000000000L, + 0xb0bc2e589204f500L, + 0x55a17ae27c9e796bL, + 0xe51d54baee9a8c6bL, + 0xab42f5c4f93cf2d6L, + 0x1bfedb9c6b3807d6L, + 0xfee38f2685a28bbdL, + 0x4e5fa17e17a67ebdL, + 0x625ccddaaaee76c7L, + 0xd2e0e38238ea83c7L, + 0x37fdb738d6700facL, + 0x874199604474faacL, + 0xc91e381e53d28411L, + 0x79a21646c1d67111L, + 0x9cbf42fc2f4cfd7aL, + 0x2c036ca4bd48087aL, + 0xc4b99bb555dced8eL, + 0x7405b5edc7d8188eL, + 0x9118e157294294e5L, + 0x21a4cf0fbb4661e5L, + 0x6ffb6e71ace01f58L, + 0xdf4740293ee4ea58L, + 0x3a5a1493d07e6633L, + 0x8ae63acb427a9333L, + 0xa6e5566fff329b49L, + 0x165978376d366e49L, + 0xf3442c8d83ace222L, + 0x43f802d511a81722L, + 0x0da7a3ab060e699fL, + 0xbd1b8df3940a9c9fL, + 0x5806d9497a9010f4L, + 0xe8baf711e894e5f4L, + 0xbdaa1139f32e4877L, + 0x0d163f61612abd77L, + 0xe80b6bdb8fb0311cL, + 0x58b745831db4c41cL, + 0x16e8e4fd0a12baa1L, + 0xa654caa598164fa1L, + 0x43499e1f768cc3caL, + 0xf3f5b047e48836caL, + 0xdff6dce359c03eb0L, + 0x6f4af2bbcbc4cbb0L, + 0x8a57a601255e47dbL, + 0x3aeb8859b75ab2dbL, + 0x74b42927a0fccc66L, + 0xc408077f32f83966L, + 0x211553c5dc62b50dL, + 0x91a97d9d4e66400dL, + 0x79138a8ca6f2a5f9L, + 0xc9afa4d434f650f9L, + 0x2cb2f06eda6cdc92L, + 0x9c0ede3648682992L, + 0xd2517f485fce572fL, + 0x62ed5110cdcaa22fL, + 0x87f005aa23502e44L, + 0x374c2bf2b154db44L, + 0x1b4f47560c1cd33eL, + 0xabf3690e9e18263eL, + 0x4eee3db47082aa55L, + 0xfe5213ece2865f55L, + 0xb00db292f52021e8L, + 0x00b19cca6724d4e8L, + 0xe5acc87089be5883L, + 0x5510e6281bbaad83L, + 0x4f8d0420becb0385L, + 0xff312a782ccff685L, + 0x1a2c7ec2c2557aeeL, + 0xaa90509a50518feeL, + 0xe4cff1e447f7f153L, + 0x5473dfbcd5f30453L, + 0xb16e8b063b698838L, + 0x01d2a55ea96d7d38L, + 0x2dd1c9fa14257542L, + 0x9d6de7a286218042L, + 0x7870b31868bb0c29L, + 0xc8cc9d40fabff929L, + 0x86933c3eed198794L, + 0x362f12667f1d7294L, + 0xd33246dc9187feffL, + 0x638e688403830bffL, + 0x8b349f95eb17ee0bL, + 0x3b88b1cd79131b0bL, + 0xde95e57797899760L, + 0x6e29cb2f058d6260L, + 0x20766a51122b1cddL, + 0x90ca4409802fe9ddL, + 0x75d710b36eb565b6L, + 0xc56b3eebfcb190b6L, + 0xe968524f41f998ccL, + 0x59d47c17d3fd6dccL, + 0xbcc928ad3d67e1a7L, + 0x0c7506f5af6314a7L, + 0x422aa78bb8c56a1aL, + 0xf29689d32ac19f1aL, + 0x178bdd69c45b1371L, + 0xa737f331565fe671L, + 0xf22715194de54bf2L, + 0x429b3b41dfe1bef2L, + 0xa7866ffb317b3299L, + 0x173a41a3a37fc799L, + 0x5965e0ddb4d9b924L, + 0xe9d9ce8526dd4c24L, + 0x0cc49a3fc847c04fL, + 0xbc78b4675a43354fL, + 0x907bd8c3e70b3d35L, + 0x20c7f69b750fc835L, + 0xc5daa2219b95445eL, + 0x75668c790991b15eL, + 0x3b392d071e37cfe3L, + 0x8b85035f8c333ae3L, + 0x6e9857e562a9b688L, + 0xde2479bdf0ad4388L, + 0x369e8eac1839a67cL, + 0x8622a0f48a3d537cL, + 0x633ff44e64a7df17L, + 0xd383da16f6a32a17L, + 0x9ddc7b68e10554aaL, + 0x2d6055307301a1aaL, + 0xc87d018a9d9b2dc1L, + 0x78c12fd20f9fd8c1L, + 0x54c24376b2d7d0bbL, + 0xe47e6d2e20d325bbL, + 0x01633994ce49a9d0L, + 0xb1df17cc5c4d5cd0L, + 0xff80b6b24beb226dL, + 0x4f3c98ead9efd76dL, + 0xaa21cc5037755b06L, + 0x1a9de208a571ae06L, + 0x9f1a08417d96070aL, + 0x2fa62619ef92f20aL, + 0xcabb72a301087e61L, + 0x7a075cfb930c8b61L, + 0x3458fd8584aaf5dcL, + 0x84e4d3dd16ae00dcL, + 0x61f98767f8348cb7L, + 0xd145a93f6a3079b7L, + 0xfd46c59bd77871cdL, + 0x4dfaebc3457c84cdL, + 0xa8e7bf79abe608a6L, + 0x185b912139e2fda6L, + 0x5604305f2e44831bL, + 0xe6b81e07bc40761bL, + 0x03a54abd52dafa70L, + 0xb31964e5c0de0f70L, + 0x5ba393f4284aea84L, + 0xeb1fbdacba4e1f84L, + 0x0e02e91654d493efL, + 0xbebec74ec6d066efL, + 0xf0e16630d1761852L, + 0x405d48684372ed52L, + 0xa5401cd2ade86139L, + 0x15fc328a3fec9439L, + 0x39ff5e2e82a49c43L, + 0x8943707610a06943L, + 0x6c5e24ccfe3ae528L, + 0xdce20a946c3e1028L, + 0x92bdabea7b986e95L, + 0x220185b2e99c9b95L, + 0xc71cd108070617feL, + 0x77a0ff509502e2feL, + 0x22b019788eb84f7dL, + 0x920c37201cbcba7dL, + 0x7711639af2263616L, + 0xc7ad4dc26022c316L, + 0x89f2ecbc7784bdabL, + 0x394ec2e4e58048abL, + 0xdc53965e0b1ac4c0L, + 0x6cefb806991e31c0L, + 0x40ecd4a2245639baL, + 0xf050fafab652ccbaL, + 0x154dae4058c840d1L, + 0xa5f18018caccb5d1L, + 0xebae2166dd6acb6cL, + 0x5b120f3e4f6e3e6cL, + 0xbe0f5b84a1f4b207L, + 0x0eb375dc33f04707L, + 0xe60982cddb64a2f3L, + 0x56b5ac95496057f3L, + 0xb3a8f82fa7fadb98L, + 0x0314d67735fe2e98L, + 0x4d4b770922585025L, + 0xfdf75951b05ca525L, + 0x18ea0deb5ec6294eL, + 0xa85623b3ccc2dc4eL, + 0x84554f17718ad434L, + 0x34e9614fe38e2134L, + 0xd1f435f50d14ad5fL, + 0x61481bad9f10585fL, + 0x2f17bad388b626e2L, + 0x9fab948b1ab2d3e2L, + 0x7ab6c031f4285f89L, + 0xca0aee69662caa89L, + 0xd0970c61c35d048fL, + 0x602b22395159f18fL, + 0x85367683bfc37de4L, + 0x358a58db2dc788e4L, + 0x7bd5f9a53a61f659L, + 0xcb69d7fda8650359L, + 0x2e74834746ff8f32L, + 0x9ec8ad1fd4fb7a32L, + 0xb2cbc1bb69b37248L, + 0x0277efe3fbb78748L, + 0xe76abb59152d0b23L, + 0x57d695018729fe23L, + 0x1989347f908f809eL, + 0xa9351a27028b759eL, + 0x4c284e9dec11f9f5L, + 0xfc9460c57e150cf5L, + 0x142e97d49681e901L, + 0xa492b98c04851c01L, + 0x418fed36ea1f906aL, + 0xf133c36e781b656aL, + 0xbf6c62106fbd1bd7L, + 0x0fd04c48fdb9eed7L, + 0xeacd18f2132362bcL, + 0x5a7136aa812797bcL, + 0x76725a0e3c6f9fc6L, + 0xc6ce7456ae6b6ac6L, + 0x23d320ec40f1e6adL, + 0x936f0eb4d2f513adL, + 0xdd30afcac5536d10L, + 0x6d8c819257579810L, + 0x8891d528b9cd147bL, + 0x382dfb702bc9e17bL, + 0x6d3d1d5830734cf8L, + 0xdd813300a277b9f8L, + 0x389c67ba4ced3593L, + 0x882049e2dee9c093L, + 0xc67fe89cc94fbe2eL, + 0x76c3c6c45b4b4b2eL, + 0x93de927eb5d1c745L, + 0x2362bc2627d53245L, + 0x0f61d0829a9d3a3fL, + 0xbfddfeda0899cf3fL, + 0x5ac0aa60e6034354L, + 0xea7c84387407b654L, + 0xa423254663a1c8e9L, + 0x149f0b1ef1a53de9L, + 0xf1825fa41f3fb182L, + 0x413e71fc8d3b4482L, + 0xa98486ed65afa176L, + 0x1938a8b5f7ab5476L, + 0xfc25fc0f1931d81dL, + 0x4c99d2578b352d1dL, + 0x02c673299c9353a0L, + 0xb27a5d710e97a6a0L, + 0x576709cbe00d2acbL, + 0xe7db27937209dfcbL, + 0xcbd84b37cf41d7b1L, + 0x7b64656f5d4522b1L, + 0x9e7931d5b3dfaedaL, + 0x2ec51f8d21db5bdaL, + 0x609abef3367d2567L, + 0xd02690aba479d067L, + 0x353bc4114ae35c0cL, + 0x8587ea49d8e7a90cL, }; + + private static final long[] M_UX2N = { + 0x0080000000000000L, + 0x0000800000000000L, + 0x0000000080000000L, + 0x9a6c9329ac4bc9b5L, + 0x10f4bb0f129310d6L, + 0x70f05dcea2ebd226L, + 0x311211205672822dL, + 0x2fc297db0f46c96eL, + 0xca4d536fabf7da84L, + 0xfb4cdc3b379ee6edL, + 0xea261148df25140aL, + 0x59ccb2c07aa6c9b4L, + 0x20b3674a839af27aL, + 0x2d8e1986da94d583L, + 0x42cdf4c20337635dL, + 0x1d78724bf0f26839L, + 0xb96c84e0afb34bd5L, + 0x5d2e1fcd2df0a3eaL, + 0xcd9506572332be42L, + 0x23bda2427f7d690fL, + 0x347a953232374f07L, + 0x1c2a807ac2a8ceeaL, + 0x9b92ad0e14fe1460L, + 0x2574114889f670b2L, + 0x4a84a6c45e3bf520L, + 0x915bbac21cd1c7ffL, + 0xb0290ec579f291f5L, + 0xcf2548505c624e6eL, + 0xb154f27bf08a8207L, + 0xce4e92344baf7d35L, + 0x51da8d7e057c5eb3L, + 0x9fb10823f5be15dfL, + 0x73b825b3ff1f71cfL, + 0x5db436c5406ebb74L, + 0xfa7ed8f3ec3f2bcaL, + 0xc4d58efdc61b9ef6L, + 0xa7e39e61e855bd45L, + 0x97ad46f9dd1bf2f1L, + 0x1a0abb01f853ee6bL, + 0x3f0827c3348f8215L, + 0x4eb68c4506134607L, + 0x4a46f6de5df34e0aL, + 0x2d855d6a1c57a8ddL, + 0x8688da58e1115812L, + 0x5232f417fc7c7300L, + 0xa4080fb2e767d8daL, + 0xd515a7e17693e562L, + 0x1181f7c862e94226L, + 0x9e23cd058204ca91L, + 0x9b8992c57a0aed82L, + 0xb2c0afb84609b6ffL, + 0x2f7160553a5ea018L, + 0x3cd378b5c99f2722L, + 0x814054ad61a3b058L, + 0xbf766189fce806d8L, + 0x85a5e898ac49f86fL, + 0x34830d11bc84f346L, + 0x9644d95b173c8c1cL, + 0x150401ac9ac759b1L, + 0xebe1f7f46fb00ebaL, + 0x8ee4ce0c2e2bd662L, + 0x4000000000000000L, + 0x2000000000000000L, + 0x0800000000000000L, }; + + /** + * Computes the CRC64 checksum for the given byte array using the Azure Storage CRC64 polynomial. + * This method processes the input data in chunks of 32 bytes for efficiency and uses lookup tables + * to update the CRC values. + * + * @param src the byte array for which the CRC64 checksum is to be computed. + * @param uCrc the initial CRC value. + * @return the computed CRC64 checksum. + */ + public static long compute(byte[] src, long uCrc) { + return compute(src, 0, src.length, uCrc); + } + + /** + * Computes the CRC64 checksum for a region of a byte array. Avoids copying when the caller has + * an array view (e.g. {@link ByteBuffer#array()} with array offset and position) or when combined + * with {@link #compute(byte[], long)} for whole-array input. + * + * @param src the byte array. + * @param offset start index (inclusive). + * @param length number of bytes to process. + * @param uCrc the initial CRC value. + * @return the computed CRC64 checksum. + */ + public static long compute(byte[] src, int offset, int length, long uCrc) { + int pData = offset; + long uSize = length; + long uBytes, uStop; + + uCrc = ~uCrc; // Flip all bits of uCrc + + uStop = uSize - (uSize % 32); + if (uStop >= 2 * 32) { + long uCrc0 = 0L; + long uCrc1 = 0L; + long uCrc2 = 0L; + long uCrc3 = 0L; + + int pLast = pData + (int) uStop - 32; + uSize -= uStop; + uCrc0 = uCrc; + + ByteBuffer buffer = ByteBuffer.wrap(src).order(ByteOrder.LITTLE_ENDIAN); + + for (; pData < pLast; pData += 32) { + long b0, b1, b2, b3; + + // Load and XOR data with CRC + b0 = buffer.getLong(pData) ^ uCrc0; + b1 = buffer.getLong(pData + 8) ^ uCrc1; + b2 = buffer.getLong(pData + 16) ^ uCrc2; + b3 = buffer.getLong(pData + 24) ^ uCrc3; + + // Unsigned updates using tables and masking + uCrc0 = M_U32[7 * 256 + ((int) (b0 & 0xFF))]; + b0 >>>= 8; + uCrc1 = M_U32[7 * 256 + ((int) (b1 & 0xFF))]; + b1 >>>= 8; + uCrc2 = M_U32[7 * 256 + ((int) (b2 & 0xFF))]; + b2 >>>= 8; + uCrc3 = M_U32[7 * 256 + ((int) (b3 & 0xFF))]; + b3 >>>= 8; + + uCrc0 ^= M_U32[6 * 256 + ((int) (b0 & 0xFF))]; + b0 >>>= 8; + uCrc1 ^= M_U32[6 * 256 + ((int) (b1 & 0xFF))]; + b1 >>>= 8; + uCrc2 ^= M_U32[6 * 256 + ((int) (b2 & 0xFF))]; + b2 >>>= 8; + uCrc3 ^= M_U32[6 * 256 + ((int) (b3 & 0xFF))]; + b3 >>>= 8; + + uCrc0 ^= M_U32[5 * 256 + ((int) (b0 & 0xFF))]; + b0 >>>= 8; + uCrc1 ^= M_U32[5 * 256 + ((int) (b1 & 0xFF))]; + b1 >>>= 8; + uCrc2 ^= M_U32[5 * 256 + ((int) (b2 & 0xFF))]; + b2 >>>= 8; + uCrc3 ^= M_U32[5 * 256 + ((int) (b3 & 0xFF))]; + b3 >>>= 8; + + uCrc0 ^= M_U32[4 * 256 + ((int) (b0 & 0xFF))]; + b0 >>>= 8; + uCrc1 ^= M_U32[4 * 256 + ((int) (b1 & 0xFF))]; + b1 >>>= 8; + uCrc2 ^= M_U32[4 * 256 + ((int) (b2 & 0xFF))]; + b2 >>>= 8; + uCrc3 ^= M_U32[4 * 256 + ((int) (b3 & 0xFF))]; + b3 >>>= 8; + + uCrc0 ^= M_U32[3 * 256 + ((int) (b0 & 0xFF))]; + b0 >>>= 8; + uCrc1 ^= M_U32[3 * 256 + ((int) (b1 & 0xFF))]; + b1 >>>= 8; + uCrc2 ^= M_U32[3 * 256 + ((int) (b2 & 0xFF))]; + b2 >>>= 8; + uCrc3 ^= M_U32[3 * 256 + ((int) (b3 & 0xFF))]; + b3 >>>= 8; + + uCrc0 ^= M_U32[2 * 256 + ((int) (b0 & 0xFF))]; + b0 >>>= 8; + uCrc1 ^= M_U32[2 * 256 + ((int) (b1 & 0xFF))]; + b1 >>>= 8; + uCrc2 ^= M_U32[2 * 256 + ((int) (b2 & 0xFF))]; + b2 >>>= 8; + uCrc3 ^= M_U32[2 * 256 + ((int) (b3 & 0xFF))]; + b3 >>>= 8; + + uCrc0 ^= M_U32[256 + ((int) (b0 & 0xFF))]; + b0 >>>= 8; + uCrc1 ^= M_U32[256 + ((int) (b1 & 0xFF))]; + b1 >>>= 8; + uCrc2 ^= M_U32[256 + ((int) (b2 & 0xFF))]; + b2 >>>= 8; + uCrc3 ^= M_U32[256 + ((int) (b3 & 0xFF))]; + b3 >>>= 8; + + uCrc0 ^= M_U32[((int) (b0 & 0xFF))]; + uCrc1 ^= M_U32[((int) (b1 & 0xFF))]; + uCrc2 ^= M_U32[((int) (b2 & 0xFF))]; + uCrc3 ^= M_U32[((int) (b3 & 0xFF))]; + } + + // Combine CRC values + uCrc = 0; + uCrc ^= ByteBuffer.wrap(src, pData, 8).order(ByteOrder.LITTLE_ENDIAN).getLong() ^ uCrc0; + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + + uCrc ^= ByteBuffer.wrap(src, pData + 8, 8).order(ByteOrder.LITTLE_ENDIAN).getLong() ^ uCrc1; + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + + uCrc ^= ByteBuffer.wrap(src, pData + 16, 8).order(ByteOrder.LITTLE_ENDIAN).getLong() ^ uCrc2; + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + + uCrc ^= ByteBuffer.wrap(src, pData + 24, 8).order(ByteOrder.LITTLE_ENDIAN).getLong() ^ uCrc3; + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + + pData += 32; + } + + // Process remaining bytes + for (uBytes = 0; uBytes < uSize; ++uBytes, ++pData) { + uCrc = (uCrc >>> 8) ^ M_U1[(int) ((uCrc ^ src[pData]) & 0xFF)]; + } + + return ~uCrc; // Flip all bits of uCrc and return as long + } + + /** + * Concatenates two CRC64 values by combining their initial and final CRC values and sizes. + * This method ensures unsigned behavior and uses the `mulX_N` method to perform necessary + * multiplications in GF(2^64). + * + * @param uInitialCrcAB The initial CRC value for the concatenated data. + * @param uInitialCrcA The initial CRC value for the first data segment. + * @param uFinalCrcA The final CRC value for the first data segment. + * @param uSizeA The size of the first data segment. + * @param uInitialCrcB The initial CRC value for the second data segment. + * @param uFinalCrcB The final CRC value for the second data segment. + * @param uSizeB The size of the second data segment. + * @return The concatenated CRC64 value. + */ + public static long concat(long uInitialCrcAB, long uInitialCrcA, long uFinalCrcA, long uSizeA, long uInitialCrcB, + long uFinalCrcB, long uSizeB) { + long uFinalCrcAB = ~uFinalCrcA; + + // Ensure unsigned behavior when comparing uInitialCrcA and uInitialCrcAB + if ((uInitialCrcA) != (uInitialCrcAB)) { + // Apply mulX_N with proper unsigned masking + uFinalCrcAB ^= multiplyCrcByPowerOfX((uInitialCrcA ^ uInitialCrcAB), uSizeA); + } + + uFinalCrcAB ^= ~uInitialCrcB; // Ensure unsigned XOR logic + uFinalCrcAB = multiplyCrcByPowerOfX(uFinalCrcAB, uSizeB); + uFinalCrcAB ^= uFinalCrcB; + + return uFinalCrcAB; // Ensure the result is treated as unsigned + } + + /** + * Multiplies a CRC value by x^n in GF(2^64). + * This method uses a lookup table to perform the multiplication efficiently. + * + * @param a The CRC value to be multiplied. + * @param uSize The power of x by which the CRC value is to be multiplied. + * @return The resulting CRC64 value after multiplication. + */ + private static long multiplyCrcByPowerOfX(long a, long uSize) { + long i = 0; + long r = a; + + while (uSize != 0) { + if ((uSize & 1) == 1) { + r = mulPolyUnrolled(r, M_UX2N[(int) i]); // Ensure result is treated as unsigned + } + uSize >>>= 1; // Unsigned right shift + i += 1; + } + + return r; + } + + /** + * Multiplies two CRC values using the polynomial in GF(2^64). + * This method performs the multiplication using an unrolled loop for efficiency. + * + * @param a The first CRC value to be multiplied. + * @param b The second CRC value to be multiplied. + * @return The resulting CRC64 value after multiplication. + */ + private static long mulPolyUnrolled(long a, long b) { + final long p = POLY; + final long p2 = (p >>> 1) ^ (p); // Use unsigned shift + final long bw = Long.SIZE; + + final long[] vt = { 0, p2, p, p ^ p2 }; + final long[] vs = { bw - 2, bw - 1 }; + long[] vb = { (b >>> 1) ^ vt[(int) ((b & 1) << 1)], b }; // Unsigned right shift + long[] vr = { 0, 0 }; + + for (long i = 0; i < bw; i += 2) { + for (int j = 0; j < 2; ++j) { + vr[j] ^= vb[j] * ((a >>> vs[j]) & 1); // Unsigned right shift + vb[j] = (vb[j] >>> 2) ^ vt[(int) (vb[j] & 3)]; // Unsigned right shift + } + a <<= 2; // Shift remains the same as a is multiplied by 4 + } + + return vr[0] ^ vr[1]; + } + +} diff --git a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageConstants.java b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageConstants.java new file mode 100644 index 000000000000..8e0ed1ff6e86 --- /dev/null +++ b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageConstants.java @@ -0,0 +1,57 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.common.implementation.contentvalidation; + +/** + * Constants used in the structured message encoding and decoding process. + */ +public final class StructuredMessageConstants { + /** + * The default version of the structured message. + */ + public static final int DEFAULT_MESSAGE_VERSION = 1; + + /** + * The length of the header for version 1 of the structured message. + */ + public static final int V1_HEADER_LENGTH = 13; + + /** + * The length of the segment header for version 1 of the structured message. + */ + public static final int V1_SEGMENT_HEADER_LENGTH = 10; + + /** + * The length of the CRC64 checksum. + */ + public static final int CRC64_LENGTH = 8; + + /** + * The default length of segments for version 1. + */ + public static final int V1_DEFAULT_SEGMENT_CONTENT_LENGTH = 4 * 1024 * 1024; // 4 MiB + + /** + * The maximum amount of data to encode at once. + */ + public static final int STATIC_MAXIMUM_ENCODED_DATA_LENGTH = 4 * 1024 * 1024; // 4 MiB + + /** + * The maximum single part upload size to use CRC64 header. + */ + public static final int MAXIMUM_SINGLE_SHOT_UPLOAD_SIZE_TO_USE_CRC64_HEADER = 4 * 1024 * 1024; // 4 MiB + + /** + * The structured body type header value indicating version 1 with CRC64 properties. + */ + public static final String STRUCTURED_BODY_TYPE_VALUE = "XSM/1.0; properties=crc64"; + + public static final String CONTENT_VALIDATION_MODE_KEY = "contentValidationMode"; + + public static final String USE_CRC64_CHECKSUM_HEADER_CONTEXT = "crc64ChecksumHeaderContext"; + + public static final String USE_STRUCTURED_MESSAGE_CONTEXT = "structuredMessageChecksumAlgorithm"; + + public static final String STRUCTURED_MESSAGE_DECODING_CONTEXT_KEY = "azure-storage-structured-message-decoding"; +} diff --git a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageDecoder.java b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageDecoder.java new file mode 100644 index 000000000000..3a6570fe07bc --- /dev/null +++ b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageDecoder.java @@ -0,0 +1,536 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.common.implementation.contentvalidation; + +import com.azure.core.util.logging.ClientLogger; + +import java.io.ByteArrayOutputStream; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import static com.azure.storage.common.implementation.contentvalidation.StructuredMessageConstants.CRC64_LENGTH; +import static com.azure.storage.common.implementation.contentvalidation.StructuredMessageConstants.DEFAULT_MESSAGE_VERSION; +import static com.azure.storage.common.implementation.contentvalidation.StructuredMessageConstants.V1_HEADER_LENGTH; +import static com.azure.storage.common.implementation.contentvalidation.StructuredMessageConstants.V1_SEGMENT_HEADER_LENGTH; + +/** + * Streaming decoder for the storage structured message format used to validate downloaded blob/file/datalake + * content with CRC64 checksums. + * + *

This class owns the actual parsing and CRC validation. The pipeline policy hands it raw {@link ByteBuffer}s as + * they arrive on the wire (via {@link #decodeChunk(ByteBuffer)}); the decoder returns only the payload bytes that + * have already been CRC-validated and tells the policy when the entire message has been consumed + * (via {@link #isComplete()}). Any malformed input or CRC mismatch surfaces as an + * {@link IllegalArgumentException} thrown from {@code decodeChunk} so the policy can translate it into a stream + * error.

+ * + *

Wire format (V1)

+ * + *

The encoded body has the following layout (all integers little-endian):

+ * 
+ *   |-- message header (13 B) ----------------------------------------|
+ *   |  version (1)  |  total message length (8)  |  flags (2)  |  numSegments (2)  |
+ *
+ *   for each segment in 1..numSegments:
+ *     |-- segment header (10 B) -|
+ *     |  segNum (2)  |  segContentLen (8)  |
+ *     |-- segment payload (segContentLen B) --|
+ *     |-- segment CRC64 footer (8 B; only if STORAGE_CRC64) --|
+ *
+ *   |-- message CRC64 footer (8 B; only if STORAGE_CRC64) --|
+ *
+ * + *

Emission guarantee

+ * + * Payload bytes for a segment are never emitted to the caller until that segment's CRC64 footer + * has been validated. This matches the emission semantics used by {@code BlobDecryptionPolicy}/{@code DecryptorV2} + * (which only emits a decrypted region after its GCM tag is verified) and ensures that no unvalidated bytes are + * exposed to consumers, even if the connection is later torn down or the download is retried. + * + *

Thread-safety

+ * + *

This class is not thread-safe. A new instance is created for every HTTP response, and the + * reactive operators in the policy ({@code concatMap}) serialize access to the single instance. Retries produce new + * HTTP responses and therefore new decoder instances, so a CRC failure on one attempt cannot pollute another.

+ */ +public class StructuredMessageDecoder { + private static final ClientLogger LOGGER = new ClientLogger(StructuredMessageDecoder.class); + + private long messageLength = -1; + private StructuredMessageFlags flags; + private int numSegments = -1; + private final long expectedEncodedMessageLength; + // Number of encoded bytes consumed so far (headers + payloads + footers). + private long messageOffset = 0; + private int currentSegmentNumber = 0; + private long currentSegmentContentLength = 0; + private long currentSegmentContentOffset = 0; + private boolean segmentHeaderRead = false; + // Running CRC64 over all payload bytes seen so far (across every segment). + private long messageCrc64 = 0; + // Running CRC64 over only the current segment's payload bytes. + private long segmentCrc64 = 0; + // Holds bytes left over from a previous decodeChunk() call when the current chunk did not contain a full + // header or footer. + private final ByteArrayOutputStream pendingBytes = new ByteArrayOutputStream(); + // Payload for the current segment, one per inbound wire buffer. Not pre-sized to the full segment length, + // avoiding unnecessary heap spikes. + // These bytes are intentionally NOT emitted until the segment's CRC footer has been validated. + private List currentSegmentPayload; + // Validated payload buffers to emit from the current decodeChunk() invocation. + private final List validatedOutput = new ArrayList<>(); + + /** + * Constructs a new StructuredMessageDecoder. + * + * @param expectedEncodedMessageLength The expected encoded structured-message length (typically HTTP + * {@code Content-Length}). + */ + public StructuredMessageDecoder(long expectedEncodedMessageLength) { + this.expectedEncodedMessageLength = expectedEncodedMessageLength; + } + + /** + * Reads the 13-byte message header (version + total length + flags + numSegments) the first time the decoder + * sees enough bytes, and validates each field. Subsequent calls are no-ops. + * + * @param buffer The buffer to read from. + * @return true if the header was successfully read (or had already been read on a previous pass); false if more + * bytes are still needed. + */ + private boolean tryReadMessageHeader(ByteBuffer buffer) { + if (messageLength != -1) { + // Header already parsed on a previous chunk; nothing to do. + return true; + } + + if (getAvailableBytes(buffer) < V1_HEADER_LENGTH) { + // Not enough bytes for the full header yet; carry over what we have. + appendToPending(buffer); + return false; + } + + ByteBuffer combined = getCombinedBuffer(buffer); + + // Byte 0: protocol version. + int messageVersion = Byte.toUnsignedInt(combined.get()); + if (messageVersion != DEFAULT_MESSAGE_VERSION) { + throw LOGGER.logExceptionAsError(new IllegalArgumentException( + enrichExceptionMessage("Unsupported structured message version: " + messageVersion))); + } + + // Bytes 1-8: total encoded message length. Must be at least the header itself, and must agree with what the + // HTTP layer told us via Content-Length – any disagreement implies a truncated/extended response. + long msgLen = combined.getLong(); + if (msgLen < V1_HEADER_LENGTH) { + throw LOGGER.logExceptionAsError( + new IllegalArgumentException(enrichExceptionMessage("Message length too small: " + msgLen))); + } + if (msgLen != expectedEncodedMessageLength) { + throw LOGGER + .logExceptionAsError(new IllegalArgumentException(enrichExceptionMessage("Structured message length " + + msgLen + " did not match content length " + expectedEncodedMessageLength))); + } + + // Bytes 9-10: flags (NONE or STORAGE_CRC64). Bytes 11-12: number of segments. + flags = StructuredMessageFlags.fromValue(Short.toUnsignedInt(combined.getShort())); + numSegments = Short.toUnsignedInt(combined.getShort()); + if (numSegments < 1) { + throw LOGGER.logExceptionAsError(new IllegalArgumentException( + enrichExceptionMessage("Structured message must have at least one segment, got: " + numSegments))); + } + + // Commit: drop the 13 bytes we just parsed from pending/buffer and record the message length. + consumeBytes(V1_HEADER_LENGTH, buffer); + messageOffset += V1_HEADER_LENGTH; + messageLength = msgLen; + + return true; + } + + /** + * Reads the 10-byte header for the next segment (segment number + segment payload length) and resets + * per-segment state so {@link #tryReadSegmentContent(ByteBuffer)} can begin filling + * {@link #currentSegmentPayload}. + * + *

Validates that segments arrive in order and that the declared segment size leaves enough room in the + * remaining message for any subsequent segment headers, payloads, footers, and the trailing message footer – + * this catches malformed/oversized segment lengths up front instead of waiting until we run off the end of the + * stream.

+ * + * @param buffer The buffer to read from. + * @return true if the segment header was read; false if more bytes are needed. + */ + private boolean tryReadSegmentHeader(ByteBuffer buffer) { + if (getAvailableBytes(buffer) < V1_SEGMENT_HEADER_LENGTH) { + appendToPending(buffer); + return false; + } + + ByteBuffer combined = getCombinedBuffer(buffer); + + // Bytes 0-1: segment number. Bytes 2-9: declared payload length of this segment. + int segmentNum = Short.toUnsignedInt(combined.getShort()); + long segmentSize = combined.getLong(); + + // Segments must arrive strictly in order so the running CRC and "segment N follows segment N-1" assumption + // hold. Anything else implies a malformed/reordered response. + if (segmentNum != currentSegmentNumber + 1) { + throw LOGGER.logExceptionAsError(new IllegalArgumentException(enrichExceptionMessage( + "Unexpected segment number. Expected: " + (currentSegmentNumber + 1) + ", got: " + segmentNum))); + } + + // Compute an upper bound on the legal segment size: whatever is left in the message, minus the bytes that + // MUST still appear after this segment's payload (this segment's footer, the headers/payloads/footers of all + // remaining segments, and the trailing message footer). + long footerSize = flags == StructuredMessageFlags.STORAGE_CRC64 ? CRC64_LENGTH : 0; + long remainingSegmentsAfterThis = (long) numSegments - segmentNum; + long reservedBytes + = footerSize + remainingSegmentsAfterThis * (V1_SEGMENT_HEADER_LENGTH + footerSize) + footerSize; + long maxSegmentSize = messageLength - messageOffset - V1_SEGMENT_HEADER_LENGTH - reservedBytes; + if (segmentSize < 0 || segmentSize > maxSegmentSize) { + throw LOGGER.logExceptionAsError(new IllegalArgumentException(enrichExceptionMessage( + "Invalid segment size detected: " + segmentSize + " (max=" + maxSegmentSize + ")"))); + } + + // Commit: drop the 10 header bytes and set up per-segment state so payload accumulation can start fresh. + consumeBytes(V1_SEGMENT_HEADER_LENGTH, buffer); + messageOffset += V1_SEGMENT_HEADER_LENGTH; + currentSegmentNumber = segmentNum; + currentSegmentContentLength = segmentSize; + currentSegmentContentOffset = 0; + currentSegmentPayload = new ArrayList<>(); + + if (flags == StructuredMessageFlags.STORAGE_CRC64) { + // Reset only the per-segment running CRC; the message-wide running CRC keeps accumulating across all + // segments so the final message footer covers the entire payload. + segmentCrc64 = 0; + } + + return true; + } + + /** + * Pulls as many payload bytes as possible (bounded by what is still owed for the current segment) from the + * pending+buffer view into {@link #currentSegmentPayload}, updating the running per-segment and per-message + * CRC64 values along the way. + * + *

Bytes accumulated here are not yet emitted to the caller. They are released only after + * {@link #tryReadSegmentFooter(ByteBuffer)} validates this segment's CRC. This is the mechanism that enforces + * "no unvalidated bytes ever leave the decoder".

+ * + * @param buffer The buffer to read from. + * @return The number of payload bytes read in this call (0 means we either had no bytes available or the + * current segment's payload was already complete). + */ + private int tryReadSegmentContent(ByteBuffer buffer) { + long remaining = currentSegmentContentLength - currentSegmentContentOffset; + if (remaining == 0) { + // Segment payload is already complete; nothing to do here. The caller will move on to read the footer. + return 0; + } + + int available = getAvailableBytes(buffer); + if (available == 0) { + return 0; + } + + // Read the minimum of "what's available right now" and "what's still owed for this segment" so we never + // accidentally consume the segment footer here. + int toRead = (int) Math.min(available, remaining); + ByteBuffer combined = getCombinedBuffer(buffer); + + // Materialize only this chunk so retained memory grows with bytes received, not the full declared segment size. + byte[] content = new byte[toRead]; + combined.get(content); + + if (flags == StructuredMessageFlags.STORAGE_CRC64) { + // Update both CRCs incrementally: the segment CRC will be checked at the segment footer, and the + // message CRC accumulates across every segment to be checked at the message footer. + segmentCrc64 = StorageCrc64Calculator.compute(content, 0, toRead, segmentCrc64); + messageCrc64 = StorageCrc64Calculator.compute(content, 0, toRead, messageCrc64); + } + + consumeBytes(toRead, buffer); + currentSegmentPayload.add(content); + + messageOffset += toRead; + currentSegmentContentOffset += toRead; + + return toRead; + } + + /** + * Validates the 8-byte segment CRC64 footer for the segment that has just finished accumulating. Pre-condition: + * {@code currentSegmentContentOffset == currentSegmentContentLength}. + * + *

This step is intentionally separate from reading the message footer: when the CRC matches, we want to be + * able to flush the buffered segment payload to the caller right away – even if the trailing message footer is + * not yet available in the current chunk.

+ * + * @param buffer The buffer to read from. + * @return true if the footer was successfully read (or no footer is required for this message); false if more + * bytes are still needed. + */ + private boolean tryReadSegmentFooter(ByteBuffer buffer) { + if (currentSegmentContentOffset != currentSegmentContentLength) { + // Segment payload is not complete yet; wait for more content. + return true; + } + + if (flags == StructuredMessageFlags.STORAGE_CRC64) { + return tryConsumeCrc64Footer(buffer, segmentCrc64, " in segment " + currentSegmentNumber); + } + + // No CRC was negotiated, so there is no footer to read; the caller can release the buffered payload. + return true; + } + + /** + * Validates the 8-byte message CRC64 footer that follows the last segment. + * + * @param buffer The buffer to read from. + * @return true if the footer was successfully read (or none is required); false if more bytes are still needed. + */ + private boolean tryReadMessageFooter(ByteBuffer buffer) { + if (flags == StructuredMessageFlags.STORAGE_CRC64) { + return tryConsumeCrc64Footer(buffer, messageCrc64, " in message footer"); + } + return true; + } + + /** + * Decodes as much as possible from the given buffer and returns any fully validated + * payload bytes that are now safe to emit downstream. + * + *

The returned buffers will only ever contain bytes from segments whose CRC (when + * enabled) has already been verified. If no segments have been fully validated by + * this invocation the method returns an empty list. Callers distinguish "more bytes + * needed" from "stream complete" via {@link #isComplete()}.

+ * + * @param buffer The buffer containing encoded data. + * @return Validated payload bytes ready to emit downstream; empty when none are ready yet. + * @throws IllegalArgumentException if the input is malformed or a CRC64 check fails. + */ + public List decodeChunk(ByteBuffer buffer) { + // Decoder always reads little-endian; force the order on the caller's buffer so all our get() calls match + // the wire format regardless of how the buffer was constructed. + buffer.order(ByteOrder.LITTLE_ENDIAN); + + // Output collected during this single invocation. Each segment whose CRC validates in this call is appended + // here and ultimately returned to the policy as one or more ByteBuffers. + validatedOutput.clear(); + + // Step 1: parse the message header on the first chunk that has enough bytes for it. If this chunk doesn't, + // bail out early. + if (!tryReadMessageHeader(buffer)) { + return finishDecodeChunk(); + } + + // Step 2: walk forward through the message until we either hit the end (messageOffset == messageLength) or + // we run out of bytes for the current structural element and have to wait for the next chunk. + while (messageOffset < messageLength) { + if (!segmentHeaderRead) { + // We are between segments. If every segment has been processed, only the trailing message footer + // can still appear in the stream – read it (or wait for it) and exit. + if (currentSegmentNumber == numSegments) { + if (!tryReadMessageFooter(buffer)) { + break; + } + break; + } + // Otherwise, parse the next segment's header. May return false if it is split across chunks. + if (!tryReadSegmentHeader(buffer)) { + break; + } + segmentHeaderRead = true; + } + + // Drain as many payload bytes as are available into the per-segment buffer. + int payloadRead = tryReadSegmentContent(buffer); + + if (currentSegmentContentOffset == currentSegmentContentLength) { + // Segment payload fully buffered. Validate the CRC footer (if any). When the footer isn't fully + // available yet, break and resume on the next chunk – currentSegmentPayload keeps its contents so + // we can still emit them on the call where the footer arrives. + if (!tryReadSegmentFooter(buffer)) { + break; + } + // Segment passed validation: it is now safe to release the buffered payload to the caller. + releaseValidatedSegmentPayload(); + segmentHeaderRead = false; + // Loop continues: either consume the next segment's header or the message footer. + } else if (payloadRead == 0 && getAvailableBytes(buffer) == 0) { + // Nothing left to read this pass and the segment is not complete – wait for the next chunk. + break; + } + } + + return finishDecodeChunk(); + } + + /** + * Hands off validated segment bytes for emission as {@link ByteBuffer} views over the accumulated payload copies + * without consolidating them into a single array. + */ + private void releaseValidatedSegmentPayload() { + List validatedCopies = currentSegmentPayload; + currentSegmentPayload = null; + for (byte[] validatedCopy : validatedCopies) { + validatedOutput.add(ByteBuffer.wrap(validatedCopy)); + } + } + + private List finishDecodeChunk() { + if (validatedOutput.isEmpty()) { + return Collections.emptyList(); + } + List result; + if (validatedOutput.size() == 1) { + result = Collections.singletonList(validatedOutput.get(0)); + } else { + result = new ArrayList<>(validatedOutput); + } + validatedOutput.clear(); + return result; + } + + /** + * @return the total number of bytes the decoder can currently see across the carry-over {@link #pendingBytes} + * plus the unread tail of the supplied buffer. Used to decide whether a structural element (header / + * footer) can be parsed in this pass or whether we must defer to the next chunk. + */ + private int getAvailableBytes(ByteBuffer buffer) { + return pendingBytes.size() + buffer.remaining(); + } + + /** + * Returns a single read-only view that logically concatenates {@link #pendingBytes} with the unread tail of + * a buffer. + * + *

The position of the supplied buffer is intentionally not advanced here – reads happen on the + * combined view, and the original buffer's position is moved later by {@link #consumeBytes(int, ByteBuffer)} + * once we know the parse succeeded.

+ * + *

When pendingBytes is empty we avoid the allocation and just return a duplicate of the buffer; + * otherwise we materialize a fresh array of size {@code pending + buffer.remaining()}.

+ */ + private ByteBuffer getCombinedBuffer(ByteBuffer buffer) { + if (pendingBytes.size() == 0) { + ByteBuffer dup = buffer.duplicate(); + dup.order(ByteOrder.LITTLE_ENDIAN); + return dup; + } + + byte[] pending = pendingBytes.toByteArray(); + ByteBuffer combined = ByteBuffer.allocate(pending.length + buffer.remaining()); + combined.order(ByteOrder.LITTLE_ENDIAN); + combined.put(pending); + combined.put(buffer.duplicate()); + combined.flip(); + return combined; + } + + /** + * Consumes the next 8 bytes as a little-endian CRC64 footer, validates it against expectedCrc64, and + * advances {@link #messageOffset}. Used for both segment and message footers. + * + *

If fewer than 8 bytes are available, the remaining buffer bytes are stashed in {@link #pendingBytes} and + * the method returns false so the caller can break out of the decode loop and wait for the next + * chunk. On a CRC mismatch, an {@link IllegalArgumentException} is thrown (the decoder is then discarded by + * the enclosing policy).

+ */ + private boolean tryConsumeCrc64Footer(ByteBuffer buffer, long expectedCrc64, String mismatchDetail) { + if (getAvailableBytes(buffer) < CRC64_LENGTH) { + // Not enough bytes yet for the footer; carry whatever we have over to the next call. + appendToPending(buffer); + return false; + } + ByteBuffer combined = getCombinedBuffer(buffer); + long reportedCrc = combined.getLong(); + if (expectedCrc64 != reportedCrc) { + throw LOGGER.logExceptionAsError(new IllegalArgumentException(enrichExceptionMessage( + "CRC64 mismatch" + mismatchDetail + ". Expected: " + expectedCrc64 + ", got: " + reportedCrc))); + } + consumeBytes(CRC64_LENGTH, buffer); + messageOffset += CRC64_LENGTH; + return true; + } + + /** + * Drains {@code bytesToConsume} bytes from the logical pending+buffer stream that + * {@link #getCombinedBuffer(ByteBuffer)} produced. + * + *

Bytes are taken from {@link #pendingBytes} first, then from the live buffer. The pending stream is + * reset whenever it is fully drained, and any leftover (when {@code bytesToConsume} was less than what was in + * pending) is rewritten so the carry-over stays compact.

+ */ + private void consumeBytes(int bytesToConsume, ByteBuffer buffer) { + int pendingSize = pendingBytes.size(); + if (bytesToConsume <= pendingSize) { + // The entire consume fits in pending: rewrite whatever survives back into pending after a reset. + byte[] remaining = pendingBytes.toByteArray(); + pendingBytes.reset(); + if (bytesToConsume < pendingSize) { + pendingBytes.write(remaining, bytesToConsume, pendingSize - bytesToConsume); + } + } else { + // Pending is fully drained and the remainder comes from the live buffer; advance its position directly. + int bytesFromBuffer = bytesToConsume - pendingSize; + pendingBytes.reset(); + buffer.position(buffer.position() + bytesFromBuffer); + } + } + + /** + * Stashes everything still unread in the buffer into {@link #pendingBytes} so it can be combined with the + * next chunk on the next call to {@link #decodeChunk(ByteBuffer)}. + * + *

This is only called when the current chunk does not contain enough bytes for the next structural element, + * so the carry-over is always small (bounded by the largest header size, currently 13 bytes).

+ */ + private void appendToPending(ByteBuffer buffer) { + while (buffer.hasRemaining()) { + pendingBytes.write(buffer.get()); + } + } + + /** + * Reports whether the decoder has finished consuming the entire structured message and validated everything it + * was supposed to validate. Used by the pipeline policy to distinguish "stream ended cleanly" from "stream was + * truncated". + * + *

The check requires all of:

+ *
    + *
  • The message header has been parsed ({@code messageLength != -1}).
    • + *
  • Every byte of the declared message has been consumed.
    • + *
  • No carry-over bytes remain in pending.
    • + *
  • No segment is currently in progress (no segment header without a matching footer).
    • + *
  • The current segment's payload accumulation is itself complete.
    • + *
+ * + * @return true if all expected bytes have been decoded and validated; false otherwise. + */ + public boolean isComplete() { + return messageLength != -1 + && messageOffset == messageLength + && pendingBytes.size() == 0 + && !segmentHeaderRead + && currentSegmentContentOffset == currentSegmentContentLength; + } + + /** + * Appends the current decoder offset to an exception message so failures can be traced back to a specific + * point in the encoded stream. + * + * @param message The original exception message. + * @return The original message with {@code [decoderOffset=N]} appended. + */ + private String enrichExceptionMessage(String message) { + return String.format("%s [decoderOffset=%d]", message, messageOffset); + } +} diff --git a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageEncoder.java b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageEncoder.java new file mode 100644 index 000000000000..35a559799e78 --- /dev/null +++ b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageEncoder.java @@ -0,0 +1,275 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.common.implementation.contentvalidation; + +import com.azure.core.util.logging.ClientLogger; +import com.azure.storage.common.implementation.StorageImplUtils; +import reactor.core.publisher.Flux; + +import static com.azure.storage.common.implementation.contentvalidation.StructuredMessageConstants.CRC64_LENGTH; +import static com.azure.storage.common.implementation.contentvalidation.StructuredMessageConstants.DEFAULT_MESSAGE_VERSION; +import static com.azure.storage.common.implementation.contentvalidation.StructuredMessageConstants.V1_HEADER_LENGTH; +import static com.azure.storage.common.implementation.contentvalidation.StructuredMessageConstants.V1_SEGMENT_HEADER_LENGTH; + +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.util.HashMap; +import java.util.Map; + +/** + * Encoder for structured messages with support for segmenting and CRC64 checksums. + */ +public class StructuredMessageEncoder { + private static final ClientLogger LOGGER = new ClientLogger(StructuredMessageEncoder.class); + private static final int CRC64_SCRATCH_BUFFER_SIZE = 64 * 1024; + + private final int messageVersion; + private final int contentLength; + private final int messageLength; + private final StructuredMessageFlags structuredMessageFlags; + private final int segmentSize; + private final int numSegments; + + private int currentContentOffset; + private int currentSegmentNumber; + private int currentSegmentOffset; + private long messageCRC64; + private final Map segmentCRC64s; + private final byte[] crc64ScratchBuffer; + + /** + * Constructs a new StructuredMessageEncoder. + * @param contentLength The length of the content to be encoded. + * @param segmentSize The size of each segment. + * @param structuredMessageFlags The structuredMessageFlags to be set. + * @throws IllegalArgumentException If the segment size is less than 1, the content length is less than 1, or the + * number of segments is greater than {@link java.lang.Short#MAX_VALUE}. + */ + public StructuredMessageEncoder(int contentLength, int segmentSize, StructuredMessageFlags structuredMessageFlags) { + if (segmentSize < 1) { + StorageImplUtils.assertInBounds("segmentSize", segmentSize, 1, Long.MAX_VALUE); + } + if (contentLength < 1) { + StorageImplUtils.assertInBounds("contentLength", contentLength, 1, Long.MAX_VALUE); + } + + this.messageVersion = DEFAULT_MESSAGE_VERSION; + this.contentLength = contentLength; + this.structuredMessageFlags = structuredMessageFlags; + this.segmentSize = segmentSize; + this.numSegments = Math.max(1, (int) Math.ceil((double) this.contentLength / this.segmentSize)); + this.messageLength = calculateMessageLength(); + this.currentContentOffset = 0; + this.currentSegmentNumber = 0; + this.currentSegmentOffset = 0; + this.messageCRC64 = 0; + this.segmentCRC64s = new HashMap<>(); + this.crc64ScratchBuffer = new byte[CRC64_SCRATCH_BUFFER_SIZE]; + + if (numSegments > Short.MAX_VALUE) { + StorageImplUtils.assertInBounds("numSegments", numSegments, 1, Short.MAX_VALUE); + } + } + + private int getMessageHeaderLength() { + return V1_HEADER_LENGTH; + } + + private int getSegmentHeaderLength() { + return V1_SEGMENT_HEADER_LENGTH; + } + + private int getSegmentFooterLength() { + return (structuredMessageFlags == StructuredMessageFlags.STORAGE_CRC64) ? CRC64_LENGTH : 0; + } + + private int getMessageFooterLength() { + return (structuredMessageFlags == StructuredMessageFlags.STORAGE_CRC64) ? CRC64_LENGTH : 0; + } + + private int getSegmentContentLength() { + // last segment size is remaining content + if (currentSegmentNumber == numSegments) { + return contentLength - ((currentSegmentNumber - 1) * segmentSize); + } else { + return segmentSize; + } + } + + private byte[] generateMessageHeader() { + // 1 byte version, 8 byte size, 2 byte structuredMessageFlags, 2 byte numSegments + ByteBuffer buffer = ByteBuffer.allocate(getMessageHeaderLength()).order(ByteOrder.LITTLE_ENDIAN); + buffer.put((byte) messageVersion); + buffer.putLong(messageLength); + buffer.putShort((short) structuredMessageFlags.getValue()); + buffer.putShort((short) numSegments); + + return buffer.array(); + } + + private byte[] generateSegmentHeader() { + int segmentContentSize = Math.min(segmentSize, contentLength - currentContentOffset); + // 2 byte number, 8 byte size + ByteBuffer buffer = ByteBuffer.allocate(getSegmentHeaderLength()).order(ByteOrder.LITTLE_ENDIAN); + buffer.putShort((short) currentSegmentNumber); + buffer.putLong(segmentContentSize); + + return buffer.array(); + } + + /** + * Encodes the given buffer into a structured message format as a stream of ByteBuffers. + * + * @param unencodedBuffer The buffer to be encoded. + * @return A Flux of encoded ByteBuffers. + * @throws IllegalArgumentException If the buffer length exceeds the content length, or the content has already been + * encoded. + */ + public Flux encode(ByteBuffer unencodedBuffer) { + StorageImplUtils.assertNotNull("unencodedBuffer", unencodedBuffer); + + return Flux.defer(() -> { + if (currentContentOffset == contentLength) { + // Already encoded; return empty (e.g. extra aggregator from staging/flush, or retry re-subscription). + return Flux.empty(); + } + + if ((unencodedBuffer.remaining() + currentContentOffset) > contentLength) { + return Flux.error( + LOGGER.logExceptionAsError(new IllegalArgumentException("Buffer length exceeds content length."))); + } + + if (!unencodedBuffer.hasRemaining()) { + return Flux.empty(); + } + + // Emit buffers lazily to avoid materializing full encoded output in memory + // Could be swapped to Flux.generate if performance is impacted and we're eagerly pushing a lot of data. + return Flux.create(sink -> { + // if we are at the beginning of the message, encode message header and emit it + if (currentContentOffset == 0) { + sink.next(ByteBuffer.wrap(generateMessageHeader())); + } + + // while there are remaining bytes in the unencoded buffer, encode the segment content + while (unencodedBuffer.hasRemaining()) { + // if we are at the beginning of a segment's content, encode segment header and emit it + if (currentSegmentOffset == 0) { + incrementCurrentSegment(); + sink.next(ByteBuffer.wrap(generateSegmentHeader())); + } + + // encode the segment content and emit it + sink.next(encodeSegmentContent(unencodedBuffer)); + + // if we are at the end of a segment's content, encode segment footer + if (currentSegmentOffset == getSegmentContentLength()) { + byte[] footer = generateSegmentFooter(); + if (footer.length > 0) { + sink.next(ByteBuffer.wrap(footer)); + } + currentSegmentOffset = 0; + } + } + + // if all content has been encoded, encode message footer and emit it + if (currentContentOffset == contentLength) { + byte[] footer = generateMessageFooter(); + if (footer.length > 0) { + sink.next(ByteBuffer.wrap(footer)); + } + } + + sink.complete(); + }); + }); + } + + private byte[] generateSegmentFooter() { + if (structuredMessageFlags == StructuredMessageFlags.STORAGE_CRC64) { + return ByteBuffer.allocate(CRC64_LENGTH) + .order(ByteOrder.LITTLE_ENDIAN) + .putLong(segmentCRC64s.get(currentSegmentNumber)) + .array(); + } + return new byte[0]; + } + + private byte[] generateMessageFooter() { + if (structuredMessageFlags == StructuredMessageFlags.STORAGE_CRC64) { + return ByteBuffer.allocate(CRC64_LENGTH).order(ByteOrder.LITTLE_ENDIAN).putLong(messageCRC64).array(); + } + return new byte[0]; + } + + private ByteBuffer encodeSegmentContent(ByteBuffer unencodedBuffer) { + // get the number of bytes to read from the unencoded buffer based on the segment content length and the current segment offset + int readSize = Math.min(unencodedBuffer.remaining(), getSegmentContentLength() - currentSegmentOffset); + + if (structuredMessageFlags == StructuredMessageFlags.STORAGE_CRC64) { + if (unencodedBuffer.hasArray()) { + // if the unencoded buffer has an array, compute the CRC64 checksum of the segment content + // this is more efficient than copying the array to a new byte array and computing the checksum + int pos = unencodedBuffer.arrayOffset() + unencodedBuffer.position(); + segmentCRC64s.put(currentSegmentNumber, StorageCrc64Calculator.compute(unencodedBuffer.array(), pos, + readSize, segmentCRC64s.get(currentSegmentNumber))); + messageCRC64 = StorageCrc64Calculator.compute(unencodedBuffer.array(), pos, readSize, messageCRC64); + } else { + updateCrc64sWithoutAccessibleArray(unencodedBuffer, readSize); + } + } + + currentContentOffset += readSize; + currentSegmentOffset += readSize; + + // Return a view (slice) to avoid allocating 4MB per segment; caller must consume before next segment. + ByteBuffer slice = unencodedBuffer.slice(); + slice.limit(readSize); + unencodedBuffer.position(unencodedBuffer.position() + readSize); + return slice.asReadOnlyBuffer(); + } + + private void updateCrc64sWithoutAccessibleArray(ByteBuffer unencodedBuffer, int readSize) { + ByteBuffer duplicate = unencodedBuffer.duplicate(); + duplicate.limit(duplicate.position() + readSize); + + long segmentCrc64 = segmentCRC64s.get(currentSegmentNumber); + long currentMessageCrc64 = messageCRC64; + + while (duplicate.hasRemaining()) { + int chunkSize = Math.min(duplicate.remaining(), crc64ScratchBuffer.length); + duplicate.get(crc64ScratchBuffer, 0, chunkSize); + segmentCrc64 = StorageCrc64Calculator.compute(crc64ScratchBuffer, 0, chunkSize, segmentCrc64); + currentMessageCrc64 = StorageCrc64Calculator.compute(crc64ScratchBuffer, 0, chunkSize, currentMessageCrc64); + } + + segmentCRC64s.put(currentSegmentNumber, segmentCrc64); + messageCRC64 = currentMessageCrc64; + } + + private int calculateMessageLength() { + int length = getMessageHeaderLength(); + + length += (getSegmentHeaderLength() + getSegmentFooterLength()) * numSegments; + length += contentLength; + length += getMessageFooterLength(); + return length; + } + + private void incrementCurrentSegment() { + currentSegmentNumber++; + if (structuredMessageFlags == StructuredMessageFlags.STORAGE_CRC64) { + segmentCRC64s.putIfAbsent(currentSegmentNumber, 0L); + } + } + + /** + * Returns the length of the message. + * + * @return The length of the message. + */ + public long getEncodedMessageLength() { + return messageLength; + } +} diff --git a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageFlags.java b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageFlags.java new file mode 100644 index 000000000000..a5ccb2974a6e --- /dev/null +++ b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageFlags.java @@ -0,0 +1,71 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.common.implementation.contentvalidation; + +/** + * Defines values for StructuredMessageFlags. + */ +public enum StructuredMessageFlags { + /** + * No flags set. + */ + NONE(0), + + /** + * StructuredMessageFlag indicating the use of CRC64. + */ + STORAGE_CRC64(1); + + /** + * The actual serialized value for a StructuredMessageFlags instance. + */ + private final int value; + + StructuredMessageFlags(int value) { + this.value = value; + } + + /** + * Parses a serialized value to a StructuredMessageFlags instance. + * + * @param value the serialized value to parse. + * @return the parsed StructuredMessageFlags object, or null if unable to parse. + */ + public static StructuredMessageFlags fromString(String value) { + if (value == null) { + return null; + } + StructuredMessageFlags[] items = StructuredMessageFlags.values(); + for (StructuredMessageFlags item : items) { + if (item.getValue() == Integer.parseInt(value)) { + return item; + } + } + return null; + } + + /** + * Parses a serialized value to a StructuredMessageFlags instance. + * @param value the serialized value to parse. + * @return the parsed StructuredMessageFlags object. + * @throws IllegalArgumentException if unable to parse. + */ + public static StructuredMessageFlags fromValue(int value) { + for (StructuredMessageFlags flag : StructuredMessageFlags.values()) { + if (flag.getValue() == value) { + return flag; + } + } + throw new IllegalArgumentException("Invalid value for StructuredMessageFlags: " + value); + } + + /** + * Returns the value for a StructuredMessageFlags instance. + * + * @return the integer value of the StructuredMessageFlags object. + */ + public int getValue() { + return value; + } +} diff --git a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/package-info.java b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/package-info.java new file mode 100644 index 000000000000..74c1334e3a7a --- /dev/null +++ b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/package-info.java @@ -0,0 +1,7 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +/** + * Package containing classes for structured message encoding and decoding. + */ +package com.azure.storage.common.implementation.contentvalidation; diff --git a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/DecodedResponse.java b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/DecodedResponse.java new file mode 100644 index 000000000000..7e88de8cd37e --- /dev/null +++ b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/DecodedResponse.java @@ -0,0 +1,86 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.common.policy; + +import com.azure.core.http.HttpHeaderName; +import com.azure.core.http.HttpHeaders; +import com.azure.core.http.HttpResponse; +import com.azure.core.util.CoreUtils; +import com.azure.core.util.FluxUtil; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; + +import java.nio.ByteBuffer; +import java.nio.charset.Charset; + +/** + * {@link HttpResponse} wrapper that exposes a decoded body stream while preserving the request and status code of + * the original response. + * + *

The policy hands this class a Flux that already represents validated, framing-stripped bytes (produced by the + * decoder pipeline). This class's only job is to make that Flux look like the body of the original + * {@link HttpResponse}. {@code Content-Length} is overridden to the decoded payload size so it matches what callers + * will actually read; all other headers are forwarded verbatim. The validation is transparent to callers.

+ */ +class DecodedResponse extends HttpResponse { + private final HttpResponse originalResponse; + private final Flux decodedBody; + private final HttpHeaders adjustedHeaders; + + /** + * Wraps {@code httpResponse} with a body backed by {@code decodedBody}. + * + *

{@code Content-Length} is overridden to {@code decodedContentLength} so callers see the size of the bytes + * they will actually read from the decoded payload, not the larger wire size of the structured message.

+ * + * @param httpResponse The original response from the storage service. + * @param decodedBody The Flux of CRC-validated, framing-stripped payload bytes produced by the decoder pipeline. + * @param decodedContentLength The size of the decoded payload that callers will consume. + */ + DecodedResponse(HttpResponse httpResponse, Flux decodedBody, long decodedContentLength) { + super(httpResponse.getRequest()); + this.originalResponse = httpResponse; + this.decodedBody = decodedBody; + HttpHeaders headers = new HttpHeaders(httpResponse.getHeaders()); + headers.set(HttpHeaderName.CONTENT_LENGTH, String.valueOf(decodedContentLength)); + this.adjustedHeaders = headers; + } + + @Override + public int getStatusCode() { + return originalResponse.getStatusCode(); + } + + @Override + @SuppressWarnings("deprecation") + public String getHeaderValue(String name) { + return adjustedHeaders.getValue(name); + } + + @Override + public HttpHeaders getHeaders() { + return adjustedHeaders; + } + + @Override + public Flux getBody() { + return decodedBody; + } + + @Override + public Mono getBodyAsByteArray() { + return FluxUtil.collectBytesInByteBufferStream(decodedBody); + } + + @Override + public Mono getBodyAsString() { + return getBodyAsByteArray() + .map(b -> CoreUtils.bomAwareToString(b, adjustedHeaders.getValue(HttpHeaderName.CONTENT_TYPE))); + } + + @Override + public Mono getBodyAsString(Charset charset) { + return FluxUtil.collectBytesInByteBufferStream(decodedBody).map(b -> new String(b, charset)); + } +} diff --git a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/StorageContentValidationDecoderPolicy.java b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/StorageContentValidationDecoderPolicy.java new file mode 100644 index 000000000000..21150174a03a --- /dev/null +++ b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/StorageContentValidationDecoderPolicy.java @@ -0,0 +1,207 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.common.policy; + +import com.azure.core.http.HttpHeaderName; +import com.azure.core.http.HttpHeaders; +import com.azure.core.http.HttpMethod; +import com.azure.core.http.HttpPipelineCallContext; +import com.azure.core.http.HttpPipelineNextPolicy; +import com.azure.core.http.HttpResponse; +import com.azure.core.http.policy.HttpPipelinePolicy; +import com.azure.core.util.logging.ClientLogger; +import com.azure.storage.common.implementation.Constants; +import com.azure.storage.common.implementation.contentvalidation.StructuredMessageConstants; +import com.azure.storage.common.implementation.contentvalidation.StructuredMessageDecoder; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; + +import java.io.IOException; +import java.nio.ByteBuffer; + +/** + * HTTP pipeline policy that decodes the storage structured message body returned for downloads when CRC64 + * content validation is active. + * + *

The policy decides when to opt in (via the context key), tells the service to + * encode the response (via the request header), constructs the decoder and the wrapper response, and + * translates decoder-level failures (malformed framing, CRC mismatch, premature end-of-stream) into reactive + * {@link IOException} errors.

+ * + *

This policy uses {@link com.azure.core.http.HttpPipelinePosition#PER_RETRY PER_RETRY} semantics by default, so + * each retry produces a fresh response that this policy wraps with a fresh decoder. A CRC failure on one attempt + * cannot pollute another, and the storage download retry logic ({@code BlobAsyncClientBase.downloadStream...}) can + * resume by reissuing range requests; each new range response is validated end-to-end on its own.

+ * + *

Because the wrapped {@link StructuredMessageDecoder} only releases payload bytes after the corresponding + * segment's CRC has been verified, the {@link DecodedResponse}'s body Flux is guaranteed to contain only validated + * bytes – callers never see a byte that could later fail validation, even when retries are involved.

+ */ +public class StorageContentValidationDecoderPolicy implements HttpPipelinePolicy { + private static final ClientLogger LOGGER = new ClientLogger(StorageContentValidationDecoderPolicy.class); + + /** + * Creates a new instance of {@link StorageContentValidationDecoderPolicy}. + */ + public StorageContentValidationDecoderPolicy() { + } + + @Override + public Mono process(HttpPipelineCallContext context, HttpPipelineNextPolicy next) { + // Check if the decoding should be applied. + if (!shouldApplyDecoding(context)) { + return next.process(); + } + + // Tell the service we want a structured-message body. + context.getHttpRequest() + .getHeaders() + .set(Constants.HeaderConstants.STRUCTURED_BODY_TYPE_HEADER_NAME, + StructuredMessageConstants.STRUCTURED_BODY_TYPE_VALUE); + + return next.process().map(httpResponse -> { + // The HTTP Content-Length is the size of the encoded structured message body. We hand it to the + // decoder which cross-checks it against the message header. + Long contentLength = getContentLength(httpResponse.getHeaders()); + + // Only 2xx GET responses with a positive content length carry a body that we can decode. + if (!isEligibleDownload(httpResponse, contentLength)) { + return httpResponse; + } + + // Confirm the service honored our structured-body request and parse the decoded length in one step, + // failing fast with a consistent error if either header is absent or not parseable as a long. + long decodedContentLength = validateAndGetDecodedContentLength(httpResponse); + + // Fresh decoder per response so retries each get a clean state machine. + StructuredMessageDecoder decoder = new StructuredMessageDecoder(contentLength); + + Flux decodedStream = decodeStream(httpResponse.getBody(), decoder); + return new DecodedResponse(httpResponse, decodedStream, decodedContentLength); + }); + } + + /** + * @return true when the request carries the boolean opt-in flag set + * by {@code ContentValidationModeResolver.addStructuredMessageDecodingToContext}. + */ + private boolean shouldApplyDecoding(HttpPipelineCallContext context) { + return context.getData(StructuredMessageConstants.STRUCTURED_MESSAGE_DECODING_CONTEXT_KEY) + .map(value -> value instanceof Boolean && (Boolean) value) + .orElse(false); + } + + /** + * Verifies the response acknowledges the structured-body request ({@code x-ms-structured-body} present) and + * parses {@code x-ms-structured-content-length} in one step. Throws with a consistent error message if either + * header is absent or the length value is not parseable as a {@code long}, matching the fail-fast behaviour + * used for other validation failures. + * + * @return the decoded payload size declared by the service. + */ + private long validateAndGetDecodedContentLength(HttpResponse httpResponse) { + String structuredBody + = httpResponse.getHeaders().getValue(Constants.HeaderConstants.STRUCTURED_BODY_TYPE_HEADER_NAME); + String structuredContentLength + = httpResponse.getHeaders().getValue(Constants.HeaderConstants.STRUCTURED_CONTENT_LENGTH_HEADER_NAME); + if (structuredBody == null || structuredContentLength == null) { + throw LOGGER.logExceptionAsError( + new IllegalStateException("Structured message was requested but the response did not acknowledge it.")); + } + try { + return Long.parseLong(structuredContentLength); + } catch (NumberFormatException e) { + throw LOGGER.logExceptionAsError(new IllegalStateException("x-ms-structured-content-length header value '" + + structuredContentLength + "' could not be parsed as a long.", e)); + } + } + + /** + * Reads {@code Content-Length} as a {@code long}, returning {@code null} when the header is missing or + * not parseable so callers can simply skip decoding for non-bodied responses. + */ + private static Long getContentLength(HttpHeaders headers) { + String value = headers.getValue(HttpHeaderName.CONTENT_LENGTH); + if (value != null) { + try { + return Long.parseLong(value); + } catch (NumberFormatException e) { + // Header invalid; treat as not eligible. + } + } + return null; + } + + /** + * @return true for a 2xx response to a GET request, the only response shape that carries a body we + * can decode. 206 (Partial Content) on retried range downloads is included. + */ + private static boolean isDownloadResponse(HttpResponse response) { + return response.getRequest().getHttpMethod() == HttpMethod.GET && response.getStatusCode() / 100 == 2; + } + + /** + * @return true when the response is one we should decode: a 2xx GET with a positive, parseable + * {@code Content-Length}. + */ + private static boolean isEligibleDownload(HttpResponse response, Long contentLength) { + return isDownloadResponse(response) && contentLength != null && contentLength > 0; + } + + /** + * Builds the body-decoding Flux: each upstream {@link ByteBuffer} is fed to the decoder in order + * ({@code concatMap} preserves order and serializes access), and a deferred stream-completion check is + * appended so a truncated body raises an error instead of completing silently. + */ + private Flux decodeStream(Flux encodedFlux, StructuredMessageDecoder decoder) { + // limitRate(1) mirrors StorageContentValidationPolicy's upload path: process one wire buffer at a time so + // the decoder can copy only the current chunk into owned storage and release the inbound buffer before the + // next segment payload bytes arrive. + return encodedFlux.limitRate(1) + .concatMap(buffer -> decodeBuffer(buffer, decoder)) + .concatWith(Mono.defer(() -> handleStreamCompletion(decoder))); + } + + /** + * Feeds a single inbound chunk to the decoder and translates its outputs into reactive emissions: + * If the decoder reports validated bytes, emit them downstream. + * If the decoder threw because the input is malformed or a CRC mismatch was detected, surface that as + * an {@link IOException}. + * If the decoder is already complete (e.g., extra trailing bytes after the message footer), drop the + * chunk silently. + */ + private Flux decodeBuffer(ByteBuffer buffer, StructuredMessageDecoder decoder) { + if (decoder.isComplete()) { + // Decoding finished on a previous chunk; ignore any trailing bytes the transport might still emit. + return Flux.empty(); + } + + if (buffer == null || !buffer.hasRemaining()) { + return Flux.empty(); + } + + try { + return Flux.fromIterable(decoder.decodeChunk(buffer)); + } catch (IllegalArgumentException e) { + return Flux.error(new IOException("Failed to decode structured message: " + e.getMessage(), e)); + } catch (Exception e) { + // Anything not foreseen by the decoder, log it. + LOGGER.error("Failed to decode structured message chunk: " + e.getMessage(), e); + return Flux.error(new IOException("Failed to decode structured message chunk: " + e.getMessage(), e)); + } + } + + /** + * Run after the upstream Flux completes. If the decoder is not in a complete state, the response body ended + * before all expected bytes arrived – surface this as an {@link IOException} so callers don't accept a + * truncated payload. + */ + private Mono handleStreamCompletion(StructuredMessageDecoder decoder) { + if (!decoder.isComplete()) { + return Mono.error(new IOException("Stream ended prematurely before structured message decoding completed")); + } + return Mono.empty(); + } + +} diff --git a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/StorageContentValidationPolicy.java b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/StorageContentValidationPolicy.java new file mode 100644 index 000000000000..a449e37a8c2f --- /dev/null +++ b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/StorageContentValidationPolicy.java @@ -0,0 +1,175 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.common.policy; + +import com.azure.core.http.HttpHeaderName; +import com.azure.core.http.HttpPipelineCallContext; +import com.azure.core.http.HttpPipelineNextPolicy; +import com.azure.core.http.HttpResponse; +import com.azure.core.http.policy.HttpPipelinePolicy; +import com.azure.core.util.FluxUtil; +import com.azure.core.util.logging.ClientLogger; +import com.azure.storage.common.implementation.contentvalidation.StorageCrc64Calculator; +import com.azure.storage.common.implementation.contentvalidation.StructuredMessageEncoder; +import com.azure.storage.common.implementation.contentvalidation.StructuredMessageFlags; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; +import reactor.core.scheduler.Schedulers; + +import java.nio.ByteBuffer; +import java.util.Base64; +import java.util.Optional; + +import static com.azure.storage.common.implementation.contentvalidation.StructuredMessageConstants.CONTENT_VALIDATION_MODE_KEY; +import static com.azure.storage.common.implementation.contentvalidation.StructuredMessageConstants.USE_CRC64_CHECKSUM_HEADER_CONTEXT; +import static com.azure.storage.common.implementation.contentvalidation.StructuredMessageConstants.USE_STRUCTURED_MESSAGE_CONTEXT; +import static com.azure.storage.common.implementation.contentvalidation.StructuredMessageConstants.V1_DEFAULT_SEGMENT_CONTENT_LENGTH; +import static com.azure.storage.common.implementation.Constants.HeaderConstants.CONTENT_CRC64_HEADER_NAME; +import static com.azure.storage.common.implementation.Constants.HeaderConstants.STRUCTURED_BODY_TYPE_HEADER_NAME; +import static com.azure.storage.common.implementation.Constants.HeaderConstants.STRUCTURED_CONTENT_LENGTH_HEADER_NAME; +import static com.azure.storage.common.implementation.contentvalidation.StructuredMessageConstants.STRUCTURED_BODY_TYPE_VALUE; + +/** + * StorageContentValidationPolicy is a policy that applies content validation to the request body. + */ +public class StorageContentValidationPolicy implements HttpPipelinePolicy { + private static final ClientLogger LOGGER = new ClientLogger(StorageContentValidationPolicy.class); + + /** + * Creates a new instance of {@link StorageContentValidationPolicy}. + */ + public StorageContentValidationPolicy() { + } + + @Override + public Mono process(HttpPipelineCallContext context, HttpPipelineNextPolicy next) { + // Defer creating the next policy Mono until validation (and any required header mutations) has completed. + // Some downstream policies may compute auth/signatures eagerly in their `process()` method. + return applyContentValidation(context).then(Mono.defer(next::process)); + } + + /** + * Applies content validation to the request body. + * + * @param context the HTTP pipeline call context + * @return a {@link Mono} that completes when content validation has been applied to the request body. + */ + private Mono applyContentValidation(HttpPipelineCallContext context) { + Optional behaviorOptional = context.getContext().getData(CONTENT_VALIDATION_MODE_KEY); + if (!behaviorOptional.isPresent()) { + return Mono.empty(); + } + + String contentValidationBehavior = behaviorOptional.get().toString(); + if (contentValidationBehavior.isEmpty()) { + return Mono.empty(); + } + + Mono validation = Mono.empty(); + + if (contentValidationBehavior.contains(USE_CRC64_CHECKSUM_HEADER_CONTEXT)) { + validation = validation.then(applyCRC64Header(context)); + } + if (contentValidationBehavior.contains(USE_STRUCTURED_MESSAGE_CONTEXT)) { + validation = validation.then(applyStructuredMessage(context)); + } + + return validation; + } + + /** + * Applies the crc64 header to the request body. + * + * @param context the HTTP pipeline call context + * @return a {@link Mono} that completes when the crc64 header has been applied to the request body. + */ + private Mono applyCRC64Header(HttpPipelineCallContext context) { + if (context.getHttpRequest().getBody() == null) { + return Mono.empty(); + } + + // Collect request body bytes once, compute CRC64 off the reactive thread, and then restore the body + // as a replayable Flux so downstream processing / sending still sees the expected content. + Flux originalBody = context.getHttpRequest().getBody(); + return FluxUtil.collectBytesInByteBufferStream(originalBody) + .flatMap(originalBytes -> Mono.fromCallable(() -> StorageCrc64Calculator.compute(originalBytes, 0)) + // Run CRC64 on boundedElastic so synchronous work over the full body does not block the reactive + // I/O thread (e.g. Netty event loop) that drives the pipeline. + .subscribeOn(Schedulers.boundedElastic()) + .flatMap(contentCRC64 -> { + // Restore body for downstream consumers. + context.getHttpRequest().setBody(Flux.just(ByteBuffer.wrap(originalBytes))); + + // Convert the 64-bit CRC value to 8 bytes in little-endian format. + byte[] crc64Bytes = new byte[8]; + for (int i = 0; i < 8; i++) { + crc64Bytes[i] = (byte) (contentCRC64 >>> (i * 8)); + } + + // Base64 encode the binary representation. + String encodedCRC64 = Base64.getEncoder().encodeToString(crc64Bytes); + context.getHttpRequest().setHeader(CONTENT_CRC64_HEADER_NAME, encodedCRC64); + return Mono.empty(); + })); + } + + /** + * Applies the structured message to the request body. + * + * @param context the HTTP pipeline call context + * @return a {@link Mono} that completes when the structured message has been applied, or fails if the request is + * not valid for structured messaging + */ + private Mono applyStructuredMessage(HttpPipelineCallContext context) { + String contentLengthValue = context.getHttpRequest().getHeaders().getValue(HttpHeaderName.CONTENT_LENGTH); + if (contentLengthValue == null || contentLengthValue.isEmpty()) { + return FluxUtil.monoError(LOGGER, + LOGGER.logExceptionAsError( + new IllegalArgumentException("Content-Length header is required to apply structured message " + + "and CRC64 encoding, but it was not present on the request."))); + } + + long parsedContentLength; + try { + parsedContentLength = Long.parseLong(contentLengthValue); + } catch (NumberFormatException ex) { + return FluxUtil.monoError(LOGGER, LOGGER.logExceptionAsError(new IllegalArgumentException( + "Content-Length header value '" + contentLengthValue + + "' is not a valid non-negative integer value required for structured message and CRC64 encoding.", + ex))); + } + + int unencodedContentLength = (int) parsedContentLength; + + Flux originalBody = context.getHttpRequest().getBody(); + + /* + * Replace the request body with a structured message: raw content wrapped with headers, segment + * boundaries, and CRC64 checksums so the service can validate integrity as it receives the stream. + * + * A fresh encoder is created on each subscribe (via defer) so retries re-encode correctly from the + * original replayable body. The encoded buffers are slices of the original data, produced lazily and + * consumed by the HTTP client without materialization. + * + * limitRate(1) keeps the encoder's segment boundaries aligned with buffer boundaries. + */ + Flux encodedBody = Flux.defer(() -> { + StructuredMessageEncoder encoder = new StructuredMessageEncoder(unencodedContentLength, + V1_DEFAULT_SEGMENT_CONTENT_LENGTH, StructuredMessageFlags.STORAGE_CRC64); + return Flux.from(originalBody).limitRate(1).concatMap(encoder::encode); + }); + + context.getHttpRequest().setBody(encodedBody); + + long encodedLength = new StructuredMessageEncoder(unencodedContentLength, V1_DEFAULT_SEGMENT_CONTENT_LENGTH, + StructuredMessageFlags.STORAGE_CRC64).getEncodedMessageLength(); + context.getHttpRequest().setHeader(HttpHeaderName.CONTENT_LENGTH, String.valueOf(encodedLength)); + context.getHttpRequest().setHeader(STRUCTURED_BODY_TYPE_HEADER_NAME, STRUCTURED_BODY_TYPE_VALUE); + context.getHttpRequest() + .setHeader(STRUCTURED_CONTENT_LENGTH_HEADER_NAME, String.valueOf(unencodedContentLength)); + + return Mono.empty(); + } + +} diff --git a/sdk/storage/azure-storage-common/src/main/java/module-info.java b/sdk/storage/azure-storage-common/src/main/java/module-info.java index f5f1cfa99b2b..412a11fe41d6 100644 --- a/sdk/storage/azure-storage-common/src/main/java/module-info.java +++ b/sdk/storage/azure-storage-common/src/main/java/module-info.java @@ -25,4 +25,7 @@ exports com.azure.storage.common.implementation.connectionstring to // FIXME this should not be a long-term solution com.azure.data.tables, com.azure.storage.blob, com.azure.storage.blob.cryptography, com.azure.storage.file.share, com.azure.storage.file.datalake, com.azure.storage.queue; + + exports com.azure.storage.common.implementation.contentvalidation to // FIXME this should not be a long-term solution + com.azure.storage.blob, com.azure.storage.file.share, com.azure.storage.file.datalake; } diff --git a/sdk/storage/azure-storage-common/src/test-shared/java/com/azure/storage/common/test/shared/policy/MockDownloadHttpResponse.java b/sdk/storage/azure-storage-common/src/test-shared/java/com/azure/storage/common/test/shared/policy/MockDownloadHttpResponse.java index 5e84dd31947d..ddd488ff0887 100644 --- a/sdk/storage/azure-storage-common/src/test-shared/java/com/azure/storage/common/test/shared/policy/MockDownloadHttpResponse.java +++ b/sdk/storage/azure-storage-common/src/test-shared/java/com/azure/storage/common/test/shared/policy/MockDownloadHttpResponse.java @@ -6,10 +6,10 @@ import com.azure.core.http.HttpHeaderName; import com.azure.core.http.HttpHeaders; import com.azure.core.http.HttpResponse; +import com.azure.core.util.FluxUtil; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; -import java.io.IOException; import java.nio.ByteBuffer; import java.nio.charset.Charset; @@ -19,14 +19,21 @@ with than was worth it. Because this type is just for BlobDownload, we don't need to accept a header type. */ public class MockDownloadHttpResponse extends HttpResponse { + private final HttpResponse originalResponse; private final int statusCode; private final HttpHeaders headers; private final Flux body; public MockDownloadHttpResponse(HttpResponse response, int statusCode, Flux body) { + this(response, statusCode, response.getHeaders(), body); + } + + public MockDownloadHttpResponse(HttpResponse response, int statusCode, HttpHeaders headers, + Flux body) { super(response.getRequest()); + this.originalResponse = response; this.statusCode = statusCode; - this.headers = response.getHeaders(); + this.headers = headers; this.body = body; } @@ -52,21 +59,26 @@ public HttpHeaders getHeaders() { @Override public Flux getBody() { - return body; + return Flux.using(() -> originalResponse, ignored -> body, HttpResponse::close); } @Override public Mono getBodyAsByteArray() { - return Mono.error(new IOException()); + return FluxUtil.collectBytesInByteBufferStream(getBody()); } @Override public Mono getBodyAsString() { - return Mono.error(new IOException()); + return getBodyAsByteArray().map(bytes -> new String(bytes, Charset.defaultCharset())); } @Override public Mono getBodyAsString(Charset charset) { - return Mono.error(new IOException()); + return getBodyAsByteArray().map(bytes -> new String(bytes, charset)); + } + + @Override + public void close() { + originalResponse.close(); } } diff --git a/sdk/storage/azure-storage-common/src/test-shared/java/com/azure/storage/common/test/shared/policy/MockPartialResponsePolicy.java b/sdk/storage/azure-storage-common/src/test-shared/java/com/azure/storage/common/test/shared/policy/MockPartialResponsePolicy.java index 1f1109d8b38b..347d3ac11a59 100644 --- a/sdk/storage/azure-storage-common/src/test-shared/java/com/azure/storage/common/test/shared/policy/MockPartialResponsePolicy.java +++ b/sdk/storage/azure-storage-common/src/test-shared/java/com/azure/storage/common/test/shared/policy/MockPartialResponsePolicy.java @@ -8,6 +8,7 @@ import com.azure.core.http.HttpMethod; import com.azure.core.http.HttpPipelineCallContext; import com.azure.core.http.HttpPipelineNextPolicy; +import com.azure.core.http.HttpPipelinePosition; import com.azure.core.http.HttpResponse; import com.azure.core.http.policy.HttpPipelinePolicy; import reactor.core.publisher.Flux; @@ -16,50 +17,137 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.util.ArrayList; +import java.util.Collections; import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; public class MockPartialResponsePolicy implements HttpPipelinePolicy { - static final HttpHeaderName RANGE_HEADER = HttpHeaderName.fromString("x-ms-range"); - private int tries; - private final List rangeHeaders = new ArrayList<>(); + static final HttpHeaderName X_MS_RANGE_HEADER = HttpHeaderName.fromString("x-ms-range"); + static final HttpHeaderName RANGE_HEADER = HttpHeaderName.RANGE; + private final AtomicInteger tries; + private final List rangeHeaders = Collections.synchronizedList(new ArrayList<>()); + private final int maxBytesPerResponse; + private final AtomicInteger hits = new AtomicInteger(); + private final String targetUrlPrefix; + /** + * Creates a MockPartialResponsePolicy that simulates network interruptions. + * + * @param tries Number of times to simulate interruptions (0 = no interruptions) + */ public MockPartialResponsePolicy(int tries) { - this.tries = tries; + this(tries, 200, null); + } + + /** + * Creates a MockPartialResponsePolicy with configurable interruption behavior. + * + * @param tries Number of times to simulate interruptions (0 = no interruptions) + * @param maxBytesPerResponse Maximum bytes to return in each interrupted response + */ + public MockPartialResponsePolicy(int tries, int maxBytesPerResponse) { + this(tries, maxBytesPerResponse, null); + } + + /** + * Creates a MockPartialResponsePolicy with configurable interruption behavior and an optional URL filter. + * + * @param tries Number of times to simulate interruptions (0 = no interruptions) + * @param maxBytesPerResponse Maximum bytes to return in each interrupted response + * @param targetUrlPrefix If non-null, only requests whose URL starts with this prefix will be interrupted. + */ + public MockPartialResponsePolicy(int tries, int maxBytesPerResponse, String targetUrlPrefix) { + this.tries = new AtomicInteger(tries); + this.maxBytesPerResponse = maxBytesPerResponse; + this.targetUrlPrefix = targetUrlPrefix; + } + + @Override + public HttpPipelinePosition getPipelinePosition() { + return HttpPipelinePosition.PER_RETRY; } @Override public Mono process(HttpPipelineCallContext context, HttpPipelineNextPolicy next) { return next.process().flatMap(response -> { HttpHeader rangeHttpHeader = response.getRequest().getHeaders().get(RANGE_HEADER); - String rangeHeader = rangeHttpHeader == null ? null : rangeHttpHeader.getValue(); + HttpHeader xMsRangeHttpHeader = response.getRequest().getHeaders().get(X_MS_RANGE_HEADER); - if (rangeHeader != null && rangeHeader.startsWith("bytes=")) { - rangeHeaders.add(rangeHeader); + if (response.getRequest().getHttpMethod() == HttpMethod.GET) { + String recordedRange = null; + if (rangeHttpHeader != null && rangeHttpHeader.getValue().startsWith("bytes=")) { + recordedRange = rangeHttpHeader.getValue(); + } else if (xMsRangeHttpHeader != null && xMsRangeHttpHeader.getValue().startsWith("bytes=")) { + recordedRange = xMsRangeHttpHeader.getValue(); + } + rangeHeaders.add(recordedRange == null ? "" : recordedRange); } - if ((response.getRequest().getHttpMethod() != HttpMethod.GET) || this.tries == 0) { + boolean urlMatches = targetUrlPrefix == null + || response.getRequest().getUrl().toString().startsWith(targetUrlPrefix); + + if ((response.getRequest().getHttpMethod() != HttpMethod.GET) || !urlMatches) { return Mono.just(response); } else { - this.tries -= 1; - return response.getBody().collectList().flatMap(bodyBuffers -> { - ByteBuffer firstBuffer = bodyBuffers.get(0); - byte firstByte = firstBuffer.get(); - - // Simulate partial response by returning the first byte only from the requested range and timeout - return Mono.just(new MockDownloadHttpResponse(response, 206, - Flux.just(ByteBuffer.wrap(new byte[] { firstByte })) - .concatWith(Flux.error(new IOException("Simulated timeout"))) - )); - }); + int remainingTries = this.tries.getAndUpdate(value -> value > 0 ? value - 1 : value); + if (remainingTries <= 0) { + return Mono.just(response); + } + hits.incrementAndGet(); + + Flux limitedBody = limitStreamToBytes(response.getBody(), maxBytesPerResponse); + return Mono.just( + new MockDownloadHttpResponse(response, response.getStatusCode(), response.getHeaders(), + limitedBody)); } }); } + private Flux limitStreamToBytes(Flux body, int maxBytes) { + return Flux.defer(() -> { + final long[] bytesEmitted = new long[] { 0 }; + return body.concatMap(buffer -> { + if (buffer == null || !buffer.hasRemaining()) { + return Flux.just(buffer); + } + + long remaining = maxBytes - bytesEmitted[0]; + if (remaining <= 0) { + return Flux.error(new IOException("Simulated timeout")); + } + + int bufferSize = buffer.remaining(); + if (bufferSize <= remaining) { + bytesEmitted[0] += bufferSize; + if (bytesEmitted[0] >= maxBytes) { + return Flux.just(buffer).concatWith(Flux.error(new IOException("Simulated timeout"))); + } + return Flux.just(buffer); + } else { + int bytesToEmit = (int) remaining; + ByteBuffer slice = buffer.duplicate(); + slice.limit(slice.position() + bytesToEmit); + + ByteBuffer limited = ByteBuffer.allocate(bytesToEmit); + limited.put(slice); + limited.flip(); + + bytesEmitted[0] += bytesToEmit; + return Flux.just(limited).concatWith(Flux.error(new IOException("Simulated timeout"))); + } + }); + }); + } + public int getTriesRemaining() { - return tries; + return tries.get(); } public List getRangeHeaders() { return rangeHeaders; } + + public int getHits() { + return hits.get(); + } } diff --git a/sdk/storage/azure-storage-common/src/test/java/com/azure/storage/common/implementation/contentvalidation/ContentValidationModeResolverTests.java b/sdk/storage/azure-storage-common/src/test/java/com/azure/storage/common/implementation/contentvalidation/ContentValidationModeResolverTests.java new file mode 100644 index 000000000000..14408a4163bb --- /dev/null +++ b/sdk/storage/azure-storage-common/src/test/java/com/azure/storage/common/implementation/contentvalidation/ContentValidationModeResolverTests.java @@ -0,0 +1,236 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.common.implementation.contentvalidation; + +import com.azure.core.util.Context; +import com.azure.core.util.ProgressListener; +import com.azure.storage.common.ContentValidationAlgorithm; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import reactor.core.publisher.Mono; +import reactor.test.StepVerifier; + +import java.util.stream.Stream; + +import static com.azure.storage.common.implementation.contentvalidation.StructuredMessageConstants.CONTENT_VALIDATION_MODE_KEY; +import static com.azure.storage.common.implementation.contentvalidation.StructuredMessageConstants.MAXIMUM_SINGLE_SHOT_UPLOAD_SIZE_TO_USE_CRC64_HEADER; +import static com.azure.storage.common.implementation.contentvalidation.StructuredMessageConstants.USE_CRC64_CHECKSUM_HEADER_CONTEXT; +import static com.azure.storage.common.implementation.contentvalidation.StructuredMessageConstants.USE_STRUCTURED_MESSAGE_CONTEXT; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class ContentValidationModeResolverTests { + + private static String modeOnContext(Context context, ContentValidationAlgorithm algorithm, long contentLength, + boolean chunkedUpload) { + return ContentValidationModeResolver.addContentValidationMode(context, algorithm, contentLength, chunkedUpload) + .getData(CONTENT_VALIDATION_MODE_KEY) + .map(Object::toString) + .orElse(null); + } + + // =========================================================================================== + // addContentValidationMode (Context) — single-part + // =========================================================================================== + + static Stream singlePartDoesNotSetModeSupplier() { + return Stream.of(Arguments.of(null, 1024), Arguments.of(ContentValidationAlgorithm.NONE, 1024), + Arguments.of(null, 8 * 1024 * 1024), Arguments.of(ContentValidationAlgorithm.NONE, 8 * 1024 * 1024)); + } + + @ParameterizedTest + @MethodSource("singlePartDoesNotSetModeSupplier") + public void singlePartDoesNotSetModeForNullOrNone(ContentValidationAlgorithm algorithm, long length) { + assertEquals(null, modeOnContext(Context.NONE, algorithm, length, false)); + } + + @Test + public void singlePartSmallUploadUsesCrc64Header() { + long underThreshold = MAXIMUM_SINGLE_SHOT_UPLOAD_SIZE_TO_USE_CRC64_HEADER - 1; + assertEquals(USE_CRC64_CHECKSUM_HEADER_CONTEXT, + modeOnContext(Context.NONE, ContentValidationAlgorithm.CRC64, underThreshold, false)); + } + + @Test + public void singlePartAtExact4MBBoundaryUsesStructuredMessage() { + assertEquals(USE_STRUCTURED_MESSAGE_CONTEXT, modeOnContext(Context.NONE, ContentValidationAlgorithm.CRC64, + MAXIMUM_SINGLE_SHOT_UPLOAD_SIZE_TO_USE_CRC64_HEADER, false)); + } + + @Test + public void singlePartLargeUploadUsesStructuredMessage() { + long overThreshold = MAXIMUM_SINGLE_SHOT_UPLOAD_SIZE_TO_USE_CRC64_HEADER + 1; + assertEquals(USE_STRUCTURED_MESSAGE_CONTEXT, + modeOnContext(Context.NONE, ContentValidationAlgorithm.CRC64, overThreshold, false)); + } + + @Test + public void singlePartAutoSmallUploadUsesCrc64Header() { + long underThreshold = MAXIMUM_SINGLE_SHOT_UPLOAD_SIZE_TO_USE_CRC64_HEADER - 1; + assertEquals(USE_CRC64_CHECKSUM_HEADER_CONTEXT, + modeOnContext(Context.NONE, ContentValidationAlgorithm.AUTO, underThreshold, false)); + } + + @Test + public void singlePartAutoAtExact4MBBoundaryUsesStructuredMessage() { + assertEquals(USE_STRUCTURED_MESSAGE_CONTEXT, modeOnContext(Context.NONE, ContentValidationAlgorithm.AUTO, + MAXIMUM_SINGLE_SHOT_UPLOAD_SIZE_TO_USE_CRC64_HEADER, false)); + } + + @Test + public void singlePartAutoLargeUploadUsesStructuredMessage() { + long overThreshold = MAXIMUM_SINGLE_SHOT_UPLOAD_SIZE_TO_USE_CRC64_HEADER + 1; + assertEquals(USE_STRUCTURED_MESSAGE_CONTEXT, + modeOnContext(Context.NONE, ContentValidationAlgorithm.AUTO, overThreshold, false)); + } + + @Test + public void addContentValidationModeNullContextUsesNone() { + long underThreshold = MAXIMUM_SINGLE_SHOT_UPLOAD_SIZE_TO_USE_CRC64_HEADER - 1; + assertEquals(USE_CRC64_CHECKSUM_HEADER_CONTEXT, + modeOnContext(null, ContentValidationAlgorithm.CRC64, underThreshold, false)); + } + + // =========================================================================================== + // addContentValidationMode (Context) — chunked + // =========================================================================================== + + static Stream chunkedDoesNotSetModeSupplier() { + return Stream.of(Arguments.of((ContentValidationAlgorithm) null), + Arguments.of(ContentValidationAlgorithm.NONE)); + } + + @ParameterizedTest + @MethodSource("chunkedDoesNotSetModeSupplier") + public void chunkedDoesNotSetModeForNonCrc64Algorithms(ContentValidationAlgorithm algorithm) { + assertEquals(null, modeOnContext(Context.NONE, algorithm, 1024, true)); + } + + @Test + public void chunkedCrc64AlwaysUsesStructuredMessage() { + assertEquals(USE_STRUCTURED_MESSAGE_CONTEXT, + modeOnContext(Context.NONE, ContentValidationAlgorithm.CRC64, 1024, true)); + } + + @Test + public void chunkedAutoAlwaysUsesStructuredMessage() { + assertEquals(USE_STRUCTURED_MESSAGE_CONTEXT, + modeOnContext(Context.NONE, ContentValidationAlgorithm.AUTO, 1024, true)); + } + + // =========================================================================================== + // addContentValidationMode (Mono) + // =========================================================================================== + + @Test + public void addContentValidationModeMonoWritesReactorContextForCrc64() { + long underThreshold = MAXIMUM_SINGLE_SHOT_UPLOAD_SIZE_TO_USE_CRC64_HEADER - 1; + Mono source = Mono.deferContextual(ctx -> Mono.just(ctx.get(CONTENT_VALIDATION_MODE_KEY))); + Mono augmented = ContentValidationModeResolver.addContentValidationMode(source, + ContentValidationAlgorithm.CRC64, underThreshold, false); + StepVerifier.create(augmented).expectNext(USE_CRC64_CHECKSUM_HEADER_CONTEXT).verifyComplete(); + } + + @Test + public void addContentValidationModeMonoWritesReactorContextForAuto() { + long underThreshold = MAXIMUM_SINGLE_SHOT_UPLOAD_SIZE_TO_USE_CRC64_HEADER - 1; + Mono source = Mono.deferContextual(ctx -> Mono.just(ctx.get(CONTENT_VALIDATION_MODE_KEY))); + Mono augmented = ContentValidationModeResolver.addContentValidationMode(source, + ContentValidationAlgorithm.AUTO, underThreshold, false); + StepVerifier.create(augmented).expectNext(USE_CRC64_CHECKSUM_HEADER_CONTEXT).verifyComplete(); + } + + @Test + public void addContentValidationModeMonoLeavesChainUnchangedWhenNoMode() { + Mono source = Mono.deferContextual(ctx -> Mono.just("ok")); + Mono augmented = ContentValidationModeResolver.addContentValidationMode(source, + ContentValidationAlgorithm.NONE, 1024, false); + StepVerifier.create(augmented).expectNext("ok").verifyComplete(); + } + + // =========================================================================================== + // validateTransactionalChecksumOptions (boolean computeMd5) + // =========================================================================================== + + @Test + public void validateComputeMd5PassesForCompatibleOptions() { + assertDoesNotThrow(() -> ContentValidationModeResolver.validateTransactionalChecksumOptions(true, null)); + assertDoesNotThrow(() -> ContentValidationModeResolver.validateTransactionalChecksumOptions(false, + ContentValidationAlgorithm.AUTO)); + assertDoesNotThrow(() -> ContentValidationModeResolver.validateTransactionalChecksumOptions(false, + ContentValidationAlgorithm.NONE)); + assertDoesNotThrow(() -> ContentValidationModeResolver.validateTransactionalChecksumOptions(false, + ContentValidationAlgorithm.CRC64)); + } + + @Test + public void validateComputeMd5ThrowsWhenAlgorithmNone() { + assertThrows(IllegalArgumentException.class, () -> ContentValidationModeResolver + .validateTransactionalChecksumOptions(true, ContentValidationAlgorithm.NONE)); + } + + @Test + public void validateComputeMd5ThrowsForCrc64() { + assertThrows(IllegalArgumentException.class, () -> ContentValidationModeResolver + .validateTransactionalChecksumOptions(true, ContentValidationAlgorithm.CRC64)); + } + + @Test + public void validateComputeMd5ThrowsForAuto() { + assertThrows(IllegalArgumentException.class, () -> ContentValidationModeResolver + .validateTransactionalChecksumOptions(true, ContentValidationAlgorithm.AUTO)); + } + + // =========================================================================================== + // validateProgressWithContentValidation + // =========================================================================================== + + @Test + public void validateProgressWithContentValidationPassesWhenNoProgress() { + assertDoesNotThrow(() -> ContentValidationModeResolver + .validateProgressWithContentValidation((ProgressListener) null, ContentValidationAlgorithm.CRC64)); + assertDoesNotThrow(() -> ContentValidationModeResolver + .validateProgressWithContentValidation((ProgressListener) null, ContentValidationAlgorithm.AUTO)); + } + + @Test + public void validateProgressWithContentValidationPassesWhenNoneOrNullAlgorithm() { + ProgressListener listener = l -> { + }; + assertDoesNotThrow(() -> ContentValidationModeResolver.validateProgressWithContentValidation(listener, null)); + assertDoesNotThrow(() -> ContentValidationModeResolver.validateProgressWithContentValidation(listener, + ContentValidationAlgorithm.NONE)); + } + + @Test + public void validateProgressWithContentValidationThrowsForCrc64() { + ProgressListener listener = l -> { + }; + IllegalArgumentException ex = assertThrows(IllegalArgumentException.class, () -> ContentValidationModeResolver + .validateProgressWithContentValidation(listener, ContentValidationAlgorithm.CRC64)); + assertEquals(ContentValidationModeResolver.PROGRESS_CONFLICTS_TRANSFER_CONTENT_VALIDATION_MESSAGE, + ex.getMessage()); + } + + @Test + public void validateProgressWithContentValidationThrowsForAuto() { + ProgressListener listener = l -> { + }; + assertThrows(IllegalArgumentException.class, () -> ContentValidationModeResolver + .validateProgressWithContentValidation(listener, ContentValidationAlgorithm.AUTO)); + } + + @Test + public void isCrc64OrAutoReflectsCrc64AndAutoOnly() { + assertTrue(ContentValidationModeResolver.isCrc64OrAuto(ContentValidationAlgorithm.CRC64)); + assertTrue(ContentValidationModeResolver.isCrc64OrAuto(ContentValidationAlgorithm.AUTO)); + assertFalse(ContentValidationModeResolver.isCrc64OrAuto(ContentValidationAlgorithm.NONE)); + assertFalse(ContentValidationModeResolver.isCrc64OrAuto(null)); + } +} diff --git a/sdk/storage/azure-storage-common/src/test/java/com/azure/storage/common/implementation/contentvalidation/MessageEncoderTests.java b/sdk/storage/azure-storage-common/src/test/java/com/azure/storage/common/implementation/contentvalidation/MessageEncoderTests.java new file mode 100644 index 000000000000..899a2b5f5425 --- /dev/null +++ b/sdk/storage/azure-storage-common/src/test/java/com/azure/storage/common/implementation/contentvalidation/MessageEncoderTests.java @@ -0,0 +1,330 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.common.implementation.contentvalidation; + +import com.azure.core.util.FluxUtil; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.junit.jupiter.api.Disabled; +import reactor.core.publisher.Flux; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.util.Arrays; +import java.util.concurrent.ThreadLocalRandom; +import java.util.stream.Stream; + +import static com.azure.storage.common.implementation.contentvalidation.StructuredMessageConstants.CRC64_LENGTH; +import static com.azure.storage.common.implementation.contentvalidation.StructuredMessageConstants.V1_DEFAULT_SEGMENT_CONTENT_LENGTH; +import static com.azure.storage.common.implementation.contentvalidation.StructuredMessageConstants.V1_HEADER_LENGTH; +import static com.azure.storage.common.implementation.contentvalidation.StructuredMessageConstants.V1_SEGMENT_HEADER_LENGTH; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +public class MessageEncoderTests { + + private static byte[] getRandomData(int size) { + byte[] result = new byte[size]; + ThreadLocalRandom.current().nextBytes(result); + return result; + } + + private static void writeSegment(int number, byte[] data, long dataCrc, ByteArrayOutputStream stream) + throws IOException { + writeSegment(number, data, stream); // Call the method without CRC + ByteBuffer segFooter = ByteBuffer.allocate(CRC64_LENGTH).order(ByteOrder.LITTLE_ENDIAN); + segFooter.putLong(dataCrc); + stream.write(segFooter.array()); // Write segment footer + } + + private static void writeSegment(int number, byte[] data, ByteArrayOutputStream stream) throws IOException { + ByteBuffer segHeader = ByteBuffer.allocate(10).order(ByteOrder.LITTLE_ENDIAN); // 2 + 8 + segHeader.putShort((short) number); + segHeader.putLong(data.length); + + stream.write(segHeader.array()); // Write segment header + stream.write(data); // Write segment content + } + + // TODO (isbr): Add tests with static inputs and expected outputs for the encoder. + // Avoid reimplementing the encoder in tests to prevent potential errors in both implementation and tests. + // Consider reusing outputs from existing scripts. This approach can also benefit future decoder tests. + + private static ByteBuffer buildStructuredMessage(ByteBuffer data, int segmentSize, + StructuredMessageFlags structuredMessageFlags) throws IOException { + int segmentCount = Math.max(1, (int) Math.ceil((double) data.capacity() / segmentSize)); + int segmentFooterLength = structuredMessageFlags == StructuredMessageFlags.STORAGE_CRC64 ? CRC64_LENGTH : 0; + + int messageLength = V1_HEADER_LENGTH + ((V1_SEGMENT_HEADER_LENGTH + segmentFooterLength) * segmentCount) + + data.capacity() + (structuredMessageFlags == StructuredMessageFlags.STORAGE_CRC64 ? CRC64_LENGTH : 0); + + long messageCRC = 0; + + // Message Header + ByteBuffer buffer = ByteBuffer.allocate(13).order(ByteOrder.LITTLE_ENDIAN); //1 + 8 + 2 + 2 + buffer.put((byte) 0x01); + buffer.putLong(messageLength); + buffer.putShort((short) structuredMessageFlags.getValue()); + buffer.putShort((short) segmentCount); + + ByteArrayOutputStream message = new ByteArrayOutputStream(); + message.write(buffer.array()); + + if (data.capacity() == 0) { + if (structuredMessageFlags == StructuredMessageFlags.STORAGE_CRC64) { + writeSegment(1, data.array(), 0, message); + } else { + writeSegment(1, data.array(), message); + } + } else { + // Segments + int[] segmentSizes = new int[segmentCount]; + Arrays.fill(segmentSizes, segmentSize); + + int offset = 0; + for (int i = 1; i <= segmentCount; i++) { + int size = segmentSizes[i - 1]; + byte[] segmentData = customCopyOfRange(data, offset, size); + offset += size; + + if (structuredMessageFlags == StructuredMessageFlags.STORAGE_CRC64) { + long segmentCrc = StorageCrc64Calculator.compute(segmentData, 0); + writeSegment(i, segmentData, segmentCrc, message); + messageCRC = StorageCrc64Calculator.compute(segmentData, messageCRC); + } else { + writeSegment(i, segmentData, message); + } + } + } + + // Message footer + if (structuredMessageFlags == StructuredMessageFlags.STORAGE_CRC64) { + byte[] crcBytes + = ByteBuffer.allocate(CRC64_LENGTH).order(ByteOrder.LITTLE_ENDIAN).putLong(messageCRC).array(); + message.write(crcBytes); + } + + return ByteBuffer.wrap(message.toByteArray()); + } + + public static byte[] customCopyOfRange(ByteBuffer original, int from, int size) { + int end = Math.min(from + size, original.capacity()); + return Arrays.copyOfRange(original.array(), from, end); + } + + private static Stream readAllSupplier() { + return Stream.of(Arguments.of(10, 1, StructuredMessageFlags.NONE), + Arguments.of(10, 1, StructuredMessageFlags.STORAGE_CRC64), + Arguments.of(1024, 1024, StructuredMessageFlags.NONE), + Arguments.of(1024, 1024, StructuredMessageFlags.STORAGE_CRC64), + Arguments.of(1024, 512, StructuredMessageFlags.NONE), + Arguments.of(1024, 512, StructuredMessageFlags.STORAGE_CRC64), + Arguments.of(1024, 200, StructuredMessageFlags.NONE), + Arguments.of(1024, 200, StructuredMessageFlags.STORAGE_CRC64), + Arguments.of(1024 * 10, 2, StructuredMessageFlags.NONE), + Arguments.of(1024 * 10, 2, StructuredMessageFlags.STORAGE_CRC64), + Arguments.of(1024 * 50, 512, StructuredMessageFlags.NONE), + Arguments.of(1024 * 50, 512, StructuredMessageFlags.STORAGE_CRC64), + Arguments.of(1024 * 1024, 512, StructuredMessageFlags.NONE), + Arguments.of(1024 * 1024, 512, StructuredMessageFlags.STORAGE_CRC64), + Arguments.of(1024 * 1024, 1024, StructuredMessageFlags.NONE), + Arguments.of(1024 * 1024, 1024, StructuredMessageFlags.STORAGE_CRC64), + Arguments.of(1024 * 1024 * 4, 1024 * 1024, StructuredMessageFlags.NONE), + Arguments.of(1024 * 1024 * 4, 1024 * 1024, StructuredMessageFlags.STORAGE_CRC64), + Arguments.of(1024 * 1024 * 8, 1024 * 1024, StructuredMessageFlags.NONE), + Arguments.of(1024 * 1024 * 8, 1024 * 1024, StructuredMessageFlags.STORAGE_CRC64), + Arguments.of(1234, 123, StructuredMessageFlags.NONE), + Arguments.of(1234, 123, StructuredMessageFlags.STORAGE_CRC64), + Arguments.of(1234 * 10, 12, StructuredMessageFlags.NONE), + Arguments.of(1234 * 10, 12, StructuredMessageFlags.STORAGE_CRC64), + Arguments.of(1234 * 1234, 567, StructuredMessageFlags.NONE), + Arguments.of(1234 * 1234, 567, StructuredMessageFlags.STORAGE_CRC64), + Arguments.of(1234 * 1234 * 8, 1234 * 1234, StructuredMessageFlags.NONE), + Arguments.of(1234 * 1234 * 8, 1234 * 1234, StructuredMessageFlags.STORAGE_CRC64)); + } + + @ParameterizedTest + @MethodSource("readAllSupplier") + public void readAll(int size, int segmentSize, StructuredMessageFlags flags) throws IOException { + byte[] data = getRandomData(size); + + ByteBuffer unencodedBuffer = ByteBuffer.wrap(data); + + StructuredMessageEncoder structuredMessageEncoder = new StructuredMessageEncoder(size, segmentSize, flags); + + byte[] actual + = FluxUtil.collectBytesInByteBufferStream(structuredMessageEncoder.encode(unencodedBuffer)).block(); + byte[] expected = buildStructuredMessage(unencodedBuffer, segmentSize, flags).array(); + + assertArrayEquals(expected, actual); + } + + private static Stream readMultipleSupplier() { + return Stream.of(Arguments.of(30, StructuredMessageFlags.NONE), + Arguments.of(30, StructuredMessageFlags.STORAGE_CRC64), Arguments.of(15, StructuredMessageFlags.NONE), + Arguments.of(15, StructuredMessageFlags.STORAGE_CRC64), Arguments.of(11, StructuredMessageFlags.NONE), + Arguments.of(11, StructuredMessageFlags.STORAGE_CRC64), Arguments.of(8, StructuredMessageFlags.NONE), + Arguments.of(8, StructuredMessageFlags.STORAGE_CRC64)); + } + + @ParameterizedTest + @MethodSource("readMultipleSupplier") + public void readMultiple(int segmentSize, StructuredMessageFlags flags) throws IOException { + byte[] data1 = getRandomData(10); + byte[] data2 = getRandomData(10); + byte[] data3 = getRandomData(10); + + ByteBuffer wrappedData1 = ByteBuffer.wrap(data1); + ByteBuffer wrappedData2 = ByteBuffer.wrap(data2); + ByteBuffer wrappedData3 = ByteBuffer.wrap(data3); + + ByteBuffer allWrappedData = ByteBuffer.allocate(30); + allWrappedData.put(data1); + allWrappedData.put(data2); + allWrappedData.put(data3); + + StructuredMessageEncoder structuredMessageEncoder = new StructuredMessageEncoder(30, segmentSize, flags); + + byte[] expected = buildStructuredMessage(allWrappedData, segmentSize, flags).array(); + + Flux allActualFlux = structuredMessageEncoder.encode(wrappedData1) + .concatWith(structuredMessageEncoder.encode(wrappedData2)) + .concatWith(structuredMessageEncoder.encode(wrappedData3)); + + byte[] actual = FluxUtil.collectBytesInByteBufferStream(allActualFlux).block(); + + assertArrayEquals(expected, actual); + } + + @Test + public void emptyBuffer() { + StructuredMessageEncoder encoder = new StructuredMessageEncoder(10, 5, StructuredMessageFlags.NONE); + ByteBuffer emptyBuffer = ByteBuffer.allocate(0); + byte[] result = FluxUtil.collectBytesInByteBufferStream(encoder.encode(emptyBuffer)).block(); + assertNotNull(result); + assertEquals(0, result.length); + } + + @Test + public void contentAlreadyEncoded() { + StructuredMessageEncoder encoder = new StructuredMessageEncoder(4, 2, StructuredMessageFlags.NONE); + FluxUtil.collectBytesInByteBufferStream(encoder.encode(ByteBuffer.wrap(new byte[] { 1, 2, 3, 4 }))).block(); + // After encoding is complete, further encode calls return empty (no error) to support retries and extra buffers. + byte[] result + = FluxUtil.collectBytesInByteBufferStream(encoder.encode(ByteBuffer.wrap(new byte[] { 1, 2 }))).block(); + assertNotNull(result); + assertEquals(0, result.length); + } + + @Test + public void bufferLengthExceedsContentLength() { + StructuredMessageEncoder encoder = new StructuredMessageEncoder(4, 2, StructuredMessageFlags.NONE); + FluxUtil.collectBytesInByteBufferStream(encoder.encode(ByteBuffer.wrap(new byte[] { 1, 2, 3 }))).block(); + assertThrows(IllegalArgumentException.class, + () -> FluxUtil.collectBytesInByteBufferStream(encoder.encode(ByteBuffer.wrap(new byte[] { 1, 2 }))) + .block()); + } + + @Test + public void segmentSizeLessThanOne() { + assertThrows(IllegalArgumentException.class, + () -> new StructuredMessageEncoder(10, 0, StructuredMessageFlags.NONE)); + } + + @Test + public void contentLengthLessThanOne() { + assertThrows(IllegalArgumentException.class, + () -> new StructuredMessageEncoder(0, 10, StructuredMessageFlags.NONE)); + } + + @Test + public void testNumSegmentsExceedsMaxValue() { + assertThrows(IllegalArgumentException.class, + () -> new StructuredMessageEncoder(Integer.MAX_VALUE, 1, StructuredMessageFlags.NONE)); + } + + // =========================================================================================== + // getEncodedMessageLength accuracy + // =========================================================================================== + + private static Stream encodedLengthSupplier() { + return Stream.of(Arguments.of(1, 1, StructuredMessageFlags.NONE), + Arguments.of(1, 1, StructuredMessageFlags.STORAGE_CRC64), + Arguments.of(100, 100, StructuredMessageFlags.NONE), + Arguments.of(100, 100, StructuredMessageFlags.STORAGE_CRC64), + Arguments.of(1024, 512, StructuredMessageFlags.NONE), + Arguments.of(1024, 512, StructuredMessageFlags.STORAGE_CRC64), + Arguments.of(4 * 1024 * 1024, V1_DEFAULT_SEGMENT_CONTENT_LENGTH, StructuredMessageFlags.STORAGE_CRC64), + Arguments.of(10 * 1024 * 1024, V1_DEFAULT_SEGMENT_CONTENT_LENGTH, StructuredMessageFlags.STORAGE_CRC64), + Arguments.of(1234, 123, StructuredMessageFlags.STORAGE_CRC64)); + } + + @ParameterizedTest + @MethodSource("encodedLengthSupplier") + public void encodedLengthMatchesActualOutput(int size, int segmentSize, StructuredMessageFlags flags) { + byte[] data = getRandomData(size); + StructuredMessageEncoder encoder = new StructuredMessageEncoder(size, segmentSize, flags); + long predictedLength = encoder.getEncodedMessageLength(); + + byte[] actual = FluxUtil.collectBytesInByteBufferStream(encoder.encode(ByteBuffer.wrap(data))).block(); + assertNotNull(actual); + assertEquals(predictedLength, actual.length, "getEncodedMessageLength() must match actual encoded output size"); + } + + // =========================================================================================== + // Direct (non-array-backed) ByteBuffer + // =========================================================================================== + + @ParameterizedTest + @MethodSource("readAllSupplier") + public void readAllDirectByteBuffer(int size, int segmentSize, StructuredMessageFlags flags) throws IOException { + byte[] data = getRandomData(size); + ByteBuffer directBuffer = ByteBuffer.allocateDirect(size); + directBuffer.put(data); + directBuffer.flip(); + + ByteBuffer arrayBuffer = ByteBuffer.wrap(data); + + StructuredMessageEncoder encoder = new StructuredMessageEncoder(size, segmentSize, flags); + byte[] actual = FluxUtil.collectBytesInByteBufferStream(encoder.encode(directBuffer)).block(); + byte[] expected = buildStructuredMessage(arrayBuffer, segmentSize, flags).array(); + + assertArrayEquals(expected, actual, "Direct ByteBuffer encoding must match array-backed encoding"); + } + + // =========================================================================================== + // Null buffer + // =========================================================================================== + + @Test + public void encodeNullBufferThrows() { + StructuredMessageEncoder encoder = new StructuredMessageEncoder(10, 5, StructuredMessageFlags.NONE); + assertThrows(NullPointerException.class, + () -> FluxUtil.collectBytesInByteBufferStream(encoder.encode(null)).block()); + } + + @Test + @Disabled("For local testing only") + public void bigEncode() throws IOException { + byte[] data = getRandomData(262144000); + + ByteBuffer unencodedBuffer = ByteBuffer.wrap(data); + + StructuredMessageEncoder structuredMessageEncoder = new StructuredMessageEncoder(262144000, + V1_DEFAULT_SEGMENT_CONTENT_LENGTH, StructuredMessageFlags.STORAGE_CRC64); + + byte[] actual + = FluxUtil.collectBytesInByteBufferStream(structuredMessageEncoder.encode(unencodedBuffer)).block(); + byte[] expected = buildStructuredMessage(unencodedBuffer, V1_DEFAULT_SEGMENT_CONTENT_LENGTH, + StructuredMessageFlags.STORAGE_CRC64).array(); + assertArrayEquals(expected, actual); + } +} diff --git a/sdk/storage/azure-storage-common/src/test/java/com/azure/storage/common/implementation/contentvalidation/StorageCrc64CalculatorTests.java b/sdk/storage/azure-storage-common/src/test/java/com/azure/storage/common/implementation/contentvalidation/StorageCrc64CalculatorTests.java new file mode 100644 index 000000000000..a07cef1f2013 --- /dev/null +++ b/sdk/storage/azure-storage-common/src/test/java/com/azure/storage/common/implementation/contentvalidation/StorageCrc64CalculatorTests.java @@ -0,0 +1,222 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.common.implementation.contentvalidation; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.MethodSource; + +import com.azure.storage.common.implementation.Constants; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.LinkedList; +import java.util.List; +import java.util.Queue; +import java.util.Random; +import java.util.stream.IntStream; +import java.math.BigInteger; +import java.util.stream.Stream; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class StorageCrc64CalculatorTests { + + @ParameterizedTest + @MethodSource("testComputeSupplier") + public void testCompute(String data, long initial, long expected) { + byte[] bytes = data.getBytes(); + long actual = StorageCrc64Calculator.compute(bytes, initial); + assertEquals(expected, actual); + } + + private static Stream testComputeSupplier() { + return Stream.of(Arguments.of("", 0, 0), Arguments.of("Hello World!", 0, 208604604655264165L), + Arguments.of("123456789!@#$%^&*()", 0, 2153758901452455624L), + Arguments.of( + "This is a test where the data is longer than 64 characters so that we can test that code path.", 0, + 2736107658526394369L)); + } + + @ParameterizedTest + @MethodSource("testComputeWithBinaryDataSupplier") + void testComputeWithBinaryData(long initial, String hexData, String expectedStr) { + byte[] hexBytes = hexStringToByteArray(hexData); + long expected = unsignedLongFromString(expectedStr); + long actual = StorageCrc64Calculator.compute(hexBytes, initial); + assertEquals(expected, actual); + } + + private static Stream testComputeWithBinaryDataSupplier() { + return Stream.of(Arguments.of(0L, "C8E11B40D793D1526018", "3386042136331673945"), + Arguments.of(208604604655264165L, "C8E11B40D793D1526018", "4570059697646401418"), + Arguments.of(2153758901452455624L, "C8E11B40D793D1526018", "13366433516720813220"), + Arguments.of(12345L, "C8E11B40D793D1526018", "5139183895903464380"), + Arguments.of(0L, "AABBCCDDEEFF", "13410969003359324163"), + Arguments.of(208604604655264165L, "AABBCCDDEEFF", "7077292804802198544"), + Arguments.of(2153758901452455624L, "AABBCCDDEEFF", "7845171950889742163"), + Arguments.of(12345L, "AABBCCDDEEFF", "1250888331951523569")); + } + + private static byte[] hexStringToByteArray(String hexData) { + int length = hexData.length(); + byte[] data = new byte[length / 2]; + for (int i = 0; i < length; i += 2) { + data[i / 2] + = (byte) ((Character.digit(hexData.charAt(i), 16) << 4) + Character.digit(hexData.charAt(i + 1), 16)); + } + return data; + } + + @ParameterizedTest + @MethodSource("testComposeSupplier") + void testCompose(int numSegments, int minBlockSize, int maxBlockSize) { + Random random = new Random(); + + List blockLengths = new ArrayList<>(); + IntStream.range(0, numSegments).forEach(i -> { + int blockSize = maxBlockSize > minBlockSize + ? random.nextInt(maxBlockSize - minBlockSize) + minBlockSize + : minBlockSize; + blockLengths.add(blockSize); + }); + + byte[] data = new byte[blockLengths.stream().mapToInt(Integer::intValue).sum()]; + random.nextBytes(data); + + long wholeCrc = StorageCrc64Calculator.compute(data, 0); + Queue blockCrcs = new LinkedList<>(); + int offset = 0; + for (int length : blockLengths) { + blockCrcs.add(StorageCrc64Calculator.compute(Arrays.copyOfRange(data, offset, offset + length), 0)); + offset += length; + } + + long composedCrc = blockCrcs.poll(); + int lengthIndex = 1; + while (!blockCrcs.isEmpty()) { + long nextBlockCrc = blockCrcs.poll(); + composedCrc = StorageCrc64Calculator.concat(0, 0, composedCrc, blockLengths.get(lengthIndex - 1), 0, + nextBlockCrc, blockLengths.get(lengthIndex)); + lengthIndex++; + } + + assertEquals(wholeCrc, composedCrc); + } + + private static Stream testComposeSupplier() { + return Stream.of(Arguments.of(2, Constants.KB, Constants.KB), Arguments.of(3, Constants.KB, Constants.KB), + Arguments.of(10, Constants.KB, Constants.KB), Arguments.of(2, Constants.KB, 4 * Constants.KB), + Arguments.of(3, Constants.KB, 4 * Constants.KB), Arguments.of(10, Constants.KB, 4 * Constants.KB), + Arguments.of(2, Constants.KB, Constants.MB), Arguments.of(3, Constants.KB, Constants.MB), + Arguments.of(2, Constants.KB, 512 * Constants.MB), Arguments.of(3, Constants.KB, 512 * Constants.MB)); + } + + @Test + void testComputeInChunks() { + int maxBlockSize = Constants.GB; + int chunkSize = 8 * Constants.KB; + String baseString = "Hello World! This is testing chunked crc."; + byte[] baseBytes = baseString.getBytes(); + int baseLength = baseBytes.length; + + long wholeCrc = 8100535992282268188L; + + long chunkedCrc = 0; + int totalLength = 0; + + while (totalLength < maxBlockSize) { + int length = Math.min(chunkSize, maxBlockSize - totalLength); + byte[] chunk = new byte[length]; + for (int i = 0; i < length; i += baseLength) { + System.arraycopy(baseBytes, 0, chunk, i, Math.min(baseLength, length - i)); + } + long chunkCrc = StorageCrc64Calculator.compute(chunk, 0); + chunkedCrc = StorageCrc64Calculator.concat(chunkedCrc, 0, chunkCrc, length, 0, chunkCrc, length); + totalLength += length; + } + + assertEquals(wholeCrc, chunkedCrc); + } + + @ParameterizedTest + @CsvSource({ + "0, 0, 0, 0, 0", + "17360427831495520774, 949533, 13068224794440996385, 99043, 2942932174470096852", + "11788770130477425887, 505156, 11825964890373840515, 543420, 11679439596881108042", + "3295333047304801182, 732633, 15304759627474960884, 315943, 31840984168952505", + "16590039424904606984, 550299, 6063316096266934453, 498277, 4430932446378441680", + "4505532069077416052, 852279, 4910763717047934640, 196297, 15119506491662968913", + "390777554329396866, 834642, 2639871931800812330, 213934, 11705441749781302341", + "3373070654000205532, 282713, 998330282635826126, 765863, 9830625244855600085", + "2124306943908111903, 737293, 11017351202683543503, 311283, 8163771928713973931", + "15994440356403990005, 85861, 13803536430055425947, 962715, 1941624554903785510", + "3705122932895036835, 444701, 17573219681510991482, 603875, 4421337306620606216" }) + void testConcat(String crc1Str, long size1, String crc2Str, long size2, String expectedStr) { + // Convert the large numbers from String to BigInteger for unsigned support + long crc1 = unsignedLongFromString(crc1Str); + long crc2 = unsignedLongFromString(crc2Str); + long expected = unsignedLongFromString(expectedStr); + + long actual = StorageCrc64Calculator.concat(0, 0, crc1, size1, 0, crc2, size2); + assertEquals(expected, actual); + } + + private long unsignedLongFromString(String value) { + // Convert a string to a signed long interpreting it as unsigned 64-bit integer + return new BigInteger(value).longValue(); + } + + @ParameterizedTest + @MethodSource("testConcatWithInitialsSupplier") + void testConcatWithInitials(String initialStr, String initial1Str, String crc1Str, String size1Str, + String initial2Str, String crc2Str, String size2Str, String expectedStr) { + // Convert large unsigned values to signed long + long initial = unsignedLongFromString(initialStr); + long initial1 = unsignedLongFromString(initial1Str); + long crc1 = unsignedLongFromString(crc1Str); + long size1 = Long.parseLong(size1Str); + long initial2 = unsignedLongFromString(initial2Str); + long crc2 = unsignedLongFromString(crc2Str); + long size2 = Long.parseLong(size2Str); + long expected = unsignedLongFromString(expectedStr); + + long actual = StorageCrc64Calculator.concat(initial, initial1, crc1, size1, initial2, crc2, size2); + assertEquals(expected, actual); + } + + private static Stream testConcatWithInitialsSupplier() { + return Stream.of(Arguments.of("0", "0", "0", "0", "0", "0", "0", "0"), + Arguments.of("556425425686929588", "346224202686926702", "16342296696377857982", "332915", + "1153230192133190692", "12153371329672466699", "715661", "1441822370130745021"), + Arguments.of("6707243468313456313", "572263087298867634", "16994544883182326144", "75745", + "9131338361339398429", "10182915179976307502", "972831", "14966971284513070994"), + Arguments.of("5753644013440131291", "5049702011265556767", "17549647897932809624", "255140", + "7204171574261853450", "1993084328138883374", "793436", "6041621697050742380"), + Arguments.of("6856094926385348025", "5380840211500611709", "9696539459657763690", "537777", + "4787042077805010903", "13660128687379374948", "510799", "17784586126519415898"), + Arguments.of("7768574238870932405", "97145001356670685", "607054043350981298", "706788", + "667444555190985522", "10677778047180339455", "341788", "5763961866513573791"), + Arguments.of("3302120679354661969", "7763531798276712053", "8827557196489825944", "490442", + "8582969104890206846", "6702182603435500761", "558134", "4787302867829109706"), + Arguments.of("7553023568245626261", "9093436341919279996", "10815569438302788871", "785480", + "8305342016037017917", "6633140726058569127", "263096", "14625483825524673467"), + Arguments.of("8894905328920043035", "9101951045389247372", "10098427678135105249", "782758", + "8101576936188464286", "8318237935995533450", "265818", "5983082645903611588"), + Arguments.of("1084935736425738155", "5378644106529179816", "13762475631325587388", "1014816", + "8473418370223760471", "10401355811619715622", "33760", "3692020299859515126"), + Arguments.of("889000539881195835", "2971048229276949174", "5346315327374690144", "307387", + "1407121768110541356", "10535852615249992663", "741189", "3634018251978804152")); + } + + @Test + void testComputeSliceMatchesFullArray() { + byte[] data = "Hello World!".getBytes(); + long expected = StorageCrc64Calculator.compute(data, 0); + long actual = StorageCrc64Calculator.compute(data, 0, data.length, 0); + assertEquals(expected, actual); + } +} diff --git a/sdk/storage/azure-storage-common/src/test/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageDecoderTests.java b/sdk/storage/azure-storage-common/src/test/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageDecoderTests.java new file mode 100644 index 000000000000..f9f460bb57fc --- /dev/null +++ b/sdk/storage/azure-storage-common/src/test/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageDecoderTests.java @@ -0,0 +1,631 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.common.implementation.contentvalidation; + +import com.azure.core.util.FluxUtil; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import reactor.core.publisher.Flux; + +import java.io.ByteArrayOutputStream; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.util.Arrays; +import java.util.List; +import java.util.Objects; +import java.util.concurrent.ThreadLocalRandom; +import java.util.stream.Stream; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Unit tests for StructuredMessageDecoder. + */ +public class StructuredMessageDecoderTests { + private static final int MESSAGE_HEADER_LENGTH = StructuredMessageConstants.V1_HEADER_LENGTH; + private static final int SEGMENT_HEADER_LENGTH = StructuredMessageConstants.V1_SEGMENT_HEADER_LENGTH; + private static final int CRC64_LENGTH = 8; + + @Test + public void readsCompleteMessageInSingleChunk() { + byte[] originalData = getRandomData(1024); + ByteBuffer encodedData = encodeToByteBuffer(originalData, 512, StructuredMessageFlags.STORAGE_CRC64); + int encodedLength = encodedData.remaining(); + + StructuredMessageDecoder decoder = new StructuredMessageDecoder(encodedLength); + List decodedPayload = decoder.decodeChunk(encodedData); + + assertTrue(decoder.isComplete()); + assertFalse(decodedPayload.isEmpty()); + assertArrayEquals(originalData, collectDecodedBytes(decodedPayload)); + } + + @Test + public void readsTopLevelMessageHeaderSplitAcrossChunks() { + byte[] originalData = getRandomData(256); + byte[] encodedBytes = encodeToBytes(originalData, 128); + int encodedLength = encodedBytes.length; + + // Split before the 13-byte message header is complete. + int messageHeaderSplitPoint = 7; + ByteBuffer chunk1 = ByteBuffer.wrap(encodedBytes, 0, messageHeaderSplitPoint); + ByteBuffer chunk2 + = ByteBuffer.wrap(encodedBytes, messageHeaderSplitPoint, encodedLength - messageHeaderSplitPoint); + + StructuredMessageDecoder decoder = new StructuredMessageDecoder(encodedLength); + List firstDecodedPayload = decoder.decodeChunk(chunk1); + assertTrue(firstDecodedPayload.isEmpty()); + assertFalse(decoder.isComplete()); + + List secondDecodedPayload = decoder.decodeChunk(chunk2); + assertFalse(secondDecodedPayload.isEmpty()); + assertTrue(decoder.isComplete()); + } + + @Test + public void readsPerSegmentHeaderSplitAcrossChunks() { + byte[] originalData = getRandomData(512); + byte[] encodedBytes = encodeToBytes(originalData, 256); + int encodedLength = encodedBytes.length; + + // Split after the full message header but before the 10-byte segment header is complete. + int segmentHeaderSplitPoint = MESSAGE_HEADER_LENGTH + 5; + ByteBuffer chunk1 = ByteBuffer.wrap(encodedBytes, 0, segmentHeaderSplitPoint); + ByteBuffer chunk2 + = ByteBuffer.wrap(encodedBytes, segmentHeaderSplitPoint, encodedLength - segmentHeaderSplitPoint); + + StructuredMessageDecoder decoder = new StructuredMessageDecoder(encodedLength); + List firstDecodedPayload = decoder.decodeChunk(chunk1); + // Segment header is incomplete, so nothing is emitted yet. + assertTrue(firstDecodedPayload.isEmpty()); + assertFalse(decoder.isComplete()); + + List secondDecodedPayload = decoder.decodeChunk(chunk2); + assertFalse(secondDecodedPayload.isEmpty()); + assertTrue(decoder.isComplete()); + } + + @Test + public void multipleChunksDecode() { + byte[] originalData = getRandomData(256); + byte[] encodedBytes = encodeToBytes(originalData, 128); + int encodedLength = encodedBytes.length; + + StructuredMessageDecoder decoder = new StructuredMessageDecoder(encodedLength); + + int chunkSize = 32; + ByteArrayOutputStream output = new ByteArrayOutputStream(); + + for (int offset = 0; offset < encodedLength; offset += chunkSize) { + int len = Math.min(chunkSize, encodedLength - offset); + ByteBuffer chunk = ByteBuffer.wrap(encodedBytes, offset, len); + + List decodedPayload = decoder.decodeChunk(chunk); + writeDecodedPayload(output, decodedPayload); + if (decoder.isComplete()) { + break; + } + } + + assertTrue(decoder.isComplete()); + assertArrayEquals(originalData, output.toByteArray()); + } + + @Test + public void decodeWithNoCrc() { + byte[] originalData = getRandomData(256); + ByteBuffer encodedData = encodeToByteBuffer(originalData, 128, StructuredMessageFlags.NONE); + int encodedLength = encodedData.remaining(); + + StructuredMessageDecoder decoder = new StructuredMessageDecoder(encodedLength); + List decodedPayload = decoder.decodeChunk(encodedData); + + assertTrue(decoder.isComplete()); + assertFalse(decodedPayload.isEmpty()); + assertArrayEquals(originalData, collectDecodedBytes(decodedPayload)); + } + + @Test + public void handlesZeroLengthBuffer() { + byte[] originalData = getRandomData(256); + byte[] encodedBytes = encodeToBytes(originalData, 128); + int encodedLength = encodedBytes.length; + + StructuredMessageDecoder decoder = new StructuredMessageDecoder(encodedLength); + ByteBuffer emptyBuffer = ByteBuffer.allocate(0); + List firstDecodedPayload = decoder.decodeChunk(emptyBuffer); + assertTrue(firstDecodedPayload.isEmpty()); + ByteBuffer dataBuffer = ByteBuffer.wrap(encodedBytes); + List decodedPayload = decoder.decodeChunk(dataBuffer); + assertFalse(decodedPayload.isEmpty()); + assertTrue(decoder.isComplete()); + } + + /** + * Payload bytes for a segment must not be emitted until the segment's CRC footer has been read and + * validated. While the footer is incomplete, decodeChunk must return null. + */ + @Test + public void withholdsPayloadUntilSegmentFooterValidated() { + byte[] originalData = getRandomData(1024); + byte[] encodedBytes = encodeToBytes(originalData, 1024); + int encodedLength = encodedBytes.length; + + // The segment payload cannot be emitted until the final segment CRC byte arrives in chunk2. + int segCrcAllButLast + = MESSAGE_HEADER_LENGTH + SEGMENT_HEADER_LENGTH + 1024 + StructuredMessageConstants.CRC64_LENGTH - 1; + ByteBuffer chunk1 = ByteBuffer.wrap(encodedBytes, 0, segCrcAllButLast); + StructuredMessageDecoder decoder = new StructuredMessageDecoder(encodedLength); + List firstDecodedPayload = decoder.decodeChunk(chunk1); + assertTrue(firstDecodedPayload.isEmpty(), "Decoder must not emit payload before segment CRC is validated"); + assertFalse(decoder.isComplete()); + + ByteBuffer chunk2 = ByteBuffer.wrap(encodedBytes, segCrcAllButLast, encodedLength - segCrcAllButLast); + List emittedPayload = decoder.decodeChunk(chunk2); + assertFalse(emittedPayload.isEmpty()); + assertTrue(decoder.isComplete()); + assertArrayEquals(originalData, collectDecodedBytes(emittedPayload)); + } + + @Test + public void throwsOnUnsupportedStructuredMessageVersion() { + byte[] data = getRandomData(64); + byte[] encodedBytes = encodeToBytes(data, 64); + + // Byte 0 of the message header is reserved for the version. Changing to an incompatible version (2) + encodedBytes[0] = (byte) (StructuredMessageConstants.DEFAULT_MESSAGE_VERSION + 1); + + StructuredMessageDecoder decoder = new StructuredMessageDecoder(encodedBytes.length); + IllegalArgumentException exception + = assertThrows(IllegalArgumentException.class, () -> decoder.decodeChunk(ByteBuffer.wrap(encodedBytes))); + assertTrue(exception.getMessage().contains("Unsupported structured message version")); + } + + @Test + public void throwsOnMessageLengthMismatch() { + byte[] data = getRandomData(128); + byte[] encodedBytes = encodeToBytes(data, 128); + + // Construct decoder with wrong expected encoded length. + StructuredMessageDecoder decoder = new StructuredMessageDecoder(encodedBytes.length + 1); + IllegalArgumentException exception + = assertThrows(IllegalArgumentException.class, () -> decoder.decodeChunk(ByteBuffer.wrap(encodedBytes))); + assertTrue(exception.getMessage().contains("did not match content length")); + } + + @Test + public void throwsOnUnexpectedSegmentNumber() { + byte[] data = getRandomData(300); + byte[] encodedBytes = encodeToBytes(data, 128); + + // Corrupt first segment number from 1 to 2 (offset 13). + ByteBuffer.wrap(encodedBytes).order(ByteOrder.LITTLE_ENDIAN).putShort(MESSAGE_HEADER_LENGTH, (short) 2); + + StructuredMessageDecoder decoder = new StructuredMessageDecoder(encodedBytes.length); + IllegalArgumentException exception + = assertThrows(IllegalArgumentException.class, () -> decoder.decodeChunk(ByteBuffer.wrap(encodedBytes))); + assertTrue(exception.getMessage().contains("Unexpected segment number")); + } + + @Test + public void throwsOnInvalidSegmentSize() { + byte[] data = getRandomData(256); + byte[] encodedBytes = encodeToBytes(data, 128); + + // Corrupt first segment size to an impossible value (8 bytes at offset 15). + ByteBuffer.wrap(encodedBytes).order(ByteOrder.LITTLE_ENDIAN).putLong(MESSAGE_HEADER_LENGTH + 2, Long.MAX_VALUE); + + StructuredMessageDecoder decoder = new StructuredMessageDecoder(encodedBytes.length); + IllegalArgumentException exception + = assertThrows(IllegalArgumentException.class, () -> decoder.decodeChunk(ByteBuffer.wrap(encodedBytes))); + assertTrue(exception.getMessage().contains("Invalid segment size detected")); + } + + @Test + public void throwsOnSegmentCrcMismatch() { + byte[] data = getRandomData(512); + byte[] encodedBytes = encodeToBytes(data, 512); + + // msgHdr(13) + segHdr(10) + payload(512) + segCrc(8) + msgCrc(8). Flip one bit of the segment CRC. + int segmentCrcOffset = MESSAGE_HEADER_LENGTH + SEGMENT_HEADER_LENGTH + data.length; + encodedBytes[segmentCrcOffset] ^= 0x01; + + StructuredMessageDecoder decoder = new StructuredMessageDecoder(encodedBytes.length); + IllegalArgumentException exception + = assertThrows(IllegalArgumentException.class, () -> decoder.decodeChunk(ByteBuffer.wrap(encodedBytes))); + assertTrue(exception.getMessage().contains("CRC64 mismatch in segment")); + } + + @Test + public void throwsOnSegmentCrcMismatchInLaterSegment() { + // Multi-segment message where segment 1 is intact but segment 2's CRC is corrupted; verifies + // CRC validation runs on every segment, not just the first. + byte[] data = getRandomData(300); + byte[] encodedBytes = encodeToBytes(data, 100); + + // Per-segment block: segHdr(10) + payload(100) + segCrc(8) = 118. + // Segment 2's CRC starts at: msgHdr(13) + segBlock(118) + segHdr(10) + payload(100). + int seg2CrcOffset + = MESSAGE_HEADER_LENGTH + (SEGMENT_HEADER_LENGTH + 100 + CRC64_LENGTH) + SEGMENT_HEADER_LENGTH + 100; + encodedBytes[seg2CrcOffset] ^= 0x01; + + StructuredMessageDecoder decoder = new StructuredMessageDecoder(encodedBytes.length); + IllegalArgumentException exception + = assertThrows(IllegalArgumentException.class, () -> decoder.decodeChunk(ByteBuffer.wrap(encodedBytes))); + assertTrue(exception.getMessage().contains("CRC64 mismatch in segment 2")); + } + + @Test + public void throwsOnMessageCrcMismatch() { + byte[] data = getRandomData(512); + byte[] encodedBytes = encodeToBytes(data, 512); + + int messageCrcOffset = encodedBytes.length - CRC64_LENGTH; + byte[] corrupted = Arrays.copyOf(encodedBytes, encodedBytes.length); + corrupted[messageCrcOffset] ^= 0x01; + + StructuredMessageDecoder decoder = new StructuredMessageDecoder(corrupted.length); + IllegalArgumentException exception + = assertThrows(IllegalArgumentException.class, () -> decoder.decodeChunk(ByteBuffer.wrap(corrupted))); + assertTrue(exception.getMessage().contains("CRC64 mismatch in message footer")); + } + + @Test + public void throwsOnUnsupportedFlags() { + // Flags value 2 is not in the StructuredMessageFlags enum (NONE=0, STORAGE_CRC64=1) and must be rejected. + byte[] data = getRandomData(64); + byte[] encodedBytes = encodeToBytes(data, 64); + + // Flags live at offset 9 (2 bytes, little-endian). + ByteBuffer.wrap(encodedBytes).order(ByteOrder.LITTLE_ENDIAN).putShort(9, (short) 2); + + StructuredMessageDecoder decoder = new StructuredMessageDecoder(encodedBytes.length); + IllegalArgumentException exception + = assertThrows(IllegalArgumentException.class, () -> decoder.decodeChunk(ByteBuffer.wrap(encodedBytes))); + assertTrue(exception.getMessage().contains("Invalid value for StructuredMessageFlags")); + } + + @Test + public void throwsOnZeroSegments() { + byte[] data = getRandomData(64); + byte[] encodedBytes = encodeToBytes(data, 64); + + // numSegments lives at offset 11 (2 bytes, little-endian). Force it to zero. + ByteBuffer.wrap(encodedBytes).order(ByteOrder.LITTLE_ENDIAN).putShort(11, (short) 0); + + StructuredMessageDecoder decoder = new StructuredMessageDecoder(encodedBytes.length); + IllegalArgumentException exception + = assertThrows(IllegalArgumentException.class, () -> decoder.decodeChunk(ByteBuffer.wrap(encodedBytes))); + assertTrue(exception.getMessage().contains("at least one segment")); + } + + @Test + public void throwsOnSkippedSegmentNumber() { + // 3 segments of 100 bytes each. Layout per segment: segHdr(10) + payload(100) + segCrc(8) = 118. + // Rewrite segment 2's number to 3 to simulate a stream that skips segment 2. + byte[] data = getRandomData(300); + byte[] encodedBytes = encodeToBytes(data, 100); + + int seg2NumberOffset = MESSAGE_HEADER_LENGTH + (SEGMENT_HEADER_LENGTH + 100 + CRC64_LENGTH); + ByteBuffer.wrap(encodedBytes).order(ByteOrder.LITTLE_ENDIAN).putShort(seg2NumberOffset, (short) 3); + + StructuredMessageDecoder decoder = new StructuredMessageDecoder(encodedBytes.length); + IllegalArgumentException exception + = assertThrows(IllegalArgumentException.class, () -> decoder.decodeChunk(ByteBuffer.wrap(encodedBytes))); + assertTrue(exception.getMessage().contains("Unexpected segment number")); + } + + @Test + public void truncatedMessageHeaderLeavesDecoderIncomplete() { + byte[] data = getRandomData(64); + byte[] encodedBytes = encodeToBytes(data, 64); + + // Feed only the first 5 bytes of the 13-byte message header. + ByteBuffer chunk = ByteBuffer.wrap(encodedBytes, 0, 5); + + StructuredMessageDecoder decoder = new StructuredMessageDecoder(encodedBytes.length); + List decodedPayload = decoder.decodeChunk(chunk); + assertTrue(decodedPayload.isEmpty()); + assertFalse(decoder.isComplete()); + } + + @Test + public void truncatedSegmentHeaderLeavesDecoderIncomplete() { + byte[] data = getRandomData(256); + byte[] encodedBytes = encodeToBytes(data, 256); + + // Feed full message header + 5 of the 10 segment-header bytes. + int truncated = MESSAGE_HEADER_LENGTH + 5; + ByteBuffer chunk = ByteBuffer.wrap(encodedBytes, 0, truncated); + + StructuredMessageDecoder decoder = new StructuredMessageDecoder(encodedBytes.length); + List decodedPayload = decoder.decodeChunk(chunk); + assertTrue(decodedPayload.isEmpty()); + assertFalse(decoder.isComplete()); + } + + @Test + public void truncatedSegmentFooterLeavesDecoderIncomplete() { + byte[] data = getRandomData(128); + byte[] encodedBytes = encodeToBytes(data, 128); + + // Layout: msgHdr(13) + segHdr(10) + payload(128) + segCrc(8) + msgCrc(8). Truncate mid-segCrc. + int truncated = MESSAGE_HEADER_LENGTH + SEGMENT_HEADER_LENGTH + 128 + 4; + ByteBuffer chunk = ByteBuffer.wrap(encodedBytes, 0, truncated); + + StructuredMessageDecoder decoder = new StructuredMessageDecoder(encodedBytes.length); + List decodedPayload = decoder.decodeChunk(chunk); + assertTrue(decodedPayload.isEmpty()); + assertFalse(decoder.isComplete()); + } + + @Test + public void truncatedMessageFooterLeavesDecoderIncomplete() { + byte[] data = getRandomData(128); + byte[] encodedBytes = encodeToBytes(data, 128); + + // Feed everything except the last 4 bytes of the 8-byte message CRC footer. + int truncated = encodedBytes.length - 4; + ByteBuffer chunk = ByteBuffer.wrap(encodedBytes, 0, truncated); + + StructuredMessageDecoder decoder = new StructuredMessageDecoder(encodedBytes.length); + List decodedPayload = decoder.decodeChunk(chunk); + // Segment payload has been released, but the message footer is still incomplete. + assertFalse(decodedPayload.isEmpty()); + assertFalse(decoder.isComplete()); + } + + @Test + public void extraBytesAfterMessageFooterAreNotConsumed() { + byte[] data = getRandomData(128); + byte[] encodedBytes = encodeToBytes(data, 128); + + // Append garbage bytes after the message footer; the decoder must stop at the declared message length. + int extras = 16; + byte[] padded = new byte[encodedBytes.length + extras]; + System.arraycopy(encodedBytes, 0, padded, 0, encodedBytes.length); + byte[] noise = getRandomData(extras); + System.arraycopy(noise, 0, padded, encodedBytes.length, extras); + ByteBuffer buffer = ByteBuffer.wrap(padded); + StructuredMessageDecoder decoder = new StructuredMessageDecoder(encodedBytes.length); + List decodedPayload = decoder.decodeChunk(buffer); + + assertTrue(decoder.isComplete()); + assertFalse(decodedPayload.isEmpty()); + assertArrayEquals(data, collectDecodedBytes(decodedPayload)); + // Trailing bytes must not be consumed; buffer position stops at the declared message length. + assertEquals(extras, buffer.remaining()); + } + + @Test + public void throwsOnEncodedPayloadLargerThanExpectedSize() { + // Wire payload is larger than the expectedEncodedMessageLength supplied to the decoder. Must be rejected by the msgLen check. + byte[] data = getRandomData(128); + byte[] encodedBytes = encodeToBytes(data, 128); + + StructuredMessageDecoder decoder = new StructuredMessageDecoder(encodedBytes.length - 8); + IllegalArgumentException exception + = assertThrows(IllegalArgumentException.class, () -> decoder.decodeChunk(ByteBuffer.wrap(encodedBytes))); + assertTrue(exception.getMessage().contains("did not match content length")); + } + + @Test + public void throwsOnNegativeMessageLength() { + // msgLen lives at offset 1 (8 bytes, little-endian). A negative value must be rejected before + // any further bounds math runs. + byte[] data = getRandomData(64); + byte[] encodedBytes = encodeToBytes(data, 64); + + ByteBuffer.wrap(encodedBytes).order(ByteOrder.LITTLE_ENDIAN).putLong(1, Long.MIN_VALUE); + + StructuredMessageDecoder decoder = new StructuredMessageDecoder(encodedBytes.length); + IllegalArgumentException exception + = assertThrows(IllegalArgumentException.class, () -> decoder.decodeChunk(ByteBuffer.wrap(encodedBytes))); + assertTrue(exception.getMessage().contains("Message length too small")); + } + + @Test + public void throwsOnNegativeSegmentSize() { + // Companion to throwsOnInvalidSegmentSize covering the negative-value branch of the segment-size check. + byte[] data = getRandomData(256); + byte[] encodedBytes = encodeToBytes(data, 256); + + // Segment size lives at offset MESSAGE_HEADER_LENGTH + 2 (after the 2-byte segment number). + ByteBuffer.wrap(encodedBytes).order(ByteOrder.LITTLE_ENDIAN).putLong(MESSAGE_HEADER_LENGTH + 2, Long.MIN_VALUE); + + StructuredMessageDecoder decoder = new StructuredMessageDecoder(encodedBytes.length); + IllegalArgumentException exception + = assertThrows(IllegalArgumentException.class, () -> decoder.decodeChunk(ByteBuffer.wrap(encodedBytes))); + assertTrue(exception.getMessage().contains("Invalid segment size detected")); + } + + @Test + public void throwsOnInjectedRandomByte() { + // Insert a single random byte at a random offset in the encoded wire bytes. The msgLen field still + // declares the original size, so any insertion must be rejected by validation. + byte[] data = getRandomData(256); + byte[] encodedBytes = encodeToBytes(data, 128); + + int insertAt = ThreadLocalRandom.current().nextInt(encodedBytes.length + 1); + byte injected = (byte) ThreadLocalRandom.current().nextInt(256); + + byte[] tampered = new byte[encodedBytes.length + 1]; + System.arraycopy(encodedBytes, 0, tampered, 0, insertAt); + tampered[insertAt] = injected; + System.arraycopy(encodedBytes, insertAt, tampered, insertAt + 1, encodedBytes.length - insertAt); + + StructuredMessageDecoder decoder = new StructuredMessageDecoder(tampered.length); + assertThrows(IllegalArgumentException.class, () -> decoder.decodeChunk(ByteBuffer.wrap(tampered))); + } + + @Test + public void throwsOnRemovedRandomBytes() { + // Remove a random run of bytes from a random offset in the encoded wire. The msgLen field still + // declares the original size, so any deletion must be rejected by validation. + byte[] data = getRandomData(256); + byte[] encodedBytes = encodeToBytes(data, 128); + + int removeCount = 1 + ThreadLocalRandom.current().nextInt(8); + int removeAt = ThreadLocalRandom.current().nextInt(encodedBytes.length - removeCount); + + byte[] tampered = new byte[encodedBytes.length - removeCount]; + System.arraycopy(encodedBytes, 0, tampered, 0, removeAt); + System.arraycopy(encodedBytes, removeAt + removeCount, tampered, removeAt, + encodedBytes.length - removeAt - removeCount); + + StructuredMessageDecoder decoder = new StructuredMessageDecoder(tampered.length); + assertThrows(IllegalArgumentException.class, () -> decoder.decodeChunk(ByteBuffer.wrap(tampered))); + } + + /** + * Multi-segment round-trip with CRC enabled. Exercises the per-segment CRC validation followed by the + * message-wide CRC concat fold: if the concat math is wrong the trailing message footer check would fail + * even though every individual segment CRC matched, so this test directly guards the concat optimization. + */ + @Test + public void multipleSegmentsRoundTripWithCrc() { + // 16 segments of 1 KiB each. Small enough to feed in one chunk; large enough that there is meaningful + // CRC accumulation across segments. + int segmentSize = 1024; + int numSegments = 16; + byte[] originalData = getRandomData(segmentSize * numSegments); + ByteBuffer encoded = encodeToByteBuffer(originalData, segmentSize, StructuredMessageFlags.STORAGE_CRC64); + + StructuredMessageDecoder decoder = new StructuredMessageDecoder(encoded.remaining()); + List decodedPayload = decoder.decodeChunk(encoded); + + assertTrue(decoder.isComplete()); + assertFalse(decodedPayload.isEmpty()); + assertArrayEquals(originalData, collectDecodedBytes(decodedPayload)); + } + + /** + * Multi-segment round-trip with CRC where the decoder is fed many small chunks rather than the whole encoded + * blob at once. This ensures the per-segment CRC computation is correct when payload bytes for a single + * segment arrive split across many decodeChunk calls (the typical production wire pattern) and that the + * O(1) concat fold at each segment boundary still produces a matching message CRC at the end. + */ + @Test + public void multipleSegmentsRoundTripWithCrcAcrossManyChunks() { + int segmentSize = 4 * 1024; + int numSegments = 8; + byte[] originalData = getRandomData(segmentSize * numSegments); + byte[] encodedBytes = encodeToBytes(originalData, segmentSize); + + StructuredMessageDecoder decoder = new StructuredMessageDecoder(encodedBytes.length); + ByteArrayOutputStream collected = new ByteArrayOutputStream(); + int chunkSize = 137; // deliberately non-power-of-two and smaller than the segment so footers split + + for (int offset = 0; offset < encodedBytes.length; offset += chunkSize) { + int len = Math.min(chunkSize, encodedBytes.length - offset); + ByteBuffer chunk = ByteBuffer.wrap(encodedBytes, offset, len); + writeDecodedPayload(collected, decoder.decodeChunk(chunk)); + } + + assertTrue(decoder.isComplete()); + assertArrayEquals(originalData, collected.toByteArray()); + } + + /** + * Verifies that emitted payload buffers are exact-sized for the copied payload they contain and do not expose + * an oversized backing array. In this single-segment scenario the decoder emits one buffer whose backing + * array length matches the payload length, guarding against accidental reintroduction of intermediate + * buffering (for example, a growing {@code ByteArrayOutputStream} that hands off a larger-than-needed + * buffer). + */ + @Test + public void singleSegmentEmissionIsExactSized() { + int segmentSize = 2 * 1024; + byte[] originalData = getRandomData(segmentSize); + ByteBuffer encoded = encodeToByteBuffer(originalData, segmentSize, StructuredMessageFlags.STORAGE_CRC64); + + StructuredMessageDecoder decoder = new StructuredMessageDecoder(encoded.remaining()); + List decoded = decoder.decodeChunk(encoded); + + assertEquals(1, decoded.size()); + assertTrue(decoder.isComplete()); + ByteBuffer payload = decoded.get(0); + assertTrue(payload.hasArray()); + assertEquals(segmentSize, payload.remaining()); + assertEquals(segmentSize, payload.array().length); + assertArrayEquals(originalData, collectDecodedBytes(decoded)); + } + + /** + * Multi-megabyte segment size exercises per-segment length from the wire header, + * not a fixed 4 MiB assumption. + */ + @ParameterizedTest + @MethodSource("segmentPayloadSizeAndTotalPayloadSizeSupplier") + public void decodesMultiMegabyteSegments(int segmentPayloadSize, int totalPayloadSize) { + byte[] originalData = getRandomData(totalPayloadSize); + ByteBuffer encodedData = encodeToByteBuffer(originalData, segmentPayloadSize, StructuredMessageFlags.NONE); + int encodedLength = encodedData.remaining(); + + StructuredMessageDecoder decoder = new StructuredMessageDecoder(encodedLength); + ByteArrayOutputStream output = new ByteArrayOutputStream(); + + writeDecodedPayload(output, decoder.decodeChunk(encodedData)); + + assertTrue(decoder.isComplete()); + assertArrayEquals(originalData, output.toByteArray()); + } + + private static Stream segmentPayloadSizeAndTotalPayloadSizeSupplier() { + return Stream.of(Arguments.of(10 * 1024 * 1024, 10 * 1024 * 1024 + 1), + Arguments.of(3 * 1024 * 1024, 3 * 1024 * 1024 + 1), Arguments.of(5 * 1024 * 1024 + 1, 15 * 1024 * 1024)); + } + + // For tests that pass the whole encoded message to decodeChunk. + private static ByteBuffer encodeToByteBuffer(byte[] originalData, int segmentLength, StructuredMessageFlags flags) { + StructuredMessageEncoder encoder = new StructuredMessageEncoder(originalData.length, segmentLength, flags); + Flux flux = encoder.encode(ByteBuffer.wrap(originalData)); + + return ByteBuffer.wrap(Objects.requireNonNull(FluxUtil.collectBytesInByteBufferStream(flux).block())); + } + + // For tests that need random access/mutation/splitting of encoded bytes. + private static byte[] encodeToBytes(byte[] originalData, int segmentLength) { + ByteBuffer encoded = encodeToByteBuffer(originalData, segmentLength, StructuredMessageFlags.STORAGE_CRC64); + byte[] encodedBytes = new byte[encoded.remaining()]; + encoded.get(encodedBytes); + return encodedBytes; + } + + private static byte[] getRandomData(int size) { + byte[] result = new byte[size]; + ThreadLocalRandom.current().nextBytes(result); + return result; + } + + private static byte[] collectDecodedBytes(List decoded) { + if (decoded.isEmpty()) { + return null; + } + ByteArrayOutputStream output = new ByteArrayOutputStream(); + for (ByteBuffer buffer : decoded) { + if (buffer != null && buffer.hasRemaining()) { + byte[] decodedBytes = new byte[buffer.remaining()]; + buffer.get(decodedBytes); + output.write(decodedBytes, 0, decodedBytes.length); + } + } + return output.toByteArray(); + } + + private static void writeDecodedPayload(ByteArrayOutputStream output, List decoded) { + byte[] bytes = collectDecodedBytes(decoded); + if (bytes != null) { + output.write(bytes, 0, bytes.length); + } + } + +} diff --git a/sdk/storage/azure-storage-common/src/test/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageFlagsTests.java b/sdk/storage/azure-storage-common/src/test/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageFlagsTests.java new file mode 100644 index 000000000000..61da6d3b2b4c --- /dev/null +++ b/sdk/storage/azure-storage-common/src/test/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageFlagsTests.java @@ -0,0 +1,39 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.common.implementation.contentvalidation; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; + +public class StructuredMessageFlagsTests { + @Test + public void testNoneFlag() { + StructuredMessageFlags flag = StructuredMessageFlags.NONE; + assertEquals(0, flag.getValue()); + } + + @Test + public void testStorageCrc64Flag() { + StructuredMessageFlags flag = StructuredMessageFlags.STORAGE_CRC64; + assertEquals(1, flag.getValue()); + } + + @Test + public void testFromStringValid() { + assertEquals(StructuredMessageFlags.NONE, StructuredMessageFlags.fromString("0")); + assertEquals(StructuredMessageFlags.STORAGE_CRC64, StructuredMessageFlags.fromString("1")); + } + + @Test + public void testFromStringInvalid() { + assertNull(StructuredMessageFlags.fromString("2")); + assertNull(StructuredMessageFlags.fromString(null)); + assertThrows(NumberFormatException.class, () -> { + StructuredMessageFlags.fromString("invalid"); + }); + } +} diff --git a/sdk/storage/azure-storage-common/src/test/java/com/azure/storage/common/policy/DecodedResponseTests.java b/sdk/storage/azure-storage-common/src/test/java/com/azure/storage/common/policy/DecodedResponseTests.java new file mode 100644 index 000000000000..e99c169a1763 --- /dev/null +++ b/sdk/storage/azure-storage-common/src/test/java/com/azure/storage/common/policy/DecodedResponseTests.java @@ -0,0 +1,241 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.common.policy; + +import com.azure.core.http.HttpHeaderName; +import com.azure.core.http.HttpHeaders; +import com.azure.core.http.HttpMethod; +import com.azure.core.http.HttpRequest; +import com.azure.core.http.HttpResponse; +import com.azure.core.test.http.MockHttpResponse; +import com.azure.core.util.BinaryData; +import org.junit.jupiter.api.Test; +import reactor.core.publisher.Flux; +import reactor.test.StepVerifier; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.MalformedURLException; +import java.net.URL; +import java.nio.ByteBuffer; +import java.nio.channels.Channels; +import java.nio.channels.WritableByteChannel; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; + +public class DecodedResponseTests { + + private static final HttpHeaderName CUSTOM_HEADER = HttpHeaderName.fromString("x-ms-custom"); + + private static HttpRequest newRequest() { + try { + return new HttpRequest(HttpMethod.GET, new URL("http://example.com/")); + } catch (MalformedURLException e) { + throw new RuntimeException(e); + } + } + + private static HttpHeaders headers(HttpHeaderName name, String value) { + return new HttpHeaders().set(name, value); + } + + private static MockHttpResponse mockResponse(int status, HttpHeaders headers, byte[] body) { + return new MockHttpResponse(newRequest(), status, headers, body); + } + + private static byte[] bytes(String s) { + return s.getBytes(StandardCharsets.UTF_8); + } + + private static Flux fluxOf(byte[] data) { + return Flux.just(ByteBuffer.wrap(data)); + } + + @Test + public void preservesRequestStatusCodeAndHeaders() { + HttpHeaders h = new HttpHeaders().set(HttpHeaderName.CONTENT_LENGTH, "100").set(CUSTOM_HEADER, "value"); + MockHttpResponse original = mockResponse(206, h, bytes("encoded")); + + DecodedResponse wrapper = new DecodedResponse(original, fluxOf(bytes("decoded")), 80L); + + assertSame(original.getRequest(), wrapper.getRequest()); + assertEquals(206, wrapper.getStatusCode()); + // Content-Length is overridden to decoded size; other headers are preserved. + assertEquals("80", wrapper.getHeaders().getValue(HttpHeaderName.CONTENT_LENGTH)); + assertEquals("value", wrapper.getHeaders().getValue(CUSTOM_HEADER)); + } + + @Test + public void getHeaderValueByStringReturnsHeaderValue() { + HttpHeaders h = headers(CUSTOM_HEADER, "value"); + DecodedResponse wrapper = new DecodedResponse(mockResponse(200, h, new byte[0]), fluxOf(new byte[0]), 0L); + + assertEquals("value", wrapper.getHeaderValue(CUSTOM_HEADER.getCaseInsensitiveName())); + assertNull(wrapper.getHeaderValue("nonexistent")); + } + + @Test + public void getBodyReturnsDecodedFlux() { + byte[] decoded = bytes("decoded body"); + DecodedResponse wrapper + = new DecodedResponse(mockResponse(200, new HttpHeaders(), bytes("encoded")), fluxOf(decoded), 0L); + + StepVerifier.create(wrapper.getBody().reduce(new ByteArrayOutputStream(), (sink, buf) -> { + byte[] copy = new byte[buf.remaining()]; + buf.get(copy); + sink.write(copy, 0, copy.length); + return sink; + })).expectNextMatches(sink -> Arrays.equals(decoded, sink.toByteArray())).verifyComplete(); + } + + @Test + public void getBodyAsByteArrayReturnsDecodedBytes() { + byte[] decoded = bytes("decoded body"); + DecodedResponse wrapper + = new DecodedResponse(mockResponse(200, new HttpHeaders(), bytes("encoded")), fluxOf(decoded), 0L); + + StepVerifier.create(wrapper.getBodyAsByteArray()) + .expectNextMatches(b -> Arrays.equals(decoded, b)) + .verifyComplete(); + } + + @Test + public void getBodyAsStringDefaultsToUtf8WhenNoCharsetSpecified() { + // The no-arg overload routes through CoreUtils.bomAwareToString, which falls back to UTF-8 when neither a + // BOM nor a Content-Type charset parameter is present. This test pins the "no headers, no BOM" path. + String text = "héllo wörld – ✓"; + DecodedResponse wrapper = new DecodedResponse(mockResponse(200, new HttpHeaders(), new byte[0]), + fluxOf(text.getBytes(StandardCharsets.UTF_8)), 0L); + + StepVerifier.create(wrapper.getBodyAsString()).expectNext(text).verifyComplete(); + } + + @Test + public void getBodyAsStringHonorsCharsetFromContentTypeHeader() { + // Per the base HttpResponse contract, the no-arg getBodyAsString() must honor a charset declared in the + // response's Content-Type header. Without the bom-aware decoding the bytes would be (mis)interpreted as + // UTF-8 and the assertion below would fail. + String text = "ümlaut"; + byte[] iso = text.getBytes(StandardCharsets.ISO_8859_1); + HttpHeaders h = new HttpHeaders().set(HttpHeaderName.CONTENT_TYPE, "text/plain; charset=ISO-8859-1"); + DecodedResponse wrapper = new DecodedResponse(mockResponse(200, h, new byte[0]), fluxOf(iso), 0L); + + StepVerifier.create(wrapper.getBodyAsString()).expectNext(text).verifyComplete(); + } + + @Test + public void getBodyAsStringDetectsUtf8BomAndStripsIt() { + // A leading UTF-8 BOM (EF BB BF) must be detected and stripped from the decoded string, matching the base + // HttpResponse contract that CoreUtils.bomAwareToString implements. + String text = "with bom"; + byte[] bom = new byte[] { (byte) 0xEF, (byte) 0xBB, (byte) 0xBF }; + byte[] payload = text.getBytes(StandardCharsets.UTF_8); + byte[] withBom = new byte[bom.length + payload.length]; + System.arraycopy(bom, 0, withBom, 0, bom.length); + System.arraycopy(payload, 0, withBom, bom.length, payload.length); + DecodedResponse wrapper + = new DecodedResponse(mockResponse(200, new HttpHeaders(), new byte[0]), fluxOf(withBom), 0L); + + StepVerifier.create(wrapper.getBodyAsString()).expectNext(text).verifyComplete(); + } + + @Test + public void getBodyAsStringDecodesUsingProvidedCharset() { + String text = "ümlaut"; + byte[] latin1 = text.getBytes(StandardCharsets.ISO_8859_1); + DecodedResponse wrapper + = new DecodedResponse(mockResponse(200, new HttpHeaders(), new byte[0]), fluxOf(latin1), 0L); + + StepVerifier.create(wrapper.getBodyAsString(StandardCharsets.ISO_8859_1)).expectNext(text).verifyComplete(); + } + + @Test + public void inheritedGetBodyAsInputStreamUsesDecodedBytes() throws IOException { + // Base getBodyAsInputStream() routes through getBodyAsByteArray(), so the override is exercised end-to-end. + byte[] decoded = bytes("decoded stream"); + DecodedResponse wrapper + = new DecodedResponse(mockResponse(200, new HttpHeaders(), bytes("encoded")), fluxOf(decoded), 0L); + + try (InputStream stream = wrapper.getBodyAsInputStream().block()) { + assertNotNull(stream); + assertArrayEquals(decoded, readAll(stream)); + } + } + + @Test + public void inheritedWriteBodyToWritesDecodedBytes() throws IOException { + byte[] decoded = bytes("write me"); + DecodedResponse wrapper + = new DecodedResponse(mockResponse(200, new HttpHeaders(), bytes("encoded")), fluxOf(decoded), 0L); + + ByteArrayOutputStream sink = new ByteArrayOutputStream(); + try (WritableByteChannel channel = Channels.newChannel(sink)) { + wrapper.writeBodyTo(channel); + } + + assertArrayEquals(decoded, sink.toByteArray()); + } + + @Test + public void inheritedBufferReturnsResponseBackedByDecodedBytes() { + byte[] decoded = bytes("buffered"); + DecodedResponse wrapper + = new DecodedResponse(mockResponse(200, new HttpHeaders(), bytes("encoded")), fluxOf(decoded), 0L); + + HttpResponse buffered = wrapper.buffer(); + assertNotNull(buffered); + StepVerifier.create(buffered.getBodyAsByteArray()) + .expectNextMatches(b -> Arrays.equals(decoded, b)) + .verifyComplete(); + } + + @Test + public void inheritedGetBodyAsBinaryDataReturnsDecodedBytes() { + // Base HttpResponse#getBodyAsBinaryData() pulls from getBody() (our override), so the resulting BinaryData + // must contain the decoded payload, not the original wire body. A divergent Content-Length header is set + // to make the wire vs decoded distinction explicit and guard against regressions in header forwarding. + byte[] decoded = bytes("decoded payload"); + long decodedSize = decoded.length; + HttpHeaders h = new HttpHeaders().set(HttpHeaderName.CONTENT_LENGTH, String.valueOf(decodedSize + 32)); + DecodedResponse wrapper + = new DecodedResponse(mockResponse(200, h, bytes("encoded wire body")), fluxOf(decoded), decodedSize); + + BinaryData data = wrapper.getBodyAsBinaryData(); + assertNotNull(data); + assertArrayEquals(decoded, data.toBytes()); + } + + @Test + public void contentLengthIsOverriddenToDecodedSize() { + long wireSize = 500L; + long decodedSize = 300L; + HttpHeaders h = new HttpHeaders().set(HttpHeaderName.CONTENT_LENGTH, String.valueOf(wireSize)) + .set(CUSTOM_HEADER, "preserve-me"); + DecodedResponse wrapper + = new DecodedResponse(mockResponse(200, h, new byte[0]), fluxOf(new byte[0]), decodedSize); + + assertEquals(String.valueOf(decodedSize), wrapper.getHeaders().getValue(HttpHeaderName.CONTENT_LENGTH)); + assertEquals("preserve-me", wrapper.getHeaders().getValue(CUSTOM_HEADER)); + // Deprecated getHeaderValue must reflect the same override. + assertEquals(String.valueOf(decodedSize), + wrapper.getHeaderValue(HttpHeaderName.CONTENT_LENGTH.getCaseInsensitiveName())); + } + + private static byte[] readAll(InputStream stream) throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + byte[] buf = new byte[1024]; + int n; + while ((n = stream.read(buf)) != -1) { + out.write(buf, 0, n); + } + return out.toByteArray(); + } +} diff --git a/sdk/storage/azure-storage-common/src/test/java/com/azure/storage/common/policy/StorageContentValidationDecoderPolicyTests.java b/sdk/storage/azure-storage-common/src/test/java/com/azure/storage/common/policy/StorageContentValidationDecoderPolicyTests.java new file mode 100644 index 000000000000..6b0b8474ee5c --- /dev/null +++ b/sdk/storage/azure-storage-common/src/test/java/com/azure/storage/common/policy/StorageContentValidationDecoderPolicyTests.java @@ -0,0 +1,183 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.common.policy; + +import com.azure.core.http.HttpClient; +import com.azure.core.http.HttpHeaderName; +import com.azure.core.http.HttpHeaders; +import com.azure.core.http.HttpMethod; +import com.azure.core.http.HttpPipeline; +import com.azure.core.http.HttpPipelineBuilder; +import com.azure.core.http.HttpRequest; +import com.azure.core.http.HttpResponse; +import com.azure.core.test.http.MockHttpResponse; +import com.azure.core.util.Context; +import com.azure.core.util.FluxUtil; +import com.azure.storage.common.implementation.Constants; +import com.azure.storage.common.implementation.contentvalidation.StructuredMessageConstants; +import com.azure.storage.common.implementation.contentvalidation.StructuredMessageEncoder; +import com.azure.storage.common.implementation.contentvalidation.StructuredMessageFlags; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Objects; +import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.Stream; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Tests {@link StorageContentValidationDecoderPolicy} together with {@link StructuredMessageEncoder} / + * wire-format payloads so the reactive decode path matches what the blob download pipeline uses. + */ +public class StorageContentValidationDecoderPolicyTests { + + /** + * End-to-end through the policy: encoded body uses multi-megabyte segment payload lengths (not the default + * 4 MiB framing only); decoded flux must match the original bytes. + */ + @ParameterizedTest + @MethodSource("segmentPayloadSizeAndTotalPayloadSizeSupplier") + public void decodesDynamicallySizedSegmentStructuredMessageThroughPipeline(int segmentPayloadSize, + int totalPayloadSize) throws IOException { + byte[] originalData = new byte[totalPayloadSize]; + ThreadLocalRandom.current().nextBytes(originalData); + + byte[] encodedBytes + = encodeStructuredMessageWireBytes(originalData, segmentPayloadSize, StructuredMessageFlags.STORAGE_CRC64); + + AtomicReference requestAfterPolicies = new AtomicReference<>(); + HttpClient httpClient = request -> { + requestAfterPolicies.set(request); + HttpHeaders headers = structuredDownloadResponseHeaders(encodedBytes.length, totalPayloadSize); + return Mono.just(new MockHttpResponse(request, 200, headers, encodedBytes)); + }; + + HttpPipeline pipeline = new HttpPipelineBuilder().policies((context, next) -> { + context.setData(StructuredMessageConstants.STRUCTURED_MESSAGE_DECODING_CONTEXT_KEY, true); + return next.process(); + }, new StorageContentValidationDecoderPolicy()).httpClient(httpClient).build(); + + HttpRequest request = new HttpRequest(HttpMethod.GET, "https://example.blob.core.windows.net/c/b"); + try (HttpResponse response = pipeline.send(request, Context.NONE).block()) { + assertNotNull(response); + assertTrue(response instanceof DecodedResponse); + byte[] decoded = Objects.requireNonNull(response.getBodyAsByteArray().block()); + assertArrayEquals(originalData, decoded); + } + + HttpRequest sent = requestAfterPolicies.get(); + assertNotNull(sent); + assertEquals(StructuredMessageConstants.STRUCTURED_BODY_TYPE_VALUE, + sent.getHeaders().getValue(Constants.HeaderConstants.STRUCTURED_BODY_TYPE_HEADER_NAME)); + } + + @Test + public void contentLengthIsOverriddenToDecodedSizeWhenDecodingApplied() throws IOException { + byte[] payload = new byte[64]; + ThreadLocalRandom.current().nextBytes(payload); + + byte[] encoded = encodeStructuredMessageWireBytes(payload, 64, StructuredMessageFlags.STORAGE_CRC64); + long encodedLen = encoded.length; + long decodedLen = payload.length; + + HttpClient httpClient = request -> Mono.just( + new MockHttpResponse(request, 200, structuredDownloadResponseHeaders(encodedLen, decodedLen), encoded)); + + HttpPipeline pipeline = new HttpPipelineBuilder().policies((context, next) -> { + context.setData(StructuredMessageConstants.STRUCTURED_MESSAGE_DECODING_CONTEXT_KEY, true); + return next.process(); + }, new StorageContentValidationDecoderPolicy()).httpClient(httpClient).build(); + + HttpRequest request = new HttpRequest(HttpMethod.GET, "https://example.blob.core.windows.net/c/b"); + try (HttpResponse response = pipeline.send(request, Context.NONE).block()) { + assertNotNull(response); + assertEquals(String.valueOf(decodedLen), response.getHeaders().getValue(HttpHeaderName.CONTENT_LENGTH)); + } + } + + @Test + public void contentLengthMatchesActualDecodedBodySize() throws IOException { + byte[] payload = new byte[128]; + ThreadLocalRandom.current().nextBytes(payload); + + byte[] encoded = encodeStructuredMessageWireBytes(payload, 64, StructuredMessageFlags.STORAGE_CRC64); + long encodedLen = encoded.length; + long decodedLen = payload.length; + + HttpClient httpClient = request -> Mono.just( + new MockHttpResponse(request, 200, structuredDownloadResponseHeaders(encodedLen, decodedLen), encoded)); + + HttpPipeline pipeline = new HttpPipelineBuilder().policies((context, next) -> { + context.setData(StructuredMessageConstants.STRUCTURED_MESSAGE_DECODING_CONTEXT_KEY, true); + return next.process(); + }, new StorageContentValidationDecoderPolicy()).httpClient(httpClient).build(); + + HttpRequest request = new HttpRequest(HttpMethod.GET, "https://example.blob.core.windows.net/c/b"); + try (HttpResponse response = pipeline.send(request, Context.NONE).block()) { + assertNotNull(response); + assertEquals(String.valueOf(decodedLen), response.getHeaders().getValue(HttpHeaderName.CONTENT_LENGTH)); + + byte[] body = Objects.requireNonNull(FluxUtil.collectBytesInByteBufferStream(response.getBody()).block()); + assertEquals(decodedLen, body.length); + assertArrayEquals(payload, body); + } + } + + @Test + public void contentLengthIsUnchangedWhenDecodingNotApplied() throws IOException { + byte[] payload = new byte[64]; + ThreadLocalRandom.current().nextBytes(payload); + + byte[] encoded = encodeStructuredMessageWireBytes(payload, 64, StructuredMessageFlags.STORAGE_CRC64); + long encodedLen = encoded.length; + + HttpHeaders responseHeaders = new HttpHeaders().set(HttpHeaderName.CONTENT_LENGTH, String.valueOf(encodedLen)); + HttpClient httpClient = request -> Mono.just(new MockHttpResponse(request, 200, responseHeaders, encoded)); + + HttpPipeline pipeline = new HttpPipelineBuilder().policies(new StorageContentValidationDecoderPolicy()) + .httpClient(httpClient) + .build(); + + HttpRequest request = new HttpRequest(HttpMethod.GET, "https://example.blob.core.windows.net/c/b"); + HttpResponse response = pipeline.send(request, Context.NONE).block(); + + assertNotNull(response); + assertEquals(String.valueOf(encodedLen), response.getHeaders().getValue(HttpHeaderName.CONTENT_LENGTH)); + } + + private static Stream segmentPayloadSizeAndTotalPayloadSizeSupplier() { + return Stream.of(Arguments.of(10 * 1024 * 1024, 10 * 1024 * 1024 + 1), // larger than 4 MiB + Arguments.of(3 * 1024 * 1024, 3 * 1024 * 1024 + 1), // smaller than 4 MiB, but not KB + Arguments.of(5 * 1024 * 1024 + 1, 15 * 1024 * 1024)); + } + + private static HttpHeaders structuredDownloadResponseHeaders(long contentLength, long structuredContentLength) { + HttpHeaders headers = new HttpHeaders(); + headers.set(HttpHeaderName.CONTENT_LENGTH, String.valueOf(contentLength)); + headers.set(Constants.HeaderConstants.STRUCTURED_BODY_TYPE_HEADER_NAME, + StructuredMessageConstants.STRUCTURED_BODY_TYPE_VALUE); + headers.set(Constants.HeaderConstants.STRUCTURED_CONTENT_LENGTH_HEADER_NAME, + String.valueOf(structuredContentLength)); + return headers; + } + + private static byte[] encodeStructuredMessageWireBytes(byte[] originalData, int segmentLength, + StructuredMessageFlags flags) throws IOException { + StructuredMessageEncoder encoder = new StructuredMessageEncoder(originalData.length, segmentLength, flags); + Flux flux = encoder.encode(ByteBuffer.wrap(originalData)); + return Objects.requireNonNull(FluxUtil.collectBytesInByteBufferStream(flux).block()); + } +} From ed84ccac075dcd786d5052c1acdbcda90fc4ef67 Mon Sep 17 00:00:00 2001 From: Isabelle <141270045+ibrandes@users.noreply.github.com> Date: Wed, 24 Jun 2026 08:27:24 -0700 Subject: [PATCH 5/5] Storage - [Content Validation] Make synchronous stageBlock request body replayable on retry (#49612) * Make synchronous stageBlock request body replayable on retry * updating assets (unrelated to the changes in this PR) * updating recordings --- sdk/storage/azure-storage-blob/assets.json | 2 +- .../storage/blob/specialized/BlockBlobClient.java | 14 ++++++++++---- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/sdk/storage/azure-storage-blob/assets.json b/sdk/storage/azure-storage-blob/assets.json index 85028bbf87f6..769ef2797967 100644 --- a/sdk/storage/azure-storage-blob/assets.json +++ b/sdk/storage/azure-storage-blob/assets.json @@ -2,5 +2,5 @@ "AssetsRepo": "Azure/azure-sdk-assets", "AssetsRepoPrefixPath": "java", "TagPrefix": "java/storage/azure-storage-blob", - "Tag": "java/storage/azure-storage-blob_afc0d343f3" + "Tag": "java/storage/azure-storage-blob_494b72fca7" } diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlockBlobClient.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlockBlobClient.java index a9a68ada5e71..2ab1287cbe63 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlockBlobClient.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlockBlobClient.java @@ -38,14 +38,17 @@ import com.azure.storage.blob.options.BlockBlobSimpleUploadOptions; import com.azure.storage.blob.options.BlockBlobStageBlockFromUrlOptions; import com.azure.storage.blob.options.BlockBlobStageBlockOptions; +import com.azure.storage.common.Utility; import com.azure.storage.common.implementation.Constants; import com.azure.storage.common.implementation.StorageImplUtils; import com.azure.storage.common.implementation.StorageSeekableByteChannel; +import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; import java.io.InputStream; import java.io.UncheckedIOException; import java.net.URL; +import java.nio.ByteBuffer; import java.nio.channels.SeekableByteChannel; import java.time.Duration; import java.util.List; @@ -783,10 +786,13 @@ public Response stageBlockWithResponse(String base64BlockId, InputStream d String leaseId, Duration timeout, Context context) { StorageImplUtils.assertNotNull("data", data); - Mono> response = client.stageBlockWithResponseInternal( - new BlockBlobStageBlockOptions(base64BlockId, BinaryData.fromStream(data, length)).setContentMd5(contentMd5) - .setLeaseId(leaseId), - context); + Flux fbb + = Utility.convertStreamToByteBuffer(data, length, BlobAsyncClient.BLOB_DEFAULT_UPLOAD_BLOCK_SIZE, true); + + Mono> response = BinaryData.fromFlux(fbb, length, false) + .flatMap(binaryData -> client.stageBlockWithResponseInternal( + new BlockBlobStageBlockOptions(base64BlockId, binaryData).setContentMd5(contentMd5).setLeaseId(leaseId), + context)); return blockWithOptionalTimeout(response, timeout); }