diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java index 3db1565c7057e..f570f82c5be18 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java @@ -438,6 +438,10 @@ public class AbfsConfiguration{ FS_AZURE_ABFS_ENABLE_CHECKSUM_VALIDATION, DefaultValue = DEFAULT_ENABLE_ABFS_CHECKSUM_VALIDATION) private boolean isChecksumValidationEnabled; + @BooleanConfigurationValidatorAnnotation(ConfigurationKey = + FS_AZURE_ENABLE_FULL_BLOB_CHECKSUM_VALIDATION, DefaultValue = DEFAULT_ENABLE_FULL_BLOB_ABFS_CHECKSUM_VALIDATION) + private boolean isFullBlobChecksumValidationEnabled; + @BooleanConfigurationValidatorAnnotation(ConfigurationKey = FS_AZURE_ENABLE_PAGINATED_DELETE, DefaultValue = DEFAULT_ENABLE_PAGINATED_DELETE) private boolean isPaginatedDeleteEnabled; @@ -1705,6 +1709,10 @@ public void setIsChecksumValidationEnabled(boolean isChecksumValidationEnabled) this.isChecksumValidationEnabled = isChecksumValidationEnabled; } + public boolean isFullBlobChecksumValidationEnabled() { + return isFullBlobChecksumValidationEnabled; + } + public long getBlobCopyProgressPollWaitMillis() { return blobCopyProgressPollWaitMillis; } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java index 50a88ab4e4587..ef39fb11b2d19 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java @@ -356,6 +356,9 @@ public final class ConfigurationKeys { /** Add extra layer of verification of the integrity of the request content during transport: {@value}. */ public static final String FS_AZURE_ABFS_ENABLE_CHECKSUM_VALIDATION = "fs.azure.enable.checksum.validation"; + /** Add extra layer of verification of the integrity of the full blob request content during transport: {@value}. */ + public static final String FS_AZURE_ENABLE_FULL_BLOB_CHECKSUM_VALIDATION = "fs.azure.enable.full.blob.checksum.validation"; + public static String accountProperty(String property, String account) { return property + DOT + account; } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java index 8bcd55aee8e35..d53e936fd5c71 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java @@ -147,6 +147,7 @@ public final class FileSystemConfigurations { public static final boolean DEFAULT_ENABLE_ABFS_RENAME_RESILIENCE = true; public static final boolean DEFAULT_ENABLE_PAGINATED_DELETE = false; public static final boolean DEFAULT_ENABLE_ABFS_CHECKSUM_VALIDATION = false; + public static final boolean DEFAULT_ENABLE_FULL_BLOB_ABFS_CHECKSUM_VALIDATION = false; /** * Limit of queued block upload operations before writes diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsBlobClient.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsBlobClient.java index e9fbfe49dee46..bb46a97835fec 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsBlobClient.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsBlobClient.java @@ -1076,9 +1076,10 @@ public AbfsRestOperation flush(byte[] buffer, if (leaseId != null) { requestHeaders.add(new AbfsHttpHeader(X_MS_LEASE_ID, leaseId)); } - if (blobMd5 != null) { - requestHeaders.add(new AbfsHttpHeader(X_MS_BLOB_CONTENT_MD5, blobMd5)); - } + String md5Value = (isFullBlobChecksumValidationEnabled() && blobMd5 != null) + ? blobMd5 + : computeMD5Hash(buffer, 0, buffer.length); + requestHeaders.add(new AbfsHttpHeader(X_MS_BLOB_CONTENT_MD5, md5Value)); final AbfsUriQueryBuilder abfsUriQueryBuilder = createDefaultUriQueryBuilder(); abfsUriQueryBuilder.addQuery(QUERY_PARAM_COMP, BLOCKLIST); abfsUriQueryBuilder.addQuery(QUERY_PARAM_CLOSE, String.valueOf(isClose)); @@ -1103,7 +1104,12 @@ public AbfsRestOperation flush(byte[] buffer, AbfsRestOperation op1 = getPathStatus(path, true, tracingContext, contextEncryptionAdapter); String metadataMd5 = op1.getResult().getResponseHeader(CONTENT_MD5); - if (blobMd5 != null && !blobMd5.equals(metadataMd5)) { + /* + * Validate the response by comparing the server's MD5 metadata against either: + * 1. The full blob content MD5 (if full blob checksum validation is enabled), or + * 2. The full block ID list buffer MD5 (fallback if blob checksum validation is disabled) + */ + if (md5Value != null && !md5Value.equals(metadataMd5)) { throw ex; } return op; diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java index faf6bf8aadc8f..107558bb6e421 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java @@ -1415,7 +1415,7 @@ protected boolean isChecksumValidationEnabled(List requestHeader /** * Conditions check for allowing checksum support for write operation. * Server will support this if client sends the MD5 Hash as a request header. - * For azure stoage service documentation and more details refer to + * For azure storage service documentation and more details refer to * Path - Update Azure Rest API. * @return true if checksum validation enabled. */ @@ -1423,6 +1423,17 @@ protected boolean isChecksumValidationEnabled() { return getAbfsConfiguration().getIsChecksumValidationEnabled(); } + /** + * Conditions check for allowing checksum support for write operation. + * Server will support this if client sends the MD5 Hash as a request header. + * For azure storage service documentation and more details refer to + * Path - Update Azure Rest API. + * @return true if full blob checksum validation enabled. + */ + protected boolean isFullBlobChecksumValidationEnabled() { + return getAbfsConfiguration().isFullBlobChecksumValidationEnabled(); + } + /** * Compute MD5Hash of the given byte array starting from given offset up to given length. * @param data byte array from which data is fetched to compute MD5 Hash. diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsDfsClient.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsDfsClient.java index 197744e79fe6c..f88df14c3423a 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsDfsClient.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsDfsClient.java @@ -867,10 +867,9 @@ public AbfsRestOperation flush(final String path, if (leaseId != null) { requestHeaders.add(new AbfsHttpHeader(X_MS_LEASE_ID, leaseId)); } - if (isChecksumValidationEnabled() && blobMd5 != null) { + if (isFullBlobChecksumValidationEnabled() && blobMd5 != null) { requestHeaders.add(new AbfsHttpHeader(X_MS_BLOB_CONTENT_MD5, blobMd5)); } - final AbfsUriQueryBuilder abfsUriQueryBuilder = createDefaultUriQueryBuilder(); abfsUriQueryBuilder.addQuery(QUERY_PARAM_ACTION, FLUSH_ACTION); abfsUriQueryBuilder.addQuery(QUERY_PARAM_POSITION, Long.toString(position)); diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsOutputStream.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsOutputStream.java index 10e7d9910c884..3abf21c778793 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsOutputStream.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsOutputStream.java @@ -223,7 +223,7 @@ public AbfsOutputStream(AbfsOutputStreamContext abfsOutputStreamContext) md5 = MessageDigest.getInstance(MD5); fullBlobContentMd5 = MessageDigest.getInstance(MD5); } catch (NoSuchAlgorithmException e) { - if (client.isChecksumValidationEnabled()) { + if (isChecksumValidationEnabled()) { throw new IOException("MD5 algorithm not available", e); } } @@ -464,10 +464,13 @@ public synchronized void write(final byte[] data, final int off, final int lengt AbfsBlock block = createBlockIfNeeded(position); int written = bufferData(block, data, off, length); // Update the incremental MD5 hash with the written data. - getMessageDigest().update(data, off, written); - + if (isChecksumValidationEnabled()) { + getMessageDigest().update(data, off, written); + } // Update the full blob MD5 hash with the written data. - getFullBlobContentMd5().update(data, off, written); + if (isFullBlobChecksumValidationEnabled()) { + getFullBlobContentMd5().update(data, off, written); + } int remainingCapacity = block.remainingCapacity(); if (written < length) { @@ -544,7 +547,7 @@ private void uploadBlockAsync(AbfsBlock blockToUpload, outputStreamStatistics.bytesToUpload(bytesLength); outputStreamStatistics.writeCurrentBuffer(); DataBlocks.BlockUploadData blockUploadData = blockToUpload.startUpload(); - String md5Hash = getMd5(); + String md5Hash = getClient().isChecksumValidationEnabled() ? getMd5() : null; final Future job = executorService.submit(() -> { AbfsPerfTracker tracker = @@ -1222,6 +1225,20 @@ public MessageDigest getFullBlobContentMd5() { return fullBlobContentMd5; } + /** + * @return true if checksum validation is enabled. + */ + public boolean isChecksumValidationEnabled() { + return getClient().isChecksumValidationEnabled(); + } + + /** + * @return true if full blob checksum validation is enabled. + */ + public boolean isFullBlobChecksumValidationEnabled() { + return getClient().isFullBlobChecksumValidationEnabled(); + } + /** * Returns the Base64-encoded MD5 checksum based on the current digest state. * This finalizes the digest calculation. Returns null if the digest is empty. diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AzureBlobBlockManager.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AzureBlobBlockManager.java index 4e3e20f00246c..befcf26f37a73 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AzureBlobBlockManager.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AzureBlobBlockManager.java @@ -95,7 +95,9 @@ protected synchronized AbfsBlock createBlockInternal(long position) setBlockCount(getBlockCount() + 1); AbfsBlock activeBlock = new AbfsBlobBlock(getAbfsOutputStream(), position, getBlockIdLength(), getBlockCount()); activeBlock.setBlockEntry(addNewEntry(activeBlock.getBlockId(), activeBlock.getOffset())); - getAbfsOutputStream().getMessageDigest().reset(); + if (getAbfsOutputStream().isChecksumValidationEnabled()) { + getAbfsOutputStream().getMessageDigest().reset(); + } setActiveBlock(activeBlock); } return getActiveBlock(); diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AzureBlobIngressHandler.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AzureBlobIngressHandler.java index eaa4ca41c781e..8610adde30b26 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AzureBlobIngressHandler.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AzureBlobIngressHandler.java @@ -180,7 +180,10 @@ protected synchronized AbfsRestOperation remoteFlush(final long offset, tracingContextFlush.setIngressHandler(BLOB_FLUSH); tracingContextFlush.setPosition(String.valueOf(offset)); LOG.trace("Flushing data at offset {} for path {}", offset, getAbfsOutputStream().getPath()); - String fullBlobMd5 = computeFullBlobMd5(); + String fullBlobMd5 = null; + if (getClient().isFullBlobChecksumValidationEnabled()) { + fullBlobMd5 = computeFullBlobMd5(); + } op = getClient().flush(blockListXml.getBytes(StandardCharsets.UTF_8), getAbfsOutputStream().getPath(), isClose, getAbfsOutputStream().getCachedSasTokenString(), leaseId, @@ -194,7 +197,9 @@ isClose, getAbfsOutputStream().getCachedSasTokenString(), leaseId, LOG.error("Error in remote flush for path {} and offset {}", getAbfsOutputStream().getPath(), offset, ex); throw ex; } finally { - getAbfsOutputStream().getFullBlobContentMd5().reset(); + if (getClient().isFullBlobChecksumValidationEnabled()) { + getAbfsOutputStream().getFullBlobContentMd5().reset(); + } } return op; } @@ -221,7 +226,7 @@ protected AbfsRestOperation remoteAppendBlobWrite(String path, AppendRequestParameters reqParams, TracingContext tracingContext) throws IOException { // Perform the remote append operation using the blob client. - AbfsRestOperation op = null; + AbfsRestOperation op; try { op = blobClient.appendBlock(path, reqParams, uploadData.toByteArray(), tracingContext); } catch (AbfsRestOperationException ex) { diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AzureDFSBlockManager.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AzureDFSBlockManager.java index c58f1d1b28461..dc0104a9ae223 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AzureDFSBlockManager.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AzureDFSBlockManager.java @@ -62,7 +62,9 @@ protected synchronized AbfsBlock createBlockInternal(long position) if (getActiveBlock() == null) { setBlockCount(getBlockCount() + 1); AbfsBlock activeBlock = new AbfsBlock(getAbfsOutputStream(), position); - getAbfsOutputStream().getMessageDigest().reset(); + if (getAbfsOutputStream().isChecksumValidationEnabled()) { + getAbfsOutputStream().getMessageDigest().reset(); + } setActiveBlock(activeBlock); } return getActiveBlock(); diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AzureDFSIngressHandler.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AzureDFSIngressHandler.java index faf293b9242ce..96a8f07cc3d50 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AzureDFSIngressHandler.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AzureDFSIngressHandler.java @@ -179,7 +179,10 @@ protected synchronized AbfsRestOperation remoteFlush(final long offset, tracingContextFlush.setIngressHandler(DFS_FLUSH); tracingContextFlush.setPosition(String.valueOf(offset)); } - String fullBlobMd5 = computeFullBlobMd5(); + String fullBlobMd5 = null; + if (getClient().isFullBlobChecksumValidationEnabled()) { + fullBlobMd5 = computeFullBlobMd5(); + } LOG.trace("Flushing data at offset {} and path {}", offset, getAbfsOutputStream().getPath()); AbfsRestOperation op; try { @@ -194,7 +197,9 @@ protected synchronized AbfsRestOperation remoteFlush(final long offset, getAbfsOutputStream().getPath(), offset, ex); throw ex; } finally { - getAbfsOutputStream().getFullBlobContentMd5().reset(); + if (getClient().isFullBlobChecksumValidationEnabled()) { + getAbfsOutputStream().getFullBlobContentMd5().reset(); + } } return op; } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/RenameAtomicity.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/RenameAtomicity.java index 24f97238bde36..43157c6d730a4 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/RenameAtomicity.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/RenameAtomicity.java @@ -211,13 +211,18 @@ void createRenamePendingJson(Path path, byte[] bytes) String blockId = generateBlockId(); String blockList = generateBlockListXml(blockId); byte[] buffer = blockList.getBytes(StandardCharsets.UTF_8); - String computedMd5 = abfsClient.computeMD5Hash(buffer, 0, buffer.length); + String computedMd5 = null; + if (abfsClient.isFullBlobChecksumValidationEnabled()) { + computedMd5 = abfsClient.computeMD5Hash(buffer, 0, buffer.length); + } AppendRequestParameters appendRequestParameters = new AppendRequestParameters(0, 0, bytes.length, AppendRequestParameters.Mode.APPEND_MODE, false, null, abfsClient.getAbfsConfiguration().isExpectHeaderEnabled(), - new BlobAppendRequestParameters(blockId, eTag), abfsClient.computeMD5Hash(bytes, 0, bytes.length)); + new BlobAppendRequestParameters(blockId, eTag), + abfsClient.isChecksumValidationEnabled() ? abfsClient.computeMD5Hash( + bytes, 0, bytes.length) : null); abfsClient.append(path.toUri().getPath(), bytes, appendRequestParameters, null, null, tracingContext); diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRename.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRename.java index e063f71e8c2da..1be40c09dbcd7 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRename.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRename.java @@ -2125,7 +2125,7 @@ private void testRenamePreRenameFailureResolution(final AzureBlobFileSystem fs) Mockito.anyBoolean(), Mockito.nullable(String.class), Mockito.nullable(String.class), Mockito.anyString(), Mockito.nullable(ContextEncryptionAdapter.class), - Mockito.any(TracingContext.class), Mockito.anyString()); + Mockito.any(TracingContext.class), Mockito.nullable(String.class)); return createAnswer.callRealMethod(); }; RenameAtomicityTestUtils.addCreatePathMock(client, diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestWasbAbfsCompatibility.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestWasbAbfsCompatibility.java index 66f105f8701f5..ec63c3dcda72f 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestWasbAbfsCompatibility.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestWasbAbfsCompatibility.java @@ -25,7 +25,8 @@ import java.util.UUID; import org.assertj.core.api.Assertions; -import org.junit.jupiter.api.Test; +import org.junit.Assume; +import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -42,6 +43,7 @@ import static java.net.HttpURLConnection.HTTP_CONFLICT; import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ABFS_ENABLE_CHECKSUM_VALIDATION; +import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ENABLE_FULL_BLOB_CHECKSUM_VALIDATION; import static org.apache.hadoop.fs.contract.ContractTestUtils.assertDeleted; import static org.apache.hadoop.fs.contract.ContractTestUtils.assertIsDirectory; import static org.apache.hadoop.fs.contract.ContractTestUtils.assertMkdirs; @@ -69,18 +71,15 @@ public class ITestWasbAbfsCompatibility extends AbstractAbfsIntegrationTest { public ITestWasbAbfsCompatibility() throws Exception { assumeThat(isIPAddress()).as("Emulator is not supported").isFalse(); + assumeHnsDisabled(); + assumeBlobServiceType(); } @Test public void testListFileStatus() throws Exception { // crate file using abfs - AzureBlobFileSystem fs = getFileSystem(); - // test only valid for non-namespace enabled account - assumeThat(getIsNamespaceEnabled(fs)) - .as("Namespace enabled account does not support this test") - .isFalse(); assumeThat(isAppendBlobEnabled()).as("Not valid for APPEND BLOB").isFalse(); - + AzureBlobFileSystem fs = getFileSystem(); NativeAzureFileSystem wasb = getWasbFileSystem(); Path testFiles = path("/testfiles"); @@ -111,42 +110,42 @@ public void testListFileStatus() throws Exception { @Test public void testReadFile() throws Exception { + Assume.assumeFalse("Not valid for APPEND BLOB", isAppendBlobEnabled()); boolean[] createFileWithAbfs = new boolean[]{false, true, false, true}; boolean[] readFileWithAbfs = new boolean[]{false, true, true, false}; - AzureBlobFileSystem abfs = getFileSystem(); - // test only valid for non-namespace enabled account - assumeThat(getIsNamespaceEnabled(abfs)) - .as("Namespace enabled account does not support this test") - .isFalse(); - assumeThat(isAppendBlobEnabled()).as("Not valid for APPEND BLOB").isFalse(); - - NativeAzureFileSystem wasb = getWasbFileSystem(); - - Path testFile = path("/testReadFile"); - for (int i = 0; i < 4; i++) { - Path path = new Path(testFile + "/~12/!008/testfile" + i); - final FileSystem createFs = createFileWithAbfs[i] ? abfs : wasb; - // Read - final FileSystem readFs = readFileWithAbfs[i] ? abfs : wasb; - // Write - try (FSDataOutputStream nativeFsStream = createFs.create(path, true)) { - nativeFsStream.write(TEST_CONTEXT.getBytes()); - nativeFsStream.flush(); - nativeFsStream.hsync(); - } - - // Check file status - ContractTestUtils.assertIsFile(createFs, path); - - try (BufferedReader br = new BufferedReader( - new InputStreamReader(readFs.open(path)))) { - String line = br.readLine(); - assertEquals(TEST_CONTEXT, line, "Wrong text from " + readFs); + Configuration conf = getRawConfiguration(); + conf.setBoolean(FS_AZURE_ENABLE_FULL_BLOB_CHECKSUM_VALIDATION, true); + FileSystem fileSystem = FileSystem.newInstance(conf); + try (AzureBlobFileSystem abfs = (AzureBlobFileSystem) fileSystem) { + NativeAzureFileSystem wasb = getWasbFileSystem(); + + Path testFile = path("/testReadFile"); + for (int i = 0; i < 4; i++) { + Path path = new Path(testFile + "/~12/!008/testfile" + i); + final FileSystem createFs = createFileWithAbfs[i] ? abfs : wasb; + // Read + final FileSystem readFs = readFileWithAbfs[i] ? abfs : wasb; + // Write + try (FSDataOutputStream nativeFsStream = createFs.create(path, true)) { + nativeFsStream.write(TEST_CONTEXT.getBytes()); + nativeFsStream.flush(); + nativeFsStream.hsync(); + } + + // Check file status + ContractTestUtils.assertIsFile(createFs, path); + + try (BufferedReader br = new BufferedReader( + new InputStreamReader(readFs.open(path)))) { + String line = br.readLine(); + assertEquals("Wrong text from " + readFs, + TEST_CONTEXT, line); + } + + // Remove file + assertDeleted(readFs, path, true); } - - // Remove file - assertDeleted(readFs, path, true); } } @@ -156,16 +155,13 @@ public void testReadFile() throws Exception { */ @Test public void testwriteFile() throws Exception { + Assume.assumeFalse("Not valid for APPEND BLOB", isAppendBlobEnabled()); AzureBlobFileSystem abfs = getFileSystem(); - assumeThat(getIsNamespaceEnabled(abfs)) - .as("Namespace enabled account does not support this test") - .isFalse(); - assumeBlobServiceType(); - assumeThat(isAppendBlobEnabled()).as("Not valid for APPEND BLOB").isFalse(); NativeAzureFileSystem wasb = getWasbFileSystem(); Path testFile = path("/testReadFile"); - Path path = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); + Path path = new Path( + testFile + "/~12/!008/testfile_" + UUID.randomUUID()); // Write try (FSDataOutputStream nativeFsStream = wasb.create(path, true)) { nativeFsStream.write(TEST_CONTEXT.getBytes()); @@ -179,7 +175,8 @@ public void testwriteFile() throws Exception { try (BufferedReader br = new BufferedReader( new InputStreamReader(abfs.open(path)))) { String line = br.readLine(); - assertEquals(TEST_CONTEXT, line, "Wrong text from " + abfs); + assertEquals("Wrong text from " + abfs, + TEST_CONTEXT, line); } try (FSDataOutputStream abfsOutputStream = abfs.append(path)) { abfsOutputStream.write(TEST_CONTEXT.getBytes()); @@ -197,16 +194,13 @@ public void testwriteFile() throws Exception { @Test public void testwriteFile1() throws Exception { + Assume.assumeFalse("Not valid for APPEND BLOB", isAppendBlobEnabled()); AzureBlobFileSystem abfs = getFileSystem(); - assumeThat(getIsNamespaceEnabled(abfs)) - .as("Namespace enabled account does not support this test") - .isFalse(); - assumeBlobServiceType(); - assumeThat(isAppendBlobEnabled()).as("Not valid for APPEND BLOB").isFalse(); NativeAzureFileSystem wasb = getWasbFileSystem(); Path testFile = path("/testReadFile"); - Path path = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); + Path path = new Path( + testFile + "/~12/!008/testfile_" + UUID.randomUUID()); // Write try (FSDataOutputStream nativeFsStream = abfs.create(path, true)) { nativeFsStream.write(TEST_CONTEXT.getBytes()); @@ -238,14 +232,11 @@ public void testwriteFile1() throws Exception { */ @Test public void testazcopywasbcompatibility() throws Exception { + Assume.assumeFalse("Not valid for APPEND BLOB", isAppendBlobEnabled()); AzureBlobFileSystem abfs = getFileSystem(); - assumeThat(getIsNamespaceEnabled(abfs)) - .as("Namespace enabled account does not support this test") - .isFalse(); - assumeThat(isAppendBlobEnabled()).as("Not valid for APPEND BLOB").isFalse(); - Path testFile = path("/testReadFile"); - Path path = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); + Path path = new Path( + testFile + "/~12/!008/testfile_" + UUID.randomUUID()); createAzCopyFile(path); try (FSDataOutputStream nativeFsStream = abfs.append(path)) { @@ -264,11 +255,6 @@ public void testDir() throws Exception { boolean[] readDirWithAbfs = new boolean[]{false, true, true, false}; AzureBlobFileSystem abfs = getFileSystem(); - // test only valid for non-namespace enabled account - assumeThat(getIsNamespaceEnabled(abfs)) - .as("Namespace enabled account does not support this test") - .isFalse(); - NativeAzureFileSystem wasb = getWasbFileSystem(); Path testDir = path("/testDir"); @@ -304,11 +290,6 @@ public void testUrlConversion() { public void testSetWorkingDirectory() throws Exception { //create folders AzureBlobFileSystem abfs = getFileSystem(); - // test only valid for non-namespace enabled account - assumeThat(getIsNamespaceEnabled(abfs)) - .as("Namespace enabled account does not support this test") - .isFalse(); - NativeAzureFileSystem wasb = getWasbFileSystem(); Path d1 = path("/d1"); @@ -341,13 +322,11 @@ public void testSetWorkingDirectory() throws Exception { @Test public void testScenario1() throws Exception { AzureBlobFileSystem abfs = getFileSystem(); - assumeThat(getIsNamespaceEnabled(abfs)) - .as("Namespace enabled account does not support this test") - .isFalse(); NativeAzureFileSystem wasb = getWasbFileSystem(); Path testFile = path("/testReadFile"); - Path path = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); + Path path = new Path( + testFile + "/~12/!008/testfile_" + UUID.randomUUID()); // Write try (FSDataOutputStream nativeFsStream = wasb.create(path, true)) { @@ -361,7 +340,8 @@ public void testScenario1() throws Exception { try (BufferedReader br = new BufferedReader( new InputStreamReader(abfs.open(path)))) { String line = br.readLine(); - assertEquals(TEST_CONTEXT, line, "Wrong text from " + abfs); + assertEquals("Wrong text from " + abfs, + TEST_CONTEXT, line); } // Remove file @@ -374,16 +354,13 @@ public void testScenario1() throws Exception { */ @Test public void testScenario2() throws Exception { + Assume.assumeFalse("Not valid for APPEND BLOB", isAppendBlobEnabled()); AzureBlobFileSystem abfs = getFileSystem(); - assumeThat(getIsNamespaceEnabled(abfs)) - .as("Namespace enabled account does not support this test") - .isFalse(); - assumeBlobServiceType(); - assumeThat(isAppendBlobEnabled()).as("Not valid for APPEND BLOB").isFalse(); NativeAzureFileSystem wasb = getWasbFileSystem(); Path testFile = path("/testReadFile"); - Path path = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); + Path path = new Path( + testFile + "/~12/!008/testfile_" + UUID.randomUUID()); // Write try (FSDataOutputStream nativeFsStream = wasb.create(path, true)) { @@ -397,7 +374,8 @@ public void testScenario2() throws Exception { try (BufferedReader br = new BufferedReader( new InputStreamReader(abfs.open(path)))) { String line = br.readLine(); - assertEquals(TEST_CONTEXT, line, "Wrong text from " + abfs); + assertEquals("Wrong text from " + abfs, + TEST_CONTEXT, line); } // Write @@ -417,33 +395,36 @@ public void testScenario2() throws Exception { */ @Test public void testScenario3() throws Exception { - AzureBlobFileSystem abfs = getFileSystem(); - assumeThat(getIsNamespaceEnabled(abfs)) - .as("Namespace enabled account does not support this test") - .isFalse(); - assumeThat(isAppendBlobEnabled()).as("Not valid for APPEND BLOB").isFalse(); - NativeAzureFileSystem wasb = getWasbFileSystem(); + Assume.assumeFalse("Not valid for APPEND BLOB", isAppendBlobEnabled()); + Configuration conf = getRawConfiguration(); + conf.setBoolean(FS_AZURE_ENABLE_FULL_BLOB_CHECKSUM_VALIDATION, true); + FileSystem fileSystem = FileSystem.newInstance(conf); + try (AzureBlobFileSystem abfs = (AzureBlobFileSystem) fileSystem) { + NativeAzureFileSystem wasb = getWasbFileSystem(); - Path testFile = path("/testReadFile"); - Path path = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); + Path testFile = path("/testReadFile"); + Path path = new Path( + testFile + "/~12/!008/testfile_" + UUID.randomUUID()); - // Write - try (FSDataOutputStream abfsOutputStream = abfs.create(path, true)) { - abfsOutputStream.write(TEST_CONTEXT.getBytes()); - abfsOutputStream.flush(); - abfsOutputStream.hsync(); - } + // Write + try (FSDataOutputStream abfsOutputStream = abfs.create(path, true)) { + abfsOutputStream.write(TEST_CONTEXT.getBytes()); + abfsOutputStream.flush(); + abfsOutputStream.hsync(); + } - // Check file status - ContractTestUtils.assertIsFile(abfs, path); + // Check file status + ContractTestUtils.assertIsFile(abfs, path); - try (BufferedReader br = new BufferedReader( - new InputStreamReader(wasb.open(path)))) { - String line = br.readLine(); - assertEquals(TEST_CONTEXT, line, "Wrong text from " + wasb); + try (BufferedReader br = new BufferedReader( + new InputStreamReader(wasb.open(path)))) { + String line = br.readLine(); + assertEquals("Wrong text from " + wasb, + TEST_CONTEXT, line); + } + // Remove file + assertDeleted(abfs, path, true); } - // Remove file - assertDeleted(abfs, path, true); } /** @@ -452,16 +433,13 @@ public void testScenario3() throws Exception { */ @Test public void testScenario4() throws Exception { + Assume.assumeFalse("Not valid for APPEND BLOB", isAppendBlobEnabled()); AzureBlobFileSystem abfs = getFileSystem(); - assumeThat(getIsNamespaceEnabled(abfs)) - .as("Namespace enabled account does not support this test") - .isFalse(); - assumeBlobServiceType(); - assumeThat(isAppendBlobEnabled()).as("Not valid for APPEND BLOB").isFalse(); NativeAzureFileSystem wasb = getWasbFileSystem(); Path testFile = path("/testReadFile"); - Path path = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); + Path path = new Path( + testFile + "/~12/!008/testfile_" + UUID.randomUUID()); // Write wasb.create(path, true); @@ -489,39 +467,37 @@ public void testScenario4() throws Exception { */ @Test public void testScenario5() throws Exception { + Assume.assumeFalse("Not valid for APPEND BLOB", isAppendBlobEnabled()); Configuration conf = getRawConfiguration(); conf.setBoolean(FS_AZURE_ABFS_ENABLE_CHECKSUM_VALIDATION, false); FileSystem fileSystem = FileSystem.newInstance(conf); - AzureBlobFileSystem abfs = (AzureBlobFileSystem) fileSystem; - assumeThat(getIsNamespaceEnabled(abfs)) - .as("Namespace enabled account does not support this test") - .isFalse(); - assumeBlobServiceType(); - assumeThat(isAppendBlobEnabled()).as("Not valid for APPEND BLOB").isFalse(); - NativeAzureFileSystem wasb = getWasbFileSystem(); + try (AzureBlobFileSystem abfs = (AzureBlobFileSystem) fileSystem) { + NativeAzureFileSystem wasb = getWasbFileSystem(); - Path testFile = path("/testReadFile"); - Path path = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); + Path testFile = path("/testReadFile"); + Path path = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); - // Write - abfs.create(path, true); - try (FSDataOutputStream nativeFsStream = wasb.append(path)) { - nativeFsStream.write(TEST_CONTEXT.getBytes()); - nativeFsStream.flush(); - nativeFsStream.hsync(); - } + // Write + abfs.create(path, true); + try (FSDataOutputStream nativeFsStream = wasb.append(path)) { + nativeFsStream.write(TEST_CONTEXT.getBytes()); + nativeFsStream.flush(); + nativeFsStream.hsync(); + } - // Check file status - ContractTestUtils.assertIsFile(abfs, path); + // Check file status + ContractTestUtils.assertIsFile(abfs, path); - try (BufferedReader br = new BufferedReader( - new InputStreamReader(abfs.open(path)))) { - String line = br.readLine(); - assertEquals(TEST_CONTEXT, line, "Wrong text from " + abfs); - } + try (BufferedReader br = new BufferedReader( + new InputStreamReader(abfs.open(path)))) { + String line = br.readLine(); + assertEquals("Wrong text from " + abfs, + TEST_CONTEXT, line); + } - // Remove file - assertDeleted(abfs, path, true); + // Remove file + assertDeleted(abfs, path, true); + } } /** @@ -530,39 +506,38 @@ public void testScenario5() throws Exception { */ @Test public void testScenario6() throws Exception { + Assume.assumeFalse("Not valid for APPEND BLOB", isAppendBlobEnabled()); Configuration conf = getRawConfiguration(); conf.setBoolean(FS_AZURE_ABFS_ENABLE_CHECKSUM_VALIDATION, true); FileSystem fileSystem = FileSystem.newInstance(conf); - AzureBlobFileSystem abfs = (AzureBlobFileSystem) fileSystem; - assumeThat(getIsNamespaceEnabled(abfs)) - .as("Namespace enabled account does not support this test") - .isFalse(); - assumeThat(isAppendBlobEnabled()).as("Not valid for APPEND BLOB").isFalse(); - assumeBlobServiceType(); - NativeAzureFileSystem wasb = getWasbFileSystem(); + try (AzureBlobFileSystem abfs = (AzureBlobFileSystem) fileSystem) { + NativeAzureFileSystem wasb = getWasbFileSystem(); - Path testFile = path("/testReadFile"); - Path path = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); + Path testFile = path("/testReadFile"); + Path path = new Path( + testFile + "/~12/!008/testfile_" + UUID.randomUUID()); - // Write - abfs.create(path, true); - try (FSDataOutputStream nativeFsStream = wasb.append(path)) { - nativeFsStream.write(TEST_CONTEXT.getBytes()); - nativeFsStream.flush(); - nativeFsStream.hsync(); - } + // Write + abfs.create(path, true); + try (FSDataOutputStream nativeFsStream = wasb.append(path)) { + nativeFsStream.write(TEST_CONTEXT.getBytes()); + nativeFsStream.flush(); + nativeFsStream.hsync(); + } - // Check file status - ContractTestUtils.assertIsFile(abfs, path); + // Check file status + ContractTestUtils.assertIsFile(abfs, path); - try (BufferedReader br = new BufferedReader( - new InputStreamReader(abfs.open(path)))) { - String line = br.readLine(); - assertEquals(TEST_CONTEXT, line, "Wrong text from " + abfs); - } + try (BufferedReader br = new BufferedReader( + new InputStreamReader(abfs.open(path)))) { + String line = br.readLine(); + assertEquals("Wrong text from " + abfs, + TEST_CONTEXT, line); + } - // Remove file - assertDeleted(abfs, path, true); + // Remove file + assertDeleted(abfs, path, true); + } } /** @@ -571,41 +546,39 @@ public void testScenario6() throws Exception { */ @Test public void testScenario7() throws Exception { + Assume.assumeFalse("Not valid for APPEND BLOB", isAppendBlobEnabled()); Configuration conf = getRawConfiguration(); conf.setBoolean(FS_AZURE_ABFS_ENABLE_CHECKSUM_VALIDATION, true); FileSystem fileSystem = FileSystem.newInstance(conf); - AzureBlobFileSystem abfs = (AzureBlobFileSystem) fileSystem; - assumeThat(getIsNamespaceEnabled(abfs)) - .as("Namespace enabled account does not support this test") - .isFalse(); - assumeThat(isAppendBlobEnabled()).as("Not valid for APPEND BLOB").isFalse(); - NativeAzureFileSystem wasb = getWasbFileSystem(); + try (AzureBlobFileSystem abfs = (AzureBlobFileSystem) fileSystem) { + NativeAzureFileSystem wasb = getWasbFileSystem(); + Path testFile = path("/testReadFile"); + Path path = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); - Path testFile = path("/testReadFile"); - Path path = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); + try (FSDataOutputStream nativeFsStream = wasb.create(path, true)) { + nativeFsStream.write(TEST_CONTEXT.getBytes()); + nativeFsStream.flush(); + nativeFsStream.hsync(); + } - try (FSDataOutputStream nativeFsStream = wasb.create(path, true)) { - nativeFsStream.write(TEST_CONTEXT.getBytes()); - nativeFsStream.flush(); - nativeFsStream.hsync(); - } + // Check file status + ContractTestUtils.assertIsFile(abfs, path); - // Check file status - ContractTestUtils.assertIsFile(abfs, path); + try (BufferedReader br = new BufferedReader( + new InputStreamReader(abfs.open(path)))) { + String line = br.readLine(); + assertEquals("Wrong text from " + abfs, + TEST_CONTEXT, line); + } + abfs.create(path, true); + FileStatus fileStatus = abfs.getFileStatus(path); + Assertions.assertThat(fileStatus.getLen()) + .as("Expected file length to be 0 after overwrite") + .isEqualTo(0L); - try (BufferedReader br = new BufferedReader( - new InputStreamReader(abfs.open(path)))) { - String line = br.readLine(); - assertEquals(TEST_CONTEXT, line, "Wrong text from " + abfs); + // Remove file + assertDeleted(abfs, path, true); } - abfs.create(path, true); - FileStatus fileStatus = abfs.getFileStatus(path); - Assertions.assertThat(fileStatus.getLen()) - .as("Expected file length to be 0 after overwrite") - .isEqualTo(0L); - - // Remove file - assertDeleted(abfs, path, true); } /** @@ -614,47 +587,48 @@ public void testScenario7() throws Exception { */ @Test public void testScenario8() throws Exception { + Assume.assumeFalse("Not valid for APPEND BLOB", isAppendBlobEnabled()); Configuration conf = getRawConfiguration(); conf.setBoolean(FS_AZURE_ABFS_ENABLE_CHECKSUM_VALIDATION, true); FileSystem fileSystem = FileSystem.newInstance(conf); - AzureBlobFileSystem abfs = (AzureBlobFileSystem) fileSystem; - assumeThat(getIsNamespaceEnabled(abfs)).isFalse(); - assumeThat(isAppendBlobEnabled()).as("Not valid for APPEND BLOB").isFalse(); - NativeAzureFileSystem wasb = getWasbFileSystem(); + try (AzureBlobFileSystem abfs = (AzureBlobFileSystem) fileSystem) { + NativeAzureFileSystem wasb = getWasbFileSystem(); - Path testFile = path("/testReadFile"); - Path path = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); + Path testFile = path("/testReadFile"); + Path path = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); - try (FSDataOutputStream nativeFsStream = wasb.create(path, true)) { - nativeFsStream.write(TEST_CONTEXT.getBytes()); - nativeFsStream.flush(); - nativeFsStream.hsync(); - } + try (FSDataOutputStream nativeFsStream = wasb.create(path, true)) { + nativeFsStream.write(TEST_CONTEXT.getBytes()); + nativeFsStream.flush(); + nativeFsStream.hsync(); + } - // Check file status - ContractTestUtils.assertIsFile(abfs, path); + // Check file status + ContractTestUtils.assertIsFile(abfs, path); - try (BufferedReader br = new BufferedReader( - new InputStreamReader(abfs.open(path)))) { - String line = br.readLine(); - assertEquals(TEST_CONTEXT, line, "Wrong text from " + abfs); - } - try { - abfs.create(path, false); - } catch (IOException e) { - AbfsRestOperationException restEx = (AbfsRestOperationException) e.getCause(); - if (restEx != null) { - Assertions.assertThat(restEx.getStatusCode()) - .as("Expected HTTP status code 409 (Conflict) when file already exists") - .isEqualTo(HTTP_CONFLICT); - } - Assertions.assertThat(e.getMessage()) - .as("Expected error message to contain 'AlreadyExists'") - .contains("AlreadyExists"); - } + try (BufferedReader br = new BufferedReader( + new InputStreamReader(abfs.open(path)))) { + String line = br.readLine(); + assertEquals("Wrong text from " + abfs, + TEST_CONTEXT, line); + } + try { + abfs.create(path, false); + } catch (IOException e) { + AbfsRestOperationException restEx = (AbfsRestOperationException) e.getCause(); + if (restEx != null) { + Assertions.assertThat(restEx.getStatusCode()) + .as("Expected HTTP status code 409 (Conflict) when file already exists") + .isEqualTo(HTTP_CONFLICT); + } + Assertions.assertThat(e.getMessage()) + .as("Expected error message to contain 'AlreadyExists'") + .contains("AlreadyExists"); + } - // Remove file - assertDeleted(abfs, path, true); + // Remove file + assertDeleted(abfs, path, true); + } } /** @@ -663,93 +637,89 @@ public void testScenario8() throws Exception { */ @Test public void testScenario9() throws Exception { + Assume.assumeFalse("Not valid for APPEND BLOB", isAppendBlobEnabled()); Configuration conf = getRawConfiguration(); conf.setBoolean(FS_AZURE_ABFS_ENABLE_CHECKSUM_VALIDATION, true); FileSystem fileSystem = FileSystem.newInstance(conf); - AzureBlobFileSystem abfs = (AzureBlobFileSystem) fileSystem; - assumeThat(getIsNamespaceEnabled(abfs)) - .as("Namespace enabled account does not support this test") - .isFalse(); - assumeBlobServiceType(); - assumeThat(isAppendBlobEnabled()).as("Not valid for APPEND BLOB").isFalse(); - NativeAzureFileSystem wasb = getWasbFileSystem(); + try (AzureBlobFileSystem abfs = (AzureBlobFileSystem) fileSystem) { + NativeAzureFileSystem wasb = getWasbFileSystem(); - Path testFile = path("/testReadFile"); - Path path = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); + Path testFile = path("/testReadFile"); + Path path = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); - try (FSDataOutputStream abfsOutputStream = abfs.create(path, true)) { - abfsOutputStream.write(TEST_CONTEXT.getBytes()); - abfsOutputStream.flush(); - abfsOutputStream.hsync(); - } + try (FSDataOutputStream abfsOutputStream = abfs.create(path, true)) { + abfsOutputStream.write(TEST_CONTEXT.getBytes()); + abfsOutputStream.flush(); + abfsOutputStream.hsync(); + } - // Check file status - ContractTestUtils.assertIsFile(abfs, path); + // Check file status + ContractTestUtils.assertIsFile(abfs, path); - try (BufferedReader br = new BufferedReader( - new InputStreamReader(abfs.open(path)))) { - String line = br.readLine(); - assertEquals(TEST_CONTEXT, line, "Wrong text from " + abfs); - } - wasb.create(path, true); - FileStatus fileStatus = abfs.getFileStatus(path); - Assertions.assertThat(fileStatus.getLen()) - .as("Expected file length to be 0 after overwrite") - .isEqualTo(0L); + try (BufferedReader br = new BufferedReader( + new InputStreamReader(abfs.open(path)))) { + String line = br.readLine(); + assertEquals("Wrong text from " + abfs, + TEST_CONTEXT, line); + } + wasb.create(path, true); + FileStatus fileStatus = abfs.getFileStatus(path); + Assertions.assertThat(fileStatus.getLen()) + .as("Expected file length to be 0 after overwrite") + .isEqualTo(0L); - // Remove file - assertDeleted(abfs, path, true); + // Remove file + assertDeleted(abfs, path, true); + } } /** * Scenario 10: Create a file using ABFS and then attempt to create the same file using WASB with overwrite=false. * Expected Outcome: WASB should fail to create the file as it already exists. The exception should indicate - * an "AlreadyExists" error with HTTP status code 409 (Conflict). + * an "AlreadyExists" error with HTTP status code 409 (Conflict). */ @Test public void testScenario10() throws Exception { Configuration conf = getRawConfiguration(); conf.setBoolean(FS_AZURE_ABFS_ENABLE_CHECKSUM_VALIDATION, true); FileSystem fileSystem = FileSystem.newInstance(conf); - AzureBlobFileSystem abfs = (AzureBlobFileSystem) fileSystem; - assumeThat(getIsNamespaceEnabled(abfs)) - .as("Namespace enabled account does not support this test") - .isFalse(); - NativeAzureFileSystem wasb = getWasbFileSystem(); - - Path testFile = path("/testReadFile"); - Path path = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); - - try (FSDataOutputStream abfsOutputStream = abfs.create(path, true)) { - abfsOutputStream.write(TEST_CONTEXT.getBytes()); - abfsOutputStream.flush(); - abfsOutputStream.hsync(); - } + try (AzureBlobFileSystem abfs = (AzureBlobFileSystem) fileSystem) { + NativeAzureFileSystem wasb = getWasbFileSystem(); + Path testFile = path("/testReadFile"); + Path path = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); + + try (FSDataOutputStream abfsOutputStream = abfs.create(path, true)) { + abfsOutputStream.write(TEST_CONTEXT.getBytes()); + abfsOutputStream.flush(); + abfsOutputStream.hsync(); + } - // Check file status - ContractTestUtils.assertIsFile(abfs, path); + // Check file status + ContractTestUtils.assertIsFile(abfs, path); - try (BufferedReader br = new BufferedReader( - new InputStreamReader(abfs.open(path)))) { - String line = br.readLine(); - assertEquals(TEST_CONTEXT, line, "Wrong text from " + abfs); - } - try { - wasb.create(path, false); - } catch (IOException e) { - AbfsRestOperationException restEx - = (AbfsRestOperationException) e.getCause(); - if (restEx != null) { - Assertions.assertThat(restEx.getStatusCode()) - .as("Expected HTTP status code 409 (Conflict) when file already exists") - .isEqualTo(HTTP_CONFLICT); - } - Assertions.assertThat(e.getMessage()) - .as("Expected error message to contain 'exists'") - .contains("exists"); + try (BufferedReader br = new BufferedReader( + new InputStreamReader(abfs.open(path)))) { + String line = br.readLine(); + assertEquals("Wrong text from " + abfs, + TEST_CONTEXT, line); + } + try { + wasb.create(path, false); + } catch (IOException e) { + AbfsRestOperationException restEx + = (AbfsRestOperationException) e.getCause(); + if (restEx != null) { + Assertions.assertThat(restEx.getStatusCode()) + .as("Expected HTTP status code 409 (Conflict) when file already exists") + .isEqualTo(HTTP_CONFLICT); + } + Assertions.assertThat(e.getMessage()) + .as("Expected error message to contain 'exists'") + .contains("exists"); + } + // Remove file + assertDeleted(abfs, path, true); } - // Remove file - assertDeleted(abfs, path, true); } /** @@ -759,37 +729,35 @@ public void testScenario10() throws Exception { */ @Test public void testScenario11() throws Exception { + Assume.assumeFalse("Not valid for APPEND BLOB", isAppendBlobEnabled()); Configuration conf = getRawConfiguration(); conf.setBoolean(FS_AZURE_ABFS_ENABLE_CHECKSUM_VALIDATION, true); FileSystem fileSystem = FileSystem.newInstance(conf); - AzureBlobFileSystem abfs = (AzureBlobFileSystem) fileSystem; - assumeThat(getIsNamespaceEnabled(abfs)) - .as("Namespace enabled account does not support this test") - .isFalse(); - assumeBlobServiceType(); - assumeThat(isAppendBlobEnabled()).as("Not valid for APPEND BLOB").isFalse(); - NativeAzureFileSystem wasb = getWasbFileSystem(); + try (AzureBlobFileSystem abfs = (AzureBlobFileSystem) fileSystem) { + NativeAzureFileSystem wasb = getWasbFileSystem(); - Path testFile = path("/testReadFile"); - Path path = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); + Path testFile = path("/testReadFile"); + Path path = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); - // Write - abfs.create(path, true); - try (FSDataOutputStream nativeFsStream = wasb.append(path)) { - nativeFsStream.write(TEST_CONTEXT.getBytes()); - nativeFsStream.flush(); - nativeFsStream.hsync(); - } + // Write + abfs.create(path, true); + try (FSDataOutputStream nativeFsStream = wasb.append(path)) { + nativeFsStream.write(TEST_CONTEXT.getBytes()); + nativeFsStream.flush(); + nativeFsStream.hsync(); + } - // Check file status - ContractTestUtils.assertIsFile(abfs, path); + // Check file status + ContractTestUtils.assertIsFile(abfs, path); - try (BufferedReader br = new BufferedReader( - new InputStreamReader(abfs.open(path)))) { - String line = br.readLine(); - assertEquals(TEST_CONTEXT, line, "Wrong text from " + abfs); + try (BufferedReader br = new BufferedReader( + new InputStreamReader(abfs.open(path)))) { + String line = br.readLine(); + assertEquals("Wrong text from " + abfs, + TEST_CONTEXT, line); + } + abfs.delete(path, true); } - abfs.delete(path, true); } /** @@ -802,31 +770,30 @@ public void testScenario12() throws Exception { Configuration conf = getRawConfiguration(); conf.setBoolean(FS_AZURE_ABFS_ENABLE_CHECKSUM_VALIDATION, true); FileSystem fileSystem = FileSystem.newInstance(conf); - AzureBlobFileSystem abfs = (AzureBlobFileSystem) fileSystem; - assumeThat(getIsNamespaceEnabled(abfs)) - .as("Namespace enabled account does not support this test") - .isFalse(); - NativeAzureFileSystem wasb = getWasbFileSystem(); + try (AzureBlobFileSystem abfs = (AzureBlobFileSystem) fileSystem) { + NativeAzureFileSystem wasb = getWasbFileSystem(); - Path testFile = path("/testReadFile"); - Path path = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); + Path testFile = path("/testReadFile"); + Path path = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); - // Write - try (FSDataOutputStream abfsOutputStream = abfs.create(path, true)) { - abfsOutputStream.write(TEST_CONTEXT.getBytes()); - abfsOutputStream.flush(); - abfsOutputStream.hsync(); - } + // Write + try (FSDataOutputStream abfsOutputStream = abfs.create(path, true)) { + abfsOutputStream.write(TEST_CONTEXT.getBytes()); + abfsOutputStream.flush(); + abfsOutputStream.hsync(); + } - // Check file status - ContractTestUtils.assertIsFile(abfs, path); + // Check file status + ContractTestUtils.assertIsFile(abfs, path); - try (BufferedReader br = new BufferedReader( - new InputStreamReader(abfs.open(path)))) { - String line = br.readLine(); - assertEquals(TEST_CONTEXT, line, "Wrong text from " + abfs); + try (BufferedReader br = new BufferedReader( + new InputStreamReader(abfs.open(path)))) { + String line = br.readLine(); + assertEquals("Wrong text from " + abfs, + TEST_CONTEXT, line); + } + wasb.delete(path, true); } - wasb.delete(path, true); } /** @@ -835,37 +802,35 @@ public void testScenario12() throws Exception { */ @Test public void testScenario13() throws Exception { + Assume.assumeFalse("Not valid for APPEND BLOB", isAppendBlobEnabled()); Configuration conf = getRawConfiguration(); conf.setBoolean(FS_AZURE_ABFS_ENABLE_CHECKSUM_VALIDATION, true); FileSystem fileSystem = FileSystem.newInstance(conf); - AzureBlobFileSystem abfs = (AzureBlobFileSystem) fileSystem; - assumeThat(getIsNamespaceEnabled(abfs)) - .as("Namespace enabled account does not support this test") - .isFalse(); - assumeBlobServiceType(); - assumeThat(isAppendBlobEnabled()).as("Not valid for APPEND BLOB").isFalse(); - NativeAzureFileSystem wasb = getWasbFileSystem(); + try (AzureBlobFileSystem abfs = (AzureBlobFileSystem) fileSystem) { + NativeAzureFileSystem wasb = getWasbFileSystem(); - Path testFile = path("/testReadFile"); - Path path = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); + Path testFile = path("/testReadFile"); + Path path = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); - // Write - abfs.create(path, true); - try (FSDataOutputStream nativeFsStream = wasb.append(path)) { - nativeFsStream.write(TEST_CONTEXT.getBytes()); - nativeFsStream.flush(); - nativeFsStream.hsync(); - } + // Write + abfs.create(path, true); + try (FSDataOutputStream nativeFsStream = wasb.append(path)) { + nativeFsStream.write(TEST_CONTEXT.getBytes()); + nativeFsStream.flush(); + nativeFsStream.hsync(); + } - // Check file status - ContractTestUtils.assertIsFile(abfs, path); + // Check file status + ContractTestUtils.assertIsFile(abfs, path); - try (BufferedReader br = new BufferedReader( - new InputStreamReader(wasb.open(path)))) { - String line = br.readLine(); - assertEquals(TEST_CONTEXT, line, "Wrong text from " + wasb); + try (BufferedReader br = new BufferedReader( + new InputStreamReader(wasb.open(path)))) { + String line = br.readLine(); + assertEquals("Wrong text from " + wasb, + TEST_CONTEXT, line); + } + abfs.delete(path, true); } - abfs.delete(path, true); } /** @@ -874,37 +839,35 @@ public void testScenario13() throws Exception { */ @Test public void testScenario14() throws Exception { + Assume.assumeFalse("Not valid for APPEND BLOB", isAppendBlobEnabled()); Configuration conf = getRawConfiguration(); conf.setBoolean(FS_AZURE_ABFS_ENABLE_CHECKSUM_VALIDATION, true); FileSystem fileSystem = FileSystem.newInstance(conf); - AzureBlobFileSystem abfs = (AzureBlobFileSystem) fileSystem; - assumeThat(getIsNamespaceEnabled(abfs)) - .as("Namespace enabled account does not support this test") - .isFalse(); - assumeBlobServiceType(); - assumeThat(isAppendBlobEnabled()).as("Not valid for APPEND BLOB").isFalse(); - NativeAzureFileSystem wasb = getWasbFileSystem(); + try (AzureBlobFileSystem abfs = (AzureBlobFileSystem) fileSystem) { + NativeAzureFileSystem wasb = getWasbFileSystem(); - Path testFile = path("/testReadFile"); - Path path = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); + Path testFile = path("/testReadFile"); + Path path = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); - // Write - abfs.create(path, true); - try (FSDataOutputStream nativeFsStream = wasb.append(path)) { - nativeFsStream.write(TEST_CONTEXT.getBytes()); - nativeFsStream.flush(); - nativeFsStream.hsync(); - } + // Write + abfs.create(path, true); + try (FSDataOutputStream nativeFsStream = wasb.append(path)) { + nativeFsStream.write(TEST_CONTEXT.getBytes()); + nativeFsStream.flush(); + nativeFsStream.hsync(); + } - // Check file status - ContractTestUtils.assertIsFile(abfs, path); + // Check file status + ContractTestUtils.assertIsFile(abfs, path); - try (BufferedReader br = new BufferedReader( - new InputStreamReader(wasb.open(path)))) { - String line = br.readLine(); - assertEquals(TEST_CONTEXT, line, "Wrong text from " + wasb); + try (BufferedReader br = new BufferedReader( + new InputStreamReader(wasb.open(path)))) { + String line = br.readLine(); + assertEquals("Wrong text from " + wasb, + TEST_CONTEXT, line); + } + wasb.delete(path, true); } - wasb.delete(path, true); } /** @@ -916,31 +879,30 @@ public void testScenario15() throws Exception { Configuration conf = getRawConfiguration(); conf.setBoolean(FS_AZURE_ABFS_ENABLE_CHECKSUM_VALIDATION, true); FileSystem fileSystem = FileSystem.newInstance(conf); - AzureBlobFileSystem abfs = (AzureBlobFileSystem) fileSystem; - assumeThat(getIsNamespaceEnabled(abfs)) - .as("Namespace enabled account does not support this test") - .isFalse(); - NativeAzureFileSystem wasb = getWasbFileSystem(); + try (AzureBlobFileSystem abfs = (AzureBlobFileSystem) fileSystem) { + NativeAzureFileSystem wasb = getWasbFileSystem(); - Path testFile = path("/testReadFile"); - Path path = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); + Path testFile = path("/testReadFile"); + Path path = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); - // Write - try (FSDataOutputStream nativeFsStream = wasb.create(path, true)) { - nativeFsStream.write(TEST_CONTEXT.getBytes()); - nativeFsStream.flush(); - nativeFsStream.hsync(); - } + // Write + try (FSDataOutputStream nativeFsStream = wasb.create(path, true)) { + nativeFsStream.write(TEST_CONTEXT.getBytes()); + nativeFsStream.flush(); + nativeFsStream.hsync(); + } - // Check file status - ContractTestUtils.assertIsFile(abfs, path); + // Check file status + ContractTestUtils.assertIsFile(abfs, path); - try (BufferedReader br = new BufferedReader( - new InputStreamReader(wasb.open(path)))) { - String line = br.readLine(); - assertEquals(TEST_CONTEXT, line, "Wrong text from " + wasb); + try (BufferedReader br = new BufferedReader( + new InputStreamReader(wasb.open(path)))) { + String line = br.readLine(); + assertEquals("Wrong text from " + wasb, + TEST_CONTEXT, line); + } + abfs.delete(path, true); } - abfs.delete(path, true); } /** @@ -949,37 +911,35 @@ public void testScenario15() throws Exception { */ @Test public void testScenario16() throws Exception { + Assume.assumeFalse("Not valid for APPEND BLOB", isAppendBlobEnabled()); Configuration conf = getRawConfiguration(); conf.setBoolean(FS_AZURE_ABFS_ENABLE_CHECKSUM_VALIDATION, true); FileSystem fileSystem = FileSystem.newInstance(conf); - AzureBlobFileSystem abfs = (AzureBlobFileSystem) fileSystem; - assumeThat(getIsNamespaceEnabled(abfs)) - .as("Namespace enabled account does not support this test") - .isFalse(); - assumeBlobServiceType(); - assumeThat(isAppendBlobEnabled()).as("Not valid for APPEND BLOB").isFalse(); - NativeAzureFileSystem wasb = getWasbFileSystem(); + try (AzureBlobFileSystem abfs = (AzureBlobFileSystem) fileSystem) { + NativeAzureFileSystem wasb = getWasbFileSystem(); - Path testFile = path("/testReadFile"); - Path path = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); + Path testFile = path("/testReadFile"); + Path path = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); - // Write - wasb.create(path, true); - try (FSDataOutputStream abfsOutputStream = abfs.append(path)) { - abfsOutputStream.write(TEST_CONTEXT.getBytes()); - abfsOutputStream.flush(); - abfsOutputStream.hsync(); - } + // Write + wasb.create(path, true); + try (FSDataOutputStream abfsOutputStream = abfs.append(path)) { + abfsOutputStream.write(TEST_CONTEXT.getBytes()); + abfsOutputStream.flush(); + abfsOutputStream.hsync(); + } - // Check file status - ContractTestUtils.assertIsFile(abfs, path); + // Check file status + ContractTestUtils.assertIsFile(abfs, path); - try (BufferedReader br = new BufferedReader( - new InputStreamReader(abfs.open(path)))) { - String line = br.readLine(); - assertEquals(TEST_CONTEXT, line, "Wrong text from " + abfs); + try (BufferedReader br = new BufferedReader( + new InputStreamReader(abfs.open(path)))) { + String line = br.readLine(); + assertEquals("Wrong text from " + abfs, + TEST_CONTEXT, line); + } + wasb.delete(path, true); } - wasb.delete(path, true); } /** @@ -991,38 +951,36 @@ public void testScenario17() throws Exception { Configuration conf = getRawConfiguration(); conf.setBoolean(FS_AZURE_ABFS_ENABLE_CHECKSUM_VALIDATION, true); FileSystem fileSystem = FileSystem.newInstance(conf); - AzureBlobFileSystem abfs = (AzureBlobFileSystem) fileSystem; - assumeThat(getIsNamespaceEnabled(abfs)) - .as("Namespace enabled account does not support this test") - .isFalse(); - Path testFile = path("/testReadFile"); - Path path = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); + try (AzureBlobFileSystem abfs = (AzureBlobFileSystem) fileSystem) { + Path testFile = path("/testReadFile"); + Path path = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); - // Write - try (FSDataOutputStream abfsOutputStream = abfs.create(path, true)) { - abfsOutputStream.write(TEST_CONTEXT.getBytes()); - abfsOutputStream.flush(); - abfsOutputStream.hsync(); - } - // --- VALIDATE FILE --- - FileStatus status = abfs.getFileStatus(path); - assertIsFile(path, status); + // Write + try (FSDataOutputStream abfsOutputStream = abfs.create(path, true)) { + abfsOutputStream.write(TEST_CONTEXT.getBytes()); + abfsOutputStream.flush(); + abfsOutputStream.hsync(); + } + // --- VALIDATE FILE --- + FileStatus status = abfs.getFileStatus(path); + assertIsFile(path, status); - // --- SET XATTR #1 --- - abfs.setXAttr(path, ATTRIBUTE_NAME_1, ATTRIBUTE_VALUE_1); - byte[] readValue = abfs.getXAttr(path, ATTRIBUTE_NAME_1); - ITestAzureBlobFileSystemAttributes.assertAttributeEqual(abfs, readValue, ATTRIBUTE_VALUE_1, "one"); + // --- SET XATTR #1 --- + abfs.setXAttr(path, ATTRIBUTE_NAME_1, ATTRIBUTE_VALUE_1); + byte[] readValue = abfs.getXAttr(path, ATTRIBUTE_NAME_1); + ITestAzureBlobFileSystemAttributes.assertAttributeEqual(abfs, readValue, ATTRIBUTE_VALUE_1, "one"); - // --- SET XATTR #2 --- - abfs.setXAttr(path, ATTRIBUTE_NAME_2, ATTRIBUTE_VALUE_2); - readValue = abfs.getXAttr(path, ATTRIBUTE_NAME_2); - ITestAzureBlobFileSystemAttributes.assertAttributeEqual(abfs, readValue, ATTRIBUTE_VALUE_2, "two"); + // --- SET XATTR #2 --- + abfs.setXAttr(path, ATTRIBUTE_NAME_2, ATTRIBUTE_VALUE_2); + readValue = abfs.getXAttr(path, ATTRIBUTE_NAME_2); + ITestAzureBlobFileSystemAttributes.assertAttributeEqual(abfs, readValue, ATTRIBUTE_VALUE_2, "two"); - // --- VERIFY XATTR #1 AGAIN --- - readValue = abfs.getXAttr(path, ATTRIBUTE_NAME_1); - ITestAzureBlobFileSystemAttributes.assertAttributeEqual(abfs, readValue, ATTRIBUTE_VALUE_1, "one"); + // --- VERIFY XATTR #1 AGAIN --- + readValue = abfs.getXAttr(path, ATTRIBUTE_NAME_1); + ITestAzureBlobFileSystemAttributes.assertAttributeEqual(abfs, readValue, ATTRIBUTE_VALUE_1, "one"); - abfs.delete(path, true); + abfs.delete(path, true); + } } /** @@ -1034,40 +992,38 @@ public void testScenario18() throws Exception { Configuration conf = getRawConfiguration(); conf.setBoolean(FS_AZURE_ABFS_ENABLE_CHECKSUM_VALIDATION, true); FileSystem fileSystem = FileSystem.newInstance(conf); - AzureBlobFileSystem abfs = (AzureBlobFileSystem) fileSystem; - assumeThat(getIsNamespaceEnabled(abfs)) - .as("Namespace enabled account does not support this test") - .isFalse(); - NativeAzureFileSystem wasb = getWasbFileSystem(); + try (AzureBlobFileSystem abfs = (AzureBlobFileSystem) fileSystem) { + NativeAzureFileSystem wasb = getWasbFileSystem(); - Path testFile = path("/testReadFile"); - Path path = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); + Path testFile = path("/testReadFile"); + Path path = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); - // Write - try (FSDataOutputStream nativeFsStream = wasb.create(path, true)) { - nativeFsStream.write(TEST_CONTEXT.getBytes()); - nativeFsStream.flush(); - nativeFsStream.hsync(); - } - // --- VALIDATE FILE --- - FileStatus status = wasb.getFileStatus(path); - assertIsFile(path, status); + // Write + try (FSDataOutputStream nativeFsStream = wasb.create(path, true)) { + nativeFsStream.write(TEST_CONTEXT.getBytes()); + nativeFsStream.flush(); + nativeFsStream.hsync(); + } + // --- VALIDATE FILE --- + FileStatus status = wasb.getFileStatus(path); + assertIsFile(path, status); - // --- SET XATTR #1 --- - wasb.setXAttr(path, ATTRIBUTE_NAME_1, ATTRIBUTE_VALUE_1); - byte[] readValue = wasb.getXAttr(path, ATTRIBUTE_NAME_1); - ITestAzureBlobFileSystemAttributes.assertAttributeEqual(abfs, readValue, ATTRIBUTE_VALUE_1, "one"); + // --- SET XATTR #1 --- + wasb.setXAttr(path, ATTRIBUTE_NAME_1, ATTRIBUTE_VALUE_1); + byte[] readValue = wasb.getXAttr(path, ATTRIBUTE_NAME_1); + ITestAzureBlobFileSystemAttributes.assertAttributeEqual(abfs, readValue, ATTRIBUTE_VALUE_1, "one"); - // --- SET XATTR #2 --- - wasb.setXAttr(path, ATTRIBUTE_NAME_2, ATTRIBUTE_VALUE_2); - readValue = wasb.getXAttr(path, ATTRIBUTE_NAME_2); - ITestAzureBlobFileSystemAttributes.assertAttributeEqual(abfs, readValue, ATTRIBUTE_VALUE_2, "two"); + // --- SET XATTR #2 --- + wasb.setXAttr(path, ATTRIBUTE_NAME_2, ATTRIBUTE_VALUE_2); + readValue = wasb.getXAttr(path, ATTRIBUTE_NAME_2); + ITestAzureBlobFileSystemAttributes.assertAttributeEqual(abfs, readValue, ATTRIBUTE_VALUE_2, "two"); - // --- VERIFY XATTR #1 AGAIN --- - readValue = wasb.getXAttr(path, ATTRIBUTE_NAME_1); - ITestAzureBlobFileSystemAttributes.assertAttributeEqual(abfs, readValue, ATTRIBUTE_VALUE_1, "one"); + // --- VERIFY XATTR #1 AGAIN --- + readValue = wasb.getXAttr(path, ATTRIBUTE_NAME_1); + ITestAzureBlobFileSystemAttributes.assertAttributeEqual(abfs, readValue, ATTRIBUTE_VALUE_1, "one"); - wasb.delete(path, true); + wasb.delete(path, true); + } } /** @@ -1079,40 +1035,38 @@ public void testScenario19() throws Exception { Configuration conf = getRawConfiguration(); conf.setBoolean(FS_AZURE_ABFS_ENABLE_CHECKSUM_VALIDATION, true); FileSystem fileSystem = FileSystem.newInstance(conf); - AzureBlobFileSystem abfs = (AzureBlobFileSystem) fileSystem; - assumeThat(getIsNamespaceEnabled(abfs)) - .as("Namespace enabled account does not support this test") - .isFalse(); - NativeAzureFileSystem wasb = getWasbFileSystem(); + try (AzureBlobFileSystem abfs = (AzureBlobFileSystem) fileSystem) { + NativeAzureFileSystem wasb = getWasbFileSystem(); - Path testFile = path("/testReadFile"); - Path path = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); + Path testFile = path("/testReadFile"); + Path path = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); - // Write - try (FSDataOutputStream nativeFsStream = wasb.create(path, true)) { - nativeFsStream.write(TEST_CONTEXT.getBytes()); - nativeFsStream.flush(); - nativeFsStream.hsync(); - } - // --- VALIDATE FILE --- - FileStatus status = wasb.getFileStatus(path); - assertIsFile(path, status); + // Write + try (FSDataOutputStream nativeFsStream = wasb.create(path, true)) { + nativeFsStream.write(TEST_CONTEXT.getBytes()); + nativeFsStream.flush(); + nativeFsStream.hsync(); + } + // --- VALIDATE FILE --- + FileStatus status = wasb.getFileStatus(path); + assertIsFile(path, status); - // --- SET XATTR #1 --- - wasb.setXAttr(path, ATTRIBUTE_NAME_1, ATTRIBUTE_VALUE_1); - byte[] readValue = abfs.getXAttr(path, ATTRIBUTE_NAME_1); - ITestAzureBlobFileSystemAttributes.assertAttributeEqual(abfs, readValue, ATTRIBUTE_VALUE_1, "one"); + // --- SET XATTR #1 --- + wasb.setXAttr(path, ATTRIBUTE_NAME_1, ATTRIBUTE_VALUE_1); + byte[] readValue = abfs.getXAttr(path, ATTRIBUTE_NAME_1); + ITestAzureBlobFileSystemAttributes.assertAttributeEqual(abfs, readValue, ATTRIBUTE_VALUE_1, "one"); - // --- SET XATTR #2 --- - wasb.setXAttr(path, ATTRIBUTE_NAME_2, ATTRIBUTE_VALUE_2); - readValue = abfs.getXAttr(path, ATTRIBUTE_NAME_2); - ITestAzureBlobFileSystemAttributes.assertAttributeEqual(abfs, readValue, ATTRIBUTE_VALUE_2, "two"); + // --- SET XATTR #2 --- + wasb.setXAttr(path, ATTRIBUTE_NAME_2, ATTRIBUTE_VALUE_2); + readValue = abfs.getXAttr(path, ATTRIBUTE_NAME_2); + ITestAzureBlobFileSystemAttributes.assertAttributeEqual(abfs, readValue, ATTRIBUTE_VALUE_2, "two"); - // --- VERIFY XATTR #1 AGAIN --- - readValue = abfs.getXAttr(path, ATTRIBUTE_NAME_1); - ITestAzureBlobFileSystemAttributes.assertAttributeEqual(abfs, readValue, ATTRIBUTE_VALUE_1, "one"); + // --- VERIFY XATTR #1 AGAIN --- + readValue = abfs.getXAttr(path, ATTRIBUTE_NAME_1); + ITestAzureBlobFileSystemAttributes.assertAttributeEqual(abfs, readValue, ATTRIBUTE_VALUE_1, "one"); - wasb.delete(path, true); + wasb.delete(path, true); + } } /** @@ -1123,49 +1077,47 @@ public void testScenario19() throws Exception { */ @Test public void testScenario20() throws Exception { + Assume.assumeFalse("Not valid for APPEND BLOB", isAppendBlobEnabled()); Configuration conf = getRawConfiguration(); conf.setBoolean(FS_AZURE_ABFS_ENABLE_CHECKSUM_VALIDATION, true); FileSystem fileSystem = FileSystem.newInstance(conf); - AzureBlobFileSystem abfs = (AzureBlobFileSystem) fileSystem; - assumeThat(getIsNamespaceEnabled(abfs)) - .as("Namespace enabled account does not support this test") - .isFalse(); - assumeThat(isAppendBlobEnabled()).as("Not valid for APPEND BLOB").isFalse(); - NativeAzureFileSystem wasb = getWasbFileSystem(); + try (AzureBlobFileSystem abfs = (AzureBlobFileSystem) fileSystem) { + NativeAzureFileSystem wasb = getWasbFileSystem(); - Path testFile = path("/testReadFile"); - Path path = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); + Path testFile = path("/testReadFile"); + Path path = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); - // Write - try (FSDataOutputStream nativeFsStream = wasb.create(path, true)) { - nativeFsStream.write(TEST_CONTEXT.getBytes()); - nativeFsStream.flush(); - nativeFsStream.hsync(); + // Write + try (FSDataOutputStream nativeFsStream = wasb.create(path, true)) { + nativeFsStream.write(TEST_CONTEXT.getBytes()); + nativeFsStream.flush(); + nativeFsStream.hsync(); + } + // --- VALIDATE FILE --- + FileStatus status = wasb.getFileStatus(path); + assertIsFile(path, status); + + // --- SET XATTR #1 --- + wasb.setXAttr(path, ATTRIBUTE_NAME_1, ATTRIBUTE_VALUE_1); + byte[] readValue = abfs.getXAttr(path, ATTRIBUTE_NAME_1); + ITestAzureBlobFileSystemAttributes.assertAttributeEqual(abfs, readValue, ATTRIBUTE_VALUE_1, "one"); + + // --- SET XATTR #2 --- + wasb.setXAttr(path, ATTRIBUTE_NAME_2, ATTRIBUTE_VALUE_2); + readValue = abfs.getXAttr(path, ATTRIBUTE_NAME_2); + ITestAzureBlobFileSystemAttributes.assertAttributeEqual(abfs, readValue, ATTRIBUTE_VALUE_2, "two"); + + // --- VERIFY XATTR #1 AGAIN --- + readValue = abfs.getXAttr(path, ATTRIBUTE_NAME_1); + ITestAzureBlobFileSystemAttributes.assertAttributeEqual(abfs, readValue, ATTRIBUTE_VALUE_1, "one"); + + abfs.create(path, true); + FileStatus fileStatus = abfs.getFileStatus(path); + Assertions.assertThat(fileStatus.getLen()) + .as("Expected file length to be 0 after overwrite") + .isEqualTo(0L); + wasb.delete(path, true); } - // --- VALIDATE FILE --- - FileStatus status = wasb.getFileStatus(path); - assertIsFile(path, status); - - // --- SET XATTR #1 --- - wasb.setXAttr(path, ATTRIBUTE_NAME_1, ATTRIBUTE_VALUE_1); - byte[] readValue = abfs.getXAttr(path, ATTRIBUTE_NAME_1); - ITestAzureBlobFileSystemAttributes.assertAttributeEqual(abfs, readValue, ATTRIBUTE_VALUE_1, "one"); - - // --- SET XATTR #2 --- - wasb.setXAttr(path, ATTRIBUTE_NAME_2, ATTRIBUTE_VALUE_2); - readValue = abfs.getXAttr(path, ATTRIBUTE_NAME_2); - ITestAzureBlobFileSystemAttributes.assertAttributeEqual(abfs, readValue, ATTRIBUTE_VALUE_2, "two"); - - // --- VERIFY XATTR #1 AGAIN --- - readValue = abfs.getXAttr(path, ATTRIBUTE_NAME_1); - ITestAzureBlobFileSystemAttributes.assertAttributeEqual(abfs, readValue, ATTRIBUTE_VALUE_1, "one"); - - abfs.create(path, true); - FileStatus fileStatus = abfs.getFileStatus(path); - Assertions.assertThat(fileStatus.getLen()) - .as("Expected file length to be 0 after overwrite") - .isEqualTo(0L); - wasb.delete(path, true); } /** @@ -1176,50 +1128,47 @@ public void testScenario20() throws Exception { */ @Test public void testScenario21() throws Exception { + Assume.assumeFalse("Not valid for APPEND BLOB", isAppendBlobEnabled()); Configuration conf = getRawConfiguration(); conf.setBoolean(FS_AZURE_ABFS_ENABLE_CHECKSUM_VALIDATION, true); FileSystem fileSystem = FileSystem.newInstance(conf); - AzureBlobFileSystem abfs = (AzureBlobFileSystem) fileSystem; - assumeThat(getIsNamespaceEnabled(abfs)) - .as("Namespace enabled account does not support this test") - .isFalse(); - assumeBlobServiceType(); - assumeThat(isAppendBlobEnabled()).as("Not valid for APPEND BLOB").isFalse(); - NativeAzureFileSystem wasb = getWasbFileSystem(); + try (AzureBlobFileSystem abfs = (AzureBlobFileSystem) fileSystem) { + NativeAzureFileSystem wasb = getWasbFileSystem(); - Path testFile = path("/testReadFile"); - Path path = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); + Path testFile = path("/testReadFile"); + Path path = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); - // Write - try (FSDataOutputStream abfsOutputStream = abfs.create(path, true)) { - abfsOutputStream.write(TEST_CONTEXT.getBytes()); - abfsOutputStream.flush(); - abfsOutputStream.hsync(); + // Write + try (FSDataOutputStream abfsOutputStream = abfs.create(path, true)) { + abfsOutputStream.write(TEST_CONTEXT.getBytes()); + abfsOutputStream.flush(); + abfsOutputStream.hsync(); + } + // --- VALIDATE FILE --- + FileStatus status = wasb.getFileStatus(path); + assertIsFile(path, status); + + // --- SET XATTR #1 --- + abfs.setXAttr(path, ATTRIBUTE_NAME_1, ATTRIBUTE_VALUE_1); + byte[] readValue = wasb.getXAttr(path, ATTRIBUTE_NAME_1); + ITestAzureBlobFileSystemAttributes.assertAttributeEqual(abfs, readValue, ATTRIBUTE_VALUE_1, "one"); + + // --- SET XATTR #2 --- + abfs.setXAttr(path, ATTRIBUTE_NAME_2, ATTRIBUTE_VALUE_2); + readValue = wasb.getXAttr(path, ATTRIBUTE_NAME_2); + ITestAzureBlobFileSystemAttributes.assertAttributeEqual(abfs, readValue, ATTRIBUTE_VALUE_2, "two"); + + // --- VERIFY XATTR #1 AGAIN --- + readValue = wasb.getXAttr(path, ATTRIBUTE_NAME_1); + ITestAzureBlobFileSystemAttributes.assertAttributeEqual(abfs, readValue, ATTRIBUTE_VALUE_1, "one"); + + wasb.create(path, true); + FileStatus fileStatus = abfs.getFileStatus(path); + Assertions.assertThat(fileStatus.getLen()) + .as("Expected file length to be 0 after overwrite") + .isEqualTo(0L); + wasb.delete(path, true); } - // --- VALIDATE FILE --- - FileStatus status = wasb.getFileStatus(path); - assertIsFile(path, status); - - // --- SET XATTR #1 --- - abfs.setXAttr(path, ATTRIBUTE_NAME_1, ATTRIBUTE_VALUE_1); - byte[] readValue = wasb.getXAttr(path, ATTRIBUTE_NAME_1); - ITestAzureBlobFileSystemAttributes.assertAttributeEqual(abfs, readValue, ATTRIBUTE_VALUE_1, "one"); - - // --- SET XATTR #2 --- - abfs.setXAttr(path, ATTRIBUTE_NAME_2, ATTRIBUTE_VALUE_2); - readValue = wasb.getXAttr(path, ATTRIBUTE_NAME_2); - ITestAzureBlobFileSystemAttributes.assertAttributeEqual(abfs, readValue, ATTRIBUTE_VALUE_2, "two"); - - // --- VERIFY XATTR #1 AGAIN --- - readValue = wasb.getXAttr(path, ATTRIBUTE_NAME_1); - ITestAzureBlobFileSystemAttributes.assertAttributeEqual(abfs, readValue, ATTRIBUTE_VALUE_1, "one"); - - wasb.create(path, true); - FileStatus fileStatus = abfs.getFileStatus(path); - Assertions.assertThat(fileStatus.getLen()) - .as("Expected file length to be 0 after overwrite") - .isEqualTo(0L); - wasb.delete(path, true); } /** @@ -1233,45 +1182,43 @@ public void testScenario22() throws Exception { Configuration conf = getRawConfiguration(); conf.setBoolean(FS_AZURE_ABFS_ENABLE_CHECKSUM_VALIDATION, true); FileSystem fileSystem = FileSystem.newInstance(conf); - AzureBlobFileSystem abfs = (AzureBlobFileSystem) fileSystem; - assumeThat(getIsNamespaceEnabled(abfs)) - .as("Namespace enabled account does not support this test") - .isFalse(); - NativeAzureFileSystem wasb = getWasbFileSystem(); + try (AzureBlobFileSystem abfs = (AzureBlobFileSystem) fileSystem) { + NativeAzureFileSystem wasb = getWasbFileSystem(); - Path testFile = path("/testReadFile"); - Path path = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); + Path testFile = path("/testReadFile"); + Path path = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); - // Write - try (FSDataOutputStream nativeFsStream = wasb.create(path, true)) { - nativeFsStream.write(TEST_CONTEXT.getBytes()); - nativeFsStream.flush(); - nativeFsStream.hsync(); + // Write + try (FSDataOutputStream nativeFsStream = wasb.create(path, true)) { + nativeFsStream.write(TEST_CONTEXT.getBytes()); + nativeFsStream.flush(); + nativeFsStream.hsync(); + } + // --- VALIDATE FILE --- + FileStatus status = wasb.getFileStatus(path); + assertIsFile(path, status); + + // --- SET XATTR #1 --- + abfs.setXAttr(path, ATTRIBUTE_NAME_1, ATTRIBUTE_VALUE_1); + byte[] readValue = wasb.getXAttr(path, ATTRIBUTE_NAME_1); + ITestAzureBlobFileSystemAttributes.assertAttributeEqual(abfs, readValue, ATTRIBUTE_VALUE_1, "one"); + + // --- SET XATTR #2 --- + abfs.setXAttr(path, ATTRIBUTE_NAME_2, ATTRIBUTE_VALUE_2); + readValue = wasb.getXAttr(path, ATTRIBUTE_NAME_2); + ITestAzureBlobFileSystemAttributes.assertAttributeEqual(abfs, readValue, ATTRIBUTE_VALUE_2, "two"); + + // --- VERIFY XATTR #1 AGAIN --- + readValue = wasb.getXAttr(path, ATTRIBUTE_NAME_1); + ITestAzureBlobFileSystemAttributes.assertAttributeEqual(abfs, readValue, ATTRIBUTE_VALUE_1, "one"); + + wasb.create(path, true); + FileStatus fileStatus = abfs.getFileStatus(path); + Assertions.assertThat(fileStatus.getLen()) + .as("Expected file length to be 0 after overwrite") + .isEqualTo(0L); + wasb.delete(path, true); } - // --- VALIDATE FILE --- - FileStatus status = wasb.getFileStatus(path); - assertIsFile(path, status); - - // --- SET XATTR #1 --- - abfs.setXAttr(path, ATTRIBUTE_NAME_1, ATTRIBUTE_VALUE_1); - byte[] readValue = wasb.getXAttr(path, ATTRIBUTE_NAME_1); - ITestAzureBlobFileSystemAttributes.assertAttributeEqual(abfs, readValue, ATTRIBUTE_VALUE_1, "one"); - - // --- SET XATTR #2 --- - abfs.setXAttr(path, ATTRIBUTE_NAME_2, ATTRIBUTE_VALUE_2); - readValue = wasb.getXAttr(path, ATTRIBUTE_NAME_2); - ITestAzureBlobFileSystemAttributes.assertAttributeEqual(abfs, readValue, ATTRIBUTE_VALUE_2, "two"); - - // --- VERIFY XATTR #1 AGAIN --- - readValue = wasb.getXAttr(path, ATTRIBUTE_NAME_1); - ITestAzureBlobFileSystemAttributes.assertAttributeEqual(abfs, readValue, ATTRIBUTE_VALUE_1, "one"); - - wasb.create(path, true); - FileStatus fileStatus = abfs.getFileStatus(path); - Assertions.assertThat(fileStatus.getLen()) - .as("Expected file length to be 0 after overwrite") - .isEqualTo(0L); - wasb.delete(path, true); } /** @@ -1285,39 +1232,37 @@ public void testScenario23() throws Exception { Configuration conf = getRawConfiguration(); conf.setBoolean(FS_AZURE_ABFS_ENABLE_CHECKSUM_VALIDATION, true); FileSystem fileSystem = FileSystem.newInstance(conf); - AzureBlobFileSystem abfs = (AzureBlobFileSystem) fileSystem; - assumeThat(getIsNamespaceEnabled(abfs)) - .as("Namespace enabled account does not support this test") - .isFalse(); - NativeAzureFileSystem wasb = getWasbFileSystem(); + try (AzureBlobFileSystem abfs = (AzureBlobFileSystem) fileSystem) { + NativeAzureFileSystem wasb = getWasbFileSystem(); - Path testFile = path("/testReadFile"); - Path path = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); - // Write - try (FSDataOutputStream nativeFsStream = wasb.create(path, true)) { - nativeFsStream.write(TEST_CONTEXT.getBytes()); - nativeFsStream.flush(); - nativeFsStream.hsync(); - } - // --- VALIDATE FILE --- - FileStatus status = wasb.getFileStatus(path); - assertIsFile(path, status); + Path testFile = path("/testReadFile"); + Path path = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); + // Write + try (FSDataOutputStream nativeFsStream = wasb.create(path, true)) { + nativeFsStream.write(TEST_CONTEXT.getBytes()); + nativeFsStream.flush(); + nativeFsStream.hsync(); + } + // --- VALIDATE FILE --- + FileStatus status = wasb.getFileStatus(path); + assertIsFile(path, status); - // --- SET XATTR #1 --- - abfs.setXAttr(path, ATTRIBUTE_NAME_1, ATTRIBUTE_VALUE_1); - byte[] readValue = abfs.getXAttr(path, ATTRIBUTE_NAME_1); - ITestAzureBlobFileSystemAttributes.assertAttributeEqual(abfs, readValue, ATTRIBUTE_VALUE_1, "one"); + // --- SET XATTR #1 --- + abfs.setXAttr(path, ATTRIBUTE_NAME_1, ATTRIBUTE_VALUE_1); + byte[] readValue = abfs.getXAttr(path, ATTRIBUTE_NAME_1); + ITestAzureBlobFileSystemAttributes.assertAttributeEqual(abfs, readValue, ATTRIBUTE_VALUE_1, "one"); - // --- SET XATTR #2 --- - wasb.setXAttr(path, ATTRIBUTE_NAME_2, ATTRIBUTE_VALUE_2, CREATE_FLAG); - readValue = abfs.getXAttr(path, ATTRIBUTE_NAME_2); - ITestAzureBlobFileSystemAttributes.assertAttributeEqual(abfs, readValue, ATTRIBUTE_VALUE_2, "two"); + // --- SET XATTR #2 --- + wasb.setXAttr(path, ATTRIBUTE_NAME_2, ATTRIBUTE_VALUE_2, CREATE_FLAG); + readValue = abfs.getXAttr(path, ATTRIBUTE_NAME_2); + ITestAzureBlobFileSystemAttributes.assertAttributeEqual(abfs, readValue, ATTRIBUTE_VALUE_2, "two"); - // --- VERIFY XATTR #1 AGAIN --- - readValue = abfs.getXAttr(path, ATTRIBUTE_NAME_1); - ITestAzureBlobFileSystemAttributes.assertAttributeEqual(abfs, readValue, ATTRIBUTE_VALUE_1, "one"); + // --- VERIFY XATTR #1 AGAIN --- + readValue = abfs.getXAttr(path, ATTRIBUTE_NAME_1); + ITestAzureBlobFileSystemAttributes.assertAttributeEqual(abfs, readValue, ATTRIBUTE_VALUE_1, "one"); - wasb.delete(path, true); + wasb.delete(path, true); + } } /** @@ -1328,44 +1273,42 @@ public void testScenario23() throws Exception { */ @Test public void testScenario24() throws Exception { + Assume.assumeFalse("Not valid for APPEND BLOB", isAppendBlobEnabled()); Configuration conf = getRawConfiguration(); conf.setBoolean(FS_AZURE_ABFS_ENABLE_CHECKSUM_VALIDATION, true); FileSystem fileSystem = FileSystem.newInstance(conf); - AzureBlobFileSystem abfs = (AzureBlobFileSystem) fileSystem; - assumeThat(getIsNamespaceEnabled(abfs)) - .as("Namespace enabled account does not support this test") - .isFalse(); - assumeThat(isAppendBlobEnabled()).as("Not valid for APPEND BLOB").isFalse(); - NativeAzureFileSystem wasb = getWasbFileSystem(); + try (AzureBlobFileSystem abfs = (AzureBlobFileSystem) fileSystem) { + NativeAzureFileSystem wasb = getWasbFileSystem(); - Path testFile = path("/testReadFile"); - Path path = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); + Path testFile = path("/testReadFile"); + Path path = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); - // Write - try (FSDataOutputStream abfsOutputStream = abfs.create(path, true)) { - abfsOutputStream.write(TEST_CONTEXT.getBytes()); - abfsOutputStream.flush(); - abfsOutputStream.hsync(); - } - // --- VALIDATE FILE --- - FileStatus status = wasb.getFileStatus(path); - assertIsFile(path, status); + // Write + try (FSDataOutputStream abfsOutputStream = abfs.create(path, true)) { + abfsOutputStream.write(TEST_CONTEXT.getBytes()); + abfsOutputStream.flush(); + abfsOutputStream.hsync(); + } + // --- VALIDATE FILE --- + FileStatus status = wasb.getFileStatus(path); + assertIsFile(path, status); - // --- SET XATTR #1 --- - wasb.setXAttr(path, ATTRIBUTE_NAME_1, ATTRIBUTE_VALUE_1); - byte[] readValue = abfs.getXAttr(path, ATTRIBUTE_NAME_1); - ITestAzureBlobFileSystemAttributes.assertAttributeEqual(abfs, readValue, ATTRIBUTE_VALUE_1, "one"); + // --- SET XATTR #1 --- + wasb.setXAttr(path, ATTRIBUTE_NAME_1, ATTRIBUTE_VALUE_1); + byte[] readValue = abfs.getXAttr(path, ATTRIBUTE_NAME_1); + ITestAzureBlobFileSystemAttributes.assertAttributeEqual(abfs, readValue, ATTRIBUTE_VALUE_1, "one"); - // --- SET XATTR #2 --- - wasb.setXAttr(path, ATTRIBUTE_NAME_2, ATTRIBUTE_VALUE_2, CREATE_FLAG); - readValue = abfs.getXAttr(path, ATTRIBUTE_NAME_2); - ITestAzureBlobFileSystemAttributes.assertAttributeEqual(abfs, readValue, ATTRIBUTE_VALUE_2, "two"); + // --- SET XATTR #2 --- + wasb.setXAttr(path, ATTRIBUTE_NAME_2, ATTRIBUTE_VALUE_2, CREATE_FLAG); + readValue = abfs.getXAttr(path, ATTRIBUTE_NAME_2); + ITestAzureBlobFileSystemAttributes.assertAttributeEqual(abfs, readValue, ATTRIBUTE_VALUE_2, "two"); - // --- VERIFY XATTR #1 AGAIN --- - readValue = abfs.getXAttr(path, ATTRIBUTE_NAME_1); - ITestAzureBlobFileSystemAttributes.assertAttributeEqual(abfs, readValue, ATTRIBUTE_VALUE_1, "one"); + // --- VERIFY XATTR #1 AGAIN --- + readValue = abfs.getXAttr(path, ATTRIBUTE_NAME_1); + ITestAzureBlobFileSystemAttributes.assertAttributeEqual(abfs, readValue, ATTRIBUTE_VALUE_1, "one"); - wasb.delete(path, true); + wasb.delete(path, true); + } } /** @@ -1379,40 +1322,38 @@ public void testScenario25() throws Exception { Configuration conf = getRawConfiguration(); conf.setBoolean(FS_AZURE_ABFS_ENABLE_CHECKSUM_VALIDATION, true); FileSystem fileSystem = FileSystem.newInstance(conf); - AzureBlobFileSystem abfs = (AzureBlobFileSystem) fileSystem; - assumeThat(getIsNamespaceEnabled(abfs)) - .as("Namespace enabled account does not support this test") - .isFalse(); - NativeAzureFileSystem wasb = getWasbFileSystem(); + try (AzureBlobFileSystem abfs = (AzureBlobFileSystem) fileSystem) { + NativeAzureFileSystem wasb = getWasbFileSystem(); - Path testFile = path("/testReadFile"); - Path path = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); + Path testFile = path("/testReadFile"); + Path path = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); - // Write - try (FSDataOutputStream nativeFsStream = wasb.create(path, true)) { - nativeFsStream.write(TEST_CONTEXT.getBytes()); - nativeFsStream.flush(); - nativeFsStream.hsync(); - } - // --- VALIDATE FILE --- - FileStatus status = wasb.getFileStatus(path); - assertIsFile(path, status); + // Write + try (FSDataOutputStream nativeFsStream = wasb.create(path, true)) { + nativeFsStream.write(TEST_CONTEXT.getBytes()); + nativeFsStream.flush(); + nativeFsStream.hsync(); + } + // --- VALIDATE FILE --- + FileStatus status = wasb.getFileStatus(path); + assertIsFile(path, status); - // --- SET XATTR #1 --- - abfs.setXAttr(path, ATTRIBUTE_NAME_1, ATTRIBUTE_VALUE_1); - byte[] readValue = abfs.getXAttr(path, ATTRIBUTE_NAME_1); - ITestAzureBlobFileSystemAttributes.assertAttributeEqual(abfs, readValue, ATTRIBUTE_VALUE_1, "one"); + // --- SET XATTR #1 --- + abfs.setXAttr(path, ATTRIBUTE_NAME_1, ATTRIBUTE_VALUE_1); + byte[] readValue = abfs.getXAttr(path, ATTRIBUTE_NAME_1); + ITestAzureBlobFileSystemAttributes.assertAttributeEqual(abfs, readValue, ATTRIBUTE_VALUE_1, "one"); - // --- SET XATTR #2 --- - abfs.setXAttr(path, ATTRIBUTE_NAME_2, ATTRIBUTE_VALUE_2, CREATE_FLAG); - readValue = abfs.getXAttr(path, ATTRIBUTE_NAME_2); - ITestAzureBlobFileSystemAttributes.assertAttributeEqual(abfs, readValue, ATTRIBUTE_VALUE_2, "two"); + // --- SET XATTR #2 --- + abfs.setXAttr(path, ATTRIBUTE_NAME_2, ATTRIBUTE_VALUE_2, CREATE_FLAG); + readValue = abfs.getXAttr(path, ATTRIBUTE_NAME_2); + ITestAzureBlobFileSystemAttributes.assertAttributeEqual(abfs, readValue, ATTRIBUTE_VALUE_2, "two"); - // --- VERIFY XATTR #1 AGAIN --- - readValue = abfs.getXAttr(path, ATTRIBUTE_NAME_1); - ITestAzureBlobFileSystemAttributes.assertAttributeEqual(abfs, readValue, ATTRIBUTE_VALUE_1, "one"); + // --- VERIFY XATTR #1 AGAIN --- + readValue = abfs.getXAttr(path, ATTRIBUTE_NAME_1); + ITestAzureBlobFileSystemAttributes.assertAttributeEqual(abfs, readValue, ATTRIBUTE_VALUE_1, "one"); - wasb.delete(path, true); + wasb.delete(path, true); + } } /** @@ -1423,44 +1364,42 @@ public void testScenario25() throws Exception { */ @Test public void testScenario26() throws Exception { + Assume.assumeFalse("Not valid for APPEND BLOB", isAppendBlobEnabled()); Configuration conf = getRawConfiguration(); conf.setBoolean(FS_AZURE_ABFS_ENABLE_CHECKSUM_VALIDATION, true); FileSystem fileSystem = FileSystem.newInstance(conf); - AzureBlobFileSystem abfs = (AzureBlobFileSystem) fileSystem; - assumeThat(getIsNamespaceEnabled(abfs)) - .as("Namespace enabled account does not support this test") - .isFalse(); - assumeThat(isAppendBlobEnabled()).as("Not valid for APPEND BLOB").isFalse(); - NativeAzureFileSystem wasb = getWasbFileSystem(); + try (AzureBlobFileSystem abfs = (AzureBlobFileSystem) fileSystem) { + NativeAzureFileSystem wasb = getWasbFileSystem(); - Path testFile = path("/testReadFile"); - Path path = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); + Path testFile = path("/testReadFile"); + Path path = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); - // Write - try (FSDataOutputStream abfsOutputStream = abfs.create(path, true)) { - abfsOutputStream.write(TEST_CONTEXT.getBytes()); - abfsOutputStream.flush(); - abfsOutputStream.hsync(); - } - // --- VALIDATE FILE --- - FileStatus status = abfs.getFileStatus(path); - assertIsFile(path, status); + // Write + try (FSDataOutputStream abfsOutputStream = abfs.create(path, true)) { + abfsOutputStream.write(TEST_CONTEXT.getBytes()); + abfsOutputStream.flush(); + abfsOutputStream.hsync(); + } + // --- VALIDATE FILE --- + FileStatus status = abfs.getFileStatus(path); + assertIsFile(path, status); - // --- SET XATTR #1 --- - wasb.setXAttr(path, ATTRIBUTE_NAME_1, ATTRIBUTE_VALUE_1); - byte[] readValue = wasb.getXAttr(path, ATTRIBUTE_NAME_1); - ITestAzureBlobFileSystemAttributes.assertAttributeEqual(abfs, readValue, ATTRIBUTE_VALUE_1, "one"); + // --- SET XATTR #1 --- + wasb.setXAttr(path, ATTRIBUTE_NAME_1, ATTRIBUTE_VALUE_1); + byte[] readValue = wasb.getXAttr(path, ATTRIBUTE_NAME_1); + ITestAzureBlobFileSystemAttributes.assertAttributeEqual(abfs, readValue, ATTRIBUTE_VALUE_1, "one"); - // --- SET XATTR #2 --- - wasb.setXAttr(path, ATTRIBUTE_NAME_2, ATTRIBUTE_VALUE_2, CREATE_FLAG); - readValue = wasb.getXAttr(path, ATTRIBUTE_NAME_2); - ITestAzureBlobFileSystemAttributes.assertAttributeEqual(abfs, readValue, ATTRIBUTE_VALUE_2, "two"); + // --- SET XATTR #2 --- + wasb.setXAttr(path, ATTRIBUTE_NAME_2, ATTRIBUTE_VALUE_2, CREATE_FLAG); + readValue = wasb.getXAttr(path, ATTRIBUTE_NAME_2); + ITestAzureBlobFileSystemAttributes.assertAttributeEqual(abfs, readValue, ATTRIBUTE_VALUE_2, "two"); - // --- VERIFY XATTR #1 AGAIN --- - readValue = wasb.getXAttr(path, ATTRIBUTE_NAME_1); - ITestAzureBlobFileSystemAttributes.assertAttributeEqual(abfs, readValue, ATTRIBUTE_VALUE_1, "one"); + // --- VERIFY XATTR #1 AGAIN --- + readValue = wasb.getXAttr(path, ATTRIBUTE_NAME_1); + ITestAzureBlobFileSystemAttributes.assertAttributeEqual(abfs, readValue, ATTRIBUTE_VALUE_1, "one"); - wasb.delete(path, true); + wasb.delete(path, true); + } } /** @@ -1469,48 +1408,47 @@ public void testScenario26() throws Exception { */ @Test public void testScenario27() throws Exception { + Assume.assumeFalse("Not valid for APPEND BLOB", isAppendBlobEnabled()); Configuration conf = getRawConfiguration(); conf.setBoolean(FS_AZURE_ABFS_ENABLE_CHECKSUM_VALIDATION, true); FileSystem fileSystem = FileSystem.newInstance(conf); - AzureBlobFileSystem abfs = (AzureBlobFileSystem) fileSystem; - assumeThat(getIsNamespaceEnabled(abfs)) - .as("Namespace enabled account does not support this test") - .isFalse(); - assumeThat(isAppendBlobEnabled()).as("Not valid for APPEND BLOB").isFalse(); - NativeAzureFileSystem wasb = getWasbFileSystem(); + try (AzureBlobFileSystem abfs = (AzureBlobFileSystem) fileSystem) { + NativeAzureFileSystem wasb = getWasbFileSystem(); - Path testFile = path("/testReadFile"); - Path testPath1 = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); - Path testPath2 = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); + Path testFile = path("/testReadFile"); + Path testPath1 = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); + Path testPath2 = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); - // Write - try (FSDataOutputStream abfsOutputStream = abfs.create(testPath1, true)) { - abfsOutputStream.write(TEST_CONTEXT.getBytes()); - abfsOutputStream.flush(); - abfsOutputStream.hsync(); - } + // Write + try (FSDataOutputStream abfsOutputStream = abfs.create(testPath1, true)) { + abfsOutputStream.write(TEST_CONTEXT.getBytes()); + abfsOutputStream.flush(); + abfsOutputStream.hsync(); + } - // Check file status - ContractTestUtils.assertIsFile(abfs, testPath1); + // Check file status + ContractTestUtils.assertIsFile(abfs, testPath1); - try (BufferedReader br = new BufferedReader( - new InputStreamReader(abfs.open(testPath1)))) { - String line = br.readLine(); - assertEquals(TEST_CONTEXT, line, "Wrong text from " + abfs); + try (BufferedReader br = new BufferedReader( + new InputStreamReader(abfs.open(testPath1)))) { + String line = br.readLine(); + assertEquals("Wrong text from " + abfs, + TEST_CONTEXT, line); + } + // --- RENAME FILE --- + boolean renamed = wasb.rename(testPath1, testPath2); + Assertions.assertThat(renamed) + .as("Rename failed") + .isTrue(); + + // --- LIST FILES IN DIRECTORY --- + Path parentDir = new Path(testFile + "/~12/!008"); + int noOfFiles = listAllFilesAndDirs(wasb, parentDir); + Assertions.assertThat(noOfFiles) + .as("Expected only 1 file or directory under path: %s", parentDir) + .isEqualTo(1); + wasb.delete(testPath2, true); } - // --- RENAME FILE --- - boolean renamed = wasb.rename(testPath1, testPath2); - Assertions.assertThat(renamed) - .as("Rename failed") - .isTrue(); - - // --- LIST FILES IN DIRECTORY --- - Path parentDir = new Path(testFile + "/~12/!008"); - int noOfFiles = listAllFilesAndDirs(wasb, parentDir); - Assertions.assertThat(noOfFiles) - .as("Expected only 1 file or directory under path: %s", parentDir) - .isEqualTo(1); - wasb.delete(testPath2, true); } /** @@ -1522,44 +1460,45 @@ public void testScenario28() throws Exception { Configuration conf = getRawConfiguration(); conf.setBoolean(FS_AZURE_ABFS_ENABLE_CHECKSUM_VALIDATION, true); FileSystem fileSystem = FileSystem.newInstance(conf); - AzureBlobFileSystem abfs = (AzureBlobFileSystem) fileSystem; - assumeThat(getIsNamespaceEnabled(abfs)) - .as("Namespace enabled account does not support this test") - .isFalse(); - NativeAzureFileSystem wasb = getWasbFileSystem(); + try (AzureBlobFileSystem abfs = (AzureBlobFileSystem) fileSystem) { + NativeAzureFileSystem wasb = getWasbFileSystem(); - Path testFile = path("/testReadFile"); - Path testPath1 = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); - Path testPath2 = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); + Path testFile = path("/testReadFile"); + Path testPath1 = new Path( + testFile + "/~12/!008/testfile_" + UUID.randomUUID()); + Path testPath2 = new Path( + testFile + "/~12/!008/testfile_" + UUID.randomUUID()); - // Write - try (FSDataOutputStream nativeFsStream = wasb.create(testPath1, true)) { - nativeFsStream.write(TEST_CONTEXT.getBytes()); - nativeFsStream.flush(); - nativeFsStream.hsync(); - } + // Write + try (FSDataOutputStream nativeFsStream = wasb.create(testPath1, true)) { + nativeFsStream.write(TEST_CONTEXT.getBytes()); + nativeFsStream.flush(); + nativeFsStream.hsync(); + } - // Check file status - ContractTestUtils.assertIsFile(abfs, testPath1); + // Check file status + ContractTestUtils.assertIsFile(abfs, testPath1); - try (BufferedReader br = new BufferedReader( - new InputStreamReader(abfs.open(testPath1)))) { - String line = br.readLine(); - assertEquals(TEST_CONTEXT, line, "Wrong text from " + abfs); + try (BufferedReader br = new BufferedReader( + new InputStreamReader(abfs.open(testPath1)))) { + String line = br.readLine(); + assertEquals("Wrong text from " + abfs, + TEST_CONTEXT, line); + } + // --- RENAME FILE --- + boolean renamed = abfs.rename(testPath1, testPath2); + Assertions.assertThat(renamed) + .as("Rename failed") + .isTrue(); + + // --- LIST FILES IN DIRECTORY --- + Path parentDir = new Path(testFile + "/~12/!008"); + int noOfFiles = listAllFilesAndDirs(abfs, parentDir); + Assertions.assertThat(noOfFiles) + .as("Expected only 1 file or directory under path: %s", parentDir) + .isEqualTo(1); + wasb.delete(testPath2, true); } - // --- RENAME FILE --- - boolean renamed = abfs.rename(testPath1, testPath2); - Assertions.assertThat(renamed) - .as("Rename failed") - .isTrue(); - - // --- LIST FILES IN DIRECTORY --- - Path parentDir = new Path(testFile + "/~12/!008"); - int noOfFiles = listAllFilesAndDirs(abfs, parentDir); - Assertions.assertThat(noOfFiles) - .as("Expected only 1 file or directory under path: %s", parentDir) - .isEqualTo(1); - wasb.delete(testPath2, true); } /** @@ -1568,50 +1507,48 @@ public void testScenario28() throws Exception { */ @Test public void testScenario29() throws Exception { + Assume.assumeFalse("Not valid for APPEND BLOB", isAppendBlobEnabled()); Configuration conf = getRawConfiguration(); conf.setBoolean(FS_AZURE_ABFS_ENABLE_CHECKSUM_VALIDATION, true); FileSystem fileSystem = FileSystem.newInstance(conf); - AzureBlobFileSystem abfs = (AzureBlobFileSystem) fileSystem; - assumeThat(getIsNamespaceEnabled(abfs)) - .as("Namespace enabled account does not support this test") - .isFalse(); - assumeBlobServiceType(); - assumeThat(isAppendBlobEnabled()).as("Not valid for APPEND BLOB").isFalse(); - NativeAzureFileSystem wasb = getWasbFileSystem(); + try (AzureBlobFileSystem abfs = (AzureBlobFileSystem) fileSystem) { + NativeAzureFileSystem wasb = getWasbFileSystem(); - Path testFile = path("/testReadFile"); - Path testPath1 = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); - Path testPath2 = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); + Path testFile = path("/testReadFile"); + Path testPath1 = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); + Path testPath2 = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); - // Write - wasb.create(testPath1, true); - try (FSDataOutputStream abfsOutputStream = abfs.append(testPath1)) { - abfsOutputStream.write(TEST_CONTEXT.getBytes()); - abfsOutputStream.flush(); - abfsOutputStream.hsync(); - } + // Write + wasb.create(testPath1, true); + try (FSDataOutputStream abfsOutputStream = abfs.append(testPath1)) { + abfsOutputStream.write(TEST_CONTEXT.getBytes()); + abfsOutputStream.flush(); + abfsOutputStream.hsync(); + } - // Check file status - ContractTestUtils.assertIsFile(abfs, testPath1); + // Check file status + ContractTestUtils.assertIsFile(abfs, testPath1); - try (BufferedReader br = new BufferedReader( - new InputStreamReader(abfs.open(testPath1)))) { - String line = br.readLine(); - assertEquals(TEST_CONTEXT, line, "Wrong text from " + abfs); + try (BufferedReader br = new BufferedReader( + new InputStreamReader(abfs.open(testPath1)))) { + String line = br.readLine(); + assertEquals("Wrong text from " + abfs, + TEST_CONTEXT, line); + } + // --- RENAME FILE --- + boolean renamed = abfs.rename(testPath1, testPath2); + Assertions.assertThat(renamed) + .as("Rename failed") + .isTrue(); + + // --- LIST FILES IN DIRECTORY --- + Path parentDir = new Path(testFile + "/~12/!008"); + int noOfFiles = listAllFilesAndDirs(abfs, parentDir); + Assertions.assertThat(noOfFiles) + .as("Expected only 1 file or directory under path: %s", parentDir) + .isEqualTo(1); + wasb.delete(testPath2, true); } - // --- RENAME FILE --- - boolean renamed = abfs.rename(testPath1, testPath2); - Assertions.assertThat(renamed) - .as("Rename failed") - .isTrue(); - - // --- LIST FILES IN DIRECTORY --- - Path parentDir = new Path(testFile + "/~12/!008"); - int noOfFiles = listAllFilesAndDirs(abfs, parentDir); - Assertions.assertThat(noOfFiles) - .as("Expected only 1 file or directory under path: %s", parentDir) - .isEqualTo(1); - wasb.delete(testPath2, true); } /** @@ -1623,51 +1560,50 @@ public void testScenario30() throws Exception { Configuration conf = getRawConfiguration(); conf.setBoolean(FS_AZURE_ABFS_ENABLE_CHECKSUM_VALIDATION, true); FileSystem fileSystem = FileSystem.newInstance(conf); - AzureBlobFileSystem abfs = (AzureBlobFileSystem) fileSystem; - assumeThat(getIsNamespaceEnabled(abfs)) - .as("Namespace enabled account does not support this test") - .isFalse(); - NativeAzureFileSystem wasb = getWasbFileSystem(); + try (AzureBlobFileSystem abfs = (AzureBlobFileSystem) fileSystem) { + NativeAzureFileSystem wasb = getWasbFileSystem(); - Path testFile = path("/testReadFile"); - Path testPath1 = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); - Path testPath2 = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); - Path testPath3 = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); + Path testFile = path("/testReadFile"); + Path testPath1 = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); + Path testPath2 = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); + Path testPath3 = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); - // Write - try (FSDataOutputStream nativeFsStream = wasb.create(testPath1, true)) { - nativeFsStream.write(TEST_CONTEXT.getBytes()); - nativeFsStream.flush(); - nativeFsStream.hsync(); - } + // Write + try (FSDataOutputStream nativeFsStream = wasb.create(testPath1, true)) { + nativeFsStream.write(TEST_CONTEXT.getBytes()); + nativeFsStream.flush(); + nativeFsStream.hsync(); + } - // Check file status - ContractTestUtils.assertIsFile(abfs, testPath1); + // Check file status + ContractTestUtils.assertIsFile(abfs, testPath1); - try (BufferedReader br = new BufferedReader( - new InputStreamReader(abfs.open(testPath1)))) { - String line = br.readLine(); - assertEquals(TEST_CONTEXT, line, "Wrong text from " + abfs); + try (BufferedReader br = new BufferedReader( + new InputStreamReader(abfs.open(testPath1)))) { + String line = br.readLine(); + assertEquals("Wrong text from " + abfs, + TEST_CONTEXT, line); + } + // --- RENAME FILE --- + boolean renamed = wasb.rename(testPath1, testPath2); + Assertions.assertThat(renamed) + .as("Rename failed") + .isTrue(); + + // --- RENAME FILE --- + boolean renamed1 = abfs.rename(testPath2, testPath3); + Assertions.assertThat(renamed1) + .as("Rename failed") + .isTrue(); + + // --- LIST FILES IN DIRECTORY --- + Path parentDir = new Path(testFile + "/~12/!008"); + int noOfFiles = listAllFilesAndDirs(abfs, parentDir); + Assertions.assertThat(noOfFiles) + .as("Expected only 1 file or directory under path: %s", parentDir) + .isEqualTo(1); + wasb.delete(testPath3, true); } - // --- RENAME FILE --- - boolean renamed = wasb.rename(testPath1, testPath2); - Assertions.assertThat(renamed) - .as("Rename failed") - .isTrue(); - - // --- RENAME FILE --- - boolean renamed1 = abfs.rename(testPath2, testPath3); - Assertions.assertThat(renamed1) - .as("Rename failed") - .isTrue(); - - // --- LIST FILES IN DIRECTORY --- - Path parentDir = new Path(testFile + "/~12/!008"); - int noOfFiles = listAllFilesAndDirs(abfs, parentDir); - Assertions.assertThat(noOfFiles) - .as("Expected only 1 file or directory under path: %s", parentDir) - .isEqualTo(1); - wasb.delete(testPath3, true); } /** @@ -1679,38 +1615,37 @@ public void testScenario31() throws Exception { Configuration conf = getRawConfiguration(); conf.setBoolean(FS_AZURE_ABFS_ENABLE_CHECKSUM_VALIDATION, true); FileSystem fileSystem = FileSystem.newInstance(conf); - AzureBlobFileSystem abfs = (AzureBlobFileSystem) fileSystem; - assumeThat(getIsNamespaceEnabled(abfs)) - .as("Namespace enabled account does not support this test") - .isFalse(); - NativeAzureFileSystem wasb = getWasbFileSystem(); + try (AzureBlobFileSystem abfs = (AzureBlobFileSystem) fileSystem) { + NativeAzureFileSystem wasb = getWasbFileSystem(); - Path testFile = path("/testReadFile"); - Path testPath1 = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); - Path testPath2 = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); + Path testFile = path("/testReadFile"); + Path testPath1 = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); + Path testPath2 = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); - // Write - try (FSDataOutputStream nativeFsStream = wasb.create(testPath1, true)) { - nativeFsStream.write(TEST_CONTEXT.getBytes()); - nativeFsStream.flush(); - nativeFsStream.hsync(); - } + // Write + try (FSDataOutputStream nativeFsStream = wasb.create(testPath1, true)) { + nativeFsStream.write(TEST_CONTEXT.getBytes()); + nativeFsStream.flush(); + nativeFsStream.hsync(); + } - // Check file status - ContractTestUtils.assertIsFile(abfs, testPath1); + // Check file status + ContractTestUtils.assertIsFile(abfs, testPath1); - try (BufferedReader br = new BufferedReader( - new InputStreamReader(abfs.open(testPath1)))) { - String line = br.readLine(); - assertEquals(TEST_CONTEXT, line, "Wrong text from " + abfs); - } - wasb.delete(testPath1, true); + try (BufferedReader br = new BufferedReader( + new InputStreamReader(abfs.open(testPath1)))) { + String line = br.readLine(); + assertEquals("Wrong text from " + abfs, + TEST_CONTEXT, line); + } + wasb.delete(testPath1, true); - // --- RENAME FILE --- - boolean renamed = abfs.rename(testPath1, testPath2); - Assertions.assertThat(renamed) - .as("Rename operation should have failed but returned true") - .isFalse(); + // --- RENAME FILE --- + boolean renamed = abfs.rename(testPath1, testPath2); + Assertions.assertThat(renamed) + .as("Rename operation should have failed but returned true") + .isFalse(); + } } /** @@ -1722,46 +1657,45 @@ public void testScenario32() throws Exception { Configuration conf = getRawConfiguration(); conf.setBoolean(FS_AZURE_ABFS_ENABLE_CHECKSUM_VALIDATION, true); FileSystem fileSystem = FileSystem.newInstance(conf); - AzureBlobFileSystem abfs = (AzureBlobFileSystem) fileSystem; - assumeThat(getIsNamespaceEnabled(abfs)) - .as("Namespace enabled account does not support this test") - .isFalse(); - NativeAzureFileSystem wasb = getWasbFileSystem(); + try (AzureBlobFileSystem abfs = (AzureBlobFileSystem) fileSystem) { + NativeAzureFileSystem wasb = getWasbFileSystem(); - Path testFile = path("/testReadFile"); - Path testFile1 = path("/testReadFile1"); - Path testPath1 = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); - Path testPath2 = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); - Path testPath3 = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); + Path testFile = path("/testReadFile"); + Path testFile1 = path("/testReadFile1"); + Path testPath1 = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); + Path testPath2 = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); + Path testPath3 = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); - // Write - wasb.mkdirs(testFile); - try (FSDataOutputStream nativeFsStream = wasb.create(testPath1, true)) { - nativeFsStream.write(TEST_CONTEXT.getBytes()); - nativeFsStream.flush(); - nativeFsStream.hsync(); - } - wasb.create(testPath2, true); - wasb.create(testPath3, true); + // Write + wasb.mkdirs(testFile); + try (FSDataOutputStream nativeFsStream = wasb.create(testPath1, true)) { + nativeFsStream.write(TEST_CONTEXT.getBytes()); + nativeFsStream.flush(); + nativeFsStream.hsync(); + } + wasb.create(testPath2, true); + wasb.create(testPath3, true); - // Check file status - ContractTestUtils.assertIsFile(abfs, testPath1); + // Check file status + ContractTestUtils.assertIsFile(abfs, testPath1); - try (BufferedReader br = new BufferedReader( - new InputStreamReader(abfs.open(testPath1)))) { - String line = br.readLine(); - assertEquals(TEST_CONTEXT, line, "Wrong text from " + abfs); + try (BufferedReader br = new BufferedReader( + new InputStreamReader(abfs.open(testPath1)))) { + String line = br.readLine(); + assertEquals("Wrong text from " + abfs, + TEST_CONTEXT, line); + } + // --- RENAME DIR --- + boolean renamed = abfs.rename(testFile, testFile1); + Assertions.assertThat(renamed) + .as("Rename failed") + .isTrue(); + // --- LIST FILES IN DIRECTORY --- + int listResult = listAllFilesAndDirs(abfs, testFile1); + Assertions.assertThat(listResult) + .as("Expected only 5 entries under path: %s", testFile1) + .isEqualTo(5); } - // --- RENAME DIR --- - boolean renamed = abfs.rename(testFile, testFile1); - Assertions.assertThat(renamed) - .as("Rename failed") - .isTrue(); - // --- LIST FILES IN DIRECTORY --- - int listResult = listAllFilesAndDirs(abfs, testFile1); - Assertions.assertThat(listResult) - .as("Expected only 5 entries under path: %s", testFile1) - .isEqualTo(5); } /** @@ -1770,50 +1704,49 @@ public void testScenario32() throws Exception { */ @Test public void testScenario33() throws Exception { + Assume.assumeFalse("Not valid for APPEND BLOB", isAppendBlobEnabled()); Configuration conf = getRawConfiguration(); conf.setBoolean(FS_AZURE_ABFS_ENABLE_CHECKSUM_VALIDATION, true); FileSystem fileSystem = FileSystem.newInstance(conf); - AzureBlobFileSystem abfs = (AzureBlobFileSystem) fileSystem; - assumeThat(getIsNamespaceEnabled(abfs)) - .as("Namespace enabled account does not support this test") - .isFalse(); - assumeThat(isAppendBlobEnabled()).as("Not valid for APPEND BLOB").isFalse(); - NativeAzureFileSystem wasb = getWasbFileSystem(); + try (AzureBlobFileSystem abfs = (AzureBlobFileSystem) fileSystem) { + NativeAzureFileSystem wasb = getWasbFileSystem(); - Path testFile = path("/testReadFile"); - Path testFile1 = path("/testReadFile1"); - Path testPath1 = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); - Path testPath2 = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); - Path testPath3 = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); + Path testFile = path("/testReadFile"); + Path testFile1 = path("/testReadFile1"); + Path testPath1 = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); + Path testPath2 = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); + Path testPath3 = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); - // Write - abfs.mkdirs(testFile); - try (FSDataOutputStream abfsOutputStream = abfs.create(testPath1, true)) { - abfsOutputStream.write(TEST_CONTEXT.getBytes()); - abfsOutputStream.flush(); - abfsOutputStream.hsync(); - } - abfs.create(testPath2, true); - abfs.create(testPath3, true); + // Write + abfs.mkdirs(testFile); + try (FSDataOutputStream abfsOutputStream = abfs.create(testPath1, true)) { + abfsOutputStream.write(TEST_CONTEXT.getBytes()); + abfsOutputStream.flush(); + abfsOutputStream.hsync(); + } + abfs.create(testPath2, true); + abfs.create(testPath3, true); - // Check file status - ContractTestUtils.assertIsFile(abfs, testPath1); + // Check file status + ContractTestUtils.assertIsFile(abfs, testPath1); - try (BufferedReader br = new BufferedReader( - new InputStreamReader(abfs.open(testPath1)))) { - String line = br.readLine(); - assertEquals(TEST_CONTEXT, line, "Wrong text from " + abfs); + try (BufferedReader br = new BufferedReader( + new InputStreamReader(abfs.open(testPath1)))) { + String line = br.readLine(); + assertEquals("Wrong text from " + abfs, + TEST_CONTEXT, line); + } + // --- RENAME DIR --- + boolean renamed = wasb.rename(testFile, testFile1); + Assertions.assertThat(renamed) + .as("Rename failed") + .isTrue(); + // --- LIST FILES IN DIRECTORY --- + int listResult = listAllFilesAndDirs(wasb, testFile1); + Assertions.assertThat(listResult) + .as("Expected only 5 entries under path: %s", testFile1) + .isEqualTo(5); } - // --- RENAME DIR --- - boolean renamed = wasb.rename(testFile, testFile1); - Assertions.assertThat(renamed) - .as("Rename failed") - .isTrue(); - // --- LIST FILES IN DIRECTORY --- - int listResult = listAllFilesAndDirs(wasb, testFile1); - Assertions.assertThat(listResult) - .as("Expected only 5 entries under path: %s", testFile1) - .isEqualTo(5); } /** @@ -1825,44 +1758,43 @@ public void testScenario34() throws Exception { Configuration conf = getRawConfiguration(); conf.setBoolean(FS_AZURE_ABFS_ENABLE_CHECKSUM_VALIDATION, true); FileSystem fileSystem = FileSystem.newInstance(conf); - AzureBlobFileSystem abfs = (AzureBlobFileSystem) fileSystem; - assumeThat(getIsNamespaceEnabled(abfs)) - .as("Namespace enabled account does not support this test") - .isFalse(); - NativeAzureFileSystem wasb = getWasbFileSystem(); + try (AzureBlobFileSystem abfs = (AzureBlobFileSystem) fileSystem) { + NativeAzureFileSystem wasb = getWasbFileSystem(); - Path testFile = path("/testReadFile"); - Path testPath1 = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); - Path testPath2 = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); - Path testPath3 = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); + Path testFile = path("/testReadFile"); + Path testPath1 = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); + Path testPath2 = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); + Path testPath3 = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); - // Write - abfs.mkdirs(testFile); - try (FSDataOutputStream abfsOutputStream = abfs.create(testPath1, true)) { - abfsOutputStream.write(TEST_CONTEXT.getBytes()); - abfsOutputStream.flush(); - abfsOutputStream.hsync(); - } - abfs.create(testPath3, true); + // Write + abfs.mkdirs(testFile); + try (FSDataOutputStream abfsOutputStream = abfs.create(testPath1, true)) { + abfsOutputStream.write(TEST_CONTEXT.getBytes()); + abfsOutputStream.flush(); + abfsOutputStream.hsync(); + } + abfs.create(testPath3, true); - // Check file status - ContractTestUtils.assertIsFile(abfs, testPath1); + // Check file status + ContractTestUtils.assertIsFile(abfs, testPath1); - try (BufferedReader br = new BufferedReader( - new InputStreamReader(abfs.open(testPath1)))) { - String line = br.readLine(); - assertEquals(TEST_CONTEXT, line, "Wrong text from " + abfs); + try (BufferedReader br = new BufferedReader( + new InputStreamReader(abfs.open(testPath1)))) { + String line = br.readLine(); + assertEquals("Wrong text from " + abfs, + TEST_CONTEXT, line); + } + // --- RENAME DIR --- + boolean renamed = wasb.rename(testPath1, testPath2); + Assertions.assertThat(renamed) + .as("Rename failed") + .isTrue(); + // --- LIST FILES IN DIRECTORY --- + int listResult = listAllFilesAndDirs(abfs, testFile); + Assertions.assertThat(listResult) + .as("Expected only 4 entries under path: %s", testFile) + .isEqualTo(4); } - // --- RENAME DIR --- - boolean renamed = wasb.rename(testPath1, testPath2); - Assertions.assertThat(renamed) - .as("Rename failed") - .isTrue(); - // --- LIST FILES IN DIRECTORY --- - int listResult = listAllFilesAndDirs(abfs, testFile); - Assertions.assertThat(listResult) - .as("Expected only 4 entries under path: %s", testFile) - .isEqualTo(4); } /** @@ -1874,44 +1806,43 @@ public void testScenario35() throws Exception { Configuration conf = getRawConfiguration(); conf.setBoolean(FS_AZURE_ABFS_ENABLE_CHECKSUM_VALIDATION, true); FileSystem fileSystem = FileSystem.newInstance(conf); - AzureBlobFileSystem abfs = (AzureBlobFileSystem) fileSystem; - assumeThat(getIsNamespaceEnabled(abfs)) - .as("Namespace enabled account does not support this test") - .isFalse(); - NativeAzureFileSystem wasb = getWasbFileSystem(); + try (AzureBlobFileSystem abfs = (AzureBlobFileSystem) fileSystem) { + NativeAzureFileSystem wasb = getWasbFileSystem(); - Path testFile = path("/testReadFile"); - Path testPath1 = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); - Path testPath2 = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); - Path testPath3 = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); + Path testFile = path("/testReadFile"); + Path testPath1 = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); + Path testPath2 = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); + Path testPath3 = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); - // Write - wasb.mkdirs(testFile); - try (FSDataOutputStream nativeFsStream = wasb.create(testPath1, true)) { - nativeFsStream.write(TEST_CONTEXT.getBytes()); - nativeFsStream.flush(); - nativeFsStream.hsync(); - } - wasb.create(testPath3, true); + // Write + wasb.mkdirs(testFile); + try (FSDataOutputStream nativeFsStream = wasb.create(testPath1, true)) { + nativeFsStream.write(TEST_CONTEXT.getBytes()); + nativeFsStream.flush(); + nativeFsStream.hsync(); + } + wasb.create(testPath3, true); - // Check file status - ContractTestUtils.assertIsFile(abfs, testPath1); + // Check file status + ContractTestUtils.assertIsFile(abfs, testPath1); - try (BufferedReader br = new BufferedReader( - new InputStreamReader(abfs.open(testPath1)))) { - String line = br.readLine(); - assertEquals(TEST_CONTEXT, line, "Wrong text from " + abfs); + try (BufferedReader br = new BufferedReader( + new InputStreamReader(abfs.open(testPath1)))) { + String line = br.readLine(); + assertEquals("Wrong text from " + abfs, + TEST_CONTEXT, line); + } + // --- RENAME DIR --- + boolean renamed = abfs.rename(testPath1, testPath2); + Assertions.assertThat(renamed) + .as("Rename failed") + .isTrue(); + // --- LIST FILES IN DIRECTORY --- + int listResult = listAllFilesAndDirs(wasb, testFile); + Assertions.assertThat(listResult) + .as("Expected only 4 entries under path: %s", testFile) + .isEqualTo(4); } - // --- RENAME DIR --- - boolean renamed = abfs.rename(testPath1, testPath2); - Assertions.assertThat(renamed) - .as("Rename failed") - .isTrue(); - // --- LIST FILES IN DIRECTORY --- - int listResult = listAllFilesAndDirs(wasb, testFile); - Assertions.assertThat(listResult) - .as("Expected only 4 entries under path: %s", testFile) - .isEqualTo(4); } /** @@ -1924,38 +1855,37 @@ public void testScenario36() throws Exception { Configuration conf = getRawConfiguration(); conf.setBoolean(FS_AZURE_ABFS_ENABLE_CHECKSUM_VALIDATION, true); FileSystem fileSystem = FileSystem.newInstance(conf); - AzureBlobFileSystem abfs = (AzureBlobFileSystem) fileSystem; - assumeThat(getIsNamespaceEnabled(abfs)) - .as("Namespace enabled account does not support this test") - .isFalse(); - NativeAzureFileSystem wasb = getWasbFileSystem(); + try (AzureBlobFileSystem abfs = (AzureBlobFileSystem) fileSystem) { + NativeAzureFileSystem wasb = getWasbFileSystem(); - Path testFile = path("/testReadFile"); - Path testPath1 = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); - Path testPath3 = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); + Path testFile = path("/testReadFile"); + Path testPath1 = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); + Path testPath3 = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); - // Write - wasb.mkdirs(testFile); - try (FSDataOutputStream nativeFsStream = wasb.create(testPath1, true)) { - nativeFsStream.write(TEST_CONTEXT.getBytes()); - nativeFsStream.flush(); - nativeFsStream.hsync(); - } - wasb.create(testPath3, true); + // Write + wasb.mkdirs(testFile); + try (FSDataOutputStream nativeFsStream = wasb.create(testPath1, true)) { + nativeFsStream.write(TEST_CONTEXT.getBytes()); + nativeFsStream.flush(); + nativeFsStream.hsync(); + } + wasb.create(testPath3, true); - // Check file status - ContractTestUtils.assertIsFile(abfs, testPath1); + // Check file status + ContractTestUtils.assertIsFile(abfs, testPath1); - try (BufferedReader br = new BufferedReader( - new InputStreamReader(abfs.open(testPath1)))) { - String line = br.readLine(); - assertEquals(TEST_CONTEXT, line, "Wrong text from " + abfs); + try (BufferedReader br = new BufferedReader( + new InputStreamReader(abfs.open(testPath1)))) { + String line = br.readLine(); + assertEquals("Wrong text from " + abfs, + TEST_CONTEXT, line); + } + // --- RENAME DIR --- + boolean renamed = abfs.rename(testFile, testFile); + Assertions.assertThat(renamed) + .as("Rename operation should have failed but returned true") + .isFalse(); } - // --- RENAME DIR --- - boolean renamed = abfs.rename(testFile, testFile); - Assertions.assertThat(renamed) - .as("Rename operation should have failed but returned true") - .isFalse(); } /** @@ -1967,39 +1897,38 @@ public void testScenario37() throws Exception { Configuration conf = getRawConfiguration(); conf.setBoolean(FS_AZURE_ABFS_ENABLE_CHECKSUM_VALIDATION, true); FileSystem fileSystem = FileSystem.newInstance(conf); - AzureBlobFileSystem abfs = (AzureBlobFileSystem) fileSystem; - assumeThat(getIsNamespaceEnabled(abfs)) - .as("Namespace enabled account does not support this test") - .isFalse(); - NativeAzureFileSystem wasb = getWasbFileSystem(); + try (AzureBlobFileSystem abfs = (AzureBlobFileSystem) fileSystem) { + NativeAzureFileSystem wasb = getWasbFileSystem(); - Path testFile = path("/testReadFile"); - Path testPath1 = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); - Path testPath2 = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); - Path testPath3 = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); + Path testFile = path("/testReadFile"); + Path testPath1 = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); + Path testPath2 = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); + Path testPath3 = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); - // Write - abfs.mkdirs(testFile); - try (FSDataOutputStream abfsOutputStream = abfs.create(testPath1, true)) { - abfsOutputStream.write(TEST_CONTEXT.getBytes()); - abfsOutputStream.flush(); - abfsOutputStream.hsync(); - } - abfs.create(testPath3, true); + // Write + abfs.mkdirs(testFile); + try (FSDataOutputStream abfsOutputStream = abfs.create(testPath1, true)) { + abfsOutputStream.write(TEST_CONTEXT.getBytes()); + abfsOutputStream.flush(); + abfsOutputStream.hsync(); + } + abfs.create(testPath3, true); - // Check file status - ContractTestUtils.assertIsFile(abfs, testPath1); + // Check file status + ContractTestUtils.assertIsFile(abfs, testPath1); - try (BufferedReader br = new BufferedReader( - new InputStreamReader(abfs.open(testPath1)))) { - String line = br.readLine(); - assertEquals(TEST_CONTEXT, line, "Wrong text from " + abfs); + try (BufferedReader br = new BufferedReader( + new InputStreamReader(abfs.open(testPath1)))) { + String line = br.readLine(); + assertEquals("Wrong text from " + abfs, + TEST_CONTEXT, line); + } + // --- RENAME NON EXISTENT FILE --- + boolean renamed = wasb.rename(testPath2, testPath3); + Assertions.assertThat(renamed) + .as("Rename operation should have failed but returned true") + .isFalse(); } - // --- RENAME NON EXISTENT FILE --- - boolean renamed = wasb.rename(testPath2, testPath3); - Assertions.assertThat(renamed) - .as("Rename operation should have failed but returned true") - .isFalse(); } /** @@ -2011,45 +1940,43 @@ public void testScenario38() throws Exception { Configuration conf = getRawConfiguration(); conf.setBoolean(FS_AZURE_ABFS_ENABLE_CHECKSUM_VALIDATION, true); FileSystem fileSystem = FileSystem.newInstance(conf); - AzureBlobFileSystem abfs = (AzureBlobFileSystem) fileSystem; - assumeThat(getIsNamespaceEnabled(abfs)) - .as("Namespace enabled account does not support this test") - .isFalse(); - NativeAzureFileSystem wasb = getWasbFileSystem(); + try (AzureBlobFileSystem abfs = (AzureBlobFileSystem) fileSystem) { + NativeAzureFileSystem wasb = getWasbFileSystem(); - Path testFile = path("/testReadFile"); - Path path = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); + Path testFile = path("/testReadFile"); + Path path = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); - // Write - try (FSDataOutputStream nativeFsStream = wasb.create(path, true)) { - nativeFsStream.write(TEST_CONTEXT.getBytes()); - nativeFsStream.flush(); - nativeFsStream.hsync(); + // Write + try (FSDataOutputStream nativeFsStream = wasb.create(path, true)) { + nativeFsStream.write(TEST_CONTEXT.getBytes()); + nativeFsStream.flush(); + nativeFsStream.hsync(); + } + // --- VALIDATE FILE --- + FileStatus status = wasb.getFileStatus(path); + assertIsFile(path, status); + + // --- SET XATTR #1 --- + wasb.setXAttr(path, ATTRIBUTE_NAME_1, ATTRIBUTE_VALUE_1); + byte[] readValue = wasb.getXAttr(path, ATTRIBUTE_NAME_1); + ITestAzureBlobFileSystemAttributes.assertAttributeEqual(abfs, readValue, ATTRIBUTE_VALUE_1, "one"); + + // --- SET XATTR #2 --- + wasb.setXAttr(path, ATTRIBUTE_NAME_2, ATTRIBUTE_VALUE_2); + readValue = wasb.getXAttr(path, ATTRIBUTE_NAME_2); + ITestAzureBlobFileSystemAttributes.assertAttributeEqual(abfs, readValue, ATTRIBUTE_VALUE_2, "two"); + + // --- VERIFY XATTR #1 AGAIN --- + readValue = wasb.getXAttr(path, ATTRIBUTE_NAME_1); + ITestAzureBlobFileSystemAttributes.assertAttributeEqual(abfs, readValue, ATTRIBUTE_VALUE_1, "one"); + + wasb.create(path, true); + FileStatus fileStatus = abfs.getFileStatus(path); + Assertions.assertThat(fileStatus.getLen()) + .as("Expected file length to be 0 after overwrite") + .isEqualTo(0L); + wasb.delete(path, true); } - // --- VALIDATE FILE --- - FileStatus status = wasb.getFileStatus(path); - assertIsFile(path, status); - - // --- SET XATTR #1 --- - wasb.setXAttr(path, ATTRIBUTE_NAME_1, ATTRIBUTE_VALUE_1); - byte[] readValue = wasb.getXAttr(path, ATTRIBUTE_NAME_1); - ITestAzureBlobFileSystemAttributes.assertAttributeEqual(abfs, readValue, ATTRIBUTE_VALUE_1, "one"); - - // --- SET XATTR #2 --- - wasb.setXAttr(path, ATTRIBUTE_NAME_2, ATTRIBUTE_VALUE_2); - readValue = wasb.getXAttr(path, ATTRIBUTE_NAME_2); - ITestAzureBlobFileSystemAttributes.assertAttributeEqual(abfs, readValue, ATTRIBUTE_VALUE_2, "two"); - - // --- VERIFY XATTR #1 AGAIN --- - readValue = wasb.getXAttr(path, ATTRIBUTE_NAME_1); - ITestAzureBlobFileSystemAttributes.assertAttributeEqual(abfs, readValue, ATTRIBUTE_VALUE_1, "one"); - - wasb.create(path, true); - FileStatus fileStatus = abfs.getFileStatus(path); - Assertions.assertThat(fileStatus.getLen()) - .as("Expected file length to be 0 after overwrite") - .isEqualTo(0L); - wasb.delete(path, true); } /** @@ -2061,46 +1988,45 @@ public void testScenario39() throws Exception { Configuration conf = getRawConfiguration(); conf.setBoolean(FS_AZURE_ABFS_ENABLE_CHECKSUM_VALIDATION, true); FileSystem fileSystem = FileSystem.newInstance(conf); - AzureBlobFileSystem abfs = (AzureBlobFileSystem) fileSystem; - assumeThat(getIsNamespaceEnabled(abfs)) - .as("Namespace enabled account does not support this test") - .isFalse(); - NativeAzureFileSystem wasb = getWasbFileSystem(); + try (AzureBlobFileSystem abfs = (AzureBlobFileSystem) fileSystem) { + NativeAzureFileSystem wasb = getWasbFileSystem(); - String testRunId = UUID.randomUUID().toString(); - Path baseDir = path("/testScenario39_" + testRunId); - Path testFile = new Path(baseDir, "testReadFile"); - Path testPath1 = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); - Path testPath2 = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); - Path testPath3 = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); + String testRunId = UUID.randomUUID().toString(); + Path baseDir = path("/testScenario39_" + testRunId); + Path testFile = new Path(baseDir, "testReadFile"); + Path testPath1 = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); + Path testPath2 = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); + Path testPath3 = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); - // Write - wasb.mkdirs(testFile); - try (FSDataOutputStream nativeFsStream = wasb.create(testPath1, true)) { - nativeFsStream.write(TEST_CONTEXT.getBytes()); - nativeFsStream.flush(); - nativeFsStream.hsync(); - } - wasb.create(testPath3, true); + // Write + wasb.mkdirs(testFile); + try (FSDataOutputStream nativeFsStream = wasb.create(testPath1, true)) { + nativeFsStream.write(TEST_CONTEXT.getBytes()); + nativeFsStream.flush(); + nativeFsStream.hsync(); + } + wasb.create(testPath3, true); - // Check file status - ContractTestUtils.assertIsFile(abfs, testPath1); + // Check file status + ContractTestUtils.assertIsFile(abfs, testPath1); - try (BufferedReader br = new BufferedReader( - new InputStreamReader(abfs.open(testPath1)))) { - String line = br.readLine(); - assertEquals(TEST_CONTEXT, line, "Wrong text from " + abfs); + try (BufferedReader br = new BufferedReader( + new InputStreamReader(abfs.open(testPath1)))) { + String line = br.readLine(); + assertEquals("Wrong text from " + abfs, + TEST_CONTEXT, line); + } + // --- RENAME DIR --- + boolean renamed = wasb.rename(testPath1, testPath2); + Assertions.assertThat(renamed) + .as("Rename failed") + .isTrue(); + // --- LIST FILES IN DIRECTORY --- + int listResult = listAllFilesAndDirs(wasb, testFile); + Assertions.assertThat(listResult) + .as("Expected only 4 entries under path: %s", testFile) + .isEqualTo(4); } - // --- RENAME DIR --- - boolean renamed = wasb.rename(testPath1, testPath2); - Assertions.assertThat(renamed) - .as("Rename failed") - .isTrue(); - // --- LIST FILES IN DIRECTORY --- - int listResult = listAllFilesAndDirs(wasb, testFile); - Assertions.assertThat(listResult) - .as("Expected only 4 entries under path: %s", testFile) - .isEqualTo(4); } /** diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/AbfsClientTestUtil.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/AbfsClientTestUtil.java index 4ab5f68bb3ca5..b9dcefc35e239 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/AbfsClientTestUtil.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/AbfsClientTestUtil.java @@ -166,7 +166,12 @@ public static void setMockAbfsRestOperationForFlushOperation( requestHeaders.add(new AbfsHttpHeader(CONTENT_LENGTH, String.valueOf(buffer.length))); requestHeaders.add(new AbfsHttpHeader(CONTENT_TYPE, APPLICATION_XML)); requestHeaders.add(new AbfsHttpHeader(IF_MATCH, eTag)); - requestHeaders.add(new AbfsHttpHeader(X_MS_BLOB_CONTENT_MD5, blobMd5)); + requestHeaders.add(new AbfsHttpHeader( + X_MS_BLOB_CONTENT_MD5, + (spiedClient.isFullBlobChecksumValidationEnabled() && blobMd5 != null) + ? blobMd5 + : spiedClient.computeMD5Hash(buffer, 0, buffer.length) + )); final AbfsUriQueryBuilder abfsUriQueryBuilder = spiedClient.createDefaultUriQueryBuilder(); abfsUriQueryBuilder.addQuery(QUERY_PARAM_COMP, BLOCKLIST); abfsUriQueryBuilder.addQuery(QUERY_PARAM_CLOSE, String.valueOf(false)); diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestAbfsOutputStream.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestAbfsOutputStream.java index 51317f8e89562..dc68fd7e4d07d 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestAbfsOutputStream.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestAbfsOutputStream.java @@ -27,6 +27,7 @@ import java.net.URL; import java.security.MessageDigest; import java.util.Arrays; +import java.util.Random; import java.util.concurrent.TimeUnit; import org.assertj.core.api.Assertions; @@ -57,6 +58,7 @@ import static java.net.HttpURLConnection.HTTP_CONFLICT; import static java.net.HttpURLConnection.HTTP_UNAVAILABLE; import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.EXPECT_100_JDK_ERROR; +import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ABFS_ENABLE_CHECKSUM_VALIDATION; import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ACCOUNT_IS_EXPECT_HEADER_ENABLED; import static org.apache.hadoop.fs.azurebfs.constants.HttpHeaderConfigurations.EXPECT; import static org.apache.hadoop.test.LambdaTestUtils.intercept; @@ -438,11 +440,10 @@ public void testNoNetworkCallsForSecondFlush() throws Exception { */ @Test public void testResetCalledOnExceptionInRemoteFlush() throws Exception { - AzureBlobFileSystem fs = Mockito.spy(getFileSystem()); - assumeThat(getIsNamespaceEnabled(fs)).isFalse(); - AzureBlobFileSystemStore store = Mockito.spy(fs.getAbfsStore()); + assumeHnsDisabled(); assumeBlobServiceType(); assumeThat(isAppendBlobEnabled()).as("Not valid for APPEND BLOB").isFalse(); + AzureBlobFileSystem fs = Mockito.spy(getFileSystem()); // Create a file and spy on AbfsOutputStream Path path = new Path("/testFile"); @@ -483,6 +484,113 @@ public void testResetCalledOnExceptionInRemoteFlush() throws Exception { //expected exception } // Verify that reset was called on the message digest - Mockito.verify(mockMessageDigest, Mockito.times(1)).reset(); + if (spiedClient.isFullBlobChecksumValidationEnabled()) { + Assertions.assertThat(Mockito.mockingDetails(mockMessageDigest).getInvocations() + .stream() + .filter(i -> i.getMethod().getName().equals("reset")) + .count()) + .as("Expected MessageDigest.reset() to be called exactly once when checksum validation is enabled") + .isEqualTo(1); + } + } + + /** + * Tests that the message digest is reset when an exception occurs during remote flush. + * Simulates a failure in the flush operation and verifies reset is called on MessageDigest. + */ + @Test + public void testNoChecksumComputedWhenConfigFalse() throws Exception { + assumeThat(isAppendBlobEnabled()).as("Not valid for APPEND BLOB").isFalse(); + assumeBlobServiceType(); + assumeHnsDisabled(); + Configuration conf = getRawConfiguration(); + conf.setBoolean(FS_AZURE_ABFS_ENABLE_CHECKSUM_VALIDATION, false); + FileSystem fileSystem = FileSystem.newInstance(conf); + try (AzureBlobFileSystem fs = (AzureBlobFileSystem) fileSystem) { + AzureBlobFileSystemStore store = Mockito.spy(fs.getAbfsStore()); + + // Create spies for the client handler and blob client + AbfsClientHandler clientHandler = Mockito.spy(store.getClientHandler()); + AbfsBlobClient blobClient = Mockito.spy(clientHandler.getBlobClient()); + + // Set up the spies to return the mocked objects + Mockito.doReturn(clientHandler).when(store).getClientHandler(); + Mockito.doReturn(blobClient).when(clientHandler).getBlobClient(); + Mockito.doReturn(blobClient).when(clientHandler).getIngressClient(); + AbfsOutputStream abfsOutputStream = Mockito.spy( + (AbfsOutputStream) fs.create(new Path("/test/file")) + .getWrappedStream()); + AzureIngressHandler ingressHandler = Mockito.spy( + abfsOutputStream.getIngressHandler()); + Mockito.doReturn(ingressHandler) + .when(abfsOutputStream) + .getIngressHandler(); + Mockito.doReturn(blobClient).when(ingressHandler).getClient(); + FSDataOutputStream os = Mockito.spy( + new FSDataOutputStream(abfsOutputStream, null)); + AbfsOutputStream out = (AbfsOutputStream) os.getWrappedStream(); + byte[] bytes = new byte[1024 * 1024 * 4]; + new Random().nextBytes(bytes); + // Write some bytes and attempt to flush, which should retry + out.write(bytes); + out.hsync(); + Assertions.assertThat(Mockito.mockingDetails(blobClient).getInvocations() + .stream() + .filter( + i -> i.getMethod().getName().equals("addCheckSumHeaderForWrite")) + .count()) + .as("Expected addCheckSumHeaderForWrite() to be called exactly 0 times") + .isZero(); + } + } + + /** + * Tests that the message digest is reset when an exception occurs during remote flush. + * Simulates a failure in the flush operation and verifies reset is called on MessageDigest. + */ + @Test + public void testChecksumComputedWhenConfigTrue() throws Exception { + assumeHnsDisabled(); + assumeBlobServiceType(); + assumeThat(isAppendBlobEnabled()).as("Not valid for APPEND BLOB") + .isFalse(); + Configuration conf = getRawConfiguration(); + conf.setBoolean(FS_AZURE_ABFS_ENABLE_CHECKSUM_VALIDATION, true); + FileSystem fileSystem = FileSystem.newInstance(conf); + try (AzureBlobFileSystem fs = (AzureBlobFileSystem) fileSystem) { + AzureBlobFileSystemStore store = Mockito.spy(fs.getAbfsStore()); + // Create spies for the client handler and blob client + AbfsClientHandler clientHandler = Mockito.spy(store.getClientHandler()); + AbfsBlobClient blobClient = Mockito.spy(clientHandler.getBlobClient()); + + // Set up the spies to return the mocked objects + Mockito.doReturn(clientHandler).when(store).getClientHandler(); + Mockito.doReturn(blobClient).when(clientHandler).getBlobClient(); + Mockito.doReturn(blobClient).when(clientHandler).getIngressClient(); + AbfsOutputStream abfsOutputStream = Mockito.spy( + (AbfsOutputStream) fs.create(new Path("/test/file")) + .getWrappedStream()); + AzureIngressHandler ingressHandler = Mockito.spy( + abfsOutputStream.getIngressHandler()); + Mockito.doReturn(ingressHandler) + .when(abfsOutputStream) + .getIngressHandler(); + Mockito.doReturn(blobClient).when(ingressHandler).getClient(); + FSDataOutputStream os = Mockito.spy( + new FSDataOutputStream(abfsOutputStream, null)); + AbfsOutputStream out = (AbfsOutputStream) os.getWrappedStream(); + byte[] bytes = new byte[1024 * 1024 * 4]; + new Random().nextBytes(bytes); + // Write some bytes and attempt to flush, which should retry + out.write(bytes); + out.hsync(); + Assertions.assertThat(Mockito.mockingDetails(blobClient).getInvocations() + .stream() + .filter( + i -> i.getMethod().getName().equals("addCheckSumHeaderForWrite")) + .count()) + .as("Expected addCheckSumHeaderForWrite() to be called exactly once") + .isEqualTo(1); + } } }