Skip to content

Commit cf428f6

Browse files
amrita-hegdeAmrita Hegde
amrita-hegde
and
Amrita Hegde
authored
Support pagination for enumerating deleted items (#26974)
Co-authored-by: Amrita Hegde <[email protected]>
1 parent dfce89d commit cf428f6

15 files changed

+632
-128
lines changed

‎src/DataLakeStore/DataLakeStore.Test/ScenarioTests/AdlsAliasTests.cs

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -70,7 +70,7 @@ public void TestAdlsTrustedIdProvider()
7070
}
7171

7272
[Fact]
73-
[Trait(Category.AcceptanceType, Category.CheckIn)]
73+
[Trait(Category.AcceptanceType, Category.LiveOnly)]
7474
public void TestAdlsFileSystem()
7575
{
7676
var workingPath = Path.GetDirectoryName(new Uri(Assembly.GetExecutingAssembly().Location).AbsolutePath);
@@ -79,7 +79,7 @@ public void TestAdlsFileSystem()
7979
}
8080

8181
[Fact]
82-
[Trait(Category.AcceptanceType, Category.CheckIn)]
82+
[Trait(Category.AcceptanceType, Category.LiveOnly)]
8383
public void TestAdlsFileSystemPermissions()
8484
{
8585
TestRunner.RunTestScript($"Test-DataLakeStoreFileSystemPermissions -location '{TestFileSystemPermissionResourceGroupLocation}'");
@@ -93,7 +93,7 @@ public void TestNegativeAdlsAccount()
9393
}
9494

9595
[Fact]
96-
[Trait(Category.AcceptanceType, Category.CheckIn)]
96+
[Trait(Category.AcceptanceType, Category.LiveOnly)]
9797
public void TestAdlsEnumerateAndRestoreDeletedItem()
9898
{
9999
TestRunner.RunTestScript($"Test-AdlsEnumerateAndRestoreDeletedItem -location '{ResourceGroupLocation}'");

‎src/DataLakeStore/DataLakeStore.Test/ScenarioTests/AdlsTests.cs

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ namespace Microsoft.Azure.Commands.DataLakeStore.Test.ScenarioTests
2424

2525
public class AdlsTests : DataLakeStoreTestRunner
2626
{
27+
private readonly string AccountName = "sss-datalakestore-test-c17";
2728
private readonly string ResourceGroupLocation = "westus";
2829
private readonly string TestFileSystemPermissionResourceGroupLocation = "ukwest";
2930
private readonly string TestFileSystemResourceGroupLocation = "ukwest";
@@ -69,7 +70,7 @@ public void TestAdlsAccountTiers()
6970
}
7071

7172
[Fact]
72-
[Trait(Category.AcceptanceType, Category.CheckIn)]
73+
[Trait(Category.AcceptanceType, Category.LiveOnly)]
7374
public void TestAdlsFileSystem()
7475
{
7576
var workingPath = Path.GetDirectoryName(new Uri(Assembly.GetExecutingAssembly().Location).AbsolutePath);
@@ -78,7 +79,7 @@ public void TestAdlsFileSystem()
7879
}
7980

8081
[Fact]
81-
[Trait(Category.AcceptanceType, Category.CheckIn)]
82+
[Trait(Category.AcceptanceType, Category.LiveOnly)]
8283
public void TestAdlsFileSystemPermissions()
8384
{
8485
TestRunner.RunTestScript($"Test-DataLakeStoreFileSystemPermissions -location '{TestFileSystemPermissionResourceGroupLocation}'");
@@ -92,10 +93,17 @@ public void TestNegativeAdlsAccount()
9293
}
9394

9495
[Fact]
95-
[Trait(Category.AcceptanceType, Category.CheckIn)]
96+
[Trait(Category.AcceptanceType, Category.LiveOnly)]
9697
public void TestAdlsEnumerateAndRestoreDeletedItem()
9798
{
98-
TestRunner.RunTestScript($"Test-EnumerateAndRestoreDataLakeStoreDeletedItem -location '{ResourceGroupLocation}'");
99+
TestRunner.RunTestScript($"Test-EnumerateAndRestoreDataLakeStoreDeletedItem -location '{ResourceGroupLocation}' -accountName '{AccountName}'");
100+
}
101+
102+
[Fact]
103+
[Trait(Category.AcceptanceType, Category.LiveOnly)]
104+
public void TestAdlsEnumerateDeletedItemsWithToken()
105+
{
106+
TestRunner.RunTestScript($"Test-EnumerateDataLakeStoreDeletedItemsWithToken -location '{ResourceGroupLocation}' -accountName '{AccountName}'");
99107
}
100108
}
101109
}

‎src/DataLakeStore/DataLakeStore.Test/ScenarioTests/AdlsTests.ps1

Lines changed: 161 additions & 77 deletions
Original file line numberDiff line numberDiff line change
@@ -722,7 +722,7 @@ function Test-DataLakeStoreFileSystem
722722
Assert-True {Remove-AzDataLakeStoreItem -Account $accountName -paths $summaryFolder -force -recurse -passthru} "Remove folder failed"
723723
Assert-Throws {Get-AzDataLakeStoreItem -Account $accountName -path $summaryFolder}
724724
Assert-True {Remove-AzDataLakeStoreItem -Account $accountName -paths $encodingFolder -force -recurse -passthru} "Remove folder failed"
725-
725+
726726
# Delete Data Lake account
727727
Assert-True {Remove-AzDataLakeStoreAccount -ResourceGroupName $resourceGroupName -Name $accountName -Force -PassThru} "Remove Account failed."
728728

@@ -760,7 +760,7 @@ function Test-DataLakeStoreFileSystemPermissions
760760
$accountName = Get-DataLakeStoreAccountName
761761
New-AzResourceGroup -Name $resourceGroupName -Location $location
762762
$accountCreated = New-AzDataLakeStoreAccount -ResourceGroupName $resourceGroupName -Name $accountName -Location $location -DisableEncryption
763-
763+
764764
Assert-AreEqual $accountName $accountCreated.Name
765765
Assert-AreEqual $location $accountCreated.Location
766766
Assert-AreEqual "Microsoft.DataLakeStore/accounts" $accountCreated.Type
@@ -998,6 +998,87 @@ function CreateAndGetVirtualNetwork ($resourceGroupName, $vnetName, $location =
998998
return $getVnet
999999
}
10001000

1001+
<#
1002+
.SYNOPSIS
1003+
Helper function to create files and folders in DataLakeStore account.
1004+
#>
1005+
function Create-FilesAndFolders
1006+
{
1007+
param
1008+
(
1009+
[string]$accountName
1010+
)
1011+
1012+
$folderToCreate1 = "/adlTestDir1"
1013+
$folderToCreate2 = "/adlTestDir2"
1014+
$fileToCreate1 = "/adlTestDir1/adlfile1"
1015+
$fileToCreate2 = "/adlTestDir2/adlfile2"
1016+
1017+
$result = New-AzDataLakeStoreItem -Account $accountName -path $folderToCreate1 -Folder
1018+
Assert-NotNull $result "No value was returned on folder creation"
1019+
1020+
$result = New-AzDataLakeStoreItem -Account $accountName -path $folderToCreate2 -Folder
1021+
Assert-NotNull $result "No value was returned on folder creation"
1022+
1023+
$result = New-AzDataLakeStoreItem -Account $accountName -path $fileToCreate1
1024+
Assert-NotNull $result "No value was returned on empty file creation"
1025+
1026+
$result = New-AzDataLakeStoreItem -Account $accountName -path $fileToCreate2
1027+
Assert-NotNull $result "No value was returned on empty file creation"
1028+
}
1029+
1030+
<#
1031+
.SYNOPSIS
1032+
Helper function to delete files and folders in DataLakeStore account.
1033+
#>
1034+
function Delete-FilesAndFolders
1035+
{
1036+
param
1037+
(
1038+
[string]$accountName
1039+
)
1040+
1041+
$itemsToDelete = @(
1042+
"/adlTestDir1/adlfile1",
1043+
"/adlTestDir2/adlfile2",
1044+
"/adlTestDir1",
1045+
"/adlTestDir2"
1046+
)
1047+
1048+
foreach ($item in $itemsToDelete) {
1049+
if (Test-AzDataLakeStoreItem -Account $accountName -Path $item) {
1050+
Write-Host "Deleting item: $item"
1051+
Assert-True { Remove-AzDataLakeStoreItem -Account $accountName -Paths $item -Force -Passthru } "Remove item failed: $item"
1052+
} else {
1053+
Write-Host "Item does not exist: $item"
1054+
}
1055+
}
1056+
}
1057+
1058+
function Restore-DeletedItems {
1059+
param (
1060+
[string]$accountName,
1061+
[array]$items
1062+
)
1063+
1064+
foreach ($item in $items) {
1065+
if ([string]::IsNullOrEmpty($item.TrashDirPath) -or [string]::IsNullOrEmpty($item.OriginalPath)) {
1066+
continue
1067+
}
1068+
1069+
# Check if the destination already exists
1070+
$relativeOriginalPath = $item.OriginalPath -replace "^adl://[^/]+", ""
1071+
if (Test-AzDataLakeStoreItem -Account $accountName -Path $relativeOriginalPath) {
1072+
continue
1073+
}
1074+
1075+
# Attempt to restore the item
1076+
Assert-True {
1077+
Restore-AzDataLakeStoreDeletedItem -Account $accountName -Path $item.TrashDirPath -Destination $item.OriginalPath -Type $item.Type -Force -Passthru
1078+
}
1079+
}
1080+
}
1081+
10011082
<#
10021083
.SYNOPSIS
10031084
Tests DataLakeStore deleted items operations (Enumerate, Restore).
@@ -1007,7 +1088,9 @@ function Test-EnumerateAndRestoreDataLakeStoreDeletedItem
10071088
param
10081089
(
10091090
$fileToCopy,
1010-
$location
1091+
$location,
1092+
$accountName
1093+
10111094
)
10121095

10131096
if ([string]::IsNullOrEmpty($location))
@@ -1017,84 +1100,85 @@ function Test-EnumerateAndRestoreDataLakeStoreDeletedItem
10171100

10181101
try
10191102
{
1020-
# Creating Account
1021-
$resourceGroupName = Get-ResourceGroupName
1022-
$accountName = Get-DataLakeStoreAccountName + "-c12" # testing accountname validation
1023-
New-AzResourceGroup -Name $resourceGroupName -Location $location
1024-
$accountCreated = New-AzDataLakeStoreAccount -ResourceGroupName $resourceGroupName -Name $accountName -Location $location
1025-
1026-
Assert-AreEqual $accountName $accountCreated.Name
1027-
Assert-AreEqual $location $accountCreated.Location
1028-
Assert-AreEqual "Microsoft.DataLakeStore/accounts" $accountCreated.Type
1029-
Assert-True {$accountCreated.Id -like "*$resourceGroupName*"}
1030-
1031-
# In loop to check if account exists
1032-
for ($i = 0; $i -le 60; $i++)
1033-
{
1034-
[array]$accountGet = Get-AzDataLakeStoreAccount -ResourceGroupName $resourceGroupName -Name $accountName
1035-
if ($accountGet[0].ProvisioningState -like "Succeeded")
1036-
{
1037-
Assert-AreEqual $accountName $accountGet[0].Name
1038-
Assert-AreEqual $location $accountGet[0].Location
1039-
Assert-AreEqual "Microsoft.DataLakeStore/accounts" $accountGet[0].Type
1040-
Assert-True {$accountGet[0].Id -like "*$resourceGroupName*"}
1041-
break
1042-
}
1043-
1044-
Write-Host "account not yet provisioned. current state: $($accountGet[0].ProvisioningState)"
1045-
[Microsoft.WindowsAzure.Commands.Utilities.Common.TestMockSupport]::Delay(30000)
1046-
Assert-False {$i -eq 60} " Data Lake Store account is not in succeeded state even after 30 min."
1047-
}
1048-
1049-
# define all the files and folders
1050-
$folderToCreate1 = "/adlfolderTest1"
1051-
$folderToCreate2 = "/adlfolderTest2"
1052-
$fileToCreate1 = "/adlfolderTest1/adlfile1"
1053-
$fileToCreate2 = "/adlfolderTest2/adlfile2"
1054-
1055-
# Create and get Empty folder
1056-
$result = New-AzDataLakeStoreItem -Account $accountName -path $folderToCreate1 -Folder
1057-
Assert-NotNull $result "No value was returned on folder creation"
1058-
1059-
$result = New-AzDataLakeStoreItem -Account $accountName -path $folderToCreate2 -Folder
1060-
Assert-NotNull $result "No value was returned on folder creation"
1103+
Create-FilesAndFolders -accountName $accountName
1104+
Delete-FilesAndFolders -accountName $accountName
10611105

1062-
# Create and get Empty File
1063-
$result = New-AzDataLakeStoreItem -Account $accountName -path $fileToCreate1
1064-
Assert-NotNull $result "No value was returned on empty file creation"
1065-
$result = New-AzDataLakeStoreItem -Account $accountName -path $fileToCreate2
1066-
Assert-NotNull $result "No value was returned on empty file creation"
1067-
1068-
# delete a file
1069-
Assert-True {Remove-AzDataLakeStoreItem -Account $accountName -paths $fileToCreate1 -force -passthru } "Remove File Failed"
1070-
Assert-Throws {Get-AzDataLakeStoreItem -Account $accountName -path $fileToCreate1}
1071-
Assert-True {Remove-AzDataLakeStoreItem -Account $accountName -paths $fileToCreate2 -force -passthru } "Remove File Failed"
1072-
Assert-Throws {Get-AzDataLakeStoreItem -Account $accountName -path $fileToCreate2}
1073-
1074-
# search delete folder
1075-
$out = Get-AzDataLakeStoreDeletedItem -Account $accountName -filter "adlfolderTest1" -Count 1000
1076-
foreach($item in $out)
1077-
{
1078-
Restore-AzDataLakeStoreDeletedItem -Account $accountName -Path $item.TrashDirPath -Destination $item.OriginalPath -Type "file" -Force -Passthru
1079-
Assert-True { Restore-AzDataLakeStoreDeletedItem -Account $accountName -Path $item.TrashDirPath -Destination $item.OriginalPath -Type "file" -Force -Passthru}
1080-
}
1081-
1082-
$out = Get-AzDataLakeStoreDeletedItem -Account $accountName -filter "adlfolderTest2" -Count 1000
1083-
foreach($item in $out)
1084-
{
1085-
Assert-True { Restore-AzDataLakeStoreDeletedItem -Account $accountName $item -Force -Passthru}
1086-
}
1087-
1088-
# Delete Data Lake account
1089-
Assert-True {Remove-AzDataLakeStoreAccount -ResourceGroupName $resourceGroupName -Name $accountName -Force -PassThru} "Remove Account failed."
1106+
# search deleted folder
1107+
$out = Get-AzDataLakeStoreDeletedItem -Account $accountName -filter "adlTestDir1" -Count 1000
1108+
Restore-DeletedItems -accountName $accountName -items $out
10901109

1091-
# Verify that it is gone by trying to get it again
1092-
Assert-Throws {Get-AzDataLakeStoreAccount -ResourceGroupName $resourceGroupName -Name $accountName}
1110+
$out = Get-AzDataLakeStoreDeletedItem -Account $accountName -filter "adlTestDir2" -Count 1000
1111+
Restore-DeletedItems -accountName $accountName -items $out
10931112
}
10941113
finally
10951114
{
1096-
# cleanup the resource group that was used in case it still exists. This is a best effort task, we ignore failures here.
1097-
Invoke-HandledCmdlet -Command {Remove-AzDataLakeStoreAccount -ResourceGroupName $resourceGroupName -Name $accountName -Force -ErrorAction SilentlyContinue} -IgnoreFailures
1098-
Invoke-HandledCmdlet -Command {Remove-AzResourceGroup -Name $resourceGroupName -Force -ErrorAction SilentlyContinue} -IgnoreFailures
1115+
Delete-FilesAndFolders -accountName $accountName
1116+
# Skip cleanup for existing account and resource group
1117+
Write-Host "Skipping cleanup for existing account and resource group."
10991118
}
1119+
}
1120+
1121+
<#
1122+
.SYNOPSIS
1123+
Tests DataLakeStore deleted items operations with token (Enumerate with pagination).
1124+
#>
1125+
function Test-EnumerateDataLakeStoreDeletedItemsWithToken
1126+
{
1127+
param
1128+
(
1129+
$fileToCopy,
1130+
$location,
1131+
$accountName
1132+
)
1133+
1134+
if ([string]::IsNullOrEmpty($location))
1135+
{
1136+
$location = Get-Location -providerNamespace "Microsoft.CognitiveServices" -resourceType "accounts" -preferredLocation "West US";
1137+
}
1138+
1139+
try
1140+
{
1141+
Create-FilesAndFolders -accountName $accountName
1142+
Delete-FilesAndFolders -accountName $accountName
1143+
1144+
# Test basic enumeration with token
1145+
$out = Get-AzDataLakeStoreDeletedItemsWithToken -Account $accountName -filter "adlTestDir1" -Count 2
1146+
Assert-NotNull $out "No deleted items found for adlTestDir1"
1147+
1148+
# Check if we have pagination results
1149+
$hasMoreResults = $false
1150+
$continuationToken = $null
1151+
1152+
foreach ($item in $out) {
1153+
if ($item.ContinuationToken -ne $null -and $item.ContinuationToken -ne "") {
1154+
$hasMoreResults = $true
1155+
$continuationToken = $item.ContinuationToken
1156+
break
1157+
}
1158+
}
1159+
1160+
# If there are more results, test pagination
1161+
if ($hasMoreResults) {
1162+
$out2 = Get-AzDataLakeStoreDeletedItemsWithToken -Account $accountName -filter "adlTestDir1" -Count 2 -ListAfter $continuationToken
1163+
Assert-NotNull $out2 "Failed to get deleted items with continuation token"
1164+
}
1165+
1166+
# Test with different filter
1167+
$out3 = Get-AzDataLakeStoreDeletedItemsWithToken -Account $accountName -filter "adlTestDir2" -Count 2
1168+
Assert-NotNull $out3 "No deleted items found for adlTestDir2"
1169+
1170+
# Verify that the items returned have the expected properties
1171+
foreach ($item in $out) {
1172+
if ($null -ne $item.DeletedItem) {
1173+
Assert-NotNull $item.DeletedItem.OriginalPath "DeletedItem should have OriginalPath"
1174+
Assert-NotNull $item.DeletedItem.TrashDirPath "DeletedItem should have TrashDirPath"
1175+
Assert-NotNull $item.DeletedItem.Type "DeletedItem should have Type"
1176+
}
1177+
}
1178+
}
1179+
finally
1180+
{
1181+
# Skip cleanup for existing account and resource group
1182+
Write-Host "Skipping cleanup for existing account and resource group."
1183+
}
11001184
}

‎src/DataLakeStore/DataLakeStore.Test/ScenarioTests/DataLakeStoreTestRunner.cs

Lines changed: 4 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -57,18 +57,10 @@ protected DataLakeStoreTestRunner(ITestOutputHelper output)
5757
(ignoreResourcesClient, resourceProviders, userAgentsToIgnore) => new UrlDecodingRecordMatcher(ignoreResourcesClient, resourceProviders, userAgentsToIgnore)
5858
)
5959
.WithManagementClients(context =>
60-
{
61-
AdlsClientFactory.IsTest = true;
62-
var creds = context.GetClientCredentials(AzureEnvironment.Endpoint.AzureDataLakeStoreFileSystemEndpointSuffix);
63-
AdlsClientFactory.CustomDelegatingHAndler = context.AddHandlers(creds, new AdlMockDelegatingHandler());
64-
AdlsClientFactory.MockCredentials = creds;
65-
return new object();
66-
}
67-
)
68-
.WithCleanupAction(
69-
() => AdlsClientFactory.IsTest = false
70-
)
60+
{
61+
return new object();
62+
})
7163
.Build();
7264
}
7365
}
74-
}
66+
}

‎src/DataLakeStore/DataLakeStore/Az.DataLakeStore.psd1

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -84,7 +84,8 @@ CmdletsToExport = 'Add-AzDataLakeStoreFirewallRule', 'Add-AzDataLakeStoreItemCon
8484
'Export-AzDataLakeStoreItem', 'Get-AzDataLakeStoreAccount',
8585
'Get-AzDataLakeStoreChildItem',
8686
'Get-AzDataLakeStoreChildItemSummary',
87-
'Get-AzDataLakeStoreDeletedItem', 'Get-AzDataLakeStoreFirewallRule',
87+
'Get-AzDataLakeStoreDeletedItem', 'Get-AzDataLakeStoreDeletedItemsWithToken',
88+
'Get-AzDataLakeStoreFirewallRule',
8889
'Get-AzDataLakeStoreItem', 'Get-AzDataLakeStoreItemAclEntry',
8990
'Get-AzDataLakeStoreItemContent', 'Get-AzDataLakeStoreItemOwner',
9091
'Get-AzDataLakeStoreItemPermission',

‎src/DataLakeStore/DataLakeStore/ChangeLog.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
- Additional information about change #1
1919
-->
2020
## Upcoming Release
21+
* Added a new cmdlet GetAzureDataLakeStoreDeletedItemWithtoken to support pagination in GetAzureDataLakeStoreDeletedItem
2122

2223
## Version 1.4.0
2324
* Upgraded nuget package to signed package.

0 commit comments

Comments
 (0)