diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/FDBStoreTimer.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/FDBStoreTimer.java index c31508c8d2..387669244b 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/FDBStoreTimer.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/FDBStoreTimer.java @@ -182,6 +182,8 @@ public enum Events implements StoreTimer.Event { RANGE_SET_CONTAINS("range set contains key"), /** The amount of time checking if a {@link com.google.common.collect.RangeSet} is empty. */ RANGE_SET_IS_EMPTY("range set is empty"), + /** The amount of time importing a single KeyValue into a path. */ + IMPORT_DATA("import KeyValue"), /** The amount of time spent clearing the space taken by an index that has been removed from the meta-data. */ REMOVE_FORMER_INDEX("remove former index"), diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/DataInKeySpacePath.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/DataInKeySpacePath.java index f2d2284730..afbd34599a 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/DataInKeySpacePath.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/DataInKeySpacePath.java @@ -24,10 +24,13 @@ import com.apple.foundationdb.annotation.API; import com.apple.foundationdb.record.RecordCoreArgumentException; import com.apple.foundationdb.record.logging.LogMessageKeys; +import com.apple.foundationdb.tuple.ByteArrayUtil2; import com.apple.foundationdb.tuple.Tuple; import javax.annotation.Nonnull; import javax.annotation.Nullable; +import java.util.Arrays; +import java.util.Objects; /** * Class representing a {@link KeyValue} pair within in {@link KeySpacePath}. @@ -67,4 +70,22 @@ public Tuple getRemainder() { return remainder; } + @Override + public boolean equals(final Object o) { + if (o == null || getClass() != o.getClass()) { + return false; + } + final DataInKeySpacePath that = (DataInKeySpacePath)o; + return Objects.equals(path, that.path) && Objects.equals(remainder, that.remainder) && Arrays.equals(value, that.value); + } + + @Override + public int hashCode() { + return Objects.hash(path, remainder, Arrays.hashCode(value)); + } + + @Override + public String toString() { + return path + "+" + remainder + "->" + ByteArrayUtil2.loggable(value); + } } diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePath.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePath.java index ff228ef1be..f9e4b7e216 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePath.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePath.java @@ -410,4 +410,26 @@ default RecordCursor exportAllData(@Nonnull FDBRecordContext @Nonnull ScanProperties scanProperties) { throw new UnsupportedOperationException("exportAllData is not supported"); } + + /** + * Imports the provided data exported via {@link #exportAllData} into this {@code KeySpacePath}. + *

+ * This will validate that any data provided in {@code dataToImport} has a path that should be in this path, + * or one of the sub-directories, if not, the future will complete exceptionally with + * {@link RecordCoreIllegalImportDataException}. + * If there is any data already existing under this path, the new data will overwrite if the keys are the same. + * This will use the logical values in the {@link DataInKeySpacePath#getPath()} and + * {@link DataInKeySpacePath#getRemainder()} to determine the key, rather + * than the raw key, meaning that this will work even if the data was exported from a different cluster. + * Note, this will not correct for any cluster-specific data, other than {@link DirectoryLayerDirectory} data; + * for example, if you have versionstamps, that data will not align on the destination. + *

+ * @param context the transaction context in which to save the data + * @param dataToImport the data to be saved to the database + * @return a future to be completed once all data has been important. + */ + @API(API.Status.EXPERIMENTAL) + @Nonnull + CompletableFuture importData(@Nonnull FDBRecordContext context, + @Nonnull Iterable dataToImport); } diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathImpl.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathImpl.java index 794672fd04..09e53cd16a 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathImpl.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathImpl.java @@ -316,6 +316,49 @@ public RecordCursor exportAllData(@Nonnull FDBRecordContext 1); } + @Nonnull + @Override + public CompletableFuture importData(@Nonnull FDBRecordContext context, + @Nonnull Iterable dataToImport) { + return toTupleAsync(context).thenCompose(targetTuple -> { + // We use a mapPipelined here to help control the rate of insertions into the directory layer if those + // are happening. + // DirectoryLayer operations are done in a separate transaction for two reasons: + // 1. To reduce conflicts + // 2. So that it can immediately be cached without having to do it in a post-commit hook, which can make + // things complicated + // So if we just spun off a future for every data item, they would conflict like crazy, and retrying all + // of those conflicts would cause this future to take way longer than if we pipeline. + // This shouldn't make much of a difference in the general case because almost all the directory layer + // lookups should be from cache. + final RecordCursor insertionWork = RecordCursor.fromIterator(dataToImport.iterator()) + .mapPipelined(dataItem -> + dataItem.getPath().toTupleAsync(context).thenAccept(itemPathTuple -> { + // Validate that this data belongs under this path + if (!TupleHelpers.isPrefix(targetTuple, itemPathTuple)) { + throw new RecordCoreIllegalImportDataException( + "Data item path does not belong under target path", + "target", targetTuple, + "item", itemPathTuple); + } + + // Reconstruct the key using the path and remainder + Tuple keyTuple = itemPathTuple; + if (dataItem.getRemainder() != null) { + keyTuple = keyTuple.addAll(dataItem.getRemainder()); + } + + // Store the data + byte[] keyBytes = keyTuple.pack(); + byte[] valueBytes = dataItem.getValue(); + context.ensureActive().set(keyBytes, valueBytes); + }), + 1); + return insertionWork.forEach(vignore -> { }) + .whenComplete((vignore, e) -> insertionWork.close()); + }); + } + /** * Returns this path properly wrapped in whatever implementation the directory the path is contained in dictates. */ diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathWrapper.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathWrapper.java index 1cec9787ea..baa020ddf3 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathWrapper.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathWrapper.java @@ -202,4 +202,11 @@ public RecordCursor exportAllData(@Nonnull FDBRecordContext @Nonnull ScanProperties scanProperties) { return inner.exportAllData(context, continuation, scanProperties); } + + @Nonnull + @Override + public CompletableFuture importData(@Nonnull FDBRecordContext context, + @Nonnull Iterable dataToImport) { + return inner.importData(context, dataToImport); + } } diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/RecordCoreIllegalImportDataException.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/RecordCoreIllegalImportDataException.java new file mode 100644 index 0000000000..7879eefab7 --- /dev/null +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/RecordCoreIllegalImportDataException.java @@ -0,0 +1,36 @@ +/* + * RecordCoreIllegalImportDataException.java + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2015-2025 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.apple.foundationdb.record.provider.foundationdb.keyspace; + +import com.apple.foundationdb.record.RecordCoreArgumentException; + +import javax.annotation.Nonnull; + +/** + * Thrown if the data being imported into {@link KeySpacePath#importData} does not belong in that path. + */ +public class RecordCoreIllegalImportDataException extends RecordCoreArgumentException { + private static final long serialVersionUID = 1L; + + public RecordCoreIllegalImportDataException(@Nonnull final String msg, @Nonnull final Object... keyValue) { + super(msg, keyValue); + } +} diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/DataInKeySpacePathTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/DataInKeySpacePathTest.java index 6e8f0fb0b7..33fd2bdff7 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/DataInKeySpacePathTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/DataInKeySpacePathTest.java @@ -31,9 +31,11 @@ import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; /** * Tests for {@link DataInKeySpacePath}. @@ -118,4 +120,105 @@ void nullValue() { assertThrows(RecordCoreArgumentException.class, () -> new DataInKeySpacePath(testPath, null, valueBytes)); } + + @Test + void testEquals() { + KeySpace root = new KeySpace( + new KeySpaceDirectory("test", KeyType.STRING, UUID.randomUUID().toString())); + + KeySpacePath testPath = root.path("test"); + Tuple remainder1 = Tuple.from("key1", "key2"); + byte[] value1 = Tuple.from("value1").pack(); + + DataInKeySpacePath data1 = new DataInKeySpacePath(testPath, remainder1, value1); + DataInKeySpacePath data2 = new DataInKeySpacePath(testPath, remainder1, value1); + + // Reflexive: object equals itself + assertEquals(data1, data1); + + // Symmetric: a.equals(b) implies b.equals(a) + assertEquals(data1, data2); + assertEquals(data2, data1); + + // Test with different remainder + Tuple remainder2 = Tuple.from("different", "key"); + DataInKeySpacePath data3 = new DataInKeySpacePath(testPath, remainder2, value1); + assertNotEquals(data1, data3); + + // Test with different value + byte[] value2 = Tuple.from("value2").pack(); + DataInKeySpacePath data4 = new DataInKeySpacePath(testPath, remainder1, value2); + assertNotEquals(data1, data4); + + // Test with different path + KeySpace root2 = new KeySpace( + new KeySpaceDirectory("test", KeyType.STRING, UUID.randomUUID().toString())); + KeySpacePath testPath2 = root2.path("test"); + DataInKeySpacePath data5 = new DataInKeySpacePath(testPath2, remainder1, value1); + assertNotEquals(data1, data5); + + // Test with null remainder + DataInKeySpacePath data6 = new DataInKeySpacePath(testPath, null, value1); + DataInKeySpacePath data7 = new DataInKeySpacePath(testPath, null, value1); + assertEquals(data6, data7); + assertNotEquals(data1, data6); + + // Test with null object + assertNotEquals(data1, null); + + // Test with different class + assertNotEquals(data1, "not a DataInKeySpacePath"); + } + + @Test + void testHashCode() { + KeySpace root = new KeySpace( + new KeySpaceDirectory("test", KeyType.STRING, UUID.randomUUID().toString())); + + KeySpacePath testPath = root.path("test"); + Tuple remainder = Tuple.from("key1", "key2"); + byte[] value = Tuple.from("value1").pack(); + + DataInKeySpacePath data1 = new DataInKeySpacePath(testPath, remainder, value); + DataInKeySpacePath data2 = new DataInKeySpacePath(testPath, remainder, value); + + // Equal objects must have equal hash codes + assertEquals(data1.hashCode(), data2.hashCode()); + + // Test with null remainder + DataInKeySpacePath data3 = new DataInKeySpacePath(testPath, null, value); + DataInKeySpacePath data4 = new DataInKeySpacePath(testPath, null, value); + assertEquals(data3.hashCode(), data4.hashCode()); + + // Different objects should generally have different hash codes (not required, but good practice) + Tuple remainder2 = Tuple.from("different", "key"); + DataInKeySpacePath data5 = new DataInKeySpacePath(testPath, remainder2, value); + assertNotEquals(data1.hashCode(), data5.hashCode()); + } + + @Test + void testToString() { + final String rootUuid = UUID.randomUUID().toString(); + KeySpace root = new KeySpace( + new KeySpaceDirectory("test", KeyType.STRING, rootUuid)); + + KeySpacePath testPath = root.path("test"); + Tuple remainder = Tuple.from("key1", "key2"); + byte[] value = Tuple.from("value1").pack(); + + DataInKeySpacePath data = new DataInKeySpacePath(testPath, remainder, value); + + String result = data.toString(); + + // Verify the string contains expected components + assertTrue(result.contains(rootUuid)); + assertTrue(result.contains("test")); + assertTrue(result.contains("key1"), "toString should contain remainder elements"); + assertTrue(result.contains("key2"), "toString should contain remainder elements"); + + // Test with null remainder + DataInKeySpacePath dataWithNullRemainder = new DataInKeySpacePath(testPath, null, value); + String resultWithNull = dataWithNullRemainder.toString(); + assertTrue(resultWithNull.contains("null"), "toString should contain 'null' for null remainder"); + } } diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathDataExportTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathDataExportTest.java index 1ff39187dd..44d8f1c6ae 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathDataExportTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathDataExportTest.java @@ -49,7 +49,6 @@ import java.util.Map; import java.util.Set; import java.util.UUID; -import java.util.concurrent.ExecutionException; import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; import java.util.stream.IntStream; @@ -69,7 +68,7 @@ class KeySpacePathDataExportTest { final FDBDatabaseExtension dbExtension = new FDBDatabaseExtension(); @Test - void exportAllDataFromSimplePath() throws ExecutionException, InterruptedException { + void exportAllDataFromSimplePath() { KeySpace root = new KeySpace( new KeySpaceDirectory("root", KeyType.STRING, UUID.randomUUID().toString()) .addSubdirectory(new KeySpaceDirectory("level1", KeyType.LONG))); @@ -83,13 +82,14 @@ void exportAllDataFromSimplePath() throws ExecutionException, InterruptedExcepti // Add data at different levels for (int i = 0; i < 5; i++) { - Tuple key = basePath.add("level1", (long) i).toTuple(context); + final KeySpacePath path = basePath.add("level1", (long)i); + Tuple key = path.toTuple(context); tr.set(key.pack(), Tuple.from("value" + i).pack()); // Add some sub-data under each key for (int j = 0; j < 3; j++) { - Tuple subKey = key.add("sub" + j); - tr.set(subKey.pack(), Tuple.from("subvalue" + i + "_" + j).pack()); + tr.set(path.toSubspace(context).pack(Tuple.from("sub" + j)), + Tuple.from("subvalue" + i + "_" + j).pack()); } } context.commit(); @@ -103,16 +103,19 @@ void exportAllDataFromSimplePath() throws ExecutionException, InterruptedExcepti // Should have 5 main entries + 15 sub-entries = 20 total assertEquals(20, allData.size()); + assertThat(allData) + .allSatisfy(data -> + assertThat(data.getPath().getDirectoryName()).isEqualTo("level1")); + // Verify the data is sorted by key - for (int i = 1; i < allData.size(); i++) { - assertTrue(getKey(allData.get(i - 1), context).compareTo(getKey(allData.get(i), context)) < 0); - } + assertThat(allData.stream().map(data -> getKey(data, context)).collect(Collectors.toList())) + .isSorted(); } } // `toTuple` does not include the remainder, I'm not sure if that is intentional, or an oversight. - private Tuple getKey(final DataInKeySpacePath dataInKeySpacePath, final FDBRecordContext context) throws ExecutionException, InterruptedException { - final ResolvedKeySpacePath resolvedKeySpacePath = dataInKeySpacePath.getPath().toResolvedPathAsync(context).get(); + private Tuple getKey(final DataInKeySpacePath dataInKeySpacePath, final FDBRecordContext context) { + final ResolvedKeySpacePath resolvedKeySpacePath = dataInKeySpacePath.getPath().toResolvedPathAsync(context).join(); if (dataInKeySpacePath.getRemainder() != null) { return resolvedKeySpacePath.toTuple().addAll(dataInKeySpacePath.getRemainder()); } else { @@ -524,9 +527,7 @@ private static void exportWithContinuations(final KeySpacePath pathToExport, final RecordCursor cursor = pathToExport.exportAllData(context, continuation.toBytes(), scanProperties); final AtomicReference> tupleResult = new AtomicReference<>(); - final List batch = cursor.map(dataInPath -> { - return Tuple.fromBytes(dataInPath.getValue()); - }).asList(tupleResult).join(); + final List batch = cursor.map(dataInPath -> Tuple.fromBytes(dataInPath.getValue())).asList(tupleResult).join(); actual.add(batch); continuation = tupleResult.get().getContinuation(); } @@ -578,7 +579,7 @@ void exportAllDataThroughKeySpacePathWrapper() { } @Test - void exportAllDataThroughKeySpacePathWrapperResolvedPaths() { + void exportAllDataThroughKeySpacePathWrapperRemainders() { final FDBDatabase database = dbExtension.getDatabase(); final EnvironmentKeySpace keySpace = EnvironmentKeySpace.setupSampleData(database); diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathImportDataTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathImportDataTest.java new file mode 100644 index 0000000000..f75826f16f --- /dev/null +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathImportDataTest.java @@ -0,0 +1,570 @@ +/* + * KeySpacePathImportDataTest.java + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2015-2025 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.apple.foundationdb.record.provider.foundationdb.keyspace; + +import com.apple.foundationdb.Transaction; +import com.apple.foundationdb.record.ScanProperties; +import com.apple.foundationdb.record.provider.foundationdb.FDBDatabase; +import com.apple.foundationdb.record.provider.foundationdb.FDBExceptions; +import com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext; +import com.apple.foundationdb.record.provider.foundationdb.keyspace.KeySpaceDirectory.KeyType; +import com.apple.foundationdb.record.test.FDBDatabaseExtension; +import com.apple.foundationdb.tuple.Tuple; +import com.apple.test.BooleanSource; +import com.apple.test.Tags; +import org.assertj.core.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; + +import javax.annotation.Nonnull; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.Locale; +import java.util.UUID; +import java.util.concurrent.CompletionException; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assumptions.assumeThat; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Tests for {@link KeySpacePath#importData(FDBRecordContext, Iterable)}. + */ +@Tag(Tags.RequiresFDB) +class KeySpacePathImportDataTest { + @RegisterExtension + final FDBDatabaseExtension dbExtension = new FDBDatabaseExtension(); + private FDBDatabase sourceDatabase; + private FDBDatabase destinationDatabase; + + @BeforeEach + void setUp() { + final List databases = dbExtension.getRandomDatabaseSubset(2); + sourceDatabase = databases.get(0); + if (databases.size() > 1) { + destinationDatabase = databases.get(1); + } else { + destinationDatabase = sourceDatabase; + } + } + + @ParameterizedTest + @EnumSource(CopyConfig.class) + void importComprehensiveData(CopyConfig copyConfig) { + // Test importing data covering ALL KeyType enum values in a single complex directory structure: + // - NULL, BYTES, STRING, LONG, FLOAT, DOUBLE, BOOLEAN, UUID + // - Constant value directories + // - Complex multi-level hierarchy + // - Different data types in remainder elements + // Given that the majority of complexity is in the components under the importData method, this is basically + // an integration test + final String rootUuid = UUID.randomUUID().toString(); + byte[] binaryId = {0x01, 0x02, 0x03, (byte) 0xFF, (byte) 0xFE}; + UUID memberId = UUID.randomUUID(); + + KeySpace root = new KeySpace( + new KeySpaceDirectory("company", KeyType.STRING, rootUuid) + .addSubdirectory(new KeySpaceDirectory("version", KeyType.LONG, 1L) + .addSubdirectory(new KeySpaceDirectory("department", KeyType.STRING) + .addSubdirectory(new KeySpaceDirectory("employee_id", KeyType.LONG) + .addSubdirectory(new KeySpaceDirectory("binary_data", KeyType.BYTES) + .addSubdirectory(new KeySpaceDirectory("null_section", KeyType.NULL) + .addSubdirectory(new KeySpaceDirectory("member", KeyType.UUID) + .addSubdirectory(new KeySpaceDirectory("active", KeyType.BOOLEAN) + .addSubdirectory(new KeySpaceDirectory("rating", KeyType.FLOAT)))))))))); + + + // Create comprehensive test data covering ALL KeyType values + KeySpacePath basePath = root.path("company").add("version").add("department", "engineering"); + + // Build paths using all KeyType values + KeySpacePath emp1Path = basePath.add("employee_id", 100L) + .add("binary_data", binaryId) + .add("null_section") + .add("member", memberId) + .add("active", true) + .add("rating", 4.5f); + + KeySpacePath emp2Path = basePath.add("employee_id", 200L) + .add("binary_data", binaryId) + .add("null_section") + .add("member", memberId) + .add("active", false) + .add("rating", 3.8f); + + try (FDBRecordContext context = sourceDatabase.openContext()) { + setInPath(emp1Path, context, Tuple.from("profile", "name"), "John Doe"); + setInPath(emp2Path, context, Tuple.from("profile", "name"), "Jane Smith"); + setInPath(emp1Path, context, Tuple.from("salary"), 75000); + setInPath(emp1Path, context, Tuple.from("info", 42, true, "complex"), "Complex Test"); + + byte[] binaryKey = emp1Path.toSubspace(context).pack(Tuple.from("binary_metadata")); + context.ensureActive().set(binaryKey, "binary_test_data".getBytes()); + + context.commit(); + } + + final FDBDatabase targetDatabase = copyData(root.path("company"), root.path("company"), copyConfig); + + // Verify all different KeyType values were handled correctly during import + try (FDBRecordContext context = targetDatabase.openContext()) { + assertEquals(Tuple.from("John Doe"), + getTupleFromPath(context, emp1Path, Tuple.from("profile", "name"))); + assertEquals(Tuple.from("Jane Smith"), + getTupleFromPath(context, emp2Path, Tuple.from("profile", "name"))); + assertEquals(Tuple.from(75000), + getTupleFromPath(context, emp1Path, Tuple.from("salary"))); + assertEquals(Tuple.from("Complex Test"), + getTupleFromPath(context, emp1Path, Tuple.from("info", 42, true, "complex"))); + + // Verify BYTES data (raw binary, not in tuple) + byte[] binaryKey = emp1Path.toSubspace(context).pack(Tuple.from("binary_metadata")); + assertArrayEquals("binary_test_data".getBytes(), context.ensureActive().get(binaryKey).join()); + } + } + + @Test + void importEmptyData() { + // Test importing an empty collection of data + // Should complete successfully without modifying the data under the path + KeySpace root = new KeySpace( + new KeySpaceDirectory("test", KeyType.STRING, UUID.randomUUID().toString())); + + KeySpacePath testPath = root.path("test"); + importData(destinationDatabase, testPath, Collections.emptyList()); // should not throw any exception + + assertTrue(getExportedData(sourceDatabase, testPath).isEmpty(), + "there should not have been any data created"); + } + + @Test + void importOverwriteExistingData() { + // Test importing data that overwrites existing keys + final String rootUuid = UUID.randomUUID().toString(); + KeySpace root = new KeySpace( + new KeySpaceDirectory("root", KeyType.STRING, rootUuid) + .addSubdirectory(new KeySpaceDirectory("data", KeyType.LONG))); + + final KeySpacePath dataPath = root.path("root").add("data", 1L); + + // Import one value + importData(destinationDatabase, root.path("root"), List.of(new DataInKeySpacePath(dataPath, + Tuple.from("record"), Tuple.from("old_value").pack()))); + verifySingleKey(destinationDatabase, dataPath, Tuple.from("record"), Tuple.from("old_value")); + + // Import a different value + importData(destinationDatabase, root.path("root"), List.of(new DataInKeySpacePath(dataPath, + Tuple.from("record"), Tuple.from("new_value").pack()))); + + // Verify the data was overwritten + verifySingleKey(destinationDatabase, dataPath, Tuple.from("record"), Tuple.from("new_value")); + } + + @Test + void leaveExistingData() { + // Test importing data leaves other data under the path untouched + final String rootUuid = UUID.randomUUID().toString(); + KeySpace root = new KeySpace( + new KeySpaceDirectory("root", KeyType.STRING, rootUuid) + .addSubdirectory(new KeySpaceDirectory("data", KeyType.LONG))); + + final KeySpacePath dataPath = root.path("root").add("data", 1L); + + // Write one key + final Tuple remainder1 = Tuple.from("recordA"); + final Tuple value1 = Tuple.from("old_value"); + importData(destinationDatabase, root.path("root"), List.of(new DataInKeySpacePath(dataPath, + remainder1, value1.pack()))); + verifySingleKey(destinationDatabase, dataPath, remainder1, value1); + + // write a different key + final Tuple remainder2 = Tuple.from("recordB"); + final Tuple value2 = Tuple.from("new_value"); + importData(destinationDatabase, root.path("root"), List.of(new DataInKeySpacePath(dataPath, + remainder2, value2.pack()))); + + // Verify the data was overwritten + verifySingleKey(destinationDatabase, dataPath, remainder1, value1); + verifySingleKey(destinationDatabase, dataPath, remainder2, value2); + } + + @Test + void reimport() { + final String rootUuid = UUID.randomUUID().toString(); + KeySpace root = new KeySpace( + new KeySpaceDirectory("root", KeyType.STRING, rootUuid) + .addSubdirectory(new KeySpaceDirectory("data", KeyType.LONG))); + + final KeySpacePath dataPath = root.path("root").add("data", 1L); + List importData = new ArrayList<>(); + final Tuple remainder = Tuple.from("record"); + importData.add(new DataInKeySpacePath(dataPath, remainder, Tuple.from("value").pack())); + + // Verify we can re-import the data multiple times + importData(destinationDatabase, root.path("root"), importData); + importData(destinationDatabase, root.path("root"), importData); + importData(destinationDatabase, root.path("root"), importData); + + verifySingleKey(destinationDatabase, dataPath, remainder, Tuple.from("value")); + } + + @ParameterizedTest + @EnumSource(CopyConfig.class) + void importDataWithDirectoryLayer(final CopyConfig copyConfig) { + // Test importing data into a keyspace using DirectoryLayer directories + // Should verify that DirectoryLayer mappings work correctly during import + final String tenantUuid = UUID.randomUUID().toString(); + KeySpace root = new KeySpace( + new DirectoryLayerDirectory("tenant", tenantUuid) + .addSubdirectory(new KeySpaceDirectory("user_id", KeyType.LONG))); + + final KeySpacePath dataPath = root.path("tenant").add("user_id", 999L); + setSingleKey(dataPath, Tuple.from("data"), Tuple.from("directory_test")); + + final FDBDatabase targetDatabase = copyData(root.path("tenant"), root.path("tenant"), copyConfig); + + verifySingleKey(targetDatabase, dataPath, Tuple.from("data"), Tuple.from("directory_test")); + } + + @Test + void importDataWithMismatchedPath() { + // Test importing data that doesn't belong to the target path + // Should throw RecordCoreIllegalImportDataException + final String root1Uuid = UUID.randomUUID().toString(); + final String root2Uuid = UUID.randomUUID().toString(); + + KeySpace keySpace = new KeySpace( + new KeySpaceDirectory("root1", KeyType.STRING, root1Uuid) + .addSubdirectory(new KeySpaceDirectory("data", KeyType.LONG)), + new KeySpaceDirectory("root2", KeyType.STRING, root2Uuid) + .addSubdirectory(new KeySpaceDirectory("data", KeyType.LONG))); + + setSingleKey(keySpace.path("root1").add("data", 1L), Tuple.from("record"), Tuple.from("other")); + + // Now try to import that into root2 + assertBadImport(keySpace.path("root1"), keySpace.path("root2")); + } + + @Test + void importDataWithInvalidPath() { + // Test importing data with paths that don't exist in the keyspace + // Should throw RecordCoreIllegalImportDataException + final String root1Uuid = UUID.randomUUID().toString(); + final String root2Uuid = UUID.randomUUID().toString(); + + KeySpace keySpace1 = new KeySpace( + new KeySpaceDirectory("root1", KeyType.STRING, root1Uuid) + .addSubdirectory(new KeySpaceDirectory("data", KeyType.LONG))); + + KeySpace keySpace2 = new KeySpace( + new KeySpaceDirectory("root2", KeyType.STRING, root2Uuid) + .addSubdirectory(new KeySpaceDirectory("data", KeyType.LONG))); + + setSingleKey(keySpace1.path("root1").add("data", 1L), Tuple.from("record"), Tuple.from("other")); + + // Now try to import that into root2 + assertBadImport(keySpace1.path("root1"), keySpace2.path("root2")); + } + + @ParameterizedTest + @EnumSource(CopyConfig.class) + void importDataWithSubdirectoryPath(final CopyConfig copyConfig) { + // Test importing data where the target path is a subdirectory of the import path + // Should succeed only if all the data is in the subdirectory + final String rootUuid = UUID.randomUUID().toString(); + KeySpace root = new KeySpace( + new KeySpaceDirectory("root", KeyType.STRING, rootUuid) + .addSubdirectory(new KeySpaceDirectory("level1", KeyType.LONG))); + + KeySpacePath level1Path = root.path("root").add("level1", 1L); + + setSingleKey(level1Path, Tuple.from("item1"), Tuple.from("value1")); + + // Export from root, import to subdirectory + final FDBDatabase targetDatabase = copyData(root.path("root"), level1Path, copyConfig); + + verifySingleKey(targetDatabase, level1Path, Tuple.from("item1"), Tuple.from("value1")); + } + + @Test + void importDataWithSubdirectoryPathFailure() { + // Test importing data where the target path is a subdirectory of the import path + // Should succeed only if all the data is in the subdirectory + final String rootUuid = UUID.randomUUID().toString(); + KeySpace root = new KeySpace( + new KeySpaceDirectory("root", KeyType.STRING, rootUuid) + .addSubdirectory(new KeySpaceDirectory("level1", KeyType.LONG))); + + KeySpacePath level1Path = root.path("root").add("level1", 1L); + + setSingleKey(level1Path, Tuple.from("item1"), Tuple.from("value1")); + setSingleKey(root.path("root").add("level1", 2L), Tuple.from("item1"), Tuple.from("value1")); + + // Export from root, import to subdirectory + assertBadImport(root.path("root"), level1Path); + } + + @Test + void importDataWithPartialMismatch() { + // Test importing data where the target path is a parent of some import data paths + // Should throw RecordCoreIllegalImportDataException for paths outside the target + final String root1Uuid = UUID.randomUUID().toString(); + final String root2Uuid = UUID.randomUUID().toString(); + + KeySpace keySpace = new KeySpace( + new KeySpaceDirectory("root1", KeyType.STRING, root1Uuid) + .addSubdirectory(new KeySpaceDirectory("child", KeyType.LONG)), + new KeySpaceDirectory("root2", KeyType.STRING, root2Uuid) + .addSubdirectory(new KeySpaceDirectory("child", KeyType.LONG))); + + KeySpacePath path1 = keySpace.path("root1").add("child", 1L); + KeySpacePath path2 = keySpace.path("root2").add("child", 2L); + setSingleKey(path1, Tuple.from("data"), Tuple.from("data1")); + setSingleKey(path2, Tuple.from("data"), Tuple.from("data2")); + + List mixedData = new ArrayList<>(); + mixedData.addAll(getExportedData(sourceDatabase, path1)); + mixedData.addAll(getExportedData(sourceDatabase, path2)); + + assertBadImport(keySpace.path("root1"), mixedData); + } + + @ParameterizedTest + @EnumSource(CopyConfig.class) + void importDataWithWrapperClasses(final CopyConfig copyConfig) { + // Test importing data using wrapper classes like EnvironmentKeySpace + // Should verify that wrapper functionality works correctly with import + final EnvironmentKeySpace keySpace = EnvironmentKeySpace.setupSampleData(sourceDatabase); + + EnvironmentKeySpace.DataPath dataStore = keySpace.root().userid(100L).application("app1").dataStore(); + + final FDBDatabase targetDatabase = copyData(keySpace.root(), keySpace.root(), copyConfig); + + verifySingleKey(targetDatabase, dataStore, Tuple.from("record2", 0), Tuple.from("user100_app1_data2_0")); + } + + @Test + void importDataWithDuplicateKeys() { + // Test importing data where the same key appears multiple times in the input + // Should verify that the last value wins for duplicate keys + final String rootUuid = UUID.randomUUID().toString(); + KeySpace root = new KeySpace( + new KeySpaceDirectory("root", KeyType.STRING, rootUuid) + .addSubdirectory(new KeySpaceDirectory("data", KeyType.LONG))); + + + KeySpacePath dataPath = root.path("root").add("data", 1L); + + // Create multiple DataInKeySpacePath objects with same key but different values + List duplicateData = Arrays.asList( + new DataInKeySpacePath(dataPath, Tuple.from("item"), Tuple.from("first_value").pack()), + new DataInKeySpacePath(dataPath, Tuple.from("item"), Tuple.from("second_value").pack()), + new DataInKeySpacePath(dataPath, Tuple.from("item"), Tuple.from("final_value").pack()) + ); + + importData(destinationDatabase, root.path("root"), duplicateData); + + verifySingleKey(destinationDatabase, dataPath, Tuple.from("item"), Tuple.from("final_value")); + } + + @ParameterizedTest + @BooleanSource({"manyPaths", "useDirectoryLayer"}) + void importTooMuchData(boolean manyPaths, boolean useDirectoryLayer) { + // Test importing too much data within a single path + // we do it both with 1 path, and many paths, because resolving a directory layer requires reads to the + // database which should slow things down, and result in different errors + // using a standard path we can write data until we get OOM, and if we have a more reasonable limit, we'll + // get a transaction-too-large. + final String rootUuid = UUID.randomUUID().toString(); + KeySpace root = new KeySpace( + new KeySpaceDirectory("root", KeyType.STRING, rootUuid) + .addSubdirectory(useDirectoryLayer + ? new DirectoryLayerDirectory("data") + : new KeySpaceDirectory("data", KeyType.STRING))); + + final KeySpacePath rootPath = root.path("root"); + + final DataGenerator data = new DataGenerator(rootPath, 100_000, manyPaths); + assertThrows(FDBExceptions.FDBStoreTransactionSizeException.class, () -> + importData(destinationDatabase, rootPath, data)); + } + + @ParameterizedTest + @BooleanSource({"manyPaths", "useDirectoryLayer"}) + void importALotOfData(boolean manyPaths, boolean useDirectoryLayer) { + // Test importing a lot of data within a single path + final String rootUuid = UUID.randomUUID().toString(); + KeySpace root = new KeySpace( + new KeySpaceDirectory("root", KeyType.STRING, rootUuid) + .addSubdirectory(useDirectoryLayer + ? new DirectoryLayerDirectory("data") + : new KeySpaceDirectory("data", KeyType.STRING))); + + final KeySpacePath rootPath = root.path("root"); + + final DataGenerator data = new DataGenerator(rootPath, 10_000, manyPaths); + importData(destinationDatabase, rootPath, data); + final List exportedData = getExportedData(destinationDatabase, rootPath); + // Note: The order won't be the same in the directory layer case, because the order is based on the resolved + // values, not the logical, or resolved values on the source + assertThat(exportedData).containsExactlyInAnyOrderElementsOf(data); + } + + private void setSingleKey(KeySpacePath path, Tuple remainder, Tuple value) { + try (FDBRecordContext context = sourceDatabase.openContext()) { + byte[] key = path.toSubspace(context).pack(remainder); + context.ensureActive().set(key, value.pack()); + context.commit(); + } + } + + private void verifySingleKey(FDBDatabase database, KeySpacePath path, Tuple remainder, Tuple expected) { + try (FDBRecordContext context = database.openContext()) { + byte[] key = path.toSubspace(context).pack(remainder); + assertEquals(expected, Tuple.fromBytes(context.ensureActive().get(key).join())); + } + } + + private static void setInPath(final KeySpacePath path, final FDBRecordContext context, + final Tuple remainder, final Object value) { + byte[] key = path.toSubspace(context).pack(remainder); + context.ensureActive().set(key, Tuple.from(value).pack()); + } + + private static Tuple getTupleFromPath(final FDBRecordContext context, final KeySpacePath path, final Tuple remainder) { + byte[] key = path.toSubspace(context).pack(remainder); + return Tuple.fromBytes(context.ensureActive().get(key).join()); + } + + enum CopyConfig { + WithinCluster, + BetweenClusters + } + + private FDBDatabase copyData(final KeySpacePath sourcePath, KeySpacePath destinationPath, final CopyConfig copyConfig) { + // Export the data + final List exportedData = getExportedData(sourceDatabase, sourcePath); + + FDBDatabase targetDatabase; + switch (copyConfig) { + case WithinCluster: + // Clear the data and import it back + clearSourcePath(sourcePath); + targetDatabase = sourceDatabase; + break; + case BetweenClusters: + assumeThat(destinationDatabase).isNotEqualTo(sourceDatabase); + targetDatabase = destinationDatabase; + break; + default: + throw new IllegalStateException("Unexpected value: " + copyConfig); + } + + // Import the data + importData(targetDatabase, destinationPath, exportedData); + return targetDatabase; + } + + private void importData(FDBDatabase targetDatabase, final KeySpacePath path, final Iterable exportedData) { + try (FDBRecordContext context = targetDatabase.openContext()) { + path.importData(context, exportedData).join(); + context.commit(); + } + } + + private void assertBadImport(KeySpacePath sourcePath, KeySpacePath destinationPath) { + List exportedData = getExportedData(sourceDatabase, sourcePath); + assertBadImport(destinationPath, exportedData); + } + + private void assertBadImport(final KeySpacePath path, final List invalidData) { + try (FDBRecordContext context = destinationDatabase.openContext()) { + Assertions.assertThatThrownBy(() -> path.importData(context, invalidData).join()) + .isInstanceOf(CompletionException.class) + .hasCauseInstanceOf(RecordCoreIllegalImportDataException.class); + } + } + + private void clearSourcePath(final KeySpacePath path) { + // Note: this does not touch the directory layer + try (FDBRecordContext context = sourceDatabase.openContext()) { + Transaction tr = context.ensureActive(); + tr.clear(path.toSubspace(context).range()); + context.commit(); + } + // just an extra check to make sure the test is working as expected + assertTrue(getExportedData(sourceDatabase, path).isEmpty(), + "Clearing should remove all the data"); + } + + @Nonnull + private List getExportedData(FDBDatabase targetDatabase, final KeySpacePath path) { + try (FDBRecordContext context = targetDatabase.openContext()) { + return path.exportAllData(context, null, ScanProperties.FORWARD_SCAN) + .asList().join(); + } + } + + private static class DataGenerator implements Iterable { + + private final KeySpacePath rootPath; + private final int limit; + private final boolean manyPaths; + + public DataGenerator(final KeySpacePath rootPath, int limit, final boolean manyPaths) { + this.rootPath = rootPath; + this.limit = limit; + this.manyPaths = manyPaths; + } + + @Override + public Iterator iterator() { + return new Iterator<>() { + int counter = 0; + + @Override + public boolean hasNext() { + return counter < limit; + } + + @Override + public DataInKeySpacePath next() { + counter++; + final KeySpacePath path = manyPaths ? rootPath.add("data", String.format(Locale.ROOT, "path %04d", counter / 100)) + : rootPath.add("data", "OnlyPath"); + return new DataInKeySpacePath( + path, + Tuple.from(counter), Tuple.from("value", counter).pack()); + } + }; + } + } +} diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/test/FDBDatabaseExtension.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/test/FDBDatabaseExtension.java index a1309989f7..282f09ebbe 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/test/FDBDatabaseExtension.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/test/FDBDatabaseExtension.java @@ -38,8 +38,12 @@ import javax.annotation.Nullable; import java.util.HashMap; import java.util.Map; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; import java.util.Objects; import java.util.concurrent.Executor; +import java.util.stream.Collectors; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -161,6 +165,18 @@ public FDBDatabase getDatabase(@Nullable String clusterFile) { }); } + /** + * Return a random subset of the databases available. + * @param maxCount the number of desired databases + * @return a random subset of the databases available. This may be less than {@code maxCount} if there aren't that many + * databases available. + */ + public List getRandomDatabaseSubset(int maxCount) { + List clusterFiles = new ArrayList<>(FDBTestEnvironment.allClusterFiles()); + Collections.shuffle(clusterFiles); + return clusterFiles.stream().limit(maxCount).map(this::getDatabase).collect(Collectors.toList()); + } + public void checkForOpenContexts() { for (final Map.Entry clusterFileToDatabase : databases.entrySet()) { assertEquals(0, clusterFileToDatabase.getValue().warnAndCloseOldTrackedOpenContexts(0),