From da1cb57a170452853cc37fcf795687dc4f431cd9 Mon Sep 17 00:00:00 2001 From: Toshihiro Suzuki Date: Wed, 28 May 2025 16:23:00 +0900 Subject: [PATCH 01/19] Add getScanner() method to transaction interfaces (#2698) --- .../jdbc/JdbcTransactionIntegrationTest.java | 15 +++++ .../java/com/scalar/db/api/CrudOperable.java | 58 ++++++++++++++++- .../main/java/com/scalar/db/api/Scanner.java | 9 +-- .../db/api/TransactionCrudOperable.java | 46 ++++++++++++++ .../api/TransactionManagerCrudOperable.java | 47 ++++++++++++++ .../common/AbstractCrudOperableScanner.java | 61 ++++++++++++++++++ ...bstractTransactionCrudOperableScanner.java | 7 +++ ...TransactionManagerCrudOperableScanner.java | 8 +++ ...nManagedDistributedTransactionManager.java | 5 ++ ...nagedTwoPhaseCommitTransactionManager.java | 5 ++ .../DecoratedDistributedTransaction.java | 5 ++ ...ecoratedDistributedTransactionManager.java | 5 ++ .../DecoratedTwoPhaseCommitTransaction.java | 5 ++ ...ratedTwoPhaseCommitTransactionManager.java | 5 ++ ...eManagedDistributedTransactionManager.java | 6 ++ ...nagedTwoPhaseCommitTransactionManager.java | 6 ++ .../scalar/db/service/TransactionService.java | 11 ++++ .../TwoPhaseCommitTransactionService.java | 11 ++++ .../consensuscommit/ConsensusCommit.java | 5 ++ .../ConsensusCommitManager.java | 5 ++ .../TwoPhaseConsensusCommit.java | 5 ++ .../TwoPhaseConsensusCommitManager.java | 5 ++ .../db/transaction/jdbc/JdbcTransaction.java | 5 ++ .../jdbc/JdbcTransactionManager.java | 5 ++ ...SingleCrudOperationTransactionManager.java | 9 ++- ...ributedTransactionIntegrationTestBase.java | 62 ++++++++++++++++++- ...eCommitTransactionIntegrationTestBase.java | 62 +++++++++++++++++++ .../ConsensusCommitIntegrationTestBase.java | 11 ++++ ...aseConsensusCommitIntegrationTestBase.java | 12 ++++ ...erationTransactionIntegrationTestBase.java | 12 +++- 30 files changed, 502 insertions(+), 11 deletions(-) create mode 100644 core/src/main/java/com/scalar/db/common/AbstractCrudOperableScanner.java create mode 100644 core/src/main/java/com/scalar/db/common/AbstractTransactionCrudOperableScanner.java create mode 100644 core/src/main/java/com/scalar/db/common/AbstractTransactionManagerCrudOperableScanner.java diff --git a/core/src/integration-test/java/com/scalar/db/transaction/jdbc/JdbcTransactionIntegrationTest.java b/core/src/integration-test/java/com/scalar/db/transaction/jdbc/JdbcTransactionIntegrationTest.java index 951ea6a57a..0ca3b064b8 100644 --- a/core/src/integration-test/java/com/scalar/db/transaction/jdbc/JdbcTransactionIntegrationTest.java +++ b/core/src/integration-test/java/com/scalar/db/transaction/jdbc/JdbcTransactionIntegrationTest.java @@ -6,6 +6,7 @@ import com.scalar.db.storage.jdbc.JdbcEnv; import java.util.Properties; import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; public class JdbcTransactionIntegrationTest extends DistributedTransactionIntegrationTestBase { @@ -24,17 +25,31 @@ protected Properties getProperties(String testName) { @Disabled("JDBC transactions don't support getState()") @Override + @Test public void getState_forSuccessfulTransaction_ShouldReturnCommittedState() {} @Disabled("JDBC transactions don't support getState()") @Override + @Test public void getState_forFailedTransaction_ShouldReturnAbortedState() {} @Disabled("JDBC transactions don't support abort()") @Override + @Test public void abort_forOngoingTransaction_ShouldAbortCorrectly() {} @Disabled("JDBC transactions don't support rollback()") @Override + @Test public void rollback_forOngoingTransaction_ShouldRollbackCorrectly() {} + + @Disabled("Implement later") + @Override + @Test + public void getScanner_ScanGivenForCommittedRecord_ShouldReturnRecords() {} + + @Disabled("Implement later") + @Override + @Test + public void manager_getScanner_ScanGivenForCommittedRecord_ShouldReturnRecords() {} } diff --git a/core/src/main/java/com/scalar/db/api/CrudOperable.java b/core/src/main/java/com/scalar/db/api/CrudOperable.java index a773f3e93a..ecbf67dd28 100644 --- a/core/src/main/java/com/scalar/db/api/CrudOperable.java +++ b/core/src/main/java/com/scalar/db/api/CrudOperable.java @@ -12,6 +12,9 @@ * An interface for transactional CRUD operations. Note that the LINEARIZABLE consistency level is * always used in transactional CRUD operations, so {@link Consistency} specified for CRUD * operations is ignored. + * + * @param the type of {@link TransactionException} that the implementation throws if the + * operation fails */ public interface CrudOperable { @@ -26,9 +29,18 @@ public interface CrudOperable { Optional get(Get get) throws E; /** - * Retrieves results from the storage through a transaction with the specified {@link Scan} - * command with a partition key and returns a list of {@link Result}. Results can be filtered by - * specifying a range of clustering keys. + * Retrieves results from the storage through a transaction with the specified {@link Scan} or + * {@link ScanAll} or {@link ScanWithIndex} command with a partition key and returns a list of + * {@link Result}. Results can be filtered by specifying a range of clustering keys. + * + *
    + *
  • {@link Scan} : by specifying a partition key, it will return results within the + * partition. Results can be filtered by specifying a range of clustering keys. + *
  • {@link ScanAll} : for a given table, it will return all its records even if they span + * several partitions. + *
  • {@link ScanWithIndex} : by specifying an index key, it will return results within the + * index. + *
* * @param scan a {@code Scan} command * @return a list of {@link Result} @@ -36,6 +48,18 @@ public interface CrudOperable { */ List scan(Scan scan) throws E; + /** + * Retrieves results from the storage through a transaction with the specified {@link Scan} or + * {@link ScanAll} or {@link ScanWithIndex} command with a partition key and returns a {@link + * Scanner} to iterate over the results. Results can be filtered by specifying a range of + * clustering keys. + * + * @param scan a {@code Scan} command + * @return a {@code Scanner} to iterate over the results + * @throws E if the transaction CRUD operation fails + */ + Scanner getScanner(Scan scan) throws E; + /** * Inserts an entry into or updates an entry in the underlying storage through a transaction with * the specified {@link Put} command. If a condition is specified in the {@link Put} command, and @@ -131,4 +155,32 @@ public interface CrudOperable { * @throws E if the transaction CRUD operation fails */ void mutate(List mutations) throws E; + + /** A scanner abstraction for iterating results. */ + interface Scanner extends AutoCloseable, Iterable { + /** + * Returns the next result. + * + * @return an {@code Optional} containing the next result if available, or empty if no more + * results + * @throws E if the operation fails + */ + Optional one() throws E; + + /** + * Returns all remaining results. + * + * @return a {@code List} containing all remaining results + * @throws E if the operation fails + */ + List all() throws E; + + /** + * Closes the scanner. + * + * @throws E if closing the scanner fails + */ + @Override + void close() throws E; + } } diff --git a/core/src/main/java/com/scalar/db/api/Scanner.java b/core/src/main/java/com/scalar/db/api/Scanner.java index 21a9b3a7cc..b863b53480 100644 --- a/core/src/main/java/com/scalar/db/api/Scanner.java +++ b/core/src/main/java/com/scalar/db/api/Scanner.java @@ -13,17 +13,18 @@ public interface Scanner extends Closeable, Iterable { /** - * Returns the first result in the results. + * Returns the next result. * - * @return the first result in the results + * @return an {@code Optional} containing the next result if available, or empty if no more + * results * @throws ExecutionException if the operation fails */ Optional one() throws ExecutionException; /** - * Returns all the results. + * Returns all remaining results. * - * @return the list of {@code Result}s + * @return a {@code List} containing all remaining results * @throws ExecutionException if the operation fails */ List all() throws ExecutionException; diff --git a/core/src/main/java/com/scalar/db/api/TransactionCrudOperable.java b/core/src/main/java/com/scalar/db/api/TransactionCrudOperable.java index c8303f7a90..d2be32919a 100644 --- a/core/src/main/java/com/scalar/db/api/TransactionCrudOperable.java +++ b/core/src/main/java/com/scalar/db/api/TransactionCrudOperable.java @@ -33,6 +33,18 @@ public interface TransactionCrudOperable extends CrudOperable { @Override List scan(Scan scan) throws CrudConflictException, CrudException; + /** + * {@inheritDoc} + * + * @throws CrudConflictException if the transaction CRUD operation fails due to transient faults + * (e.g., a conflict error). You can retry the transaction from the beginning + * @throws CrudException if the transaction CRUD operation fails due to transient or nontransient + * faults. You can try retrying the transaction from the beginning, but the transaction may + * still fail if the cause is nontranient + */ + @Override + Scanner getScanner(Scan scan) throws CrudConflictException, CrudException; + /** * {@inheritDoc} * @@ -154,4 +166,38 @@ void delete(List deletes) @Override void mutate(List mutations) throws CrudConflictException, CrudException, UnsatisfiedConditionException; + + interface Scanner extends CrudOperable.Scanner { + /** + * {@inheritDoc} + * + * @throws CrudConflictException if the transaction CRUD operation fails due to transient faults + * (e.g., a conflict error). You can retry the transaction from the beginning + * @throws CrudException if the transaction CRUD operation fails due to transient or + * nontransient faults. You can try retrying the transaction from the beginning, but the + * transaction may still fail if the cause is nontranient + */ + @Override + Optional one() throws CrudConflictException, CrudException; + + /** + * {@inheritDoc} + * + * @throws CrudConflictException if the transaction CRUD operation fails due to transient faults + * (e.g., a conflict error). You can retry the transaction from the beginning + * @throws CrudException if the transaction CRUD operation fails due to transient or + * nontransient faults. You can try retrying the transaction from the beginning, but the + * transaction may still fail if the cause is nontranient + */ + @Override + List all() throws CrudConflictException, CrudException; + + /** + * {@inheritDoc} + * + * @throws CrudException if closing the scanner fails + */ + @Override + void close() throws CrudException; + } } diff --git a/core/src/main/java/com/scalar/db/api/TransactionManagerCrudOperable.java b/core/src/main/java/com/scalar/db/api/TransactionManagerCrudOperable.java index eb285a2d54..608d80cdf3 100644 --- a/core/src/main/java/com/scalar/db/api/TransactionManagerCrudOperable.java +++ b/core/src/main/java/com/scalar/db/api/TransactionManagerCrudOperable.java @@ -39,6 +39,18 @@ Optional get(Get get) List scan(Scan scan) throws CrudConflictException, CrudException, UnknownTransactionStatusException; + /** + * {@inheritDoc} + * + * @throws CrudConflictException if the transaction CRUD operation fails due to transient faults + * (e.g., a conflict error). You can retry the transaction from the beginning + * @throws CrudException if the transaction CRUD operation fails due to transient or nontransient + * faults. You can try retrying the transaction from the beginning, but the transaction may + * still fail if the cause is nontranient + */ + @Override + Scanner getScanner(Scan scan) throws CrudConflictException, CrudException; + /** * {@inheritDoc} * @@ -177,4 +189,39 @@ void delete(List deletes) void mutate(List mutations) throws CrudConflictException, CrudException, UnsatisfiedConditionException, UnknownTransactionStatusException; + + interface Scanner extends CrudOperable.Scanner { + /** + * {@inheritDoc} + * + * @throws CrudConflictException if the transaction CRUD operation fails due to transient faults + * (e.g., a conflict error). You can retry the transaction from the beginning + * @throws CrudException if the transaction CRUD operation fails due to transient or + * nontransient faults. You can try retrying the transaction from the beginning, but the + * transaction may still fail if the cause is nontranient + */ + @Override + Optional one() throws CrudConflictException, CrudException; + + /** + * {@inheritDoc} + * + * @throws CrudConflictException if the transaction CRUD operation fails due to transient faults + * (e.g., a conflict error). You can retry the transaction from the beginning + * @throws CrudException if the transaction CRUD operation fails due to transient or + * nontransient faults. You can try retrying the transaction from the beginning, but the + * transaction may still fail if the cause is nontranient + */ + @Override + List all() throws CrudConflictException, CrudException; + + /** + * {@inheritDoc} + * + * @throws CrudException if closing the scanner fails + * @throws UnknownTransactionStatusException if the status of the commit is unknown + */ + @Override + void close() throws CrudException, UnknownTransactionStatusException; + } } diff --git a/core/src/main/java/com/scalar/db/common/AbstractCrudOperableScanner.java b/core/src/main/java/com/scalar/db/common/AbstractCrudOperableScanner.java new file mode 100644 index 0000000000..1e5a6fc6be --- /dev/null +++ b/core/src/main/java/com/scalar/db/common/AbstractCrudOperableScanner.java @@ -0,0 +1,61 @@ +package com.scalar.db.common; + +import com.google.errorprone.annotations.concurrent.LazyInit; +import com.scalar.db.api.CrudOperable; +import com.scalar.db.api.Result; +import com.scalar.db.exception.transaction.TransactionException; +import java.util.Iterator; +import java.util.NoSuchElementException; +import java.util.Objects; +import javax.annotation.Nonnull; +import javax.annotation.concurrent.NotThreadSafe; + +public abstract class AbstractCrudOperableScanner + implements CrudOperable.Scanner { + + @LazyInit private ScannerIterator scannerIterator; + + @Override + @Nonnull + public Iterator iterator() { + if (scannerIterator == null) { + scannerIterator = new ScannerIterator(this); + } + return scannerIterator; + } + + @NotThreadSafe + public class ScannerIterator implements Iterator { + + private final CrudOperable.Scanner scanner; + private Result next; + + public ScannerIterator(CrudOperable.Scanner scanner) { + this.scanner = Objects.requireNonNull(scanner); + } + + @Override + public boolean hasNext() { + if (next != null) { + return true; + } + + try { + return (next = scanner.one().orElse(null)) != null; + } catch (TransactionException e) { + throw new RuntimeException(e.getMessage(), e); + } + } + + @Override + public Result next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + + Result ret = next; + next = null; + return ret; + } + } +} diff --git a/core/src/main/java/com/scalar/db/common/AbstractTransactionCrudOperableScanner.java b/core/src/main/java/com/scalar/db/common/AbstractTransactionCrudOperableScanner.java new file mode 100644 index 0000000000..34e1d54568 --- /dev/null +++ b/core/src/main/java/com/scalar/db/common/AbstractTransactionCrudOperableScanner.java @@ -0,0 +1,7 @@ +package com.scalar.db.common; + +import com.scalar.db.api.TransactionCrudOperable; +import com.scalar.db.exception.transaction.CrudException; + +public abstract class AbstractTransactionCrudOperableScanner + extends AbstractCrudOperableScanner implements TransactionCrudOperable.Scanner {} diff --git a/core/src/main/java/com/scalar/db/common/AbstractTransactionManagerCrudOperableScanner.java b/core/src/main/java/com/scalar/db/common/AbstractTransactionManagerCrudOperableScanner.java new file mode 100644 index 0000000000..5dcbdb3479 --- /dev/null +++ b/core/src/main/java/com/scalar/db/common/AbstractTransactionManagerCrudOperableScanner.java @@ -0,0 +1,8 @@ +package com.scalar.db.common; + +import com.scalar.db.api.TransactionManagerCrudOperable; +import com.scalar.db.exception.transaction.TransactionException; + +public abstract class AbstractTransactionManagerCrudOperableScanner + extends AbstractCrudOperableScanner + implements TransactionManagerCrudOperable.Scanner {} diff --git a/core/src/main/java/com/scalar/db/common/ActiveTransactionManagedDistributedTransactionManager.java b/core/src/main/java/com/scalar/db/common/ActiveTransactionManagedDistributedTransactionManager.java index e3ef02c635..ea592e5b41 100644 --- a/core/src/main/java/com/scalar/db/common/ActiveTransactionManagedDistributedTransactionManager.java +++ b/core/src/main/java/com/scalar/db/common/ActiveTransactionManagedDistributedTransactionManager.java @@ -121,6 +121,11 @@ public synchronized List scan(Scan scan) throws CrudException { return super.scan(scan); } + @Override + public synchronized Scanner getScanner(Scan scan) throws CrudException { + return super.getScanner(scan); + } + /** @deprecated As of release 3.13.0. Will be removed in release 5.0.0. */ @Deprecated @Override diff --git a/core/src/main/java/com/scalar/db/common/ActiveTransactionManagedTwoPhaseCommitTransactionManager.java b/core/src/main/java/com/scalar/db/common/ActiveTransactionManagedTwoPhaseCommitTransactionManager.java index a6a4c6b9ee..b0543433d3 100644 --- a/core/src/main/java/com/scalar/db/common/ActiveTransactionManagedTwoPhaseCommitTransactionManager.java +++ b/core/src/main/java/com/scalar/db/common/ActiveTransactionManagedTwoPhaseCommitTransactionManager.java @@ -127,6 +127,11 @@ public synchronized List scan(Scan scan) throws CrudException { return super.scan(scan); } + @Override + public synchronized Scanner getScanner(Scan scan) throws CrudException { + return super.getScanner(scan); + } + /** @deprecated As of release 3.13.0. Will be removed in release 5.0.0. */ @Deprecated @Override diff --git a/core/src/main/java/com/scalar/db/common/DecoratedDistributedTransaction.java b/core/src/main/java/com/scalar/db/common/DecoratedDistributedTransaction.java index ca6d0cdd76..ca997988c6 100644 --- a/core/src/main/java/com/scalar/db/common/DecoratedDistributedTransaction.java +++ b/core/src/main/java/com/scalar/db/common/DecoratedDistributedTransaction.java @@ -78,6 +78,11 @@ public List scan(Scan scan) throws CrudException { return transaction.scan(scan); } + @Override + public Scanner getScanner(Scan scan) throws CrudException { + return transaction.getScanner(scan); + } + /** @deprecated As of release 3.13.0. Will be removed in release 5.0.0. */ @Deprecated @Override diff --git a/core/src/main/java/com/scalar/db/common/DecoratedDistributedTransactionManager.java b/core/src/main/java/com/scalar/db/common/DecoratedDistributedTransactionManager.java index c2caadd9eb..dac3cfa2c7 100644 --- a/core/src/main/java/com/scalar/db/common/DecoratedDistributedTransactionManager.java +++ b/core/src/main/java/com/scalar/db/common/DecoratedDistributedTransactionManager.java @@ -157,6 +157,11 @@ public List scan(Scan scan) throws CrudException, UnknownTransactionStat return transactionManager.scan(scan); } + @Override + public Scanner getScanner(Scan scan) throws CrudException { + return transactionManager.getScanner(scan); + } + /** @deprecated As of release 3.13.0. Will be removed in release 5.0.0. */ @Deprecated @Override diff --git a/core/src/main/java/com/scalar/db/common/DecoratedTwoPhaseCommitTransaction.java b/core/src/main/java/com/scalar/db/common/DecoratedTwoPhaseCommitTransaction.java index 097e4f032c..04a48f4314 100644 --- a/core/src/main/java/com/scalar/db/common/DecoratedTwoPhaseCommitTransaction.java +++ b/core/src/main/java/com/scalar/db/common/DecoratedTwoPhaseCommitTransaction.java @@ -80,6 +80,11 @@ public List scan(Scan scan) throws CrudException { return transaction.scan(scan); } + @Override + public Scanner getScanner(Scan scan) throws CrudException { + return transaction.getScanner(scan); + } + /** @deprecated As of release 3.13.0. Will be removed in release 5.0.0. */ @Deprecated @Override diff --git a/core/src/main/java/com/scalar/db/common/DecoratedTwoPhaseCommitTransactionManager.java b/core/src/main/java/com/scalar/db/common/DecoratedTwoPhaseCommitTransactionManager.java index edcbd6e7a6..ce479795f1 100644 --- a/core/src/main/java/com/scalar/db/common/DecoratedTwoPhaseCommitTransactionManager.java +++ b/core/src/main/java/com/scalar/db/common/DecoratedTwoPhaseCommitTransactionManager.java @@ -111,6 +111,11 @@ public List scan(Scan scan) throws CrudException, UnknownTransactionStat return transactionManager.scan(scan); } + @Override + public Scanner getScanner(Scan scan) throws CrudException { + return transactionManager.getScanner(scan); + } + /** @deprecated As of release 3.13.0. Will be removed in release 5.0.0. */ @Deprecated @Override diff --git a/core/src/main/java/com/scalar/db/common/StateManagedDistributedTransactionManager.java b/core/src/main/java/com/scalar/db/common/StateManagedDistributedTransactionManager.java index 7d7adec72e..52866cb405 100644 --- a/core/src/main/java/com/scalar/db/common/StateManagedDistributedTransactionManager.java +++ b/core/src/main/java/com/scalar/db/common/StateManagedDistributedTransactionManager.java @@ -70,6 +70,12 @@ public List scan(Scan scan) throws CrudException { return super.scan(scan); } + @Override + public Scanner getScanner(Scan scan) throws CrudException { + checkIfActive(); + return super.getScanner(scan); + } + /** @deprecated As of release 3.13.0. Will be removed in release 5.0.0. */ @Deprecated @Override diff --git a/core/src/main/java/com/scalar/db/common/StateManagedTwoPhaseCommitTransactionManager.java b/core/src/main/java/com/scalar/db/common/StateManagedTwoPhaseCommitTransactionManager.java index 7ba79ddede..1d79240d04 100644 --- a/core/src/main/java/com/scalar/db/common/StateManagedTwoPhaseCommitTransactionManager.java +++ b/core/src/main/java/com/scalar/db/common/StateManagedTwoPhaseCommitTransactionManager.java @@ -76,6 +76,12 @@ public List scan(Scan scan) throws CrudException { return super.scan(scan); } + @Override + public Scanner getScanner(Scan scan) throws CrudException { + checkIfActive(); + return super.getScanner(scan); + } + /** @deprecated As of release 3.13.0. Will be removed in release 5.0.0. */ @Deprecated @Override diff --git a/core/src/main/java/com/scalar/db/service/TransactionService.java b/core/src/main/java/com/scalar/db/service/TransactionService.java index fd68d2dc22..8acc748eaa 100644 --- a/core/src/main/java/com/scalar/db/service/TransactionService.java +++ b/core/src/main/java/com/scalar/db/service/TransactionService.java @@ -167,11 +167,20 @@ public List scan(Scan scan) throws CrudException, UnknownTransactionStat return manager.scan(scan); } + @Override + public Scanner getScanner(Scan scan) throws CrudException { + return manager.getScanner(scan); + } + + /** @deprecated As of release 3.13.0. Will be removed in release 5.0.0. */ + @Deprecated @Override public void put(Put put) throws CrudException, UnknownTransactionStatusException { manager.put(put); } + /** @deprecated As of release 3.13.0. Will be removed in release 5.0.0. */ + @Deprecated @Override public void put(List puts) throws CrudException, UnknownTransactionStatusException { manager.put(puts); @@ -197,6 +206,8 @@ public void delete(Delete delete) throws CrudException, UnknownTransactionStatus manager.delete(delete); } + /** @deprecated As of release 3.13.0. Will be removed in release 5.0.0. */ + @Deprecated @Override public void delete(List deletes) throws CrudException, UnknownTransactionStatusException { manager.delete(deletes); diff --git a/core/src/main/java/com/scalar/db/service/TwoPhaseCommitTransactionService.java b/core/src/main/java/com/scalar/db/service/TwoPhaseCommitTransactionService.java index a2a2439f2c..6cf6f68a98 100644 --- a/core/src/main/java/com/scalar/db/service/TwoPhaseCommitTransactionService.java +++ b/core/src/main/java/com/scalar/db/service/TwoPhaseCommitTransactionService.java @@ -124,11 +124,20 @@ public List scan(Scan scan) throws CrudException, UnknownTransactionStat return manager.scan(scan); } + @Override + public Scanner getScanner(Scan scan) throws CrudException { + return manager.getScanner(scan); + } + + /** @deprecated As of release 3.13.0. Will be removed in release 5.0.0. */ + @Deprecated @Override public void put(Put put) throws CrudException, UnknownTransactionStatusException { manager.put(put); } + /** @deprecated As of release 3.13.0. Will be removed in release 5.0.0. */ + @Deprecated @Override public void put(List puts) throws CrudException, UnknownTransactionStatusException { manager.put(puts); @@ -154,6 +163,8 @@ public void delete(Delete delete) throws CrudException, UnknownTransactionStatus manager.delete(delete); } + /** @deprecated As of release 3.13.0. Will be removed in release 5.0.0. */ + @Deprecated @Override public void delete(List deletes) throws CrudException, UnknownTransactionStatusException { manager.delete(deletes); diff --git a/core/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommit.java b/core/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommit.java index b215ccc928..cd4c2fb4a6 100644 --- a/core/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommit.java +++ b/core/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommit.java @@ -96,6 +96,11 @@ public List scan(Scan scan) throws CrudException { } } + @Override + public Scanner getScanner(Scan scan) throws CrudException { + throw new UnsupportedOperationException("Implement later"); + } + /** @deprecated As of release 3.13.0. Will be removed in release 5.0.0. */ @Deprecated @Override diff --git a/core/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitManager.java b/core/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitManager.java index 93e2056065..46a13035cb 100644 --- a/core/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitManager.java +++ b/core/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitManager.java @@ -229,6 +229,11 @@ public List scan(Scan scan) throws CrudException, UnknownTransactionStat return executeTransaction(t -> t.scan(copyAndSetTargetToIfNot(scan))); } + @Override + public Scanner getScanner(Scan scan) throws CrudException { + throw new UnsupportedOperationException("Implement later"); + } + @Deprecated @Override public void put(Put put) throws CrudException, UnknownTransactionStatusException { diff --git a/core/src/main/java/com/scalar/db/transaction/consensuscommit/TwoPhaseConsensusCommit.java b/core/src/main/java/com/scalar/db/transaction/consensuscommit/TwoPhaseConsensusCommit.java index fe25742dd7..d8edb63291 100644 --- a/core/src/main/java/com/scalar/db/transaction/consensuscommit/TwoPhaseConsensusCommit.java +++ b/core/src/main/java/com/scalar/db/transaction/consensuscommit/TwoPhaseConsensusCommit.java @@ -86,6 +86,11 @@ public List scan(Scan scan) throws CrudException { } } + @Override + public Scanner getScanner(Scan scan) throws CrudException { + throw new UnsupportedOperationException("Implement later"); + } + /** @deprecated As of release 3.13.0. Will be removed in release 5.0.0. */ @Deprecated @Override diff --git a/core/src/main/java/com/scalar/db/transaction/consensuscommit/TwoPhaseConsensusCommitManager.java b/core/src/main/java/com/scalar/db/transaction/consensuscommit/TwoPhaseConsensusCommitManager.java index 5afc3e98df..3c9a47aa60 100644 --- a/core/src/main/java/com/scalar/db/transaction/consensuscommit/TwoPhaseConsensusCommitManager.java +++ b/core/src/main/java/com/scalar/db/transaction/consensuscommit/TwoPhaseConsensusCommitManager.java @@ -186,6 +186,11 @@ public List scan(Scan scan) throws CrudException, UnknownTransactionStat return executeTransaction(t -> t.scan(copyAndSetTargetToIfNot(scan))); } + @Override + public Scanner getScanner(Scan scan) throws CrudException { + throw new UnsupportedOperationException("Implement later"); + } + @Deprecated @Override public void put(Put put) throws CrudException, UnknownTransactionStatusException { diff --git a/core/src/main/java/com/scalar/db/transaction/jdbc/JdbcTransaction.java b/core/src/main/java/com/scalar/db/transaction/jdbc/JdbcTransaction.java index 27e2cdc7f8..a469c47003 100644 --- a/core/src/main/java/com/scalar/db/transaction/jdbc/JdbcTransaction.java +++ b/core/src/main/java/com/scalar/db/transaction/jdbc/JdbcTransaction.java @@ -94,6 +94,11 @@ public List scan(Scan scan) throws CrudException { } } + @Override + public Scanner getScanner(Scan scan) throws CrudException { + throw new UnsupportedOperationException("Implement later"); + } + /** @deprecated As of release 3.13.0. Will be removed in release 5.0.0. */ @Deprecated @Override diff --git a/core/src/main/java/com/scalar/db/transaction/jdbc/JdbcTransactionManager.java b/core/src/main/java/com/scalar/db/transaction/jdbc/JdbcTransactionManager.java index 551725b264..df7f02197d 100644 --- a/core/src/main/java/com/scalar/db/transaction/jdbc/JdbcTransactionManager.java +++ b/core/src/main/java/com/scalar/db/transaction/jdbc/JdbcTransactionManager.java @@ -168,6 +168,11 @@ public List scan(Scan scan) throws CrudException, UnknownTransactionStat return executeTransaction(t -> t.scan(copyAndSetTargetToIfNot(scan))); } + @Override + public Scanner getScanner(Scan scan) throws CrudException { + throw new UnsupportedOperationException("Implement later"); + } + @Deprecated @Override public void put(Put put) throws CrudException, UnknownTransactionStatusException { diff --git a/core/src/main/java/com/scalar/db/transaction/singlecrudoperation/SingleCrudOperationTransactionManager.java b/core/src/main/java/com/scalar/db/transaction/singlecrudoperation/SingleCrudOperationTransactionManager.java index cb488cfa55..03857cac8a 100644 --- a/core/src/main/java/com/scalar/db/transaction/singlecrudoperation/SingleCrudOperationTransactionManager.java +++ b/core/src/main/java/com/scalar/db/transaction/singlecrudoperation/SingleCrudOperationTransactionManager.java @@ -20,7 +20,6 @@ import com.scalar.db.api.PutIf; import com.scalar.db.api.Result; import com.scalar.db.api.Scan; -import com.scalar.db.api.Scanner; import com.scalar.db.api.SerializableStrategy; import com.scalar.db.api.TransactionState; import com.scalar.db.api.Update; @@ -156,13 +155,19 @@ public Optional get(Get get) throws CrudException { public List scan(Scan scan) throws CrudException { scan = copyAndSetTargetToIfNot(scan); - try (Scanner scanner = storage.scan(scan.withConsistency(Consistency.LINEARIZABLE))) { + try (com.scalar.db.api.Scanner scanner = + storage.scan(scan.withConsistency(Consistency.LINEARIZABLE))) { return scanner.all(); } catch (ExecutionException | IOException e) { throw new CrudException(e.getMessage(), e, null); } } + @Override + public Scanner getScanner(Scan scan) throws CrudException { + throw new UnsupportedOperationException("Implement later"); + } + /** @deprecated As of release 3.13.0. Will be removed in release 5.0.0. */ @Deprecated @Override diff --git a/integration-test/src/main/java/com/scalar/db/api/DistributedTransactionIntegrationTestBase.java b/integration-test/src/main/java/com/scalar/db/api/DistributedTransactionIntegrationTestBase.java index 7779090ece..5e3c9910e0 100644 --- a/integration-test/src/main/java/com/scalar/db/api/DistributedTransactionIntegrationTestBase.java +++ b/integration-test/src/main/java/com/scalar/db/api/DistributedTransactionIntegrationTestBase.java @@ -652,7 +652,7 @@ public void scan_ScanAllWithProjectionsGiven_ShouldRetrieveSpecifiedValues() } @Test - public void scanAll_ScanAllGivenForNonExisting_ShouldReturnEmpty() throws TransactionException { + public void scan_ScanAllGivenForNonExisting_ShouldReturnEmpty() throws TransactionException { // Arrange DistributedTransaction transaction = manager.start(); ScanAll scanAll = prepareScanAll(); @@ -1093,6 +1093,30 @@ public void rollback_forOngoingTransaction_ShouldRollbackCorrectly() throws Tran results, Collections.singletonList(expectedResult)); } + @Test + public void getScanner_ScanGivenForCommittedRecord_ShouldReturnRecords() + throws TransactionException { + // Arrange + populateRecords(); + DistributedTransaction transaction = manager.start(); + Scan scan = prepareScan(0, 0, 2); + + // Act Assert + TransactionCrudOperable.Scanner scanner = transaction.getScanner(scan); + + Optional result1 = scanner.one(); + assertThat(result1).isPresent(); + assertResult(0, 0, result1.get()); + + Optional result2 = scanner.one(); + assertThat(result2).isPresent(); + assertResult(0, 1, result2.get()); + + scanner.close(); + + transaction.commit(); + } + @Test public void resume_WithBeginningTransaction_ShouldReturnBegunTransaction() throws TransactionException { @@ -2006,6 +2030,42 @@ public void manager_scan_ScanGivenForCommittedRecord_ShouldReturnRecords() assertThat(results.get(2).getInt(SOME_COLUMN)).isEqualTo(2); } + @Test + public void manager_getScanner_ScanGivenForCommittedRecord_ShouldReturnRecords() + throws TransactionException { + // Arrange + populateRecords(); + Scan scan = prepareScan(1, 0, 2); + + // Act Assert + TransactionManagerCrudOperable.Scanner scanner = manager.getScanner(scan); + + Optional result1 = scanner.one(); + assertThat(result1).isPresent(); + assertThat(result1.get().getInt(ACCOUNT_ID)).isEqualTo(1); + assertThat(result1.get().getInt(ACCOUNT_TYPE)).isEqualTo(0); + assertThat(getBalance(result1.get())).isEqualTo(INITIAL_BALANCE); + assertThat(result1.get().getInt(SOME_COLUMN)).isEqualTo(0); + + Optional result2 = scanner.one(); + assertThat(result2).isPresent(); + assertThat(result2.get().getInt(ACCOUNT_ID)).isEqualTo(1); + assertThat(result2.get().getInt(ACCOUNT_TYPE)).isEqualTo(1); + assertThat(getBalance(result2.get())).isEqualTo(INITIAL_BALANCE); + assertThat(result2.get().getInt(SOME_COLUMN)).isEqualTo(1); + + Optional result3 = scanner.one(); + assertThat(result3).isPresent(); + assertThat(result3.get().getInt(ACCOUNT_ID)).isEqualTo(1); + assertThat(result3.get().getInt(ACCOUNT_TYPE)).isEqualTo(2); + assertThat(getBalance(result3.get())).isEqualTo(INITIAL_BALANCE); + assertThat(result3.get().getInt(SOME_COLUMN)).isEqualTo(2); + + assertThat(scanner.one()).isNotPresent(); + + scanner.close(); + } + @Test public void manager_put_PutGivenForNonExisting_ShouldCreateRecord() throws TransactionException { // Arrange diff --git a/integration-test/src/main/java/com/scalar/db/api/TwoPhaseCommitTransactionIntegrationTestBase.java b/integration-test/src/main/java/com/scalar/db/api/TwoPhaseCommitTransactionIntegrationTestBase.java index 6587cd79a1..f6f7febcbe 100644 --- a/integration-test/src/main/java/com/scalar/db/api/TwoPhaseCommitTransactionIntegrationTestBase.java +++ b/integration-test/src/main/java/com/scalar/db/api/TwoPhaseCommitTransactionIntegrationTestBase.java @@ -1326,6 +1326,32 @@ public void scan_ScanAllGivenForNonExisting_ShouldReturnEmpty() throws Transacti results, Collections.singletonList(expectedResult)); } + @Test + public void getScanner_ScanGivenForCommittedRecord_ShouldReturnRecords() + throws TransactionException { + // Arrange + populateRecords(manager1, namespace1, TABLE_1); + TwoPhaseCommitTransaction transaction = manager1.start(); + Scan scan = prepareScan(0, 0, 2, namespace1, TABLE_1); + + // Act Assert + TransactionCrudOperable.Scanner scanner = transaction.getScanner(scan); + + Optional result1 = scanner.one(); + assertThat(result1).isPresent(); + assertResult(0, 0, result1.get()); + + Optional result2 = scanner.one(); + assertThat(result2).isPresent(); + assertResult(0, 1, result2.get()); + + scanner.close(); + + transaction.prepare(); + transaction.validate(); + transaction.commit(); + } + @Test public void resume_WithBeginningTransaction_ShouldReturnBegunTransaction() throws TransactionException { @@ -2279,6 +2305,42 @@ public void manager_scan_ScanGivenForCommittedRecord_ShouldReturnRecords() assertThat(results.get(2).getInt(SOME_COLUMN)).isEqualTo(2); } + @Test + public void manager_getScanner_ScanGivenForCommittedRecord_ShouldReturnRecords() + throws TransactionException { + // Arrange + populateRecords(manager1, namespace1, TABLE_1); + Scan scan = prepareScan(1, 0, 2, namespace1, TABLE_1); + + // Act Assert + TransactionManagerCrudOperable.Scanner scanner = manager1.getScanner(scan); + + Optional result1 = scanner.one(); + assertThat(result1).isPresent(); + assertThat(result1.get().getInt(ACCOUNT_ID)).isEqualTo(1); + assertThat(result1.get().getInt(ACCOUNT_TYPE)).isEqualTo(0); + assertThat(getBalance(result1.get())).isEqualTo(INITIAL_BALANCE); + assertThat(result1.get().getInt(SOME_COLUMN)).isEqualTo(0); + + Optional result2 = scanner.one(); + assertThat(result2).isPresent(); + assertThat(result2.get().getInt(ACCOUNT_ID)).isEqualTo(1); + assertThat(result2.get().getInt(ACCOUNT_TYPE)).isEqualTo(1); + assertThat(getBalance(result2.get())).isEqualTo(INITIAL_BALANCE); + assertThat(result2.get().getInt(SOME_COLUMN)).isEqualTo(1); + + Optional result3 = scanner.one(); + assertThat(result3).isPresent(); + assertThat(result3.get().getInt(ACCOUNT_ID)).isEqualTo(1); + assertThat(result3.get().getInt(ACCOUNT_TYPE)).isEqualTo(2); + assertThat(getBalance(result3.get())).isEqualTo(INITIAL_BALANCE); + assertThat(result3.get().getInt(SOME_COLUMN)).isEqualTo(2); + + assertThat(scanner.one()).isNotPresent(); + + scanner.close(); + } + @Test public void manager_put_PutGivenForNonExisting_ShouldCreateRecord() throws TransactionException { // Arrange diff --git a/integration-test/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitIntegrationTestBase.java b/integration-test/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitIntegrationTestBase.java index bf5abaae05..9854894ac4 100644 --- a/integration-test/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitIntegrationTestBase.java +++ b/integration-test/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitIntegrationTestBase.java @@ -15,6 +15,7 @@ import com.scalar.db.io.Key; import java.util.Optional; import java.util.Properties; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; public abstract class ConsensusCommitIntegrationTestBase @@ -929,4 +930,14 @@ public void deleteAndDelete_forSameRecord_shouldWorkCorrectly() throws Transacti Optional optResult = get(prepareGet(0, 0)); assertThat(optResult).isNotPresent(); } + + @Disabled("Implement later") + @Override + @Test + public void getScanner_ScanGivenForCommittedRecord_ShouldReturnRecords() {} + + @Disabled("Implement later") + @Override + @Test + public void manager_getScanner_ScanGivenForCommittedRecord_ShouldReturnRecords() {} } diff --git a/integration-test/src/main/java/com/scalar/db/transaction/consensuscommit/TwoPhaseConsensusCommitIntegrationTestBase.java b/integration-test/src/main/java/com/scalar/db/transaction/consensuscommit/TwoPhaseConsensusCommitIntegrationTestBase.java index 781087ed79..d57c31302c 100644 --- a/integration-test/src/main/java/com/scalar/db/transaction/consensuscommit/TwoPhaseConsensusCommitIntegrationTestBase.java +++ b/integration-test/src/main/java/com/scalar/db/transaction/consensuscommit/TwoPhaseConsensusCommitIntegrationTestBase.java @@ -2,6 +2,8 @@ import com.scalar.db.api.TwoPhaseCommitTransactionIntegrationTestBase; import java.util.Properties; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; public abstract class TwoPhaseConsensusCommitIntegrationTestBase extends TwoPhaseCommitTransactionIntegrationTestBase { @@ -38,4 +40,14 @@ protected final Properties getProperties2(String testName) { protected Properties getProps2(String testName) { return getProps1(testName); } + + @Disabled("Implement later") + @Override + @Test + public void getScanner_ScanGivenForCommittedRecord_ShouldReturnRecords() {} + + @Disabled("Implement later") + @Override + @Test + public void manager_getScanner_ScanGivenForCommittedRecord_ShouldReturnRecords() {} } diff --git a/integration-test/src/main/java/com/scalar/db/transaction/singlecrudoperation/SingleCrudOperationTransactionIntegrationTestBase.java b/integration-test/src/main/java/com/scalar/db/transaction/singlecrudoperation/SingleCrudOperationTransactionIntegrationTestBase.java index d9c27237c1..6c50954e07 100644 --- a/integration-test/src/main/java/com/scalar/db/transaction/singlecrudoperation/SingleCrudOperationTransactionIntegrationTestBase.java +++ b/integration-test/src/main/java/com/scalar/db/transaction/singlecrudoperation/SingleCrudOperationTransactionIntegrationTestBase.java @@ -126,7 +126,12 @@ public void scan_ScanAllWithProjectionsGiven_ShouldRetrieveSpecifiedValues() {} @Disabled("Single CRUD operation transactions don't support beginning a transaction") @Override @Test - public void scanAll_ScanAllGivenForNonExisting_ShouldReturnEmpty() {} + public void scan_ScanAllGivenForNonExisting_ShouldReturnEmpty() {} + + @Disabled("Single CRUD operation transactions don't support beginning a transaction") + @Override + @Test + public void getScanner_ScanGivenForCommittedRecord_ShouldReturnRecords() {} @Disabled("Single CRUD operation transactions don't support beginning a transaction") @Override @@ -405,6 +410,11 @@ public void abort_forOngoingTransaction_ShouldAbortCorrectly() {} @Test public void rollback_forOngoingTransaction_ShouldRollbackCorrectly() {} + @Disabled("Implement later") + @Override + @Test + public void manager_getScanner_ScanGivenForCommittedRecord_ShouldReturnRecords() {} + @Disabled( "Single CRUD operation transactions don't support executing multiple mutations in a transaction") @Override From 8ea08fab028a4ea755411672266455af75954c59 Mon Sep 17 00:00:00 2001 From: Toshihiro Suzuki Date: Thu, 29 May 2025 14:18:56 +0900 Subject: [PATCH 02/19] Implement scanner API for single CRUD transactions (#2701) --- ...SingleCrudOperationTransactionManager.java | 39 +++- ...leCrudOperationTransactionManagerTest.java | 175 ++++++++++++++++++ ...erationTransactionIntegrationTestBase.java | 5 - 3 files changed, 213 insertions(+), 6 deletions(-) diff --git a/core/src/main/java/com/scalar/db/transaction/singlecrudoperation/SingleCrudOperationTransactionManager.java b/core/src/main/java/com/scalar/db/transaction/singlecrudoperation/SingleCrudOperationTransactionManager.java index 03857cac8a..cbc136d120 100644 --- a/core/src/main/java/com/scalar/db/transaction/singlecrudoperation/SingleCrudOperationTransactionManager.java +++ b/core/src/main/java/com/scalar/db/transaction/singlecrudoperation/SingleCrudOperationTransactionManager.java @@ -27,6 +27,7 @@ import com.scalar.db.api.UpdateIfExists; import com.scalar.db.api.Upsert; import com.scalar.db.common.AbstractDistributedTransactionManager; +import com.scalar.db.common.AbstractTransactionManagerCrudOperableScanner; import com.scalar.db.common.error.CoreError; import com.scalar.db.config.DatabaseConfig; import com.scalar.db.exception.storage.ExecutionException; @@ -165,7 +166,43 @@ public List scan(Scan scan) throws CrudException { @Override public Scanner getScanner(Scan scan) throws CrudException { - throw new UnsupportedOperationException("Implement later"); + scan = copyAndSetTargetToIfNot(scan); + + com.scalar.db.api.Scanner scanner; + try { + scanner = storage.scan(scan); + } catch (ExecutionException e) { + throw new CrudException(e.getMessage(), e, null); + } + + return new AbstractTransactionManagerCrudOperableScanner() { + @Override + public Optional one() throws CrudException { + try { + return scanner.one(); + } catch (ExecutionException e) { + throw new CrudException(e.getMessage(), e, null); + } + } + + @Override + public List all() throws CrudException { + try { + return scanner.all(); + } catch (ExecutionException e) { + throw new CrudException(e.getMessage(), e, null); + } + } + + @Override + public void close() throws CrudException { + try { + scanner.close(); + } catch (IOException e) { + throw new CrudException(e.getMessage(), e, null); + } + } + }; } /** @deprecated As of release 3.13.0. Will be removed in release 5.0.0. */ diff --git a/core/src/test/java/com/scalar/db/transaction/singlecrudoperation/SingleCrudOperationTransactionManagerTest.java b/core/src/test/java/com/scalar/db/transaction/singlecrudoperation/SingleCrudOperationTransactionManagerTest.java index c96d609485..42ef3d1c96 100644 --- a/core/src/test/java/com/scalar/db/transaction/singlecrudoperation/SingleCrudOperationTransactionManagerTest.java +++ b/core/src/test/java/com/scalar/db/transaction/singlecrudoperation/SingleCrudOperationTransactionManagerTest.java @@ -18,6 +18,7 @@ import com.scalar.db.api.Result; import com.scalar.db.api.Scan; import com.scalar.db.api.Scanner; +import com.scalar.db.api.TransactionManagerCrudOperable; import com.scalar.db.api.Update; import com.scalar.db.api.Upsert; import com.scalar.db.config.DatabaseConfig; @@ -28,7 +29,9 @@ import com.scalar.db.exception.transaction.TransactionException; import com.scalar.db.exception.transaction.UnsatisfiedConditionException; import com.scalar.db.io.Key; +import java.io.IOException; import java.util.Arrays; +import java.util.Iterator; import java.util.List; import java.util.Optional; import org.junit.jupiter.api.BeforeEach; @@ -151,6 +154,178 @@ public void scan_ExecutionExceptionThrownByStorage_ShouldThrowCrudException() .hasCause(exception); } + @Test + public void getScannerAndScannerOne_ShouldReturnScannerAndShouldReturnProperResult() + throws ExecutionException, TransactionException, IOException { + // Arrange + Scan scan = + Scan.newBuilder().namespace("ns").table("tbl").partitionKey(Key.ofInt("id", 0)).build(); + + Result result1 = mock(Result.class); + Result result2 = mock(Result.class); + Result result3 = mock(Result.class); + + Scanner scanner = mock(Scanner.class); + when(scanner.one()) + .thenReturn(Optional.of(result1)) + .thenReturn(Optional.of(result2)) + .thenReturn(Optional.of(result3)) + .thenReturn(Optional.empty()); + + when(storage.scan(scan)).thenReturn(scanner); + + // Act Assert + TransactionManagerCrudOperable.Scanner actual = transactionManager.getScanner(scan); + assertThat(actual.one()).hasValue(result1); + assertThat(actual.one()).hasValue(result2); + assertThat(actual.one()).hasValue(result3); + assertThat(actual.one()).isEmpty(); + actual.close(); + + verify(storage).scan(scan); + verify(scanner).close(); + } + + @Test + public void getScannerAndScannerAll_ShouldReturnScannerAndShouldReturnProperResults() + throws ExecutionException, TransactionException, IOException { + // Arrange + Scan scan = + Scan.newBuilder().namespace("ns").table("tbl").partitionKey(Key.ofInt("id", 0)).build(); + + Result result1 = mock(Result.class); + Result result2 = mock(Result.class); + Result result3 = mock(Result.class); + + Scanner scanner = mock(Scanner.class); + when(scanner.all()).thenReturn(Arrays.asList(result1, result2, result3)); + + when(storage.scan(scan)).thenReturn(scanner); + + // Act Assert + TransactionManagerCrudOperable.Scanner actual = transactionManager.getScanner(scan); + assertThat(actual.all()).containsExactly(result1, result2, result3); + actual.close(); + + verify(storage).scan(scan); + verify(scanner).close(); + } + + @Test + public void getScannerAndScannerIterator_ShouldReturnScannerAndShouldReturnProperResults() + throws ExecutionException, TransactionException, IOException { + // Arrange + Scan scan = + Scan.newBuilder().namespace("ns").table("tbl").partitionKey(Key.ofInt("id", 0)).build(); + + Result result1 = mock(Result.class); + Result result2 = mock(Result.class); + Result result3 = mock(Result.class); + + Scanner scanner = mock(Scanner.class); + when(scanner.one()) + .thenReturn(Optional.of(result1)) + .thenReturn(Optional.of(result2)) + .thenReturn(Optional.of(result3)) + .thenReturn(Optional.empty()); + + when(storage.scan(scan)).thenReturn(scanner); + + // Act Assert + TransactionManagerCrudOperable.Scanner actual = transactionManager.getScanner(scan); + + Iterator iterator = actual.iterator(); + assertThat(iterator.hasNext()).isTrue(); + assertThat(iterator.next()).isEqualTo(result1); + assertThat(iterator.hasNext()).isTrue(); + assertThat(iterator.next()).isEqualTo(result2); + assertThat(iterator.hasNext()).isTrue(); + assertThat(iterator.next()).isEqualTo(result3); + assertThat(iterator.hasNext()).isFalse(); + actual.close(); + + verify(storage).scan(scan); + verify(scanner).close(); + } + + @Test + public void getScanner_WhenExecutionExceptionThrownByJdbcService_ShouldThrowCrudException() + throws ExecutionException { + // Arrange + Scan scan = + Scan.newBuilder().namespace("ns").table("tbl").partitionKey(Key.ofInt("id", 0)).build(); + + ExecutionException executionException = mock(ExecutionException.class); + when(executionException.getMessage()).thenReturn("error"); + when(storage.scan(scan)).thenThrow(executionException); + + // Act Assert + assertThatThrownBy(() -> transactionManager.getScanner(scan)).isInstanceOf(CrudException.class); + } + + @Test + public void + getScannerAndScannerOne_WhenExecutionExceptionThrownByScannerOne_ShouldThrowCrudException() + throws ExecutionException, CrudException { + // Arrange + Scan scan = + Scan.newBuilder().namespace("ns").table("tbl").partitionKey(Key.ofInt("id", 0)).build(); + + Scanner scanner = mock(Scanner.class); + + ExecutionException executionException = mock(ExecutionException.class); + when(executionException.getMessage()).thenReturn("error"); + when(scanner.one()).thenThrow(executionException); + + when(storage.scan(scan)).thenReturn(scanner); + + // Act Assert + TransactionManagerCrudOperable.Scanner actual = transactionManager.getScanner(scan); + assertThatThrownBy(actual::one).isInstanceOf(CrudException.class); + } + + @Test + public void + getScannerAndScannerAll_WhenExecutionExceptionThrownByScannerAll_ShouldThrowCrudException() + throws ExecutionException, CrudException { + // Arrange + Scan scan = + Scan.newBuilder().namespace("ns").table("tbl").partitionKey(Key.ofInt("id", 0)).build(); + + Scanner scanner = mock(Scanner.class); + + ExecutionException executionException = mock(ExecutionException.class); + when(executionException.getMessage()).thenReturn("error"); + when(scanner.all()).thenThrow(executionException); + + when(storage.scan(scan)).thenReturn(scanner); + + // Act Assert + TransactionManagerCrudOperable.Scanner actual = transactionManager.getScanner(scan); + assertThatThrownBy(actual::all).isInstanceOf(CrudException.class); + } + + @Test + public void + getScannerAndScannerClose_WhenIOExceptionThrownByScannerClose_ShouldThrowCrudException() + throws ExecutionException, CrudException, IOException { + // Arrange + Scan scan = + Scan.newBuilder().namespace("ns").table("tbl").partitionKey(Key.ofInt("id", 0)).build(); + + Scanner scanner = mock(Scanner.class); + + IOException ioException = mock(IOException.class); + when(ioException.getMessage()).thenReturn("error"); + doThrow(ioException).when(scanner).close(); + + when(storage.scan(scan)).thenReturn(scanner); + + // Act Assert + TransactionManagerCrudOperable.Scanner actual = transactionManager.getScanner(scan); + assertThatThrownBy(actual::close).isInstanceOf(CrudException.class); + } + @Test public void put_ShouldCallStorageProperly() throws ExecutionException, TransactionException { // Arrange diff --git a/integration-test/src/main/java/com/scalar/db/transaction/singlecrudoperation/SingleCrudOperationTransactionIntegrationTestBase.java b/integration-test/src/main/java/com/scalar/db/transaction/singlecrudoperation/SingleCrudOperationTransactionIntegrationTestBase.java index 6c50954e07..c3e9fc249e 100644 --- a/integration-test/src/main/java/com/scalar/db/transaction/singlecrudoperation/SingleCrudOperationTransactionIntegrationTestBase.java +++ b/integration-test/src/main/java/com/scalar/db/transaction/singlecrudoperation/SingleCrudOperationTransactionIntegrationTestBase.java @@ -410,11 +410,6 @@ public void abort_forOngoingTransaction_ShouldAbortCorrectly() {} @Test public void rollback_forOngoingTransaction_ShouldRollbackCorrectly() {} - @Disabled("Implement later") - @Override - @Test - public void manager_getScanner_ScanGivenForCommittedRecord_ShouldReturnRecords() {} - @Disabled( "Single CRUD operation transactions don't support executing multiple mutations in a transaction") @Override From 5042c49ef41d3216a78d3d9b55314981334d62ff Mon Sep 17 00:00:00 2001 From: Toshihiro Suzuki Date: Thu, 29 May 2025 15:23:39 +0900 Subject: [PATCH 03/19] Implement scanner API for JDBC transactions (#2702) --- .../jdbc/JdbcTransactionIntegrationTest.java | 10 - .../com/scalar/db/common/error/CoreError.java | 2 + .../scalar/db/storage/jdbc/JdbcService.java | 10 +- .../scalar/db/storage/jdbc/ScannerImpl.java | 16 +- .../db/transaction/jdbc/JdbcTransaction.java | 43 +- .../jdbc/JdbcTransactionManager.java | 91 ++++- .../db/storage/jdbc/JdbcDatabaseTest.java | 3 +- .../jdbc/JdbcTransactionManagerTest.java | 366 ++++++++++++++++++ .../transaction/jdbc/JdbcTransactionTest.java | 227 +++++++++++ 9 files changed, 748 insertions(+), 20 deletions(-) diff --git a/core/src/integration-test/java/com/scalar/db/transaction/jdbc/JdbcTransactionIntegrationTest.java b/core/src/integration-test/java/com/scalar/db/transaction/jdbc/JdbcTransactionIntegrationTest.java index 0ca3b064b8..37bebaf726 100644 --- a/core/src/integration-test/java/com/scalar/db/transaction/jdbc/JdbcTransactionIntegrationTest.java +++ b/core/src/integration-test/java/com/scalar/db/transaction/jdbc/JdbcTransactionIntegrationTest.java @@ -42,14 +42,4 @@ public void abort_forOngoingTransaction_ShouldAbortCorrectly() {} @Override @Test public void rollback_forOngoingTransaction_ShouldRollbackCorrectly() {} - - @Disabled("Implement later") - @Override - @Test - public void getScanner_ScanGivenForCommittedRecord_ShouldReturnRecords() {} - - @Disabled("Implement later") - @Override - @Test - public void manager_getScanner_ScanGivenForCommittedRecord_ShouldReturnRecords() {} } diff --git a/core/src/main/java/com/scalar/db/common/error/CoreError.java b/core/src/main/java/com/scalar/db/common/error/CoreError.java index acaa4566dd..1e465bd550 100644 --- a/core/src/main/java/com/scalar/db/common/error/CoreError.java +++ b/core/src/main/java/com/scalar/db/common/error/CoreError.java @@ -1182,6 +1182,8 @@ public enum CoreError implements ScalarDbError { Category.INTERNAL_ERROR, "0052", "Failed to read JSON file. Details: %s.", "", ""), DATA_LOADER_JSONLINES_FILE_READ_FAILED( Category.INTERNAL_ERROR, "0053", "Failed to read JSON Lines file. Details: %s.", "", ""), + JDBC_TRANSACTION_GETTING_SCANNER_FAILED( + Category.INTERNAL_ERROR, "0054", "Getting the scanner failed. Details: %s", "", ""), // // Errors for the unknown transaction status error category diff --git a/core/src/main/java/com/scalar/db/storage/jdbc/JdbcService.java b/core/src/main/java/com/scalar/db/storage/jdbc/JdbcService.java index 82698ba5c9..852812ac94 100644 --- a/core/src/main/java/com/scalar/db/storage/jdbc/JdbcService.java +++ b/core/src/main/java/com/scalar/db/storage/jdbc/JdbcService.java @@ -96,9 +96,14 @@ public Optional get(Get get, Connection connection) } } - @SuppressFBWarnings("OBL_UNSATISFIED_OBLIGATION_EXCEPTION_EDGE") public Scanner getScanner(Scan scan, Connection connection) throws SQLException, ExecutionException { + return getScanner(scan, connection, true); + } + + @SuppressFBWarnings("OBL_UNSATISFIED_OBLIGATION_EXCEPTION_EDGE") + public Scanner getScanner(Scan scan, Connection connection, boolean closeConnectionOnScannerClose) + throws SQLException, ExecutionException { operationChecker.check(scan); TableMetadata tableMetadata = tableMetadataManager.getTableMetadata(scan); @@ -111,7 +116,8 @@ public Scanner getScanner(Scan scan, Connection connection) new ResultInterpreter(scan.getProjections(), tableMetadata, rdbEngine), connection, preparedStatement, - resultSet); + resultSet, + closeConnectionOnScannerClose); } public List scan(Scan scan, Connection connection) diff --git a/core/src/main/java/com/scalar/db/storage/jdbc/ScannerImpl.java b/core/src/main/java/com/scalar/db/storage/jdbc/ScannerImpl.java index 3d48e2a2f9..d9dc38268e 100644 --- a/core/src/main/java/com/scalar/db/storage/jdbc/ScannerImpl.java +++ b/core/src/main/java/com/scalar/db/storage/jdbc/ScannerImpl.java @@ -25,17 +25,20 @@ public class ScannerImpl extends AbstractScanner { private final Connection connection; private final PreparedStatement preparedStatement; private final ResultSet resultSet; + private final boolean closeConnectionOnClose; @SuppressFBWarnings("EI_EXPOSE_REP2") public ScannerImpl( ResultInterpreter resultInterpreter, Connection connection, PreparedStatement preparedStatement, - ResultSet resultSet) { + ResultSet resultSet, + boolean closeConnectionOnClose) { this.resultInterpreter = Objects.requireNonNull(resultInterpreter); this.connection = Objects.requireNonNull(connection); this.preparedStatement = Objects.requireNonNull(preparedStatement); this.resultSet = Objects.requireNonNull(resultSet); + this.closeConnectionOnClose = closeConnectionOnClose; } @Override @@ -75,10 +78,13 @@ public void close() { } catch (SQLException e) { logger.warn("Failed to close the preparedStatement", e); } - try { - connection.close(); - } catch (SQLException e) { - logger.warn("Failed to close the connection", e); + + if (closeConnectionOnClose) { + try { + connection.close(); + } catch (SQLException e) { + logger.warn("Failed to close the connection", e); + } } } } diff --git a/core/src/main/java/com/scalar/db/transaction/jdbc/JdbcTransaction.java b/core/src/main/java/com/scalar/db/transaction/jdbc/JdbcTransaction.java index a469c47003..b29bf7ae31 100644 --- a/core/src/main/java/com/scalar/db/transaction/jdbc/JdbcTransaction.java +++ b/core/src/main/java/com/scalar/db/transaction/jdbc/JdbcTransaction.java @@ -20,6 +20,7 @@ import com.scalar.db.api.UpdateIfExists; import com.scalar.db.api.Upsert; import com.scalar.db.common.AbstractDistributedTransaction; +import com.scalar.db.common.AbstractTransactionCrudOperableScanner; import com.scalar.db.common.error.CoreError; import com.scalar.db.exception.storage.ExecutionException; import com.scalar.db.exception.transaction.CommitConflictException; @@ -32,6 +33,7 @@ import com.scalar.db.storage.jdbc.JdbcService; import com.scalar.db.storage.jdbc.RdbEngineStrategy; import com.scalar.db.util.ScalarDbUtils; +import java.io.IOException; import java.sql.Connection; import java.sql.SQLException; import java.util.List; @@ -96,7 +98,46 @@ public List scan(Scan scan) throws CrudException { @Override public Scanner getScanner(Scan scan) throws CrudException { - throw new UnsupportedOperationException("Implement later"); + scan = copyAndSetTargetToIfNot(scan); + + com.scalar.db.api.Scanner scanner; + try { + scanner = jdbcService.getScanner(scan, connection, false); + } catch (SQLException e) { + throw createCrudException( + e, CoreError.JDBC_TRANSACTION_GETTING_SCANNER_FAILED.buildMessage(e.getMessage())); + } catch (ExecutionException e) { + throw new CrudException(e.getMessage(), e, txId); + } + + return new AbstractTransactionCrudOperableScanner() { + @Override + public Optional one() throws CrudException { + try { + return scanner.one(); + } catch (ExecutionException e) { + throw new CrudException(e.getMessage(), e, txId); + } + } + + @Override + public List all() throws CrudException { + try { + return scanner.all(); + } catch (ExecutionException e) { + throw new CrudException(e.getMessage(), e, txId); + } + } + + @Override + public void close() throws CrudException { + try { + scanner.close(); + } catch (IOException e) { + throw new CrudException(e.getMessage(), e, txId); + } + } + }; } /** @deprecated As of release 3.13.0. Will be removed in release 5.0.0. */ diff --git a/core/src/main/java/com/scalar/db/transaction/jdbc/JdbcTransactionManager.java b/core/src/main/java/com/scalar/db/transaction/jdbc/JdbcTransactionManager.java index df7f02197d..d65b512106 100644 --- a/core/src/main/java/com/scalar/db/transaction/jdbc/JdbcTransactionManager.java +++ b/core/src/main/java/com/scalar/db/transaction/jdbc/JdbcTransactionManager.java @@ -12,10 +12,12 @@ import com.scalar.db.api.Result; import com.scalar.db.api.Scan; import com.scalar.db.api.SerializableStrategy; +import com.scalar.db.api.TransactionCrudOperable; import com.scalar.db.api.TransactionState; import com.scalar.db.api.Update; import com.scalar.db.api.Upsert; import com.scalar.db.common.AbstractDistributedTransactionManager; +import com.scalar.db.common.AbstractTransactionManagerCrudOperableScanner; import com.scalar.db.common.TableMetadataManager; import com.scalar.db.common.checker.OperationChecker; import com.scalar.db.common.error.CoreError; @@ -38,6 +40,7 @@ import java.util.List; import java.util.Optional; import java.util.UUID; +import java.util.concurrent.atomic.AtomicBoolean; import javax.annotation.concurrent.ThreadSafe; import org.apache.commons.dbcp2.BasicDataSource; import org.slf4j.Logger; @@ -170,9 +173,93 @@ public List scan(Scan scan) throws CrudException, UnknownTransactionStat @Override public Scanner getScanner(Scan scan) throws CrudException { - throw new UnsupportedOperationException("Implement later"); + DistributedTransaction transaction; + try { + transaction = begin(); + } catch (TransactionNotFoundException e) { + throw new CrudConflictException(e.getMessage(), e, e.getTransactionId().orElse(null)); + } catch (TransactionException e) { + throw new CrudException(e.getMessage(), e, e.getTransactionId().orElse(null)); + } + + TransactionCrudOperable.Scanner scanner; + try { + scanner = transaction.getScanner(copyAndSetTargetToIfNot(scan)); + } catch (CrudException e) { + rollbackTransaction(transaction); + throw e; + } + + return new AbstractTransactionManagerCrudOperableScanner() { + + private final AtomicBoolean closed = new AtomicBoolean(); + + @Override + public Optional one() throws CrudException { + try { + return scanner.one(); + } catch (CrudException e) { + closed.set(true); + + try { + scanner.close(); + } catch (CrudException ex) { + e.addSuppressed(ex); + } + + rollbackTransaction(transaction); + throw e; + } + } + + @Override + public List all() throws CrudException { + try { + return scanner.all(); + } catch (CrudException e) { + closed.set(true); + + try { + scanner.close(); + } catch (CrudException ex) { + e.addSuppressed(ex); + } + + rollbackTransaction(transaction); + throw e; + } + } + + @Override + public void close() throws CrudException, UnknownTransactionStatusException { + if (closed.get()) { + return; + } + closed.set(true); + + try { + scanner.close(); + } catch (CrudException e) { + rollbackTransaction(transaction); + throw e; + } + + try { + transaction.commit(); + } catch (CommitConflictException e) { + rollbackTransaction(transaction); + throw new CrudConflictException(e.getMessage(), e, e.getTransactionId().orElse(null)); + } catch (UnknownTransactionStatusException e) { + throw e; + } catch (TransactionException e) { + rollbackTransaction(transaction); + throw new CrudException(e.getMessage(), e, e.getTransactionId().orElse(null)); + } + } + }; } + /** @deprecated As of release 3.13.0. Will be removed in release 5.0.0. */ @Deprecated @Override public void put(Put put) throws CrudException, UnknownTransactionStatusException { @@ -183,6 +270,7 @@ public void put(Put put) throws CrudException, UnknownTransactionStatusException }); } + /** @deprecated As of release 3.13.0. Will be removed in release 5.0.0. */ @Deprecated @Override public void put(List puts) throws CrudException, UnknownTransactionStatusException { @@ -229,6 +317,7 @@ public void delete(Delete delete) throws CrudException, UnknownTransactionStatus }); } + /** @deprecated As of release 3.13.0. Will be removed in release 5.0.0. */ @Deprecated @Override public void delete(List deletes) throws CrudException, UnknownTransactionStatusException { diff --git a/core/src/test/java/com/scalar/db/storage/jdbc/JdbcDatabaseTest.java b/core/src/test/java/com/scalar/db/storage/jdbc/JdbcDatabaseTest.java index 82b40061a6..7710f6a426 100644 --- a/core/src/test/java/com/scalar/db/storage/jdbc/JdbcDatabaseTest.java +++ b/core/src/test/java/com/scalar/db/storage/jdbc/JdbcDatabaseTest.java @@ -98,7 +98,8 @@ public void whenGetOperationExecuted_shouldCallJdbcService() throws Exception { public void whenScanOperationExecutedAndScannerClosed_shouldCallJdbcService() throws Exception { // Arrange when(jdbcService.getScanner(any(), any())) - .thenReturn(new ScannerImpl(resultInterpreter, connection, preparedStatement, resultSet)); + .thenReturn( + new ScannerImpl(resultInterpreter, connection, preparedStatement, resultSet, true)); // Act Scan scan = new Scan(new Key("p1", "val")).forNamespace(NAMESPACE).forTable(TABLE); diff --git a/core/src/test/java/com/scalar/db/transaction/jdbc/JdbcTransactionManagerTest.java b/core/src/test/java/com/scalar/db/transaction/jdbc/JdbcTransactionManagerTest.java index 813d757334..80d0375e6d 100644 --- a/core/src/test/java/com/scalar/db/transaction/jdbc/JdbcTransactionManagerTest.java +++ b/core/src/test/java/com/scalar/db/transaction/jdbc/JdbcTransactionManagerTest.java @@ -20,6 +20,8 @@ import com.scalar.db.api.Put; import com.scalar.db.api.Result; import com.scalar.db.api.Scan; +import com.scalar.db.api.TransactionCrudOperable; +import com.scalar.db.api.TransactionManagerCrudOperable; import com.scalar.db.api.Update; import com.scalar.db.api.Upsert; import com.scalar.db.common.ActiveTransactionManagedDistributedTransactionManager; @@ -41,6 +43,7 @@ import java.sql.SQLException; import java.util.Arrays; import java.util.Collections; +import java.util.Iterator; import java.util.List; import java.util.Optional; import org.apache.commons.dbcp2.BasicDataSource; @@ -178,6 +181,369 @@ public void scan_withConflictError_shouldThrowCrudConflictException() .isInstanceOf(CrudConflictException.class); } + @Test + public void getScannerAndScannerOne_ShouldReturnScannerAndReturnProperResult() throws Exception { + // Arrange + DistributedTransaction transaction = mock(DistributedTransaction.class); + + JdbcTransactionManager spied = spy(manager); + doReturn(transaction).when(spied).begin(); + + Scan scan = + Scan.newBuilder() + .namespace(NAMESPACE) + .table(TABLE) + .partitionKey(Key.ofText("p1", "val")) + .build(); + + TransactionCrudOperable.Scanner scanner = mock(TransactionCrudOperable.Scanner.class); + when(transaction.getScanner(scan)).thenReturn(scanner); + + Result result1 = mock(Result.class); + Result result2 = mock(Result.class); + Result result3 = mock(Result.class); + + when(scanner.one()) + .thenReturn(Optional.of(result1)) + .thenReturn(Optional.of(result2)) + .thenReturn(Optional.of(result3)) + .thenReturn(Optional.empty()); + + // Act Assert + TransactionManagerCrudOperable.Scanner actual = spied.getScanner(scan); + assertThat(actual.one()).hasValue(result1); + assertThat(actual.one()).hasValue(result2); + assertThat(actual.one()).hasValue(result3); + assertThat(actual.one()).isEmpty(); + actual.close(); + + verify(spied).begin(); + verify(transaction).commit(); + verify(scanner).close(); + } + + @Test + public void getScannerAndScannerAll_ShouldReturnScannerAndReturnProperResults() throws Exception { + // Arrange + DistributedTransaction transaction = mock(DistributedTransaction.class); + + JdbcTransactionManager spied = spy(manager); + doReturn(transaction).when(spied).begin(); + + Scan scan = + Scan.newBuilder() + .namespace(NAMESPACE) + .table(TABLE) + .partitionKey(Key.ofText("p1", "val")) + .build(); + + TransactionCrudOperable.Scanner scanner = mock(TransactionCrudOperable.Scanner.class); + when(transaction.getScanner(scan)).thenReturn(scanner); + + Result result1 = mock(Result.class); + Result result2 = mock(Result.class); + Result result3 = mock(Result.class); + + when(scanner.all()) + .thenReturn(Arrays.asList(result1, result2, result3)) + .thenReturn(Collections.emptyList()); + + // Act Assert + TransactionManagerCrudOperable.Scanner actual = spied.getScanner(scan); + List results = actual.all(); + assertThat(results).containsExactly(result1, result2, result3); + assertThat(actual.all()).isEmpty(); + actual.close(); + + verify(spied).begin(); + verify(transaction).commit(); + verify(scanner).close(); + } + + @Test + public void getScannerAndScannerIterator_ShouldReturnScannerAndReturnProperResults() + throws Exception { + // Arrange + DistributedTransaction transaction = mock(DistributedTransaction.class); + + JdbcTransactionManager spied = spy(manager); + doReturn(transaction).when(spied).begin(); + + Scan scan = + Scan.newBuilder() + .namespace(NAMESPACE) + .table(TABLE) + .partitionKey(Key.ofText("p1", "val")) + .build(); + + TransactionCrudOperable.Scanner scanner = mock(TransactionCrudOperable.Scanner.class); + when(transaction.getScanner(scan)).thenReturn(scanner); + + Result result1 = mock(Result.class); + Result result2 = mock(Result.class); + Result result3 = mock(Result.class); + + when(scanner.one()) + .thenReturn(Optional.of(result1)) + .thenReturn(Optional.of(result2)) + .thenReturn(Optional.of(result3)) + .thenReturn(Optional.empty()); + + // Act Assert + TransactionManagerCrudOperable.Scanner actual = spied.getScanner(scan); + + Iterator iterator = actual.iterator(); + assertThat(iterator.hasNext()).isTrue(); + assertThat(iterator.next()).isEqualTo(result1); + assertThat(iterator.hasNext()).isTrue(); + assertThat(iterator.next()).isEqualTo(result2); + assertThat(iterator.hasNext()).isTrue(); + assertThat(iterator.next()).isEqualTo(result3); + assertThat(iterator.hasNext()).isFalse(); + actual.close(); + + verify(spied).begin(); + verify(transaction).commit(); + verify(scanner).close(); + } + + @Test + public void + getScanner_TransactionNotFoundExceptionThrownByTransactionBegin_ShouldThrowCrudConflictException() + throws TransactionException { + // Arrange + JdbcTransactionManager spied = spy(manager); + doThrow(TransactionNotFoundException.class).when(spied).begin(); + + Scan scan = mock(Scan.class); + + // Act Assert + assertThatThrownBy(() -> spied.getScanner(scan)).isInstanceOf(CrudConflictException.class); + + verify(spied).begin(); + } + + @Test + public void getScanner_TransactionExceptionThrownByTransactionBegin_ShouldThrowCrudException() + throws TransactionException { + // Arrange + JdbcTransactionManager spied = spy(manager); + doThrow(TransactionException.class).when(spied).begin(); + + Scan scan = mock(Scan.class); + + // Act Assert + assertThatThrownBy(() -> spied.getScanner(scan)).isInstanceOf(CrudException.class); + + verify(spied).begin(); + } + + @Test + public void + getScanner_CrudExceptionThrownByTransactionGetScanner_ShouldRollbackTransactionAndThrowCrudException() + throws TransactionException { + // Arrange + DistributedTransaction transaction = mock(DistributedTransaction.class); + + JdbcTransactionManager spied = spy(manager); + doReturn(transaction).when(spied).begin(); + + Scan scan = + Scan.newBuilder() + .namespace(NAMESPACE) + .table(TABLE) + .partitionKey(Key.ofText("p1", "val")) + .build(); + + when(transaction.getScanner(scan)).thenThrow(CrudException.class); + + // Act Assert + assertThatThrownBy(() -> spied.getScanner(scan)).isInstanceOf(CrudException.class); + + verify(spied).begin(); + verify(transaction).rollback(); + } + + @Test + public void + getScannerAndScannerOne_CrudExceptionThrownByScannerOne_ShouldRollbackTransactionAndThrowCrudException() + throws TransactionException { + // Arrange + DistributedTransaction transaction = mock(DistributedTransaction.class); + + JdbcTransactionManager spied = spy(manager); + doReturn(transaction).when(spied).begin(); + + Scan scan = + Scan.newBuilder() + .namespace(NAMESPACE) + .table(TABLE) + .partitionKey(Key.ofText("p1", "val")) + .build(); + + TransactionCrudOperable.Scanner scanner = mock(TransactionCrudOperable.Scanner.class); + when(transaction.getScanner(scan)).thenReturn(scanner); + + when(scanner.one()).thenThrow(CrudException.class); + + // Act Assert + TransactionManagerCrudOperable.Scanner actual = spied.getScanner(scan); + assertThatThrownBy(actual::one).isInstanceOf(CrudException.class); + + verify(spied).begin(); + verify(scanner).close(); + verify(transaction).rollback(); + } + + @Test + public void + getScannerAndScannerAll_CrudExceptionThrownByScannerAll_ShouldRollbackTransactionAndThrowCrudException() + throws TransactionException { + // Arrange + DistributedTransaction transaction = mock(DistributedTransaction.class); + JdbcTransactionManager spied = spy(manager); + doReturn(transaction).when(spied).begin(); + + Scan scan = + Scan.newBuilder() + .namespace(NAMESPACE) + .table(TABLE) + .partitionKey(Key.ofText("p1", "val")) + .build(); + + TransactionCrudOperable.Scanner scanner = mock(TransactionCrudOperable.Scanner.class); + when(transaction.getScanner(scan)).thenReturn(scanner); + + when(scanner.all()).thenThrow(CrudException.class); + + // Act Assert + TransactionManagerCrudOperable.Scanner actual = spied.getScanner(scan); + assertThatThrownBy(actual::all).isInstanceOf(CrudException.class); + + verify(spied).begin(); + verify(scanner).close(); + verify(transaction).rollback(); + } + + @Test + public void + getScannerAndScannerClose_CrudExceptionThrownByScannerClose_ShouldRollbackTransactionAndThrowCrudException() + throws TransactionException { + // Arrange + DistributedTransaction transaction = mock(DistributedTransaction.class); + JdbcTransactionManager spied = spy(manager); + doReturn(transaction).when(spied).begin(); + + Scan scan = + Scan.newBuilder() + .namespace(NAMESPACE) + .table(TABLE) + .partitionKey(Key.ofText("p1", "val")) + .build(); + + TransactionCrudOperable.Scanner scanner = mock(TransactionCrudOperable.Scanner.class); + when(transaction.getScanner(scan)).thenReturn(scanner); + + doThrow(CrudException.class).when(scanner).close(); + + // Act Assert + TransactionManagerCrudOperable.Scanner actual = spied.getScanner(scan); + assertThatThrownBy(actual::close).isInstanceOf(CrudException.class); + + verify(spied).begin(); + verify(scanner).close(); + verify(transaction).rollback(); + } + + @Test + public void + getScannerAndScannerClose_CommitConflictExceptionThrownByTransactionCommit_ShouldRollbackTransactionAndThrowCrudConflictException() + throws TransactionException { + // Arrange + DistributedTransaction transaction = mock(DistributedTransaction.class); + + JdbcTransactionManager spied = spy(manager); + doReturn(transaction).when(spied).begin(); + doThrow(CommitConflictException.class).when(transaction).commit(); + + Scan scan = + Scan.newBuilder() + .namespace(NAMESPACE) + .table(TABLE) + .partitionKey(Key.ofText("p1", "val")) + .build(); + + TransactionCrudOperable.Scanner scanner = mock(TransactionCrudOperable.Scanner.class); + when(transaction.getScanner(scan)).thenReturn(scanner); + + // Act Assert + TransactionManagerCrudOperable.Scanner actual = spied.getScanner(scan); + assertThatThrownBy(actual::close).isInstanceOf(CrudConflictException.class); + + verify(spied).begin(); + verify(scanner).close(); + verify(transaction).rollback(); + } + + @Test + public void + getScannerAndScannerClose_UnknownTransactionStatusExceptionByTransactionCommit_ShouldThrowUnknownTransactionStatusException() + throws TransactionException { + // Arrange + DistributedTransaction transaction = mock(DistributedTransaction.class); + + JdbcTransactionManager spied = spy(manager); + doReturn(transaction).when(spied).begin(); + doThrow(UnknownTransactionStatusException.class).when(transaction).commit(); + + Scan scan = + Scan.newBuilder() + .namespace(NAMESPACE) + .table(TABLE) + .partitionKey(Key.ofText("p1", "val")) + .build(); + + TransactionCrudOperable.Scanner scanner = mock(TransactionCrudOperable.Scanner.class); + when(transaction.getScanner(scan)).thenReturn(scanner); + + // Act Assert + TransactionManagerCrudOperable.Scanner actual = spied.getScanner(scan); + assertThatThrownBy(actual::close).isInstanceOf(UnknownTransactionStatusException.class); + + verify(spied).begin(); + verify(scanner).close(); + } + + @Test + public void + getScannerAndScannerClose_CommitExceptionThrownByTransactionCommit_ShouldRollbackTransactionAndThrowCrudException() + throws TransactionException { + // Arrange + DistributedTransaction transaction = mock(DistributedTransaction.class); + + JdbcTransactionManager spied = spy(manager); + doReturn(transaction).when(spied).begin(); + doThrow(CommitException.class).when(transaction).commit(); + + Scan scan = + Scan.newBuilder() + .namespace(NAMESPACE) + .table(TABLE) + .partitionKey(Key.ofText("p1", "val")) + .build(); + + TransactionCrudOperable.Scanner scanner = mock(TransactionCrudOperable.Scanner.class); + when(transaction.getScanner(scan)).thenReturn(scanner); + + // Act Assert + TransactionManagerCrudOperable.Scanner actual = spied.getScanner(scan); + assertThatThrownBy(actual::close).isInstanceOf(CrudException.class); + + verify(spied).begin(); + verify(scanner).close(); + verify(transaction).rollback(); + } + @Test public void whenPutOperationsExecutedAndJdbcServiceThrowsSQLException_shouldThrowCrudException() throws Exception { diff --git a/core/src/test/java/com/scalar/db/transaction/jdbc/JdbcTransactionTest.java b/core/src/test/java/com/scalar/db/transaction/jdbc/JdbcTransactionTest.java index e09b04f8e4..2423457186 100644 --- a/core/src/test/java/com/scalar/db/transaction/jdbc/JdbcTransactionTest.java +++ b/core/src/test/java/com/scalar/db/transaction/jdbc/JdbcTransactionTest.java @@ -1,8 +1,10 @@ package com.scalar.db.transaction.jdbc; +import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatCode; import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -12,6 +14,10 @@ import com.scalar.db.api.Insert; import com.scalar.db.api.MutationCondition; import com.scalar.db.api.Put; +import com.scalar.db.api.Result; +import com.scalar.db.api.Scan; +import com.scalar.db.api.Scanner; +import com.scalar.db.api.TransactionCrudOperable; import com.scalar.db.api.Update; import com.scalar.db.api.Upsert; import com.scalar.db.exception.storage.ExecutionException; @@ -21,8 +27,12 @@ import com.scalar.db.io.Key; import com.scalar.db.storage.jdbc.JdbcService; import com.scalar.db.storage.jdbc.RdbEngineStrategy; +import java.io.IOException; import java.sql.Connection; import java.sql.SQLException; +import java.util.Arrays; +import java.util.Iterator; +import java.util.Optional; import java.util.stream.Stream; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -70,6 +80,223 @@ public void setUp() throws Exception { transaction = new JdbcTransaction(ANY_TX_ID, jdbcService, connection, rdbEngineStrategy); } + @Test + public void getScannerAndScannerOne_ShouldReturnScannerAndShouldReturnProperResult() + throws SQLException, ExecutionException, CrudException, IOException { + // Arrange + Scan scan = + Scan.newBuilder() + .namespace(ANY_NAMESPACE) + .table(ANY_TABLE_NAME) + .partitionKey(Key.ofText("p1", "val")) + .build(); + + Result result1 = mock(Result.class); + Result result2 = mock(Result.class); + Result result3 = mock(Result.class); + + Scanner scanner = mock(Scanner.class); + when(scanner.one()) + .thenReturn(Optional.of(result1)) + .thenReturn(Optional.of(result2)) + .thenReturn(Optional.of(result3)) + .thenReturn(Optional.empty()); + + when(jdbcService.getScanner(scan, connection, false)).thenReturn(scanner); + + // Act Assert + TransactionCrudOperable.Scanner actual = transaction.getScanner(scan); + assertThat(actual.one()).hasValue(result1); + assertThat(actual.one()).hasValue(result2); + assertThat(actual.one()).hasValue(result3); + assertThat(actual.one()).isEmpty(); + actual.close(); + + verify(jdbcService).getScanner(scan, connection, false); + verify(scanner).close(); + } + + @Test + public void getScannerAndScannerAll_ShouldReturnScannerAndShouldReturnProperResults() + throws SQLException, ExecutionException, CrudException, IOException { + // Arrange + Scan scan = + Scan.newBuilder() + .namespace(ANY_NAMESPACE) + .table(ANY_TABLE_NAME) + .partitionKey(Key.ofText("p1", "val")) + .build(); + + Result result1 = mock(Result.class); + Result result2 = mock(Result.class); + Result result3 = mock(Result.class); + + Scanner scanner = mock(Scanner.class); + when(scanner.all()).thenReturn(Arrays.asList(result1, result2, result3)); + + when(jdbcService.getScanner(scan, connection, false)).thenReturn(scanner); + + // Act Assert + TransactionCrudOperable.Scanner actual = transaction.getScanner(scan); + assertThat(actual.all()).containsExactly(result1, result2, result3); + actual.close(); + + verify(jdbcService).getScanner(scan, connection, false); + verify(scanner).close(); + } + + @Test + public void getScannerAndScannerIterator_ShouldReturnScannerAndShouldReturnProperResults() + throws SQLException, ExecutionException, CrudException, IOException { + // Arrange + Scan scan = + Scan.newBuilder() + .namespace(ANY_NAMESPACE) + .table(ANY_TABLE_NAME) + .partitionKey(Key.ofText("p1", "val")) + .build(); + + Result result1 = mock(Result.class); + Result result2 = mock(Result.class); + Result result3 = mock(Result.class); + + Scanner scanner = mock(Scanner.class); + when(scanner.one()) + .thenReturn(Optional.of(result1)) + .thenReturn(Optional.of(result2)) + .thenReturn(Optional.of(result3)) + .thenReturn(Optional.empty()); + + when(jdbcService.getScanner(scan, connection, false)).thenReturn(scanner); + + // Act Assert + TransactionCrudOperable.Scanner actual = transaction.getScanner(scan); + + Iterator iterator = actual.iterator(); + assertThat(iterator.hasNext()).isTrue(); + assertThat(iterator.next()).isEqualTo(result1); + assertThat(iterator.hasNext()).isTrue(); + assertThat(iterator.next()).isEqualTo(result2); + assertThat(iterator.hasNext()).isTrue(); + assertThat(iterator.next()).isEqualTo(result3); + assertThat(iterator.hasNext()).isFalse(); + actual.close(); + + verify(jdbcService).getScanner(scan, connection, false); + verify(scanner).close(); + } + + @Test + public void getScanner_WhenSQLExceptionThrownByJdbcService_ShouldThrowCrudException() + throws SQLException, ExecutionException { + // Arrange + Scan scan = + Scan.newBuilder() + .namespace(ANY_NAMESPACE) + .table(ANY_TABLE_NAME) + .partitionKey(Key.ofText("p1", "val")) + .build(); + + when(jdbcService.getScanner(scan, connection, false)).thenThrow(SQLException.class); + + // Act Assert + assertThatThrownBy(() -> transaction.getScanner(scan)).isInstanceOf(CrudException.class); + } + + @Test + public void getScanner_WhenExecutionExceptionThrownByJdbcService_ShouldThrowCrudException() + throws SQLException, ExecutionException { + // Arrange + Scan scan = + Scan.newBuilder() + .namespace(ANY_NAMESPACE) + .table(ANY_TABLE_NAME) + .partitionKey(Key.ofText("p1", "val")) + .build(); + + ExecutionException executionException = mock(ExecutionException.class); + when(executionException.getMessage()).thenReturn("error"); + when(jdbcService.getScanner(scan, connection, false)).thenThrow(executionException); + + // Act Assert + assertThatThrownBy(() -> transaction.getScanner(scan)).isInstanceOf(CrudException.class); + } + + @Test + public void + getScannerAndScannerOne_WhenExecutionExceptionThrownByScannerOne_ShouldThrowCrudException() + throws SQLException, ExecutionException, CrudException { + // Arrange + Scan scan = + Scan.newBuilder() + .namespace(ANY_NAMESPACE) + .table(ANY_TABLE_NAME) + .partitionKey(Key.ofText("p1", "val")) + .build(); + + Scanner scanner = mock(Scanner.class); + + ExecutionException executionException = mock(ExecutionException.class); + when(executionException.getMessage()).thenReturn("error"); + when(scanner.one()).thenThrow(executionException); + + when(jdbcService.getScanner(scan, connection, false)).thenReturn(scanner); + + // Act Assert + TransactionCrudOperable.Scanner actual = transaction.getScanner(scan); + assertThatThrownBy(actual::one).isInstanceOf(CrudException.class); + } + + @Test + public void + getScannerAndScannerAll_WhenExecutionExceptionThrownByScannerAll_ShouldThrowCrudException() + throws SQLException, ExecutionException, CrudException { + // Arrange + Scan scan = + Scan.newBuilder() + .namespace(ANY_NAMESPACE) + .table(ANY_TABLE_NAME) + .partitionKey(Key.ofText("p1", "val")) + .build(); + + Scanner scanner = mock(Scanner.class); + + ExecutionException executionException = mock(ExecutionException.class); + when(executionException.getMessage()).thenReturn("error"); + when(scanner.all()).thenThrow(executionException); + + when(jdbcService.getScanner(scan, connection, false)).thenReturn(scanner); + + // Act Assert + TransactionCrudOperable.Scanner actual = transaction.getScanner(scan); + assertThatThrownBy(actual::all).isInstanceOf(CrudException.class); + } + + @Test + public void + getScannerAndScannerClose_WhenIOExceptionThrownByScannerClose_ShouldThrowCrudException() + throws SQLException, ExecutionException, CrudException, IOException { + // Arrange + Scan scan = + Scan.newBuilder() + .namespace(ANY_NAMESPACE) + .table(ANY_TABLE_NAME) + .partitionKey(Key.ofText("p1", "val")) + .build(); + + Scanner scanner = mock(Scanner.class); + + IOException ioException = mock(IOException.class); + when(ioException.getMessage()).thenReturn("error"); + doThrow(ioException).when(scanner).close(); + + when(jdbcService.getScanner(scan, connection, false)).thenReturn(scanner); + + // Act Assert + TransactionCrudOperable.Scanner actual = transaction.getScanner(scan); + assertThatThrownBy(actual::close).isInstanceOf(CrudException.class); + } + @Test public void put_putDoesNotSucceed_shouldThrowUnsatisfiedConditionException() throws SQLException, ExecutionException { From 7b91326649d64218b5328d352cc78d0f66a6c140 Mon Sep 17 00:00:00 2001 From: Toshihiro Suzuki Date: Mon, 2 Jun 2025 17:44:04 +0900 Subject: [PATCH 04/19] Implement scanner API for Consensus Commit (#2711) --- .../com/scalar/db/common/error/CoreError.java | 12 + .../consensuscommit/ConsensusCommit.java | 48 ++- .../ConsensusCommitManager.java | 84 +++- .../consensuscommit/CrudHandler.java | 202 +++++++++ .../transaction/consensuscommit/Snapshot.java | 61 ++- .../TwoPhaseConsensusCommit.java | 49 ++- .../TwoPhaseConsensusCommitManager.java | 88 +++- .../ConsensusCommitManagerTest.java | 336 +++++++++++++++ .../consensuscommit/ConsensusCommitTest.java | 107 ++++- .../consensuscommit/CrudHandlerTest.java | 273 +++++++++--- .../consensuscommit/SnapshotTest.java | 34 +- .../TwoPhaseConsensusCommitManagerTest.java | 402 ++++++++++++++++++ .../TwoPhaseConsensusCommitTest.java | 118 ++++- .../ConsensusCommitIntegrationTestBase.java | 11 - ...nsusCommitSpecificIntegrationTestBase.java | 215 ++++++++++ ...aseConsensusCommitIntegrationTestBase.java | 12 - 16 files changed, 1950 insertions(+), 102 deletions(-) diff --git a/core/src/main/java/com/scalar/db/common/error/CoreError.java b/core/src/main/java/com/scalar/db/common/error/CoreError.java index 1e465bd550..7c5a5fd1d4 100644 --- a/core/src/main/java/com/scalar/db/common/error/CoreError.java +++ b/core/src/main/java/com/scalar/db/common/error/CoreError.java @@ -911,6 +911,18 @@ public enum CoreError implements ScalarDbError { Category.USER_ERROR, "0203", "Delimiter must not be null", "", ""), DATA_LOADER_CONFIG_FILE_PATH_BLANK( Category.USER_ERROR, "0204", "Config file path must not be blank", "", ""), + CONSENSUS_COMMIT_SCANNER_NOT_CLOSED( + Category.USER_ERROR, + "0205", + "Some scanners were not closed. All scanners must be closed before committing the transaction.", + "", + ""), + TWO_PHASE_CONSENSUS_COMMIT_SCANNER_NOT_CLOSED( + Category.USER_ERROR, + "0206", + "Some scanners were not closed. All scanners must be closed before preparing the transaction.", + "", + ""), // // Errors for the concurrency error category diff --git a/core/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommit.java b/core/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommit.java index cd4c2fb4a6..c2f6f12797 100644 --- a/core/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommit.java +++ b/core/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommit.java @@ -24,8 +24,10 @@ import com.scalar.db.exception.transaction.UnsatisfiedConditionException; import com.scalar.db.util.ScalarDbUtils; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import java.util.Iterator; import java.util.List; import java.util.Optional; +import javax.annotation.Nonnull; import javax.annotation.Nullable; import javax.annotation.concurrent.NotThreadSafe; import org.slf4j.Logger; @@ -98,7 +100,41 @@ public List scan(Scan scan) throws CrudException { @Override public Scanner getScanner(Scan scan) throws CrudException { - throw new UnsupportedOperationException("Implement later"); + scan = copyAndSetTargetToIfNot(scan); + Scanner scanner = crud.getScanner(scan); + + return new Scanner() { + @Override + public Optional one() throws CrudException { + try { + return scanner.one(); + } catch (UncommittedRecordException e) { + lazyRecovery(e); + throw e; + } + } + + @Override + public List all() throws CrudException { + try { + return scanner.all(); + } catch (UncommittedRecordException e) { + lazyRecovery(e); + throw e; + } + } + + @Override + public void close() throws CrudException { + scanner.close(); + } + + @Nonnull + @Override + public Iterator iterator() { + return scanner.iterator(); + } + }; } /** @deprecated As of release 3.13.0. Will be removed in release 5.0.0. */ @@ -213,6 +249,10 @@ public void mutate(List mutations) throws CrudException { @Override public void commit() throws CommitException, UnknownTransactionStatusException { + if (!crud.areAllScannersClosed()) { + throw new IllegalStateException(CoreError.CONSENSUS_COMMIT_SCANNER_NOT_CLOSED.buildMessage()); + } + // Execute implicit pre-read try { crud.readIfImplicitPreReadEnabled(); @@ -234,6 +274,12 @@ public void commit() throws CommitException, UnknownTransactionStatusException { @Override public void rollback() { + try { + crud.closeScanners(); + } catch (CrudException e) { + logger.warn("Failed to close the scanner", e); + } + if (groupCommitter != null) { groupCommitter.remove(crud.getSnapshot().getId()); } diff --git a/core/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitManager.java b/core/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitManager.java index 46a13035cb..9eff205532 100644 --- a/core/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitManager.java +++ b/core/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitManager.java @@ -16,10 +16,12 @@ import com.scalar.db.api.Put; import com.scalar.db.api.Result; import com.scalar.db.api.Scan; +import com.scalar.db.api.TransactionCrudOperable; import com.scalar.db.api.TransactionState; import com.scalar.db.api.Update; import com.scalar.db.api.Upsert; import com.scalar.db.common.AbstractDistributedTransactionManager; +import com.scalar.db.common.AbstractTransactionManagerCrudOperableScanner; import com.scalar.db.config.DatabaseConfig; import com.scalar.db.exception.transaction.CommitConflictException; import com.scalar.db.exception.transaction.CrudConflictException; @@ -34,6 +36,7 @@ import java.util.List; import java.util.Optional; import java.util.UUID; +import java.util.concurrent.atomic.AtomicBoolean; import javax.annotation.Nullable; import javax.annotation.concurrent.ThreadSafe; import org.slf4j.Logger; @@ -231,9 +234,86 @@ public List scan(Scan scan) throws CrudException, UnknownTransactionStat @Override public Scanner getScanner(Scan scan) throws CrudException { - throw new UnsupportedOperationException("Implement later"); + DistributedTransaction transaction = begin(); + + TransactionCrudOperable.Scanner scanner; + try { + scanner = transaction.getScanner(copyAndSetTargetToIfNot(scan)); + } catch (CrudException e) { + rollbackTransaction(transaction); + throw e; + } + + return new AbstractTransactionManagerCrudOperableScanner() { + + private final AtomicBoolean closed = new AtomicBoolean(); + + @Override + public Optional one() throws CrudException { + try { + return scanner.one(); + } catch (CrudException e) { + closed.set(true); + + try { + scanner.close(); + } catch (CrudException ex) { + e.addSuppressed(ex); + } + + rollbackTransaction(transaction); + throw e; + } + } + + @Override + public List all() throws CrudException { + try { + return scanner.all(); + } catch (CrudException e) { + closed.set(true); + + try { + scanner.close(); + } catch (CrudException ex) { + e.addSuppressed(ex); + } + + rollbackTransaction(transaction); + throw e; + } + } + + @Override + public void close() throws CrudException, UnknownTransactionStatusException { + if (closed.get()) { + return; + } + closed.set(true); + + try { + scanner.close(); + } catch (CrudException e) { + rollbackTransaction(transaction); + throw e; + } + + try { + transaction.commit(); + } catch (CommitConflictException e) { + rollbackTransaction(transaction); + throw new CrudConflictException(e.getMessage(), e, e.getTransactionId().orElse(null)); + } catch (UnknownTransactionStatusException e) { + throw e; + } catch (TransactionException e) { + rollbackTransaction(transaction); + throw new CrudException(e.getMessage(), e, e.getTransactionId().orElse(null)); + } + } + }; } + /** @deprecated As of release 3.13.0. Will be removed in release 5.0.0. */ @Deprecated @Override public void put(Put put) throws CrudException, UnknownTransactionStatusException { @@ -244,6 +324,7 @@ public void put(Put put) throws CrudException, UnknownTransactionStatusException }); } + /** @deprecated As of release 3.13.0. Will be removed in release 5.0.0. */ @Deprecated @Override public void put(List puts) throws CrudException, UnknownTransactionStatusException { @@ -290,6 +371,7 @@ public void delete(Delete delete) throws CrudException, UnknownTransactionStatus }); } + /** @deprecated As of release 3.13.0. Will be removed in release 5.0.0. */ @Deprecated @Override public void delete(List deletes) throws CrudException, UnknownTransactionStatusException { diff --git a/core/src/main/java/com/scalar/db/transaction/consensuscommit/CrudHandler.java b/core/src/main/java/com/scalar/db/transaction/consensuscommit/CrudHandler.java index c66595ca81..ebe5da6a13 100644 --- a/core/src/main/java/com/scalar/db/transaction/consensuscommit/CrudHandler.java +++ b/core/src/main/java/com/scalar/db/transaction/consensuscommit/CrudHandler.java @@ -16,6 +16,8 @@ import com.scalar.db.api.Scanner; import com.scalar.db.api.Selection; import com.scalar.db.api.TableMetadata; +import com.scalar.db.api.TransactionCrudOperable; +import com.scalar.db.common.AbstractTransactionCrudOperableScanner; import com.scalar.db.common.error.CoreError; import com.scalar.db.exception.storage.ExecutionException; import com.scalar.db.exception.transaction.CrudException; @@ -23,10 +25,13 @@ import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import java.io.IOException; import java.util.ArrayList; +import java.util.Iterator; import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; +import java.util.Map; import java.util.Optional; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Collectors; import javax.annotation.Nullable; import javax.annotation.concurrent.NotThreadSafe; @@ -42,6 +47,7 @@ public class CrudHandler { private final boolean isIncludeMetadataEnabled; private final MutationConditionsValidator mutationConditionsValidator; private final ParallelExecutor parallelExecutor; + private final List scanners = new ArrayList<>(); @SuppressFBWarnings("EI_EXPOSE_REP2") public CrudHandler( @@ -211,6 +217,37 @@ private void processScanResult(Snapshot.Key key, Scan scan, TransactionResult re snapshot.putIntoReadSet(key, Optional.of(result)); } + public TransactionCrudOperable.Scanner getScanner(Scan originalScan) throws CrudException { + List originalProjections = new ArrayList<>(originalScan.getProjections()); + Scan scan = (Scan) prepareStorageSelection(originalScan); + + ConsensusCommitScanner scanner; + + Optional> resultsInSnapshot = + snapshot.getResults(scan); + if (resultsInSnapshot.isPresent()) { + scanner = + new ConsensusCommitSnapshotScanner(scan, originalProjections, resultsInSnapshot.get()); + } else { + scanner = new ConsensusCommitStorageScanner(scan, originalProjections); + } + + scanners.add(scanner); + return scanner; + } + + public boolean areAllScannersClosed() { + return scanners.stream().allMatch(ConsensusCommitScanner::isClosed); + } + + public void closeScanners() throws CrudException { + for (ConsensusCommitScanner scanner : scanners) { + if (!scanner.isClosed()) { + scanner.close(); + } + } + } + public void put(Put put) throws CrudException { Snapshot.Key key = new Snapshot.Key(put); @@ -360,4 +397,169 @@ private TableMetadata getTableMetadata(Operation operation) throws CrudException public Snapshot getSnapshot() { return snapshot; } + + private interface ConsensusCommitScanner extends TransactionCrudOperable.Scanner { + boolean isClosed(); + } + + @NotThreadSafe + private class ConsensusCommitStorageScanner extends AbstractTransactionCrudOperableScanner + implements ConsensusCommitScanner { + + private final Scan scan; + private final List originalProjections; + private final Scanner scanner; + + private final LinkedHashMap results = new LinkedHashMap<>(); + private final AtomicBoolean fullyScanned = new AtomicBoolean(); + private final AtomicBoolean closed = new AtomicBoolean(); + + public ConsensusCommitStorageScanner(Scan scan, List originalProjections) + throws CrudException { + this.scan = scan; + this.originalProjections = originalProjections; + scanner = scanFromStorage(scan); + } + + @Override + public Optional one() throws CrudException { + try { + Optional r = scanner.one(); + + if (!r.isPresent()) { + fullyScanned.set(true); + return Optional.empty(); + } + + Snapshot.Key key = new Snapshot.Key(scan, r.get()); + TransactionResult result = new TransactionResult(r.get()); + processScanResult(key, scan, result); + results.put(key, result); + + TableMetadata metadata = getTableMetadata(scan); + return Optional.of( + new FilteredResult(result, originalProjections, metadata, isIncludeMetadataEnabled)); + } catch (ExecutionException e) { + closeScanner(); + throw new CrudException( + CoreError.CONSENSUS_COMMIT_SCANNING_RECORDS_FROM_STORAGE_FAILED.buildMessage(), + e, + snapshot.getId()); + } catch (CrudException e) { + closeScanner(); + throw e; + } + } + + @Override + public List all() throws CrudException { + List results = new ArrayList<>(); + + while (true) { + Optional result = one(); + if (!result.isPresent()) { + break; + } + results.add(result.get()); + } + + return results; + } + + @Override + public void close() { + if (closed.get()) { + return; + } + + closeScanner(); + + if (fullyScanned.get()) { + // If the scanner is fully scanned, we can treat it as a normal scan, and put the results + // into the scan set + snapshot.putIntoScanSet(scan, results); + } else { + // If the scanner is not fully scanned, put the results into the scanner set + snapshot.putIntoScannerSet(scan, results); + } + + snapshot.verifyNoOverlap(scan, results); + } + + @Override + public boolean isClosed() { + return closed.get(); + } + + private void closeScanner() { + closed.set(true); + try { + scanner.close(); + } catch (IOException e) { + logger.warn("Failed to close the scanner", e); + } + } + } + + @NotThreadSafe + private class ConsensusCommitSnapshotScanner extends AbstractTransactionCrudOperableScanner + implements ConsensusCommitScanner { + + private final Scan scan; + private final List originalProjections; + private final Iterator> resultsIterator; + + private final LinkedHashMap results = new LinkedHashMap<>(); + private boolean closed; + + public ConsensusCommitSnapshotScanner( + Scan scan, + List originalProjections, + LinkedHashMap resultsInSnapshot) { + this.scan = scan; + this.originalProjections = originalProjections; + resultsIterator = resultsInSnapshot.entrySet().iterator(); + } + + @Override + public Optional one() throws CrudException { + if (!resultsIterator.hasNext()) { + return Optional.empty(); + } + + Map.Entry entry = resultsIterator.next(); + results.put(entry.getKey(), entry.getValue()); + + TableMetadata metadata = getTableMetadata(scan); + return Optional.of( + new FilteredResult( + entry.getValue(), originalProjections, metadata, isIncludeMetadataEnabled)); + } + + @Override + public List all() throws CrudException { + List results = new ArrayList<>(); + + while (true) { + Optional result = one(); + if (!result.isPresent()) { + break; + } + results.add(result.get()); + } + + return results; + } + + @Override + public void close() { + closed = true; + snapshot.verifyNoOverlap(scan, results); + } + + @Override + public boolean isClosed() { + return closed; + } + } } diff --git a/core/src/main/java/com/scalar/db/transaction/consensuscommit/Snapshot.java b/core/src/main/java/com/scalar/db/transaction/consensuscommit/Snapshot.java index cf535281ab..24b16e59c7 100644 --- a/core/src/main/java/com/scalar/db/transaction/consensuscommit/Snapshot.java +++ b/core/src/main/java/com/scalar/db/transaction/consensuscommit/Snapshot.java @@ -64,6 +64,9 @@ public class Snapshot { private final Map writeSet; private final Map deleteSet; + // The scanner set used to store information about scanners that are not fully scanned + private final List scannerSet; + public Snapshot( String id, Isolation isolation, @@ -78,6 +81,7 @@ public Snapshot( scanSet = new HashMap<>(); writeSet = new HashMap<>(); deleteSet = new HashMap<>(); + scannerSet = new ArrayList<>(); } @VisibleForTesting @@ -90,7 +94,8 @@ public Snapshot( ConcurrentMap> getSet, Map> scanSet, Map writeSet, - Map deleteSet) { + Map deleteSet, + List scannerSet) { this.id = id; this.isolation = isolation; this.tableMetadataManager = tableMetadataManager; @@ -100,6 +105,7 @@ public Snapshot( this.scanSet = scanSet; this.writeSet = writeSet; this.deleteSet = deleteSet; + this.scannerSet = scannerSet; } @Nonnull @@ -173,6 +179,10 @@ public void putIntoDeleteSet(Key key, Delete delete) { deleteSet.put(key, delete); } + public void putIntoScannerSet(Scan scan, LinkedHashMap results) { + scannerSet.add(new ScannerInfo(scan, results)); + } + public List getPutsInWriteSet() { return new ArrayList<>(writeSet.values()); } @@ -485,7 +495,12 @@ void toSerializable(DistributedStorage storage) // Scan set is re-validated to check if there is no anti-dependency for (Map.Entry> entry : scanSet.entrySet()) { - tasks.add(() -> validateScanResults(storage, entry.getKey(), entry.getValue())); + tasks.add(() -> validateScanResults(storage, entry.getKey(), entry.getValue(), false)); + } + + // Scanner set is re-validated to check if there is no anti-dependency + for (ScannerInfo scannerInfo : scannerSet) { + tasks.add(() -> validateScanResults(storage, scannerInfo.scan, scannerInfo.results, true)); } // Get set is re-validated to check if there is no anti-dependency @@ -527,11 +542,16 @@ void toSerializable(DistributedStorage storage) * @param storage a distributed storage * @param scan the scan to be validated * @param results the results of the scan + * @param notFullyScannedScanner if this is a validation for a scanner that has not been fully + * scanned * @throws ExecutionException if a storage operation fails * @throws ValidationConflictException if the scan results are changed by another transaction */ private void validateScanResults( - DistributedStorage storage, Scan scan, LinkedHashMap results) + DistributedStorage storage, + Scan scan, + LinkedHashMap results, + boolean notFullyScannedScanner) throws ExecutionException, ValidationConflictException { Scanner scanner = null; try { @@ -604,6 +624,11 @@ private void validateScanResults( return; } + if (notFullyScannedScanner) { + // If the scanner is not fully scanned, no further checks are needed + return; + } + // Check if there are any remaining records in the latest scan results while (latestResult.isPresent()) { TransactionResult latestTxResult = new TransactionResult(latestResult.get()); @@ -654,7 +679,7 @@ private void validateGetWithIndexResult( originalResult.ifPresent(r -> results.put(new Snapshot.Key(scanWithIndex, r), r)); // Validate the result to check if there is no anti-dependency - validateScanResults(storage, scanWithIndex, results); + validateScanResults(storage, scanWithIndex, results, false); } private void validateGetResult( @@ -842,4 +867,32 @@ public int hashCode() { return Objects.hash(transactionId, readSetMap, writeSet, deleteSet); } } + + @VisibleForTesting + static class ScannerInfo { + public final Scan scan; + public final LinkedHashMap results; + + public ScannerInfo(Scan scan, LinkedHashMap results) { + this.scan = scan; + this.results = results; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof ScannerInfo)) { + return false; + } + ScannerInfo that = (ScannerInfo) o; + return Objects.equals(scan, that.scan) && Objects.equals(results, that.results); + } + + @Override + public int hashCode() { + return Objects.hash(scan, results); + } + } } diff --git a/core/src/main/java/com/scalar/db/transaction/consensuscommit/TwoPhaseConsensusCommit.java b/core/src/main/java/com/scalar/db/transaction/consensuscommit/TwoPhaseConsensusCommit.java index d8edb63291..dd72f03b4d 100644 --- a/core/src/main/java/com/scalar/db/transaction/consensuscommit/TwoPhaseConsensusCommit.java +++ b/core/src/main/java/com/scalar/db/transaction/consensuscommit/TwoPhaseConsensusCommit.java @@ -27,8 +27,10 @@ import com.scalar.db.exception.transaction.ValidationException; import com.scalar.db.util.ScalarDbUtils; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import java.util.Iterator; import java.util.List; import java.util.Optional; +import javax.annotation.Nonnull; import javax.annotation.concurrent.NotThreadSafe; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -88,7 +90,41 @@ public List scan(Scan scan) throws CrudException { @Override public Scanner getScanner(Scan scan) throws CrudException { - throw new UnsupportedOperationException("Implement later"); + scan = copyAndSetTargetToIfNot(scan); + Scanner scanner = crud.getScanner(scan); + + return new Scanner() { + @Override + public Optional one() throws CrudException { + try { + return scanner.one(); + } catch (UncommittedRecordException e) { + lazyRecovery(e); + throw e; + } + } + + @Override + public List all() throws CrudException { + try { + return scanner.all(); + } catch (UncommittedRecordException e) { + lazyRecovery(e); + throw e; + } + } + + @Override + public void close() throws CrudException { + scanner.close(); + } + + @Nonnull + @Override + public Iterator iterator() { + return scanner.iterator(); + } + }; } /** @deprecated As of release 3.13.0. Will be removed in release 5.0.0. */ @@ -211,6 +247,11 @@ public void mutate(List mutations) throws CrudException { @Override public void prepare() throws PreparationException { + if (!crud.areAllScannersClosed()) { + throw new IllegalStateException( + CoreError.TWO_PHASE_CONSENSUS_COMMIT_SCANNER_NOT_CLOSED.buildMessage()); + } + // Execute implicit pre-read try { crud.readIfImplicitPreReadEnabled(); @@ -261,6 +302,12 @@ public void commit() throws CommitConflictException, UnknownTransactionStatusExc @Override public void rollback() throws RollbackException { + try { + crud.closeScanners(); + } catch (CrudException e) { + logger.warn("Failed to close the scanner", e); + } + if (!needRollback) { return; } diff --git a/core/src/main/java/com/scalar/db/transaction/consensuscommit/TwoPhaseConsensusCommitManager.java b/core/src/main/java/com/scalar/db/transaction/consensuscommit/TwoPhaseConsensusCommitManager.java index 3c9a47aa60..1097f0b62f 100644 --- a/core/src/main/java/com/scalar/db/transaction/consensuscommit/TwoPhaseConsensusCommitManager.java +++ b/core/src/main/java/com/scalar/db/transaction/consensuscommit/TwoPhaseConsensusCommitManager.java @@ -13,10 +13,12 @@ import com.scalar.db.api.Put; import com.scalar.db.api.Result; import com.scalar.db.api.Scan; +import com.scalar.db.api.TransactionCrudOperable; import com.scalar.db.api.TransactionState; import com.scalar.db.api.TwoPhaseCommitTransaction; import com.scalar.db.api.Update; import com.scalar.db.api.Upsert; +import com.scalar.db.common.AbstractTransactionManagerCrudOperableScanner; import com.scalar.db.common.AbstractTwoPhaseCommitTransactionManager; import com.scalar.db.common.error.CoreError; import com.scalar.db.config.DatabaseConfig; @@ -35,6 +37,7 @@ import java.util.List; import java.util.Optional; import java.util.UUID; +import java.util.concurrent.atomic.AtomicBoolean; import javax.annotation.concurrent.ThreadSafe; import javax.inject.Inject; import org.slf4j.Logger; @@ -188,9 +191,90 @@ public List scan(Scan scan) throws CrudException, UnknownTransactionStat @Override public Scanner getScanner(Scan scan) throws CrudException { - throw new UnsupportedOperationException("Implement later"); + TwoPhaseCommitTransaction transaction = begin(); + + TransactionCrudOperable.Scanner scanner; + try { + scanner = transaction.getScanner(copyAndSetTargetToIfNot(scan)); + } catch (CrudException e) { + rollbackTransaction(transaction); + throw e; + } + + return new AbstractTransactionManagerCrudOperableScanner() { + + private final AtomicBoolean closed = new AtomicBoolean(); + + @Override + public Optional one() throws CrudException { + try { + return scanner.one(); + } catch (CrudException e) { + closed.set(true); + + try { + scanner.close(); + } catch (CrudException ex) { + e.addSuppressed(ex); + } + + rollbackTransaction(transaction); + throw e; + } + } + + @Override + public List all() throws CrudException { + try { + return scanner.all(); + } catch (CrudException e) { + closed.set(true); + + try { + scanner.close(); + } catch (CrudException ex) { + e.addSuppressed(ex); + } + + rollbackTransaction(transaction); + throw e; + } + } + + @Override + public void close() throws CrudException, UnknownTransactionStatusException { + if (closed.get()) { + return; + } + closed.set(true); + + try { + scanner.close(); + } catch (CrudException e) { + rollbackTransaction(transaction); + throw e; + } + + try { + transaction.prepare(); + transaction.validate(); + transaction.commit(); + } catch (PreparationConflictException + | ValidationConflictException + | CommitConflictException e) { + rollbackTransaction(transaction); + throw new CrudConflictException(e.getMessage(), e, e.getTransactionId().orElse(null)); + } catch (UnknownTransactionStatusException e) { + throw e; + } catch (TransactionException e) { + rollbackTransaction(transaction); + throw new CrudException(e.getMessage(), e, e.getTransactionId().orElse(null)); + } + } + }; } + /** @deprecated As of release 3.13.0. Will be removed in release 5.0.0. */ @Deprecated @Override public void put(Put put) throws CrudException, UnknownTransactionStatusException { @@ -201,6 +285,7 @@ public void put(Put put) throws CrudException, UnknownTransactionStatusException }); } + /** @deprecated As of release 3.13.0. Will be removed in release 5.0.0. */ @Deprecated @Override public void put(List puts) throws CrudException, UnknownTransactionStatusException { @@ -247,6 +332,7 @@ public void delete(Delete delete) throws CrudException, UnknownTransactionStatus }); } + /** @deprecated As of release 3.13.0. Will be removed in release 5.0.0. */ @Deprecated @Override public void delete(List deletes) throws CrudException, UnknownTransactionStatusException { diff --git a/core/src/test/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitManagerTest.java b/core/src/test/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitManagerTest.java index 4b363a1b35..0260c70309 100644 --- a/core/src/test/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitManagerTest.java +++ b/core/src/test/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitManagerTest.java @@ -22,6 +22,8 @@ import com.scalar.db.api.Put; import com.scalar.db.api.Result; import com.scalar.db.api.Scan; +import com.scalar.db.api.TransactionCrudOperable; +import com.scalar.db.api.TransactionManagerCrudOperable; import com.scalar.db.api.TransactionState; import com.scalar.db.api.Update; import com.scalar.db.api.Upsert; @@ -38,6 +40,8 @@ import com.scalar.db.transaction.consensuscommit.Coordinator.State; import com.scalar.db.transaction.consensuscommit.CoordinatorGroupCommitter.CoordinatorGroupCommitKeyManipulator; import java.util.Arrays; +import java.util.Collections; +import java.util.Iterator; import java.util.List; import java.util.Optional; import org.junit.jupiter.api.BeforeEach; @@ -666,6 +670,338 @@ public void scan_ShouldScan() throws TransactionException { assertThat(actual).isEqualTo(results); } + @Test + public void getScannerAndScannerOne_ShouldReturnScannerAndReturnProperResult() throws Exception { + // Arrange + DistributedTransaction transaction = mock(DistributedTransaction.class); + + ConsensusCommitManager spied = spy(manager); + doReturn(transaction).when(spied).begin(); + + Scan scan = + Scan.newBuilder() + .namespace("ns") + .table("tbl") + .partitionKey(Key.ofText("p1", "val")) + .build(); + + TransactionCrudOperable.Scanner scanner = mock(TransactionCrudOperable.Scanner.class); + when(transaction.getScanner(scan)).thenReturn(scanner); + + Result result1 = mock(Result.class); + Result result2 = mock(Result.class); + Result result3 = mock(Result.class); + + when(scanner.one()) + .thenReturn(Optional.of(result1)) + .thenReturn(Optional.of(result2)) + .thenReturn(Optional.of(result3)) + .thenReturn(Optional.empty()); + + // Act Assert + TransactionManagerCrudOperable.Scanner actual = spied.getScanner(scan); + assertThat(actual.one()).hasValue(result1); + assertThat(actual.one()).hasValue(result2); + assertThat(actual.one()).hasValue(result3); + assertThat(actual.one()).isEmpty(); + actual.close(); + + verify(spied).begin(); + verify(transaction).commit(); + verify(scanner).close(); + } + + @Test + public void getScannerAndScannerAll_ShouldReturnScannerAndReturnProperResults() throws Exception { + // Arrange + DistributedTransaction transaction = mock(DistributedTransaction.class); + + ConsensusCommitManager spied = spy(manager); + doReturn(transaction).when(spied).begin(); + + Scan scan = + Scan.newBuilder() + .namespace("ns") + .table("tbl") + .partitionKey(Key.ofText("p1", "val")) + .build(); + + TransactionCrudOperable.Scanner scanner = mock(TransactionCrudOperable.Scanner.class); + when(transaction.getScanner(scan)).thenReturn(scanner); + + Result result1 = mock(Result.class); + Result result2 = mock(Result.class); + Result result3 = mock(Result.class); + + when(scanner.all()) + .thenReturn(Arrays.asList(result1, result2, result3)) + .thenReturn(Collections.emptyList()); + + // Act Assert + TransactionManagerCrudOperable.Scanner actual = spied.getScanner(scan); + List results = actual.all(); + assertThat(results).containsExactly(result1, result2, result3); + assertThat(actual.all()).isEmpty(); + actual.close(); + + verify(spied).begin(); + verify(transaction).commit(); + verify(scanner).close(); + } + + @Test + public void getScannerAndScannerIterator_ShouldReturnScannerAndReturnProperResults() + throws Exception { + // Arrange + DistributedTransaction transaction = mock(DistributedTransaction.class); + + ConsensusCommitManager spied = spy(manager); + doReturn(transaction).when(spied).begin(); + + Scan scan = + Scan.newBuilder() + .namespace("ns") + .table("tbl") + .partitionKey(Key.ofText("p1", "val")) + .build(); + + TransactionCrudOperable.Scanner scanner = mock(TransactionCrudOperable.Scanner.class); + when(transaction.getScanner(scan)).thenReturn(scanner); + + Result result1 = mock(Result.class); + Result result2 = mock(Result.class); + Result result3 = mock(Result.class); + + when(scanner.one()) + .thenReturn(Optional.of(result1)) + .thenReturn(Optional.of(result2)) + .thenReturn(Optional.of(result3)) + .thenReturn(Optional.empty()); + + // Act Assert + TransactionManagerCrudOperable.Scanner actual = spied.getScanner(scan); + + Iterator iterator = actual.iterator(); + assertThat(iterator.hasNext()).isTrue(); + assertThat(iterator.next()).isEqualTo(result1); + assertThat(iterator.hasNext()).isTrue(); + assertThat(iterator.next()).isEqualTo(result2); + assertThat(iterator.hasNext()).isTrue(); + assertThat(iterator.next()).isEqualTo(result3); + assertThat(iterator.hasNext()).isFalse(); + actual.close(); + + verify(spied).begin(); + verify(transaction).commit(); + verify(scanner).close(); + } + + @Test + public void + getScanner_CrudExceptionThrownByTransactionGetScanner_ShouldRollbackTransactionAndThrowCrudException() + throws TransactionException { + // Arrange + DistributedTransaction transaction = mock(DistributedTransaction.class); + + ConsensusCommitManager spied = spy(manager); + doReturn(transaction).when(spied).begin(); + + Scan scan = + Scan.newBuilder() + .namespace("ns") + .table("tbl") + .partitionKey(Key.ofText("p1", "val")) + .build(); + + when(transaction.getScanner(scan)).thenThrow(CrudException.class); + + // Act Assert + assertThatThrownBy(() -> spied.getScanner(scan)).isInstanceOf(CrudException.class); + + verify(spied).begin(); + verify(transaction).rollback(); + } + + @Test + public void + getScannerAndScannerOne_CrudExceptionThrownByScannerOne_ShouldRollbackTransactionAndThrowCrudException() + throws TransactionException { + // Arrange + DistributedTransaction transaction = mock(DistributedTransaction.class); + + ConsensusCommitManager spied = spy(manager); + doReturn(transaction).when(spied).begin(); + + Scan scan = + Scan.newBuilder() + .namespace("ns") + .table("tbl") + .partitionKey(Key.ofText("p1", "val")) + .build(); + + TransactionCrudOperable.Scanner scanner = mock(TransactionCrudOperable.Scanner.class); + when(transaction.getScanner(scan)).thenReturn(scanner); + + when(scanner.one()).thenThrow(CrudException.class); + + // Act Assert + TransactionManagerCrudOperable.Scanner actual = spied.getScanner(scan); + assertThatThrownBy(actual::one).isInstanceOf(CrudException.class); + + verify(spied).begin(); + verify(scanner).close(); + verify(transaction).rollback(); + } + + @Test + public void + getScannerAndScannerAll_CrudExceptionThrownByScannerAll_ShouldRollbackTransactionAndThrowCrudException() + throws TransactionException { + // Arrange + DistributedTransaction transaction = mock(DistributedTransaction.class); + ConsensusCommitManager spied = spy(manager); + doReturn(transaction).when(spied).begin(); + + Scan scan = + Scan.newBuilder() + .namespace("ns") + .table("tbl") + .partitionKey(Key.ofText("p1", "val")) + .build(); + + TransactionCrudOperable.Scanner scanner = mock(TransactionCrudOperable.Scanner.class); + when(transaction.getScanner(scan)).thenReturn(scanner); + + when(scanner.all()).thenThrow(CrudException.class); + + // Act Assert + TransactionManagerCrudOperable.Scanner actual = spied.getScanner(scan); + assertThatThrownBy(actual::all).isInstanceOf(CrudException.class); + + verify(spied).begin(); + verify(scanner).close(); + verify(transaction).rollback(); + } + + @Test + public void + getScannerAndScannerClose_CrudExceptionThrownByScannerClose_ShouldRollbackTransactionAndThrowCrudException() + throws TransactionException { + // Arrange + DistributedTransaction transaction = mock(DistributedTransaction.class); + ConsensusCommitManager spied = spy(manager); + doReturn(transaction).when(spied).begin(); + + Scan scan = + Scan.newBuilder() + .namespace("ns") + .table("tbl") + .partitionKey(Key.ofText("p1", "val")) + .build(); + + TransactionCrudOperable.Scanner scanner = mock(TransactionCrudOperable.Scanner.class); + when(transaction.getScanner(scan)).thenReturn(scanner); + + doThrow(CrudException.class).when(scanner).close(); + + // Act Assert + TransactionManagerCrudOperable.Scanner actual = spied.getScanner(scan); + assertThatThrownBy(actual::close).isInstanceOf(CrudException.class); + + verify(spied).begin(); + verify(scanner).close(); + verify(transaction).rollback(); + } + + @Test + public void + getScannerAndScannerClose_CommitConflictExceptionThrownByTransactionCommit_ShouldRollbackTransactionAndThrowCrudConflictException() + throws TransactionException { + // Arrange + DistributedTransaction transaction = mock(DistributedTransaction.class); + + ConsensusCommitManager spied = spy(manager); + doReturn(transaction).when(spied).begin(); + doThrow(CommitConflictException.class).when(transaction).commit(); + + Scan scan = + Scan.newBuilder() + .namespace("ns") + .table("tbl") + .partitionKey(Key.ofText("p1", "val")) + .build(); + + TransactionCrudOperable.Scanner scanner = mock(TransactionCrudOperable.Scanner.class); + when(transaction.getScanner(scan)).thenReturn(scanner); + + // Act Assert + TransactionManagerCrudOperable.Scanner actual = spied.getScanner(scan); + assertThatThrownBy(actual::close).isInstanceOf(CrudConflictException.class); + + verify(spied).begin(); + verify(scanner).close(); + verify(transaction).rollback(); + } + + @Test + public void + getScannerAndScannerClose_UnknownTransactionStatusExceptionByTransactionCommit_ShouldThrowUnknownTransactionStatusException() + throws TransactionException { + // Arrange + DistributedTransaction transaction = mock(DistributedTransaction.class); + + ConsensusCommitManager spied = spy(manager); + doReturn(transaction).when(spied).begin(); + doThrow(UnknownTransactionStatusException.class).when(transaction).commit(); + + Scan scan = + Scan.newBuilder() + .namespace("ns") + .table("tbl") + .partitionKey(Key.ofText("p1", "val")) + .build(); + + TransactionCrudOperable.Scanner scanner = mock(TransactionCrudOperable.Scanner.class); + when(transaction.getScanner(scan)).thenReturn(scanner); + + // Act Assert + TransactionManagerCrudOperable.Scanner actual = spied.getScanner(scan); + assertThatThrownBy(actual::close).isInstanceOf(UnknownTransactionStatusException.class); + + verify(spied).begin(); + verify(scanner).close(); + } + + @Test + public void + getScannerAndScannerClose_CommitExceptionThrownByTransactionCommit_ShouldRollbackTransactionAndThrowCrudException() + throws TransactionException { + // Arrange + DistributedTransaction transaction = mock(DistributedTransaction.class); + + ConsensusCommitManager spied = spy(manager); + doReturn(transaction).when(spied).begin(); + doThrow(CommitException.class).when(transaction).commit(); + + Scan scan = + Scan.newBuilder() + .namespace("ns") + .table("tbl") + .partitionKey(Key.ofText("p1", "val")) + .build(); + + TransactionCrudOperable.Scanner scanner = mock(TransactionCrudOperable.Scanner.class); + when(transaction.getScanner(scan)).thenReturn(scanner); + + // Act Assert + TransactionManagerCrudOperable.Scanner actual = spied.getScanner(scan); + assertThatThrownBy(actual::close).isInstanceOf(CrudException.class); + + verify(spied).begin(); + verify(scanner).close(); + verify(transaction).rollback(); + } + @Test public void put_ShouldPut() throws TransactionException { // Arrange diff --git a/core/src/test/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitTest.java b/core/src/test/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitTest.java index 536d2ebfe5..f81a85b70b 100644 --- a/core/src/test/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitTest.java +++ b/core/src/test/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitTest.java @@ -21,6 +21,7 @@ import com.scalar.db.api.Put; import com.scalar.db.api.Result; import com.scalar.db.api.Scan; +import com.scalar.db.api.TransactionCrudOperable; import com.scalar.db.api.Update; import com.scalar.db.api.Upsert; import com.scalar.db.exception.storage.ExecutionException; @@ -68,6 +69,9 @@ public class ConsensusCommitTest { @BeforeEach public void setUp() throws Exception { MockitoAnnotations.openMocks(this).close(); + + // Arrange + when(crud.areAllScannersClosed()).thenReturn(true); } private Get prepareGet() { @@ -164,6 +168,93 @@ public void scan_ScanForUncommittedRecordGiven_ShouldRecoverRecord() throws Crud verify(recovery).recover(scan, result); } + @Test + public void getScannerAndScannerOne_ShouldCallCrudHandlerGetScannerAndScannerOne() + throws CrudException { + // Arrange + Scan scan = prepareScan(); + TransactionCrudOperable.Scanner scanner = mock(TransactionCrudOperable.Scanner.class); + Result result = mock(Result.class); + when(scanner.one()).thenReturn(Optional.of(result)); + when(crud.getScanner(scan)).thenReturn(scanner); + + // Act + TransactionCrudOperable.Scanner actualScanner = consensus.getScanner(scan); + Optional actualResult = actualScanner.one(); + + // Assert + assertThat(actualResult).hasValue(result); + verify(crud).getScanner(scan); + verify(scanner).one(); + } + + @Test + public void + getScannerAndScannerOne_UncommittedRecordExceptionThrownByScannerOne_ShouldRecoverRecord() + throws CrudException { + // Arrange + Scan scan = prepareScan(); + + UncommittedRecordException toThrow = mock(UncommittedRecordException.class); + TransactionResult result = mock(TransactionResult.class); + when(toThrow.getSelection()).thenReturn(scan); + when(toThrow.getResults()).thenReturn(Collections.singletonList(result)); + + TransactionCrudOperable.Scanner scanner = mock(TransactionCrudOperable.Scanner.class); + when(scanner.one()).thenThrow(toThrow); + when(crud.getScanner(scan)).thenReturn(scanner); + + // Act Assert + TransactionCrudOperable.Scanner actualScanner = consensus.getScanner(scan); + assertThatThrownBy(actualScanner::one).isInstanceOf(UncommittedRecordException.class); + + verify(recovery).recover(scan, result); + } + + @Test + public void getScannerAndScannerAll_ShouldCallCrudHandlerGetScannerAndScannerAll() + throws CrudException { + // Arrange + Scan scan = prepareScan(); + TransactionCrudOperable.Scanner scanner = mock(TransactionCrudOperable.Scanner.class); + Result result1 = mock(Result.class); + Result result2 = mock(Result.class); + when(scanner.all()).thenReturn(Arrays.asList(result1, result2)); + when(crud.getScanner(scan)).thenReturn(scanner); + + // Act + TransactionCrudOperable.Scanner actualScanner = consensus.getScanner(scan); + List actualResults = actualScanner.all(); + + // Assert + assertThat(actualResults).containsExactly(result1, result2); + verify(crud).getScanner(scan); + verify(scanner).all(); + } + + @Test + public void + getScannerAndScannerAll_UncommittedRecordExceptionThrownByScannerAll_ShouldRecoverRecord() + throws CrudException { + // Arrange + Scan scan = prepareScan(); + + UncommittedRecordException toThrow = mock(UncommittedRecordException.class); + TransactionResult result = mock(TransactionResult.class); + when(toThrow.getSelection()).thenReturn(scan); + when(toThrow.getResults()).thenReturn(Collections.singletonList(result)); + + TransactionCrudOperable.Scanner scanner = mock(TransactionCrudOperable.Scanner.class); + when(scanner.all()).thenThrow(toThrow); + when(crud.getScanner(scan)).thenReturn(scanner); + + // Act Assert + TransactionCrudOperable.Scanner actualScanner = consensus.getScanner(scan); + assertThatThrownBy(actualScanner::all).isInstanceOf(UncommittedRecordException.class); + + verify(recovery).recover(scan, result); + } + @Test public void put_PutGiven_ShouldCallCrudHandlerPut() throws ExecutionException, CrudException { // Arrange @@ -674,21 +765,30 @@ public void commit_ProcessedCrudGiven_ShouldCommitWithSnapshot() } @Test - public void rollback_WithoutGroupCommitter_ShouldDoNothing() - throws UnknownTransactionStatusException { + public void commit_ScannerNotClosed_ShouldThrowIllegalStateException() { + // Arrange + when(crud.areAllScannersClosed()).thenReturn(false); + + // Act Assert + assertThatThrownBy(() -> consensus.commit()).isInstanceOf(IllegalStateException.class); + } + + @Test + public void rollback_ShouldDoNothing() throws CrudException, UnknownTransactionStatusException { // Arrange // Act consensus.rollback(); // Assert + verify(crud).closeScanners(); verify(commit, never()).rollbackRecords(any(Snapshot.class)); verify(commit, never()).abortState(anyString()); } @Test public void rollback_WithGroupCommitter_ShouldRemoveTxFromGroupCommitter() - throws UnknownTransactionStatusException { + throws CrudException, UnknownTransactionStatusException { // Arrange String txId = "tx-id"; Snapshot snapshot = mock(Snapshot.class); @@ -702,6 +802,7 @@ public void rollback_WithGroupCommitter_ShouldRemoveTxFromGroupCommitter() consensusWithGroupCommit.rollback(); // Assert + verify(crud).closeScanners(); verify(groupCommitter).remove(txId); verify(commit, never()).rollbackRecords(any(Snapshot.class)); verify(commit, never()).abortState(anyString()); diff --git a/core/src/test/java/com/scalar/db/transaction/consensuscommit/CrudHandlerTest.java b/core/src/test/java/com/scalar/db/transaction/consensuscommit/CrudHandlerTest.java index 933d40b5d6..689ede3bb9 100644 --- a/core/src/test/java/com/scalar/db/transaction/consensuscommit/CrudHandlerTest.java +++ b/core/src/test/java/com/scalar/db/transaction/consensuscommit/CrudHandlerTest.java @@ -24,6 +24,7 @@ import com.scalar.db.api.ScanAll; import com.scalar.db.api.Scanner; import com.scalar.db.api.TableMetadata; +import com.scalar.db.api.TransactionCrudOperable; import com.scalar.db.api.TransactionState; import com.scalar.db.common.ResultImpl; import com.scalar.db.exception.storage.ExecutionException; @@ -33,6 +34,8 @@ import com.scalar.db.io.Key; import com.scalar.db.io.TextColumn; import com.scalar.db.util.ScalarDbUtils; +import java.io.IOException; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; @@ -44,8 +47,9 @@ import java.util.concurrent.ConcurrentMap; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; import org.mockito.ArgumentCaptor; -import org.mockito.ArgumentMatchers; import org.mockito.Mock; import org.mockito.MockitoAnnotations; @@ -231,8 +235,8 @@ public void get_GetExistsInSnapshot_ShouldReturnFromSnapshot() throws CrudExcept assertThat(exception.getResults().get(0)).isEqualTo(result); }); - verify(snapshot, never()).putIntoReadSet(any(), ArgumentMatchers.any()); - verify(snapshot, never()).putIntoGetSet(any(), ArgumentMatchers.any()); + verify(snapshot, never()).putIntoReadSet(any(), any()); + verify(snapshot, never()).putIntoGetSet(any(), any()); } @Test @@ -345,23 +349,29 @@ public void get_ForNonExistingTable_ShouldThrowIllegalArgumentException() assertThatThrownBy(() -> handler.get(get)).isInstanceOf(IllegalArgumentException.class); } - @Test - public void scan_ResultGivenFromStorage_ShouldUpdateSnapshotAndReturn() - throws ExecutionException, CrudException { + @ParameterizedTest + @EnumSource(ScanType.class) + void scanOrGetScanner_ResultGivenFromStorage_ShouldUpdateSnapshotAndReturn(ScanType scanType) + throws ExecutionException, CrudException, IOException { // Arrange Scan scan = prepareScan(); Scan scanForStorage = toScanForStorageFrom(scan); result = prepareResult(TransactionState.COMMITTED); Snapshot.Key key = new Snapshot.Key(scan, result); TransactionResult expected = new TransactionResult(result); - when(scanner.iterator()).thenReturn(Collections.singletonList(result).iterator()); + if (scanType == ScanType.SCAN) { + when(scanner.iterator()).thenReturn(Collections.singletonList(result).iterator()); + } else { + when(scanner.one()).thenReturn(Optional.of(result)).thenReturn(Optional.empty()); + } when(storage.scan(scanForStorage)).thenReturn(scanner); when(snapshot.getResult(any())).thenReturn(Optional.of(expected)); // Act - List results = handler.scan(scan); + List results = scanOrGetScanner(scan, scanType); // Assert + verify(scanner).close(); verify(snapshot).putIntoReadSet(key, Optional.of(expected)); verify(snapshot).putIntoScanSet(scan, Maps.newLinkedHashMap(ImmutableMap.of(key, expected))); verify(snapshot).verifyNoOverlap(scan, ImmutableMap.of(key, expected)); @@ -370,19 +380,24 @@ public void scan_ResultGivenFromStorage_ShouldUpdateSnapshotAndReturn() .isEqualTo(new FilteredResult(expected, Collections.emptyList(), TABLE_METADATA, false)); } - @Test - public void - scan_PreparedResultGivenFromStorage_ShouldNeverUpdateSnapshotThrowUncommittedRecordException() - throws ExecutionException { + @ParameterizedTest + @EnumSource(ScanType.class) + void + scanOrGetScanner_PreparedResultGivenFromStorage_ShouldNeverUpdateSnapshotThrowUncommittedRecordException( + ScanType scanType) throws ExecutionException, IOException { // Arrange Scan scan = prepareScan(); Scan scanForStorage = toScanForStorageFrom(scan); result = prepareResult(TransactionState.PREPARED); - when(scanner.iterator()).thenReturn(Collections.singletonList(result).iterator()); + if (scanType == ScanType.SCAN) { + when(scanner.iterator()).thenReturn(Collections.singletonList(result).iterator()); + } else { + when(scanner.one()).thenReturn(Optional.of(result)).thenReturn(Optional.empty()); + } when(storage.scan(scanForStorage)).thenReturn(scanner); // Act Assert - assertThatThrownBy(() -> handler.scan(scan)) + assertThatThrownBy(() -> scanOrGetScanner(scan, scanType)) .isInstanceOf(UncommittedRecordException.class) .satisfies( e -> { @@ -392,13 +407,15 @@ public void scan_ResultGivenFromStorage_ShouldUpdateSnapshotAndReturn() assertThat(exception.getResults().get(0)).isEqualTo(result); }); - verify(snapshot, never()).putIntoReadSet(any(), ArgumentMatchers.any()); - verify(snapshot, never()).putIntoScanSet(any(), ArgumentMatchers.any()); + verify(scanner).close(); + verify(snapshot, never()).putIntoReadSet(any(), any()); + verify(snapshot, never()).putIntoScanSet(any(), any()); } - @Test - public void scan_CalledTwice_SecondTimeShouldReturnTheSameFromSnapshot() - throws ExecutionException, CrudException { + @ParameterizedTest + @EnumSource(ScanType.class) + void scanOrGetScanner_CalledTwice_SecondTimeShouldReturnTheSameFromSnapshot(ScanType scanType) + throws ExecutionException, CrudException, IOException { // Arrange Scan originalScan = prepareScan(); Scan scanForStorage = toScanForStorageFrom(originalScan); @@ -406,7 +423,11 @@ public void scan_CalledTwice_SecondTimeShouldReturnTheSameFromSnapshot() Scan scan2 = prepareScan(); result = prepareResult(TransactionState.COMMITTED); TransactionResult expected = new TransactionResult(result); - when(scanner.iterator()).thenReturn(Collections.singletonList(result).iterator()); + if (scanType == ScanType.SCAN) { + when(scanner.iterator()).thenReturn(Collections.singletonList(result).iterator()); + } else { + when(scanner.one()).thenReturn(Optional.of(result)).thenReturn(Optional.empty()); + } when(storage.scan(scanForStorage)).thenReturn(scanner); Snapshot.Key key = new Snapshot.Key(scanForStorage, result); when(snapshot.getResults(scanForStorage)) @@ -415,10 +436,11 @@ public void scan_CalledTwice_SecondTimeShouldReturnTheSameFromSnapshot() when(snapshot.getResult(key)).thenReturn(Optional.of(expected)); // Act - List results1 = handler.scan(scan1); - List results2 = handler.scan(scan2); + List results1 = scanOrGetScanner(scan1, scanType); + List results2 = scanOrGetScanner(scan2, scanType); // Assert + verify(scanner).close(); verify(snapshot).putIntoReadSet(key, Optional.of(expected)); verify(snapshot) .putIntoScanSet(scanForStorage, Maps.newLinkedHashMap(ImmutableMap.of(key, expected))); @@ -430,9 +452,10 @@ public void scan_CalledTwice_SecondTimeShouldReturnTheSameFromSnapshot() verify(storage).scan(scanForStorage); } - @Test - public void scan_CalledTwiceUnderRealSnapshot_SecondTimeShouldReturnTheSameFromSnapshot() - throws ExecutionException, CrudException { + @ParameterizedTest + @EnumSource(ScanType.class) + void scan_CalledTwiceUnderRealSnapshot_SecondTimeShouldReturnTheSameFromSnapshot( + ScanType scanType) throws ExecutionException, CrudException, IOException { // Arrange Scan originalScan = prepareScan(); Scan scanForStorage = toScanForStorageFrom(originalScan); @@ -442,30 +465,41 @@ public void scan_CalledTwiceUnderRealSnapshot_SecondTimeShouldReturnTheSameFromS TransactionResult expected = new TransactionResult(result); snapshot = new Snapshot(ANY_TX_ID, Isolation.SNAPSHOT, tableMetadataManager, parallelExecutor); handler = new CrudHandler(storage, snapshot, tableMetadataManager, false, parallelExecutor); - when(scanner.iterator()).thenReturn(Collections.singletonList(result).iterator()); + if (scanType == ScanType.SCAN) { + when(scanner.iterator()).thenReturn(Collections.singletonList(result).iterator()); + } else { + when(scanner.one()).thenReturn(Optional.of(result)).thenReturn(Optional.empty()); + } when(storage.scan(scanForStorage)).thenReturn(scanner); // Act - List results1 = handler.scan(scan1); - List results2 = handler.scan(scan2); + List results1 = scanOrGetScanner(scan1, scanType); + List results2 = scanOrGetScanner(scan2, scanType); // Assert assertThat(results1.size()).isEqualTo(1); assertThat(results1.get(0)) .isEqualTo(new FilteredResult(expected, Collections.emptyList(), TABLE_METADATA, false)); assertThat(results1).isEqualTo(results2); + + verify(scanner).close(); verify(storage, never()).scan(originalScan); verify(storage).scan(scanForStorage); } - @Test - public void scan_GetCalledAfterScan_ShouldReturnFromStorage() - throws ExecutionException, CrudException { + @ParameterizedTest + @EnumSource(ScanType.class) + void scanOrGetScanner_GetCalledAfterScan_ShouldReturnFromStorage(ScanType scanType) + throws ExecutionException, CrudException, IOException { // Arrange Scan scan = prepareScan(); Scan scanForStorage = toScanForStorageFrom(scan); result = prepareResult(TransactionState.COMMITTED); - when(scanner.iterator()).thenReturn(Collections.singletonList(result).iterator()); + if (scanType == ScanType.SCAN) { + when(scanner.iterator()).thenReturn(Collections.singletonList(result).iterator()); + } else { + when(scanner.one()).thenReturn(Optional.of(result)).thenReturn(Optional.empty()); + } when(storage.scan(scanForStorage)).thenReturn(scanner); Get get = prepareGet(); Snapshot.Key key = new Snapshot.Key(get); @@ -476,46 +510,55 @@ public void scan_GetCalledAfterScan_ShouldReturnFromStorage() when(snapshot.getResult(key)).thenReturn(transactionResult); // Act - List results = handler.scan(scan); + List results = scanOrGetScanner(scan, scanType); Optional result = handler.get(get); // Assert verify(storage).scan(scanForStorage); - verify(storage).get(getForStorage); + verify(scanner).close(); + assertThat(results.size()).isEqualTo(1); assertThat(result).isPresent(); assertThat(results.get(0)).isEqualTo(result.get()); } - @Test - public void scan_GetCalledAfterScanUnderRealSnapshot_ShouldReturnFromStorage() - throws ExecutionException, CrudException { + @ParameterizedTest + @EnumSource(ScanType.class) + void scanOrGetScanner_GetCalledAfterScanUnderRealSnapshot_ShouldReturnFromStorage( + ScanType scanType) throws ExecutionException, CrudException, IOException { // Arrange Scan scan = toScanForStorageFrom(prepareScan()); result = prepareResult(TransactionState.COMMITTED); snapshot = new Snapshot(ANY_TX_ID, Isolation.SNAPSHOT, tableMetadataManager, parallelExecutor); handler = new CrudHandler(storage, snapshot, tableMetadataManager, false, parallelExecutor); - when(scanner.iterator()).thenReturn(Collections.singletonList(result).iterator()); + if (scanType == ScanType.SCAN) { + when(scanner.iterator()).thenReturn(Collections.singletonList(result).iterator()); + } else { + when(scanner.one()).thenReturn(Optional.of(result)).thenReturn(Optional.empty()); + } when(storage.scan(scan)).thenReturn(scanner); Get get = prepareGet(); when(storage.get(get)).thenReturn(Optional.of(result)); // Act - List results = handler.scan(scan); + List results = scanOrGetScanner(scan, scanType); Optional result = handler.get(get); // Assert verify(storage).scan(scan); verify(storage).get(get); + verify(scanner).close(); + assertThat(results.size()).isEqualTo(1); assertThat(result).isPresent(); assertThat(results.get(0)).isEqualTo(result.get()); } - @Test - public void scan_CalledAfterDeleteUnderRealSnapshot_ShouldThrowIllegalArgumentException() - throws ExecutionException, CrudException { + @ParameterizedTest + @EnumSource(ScanType.class) + void scanOrGetScanner_CalledAfterDeleteUnderRealSnapshot_ShouldThrowIllegalArgumentException( + ScanType scanType) throws ExecutionException, CrudException, IOException { // Arrange Scan scan = prepareScan(); Scan scanForStorage = toScanForStorageFrom(scan); @@ -551,9 +594,17 @@ public void scan_CalledAfterDeleteUnderRealSnapshot_ShouldThrowIllegalArgumentEx new ConcurrentHashMap<>(), new HashMap<>(), new HashMap<>(), - deleteSet); + deleteSet, + new ArrayList<>()); handler = new CrudHandler(storage, snapshot, tableMetadataManager, false, parallelExecutor); - when(scanner.iterator()).thenReturn(Arrays.asList(result, result2).iterator()); + if (scanType == ScanType.SCAN) { + when(scanner.iterator()).thenReturn(Arrays.asList(result, result2).iterator()); + } else { + when(scanner.one()) + .thenReturn(Optional.of(result)) + .thenReturn(Optional.of(result2)) + .thenReturn(Optional.empty()); + } when(storage.scan(scanForStorage)).thenReturn(scanner); Delete delete = @@ -568,26 +619,35 @@ public void scan_CalledAfterDeleteUnderRealSnapshot_ShouldThrowIllegalArgumentEx assertThat(deleteSet.size()).isEqualTo(1); assertThat(deleteSet).containsKey(new Snapshot.Key(delete)); - assertThatThrownBy(() -> handler.scan(scan)).isInstanceOf(IllegalArgumentException.class); + assertThatThrownBy(() -> scanOrGetScanner(scan, scanType)) + .isInstanceOf(IllegalArgumentException.class); + + verify(scanner).close(); } - @Test - public void - scan_CrossPartitionScanAndResultFromStorageGiven_ShouldUpdateSnapshotAndVerifyNoOverlapThenReturn() - throws ExecutionException, CrudException { + @ParameterizedTest + @EnumSource(ScanType.class) + void + scanOrGetScanner_CrossPartitionScanAndResultFromStorageGiven_ShouldUpdateSnapshotAndVerifyNoOverlapThenReturn( + ScanType scanType) throws ExecutionException, CrudException, IOException { // Arrange Scan scan = prepareCrossPartitionScan(); result = prepareResult(TransactionState.COMMITTED); Snapshot.Key key = new Snapshot.Key(scan, result); - when(scanner.iterator()).thenReturn(Collections.singletonList(result).iterator()); + if (scanType == ScanType.SCAN) { + when(scanner.iterator()).thenReturn(Collections.singletonList(result).iterator()); + } else { + when(scanner.one()).thenReturn(Optional.of(result)).thenReturn(Optional.empty()); + } when(storage.scan(any(ScanAll.class))).thenReturn(scanner); TransactionResult transactionResult = new TransactionResult(result); when(snapshot.getResult(key)).thenReturn(Optional.of(transactionResult)); // Act - List results = handler.scan(scan); + List results = scanOrGetScanner(scan, scanType); // Assert + verify(scanner).close(); verify(snapshot).putIntoReadSet(key, Optional.of(transactionResult)); verify(snapshot) .putIntoScanSet(scan, Maps.newLinkedHashMap(ImmutableMap.of(key, transactionResult))); @@ -598,18 +658,23 @@ public void scan_CalledAfterDeleteUnderRealSnapshot_ShouldThrowIllegalArgumentEx new FilteredResult(transactionResult, Collections.emptyList(), TABLE_METADATA, false)); } - @Test - public void - scan_CrossPartitionScanAndPreparedResultFromStorageGiven_ShouldNeverUpdateSnapshotNorVerifyNoOverlapButThrowUncommittedRecordException() - throws ExecutionException { + @ParameterizedTest + @EnumSource(ScanType.class) + void + scanOrGetScanner_CrossPartitionScanAndPreparedResultFromStorageGiven_ShouldNeverUpdateSnapshotNorVerifyNoOverlapButThrowUncommittedRecordException( + ScanType scanType) throws ExecutionException, IOException { // Arrange Scan scan = prepareCrossPartitionScan(); result = prepareResult(TransactionState.PREPARED); - when(scanner.iterator()).thenReturn(Collections.singletonList(result).iterator()); + if (scanType == ScanType.SCAN) { + when(scanner.iterator()).thenReturn(Collections.singletonList(result).iterator()); + } else { + when(scanner.one()).thenReturn(Optional.of(result)).thenReturn(Optional.empty()); + } when(storage.scan(any(ScanAll.class))).thenReturn(scanner); // Act Assert - assertThatThrownBy(() -> handler.scan(scan)) + assertThatThrownBy(() -> scanOrGetScanner(scan, scanType)) .isInstanceOf(UncommittedRecordException.class) .satisfies( e -> { @@ -619,14 +684,16 @@ public void scan_CalledAfterDeleteUnderRealSnapshot_ShouldThrowIllegalArgumentEx assertThat(exception.getResults().get(0)).isEqualTo(result); }); - verify(snapshot, never()).putIntoReadSet(any(Snapshot.Key.class), ArgumentMatchers.any()); + verify(scanner).close(); + verify(snapshot, never()).putIntoReadSet(any(Snapshot.Key.class), any()); + verify(snapshot, never()).putIntoScannerSet(any(Scan.class), any()); verify(snapshot, never()).verifyNoOverlap(any(), any()); } @Test public void scan_RuntimeExceptionCausedByExecutionExceptionThrownByIteratorHasNext_ShouldThrowCrudException() - throws ExecutionException { + throws ExecutionException, IOException { // Arrange Scan scan = prepareScan(); Scan scanForStorage = toScanForStorageFrom(scan); @@ -643,11 +710,13 @@ public void scan_CalledAfterDeleteUnderRealSnapshot_ShouldThrowIllegalArgumentEx assertThatThrownBy(() -> handler.scan(scan)) .isInstanceOf(CrudException.class) .hasCause(executionException); + + verify(scanner).close(); } @Test public void scan_RuntimeExceptionThrownByIteratorHasNext_ShouldThrowCrudException() - throws ExecutionException { + throws ExecutionException, IOException { // Arrange Scan scan = prepareScan(); Scan scanForStorage = toScanForStorageFrom(scan); @@ -662,6 +731,60 @@ public void scan_RuntimeExceptionThrownByIteratorHasNext_ShouldThrowCrudExceptio assertThatThrownBy(() -> handler.scan(scan)) .isInstanceOf(CrudException.class) .hasCause(runtimeException); + + verify(scanner).close(); + } + + @Test + public void getScanner_ExecutionExceptionThrownByScannerOne_ShouldThrowCrudException() + throws ExecutionException, IOException, CrudException { + // Arrange + Scan scan = prepareScan(); + Scan scanForStorage = toScanForStorageFrom(scan); + ExecutionException executionException = mock(ExecutionException.class); + when(scanner.one()).thenThrow(executionException); + when(storage.scan(scanForStorage)).thenReturn(scanner); + + // Act Assert + TransactionCrudOperable.Scanner actualScanner = handler.getScanner(scan); + assertThatThrownBy(actualScanner::one) + .isInstanceOf(CrudException.class) + .hasCause(executionException); + + verify(scanner).close(); + } + + @Test + public void + getScanner_ScannerNotFullyScanned_ShouldPutReadSetAndScannerSetInSnapshotAndVerifyScan() + throws ExecutionException, CrudException, IOException { + // Arrange + Scan scan = prepareScan(); + Scan scanForStorage = toScanForStorageFrom(scan); + Result result1 = prepareResult(TransactionState.COMMITTED); + Result result2 = prepareResult(TransactionState.COMMITTED); + Snapshot.Key key1 = new Snapshot.Key(scan, result1); + TransactionResult txResult1 = new TransactionResult(result1); + when(scanner.one()) + .thenReturn(Optional.of(result1)) + .thenReturn(Optional.of(result2)) + .thenReturn(Optional.empty()); + when(storage.scan(scanForStorage)).thenReturn(scanner); + + // Act + TransactionCrudOperable.Scanner actualScanner = handler.getScanner(scan); + Optional actualResult = actualScanner.one(); + actualScanner.close(); + + // Assert + verify(scanner).close(); + verify(snapshot).putIntoReadSet(key1, Optional.of(txResult1)); + verify(snapshot) + .putIntoScannerSet(scan, Maps.newLinkedHashMap(ImmutableMap.of(key1, txResult1))); + verify(snapshot).verifyNoOverlap(scan, ImmutableMap.of(key1, txResult1)); + + assertThat(actualResult) + .hasValue(new FilteredResult(txResult1, Collections.emptyList(), TABLE_METADATA, false)); } @Test @@ -1191,4 +1314,34 @@ public void readIfImplicitPreReadEnabled_ShouldCallAppropriateMethods() throws C assertThat(transactionIdCaptor.getValue()).isEqualTo(ANY_TX_ID); } + + private List scanOrGetScanner(Scan scan, ScanType scanType) throws CrudException { + if (scanType == ScanType.SCAN) { + return handler.scan(scan); + } + + try (TransactionCrudOperable.Scanner scanner = handler.getScanner(scan)) { + switch (scanType) { + case SCANNER_ONE: + List results = new ArrayList<>(); + while (true) { + Optional result = scanner.one(); + if (!result.isPresent()) { + return results; + } + results.add(result.get()); + } + case SCANNER_ALL: + return scanner.all(); + default: + throw new AssertionError(); + } + } + } + + enum ScanType { + SCAN, + SCANNER_ONE, + SCANNER_ALL + } } diff --git a/core/src/test/java/com/scalar/db/transaction/consensuscommit/SnapshotTest.java b/core/src/test/java/com/scalar/db/transaction/consensuscommit/SnapshotTest.java index 019df03c64..06b5d0aa1c 100644 --- a/core/src/test/java/com/scalar/db/transaction/consensuscommit/SnapshotTest.java +++ b/core/src/test/java/com/scalar/db/transaction/consensuscommit/SnapshotTest.java @@ -41,12 +41,14 @@ import com.scalar.db.io.Value; import com.scalar.db.transaction.consensuscommit.Snapshot.ReadWriteSets; import com.scalar.db.util.ScalarDbUtils; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; @@ -98,6 +100,7 @@ public class SnapshotTest { private Map> scanSet; private Map writeSet; private Map deleteSet; + private List scannerSet; @Mock private ConsensusCommitConfig config; @Mock private PrepareMutationComposer prepareComposer; @@ -122,6 +125,7 @@ private Snapshot prepareSnapshot(Isolation isolation) { scanSet = new HashMap<>(); writeSet = new HashMap<>(); deleteSet = new HashMap<>(); + scannerSet = new ArrayList<>(); return spy( new Snapshot( @@ -133,7 +137,8 @@ private Snapshot prepareSnapshot(Isolation isolation) { getSet, scanSet, writeSet, - deleteSet)); + deleteSet, + scannerSet)); } private TransactionResult prepareResult(String txId) { @@ -1614,6 +1619,33 @@ public void toSerializable_ScanWithLimitInScanSet_ShouldProcessWithoutExceptions verify(storage).scan(scanWithProjectionsWithoutLimit); } + @Test + public void toSerializable_ScannerSetNotChanged_ShouldProcessWithoutExceptions() + throws ExecutionException { + // Arrange + snapshot = prepareSnapshot(Isolation.SERIALIZABLE); + Scan scan = prepareScan(); + TransactionResult result1 = prepareResult(ANY_ID + "x", ANY_TEXT_1, ANY_TEXT_2); + TransactionResult result2 = prepareResult(ANY_ID + "x", ANY_TEXT_1, ANY_TEXT_3); + Snapshot.Key key1 = new Snapshot.Key(scan, result1); + snapshot.putIntoScannerSet(scan, Maps.newLinkedHashMap(ImmutableMap.of(key1, result1))); + DistributedStorage storage = mock(DistributedStorage.class); + Scan scanWithProjections = + Scan.newBuilder(scan).projections(Attribute.ID, ANY_NAME_1, ANY_NAME_2).build(); + Scanner scanner = mock(Scanner.class); + when(scanner.one()) + .thenReturn(Optional.of(result1)) + .thenReturn(Optional.of(result2)) + .thenReturn(Optional.empty()); + when(storage.scan(scanWithProjections)).thenReturn(scanner); + + // Act Assert + assertThatCode(() -> snapshot.toSerializable(storage)).doesNotThrowAnyException(); + + // Assert + verify(storage).scan(scanWithProjections); + } + @Test public void verifyNoOverlap_ScanGivenAndDeleteKeyAlreadyPresentInDeleteSet_ShouldThrowIllegalArgumentException() { diff --git a/core/src/test/java/com/scalar/db/transaction/consensuscommit/TwoPhaseConsensusCommitManagerTest.java b/core/src/test/java/com/scalar/db/transaction/consensuscommit/TwoPhaseConsensusCommitManagerTest.java index ba6c29ad56..880e7466c6 100644 --- a/core/src/test/java/com/scalar/db/transaction/consensuscommit/TwoPhaseConsensusCommitManagerTest.java +++ b/core/src/test/java/com/scalar/db/transaction/consensuscommit/TwoPhaseConsensusCommitManagerTest.java @@ -19,6 +19,8 @@ import com.scalar.db.api.Put; import com.scalar.db.api.Result; import com.scalar.db.api.Scan; +import com.scalar.db.api.TransactionCrudOperable; +import com.scalar.db.api.TransactionManagerCrudOperable; import com.scalar.db.api.TransactionState; import com.scalar.db.api.TwoPhaseCommitTransaction; import com.scalar.db.api.TwoPhaseCommitTransactionManager; @@ -40,6 +42,8 @@ import com.scalar.db.io.Key; import com.scalar.db.transaction.consensuscommit.Coordinator.State; import java.util.Arrays; +import java.util.Collections; +import java.util.Iterator; import java.util.List; import java.util.Optional; import org.junit.jupiter.api.BeforeEach; @@ -702,6 +706,404 @@ public void scan_ShouldScan() throws TransactionException { assertThat(actual).isEqualTo(results); } + @Test + public void getScannerAndScannerOne_ShouldReturnScannerAndReturnProperResult() throws Exception { + // Arrange + TwoPhaseCommitTransaction transaction = mock(TwoPhaseCommitTransaction.class); + + TwoPhaseConsensusCommitManager spied = spy(manager); + doReturn(transaction).when(spied).begin(); + + Scan scan = + Scan.newBuilder() + .namespace("ns") + .table("tbl") + .partitionKey(Key.ofText("p1", "val")) + .build(); + + TransactionCrudOperable.Scanner scanner = mock(TransactionCrudOperable.Scanner.class); + when(transaction.getScanner(scan)).thenReturn(scanner); + + Result result1 = mock(Result.class); + Result result2 = mock(Result.class); + Result result3 = mock(Result.class); + + when(scanner.one()) + .thenReturn(Optional.of(result1)) + .thenReturn(Optional.of(result2)) + .thenReturn(Optional.of(result3)) + .thenReturn(Optional.empty()); + + // Act Assert + TransactionManagerCrudOperable.Scanner actual = spied.getScanner(scan); + assertThat(actual.one()).hasValue(result1); + assertThat(actual.one()).hasValue(result2); + assertThat(actual.one()).hasValue(result3); + assertThat(actual.one()).isEmpty(); + actual.close(); + + verify(spied).begin(); + verify(transaction).prepare(); + verify(transaction).validate(); + verify(transaction).commit(); + verify(scanner).close(); + } + + @Test + public void getScannerAndScannerAll_ShouldReturnScannerAndReturnProperResults() throws Exception { + // Arrange + TwoPhaseCommitTransaction transaction = mock(TwoPhaseCommitTransaction.class); + + TwoPhaseConsensusCommitManager spied = spy(manager); + doReturn(transaction).when(spied).begin(); + + Scan scan = + Scan.newBuilder() + .namespace("ns") + .table("tbl") + .partitionKey(Key.ofText("p1", "val")) + .build(); + + TransactionCrudOperable.Scanner scanner = mock(TransactionCrudOperable.Scanner.class); + when(transaction.getScanner(scan)).thenReturn(scanner); + + Result result1 = mock(Result.class); + Result result2 = mock(Result.class); + Result result3 = mock(Result.class); + + when(scanner.all()) + .thenReturn(Arrays.asList(result1, result2, result3)) + .thenReturn(Collections.emptyList()); + + // Act Assert + TransactionManagerCrudOperable.Scanner actual = spied.getScanner(scan); + List results = actual.all(); + assertThat(results).containsExactly(result1, result2, result3); + assertThat(actual.all()).isEmpty(); + actual.close(); + + verify(spied).begin(); + verify(transaction).prepare(); + verify(transaction).validate(); + verify(transaction).commit(); + verify(scanner).close(); + } + + @Test + public void getScannerAndScannerIterator_ShouldReturnScannerAndReturnProperResults() + throws Exception { + // Arrange + TwoPhaseCommitTransaction transaction = mock(TwoPhaseCommitTransaction.class); + + TwoPhaseConsensusCommitManager spied = spy(manager); + doReturn(transaction).when(spied).begin(); + + Scan scan = + Scan.newBuilder() + .namespace("ns") + .table("tbl") + .partitionKey(Key.ofText("p1", "val")) + .build(); + + TransactionCrudOperable.Scanner scanner = mock(TransactionCrudOperable.Scanner.class); + when(transaction.getScanner(scan)).thenReturn(scanner); + + Result result1 = mock(Result.class); + Result result2 = mock(Result.class); + Result result3 = mock(Result.class); + + when(scanner.one()) + .thenReturn(Optional.of(result1)) + .thenReturn(Optional.of(result2)) + .thenReturn(Optional.of(result3)) + .thenReturn(Optional.empty()); + + // Act Assert + TransactionManagerCrudOperable.Scanner actual = spied.getScanner(scan); + + Iterator iterator = actual.iterator(); + assertThat(iterator.hasNext()).isTrue(); + assertThat(iterator.next()).isEqualTo(result1); + assertThat(iterator.hasNext()).isTrue(); + assertThat(iterator.next()).isEqualTo(result2); + assertThat(iterator.hasNext()).isTrue(); + assertThat(iterator.next()).isEqualTo(result3); + assertThat(iterator.hasNext()).isFalse(); + actual.close(); + + verify(spied).begin(); + verify(transaction).prepare(); + verify(transaction).validate(); + verify(transaction).commit(); + verify(scanner).close(); + } + + @Test + public void + getScanner_CrudExceptionThrownByTransactionGetScanner_ShouldRollbackTransactionAndThrowCrudException() + throws TransactionException { + // Arrange + TwoPhaseCommitTransaction transaction = mock(TwoPhaseCommitTransaction.class); + + TwoPhaseConsensusCommitManager spied = spy(manager); + doReturn(transaction).when(spied).begin(); + + Scan scan = + Scan.newBuilder() + .namespace("ns") + .table("tbl") + .partitionKey(Key.ofText("p1", "val")) + .build(); + + when(transaction.getScanner(scan)).thenThrow(CrudException.class); + + // Act Assert + assertThatThrownBy(() -> spied.getScanner(scan)).isInstanceOf(CrudException.class); + + verify(spied).begin(); + verify(transaction).rollback(); + } + + @Test + public void + getScannerAndScannerOne_CrudExceptionThrownByScannerOne_ShouldRollbackTransactionAndThrowCrudException() + throws TransactionException { + // Arrange + TwoPhaseCommitTransaction transaction = mock(TwoPhaseCommitTransaction.class); + + TwoPhaseConsensusCommitManager spied = spy(manager); + doReturn(transaction).when(spied).begin(); + + Scan scan = + Scan.newBuilder() + .namespace("ns") + .table("tbl") + .partitionKey(Key.ofText("p1", "val")) + .build(); + + TransactionCrudOperable.Scanner scanner = mock(TransactionCrudOperable.Scanner.class); + when(transaction.getScanner(scan)).thenReturn(scanner); + + when(scanner.one()).thenThrow(CrudException.class); + + // Act Assert + TransactionManagerCrudOperable.Scanner actual = spied.getScanner(scan); + assertThatThrownBy(actual::one).isInstanceOf(CrudException.class); + + verify(spied).begin(); + verify(scanner).close(); + verify(transaction).rollback(); + } + + @Test + public void + getScannerAndScannerAll_CrudExceptionThrownByScannerAll_ShouldRollbackTransactionAndThrowCrudException() + throws TransactionException { + // Arrange + TwoPhaseCommitTransaction transaction = mock(TwoPhaseCommitTransaction.class); + TwoPhaseConsensusCommitManager spied = spy(manager); + doReturn(transaction).when(spied).begin(); + + Scan scan = + Scan.newBuilder() + .namespace("ns") + .table("tbl") + .partitionKey(Key.ofText("p1", "val")) + .build(); + + TransactionCrudOperable.Scanner scanner = mock(TransactionCrudOperable.Scanner.class); + when(transaction.getScanner(scan)).thenReturn(scanner); + + when(scanner.all()).thenThrow(CrudException.class); + + // Act Assert + TransactionManagerCrudOperable.Scanner actual = spied.getScanner(scan); + assertThatThrownBy(actual::all).isInstanceOf(CrudException.class); + + verify(spied).begin(); + verify(scanner).close(); + verify(transaction).rollback(); + } + + @Test + public void + getScannerAndScannerClose_CrudExceptionThrownByScannerClose_ShouldRollbackTransactionAndThrowCrudException() + throws TransactionException { + // Arrange + TwoPhaseCommitTransaction transaction = mock(TwoPhaseCommitTransaction.class); + TwoPhaseConsensusCommitManager spied = spy(manager); + doReturn(transaction).when(spied).begin(); + + Scan scan = + Scan.newBuilder() + .namespace("ns") + .table("tbl") + .partitionKey(Key.ofText("p1", "val")) + .build(); + + TransactionCrudOperable.Scanner scanner = mock(TransactionCrudOperable.Scanner.class); + when(transaction.getScanner(scan)).thenReturn(scanner); + + doThrow(CrudException.class).when(scanner).close(); + + // Act Assert + TransactionManagerCrudOperable.Scanner actual = spied.getScanner(scan); + assertThatThrownBy(actual::close).isInstanceOf(CrudException.class); + + verify(spied).begin(); + verify(scanner).close(); + verify(transaction).rollback(); + } + + @Test + public void + getScannerAndScannerClose_PreparationConflictExceptionThrownByTransactionPrepare_ShouldRollbackTransactionAndThrowCrudConflictException() + throws TransactionException { + // Arrange + TwoPhaseCommitTransaction transaction = mock(TwoPhaseCommitTransaction.class); + + TwoPhaseConsensusCommitManager spied = spy(manager); + doReturn(transaction).when(spied).begin(); + doThrow(PreparationConflictException.class).when(transaction).prepare(); + + Scan scan = + Scan.newBuilder() + .namespace("ns") + .table("tbl") + .partitionKey(Key.ofText("p1", "val")) + .build(); + + TransactionCrudOperable.Scanner scanner = mock(TransactionCrudOperable.Scanner.class); + when(transaction.getScanner(scan)).thenReturn(scanner); + + // Act Assert + TransactionManagerCrudOperable.Scanner actual = spied.getScanner(scan); + assertThatThrownBy(actual::close).isInstanceOf(CrudConflictException.class); + + verify(spied).begin(); + verify(scanner).close(); + verify(transaction).rollback(); + } + + @Test + public void + getScannerAndScannerClose_ValidationConflictExceptionThrownByTransactionValidate_ShouldRollbackTransactionAndThrowCrudConflictException() + throws TransactionException { + // Arrange + TwoPhaseCommitTransaction transaction = mock(TwoPhaseCommitTransaction.class); + + TwoPhaseConsensusCommitManager spied = spy(manager); + doReturn(transaction).when(spied).begin(); + doThrow(ValidationConflictException.class).when(transaction).validate(); + + Scan scan = + Scan.newBuilder() + .namespace("ns") + .table("tbl") + .partitionKey(Key.ofText("p1", "val")) + .build(); + + TransactionCrudOperable.Scanner scanner = mock(TransactionCrudOperable.Scanner.class); + when(transaction.getScanner(scan)).thenReturn(scanner); + + // Act Assert + TransactionManagerCrudOperable.Scanner actual = spied.getScanner(scan); + assertThatThrownBy(actual::close).isInstanceOf(CrudConflictException.class); + + verify(spied).begin(); + verify(scanner).close(); + verify(transaction).rollback(); + } + + @Test + public void + getScannerAndScannerClose_CommitConflictExceptionThrownByTransactionCommit_ShouldRollbackTransactionAndThrowCrudConflictException() + throws TransactionException { + // Arrange + TwoPhaseCommitTransaction transaction = mock(TwoPhaseCommitTransaction.class); + + TwoPhaseConsensusCommitManager spied = spy(manager); + doReturn(transaction).when(spied).begin(); + doThrow(CommitConflictException.class).when(transaction).commit(); + + Scan scan = + Scan.newBuilder() + .namespace("ns") + .table("tbl") + .partitionKey(Key.ofText("p1", "val")) + .build(); + + TransactionCrudOperable.Scanner scanner = mock(TransactionCrudOperable.Scanner.class); + when(transaction.getScanner(scan)).thenReturn(scanner); + + // Act Assert + TransactionManagerCrudOperable.Scanner actual = spied.getScanner(scan); + assertThatThrownBy(actual::close).isInstanceOf(CrudConflictException.class); + + verify(spied).begin(); + verify(scanner).close(); + verify(transaction).rollback(); + } + + @Test + public void + getScannerAndScannerClose_UnknownTransactionStatusExceptionByTransactionCommit_ShouldThrowUnknownTransactionStatusException() + throws TransactionException { + // Arrange + TwoPhaseCommitTransaction transaction = mock(TwoPhaseCommitTransaction.class); + + TwoPhaseConsensusCommitManager spied = spy(manager); + doReturn(transaction).when(spied).begin(); + doThrow(UnknownTransactionStatusException.class).when(transaction).commit(); + + Scan scan = + Scan.newBuilder() + .namespace("ns") + .table("tbl") + .partitionKey(Key.ofText("p1", "val")) + .build(); + + TransactionCrudOperable.Scanner scanner = mock(TransactionCrudOperable.Scanner.class); + when(transaction.getScanner(scan)).thenReturn(scanner); + + // Act Assert + TransactionManagerCrudOperable.Scanner actual = spied.getScanner(scan); + assertThatThrownBy(actual::close).isInstanceOf(UnknownTransactionStatusException.class); + + verify(spied).begin(); + verify(scanner).close(); + } + + @Test + public void + getScannerAndScannerClose_CommitExceptionThrownByTransactionCommit_ShouldRollbackTransactionAndThrowCrudException() + throws TransactionException { + // Arrange + TwoPhaseCommitTransaction transaction = mock(TwoPhaseCommitTransaction.class); + + TwoPhaseConsensusCommitManager spied = spy(manager); + doReturn(transaction).when(spied).begin(); + doThrow(CommitException.class).when(transaction).commit(); + + Scan scan = + Scan.newBuilder() + .namespace("ns") + .table("tbl") + .partitionKey(Key.ofText("p1", "val")) + .build(); + + TransactionCrudOperable.Scanner scanner = mock(TransactionCrudOperable.Scanner.class); + when(transaction.getScanner(scan)).thenReturn(scanner); + + // Act Assert + TransactionManagerCrudOperable.Scanner actual = spied.getScanner(scan); + assertThatThrownBy(actual::close).isInstanceOf(CrudException.class); + + verify(spied).begin(); + verify(scanner).close(); + verify(transaction).rollback(); + } + @Test public void put_ShouldPut() throws TransactionException { // Arrange diff --git a/core/src/test/java/com/scalar/db/transaction/consensuscommit/TwoPhaseConsensusCommitTest.java b/core/src/test/java/com/scalar/db/transaction/consensuscommit/TwoPhaseConsensusCommitTest.java index bd12bbea43..0b80bf8b22 100644 --- a/core/src/test/java/com/scalar/db/transaction/consensuscommit/TwoPhaseConsensusCommitTest.java +++ b/core/src/test/java/com/scalar/db/transaction/consensuscommit/TwoPhaseConsensusCommitTest.java @@ -17,6 +17,7 @@ import com.scalar.db.api.Put; import com.scalar.db.api.Result; import com.scalar.db.api.Scan; +import com.scalar.db.api.TransactionCrudOperable; import com.scalar.db.api.TransactionState; import com.scalar.db.api.Update; import com.scalar.db.api.Upsert; @@ -28,6 +29,7 @@ import com.scalar.db.exception.transaction.PreparationConflictException; import com.scalar.db.exception.transaction.PreparationException; import com.scalar.db.exception.transaction.RollbackException; +import com.scalar.db.exception.transaction.TransactionException; import com.scalar.db.exception.transaction.UnknownTransactionStatusException; import com.scalar.db.exception.transaction.UnsatisfiedConditionException; import com.scalar.db.exception.transaction.ValidationException; @@ -65,7 +67,10 @@ public class TwoPhaseConsensusCommitTest { public void setUp() throws Exception { MockitoAnnotations.openMocks(this).close(); + // Arrange transaction = new TwoPhaseConsensusCommit(crud, commit, recovery, mutationOperationChecker); + + when(crud.areAllScannersClosed()).thenReturn(true); } private Get prepareGet() { @@ -166,6 +171,93 @@ public void scan_ScanForUncommittedRecordGiven_ShouldRecoverRecord() throws Crud verify(recovery).recover(scan, result); } + @Test + public void getScannerAndScannerOne_ShouldCallCrudHandlerGetScannerAndScannerOne() + throws CrudException { + // Arrange + Scan scan = prepareScan(); + TransactionCrudOperable.Scanner scanner = mock(TransactionCrudOperable.Scanner.class); + Result result = mock(Result.class); + when(scanner.one()).thenReturn(Optional.of(result)); + when(crud.getScanner(scan)).thenReturn(scanner); + + // Act + TransactionCrudOperable.Scanner actualScanner = transaction.getScanner(scan); + Optional actualResult = actualScanner.one(); + + // Assert + assertThat(actualResult).hasValue(result); + verify(crud).getScanner(scan); + verify(scanner).one(); + } + + @Test + public void + getScannerAndScannerOne_UncommittedRecordExceptionThrownByScannerOne_ShouldRecoverRecord() + throws CrudException { + // Arrange + Scan scan = prepareScan(); + + UncommittedRecordException toThrow = mock(UncommittedRecordException.class); + TransactionResult result = mock(TransactionResult.class); + when(toThrow.getSelection()).thenReturn(scan); + when(toThrow.getResults()).thenReturn(Collections.singletonList(result)); + + TransactionCrudOperable.Scanner scanner = mock(TransactionCrudOperable.Scanner.class); + when(scanner.one()).thenThrow(toThrow); + when(crud.getScanner(scan)).thenReturn(scanner); + + // Act Assert + TransactionCrudOperable.Scanner actualScanner = transaction.getScanner(scan); + assertThatThrownBy(actualScanner::one).isInstanceOf(UncommittedRecordException.class); + + verify(recovery).recover(scan, result); + } + + @Test + public void getScannerAndScannerAll_ShouldCallCrudHandlerGetScannerAndScannerAll() + throws CrudException { + // Arrange + Scan scan = prepareScan(); + TransactionCrudOperable.Scanner scanner = mock(TransactionCrudOperable.Scanner.class); + Result result1 = mock(Result.class); + Result result2 = mock(Result.class); + when(scanner.all()).thenReturn(Arrays.asList(result1, result2)); + when(crud.getScanner(scan)).thenReturn(scanner); + + // Act + TransactionCrudOperable.Scanner actualScanner = transaction.getScanner(scan); + List actualResults = actualScanner.all(); + + // Assert + assertThat(actualResults).containsExactly(result1, result2); + verify(crud).getScanner(scan); + verify(scanner).all(); + } + + @Test + public void + getScannerAndScannerAll_UncommittedRecordExceptionThrownByScannerAll_ShouldRecoverRecord() + throws CrudException { + // Arrange + Scan scan = prepareScan(); + + UncommittedRecordException toThrow = mock(UncommittedRecordException.class); + TransactionResult result = mock(TransactionResult.class); + when(toThrow.getSelection()).thenReturn(scan); + when(toThrow.getResults()).thenReturn(Collections.singletonList(result)); + + TransactionCrudOperable.Scanner scanner = mock(TransactionCrudOperable.Scanner.class); + when(scanner.all()).thenThrow(toThrow); + when(crud.getScanner(scan)).thenReturn(scanner); + + // Act Assert + TransactionCrudOperable.Scanner actualScanner = transaction.getScanner(scan); + assertThatThrownBy(actualScanner::all).isInstanceOf(UncommittedRecordException.class); + + verify(recovery).recover(scan, result); + } + @Test public void put_PutGiven_ShouldCallCrudHandlerPut() throws ExecutionException, CrudException { // Arrange @@ -673,6 +765,15 @@ public void prepare_ProcessedCrudGiven_ShouldPrepareWithSnapshot() assertThatThrownBy(transaction::prepare).isInstanceOf(PreparationException.class); } + @Test + public void prepare_ScannerNotClosed_ShouldThrowIllegalStateException() { + // Arrange + when(crud.areAllScannersClosed()).thenReturn(false); + + // Act Assert + assertThatThrownBy(() -> transaction.prepare()).isInstanceOf(IllegalStateException.class); + } + @Test public void validate_ProcessedCrudGiven_ShouldPerformValidationWithSnapshot() throws ValidationException, PreparationException { @@ -735,8 +836,7 @@ public void commit_SerializableUsedAndPreparedState_ShouldThrowIllegalStateExcep } @Test - public void rollback_ShouldAbortStateAndRollbackRecords() - throws RollbackException, UnknownTransactionStatusException, PreparationException { + public void rollback_ShouldAbortStateAndRollbackRecords() throws TransactionException { // Arrange transaction.prepare(); when(crud.getSnapshot()).thenReturn(snapshot); @@ -745,13 +845,14 @@ public void rollback_ShouldAbortStateAndRollbackRecords() transaction.rollback(); // Assert + verify(crud).closeScanners(); verify(commit).abortState(snapshot.getId()); verify(commit).rollbackRecords(snapshot); } @Test public void rollback_CalledAfterPrepareFails_ShouldAbortStateAndRollbackRecords() - throws PreparationException, UnknownTransactionStatusException, RollbackException { + throws TransactionException { // Arrange when(crud.getSnapshot()).thenReturn(snapshot); doThrow(PreparationException.class).when(commit).prepare(snapshot); @@ -761,14 +862,14 @@ public void rollback_CalledAfterPrepareFails_ShouldAbortStateAndRollbackRecords( transaction.rollback(); // Assert + verify(crud).closeScanners(); verify(commit).abortState(snapshot.getId()); verify(commit).rollbackRecords(snapshot); } @Test public void rollback_CalledAfterCommitFails_ShouldNeverAbortStateAndRollbackRecords() - throws CommitException, UnknownTransactionStatusException, RollbackException, - PreparationException { + throws TransactionException { // Arrange transaction.prepare(); when(crud.getSnapshot()).thenReturn(snapshot); @@ -779,6 +880,7 @@ public void rollback_CalledAfterCommitFails_ShouldNeverAbortStateAndRollbackReco transaction.rollback(); // Assert + verify(crud).closeScanners(); verify(commit, never()).abortState(snapshot.getId()); verify(commit, never()).rollbackRecords(snapshot); } @@ -786,7 +888,7 @@ public void rollback_CalledAfterCommitFails_ShouldNeverAbortStateAndRollbackReco @Test public void rollback_UnknownTransactionStatusExceptionThrownByAbortState_ShouldThrowRollbackException() - throws UnknownTransactionStatusException, PreparationException { + throws TransactionException { // Arrange transaction.prepare(); when(crud.getSnapshot()).thenReturn(snapshot); @@ -795,12 +897,13 @@ public void rollback_CalledAfterCommitFails_ShouldNeverAbortStateAndRollbackReco // Act Assert assertThatThrownBy(transaction::rollback).isInstanceOf(RollbackException.class); + verify(crud).closeScanners(); verify(commit, never()).rollbackRecords(snapshot); } @Test public void rollback_CommittedStateReturnedByAbortState_ShouldThrowRollbackException() - throws UnknownTransactionStatusException, PreparationException { + throws TransactionException { // Arrange transaction.prepare(); when(crud.getSnapshot()).thenReturn(snapshot); @@ -809,6 +912,7 @@ public void rollback_CommittedStateReturnedByAbortState_ShouldThrowRollbackExcep // Act Assert assertThatThrownBy(transaction::rollback).isInstanceOf(RollbackException.class); + verify(crud).closeScanners(); verify(commit, never()).rollbackRecords(snapshot); } } diff --git a/integration-test/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitIntegrationTestBase.java b/integration-test/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitIntegrationTestBase.java index 9854894ac4..bf5abaae05 100644 --- a/integration-test/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitIntegrationTestBase.java +++ b/integration-test/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitIntegrationTestBase.java @@ -15,7 +15,6 @@ import com.scalar.db.io.Key; import java.util.Optional; import java.util.Properties; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; public abstract class ConsensusCommitIntegrationTestBase @@ -930,14 +929,4 @@ public void deleteAndDelete_forSameRecord_shouldWorkCorrectly() throws Transacti Optional optResult = get(prepareGet(0, 0)); assertThat(optResult).isNotPresent(); } - - @Disabled("Implement later") - @Override - @Test - public void getScanner_ScanGivenForCommittedRecord_ShouldReturnRecords() {} - - @Disabled("Implement later") - @Override - @Test - public void manager_getScanner_ScanGivenForCommittedRecord_ShouldReturnRecords() {} } diff --git a/integration-test/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitSpecificIntegrationTestBase.java b/integration-test/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitSpecificIntegrationTestBase.java index d552d704e3..e4a31e9673 100644 --- a/integration-test/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitSpecificIntegrationTestBase.java +++ b/integration-test/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitSpecificIntegrationTestBase.java @@ -32,6 +32,7 @@ import com.scalar.db.api.ScanAll; import com.scalar.db.api.Selection; import com.scalar.db.api.TableMetadata; +import com.scalar.db.api.TransactionCrudOperable; import com.scalar.db.api.TransactionState; import com.scalar.db.api.Update; import com.scalar.db.config.DatabaseConfig; @@ -4184,6 +4185,220 @@ public void get_GetWithIndexGiven_WithSerializable_ShouldNotThrowAnyException() assertThatCode(transaction::commit).doesNotThrowAnyException(); } + @Test + public void getScanner_WithSerializable_ShouldNotThrowAnyException() throws TransactionException { + // Arrange + manager.mutate( + Arrays.asList( + Insert.newBuilder() + .namespace(namespace1) + .table(TABLE_1) + .partitionKey(Key.ofInt(ACCOUNT_ID, 0)) + .clusteringKey(Key.ofInt(ACCOUNT_TYPE, 0)) + .intValue(BALANCE, INITIAL_BALANCE) + .build(), + Insert.newBuilder() + .namespace(namespace1) + .table(TABLE_1) + .partitionKey(Key.ofInt(ACCOUNT_ID, 0)) + .clusteringKey(Key.ofInt(ACCOUNT_TYPE, 1)) + .intValue(BALANCE, INITIAL_BALANCE) + .build(), + Insert.newBuilder() + .namespace(namespace1) + .table(TABLE_1) + .partitionKey(Key.ofInt(ACCOUNT_ID, 0)) + .clusteringKey(Key.ofInt(ACCOUNT_TYPE, 2)) + .intValue(BALANCE, INITIAL_BALANCE) + .build())); + + Scan scan = prepareScan(0, namespace1, TABLE_1); + DistributedTransaction transaction = manager.begin(Isolation.SERIALIZABLE); + + // Act Assert + TransactionCrudOperable.Scanner scanner = transaction.getScanner(scan); + Optional result1 = scanner.one(); + assertThat(result1).isNotEmpty(); + assertThat(result1.get().getInt(ACCOUNT_ID)).isEqualTo(0); + assertThat(result1.get().getInt(ACCOUNT_TYPE)).isEqualTo(0); + assertThat(result1.get().getInt(BALANCE)).isEqualTo(INITIAL_BALANCE); + + Optional result2 = scanner.one(); + assertThat(result2).isNotEmpty(); + assertThat(result2.get().getInt(ACCOUNT_ID)).isEqualTo(0); + assertThat(result2.get().getInt(ACCOUNT_TYPE)).isEqualTo(1); + assertThat(result2.get().getInt(BALANCE)).isEqualTo(INITIAL_BALANCE); + + scanner.close(); + + assertThatCode(transaction::commit).doesNotThrowAnyException(); + } + + @Test + public void + getScanner_FirstInsertedRecordByAnotherTransaction_WithSerializable_ShouldNotThrowCommitConflictException() + throws TransactionException { + // Arrange + manager.mutate( + Arrays.asList( + Insert.newBuilder() + .namespace(namespace1) + .table(TABLE_1) + .partitionKey(Key.ofInt(ACCOUNT_ID, 0)) + .clusteringKey(Key.ofInt(ACCOUNT_TYPE, 1)) + .intValue(BALANCE, INITIAL_BALANCE) + .build(), + Insert.newBuilder() + .namespace(namespace1) + .table(TABLE_1) + .partitionKey(Key.ofInt(ACCOUNT_ID, 0)) + .clusteringKey(Key.ofInt(ACCOUNT_TYPE, 2)) + .intValue(BALANCE, INITIAL_BALANCE) + .build())); + + Scan scan = prepareScan(0, namespace1, TABLE_1); + DistributedTransaction transaction = manager.begin(Isolation.SERIALIZABLE); + + // Act Assert + TransactionCrudOperable.Scanner scanner = transaction.getScanner(scan); + Optional result1 = scanner.one(); + assertThat(result1).isNotEmpty(); + assertThat(result1.get().getInt(ACCOUNT_ID)).isEqualTo(0); + assertThat(result1.get().getInt(ACCOUNT_TYPE)).isEqualTo(1); + assertThat(result1.get().getInt(BALANCE)).isEqualTo(INITIAL_BALANCE); + + Optional result2 = scanner.one(); + assertThat(result2).isNotEmpty(); + assertThat(result2.get().getInt(ACCOUNT_ID)).isEqualTo(0); + assertThat(result2.get().getInt(ACCOUNT_TYPE)).isEqualTo(2); + assertThat(result2.get().getInt(BALANCE)).isEqualTo(INITIAL_BALANCE); + + scanner.close(); + + DistributedTransaction another = manager.begin(Isolation.SERIALIZABLE); + another.insert( + Insert.newBuilder() + .namespace(namespace1) + .table(TABLE_1) + .partitionKey(Key.ofInt(ACCOUNT_ID, 0)) + .clusteringKey(Key.ofInt(ACCOUNT_TYPE, 0)) + .intValue(BALANCE, INITIAL_BALANCE) + .build()); + another.commit(); + + assertThatThrownBy(transaction::commit).isInstanceOf(CommitConflictException.class); + } + + @Test + public void + getScanner_RecordInsertedByAnotherTransaction_WithSerializable_ShouldNotThrowAnyException() + throws TransactionException { + // Arrange + manager.mutate( + Arrays.asList( + Insert.newBuilder() + .namespace(namespace1) + .table(TABLE_1) + .partitionKey(Key.ofInt(ACCOUNT_ID, 0)) + .clusteringKey(Key.ofInt(ACCOUNT_TYPE, 0)) + .intValue(BALANCE, INITIAL_BALANCE) + .build(), + Insert.newBuilder() + .namespace(namespace1) + .table(TABLE_1) + .partitionKey(Key.ofInt(ACCOUNT_ID, 0)) + .clusteringKey(Key.ofInt(ACCOUNT_TYPE, 1)) + .intValue(BALANCE, INITIAL_BALANCE) + .build())); + + Scan scan = prepareScan(0, namespace1, TABLE_1); + DistributedTransaction transaction = manager.begin(Isolation.SERIALIZABLE); + + // Act Assert + TransactionCrudOperable.Scanner scanner = transaction.getScanner(scan); + Optional result1 = scanner.one(); + assertThat(result1).isNotEmpty(); + assertThat(result1.get().getInt(ACCOUNT_ID)).isEqualTo(0); + assertThat(result1.get().getInt(ACCOUNT_TYPE)).isEqualTo(0); + assertThat(result1.get().getInt(BALANCE)).isEqualTo(INITIAL_BALANCE); + + Optional result2 = scanner.one(); + assertThat(result2).isNotEmpty(); + assertThat(result2.get().getInt(ACCOUNT_ID)).isEqualTo(0); + assertThat(result2.get().getInt(ACCOUNT_TYPE)).isEqualTo(1); + assertThat(result2.get().getInt(BALANCE)).isEqualTo(INITIAL_BALANCE); + + scanner.close(); + + DistributedTransaction another = manager.begin(Isolation.SERIALIZABLE); + another.insert( + Insert.newBuilder() + .namespace(namespace1) + .table(TABLE_1) + .partitionKey(Key.ofInt(ACCOUNT_ID, 0)) + .clusteringKey(Key.ofInt(ACCOUNT_TYPE, 2)) + .intValue(BALANCE, INITIAL_BALANCE) + .build()); + another.commit(); + + assertThatCode(transaction::commit).doesNotThrowAnyException(); + } + + @Test + public void + getScanner_RecordUpdatedByAnotherTransaction_WithSerializable_ShouldThrowCommitConflictException() + throws TransactionException { + // Arrange + manager.mutate( + Arrays.asList( + Insert.newBuilder() + .namespace(namespace1) + .table(TABLE_1) + .partitionKey(Key.ofInt(ACCOUNT_ID, 0)) + .clusteringKey(Key.ofInt(ACCOUNT_TYPE, 0)) + .intValue(BALANCE, INITIAL_BALANCE) + .build(), + Insert.newBuilder() + .namespace(namespace1) + .table(TABLE_1) + .partitionKey(Key.ofInt(ACCOUNT_ID, 0)) + .clusteringKey(Key.ofInt(ACCOUNT_TYPE, 1)) + .intValue(BALANCE, INITIAL_BALANCE) + .build())); + + Scan scan = prepareScan(0, namespace1, TABLE_1); + DistributedTransaction transaction = manager.begin(Isolation.SERIALIZABLE); + + // Act Assert + TransactionCrudOperable.Scanner scanner = transaction.getScanner(scan); + Optional result1 = scanner.one(); + assertThat(result1).isNotEmpty(); + assertThat(result1.get().getInt(ACCOUNT_ID)).isEqualTo(0); + assertThat(result1.get().getInt(ACCOUNT_TYPE)).isEqualTo(0); + assertThat(result1.get().getInt(BALANCE)).isEqualTo(INITIAL_BALANCE); + + Optional result2 = scanner.one(); + assertThat(result2).isNotEmpty(); + assertThat(result2.get().getInt(ACCOUNT_ID)).isEqualTo(0); + assertThat(result2.get().getInt(ACCOUNT_TYPE)).isEqualTo(1); + assertThat(result2.get().getInt(BALANCE)).isEqualTo(INITIAL_BALANCE); + + scanner.close(); + + DistributedTransaction another = manager.begin(Isolation.SERIALIZABLE); + another.update( + Update.newBuilder() + .namespace(namespace1) + .table(TABLE_1) + .partitionKey(Key.ofInt(ACCOUNT_ID, 0)) + .clusteringKey(Key.ofInt(ACCOUNT_TYPE, 0)) + .intValue(BALANCE, 0) + .build()); + another.commit(); + + assertThatThrownBy(transaction::commit).isInstanceOf(CommitConflictException.class); + } + @Test public void get_GetWithIndexGiven_NoRecordsInIndexRange_WithSerializable_ShouldNotThrowAnyException() diff --git a/integration-test/src/main/java/com/scalar/db/transaction/consensuscommit/TwoPhaseConsensusCommitIntegrationTestBase.java b/integration-test/src/main/java/com/scalar/db/transaction/consensuscommit/TwoPhaseConsensusCommitIntegrationTestBase.java index d57c31302c..781087ed79 100644 --- a/integration-test/src/main/java/com/scalar/db/transaction/consensuscommit/TwoPhaseConsensusCommitIntegrationTestBase.java +++ b/integration-test/src/main/java/com/scalar/db/transaction/consensuscommit/TwoPhaseConsensusCommitIntegrationTestBase.java @@ -2,8 +2,6 @@ import com.scalar.db.api.TwoPhaseCommitTransactionIntegrationTestBase; import java.util.Properties; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Test; public abstract class TwoPhaseConsensusCommitIntegrationTestBase extends TwoPhaseCommitTransactionIntegrationTestBase { @@ -40,14 +38,4 @@ protected final Properties getProperties2(String testName) { protected Properties getProps2(String testName) { return getProps1(testName); } - - @Disabled("Implement later") - @Override - @Test - public void getScanner_ScanGivenForCommittedRecord_ShouldReturnRecords() {} - - @Disabled("Implement later") - @Override - @Test - public void manager_getScanner_ScanGivenForCommittedRecord_ShouldReturnRecords() {} } From 9ad1219440ba9b0ad09d2ea167448bf7b013f507 Mon Sep 17 00:00:00 2001 From: Toshihiro Suzuki Date: Tue, 3 Jun 2025 17:15:28 +0900 Subject: [PATCH 05/19] Add more integration tests for scanner API (#2724) --- .../ConsensusCommitCassandraEnv.java | 2 +- .../cosmos/ConsensusCommitCosmosEnv.java | 2 +- .../dynamo/ConsensusCommitDynamoEnv.java | 2 +- .../storage/jdbc/ConsensusCommitJdbcEnv.java | 2 +- ...tadataIntegrationTestWithMultiStorage.java | 2 +- ...ecificIntegrationTestWithMultiStorage.java | 2 +- ...ributedTransactionIntegrationTestBase.java | 202 ++++++++---- ...eCommitTransactionIntegrationTestBase.java | 290 ++++++++++++------ .../ConsensusCommitTestUtils.java | 2 +- ...erationTransactionIntegrationTestBase.java | 87 ++++-- 10 files changed, 418 insertions(+), 175 deletions(-) rename {core/src/integration-test/java/com/scalar/db/common => integration-test/src/main/java/com/scalar/db/transaction/consensuscommit}/ConsensusCommitTestUtils.java (98%) diff --git a/core/src/integration-test/java/com/scalar/db/storage/cassandra/ConsensusCommitCassandraEnv.java b/core/src/integration-test/java/com/scalar/db/storage/cassandra/ConsensusCommitCassandraEnv.java index d714c83e05..31e8d585e1 100644 --- a/core/src/integration-test/java/com/scalar/db/storage/cassandra/ConsensusCommitCassandraEnv.java +++ b/core/src/integration-test/java/com/scalar/db/storage/cassandra/ConsensusCommitCassandraEnv.java @@ -1,7 +1,7 @@ package com.scalar.db.storage.cassandra; -import com.scalar.db.common.ConsensusCommitTestUtils; import com.scalar.db.transaction.consensuscommit.ConsensusCommitIntegrationTestUtils; +import com.scalar.db.transaction.consensuscommit.ConsensusCommitTestUtils; import java.util.Properties; public final class ConsensusCommitCassandraEnv { diff --git a/core/src/integration-test/java/com/scalar/db/storage/cosmos/ConsensusCommitCosmosEnv.java b/core/src/integration-test/java/com/scalar/db/storage/cosmos/ConsensusCommitCosmosEnv.java index a5738526ee..d9b61a2be8 100644 --- a/core/src/integration-test/java/com/scalar/db/storage/cosmos/ConsensusCommitCosmosEnv.java +++ b/core/src/integration-test/java/com/scalar/db/storage/cosmos/ConsensusCommitCosmosEnv.java @@ -1,7 +1,7 @@ package com.scalar.db.storage.cosmos; -import com.scalar.db.common.ConsensusCommitTestUtils; import com.scalar.db.transaction.consensuscommit.ConsensusCommitIntegrationTestUtils; +import com.scalar.db.transaction.consensuscommit.ConsensusCommitTestUtils; import java.util.Map; import java.util.Properties; diff --git a/core/src/integration-test/java/com/scalar/db/storage/dynamo/ConsensusCommitDynamoEnv.java b/core/src/integration-test/java/com/scalar/db/storage/dynamo/ConsensusCommitDynamoEnv.java index 644785cec4..dd636cae75 100644 --- a/core/src/integration-test/java/com/scalar/db/storage/dynamo/ConsensusCommitDynamoEnv.java +++ b/core/src/integration-test/java/com/scalar/db/storage/dynamo/ConsensusCommitDynamoEnv.java @@ -1,7 +1,7 @@ package com.scalar.db.storage.dynamo; -import com.scalar.db.common.ConsensusCommitTestUtils; import com.scalar.db.transaction.consensuscommit.ConsensusCommitIntegrationTestUtils; +import com.scalar.db.transaction.consensuscommit.ConsensusCommitTestUtils; import java.util.Map; import java.util.Properties; diff --git a/core/src/integration-test/java/com/scalar/db/storage/jdbc/ConsensusCommitJdbcEnv.java b/core/src/integration-test/java/com/scalar/db/storage/jdbc/ConsensusCommitJdbcEnv.java index e098a847e9..a1650f8cc5 100644 --- a/core/src/integration-test/java/com/scalar/db/storage/jdbc/ConsensusCommitJdbcEnv.java +++ b/core/src/integration-test/java/com/scalar/db/storage/jdbc/ConsensusCommitJdbcEnv.java @@ -1,7 +1,7 @@ package com.scalar.db.storage.jdbc; -import com.scalar.db.common.ConsensusCommitTestUtils; import com.scalar.db.transaction.consensuscommit.ConsensusCommitIntegrationTestUtils; +import com.scalar.db.transaction.consensuscommit.ConsensusCommitTestUtils; import java.util.Properties; public final class ConsensusCommitJdbcEnv { diff --git a/core/src/integration-test/java/com/scalar/db/storage/multistorage/ConsensusCommitNullMetadataIntegrationTestWithMultiStorage.java b/core/src/integration-test/java/com/scalar/db/storage/multistorage/ConsensusCommitNullMetadataIntegrationTestWithMultiStorage.java index 7f8355fc49..ab58f990d4 100644 --- a/core/src/integration-test/java/com/scalar/db/storage/multistorage/ConsensusCommitNullMetadataIntegrationTestWithMultiStorage.java +++ b/core/src/integration-test/java/com/scalar/db/storage/multistorage/ConsensusCommitNullMetadataIntegrationTestWithMultiStorage.java @@ -1,9 +1,9 @@ package com.scalar.db.storage.multistorage; -import com.scalar.db.common.ConsensusCommitTestUtils; import com.scalar.db.config.DatabaseConfig; import com.scalar.db.storage.cassandra.CassandraAdmin; import com.scalar.db.transaction.consensuscommit.ConsensusCommitNullMetadataIntegrationTestBase; +import com.scalar.db.transaction.consensuscommit.ConsensusCommitTestUtils; import com.scalar.db.transaction.consensuscommit.Coordinator; import java.util.Collections; import java.util.Map; diff --git a/core/src/integration-test/java/com/scalar/db/storage/multistorage/ConsensusCommitSpecificIntegrationTestWithMultiStorage.java b/core/src/integration-test/java/com/scalar/db/storage/multistorage/ConsensusCommitSpecificIntegrationTestWithMultiStorage.java index 9938be190b..75c1b74901 100644 --- a/core/src/integration-test/java/com/scalar/db/storage/multistorage/ConsensusCommitSpecificIntegrationTestWithMultiStorage.java +++ b/core/src/integration-test/java/com/scalar/db/storage/multistorage/ConsensusCommitSpecificIntegrationTestWithMultiStorage.java @@ -1,9 +1,9 @@ package com.scalar.db.storage.multistorage; -import com.scalar.db.common.ConsensusCommitTestUtils; import com.scalar.db.config.DatabaseConfig; import com.scalar.db.storage.cassandra.CassandraAdmin; import com.scalar.db.transaction.consensuscommit.ConsensusCommitSpecificIntegrationTestBase; +import com.scalar.db.transaction.consensuscommit.ConsensusCommitTestUtils; import com.scalar.db.transaction.consensuscommit.Coordinator; import java.util.Collections; import java.util.Map; diff --git a/integration-test/src/main/java/com/scalar/db/api/DistributedTransactionIntegrationTestBase.java b/integration-test/src/main/java/com/scalar/db/api/DistributedTransactionIntegrationTestBase.java index 5e3c9910e0..2577ca9163 100644 --- a/integration-test/src/main/java/com/scalar/db/api/DistributedTransactionIntegrationTestBase.java +++ b/integration-test/src/main/java/com/scalar/db/api/DistributedTransactionIntegrationTestBase.java @@ -57,6 +57,8 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInstance; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -273,15 +275,17 @@ public void get_GetWithUnmatchedConjunctionsGivenForCommittedRecord_ShouldReturn assertThat(result.isPresent()).isFalse(); } - @Test - public void scan_ScanGivenForCommittedRecord_ShouldReturnRecords() throws TransactionException { + @ParameterizedTest + @EnumSource(ScanType.class) + public void scanOrGetScanner_ScanGivenForCommittedRecord_ShouldReturnRecords(ScanType scanType) + throws TransactionException { // Arrange populateRecords(); DistributedTransaction transaction = manager.start(); Scan scan = prepareScan(1, 0, 2); // Act - List results = transaction.scan(scan); + List results = scanOrGetScanner(transaction, scan, scanType); transaction.commit(); // Assert @@ -291,9 +295,10 @@ public void scan_ScanGivenForCommittedRecord_ShouldReturnRecords() throws Transa assertResult(1, 2, results.get(2)); } - @Test - public void scan_ScanWithConjunctionsGivenForCommittedRecord_ShouldReturnRecords() - throws TransactionException { + @ParameterizedTest + @EnumSource(ScanType.class) + public void scanOrGetScanner_ScanWithConjunctionsGivenForCommittedRecord_ShouldReturnRecords( + ScanType scanType) throws TransactionException { // Arrange populateRecords(); DistributedTransaction transaction = manager.start(); @@ -303,7 +308,7 @@ public void scan_ScanWithConjunctionsGivenForCommittedRecord_ShouldReturnRecords .build(); // Act - List results = transaction.scan(scan); + List results = scanOrGetScanner(transaction, scan, scanType); transaction.commit(); // Assert @@ -320,9 +325,10 @@ public void scan_ScanWithConjunctionsGivenForCommittedRecord_ShouldReturnRecords assertThat(results.get(1).getInt(SOME_COLUMN)).isEqualTo(2); } - @Test - public void scan_ScanWithProjectionsGivenForCommittedRecord_ShouldReturnRecords() - throws TransactionException { + @ParameterizedTest + @EnumSource(ScanType.class) + public void scanOrGetScanner_ScanWithProjectionsGivenForCommittedRecord_ShouldReturnRecords( + ScanType scanType) throws TransactionException { // Arrange populateRecords(); DistributedTransaction transaction = manager.start(); @@ -333,7 +339,7 @@ public void scan_ScanWithProjectionsGivenForCommittedRecord_ShouldReturnRecords( .withProjection(BALANCE); // Act - List results = transaction.scan(scan); + List results = scanOrGetScanner(transaction, scan, scanType); transaction.commit(); // Assert @@ -354,16 +360,17 @@ public void scan_ScanWithProjectionsGivenForCommittedRecord_ShouldReturnRecords( assertThat(results.get(2).getInt(ACCOUNT_TYPE)).isEqualTo(2); } - @Test - public void scan_ScanWithOrderingGivenForCommittedRecord_ShouldReturnRecords() - throws TransactionException { + @ParameterizedTest + @EnumSource(ScanType.class) + public void scanOrGetScanner_ScanWithOrderingGivenForCommittedRecord_ShouldReturnRecords( + ScanType scanType) throws TransactionException { // Arrange populateRecords(); DistributedTransaction transaction = manager.start(); Scan scan = prepareScan(1, 0, 2).withOrdering(Ordering.desc(ACCOUNT_TYPE)); // Act - List results = transaction.scan(scan); + List results = scanOrGetScanner(transaction, scan, scanType); transaction.commit(); // Assert @@ -384,16 +391,17 @@ public void scan_ScanWithOrderingGivenForCommittedRecord_ShouldReturnRecords() assertThat(results.get(2).getInt(SOME_COLUMN)).isEqualTo(0); } - @Test - public void scan_ScanWithLimitGivenForCommittedRecord_ShouldReturnRecords() - throws TransactionException { + @ParameterizedTest + @EnumSource(ScanType.class) + public void scanOrGetScanner_ScanWithLimitGivenForCommittedRecord_ShouldReturnRecords( + ScanType scanType) throws TransactionException { // Arrange populateRecords(); DistributedTransaction transaction = manager.start(); Scan scan = prepareScan(1, 0, 2).withLimit(2); // Act - List results = transaction.scan(scan); + List results = scanOrGetScanner(transaction, scan, scanType); transaction.commit(); // Assert @@ -424,15 +432,17 @@ public void get_GetGivenForNonExisting_ShouldReturnEmpty() throws TransactionExc assertThat(result.isPresent()).isFalse(); } - @Test - public void scan_ScanGivenForNonExisting_ShouldReturnEmpty() throws TransactionException { + @ParameterizedTest + @EnumSource(ScanType.class) + public void scanOrGetScanner_ScanGivenForNonExisting_ShouldReturnEmpty(ScanType scanType) + throws TransactionException { // Arrange populateRecords(); DistributedTransaction transaction = manager.start(); Scan scan = prepareScan(0, 4, 6); // Act - List results = transaction.scan(scan); + List results = scanOrGetScanner(transaction, scan, scanType); transaction.commit(); // Assert @@ -484,8 +494,10 @@ public void get_GetGivenForIndexColumn_ShouldReturnRecords() throws TransactionE assertThat(result2).isEqualTo(result1); } - @Test - public void scan_ScanGivenForIndexColumn_ShouldReturnRecords() throws TransactionException { + @ParameterizedTest + @EnumSource(ScanType.class) + public void scanOrGetScanner_ScanGivenForIndexColumn_ShouldReturnRecords(ScanType scanType) + throws TransactionException { // Arrange populateRecords(); DistributedTransaction transaction = manager.start(); @@ -517,8 +529,8 @@ public void scan_ScanGivenForIndexColumn_ShouldReturnRecords() throws Transactio .build()); // Act - List results1 = transaction.scan(scanBuiltByConstructor); - List results2 = transaction.scan(scanBuiltByBuilder); + List results1 = scanOrGetScanner(transaction, scanBuiltByConstructor, scanType); + List results2 = scanOrGetScanner(transaction, scanBuiltByBuilder, scanType); transaction.commit(); // Assert @@ -526,9 +538,10 @@ public void scan_ScanGivenForIndexColumn_ShouldReturnRecords() throws Transactio TestUtils.assertResultsContainsExactlyInAnyOrder(results2, expectedResults); } - @Test - public void scan_ScanGivenForIndexColumnWithConjunctions_ShouldReturnRecords() - throws TransactionException { + @ParameterizedTest + @EnumSource(ScanType.class) + public void scanOrGetScanner_ScanGivenForIndexColumnWithConjunctions_ShouldReturnRecords( + ScanType scanType) throws TransactionException { // Arrange populateRecords(); DistributedTransaction transaction = manager.start(); @@ -541,7 +554,7 @@ public void scan_ScanGivenForIndexColumnWithConjunctions_ShouldReturnRecords() .build(); // Act - List results = transaction.scan(scan); + List results = scanOrGetScanner(transaction, scan, scanType); transaction.commit(); // Assert @@ -554,8 +567,9 @@ public void scan_ScanGivenForIndexColumnWithConjunctions_ShouldReturnRecords() assertThat(results.get(0).getInt(SOME_COLUMN)).isEqualTo(6); } - @Test - public void scan_ScanAllGivenForCommittedRecord_ShouldReturnRecords() + @ParameterizedTest + @EnumSource(ScanType.class) + public void scanOrGetScanner_ScanAllGivenForCommittedRecord_ShouldReturnRecords(ScanType scanType) throws TransactionException { // Arrange populateRecords(); @@ -563,7 +577,7 @@ public void scan_ScanAllGivenForCommittedRecord_ShouldReturnRecords() ScanAll scanAll = prepareScanAll(); // Act - List results = transaction.scan(scanAll); + List results = scanOrGetScanner(transaction, scanAll, scanType); transaction.commit(); // Assert @@ -583,9 +597,10 @@ public void scan_ScanAllGivenForCommittedRecord_ShouldReturnRecords() TestUtils.assertResultsContainsExactlyInAnyOrder(results, expectedResults); } - @Test - public void scan_ScanAllGivenWithLimit_ShouldReturnLimitedAmountOfRecords() - throws TransactionException { + @ParameterizedTest + @EnumSource(ScanType.class) + public void scanOrGetScanner_ScanAllGivenWithLimit_ShouldReturnLimitedAmountOfRecords( + ScanType scanType) throws TransactionException { // Arrange insert(prepareInsert(1, 1), prepareInsert(1, 2), prepareInsert(2, 1), prepareInsert(3, 0)); @@ -593,7 +608,7 @@ public void scan_ScanAllGivenWithLimit_ShouldReturnLimitedAmountOfRecords() ScanAll scanAll = prepareScanAll().withLimit(2); // Act - List results = scanAllTransaction.scan(scanAll); + List results = scanOrGetScanner(scanAllTransaction, scanAll, scanType); scanAllTransaction.commit(); // Assert @@ -623,16 +638,17 @@ public void scan_ScanAllGivenWithLimit_ShouldReturnLimitedAmountOfRecords() assertThat(results).hasSize(2); } - @Test - public void scan_ScanAllWithProjectionsGiven_ShouldRetrieveSpecifiedValues() - throws TransactionException { + @ParameterizedTest + @EnumSource(ScanType.class) + public void scanOrGetScanner_ScanAllWithProjectionsGiven_ShouldRetrieveSpecifiedValues( + ScanType scanType) throws TransactionException { // Arrange populateRecords(); DistributedTransaction transaction = manager.start(); ScanAll scanAll = prepareScanAll().withProjection(ACCOUNT_TYPE).withProjection(BALANCE); // Act - List results = transaction.scan(scanAll); + List results = scanOrGetScanner(transaction, scanAll, scanType); transaction.commit(); // Assert @@ -651,14 +667,16 @@ public void scan_ScanAllWithProjectionsGiven_ShouldRetrieveSpecifiedValues() TestUtils.assertResultsContainsExactlyInAnyOrder(results, expectedResults); } - @Test - public void scan_ScanAllGivenForNonExisting_ShouldReturnEmpty() throws TransactionException { + @ParameterizedTest + @EnumSource(ScanType.class) + public void scanOrGetScanner_ScanAllGivenForNonExisting_ShouldReturnEmpty(ScanType scanType) + throws TransactionException { // Arrange DistributedTransaction transaction = manager.start(); ScanAll scanAll = prepareScanAll(); // Act - List results = transaction.scan(scanAll); + List results = scanOrGetScanner(transaction, scanAll, scanType); transaction.commit(); // Assert @@ -1048,17 +1066,18 @@ public void rollback_forOngoingTransaction_ShouldRollbackCorrectly() throws Tran assertThat(result.get().isNull(SOME_COLUMN)).isTrue(); } - @Test + @ParameterizedTest + @EnumSource(ScanType.class) public void - scan_ScanWithProjectionsGivenOnNonPrimaryKeyColumnsForCommittedRecord_ShouldReturnOnlyProjectedColumns() - throws TransactionException { + scanOrGetScanner_ScanWithProjectionsGivenOnNonPrimaryKeyColumnsForCommittedRecord_ShouldReturnOnlyProjectedColumns( + ScanType scanType) throws TransactionException { // Arrange populateSingleRecord(); DistributedTransaction transaction = manager.begin(); Scan scan = prepareScan(0, 0, 0).withProjections(Arrays.asList(BALANCE, SOME_COLUMN)); // Act - List results = transaction.scan(scan); + List results = scanOrGetScanner(transaction, scan, scanType); transaction.commit(); // Assert @@ -1070,17 +1089,18 @@ public void rollback_forOngoingTransaction_ShouldRollbackCorrectly() throws Tran }); } - @Test + @ParameterizedTest + @EnumSource(ScanType.class) public void - scan_ScanAllWithProjectionsGivenOnNonPrimaryKeyColumnsForCommittedRecord_ShouldReturnOnlyProjectedColumns() - throws TransactionException { + scanOrGetScanner_ScanAllWithProjectionsGivenOnNonPrimaryKeyColumnsForCommittedRecord_ShouldReturnOnlyProjectedColumns( + ScanType scanType) throws TransactionException { // Arrange populateSingleRecord(); DistributedTransaction transaction = manager.begin(); ScanAll scanAll = prepareScanAll().withProjections(Arrays.asList(BALANCE, SOME_COLUMN)); // Act - List results = transaction.scan(scanAll); + List results = scanOrGetScanner(transaction, scanAll, scanType); transaction.commit(); // Assert @@ -1213,6 +1233,29 @@ public void scan_DefaultNamespaceGiven_ShouldWorkProperly() throws TransactionEx } } + @Test + public void getScanner_DefaultNamespaceGiven_ShouldWorkProperly() throws TransactionException { + Properties properties = getProperties(getTestName()); + properties.put(DatabaseConfig.DEFAULT_NAMESPACE_NAME, namespace); + try (DistributedTransactionManager managerWithDefaultNamespace = + TransactionFactory.create(properties).getTransactionManager()) { + // Arrange + populateRecords(); + Scan scan = Scan.newBuilder().table(TABLE).all().build(); + + // Act Assert + Assertions.assertThatCode( + () -> { + DistributedTransaction tx = managerWithDefaultNamespace.start(); + TransactionCrudOperable.Scanner scanner = tx.getScanner(scan); + scanner.all(); + scanner.close(); + tx.commit(); + }) + .doesNotThrowAnyException(); + } + } + @Test public void put_DefaultNamespaceGiven_ShouldWorkProperly() throws TransactionException { Properties properties = getProperties(getTestName()); @@ -2310,6 +2353,30 @@ public void manager_scan_DefaultNamespaceGiven_ShouldWorkProperly() throws Trans } } + @Test + public void manager_getScanner_DefaultNamespaceGiven_ShouldWorkProperly() + throws TransactionException { + // Arrange + Properties properties = getProperties(getTestName()); + properties.put(DatabaseConfig.DEFAULT_NAMESPACE_NAME, namespace); + try (DistributedTransactionManager managerWithDefaultNamespace = + TransactionFactory.create(properties).getTransactionManager()) { + // Arrange + populateRecords(); + Scan scan = Scan.newBuilder().table(TABLE).all().build(); + + // Act Assert + Assertions.assertThatCode( + () -> { + TransactionManagerCrudOperable.Scanner scanner = + managerWithDefaultNamespace.getScanner(scan); + scanner.all(); + scanner.close(); + }) + .doesNotThrowAnyException(); + } + } + @Test public void manager_put_DefaultNamespaceGiven_ShouldWorkProperly() throws TransactionException { Properties properties = getProperties(getTestName()); @@ -2730,4 +2797,35 @@ protected List> prepareNonKeyColumns(int accountId, int accountType) { } return columns.build(); } + + protected List scanOrGetScanner( + DistributedTransaction transaction, Scan scan, ScanType scanType) throws CrudException { + if (scanType == ScanType.SCAN) { + return transaction.scan(scan); + } + + try (TransactionCrudOperable.Scanner scanner = transaction.getScanner(scan)) { + switch (scanType) { + case SCANNER_ONE: + List results = new ArrayList<>(); + while (true) { + Optional result = scanner.one(); + if (!result.isPresent()) { + return results; + } + results.add(result.get()); + } + case SCANNER_ALL: + return scanner.all(); + default: + throw new AssertionError(); + } + } + } + + public enum ScanType { + SCAN, + SCANNER_ONE, + SCANNER_ALL + } } diff --git a/integration-test/src/main/java/com/scalar/db/api/TwoPhaseCommitTransactionIntegrationTestBase.java b/integration-test/src/main/java/com/scalar/db/api/TwoPhaseCommitTransactionIntegrationTestBase.java index f6f7febcbe..c81bfe92d2 100644 --- a/integration-test/src/main/java/com/scalar/db/api/TwoPhaseCommitTransactionIntegrationTestBase.java +++ b/integration-test/src/main/java/com/scalar/db/api/TwoPhaseCommitTransactionIntegrationTestBase.java @@ -57,6 +57,8 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInstance; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -312,15 +314,17 @@ public void get_GetWithUnmatchedConjunctionsGivenForCommittedRecord_ShouldReturn assertThat(result.isPresent()).isFalse(); } - @Test - public void scan_ScanGivenForCommittedRecord_ShouldReturnRecords() throws TransactionException { + @ParameterizedTest + @EnumSource(ScanType.class) + public void scanOrGetScanner_ScanGivenForCommittedRecord_ShouldReturnRecords(ScanType scanType) + throws TransactionException { // Arrange populateRecords(manager1, namespace1, TABLE_1); TwoPhaseCommitTransaction transaction = manager1.start(); Scan scan = prepareScan(1, 0, 2, namespace1, TABLE_1); // Act - List results = transaction.scan(scan); + List results = scanOrGetScanner(transaction, scan, scanType); transaction.prepare(); transaction.validate(); transaction.commit(); @@ -332,9 +336,10 @@ public void scan_ScanGivenForCommittedRecord_ShouldReturnRecords() throws Transa assertResult(1, 2, results.get(2)); } - @Test - public void scan_ScanWithConjunctionsGivenForCommittedRecord_ShouldReturnRecords() - throws TransactionException { + @ParameterizedTest + @EnumSource(ScanType.class) + public void scanOrGetScanner_ScanWithConjunctionsGivenForCommittedRecord_ShouldReturnRecords( + ScanType scanType) throws TransactionException { // Arrange populateRecords(manager1, namespace1, TABLE_1); TwoPhaseCommitTransaction transaction = manager1.start(); @@ -344,7 +349,7 @@ public void scan_ScanWithConjunctionsGivenForCommittedRecord_ShouldReturnRecords .build(); // Act - List results = transaction.scan(scan); + List results = scanOrGetScanner(transaction, scan, scanType); transaction.prepare(); transaction.validate(); transaction.commit(); @@ -363,9 +368,10 @@ public void scan_ScanWithConjunctionsGivenForCommittedRecord_ShouldReturnRecords assertThat(results.get(1).getInt(SOME_COLUMN)).isEqualTo(2); } - @Test - public void scan_ScanWithProjectionsGivenForCommittedRecord_ShouldReturnRecords() - throws TransactionException { + @ParameterizedTest + @EnumSource(ScanType.class) + public void scanOrGetScanner_ScanWithProjectionsGivenForCommittedRecord_ShouldReturnRecords( + ScanType scanType) throws TransactionException { // Arrange populateRecords(manager1, namespace1, TABLE_1); TwoPhaseCommitTransaction transaction = manager1.start(); @@ -376,7 +382,7 @@ public void scan_ScanWithProjectionsGivenForCommittedRecord_ShouldReturnRecords( .withProjection(BALANCE); // Act - List results = transaction.scan(scan); + List results = scanOrGetScanner(transaction, scan, scanType); transaction.prepare(); transaction.validate(); transaction.commit(); @@ -399,16 +405,17 @@ public void scan_ScanWithProjectionsGivenForCommittedRecord_ShouldReturnRecords( assertThat(results.get(2).contains(SOME_COLUMN)).isFalse(); } - @Test - public void scan_ScanWithOrderingGivenForCommittedRecord_ShouldReturnRecords() - throws TransactionException { + @ParameterizedTest + @EnumSource(ScanType.class) + public void scanOrGetScanner_ScanWithOrderingGivenForCommittedRecord_ShouldReturnRecords( + ScanType scanType) throws TransactionException { // Arrange populateRecords(manager1, namespace1, TABLE_1); TwoPhaseCommitTransaction transaction = manager1.start(); Scan scan = prepareScan(1, 0, 2, namespace1, TABLE_1).withOrdering(Ordering.desc(ACCOUNT_TYPE)); // Act - List results = transaction.scan(scan); + List results = scanOrGetScanner(transaction, scan, scanType); transaction.prepare(); transaction.validate(); transaction.commit(); @@ -431,16 +438,17 @@ public void scan_ScanWithOrderingGivenForCommittedRecord_ShouldReturnRecords() assertThat(results.get(2).getInt(SOME_COLUMN)).isEqualTo(0); } - @Test - public void scan_ScanWithLimitGivenForCommittedRecord_ShouldReturnRecords() - throws TransactionException { + @ParameterizedTest + @EnumSource(ScanType.class) + public void scanOrGetScanner_ScanWithLimitGivenForCommittedRecord_ShouldReturnRecords( + ScanType scanType) throws TransactionException { // Arrange populateRecords(manager1, namespace1, TABLE_1); TwoPhaseCommitTransaction transaction = manager1.start(); Scan scan = prepareScan(1, 0, 2, namespace1, TABLE_1).withLimit(2); // Act - List results = transaction.scan(scan); + List results = scanOrGetScanner(transaction, scan, scanType); transaction.prepare(); transaction.validate(); transaction.commit(); @@ -475,15 +483,17 @@ public void get_GetGivenForNonExisting_ShouldReturnEmpty() throws TransactionExc assertThat(result.isPresent()).isFalse(); } - @Test - public void scan_ScanGivenForNonExisting_ShouldReturnEmpty() throws TransactionException { + @ParameterizedTest + @EnumSource(ScanType.class) + public void scanOrGetScanner_ScanGivenForNonExisting_ShouldReturnEmpty(ScanType scanType) + throws TransactionException { // Arrange populateRecords(manager1, namespace1, TABLE_1); TwoPhaseCommitTransaction transaction = manager1.start(); Scan scan = prepareScan(0, 4, 4, namespace1, TABLE_1); // Act - List results = transaction.scan(scan); + List results = scanOrGetScanner(transaction, scan, scanType); transaction.prepare(); transaction.validate(); transaction.commit(); @@ -541,8 +551,10 @@ public void get_GetGivenForIndexColumn_ShouldReturnRecords() throws TransactionE assertThat(result2).isEqualTo(result1); } - @Test - public void scan_ScanGivenForIndexColumn_ShouldReturnRecords() throws TransactionException { + @ParameterizedTest + @EnumSource(ScanType.class) + public void scanOrGetScanner_ScanGivenForIndexColumn_ShouldReturnRecords(ScanType scanType) + throws TransactionException { // Arrange populateRecords(manager1, namespace1, TABLE_1); TwoPhaseCommitTransaction transaction = manager1.start(); @@ -574,8 +586,8 @@ public void scan_ScanGivenForIndexColumn_ShouldReturnRecords() throws Transactio .build()); // Act - List results1 = transaction.scan(scanBuiltByConstructor); - List results2 = transaction.scan(scanBuiltByBuilder); + List results1 = scanOrGetScanner(transaction, scanBuiltByConstructor, scanType); + List results2 = scanOrGetScanner(transaction, scanBuiltByBuilder, scanType); transaction.prepare(); transaction.validate(); transaction.commit(); @@ -1118,8 +1130,9 @@ public void abort_forOngoingTransaction_ShouldAbortCorrectly() throws Transactio assertThat(state).isEqualTo(TransactionState.ABORTED); } - @Test - public void scan_ScanAllGivenForCommittedRecord_ShouldReturnRecords() + @ParameterizedTest + @EnumSource(ScanType.class) + public void scanOrGetScanner_ScanAllGivenForCommittedRecord_ShouldReturnRecords(ScanType scanType) throws TransactionException { // Arrange populateRecords(manager1, namespace1, TABLE_1); @@ -1127,7 +1140,7 @@ public void scan_ScanAllGivenForCommittedRecord_ShouldReturnRecords() ScanAll scanAll = prepareScanAll(namespace1, TABLE_1); // Act - List results = transaction.scan(scanAll); + List results = scanOrGetScanner(transaction, scanAll, scanType); transaction.prepare(); transaction.validate(); transaction.commit(); @@ -1149,9 +1162,10 @@ public void scan_ScanAllGivenForCommittedRecord_ShouldReturnRecords() TestUtils.assertResultsContainsExactlyInAnyOrder(results, expectedResults); } - @Test - public void scan_ScanAllGivenWithLimit_ShouldReturnLimitedAmountOfRecords() - throws TransactionException { + @ParameterizedTest + @EnumSource(ScanType.class) + public void scanOrGetScanner_ScanAllGivenWithLimit_ShouldReturnLimitedAmountOfRecords( + ScanType scanType) throws TransactionException { // Arrange TwoPhaseCommitTransaction putTransaction = manager1.begin(); insert( @@ -1167,7 +1181,7 @@ public void scan_ScanAllGivenWithLimit_ShouldReturnLimitedAmountOfRecords() ScanAll scanAll = prepareScanAll(namespace1, TABLE_1).withLimit(2); // Act - List results = scanAllTransaction.scan(scanAll); + List results = scanOrGetScanner(scanAllTransaction, scanAll, scanType); scanAllTransaction.prepare(); scanAllTransaction.validate(); scanAllTransaction.commit(); @@ -1199,9 +1213,10 @@ public void scan_ScanAllGivenWithLimit_ShouldReturnLimitedAmountOfRecords() assertThat(results).hasSize(2); } - @Test - public void scan_ScanAllWithProjectionsGiven_ShouldRetrieveSpecifiedValues() - throws TransactionException { + @ParameterizedTest + @EnumSource(ScanType.class) + public void scanOrGetScanner_ScanAllWithProjectionsGiven_ShouldRetrieveSpecifiedValues( + ScanType scanType) throws TransactionException { // Arrange populateRecords(manager1, namespace1, TABLE_1); TwoPhaseCommitTransaction transaction = manager1.begin(); @@ -1209,7 +1224,7 @@ public void scan_ScanAllWithProjectionsGiven_ShouldRetrieveSpecifiedValues() prepareScanAll(namespace1, TABLE_1).withProjection(ACCOUNT_TYPE).withProjection(BALANCE); // Act - List results = transaction.scan(scanAll); + List results = scanOrGetScanner(transaction, scanAll, scanType); transaction.prepare(); transaction.validate(); transaction.commit(); @@ -1235,14 +1250,16 @@ public void scan_ScanAllWithProjectionsGiven_ShouldRetrieveSpecifiedValues() }); } - @Test - public void scan_ScanAllGivenForNonExisting_ShouldReturnEmpty() throws TransactionException { + @ParameterizedTest + @EnumSource(ScanType.class) + public void scanOrGetScanner_ScanAllGivenForNonExisting_ShouldReturnEmpty(ScanType scanType) + throws TransactionException { // Arrange TwoPhaseCommitTransaction transaction = manager1.begin(); ScanAll scanAll = prepareScanAll(namespace1, TABLE_1); // Act - List results = transaction.scan(scanAll); + List results = scanOrGetScanner(transaction, scanAll, scanType); transaction.prepare(); transaction.validate(); transaction.commit(); @@ -1274,10 +1291,11 @@ public void scan_ScanAllGivenForNonExisting_ShouldReturnEmpty() throws Transacti assertThat(result.get().isNull(SOME_COLUMN)).isTrue(); } - @Test + @ParameterizedTest + @EnumSource(ScanType.class) public void - scan_ScanWithProjectionsGivenOnNonPrimaryKeyColumnsForCommittedRecord_ShouldReturnOnlyProjectedColumns() - throws TransactionException { + scanOrGetScanner_ScanWithProjectionsGivenOnNonPrimaryKeyColumnsForCommittedRecord_ShouldReturnOnlyProjectedColumns( + ScanType scanType) throws TransactionException { // Arrange TwoPhaseCommitTransaction transaction = manager1.begin(); populateSingleRecord(namespace1, TABLE_1); @@ -1286,7 +1304,7 @@ public void scan_ScanAllGivenForNonExisting_ShouldReturnEmpty() throws Transacti .withProjections(Arrays.asList(BALANCE, SOME_COLUMN)); // Act - List results = transaction.scan(scan); + List results = scanOrGetScanner(transaction, scan, scanType); transaction.prepare(); transaction.validate(); transaction.commit(); @@ -1300,10 +1318,11 @@ public void scan_ScanAllGivenForNonExisting_ShouldReturnEmpty() throws Transacti }); } - @Test + @ParameterizedTest + @EnumSource(ScanType.class) public void - scan_ScanAllWithProjectionsGivenOnNonPrimaryKeyColumnsForCommittedRecord_ShouldReturnOnlyProjectedColumns() - throws TransactionException { + scanOrGetScanner_ScanAllWithProjectionsGivenOnNonPrimaryKeyColumnsForCommittedRecord_ShouldReturnOnlyProjectedColumns( + ScanType scanType) throws TransactionException { // Arrange populateSingleRecord(namespace1, TABLE_1); TwoPhaseCommitTransaction transaction = manager1.begin(); @@ -1311,7 +1330,7 @@ public void scan_ScanAllGivenForNonExisting_ShouldReturnEmpty() throws Transacti prepareScanAll(namespace1, TABLE_1).withProjections(Arrays.asList(BALANCE, SOME_COLUMN)); // Act - List results = transaction.scan(scanAll); + List results = scanOrGetScanner(transaction, scanAll, scanType); transaction.prepare(); transaction.validate(); transaction.commit(); @@ -1407,8 +1426,8 @@ public void resume_WithBeginningAndCommittingTransaction_ShouldThrowTransactionN public void get_DefaultNamespaceGiven_ShouldWorkProperly() throws TransactionException { Properties properties = getProperties1(getTestName()); properties.put(DatabaseConfig.DEFAULT_NAMESPACE_NAME, namespace1); - try (DistributedTransactionManager managerWithDefaultNamespace = - TransactionFactory.create(properties).getTransactionManager()) { + try (TwoPhaseCommitTransactionManager managerWithDefaultNamespace = + TransactionFactory.create(properties).getTwoPhaseCommitTransactionManager()) { // Arrange populateRecords(manager1, namespace1, TABLE_1); Get get = @@ -1421,8 +1440,10 @@ public void get_DefaultNamespaceGiven_ShouldWorkProperly() throws TransactionExc // Act Assert Assertions.assertThatCode( () -> { - DistributedTransaction tx = managerWithDefaultNamespace.start(); + TwoPhaseCommitTransaction tx = managerWithDefaultNamespace.start(); tx.get(get); + tx.prepare(); + tx.validate(); tx.commit(); }) .doesNotThrowAnyException(); @@ -1433,8 +1454,8 @@ public void get_DefaultNamespaceGiven_ShouldWorkProperly() throws TransactionExc public void scan_DefaultNamespaceGiven_ShouldWorkProperly() throws TransactionException { Properties properties = getProperties1(getTestName()); properties.put(DatabaseConfig.DEFAULT_NAMESPACE_NAME, namespace1); - try (DistributedTransactionManager managerWithDefaultNamespace = - TransactionFactory.create(properties).getTransactionManager()) { + try (TwoPhaseCommitTransactionManager managerWithDefaultNamespace = + TransactionFactory.create(properties).getTwoPhaseCommitTransactionManager()) { // Arrange populateRecords(manager1, namespace1, TABLE_1); Scan scan = Scan.newBuilder().table(TABLE_1).all().build(); @@ -1442,8 +1463,35 @@ public void scan_DefaultNamespaceGiven_ShouldWorkProperly() throws TransactionEx // Act Assert Assertions.assertThatCode( () -> { - DistributedTransaction tx = managerWithDefaultNamespace.start(); + TwoPhaseCommitTransaction tx = managerWithDefaultNamespace.start(); tx.scan(scan); + tx.prepare(); + tx.validate(); + tx.commit(); + }) + .doesNotThrowAnyException(); + } + } + + @Test + public void getScanner_DefaultNamespaceGiven_ShouldWorkProperly() throws TransactionException { + Properties properties = getProperties1(getTestName()); + properties.put(DatabaseConfig.DEFAULT_NAMESPACE_NAME, namespace1); + try (TwoPhaseCommitTransactionManager managerWithDefaultNamespace = + TransactionFactory.create(properties).getTwoPhaseCommitTransactionManager()) { + // Arrange + populateRecords(manager1, namespace1, TABLE_1); + Scan scan = Scan.newBuilder().table(TABLE_1).all().build(); + + // Act Assert + Assertions.assertThatCode( + () -> { + TwoPhaseCommitTransaction tx = managerWithDefaultNamespace.start(); + TransactionCrudOperable.Scanner scanner = tx.getScanner(scan); + scanner.all(); + scanner.close(); + tx.prepare(); + tx.validate(); tx.commit(); }) .doesNotThrowAnyException(); @@ -1454,8 +1502,8 @@ public void scan_DefaultNamespaceGiven_ShouldWorkProperly() throws TransactionEx public void put_DefaultNamespaceGiven_ShouldWorkProperly() throws TransactionException { Properties properties = getProperties1(getTestName()); properties.put(DatabaseConfig.DEFAULT_NAMESPACE_NAME, namespace1); - try (DistributedTransactionManager managerWithDefaultNamespace = - TransactionFactory.create(properties).getTransactionManager()) { + try (TwoPhaseCommitTransactionManager managerWithDefaultNamespace = + TransactionFactory.create(properties).getTwoPhaseCommitTransactionManager()) { // Arrange populateRecords(manager1, namespace1, TABLE_1); Put put = @@ -1470,8 +1518,10 @@ public void put_DefaultNamespaceGiven_ShouldWorkProperly() throws TransactionExc // Act Assert Assertions.assertThatCode( () -> { - DistributedTransaction tx = managerWithDefaultNamespace.start(); + TwoPhaseCommitTransaction tx = managerWithDefaultNamespace.start(); tx.put(put); + tx.prepare(); + tx.validate(); tx.commit(); }) .doesNotThrowAnyException(); @@ -1482,8 +1532,8 @@ public void put_DefaultNamespaceGiven_ShouldWorkProperly() throws TransactionExc public void insert_DefaultNamespaceGiven_ShouldWorkProperly() throws TransactionException { Properties properties = getProperties1(getTestName()); properties.put(DatabaseConfig.DEFAULT_NAMESPACE_NAME, namespace1); - try (DistributedTransactionManager managerWithDefaultNamespace = - TransactionFactory.create(properties).getTransactionManager()) { + try (TwoPhaseCommitTransactionManager managerWithDefaultNamespace = + TransactionFactory.create(properties).getTwoPhaseCommitTransactionManager()) { // Arrange populateRecords(manager1, namespace1, TABLE_1); Insert insert = @@ -1497,8 +1547,10 @@ public void insert_DefaultNamespaceGiven_ShouldWorkProperly() throws Transaction // Act Assert Assertions.assertThatCode( () -> { - DistributedTransaction tx = managerWithDefaultNamespace.start(); + TwoPhaseCommitTransaction tx = managerWithDefaultNamespace.start(); tx.insert(insert); + tx.prepare(); + tx.validate(); tx.commit(); }) .doesNotThrowAnyException(); @@ -1509,8 +1561,8 @@ public void insert_DefaultNamespaceGiven_ShouldWorkProperly() throws Transaction public void upsert_DefaultNamespaceGiven_ShouldWorkProperly() throws TransactionException { Properties properties = getProperties1(getTestName()); properties.put(DatabaseConfig.DEFAULT_NAMESPACE_NAME, namespace1); - try (DistributedTransactionManager managerWithDefaultNamespace = - TransactionFactory.create(properties).getTransactionManager()) { + try (TwoPhaseCommitTransactionManager managerWithDefaultNamespace = + TransactionFactory.create(properties).getTwoPhaseCommitTransactionManager()) { // Arrange populateRecords(manager1, namespace1, TABLE_1); Upsert upsert = @@ -1524,8 +1576,10 @@ public void upsert_DefaultNamespaceGiven_ShouldWorkProperly() throws Transaction // Act Assert Assertions.assertThatCode( () -> { - DistributedTransaction tx = managerWithDefaultNamespace.start(); + TwoPhaseCommitTransaction tx = managerWithDefaultNamespace.start(); tx.upsert(upsert); + tx.prepare(); + tx.validate(); tx.commit(); }) .doesNotThrowAnyException(); @@ -1536,8 +1590,8 @@ public void upsert_DefaultNamespaceGiven_ShouldWorkProperly() throws Transaction public void update_DefaultNamespaceGiven_ShouldWorkProperly() throws TransactionException { Properties properties = getProperties1(getTestName()); properties.put(DatabaseConfig.DEFAULT_NAMESPACE_NAME, namespace1); - try (DistributedTransactionManager managerWithDefaultNamespace = - TransactionFactory.create(properties).getTransactionManager()) { + try (TwoPhaseCommitTransactionManager managerWithDefaultNamespace = + TransactionFactory.create(properties).getTwoPhaseCommitTransactionManager()) { // Arrange populateRecords(manager1, namespace1, TABLE_1); Update update = @@ -1551,8 +1605,10 @@ public void update_DefaultNamespaceGiven_ShouldWorkProperly() throws Transaction // Act Assert Assertions.assertThatCode( () -> { - DistributedTransaction tx = managerWithDefaultNamespace.start(); + TwoPhaseCommitTransaction tx = managerWithDefaultNamespace.start(); tx.update(update); + tx.prepare(); + tx.validate(); tx.commit(); }) .doesNotThrowAnyException(); @@ -1563,8 +1619,8 @@ public void update_DefaultNamespaceGiven_ShouldWorkProperly() throws Transaction public void delete_DefaultNamespaceGiven_ShouldWorkProperly() throws TransactionException { Properties properties = getProperties1(getTestName()); properties.put(DatabaseConfig.DEFAULT_NAMESPACE_NAME, namespace1); - try (DistributedTransactionManager managerWithDefaultNamespace = - TransactionFactory.create(properties).getTransactionManager()) { + try (TwoPhaseCommitTransactionManager managerWithDefaultNamespace = + TransactionFactory.create(properties).getTwoPhaseCommitTransactionManager()) { // Arrange populateRecords(manager1, namespace1, TABLE_1); Delete delete = @@ -1577,8 +1633,10 @@ public void delete_DefaultNamespaceGiven_ShouldWorkProperly() throws Transaction // Act Assert Assertions.assertThatCode( () -> { - DistributedTransaction tx = managerWithDefaultNamespace.start(); + TwoPhaseCommitTransaction tx = managerWithDefaultNamespace.start(); tx.delete(delete); + tx.prepare(); + tx.validate(); tx.commit(); }) .doesNotThrowAnyException(); @@ -1589,8 +1647,8 @@ public void delete_DefaultNamespaceGiven_ShouldWorkProperly() throws Transaction public void mutate_DefaultNamespaceGiven_ShouldWorkProperly() throws TransactionException { Properties properties = getProperties1(getTestName()); properties.put(DatabaseConfig.DEFAULT_NAMESPACE_NAME, namespace1); - try (DistributedTransactionManager managerWithDefaultNamespace = - TransactionFactory.create(properties).getTransactionManager()) { + try (TwoPhaseCommitTransactionManager managerWithDefaultNamespace = + TransactionFactory.create(properties).getTwoPhaseCommitTransactionManager()) { // Arrange populateRecords(manager1, namespace1, TABLE_1); Mutation putAsMutation1 = @@ -1611,8 +1669,10 @@ public void mutate_DefaultNamespaceGiven_ShouldWorkProperly() throws Transaction // Act Assert Assertions.assertThatCode( () -> { - DistributedTransaction tx = managerWithDefaultNamespace.start(); + TwoPhaseCommitTransaction tx = managerWithDefaultNamespace.start(); tx.mutate(ImmutableList.of(putAsMutation1, deleteAsMutation2)); + tx.prepare(); + tx.validate(); tx.commit(); }) .doesNotThrowAnyException(); @@ -2552,8 +2612,8 @@ public void manager_delete_DeleteGivenForExisting_ShouldDeleteRecord() public void manager_get_DefaultNamespaceGiven_ShouldWorkProperly() throws TransactionException { Properties properties = getProperties1(getTestName()); properties.put(DatabaseConfig.DEFAULT_NAMESPACE_NAME, namespace1); - try (DistributedTransactionManager managerWithDefaultNamespace = - TransactionFactory.create(properties).getTransactionManager()) { + try (TwoPhaseCommitTransactionManager managerWithDefaultNamespace = + TransactionFactory.create(properties).getTwoPhaseCommitTransactionManager()) { // Arrange populateRecords(manager1, namespace1, TABLE_1); Get get = @@ -2573,8 +2633,8 @@ public void manager_get_DefaultNamespaceGiven_ShouldWorkProperly() throws Transa public void manager_scan_DefaultNamespaceGiven_ShouldWorkProperly() throws TransactionException { Properties properties = getProperties1(getTestName()); properties.put(DatabaseConfig.DEFAULT_NAMESPACE_NAME, namespace1); - try (DistributedTransactionManager managerWithDefaultNamespace = - TransactionFactory.create(properties).getTransactionManager()) { + try (TwoPhaseCommitTransactionManager managerWithDefaultNamespace = + TransactionFactory.create(properties).getTwoPhaseCommitTransactionManager()) { // Arrange populateRecords(manager1, namespace1, TABLE_1); Scan scan = Scan.newBuilder().table(TABLE_1).all().build(); @@ -2585,12 +2645,35 @@ public void manager_scan_DefaultNamespaceGiven_ShouldWorkProperly() throws Trans } } + @Test + public void manager_getScanner_DefaultNamespaceGiven_ShouldWorkProperly() + throws TransactionException { + Properties properties = getProperties1(getTestName()); + properties.put(DatabaseConfig.DEFAULT_NAMESPACE_NAME, namespace1); + try (TwoPhaseCommitTransactionManager managerWithDefaultNamespace = + TransactionFactory.create(properties).getTwoPhaseCommitTransactionManager()) { + // Arrange + populateRecords(manager1, namespace1, TABLE_1); + Scan scan = Scan.newBuilder().table(TABLE_1).all().build(); + + // Act Assert + Assertions.assertThatCode( + () -> { + TransactionManagerCrudOperable.Scanner scanner = + managerWithDefaultNamespace.getScanner(scan); + scanner.all(); + scanner.close(); + }) + .doesNotThrowAnyException(); + } + } + @Test public void manager_put_DefaultNamespaceGiven_ShouldWorkProperly() throws TransactionException { Properties properties = getProperties1(getTestName()); properties.put(DatabaseConfig.DEFAULT_NAMESPACE_NAME, namespace1); - try (DistributedTransactionManager managerWithDefaultNamespace = - TransactionFactory.create(properties).getTransactionManager()) { + try (TwoPhaseCommitTransactionManager managerWithDefaultNamespace = + TransactionFactory.create(properties).getTwoPhaseCommitTransactionManager()) { // Arrange populateRecords(manager1, namespace1, TABLE_1); Put put = @@ -2613,8 +2696,8 @@ public void manager_insert_DefaultNamespaceGiven_ShouldWorkProperly() throws TransactionException { Properties properties = getProperties1(getTestName()); properties.put(DatabaseConfig.DEFAULT_NAMESPACE_NAME, namespace1); - try (DistributedTransactionManager managerWithDefaultNamespace = - TransactionFactory.create(properties).getTransactionManager()) { + try (TwoPhaseCommitTransactionManager managerWithDefaultNamespace = + TransactionFactory.create(properties).getTwoPhaseCommitTransactionManager()) { // Arrange populateRecords(manager1, namespace1, TABLE_1); Insert insert = @@ -2636,8 +2719,8 @@ public void manager_upsert_DefaultNamespaceGiven_ShouldWorkProperly() throws TransactionException { Properties properties = getProperties1(getTestName()); properties.put(DatabaseConfig.DEFAULT_NAMESPACE_NAME, namespace1); - try (DistributedTransactionManager managerWithDefaultNamespace = - TransactionFactory.create(properties).getTransactionManager()) { + try (TwoPhaseCommitTransactionManager managerWithDefaultNamespace = + TransactionFactory.create(properties).getTwoPhaseCommitTransactionManager()) { // Arrange populateRecords(manager1, namespace1, TABLE_1); Upsert upsert = @@ -2659,8 +2742,8 @@ public void manager_update_DefaultNamespaceGiven_ShouldWorkProperly() throws TransactionException { Properties properties = getProperties1(getTestName()); properties.put(DatabaseConfig.DEFAULT_NAMESPACE_NAME, namespace1); - try (DistributedTransactionManager managerWithDefaultNamespace = - TransactionFactory.create(properties).getTransactionManager()) { + try (TwoPhaseCommitTransactionManager managerWithDefaultNamespace = + TransactionFactory.create(properties).getTwoPhaseCommitTransactionManager()) { // Arrange populateRecords(manager1, namespace1, TABLE_1); Update update = @@ -2682,8 +2765,8 @@ public void manager_delete_DefaultNamespaceGiven_ShouldWorkProperly() throws TransactionException { Properties properties = getProperties1(getTestName()); properties.put(DatabaseConfig.DEFAULT_NAMESPACE_NAME, namespace1); - try (DistributedTransactionManager managerWithDefaultNamespace = - TransactionFactory.create(properties).getTransactionManager()) { + try (TwoPhaseCommitTransactionManager managerWithDefaultNamespace = + TransactionFactory.create(properties).getTwoPhaseCommitTransactionManager()) { // Arrange populateRecords(manager1, namespace1, TABLE_1); Delete delete = @@ -2704,8 +2787,8 @@ public void manager_mutate_DefaultNamespaceGiven_ShouldWorkProperly() throws TransactionException { Properties properties = getProperties1(getTestName()); properties.put(DatabaseConfig.DEFAULT_NAMESPACE_NAME, namespace1); - try (DistributedTransactionManager managerWithDefaultNamespace = - TransactionFactory.create(properties).getTransactionManager()) { + try (TwoPhaseCommitTransactionManager managerWithDefaultNamespace = + TransactionFactory.create(properties).getTwoPhaseCommitTransactionManager()) { // Arrange populateRecords(manager1, namespace1, TABLE_1); Mutation putAsMutation1 = @@ -3034,4 +3117,35 @@ protected List> prepareNonKeyColumns(int accountId, int accountType) { } return columns.build(); } + + protected List scanOrGetScanner( + TwoPhaseCommitTransaction transaction, Scan scan, ScanType scanType) throws CrudException { + if (scanType == ScanType.SCAN) { + return transaction.scan(scan); + } + + try (TransactionCrudOperable.Scanner scanner = transaction.getScanner(scan)) { + switch (scanType) { + case SCANNER_ONE: + List results = new ArrayList<>(); + while (true) { + Optional result = scanner.one(); + if (!result.isPresent()) { + return results; + } + results.add(result.get()); + } + case SCANNER_ALL: + return scanner.all(); + default: + throw new AssertionError(); + } + } + } + + public enum ScanType { + SCAN, + SCANNER_ONE, + SCANNER_ALL + } } diff --git a/core/src/integration-test/java/com/scalar/db/common/ConsensusCommitTestUtils.java b/integration-test/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitTestUtils.java similarity index 98% rename from core/src/integration-test/java/com/scalar/db/common/ConsensusCommitTestUtils.java rename to integration-test/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitTestUtils.java index 844b3276c0..f7ec3b18b8 100644 --- a/core/src/integration-test/java/com/scalar/db/common/ConsensusCommitTestUtils.java +++ b/integration-test/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitTestUtils.java @@ -1,4 +1,4 @@ -package com.scalar.db.common; +package com.scalar.db.transaction.consensuscommit; import static com.scalar.db.transaction.consensuscommit.ConsensusCommitConfig.COORDINATOR_GROUP_COMMIT_DELAYED_SLOT_MOVE_TIMEOUT_MILLIS; import static com.scalar.db.transaction.consensuscommit.ConsensusCommitConfig.COORDINATOR_GROUP_COMMIT_ENABLED; diff --git a/integration-test/src/main/java/com/scalar/db/transaction/singlecrudoperation/SingleCrudOperationTransactionIntegrationTestBase.java b/integration-test/src/main/java/com/scalar/db/transaction/singlecrudoperation/SingleCrudOperationTransactionIntegrationTestBase.java index c3e9fc249e..e9df3c8941 100644 --- a/integration-test/src/main/java/com/scalar/db/transaction/singlecrudoperation/SingleCrudOperationTransactionIntegrationTestBase.java +++ b/integration-test/src/main/java/com/scalar/db/transaction/singlecrudoperation/SingleCrudOperationTransactionIntegrationTestBase.java @@ -9,6 +9,8 @@ import java.util.Properties; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; public abstract class SingleCrudOperationTransactionIntegrationTestBase extends DistributedTransactionIntegrationTestBase { @@ -70,23 +72,30 @@ public void get_GetWithUnmatchedConjunctionsGivenForCommittedRecord_ShouldReturn @Disabled("Single CRUD operation transactions don't support beginning a transaction") @Override - @Test - public void scan_ScanGivenForCommittedRecord_ShouldReturnRecords() {} + @ParameterizedTest + @EnumSource(ScanType.class) + public void scanOrGetScanner_ScanGivenForCommittedRecord_ShouldReturnRecords(ScanType scanType) {} @Disabled("Single CRUD operation transactions don't support beginning a transaction") @Override - @Test - public void scan_ScanWithProjectionsGivenForCommittedRecord_ShouldReturnRecords() {} + @ParameterizedTest + @EnumSource(ScanType.class) + public void scanOrGetScanner_ScanWithProjectionsGivenForCommittedRecord_ShouldReturnRecords( + ScanType scanType) {} @Disabled("Single CRUD operation transactions don't support beginning a transaction") @Override - @Test - public void scan_ScanWithOrderingGivenForCommittedRecord_ShouldReturnRecords() {} + @ParameterizedTest + @EnumSource(ScanType.class) + public void scanOrGetScanner_ScanWithOrderingGivenForCommittedRecord_ShouldReturnRecords( + ScanType scanType) {} @Disabled("Single CRUD operation transactions don't support beginning a transaction") @Override - @Test - public void scan_ScanWithLimitGivenForCommittedRecord_ShouldReturnRecords() {} + @ParameterizedTest + @EnumSource(ScanType.class) + public void scanOrGetScanner_ScanWithLimitGivenForCommittedRecord_ShouldReturnRecords( + ScanType scanType) {} @Disabled("Single CRUD operation transactions don't support beginning a transaction") @Override @@ -95,8 +104,9 @@ public void get_GetGivenForNonExisting_ShouldReturnEmpty() {} @Disabled("Single CRUD operation transactions don't support beginning a transaction") @Override - @Test - public void scan_ScanGivenForNonExisting_ShouldReturnEmpty() {} + @ParameterizedTest + @EnumSource(ScanType.class) + public void scanOrGetScanner_ScanGivenForNonExisting_ShouldReturnEmpty(ScanType scanType) {} @Disabled("Single CRUD operation transactions don't support beginning a transaction") @Override @@ -105,28 +115,36 @@ public void get_GetGivenForIndexColumn_ShouldReturnRecords() {} @Disabled("Single CRUD operation transactions don't support beginning a transaction") @Override - @Test - public void scan_ScanGivenForIndexColumn_ShouldReturnRecords() {} + @ParameterizedTest + @EnumSource(ScanType.class) + public void scanOrGetScanner_ScanGivenForIndexColumn_ShouldReturnRecords(ScanType scanType) {} @Disabled("Single CRUD operation transactions don't support beginning a transaction") @Override - @Test - public void scan_ScanAllGivenForCommittedRecord_ShouldReturnRecords() {} + @ParameterizedTest + @EnumSource(ScanType.class) + public void scanOrGetScanner_ScanAllGivenForCommittedRecord_ShouldReturnRecords( + ScanType scanType) {} @Disabled("Single CRUD operation transactions don't support beginning a transaction") @Override - @Test - public void scan_ScanAllGivenWithLimit_ShouldReturnLimitedAmountOfRecords() {} + @ParameterizedTest + @EnumSource(ScanType.class) + public void scanOrGetScanner_ScanAllGivenWithLimit_ShouldReturnLimitedAmountOfRecords( + ScanType scanType) {} @Disabled("Single CRUD operation transactions don't support beginning a transaction") @Override - @Test - public void scan_ScanAllWithProjectionsGiven_ShouldRetrieveSpecifiedValues() {} + @ParameterizedTest + @EnumSource(ScanType.class) + public void scanOrGetScanner_ScanAllWithProjectionsGiven_ShouldRetrieveSpecifiedValues( + ScanType scanType) {} @Disabled("Single CRUD operation transactions don't support beginning a transaction") @Override - @Test - public void scan_ScanAllGivenForNonExisting_ShouldReturnEmpty() {} + @ParameterizedTest + @EnumSource(ScanType.class) + public void scanOrGetScanner_ScanAllGivenForNonExisting_ShouldReturnEmpty(ScanType scanType) {} @Disabled("Single CRUD operation transactions don't support beginning a transaction") @Override @@ -201,15 +219,19 @@ public void mutateAndCommit_ShouldMutateRecordsProperly() {} @Disabled("Single CRUD operation transactions don't support beginning a transaction") @Override - @Test + @ParameterizedTest + @EnumSource(ScanType.class) public void - scan_ScanWithProjectionsGivenOnNonPrimaryKeyColumnsForCommittedRecord_ShouldReturnOnlyProjectedColumns() {} + scanOrGetScanner_ScanWithProjectionsGivenOnNonPrimaryKeyColumnsForCommittedRecord_ShouldReturnOnlyProjectedColumns( + ScanType scanType) {} @Disabled("Single CRUD operation transactions don't support beginning a transaction") @Override - @Test + @ParameterizedTest + @EnumSource(ScanType.class) public void - scan_ScanAllWithProjectionsGivenOnNonPrimaryKeyColumnsForCommittedRecord_ShouldReturnOnlyProjectedColumns() {} + scanOrGetScanner_ScanAllWithProjectionsGivenOnNonPrimaryKeyColumnsForCommittedRecord_ShouldReturnOnlyProjectedColumns( + ScanType scanType) {} @Disabled("Single CRUD operation transactions don't support resuming a transaction") @Override @@ -243,6 +265,11 @@ public void get_DefaultNamespaceGiven_ShouldWorkProperly() {} @Test public void scan_DefaultNamespaceGiven_ShouldWorkProperly() {} + @Disabled("Single CRUD operation transactions don't support beginning a transaction") + @Override + @Test + public void getScanner_DefaultNamespaceGiven_ShouldWorkProperly() {} + @Disabled("Single CRUD operation transactions don't support beginning a transaction") @Override @Test @@ -418,11 +445,15 @@ public void manager_mutate_DefaultNamespaceGiven_ShouldWorkProperly() {} @Disabled("Single CRUD operation transactions don't support beginning a transaction") @Override - @Test - public void scan_ScanWithConjunctionsGivenForCommittedRecord_ShouldReturnRecords() {} + @ParameterizedTest + @EnumSource(ScanType.class) + public void scanOrGetScanner_ScanWithConjunctionsGivenForCommittedRecord_ShouldReturnRecords( + ScanType scanType) {} @Disabled("Single CRUD operation transactions don't support beginning a transaction") @Override - @Test - public void scan_ScanGivenForIndexColumnWithConjunctions_ShouldReturnRecords() {} + @ParameterizedTest + @EnumSource(ScanType.class) + public void scanOrGetScanner_ScanGivenForIndexColumnWithConjunctions_ShouldReturnRecords( + ScanType scanType) {} } From e32fd446be0287db6c2f647978cbf401c30ab820 Mon Sep 17 00:00:00 2001 From: Jishnu J Date: Wed, 4 Jun 2025 18:26:48 +0530 Subject: [PATCH 06/19] Initial core and cli changes --- .../cli/command/dataexport/ExportCommand.java | 65 +++++++++++-- .../dataexport/ExportCommandOptions.java | 8 ++ .../core/dataexport/CsvExportManager.java | 14 ++- .../core/dataexport/ExportManager.java | 82 +++++++++++++++- .../core/dataexport/ExportOptions.java | 2 + .../core/dataexport/JsonExportManager.java | 14 ++- .../dataexport/JsonLineExportManager.java | 14 ++- .../core/dataimport/dao/ScalarDbDao.java | 60 ++++++++++++ .../tablemetadata/TableMetadataService.java | 18 +++- .../core/dataexport/CsvExportManagerTest.java | 95 ++++++++++++++++++- .../dataexport/JsonExportManagerTest.java | 95 ++++++++++++++++++- .../dataexport/JsonLineExportManagerTest.java | 95 ++++++++++++++++++- .../TableMetadataServiceTest.java | 35 ++++++- 13 files changed, 569 insertions(+), 28 deletions(-) diff --git a/data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/command/dataexport/ExportCommand.java b/data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/command/dataexport/ExportCommand.java index fdedbeef2c..7b15b9ccdf 100755 --- a/data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/command/dataexport/ExportCommand.java +++ b/data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/command/dataexport/ExportCommand.java @@ -4,6 +4,7 @@ import static java.nio.file.StandardOpenOption.CREATE; import com.scalar.db.api.DistributedStorage; +import com.scalar.db.api.DistributedTransactionManager; import com.scalar.db.api.TableMetadata; import com.scalar.db.common.error.CoreError; import com.scalar.db.dataloader.cli.exception.DirectoryValidationException; @@ -12,6 +13,7 @@ import com.scalar.db.dataloader.cli.util.InvalidFilePathException; import com.scalar.db.dataloader.core.ColumnKeyValue; import com.scalar.db.dataloader.core.FileFormat; +import com.scalar.db.dataloader.core.ScalarDbMode; import com.scalar.db.dataloader.core.ScanRange; import com.scalar.db.dataloader.core.dataexport.CsvExportManager; import com.scalar.db.dataloader.core.dataexport.ExportManager; @@ -27,7 +29,9 @@ import com.scalar.db.dataloader.core.util.KeyUtils; import com.scalar.db.io.Key; import com.scalar.db.service.StorageFactory; +import com.scalar.db.service.TransactionFactory; import java.io.BufferedWriter; +import java.io.IOException; import java.nio.charset.Charset; import java.nio.file.Files; import java.nio.file.Paths; @@ -56,15 +60,14 @@ public Integer call() throws Exception { try { validateOutputDirectory(); FileUtils.validateFilePath(scalarDbPropertiesFilePath); - - StorageFactory storageFactory = StorageFactory.create(scalarDbPropertiesFilePath); - TableMetadataService metaDataService = - new TableMetadataService(storageFactory.getStorageAdmin()); + TableMetadataService tableMetadataService = + createTableMetadataService(scalarDbMode, scalarDbPropertiesFilePath); ScalarDbDao scalarDbDao = new ScalarDbDao(); - ExportManager exportManager = createExportManager(storageFactory, scalarDbDao, outputFormat); + ExportManager exportManager = + createExportManager(scalarDbMode, scalarDbDao, outputFormat, scalarDbPropertiesFilePath); - TableMetadata tableMetadata = metaDataService.getTableMetadata(namespace, table); + TableMetadata tableMetadata = tableMetadataService.getTableMetadata(namespace, table); Key partitionKey = partitionKeyValue != null ? getKeysFromList(partitionKeyValue, tableMetadata) : null; @@ -122,11 +125,57 @@ private void validateOutputDirectory() throws DirectoryValidationException { } } + private TableMetadataService createTableMetadataService( + ScalarDbMode scalarDbMode, String scalarDbPropertiesFilePath) throws IOException { + if (scalarDbMode.equals(ScalarDbMode.TRANSACTION)) { + TransactionFactory transactionFactory = TransactionFactory.create(scalarDbPropertiesFilePath); + return new TableMetadataService(transactionFactory.getTransactionAdmin()); + } + StorageFactory storageFactory = StorageFactory.create(scalarDbPropertiesFilePath); + return new TableMetadataService(storageFactory.getStorageAdmin()); + } + private ExportManager createExportManager( - StorageFactory storageFactory, ScalarDbDao scalarDbDao, FileFormat fileFormat) { + ScalarDbMode scalarDbMode, + ScalarDbDao scalarDbDao, + FileFormat fileFormat, + String scalarDbPropertiesFilePath) + throws IOException { ProducerTaskFactory taskFactory = new ProducerTaskFactory(delimiter, includeTransactionMetadata, prettyPrintJson); - DistributedStorage storage = storageFactory.getStorage(); + if (scalarDbMode.equals(ScalarDbMode.TRANSACTION)) { + DistributedStorage storage = StorageFactory.create(scalarDbPropertiesFilePath).getStorage(); + return createExportManagerWithStorage(storage, scalarDbDao, fileFormat, taskFactory); + } else { + DistributedTransactionManager distributedTransactionManager = + TransactionFactory.create(scalarDbPropertiesFilePath).getTransactionManager(); + return createExportManagerWithTransaction( + distributedTransactionManager, scalarDbDao, fileFormat, taskFactory); + } + } + + private ExportManager createExportManagerWithTransaction( + DistributedTransactionManager distributedTransactionManager, + ScalarDbDao scalarDbDao, + FileFormat fileFormat, + ProducerTaskFactory taskFactory) { + switch (fileFormat) { + case JSON: + return new JsonExportManager(distributedTransactionManager, scalarDbDao, taskFactory); + case JSONL: + return new JsonLineExportManager(distributedTransactionManager, scalarDbDao, taskFactory); + case CSV: + return new CsvExportManager(distributedTransactionManager, scalarDbDao, taskFactory); + default: + throw new AssertionError("Invalid file format" + fileFormat); + } + } + + private ExportManager createExportManagerWithStorage( + DistributedStorage storage, + ScalarDbDao scalarDbDao, + FileFormat fileFormat, + ProducerTaskFactory taskFactory) { switch (fileFormat) { case JSON: return new JsonExportManager(storage, scalarDbDao, taskFactory); diff --git a/data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/command/dataexport/ExportCommandOptions.java b/data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/command/dataexport/ExportCommandOptions.java index 5cbe8f6c78..3846bd6777 100755 --- a/data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/command/dataexport/ExportCommandOptions.java +++ b/data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/command/dataexport/ExportCommandOptions.java @@ -3,6 +3,7 @@ import com.scalar.db.api.Scan; import com.scalar.db.dataloader.core.ColumnKeyValue; import com.scalar.db.dataloader.core.FileFormat; +import com.scalar.db.dataloader.core.ScalarDbMode; import java.util.ArrayList; import java.util.List; import picocli.CommandLine; @@ -144,4 +145,11 @@ public class ExportCommandOptions { description = "Size of the data chunk to process in a single task (default: 200)", defaultValue = "200") protected int dataChunkSize; + + @CommandLine.Option( + names = {"--mode", "-sm"}, + description = "ScalarDB mode (STORAGE, TRANSACTION) (default: STORAGE)", + paramLabel = "", + defaultValue = "STORAGE") + protected ScalarDbMode scalarDbMode; } diff --git a/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataexport/CsvExportManager.java b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataexport/CsvExportManager.java index 9e0dc4ba46..cbf482e7a1 100644 --- a/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataexport/CsvExportManager.java +++ b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataexport/CsvExportManager.java @@ -1,6 +1,7 @@ package com.scalar.db.dataloader.core.dataexport; import com.scalar.db.api.DistributedStorage; +import com.scalar.db.api.DistributedTransactionManager; import com.scalar.db.api.TableMetadata; import com.scalar.db.dataloader.core.dataexport.producer.ProducerTaskFactory; import com.scalar.db.dataloader.core.dataimport.dao.ScalarDbDao; @@ -13,8 +14,17 @@ public class CsvExportManager extends ExportManager { public CsvExportManager( - DistributedStorage storage, ScalarDbDao dao, ProducerTaskFactory producerTaskFactory) { - super(storage, dao, producerTaskFactory); + DistributedStorage distributedStorage, + ScalarDbDao dao, + ProducerTaskFactory producerTaskFactory) { + super(distributedStorage, dao, producerTaskFactory); + } + + public CsvExportManager( + DistributedTransactionManager distributedTransactionManager, + ScalarDbDao dao, + ProducerTaskFactory producerTaskFactory) { + super(distributedTransactionManager, dao, producerTaskFactory); } /** diff --git a/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataexport/ExportManager.java b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataexport/ExportManager.java index 13f33a319a..09a03edad5 100644 --- a/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataexport/ExportManager.java +++ b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataexport/ExportManager.java @@ -1,10 +1,12 @@ package com.scalar.db.dataloader.core.dataexport; import com.scalar.db.api.DistributedStorage; +import com.scalar.db.api.DistributedTransactionManager; import com.scalar.db.api.Result; import com.scalar.db.api.Scanner; import com.scalar.db.api.TableMetadata; import com.scalar.db.dataloader.core.FileFormat; +import com.scalar.db.dataloader.core.ScalarDbMode; import com.scalar.db.dataloader.core.dataexport.producer.ProducerTask; import com.scalar.db.dataloader.core.dataexport.producer.ProducerTaskFactory; import com.scalar.db.dataloader.core.dataexport.validation.ExportOptionsValidationException; @@ -32,11 +34,31 @@ public abstract class ExportManager { private static final Logger logger = LoggerFactory.getLogger(ExportManager.class); - private final DistributedStorage storage; + private final DistributedStorage distributedStorage; + private final DistributedTransactionManager distributedTransactionManager; private final ScalarDbDao dao; private final ProducerTaskFactory producerTaskFactory; private final Object lock = new Object(); + public ExportManager( + DistributedStorage distributedStorage, + ScalarDbDao dao, + ProducerTaskFactory producerTaskFactory) { + this.distributedStorage = distributedStorage; + this.distributedTransactionManager = null; + this.dao = dao; + this.producerTaskFactory = producerTaskFactory; + } + + public ExportManager( + DistributedTransactionManager distributedTransactionManager, + ScalarDbDao dao, + ProducerTaskFactory producerTaskFactory) { + this.distributedStorage = null; + this.distributedTransactionManager = distributedTransactionManager; + this.dao = dao; + this.producerTaskFactory = producerTaskFactory; + } /** * Create and add header part for the export file * @@ -83,8 +105,8 @@ public ExportReport startExport( BufferedWriter bufferedWriter = new BufferedWriter(writer); boolean isJson = exportOptions.getOutputFileFormat() == FileFormat.JSON; - try (Scanner scanner = createScanner(exportOptions, dao, storage)) { - + try (Scanner scanner = + createScanner(exportOptions, dao, distributedStorage, distributedTransactionManager)) { Iterator iterator = scanner.iterator(); AtomicBoolean isFirstBatch = new AtomicBoolean(true); @@ -118,6 +140,7 @@ public ExportReport startExport( } catch (ExportOptionsValidationException | IOException | ScalarDbDaoException e) { logger.error("Error during export: {}", e.getMessage()); } + closeResources(); return exportReport; } @@ -208,6 +231,18 @@ private void handleTransactionMetadata(ExportOptions exportOptions, TableMetadat } } + private Scanner createScanner( + ExportOptions exportOptions, + ScalarDbDao dao, + DistributedStorage storage, + DistributedTransactionManager transactionManager) + throws ScalarDbDaoException { + if (exportOptions.getScalarDbMode().equals(ScalarDbMode.TRANSACTION)) { + return createScannerWithTransaction(exportOptions, dao, transactionManager); + } + return createScannerWithStorage(exportOptions, dao, storage); + } + /** * To create a scanner object * @@ -217,7 +252,7 @@ private void handleTransactionMetadata(ExportOptions exportOptions, TableMetadat * @return created scanner * @throws ScalarDbDaoException throws if any issue occurs in creating scanner object */ - private Scanner createScanner( + private Scanner createScannerWithStorage( ExportOptions exportOptions, ScalarDbDao dao, DistributedStorage storage) throws ScalarDbDaoException { boolean isScanAll = exportOptions.getScanPartitionKey() == null; @@ -240,4 +275,43 @@ private Scanner createScanner( storage); } } + + private Scanner createScannerWithTransaction( + ExportOptions exportOptions, + ScalarDbDao dao, + DistributedTransactionManager distributedTransactionManager) + throws ScalarDbDaoException { + boolean isScanAll = exportOptions.getScanPartitionKey() == null; + if (isScanAll) { + return dao.createScanner( + exportOptions.getNamespace(), + exportOptions.getTableName(), + exportOptions.getProjectionColumns(), + exportOptions.getLimit(), + distributedTransactionManager); + } else { + return dao.createScanner( + exportOptions.getNamespace(), + exportOptions.getTableName(), + exportOptions.getScanPartitionKey(), + exportOptions.getScanRange(), + exportOptions.getSortOrders(), + exportOptions.getProjectionColumns(), + exportOptions.getLimit(), + distributedTransactionManager); + } + } + + /** Close resources properly once the process is completed */ + public void closeResources() { + try { + if (distributedStorage != null) { + distributedStorage.close(); + } else if (distributedTransactionManager != null) { + distributedTransactionManager.close(); + } + } catch (Throwable e) { + throw new RuntimeException("Failed to close the resource", e); + } + } } diff --git a/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataexport/ExportOptions.java b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataexport/ExportOptions.java index da515cf3c2..c6279ae34b 100644 --- a/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataexport/ExportOptions.java +++ b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataexport/ExportOptions.java @@ -2,6 +2,7 @@ import com.scalar.db.api.Scan; import com.scalar.db.dataloader.core.FileFormat; +import com.scalar.db.dataloader.core.ScalarDbMode; import com.scalar.db.dataloader.core.ScanRange; import com.scalar.db.io.Key; import java.util.Collections; @@ -30,6 +31,7 @@ public class ExportOptions { @Builder.Default private final boolean includeTransactionMetadata = false; @Builder.Default private List projectionColumns = Collections.emptyList(); private List sortOrders; + @Builder.Default private final ScalarDbMode scalarDbMode = ScalarDbMode.STORAGE; public static ExportOptionsBuilder builder( String namespace, String tableName, Key scanPartitionKey, FileFormat outputFileFormat) { diff --git a/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataexport/JsonExportManager.java b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataexport/JsonExportManager.java index 34e382dd5e..880b156693 100644 --- a/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataexport/JsonExportManager.java +++ b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataexport/JsonExportManager.java @@ -1,6 +1,7 @@ package com.scalar.db.dataloader.core.dataexport; import com.scalar.db.api.DistributedStorage; +import com.scalar.db.api.DistributedTransactionManager; import com.scalar.db.api.TableMetadata; import com.scalar.db.dataloader.core.dataexport.producer.ProducerTaskFactory; import com.scalar.db.dataloader.core.dataimport.dao.ScalarDbDao; @@ -9,8 +10,17 @@ public class JsonExportManager extends ExportManager { public JsonExportManager( - DistributedStorage storage, ScalarDbDao dao, ProducerTaskFactory producerTaskFactory) { - super(storage, dao, producerTaskFactory); + DistributedStorage distributedStorage, + ScalarDbDao dao, + ProducerTaskFactory producerTaskFactory) { + super(distributedStorage, dao, producerTaskFactory); + } + + public JsonExportManager( + DistributedTransactionManager distributedTransactionManager, + ScalarDbDao dao, + ProducerTaskFactory producerTaskFactory) { + super(distributedTransactionManager, dao, producerTaskFactory); } /** diff --git a/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataexport/JsonLineExportManager.java b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataexport/JsonLineExportManager.java index 8bc5fabe07..47fc0b9e5c 100644 --- a/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataexport/JsonLineExportManager.java +++ b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataexport/JsonLineExportManager.java @@ -1,6 +1,7 @@ package com.scalar.db.dataloader.core.dataexport; import com.scalar.db.api.DistributedStorage; +import com.scalar.db.api.DistributedTransactionManager; import com.scalar.db.api.TableMetadata; import com.scalar.db.dataloader.core.dataexport.producer.ProducerTaskFactory; import com.scalar.db.dataloader.core.dataimport.dao.ScalarDbDao; @@ -9,8 +10,17 @@ public class JsonLineExportManager extends ExportManager { public JsonLineExportManager( - DistributedStorage storage, ScalarDbDao dao, ProducerTaskFactory producerTaskFactory) { - super(storage, dao, producerTaskFactory); + DistributedStorage distributedStorage, + ScalarDbDao dao, + ProducerTaskFactory producerTaskFactory) { + super(distributedStorage, dao, producerTaskFactory); + } + + public JsonLineExportManager( + DistributedTransactionManager distributedTransactionManager, + ScalarDbDao dao, + ProducerTaskFactory producerTaskFactory) { + super(distributedTransactionManager, dao, producerTaskFactory); } /** diff --git a/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/dao/ScalarDbDao.java b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/dao/ScalarDbDao.java index 8066141ec2..6ef428d783 100644 --- a/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/dao/ScalarDbDao.java +++ b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/dao/ScalarDbDao.java @@ -2,6 +2,7 @@ import com.scalar.db.api.DistributedStorage; import com.scalar.db.api.DistributedTransaction; +import com.scalar.db.api.DistributedTransactionManager; import com.scalar.db.api.Get; import com.scalar.db.api.GetBuilder; import com.scalar.db.api.Put; @@ -248,6 +249,34 @@ public Scanner createScanner( } } + /** + * Create a ScalarDB scanner instance + * + * @param namespace ScalarDB namespace + * @param table ScalarDB table name + * @param projectionColumns List of column projection to use during scan + * @param limit Scan limit value + * @param distributedTransactionManager Distributed transaction manager object + * @return ScalarDB Scanner object + * @throws ScalarDbDaoException if scan fails + */ + public Scanner createScanner( + String namespace, + String table, + List projectionColumns, + int limit, + DistributedTransactionManager distributedTransactionManager) + throws ScalarDbDaoException { + Scan scan = + createScan(namespace, table, null, null, new ArrayList<>(), projectionColumns, limit); + try { + return (Scanner) distributedTransactionManager.getScanner(scan); + } catch (CrudException e) { + throw new ScalarDbDaoException( + CoreError.DATA_LOADER_ERROR_SCAN.buildMessage(e.getMessage()), e); + } + } + /** * Create a ScalarDB scanner instance * @@ -282,6 +311,37 @@ public Scanner createScanner( } } + /** + * Create a ScalarDB scanner instance + * + * @param namespace ScalarDB namespace + * @param table ScalarDB table name + * @param partitionKey Partition key used in ScalarDB scan + * @param scanRange Optional range to set ScalarDB scan start and end values + * @param sortOrders Optional scan clustering key sorting values + * @param projectionColumns List of column projection to use during scan + * @param limit Scan limit value + * @param distributedTransactionManager Distributed transaction manager object + * @return ScalarDB Scanner object + */ + public Scanner createScanner( + String namespace, + String table, + @Nullable Key partitionKey, + @Nullable ScanRange scanRange, + @Nullable List sortOrders, + @Nullable List projectionColumns, + int limit, + DistributedTransactionManager distributedTransactionManager) { + Scan scan = + createScan(namespace, table, partitionKey, scanRange, sortOrders, projectionColumns, limit); + try { + return (Scanner) distributedTransactionManager.getScanner(scan); + } catch (CrudException e) { + throw new RuntimeException(e); + } + } + /** * Create ScalarDB scan instance * diff --git a/data-loader/core/src/main/java/com/scalar/db/dataloader/core/tablemetadata/TableMetadataService.java b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/tablemetadata/TableMetadataService.java index 8816945800..41c292ec96 100644 --- a/data-loader/core/src/main/java/com/scalar/db/dataloader/core/tablemetadata/TableMetadataService.java +++ b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/tablemetadata/TableMetadataService.java @@ -1,6 +1,7 @@ package com.scalar.db.dataloader.core.tablemetadata; import com.scalar.db.api.DistributedStorageAdmin; +import com.scalar.db.api.DistributedTransactionAdmin; import com.scalar.db.api.TableMetadata; import com.scalar.db.common.error.CoreError; import com.scalar.db.dataloader.core.util.TableMetadataUtil; @@ -18,7 +19,17 @@ public class TableMetadataService { private final DistributedStorageAdmin storageAdmin; + private final DistributedTransactionAdmin transactionAdmin; + public TableMetadataService(DistributedStorageAdmin storageAdmin) { + this.transactionAdmin = null; + this.storageAdmin = storageAdmin; + } + + public TableMetadataService(DistributedTransactionAdmin transactionAdmin) { + this.transactionAdmin = transactionAdmin; + this.storageAdmin = null; + } /** * Retrieves the {@link TableMetadata} for a specific namespace and table name. * @@ -31,7 +42,12 @@ public class TableMetadataService { public TableMetadata getTableMetadata(String namespace, String tableName) throws TableMetadataException { try { - TableMetadata tableMetadata = storageAdmin.getTableMetadata(namespace, tableName); + TableMetadata tableMetadata = null; + if (storageAdmin != null) { + tableMetadata = storageAdmin.getTableMetadata(namespace, tableName); + } else if (transactionAdmin != null) { + tableMetadata = transactionAdmin.getTableMetadata(namespace, tableName); + } if (tableMetadata == null) { throw new TableMetadataException( CoreError.DATA_LOADER_MISSING_NAMESPACE_OR_TABLE.buildMessage(namespace, tableName)); diff --git a/data-loader/core/src/test/java/com/scalar/db/dataloader/core/dataexport/CsvExportManagerTest.java b/data-loader/core/src/test/java/com/scalar/db/dataloader/core/dataexport/CsvExportManagerTest.java index ca65c10010..498102228f 100644 --- a/data-loader/core/src/test/java/com/scalar/db/dataloader/core/dataexport/CsvExportManagerTest.java +++ b/data-loader/core/src/test/java/com/scalar/db/dataloader/core/dataexport/CsvExportManagerTest.java @@ -1,11 +1,13 @@ package com.scalar.db.dataloader.core.dataexport; import com.scalar.db.api.DistributedStorage; +import com.scalar.db.api.DistributedTransactionManager; import com.scalar.db.api.Result; import com.scalar.db.api.Scanner; import com.scalar.db.api.TableMetadata; import com.scalar.db.common.ResultImpl; import com.scalar.db.dataloader.core.FileFormat; +import com.scalar.db.dataloader.core.ScalarDbMode; import com.scalar.db.dataloader.core.ScanRange; import com.scalar.db.dataloader.core.UnitTestUtils; import com.scalar.db.dataloader.core.dataexport.producer.ProducerTaskFactory; @@ -33,6 +35,7 @@ public class CsvExportManagerTest { TableMetadata mockData; DistributedStorage storage; + DistributedTransactionManager manager; @Spy ScalarDbDao dao; ProducerTaskFactory producerTaskFactory; ExportManager exportManager; @@ -40,13 +43,14 @@ public class CsvExportManagerTest { @BeforeEach void setup() { storage = Mockito.mock(DistributedStorage.class); + manager = Mockito.mock(DistributedTransactionManager.class); mockData = UnitTestUtils.createTestTableMetadata(); dao = Mockito.mock(ScalarDbDao.class); producerTaskFactory = new ProducerTaskFactory(null, false, true); } @Test - void startExport_givenValidDataWithoutPartitionKey_shouldGenerateOutputFile() + void startExport_givenValidDataWithoutPartitionKey_withStorage_shouldGenerateOutputFile() throws IOException, ScalarDbDaoException { exportManager = new JsonLineExportManager(storage, dao, producerTaskFactory); Scanner scanner = Mockito.mock(Scanner.class); @@ -84,7 +88,7 @@ void startExport_givenValidDataWithoutPartitionKey_shouldGenerateOutputFile() } @Test - void startExport_givenPartitionKey_shouldGenerateOutputFile() + void startExport_givenPartitionKey_withStorage_shouldGenerateOutputFile() throws IOException, ScalarDbDaoException { producerTaskFactory = new ProducerTaskFactory(",", false, false); exportManager = new CsvExportManager(storage, dao, producerTaskFactory); @@ -129,4 +133,91 @@ void startExport_givenPartitionKey_shouldGenerateOutputFile() Assertions.assertTrue(file.exists()); Assertions.assertTrue(file.delete()); } + + @Test + void startExport_givenValidDataWithoutPartitionKey_withTransaction_shouldGenerateOutputFile() + throws IOException, ScalarDbDaoException { + exportManager = new JsonLineExportManager(manager, dao, producerTaskFactory); + Scanner scanner = Mockito.mock(Scanner.class); + String filePath = Paths.get("").toAbsolutePath() + "/output.csv"; + Map> values = UnitTestUtils.createTestValues(); + Result result = new ResultImpl(values, mockData); + List results = Collections.singletonList(result); + ExportOptions exportOptions = + ExportOptions.builder("namespace", "table", null, FileFormat.CSV) + .sortOrders(Collections.emptyList()) + .scanRange(new ScanRange(null, null, false, false)) + .scalarDbMode(ScalarDbMode.TRANSACTION) + .build(); + + Mockito.when( + dao.createScanner( + exportOptions.getNamespace(), + exportOptions.getTableName(), + exportOptions.getProjectionColumns(), + exportOptions.getLimit(), + manager)) + .thenReturn(scanner); + Mockito.when(scanner.iterator()).thenReturn(results.iterator()); + try (BufferedWriter writer = + new BufferedWriter( + Files.newBufferedWriter( + Paths.get(filePath), + Charset.defaultCharset(), // Explicitly use the default charset + StandardOpenOption.CREATE, + StandardOpenOption.APPEND))) { + exportManager.startExport(exportOptions, mockData, writer); + } + File file = new File(filePath); + Assertions.assertTrue(file.exists()); + Assertions.assertTrue(file.delete()); + } + + @Test + void startExport_givenPartitionKey_withTransaction_shouldGenerateOutputFile() + throws IOException, ScalarDbDaoException { + producerTaskFactory = new ProducerTaskFactory(",", false, false); + exportManager = new CsvExportManager(manager, dao, producerTaskFactory); + Scanner scanner = Mockito.mock(Scanner.class); + String filePath = Paths.get("").toAbsolutePath() + "/output.csv"; + Map> values = UnitTestUtils.createTestValues(); + Result result = new ResultImpl(values, mockData); + List results = Collections.singletonList(result); + + ExportOptions exportOptions = + ExportOptions.builder( + "namespace", + "table", + Key.newBuilder().add(IntColumn.of("col1", 1)).build(), + FileFormat.CSV) + .sortOrders(Collections.emptyList()) + .scanRange(new ScanRange(null, null, false, false)) + .scalarDbMode(ScalarDbMode.TRANSACTION) + .build(); + + Mockito.when( + dao.createScanner( + exportOptions.getNamespace(), + exportOptions.getTableName(), + exportOptions.getScanPartitionKey(), + exportOptions.getScanRange(), + exportOptions.getSortOrders(), + exportOptions.getProjectionColumns(), + exportOptions.getLimit(), + manager)) + .thenReturn(scanner); + Mockito.when(scanner.iterator()).thenReturn(results.iterator()); + try (BufferedWriter writer = + new BufferedWriter( + Files.newBufferedWriter( + Paths.get(filePath), + Charset.defaultCharset(), // Explicitly use the default charset + StandardOpenOption.CREATE, + StandardOpenOption.APPEND))) { + exportManager.startExport(exportOptions, mockData, writer); + } + File file = new File(filePath); + Assertions.assertTrue(file.exists()); + Assertions.assertTrue(file.delete()); + } } diff --git a/data-loader/core/src/test/java/com/scalar/db/dataloader/core/dataexport/JsonExportManagerTest.java b/data-loader/core/src/test/java/com/scalar/db/dataloader/core/dataexport/JsonExportManagerTest.java index c1ef7ead1a..e03815a429 100644 --- a/data-loader/core/src/test/java/com/scalar/db/dataloader/core/dataexport/JsonExportManagerTest.java +++ b/data-loader/core/src/test/java/com/scalar/db/dataloader/core/dataexport/JsonExportManagerTest.java @@ -1,11 +1,13 @@ package com.scalar.db.dataloader.core.dataexport; import com.scalar.db.api.DistributedStorage; +import com.scalar.db.api.DistributedTransactionManager; import com.scalar.db.api.Result; import com.scalar.db.api.Scanner; import com.scalar.db.api.TableMetadata; import com.scalar.db.common.ResultImpl; import com.scalar.db.dataloader.core.FileFormat; +import com.scalar.db.dataloader.core.ScalarDbMode; import com.scalar.db.dataloader.core.ScanRange; import com.scalar.db.dataloader.core.UnitTestUtils; import com.scalar.db.dataloader.core.dataexport.producer.ProducerTaskFactory; @@ -34,6 +36,7 @@ public class JsonExportManagerTest { TableMetadata mockData; DistributedStorage storage; + DistributedTransactionManager manager; @Spy ScalarDbDao dao; ProducerTaskFactory producerTaskFactory; ExportManager exportManager; @@ -41,13 +44,14 @@ public class JsonExportManagerTest { @BeforeEach void setup() { storage = Mockito.mock(DistributedStorage.class); + manager = Mockito.mock(DistributedTransactionManager.class); mockData = UnitTestUtils.createTestTableMetadata(); dao = Mockito.mock(ScalarDbDao.class); producerTaskFactory = new ProducerTaskFactory(null, false, true); } @Test - void startExport_givenValidDataWithoutPartitionKey_shouldGenerateOutputFile() + void startExport_givenValidDataWithoutPartitionKey_withStorage_shouldGenerateOutputFile() throws IOException, ScalarDbDaoException { exportManager = new JsonExportManager(storage, dao, producerTaskFactory); Scanner scanner = Mockito.mock(Scanner.class); @@ -86,7 +90,7 @@ void startExport_givenValidDataWithoutPartitionKey_shouldGenerateOutputFile() } @Test - void startExport_givenPartitionKey_shouldGenerateOutputFile() + void startExport_givenPartitionKey_withStorage_shouldGenerateOutputFile() throws IOException, ScalarDbDaoException { exportManager = new JsonExportManager(storage, dao, producerTaskFactory); Scanner scanner = Mockito.mock(Scanner.class); @@ -130,4 +134,91 @@ void startExport_givenPartitionKey_shouldGenerateOutputFile() Assertions.assertTrue(file.exists()); Assertions.assertTrue(file.delete()); } + + @Test + void + startExport_givenValidDataWithoutPartitionKey_withTransaction_withStorage_shouldGenerateOutputFile() + throws IOException, ScalarDbDaoException { + exportManager = new JsonExportManager(manager, dao, producerTaskFactory); + Scanner scanner = Mockito.mock(Scanner.class); + String filePath = Paths.get("").toAbsolutePath() + "/output.json"; + Map> values = UnitTestUtils.createTestValues(); + Result result = new ResultImpl(values, mockData); + List results = Collections.singletonList(result); + + ExportOptions exportOptions = + ExportOptions.builder("namespace", "table", null, FileFormat.JSON) + .sortOrders(Collections.emptyList()) + .scanRange(new ScanRange(null, null, false, false)) + .scalarDbMode(ScalarDbMode.TRANSACTION) + .build(); + + Mockito.when( + dao.createScanner( + exportOptions.getNamespace(), + exportOptions.getTableName(), + exportOptions.getProjectionColumns(), + exportOptions.getLimit(), + manager)) + .thenReturn(scanner); + Mockito.when(scanner.iterator()).thenReturn(results.iterator()); + try (BufferedWriter writer = + new BufferedWriter( + Files.newBufferedWriter( + Paths.get(filePath), + Charset.defaultCharset(), // Explicitly use the default charset + StandardOpenOption.CREATE, + StandardOpenOption.APPEND))) { + exportManager.startExport(exportOptions, mockData, writer); + } + File file = new File(filePath); + Assertions.assertTrue(file.exists()); + Assertions.assertTrue(file.delete()); + } + + @Test + void startExport_givenPartitionKey_withTransaction_shouldGenerateOutputFile() throws IOException { + exportManager = new JsonExportManager(manager, dao, producerTaskFactory); + Scanner scanner = Mockito.mock(Scanner.class); + String filePath = Paths.get("").toAbsolutePath() + "/output.json"; + Map> values = UnitTestUtils.createTestValues(); + Result result = new ResultImpl(values, mockData); + List results = Collections.singletonList(result); + + ExportOptions exportOptions = + ExportOptions.builder( + "namespace", + "table", + Key.newBuilder().add(IntColumn.of("col1", 1)).build(), + FileFormat.JSON) + .sortOrders(Collections.emptyList()) + .scanRange(new ScanRange(null, null, false, false)) + .scalarDbMode(ScalarDbMode.TRANSACTION) + .build(); + + Mockito.when( + dao.createScanner( + exportOptions.getNamespace(), + exportOptions.getTableName(), + exportOptions.getScanPartitionKey(), + exportOptions.getScanRange(), + exportOptions.getSortOrders(), + exportOptions.getProjectionColumns(), + exportOptions.getLimit(), + manager)) + .thenReturn(scanner); + Mockito.when(scanner.iterator()).thenReturn(results.iterator()); + try (BufferedWriter writer = + new BufferedWriter( + Files.newBufferedWriter( + Paths.get(filePath), + Charset.defaultCharset(), // Explicitly use the default charset + StandardOpenOption.CREATE, + StandardOpenOption.APPEND))) { + exportManager.startExport(exportOptions, mockData, writer); + } + File file = new File(filePath); + Assertions.assertTrue(file.exists()); + Assertions.assertTrue(file.delete()); + } } diff --git a/data-loader/core/src/test/java/com/scalar/db/dataloader/core/dataexport/JsonLineExportManagerTest.java b/data-loader/core/src/test/java/com/scalar/db/dataloader/core/dataexport/JsonLineExportManagerTest.java index 31e4326a33..064cca7358 100644 --- a/data-loader/core/src/test/java/com/scalar/db/dataloader/core/dataexport/JsonLineExportManagerTest.java +++ b/data-loader/core/src/test/java/com/scalar/db/dataloader/core/dataexport/JsonLineExportManagerTest.java @@ -1,11 +1,13 @@ package com.scalar.db.dataloader.core.dataexport; import com.scalar.db.api.DistributedStorage; +import com.scalar.db.api.DistributedTransactionManager; import com.scalar.db.api.Result; import com.scalar.db.api.Scanner; import com.scalar.db.api.TableMetadata; import com.scalar.db.common.ResultImpl; import com.scalar.db.dataloader.core.FileFormat; +import com.scalar.db.dataloader.core.ScalarDbMode; import com.scalar.db.dataloader.core.ScanRange; import com.scalar.db.dataloader.core.UnitTestUtils; import com.scalar.db.dataloader.core.dataexport.producer.ProducerTaskFactory; @@ -33,6 +35,7 @@ public class JsonLineExportManagerTest { TableMetadata mockData; DistributedStorage storage; + DistributedTransactionManager manager; @Spy ScalarDbDao dao; ProducerTaskFactory producerTaskFactory; ExportManager exportManager; @@ -40,13 +43,14 @@ public class JsonLineExportManagerTest { @BeforeEach void setup() { storage = Mockito.mock(DistributedStorage.class); + manager = Mockito.mock(DistributedTransactionManager.class); mockData = UnitTestUtils.createTestTableMetadata(); dao = Mockito.mock(ScalarDbDao.class); producerTaskFactory = new ProducerTaskFactory(null, false, true); } @Test - void startExport_givenValidDataWithoutPartitionKey_shouldGenerateOutputFile() + void startExport_givenValidDataWithoutPartitionKey_withStorage_shouldGenerateOutputFile() throws IOException, ScalarDbDaoException { exportManager = new JsonLineExportManager(storage, dao, producerTaskFactory); Scanner scanner = Mockito.mock(Scanner.class); @@ -85,7 +89,7 @@ void startExport_givenValidDataWithoutPartitionKey_shouldGenerateOutputFile() } @Test - void startExport_givenPartitionKey_shouldGenerateOutputFile() + void startExport_givenPartitionKey_withStorage_shouldGenerateOutputFile() throws IOException, ScalarDbDaoException { exportManager = new JsonLineExportManager(storage, dao, producerTaskFactory); Scanner scanner = Mockito.mock(Scanner.class); @@ -129,4 +133,91 @@ void startExport_givenPartitionKey_shouldGenerateOutputFile() Assertions.assertTrue(file.exists()); Assertions.assertTrue(file.delete()); } + + @Test + void + startExport_givenValidDataWithoutPartitionKey_withTransaction_withStorage_shouldGenerateOutputFile() + throws IOException, ScalarDbDaoException { + exportManager = new JsonLineExportManager(manager, dao, producerTaskFactory); + Scanner scanner = Mockito.mock(Scanner.class); + String filePath = Paths.get("").toAbsolutePath() + "/output.jsonl"; + Map> values = UnitTestUtils.createTestValues(); + Result result = new ResultImpl(values, mockData); + List results = Collections.singletonList(result); + + ExportOptions exportOptions = + ExportOptions.builder("namespace", "table", null, FileFormat.JSONL) + .sortOrders(Collections.emptyList()) + .scanRange(new ScanRange(null, null, false, false)) + .scalarDbMode(ScalarDbMode.TRANSACTION) + .build(); + + Mockito.when( + dao.createScanner( + exportOptions.getNamespace(), + exportOptions.getTableName(), + exportOptions.getProjectionColumns(), + exportOptions.getLimit(), + manager)) + .thenReturn(scanner); + Mockito.when(scanner.iterator()).thenReturn(results.iterator()); + try (BufferedWriter writer = + new BufferedWriter( + Files.newBufferedWriter( + Paths.get(filePath), + Charset.defaultCharset(), // Explicitly use the default charset + StandardOpenOption.CREATE, + StandardOpenOption.APPEND))) { + exportManager.startExport(exportOptions, mockData, writer); + } + File file = new File(filePath); + Assertions.assertTrue(file.exists()); + Assertions.assertTrue(file.delete()); + } + + @Test + void startExport_givenPartitionKey_withTransaction_shouldGenerateOutputFile() throws IOException { + exportManager = new JsonLineExportManager(manager, dao, producerTaskFactory); + Scanner scanner = Mockito.mock(Scanner.class); + String filePath = Paths.get("").toAbsolutePath() + "/output.jsonl"; + Map> values = UnitTestUtils.createTestValues(); + Result result = new ResultImpl(values, mockData); + List results = Collections.singletonList(result); + + ExportOptions exportOptions = + ExportOptions.builder( + "namespace", + "table", + Key.newBuilder().add(IntColumn.of("col1", 1)).build(), + FileFormat.JSONL) + .sortOrders(Collections.emptyList()) + .scanRange(new ScanRange(null, null, false, false)) + .scalarDbMode(ScalarDbMode.TRANSACTION) + .build(); + + Mockito.when( + dao.createScanner( + exportOptions.getNamespace(), + exportOptions.getTableName(), + exportOptions.getScanPartitionKey(), + exportOptions.getScanRange(), + exportOptions.getSortOrders(), + exportOptions.getProjectionColumns(), + exportOptions.getLimit(), + manager)) + .thenReturn(scanner); + Mockito.when(scanner.iterator()).thenReturn(results.iterator()); + try (BufferedWriter writer = + new BufferedWriter( + Files.newBufferedWriter( + Paths.get(filePath), + Charset.defaultCharset(), // Explicitly use the default charset + StandardOpenOption.CREATE, + StandardOpenOption.APPEND))) { + exportManager.startExport(exportOptions, mockData, writer); + } + File file = new File(filePath); + Assertions.assertTrue(file.exists()); + Assertions.assertTrue(file.delete()); + } } diff --git a/data-loader/core/src/test/java/com/scalar/db/dataloader/core/tablemetadata/TableMetadataServiceTest.java b/data-loader/core/src/test/java/com/scalar/db/dataloader/core/tablemetadata/TableMetadataServiceTest.java index 9bcd06bf9b..e68fad90d9 100644 --- a/data-loader/core/src/test/java/com/scalar/db/dataloader/core/tablemetadata/TableMetadataServiceTest.java +++ b/data-loader/core/src/test/java/com/scalar/db/dataloader/core/tablemetadata/TableMetadataServiceTest.java @@ -3,6 +3,7 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy; import com.scalar.db.api.DistributedStorageAdmin; +import com.scalar.db.api.DistributedTransactionAdmin; import com.scalar.db.api.TableMetadata; import com.scalar.db.common.error.CoreError; import com.scalar.db.dataloader.core.UnitTestUtils; @@ -18,6 +19,7 @@ class TableMetadataServiceTest { DistributedStorageAdmin storageAdmin; + DistributedTransactionAdmin transactionAdmin; TableMetadataService tableMetadataService; @BeforeEach @@ -25,13 +27,39 @@ void setup() throws ExecutionException { storageAdmin = Mockito.mock(DistributedStorageAdmin.class); Mockito.when(storageAdmin.getTableMetadata("namespace", "table")) .thenReturn(UnitTestUtils.createTestTableMetadata()); - tableMetadataService = new TableMetadataService(storageAdmin); + transactionAdmin = Mockito.mock(DistributedTransactionAdmin.class); + Mockito.when(transactionAdmin.getTableMetadata("namespace", "table")) + .thenReturn(UnitTestUtils.createTestTableMetadata()); } @Test - void getTableMetadata_withValidNamespaceAndTable_shouldReturnTableMetadataMap() + void getTableMetadata_withStorage_withValidNamespaceAndTable_shouldReturnTableMetadataMap() throws TableMetadataException { + tableMetadataService = new TableMetadataService(storageAdmin); + Map expected = new HashMap<>(); + expected.put("namespace.table", UnitTestUtils.createTestTableMetadata()); + TableMetadataRequest tableMetadataRequest = new TableMetadataRequest("namespace", "table"); + Map output = + tableMetadataService.getTableMetadata(Collections.singleton(tableMetadataRequest)); + Assertions.assertEquals(expected.get("namespace.table"), output.get("namespace.table")); + } + @Test + void getTableMetadata_withStorage_withInvalidNamespaceAndTable_shouldThrowException() { + tableMetadataService = new TableMetadataService(storageAdmin); + TableMetadataRequest tableMetadataRequest = new TableMetadataRequest("namespace2", "table2"); + assertThatThrownBy( + () -> + tableMetadataService.getTableMetadata(Collections.singleton(tableMetadataRequest))) + .isInstanceOf(TableMetadataException.class) + .hasMessage( + CoreError.DATA_LOADER_MISSING_NAMESPACE_OR_TABLE.buildMessage("namespace2", "table2")); + } + + @Test + void getTableMetadata_withTransaction_withValidNamespaceAndTable_shouldReturnTableMetadataMap() + throws TableMetadataException { + tableMetadataService = new TableMetadataService(transactionAdmin); Map expected = new HashMap<>(); expected.put("namespace.table", UnitTestUtils.createTestTableMetadata()); TableMetadataRequest tableMetadataRequest = new TableMetadataRequest("namespace", "table"); @@ -41,7 +69,8 @@ void getTableMetadata_withValidNamespaceAndTable_shouldReturnTableMetadataMap() } @Test - void getTableMetadata_withInvalidNamespaceAndTable_shouldThrowException() { + void getTableMetadata_withTransaction_withInvalidNamespaceAndTable_shouldThrowException() { + tableMetadataService = new TableMetadataService(transactionAdmin); TableMetadataRequest tableMetadataRequest = new TableMetadataRequest("namespace2", "table2"); assertThatThrownBy( () -> From 24ce20c379a43dfe5f146ececbb19c6d61bd7ae4 Mon Sep 17 00:00:00 2001 From: Jishnu J Date: Thu, 5 Jun 2025 10:19:30 +0530 Subject: [PATCH 07/19] minor test change --- .../cli/command/dataexport/ExportCommandTest.java | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/data-loader/cli/src/test/java/com/scalar/db/dataloader/cli/command/dataexport/ExportCommandTest.java b/data-loader/cli/src/test/java/com/scalar/db/dataloader/cli/command/dataexport/ExportCommandTest.java index 73934f340d..65cd9e137a 100755 --- a/data-loader/cli/src/test/java/com/scalar/db/dataloader/cli/command/dataexport/ExportCommandTest.java +++ b/data-loader/cli/src/test/java/com/scalar/db/dataloader/cli/command/dataexport/ExportCommandTest.java @@ -6,6 +6,7 @@ import com.scalar.db.dataloader.core.FileFormat; import java.io.File; import java.nio.file.Paths; +import com.scalar.db.dataloader.core.ScalarDbMode; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -53,4 +54,18 @@ void call_withInvalidScalarDBConfigurationFile_shouldReturnOne() throws Exceptio exportCommand.outputFormat = FileFormat.JSON; Assertions.assertEquals(1, exportCommand.call()); } + + @Test + void call_withScalarDBModeTransaction_WithInvalidConfigurationFile_shouldReturnOne() throws Exception { + ExportCommand exportCommand = new ExportCommand(); + exportCommand.configFilePath = "scalardb.properties"; + exportCommand.dataChunkSize = 100; + exportCommand.namespace = "scalar"; + exportCommand.table = "asset"; + exportCommand.outputDirectory = ""; + exportCommand.outputFileName = "sample.json"; + exportCommand.outputFormat = FileFormat.JSON; + exportCommand.scalarDbMode = ScalarDbMode.TRANSACTION; + Assertions.assertEquals(1, exportCommand.call()); + } } From 68059f8e21b7a6fa560d6c773aaf4f0b9ac870f4 Mon Sep 17 00:00:00 2001 From: Jishnu J Date: Thu, 5 Jun 2025 14:20:34 +0530 Subject: [PATCH 08/19] Added unit test for Dao --- .../core/dataimport/dao/ScalarDbDaoTest.java | 74 ++++++++++++++++++- 1 file changed, 73 insertions(+), 1 deletion(-) diff --git a/data-loader/core/src/test/java/com/scalar/db/dataloader/core/dataimport/dao/ScalarDbDaoTest.java b/data-loader/core/src/test/java/com/scalar/db/dataloader/core/dataimport/dao/ScalarDbDaoTest.java index cc1798e2f8..f39dc4f130 100644 --- a/data-loader/core/src/test/java/com/scalar/db/dataloader/core/dataimport/dao/ScalarDbDaoTest.java +++ b/data-loader/core/src/test/java/com/scalar/db/dataloader/core/dataimport/dao/ScalarDbDaoTest.java @@ -2,10 +2,22 @@ import static com.scalar.db.dataloader.core.UnitTestUtils.*; import static org.assertj.core.api.Assertions.assertThat; - +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.mockito.Mockito.withSettings; + +import com.scalar.db.api.DistributedStorage; +import com.scalar.db.api.DistributedTransactionManager; import com.scalar.db.api.Scan; import com.scalar.db.api.ScanBuilder; +import com.scalar.db.api.Scanner; +import com.scalar.db.api.TransactionManagerCrudOperable; import com.scalar.db.dataloader.core.ScanRange; +import com.scalar.db.exception.storage.ExecutionException; +import com.scalar.db.exception.transaction.CrudException; import com.scalar.db.io.Key; import java.util.*; import org.junit.jupiter.api.BeforeEach; @@ -15,10 +27,14 @@ class ScalarDbDaoTest { private static final int TEST_VALUE_INT_MIN = 1; private ScalarDbDao dao; + private DistributedTransactionManager distributedTransactionManager; + private DistributedStorage distributedStorage; @BeforeEach void setUp() { this.dao = new ScalarDbDao(); + this.distributedStorage = mock(DistributedStorage.class); + this.distributedTransactionManager = mock(DistributedTransactionManager.class); } @Test @@ -151,6 +167,62 @@ void createScan_scanAllWithLimitAndProjection_shouldCreateScanAllObjectWithLimit assertThat(scan.toString()).isEqualTo(expectedResult.toString()); } + @Test + void createScanner_withTransactionManager_ShouldCreateScannerObject() + throws CrudException, ScalarDbDaoException { + // Create Scan Object + TransactionManagerCrudOperable.Scanner mockScanner = + mock( + TransactionManagerCrudOperable.Scanner.class, + withSettings().extraInterfaces(Scanner.class)); + when(distributedTransactionManager.getScanner(any())).thenReturn(mockScanner); + Scanner result = + this.dao.createScanner( + TEST_NAMESPACE, + TEST_TABLE_NAME, + null, + new ScanRange(null, null, false, false), + new ArrayList<>(), + new ArrayList<>(), + 0, + distributedTransactionManager); + // Assert + assertNotNull(result); + assertEquals(mockScanner, result); + result = + this.dao.createScanner( + TEST_NAMESPACE, TEST_TABLE_NAME, null, 0, distributedTransactionManager); + // Assert + assertNotNull(result); + assertEquals(mockScanner, result); + } + + @Test + void createScanner_withStorage_ShouldCreateScannerObject() + throws CrudException, ExecutionException, ScalarDbDaoException { + // Create Scan Object + Scanner mockScanner = mock(Scanner.class); + when(distributedStorage.scan(any())).thenReturn(mockScanner); + Scanner result = + this.dao.createScanner( + TEST_NAMESPACE, + TEST_TABLE_NAME, + null, + new ScanRange(null, null, false, false), + new ArrayList<>(), + new ArrayList<>(), + 0, + distributedStorage); + // Assert + assertNotNull(result); + assertEquals(mockScanner, result); + + result = this.dao.createScanner(TEST_NAMESPACE, TEST_TABLE_NAME, null, 0, distributedStorage); + // Assert + assertNotNull(result); + assertEquals(mockScanner, result); + } + /** * Create Scan Object * From f4b0571436b0ab17c48866acdfcea2fce8e75019 Mon Sep 17 00:00:00 2001 From: Jishnu J Date: Thu, 5 Jun 2025 16:17:59 +0530 Subject: [PATCH 09/19] Changes --- .../cli/command/dataexport/ExportCommand.java | 28 +++++++ .../core/dataexport/ExportManager.java | 76 ++++++++++++----- .../core/dataimport/dao/ScalarDbDao.java | 13 ++- .../core/dataexport/CsvExportManagerTest.java | 84 ++++++++++--------- .../dataexport/JsonExportManagerTest.java | 13 ++- .../dataexport/JsonLineExportManagerTest.java | 13 ++- .../core/dataimport/dao/ScalarDbDaoTest.java | 22 ++--- 7 files changed, 162 insertions(+), 87 deletions(-) diff --git a/data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/command/dataexport/ExportCommand.java b/data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/command/dataexport/ExportCommand.java index 7b15b9ccdf..75dc9782d8 100755 --- a/data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/command/dataexport/ExportCommand.java +++ b/data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/command/dataexport/ExportCommand.java @@ -135,6 +135,16 @@ private TableMetadataService createTableMetadataService( return new TableMetadataService(storageFactory.getStorageAdmin()); } + /** + * Creates an {@link ExportManager} instance based on ScalarDB mode and file format. + * + * @param scalarDbMode The ScalarDB mode (TRANSACTION or STORAGE). + * @param scalarDbDao The DAO for accessing ScalarDB. + * @param fileFormat The output file format (CSV, JSON, JSONL). + * @param scalarDbPropertiesFilePath Path to the ScalarDB properties file. + * @return A configured {@link ExportManager}. + * @throws IOException If there is an error reading the properties file. + */ private ExportManager createExportManager( ScalarDbMode scalarDbMode, ScalarDbDao scalarDbDao, @@ -154,6 +164,15 @@ private ExportManager createExportManager( } } + /** + * Returns an {@link ExportManager} that uses {@link DistributedTransactionManager}. + * + * @param distributedTransactionManager distributed transaction manager object + * @param scalarDbDao The DAO for accessing ScalarDB. + * @param fileFormat The output file format (CSV, JSON, JSONL). + * @param taskFactory Producer task factory object + * @return A configured {@link ExportManager}. + */ private ExportManager createExportManagerWithTransaction( DistributedTransactionManager distributedTransactionManager, ScalarDbDao scalarDbDao, @@ -171,6 +190,15 @@ private ExportManager createExportManagerWithTransaction( } } + /** + * Returns an {@link ExportManager} that uses {@link DistributedStorage}. + * + * @param storage distributed storage object + * @param scalarDbDao The DAO for accessing ScalarDB. + * @param fileFormat The output file format (CSV, JSON, JSONL). + * @param taskFactory Producer task factory object + * @return A configured {@link ExportManager}. + */ private ExportManager createExportManagerWithStorage( DistributedStorage storage, ScalarDbDao scalarDbDao, diff --git a/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataexport/ExportManager.java b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataexport/ExportManager.java index 09a03edad5..a3299d5c8b 100644 --- a/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataexport/ExportManager.java +++ b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataexport/ExportManager.java @@ -1,6 +1,7 @@ package com.scalar.db.dataloader.core.dataexport; import com.scalar.db.api.DistributedStorage; +import com.scalar.db.api.DistributedTransaction; import com.scalar.db.api.DistributedTransactionManager; import com.scalar.db.api.Result; import com.scalar.db.api.Scanner; @@ -14,6 +15,7 @@ import com.scalar.db.dataloader.core.dataimport.dao.ScalarDbDao; import com.scalar.db.dataloader.core.dataimport.dao.ScalarDbDaoException; import com.scalar.db.dataloader.core.util.TableMetadataUtil; +import com.scalar.db.exception.transaction.TransactionException; import com.scalar.db.io.DataType; import java.io.BufferedWriter; import java.io.IOException; @@ -132,7 +134,7 @@ public ExportReport startExport( // TODO: handle this } processFooter(exportOptions, tableMetadata, bufferedWriter); - } catch (InterruptedException | IOException e) { + } catch (InterruptedException | IOException | TransactionException e) { logger.error("Error during export: {}", e.getMessage()); } finally { bufferedWriter.flush(); @@ -236,7 +238,7 @@ private Scanner createScanner( ScalarDbDao dao, DistributedStorage storage, DistributedTransactionManager transactionManager) - throws ScalarDbDaoException { + throws ScalarDbDaoException, TransactionException { if (exportOptions.getScalarDbMode().equals(ScalarDbMode.TRANSACTION)) { return createScannerWithTransaction(exportOptions, dao, transactionManager); } @@ -276,29 +278,63 @@ private Scanner createScannerWithStorage( } } + /** + * Creates a {@link Scanner} using a {@link DistributedTransaction} based on the provided export + * options. This method initiates a read-only transaction to ensure a consistent snapshot of the + * data during scan. + * + * @param exportOptions Options specifying how to scan the table (e.g., partition key, range, + * projection). + * @param dao The ScalarDb data access object to create the scanner. + * @param distributedTransactionManager The transaction manager used to start a new transaction. + * @return A {@link Scanner} for reading data from the specified table. + * @throws ScalarDbDaoException If an error occurs while creating the scanner. + * @throws TransactionException If an error occurs during transaction management (start or + * commit). + */ private Scanner createScannerWithTransaction( ExportOptions exportOptions, ScalarDbDao dao, DistributedTransactionManager distributedTransactionManager) - throws ScalarDbDaoException { + throws ScalarDbDaoException, TransactionException { + boolean isScanAll = exportOptions.getScanPartitionKey() == null; - if (isScanAll) { - return dao.createScanner( - exportOptions.getNamespace(), - exportOptions.getTableName(), - exportOptions.getProjectionColumns(), - exportOptions.getLimit(), - distributedTransactionManager); - } else { - return dao.createScanner( - exportOptions.getNamespace(), - exportOptions.getTableName(), - exportOptions.getScanPartitionKey(), - exportOptions.getScanRange(), - exportOptions.getSortOrders(), - exportOptions.getProjectionColumns(), - exportOptions.getLimit(), - distributedTransactionManager); + DistributedTransaction transaction = distributedTransactionManager.start(); + + try { + Scanner scanner; + if (isScanAll) { + scanner = + dao.createScanner( + exportOptions.getNamespace(), + exportOptions.getTableName(), + exportOptions.getProjectionColumns(), + exportOptions.getLimit(), + transaction); + } else { + scanner = + dao.createScanner( + exportOptions.getNamespace(), + exportOptions.getTableName(), + exportOptions.getScanPartitionKey(), + exportOptions.getScanRange(), + exportOptions.getSortOrders(), + exportOptions.getProjectionColumns(), + exportOptions.getLimit(), + transaction); + } + + transaction.commit(); + return scanner; + + } catch (Exception e) { + try { + transaction.abort(); + } catch (TransactionException abortException) { + logger.error( + "Failed to abort transaction: {}", abortException.getMessage(), abortException); + } + throw e; } } diff --git a/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/dao/ScalarDbDao.java b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/dao/ScalarDbDao.java index 6ef428d783..864c9e14ac 100644 --- a/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/dao/ScalarDbDao.java +++ b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/dao/ScalarDbDao.java @@ -2,7 +2,6 @@ import com.scalar.db.api.DistributedStorage; import com.scalar.db.api.DistributedTransaction; -import com.scalar.db.api.DistributedTransactionManager; import com.scalar.db.api.Get; import com.scalar.db.api.GetBuilder; import com.scalar.db.api.Put; @@ -256,7 +255,7 @@ public Scanner createScanner( * @param table ScalarDB table name * @param projectionColumns List of column projection to use during scan * @param limit Scan limit value - * @param distributedTransactionManager Distributed transaction manager object + * @param transaction Distributed transaction object * @return ScalarDB Scanner object * @throws ScalarDbDaoException if scan fails */ @@ -265,12 +264,12 @@ public Scanner createScanner( String table, List projectionColumns, int limit, - DistributedTransactionManager distributedTransactionManager) + DistributedTransaction transaction) throws ScalarDbDaoException { Scan scan = createScan(namespace, table, null, null, new ArrayList<>(), projectionColumns, limit); try { - return (Scanner) distributedTransactionManager.getScanner(scan); + return (Scanner) transaction.getScanner(scan); } catch (CrudException e) { throw new ScalarDbDaoException( CoreError.DATA_LOADER_ERROR_SCAN.buildMessage(e.getMessage()), e); @@ -321,7 +320,7 @@ public Scanner createScanner( * @param sortOrders Optional scan clustering key sorting values * @param projectionColumns List of column projection to use during scan * @param limit Scan limit value - * @param distributedTransactionManager Distributed transaction manager object + * @param transaction Distributed transaction object * @return ScalarDB Scanner object */ public Scanner createScanner( @@ -332,11 +331,11 @@ public Scanner createScanner( @Nullable List sortOrders, @Nullable List projectionColumns, int limit, - DistributedTransactionManager distributedTransactionManager) { + DistributedTransaction transaction) { Scan scan = createScan(namespace, table, partitionKey, scanRange, sortOrders, projectionColumns, limit); try { - return (Scanner) distributedTransactionManager.getScanner(scan); + return (Scanner) transaction.getScanner(scan); } catch (CrudException e) { throw new RuntimeException(e); } diff --git a/data-loader/core/src/test/java/com/scalar/db/dataloader/core/dataexport/CsvExportManagerTest.java b/data-loader/core/src/test/java/com/scalar/db/dataloader/core/dataexport/CsvExportManagerTest.java index 498102228f..8cb1b7478b 100644 --- a/data-loader/core/src/test/java/com/scalar/db/dataloader/core/dataexport/CsvExportManagerTest.java +++ b/data-loader/core/src/test/java/com/scalar/db/dataloader/core/dataexport/CsvExportManagerTest.java @@ -1,6 +1,9 @@ package com.scalar.db.dataloader.core.dataexport; +import static org.mockito.Mockito.when; + import com.scalar.db.api.DistributedStorage; +import com.scalar.db.api.DistributedTransaction; import com.scalar.db.api.DistributedTransactionManager; import com.scalar.db.api.Result; import com.scalar.db.api.Scanner; @@ -13,6 +16,7 @@ import com.scalar.db.dataloader.core.dataexport.producer.ProducerTaskFactory; import com.scalar.db.dataloader.core.dataimport.dao.ScalarDbDao; import com.scalar.db.dataloader.core.dataimport.dao.ScalarDbDaoException; +import com.scalar.db.exception.transaction.TransactionException; import com.scalar.db.io.Column; import com.scalar.db.io.IntColumn; import com.scalar.db.io.Key; @@ -36,16 +40,19 @@ public class CsvExportManagerTest { TableMetadata mockData; DistributedStorage storage; DistributedTransactionManager manager; + DistributedTransaction transaction; @Spy ScalarDbDao dao; ProducerTaskFactory producerTaskFactory; ExportManager exportManager; @BeforeEach - void setup() { + void setup() throws TransactionException { storage = Mockito.mock(DistributedStorage.class); manager = Mockito.mock(DistributedTransactionManager.class); + transaction = Mockito.mock(DistributedTransaction.class); mockData = UnitTestUtils.createTestTableMetadata(); dao = Mockito.mock(ScalarDbDao.class); + when(manager.start()).thenReturn(transaction); producerTaskFactory = new ProducerTaskFactory(null, false, true); } @@ -64,15 +71,14 @@ void startExport_givenValidDataWithoutPartitionKey_withStorage_shouldGenerateOut .scanRange(new ScanRange(null, null, false, false)) .build(); - Mockito.when( - dao.createScanner( - exportOptions.getNamespace(), - exportOptions.getTableName(), - exportOptions.getProjectionColumns(), - exportOptions.getLimit(), - storage)) + when(dao.createScanner( + exportOptions.getNamespace(), + exportOptions.getTableName(), + exportOptions.getProjectionColumns(), + exportOptions.getLimit(), + storage)) .thenReturn(scanner); - Mockito.when(scanner.iterator()).thenReturn(results.iterator()); + when(scanner.iterator()).thenReturn(results.iterator()); try (BufferedWriter writer = new BufferedWriter( Files.newBufferedWriter( @@ -108,18 +114,17 @@ void startExport_givenPartitionKey_withStorage_shouldGenerateOutputFile() .scanRange(new ScanRange(null, null, false, false)) .build(); - Mockito.when( - dao.createScanner( - exportOptions.getNamespace(), - exportOptions.getTableName(), - exportOptions.getScanPartitionKey(), - exportOptions.getScanRange(), - exportOptions.getSortOrders(), - exportOptions.getProjectionColumns(), - exportOptions.getLimit(), - storage)) + when(dao.createScanner( + exportOptions.getNamespace(), + exportOptions.getTableName(), + exportOptions.getScanPartitionKey(), + exportOptions.getScanRange(), + exportOptions.getSortOrders(), + exportOptions.getProjectionColumns(), + exportOptions.getLimit(), + storage)) .thenReturn(scanner); - Mockito.when(scanner.iterator()).thenReturn(results.iterator()); + when(scanner.iterator()).thenReturn(results.iterator()); try (BufferedWriter writer = new BufferedWriter( Files.newBufferedWriter( @@ -150,15 +155,14 @@ void startExport_givenValidDataWithoutPartitionKey_withTransaction_shouldGenerat .scalarDbMode(ScalarDbMode.TRANSACTION) .build(); - Mockito.when( - dao.createScanner( - exportOptions.getNamespace(), - exportOptions.getTableName(), - exportOptions.getProjectionColumns(), - exportOptions.getLimit(), - manager)) + when(dao.createScanner( + exportOptions.getNamespace(), + exportOptions.getTableName(), + exportOptions.getProjectionColumns(), + exportOptions.getLimit(), + transaction)) .thenReturn(scanner); - Mockito.when(scanner.iterator()).thenReturn(results.iterator()); + when(scanner.iterator()).thenReturn(results.iterator()); try (BufferedWriter writer = new BufferedWriter( Files.newBufferedWriter( @@ -174,8 +178,7 @@ void startExport_givenValidDataWithoutPartitionKey_withTransaction_shouldGenerat } @Test - void startExport_givenPartitionKey_withTransaction_shouldGenerateOutputFile() - throws IOException, ScalarDbDaoException { + void startExport_givenPartitionKey_withTransaction_shouldGenerateOutputFile() throws IOException { producerTaskFactory = new ProducerTaskFactory(",", false, false); exportManager = new CsvExportManager(manager, dao, producerTaskFactory); Scanner scanner = Mockito.mock(Scanner.class); @@ -195,18 +198,17 @@ void startExport_givenPartitionKey_withTransaction_shouldGenerateOutputFile() .scalarDbMode(ScalarDbMode.TRANSACTION) .build(); - Mockito.when( - dao.createScanner( - exportOptions.getNamespace(), - exportOptions.getTableName(), - exportOptions.getScanPartitionKey(), - exportOptions.getScanRange(), - exportOptions.getSortOrders(), - exportOptions.getProjectionColumns(), - exportOptions.getLimit(), - manager)) + when(dao.createScanner( + exportOptions.getNamespace(), + exportOptions.getTableName(), + exportOptions.getScanPartitionKey(), + exportOptions.getScanRange(), + exportOptions.getSortOrders(), + exportOptions.getProjectionColumns(), + exportOptions.getLimit(), + transaction)) .thenReturn(scanner); - Mockito.when(scanner.iterator()).thenReturn(results.iterator()); + when(scanner.iterator()).thenReturn(results.iterator()); try (BufferedWriter writer = new BufferedWriter( Files.newBufferedWriter( diff --git a/data-loader/core/src/test/java/com/scalar/db/dataloader/core/dataexport/JsonExportManagerTest.java b/data-loader/core/src/test/java/com/scalar/db/dataloader/core/dataexport/JsonExportManagerTest.java index e03815a429..7cbcd70a4d 100644 --- a/data-loader/core/src/test/java/com/scalar/db/dataloader/core/dataexport/JsonExportManagerTest.java +++ b/data-loader/core/src/test/java/com/scalar/db/dataloader/core/dataexport/JsonExportManagerTest.java @@ -1,6 +1,9 @@ package com.scalar.db.dataloader.core.dataexport; +import static org.mockito.Mockito.when; + import com.scalar.db.api.DistributedStorage; +import com.scalar.db.api.DistributedTransaction; import com.scalar.db.api.DistributedTransactionManager; import com.scalar.db.api.Result; import com.scalar.db.api.Scanner; @@ -13,6 +16,7 @@ import com.scalar.db.dataloader.core.dataexport.producer.ProducerTaskFactory; import com.scalar.db.dataloader.core.dataimport.dao.ScalarDbDao; import com.scalar.db.dataloader.core.dataimport.dao.ScalarDbDaoException; +import com.scalar.db.exception.transaction.TransactionException; import com.scalar.db.io.Column; import com.scalar.db.io.IntColumn; import com.scalar.db.io.Key; @@ -37,17 +41,20 @@ public class JsonExportManagerTest { TableMetadata mockData; DistributedStorage storage; DistributedTransactionManager manager; + DistributedTransaction transaction; @Spy ScalarDbDao dao; ProducerTaskFactory producerTaskFactory; ExportManager exportManager; @BeforeEach - void setup() { + void setup() throws TransactionException { storage = Mockito.mock(DistributedStorage.class); + transaction = Mockito.mock(DistributedTransaction.class); manager = Mockito.mock(DistributedTransactionManager.class); mockData = UnitTestUtils.createTestTableMetadata(); dao = Mockito.mock(ScalarDbDao.class); producerTaskFactory = new ProducerTaskFactory(null, false, true); + when(manager.start()).thenReturn(transaction); } @Test @@ -159,7 +166,7 @@ void startExport_givenPartitionKey_withStorage_shouldGenerateOutputFile() exportOptions.getTableName(), exportOptions.getProjectionColumns(), exportOptions.getLimit(), - manager)) + transaction)) .thenReturn(scanner); Mockito.when(scanner.iterator()).thenReturn(results.iterator()); try (BufferedWriter writer = @@ -205,7 +212,7 @@ void startExport_givenPartitionKey_withTransaction_shouldGenerateOutputFile() th exportOptions.getSortOrders(), exportOptions.getProjectionColumns(), exportOptions.getLimit(), - manager)) + transaction)) .thenReturn(scanner); Mockito.when(scanner.iterator()).thenReturn(results.iterator()); try (BufferedWriter writer = diff --git a/data-loader/core/src/test/java/com/scalar/db/dataloader/core/dataexport/JsonLineExportManagerTest.java b/data-loader/core/src/test/java/com/scalar/db/dataloader/core/dataexport/JsonLineExportManagerTest.java index 064cca7358..8e619bd08f 100644 --- a/data-loader/core/src/test/java/com/scalar/db/dataloader/core/dataexport/JsonLineExportManagerTest.java +++ b/data-loader/core/src/test/java/com/scalar/db/dataloader/core/dataexport/JsonLineExportManagerTest.java @@ -1,6 +1,9 @@ package com.scalar.db.dataloader.core.dataexport; +import static org.mockito.Mockito.when; + import com.scalar.db.api.DistributedStorage; +import com.scalar.db.api.DistributedTransaction; import com.scalar.db.api.DistributedTransactionManager; import com.scalar.db.api.Result; import com.scalar.db.api.Scanner; @@ -13,6 +16,7 @@ import com.scalar.db.dataloader.core.dataexport.producer.ProducerTaskFactory; import com.scalar.db.dataloader.core.dataimport.dao.ScalarDbDao; import com.scalar.db.dataloader.core.dataimport.dao.ScalarDbDaoException; +import com.scalar.db.exception.transaction.TransactionException; import com.scalar.db.io.Column; import com.scalar.db.io.IntColumn; import com.scalar.db.io.Key; @@ -35,18 +39,21 @@ public class JsonLineExportManagerTest { TableMetadata mockData; DistributedStorage storage; + DistributedTransaction transaction; DistributedTransactionManager manager; @Spy ScalarDbDao dao; ProducerTaskFactory producerTaskFactory; ExportManager exportManager; @BeforeEach - void setup() { + void setup() throws TransactionException { storage = Mockito.mock(DistributedStorage.class); + transaction = Mockito.mock(DistributedTransaction.class); manager = Mockito.mock(DistributedTransactionManager.class); mockData = UnitTestUtils.createTestTableMetadata(); dao = Mockito.mock(ScalarDbDao.class); producerTaskFactory = new ProducerTaskFactory(null, false, true); + when(manager.start()).thenReturn(transaction); } @Test @@ -158,7 +165,7 @@ void startExport_givenPartitionKey_withStorage_shouldGenerateOutputFile() exportOptions.getTableName(), exportOptions.getProjectionColumns(), exportOptions.getLimit(), - manager)) + transaction)) .thenReturn(scanner); Mockito.when(scanner.iterator()).thenReturn(results.iterator()); try (BufferedWriter writer = @@ -204,7 +211,7 @@ void startExport_givenPartitionKey_withTransaction_shouldGenerateOutputFile() th exportOptions.getSortOrders(), exportOptions.getProjectionColumns(), exportOptions.getLimit(), - manager)) + transaction)) .thenReturn(scanner); Mockito.when(scanner.iterator()).thenReturn(results.iterator()); try (BufferedWriter writer = diff --git a/data-loader/core/src/test/java/com/scalar/db/dataloader/core/dataimport/dao/ScalarDbDaoTest.java b/data-loader/core/src/test/java/com/scalar/db/dataloader/core/dataimport/dao/ScalarDbDaoTest.java index f39dc4f130..0aa74248db 100644 --- a/data-loader/core/src/test/java/com/scalar/db/dataloader/core/dataimport/dao/ScalarDbDaoTest.java +++ b/data-loader/core/src/test/java/com/scalar/db/dataloader/core/dataimport/dao/ScalarDbDaoTest.java @@ -10,11 +10,11 @@ import static org.mockito.Mockito.withSettings; import com.scalar.db.api.DistributedStorage; -import com.scalar.db.api.DistributedTransactionManager; +import com.scalar.db.api.DistributedTransaction; import com.scalar.db.api.Scan; import com.scalar.db.api.ScanBuilder; import com.scalar.db.api.Scanner; -import com.scalar.db.api.TransactionManagerCrudOperable; +import com.scalar.db.api.TransactionCrudOperable; import com.scalar.db.dataloader.core.ScanRange; import com.scalar.db.exception.storage.ExecutionException; import com.scalar.db.exception.transaction.CrudException; @@ -27,14 +27,14 @@ class ScalarDbDaoTest { private static final int TEST_VALUE_INT_MIN = 1; private ScalarDbDao dao; - private DistributedTransactionManager distributedTransactionManager; + private DistributedTransaction transaction; private DistributedStorage distributedStorage; @BeforeEach void setUp() { this.dao = new ScalarDbDao(); this.distributedStorage = mock(DistributedStorage.class); - this.distributedTransactionManager = mock(DistributedTransactionManager.class); + this.transaction = mock(DistributedTransaction.class); } @Test @@ -171,11 +171,9 @@ void createScan_scanAllWithLimitAndProjection_shouldCreateScanAllObjectWithLimit void createScanner_withTransactionManager_ShouldCreateScannerObject() throws CrudException, ScalarDbDaoException { // Create Scan Object - TransactionManagerCrudOperable.Scanner mockScanner = - mock( - TransactionManagerCrudOperable.Scanner.class, - withSettings().extraInterfaces(Scanner.class)); - when(distributedTransactionManager.getScanner(any())).thenReturn(mockScanner); + TransactionCrudOperable.Scanner mockScanner = + mock(TransactionCrudOperable.Scanner.class, withSettings().extraInterfaces(Scanner.class)); + when(transaction.getScanner(any())).thenReturn(mockScanner); Scanner result = this.dao.createScanner( TEST_NAMESPACE, @@ -185,13 +183,11 @@ void createScanner_withTransactionManager_ShouldCreateScannerObject() new ArrayList<>(), new ArrayList<>(), 0, - distributedTransactionManager); + transaction); // Assert assertNotNull(result); assertEquals(mockScanner, result); - result = - this.dao.createScanner( - TEST_NAMESPACE, TEST_TABLE_NAME, null, 0, distributedTransactionManager); + result = this.dao.createScanner(TEST_NAMESPACE, TEST_TABLE_NAME, null, 0, transaction); // Assert assertNotNull(result); assertEquals(mockScanner, result); From 735c47e4c34696c03ac421f33d39ce23b8cfa87a Mon Sep 17 00:00:00 2001 From: Jishnu J Date: Thu, 5 Jun 2025 16:49:21 +0530 Subject: [PATCH 10/19] Updated javadoc --- .../core/dataexport/ExportManager.java | 43 +++++++++++++++---- 1 file changed, 35 insertions(+), 8 deletions(-) diff --git a/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataexport/ExportManager.java b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataexport/ExportManager.java index a3299d5c8b..6cc5ee4273 100644 --- a/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataexport/ExportManager.java +++ b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataexport/ExportManager.java @@ -233,6 +233,23 @@ private void handleTransactionMetadata(ExportOptions exportOptions, TableMetadat } } + /** + * Creates a ScalarDB {@link Scanner} instance based on the configured ScalarDB mode. + * + *

If the {@link ScalarDbMode} specified in {@code exportOptions} is {@code TRANSACTION}, a + * scanner is created using the {@link DistributedTransactionManager}. Otherwise, a scanner is + * created using the {@link DistributedStorage}. + * + * @param exportOptions Options containing configuration for the export operation, including the + * ScalarDB mode + * @param dao The {@link ScalarDbDao} used to access ScalarDB + * @param storage The {@link DistributedStorage} instance used for storage-level operations + * @param transactionManager The {@link DistributedTransactionManager} instance used for + * transaction-level operations + * @return A {@link Scanner} instance for reading data from ScalarDB + * @throws ScalarDbDaoException If an error occurs while creating the scanner with the DAO + * @throws TransactionException If an error occurs during transactional scanner creation + */ private Scanner createScanner( ExportOptions exportOptions, ScalarDbDao dao, @@ -246,13 +263,20 @@ private Scanner createScanner( } /** - * To create a scanner object + * Creates a ScalarDB {@link Scanner} using the {@link DistributedStorage} interface based on the + * scan configuration provided in {@link ExportOptions}. * - * @param exportOptions export options - * @param dao ScalarDB dao object - * @param storage distributed storage object - * @return created scanner - * @throws ScalarDbDaoException throws if any issue occurs in creating scanner object + *

If no partition key is specified in the {@code exportOptions}, a full table scan is + * performed. Otherwise, a partition-specific scan is performed using the provided partition key, + * optional scan range, and sort orders. + * + * @param exportOptions Options containing configuration for the export operation, including + * namespace, table name, projection columns, limit, and scan parameters + * @param dao The {@link ScalarDbDao} used to construct the scan operation + * @param storage The {@link DistributedStorage} instance used to execute the scan + * @return A {@link Scanner} instance for reading data from ScalarDB using storage-level + * operations + * @throws ScalarDbDaoException If an error occurs while creating the scanner */ private Scanner createScannerWithStorage( ExportOptions exportOptions, ScalarDbDao dao, DistributedStorage storage) @@ -280,8 +304,11 @@ private Scanner createScannerWithStorage( /** * Creates a {@link Scanner} using a {@link DistributedTransaction} based on the provided export - * options. This method initiates a read-only transaction to ensure a consistent snapshot of the - * data during scan. + * options. + * + *

If no partition key is specified in the {@code exportOptions}, a full table scan is + * performed. Otherwise, a partition-specific scan is performed using the provided partition key, + * optional scan range, and sort orders. * * @param exportOptions Options specifying how to scan the table (e.g., partition key, range, * projection). From 43ef7bc347fbfee452cdc82fc49ac047932bee69 Mon Sep 17 00:00:00 2001 From: Jishnu J Date: Thu, 5 Jun 2025 16:57:19 +0530 Subject: [PATCH 11/19] Spotless applied on CLI --- .../dataloader/cli/command/dataexport/ExportCommandTest.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/data-loader/cli/src/test/java/com/scalar/db/dataloader/cli/command/dataexport/ExportCommandTest.java b/data-loader/cli/src/test/java/com/scalar/db/dataloader/cli/command/dataexport/ExportCommandTest.java index 65cd9e137a..7713255572 100755 --- a/data-loader/cli/src/test/java/com/scalar/db/dataloader/cli/command/dataexport/ExportCommandTest.java +++ b/data-loader/cli/src/test/java/com/scalar/db/dataloader/cli/command/dataexport/ExportCommandTest.java @@ -4,9 +4,9 @@ import com.scalar.db.common.error.CoreError; import com.scalar.db.dataloader.core.FileFormat; +import com.scalar.db.dataloader.core.ScalarDbMode; import java.io.File; import java.nio.file.Paths; -import com.scalar.db.dataloader.core.ScalarDbMode; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -56,7 +56,8 @@ void call_withInvalidScalarDBConfigurationFile_shouldReturnOne() throws Exceptio } @Test - void call_withScalarDBModeTransaction_WithInvalidConfigurationFile_shouldReturnOne() throws Exception { + void call_withScalarDBModeTransaction_WithInvalidConfigurationFile_shouldReturnOne() + throws Exception { ExportCommand exportCommand = new ExportCommand(); exportCommand.configFilePath = "scalardb.properties"; exportCommand.dataChunkSize = 100; From cbba68c3af035efb34cdcaaded944b4e36c47da6 Mon Sep 17 00:00:00 2001 From: Jishnu J Date: Wed, 11 Jun 2025 09:47:42 +0530 Subject: [PATCH 12/19] Fix --- .../cli/command/dataexport/ExportCommand.java | 2 +- .../ConsoleImportProgressListener.java | 128 ++++++++++++++++++ .../command/dataimport/ImportListener.java | 111 +++++++++++++++ 3 files changed, 240 insertions(+), 1 deletion(-) create mode 100644 data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/command/dataimport/ConsoleImportProgressListener.java create mode 100644 data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/command/dataimport/ImportListener.java diff --git a/data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/command/dataexport/ExportCommand.java b/data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/command/dataexport/ExportCommand.java index 02c12399af..94fe3a9bf9 100755 --- a/data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/command/dataexport/ExportCommand.java +++ b/data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/command/dataexport/ExportCommand.java @@ -159,7 +159,7 @@ private ExportManager createExportManager( throws IOException { ProducerTaskFactory taskFactory = new ProducerTaskFactory(delimiter, includeTransactionMetadata, prettyPrintJson); - if (scalarDbMode.equals(ScalarDbMode.TRANSACTION)) { + if (scalarDbMode.equals(ScalarDbMode.STORAGE)) { DistributedStorage storage = StorageFactory.create(scalarDbPropertiesFilePath).getStorage(); return createExportManagerWithStorage(storage, scalarDbDao, fileFormat, taskFactory); } else { diff --git a/data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/command/dataimport/ConsoleImportProgressListener.java b/data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/command/dataimport/ConsoleImportProgressListener.java new file mode 100644 index 0000000000..c647339a9d --- /dev/null +++ b/data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/command/dataimport/ConsoleImportProgressListener.java @@ -0,0 +1,128 @@ +package com.scalar.db.dataloader.cli.command.dataimport; + +import com.scalar.db.dataloader.core.dataimport.ImportEventListener; +import com.scalar.db.dataloader.core.dataimport.datachunk.ImportDataChunkStatus; +import com.scalar.db.dataloader.core.dataimport.task.result.ImportTaskResult; +import com.scalar.db.dataloader.core.dataimport.transactionbatch.ImportTransactionBatchResult; +import com.scalar.db.dataloader.core.dataimport.transactionbatch.ImportTransactionBatchStatus; +import java.time.Duration; +import java.util.Map; +import java.util.concurrent.*; +import java.util.concurrent.atomic.AtomicLong; + +public class ConsoleImportProgressListener implements ImportEventListener { + + private final ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor(); + private final Duration updateInterval; + private final long startTime; + private final Map chunkLogs = new ConcurrentHashMap<>(); + private final Map chunkFailureLogs = new ConcurrentHashMap<>(); + private final AtomicLong totalRecords = new AtomicLong(); + private volatile boolean completed = false; + + public ConsoleImportProgressListener(Duration updateInterval) { + this.updateInterval = updateInterval; + this.startTime = System.currentTimeMillis(); + scheduler.scheduleAtFixedRate( + this::render, 0, updateInterval.toMillis(), TimeUnit.MILLISECONDS); + } + + @Override + public void onDataChunkStarted(ImportDataChunkStatus status) { + chunkLogs.put( + status.getDataChunkId(), + String.format( + "🔄 Chunk %d: Processing... %d records so far", + status.getDataChunkId(), status.getTotalRecords())); + } + + @Override + public void onDataChunkCompleted(ImportDataChunkStatus status) { + long elapsed = System.currentTimeMillis() - status.getStartTime().toEpochMilli(); + totalRecords.addAndGet(status.getTotalRecords()); + if (status.getSuccessCount() > 0) { + chunkLogs.put( + status.getDataChunkId(), + String.format( + "✓ Chunk %d: %,d records imported (%.1fs), %d records imported successfully, import of %d records failed", + status.getDataChunkId(), + status.getTotalRecords(), + elapsed / 1000.0, + status.getSuccessCount(), + status.getFailureCount())); + } + // if (status.getFailureCount() > 0) { + // chunkFailureLogs.put( + // status.getDataChunkId(), + // String.format( + // "❌ Chunk %d: Failed - %d records failed to be imported) ", + // status.getDataChunkId(), status.getFailureCount())); + // } + } + + @Override + public void onAllDataChunksCompleted() { + completed = true; + scheduler.shutdown(); + render(); // Final render + } + + @Override + public void onTransactionBatchStarted(ImportTransactionBatchStatus batchStatus) { + // Optional: Implement if you want to show more granular batch progress + } + + @Override + public void onTransactionBatchCompleted(ImportTransactionBatchResult batchResult) { + if (!batchResult.isSuccess()) { + chunkFailureLogs.put( + batchResult.getDataChunkId(), + String.format( + "❌ Chunk %d: Transaction batch %d Failed - %d records failed to be imported) ", + batchResult.getDataChunkId(), + batchResult.getTransactionBatchId(), + batchResult.getRecords().size())); + } + // Optional: Implement error reporting or success/failure count + } + + @Override + public void onTaskComplete(ImportTaskResult taskResult) { + // Optional: Summary or stats after final chunk + } + + private void render() { + StringBuilder builder = new StringBuilder(); + long now = System.currentTimeMillis(); + long elapsed = now - startTime; + double recPerSec = (totalRecords.get() * 1000.0) / (elapsed == 0 ? 1 : elapsed); + + builder.append( + String.format( + "\rImporting... %,d records | %.0f rec/s | %s\n", + totalRecords.get(), recPerSec, formatElapsed(elapsed))); + + chunkLogs.values().stream() + .sorted() // Optional: stable ordering + .forEach(line -> builder.append(line).append("\n")); + chunkFailureLogs.values().stream() + .sorted() // Optional: stable ordering + .forEach(line -> builder.append(line).append("\n")); + + clearConsole(); + System.out.print(builder); + System.out.flush(); + } + + private String formatElapsed(long elapsedMillis) { + long seconds = (elapsedMillis / 1000) % 60; + long minutes = (elapsedMillis / 1000) / 60; + return String.format("%dm %ds elapsed", minutes, seconds); + } + + private void clearConsole() { + // Clear screen for updated multiline rendering + System.out.print("\033[H\033[2J"); // ANSI escape for clearing screen + System.out.flush(); + } +} diff --git a/data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/command/dataimport/ImportListener.java b/data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/command/dataimport/ImportListener.java new file mode 100644 index 0000000000..b45a13e0b6 --- /dev/null +++ b/data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/command/dataimport/ImportListener.java @@ -0,0 +1,111 @@ +package com.scalar.db.dataloader.cli.command.dataimport; + +import com.scalar.db.dataloader.core.dataimport.ImportEventListener; +import com.scalar.db.dataloader.core.dataimport.datachunk.ImportDataChunkStatus; +import com.scalar.db.dataloader.core.dataimport.task.result.ImportTaskResult; +import com.scalar.db.dataloader.core.dataimport.transactionbatch.ImportTransactionBatchResult; +import com.scalar.db.dataloader.core.dataimport.transactionbatch.ImportTransactionBatchStatus; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicInteger; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class ImportListener implements ImportEventListener { + + private static final Logger logger = LoggerFactory.getLogger(ImportListener.class); + private final AtomicInteger totalSuccessImportCount = new AtomicInteger(0); + private final AtomicInteger totalFailureImportCount = new AtomicInteger(0); + private final ConcurrentHashMap dataChunkImportCountMap = + new ConcurrentHashMap<>(); + private static final String DATA_CHUNK_COMPLETED_SUCCESS_MSG = + "\u2713 Chunk %d: %d records imported (%ds) successfully"; + private static final String DATA_CHUNK_COMPLETED_FAILURE_MSG = + "\u274C Chunk %d: %d records failed to import"; + private static final String DATA_CHUNK_IN_PROCESS_MSG = + "\uD83D\uDD04 Chunk %d: Processing... %d records so far imported"; + + /** + * Called when processing of a data chunk begins. + * + * @param status the current status of the data chunk being processed + */ + @Override + public void onDataChunkStarted(ImportDataChunkStatus status) {} + + /** + * Called when processing of a data chunk is completed. + * + * @param status the final status of the completed data chunk + */ + @Override + public void onDataChunkCompleted(ImportDataChunkStatus status) { + if (status.getSuccessCount() > 0) { + logger.info( + String.format( + DATA_CHUNK_COMPLETED_SUCCESS_MSG, + status.getDataChunkId(), + status.getSuccessCount(), + status.getTotalDurationInMilliSeconds() / 1000)); + this.totalSuccessImportCount.addAndGet(status.getSuccessCount()); + } + if (status.getFailureCount() > 0) { + logger.info( + String.format( + DATA_CHUNK_COMPLETED_FAILURE_MSG, status.getDataChunkId(), status.getFailureCount())); + this.totalFailureImportCount.addAndGet(status.getFailureCount()); + } + } + + /** + * Called when all data chunks have been processed. This indicates that the entire chunked import + * process is complete. + */ + @Override + public void onAllDataChunksCompleted() { + System.out.println("Logger Factory: " + LoggerFactory.getILoggerFactory().getClass()); + if (totalSuccessImportCount.get() > 0) {} + } + + /** + * Called when processing of a transaction batch begins. + * + * @param batchStatus the initial status of the transaction batch + */ + @Override + public void onTransactionBatchStarted(ImportTransactionBatchStatus batchStatus) {} + + /** + * Called when processing of a transaction batch is completed. + * + * @param batchResult the result of the completed transaction batch + */ + @Override + public void onTransactionBatchCompleted(ImportTransactionBatchResult batchResult) { + if (batchResult.isSuccess()) { + Integer dataChunkId = batchResult.getDataChunkId(); + dataChunkImportCountMap.compute( + dataChunkId, + (k, v) -> { + if (v == null) { + return new AtomicInteger(batchResult.getRecords().size()); + } else { + v.addAndGet(batchResult.getRecords().size()); + return v; + } + }); + logger.info( + String.format( + DATA_CHUNK_IN_PROCESS_MSG, + dataChunkId, + dataChunkImportCountMap.get(dataChunkId).get())); + } + } + + /** + * Called when an import task is completed. + * + * @param taskResult the result of the completed import task + */ + @Override + public void onTaskComplete(ImportTaskResult taskResult) {} +} From 1c62a14c1b4c4e7cb7b094cf2acfcc86ec7ef8e0 Mon Sep 17 00:00:00 2001 From: Jishnu J Date: Wed, 11 Jun 2025 10:54:53 +0530 Subject: [PATCH 13/19] Changes --- .../cli/command/dataexport/ExportCommand.java | 5 +- .../core/dataexport/ExportManager.java | 275 ++++++++++-------- .../dataexport/ScannerWithTransaction.java | 11 + .../core/dataimport/dao/ScalarDbDao.java | 9 +- .../core/dataexport/CsvExportManagerTest.java | 5 +- .../dataexport/JsonExportManagerTest.java | 5 +- .../dataexport/JsonLineExportManagerTest.java | 5 +- .../core/dataimport/dao/ScalarDbDaoTest.java | 6 +- 8 files changed, 190 insertions(+), 131 deletions(-) create mode 100644 data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataexport/ScannerWithTransaction.java diff --git a/data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/command/dataexport/ExportCommand.java b/data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/command/dataexport/ExportCommand.java index 94fe3a9bf9..9d8f9fb684 100755 --- a/data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/command/dataexport/ExportCommand.java +++ b/data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/command/dataexport/ExportCommand.java @@ -72,7 +72,7 @@ public Integer call() throws Exception { ExportManager exportManager = createExportManager(scalarDbMode, scalarDbDao, outputFormat, scalarDbPropertiesFilePath); - + System.out.println(scalarDbMode); TableMetadata tableMetadata = tableMetadataService.getTableMetadata(namespace, table); Key partitionKey = @@ -233,7 +233,8 @@ private ExportOptions buildExportOptions(Key partitionKey, ScanRange scanRange) .maxThreadCount(maxThreads) .dataChunkSize(dataChunkSize) .prettyPrintJson(prettyPrintJson) - .scanRange(scanRange); + .scanRange(scanRange) + .scalarDbMode(scalarDbMode); if (projectionColumns != null) { builder.projectionColumns(projectionColumns); diff --git a/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataexport/ExportManager.java b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataexport/ExportManager.java index 6cc5ee4273..b0e94472b3 100644 --- a/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataexport/ExportManager.java +++ b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataexport/ExportManager.java @@ -6,6 +6,7 @@ import com.scalar.db.api.Result; import com.scalar.db.api.Scanner; import com.scalar.db.api.TableMetadata; +import com.scalar.db.api.TransactionCrudOperable; import com.scalar.db.dataloader.core.FileFormat; import com.scalar.db.dataloader.core.ScalarDbMode; import com.scalar.db.dataloader.core.dataexport.producer.ProducerTask; @@ -92,60 +93,141 @@ abstract void processFooter( public ExportReport startExport( ExportOptions exportOptions, TableMetadata tableMetadata, Writer writer) { ExportReport exportReport = new ExportReport(); + ExecutorService executorService = null; + try { validateExportOptions(exportOptions, tableMetadata); - Map dataTypeByColumnName = tableMetadata.getColumnDataTypes(); handleTransactionMetadata(exportOptions, tableMetadata); - processHeader(exportOptions, tableMetadata, writer); - int maxThreadCount = - exportOptions.getMaxThreadCount() == 0 - ? Runtime.getRuntime().availableProcessors() - : exportOptions.getMaxThreadCount(); - ExecutorService executorService = Executors.newFixedThreadPool(maxThreadCount); + try (BufferedWriter bufferedWriter = new BufferedWriter(writer)) { + processHeader(exportOptions, tableMetadata, bufferedWriter); - BufferedWriter bufferedWriter = new BufferedWriter(writer); - boolean isJson = exportOptions.getOutputFileFormat() == FileFormat.JSON; + int threadCount = + exportOptions.getMaxThreadCount() > 0 + ? exportOptions.getMaxThreadCount() + : Runtime.getRuntime().availableProcessors(); + executorService = Executors.newFixedThreadPool(threadCount); - try (Scanner scanner = - createScanner(exportOptions, dao, distributedStorage, distributedTransactionManager)) { - Iterator iterator = scanner.iterator(); AtomicBoolean isFirstBatch = new AtomicBoolean(true); + Map dataTypeByColumnName = tableMetadata.getColumnDataTypes(); - while (iterator.hasNext()) { - List dataChunk = fetchDataChunk(iterator, exportOptions.getDataChunkSize()); - executorService.submit( - () -> - processDataChunk( - exportOptions, - tableMetadata, - dataTypeByColumnName, - dataChunk, - bufferedWriter, - isJson, - isFirstBatch, - exportReport)); - } - executorService.shutdown(); - if (executorService.awaitTermination(Long.MAX_VALUE, TimeUnit.NANOSECONDS)) { - logger.info("All tasks completed"); - } else { - logger.error("Timeout occurred while waiting for tasks to complete"); - // TODO: handle this + if (exportOptions.getScalarDbMode() == ScalarDbMode.STORAGE) { + try (Scanner scanner = createScannerWithStorage(exportOptions, dao, distributedStorage)) { + submitTasks( + scanner.iterator(), + executorService, + exportOptions, + tableMetadata, + dataTypeByColumnName, + bufferedWriter, + isFirstBatch, + exportReport); + } + } else if (exportOptions.getScalarDbMode() == ScalarDbMode.TRANSACTION + && distributedTransactionManager != null) { + ScannerWithTransaction scannerWithTx = + createScannerWithTransaction(exportOptions, dao, distributedTransactionManager); + + try (TransactionCrudOperable.Scanner scanner = scannerWithTx.getScanner()) { + submitTasks( + scanner.iterator(), + executorService, + exportOptions, + tableMetadata, + dataTypeByColumnName, + bufferedWriter, + isFirstBatch, + exportReport); + } finally { + scannerWithTx.getTransaction().commit(); + } } + + shutdownExecutor(executorService); processFooter(exportOptions, tableMetadata, bufferedWriter); - } catch (InterruptedException | IOException | TransactionException e) { - logger.error("Error during export: {}", e.getMessage()); - } finally { - bufferedWriter.flush(); } - } catch (ExportOptionsValidationException | IOException | ScalarDbDaoException e) { - logger.error("Error during export: {}", e.getMessage()); + + } catch (Exception e) { + logger.error("Export failed", e); + } finally { + if (executorService != null && !executorService.isShutdown()) { + executorService.shutdownNow(); + } + closeResources(); } - closeResources(); + return exportReport; } + /** + * Submits asynchronous tasks for processing chunks of data to the given executor service. + * + *

This method reads data from the provided {@code iterator} in chunks (based on the configured + * chunk size) and submits each chunk as a separate task for processing. Each task invokes {@code + * processDataChunk()} to write the data to the output format. + * + *

Any exceptions thrown during chunk processing are logged but do not halt the submission of + * other tasks. + * + * @param iterator the iterator over database results + * @param executorService the executor service to run the processing tasks + * @param exportOptions configuration for export operation + * @param tableMetadata metadata for the table being exported + * @param dataTypeByColumnName mapping of column names to their data types + * @param writer the writer to which export output is written + * @param isFirstBatch an atomic flag used to track if the current chunk is the first one (used + * for formatting) + * @param exportReport the report object that accumulates export statistics + */ + private void submitTasks( + Iterator iterator, + ExecutorService executorService, + ExportOptions exportOptions, + TableMetadata tableMetadata, + Map dataTypeByColumnName, + BufferedWriter writer, + AtomicBoolean isFirstBatch, + ExportReport exportReport) { + while (iterator.hasNext()) { + List chunk = fetchDataChunk(iterator, exportOptions.getDataChunkSize()); + executorService.submit( + () -> { + try { + processDataChunk( + exportOptions, + tableMetadata, + dataTypeByColumnName, + chunk, + writer, + exportOptions.getOutputFileFormat() == FileFormat.JSON, + isFirstBatch, + exportReport); + } catch (Exception e) { + logger.error("Error processing data chunk", e); + } + }); + } + } + + /** + * Shuts down the given executor service gracefully, waiting for tasks to complete. + * + *

This method initiates an orderly shutdown where previously submitted tasks are executed, but + * no new tasks will be accepted. It then waits for all tasks to finish within a specified + * timeout. If the tasks do not complete in time, a warning is logged. + * + * @param executorService the ExecutorService to shut down + * @throws InterruptedException if the current thread is interrupted while waiting + */ + private void shutdownExecutor(ExecutorService executorService) throws InterruptedException { + executorService.shutdown(); + if (!executorService.awaitTermination(60, TimeUnit.MINUTES)) { + logger.warn("Timeout while waiting for export tasks to finish."); + } else { + logger.info("All export tasks completed."); + } + } + /** * To process result data chunk * @@ -233,35 +315,6 @@ private void handleTransactionMetadata(ExportOptions exportOptions, TableMetadat } } - /** - * Creates a ScalarDB {@link Scanner} instance based on the configured ScalarDB mode. - * - *

If the {@link ScalarDbMode} specified in {@code exportOptions} is {@code TRANSACTION}, a - * scanner is created using the {@link DistributedTransactionManager}. Otherwise, a scanner is - * created using the {@link DistributedStorage}. - * - * @param exportOptions Options containing configuration for the export operation, including the - * ScalarDB mode - * @param dao The {@link ScalarDbDao} used to access ScalarDB - * @param storage The {@link DistributedStorage} instance used for storage-level operations - * @param transactionManager The {@link DistributedTransactionManager} instance used for - * transaction-level operations - * @return A {@link Scanner} instance for reading data from ScalarDB - * @throws ScalarDbDaoException If an error occurs while creating the scanner with the DAO - * @throws TransactionException If an error occurs during transactional scanner creation - */ - private Scanner createScanner( - ExportOptions exportOptions, - ScalarDbDao dao, - DistributedStorage storage, - DistributedTransactionManager transactionManager) - throws ScalarDbDaoException, TransactionException { - if (exportOptions.getScalarDbMode().equals(ScalarDbMode.TRANSACTION)) { - return createScannerWithTransaction(exportOptions, dao, transactionManager); - } - return createScannerWithStorage(exportOptions, dao, storage); - } - /** * Creates a ScalarDB {@link Scanner} using the {@link DistributedStorage} interface based on the * scan configuration provided in {@link ExportOptions}. @@ -303,23 +356,27 @@ private Scanner createScannerWithStorage( } /** - * Creates a {@link Scanner} using a {@link DistributedTransaction} based on the provided export - * options. + * Creates a {@link ScannerWithTransaction} object that encapsulates a transactional scanner and + * its associated transaction for reading data from a ScalarDB table. * - *

If no partition key is specified in the {@code exportOptions}, a full table scan is - * performed. Otherwise, a partition-specific scan is performed using the provided partition key, + *

If no partition key is provided in the {@link ExportOptions}, a full table scan is + * performed. Otherwise, a partition-specific scan is created using the provided partition key, * optional scan range, and sort orders. * - * @param exportOptions Options specifying how to scan the table (e.g., partition key, range, - * projection). - * @param dao The ScalarDb data access object to create the scanner. - * @param distributedTransactionManager The transaction manager used to start a new transaction. - * @return A {@link Scanner} for reading data from the specified table. - * @throws ScalarDbDaoException If an error occurs while creating the scanner. - * @throws TransactionException If an error occurs during transaction management (start or - * commit). + *

The method starts a new transaction using the given {@link DistributedTransactionManager}, + * which will be associated with the returned scanner. This allows data export operations to be + * executed in a consistent transactional context. + * + * @param exportOptions the options specifying how to scan the table, such as namespace, table + * name, projection columns, scan partition key, range, sort orders, and limit. + * @param dao the {@link ScalarDbDao} used to construct the transactional scanner. + * @param distributedTransactionManager the transaction manager used to start a new transaction. + * @return a {@link ScannerWithTransaction} instance that wraps both the transaction and the + * scanner. + * @throws ScalarDbDaoException if an error occurs while creating the scanner with the DAO. + * @throws TransactionException if an error occurs when starting the transaction. */ - private Scanner createScannerWithTransaction( + private ScannerWithTransaction createScannerWithTransaction( ExportOptions exportOptions, ScalarDbDao dao, DistributedTransactionManager distributedTransactionManager) @@ -328,41 +385,29 @@ private Scanner createScannerWithTransaction( boolean isScanAll = exportOptions.getScanPartitionKey() == null; DistributedTransaction transaction = distributedTransactionManager.start(); - try { - Scanner scanner; - if (isScanAll) { - scanner = - dao.createScanner( - exportOptions.getNamespace(), - exportOptions.getTableName(), - exportOptions.getProjectionColumns(), - exportOptions.getLimit(), - transaction); - } else { - scanner = - dao.createScanner( - exportOptions.getNamespace(), - exportOptions.getTableName(), - exportOptions.getScanPartitionKey(), - exportOptions.getScanRange(), - exportOptions.getSortOrders(), - exportOptions.getProjectionColumns(), - exportOptions.getLimit(), - transaction); - } - - transaction.commit(); - return scanner; - - } catch (Exception e) { - try { - transaction.abort(); - } catch (TransactionException abortException) { - logger.error( - "Failed to abort transaction: {}", abortException.getMessage(), abortException); - } - throw e; + TransactionCrudOperable.Scanner scanner; + if (isScanAll) { + scanner = + dao.createScanner( + exportOptions.getNamespace(), + exportOptions.getTableName(), + exportOptions.getProjectionColumns(), + exportOptions.getLimit(), + transaction); + } else { + scanner = + dao.createScanner( + exportOptions.getNamespace(), + exportOptions.getTableName(), + exportOptions.getScanPartitionKey(), + exportOptions.getScanRange(), + exportOptions.getSortOrders(), + exportOptions.getProjectionColumns(), + exportOptions.getLimit(), + transaction); } + + return new ScannerWithTransaction(transaction, scanner); } /** Close resources properly once the process is completed */ diff --git a/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataexport/ScannerWithTransaction.java b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataexport/ScannerWithTransaction.java new file mode 100644 index 0000000000..dffaa428cd --- /dev/null +++ b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataexport/ScannerWithTransaction.java @@ -0,0 +1,11 @@ +package com.scalar.db.dataloader.core.dataexport; + +import com.scalar.db.api.DistributedTransaction; +import com.scalar.db.api.TransactionCrudOperable; +import lombok.Value; + +@Value +public class ScannerWithTransaction { + DistributedTransaction transaction; + TransactionCrudOperable.Scanner scanner; +} diff --git a/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/dao/ScalarDbDao.java b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/dao/ScalarDbDao.java index 864c9e14ac..ddff5ef101 100644 --- a/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/dao/ScalarDbDao.java +++ b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/dao/ScalarDbDao.java @@ -10,6 +10,7 @@ import com.scalar.db.api.Scan; import com.scalar.db.api.ScanBuilder; import com.scalar.db.api.Scanner; +import com.scalar.db.api.TransactionCrudOperable; import com.scalar.db.common.error.CoreError; import com.scalar.db.dataloader.core.ScanRange; import com.scalar.db.exception.storage.ExecutionException; @@ -259,7 +260,7 @@ public Scanner createScanner( * @return ScalarDB Scanner object * @throws ScalarDbDaoException if scan fails */ - public Scanner createScanner( + public TransactionCrudOperable.Scanner createScanner( String namespace, String table, List projectionColumns, @@ -269,7 +270,7 @@ public Scanner createScanner( Scan scan = createScan(namespace, table, null, null, new ArrayList<>(), projectionColumns, limit); try { - return (Scanner) transaction.getScanner(scan); + return transaction.getScanner(scan); } catch (CrudException e) { throw new ScalarDbDaoException( CoreError.DATA_LOADER_ERROR_SCAN.buildMessage(e.getMessage()), e); @@ -323,7 +324,7 @@ public Scanner createScanner( * @param transaction Distributed transaction object * @return ScalarDB Scanner object */ - public Scanner createScanner( + public TransactionCrudOperable.Scanner createScanner( String namespace, String table, @Nullable Key partitionKey, @@ -335,7 +336,7 @@ public Scanner createScanner( Scan scan = createScan(namespace, table, partitionKey, scanRange, sortOrders, projectionColumns, limit); try { - return (Scanner) transaction.getScanner(scan); + return transaction.getScanner(scan); } catch (CrudException e) { throw new RuntimeException(e); } diff --git a/data-loader/core/src/test/java/com/scalar/db/dataloader/core/dataexport/CsvExportManagerTest.java b/data-loader/core/src/test/java/com/scalar/db/dataloader/core/dataexport/CsvExportManagerTest.java index 8cb1b7478b..9f39ff4915 100644 --- a/data-loader/core/src/test/java/com/scalar/db/dataloader/core/dataexport/CsvExportManagerTest.java +++ b/data-loader/core/src/test/java/com/scalar/db/dataloader/core/dataexport/CsvExportManagerTest.java @@ -8,6 +8,7 @@ import com.scalar.db.api.Result; import com.scalar.db.api.Scanner; import com.scalar.db.api.TableMetadata; +import com.scalar.db.api.TransactionCrudOperable; import com.scalar.db.common.ResultImpl; import com.scalar.db.dataloader.core.FileFormat; import com.scalar.db.dataloader.core.ScalarDbMode; @@ -143,7 +144,7 @@ void startExport_givenPartitionKey_withStorage_shouldGenerateOutputFile() void startExport_givenValidDataWithoutPartitionKey_withTransaction_shouldGenerateOutputFile() throws IOException, ScalarDbDaoException { exportManager = new JsonLineExportManager(manager, dao, producerTaskFactory); - Scanner scanner = Mockito.mock(Scanner.class); + TransactionCrudOperable.Scanner scanner = Mockito.mock(TransactionCrudOperable.Scanner.class); String filePath = Paths.get("").toAbsolutePath() + "/output.csv"; Map> values = UnitTestUtils.createTestValues(); Result result = new ResultImpl(values, mockData); @@ -181,7 +182,7 @@ void startExport_givenValidDataWithoutPartitionKey_withTransaction_shouldGenerat void startExport_givenPartitionKey_withTransaction_shouldGenerateOutputFile() throws IOException { producerTaskFactory = new ProducerTaskFactory(",", false, false); exportManager = new CsvExportManager(manager, dao, producerTaskFactory); - Scanner scanner = Mockito.mock(Scanner.class); + TransactionCrudOperable.Scanner scanner = Mockito.mock(TransactionCrudOperable.Scanner.class); String filePath = Paths.get("").toAbsolutePath() + "/output.csv"; Map> values = UnitTestUtils.createTestValues(); Result result = new ResultImpl(values, mockData); diff --git a/data-loader/core/src/test/java/com/scalar/db/dataloader/core/dataexport/JsonExportManagerTest.java b/data-loader/core/src/test/java/com/scalar/db/dataloader/core/dataexport/JsonExportManagerTest.java index 7cbcd70a4d..9b6277855d 100644 --- a/data-loader/core/src/test/java/com/scalar/db/dataloader/core/dataexport/JsonExportManagerTest.java +++ b/data-loader/core/src/test/java/com/scalar/db/dataloader/core/dataexport/JsonExportManagerTest.java @@ -8,6 +8,7 @@ import com.scalar.db.api.Result; import com.scalar.db.api.Scanner; import com.scalar.db.api.TableMetadata; +import com.scalar.db.api.TransactionCrudOperable; import com.scalar.db.common.ResultImpl; import com.scalar.db.dataloader.core.FileFormat; import com.scalar.db.dataloader.core.ScalarDbMode; @@ -147,7 +148,7 @@ void startExport_givenPartitionKey_withStorage_shouldGenerateOutputFile() startExport_givenValidDataWithoutPartitionKey_withTransaction_withStorage_shouldGenerateOutputFile() throws IOException, ScalarDbDaoException { exportManager = new JsonExportManager(manager, dao, producerTaskFactory); - Scanner scanner = Mockito.mock(Scanner.class); + TransactionCrudOperable.Scanner scanner = Mockito.mock(TransactionCrudOperable.Scanner.class); String filePath = Paths.get("").toAbsolutePath() + "/output.json"; Map> values = UnitTestUtils.createTestValues(); Result result = new ResultImpl(values, mockData); @@ -186,7 +187,7 @@ void startExport_givenPartitionKey_withStorage_shouldGenerateOutputFile() @Test void startExport_givenPartitionKey_withTransaction_shouldGenerateOutputFile() throws IOException { exportManager = new JsonExportManager(manager, dao, producerTaskFactory); - Scanner scanner = Mockito.mock(Scanner.class); + TransactionCrudOperable.Scanner scanner = Mockito.mock(TransactionCrudOperable.Scanner.class); String filePath = Paths.get("").toAbsolutePath() + "/output.json"; Map> values = UnitTestUtils.createTestValues(); Result result = new ResultImpl(values, mockData); diff --git a/data-loader/core/src/test/java/com/scalar/db/dataloader/core/dataexport/JsonLineExportManagerTest.java b/data-loader/core/src/test/java/com/scalar/db/dataloader/core/dataexport/JsonLineExportManagerTest.java index 8e619bd08f..a39945e580 100644 --- a/data-loader/core/src/test/java/com/scalar/db/dataloader/core/dataexport/JsonLineExportManagerTest.java +++ b/data-loader/core/src/test/java/com/scalar/db/dataloader/core/dataexport/JsonLineExportManagerTest.java @@ -8,6 +8,7 @@ import com.scalar.db.api.Result; import com.scalar.db.api.Scanner; import com.scalar.db.api.TableMetadata; +import com.scalar.db.api.TransactionCrudOperable; import com.scalar.db.common.ResultImpl; import com.scalar.db.dataloader.core.FileFormat; import com.scalar.db.dataloader.core.ScalarDbMode; @@ -146,7 +147,7 @@ void startExport_givenPartitionKey_withStorage_shouldGenerateOutputFile() startExport_givenValidDataWithoutPartitionKey_withTransaction_withStorage_shouldGenerateOutputFile() throws IOException, ScalarDbDaoException { exportManager = new JsonLineExportManager(manager, dao, producerTaskFactory); - Scanner scanner = Mockito.mock(Scanner.class); + TransactionCrudOperable.Scanner scanner = Mockito.mock(TransactionCrudOperable.Scanner.class); String filePath = Paths.get("").toAbsolutePath() + "/output.jsonl"; Map> values = UnitTestUtils.createTestValues(); Result result = new ResultImpl(values, mockData); @@ -185,7 +186,7 @@ void startExport_givenPartitionKey_withStorage_shouldGenerateOutputFile() @Test void startExport_givenPartitionKey_withTransaction_shouldGenerateOutputFile() throws IOException { exportManager = new JsonLineExportManager(manager, dao, producerTaskFactory); - Scanner scanner = Mockito.mock(Scanner.class); + TransactionCrudOperable.Scanner scanner = Mockito.mock(TransactionCrudOperable.Scanner.class); String filePath = Paths.get("").toAbsolutePath() + "/output.jsonl"; Map> values = UnitTestUtils.createTestValues(); Result result = new ResultImpl(values, mockData); diff --git a/data-loader/core/src/test/java/com/scalar/db/dataloader/core/dataimport/dao/ScalarDbDaoTest.java b/data-loader/core/src/test/java/com/scalar/db/dataloader/core/dataimport/dao/ScalarDbDaoTest.java index 0aa74248db..2a7cd902d6 100644 --- a/data-loader/core/src/test/java/com/scalar/db/dataloader/core/dataimport/dao/ScalarDbDaoTest.java +++ b/data-loader/core/src/test/java/com/scalar/db/dataloader/core/dataimport/dao/ScalarDbDaoTest.java @@ -7,7 +7,6 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -import static org.mockito.Mockito.withSettings; import com.scalar.db.api.DistributedStorage; import com.scalar.db.api.DistributedTransaction; @@ -171,10 +170,9 @@ void createScan_scanAllWithLimitAndProjection_shouldCreateScanAllObjectWithLimit void createScanner_withTransactionManager_ShouldCreateScannerObject() throws CrudException, ScalarDbDaoException { // Create Scan Object - TransactionCrudOperable.Scanner mockScanner = - mock(TransactionCrudOperable.Scanner.class, withSettings().extraInterfaces(Scanner.class)); + TransactionCrudOperable.Scanner mockScanner = mock(TransactionCrudOperable.Scanner.class); when(transaction.getScanner(any())).thenReturn(mockScanner); - Scanner result = + TransactionCrudOperable.Scanner result = this.dao.createScanner( TEST_NAMESPACE, TEST_TABLE_NAME, From 72d590743371004e67ed07c6e5578c3a26eed1f6 Mon Sep 17 00:00:00 2001 From: Jishnu J Date: Wed, 11 Jun 2025 10:56:55 +0530 Subject: [PATCH 14/19] Removed unwanted files --- .../ConsoleImportProgressListener.java | 128 ------------------ .../command/dataimport/ImportListener.java | 111 --------------- 2 files changed, 239 deletions(-) delete mode 100644 data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/command/dataimport/ConsoleImportProgressListener.java delete mode 100644 data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/command/dataimport/ImportListener.java diff --git a/data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/command/dataimport/ConsoleImportProgressListener.java b/data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/command/dataimport/ConsoleImportProgressListener.java deleted file mode 100644 index c647339a9d..0000000000 --- a/data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/command/dataimport/ConsoleImportProgressListener.java +++ /dev/null @@ -1,128 +0,0 @@ -package com.scalar.db.dataloader.cli.command.dataimport; - -import com.scalar.db.dataloader.core.dataimport.ImportEventListener; -import com.scalar.db.dataloader.core.dataimport.datachunk.ImportDataChunkStatus; -import com.scalar.db.dataloader.core.dataimport.task.result.ImportTaskResult; -import com.scalar.db.dataloader.core.dataimport.transactionbatch.ImportTransactionBatchResult; -import com.scalar.db.dataloader.core.dataimport.transactionbatch.ImportTransactionBatchStatus; -import java.time.Duration; -import java.util.Map; -import java.util.concurrent.*; -import java.util.concurrent.atomic.AtomicLong; - -public class ConsoleImportProgressListener implements ImportEventListener { - - private final ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor(); - private final Duration updateInterval; - private final long startTime; - private final Map chunkLogs = new ConcurrentHashMap<>(); - private final Map chunkFailureLogs = new ConcurrentHashMap<>(); - private final AtomicLong totalRecords = new AtomicLong(); - private volatile boolean completed = false; - - public ConsoleImportProgressListener(Duration updateInterval) { - this.updateInterval = updateInterval; - this.startTime = System.currentTimeMillis(); - scheduler.scheduleAtFixedRate( - this::render, 0, updateInterval.toMillis(), TimeUnit.MILLISECONDS); - } - - @Override - public void onDataChunkStarted(ImportDataChunkStatus status) { - chunkLogs.put( - status.getDataChunkId(), - String.format( - "🔄 Chunk %d: Processing... %d records so far", - status.getDataChunkId(), status.getTotalRecords())); - } - - @Override - public void onDataChunkCompleted(ImportDataChunkStatus status) { - long elapsed = System.currentTimeMillis() - status.getStartTime().toEpochMilli(); - totalRecords.addAndGet(status.getTotalRecords()); - if (status.getSuccessCount() > 0) { - chunkLogs.put( - status.getDataChunkId(), - String.format( - "✓ Chunk %d: %,d records imported (%.1fs), %d records imported successfully, import of %d records failed", - status.getDataChunkId(), - status.getTotalRecords(), - elapsed / 1000.0, - status.getSuccessCount(), - status.getFailureCount())); - } - // if (status.getFailureCount() > 0) { - // chunkFailureLogs.put( - // status.getDataChunkId(), - // String.format( - // "❌ Chunk %d: Failed - %d records failed to be imported) ", - // status.getDataChunkId(), status.getFailureCount())); - // } - } - - @Override - public void onAllDataChunksCompleted() { - completed = true; - scheduler.shutdown(); - render(); // Final render - } - - @Override - public void onTransactionBatchStarted(ImportTransactionBatchStatus batchStatus) { - // Optional: Implement if you want to show more granular batch progress - } - - @Override - public void onTransactionBatchCompleted(ImportTransactionBatchResult batchResult) { - if (!batchResult.isSuccess()) { - chunkFailureLogs.put( - batchResult.getDataChunkId(), - String.format( - "❌ Chunk %d: Transaction batch %d Failed - %d records failed to be imported) ", - batchResult.getDataChunkId(), - batchResult.getTransactionBatchId(), - batchResult.getRecords().size())); - } - // Optional: Implement error reporting or success/failure count - } - - @Override - public void onTaskComplete(ImportTaskResult taskResult) { - // Optional: Summary or stats after final chunk - } - - private void render() { - StringBuilder builder = new StringBuilder(); - long now = System.currentTimeMillis(); - long elapsed = now - startTime; - double recPerSec = (totalRecords.get() * 1000.0) / (elapsed == 0 ? 1 : elapsed); - - builder.append( - String.format( - "\rImporting... %,d records | %.0f rec/s | %s\n", - totalRecords.get(), recPerSec, formatElapsed(elapsed))); - - chunkLogs.values().stream() - .sorted() // Optional: stable ordering - .forEach(line -> builder.append(line).append("\n")); - chunkFailureLogs.values().stream() - .sorted() // Optional: stable ordering - .forEach(line -> builder.append(line).append("\n")); - - clearConsole(); - System.out.print(builder); - System.out.flush(); - } - - private String formatElapsed(long elapsedMillis) { - long seconds = (elapsedMillis / 1000) % 60; - long minutes = (elapsedMillis / 1000) / 60; - return String.format("%dm %ds elapsed", minutes, seconds); - } - - private void clearConsole() { - // Clear screen for updated multiline rendering - System.out.print("\033[H\033[2J"); // ANSI escape for clearing screen - System.out.flush(); - } -} diff --git a/data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/command/dataimport/ImportListener.java b/data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/command/dataimport/ImportListener.java deleted file mode 100644 index b45a13e0b6..0000000000 --- a/data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/command/dataimport/ImportListener.java +++ /dev/null @@ -1,111 +0,0 @@ -package com.scalar.db.dataloader.cli.command.dataimport; - -import com.scalar.db.dataloader.core.dataimport.ImportEventListener; -import com.scalar.db.dataloader.core.dataimport.datachunk.ImportDataChunkStatus; -import com.scalar.db.dataloader.core.dataimport.task.result.ImportTaskResult; -import com.scalar.db.dataloader.core.dataimport.transactionbatch.ImportTransactionBatchResult; -import com.scalar.db.dataloader.core.dataimport.transactionbatch.ImportTransactionBatchStatus; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.atomic.AtomicInteger; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class ImportListener implements ImportEventListener { - - private static final Logger logger = LoggerFactory.getLogger(ImportListener.class); - private final AtomicInteger totalSuccessImportCount = new AtomicInteger(0); - private final AtomicInteger totalFailureImportCount = new AtomicInteger(0); - private final ConcurrentHashMap dataChunkImportCountMap = - new ConcurrentHashMap<>(); - private static final String DATA_CHUNK_COMPLETED_SUCCESS_MSG = - "\u2713 Chunk %d: %d records imported (%ds) successfully"; - private static final String DATA_CHUNK_COMPLETED_FAILURE_MSG = - "\u274C Chunk %d: %d records failed to import"; - private static final String DATA_CHUNK_IN_PROCESS_MSG = - "\uD83D\uDD04 Chunk %d: Processing... %d records so far imported"; - - /** - * Called when processing of a data chunk begins. - * - * @param status the current status of the data chunk being processed - */ - @Override - public void onDataChunkStarted(ImportDataChunkStatus status) {} - - /** - * Called when processing of a data chunk is completed. - * - * @param status the final status of the completed data chunk - */ - @Override - public void onDataChunkCompleted(ImportDataChunkStatus status) { - if (status.getSuccessCount() > 0) { - logger.info( - String.format( - DATA_CHUNK_COMPLETED_SUCCESS_MSG, - status.getDataChunkId(), - status.getSuccessCount(), - status.getTotalDurationInMilliSeconds() / 1000)); - this.totalSuccessImportCount.addAndGet(status.getSuccessCount()); - } - if (status.getFailureCount() > 0) { - logger.info( - String.format( - DATA_CHUNK_COMPLETED_FAILURE_MSG, status.getDataChunkId(), status.getFailureCount())); - this.totalFailureImportCount.addAndGet(status.getFailureCount()); - } - } - - /** - * Called when all data chunks have been processed. This indicates that the entire chunked import - * process is complete. - */ - @Override - public void onAllDataChunksCompleted() { - System.out.println("Logger Factory: " + LoggerFactory.getILoggerFactory().getClass()); - if (totalSuccessImportCount.get() > 0) {} - } - - /** - * Called when processing of a transaction batch begins. - * - * @param batchStatus the initial status of the transaction batch - */ - @Override - public void onTransactionBatchStarted(ImportTransactionBatchStatus batchStatus) {} - - /** - * Called when processing of a transaction batch is completed. - * - * @param batchResult the result of the completed transaction batch - */ - @Override - public void onTransactionBatchCompleted(ImportTransactionBatchResult batchResult) { - if (batchResult.isSuccess()) { - Integer dataChunkId = batchResult.getDataChunkId(); - dataChunkImportCountMap.compute( - dataChunkId, - (k, v) -> { - if (v == null) { - return new AtomicInteger(batchResult.getRecords().size()); - } else { - v.addAndGet(batchResult.getRecords().size()); - return v; - } - }); - logger.info( - String.format( - DATA_CHUNK_IN_PROCESS_MSG, - dataChunkId, - dataChunkImportCountMap.get(dataChunkId).get())); - } - } - - /** - * Called when an import task is completed. - * - * @param taskResult the result of the completed import task - */ - @Override - public void onTaskComplete(ImportTaskResult taskResult) {} -} From a3ae85eb7348a7a1ddaecf13447e30366a82c675 Mon Sep 17 00:00:00 2001 From: Jishnu J Date: Wed, 11 Jun 2025 11:10:34 +0530 Subject: [PATCH 15/19] Spotbugs issue fixed --- .../dataloader/cli/command/dataexport/ExportCommand.java | 1 - .../db/dataloader/core/dataexport/ExportManager.java | 9 +++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/command/dataexport/ExportCommand.java b/data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/command/dataexport/ExportCommand.java index 9d8f9fb684..2e3b5c5132 100755 --- a/data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/command/dataexport/ExportCommand.java +++ b/data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/command/dataexport/ExportCommand.java @@ -72,7 +72,6 @@ public Integer call() throws Exception { ExportManager exportManager = createExportManager(scalarDbMode, scalarDbDao, outputFormat, scalarDbPropertiesFilePath); - System.out.println(scalarDbMode); TableMetadata tableMetadata = tableMetadataService.getTableMetadata(namespace, table); Key partitionKey = diff --git a/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataexport/ExportManager.java b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataexport/ExportManager.java index b0e94472b3..03abbaea9d 100644 --- a/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataexport/ExportManager.java +++ b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataexport/ExportManager.java @@ -145,10 +145,15 @@ public ExportReport startExport( shutdownExecutor(executorService); processFooter(exportOptions, tableMetadata, bufferedWriter); + } catch (TransactionException e) { + throw new RuntimeException(e); } - } catch (Exception e) { - logger.error("Export failed", e); + } catch (ExportOptionsValidationException + | IOException + | ScalarDbDaoException + | InterruptedException e) { + logger.error("Error during export: {}", e.getMessage()); } finally { if (executorService != null && !executorService.isShutdown()) { executorService.shutdownNow(); From 303b2885a72e0395e9248eb8f8b9556196c0a9d9 Mon Sep 17 00:00:00 2001 From: Jishnu J Date: Wed, 11 Jun 2025 15:49:01 +0530 Subject: [PATCH 16/19] Table metadata service split to storage and transaction --- .../cli/command/dataexport/ExportCommand.java | 6 +- .../cli/command/dataimport/ImportCommand.java | 3 +- .../tablemetadata/TableMetadataService.java | 60 +++++--------- .../TableMetadataStorageService.java | 37 +++++++++ .../TableMetadataTransactionService.java | 37 +++++++++ .../TableMetadataServiceTest.java | 82 ------------------- .../TableMetadataStorageServiceTest.java | 58 +++++++++++++ .../TableMetadataTransactionServiceTest.java | 58 +++++++++++++ 8 files changed, 218 insertions(+), 123 deletions(-) create mode 100644 data-loader/core/src/main/java/com/scalar/db/dataloader/core/tablemetadata/TableMetadataStorageService.java create mode 100644 data-loader/core/src/main/java/com/scalar/db/dataloader/core/tablemetadata/TableMetadataTransactionService.java delete mode 100644 data-loader/core/src/test/java/com/scalar/db/dataloader/core/tablemetadata/TableMetadataServiceTest.java create mode 100644 data-loader/core/src/test/java/com/scalar/db/dataloader/core/tablemetadata/TableMetadataStorageServiceTest.java create mode 100644 data-loader/core/src/test/java/com/scalar/db/dataloader/core/tablemetadata/TableMetadataTransactionServiceTest.java diff --git a/data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/command/dataexport/ExportCommand.java b/data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/command/dataexport/ExportCommand.java index 2e3b5c5132..f9aa7deaf2 100755 --- a/data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/command/dataexport/ExportCommand.java +++ b/data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/command/dataexport/ExportCommand.java @@ -27,6 +27,8 @@ import com.scalar.db.dataloader.core.exception.KeyParsingException; import com.scalar.db.dataloader.core.tablemetadata.TableMetadataException; import com.scalar.db.dataloader.core.tablemetadata.TableMetadataService; +import com.scalar.db.dataloader.core.tablemetadata.TableMetadataStorageService; +import com.scalar.db.dataloader.core.tablemetadata.TableMetadataTransactionService; import com.scalar.db.dataloader.core.util.KeyUtils; import com.scalar.db.io.Key; import com.scalar.db.service.StorageFactory; @@ -134,10 +136,10 @@ private TableMetadataService createTableMetadataService( ScalarDbMode scalarDbMode, String scalarDbPropertiesFilePath) throws IOException { if (scalarDbMode.equals(ScalarDbMode.TRANSACTION)) { TransactionFactory transactionFactory = TransactionFactory.create(scalarDbPropertiesFilePath); - return new TableMetadataService(transactionFactory.getTransactionAdmin()); + return new TableMetadataTransactionService(transactionFactory.getTransactionAdmin()); } StorageFactory storageFactory = StorageFactory.create(scalarDbPropertiesFilePath); - return new TableMetadataService(storageFactory.getStorageAdmin()); + return new TableMetadataStorageService(storageFactory.getStorageAdmin()); } /** diff --git a/data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/command/dataimport/ImportCommand.java b/data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/command/dataimport/ImportCommand.java index a505a42ade..504259bf07 100755 --- a/data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/command/dataimport/ImportCommand.java +++ b/data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/command/dataimport/ImportCommand.java @@ -24,6 +24,7 @@ import com.scalar.db.dataloader.core.dataimport.processor.ImportProcessorFactory; import com.scalar.db.dataloader.core.tablemetadata.TableMetadataException; import com.scalar.db.dataloader.core.tablemetadata.TableMetadataService; +import com.scalar.db.dataloader.core.tablemetadata.TableMetadataStorageService; import com.scalar.db.dataloader.core.util.TableMetadataUtil; import com.scalar.db.service.StorageFactory; import com.scalar.db.service.TransactionFactory; @@ -109,7 +110,7 @@ private Map createTableMetadataMap( File configFile = new File(configFilePath); StorageFactory storageFactory = StorageFactory.create(configFile); try (DistributedStorageAdmin storageAdmin = storageFactory.getStorageAdmin()) { - TableMetadataService tableMetadataService = new TableMetadataService(storageAdmin); + TableMetadataService tableMetadataService = new TableMetadataStorageService(storageAdmin); Map tableMetadataMap = new HashMap<>(); if (controlFile != null) { for (ControlFileTable table : controlFile.getTables()) { diff --git a/data-loader/core/src/main/java/com/scalar/db/dataloader/core/tablemetadata/TableMetadataService.java b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/tablemetadata/TableMetadataService.java index 41c292ec96..a53a62528d 100644 --- a/data-loader/core/src/main/java/com/scalar/db/dataloader/core/tablemetadata/TableMetadataService.java +++ b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/tablemetadata/TableMetadataService.java @@ -1,35 +1,19 @@ package com.scalar.db.dataloader.core.tablemetadata; -import com.scalar.db.api.DistributedStorageAdmin; -import com.scalar.db.api.DistributedTransactionAdmin; import com.scalar.db.api.TableMetadata; import com.scalar.db.common.error.CoreError; import com.scalar.db.dataloader.core.util.TableMetadataUtil; -import com.scalar.db.exception.storage.ExecutionException; import java.util.Collection; import java.util.HashMap; import java.util.Map; -import lombok.RequiredArgsConstructor; /** - * Service for retrieving {@link TableMetadata} from ScalarDB. Provides methods to fetch metadata - * for individual tables or a collection of tables. + * Abstract base class for retrieving {@link TableMetadata} from ScalarDB. Provides shared logic for + * fetching metadata for a collection of tables. Subclasses must implement the specific logic for + * fetching individual table metadata. */ -@RequiredArgsConstructor -public class TableMetadataService { +public abstract class TableMetadataService { - private final DistributedStorageAdmin storageAdmin; - private final DistributedTransactionAdmin transactionAdmin; - - public TableMetadataService(DistributedStorageAdmin storageAdmin) { - this.transactionAdmin = null; - this.storageAdmin = storageAdmin; - } - - public TableMetadataService(DistributedTransactionAdmin transactionAdmin) { - this.transactionAdmin = transactionAdmin; - this.storageAdmin = null; - } /** * Retrieves the {@link TableMetadata} for a specific namespace and table name. * @@ -41,22 +25,12 @@ public TableMetadataService(DistributedTransactionAdmin transactionAdmin) { */ public TableMetadata getTableMetadata(String namespace, String tableName) throws TableMetadataException { - try { - TableMetadata tableMetadata = null; - if (storageAdmin != null) { - tableMetadata = storageAdmin.getTableMetadata(namespace, tableName); - } else if (transactionAdmin != null) { - tableMetadata = transactionAdmin.getTableMetadata(namespace, tableName); - } - if (tableMetadata == null) { - throw new TableMetadataException( - CoreError.DATA_LOADER_MISSING_NAMESPACE_OR_TABLE.buildMessage(namespace, tableName)); - } - return tableMetadata; - } catch (ExecutionException e) { + TableMetadata metadata = getTableMetadataInternal(namespace, tableName); + if (metadata == null) { throw new TableMetadataException( - CoreError.DATA_LOADER_TABLE_METADATA_RETRIEVAL_FAILED.buildMessage(e.getMessage()), e); + CoreError.DATA_LOADER_MISSING_NAMESPACE_OR_TABLE.buildMessage(namespace, tableName)); } + return metadata; } /** @@ -75,15 +49,25 @@ public TableMetadata getTableMetadata(String namespace, String tableName) public Map getTableMetadata(Collection requests) throws TableMetadataException { Map metadataMap = new HashMap<>(); - for (TableMetadataRequest request : requests) { String namespace = request.getNamespace(); String tableName = request.getTable(); - TableMetadata tableMetadata = getTableMetadata(namespace, tableName); + TableMetadata metadata = getTableMetadata(namespace, tableName); String key = TableMetadataUtil.getTableLookupKey(namespace, tableName); - metadataMap.put(key, tableMetadata); + metadataMap.put(key, metadata); } - return metadataMap; } + + /** + * Abstract method for retrieving table metadata for a specific namespace and table. Subclasses + * must implement this to define how the metadata is fetched. + * + * @param namespace The namespace of the table. + * @param tableName The table name. + * @return The {@link TableMetadata} object, or null if not found. + * @throws TableMetadataException if an error occurs during metadata retrieval. + */ + protected abstract TableMetadata getTableMetadataInternal(String namespace, String tableName) + throws TableMetadataException; } diff --git a/data-loader/core/src/main/java/com/scalar/db/dataloader/core/tablemetadata/TableMetadataStorageService.java b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/tablemetadata/TableMetadataStorageService.java new file mode 100644 index 0000000000..a40b39b14c --- /dev/null +++ b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/tablemetadata/TableMetadataStorageService.java @@ -0,0 +1,37 @@ +package com.scalar.db.dataloader.core.tablemetadata; + +import com.scalar.db.api.DistributedStorageAdmin; +import com.scalar.db.api.TableMetadata; +import com.scalar.db.common.error.CoreError; +import com.scalar.db.exception.storage.ExecutionException; +import lombok.RequiredArgsConstructor; + +/** + * Implementation of {@link TableMetadataService} that retrieves table metadata using {@link + * DistributedStorageAdmin}. + */ +@RequiredArgsConstructor +public class TableMetadataStorageService extends TableMetadataService { + + private final DistributedStorageAdmin storageAdmin; + + /** + * Retrieves the {@link TableMetadata} for a given namespace and table using the {@link + * DistributedStorageAdmin}. + * + * @param namespace The namespace of the table. + * @param tableName The name of the table. + * @return The {@link TableMetadata} for the specified table, or null if not found. + * @throws TableMetadataException If an error occurs while fetching metadata. + */ + @Override + protected TableMetadata getTableMetadataInternal(String namespace, String tableName) + throws TableMetadataException { + try { + return storageAdmin.getTableMetadata(namespace, tableName); + } catch (ExecutionException e) { + throw new TableMetadataException( + CoreError.DATA_LOADER_TABLE_METADATA_RETRIEVAL_FAILED.buildMessage(e.getMessage()), e); + } + } +} diff --git a/data-loader/core/src/main/java/com/scalar/db/dataloader/core/tablemetadata/TableMetadataTransactionService.java b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/tablemetadata/TableMetadataTransactionService.java new file mode 100644 index 0000000000..f5a8dd68c6 --- /dev/null +++ b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/tablemetadata/TableMetadataTransactionService.java @@ -0,0 +1,37 @@ +package com.scalar.db.dataloader.core.tablemetadata; + +import com.scalar.db.api.DistributedTransactionAdmin; +import com.scalar.db.api.TableMetadata; +import com.scalar.db.common.error.CoreError; +import com.scalar.db.exception.storage.ExecutionException; +import lombok.RequiredArgsConstructor; + +/** + * Implementation of {@link TableMetadataService} that retrieves table metadata using {@link + * DistributedTransactionAdmin}. + */ +@RequiredArgsConstructor +public class TableMetadataTransactionService extends TableMetadataService { + + private final DistributedTransactionAdmin transactionAdmin; + + /** + * Retrieves the {@link TableMetadata} for a given namespace and table using the {@link + * DistributedTransactionAdmin}. + * + * @param namespace The namespace of the table. + * @param tableName The name of the table. + * @return The {@link TableMetadata} for the specified table, or null if not found. + * @throws TableMetadataException If an error occurs while fetching metadata. + */ + @Override + protected TableMetadata getTableMetadataInternal(String namespace, String tableName) + throws TableMetadataException { + try { + return transactionAdmin.getTableMetadata(namespace, tableName); + } catch (ExecutionException e) { + throw new TableMetadataException( + CoreError.DATA_LOADER_TABLE_METADATA_RETRIEVAL_FAILED.buildMessage(e.getMessage()), e); + } + } +} diff --git a/data-loader/core/src/test/java/com/scalar/db/dataloader/core/tablemetadata/TableMetadataServiceTest.java b/data-loader/core/src/test/java/com/scalar/db/dataloader/core/tablemetadata/TableMetadataServiceTest.java deleted file mode 100644 index e68fad90d9..0000000000 --- a/data-loader/core/src/test/java/com/scalar/db/dataloader/core/tablemetadata/TableMetadataServiceTest.java +++ /dev/null @@ -1,82 +0,0 @@ -package com.scalar.db.dataloader.core.tablemetadata; - -import static org.assertj.core.api.Assertions.assertThatThrownBy; - -import com.scalar.db.api.DistributedStorageAdmin; -import com.scalar.db.api.DistributedTransactionAdmin; -import com.scalar.db.api.TableMetadata; -import com.scalar.db.common.error.CoreError; -import com.scalar.db.dataloader.core.UnitTestUtils; -import com.scalar.db.exception.storage.ExecutionException; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.mockito.Mockito; - -class TableMetadataServiceTest { - - DistributedStorageAdmin storageAdmin; - DistributedTransactionAdmin transactionAdmin; - TableMetadataService tableMetadataService; - - @BeforeEach - void setup() throws ExecutionException { - storageAdmin = Mockito.mock(DistributedStorageAdmin.class); - Mockito.when(storageAdmin.getTableMetadata("namespace", "table")) - .thenReturn(UnitTestUtils.createTestTableMetadata()); - transactionAdmin = Mockito.mock(DistributedTransactionAdmin.class); - Mockito.when(transactionAdmin.getTableMetadata("namespace", "table")) - .thenReturn(UnitTestUtils.createTestTableMetadata()); - } - - @Test - void getTableMetadata_withStorage_withValidNamespaceAndTable_shouldReturnTableMetadataMap() - throws TableMetadataException { - tableMetadataService = new TableMetadataService(storageAdmin); - Map expected = new HashMap<>(); - expected.put("namespace.table", UnitTestUtils.createTestTableMetadata()); - TableMetadataRequest tableMetadataRequest = new TableMetadataRequest("namespace", "table"); - Map output = - tableMetadataService.getTableMetadata(Collections.singleton(tableMetadataRequest)); - Assertions.assertEquals(expected.get("namespace.table"), output.get("namespace.table")); - } - - @Test - void getTableMetadata_withStorage_withInvalidNamespaceAndTable_shouldThrowException() { - tableMetadataService = new TableMetadataService(storageAdmin); - TableMetadataRequest tableMetadataRequest = new TableMetadataRequest("namespace2", "table2"); - assertThatThrownBy( - () -> - tableMetadataService.getTableMetadata(Collections.singleton(tableMetadataRequest))) - .isInstanceOf(TableMetadataException.class) - .hasMessage( - CoreError.DATA_LOADER_MISSING_NAMESPACE_OR_TABLE.buildMessage("namespace2", "table2")); - } - - @Test - void getTableMetadata_withTransaction_withValidNamespaceAndTable_shouldReturnTableMetadataMap() - throws TableMetadataException { - tableMetadataService = new TableMetadataService(transactionAdmin); - Map expected = new HashMap<>(); - expected.put("namespace.table", UnitTestUtils.createTestTableMetadata()); - TableMetadataRequest tableMetadataRequest = new TableMetadataRequest("namespace", "table"); - Map output = - tableMetadataService.getTableMetadata(Collections.singleton(tableMetadataRequest)); - Assertions.assertEquals(expected.get("namespace.table"), output.get("namespace.table")); - } - - @Test - void getTableMetadata_withTransaction_withInvalidNamespaceAndTable_shouldThrowException() { - tableMetadataService = new TableMetadataService(transactionAdmin); - TableMetadataRequest tableMetadataRequest = new TableMetadataRequest("namespace2", "table2"); - assertThatThrownBy( - () -> - tableMetadataService.getTableMetadata(Collections.singleton(tableMetadataRequest))) - .isInstanceOf(TableMetadataException.class) - .hasMessage( - CoreError.DATA_LOADER_MISSING_NAMESPACE_OR_TABLE.buildMessage("namespace2", "table2")); - } -} diff --git a/data-loader/core/src/test/java/com/scalar/db/dataloader/core/tablemetadata/TableMetadataStorageServiceTest.java b/data-loader/core/src/test/java/com/scalar/db/dataloader/core/tablemetadata/TableMetadataStorageServiceTest.java new file mode 100644 index 0000000000..19c8973756 --- /dev/null +++ b/data-loader/core/src/test/java/com/scalar/db/dataloader/core/tablemetadata/TableMetadataStorageServiceTest.java @@ -0,0 +1,58 @@ +package com.scalar.db.dataloader.core.tablemetadata; + +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import com.scalar.db.api.DistributedStorageAdmin; +import com.scalar.db.api.TableMetadata; +import com.scalar.db.common.error.CoreError; +import com.scalar.db.dataloader.core.UnitTestUtils; +import com.scalar.db.exception.storage.ExecutionException; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +class TableMetadataStorageServiceTest { + + DistributedStorageAdmin storageAdmin; + TableMetadataService tableMetadataService; + + @BeforeEach + void setup() throws ExecutionException { + storageAdmin = Mockito.mock(DistributedStorageAdmin.class); + Mockito.when(storageAdmin.getTableMetadata("namespace", "table")) + .thenReturn(UnitTestUtils.createTestTableMetadata()); + + tableMetadataService = new TableMetadataStorageService(storageAdmin); + } + + @Test + void getTableMetadata_withValidNamespaceAndTable_shouldReturnTableMetadataMap() + throws TableMetadataException { + Map expected = new HashMap<>(); + expected.put("namespace.table", UnitTestUtils.createTestTableMetadata()); + + TableMetadataRequest tableMetadataRequest = new TableMetadataRequest("namespace", "table"); + Map output = + tableMetadataService.getTableMetadata(Collections.singleton(tableMetadataRequest)); + + Assertions.assertEquals(expected.get("namespace.table"), output.get("namespace.table")); + } + + @Test + void getTableMetadata_withInvalidNamespaceAndTable_shouldThrowException() + throws ExecutionException { + Mockito.when(storageAdmin.getTableMetadata("namespace2", "table2")).thenReturn(null); + + TableMetadataRequest tableMetadataRequest = new TableMetadataRequest("namespace2", "table2"); + assertThatThrownBy( + () -> + tableMetadataService.getTableMetadata(Collections.singleton(tableMetadataRequest))) + .isInstanceOf(TableMetadataException.class) + .hasMessage( + CoreError.DATA_LOADER_MISSING_NAMESPACE_OR_TABLE.buildMessage("namespace2", "table2")); + } +} diff --git a/data-loader/core/src/test/java/com/scalar/db/dataloader/core/tablemetadata/TableMetadataTransactionServiceTest.java b/data-loader/core/src/test/java/com/scalar/db/dataloader/core/tablemetadata/TableMetadataTransactionServiceTest.java new file mode 100644 index 0000000000..fde6b064f9 --- /dev/null +++ b/data-loader/core/src/test/java/com/scalar/db/dataloader/core/tablemetadata/TableMetadataTransactionServiceTest.java @@ -0,0 +1,58 @@ +package com.scalar.db.dataloader.core.tablemetadata; + +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import com.scalar.db.api.DistributedTransactionAdmin; +import com.scalar.db.api.TableMetadata; +import com.scalar.db.common.error.CoreError; +import com.scalar.db.dataloader.core.UnitTestUtils; +import com.scalar.db.exception.storage.ExecutionException; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +class TableMetadataTransactionServiceTest { + + DistributedTransactionAdmin transactionAdmin; + TableMetadataService tableMetadataService; + + @BeforeEach + void setup() throws ExecutionException { + transactionAdmin = Mockito.mock(DistributedTransactionAdmin.class); + Mockito.when(transactionAdmin.getTableMetadata("namespace", "table")) + .thenReturn(UnitTestUtils.createTestTableMetadata()); + + tableMetadataService = new TableMetadataTransactionService(transactionAdmin); + } + + @Test + void getTableMetadata_withValidNamespaceAndTable_shouldReturnTableMetadataMap() + throws TableMetadataException { + Map expected = new HashMap<>(); + expected.put("namespace.table", UnitTestUtils.createTestTableMetadata()); + + TableMetadataRequest tableMetadataRequest = new TableMetadataRequest("namespace", "table"); + Map output = + tableMetadataService.getTableMetadata(Collections.singleton(tableMetadataRequest)); + + Assertions.assertEquals(expected.get("namespace.table"), output.get("namespace.table")); + } + + @Test + void getTableMetadata_withInvalidNamespaceAndTable_shouldThrowException() + throws ExecutionException { + Mockito.when(transactionAdmin.getTableMetadata("namespace2", "table2")).thenReturn(null); + + TableMetadataRequest tableMetadataRequest = new TableMetadataRequest("namespace2", "table2"); + assertThatThrownBy( + () -> + tableMetadataService.getTableMetadata(Collections.singleton(tableMetadataRequest))) + .isInstanceOf(TableMetadataException.class) + .hasMessage( + CoreError.DATA_LOADER_MISSING_NAMESPACE_OR_TABLE.buildMessage("namespace2", "table2")); + } +} From 36f9b4369b4aae70621106e5524aace3328d70a0 Mon Sep 17 00:00:00 2001 From: Jishnu J Date: Thu, 12 Jun 2025 11:20:54 +0530 Subject: [PATCH 17/19] Changed implemenatation to use DistributedTransactionManager --- .../core/dataexport/ExportManager.java | 51 ++++++++----------- .../dataexport/ScannerWithTransaction.java | 11 ---- .../core/dataimport/dao/ScalarDbDao.java | 11 ++-- .../core/dataexport/CsvExportManagerTest.java | 12 +++-- .../dataexport/JsonExportManagerTest.java | 12 +++-- .../dataexport/JsonLineExportManagerTest.java | 21 +++----- .../core/dataimport/dao/ScalarDbDaoTest.java | 20 ++++---- 7 files changed, 60 insertions(+), 78 deletions(-) delete mode 100644 data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataexport/ScannerWithTransaction.java diff --git a/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataexport/ExportManager.java b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataexport/ExportManager.java index 03abbaea9d..8889bf580a 100644 --- a/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataexport/ExportManager.java +++ b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataexport/ExportManager.java @@ -1,12 +1,11 @@ package com.scalar.db.dataloader.core.dataexport; import com.scalar.db.api.DistributedStorage; -import com.scalar.db.api.DistributedTransaction; import com.scalar.db.api.DistributedTransactionManager; import com.scalar.db.api.Result; import com.scalar.db.api.Scanner; import com.scalar.db.api.TableMetadata; -import com.scalar.db.api.TransactionCrudOperable; +import com.scalar.db.api.TransactionManagerCrudOperable; import com.scalar.db.dataloader.core.FileFormat; import com.scalar.db.dataloader.core.ScalarDbMode; import com.scalar.db.dataloader.core.dataexport.producer.ProducerTask; @@ -125,10 +124,9 @@ public ExportReport startExport( } } else if (exportOptions.getScalarDbMode() == ScalarDbMode.TRANSACTION && distributedTransactionManager != null) { - ScannerWithTransaction scannerWithTx = - createScannerWithTransaction(exportOptions, dao, distributedTransactionManager); - try (TransactionCrudOperable.Scanner scanner = scannerWithTx.getScanner()) { + try (TransactionManagerCrudOperable.Scanner scanner = + createScannerWithTransaction(exportOptions, dao, distributedTransactionManager)) { submitTasks( scanner.iterator(), executorService, @@ -138,8 +136,6 @@ public ExportReport startExport( bufferedWriter, isFirstBatch, exportReport); - } finally { - scannerWithTx.getTransaction().commit(); } } @@ -361,36 +357,31 @@ private Scanner createScannerWithStorage( } /** - * Creates a {@link ScannerWithTransaction} object that encapsulates a transactional scanner and - * its associated transaction for reading data from a ScalarDB table. + * Creates a {@link TransactionManagerCrudOperable.Scanner} instance using the given {@link + * ExportOptions}, {@link ScalarDbDao}, and {@link DistributedTransactionManager}. * - *

If no partition key is provided in the {@link ExportOptions}, a full table scan is - * performed. Otherwise, a partition-specific scan is created using the provided partition key, - * optional scan range, and sort orders. - * - *

The method starts a new transaction using the given {@link DistributedTransactionManager}, - * which will be associated with the returned scanner. This allows data export operations to be - * executed in a consistent transactional context. + *

If {@code scanPartitionKey} is not specified in {@code exportOptions}, a full table scan is + * performed using the specified projection columns and limit. Otherwise, the scan is executed + * with the specified partition key, range, sort orders, projection columns, and limit. * - * @param exportOptions the options specifying how to scan the table, such as namespace, table - * name, projection columns, scan partition key, range, sort orders, and limit. - * @param dao the {@link ScalarDbDao} used to construct the transactional scanner. - * @param distributedTransactionManager the transaction manager used to start a new transaction. - * @return a {@link ScannerWithTransaction} instance that wraps both the transaction and the - * scanner. - * @throws ScalarDbDaoException if an error occurs while creating the scanner with the DAO. - * @throws TransactionException if an error occurs when starting the transaction. + * @param exportOptions the export options containing scan configuration such as namespace, table + * name, partition key, projection columns, limit, range, and sort order + * @param dao the ScalarDB DAO used to create the scanner + * @param distributedTransactionManager the transaction manager to use for the scan operation + * @return a {@link TransactionManagerCrudOperable.Scanner} for retrieving rows in transaction + * mode + * @throws ScalarDbDaoException if an error occurs while creating the scanner + * @throws TransactionException if a transaction-related error occurs during scanner creation */ - private ScannerWithTransaction createScannerWithTransaction( + private TransactionManagerCrudOperable.Scanner createScannerWithTransaction( ExportOptions exportOptions, ScalarDbDao dao, DistributedTransactionManager distributedTransactionManager) throws ScalarDbDaoException, TransactionException { boolean isScanAll = exportOptions.getScanPartitionKey() == null; - DistributedTransaction transaction = distributedTransactionManager.start(); - TransactionCrudOperable.Scanner scanner; + TransactionManagerCrudOperable.Scanner scanner; if (isScanAll) { scanner = dao.createScanner( @@ -398,7 +389,7 @@ private ScannerWithTransaction createScannerWithTransaction( exportOptions.getTableName(), exportOptions.getProjectionColumns(), exportOptions.getLimit(), - transaction); + distributedTransactionManager); } else { scanner = dao.createScanner( @@ -409,10 +400,10 @@ private ScannerWithTransaction createScannerWithTransaction( exportOptions.getSortOrders(), exportOptions.getProjectionColumns(), exportOptions.getLimit(), - transaction); + distributedTransactionManager); } - return new ScannerWithTransaction(transaction, scanner); + return scanner; } /** Close resources properly once the process is completed */ diff --git a/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataexport/ScannerWithTransaction.java b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataexport/ScannerWithTransaction.java deleted file mode 100644 index dffaa428cd..0000000000 --- a/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataexport/ScannerWithTransaction.java +++ /dev/null @@ -1,11 +0,0 @@ -package com.scalar.db.dataloader.core.dataexport; - -import com.scalar.db.api.DistributedTransaction; -import com.scalar.db.api.TransactionCrudOperable; -import lombok.Value; - -@Value -public class ScannerWithTransaction { - DistributedTransaction transaction; - TransactionCrudOperable.Scanner scanner; -} diff --git a/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/dao/ScalarDbDao.java b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/dao/ScalarDbDao.java index ddff5ef101..d107d705db 100644 --- a/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/dao/ScalarDbDao.java +++ b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/dao/ScalarDbDao.java @@ -2,6 +2,7 @@ import com.scalar.db.api.DistributedStorage; import com.scalar.db.api.DistributedTransaction; +import com.scalar.db.api.DistributedTransactionManager; import com.scalar.db.api.Get; import com.scalar.db.api.GetBuilder; import com.scalar.db.api.Put; @@ -10,7 +11,7 @@ import com.scalar.db.api.Scan; import com.scalar.db.api.ScanBuilder; import com.scalar.db.api.Scanner; -import com.scalar.db.api.TransactionCrudOperable; +import com.scalar.db.api.TransactionManagerCrudOperable; import com.scalar.db.common.error.CoreError; import com.scalar.db.dataloader.core.ScanRange; import com.scalar.db.exception.storage.ExecutionException; @@ -260,12 +261,12 @@ public Scanner createScanner( * @return ScalarDB Scanner object * @throws ScalarDbDaoException if scan fails */ - public TransactionCrudOperable.Scanner createScanner( + public TransactionManagerCrudOperable.Scanner createScanner( String namespace, String table, List projectionColumns, int limit, - DistributedTransaction transaction) + DistributedTransactionManager transaction) throws ScalarDbDaoException { Scan scan = createScan(namespace, table, null, null, new ArrayList<>(), projectionColumns, limit); @@ -324,7 +325,7 @@ public Scanner createScanner( * @param transaction Distributed transaction object * @return ScalarDB Scanner object */ - public TransactionCrudOperable.Scanner createScanner( + public TransactionManagerCrudOperable.Scanner createScanner( String namespace, String table, @Nullable Key partitionKey, @@ -332,7 +333,7 @@ public TransactionCrudOperable.Scanner createScanner( @Nullable List sortOrders, @Nullable List projectionColumns, int limit, - DistributedTransaction transaction) { + DistributedTransactionManager transaction) { Scan scan = createScan(namespace, table, partitionKey, scanRange, sortOrders, projectionColumns, limit); try { diff --git a/data-loader/core/src/test/java/com/scalar/db/dataloader/core/dataexport/CsvExportManagerTest.java b/data-loader/core/src/test/java/com/scalar/db/dataloader/core/dataexport/CsvExportManagerTest.java index 9f39ff4915..22b0830f2d 100644 --- a/data-loader/core/src/test/java/com/scalar/db/dataloader/core/dataexport/CsvExportManagerTest.java +++ b/data-loader/core/src/test/java/com/scalar/db/dataloader/core/dataexport/CsvExportManagerTest.java @@ -8,7 +8,7 @@ import com.scalar.db.api.Result; import com.scalar.db.api.Scanner; import com.scalar.db.api.TableMetadata; -import com.scalar.db.api.TransactionCrudOperable; +import com.scalar.db.api.TransactionManagerCrudOperable; import com.scalar.db.common.ResultImpl; import com.scalar.db.dataloader.core.FileFormat; import com.scalar.db.dataloader.core.ScalarDbMode; @@ -144,7 +144,8 @@ void startExport_givenPartitionKey_withStorage_shouldGenerateOutputFile() void startExport_givenValidDataWithoutPartitionKey_withTransaction_shouldGenerateOutputFile() throws IOException, ScalarDbDaoException { exportManager = new JsonLineExportManager(manager, dao, producerTaskFactory); - TransactionCrudOperable.Scanner scanner = Mockito.mock(TransactionCrudOperable.Scanner.class); + TransactionManagerCrudOperable.Scanner scanner = + Mockito.mock(TransactionManagerCrudOperable.Scanner.class); String filePath = Paths.get("").toAbsolutePath() + "/output.csv"; Map> values = UnitTestUtils.createTestValues(); Result result = new ResultImpl(values, mockData); @@ -161,7 +162,7 @@ void startExport_givenValidDataWithoutPartitionKey_withTransaction_shouldGenerat exportOptions.getTableName(), exportOptions.getProjectionColumns(), exportOptions.getLimit(), - transaction)) + manager)) .thenReturn(scanner); when(scanner.iterator()).thenReturn(results.iterator()); try (BufferedWriter writer = @@ -182,7 +183,8 @@ void startExport_givenValidDataWithoutPartitionKey_withTransaction_shouldGenerat void startExport_givenPartitionKey_withTransaction_shouldGenerateOutputFile() throws IOException { producerTaskFactory = new ProducerTaskFactory(",", false, false); exportManager = new CsvExportManager(manager, dao, producerTaskFactory); - TransactionCrudOperable.Scanner scanner = Mockito.mock(TransactionCrudOperable.Scanner.class); + TransactionManagerCrudOperable.Scanner scanner = + Mockito.mock(TransactionManagerCrudOperable.Scanner.class); String filePath = Paths.get("").toAbsolutePath() + "/output.csv"; Map> values = UnitTestUtils.createTestValues(); Result result = new ResultImpl(values, mockData); @@ -207,7 +209,7 @@ void startExport_givenPartitionKey_withTransaction_shouldGenerateOutputFile() th exportOptions.getSortOrders(), exportOptions.getProjectionColumns(), exportOptions.getLimit(), - transaction)) + manager)) .thenReturn(scanner); when(scanner.iterator()).thenReturn(results.iterator()); try (BufferedWriter writer = diff --git a/data-loader/core/src/test/java/com/scalar/db/dataloader/core/dataexport/JsonExportManagerTest.java b/data-loader/core/src/test/java/com/scalar/db/dataloader/core/dataexport/JsonExportManagerTest.java index 9b6277855d..bc1f64296a 100644 --- a/data-loader/core/src/test/java/com/scalar/db/dataloader/core/dataexport/JsonExportManagerTest.java +++ b/data-loader/core/src/test/java/com/scalar/db/dataloader/core/dataexport/JsonExportManagerTest.java @@ -8,7 +8,7 @@ import com.scalar.db.api.Result; import com.scalar.db.api.Scanner; import com.scalar.db.api.TableMetadata; -import com.scalar.db.api.TransactionCrudOperable; +import com.scalar.db.api.TransactionManagerCrudOperable; import com.scalar.db.common.ResultImpl; import com.scalar.db.dataloader.core.FileFormat; import com.scalar.db.dataloader.core.ScalarDbMode; @@ -148,7 +148,8 @@ void startExport_givenPartitionKey_withStorage_shouldGenerateOutputFile() startExport_givenValidDataWithoutPartitionKey_withTransaction_withStorage_shouldGenerateOutputFile() throws IOException, ScalarDbDaoException { exportManager = new JsonExportManager(manager, dao, producerTaskFactory); - TransactionCrudOperable.Scanner scanner = Mockito.mock(TransactionCrudOperable.Scanner.class); + TransactionManagerCrudOperable.Scanner scanner = + Mockito.mock(TransactionManagerCrudOperable.Scanner.class); String filePath = Paths.get("").toAbsolutePath() + "/output.json"; Map> values = UnitTestUtils.createTestValues(); Result result = new ResultImpl(values, mockData); @@ -167,7 +168,7 @@ void startExport_givenPartitionKey_withStorage_shouldGenerateOutputFile() exportOptions.getTableName(), exportOptions.getProjectionColumns(), exportOptions.getLimit(), - transaction)) + manager)) .thenReturn(scanner); Mockito.when(scanner.iterator()).thenReturn(results.iterator()); try (BufferedWriter writer = @@ -187,7 +188,8 @@ void startExport_givenPartitionKey_withStorage_shouldGenerateOutputFile() @Test void startExport_givenPartitionKey_withTransaction_shouldGenerateOutputFile() throws IOException { exportManager = new JsonExportManager(manager, dao, producerTaskFactory); - TransactionCrudOperable.Scanner scanner = Mockito.mock(TransactionCrudOperable.Scanner.class); + TransactionManagerCrudOperable.Scanner scanner = + Mockito.mock(TransactionManagerCrudOperable.Scanner.class); String filePath = Paths.get("").toAbsolutePath() + "/output.json"; Map> values = UnitTestUtils.createTestValues(); Result result = new ResultImpl(values, mockData); @@ -213,7 +215,7 @@ void startExport_givenPartitionKey_withTransaction_shouldGenerateOutputFile() th exportOptions.getSortOrders(), exportOptions.getProjectionColumns(), exportOptions.getLimit(), - transaction)) + manager)) .thenReturn(scanner); Mockito.when(scanner.iterator()).thenReturn(results.iterator()); try (BufferedWriter writer = diff --git a/data-loader/core/src/test/java/com/scalar/db/dataloader/core/dataexport/JsonLineExportManagerTest.java b/data-loader/core/src/test/java/com/scalar/db/dataloader/core/dataexport/JsonLineExportManagerTest.java index a39945e580..e06aa09f6a 100644 --- a/data-loader/core/src/test/java/com/scalar/db/dataloader/core/dataexport/JsonLineExportManagerTest.java +++ b/data-loader/core/src/test/java/com/scalar/db/dataloader/core/dataexport/JsonLineExportManagerTest.java @@ -1,14 +1,11 @@ package com.scalar.db.dataloader.core.dataexport; -import static org.mockito.Mockito.when; - import com.scalar.db.api.DistributedStorage; -import com.scalar.db.api.DistributedTransaction; import com.scalar.db.api.DistributedTransactionManager; import com.scalar.db.api.Result; import com.scalar.db.api.Scanner; import com.scalar.db.api.TableMetadata; -import com.scalar.db.api.TransactionCrudOperable; +import com.scalar.db.api.TransactionManagerCrudOperable; import com.scalar.db.common.ResultImpl; import com.scalar.db.dataloader.core.FileFormat; import com.scalar.db.dataloader.core.ScalarDbMode; @@ -17,7 +14,6 @@ import com.scalar.db.dataloader.core.dataexport.producer.ProducerTaskFactory; import com.scalar.db.dataloader.core.dataimport.dao.ScalarDbDao; import com.scalar.db.dataloader.core.dataimport.dao.ScalarDbDaoException; -import com.scalar.db.exception.transaction.TransactionException; import com.scalar.db.io.Column; import com.scalar.db.io.IntColumn; import com.scalar.db.io.Key; @@ -40,21 +36,18 @@ public class JsonLineExportManagerTest { TableMetadata mockData; DistributedStorage storage; - DistributedTransaction transaction; DistributedTransactionManager manager; @Spy ScalarDbDao dao; ProducerTaskFactory producerTaskFactory; ExportManager exportManager; @BeforeEach - void setup() throws TransactionException { + void setup() { storage = Mockito.mock(DistributedStorage.class); - transaction = Mockito.mock(DistributedTransaction.class); manager = Mockito.mock(DistributedTransactionManager.class); mockData = UnitTestUtils.createTestTableMetadata(); dao = Mockito.mock(ScalarDbDao.class); producerTaskFactory = new ProducerTaskFactory(null, false, true); - when(manager.start()).thenReturn(transaction); } @Test @@ -147,7 +140,8 @@ void startExport_givenPartitionKey_withStorage_shouldGenerateOutputFile() startExport_givenValidDataWithoutPartitionKey_withTransaction_withStorage_shouldGenerateOutputFile() throws IOException, ScalarDbDaoException { exportManager = new JsonLineExportManager(manager, dao, producerTaskFactory); - TransactionCrudOperable.Scanner scanner = Mockito.mock(TransactionCrudOperable.Scanner.class); + TransactionManagerCrudOperable.Scanner scanner = + Mockito.mock(TransactionManagerCrudOperable.Scanner.class); String filePath = Paths.get("").toAbsolutePath() + "/output.jsonl"; Map> values = UnitTestUtils.createTestValues(); Result result = new ResultImpl(values, mockData); @@ -166,7 +160,7 @@ void startExport_givenPartitionKey_withStorage_shouldGenerateOutputFile() exportOptions.getTableName(), exportOptions.getProjectionColumns(), exportOptions.getLimit(), - transaction)) + manager)) .thenReturn(scanner); Mockito.when(scanner.iterator()).thenReturn(results.iterator()); try (BufferedWriter writer = @@ -186,7 +180,8 @@ void startExport_givenPartitionKey_withStorage_shouldGenerateOutputFile() @Test void startExport_givenPartitionKey_withTransaction_shouldGenerateOutputFile() throws IOException { exportManager = new JsonLineExportManager(manager, dao, producerTaskFactory); - TransactionCrudOperable.Scanner scanner = Mockito.mock(TransactionCrudOperable.Scanner.class); + TransactionManagerCrudOperable.Scanner scanner = + Mockito.mock(TransactionManagerCrudOperable.Scanner.class); String filePath = Paths.get("").toAbsolutePath() + "/output.jsonl"; Map> values = UnitTestUtils.createTestValues(); Result result = new ResultImpl(values, mockData); @@ -212,7 +207,7 @@ void startExport_givenPartitionKey_withTransaction_shouldGenerateOutputFile() th exportOptions.getSortOrders(), exportOptions.getProjectionColumns(), exportOptions.getLimit(), - transaction)) + manager)) .thenReturn(scanner); Mockito.when(scanner.iterator()).thenReturn(results.iterator()); try (BufferedWriter writer = diff --git a/data-loader/core/src/test/java/com/scalar/db/dataloader/core/dataimport/dao/ScalarDbDaoTest.java b/data-loader/core/src/test/java/com/scalar/db/dataloader/core/dataimport/dao/ScalarDbDaoTest.java index 2a7cd902d6..282b5f2f72 100644 --- a/data-loader/core/src/test/java/com/scalar/db/dataloader/core/dataimport/dao/ScalarDbDaoTest.java +++ b/data-loader/core/src/test/java/com/scalar/db/dataloader/core/dataimport/dao/ScalarDbDaoTest.java @@ -9,11 +9,11 @@ import static org.mockito.Mockito.when; import com.scalar.db.api.DistributedStorage; -import com.scalar.db.api.DistributedTransaction; +import com.scalar.db.api.DistributedTransactionManager; import com.scalar.db.api.Scan; import com.scalar.db.api.ScanBuilder; import com.scalar.db.api.Scanner; -import com.scalar.db.api.TransactionCrudOperable; +import com.scalar.db.api.TransactionManagerCrudOperable; import com.scalar.db.dataloader.core.ScanRange; import com.scalar.db.exception.storage.ExecutionException; import com.scalar.db.exception.transaction.CrudException; @@ -21,19 +21,20 @@ import java.util.*; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.mockito.Mockito; class ScalarDbDaoTest { private static final int TEST_VALUE_INT_MIN = 1; private ScalarDbDao dao; - private DistributedTransaction transaction; + private DistributedTransactionManager manager; private DistributedStorage distributedStorage; @BeforeEach void setUp() { this.dao = new ScalarDbDao(); this.distributedStorage = mock(DistributedStorage.class); - this.transaction = mock(DistributedTransaction.class); + this.manager = mock(DistributedTransactionManager.class); } @Test @@ -170,9 +171,10 @@ void createScan_scanAllWithLimitAndProjection_shouldCreateScanAllObjectWithLimit void createScanner_withTransactionManager_ShouldCreateScannerObject() throws CrudException, ScalarDbDaoException { // Create Scan Object - TransactionCrudOperable.Scanner mockScanner = mock(TransactionCrudOperable.Scanner.class); - when(transaction.getScanner(any())).thenReturn(mockScanner); - TransactionCrudOperable.Scanner result = + TransactionManagerCrudOperable.Scanner mockScanner = + Mockito.mock(TransactionManagerCrudOperable.Scanner.class); + when(manager.getScanner(any())).thenReturn(mockScanner); + TransactionManagerCrudOperable.Scanner result = this.dao.createScanner( TEST_NAMESPACE, TEST_TABLE_NAME, @@ -181,11 +183,11 @@ void createScanner_withTransactionManager_ShouldCreateScannerObject() new ArrayList<>(), new ArrayList<>(), 0, - transaction); + manager); // Assert assertNotNull(result); assertEquals(mockScanner, result); - result = this.dao.createScanner(TEST_NAMESPACE, TEST_TABLE_NAME, null, 0, transaction); + result = this.dao.createScanner(TEST_NAMESPACE, TEST_TABLE_NAME, null, 0, manager); // Assert assertNotNull(result); assertEquals(mockScanner, result); From 374e833584f57c909686cc577481dbf58fbd8c57 Mon Sep 17 00:00:00 2001 From: Jishnu J Date: Fri, 13 Jun 2025 11:48:33 +0530 Subject: [PATCH 18/19] Renamed CLI input short names for mode and inlude metadata --- .../cli/command/dataexport/ExportCommandOptions.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/command/dataexport/ExportCommandOptions.java b/data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/command/dataexport/ExportCommandOptions.java index 3846bd6777..1f7fad7a43 100755 --- a/data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/command/dataexport/ExportCommandOptions.java +++ b/data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/command/dataexport/ExportCommandOptions.java @@ -62,7 +62,7 @@ public class ExportCommandOptions { protected FileFormat outputFormat; @CommandLine.Option( - names = {"--include-metadata", "-m"}, + names = {"--include-metadata", "-im"}, description = "Include transaction metadata in the exported data (default: false)", defaultValue = "false") protected boolean includeTransactionMetadata; @@ -147,7 +147,7 @@ public class ExportCommandOptions { protected int dataChunkSize; @CommandLine.Option( - names = {"--mode", "-sm"}, + names = {"--mode", "-m"}, description = "ScalarDB mode (STORAGE, TRANSACTION) (default: STORAGE)", paramLabel = "", defaultValue = "STORAGE") From 761deca9f1f3833515d2753ad98566161dda3f5e Mon Sep 17 00:00:00 2001 From: Jishnu J Date: Mon, 16 Jun 2025 11:23:29 +0530 Subject: [PATCH 19/19] Javadocs added --- .../cli/command/dataexport/ExportCommand.java | 15 +++++++++++++ .../core/dataexport/CsvExportManager.java | 22 +++++++++++++++++++ .../core/dataexport/ExportManager.java | 18 +++++++++++++++ .../core/dataexport/JsonExportManager.java | 22 +++++++++++++++++++ .../dataexport/JsonLineExportManager.java | 22 +++++++++++++++++++ 5 files changed, 99 insertions(+) diff --git a/data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/command/dataexport/ExportCommand.java b/data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/command/dataexport/ExportCommand.java index f9aa7deaf2..e5b37be458 100755 --- a/data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/command/dataexport/ExportCommand.java +++ b/data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/command/dataexport/ExportCommand.java @@ -132,6 +132,21 @@ private void validateOutputDirectory() throws DirectoryValidationException { } } + /** + * Creates a {@link TableMetadataService} instance based on the specified {@link ScalarDbMode} and + * ScalarDB configuration file. + * + *

If the mode is {@code TRANSACTION}, this method initializes a {@link TransactionFactory} and + * uses its transaction admin to create a {@link TableMetadataTransactionService}. Otherwise, it + * initializes a {@link StorageFactory} and creates a {@link TableMetadataStorageService} using + * its storage admin. + * + * @param scalarDbMode the mode ScalarDB is running in (either {@code STORAGE} or {@code + * TRANSACTION}) + * @param scalarDbPropertiesFilePath the path to the ScalarDB properties file + * @return an appropriate {@link TableMetadataService} based on the mode + * @throws IOException if reading the ScalarDB properties file fails + */ private TableMetadataService createTableMetadataService( ScalarDbMode scalarDbMode, String scalarDbPropertiesFilePath) throws IOException { if (scalarDbMode.equals(ScalarDbMode.TRANSACTION)) { diff --git a/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataexport/CsvExportManager.java b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataexport/CsvExportManager.java index cbf482e7a1..2727faf3bc 100644 --- a/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataexport/CsvExportManager.java +++ b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataexport/CsvExportManager.java @@ -13,6 +13,18 @@ import java.util.List; public class CsvExportManager extends ExportManager { + /** + * Constructs a {@code CsvExportManager} for exporting data using a {@link DistributedStorage} + * instance. + * + *

This constructor is used when exporting data in non-transactional (storage) mode. + * + * @param distributedStorage the {@link DistributedStorage} used to read data directly from + * storage + * @param dao the {@link ScalarDbDao} used to interact with ScalarDB for exporting data + * @param producerTaskFactory the factory used to create producer tasks for generating + * CSV-formatted output + */ public CsvExportManager( DistributedStorage distributedStorage, ScalarDbDao dao, @@ -20,6 +32,16 @@ public CsvExportManager( super(distributedStorage, dao, producerTaskFactory); } + /** + * Constructs a {@code CsvExportManager} for exporting data using a {@link + * DistributedTransactionManager}. + * + * @param distributedTransactionManager the transaction manager used to read data with + * transactional guarantees + * @param dao the {@link ScalarDbDao} used to interact with ScalarDB for exporting data + * @param producerTaskFactory the factory used to create producer tasks for generating + * CSV-formatted output + */ public CsvExportManager( DistributedTransactionManager distributedTransactionManager, ScalarDbDao dao, diff --git a/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataexport/ExportManager.java b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataexport/ExportManager.java index 8889bf580a..b8af3865cc 100644 --- a/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataexport/ExportManager.java +++ b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataexport/ExportManager.java @@ -42,6 +42,15 @@ public abstract class ExportManager { private final ProducerTaskFactory producerTaskFactory; private final Object lock = new Object(); + /** + * Constructs an {@code ExportManager} that uses a {@link DistributedStorage} instance for + * non-transactional data export operations. + * + * @param distributedStorage the {@link DistributedStorage} used to read data directly from + * storage + * @param dao the {@link ScalarDbDao} used to perform data operations + * @param producerTaskFactory the factory for creating producer tasks to format the exported data + */ public ExportManager( DistributedStorage distributedStorage, ScalarDbDao dao, @@ -52,6 +61,15 @@ public ExportManager( this.producerTaskFactory = producerTaskFactory; } + /** + * Constructs an {@code ExportManager} that uses a {@link DistributedTransactionManager} instance + * for transactional data export operations. + * + * @param distributedTransactionManager the {@link DistributedTransactionManager} used to read + * data with transactional guarantees + * @param dao the {@link ScalarDbDao} used to perform data operations + * @param producerTaskFactory the factory for creating producer tasks to format the exported data + */ public ExportManager( DistributedTransactionManager distributedTransactionManager, ScalarDbDao dao, diff --git a/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataexport/JsonExportManager.java b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataexport/JsonExportManager.java index 880b156693..45ae6e1fe7 100644 --- a/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataexport/JsonExportManager.java +++ b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataexport/JsonExportManager.java @@ -9,6 +9,18 @@ import java.io.Writer; public class JsonExportManager extends ExportManager { + /** + * Constructs a {@code JsonExportManager} for exporting data using a {@link DistributedStorage} + * instance. + * + *

This constructor is used when exporting data in non-transactional (storage) mode. + * + * @param distributedStorage the {@link DistributedStorage} used to read data directly from + * storage + * @param dao the {@link ScalarDbDao} used to interact with ScalarDB for exporting data + * @param producerTaskFactory the factory used to create producer tasks for generating + * CSV-formatted output + */ public JsonExportManager( DistributedStorage distributedStorage, ScalarDbDao dao, @@ -16,6 +28,16 @@ public JsonExportManager( super(distributedStorage, dao, producerTaskFactory); } + /** + * Constructs a {@code JsonExportManager} for exporting data using a {@link + * DistributedTransactionManager}. + * + * @param distributedTransactionManager the transaction manager used to read data with + * transactional guarantees + * @param dao the {@link ScalarDbDao} used to interact with ScalarDB for exporting data + * @param producerTaskFactory the factory used to create producer tasks for generating + * CSV-formatted output + */ public JsonExportManager( DistributedTransactionManager distributedTransactionManager, ScalarDbDao dao, diff --git a/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataexport/JsonLineExportManager.java b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataexport/JsonLineExportManager.java index 47fc0b9e5c..1c81d028f8 100644 --- a/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataexport/JsonLineExportManager.java +++ b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataexport/JsonLineExportManager.java @@ -9,6 +9,18 @@ import java.io.Writer; public class JsonLineExportManager extends ExportManager { + /** + * Constructs a {@code JsonLineExportManager} for exporting data using a {@link + * DistributedStorage} instance. + * + *

This constructor is used when exporting data in non-transactional (storage) mode. + * + * @param distributedStorage the {@link DistributedStorage} used to read data directly from + * storage + * @param dao the {@link ScalarDbDao} used to interact with ScalarDB for exporting data + * @param producerTaskFactory the factory used to create producer tasks for generating + * CSV-formatted output + */ public JsonLineExportManager( DistributedStorage distributedStorage, ScalarDbDao dao, @@ -16,6 +28,16 @@ public JsonLineExportManager( super(distributedStorage, dao, producerTaskFactory); } + /** + * Constructs a {@code JsonLineExportManager} for exporting data using a {@link + * DistributedTransactionManager}. + * + * @param distributedTransactionManager the transaction manager used to read data with + * transactional guarantees + * @param dao the {@link ScalarDbDao} used to interact with ScalarDB for exporting data + * @param producerTaskFactory the factory used to create producer tasks for generating + * CSV-formatted output + */ public JsonLineExportManager( DistributedTransactionManager distributedTransactionManager, ScalarDbDao dao,