diff --git a/configs/milvus.yaml b/configs/milvus.yaml index 46a5fd778ad51..59fcf91ba4b2f 100644 --- a/configs/milvus.yaml +++ b/configs/milvus.yaml @@ -115,7 +115,7 @@ minio: secretAccessKey: minioadmin useSSL: false # Switch value to control if to access the MinIO or S3 service through SSL. ssl: - tlsCACert: /path/to/public.crt # path to your CACert file + tlsCACert: # path to your CACert file # Name of the bucket where Milvus stores data in MinIO or S3. # Milvus 2.0.0 does not support storing data in multiple buckets. # Bucket with this name will be created if it does not exist. If the bucket already exists and is accessible, it will be used directly. Otherwise, there will be an error. diff --git a/internal/core/src/storage/ChunkManager.cpp b/internal/core/src/storage/ChunkManager.cpp index c6d8908625e0e..b7ae673ae01f4 100644 --- a/internal/core/src/storage/ChunkManager.cpp +++ b/internal/core/src/storage/ChunkManager.cpp @@ -19,14 +19,8 @@ #include #include #include -#include -#include -#include -#include -#include -#include -#include -#include +#include +#include #include "storage/MinioChunkManager.h" #include "storage/AliyunSTSClient.h" @@ -40,53 +34,13 @@ namespace milvus::storage { -Aws::String -ConvertToAwsString(const std::string& str) { - return Aws::String(str.c_str(), str.size()); -} - -Aws::Client::ClientConfiguration -generateConfig(const StorageConfig& storage_config) { - // The ClientConfiguration default constructor will take a long time. - // For more details, please refer to https://github.com/aws/aws-sdk-cpp/issues/1440 - static Aws::Client::ClientConfiguration g_config; - Aws::Client::ClientConfiguration config = g_config; - config.endpointOverride = ConvertToAwsString(storage_config.address); - - // Three cases: - // 1. no ssl, verifySSL=false - // 2. self-signed certificate, verifySSL=false - // 3. CA-signed certificate, verifySSL=true - if (storage_config.useSSL) { - config.scheme = Aws::Http::Scheme::HTTPS; - config.verifySSL = true; - if (!storage_config.sslCACert.empty()) { - config.caPath = ConvertToAwsString(storage_config.sslCACert); - config.verifySSL = false; - } - } else { - config.scheme = Aws::Http::Scheme::HTTP; - config.verifySSL = false; - } - - if (!storage_config.region.empty()) { - config.region = ConvertToAwsString(storage_config.region); - } - - config.requestTimeoutMs = storage_config.requestTimeoutMs == 0 - ? DEFAULT_CHUNK_MANAGER_REQUEST_TIMEOUT_MS - : storage_config.requestTimeoutMs; - - return config; -} - AwsChunkManager::AwsChunkManager(const StorageConfig& storage_config) { default_bucket_name_ = storage_config.bucket_name; remote_root_path_ = storage_config.root_path; InitSDKAPIDefault(storage_config.log_level); - Aws::Client::ClientConfiguration config = generateConfig(storage_config); + Aws::S3Crt::ClientConfiguration config = generateConfig(storage_config); if (storage_config.useIAM) { auto provider = std::make_shared(); @@ -98,7 +52,7 @@ AwsChunkManager::AwsChunkManager(const StorageConfig& storage_config) { AssertInfo(!aws_credentials.GetSessionToken().empty(), "if use iam, token should not be empty"); - client_ = std::make_shared( + client_ = std::make_shared( provider, config, Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never, @@ -134,10 +88,10 @@ GcpChunkManager::GcpChunkManager(const StorageConfig& storage_config) { InitSDKAPIDefault(storage_config.log_level); - Aws::Client::ClientConfiguration config = generateConfig(storage_config); + Aws::S3Crt::ClientConfiguration config = generateConfig(storage_config); if (storage_config.useIAM) { // Using S3 client instead of google client because of compatible protocol - client_ = std::make_shared( + client_ = std::make_shared( config, Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never, storage_config.useVirtualHost); @@ -162,11 +116,12 @@ AliyunChunkManager::AliyunChunkManager(const StorageConfig& storage_config) { InitSDKAPIDefault(storage_config.log_level); - Aws::Client::ClientConfiguration config = generateConfig(storage_config); - // For aliyun oss, support use virtual host mode StorageConfig mutable_config = storage_config; mutable_config.useVirtualHost = true; + + Aws::S3Crt::ClientConfiguration config = generateConfig(mutable_config); + if (storage_config.useIAM) { auto aliyun_provider = Aws::MakeShared< Aws::Auth::AliyunSTSAssumeRoleWebIdentityCredentialsProvider>( @@ -178,7 +133,7 @@ AliyunChunkManager::AliyunChunkManager(const StorageConfig& storage_config) { "if use iam, secret key should not be empty"); AssertInfo(!aliyun_credentials.GetSessionToken().empty(), "if use iam, token should not be empty"); - client_ = std::make_shared( + client_ = std::make_shared( aliyun_provider, config, Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never, @@ -205,10 +160,10 @@ TencentCloudChunkManager::TencentCloudChunkManager( InitSDKAPIDefault(storage_config.log_level); - Aws::Client::ClientConfiguration config = generateConfig(storage_config); - StorageConfig mutable_config = storage_config; mutable_config.useVirtualHost = true; + Aws::S3Crt::ClientConfiguration config = generateConfig(mutable_config); + if (storage_config.useIAM) { auto tencent_cloud_provider = Aws::MakeShared< Aws::Auth::TencentCloudSTSAssumeRoleWebIdentityCredentialsProvider>( @@ -221,7 +176,7 @@ TencentCloudChunkManager::TencentCloudChunkManager( "if use iam, secret key should not be empty"); AssertInfo(!tencent_cloud_credentials.GetSessionToken().empty(), "if use iam, token should not be empty"); - client_ = std::make_shared( + client_ = std::make_shared( tencent_cloud_provider, config, Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never, diff --git a/internal/core/src/storage/MinioChunkManager.cpp b/internal/core/src/storage/MinioChunkManager.cpp index 9085c621aa360..ced3da958d539 100644 --- a/internal/core/src/storage/MinioChunkManager.cpp +++ b/internal/core/src/storage/MinioChunkManager.cpp @@ -17,18 +17,21 @@ #include "storage/MinioChunkManager.h" #include +#include +#include #include #include #include #include -#include -#include -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include +#include +#include +#include #include "storage/AliyunSTSClient.h" #include "storage/AliyunCredentialsProvider.h" @@ -61,29 +64,6 @@ SwallowHandler(int signal) { #pragma GCC diagnostic pop } -/** - * @brief convert std::string to Aws::String - * because Aws has String type internally - * but has a copy of string content unfortunately - * TODO: remove this convert - * @param str - * @return Aws::String - */ -inline Aws::String -ConvertToAwsString(const std::string& str) { - return Aws::String(str.c_str(), str.size()); -} - -/** - * @brief convert Aws::string to std::string - * @param aws_str - * @return std::string - */ -inline std::string -ConvertFromAwsString(const Aws::String& aws_str) { - return std::string(aws_str.c_str(), aws_str.size()); -} - void AwsLogger::ProcessFormattedStatement(Aws::String&& statement) { LOG_INFO("[AWS LOG] {}", statement); @@ -198,7 +178,7 @@ MinioChunkManager::ShutdownSDKAPI() { void MinioChunkManager::BuildS3Client( const StorageConfig& storage_config, - const Aws::Client::ClientConfiguration& config) { + const Aws::S3Crt::ClientConfiguration& config) { if (storage_config.useIAM) { auto provider = std::make_shared(); @@ -210,7 +190,7 @@ MinioChunkManager::BuildS3Client( AssertInfo(!aws_credentials.GetSessionToken().empty(), "if use iam, token should not be empty"); - client_ = std::make_shared( + client_ = std::make_shared( provider, config, Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never, @@ -226,7 +206,9 @@ MinioChunkManager::PreCheck(const StorageConfig& config) { config.ToString()); try { // Just test connection not check real list, avoid cost resource. - ListWithPrefix("justforconnectioncheck"); + if (BucketExists(default_bucket_name_)) { + ListWithPrefix("justforconnectioncheck"); + } } catch (SegcoreError& e) { auto err_message = fmt::format( "precheck chunk manager client failed, " @@ -244,13 +226,13 @@ MinioChunkManager::PreCheck(const StorageConfig& config) { void MinioChunkManager::BuildAccessKeyClient( const StorageConfig& storage_config, - const Aws::Client::ClientConfiguration& config) { + const Aws::S3Crt::ClientConfiguration& config) { AssertInfo(!storage_config.access_key_id.empty(), "if not use iam, access key should not be empty"); AssertInfo(!storage_config.access_key_value.empty(), "if not use iam, access value should not be empty"); - client_ = std::make_shared( + client_ = std::make_shared( Aws::Auth::AWSCredentials( ConvertToAwsString(storage_config.access_key_id), ConvertToAwsString(storage_config.access_key_value)), @@ -262,10 +244,13 @@ MinioChunkManager::BuildAccessKeyClient( void MinioChunkManager::BuildAliyunCloudClient( const StorageConfig& storage_config, - const Aws::Client::ClientConfiguration& config) { + const Aws::S3Crt::ClientConfiguration& config) { // For aliyun oss, support use virtual host mode StorageConfig mutable_config = storage_config; mutable_config.useVirtualHost = true; + + Aws::S3Crt::ClientConfiguration mutable_crt_config = config; + mutable_crt_config.useVirtualAddressing = true; if (storage_config.useIAM) { auto aliyun_provider = Aws::MakeShared< Aws::Auth::AliyunSTSAssumeRoleWebIdentityCredentialsProvider>( @@ -277,23 +262,23 @@ MinioChunkManager::BuildAliyunCloudClient( "if use iam, secret key should not be empty"); AssertInfo(!aliyun_credentials.GetSessionToken().empty(), "if use iam, token should not be empty"); - client_ = std::make_shared( + client_ = std::make_shared( aliyun_provider, - config, + mutable_crt_config, Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never, mutable_config.useVirtualHost); } else { - BuildAccessKeyClient(mutable_config, config); + BuildAccessKeyClient(mutable_config, mutable_crt_config); } } void MinioChunkManager::BuildGoogleCloudClient( const StorageConfig& storage_config, - const Aws::Client::ClientConfiguration& config) { + const Aws::S3Crt::ClientConfiguration& config) { if (storage_config.useIAM) { // Using S3 client instead of google client because of compatible protocol - client_ = std::make_shared( + client_ = std::make_shared( config, Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never, storage_config.useVirtualHost); @@ -316,35 +301,7 @@ MinioChunkManager::MinioChunkManager(const StorageConfig& storage_config) InitSDKAPI(storageType, storage_config.useIAM, storage_config.log_level); - // The ClientConfiguration default constructor will take a long time. - // For more details, please refer to https://github.com/aws/aws-sdk-cpp/issues/1440 - static Aws::Client::ClientConfiguration g_config; - Aws::Client::ClientConfiguration config = g_config; - config.endpointOverride = ConvertToAwsString(storage_config.address); - - // Three cases: - // 1. no ssl, verifySSL=false - // 2. self-signed certificate, verifySSL=false - // 3. CA-signed certificate, verifySSL=true - if (storage_config.useSSL) { - config.scheme = Aws::Http::Scheme::HTTPS; - config.verifySSL = true; - if (!storage_config.sslCACert.empty()) { - config.caPath = ConvertToAwsString(storage_config.sslCACert); - config.verifySSL = false; - } - } else { - config.scheme = Aws::Http::Scheme::HTTP; - config.verifySSL = false; - } - - config.requestTimeoutMs = storage_config.requestTimeoutMs == 0 - ? DEFAULT_CHUNK_MANAGER_REQUEST_TIMEOUT_MS - : storage_config.requestTimeoutMs; - - if (!storage_config.region.empty()) { - config.region = ConvertToAwsString(storage_config.region); - } + Aws::S3Crt::ClientConfiguration config = generateConfig(storage_config); if (storageType == RemoteStorageType::S3) { BuildS3Client(storage_config, config); @@ -404,7 +361,7 @@ MinioChunkManager::Write(const std::string& filepath, bool MinioChunkManager::BucketExists(const std::string& bucket_name) { - Aws::S3::Model::HeadBucketRequest request; + Aws::S3Crt::Model::HeadBucketRequest request; request.SetBucket(bucket_name.c_str()); auto outcome = client_->HeadBucket(request); @@ -414,8 +371,8 @@ MinioChunkManager::BucketExists(const std::string& bucket_name) { auto error_type = err.GetErrorType(); // only throw if the error is not nosuchbucket // if bucket not exist, HeadBucket return errorType RESOURCE_NOT_FOUND - if (error_type != Aws::S3::S3Errors::NO_SUCH_BUCKET && - error_type != Aws::S3::S3Errors::RESOURCE_NOT_FOUND) { + if (error_type != Aws::S3Crt::S3CrtErrors::NO_SUCH_BUCKET && + error_type != Aws::S3Crt::S3CrtErrors::RESOURCE_NOT_FOUND) { ThrowS3Error("BucketExists", err, "params, bucket={}", bucket_name); } return false; @@ -440,7 +397,7 @@ MinioChunkManager::ListBuckets() { bool MinioChunkManager::CreateBucket(const std::string& bucket_name) { - Aws::S3::Model::CreateBucketRequest request; + Aws::S3Crt::Model::CreateBucketRequest request; request.SetBucket(bucket_name.c_str()); auto outcome = client_->CreateBucket(request); @@ -448,7 +405,7 @@ MinioChunkManager::CreateBucket(const std::string& bucket_name) { if (!outcome.IsSuccess()) { const auto& err = outcome.GetError(); if (err.GetErrorType() != - Aws::S3::S3Errors::BUCKET_ALREADY_OWNED_BY_YOU) { + Aws::S3Crt::S3CrtErrors::BUCKET_ALREADY_OWNED_BY_YOU) { ThrowS3Error("CreateBucket", err, "params, bucket={}", bucket_name); } return false; @@ -458,7 +415,7 @@ MinioChunkManager::CreateBucket(const std::string& bucket_name) { bool MinioChunkManager::DeleteBucket(const std::string& bucket_name) { - Aws::S3::Model::DeleteBucketRequest request; + Aws::S3Crt::Model::DeleteBucketRequest request; request.SetBucket(bucket_name.c_str()); auto outcome = client_->DeleteBucket(request); @@ -466,8 +423,8 @@ MinioChunkManager::DeleteBucket(const std::string& bucket_name) { if (!outcome.IsSuccess()) { const auto& err = outcome.GetError(); auto error_type = err.GetErrorType(); - if (error_type != Aws::S3::S3Errors::NO_SUCH_BUCKET && - error_type != Aws::S3::S3Errors::RESOURCE_NOT_FOUND) { + if (error_type != Aws::S3Crt::S3CrtErrors::NO_SUCH_BUCKET && + error_type != Aws::S3Crt::S3CrtErrors::RESOURCE_NOT_FOUND) { ThrowS3Error("DeleteBucket", err, "params, bucket={}", bucket_name); } return false; @@ -478,7 +435,7 @@ MinioChunkManager::DeleteBucket(const std::string& bucket_name) { bool MinioChunkManager::ObjectExists(const std::string& bucket_name, const std::string& object_name) { - Aws::S3::Model::HeadObjectRequest request; + Aws::S3Crt::Model::HeadObjectRequest request; request.SetBucket(bucket_name.c_str()); request.SetKey(object_name.c_str()); @@ -509,7 +466,7 @@ MinioChunkManager::ObjectExists(const std::string& bucket_name, uint64_t MinioChunkManager::GetObjectSize(const std::string& bucket_name, const std::string& object_name) { - Aws::S3::Model::HeadObjectRequest request; + Aws::S3Crt::Model::HeadObjectRequest request; request.SetBucket(bucket_name.c_str()); request.SetKey(object_name.c_str()); @@ -535,7 +492,7 @@ MinioChunkManager::GetObjectSize(const std::string& bucket_name, bool MinioChunkManager::DeleteObject(const std::string& bucket_name, const std::string& object_name) { - Aws::S3::Model::DeleteObjectRequest request; + Aws::S3Crt::Model::DeleteObjectRequest request; request.SetBucket(bucket_name.c_str()); request.SetKey(object_name.c_str()); @@ -568,7 +525,7 @@ MinioChunkManager::PutObjectBuffer(const std::string& bucket_name, const std::string& object_name, void* buf, uint64_t size) { - Aws::S3::Model::PutObjectRequest request; + Aws::S3Crt::Model::PutObjectRequest request; request.SetBucket(bucket_name.c_str()); request.SetKey(object_name.c_str()); @@ -643,7 +600,7 @@ MinioChunkManager::GetObjectBuffer(const std::string& bucket_name, const std::string& object_name, void* buf, uint64_t size) { - Aws::S3::Model::GetObjectRequest request; + Aws::S3Crt::Model::GetObjectRequest request; request.SetBucket(bucket_name.c_str()); request.SetKey(object_name.c_str()); @@ -684,7 +641,7 @@ std::vector MinioChunkManager::ListObjects(const std::string& bucket_name, const std::string& prefix) { std::vector objects_vec; - Aws::S3::Model::ListObjectsRequest request; + Aws::S3Crt::Model::ListObjectsRequest request; request.WithBucket(bucket_name); if (prefix != "") { request.SetPrefix(prefix); diff --git a/internal/core/src/storage/MinioChunkManager.h b/internal/core/src/storage/MinioChunkManager.h index a49ff04e1c728..d098977a850b4 100644 --- a/internal/core/src/storage/MinioChunkManager.h +++ b/internal/core/src/storage/MinioChunkManager.h @@ -20,6 +20,9 @@ #include #include #include +#include +#include +#include #include #include @@ -29,7 +32,9 @@ #include #include #include -#include +#include +#include +#include #include #include #include @@ -43,6 +48,7 @@ #include "storage/ChunkManager.h" #include "storage/Types.h" #include "log/Log.h" +#include "common/Consts.h" namespace milvus::storage { @@ -55,7 +61,7 @@ enum class RemoteStorageType { template static std::string S3ErrorMessage(const std::string& func, - const Aws::S3::S3Error& err, + const Aws::S3Crt::S3CrtError& err, const std::string& fmt_string, Args&&... args) { std::ostringstream oss; @@ -69,7 +75,7 @@ S3ErrorMessage(const std::string& func, template static SegcoreError ThrowS3Error(const std::string& func, - const Aws::S3::S3Error& err, + const Aws::S3Crt::S3CrtError& err, const std::string& fmt_string, Args&&... args) { std::string error_message = S3ErrorMessage(func, err, fmt_string, args...); @@ -78,9 +84,70 @@ ThrowS3Error(const std::string& func, } static bool -IsNotFound(const Aws::S3::S3Errors& s3err) { - return (s3err == Aws::S3::S3Errors::NO_SUCH_KEY || - s3err == Aws::S3::S3Errors::RESOURCE_NOT_FOUND); +IsNotFound(const Aws::S3Crt::S3CrtErrors& s3err) { + return (s3err == Aws::S3Crt::S3CrtErrors::NO_SUCH_KEY || + s3err == Aws::S3Crt::S3CrtErrors::RESOURCE_NOT_FOUND); +} + +/** + * @brief convert std::string to Aws::String + * because Aws has String type internally + * but has a copy of string content unfortunately + * TODO: remove this convert + * @param str + * @return Aws::String + */ +inline Aws::String +ConvertToAwsString(const std::string& str) { + return Aws::String(str.c_str(), str.size()); +} + +/** + * @brief convert Aws::string to std::string + * @param aws_str + * @return std::string + */ +inline std::string +ConvertFromAwsString(const Aws::String& aws_str) { + return std::string(aws_str.c_str(), aws_str.size()); +} + +static Aws::S3Crt::ClientConfiguration +generateConfig(const StorageConfig& storage_config) { + // The ClientConfiguration default constructor will take a long time. + // For more details, please refer to https://github.com/aws/aws-sdk-cpp/issues/1440 + static Aws::S3Crt::ClientConfiguration g_config; + Aws::S3Crt::ClientConfiguration config = g_config; + config.endpointOverride = ConvertToAwsString(storage_config.address); + + // Three cases: + // 1. no ssl, verifySSL=false + // 2. self-signed certificate, verifySSL=false + // 3. CA-signed certificate, verifySSL=true + if (storage_config.useSSL) { + config.scheme = Aws::Http::Scheme::HTTPS; + config.verifySSL = true; + if (!storage_config.sslCACert.empty()) { + config.caPath = ConvertToAwsString(storage_config.sslCACert); + config.verifySSL = false; + } + } else { + config.scheme = Aws::Http::Scheme::HTTP; + config.verifySSL = false; + } + + if (!storage_config.region.empty()) { + config.region = ConvertToAwsString(storage_config.region); + } + + config.useVirtualAddressing = storage_config.useVirtualHost; + config.throughputTargetGbps = 30; + config.partSize = 8 * 1024 * 1024; // 8MB + config.requestTimeoutMs = storage_config.requestTimeoutMs == 0 + ? DEFAULT_CHUNK_MANAGER_REQUEST_TIMEOUT_MS + : storage_config.requestTimeoutMs; + + return config; } /** @@ -225,23 +292,23 @@ class MinioChunkManager : public ChunkManager { ShutdownSDKAPI(); void BuildS3Client(const StorageConfig& storage_config, - const Aws::Client::ClientConfiguration& config); + const Aws::S3Crt::ClientConfiguration& config); void BuildAliyunCloudClient(const StorageConfig& storage_config, - const Aws::Client::ClientConfiguration& config); + const Aws::S3Crt::ClientConfiguration& config); void BuildGoogleCloudClient(const StorageConfig& storage_config, - const Aws::Client::ClientConfiguration& config); + const Aws::S3Crt::ClientConfiguration& config); protected: void BuildAccessKeyClient(const StorageConfig& storage_config, - const Aws::Client::ClientConfiguration& config); + const Aws::S3Crt::ClientConfiguration& config); Aws::SDKOptions sdk_options_; static std::atomic init_count_; static std::mutex client_mutex_; - std::shared_ptr client_; + std::shared_ptr client_; std::string default_bucket_name_; std::string remote_root_path_; }; diff --git a/internal/core/src/storage/MinioChunkManagerTest.cpp b/internal/core/src/storage/MinioChunkManagerTest.cpp index 9361ff4f021d2..bad5d48b45e41 100644 --- a/internal/core/src/storage/MinioChunkManagerTest.cpp +++ b/internal/core/src/storage/MinioChunkManagerTest.cpp @@ -12,6 +12,7 @@ #include #include #include +#include #include "storage/MinioChunkManager.h" #include "test_utils/indexbuilder_test_utils.h" @@ -38,53 +39,6 @@ class MinioChunkManagerTest : public testing::Test { StorageConfig configs_; }; -//StorageConfig -//get_aliyun_cloud_storage_config() { -// auto endpoint = "oss-cn-shanghai.aliyuncs.com:443"; -// auto accessKey = ""; -// auto accessValue = ""; -// auto rootPath = "files"; -// auto useSSL = false; -// auto sslCACert = ""; -// auto useIam = true; -// auto iamEndPoint = ""; -// auto bucketName = "vdc-infra-poc"; -// auto cloudProvider = "aliyun"; -// auto logLevel = "error"; -// auto region = ""; -// -// return StorageConfig{endpoint, -// bucketName, -// accessKey, -// accessValue, -// rootPath, -// "minio", -// cloudProvider, -// iamEndPoint, -// logLevel, -// region, -// useSSL, -// sslCACert, -// useIam}; -//} - -//class AliyunChunkManagerTest : public testing::Test { -// public: -// AliyunChunkManagerTest() { -// } -// ~AliyunChunkManagerTest() { -// } -// -// virtual void -// SetUp() { -// chunk_manager_ = std::make_unique( -// get_aliyun_cloud_storage_config()); -// } -// -// protected: -// MinioChunkManagerPtr chunk_manager_; -//}; - TEST_F(MinioChunkManagerTest, InitFailed) { auto configs = StorageConfig{}; // wrong address @@ -116,7 +70,7 @@ TEST_F(MinioChunkManagerTest, BucketNegtive) { } TEST_F(MinioChunkManagerTest, ObjectExist) { - string testBucketName = configs_.bucket_name; + string testBucketName = "test-object-exist"; string objPath = "1/3"; chunk_manager_->SetBucketName(testBucketName); if (!chunk_manager_->BucketExists(testBucketName)) { @@ -129,7 +83,7 @@ TEST_F(MinioChunkManagerTest, ObjectExist) { } TEST_F(MinioChunkManagerTest, WritePositive) { - string testBucketName = configs_.bucket_name; + string testBucketName = "test-write-positive"; chunk_manager_->SetBucketName(testBucketName); EXPECT_EQ(chunk_manager_->GetBucketName(), testBucketName); @@ -163,7 +117,7 @@ TEST_F(MinioChunkManagerTest, WritePositive) { } TEST_F(MinioChunkManagerTest, ReadPositive) { - string testBucketName = configs_.bucket_name; + string testBucketName = "test-read-positive"; chunk_manager_->SetBucketName(testBucketName); EXPECT_EQ(chunk_manager_->GetBucketName(), testBucketName); @@ -212,7 +166,7 @@ TEST_F(MinioChunkManagerTest, ReadPositive) { } TEST_F(MinioChunkManagerTest, ReadNotExist) { - string testBucketName = configs_.bucket_name; + string testBucketName = "test-read-not-exist"; chunk_manager_->SetBucketName(testBucketName); EXPECT_EQ(chunk_manager_->GetBucketName(), testBucketName); @@ -250,12 +204,10 @@ TEST_F(MinioChunkManagerTest, RemovePositive) { bool exist = chunk_manager_->Exist(path); EXPECT_EQ(exist, true); - bool deleted = chunk_manager_->Remove(path); - EXPECT_EQ(deleted, true); + chunk_manager_->Remove(path); // test double deleted - deleted = chunk_manager_->Remove(path); - EXPECT_EQ(deleted, false); + chunk_manager_->Remove(path); exist = chunk_manager_->Exist(path); EXPECT_EQ(exist, false); @@ -286,8 +238,7 @@ TEST_F(MinioChunkManagerTest, ListWithPrefixPositive) { EXPECT_EQ(objs[0], "1/7/4"); EXPECT_EQ(objs[1], "1/7/8"); - objs = chunk_manager_->ListWithPrefix("//1/7"); - EXPECT_EQ(objs.size(), 2); + EXPECT_THROW(chunk_manager_->ListWithPrefix("//1/7"), SegcoreError); objs = chunk_manager_->ListWithPrefix("1"); EXPECT_EQ(objs.size(), 3); diff --git a/internal/core/src/storage/RemoteChunkManagerTest.cpp b/internal/core/src/storage/RemoteChunkManagerTest.cpp index b9d58d234801d..9418556cb7682 100644 --- a/internal/core/src/storage/RemoteChunkManagerTest.cpp +++ b/internal/core/src/storage/RemoteChunkManagerTest.cpp @@ -30,10 +30,6 @@ get_default_remote_storage_config() { StorageConfig storage_config; storage_config.storage_type = "remote"; storage_config.address = "localhost:9000"; - char const* tmp = getenv("MINIO_ADDRESS"); - if (tmp != NULL) { - storage_config.address = string(tmp); - } storage_config.bucket_name = get_default_bucket_name(); storage_config.access_key_id = "minioadmin"; storage_config.access_key_value = "minioadmin"; @@ -79,27 +75,11 @@ TEST_F(RemoteChunkManagerTest, BasicFunctions) { the_chunk_manager_ = CreateChunkManager(configs_); EXPECT_TRUE(the_chunk_manager_->GetName() == "GcpChunkManager"); - configs_.cloud_provider = "aliyun"; - the_chunk_manager_ = CreateChunkManager(configs_); - EXPECT_TRUE(the_chunk_manager_->GetName() == "AliyunChunkManager"); - -#ifdef AZURE_BUILD_DIR - configs_.cloud_provider = "azure"; - the_chunk_manager_ = CreateChunkManager(configs_); - EXPECT_TRUE(the_chunk_manager_->GetName() == "AzureChunkManager"); -#endif - -#ifdef ENABLE_GCP_NATIVE - configs_.cloud_provider = "gcpnative"; - the_chunk_manager_ = CreateChunkManager(configs_); - EXPECT_TRUE(the_chunk_manager_->GetName() == "GcpNativeChunkManager"); -#endif - configs_.cloud_provider = ""; } TEST_F(RemoteChunkManagerTest, BucketPositive) { - string testBucketName = get_default_bucket_name(); + string testBucketName = "test-bucket"; aws_chunk_manager_->SetBucketName(testBucketName); bool exist = aws_chunk_manager_->BucketExists(testBucketName); EXPECT_EQ(exist, false); @@ -110,7 +90,7 @@ TEST_F(RemoteChunkManagerTest, BucketPositive) { } TEST_F(RemoteChunkManagerTest, BucketNegtive) { - string testBucketName = get_default_bucket_name(); + string testBucketName = "test-bucket-ng"; aws_chunk_manager_->SetBucketName(testBucketName); aws_chunk_manager_->DeleteBucket(testBucketName); @@ -125,7 +105,7 @@ TEST_F(RemoteChunkManagerTest, BucketNegtive) { } TEST_F(RemoteChunkManagerTest, ObjectExist) { - string testBucketName = get_default_bucket_name(); + string testBucketName = "test-object-exist"; string objPath = "1/3"; aws_chunk_manager_->SetBucketName(testBucketName); if (!aws_chunk_manager_->BucketExists(testBucketName)) { @@ -140,7 +120,7 @@ TEST_F(RemoteChunkManagerTest, ObjectExist) { } TEST_F(RemoteChunkManagerTest, WritePositive) { - string testBucketName = get_default_bucket_name(); + string testBucketName = "test-write-positive"; aws_chunk_manager_->SetBucketName(testBucketName); EXPECT_EQ(aws_chunk_manager_->GetBucketName(), testBucketName); @@ -173,7 +153,7 @@ TEST_F(RemoteChunkManagerTest, WritePositive) { } TEST_F(RemoteChunkManagerTest, ReadPositive) { - string testBucketName = get_default_bucket_name(); + string testBucketName = "test-read-positive"; aws_chunk_manager_->SetBucketName(testBucketName); EXPECT_EQ(aws_chunk_manager_->GetBucketName(), testBucketName); @@ -222,7 +202,7 @@ TEST_F(RemoteChunkManagerTest, ReadPositive) { } TEST_F(RemoteChunkManagerTest, RemovePositive) { - string testBucketName = get_default_bucket_name(); + string testBucketName = "test-remove-positive"; aws_chunk_manager_->SetBucketName(testBucketName); EXPECT_EQ(aws_chunk_manager_->GetBucketName(), testBucketName); @@ -245,7 +225,7 @@ TEST_F(RemoteChunkManagerTest, RemovePositive) { } TEST_F(RemoteChunkManagerTest, ListWithPrefixPositive) { - string testBucketName = get_default_bucket_name(); + string testBucketName = "test-list-with-prefix-positive"; aws_chunk_manager_->SetBucketName(testBucketName); EXPECT_EQ(aws_chunk_manager_->GetBucketName(), testBucketName); @@ -267,8 +247,7 @@ TEST_F(RemoteChunkManagerTest, ListWithPrefixPositive) { EXPECT_EQ(objs[0], "1/7/4"); EXPECT_EQ(objs[1], "1/7/8"); - objs = aws_chunk_manager_->ListWithPrefix("//1/7"); - EXPECT_EQ(objs.size(), 2); + EXPECT_THROW(aws_chunk_manager_->ListWithPrefix("//1/7"), SegcoreError); objs = aws_chunk_manager_->ListWithPrefix("1"); EXPECT_EQ(objs.size(), 3); diff --git a/internal/core/unittest/CMakeLists.txt b/internal/core/unittest/CMakeLists.txt index 2e7a2bf3b4325..348c5544a9ad3 100644 --- a/internal/core/unittest/CMakeLists.txt +++ b/internal/core/unittest/CMakeLists.txt @@ -57,12 +57,6 @@ if ( NOT (INDEX_ENGINE STREQUAL "cardinal") ) list(FILTER MILVUS_TEST_FILES EXCLUDE REGEX "KmeansClusteringTest\\.cpp$") endif() -# need update aws-sdk-cpp, see more from https://github.com/aws/aws-sdk-cpp/issues/1757. -# now we always remove this file from MILVUS_TEST_FILES thus it is never compiled. -# once done, compile this test file only if `BUILD_DISK_ANN STREQUAL "ON"`. -# if ( BUILD_DISK_ANN STREQUAL "OFF" ) - list(FILTER MILVUS_TEST_FILES EXCLUDE REGEX "MinioChunkManagerTest\\.cpp$") -# endif() # bitset has its own test binary list(FILTER MILVUS_TEST_FILES EXCLUDE REGEX "BitsetTest\\.cpp$") @@ -81,9 +75,6 @@ else() list(FILTER MILVUS_TEST_FILES EXCLUDE REGEX "AzureChunkManagerTest\\.cpp$") list(FILTER MILVUS_TEST_FILES EXCLUDE REGEX "AzureBlobChunkManagerTest\\.cpp$") endif() -# need update aws-sdk-cpp, see more from https://github.com/aws/aws-sdk-cpp/issues/2119 -# once done, move this line to the else branch of `if (DEFINED AZURE_BUILD_DIR)` -list(FILTER MILVUS_TEST_FILES EXCLUDE REGEX "RemoteChunkManagerTest\\.cpp$") if (ENABLE_GCP_NATIVE) add_definitions(-DENABLE_GCP_NATIVE)