diff --git a/be/src/agent/task_worker_pool.cpp b/be/src/agent/task_worker_pool.cpp index 23d707702db049..3edbbcc2c4e18d 100644 --- a/be/src/agent/task_worker_pool.cpp +++ b/be/src/agent/task_worker_pool.cpp @@ -58,10 +58,10 @@ #include "common/logging.h" #include "common/metrics/doris_metrics.h" #include "common/status.h" +#include "cpp/client/obj_storage_client.h" #include "io/fs/file_system.h" #include "io/fs/hdfs_file_system.h" #include "io/fs/local_file_system.h" -#include "io/fs/obj_storage_client.h" #include "io/fs/path.h" #include "io/fs/remote_file_system.h" #include "io/fs/s3_file_system.h" diff --git a/be/src/cloud/cloud_meta_mgr.cpp b/be/src/cloud/cloud_meta_mgr.cpp index dd1290f7024a54..c2c6d834989275 100644 --- a/be/src/cloud/cloud_meta_mgr.cpp +++ b/be/src/cloud/cloud_meta_mgr.cpp @@ -54,8 +54,8 @@ #include "common/config.h" #include "common/logging.h" #include "common/status.h" +#include "cpp/client/obj_storage_client.h" #include "cpp/sync_point.h" -#include "io/fs/obj_storage_client.h" #include "load/stream_load/stream_load_context.h" #include "runtime/cluster_info.h" #include "runtime/exec_env.h" diff --git a/be/src/common/status.h b/be/src/common/status.h index d29b66459a3832..8536332699a0dd 100644 --- a/be/src/common/status.h +++ b/be/src/common/status.h @@ -25,14 +25,8 @@ namespace doris { -namespace io { -struct ObjectStorageStatus; -} - class Status; -extern io::ObjectStorageStatus convert_to_obj_response(Status st); - class PStatus; namespace ErrorCode { @@ -577,8 +571,6 @@ class [[nodiscard]] Status { std::pair retrieve_error_msg() { return {_code, std::move(_err_msg->_msg)}; } - friend io::ObjectStorageStatus convert_to_obj_response(Status st); - private: int _code; struct ErrMsg { diff --git a/be/src/exprs/function/ai/embed.h b/be/src/exprs/function/ai/embed.h index d10349fb69b229..146885d622e732 100644 --- a/be/src/exprs/function/ai/embed.h +++ b/be/src/exprs/function/ai/embed.h @@ -435,18 +435,15 @@ class FunctionEmbed : public AIFunction { S3ClientConf s3_client_conf; RETURN_IF_ERROR(init_s3_client_conf_from_json(file_input, s3_client_conf)); - auto s3_client = S3ClientFactory::instance().create(s3_client_conf); - if (s3_client == nullptr) { - return Status::InternalError("Failed to create S3 client for EMBED file input"); - } + auto s3_client = DORIS_TRY(S3ClientFactory::instance().create(s3_client_conf)); S3URI s3_uri(uri); RETURN_IF_ERROR(s3_uri.parse()); std::string bucket = s3_uri.get_bucket(); std::string key = s3_uri.get_key(); DORIS_CHECK(!bucket.empty() && !key.empty()); - media_url = s3_client->generate_presigned_url({.bucket = bucket, .key = key}, ttl_seconds, - s3_client_conf); + media_url = s3_client->generate_presigned_url({.bucket = bucket, .key = key, .prefix = ""}, + ttl_seconds); return Status::OK(); } }; diff --git a/be/src/io/CMakeLists.txt b/be/src/io/CMakeLists.txt index 56c2eeb94a3819..2ebe38e6d5c9bf 100644 --- a/be/src/io/CMakeLists.txt +++ b/be/src/io/CMakeLists.txt @@ -22,9 +22,6 @@ set(LIBRARY_OUTPUT_PATH "${BUILD_DIR}/src/io") set(EXECUTABLE_OUTPUT_PATH "${BUILD_DIR}/src/io") file(GLOB_RECURSE IO_FILES CONFIGURE_DEPENDS *.cpp) -if(BUILD_AZURE STREQUAL "OFF") - list(REMOVE_ITEM IO_FILES "${CMAKE_CURRENT_SOURCE_DIR}/fs/azure_obj_storage_client.cpp") -endif() if(ENABLE_TDE) list(REMOVE_ITEM IO_FILES "${CMAKE_CURRENT_SOURCE_DIR}/fs/encrypted_fs_factory.cpp") diff --git a/be/src/io/fs/azure_obj_storage_client.cpp b/be/src/io/fs/azure_obj_storage_client.cpp deleted file mode 100644 index 9702c87b3b304b..00000000000000 --- a/be/src/io/fs/azure_obj_storage_client.cpp +++ /dev/null @@ -1,430 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "io/fs/azure_obj_storage_client.h" - -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "common/exception.h" -#include "common/logging.h" -#include "common/status.h" -#include "cpp/obj_retry_strategy.h" -#include "io/fs/obj_storage_client.h" -#include "util/bvar_helper.h" -#include "util/coding.h" -#include "util/s3_util.h" - -using namespace Azure::Storage::Blobs; - -namespace { -std::string wrap_object_storage_path_msg(const doris::io::ObjectStoragePathOptions& opts) { - return fmt::format("bucket {}, key {}, prefix {}, path {}", opts.bucket, opts.key, opts.prefix, - opts.path.native()); -} - -std::string to_lower_ascii(std::string_view input) { - std::string lowered(input); - std::transform(lowered.begin(), lowered.end(), lowered.begin(), - [](unsigned char ch) { return static_cast(std::tolower(ch)); }); - return lowered; -} - -auto base64_encode_part_num(int part_num) { - uint8_t buf[4]; - doris::encode_fixed32_le(buf, static_cast(part_num)); - return Aws::Utils::HashingUtils::Base64Encode({buf, sizeof(buf)}); -} - -// Rate limiting is applied by RateLimitedObjStorageClient, the decorator that -// S3ClientFactory wraps around this client when the bucket is subject to limiting. - -constexpr char SAS_TOKEN_URL_TEMPLATE[] = "{}/{}/{}{}"; -constexpr char BlobNotFound[] = "BlobNotFound"; -} // namespace - -namespace doris::io { - -// As Azure's doc said, the batch size is 256 -// You can find out the num in https://learn.microsoft.com/en-us/rest/api/storageservices/blob-batch?tabs=microsoft-entra-id -// > Each batch request supports a maximum of 256 subrequests. -constexpr size_t BlobBatchMaxOperations = 256; - -bool is_azure_tls_ca_error_message(std::string_view message) { - std::string lower = to_lower_ascii(message); - return lower.find("ssl ca cert") != std::string::npos || - lower.find("peer failed verification") != std::string::npos || - lower.find("unable to get local issuer certificate") != std::string::npos || - lower.find("problem with the ssl ca cert") != std::string::npos; -} - -std::string build_azure_tls_debug_suffix(std::string_view error_message, - std::string_view tls_debug_context) { - if (tls_debug_context.empty() || !is_azure_tls_ca_error_message(error_message)) { - return ""; - } - return fmt::format(", {}", tls_debug_context); -} - -template -ObjectStorageResponse do_azure_client_call(Func f, const ObjectStoragePathOptions& opts, - std::string_view tls_debug_context) { - try { - f(); - } catch (Azure::Core::RequestFailedException& e) { - doris::record_object_request_failed(static_cast(e.StatusCode)); - auto tls_debug_suffix = build_azure_tls_debug_suffix( - fmt::format("{} {}", e.what(), e.Message), tls_debug_context); - auto msg = fmt::format( - "Azure request failed because {}, error msg {}, http code {}, path msg {}{}", - e.what(), e.Message, static_cast(e.StatusCode), - wrap_object_storage_path_msg(opts), tls_debug_suffix); - LOG_WARNING(msg); - return {.status = convert_to_obj_response(Status::InternalError(std::move(msg))), - .http_code = static_cast(e.StatusCode), - .request_id = std::move(e.RequestId)}; - } catch (std::exception& e) { - auto msg = fmt::format("Azure request failed because {}, path msg {}{}", e.what(), - wrap_object_storage_path_msg(opts), - build_azure_tls_debug_suffix(e.what(), tls_debug_context)); - LOG_WARNING(msg); - return {.status = convert_to_obj_response(Status::InternalError(std::move(msg)))}; - } - return ObjectStorageResponse::OK(); -} - -struct AzureBatchDeleter { - AzureBatchDeleter(BlobContainerClient* client, const ObjectStoragePathOptions& opts, - std::string_view tls_debug_context) - : _client(client), - _batch(client->CreateBatch()), - _opts(opts), - _tls_debug_context(tls_debug_context) {} - // Submit one blob to be deleted in `AzureBatchDeleter::execute` - void delete_blob(const std::string& blob_name) { - deferred_resps.emplace_back(_batch.DeleteBlob(blob_name)); - } - ObjectStorageResponse execute() { - if (deferred_resps.empty()) { - return ObjectStorageResponse::OK(); - } - auto resp = do_azure_client_call( - [&]() { - SCOPED_BVAR_LATENCY(s3_bvar::s3_delete_objects_latency); - _client->SubmitBatch(_batch); - }, - _opts, _tls_debug_context); - if (resp.status.code != ErrorCode::OK) { - return resp; - } - - for (auto&& defer_response : deferred_resps) { - try { - auto r = defer_response.GetResponse(); - if (!r.Value.Deleted) { - auto msg = fmt::format("Azure batch delete failed, path msg {}", - wrap_object_storage_path_msg(_opts)); - LOG_WARNING(msg); - return {.status = convert_to_obj_response( - Status::InternalError(std::move(msg)))}; - } - } catch (Azure::Core::RequestFailedException& e) { - if (Azure::Core::Http::HttpStatusCode::NotFound == e.StatusCode && - 0 == strcmp(e.ErrorCode.c_str(), BlobNotFound)) { - continue; - } - doris::record_object_request_failed(static_cast(e.StatusCode)); - auto msg = fmt::format( - "Azure request failed because {}, error msg {}, http code {}, path msg " - "{}{}", - e.what(), e.Message, static_cast(e.StatusCode), - wrap_object_storage_path_msg(_opts), - build_azure_tls_debug_suffix(fmt::format("{} {}", e.what(), e.Message), - _tls_debug_context)); - LOG_WARNING(msg); - return {.status = convert_to_obj_response( - Status::InternalError(std::move(msg))), - .http_code = static_cast(e.StatusCode), - .request_id = std::move(e.RequestId)}; - } - } - - return ObjectStorageResponse::OK(); - } - -private: - BlobContainerClient* _client; - BlobContainerBatch _batch; - const ObjectStoragePathOptions& _opts; - std::string_view _tls_debug_context; - std::vector> deferred_resps; -}; - -// Azure would do nothing -ObjectStorageUploadResponse AzureObjStorageClient::create_multipart_upload( - const ObjectStoragePathOptions& opts) { - return ObjectStorageUploadResponse { - .resp = ObjectStorageResponse::OK(), - }; -} - -ObjectStorageResponse AzureObjStorageClient::put_object(const ObjectStoragePathOptions& opts, - std::string_view stream) { - auto client = _client->GetBlockBlobClient(opts.key); - return do_azure_client_call( - [&]() { - SCOPED_BVAR_LATENCY(s3_bvar::s3_put_latency); - client.UploadFrom(reinterpret_cast(stream.data()), stream.size()); - }, - opts, _tls_debug_context); -} - -ObjectStorageUploadResponse AzureObjStorageClient::upload_part(const ObjectStoragePathOptions& opts, - std::string_view stream, - int part_num) { - auto client = _client->GetBlockBlobClient(opts.key); - auto resp = do_azure_client_call( - [&]() { - Azure::Core::IO::MemoryBodyStream memory_body( - reinterpret_cast(stream.data()), stream.size()); - // The blockId must be base64 encoded - SCOPED_BVAR_LATENCY(s3_bvar::s3_multi_part_upload_latency); - client.StageBlock(base64_encode_part_num(part_num), memory_body); - }, - opts, _tls_debug_context); - return ObjectStorageUploadResponse { - .resp = resp, - }; -} - -ObjectStorageResponse AzureObjStorageClient::complete_multipart_upload( - const ObjectStoragePathOptions& opts, - const std::vector& completed_parts) { - auto client = _client->GetBlockBlobClient(opts.key); - std::vector string_block_ids; - std::ranges::transform( - completed_parts, std::back_inserter(string_block_ids), - [](const ObjectCompleteMultiPart& i) { return base64_encode_part_num(i.part_num); }); - return do_azure_client_call( - [&]() { - SCOPED_BVAR_LATENCY(s3_bvar::s3_multi_part_upload_latency); - client.CommitBlockList(string_block_ids); - }, - opts, _tls_debug_context); -} - -ObjectStorageHeadResponse AzureObjStorageClient::head_object(const ObjectStoragePathOptions& opts) { - Models::BlobProperties properties {}; - auto resp = do_azure_client_call( - [&]() { - SCOPED_BVAR_LATENCY(s3_bvar::s3_head_latency); - properties = _client->GetBlockBlobClient(opts.key).GetProperties().Value; - }, - opts, _tls_debug_context); - if (resp.http_code == static_cast(Azure::Core::Http::HttpStatusCode::NotFound)) { - return ObjectStorageHeadResponse { - .resp = {.status = convert_to_obj_response( - Status::Error(""))}, - .file_size = properties.BlobSize, - }; - } - - return ObjectStorageHeadResponse { - .resp = resp, - .file_size = properties.BlobSize, - }; -} - -ObjectStorageResponse AzureObjStorageClient::get_object(const ObjectStoragePathOptions& opts, - void* buffer, size_t offset, - size_t bytes_read, size_t* size_return) { - auto client = _client->GetBlockBlobClient(opts.key); - return do_azure_client_call( - [&]() { - DownloadBlobToOptions download_opts; - Azure::Core::Http::HttpRange range {static_cast(offset), bytes_read}; - download_opts.Range = range; - SCOPED_BVAR_LATENCY(s3_bvar::s3_get_latency); - auto resp = client.DownloadTo(reinterpret_cast(buffer), bytes_read, - download_opts); - *size_return = resp.Value.ContentRange.Length.Value(); - }, - opts, _tls_debug_context); -} - -ObjectStorageResponse AzureObjStorageClient::list_objects(const ObjectStoragePathOptions& opts, - std::vector* files) { - auto get_file_file = [&](ListBlobsPagedResponse& resp) { - std::ranges::transform(resp.Blobs, std::back_inserter(*files), [](auto&& blob_item) { - return FileInfo { - .file_name = blob_item.Name, .file_size = blob_item.BlobSize, .is_file = true}; - }); - }; - return do_azure_client_call( - [&]() { - ListBlobsOptions list_opts; - list_opts.Prefix = opts.prefix; - ListBlobsPagedResponse resp; - { - SCOPED_BVAR_LATENCY(s3_bvar::s3_list_latency); - resp = _client->ListBlobs(list_opts); - } - get_file_file(resp); - while (resp.NextPageToken.HasValue()) { - list_opts.ContinuationToken = resp.NextPageToken; - { - SCOPED_BVAR_LATENCY(s3_bvar::s3_list_latency); - resp = _client->ListBlobs(list_opts); - } - get_file_file(resp); - } - }, - opts, _tls_debug_context); -} - -// As Azure's doc said, the batch size is 256 -// You can find out the num in https://learn.microsoft.com/en-us/rest/api/storageservices/blob-batch?tabs=microsoft-entra-id -// > Each batch request supports a maximum of 256 subrequests. -ObjectStorageResponse AzureObjStorageClient::delete_objects(const ObjectStoragePathOptions& opts, - std::vector objs) { - // TODO(ByteYue) : use range to adate this code when compiler is ready - // auto chunkedView = objs | std::views::chunk(BlobBatchMaxOperations); - auto begin = std::begin(objs); - auto end = std::end(objs); - - while (begin != end) { - auto deleter = AzureBatchDeleter(_client.get(), opts, _tls_debug_context); - auto chunk_end = begin; - std::advance(chunk_end, std::min(BlobBatchMaxOperations, - static_cast(std::distance(begin, end)))); - - std::ranges::for_each(std::ranges::subrange(begin, chunk_end), - [&](const std::string& obj) { deleter.delete_blob(obj); }); - begin = chunk_end; - if (auto resp = deleter.execute(); resp.status.code != ErrorCode::OK) { - return resp; - } - } - return ObjectStorageResponse::OK(); -} - -ObjectStorageResponse AzureObjStorageClient::delete_object(const ObjectStoragePathOptions& opts) { - return do_azure_client_call( - [&]() { - SCOPED_BVAR_LATENCY(s3_bvar::s3_delete_object_latency); - auto resp = _client->DeleteBlob(opts.key); - if (!resp.Value.Deleted) { - throw Exception(Status::IOError("Delete azure blob failed")); - } - }, - opts, _tls_debug_context); -} - -ObjectStorageResponse AzureObjStorageClient::delete_objects_recursively( - const ObjectStoragePathOptions& opts) { - ListBlobsOptions list_opts; - list_opts.Prefix = opts.prefix; - list_opts.PageSizeHint = BlobBatchMaxOperations; - auto delete_func = [&](const std::vector& blobs) -> ObjectStorageResponse { - auto deleter = AzureBatchDeleter(_client.get(), opts, _tls_debug_context); - auto batch = _client->CreateBatch(); - for (auto&& blob_item : blobs) { - deleter.delete_blob(blob_item.Name); - } - if (auto response = deleter.execute(); response.status.code != ErrorCode::OK) { - return response; - } - return ObjectStorageResponse::OK(); - }; - - ListBlobsPagedResponse resp; - auto list_resp = do_azure_client_call( - [&]() { - SCOPED_BVAR_LATENCY(s3_bvar::s3_list_latency); - resp = _client->ListBlobs(list_opts); - }, - opts, _tls_debug_context); - if (list_resp.status.code != ErrorCode::OK) { - return list_resp; - } - - if (auto response = delete_func(resp.Blobs); response.status.code != ErrorCode::OK) { - return response; - } - - while (resp.NextPageToken.HasValue()) { - list_opts.ContinuationToken = resp.NextPageToken; - list_resp = do_azure_client_call( - [&]() { - SCOPED_BVAR_LATENCY(s3_bvar::s3_list_latency); - resp = _client->ListBlobs(list_opts); - }, - opts, _tls_debug_context); - if (list_resp.status.code != ErrorCode::OK) { - return list_resp; - } - - if (auto response = delete_func(resp.Blobs); response.status.code != ErrorCode::OK) { - return response; - } - } - return ObjectStorageResponse::OK(); -} - -std::string AzureObjStorageClient::generate_presigned_url(const ObjectStoragePathOptions& opts, - int64_t expiration_secs, - const S3ClientConf& conf) { - Azure::Storage::Sas::BlobSasBuilder sas_builder; - sas_builder.ExpiresOn = - std::chrono::system_clock::now() + std::chrono::seconds(expiration_secs); - sas_builder.BlobContainerName = opts.bucket; - sas_builder.BlobName = opts.key; - sas_builder.Resource = Azure::Storage::Sas::BlobSasResource::Blob; - sas_builder.Protocol = Azure::Storage::Sas::SasProtocol::HttpsOnly; - sas_builder.SetPermissions(Azure::Storage::Sas::BlobSasPermissions::Read); - - std::string sasToken = sas_builder.GenerateSasToken( - Azure::Storage::StorageSharedKeyCredential(conf.ak, conf.sk)); - - std::string endpoint = conf.endpoint; - auto sasURL = fmt::format(SAS_TOKEN_URL_TEMPLATE, endpoint, conf.bucket, opts.key, sasToken); - if (sasURL.find("://") == std::string::npos) { - sasURL = "https://" + sasURL; - } - return sasURL; -} -} // namespace doris::io diff --git a/be/src/io/fs/azure_obj_storage_client.h b/be/src/io/fs/azure_obj_storage_client.h deleted file mode 100644 index 7d1cecc502e44d..00000000000000 --- a/be/src/io/fs/azure_obj_storage_client.h +++ /dev/null @@ -1,70 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include -#include - -#include "io/fs/obj_storage_client.h" - -namespace Azure::Storage::Blobs { -class BlobContainerClient; -} // namespace Azure::Storage::Blobs - -namespace doris::io { - -class ObjClientHolder; - -bool is_azure_tls_ca_error_message(std::string_view message); -std::string build_azure_tls_debug_suffix(std::string_view error_message, - std::string_view tls_debug_context); - -class AzureObjStorageClient final : public ObjStorageClient { -public: - AzureObjStorageClient(std::shared_ptr client, - std::string tls_debug_context = {}) - : _client(std::move(client)), _tls_debug_context(std::move(tls_debug_context)) {} - ~AzureObjStorageClient() override = default; - ObjectStorageUploadResponse create_multipart_upload( - const ObjectStoragePathOptions& opts) override; - ObjectStorageResponse put_object(const ObjectStoragePathOptions& opts, - std::string_view stream) override; - ObjectStorageUploadResponse upload_part(const ObjectStoragePathOptions& opts, std::string_view, - int partNum) override; - ObjectStorageResponse complete_multipart_upload( - const ObjectStoragePathOptions& opts, - const std::vector& completed_parts) override; - ObjectStorageHeadResponse head_object(const ObjectStoragePathOptions& opts) override; - ObjectStorageResponse get_object(const ObjectStoragePathOptions& opts, void* buffer, - size_t offset, size_t bytes_read, - size_t* size_return) override; - ObjectStorageResponse list_objects(const ObjectStoragePathOptions& opts, - std::vector* files) override; - ObjectStorageResponse delete_objects(const ObjectStoragePathOptions& opts, - std::vector objs) override; - ObjectStorageResponse delete_object(const ObjectStoragePathOptions& opts) override; - ObjectStorageResponse delete_objects_recursively(const ObjectStoragePathOptions& opts) override; - std::string generate_presigned_url(const ObjectStoragePathOptions& opts, - int64_t expiration_secs, const S3ClientConf& conf) override; - -private: - std::shared_ptr _client; - std::string _tls_debug_context; -}; - -} // namespace doris::io diff --git a/be/src/io/fs/connectivity/s3_connectivity_tester.cpp b/be/src/io/fs/connectivity/s3_connectivity_tester.cpp index 347c1b40fb2c18..a49599d6a6df3d 100644 --- a/be/src/io/fs/connectivity/s3_connectivity_tester.cpp +++ b/be/src/io/fs/connectivity/s3_connectivity_tester.cpp @@ -39,10 +39,7 @@ Status S3ConnectivityTester::test(const std::map& prop S3Conf s3_conf; RETURN_IF_ERROR(S3ClientFactory::convert_properties_to_s3_conf(properties, s3_uri, &s3_conf)); - auto obj_client = S3ClientFactory::instance().create(s3_conf.client_conf); - if (!obj_client) { - return Status::InternalError("Failed to create S3 client"); - } + auto obj_client = DORIS_TRY(S3ClientFactory::instance().create(s3_conf.client_conf)); auto resp = obj_client->head_object({.bucket = bucket, .key = ""}); if (resp.resp.status.code != ErrorCode::OK && resp.resp.status.code != ErrorCode::NOT_FOUND) { diff --git a/be/src/io/fs/err_utils.cpp b/be/src/io/fs/err_utils.cpp index 96ac7b817e9d56..c9ce59c0897193 100644 --- a/be/src/io/fs/err_utils.cpp +++ b/be/src/io/fs/err_utils.cpp @@ -27,17 +27,10 @@ #include "common/status.h" #include "io/fs/hdfs.h" -#include "io/fs/obj_storage_client.h" namespace doris { using namespace ErrorCode; -io::ObjectStorageStatus convert_to_obj_response(Status st) { - int code = st._code; - std::string msg = st._err_msg == nullptr ? "" : std::move(st._err_msg->_msg); - return io::ObjectStorageStatus {.code = code, .msg = std::move(msg)}; -} - namespace io { std::string errno_to_str() { diff --git a/be/src/io/fs/obj_storage_client.h b/be/src/io/fs/obj_storage_client.h deleted file mode 100644 index fa239ca3282e2a..00000000000000 --- a/be/src/io/fs/obj_storage_client.h +++ /dev/null @@ -1,135 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include - -#include "io/fs/file_system.h" -#include "io/fs/path.h" -namespace doris { -class Status; -struct S3ClientConf; -namespace io { - -// Names are in lexico order. -enum class ObjStorageType : uint8_t { - UNKNOWN = 0, - AWS = 1, - AZURE, - BOS, - COS, - OSS, - OBS, - GCP, - TOS, -}; - -struct ObjectStoragePathOptions { - Path path = ""; - std::string bucket = std::string(); // blob container in azure - std::string key = std::string(); // blob name in azure - std::string prefix = std::string(); // for batch delete and recursive delete - std::optional upload_id = std::nullopt; // only used for S3 upload -}; - -struct ObjectCompleteMultiPart { - int part_num = 0; - std::string etag = std::string(); -}; - -struct ObjectStorageStatus { - int code = 0; - std::string msg = std::string(); -}; - -// We only store error code along with err_msg instead of Status to unify BE and recycler's error handle logic -struct ObjectStorageResponse { - ObjectStorageStatus status {}; - int http_code {200}; - std::string request_id = std::string(); - static ObjectStorageResponse OK() { - // clang-format off - return { - .status { .code = 0, }, - .http_code = 200, - }; - // clang-format on - } -}; - -struct ObjectStorageUploadResponse { - ObjectStorageResponse resp {}; - std::optional upload_id = std::nullopt; - std::optional etag = std::nullopt; -}; - -struct ObjectStorageHeadResponse : ObjectStorageResponse { - ObjectStorageResponse resp {}; - long long file_size {0}; -}; - -class ObjStorageClient { -public: - virtual ~ObjStorageClient() = default; - // Create a multi-part upload request. On AWS-compatible systems, it will return an upload ID, but not on Azure. - // The input parameters should include the bucket and key for the object storage. - virtual ObjectStorageUploadResponse create_multipart_upload( - const ObjectStoragePathOptions& opts) = 0; - // To directly upload a piece of data to object storage and generate a user-visible file. - // You need to clearly specify the bucket and key - virtual ObjectStorageResponse put_object(const ObjectStoragePathOptions& opts, - std::string_view stream) = 0; - // To upload a part of a large file to object storage as a temporary file, which is not visible to the user - // The temporary file's ID is the value of the part_num passed in - // You need to specify the bucket and key along with the upload_id if it's AWS-compatible system - // For the same bucket and key, as well as the same part_num, it will directly replace the original temporary file. - virtual ObjectStorageUploadResponse upload_part(const ObjectStoragePathOptions& opts, - std::string_view stream, int part_num) = 0; - // To combine the previously uploaded multiple file parts into a complete file, the file name is the name of the key passed in. - // If it is an AWS-compatible system, the upload_id needs to be included. - // After a successful execution, the large file can be accessed in the object storage - virtual ObjectStorageResponse complete_multipart_upload( - const ObjectStoragePathOptions& opts, - const std::vector& completed_parts) = 0; - // According to the passed bucket and key, it will access whether the corresponding file exists in the object storage. - // If it exists, it will return the corresponding file size - virtual ObjectStorageHeadResponse head_object(const ObjectStoragePathOptions& opts) = 0; - // According to the bucket and key, it finds the corresponding file in the object storage - // and starting from the offset, it reads bytes_read bytes into the buffer, with size_return recording the actual number of bytes read - virtual ObjectStorageResponse get_object(const ObjectStoragePathOptions& opts, void* buffer, - size_t offset, size_t bytes_read, - size_t* size_return) = 0; - // According to the passed bucket and prefix, it traverses and retrieves all files under the prefix, and returns the name and file size of all files. - // **Notice**: The files returned by this function contains the full key in object storage. - virtual ObjectStorageResponse list_objects(const ObjectStoragePathOptions& opts, - std::vector* files) = 0; - // According to the bucket and prefix specified by the user, it performs batch deletion based on the object names in the object array. - virtual ObjectStorageResponse delete_objects(const ObjectStoragePathOptions& opts, - std::vector objs) = 0; - // Delete the file named key in the object storage bucket. - virtual ObjectStorageResponse delete_object(const ObjectStoragePathOptions& opts) = 0; - // According to the prefix, recursively delete all files under the prefix. - virtual ObjectStorageResponse delete_objects_recursively( - const ObjectStoragePathOptions& opts) = 0; - // Return a presigned URL for users to access the object - virtual std::string generate_presigned_url(const ObjectStoragePathOptions& opts, - int64_t expiration_secs, - const S3ClientConf& conf) = 0; -}; -} // namespace io -} // namespace doris diff --git a/be/src/io/fs/rate_limited_obj_storage_client.cpp b/be/src/io/fs/rate_limited_obj_storage_client.cpp deleted file mode 100644 index 1b8730847162df..00000000000000 --- a/be/src/io/fs/rate_limited_obj_storage_client.cpp +++ /dev/null @@ -1,145 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "io/fs/rate_limited_obj_storage_client.h" - -#include "common/logging.h" -#include "common/status.h" -#include "util/s3_rate_limiter_manager.h" - -namespace doris::io { -namespace { - -ObjectStorageResponse rate_limited_response(S3RateLimitType type, S3RateLimitRejectReason reason) { - CHECK(reason != S3RateLimitRejectReason::NONE); - const auto* limit_type = reason == S3RateLimitRejectReason::QPS ? "QPS" : "bytes"; - return {.status = convert_to_obj_response(Status::Error( - "s3 {} request exceeds {} limit, rejected by BE rate limiter", to_string(type), - limit_type)), - // The BE rate limiter rejected the request before it reached the provider. - .http_code = 0}; -} - -} // namespace - -ObjectStorageUploadResponse RateLimitedObjStorageClient::create_multipart_upload( - const ObjectStoragePathOptions& opts) { - S3RateLimitGuard guard(S3RateLimitType::PUT, 0); - if (!guard.ok()) { - return {.resp = rate_limited_response(S3RateLimitType::PUT, guard.reject_reason())}; - } - return _inner->create_multipart_upload(opts); -} - -ObjectStorageResponse RateLimitedObjStorageClient::put_object(const ObjectStoragePathOptions& opts, - std::string_view stream) { - S3RateLimitGuard guard(S3RateLimitType::PUT, stream.size()); - if (!guard.ok()) { - return rate_limited_response(S3RateLimitType::PUT, guard.reject_reason()); - } - return _inner->put_object(opts, stream); -} - -ObjectStorageUploadResponse RateLimitedObjStorageClient::upload_part( - const ObjectStoragePathOptions& opts, std::string_view stream, int part_num) { - S3RateLimitGuard guard(S3RateLimitType::PUT, stream.size()); - if (!guard.ok()) { - return {.resp = rate_limited_response(S3RateLimitType::PUT, guard.reject_reason())}; - } - return _inner->upload_part(opts, stream, part_num); -} - -ObjectStorageResponse RateLimitedObjStorageClient::complete_multipart_upload( - const ObjectStoragePathOptions& opts, - const std::vector& completed_parts) { - S3RateLimitGuard guard(S3RateLimitType::PUT, 0); - if (!guard.ok()) { - return rate_limited_response(S3RateLimitType::PUT, guard.reject_reason()); - } - return _inner->complete_multipart_upload(opts, completed_parts); -} - -ObjectStorageHeadResponse RateLimitedObjStorageClient::head_object( - const ObjectStoragePathOptions& opts) { - S3RateLimitGuard guard(S3RateLimitType::GET, 0); - if (!guard.ok()) { - ObjectStorageHeadResponse response; - response.resp = rate_limited_response(S3RateLimitType::GET, guard.reject_reason()); - return response; - } - return _inner->head_object(opts); -} - -ObjectStorageResponse RateLimitedObjStorageClient::get_object(const ObjectStoragePathOptions& opts, - void* buffer, size_t offset, - size_t bytes_read, - size_t* size_return) { - S3RateLimitGuard guard(S3RateLimitType::GET, bytes_read); - if (!guard.ok()) { - return rate_limited_response(S3RateLimitType::GET, guard.reject_reason()); - } - auto resp = _inner->get_object(opts, buffer, offset, bytes_read, size_return); - if (resp.status.code == 0) { - // Refund the difference for short reads (e.g. requested range crosses EOF). - guard.settle(*size_return); - } - return resp; -} - -ObjectStorageResponse RateLimitedObjStorageClient::list_objects( - const ObjectStoragePathOptions& opts, std::vector* files) { - S3RateLimitGuard guard(S3RateLimitType::GET, 0); - if (!guard.ok()) { - return rate_limited_response(S3RateLimitType::GET, guard.reject_reason()); - } - return _inner->list_objects(opts, files); -} - -ObjectStorageResponse RateLimitedObjStorageClient::delete_objects( - const ObjectStoragePathOptions& opts, std::vector objs) { - S3RateLimitGuard guard(S3RateLimitType::PUT, 0); - if (!guard.ok()) { - return rate_limited_response(S3RateLimitType::PUT, guard.reject_reason()); - } - return _inner->delete_objects(opts, std::move(objs)); -} - -ObjectStorageResponse RateLimitedObjStorageClient::delete_object( - const ObjectStoragePathOptions& opts) { - S3RateLimitGuard guard(S3RateLimitType::PUT, 0); - if (!guard.ok()) { - return rate_limited_response(S3RateLimitType::PUT, guard.reject_reason()); - } - return _inner->delete_object(opts); -} - -ObjectStorageResponse RateLimitedObjStorageClient::delete_objects_recursively( - const ObjectStoragePathOptions& opts) { - S3RateLimitGuard guard(S3RateLimitType::PUT, 0); - if (!guard.ok()) { - return rate_limited_response(S3RateLimitType::PUT, guard.reject_reason()); - } - return _inner->delete_objects_recursively(opts); -} - -std::string RateLimitedObjStorageClient::generate_presigned_url( - const ObjectStoragePathOptions& opts, int64_t expiration_secs, const S3ClientConf& conf) { - // Generating a presigned URL is a local computation, no request goes out. - return _inner->generate_presigned_url(opts, expiration_secs, conf); -} - -} // namespace doris::io diff --git a/be/src/io/fs/rate_limited_obj_storage_client.h b/be/src/io/fs/rate_limited_obj_storage_client.h deleted file mode 100644 index 00725d7edcb299..00000000000000 --- a/be/src/io/fs/rate_limited_obj_storage_client.h +++ /dev/null @@ -1,70 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include - -#include "io/fs/obj_storage_client.h" - -namespace doris::io { - -// Decorator that applies the process-wide S3 GET/PUT QPS and bandwidth rate limiters -// in front of any ObjStorageClient. This is the single place where rate limiting is -// wired into the object storage path: provider clients (S3, Azure, future GCP, ...) -// contain no rate limiting code, and S3ClientFactory decides at construction time -// whether to wrap a client (internal storage-vault buckets) or return it bare -// (external buckets: S3 load, TVF, external catalogs in cloud mode). -// -// Each public API call is charged once against the QPS bucket, and data-carrying -// calls additionally reserve their payload size from the bytes bucket (reconciled -// with the actually transferred size for reads). Note that APIs which internally -// paginate (list_objects, delete_objects_recursively) are charged once per logical -// call, not once per underlying HTTP request. -class RateLimitedObjStorageClient final : public ObjStorageClient { -public: - explicit RateLimitedObjStorageClient(std::shared_ptr inner) - : _inner(std::move(inner)) {} - ~RateLimitedObjStorageClient() override = default; - - ObjectStorageUploadResponse create_multipart_upload( - const ObjectStoragePathOptions& opts) override; - ObjectStorageResponse put_object(const ObjectStoragePathOptions& opts, - std::string_view stream) override; - ObjectStorageUploadResponse upload_part(const ObjectStoragePathOptions& opts, - std::string_view stream, int part_num) override; - ObjectStorageResponse complete_multipart_upload( - const ObjectStoragePathOptions& opts, - const std::vector& completed_parts) override; - ObjectStorageHeadResponse head_object(const ObjectStoragePathOptions& opts) override; - ObjectStorageResponse get_object(const ObjectStoragePathOptions& opts, void* buffer, - size_t offset, size_t bytes_read, - size_t* size_return) override; - ObjectStorageResponse list_objects(const ObjectStoragePathOptions& opts, - std::vector* files) override; - ObjectStorageResponse delete_objects(const ObjectStoragePathOptions& opts, - std::vector objs) override; - ObjectStorageResponse delete_object(const ObjectStoragePathOptions& opts) override; - ObjectStorageResponse delete_objects_recursively(const ObjectStoragePathOptions& opts) override; - std::string generate_presigned_url(const ObjectStoragePathOptions& opts, - int64_t expiration_secs, const S3ClientConf& conf) override; - -private: - std::shared_ptr _inner; -}; - -} // namespace doris::io diff --git a/be/src/io/fs/s3_file_bufferpool.cpp b/be/src/io/fs/s3_file_bufferpool.cpp index 11f90d6c88b647..f2211ade32d16c 100644 --- a/be/src/io/fs/s3_file_bufferpool.cpp +++ b/be/src/io/fs/s3_file_bufferpool.cpp @@ -28,10 +28,10 @@ #include "common/logging.h" #include "common/status.h" #include "core/arena.h" +#include "cpp/client/s3_common.h" #include "cpp/sync_point.h" #include "io/cache/file_block.h" #include "io/cache/file_cache_common.h" -#include "io/fs/s3_common.h" #include "runtime/exec_env.h" #include "runtime/thread_context.h" #include "util/defer_op.h" diff --git a/be/src/io/fs/s3_file_reader.cpp b/be/src/io/fs/s3_file_reader.cpp index 8a6e5c0fdc4978..993e9f186f1503 100644 --- a/be/src/io/fs/s3_file_reader.cpp +++ b/be/src/io/fs/s3_file_reader.cpp @@ -33,10 +33,10 @@ #include "common/compiler_util.h" // IWYU pragma: keep #include "common/metrics/doris_metrics.h" +#include "cpp/client/obj_storage_client.h" +#include "cpp/client/s3_common.h" #include "io/cache/block_file_cache.h" #include "io/fs/err_utils.h" -#include "io/fs/obj_storage_client.h" -#include "io/fs/s3_common.h" #include "runtime/file_scan_profile.h" #include "runtime/runtime_profile.h" #include "runtime/thread_context.h" diff --git a/be/src/io/fs/s3_file_system.cpp b/be/src/io/fs/s3_file_system.cpp index 45251eab6e9d2a..6f7467c09fd520 100644 --- a/be/src/io/fs/s3_file_system.cpp +++ b/be/src/io/fs/s3_file_system.cpp @@ -35,16 +35,16 @@ #include "common/config.h" #include "common/logging.h" #include "common/status.h" +#include "cpp/client/obj_storage_client.h" +#include "cpp/client/s3_common.h" #include "cpp/sync_point.h" #include "io/fs/err_utils.h" #include "io/fs/file_system.h" #include "io/fs/file_writer.h" #include "io/fs/local_file_system.h" #include "io/fs/remote_file_system.h" -#include "io/fs/s3_common.h" #include "io/fs/s3_file_reader.h" #include "io/fs/s3_file_writer.h" -#include "io/fs/s3_obj_storage_client.h" #include "runtime/exec_env.h" #include "runtime/thread_context.h" #include "util/s3_uri.h" @@ -76,11 +76,7 @@ ObjClientHolder::ObjClientHolder(S3ClientConf conf) : _conf(std::move(conf)) {} ObjClientHolder::~ObjClientHolder() = default; Status ObjClientHolder::init() { - _client = S3ClientFactory::instance().create(_conf); - if (!_client) { - return Status::InvalidArgument("failed to init s3 client with conf {}", _conf.to_string()); - } - + _client = DORIS_TRY(S3ClientFactory::instance().create(_conf)); return Status::OK(); } @@ -111,10 +107,7 @@ Status ObjClientHolder::reset(const S3ClientConf& conf) { } } - auto client = S3ClientFactory::instance().create(reset_conf); - if (!client) { - return Status::InvalidArgument("failed to init s3 client with conf {}", conf.to_string()); - } + auto client = DORIS_TRY(S3ClientFactory::instance().create(reset_conf)); LOG(WARNING) << "reset s3 client with new conf: " << conf.to_string(); @@ -309,16 +302,27 @@ Status S3FileSystem::list_impl(const Path& dir, bool only_file, std::vectorlist_objects( {.bucket = _bucket, .prefix = prefix,}, files); - // clang-format on - if (resp.status.code == ErrorCode::OK) { - for (auto&& file : *files) { - file.file_name.erase(0, prefix.size()); + ObjectListIterator list_iter(client, { + .bucket = _bucket, + .prefix = prefix, + }); + + for (;;) { + auto resp = list_iter.next(); + if (!resp.results_.has_value()) { + if (!resp.resp.ok()) { + return {resp.resp.status.code, std::move(resp.resp.status.msg)}; + } + break; } + auto obj = std::move(*resp.results_); + obj.file_path.erase(0, prefix.size()); + bool is_dir = obj.file_path.empty() || obj.file_path.back() == '/'; + files->emplace_back(FileInfo { + .file_name = std::move(obj.file_path), .file_size = obj.size, .is_file = !is_dir}); } - return {resp.status.code, std::move(resp.status.msg)}; + return Status::OK(); } Status S3FileSystem::rename_impl(const Path& orig_name, const Path& new_name) { @@ -453,12 +457,18 @@ std::string S3FileSystem::generate_presigned_url(const Path& path, int64_t expir new_s3_conf.endpoint.erase( _client->s3_client_conf().endpoint.size() - OSS_PRIVATE_ENDPOINT_SUFFIX.size(), LEN_OF_OSS_PRIVATE_SUFFIX); - client = S3ClientFactory::instance().create(new_s3_conf); + auto client_result = S3ClientFactory::instance().create(new_s3_conf); + if (!client_result) { + LOG(WARNING) << "failed to create S3 client for presigned URL: " + << client_result.error(); + return {}; + } + client = std::move(client_result).value(); } else { client = _client->get(); } - return client->generate_presigned_url({.bucket = _bucket, .key = key}, expiration_secs, - _client->s3_client_conf()); + return client->generate_presigned_url({.bucket = _bucket, .key = key, .prefix = ""}, + expiration_secs); } } // namespace doris::io diff --git a/be/src/io/fs/s3_file_system.h b/be/src/io/fs/s3_file_system.h index f6efa5053324ff..d1b969eedeca95 100644 --- a/be/src/io/fs/s3_file_system.h +++ b/be/src/io/fs/s3_file_system.h @@ -38,7 +38,6 @@ class PooledThreadExecutor; } // namespace Aws::Utils::Threading namespace doris::io { -class ObjStorageClient; // In runtime, AK and SK may be modified, and the original `S3Client` instance will be replaced. // The `S3FileReader` cached by the `Segment` must hold a shared `ObjClientHolder` in order to // access S3 data with latest AK SK. diff --git a/be/src/io/fs/s3_file_writer.cpp b/be/src/io/fs/s3_file_writer.cpp index f8b836607a14a6..6d14de358615b3 100644 --- a/be/src/io/fs/s3_file_writer.cpp +++ b/be/src/io/fs/s3_file_writer.cpp @@ -30,6 +30,7 @@ #include "common/config.h" #include "common/status.h" +#include "cpp/client/s3_obj_storage_backend.h" #include "cpp/sync_point.h" #include "io/cache/block_file_cache.h" #include "io/cache/block_file_cache_factory.h" @@ -39,7 +40,6 @@ #include "io/fs/path.h" #include "io/fs/s3_file_bufferpool.h" #include "io/fs/s3_file_system.h" -#include "io/fs/s3_obj_storage_client.h" #include "runtime/exec_env.h" #include "util/debug_points.h" #include "util/s3_util.h" diff --git a/be/src/io/fs/s3_file_writer.h b/be/src/io/fs/s3_file_writer.h index 5a8075e03cf404..441742451e0fe4 100644 --- a/be/src/io/fs/s3_file_writer.h +++ b/be/src/io/fs/s3_file_writer.h @@ -26,9 +26,9 @@ #include #include "common/status.h" +#include "cpp/client/obj_storage_client.h" #include "io/fs/file_system.h" #include "io/fs/file_writer.h" -#include "io/fs/obj_storage_client.h" #include "io/fs/path.h" #include "io/fs/s3_file_bufferpool.h" diff --git a/be/src/io/fs/s3_obj_storage_client.cpp b/be/src/io/fs/s3_obj_storage_client.cpp deleted file mode 100644 index 0c0b0370f8097f..00000000000000 --- a/be/src/io/fs/s3_obj_storage_client.cpp +++ /dev/null @@ -1,493 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "io/fs/s3_obj_storage_client.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include - -#include "common/logging.h" -#include "common/status.h" -#include "cpp/obj_retry_strategy.h" -#include "cpp/sync_point.h" -#include "io/fs/err_utils.h" -#include "io/fs/s3_common.h" -#include "util/bvar_helper.h" - -// Rate limiting is applied by RateLimitedObjStorageClient, the decorator that -// S3ClientFactory wraps around this client when the bucket is subject to limiting. -namespace { -void record_s3_request_failed(const Aws::S3::S3Error& error) { - doris::record_object_request_failed(static_cast(error.GetResponseCode())); -} -} // namespace - -namespace Aws::S3::Model { -class DeleteObjectRequest; -} // namespace Aws::S3::Model - -using Aws::S3::Model::CompletedPart; -using Aws::S3::Model::CompletedMultipartUpload; -using Aws::S3::Model::CompleteMultipartUploadRequest; -using Aws::S3::Model::CreateMultipartUploadRequest; -using Aws::S3::Model::UploadPartRequest; -using Aws::S3::Model::UploadPartOutcome; - -namespace doris::io { -using namespace Aws::S3::Model; - -static constexpr int S3_REQUEST_THRESHOLD_MS = 5000; - -ObjectStorageUploadResponse S3ObjStorageClient::create_multipart_upload( - const ObjectStoragePathOptions& opts) { - CreateMultipartUploadRequest request; - request.WithBucket(opts.bucket).WithKey(opts.key); - request.SetContentType("application/octet-stream"); - - MonotonicStopWatch watch; - watch.start(); - - auto outcome = SYNC_POINT_HOOK_RETURN_VALUE(_client->CreateMultipartUpload(request), - "s3_file_writer::create_multi_part_upload", - std::cref(request).get()); - SYNC_POINT_CALLBACK("s3_file_writer::_open", &outcome); - watch.stop(); - - s3_bvar::s3_multi_part_upload_latency << watch.elapsed_time_microseconds(); - const auto& request_id = outcome.IsSuccess() ? outcome.GetResult().GetRequestId() - : outcome.GetError().GetRequestId(); - - LOG_IF(INFO, watch.elapsed_time_milliseconds() > S3_REQUEST_THRESHOLD_MS) - << "CreateMultipartUpload cost=" << watch.elapsed_time_milliseconds() << "ms" - << ", request_id=" << request_id << ", bucket=" << opts.bucket << ", key=" << opts.key; - - if (!outcome.IsSuccess()) { - record_s3_request_failed(outcome.GetError()); - auto st = s3fs_error(outcome.GetError(), fmt::format("failed to CreateMultipartUpload: {} ", - opts.path.native())); - LOG(WARNING) << st << " request_id=" << request_id; - return ObjectStorageUploadResponse { - .resp = {convert_to_obj_response(std::move(st)), - static_cast(outcome.GetError().GetResponseCode()), - outcome.GetError().GetRequestId()}, - }; - } - - return ObjectStorageUploadResponse {.upload_id {outcome.GetResult().GetUploadId()}}; -} - -ObjectStorageResponse S3ObjStorageClient::put_object(const ObjectStoragePathOptions& opts, - std::string_view stream) { - Aws::S3::Model::PutObjectRequest request; - request.WithBucket(opts.bucket).WithKey(opts.key); - auto string_view_stream = std::make_shared(stream.data(), stream.size()); - Aws::Utils::ByteBuffer part_md5(Aws::Utils::HashingUtils::CalculateMD5(*string_view_stream)); - request.SetContentMD5(Aws::Utils::HashingUtils::Base64Encode(part_md5)); - request.SetBody(string_view_stream); - request.SetContentLength(stream.size()); - request.SetContentType("application/octet-stream"); - - MonotonicStopWatch watch; - watch.start(); - auto outcome = - SYNC_POINT_HOOK_RETURN_VALUE(_client->PutObject(request), "s3_file_writer::put_object", - std::cref(request).get(), &stream); - - watch.stop(); - - s3_bvar::s3_put_latency << watch.elapsed_time_microseconds(); - const auto& request_id = outcome.IsSuccess() ? outcome.GetResult().GetRequestId() - : outcome.GetError().GetRequestId(); - - if (!outcome.IsSuccess()) { - record_s3_request_failed(outcome.GetError()); - auto st = s3fs_error(outcome.GetError(), - fmt::format("failed to put object: {}", opts.path.native())); - LOG(WARNING) << st << ", request_id=" << request_id; - return ObjectStorageResponse {convert_to_obj_response(std::move(st)), - static_cast(outcome.GetError().GetResponseCode()), - request_id}; - } - - LOG_IF(INFO, watch.elapsed_time_milliseconds() > S3_REQUEST_THRESHOLD_MS) - << "PutObject cost=" << watch.elapsed_time_milliseconds() << "ms" - << ", request_id=" << request_id << ", bucket=" << opts.bucket << ", key=" << opts.key; - return ObjectStorageResponse::OK(); -} - -ObjectStorageUploadResponse S3ObjStorageClient::upload_part(const ObjectStoragePathOptions& opts, - std::string_view stream, int part_num) { - UploadPartRequest request; - request.WithBucket(opts.bucket) - .WithKey(opts.key) - .WithPartNumber(part_num) - .WithUploadId(*opts.upload_id); - auto string_view_stream = std::make_shared(stream.data(), stream.size()); - - request.SetBody(string_view_stream); - - Aws::Utils::ByteBuffer part_md5(Aws::Utils::HashingUtils::CalculateMD5(*string_view_stream)); - request.SetContentMD5(Aws::Utils::HashingUtils::Base64Encode(part_md5)); - - request.SetContentLength(stream.size()); - request.SetContentType("application/octet-stream"); - - MonotonicStopWatch watch; - watch.start(); - auto outcome = SYNC_POINT_HOOK_RETURN_VALUE(_client->UploadPart(request), - "s3_file_writer::upload_part", - std::cref(request).get(), &stream); - - watch.stop(); - - s3_bvar::s3_multi_part_upload_latency << watch.elapsed_time_microseconds(); - const auto& request_id = outcome.IsSuccess() ? outcome.GetResult().GetRequestId() - : outcome.GetError().GetRequestId(); - - TEST_SYNC_POINT_CALLBACK("S3FileWriter::_upload_one_part", &outcome); - if (!outcome.IsSuccess()) { - record_s3_request_failed(outcome.GetError()); - auto st = Status::IOError( - "failed to UploadPart bucket={}, key={}, part_num={}, upload_id={}, message={}, " - "exception_name={}, response_code={}, request_id={}", - opts.bucket, opts.path.native(), part_num, *opts.upload_id, - outcome.GetError().GetMessage(), outcome.GetError().GetExceptionName(), - outcome.GetError().GetResponseCode(), request_id); - - LOG(WARNING) << st << ", request_id=" << request_id; - return ObjectStorageUploadResponse { - .resp = {convert_to_obj_response(std::move(st)), - static_cast(outcome.GetError().GetResponseCode()), - outcome.GetError().GetRequestId()}}; - } - - LOG_IF(INFO, watch.elapsed_time_milliseconds() > S3_REQUEST_THRESHOLD_MS) - << "UploadPart cost=" << watch.elapsed_time_milliseconds() << "ms" - << ", request_id=" << request_id << ", bucket=" << opts.bucket << ", key=" << opts.key - << ", part_num=" << part_num << ", upload_id=" << *opts.upload_id; - return ObjectStorageUploadResponse {.etag = outcome.GetResult().GetETag()}; -} - -ObjectStorageResponse S3ObjStorageClient::complete_multipart_upload( - const ObjectStoragePathOptions& opts, - const std::vector& completed_parts) { - CompleteMultipartUploadRequest request; - request.WithBucket(opts.bucket).WithKey(opts.key).WithUploadId(*opts.upload_id); - - CompletedMultipartUpload completed_upload; - std::vector complete_parts; - std::ranges::transform(completed_parts, std::back_inserter(complete_parts), - [](const ObjectCompleteMultiPart& part_ptr) { - CompletedPart part; - part.SetPartNumber(part_ptr.part_num); - part.SetETag(part_ptr.etag); - return part; - }); - completed_upload.SetParts(std::move(complete_parts)); - request.WithMultipartUpload(completed_upload); - - TEST_SYNC_POINT_RETURN_WITH_VALUE("S3FileWriter::_complete:3", ObjectStorageResponse(), this); - - MonotonicStopWatch watch; - watch.start(); - auto outcome = SYNC_POINT_HOOK_RETURN_VALUE(_client->CompleteMultipartUpload(request), - "s3_file_writer::complete_multi_part", - std::cref(request).get()); - - watch.stop(); - s3_bvar::s3_multi_part_upload_latency << watch.elapsed_time_microseconds(); - const auto& request_id = outcome.IsSuccess() ? outcome.GetResult().GetRequestId() - : outcome.GetError().GetRequestId(); - - if (!outcome.IsSuccess()) { - record_s3_request_failed(outcome.GetError()); - auto st = s3fs_error(outcome.GetError(), - fmt::format("failed to CompleteMultipartUpload: {}, upload_id={}", - opts.path.native(), *opts.upload_id)); - LOG(WARNING) << st << ", request_id=" << request_id; - return {convert_to_obj_response(std::move(st)), - static_cast(outcome.GetError().GetResponseCode()), - outcome.GetError().GetRequestId()}; - } - - LOG_IF(INFO, watch.elapsed_time_milliseconds() > S3_REQUEST_THRESHOLD_MS) - << "CompleteMultipartUpload cost=" << watch.elapsed_time_milliseconds() << "ms" - << ", request_id=" << request_id << ", bucket=" << opts.bucket << ", key=" << opts.key - << ", upload_id=" << *opts.upload_id; - return ObjectStorageResponse::OK(); -} - -ObjectStorageHeadResponse S3ObjStorageClient::head_object(const ObjectStoragePathOptions& opts) { - Aws::S3::Model::HeadObjectRequest request; - request.WithBucket(opts.bucket).WithKey(opts.key); - - SCOPED_BVAR_LATENCY(s3_bvar::s3_head_latency); - auto outcome = SYNC_POINT_HOOK_RETURN_VALUE( - _client->HeadObject(request), "s3_file_system::head_object", std::ref(request).get()); - if (outcome.IsSuccess()) { - return {.resp = {convert_to_obj_response(Status::OK())}, - .file_size = outcome.GetResult().GetContentLength()}; - } else if (outcome.GetError().GetResponseCode() == Aws::Http::HttpResponseCode::NOT_FOUND) { - return {.resp = {convert_to_obj_response(Status::Error(""))}}; - } else { - record_s3_request_failed(outcome.GetError()); - return {.resp = {convert_to_obj_response( - s3fs_error(outcome.GetError(), - fmt::format("failed to check exists {}", opts.key))), - static_cast(outcome.GetError().GetResponseCode()), - outcome.GetError().GetRequestId()}}; - } -} - -ObjectStorageResponse S3ObjStorageClient::get_object(const ObjectStoragePathOptions& opts, - void* buffer, size_t offset, size_t bytes_read, - size_t* size_return) { - Aws::S3::Model::GetObjectRequest request; - request.WithBucket(opts.bucket).WithKey(opts.key); - request.SetRange(fmt::format("bytes={}-{}", offset, offset + bytes_read - 1)); - request.SetResponseStreamFactory(AwsWriteableStreamFactory(buffer, bytes_read)); - - SCOPED_BVAR_LATENCY(s3_bvar::s3_get_latency); - auto outcome = _client->GetObject(request); - if (!outcome.IsSuccess()) { - record_s3_request_failed(outcome.GetError()); - return {convert_to_obj_response(s3fs_error( - outcome.GetError(), fmt::format("failed to read from {}", opts.key))), - static_cast(outcome.GetError().GetResponseCode()), - outcome.GetError().GetRequestId()}; - } - *size_return = outcome.GetResult().GetContentLength(); - // case for incomplete read - SYNC_POINT_CALLBACK("s3_obj_storage_client::get_object", size_return); - if (*size_return != bytes_read) { - return {convert_to_obj_response(Status::InternalError( - "failed to read from {}(bytes read: {}, bytes req: {}), request_id: {}", opts.key, - *size_return, bytes_read, outcome.GetResult().GetRequestId()))}; - } - return ObjectStorageResponse::OK(); -} - -ObjectStorageResponse S3ObjStorageClient::list_objects(const ObjectStoragePathOptions& opts, - std::vector* files) { - Aws::S3::Model::ListObjectsV2Request request; - request.WithBucket(opts.bucket).WithPrefix(opts.prefix); - bool is_trucated = false; - do { - Aws::S3::Model::ListObjectsV2Outcome outcome; - { - SCOPED_BVAR_LATENCY(s3_bvar::s3_list_latency); - outcome = _client->ListObjectsV2(request); - } - if (!outcome.IsSuccess()) { - files->clear(); - // Treat NoSuchKey as empty response for compatibility with some S3-compatible storage providers - // e.g. TOS by ByteDance Cloud (Volcano Engine) - if (outcome.GetError().GetErrorType() == Aws::S3::S3Errors::NO_SUCH_KEY) { - LOG(INFO) << "NoSuchKey error when listing objects, treat as empty response" - << ", prefix=" << opts.prefix - << ", request_id=" << outcome.GetError().GetRequestId(); - return ObjectStorageResponse::OK(); - } - - record_s3_request_failed(outcome.GetError()); - return {convert_to_obj_response(s3fs_error( - outcome.GetError(), fmt::format("failed to list {}", opts.prefix))), - static_cast(outcome.GetError().GetResponseCode()), - outcome.GetError().GetRequestId()}; - } - for (const auto& obj : outcome.GetResult().GetContents()) { - std::string key = obj.GetKey(); - bool is_dir = (key.back() == '/'); - FileInfo file_info; - file_info.file_name = obj.GetKey(); - file_info.file_size = obj.GetSize(); - file_info.is_file = !is_dir; - files->push_back(std::move(file_info)); - } - is_trucated = outcome.GetResult().GetIsTruncated(); - if (is_trucated && outcome.GetResult().GetNextContinuationToken().empty()) { - return {convert_to_obj_response( - Status::InternalError("failed to list {}, is_trucated is true, but next " - "continuation token is empty, request_id={}", - opts.prefix, outcome.GetResult().GetRequestId()))}; - } - - request.SetContinuationToken(outcome.GetResult().GetNextContinuationToken()); - } while (is_trucated); - return ObjectStorageResponse::OK(); -} - -ObjectStorageResponse S3ObjStorageClient::delete_objects(const ObjectStoragePathOptions& opts, - std::vector objs) { - Aws::S3::Model::DeleteObjectsRequest delete_request; - delete_request.SetBucket(opts.bucket); - Aws::S3::Model::Delete del; - Aws::Vector objects; - std::ranges::transform(objs, std::back_inserter(objects), [](auto&& obj_key) { - Aws::S3::Model::ObjectIdentifier obj_identifier; - obj_identifier.SetKey(std::move(obj_key)); - return obj_identifier; - }); - del.WithObjects(std::move(objects)).SetQuiet(true); - delete_request.SetDelete(std::move(del)); - SCOPED_BVAR_LATENCY(s3_bvar::s3_delete_objects_latency); - auto delete_outcome = _client->DeleteObjects(delete_request); - if (!delete_outcome.IsSuccess()) { - record_s3_request_failed(delete_outcome.GetError()); - return {convert_to_obj_response( - s3fs_error(delete_outcome.GetError(), - fmt::format("failed to delete dir {}", opts.key))), - static_cast(delete_outcome.GetError().GetResponseCode()), - delete_outcome.GetError().GetRequestId()}; - } - // case for partial delete object failure - SYNC_POINT_CALLBACK("s3_obj_storage_client::delete_objects", &delete_outcome); - if (!delete_outcome.GetResult().GetErrors().empty()) { - const auto& e = delete_outcome.GetResult().GetErrors().front(); - return {convert_to_obj_response( - Status::InternalError("failed to delete object {}: {}, request_id={}", e.GetKey(), - e.GetMessage(), delete_outcome.GetResult().GetRequestId()))}; - } - return ObjectStorageResponse::OK(); -} - -ObjectStorageResponse S3ObjStorageClient::delete_object(const ObjectStoragePathOptions& opts) { - Aws::S3::Model::DeleteObjectRequest request; - request.WithBucket(opts.bucket).WithKey(opts.key); - - SCOPED_BVAR_LATENCY(s3_bvar::s3_delete_object_latency); - auto outcome = _client->DeleteObject(request); - if (outcome.IsSuccess() || - outcome.GetError().GetResponseCode() == Aws::Http::HttpResponseCode::NOT_FOUND) { - return ObjectStorageResponse::OK(); - } - record_s3_request_failed(outcome.GetError()); - return {convert_to_obj_response(s3fs_error(outcome.GetError(), - fmt::format("failed to delete file {}", opts.key))), - static_cast(outcome.GetError().GetResponseCode()), - outcome.GetError().GetRequestId()}; -} - -ObjectStorageResponse S3ObjStorageClient::delete_objects_recursively( - const ObjectStoragePathOptions& opts) { - Aws::S3::Model::ListObjectsV2Request request; - request.WithBucket(opts.bucket).WithPrefix(opts.prefix); - Aws::S3::Model::DeleteObjectsRequest delete_request; - delete_request.SetBucket(opts.bucket); - bool is_trucated = false; - do { - Aws::S3::Model::ListObjectsV2Outcome outcome; - { - SCOPED_BVAR_LATENCY(s3_bvar::s3_list_latency); - outcome = _client->ListObjectsV2(request); - } - if (!outcome.IsSuccess()) { - record_s3_request_failed(outcome.GetError()); - return {convert_to_obj_response(s3fs_error( - outcome.GetError(), - fmt::format("failed to list objects when delete dir {}", opts.prefix))), - static_cast(outcome.GetError().GetResponseCode()), - outcome.GetError().GetRequestId()}; - } - const auto& result = outcome.GetResult(); - Aws::Vector objects; - objects.reserve(result.GetContents().size()); - for (const auto& obj : result.GetContents()) { - objects.emplace_back().SetKey(obj.GetKey()); - } - if (!objects.empty()) { - Aws::S3::Model::Delete del; - del.WithObjects(std::move(objects)).SetQuiet(true); - delete_request.SetDelete(std::move(del)); - SCOPED_BVAR_LATENCY(s3_bvar::s3_delete_objects_latency); - auto delete_outcome = _client->DeleteObjects(delete_request); - if (!delete_outcome.IsSuccess()) { - record_s3_request_failed(delete_outcome.GetError()); - return {convert_to_obj_response( - s3fs_error(delete_outcome.GetError(), - fmt::format("failed to delete dir {}", opts.key))), - static_cast(delete_outcome.GetError().GetResponseCode()), - delete_outcome.GetError().GetRequestId()}; - } - // case for partial delete object failure - SYNC_POINT_CALLBACK("s3_obj_storage_client::delete_objects_recursively", - &delete_outcome); - if (!delete_outcome.GetResult().GetErrors().empty()) { - const auto& e = delete_outcome.GetResult().GetErrors().front(); - return {convert_to_obj_response(Status::InternalError( - "failed to delete object {}: {}, request_id={}", opts.key, e.GetMessage(), - delete_outcome.GetResult().GetRequestId()))}; - } - } - is_trucated = result.GetIsTruncated(); - request.SetContinuationToken(result.GetNextContinuationToken()); - } while (is_trucated); - return ObjectStorageResponse::OK(); -} - -std::string S3ObjStorageClient::generate_presigned_url(const ObjectStoragePathOptions& opts, - int64_t expiration_secs, - const S3ClientConf&) { - return _client->GeneratePresignedUrl(opts.bucket, opts.key, Aws::Http::HttpMethod::HTTP_GET, - expiration_secs); -} - -} // namespace doris::io diff --git a/be/src/io/fs/s3_obj_storage_client.h b/be/src/io/fs/s3_obj_storage_client.h deleted file mode 100644 index 45294226594d81..00000000000000 --- a/be/src/io/fs/s3_obj_storage_client.h +++ /dev/null @@ -1,63 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include "io/fs/obj_storage_client.h" -#include "io/fs/s3_file_system.h" - -namespace Aws::S3 { -class S3Client; -namespace Model { -class CompletedPart; -} -} // namespace Aws::S3 - -namespace doris::io { -class ObjClientHolder; - -class S3ObjStorageClient final : public ObjStorageClient { -public: - S3ObjStorageClient(std::shared_ptr client) : _client(std::move(client)) {} - ~S3ObjStorageClient() override = default; - ObjectStorageUploadResponse create_multipart_upload( - const ObjectStoragePathOptions& opts) override; - ObjectStorageResponse put_object(const ObjectStoragePathOptions& opts, - std::string_view stream) override; - ObjectStorageUploadResponse upload_part(const ObjectStoragePathOptions& opts, std::string_view, - int partNum) override; - ObjectStorageResponse complete_multipart_upload( - const ObjectStoragePathOptions& opts, - const std::vector& completed_parts) override; - ObjectStorageHeadResponse head_object(const ObjectStoragePathOptions& opts) override; - ObjectStorageResponse get_object(const ObjectStoragePathOptions& opts, void* buffer, - size_t offset, size_t bytes_read, - size_t* size_return) override; - ObjectStorageResponse list_objects(const ObjectStoragePathOptions& opts, - std::vector* files) override; - ObjectStorageResponse delete_objects(const ObjectStoragePathOptions& opts, - std::vector objs) override; - ObjectStorageResponse delete_object(const ObjectStoragePathOptions& opts) override; - ObjectStorageResponse delete_objects_recursively(const ObjectStoragePathOptions& opts) override; - std::string generate_presigned_url(const ObjectStoragePathOptions& opts, - int64_t expiration_secs, const S3ClientConf&) override; - -private: - std::shared_ptr _client; -}; - -} // namespace doris::io diff --git a/be/src/load/routine_load/data_consumer.cpp b/be/src/load/routine_load/data_consumer.cpp index 516649f0f35b8e..b305ba6c9aae38 100644 --- a/be/src/load/routine_load/data_consumer.cpp +++ b/be/src/load/routine_load/data_consumer.cpp @@ -752,12 +752,16 @@ Status KinesisDataConsumer::_create_kinesis_client(std::shared_ptr(credentials_provider, aws_config); + _kinesis_client = std::make_shared(std::move(credentials.provider), + aws_config); if (!_kinesis_client) { return Status::InternalError( diff --git a/be/src/util/s3_rate_limiter_manager.h b/be/src/util/s3_rate_limiter_manager.h index ed668b02f0db0f..9f2220745ea79b 100644 --- a/be/src/util/s3_rate_limiter_manager.h +++ b/be/src/util/s3_rate_limiter_manager.h @@ -84,7 +84,10 @@ class S3RateLimiterManager { std::array, 2> _bytes_limiters; }; -// RAII admission for one logical object storage request. +// RAII admission for one logical object-storage call. The common ObjStorageClient facade +// constructs the guard before dispatching to the provider client. A lazy list page and a +// provider-sized delete batch are each treated as one logical call, including the list pages and +// delete batches issued by recursive deletion. // // The constructor charges the QPS bucket (may sleep when throttled; rejected only by // the legacy token_limit cumulative cap) and then reserves `estimated_bytes` from the diff --git a/be/src/util/s3_util.cpp b/be/src/util/s3_util.cpp index fe9af8f582e8f9..b4f54d1f4c29d4 100644 --- a/be/src/util/s3_util.cpp +++ b/be/src/util/s3_util.cpp @@ -20,19 +20,11 @@ #include #include #include -#include #include -#include #include #include #include -#include #include -#include -#include -#include - -#include #include "util/string_util.h" @@ -52,36 +44,23 @@ #include "common/config.h" #include "common/logging.h" #include "common/status.h" +#include "cpp/client/auth/aws_credential_factory.h" +#ifdef USE_AZURE +#include "cpp/client/auth/azure_auth_factory.h" +#include "cpp/client/azure_obj_storage_backend.h" +#endif +#include "cloud/config.h" #include "cpp/aws_logger.h" -#include "cpp/custom_aws_credentials_provider_chain.h" +#include "cpp/client/s3_obj_storage_backend.h" #include "cpp/obj_retry_strategy.h" #include "cpp/sync_point.h" #include "cpp/util.h" -#ifdef USE_AZURE -#include "io/fs/azure_obj_storage_client.h" -#endif -#include "cloud/config.h" #include "exec/scan/scanner_scheduler.h" -#include "io/fs/obj_storage_client.h" -#include "io/fs/rate_limited_obj_storage_client.h" -#include "io/fs/s3_obj_storage_client.h" #include "runtime/exec_env.h" +#include "util/s3_rate_limiter_manager.h" #include "util/s3_uri.h" namespace doris { -namespace s3_bvar { -bvar::LatencyRecorder s3_get_latency("s3_get"); -bvar::LatencyRecorder s3_put_latency("s3_put"); -bvar::LatencyRecorder s3_delete_object_latency("s3_delete_object"); -bvar::LatencyRecorder s3_delete_objects_latency("s3_delete_objects"); -bvar::LatencyRecorder s3_head_latency("s3_head"); -bvar::LatencyRecorder s3_multi_part_upload_latency("s3_multi_part_upload"); -bvar::LatencyRecorder s3_list_latency("s3_list"); -bvar::LatencyRecorder s3_list_object_versions_latency("s3_list_object_versions"); -bvar::LatencyRecorder s3_get_bucket_version_latency("s3_get_bucket_version"); -bvar::LatencyRecorder s3_copy_object_latency("s3_copy_object"); -}; // namespace s3_bvar - namespace { doris::Status is_s3_conf_valid(const S3ClientConf& conf) { @@ -108,6 +87,41 @@ doris::Status is_s3_conf_valid(const S3ClientConf& conf) { return Status::OK(); } +ObjectStorageResponse make_be_rate_limit_response(S3RateLimitType type, + S3RateLimitRejectReason reason) { + const auto* limit_type = reason == S3RateLimitRejectReason::QPS ? "QPS" : "bytes"; + return { + .status = + ObjectStorageStatus { + ErrorCode::EXCEEDED_LIMIT, + fmt::format( + "s3 {} request exceeds {} limit, rejected by BE rate limiter", + to_string(type), limit_type)}, + // A local admission rejection is not an S3 HTTP 429. Keep the merged #65420 behavior + // so S3 readers do not retry it as provider throttling. + .http_code = 0, + }; +} + +class BeObjStorageRateLimitPolicy final : public ObjStorageRateLimitPolicy { +public: + ObjStorageRateLimitToken acquire(ObjStorageRequestType type, + size_t estimated_bytes) const override { + const auto limiter_type = + type == ObjStorageRequestType::GET ? S3RateLimitType::GET : S3RateLimitType::PUT; + auto guard = std::make_shared(limiter_type, estimated_bytes); + if (!guard->ok()) { + return ObjStorageRateLimitToken { + .resp = make_be_rate_limit_response(limiter_type, guard->reject_reason()), + }; + } + return ObjStorageRateLimitToken { + .settle = [guard = std::move(guard)]( + size_t actual_bytes) { guard->settle(actual_bytes); }, + }; + } +}; + // Return true is convert `str` to int successfully bool to_int(std::string_view str, int& res) { auto [_, ec] = std::from_chars(str.data(), str.data() + str.size(), res); @@ -207,10 +221,8 @@ S3ClientFactory& S3ClientFactory::instance() { return ret; } -std::shared_ptr S3ClientFactory::create(const S3ClientConf& s3_conf) { - if (!is_s3_conf_valid(s3_conf).ok()) { - return nullptr; - } +Result> S3ClientFactory::create(const S3ClientConf& s3_conf) { + RETURN_IF_ERROR_RESULT(is_s3_conf_valid(s3_conf)); #ifdef BE_TEST { @@ -229,17 +241,19 @@ std::shared_ptr S3ClientFactory::create(const S3ClientConf } } - auto obj_client = (s3_conf.provider == io::ObjStorageType::AZURE) - ? _create_azure_client(s3_conf) - : _create_s3_client(s3_conf); - - // Rate limiting lives in one decorator, decided here at construction time: - // in cloud mode only internal storage-vault buckets are limited; external buckets - // (S3 load, TVF, external catalogs) get the bare client. In non-cloud mode every - // client is wrapped, preserving the legacy behavior. - if (obj_client != nullptr && (!config::is_cloud_mode() || s3_conf.is_internal_bucket)) { - obj_client = std::make_shared(std::move(obj_client)); + auto backend_result = (s3_conf.provider == io::ObjStorageType::AZURE) + ? _create_azure_backend(s3_conf) + : _create_s3_backend(s3_conf); + if (!backend_result.has_value()) { + return ResultError(std::move(backend_result).error()); } + auto backend = std::move(backend_result).value(); + std::shared_ptr rate_limit_policy; + if (!config::is_cloud_mode() || s3_conf.is_internal_bucket) { + rate_limit_policy = std::make_shared(); + } + auto obj_client = std::make_shared(std::move(backend), + std::move(rate_limit_policy)); { std::lock_guard l(_lock); @@ -261,12 +275,9 @@ void S3ClientFactory::clear_client_creator_for_test() { } #endif -std::shared_ptr S3ClientFactory::_create_azure_client( +Result> S3ClientFactory::_create_azure_backend( const S3ClientConf& s3_conf) { #ifdef USE_AZURE - auto cred = - std::make_shared(s3_conf.ak, s3_conf.sk); - const std::string container_name = s3_conf.bucket; std::string uri = fmt::format("{}/{}", s3_conf.endpoint, container_name); if (s3_conf.endpoint.find("://") == std::string::npos) { @@ -291,132 +302,59 @@ std::shared_ptr S3ClientFactory::_create_azure_client( VLOG_DEBUG << "uri:" << uri << ", normalized_uri:" << normalized_uri; std::string tls_debug_context = build_azure_tls_debug_context(_ca_cert_file_path); - auto containerClient = std::make_shared( - uri, cred, std::move(options)); + auto built = AzureAuthFactory::create(uri, + { + .type = AzureCredentialType::SHARED_KEY, + .account_name = s3_conf.ak, + .account_key = s3_conf.sk, + }, + std::move(options)); + if (!built) { + return ResultError( + Status::InvalidArgument("failed to create Azure client: {}", built.error)); + } LOG_INFO("create one azure client with {}", s3_conf.to_string()); - return std::make_shared(std::move(containerClient), - std::move(tls_debug_context)); + return std::make_shared( + std::move(built.container_client), + ObjectClientConfig { + .endpoint = s3_conf.endpoint, + .ak = s3_conf.ak, + .sk = s3_conf.sk, + .tls_debug_context = std::move(tls_debug_context), + }, + std::move(built.shared_key_credential)); #else - LOG_FATAL("BE is not compiled with azure support, export BUILD_AZURE=ON before building"); - return nullptr; + return ResultError(Status::NotSupported( + "BE is not compiled with azure support, export BUILD_AZURE=ON before building")); #endif } -std::shared_ptr -S3ClientFactory::_get_aws_credentials_provider_v1(const S3ClientConf& s3_conf) { - if (!s3_conf.ak.empty() && !s3_conf.sk.empty()) { - Aws::Auth::AWSCredentials aws_cred(s3_conf.ak, s3_conf.sk); - DCHECK(!aws_cred.IsExpiredOrEmpty()); - if (!s3_conf.token.empty()) { - aws_cred.SetSessionToken(s3_conf.token); - } - return std::make_shared(std::move(aws_cred)); - } - - if (s3_conf.cred_provider_type == CredProviderType::InstanceProfile) { - if (s3_conf.role_arn.empty()) { - return std::make_shared(); - } - - Aws::Client::ClientConfiguration clientConfiguration = - S3ClientFactory::getClientConfiguration(); - - if (_ca_cert_file_path.empty()) { - _ca_cert_file_path = - get_valid_ca_cert_path(doris::split(config::ca_cert_file_paths, ";")); - } - if (!_ca_cert_file_path.empty()) { - clientConfiguration.caFile = _ca_cert_file_path; - } - - auto stsClient = std::make_shared( - std::make_shared(), - clientConfiguration); - - return std::make_shared( - s3_conf.role_arn, Aws::String(), s3_conf.external_id, - Aws::Auth::DEFAULT_CREDS_LOAD_FREQ_SECONDS, stsClient); - } - - // Support anonymous access for public datasets when no credentials are provided - if (s3_conf.ak.empty() && s3_conf.sk.empty()) { - return std::make_shared(); - } - - return std::make_shared(); -} - -std::shared_ptr S3ClientFactory::_create_credentials_provider( - CredProviderType type) { - switch (type) { - case CredProviderType::Env: - return std::make_shared(); - case CredProviderType::SystemProperties: - return std::make_shared(); - case CredProviderType::WebIdentity: - return std::make_shared(); - case CredProviderType::Container: - return std::make_shared( - Aws::Environment::GetEnv("AWS_CONTAINER_CREDENTIALS_RELATIVE_URI").c_str()); - case CredProviderType::InstanceProfile: - return std::make_shared(); - case CredProviderType::Anonymous: - return std::make_shared(); - case CredProviderType::Default: - default: - return std::make_shared(); - } -} - -std::shared_ptr -S3ClientFactory::_get_aws_credentials_provider_v2(const S3ClientConf& s3_conf) { - if (!s3_conf.ak.empty() && !s3_conf.sk.empty()) { - Aws::Auth::AWSCredentials aws_cred(s3_conf.ak, s3_conf.sk); - DCHECK(!aws_cred.IsExpiredOrEmpty()); - if (!s3_conf.token.empty()) { - aws_cred.SetSessionToken(s3_conf.token); - } - return std::make_shared(std::move(aws_cred)); - } - - // Handle role_arn for assume role scenario - if (!s3_conf.role_arn.empty()) { - Aws::Client::ClientConfiguration clientConfiguration = - S3ClientFactory::getClientConfiguration(); - - if (_ca_cert_file_path.empty()) { - _ca_cert_file_path = - get_valid_ca_cert_path(doris::split(config::ca_cert_file_paths, ";")); - } - if (!_ca_cert_file_path.empty()) { - clientConfiguration.caFile = _ca_cert_file_path; - } - - auto baseProvider = _create_credentials_provider(s3_conf.cred_provider_type); - auto stsClient = std::make_shared(baseProvider, clientConfiguration); - - return std::make_shared( - s3_conf.role_arn, Aws::String(), s3_conf.external_id, - Aws::Auth::DEFAULT_CREDS_LOAD_FREQ_SECONDS, stsClient); - } - - // Return provider based on cred_provider_type - return _create_credentials_provider(s3_conf.cred_provider_type); -} - -std::shared_ptr S3ClientFactory::get_aws_credentials_provider( - const S3ClientConf& s3_conf) { - if (config::aws_credentials_provider_version == "v2") { - return _get_aws_credentials_provider_v2(s3_conf); - } - return _get_aws_credentials_provider_v1(s3_conf); +AwsCredentialResult S3ClientFactory::create_aws_credentials_provider(const S3ClientConf& s3_conf) { + auto sts_config = S3ClientFactory::getClientConfiguration(); + if (!_ca_cert_file_path.empty()) { + sts_config.caFile = _ca_cert_file_path; + } + return AwsCredentialFactory::create({ + .version = config::aws_credentials_provider_version == "v2" + ? AwsCredentialProviderVersion::V2 + : AwsCredentialProviderVersion::V1, + .access_key = s3_conf.ak, + .secret_key = s3_conf.sk, + .session_token = s3_conf.token, + .provider_type = s3_conf.cred_provider_type, + .role_arn = s3_conf.role_arn, + .external_id = s3_conf.external_id, + .empty_credentials = EmptyCredentialsBehavior::ANONYMOUS, + .sts_client_config = std::move(sts_config), + }); } -std::shared_ptr S3ClientFactory::_create_s3_client( +Result> S3ClientFactory::_create_s3_backend( const S3ClientConf& s3_conf) { TEST_SYNC_POINT_RETURN_WITH_VALUE( "s3_client_factory::create", - std::make_shared(std::make_shared())); + std::make_shared(std::make_shared(), + ObjectClientConfig {})); Aws::Client::ClientConfiguration aws_config = S3ClientFactory::getClientConfiguration(); if (s3_conf.need_override_endpoint) { aws_config.endpointOverride = s3_conf.endpoint; @@ -451,14 +389,24 @@ std::shared_ptr S3ClientFactory::_create_s3_client( aws_config.retryStrategy = std::make_shared( config::max_s3_client_retry /*scaleFactor = 25*/, /*retry_slow_down=*/true); + auto credentials = create_aws_credentials_provider(s3_conf); + if (!credentials) { + return ResultError(Status::InvalidArgument("failed to create AWS credential provider: {}", + credentials.error)); + } std::shared_ptr new_client = std::make_shared( - get_aws_credentials_provider(s3_conf), std::move(aws_config), + std::move(credentials.provider), std::move(aws_config), Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never, s3_conf.use_virtual_addressing); - auto obj_client = std::make_shared(std::move(new_client)); + auto backend = std::make_shared(std::move(new_client), + ObjectClientConfig { + .endpoint = s3_conf.endpoint, + .ak = s3_conf.ak, + .sk = s3_conf.sk, + }); LOG_INFO("create one s3 client with {}", s3_conf.to_string()); - return obj_client; + return backend; } Status S3ClientFactory::convert_properties_to_s3_conf( @@ -576,7 +524,7 @@ S3Conf S3Conf::get_s3_conf(const cloud::ObjectStoreInfoPB& info) { .region = info.region(), .ak = info.ak(), .sk = info.sk(), - .token {}, + .token = {}, .bucket = info.bucket(), .provider = io::ObjStorageType::AWS, .use_virtual_addressing = diff --git a/be/src/util/s3_util.h b/be/src/util/s3_util.h index ef938d00c15122..28b79c9e05fc9b 100644 --- a/be/src/util/s3_util.h +++ b/be/src/util/s3_util.h @@ -35,8 +35,8 @@ #include "common/status.h" #include "core/string_ref.h" #include "cpp/aws_common.h" -#include "cpp/token_bucket_rate_limiter.h" -#include "io/fs/obj_storage_client.h" +#include "cpp/client/auth/aws_credential_factory.h" +#include "cpp/client/obj_storage_client.h" namespace Aws::S3 { class S3Client; @@ -49,19 +49,6 @@ class Adder; namespace doris { -namespace s3_bvar { -extern bvar::LatencyRecorder s3_get_latency; -extern bvar::LatencyRecorder s3_put_latency; -extern bvar::LatencyRecorder s3_delete_object_latency; -extern bvar::LatencyRecorder s3_delete_objects_latency; -extern bvar::LatencyRecorder s3_head_latency; -extern bvar::LatencyRecorder s3_multi_part_upload_latency; -extern bvar::LatencyRecorder s3_list_latency; -extern bvar::LatencyRecorder s3_list_object_versions_latency; -extern bvar::LatencyRecorder s3_get_bucket_version_latency; -extern bvar::LatencyRecorder s3_copy_object_latency; -}; // namespace s3_bvar - std::string hide_access_key(const std::string& ak); class S3URI; @@ -121,9 +108,9 @@ struct S3ClientConf { "(ak={}, token={}, endpoint={}, region={}, bucket={}, max_connections={}, " "request_timeout_ms={}, connect_timeout_ms={}, use_virtual_addressing={}, " "cred_provider_type={},role_arn={}, external_id={}, is_internal_bucket={}", - hide_access_key(ak), token, endpoint, region, bucket, max_connections, - request_timeout_ms, connect_timeout_ms, use_virtual_addressing, cred_provider_type, - role_arn, external_id, is_internal_bucket); + hide_access_key(ak), token.empty() ? "" : "******", endpoint, region, bucket, + max_connections, request_timeout_ms, connect_timeout_ms, use_virtual_addressing, + cred_provider_type, role_arn, external_id, is_internal_bucket); } }; @@ -154,7 +141,7 @@ class S3ClientFactory { static S3ClientFactory& instance(); - std::shared_ptr create(const S3ClientConf& s3_conf); + Result> create(const S3ClientConf& s3_conf); static Status convert_properties_to_s3_conf(const std::map& prop, const S3URI& s3_uri, S3Conf* s3_conf); @@ -170,8 +157,7 @@ class S3ClientFactory { return instance; } - std::shared_ptr get_aws_credentials_provider( - const S3ClientConf& s3_conf); + AwsCredentialResult create_aws_credentials_provider(const S3ClientConf& s3_conf); #ifdef BE_TEST void set_client_creator_for_test( @@ -181,15 +167,9 @@ class S3ClientFactory { #endif private: - std::shared_ptr _create_s3_client(const S3ClientConf& s3_conf); - std::shared_ptr _create_azure_client(const S3ClientConf& s3_conf); - std::shared_ptr _get_aws_credentials_provider_v1( + Result> _create_s3_backend(const S3ClientConf& s3_conf); + Result> _create_azure_backend( const S3ClientConf& s3_conf); - std::shared_ptr _get_aws_credentials_provider_v2( - const S3ClientConf& s3_conf); - std::shared_ptr _create_credentials_provider( - CredProviderType type); - S3ClientFactory(); Aws::SDKOptions _aws_options; diff --git a/be/test/ai/embed_test.cpp b/be/test/ai/embed_test.cpp index c9bd32ed17cb7a..1a61946faf0f0d 100644 --- a/be/test/ai/embed_test.cpp +++ b/be/test/ai/embed_test.cpp @@ -30,9 +30,9 @@ #include "core/data_type/data_type_jsonb.h" #include "core/data_type/data_type_number.h" #include "core/value/jsonb_value.h" +#include "cpp/client/obj_storage_client.h" #include "exprs/function/ai/ai_adapter.h" #include "exprs/function/simple_function_factory.h" -#include "io/fs/obj_storage_client.h" #include "testutil/column_helper.h" #include "testutil/mock/mock_runtime_state.h" @@ -47,7 +47,7 @@ class MockHttpClient : public HttpClient { std::string _content_type; }; -class MockEmbedObjStorageClient : public io::ObjStorageClient { +class MockEmbedObjStorageBackend : public io::ObjStorageBackend { public: io::ObjectStorageUploadResponse create_multipart_upload( const io::ObjectStoragePathOptions& /*opts*/) override { @@ -82,9 +82,9 @@ class MockEmbedObjStorageClient : public io::ObjStorageClient { return io::ObjectStorageResponse::OK(); } - io::ObjectStorageResponse list_objects(const io::ObjectStoragePathOptions& /*opts*/, - std::vector* /*files*/) override { - return io::ObjectStorageResponse::OK(); + io::ObjectStorageListPage list_objects(const io::ObjectStoragePathOptions& /*opts*/, + std::string_view /*continuation_token*/) override { + return {.resp = io::ObjectStorageResponse::OK()}; } io::ObjectStorageResponse delete_objects(const io::ObjectStoragePathOptions& /*opts*/, @@ -96,16 +96,10 @@ class MockEmbedObjStorageClient : public io::ObjStorageClient { return io::ObjectStorageResponse::OK(); } - io::ObjectStorageResponse delete_objects_recursively( - const io::ObjectStoragePathOptions& /*opts*/) override { - return io::ObjectStorageResponse::OK(); - } - std::string generate_presigned_url(const io::ObjectStoragePathOptions& opts, - int64_t expiration_secs, const S3ClientConf& conf) override { + int64_t expiration_secs) override { last_opts = opts; last_expiration_secs = expiration_secs; - last_conf = conf; return fmt::format("mock-s3://{}/{}?ttl={}", opts.bucket, opts.key, expiration_secs); } @@ -618,9 +612,12 @@ TEST(EMBED_TEST, embed_function_multimodal_s3_presigned_url) { query_ctx.get()); auto ctx = FunctionContext::create_context(&runtime_state, {}, {}); - auto mock_client = std::make_shared(); + auto mock_client = std::make_shared(); S3ClientFactory::instance().set_client_creator_for_test( - [mock_client](const S3ClientConf&) { return mock_client; }); + [mock_client](const S3ClientConf& conf) { + mock_client->last_conf = conf; + return std::make_shared(mock_client); + }); std::vector resources = {"mock_resource"}; std::vector file_json_rows = {R"({ diff --git a/be/test/io/client/s3_file_system_test.cpp b/be/test/io/client/s3_file_system_test.cpp index 375ff3ff57f8e2..a8451e0e44d1d9 100644 --- a/be/test/io/client/s3_file_system_test.cpp +++ b/be/test/io/client/s3_file_system_test.cpp @@ -29,11 +29,11 @@ #include #include "common/config.h" +#include "cpp/client/obj_storage_client.h" #include "cpp/sync_point.h" #include "io/fs/file_reader.h" #include "io/fs/file_system.h" #include "io/fs/file_writer.h" -#include "io/fs/obj_storage_client.h" #include "runtime/exec_env.h" #include "util/defer_op.h" #include "util/s3_rate_limiter_manager.h" diff --git a/be/test/io/fs/azure_obj_storage_client_test.cpp b/be/test/io/fs/azure_obj_storage_client_test.cpp index 7591b4bf2ea997..cfe4b0e2e85c37 100644 --- a/be/test/io/fs/azure_obj_storage_client_test.cpp +++ b/be/test/io/fs/azure_obj_storage_client_test.cpp @@ -15,12 +15,14 @@ // specific language governing permissions and limitations // under the License. -#include "io/fs/azure_obj_storage_client.h" +#ifdef USE_AZURE +#include "cpp/client/azure_obj_storage_backend.h" +#endif #include +#include "cpp/client/obj_storage_client.h" #include "io/fs/file_system.h" -#include "io/fs/obj_storage_client.h" #include "util/s3_util.h" #ifdef USE_AZURE @@ -59,7 +61,7 @@ TEST(AzureObjStorageClientTlsHelperTest, appends_debug_suffix_only_for_tls_ca_er class AzureObjStorageClientTest : public testing::Test { protected: - static std::shared_ptr obj_storage_client; + static std::shared_ptr obj_storage_client; static void SetUpTestSuite() { if (!std::getenv("AZURE_ACCOUNT_NAME") || !std::getenv("AZURE_ACCOUNT_KEY") || @@ -74,16 +76,18 @@ class AzureObjStorageClientTest : public testing::Test { // Initialize Azure SDK [[maybe_unused]] auto& s3ClientFactory = S3ClientFactory::instance(); - AzureObjStorageClientTest::obj_storage_client = S3ClientFactory::instance().create( + auto client_result = S3ClientFactory::instance().create( {.endpoint = fmt::format("https://{}.blob.core.windows.net", accountName), .region = "dummy-region", .ak = accountName, .sk = accountKey, .token = "", .bucket = containerName, - .provider = io::ObjStorageType::AZURE, + .provider = ObjStorageType::AZURE, .role_arn = "", .external_id = ""}); + ASSERT_TRUE(client_result.has_value()) << client_result.error(); + AzureObjStorageClientTest::obj_storage_client = std::move(client_result).value(); } void SetUp() override { @@ -93,7 +97,7 @@ class AzureObjStorageClientTest : public testing::Test { } }; -std::shared_ptr AzureObjStorageClientTest::obj_storage_client = nullptr; +std::shared_ptr AzureObjStorageClientTest::obj_storage_client = nullptr; TEST_F(AzureObjStorageClientTest, put_list_delete_object) { LOG(INFO) << "AzureObjStorageClientTest::put_list_delete_object"; @@ -104,10 +108,16 @@ TEST_F(AzureObjStorageClientTest, put_list_delete_object) { std::vector files; // clang-format off - response = AzureObjStorageClientTest::obj_storage_client->list_objects({.bucket = "dummy", - .prefix = "AzureObjStorageClientTest/put_list_delete_object",}, &files); + ObjectListIterator iter(AzureObjStorageClientTest::obj_storage_client, {.bucket = "dummy", + .prefix = "AzureObjStorageClientTest/put_list_delete_object"}); // clang-format on - EXPECT_EQ(response.status.code, ErrorCode::OK); + for (auto obj = iter.next(); obj.results_.has_value(); obj = iter.next()) { + EXPECT_TRUE(obj.resp.ok()); + files.push_back({.file_name = obj.results_->file_path, + .file_size = obj.results_->size, + .is_file = true}); + } + EXPECT_TRUE(iter.is_valid()); EXPECT_EQ(files.size(), 1); files.clear(); @@ -116,10 +126,16 @@ TEST_F(AzureObjStorageClientTest, put_list_delete_object) { EXPECT_EQ(response.status.code, ErrorCode::OK); // clang-format off - response = AzureObjStorageClientTest::obj_storage_client->list_objects({.bucket = "dummy", - .prefix = "AzureObjStorageClientTest/put_list_delete_object",}, &files); + iter = ObjectListIterator(AzureObjStorageClientTest::obj_storage_client, {.bucket = "dummy", + .prefix = "AzureObjStorageClientTest/put_list_delete_object"}); // clang-format on - EXPECT_EQ(response.status.code, ErrorCode::OK); + for (auto obj = iter.next(); obj.results_.has_value(); obj = iter.next()) { + EXPECT_TRUE(obj.resp.ok()); + files.push_back({.file_name = obj.results_->file_path, + .file_size = obj.results_->size, + .is_file = true}); + } + EXPECT_TRUE(iter.is_valid()); EXPECT_EQ(files.size(), 0); } @@ -138,22 +154,34 @@ TEST_F(AzureObjStorageClientTest, delete_objects_recursively) { std::vector files; // clang-format off - auto response = AzureObjStorageClientTest::obj_storage_client->list_objects({.bucket = "dummy", - .prefix = "AzureObjStorageClientTest/delete_objects_recursively",}, &files); + ObjectListIterator iter(AzureObjStorageClientTest::obj_storage_client, {.bucket = "dummy", + .prefix = "AzureObjStorageClientTest/delete_objects_recursively"}); // clang-format on - EXPECT_EQ(response.status.code, ErrorCode::OK); + for (auto obj = iter.next(); obj.results_.has_value(); obj = iter.next()) { + EXPECT_TRUE(obj.resp.ok()); + files.push_back({.file_name = obj.results_->file_path, + .file_size = obj.results_->size, + .is_file = true}); + } + EXPECT_TRUE(iter.is_valid()); EXPECT_EQ(files.size(), 22); files.clear(); - response = AzureObjStorageClientTest::obj_storage_client->delete_objects_recursively( + auto response = AzureObjStorageClientTest::obj_storage_client->delete_objects_recursively( {.prefix = "AzureObjStorageClientTest/delete_objects_recursively"}); EXPECT_EQ(response.status.code, ErrorCode::OK); // clang-format off - response = AzureObjStorageClientTest::obj_storage_client->list_objects({.bucket = "dummy", - .prefix = "AzureObjStorageClientTest/delete_objects_recursively",}, &files); + iter = ObjectListIterator(AzureObjStorageClientTest::obj_storage_client, {.bucket = "dummy", + .prefix = "AzureObjStorageClientTest/delete_objects_recursively"}); // clang-format on - EXPECT_EQ(response.status.code, ErrorCode::OK); + for (auto obj = iter.next(); obj.results_.has_value(); obj = iter.next()) { + EXPECT_TRUE(obj.resp.ok()); + files.push_back({.file_name = obj.results_->file_path, + .file_size = obj.results_->size, + .is_file = true}); + } + EXPECT_TRUE(iter.is_valid()); EXPECT_EQ(files.size(), 0); } #else diff --git a/be/test/io/fs/obj_storage_client_test.cpp b/be/test/io/fs/obj_storage_client_test.cpp new file mode 100644 index 00000000000000..713a1316810d4f --- /dev/null +++ b/be/test/io/fs/obj_storage_client_test.cpp @@ -0,0 +1,190 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "cpp/client/obj_storage_client.h" + +#include + +#include +#include +#include +#include + +namespace doris { +namespace { + +class FakeObjStorageBackend final : public ObjStorageBackend { +public: + ObjectStorageUploadResponse create_multipart_upload(const ObjectStoragePathOptions&) override { + ++calls; + return {}; + } + + ObjectStorageResponse put_object(const ObjectStoragePathOptions&, std::string_view) override { + ++calls; + return ObjectStorageResponse::OK(); + } + + ObjectStorageUploadResponse upload_part(const ObjectStoragePathOptions&, std::string_view, + int) override { + ++calls; + return {}; + } + + ObjectStorageResponse complete_multipart_upload( + const ObjectStoragePathOptions&, const std::vector&) override { + ++calls; + return ObjectStorageResponse::OK(); + } + + ObjectStorageHeadResponse head_object(const ObjectStoragePathOptions&) override { + ++calls; + return {}; + } + + ObjectStorageResponse get_object(const ObjectStoragePathOptions&, void*, size_t, size_t, + size_t* size_return) override { + ++calls; + *size_return = 4; + return ObjectStorageResponse::OK(); + } + + ObjectStorageListPage list_objects(const ObjectStoragePathOptions&, std::string_view) override { + ++calls; + return {}; + } + + ObjectStorageResponse delete_objects(const ObjectStoragePathOptions&, + std::vector) override { + ++calls; + return ObjectStorageResponse::OK(); + } + + ObjectStorageResponse delete_object(const ObjectStoragePathOptions&) override { + ++calls; + return ObjectStorageResponse::OK(); + } + + ObjStorageCapabilities capabilities() const override { return {.max_delete_batch = 2}; } + + std::string generate_presigned_url(const ObjectStoragePathOptions&, int64_t) override { + ++presigned_url_calls; + return "url"; + } + + ObjectStorageResponse get_life_cycle(const std::string&, int64_t*) override { + ++calls; + return ObjectStorageResponse::OK(); + } + + ObjectStorageResponse check_versioning(const std::string&) override { + ++calls; + return ObjectStorageResponse::OK(); + } + + ObjectStorageResponse abort_multipart_upload(const ObjectStoragePathOptions&, + const std::string&) override { + ++calls; + return ObjectStorageResponse::OK(); + } + + int calls = 0; + int presigned_url_calls = 0; +}; + +class RecordingRateLimitPolicy final : public ObjStorageRateLimitPolicy { +public: + struct Request { + ObjStorageRequestType type; + size_t estimated_bytes; + }; + + ObjStorageRateLimitToken acquire(ObjStorageRequestType type, + size_t estimated_bytes) const override { + requests.push_back({type, estimated_bytes}); + if (reject) { + return {.resp = ObjectStorageResponse::rate_limit("rejected by test policy")}; + } + return {.settle = [this](size_t actual_bytes) { settled_bytes.push_back(actual_bytes); }}; + } + + bool reject = false; + mutable std::vector requests; + mutable std::vector settled_bytes; +}; + +TEST(ObjStorageClientTest, AppliesAdmissionPolicyToEveryBackendRequest) { + auto backend = std::make_shared(); + auto policy = std::make_shared(); + ObjStorageClient client(backend, policy); + ObjectStoragePathOptions opts {.bucket = "bucket", .key = "key"}; + + EXPECT_TRUE(client.create_multipart_upload(opts).resp.ok()); + EXPECT_TRUE(client.put_object(opts, "abc").ok()); + EXPECT_TRUE(client.upload_part(opts, "part", 1).resp.ok()); + EXPECT_TRUE(client.complete_multipart_upload(opts, {}).ok()); + EXPECT_TRUE(client.head_object(opts).resp.ok()); + char buffer[10]; + size_t size_return = 0; + EXPECT_TRUE(client.get_object(opts, buffer, 0, sizeof(buffer), &size_return).ok()); + EXPECT_EQ(size_return, 4); + EXPECT_TRUE(client.list_objects(opts).resp.ok()); + EXPECT_TRUE(client.delete_objects(opts, {"one", "two", "three"}).ok()); + EXPECT_TRUE(client.delete_object(opts).ok()); + int64_t expiration_days = 0; + EXPECT_TRUE(client.get_life_cycle("bucket", &expiration_days).ok()); + EXPECT_TRUE(client.check_versioning("bucket").ok()); + EXPECT_TRUE(client.abort_multipart_upload(opts, "upload-id").ok()); + EXPECT_EQ(client.generate_presigned_url(opts, 60), "url"); + + const std::vector expected = { + {ObjStorageRequestType::PUT, 0}, {ObjStorageRequestType::PUT, 3}, + {ObjStorageRequestType::PUT, 4}, {ObjStorageRequestType::PUT, 0}, + {ObjStorageRequestType::GET, 0}, {ObjStorageRequestType::GET, 10}, + {ObjStorageRequestType::GET, 0}, {ObjStorageRequestType::PUT, 0}, + {ObjStorageRequestType::PUT, 0}, {ObjStorageRequestType::PUT, 0}, + {ObjStorageRequestType::GET, 0}, {ObjStorageRequestType::GET, 0}, + {ObjStorageRequestType::PUT, 0}, + }; + ASSERT_EQ(policy->requests.size(), expected.size()); + for (size_t i = 0; i < expected.size(); ++i) { + EXPECT_EQ(policy->requests[i].type, expected[i].type) << "request index " << i; + EXPECT_EQ(policy->requests[i].estimated_bytes, expected[i].estimated_bytes) + << "request index " << i; + } + EXPECT_EQ(policy->settled_bytes, std::vector({4})); + EXPECT_EQ(backend->calls, 13); + EXPECT_EQ(backend->presigned_url_calls, 1); +} + +TEST(ObjStorageClientTest, RejectsBeforeDispatchingToBackend) { + auto backend = std::make_shared(); + auto policy = std::make_shared(); + policy->reject = true; + ObjStorageClient client(backend, policy); + ObjectStoragePathOptions opts {.bucket = "bucket", .key = "key"}; + + auto put_response = client.put_object(opts, "abc"); + EXPECT_EQ(put_response.status.code, static_cast(TStatusCode::LIMIT_REACH)); + auto head_response = client.head_object(opts); + EXPECT_EQ(head_response.resp.status.code, static_cast(TStatusCode::LIMIT_REACH)); + EXPECT_EQ(backend->calls, 0); + EXPECT_EQ(policy->requests.size(), 2); +} + +} // namespace +} // namespace doris diff --git a/be/test/io/fs/packed_file_concurrency_test.cpp b/be/test/io/fs/packed_file_concurrency_test.cpp index 14c41d53a7b442..e5ef12bcb0cd69 100644 --- a/be/test/io/fs/packed_file_concurrency_test.cpp +++ b/be/test/io/fs/packed_file_concurrency_test.cpp @@ -202,9 +202,9 @@ void reset_mock_s3_store() { store.objects.clear(); } -class MockObjStorageClient : public ObjStorageClient { +class MockObjStorageBackend : public ObjStorageBackend { public: - explicit MockObjStorageClient(MockS3Store* store) : _store(store) {} + explicit MockObjStorageBackend(MockS3Store* store) : _store(store) {} ObjectStorageUploadResponse create_multipart_upload( const ObjectStoragePathOptions& opts) override { @@ -302,20 +302,21 @@ class MockObjStorageClient : public ObjStorageClient { return ObjectStorageResponse::OK(); } - ObjectStorageResponse list_objects(const ObjectStoragePathOptions& opts, - std::vector* files) override { + ObjectStorageListPage list_objects(const ObjectStoragePathOptions& opts, + std::string_view /*continuation_token*/) override { std::lock_guard lock(_store->mutex); - std::string prefix = _store->make_key(opts.bucket, opts.prefix); + const auto& object_prefix = opts.prefix.empty() ? opts.key : opts.prefix; + std::string prefix = _store->make_key(opts.bucket, object_prefix); + ObjectStorageListPage page {.resp = ObjectStorageResponse::OK()}; for (const auto& [key, data] : _store->objects) { if (key.rfind(prefix, 0) == 0) { - FileInfo info; - info.file_name = key.substr(prefix.size()); - info.file_size = data.size(); - info.is_file = true; - files->push_back(std::move(info)); + page.objects.emplace_back(ObjectMeta { + .file_path = key.substr(opts.bucket.size() + 1), + .size = static_cast(data.size()), + }); } } - return ObjectStorageResponse::OK(); + return page; } ObjectStorageResponse delete_objects(const ObjectStoragePathOptions& opts, @@ -333,30 +334,17 @@ class MockObjStorageClient : public ObjStorageClient { return ObjectStorageResponse::OK(); } - ObjectStorageResponse delete_objects_recursively( - const ObjectStoragePathOptions& opts) override { - std::lock_guard lock(_store->mutex); - std::string prefix = _store->make_key(opts.bucket, opts.prefix); - for (auto it = _store->objects.begin(); it != _store->objects.end();) { - if (it->first.rfind(prefix, 0) == 0) { - it = _store->objects.erase(it); - } else { - ++it; - } - } - return ObjectStorageResponse::OK(); - } - std::string generate_presigned_url(const ObjectStoragePathOptions& opts, - int64_t /*expiration_secs*/, - const S3ClientConf& /*conf*/) override { + int64_t /*expiration_secs*/) override { return fmt::format("mock://{}/{}", opts.bucket, opts.key); } private: static ObjectStorageResponse make_error(std::string msg, int http_code = 500) { ObjectStorageResponse resp; - resp.status.code = static_cast(ErrorCode::INTERNAL_ERROR); + resp.status.code = http_code == static_cast(Aws::Http::HttpResponseCode::NOT_FOUND) + ? ObjectStorageStatus::NOT_FOUND + : static_cast(ErrorCode::INTERNAL_ERROR); resp.status.msg = std::move(msg); resp.http_code = http_code; return resp; @@ -365,11 +353,11 @@ class MockObjStorageClient : public ObjStorageClient { MockS3Store* _store; }; -std::shared_ptr g_mock_obj_client; +std::shared_ptr g_mock_obj_backend; void install_mock_environment() { - g_mock_obj_client = std::make_shared(&mock_s3_store()); - auto client = g_mock_obj_client; + g_mock_obj_backend = std::make_shared(&mock_s3_store()); + auto client = std::make_shared(g_mock_obj_backend); S3ClientFactory::instance().set_client_creator_for_test( [client](const S3ClientConf&) { return client; }); @@ -384,7 +372,7 @@ void install_mock_environment() { void remove_mock_environment() { S3ClientFactory::instance().clear_client_creator_for_test(); - g_mock_obj_client.reset(); + g_mock_obj_backend.reset(); auto* sp = SyncPoint::get_instance(); sp->clear_call_back("PackedFileManager::update_meta_service"); diff --git a/be/test/io/fs/rate_limited_obj_storage_client_test.cpp b/be/test/io/fs/rate_limited_obj_storage_client_test.cpp deleted file mode 100644 index 657b139c0a4fcd..00000000000000 --- a/be/test/io/fs/rate_limited_obj_storage_client_test.cpp +++ /dev/null @@ -1,501 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "io/fs/rate_limited_obj_storage_client.h" - -#include - -#include "common/config.h" -#include "util/s3_rate_limiter_manager.h" -#include "util/s3_util.h" - -namespace doris { - -extern bvar::Adder s3_get_bytes_rate_limit_rejected_count; -extern bvar::Adder s3_put_bytes_rate_limit_rejected_count; - -} // namespace doris - -namespace doris::io { -namespace { - -constexpr size_t kNoThrottleBytesPerSecond = 1ULL << 40; - -// Provider-free fake: counts calls and reports a configurable read size. -class FakeObjStorageClient : public ObjStorageClient { -public: - ObjectStorageUploadResponse create_multipart_upload( - const ObjectStoragePathOptions& opts) override { - ++calls; - ++create_multipart_upload_calls; - create_multipart_upload_provider_calls += - create_multipart_upload_provider_calls_per_logical_call; - return {}; - } - ObjectStorageResponse put_object(const ObjectStoragePathOptions& opts, - std::string_view stream) override { - ++calls; - return ObjectStorageResponse::OK(); - } - ObjectStorageUploadResponse upload_part(const ObjectStoragePathOptions& opts, - std::string_view stream, int part_num) override { - ++calls; - return {}; - } - ObjectStorageResponse complete_multipart_upload( - const ObjectStoragePathOptions& opts, - const std::vector& completed_parts) override { - ++calls; - return ObjectStorageResponse::OK(); - } - ObjectStorageHeadResponse head_object(const ObjectStoragePathOptions& opts) override { - ++calls; - return {}; - } - ObjectStorageResponse get_object(const ObjectStoragePathOptions& opts, void* buffer, - size_t offset, size_t bytes_read, - size_t* size_return) override { - ++calls; - *size_return = actual_read_size; - return ObjectStorageResponse::OK(); - } - ObjectStorageResponse list_objects(const ObjectStoragePathOptions& opts, - std::vector* files) override { - ++calls; - return ObjectStorageResponse::OK(); - } - ObjectStorageResponse delete_objects(const ObjectStoragePathOptions& opts, - std::vector objs) override { - ++calls; - return ObjectStorageResponse::OK(); - } - ObjectStorageResponse delete_object(const ObjectStoragePathOptions& opts) override { - ++calls; - return ObjectStorageResponse::OK(); - } - ObjectStorageResponse delete_objects_recursively( - const ObjectStoragePathOptions& opts) override { - ++calls; - ++delete_objects_recursively_calls; - delete_objects_recursively_provider_calls += - delete_objects_recursively_provider_calls_per_logical_call; - return ObjectStorageResponse::OK(); - } - std::string generate_presigned_url(const ObjectStoragePathOptions& opts, - int64_t expiration_secs, const S3ClientConf& conf) override { - ++calls; - return "presigned"; - } - - int calls = 0; - size_t actual_read_size = 0; - int create_multipart_upload_calls = 0; - int create_multipart_upload_provider_calls = 0; - int create_multipart_upload_provider_calls_per_logical_call = 1; - int delete_objects_recursively_calls = 0; - int delete_objects_recursively_provider_calls = 0; - int delete_objects_recursively_provider_calls_per_logical_call = 1; -}; - -struct RateLimiterConfigGuard { - bool enable = config::enable_s3_rate_limiter; - - ~RateLimiterConfigGuard() { - config::enable_s3_rate_limiter = enable; - S3RateLimiterManager::instance().refresh(); - } -}; - -} // namespace - -TEST(RateLimitedObjStorageClientTest, forwards_all_calls_when_disabled) { - RateLimiterConfigGuard guard; - config::enable_s3_rate_limiter = false; - - auto fake = std::make_shared(); - RateLimitedObjStorageClient client(fake); - ObjectStoragePathOptions opts {.bucket = "b", .key = "k"}; - - size_t size_return = 0; - EXPECT_EQ(0, client.create_multipart_upload(opts).resp.status.code); - EXPECT_EQ(0, client.put_object(opts, "data").status.code); - EXPECT_EQ(0, client.upload_part(opts, "data", 1).resp.status.code); - EXPECT_EQ(0, client.complete_multipart_upload(opts, {}).status.code); - EXPECT_EQ(0, client.head_object(opts).resp.status.code); - EXPECT_EQ(0, client.get_object(opts, nullptr, 0, 4, &size_return).status.code); - std::vector files; - EXPECT_EQ(0, client.list_objects(opts, &files).status.code); - EXPECT_EQ(0, client.delete_objects(opts, {}).status.code); - EXPECT_EQ(0, client.delete_object(opts).status.code); - EXPECT_EQ(0, client.delete_objects_recursively(opts).status.code); - EXPECT_EQ("presigned", client.generate_presigned_url(opts, 60, S3ClientConf {})); - EXPECT_EQ(11, fake->calls); -} - -TEST(RateLimitedObjStorageClientTest, get_rejected_by_count_limit_does_not_reach_inner) { - RateLimiterConfigGuard guard; - config::enable_s3_rate_limiter = true; - auto& manager = S3RateLimiterManager::instance(); - manager.qps_limiter(S3RateLimitType::GET)->reset(0, 0, 1); - manager.qps_limiter(S3RateLimitType::PUT)->reset(0, 0, 0); - manager.bytes_limiter(S3RateLimitType::GET)->reset(0, 0, 0); - manager.bytes_limiter(S3RateLimitType::PUT)->reset(0, 0, 0); - - auto fake = std::make_shared(); - RateLimitedObjStorageClient client(fake); - ObjectStoragePathOptions opts {.bucket = "b", .key = "k"}; - - EXPECT_EQ(0, client.head_object(opts).resp.status.code); - EXPECT_EQ(1, fake->calls); - - auto resp = client.head_object(opts); - EXPECT_EQ(ErrorCode::EXCEEDED_LIMIT, resp.resp.status.code); - EXPECT_EQ(0, resp.resp.http_code); - EXPECT_NE(std::string::npos, resp.resp.status.msg.find("exceeds QPS limit")); - EXPECT_EQ(1, fake->calls); // rejected before reaching the provider - - // PUT uses an independent bucket and is unaffected. - EXPECT_EQ(0, client.put_object(opts, "data").status.code); - EXPECT_EQ(2, fake->calls); -} - -TEST(RateLimitedObjStorageClientTest, put_rejected_by_count_limit_does_not_reach_inner) { - RateLimiterConfigGuard guard; - config::enable_s3_rate_limiter = true; - auto& manager = S3RateLimiterManager::instance(); - manager.qps_limiter(S3RateLimitType::GET)->reset(0, 0, 0); - manager.qps_limiter(S3RateLimitType::PUT)->reset(0, 0, 1); - manager.bytes_limiter(S3RateLimitType::GET)->reset(0, 0, 0); - manager.bytes_limiter(S3RateLimitType::PUT)->reset(0, 0, 0); - - auto fake = std::make_shared(); - RateLimitedObjStorageClient client(fake); - ObjectStoragePathOptions opts {.bucket = "b", .key = "k"}; - - EXPECT_EQ(0, client.put_object(opts, "data").status.code); - EXPECT_EQ(1, fake->calls); - - auto resp = client.put_object(opts, "data"); - EXPECT_EQ(ErrorCode::EXCEEDED_LIMIT, resp.status.code); - EXPECT_EQ(0, resp.http_code); - EXPECT_NE(std::string::npos, resp.status.msg.find("exceeds QPS limit")); - EXPECT_EQ(1, fake->calls); // rejected before reaching the provider - - // GET uses an independent bucket and is unaffected. - EXPECT_EQ(0, client.head_object(opts).resp.status.code); - EXPECT_EQ(2, fake->calls); -} - -TEST(RateLimitedObjStorageClientTest, head_and_list_map_to_get_qps_without_bytes) { - RateLimiterConfigGuard guard; - config::enable_s3_rate_limiter = true; - auto& manager = S3RateLimiterManager::instance(); - auto* get_bytes = manager.bytes_limiter(S3RateLimitType::GET); - manager.qps_limiter(S3RateLimitType::GET) - ->reset(kNoThrottleBytesPerSecond, kNoThrottleBytesPerSecond, 2); - get_bytes->reset(kNoThrottleBytesPerSecond, kNoThrottleBytesPerSecond, 1); - - auto fake = std::make_shared(); - RateLimitedObjStorageClient client(fake); - ObjectStoragePathOptions opts {.bucket = "b", .key = "k", .prefix = "p"}; - - EXPECT_EQ(0, client.head_object(opts).resp.status.code); - std::vector files; - EXPECT_EQ(0, client.list_objects(opts, &files).status.code); - - auto rejected = client.head_object(opts); - EXPECT_EQ(ErrorCode::EXCEEDED_LIMIT, rejected.resp.status.code); - EXPECT_EQ(0, rejected.resp.http_code); - EXPECT_NE(std::string::npos, rejected.resp.status.msg.find("exceeds QPS limit")); - EXPECT_EQ(2, fake->calls); - - // Neither HEAD nor LIST carries payload bytes. - EXPECT_EQ(0, get_bytes->add(1)); - EXPECT_EQ(-1, get_bytes->add(1)); -} - -TEST(RateLimitedObjStorageClientTest, get_object_maps_to_get_qps_and_get_bytes) { - RateLimiterConfigGuard guard; - config::enable_s3_rate_limiter = true; - auto& manager = S3RateLimiterManager::instance(); - auto* get_bytes = manager.bytes_limiter(S3RateLimitType::GET); - manager.qps_limiter(S3RateLimitType::GET) - ->reset(kNoThrottleBytesPerSecond, kNoThrottleBytesPerSecond, 1); - get_bytes->reset(kNoThrottleBytesPerSecond, kNoThrottleBytesPerSecond, 4); - - auto fake = std::make_shared(); - fake->actual_read_size = 4; - RateLimitedObjStorageClient client(fake); - ObjectStoragePathOptions opts {.bucket = "b", .key = "k"}; - - size_t size_return = 0; - EXPECT_EQ(0, client.get_object(opts, nullptr, 0, 4, &size_return).status.code); - EXPECT_EQ(4, size_return); - - auto rejected = client.get_object(opts, nullptr, 0, 4, &size_return); - EXPECT_EQ(ErrorCode::EXCEEDED_LIMIT, rejected.status.code); - EXPECT_EQ(0, rejected.http_code); - EXPECT_NE(std::string::npos, rejected.status.msg.find("exceeds QPS limit")); - EXPECT_EQ(1, fake->calls); - - // The admitted GET charged exactly its returned payload bytes. - EXPECT_EQ(-1, get_bytes->add(1)); -} - -TEST(RateLimitedObjStorageClientTest, get_object_settles_short_read) { - RateLimiterConfigGuard guard; - config::enable_s3_rate_limiter = true; - auto& manager = S3RateLimiterManager::instance(); - manager.qps_limiter(S3RateLimitType::GET)->reset(0, 0, 0); - auto* bytes = manager.bytes_limiter(S3RateLimitType::GET); - bytes->reset(kNoThrottleBytesPerSecond, kNoThrottleBytesPerSecond, 1000); - - auto fake = std::make_shared(); - fake->actual_read_size = 100; // short read: 600 requested, 100 returned - RateLimitedObjStorageClient client(fake); - ObjectStoragePathOptions opts {.bucket = "b", .key = "k"}; - - size_t size_return = 0; - EXPECT_EQ(0, client.get_object(opts, nullptr, 0, 600, &size_return).status.code); - EXPECT_EQ(100, size_return); - - // Only 100 bytes remain cumulatively charged, so exactly 900 more are admitted. - EXPECT_EQ(0, bytes->add(900)); - EXPECT_EQ(-1, bytes->add(1)); -} - -TEST(RateLimitedObjStorageClientTest, put_object_charges_payload_bytes) { - RateLimiterConfigGuard guard; - config::enable_s3_rate_limiter = true; - auto& manager = S3RateLimiterManager::instance(); - manager.qps_limiter(S3RateLimitType::PUT)->reset(0, 0, 0); - auto* bytes = manager.bytes_limiter(S3RateLimitType::PUT); - bytes->reset(kNoThrottleBytesPerSecond, kNoThrottleBytesPerSecond, 1000); - - auto fake = std::make_shared(); - RateLimitedObjStorageClient client(fake); - ObjectStoragePathOptions opts {.bucket = "b", .key = "k"}; - - std::string payload(600, 'x'); - EXPECT_EQ(0, client.put_object(opts, payload).status.code); - EXPECT_EQ(0, bytes->add(400)); // exactly the cumulative count remainder - EXPECT_EQ(-1, bytes->add(1)); -} - -TEST(RateLimitedObjStorageClientTest, put_object_maps_to_put_qps_and_put_bytes) { - RateLimiterConfigGuard guard; - config::enable_s3_rate_limiter = true; - auto& manager = S3RateLimiterManager::instance(); - auto* put_bytes = manager.bytes_limiter(S3RateLimitType::PUT); - manager.qps_limiter(S3RateLimitType::PUT) - ->reset(kNoThrottleBytesPerSecond, kNoThrottleBytesPerSecond, 1); - put_bytes->reset(kNoThrottleBytesPerSecond, kNoThrottleBytesPerSecond, 4); - - auto fake = std::make_shared(); - RateLimitedObjStorageClient client(fake); - ObjectStoragePathOptions opts {.bucket = "b", .key = "k"}; - - EXPECT_EQ(0, client.put_object(opts, "data").status.code); - auto rejected = client.put_object(opts, "data"); - EXPECT_EQ(ErrorCode::EXCEEDED_LIMIT, rejected.status.code); - EXPECT_EQ(0, rejected.http_code); - EXPECT_NE(std::string::npos, rejected.status.msg.find("exceeds QPS limit")); - EXPECT_EQ(1, fake->calls); - - EXPECT_EQ(-1, put_bytes->add(1)); -} - -TEST(RateLimitedObjStorageClientTest, upload_part_maps_to_put_qps_and_put_bytes) { - RateLimiterConfigGuard guard; - config::enable_s3_rate_limiter = true; - auto& manager = S3RateLimiterManager::instance(); - auto* put_bytes = manager.bytes_limiter(S3RateLimitType::PUT); - manager.qps_limiter(S3RateLimitType::PUT) - ->reset(kNoThrottleBytesPerSecond, kNoThrottleBytesPerSecond, 1); - put_bytes->reset(kNoThrottleBytesPerSecond, kNoThrottleBytesPerSecond, 4); - - auto fake = std::make_shared(); - RateLimitedObjStorageClient client(fake); - ObjectStoragePathOptions opts {.bucket = "b", .key = "k", .upload_id = "upload"}; - - EXPECT_EQ(0, client.upload_part(opts, "data", 1).resp.status.code); - auto rejected = client.upload_part(opts, "data", 2); - EXPECT_EQ(ErrorCode::EXCEEDED_LIMIT, rejected.resp.status.code); - EXPECT_EQ(0, rejected.resp.http_code); - EXPECT_NE(std::string::npos, rejected.resp.status.msg.find("exceeds QPS limit")); - EXPECT_EQ(1, fake->calls); - - EXPECT_EQ(-1, put_bytes->add(1)); -} - -TEST(RateLimitedObjStorageClientTest, multipart_control_apis_map_to_put_qps_without_bytes) { - RateLimiterConfigGuard guard; - config::enable_s3_rate_limiter = true; - auto& manager = S3RateLimiterManager::instance(); - auto* put_bytes = manager.bytes_limiter(S3RateLimitType::PUT); - manager.qps_limiter(S3RateLimitType::PUT) - ->reset(kNoThrottleBytesPerSecond, kNoThrottleBytesPerSecond, 2); - put_bytes->reset(kNoThrottleBytesPerSecond, kNoThrottleBytesPerSecond, 1); - - auto fake = std::make_shared(); - RateLimitedObjStorageClient client(fake); - ObjectStoragePathOptions opts {.bucket = "b", .key = "k", .upload_id = "upload"}; - - EXPECT_EQ(0, client.create_multipart_upload(opts).resp.status.code); - EXPECT_EQ(0, client.complete_multipart_upload(opts, {}).status.code); - - auto rejected = client.create_multipart_upload(opts); - EXPECT_EQ(ErrorCode::EXCEEDED_LIMIT, rejected.resp.status.code); - EXPECT_EQ(0, rejected.resp.http_code); - EXPECT_NE(std::string::npos, rejected.resp.status.msg.find("exceeds QPS limit")); - EXPECT_EQ(2, fake->calls); - - EXPECT_EQ(0, put_bytes->add(1)); - EXPECT_EQ(-1, put_bytes->add(1)); -} - -TEST(RateLimitedObjStorageClientTest, delete_apis_map_to_put_qps_without_bytes) { - RateLimiterConfigGuard guard; - config::enable_s3_rate_limiter = true; - auto& manager = S3RateLimiterManager::instance(); - auto* put_bytes = manager.bytes_limiter(S3RateLimitType::PUT); - manager.qps_limiter(S3RateLimitType::PUT) - ->reset(kNoThrottleBytesPerSecond, kNoThrottleBytesPerSecond, 3); - put_bytes->reset(kNoThrottleBytesPerSecond, kNoThrottleBytesPerSecond, 1); - - auto fake = std::make_shared(); - RateLimitedObjStorageClient client(fake); - ObjectStoragePathOptions opts {.bucket = "b", .key = "k", .prefix = "p"}; - - EXPECT_EQ(0, client.delete_object(opts).status.code); - EXPECT_EQ(0, client.delete_objects(opts, {"a", "b"}).status.code); - EXPECT_EQ(0, client.delete_objects_recursively(opts).status.code); - - auto rejected = client.delete_object(opts); - EXPECT_EQ(ErrorCode::EXCEEDED_LIMIT, rejected.status.code); - EXPECT_EQ(0, rejected.http_code); - EXPECT_NE(std::string::npos, rejected.status.msg.find("exceeds QPS limit")); - EXPECT_EQ(3, fake->calls); - - EXPECT_EQ(0, put_bytes->add(1)); - EXPECT_EQ(-1, put_bytes->add(1)); -} - -TEST(RateLimitedObjStorageClientTest, bytes_rejections_have_distinct_text_and_metrics) { - RateLimiterConfigGuard guard; - config::enable_s3_rate_limiter = true; - auto& manager = S3RateLimiterManager::instance(); - manager.qps_limiter(S3RateLimitType::GET)->reset(0, 0, 0); - manager.qps_limiter(S3RateLimitType::PUT)->reset(0, 0, 0); - manager.bytes_limiter(S3RateLimitType::GET) - ->reset(kNoThrottleBytesPerSecond, kNoThrottleBytesPerSecond, 1); - manager.bytes_limiter(S3RateLimitType::PUT) - ->reset(kNoThrottleBytesPerSecond, kNoThrottleBytesPerSecond, 1); - - const int64_t get_rejected_before = s3_get_bytes_rate_limit_rejected_count.get_value(); - const int64_t put_rejected_before = s3_put_bytes_rate_limit_rejected_count.get_value(); - - auto fake = std::make_shared(); - RateLimitedObjStorageClient client(fake); - ObjectStoragePathOptions opts {.bucket = "b", .key = "k"}; - - size_t size_return = 0; - auto get_resp = client.get_object(opts, nullptr, 0, 2, &size_return); - EXPECT_EQ(ErrorCode::EXCEEDED_LIMIT, get_resp.status.code); - EXPECT_EQ(0, get_resp.http_code); - EXPECT_NE(std::string::npos, get_resp.status.msg.find("exceeds bytes limit")); - EXPECT_EQ(get_rejected_before + 1, s3_get_bytes_rate_limit_rejected_count.get_value()); - - auto put_resp = client.put_object(opts, "xx"); - EXPECT_EQ(ErrorCode::EXCEEDED_LIMIT, put_resp.status.code); - EXPECT_EQ(0, put_resp.http_code); - EXPECT_NE(std::string::npos, put_resp.status.msg.find("exceeds bytes limit")); - EXPECT_EQ(put_rejected_before + 1, s3_put_bytes_rate_limit_rejected_count.get_value()); - - EXPECT_EQ(0, fake->calls); -} - -TEST(RateLimitedObjStorageClientTest, recursive_delete_charges_one_put_qps) { - RateLimiterConfigGuard guard; - config::enable_s3_rate_limiter = true; - auto& manager = S3RateLimiterManager::instance(); - manager.qps_limiter(S3RateLimitType::GET)->reset(0, 0, 1); - manager.qps_limiter(S3RateLimitType::PUT)->reset(0, 0, 1); - manager.bytes_limiter(S3RateLimitType::PUT)->reset(0, 0, 0); - - auto fake = std::make_shared(); - fake->delete_objects_recursively_provider_calls_per_logical_call = 4; - RateLimitedObjStorageClient client(fake); - ObjectStoragePathOptions opts {.bucket = "b", .prefix = "p"}; - - // Exhaust GET first. Recursive delete still succeeds because the logical API is PUT. - EXPECT_EQ(0, client.head_object(opts).resp.status.code); - EXPECT_EQ(0, client.delete_objects_recursively(opts).status.code); - EXPECT_EQ(1, fake->delete_objects_recursively_calls); - EXPECT_EQ(4, fake->delete_objects_recursively_provider_calls); - - auto resp = client.delete_objects_recursively(opts); - EXPECT_EQ(ErrorCode::EXCEEDED_LIMIT, resp.status.code); - EXPECT_EQ(0, resp.http_code); - EXPECT_EQ(1, fake->delete_objects_recursively_calls); - EXPECT_EQ(4, fake->delete_objects_recursively_provider_calls); -} - -TEST(RateLimitedObjStorageClientTest, azure_noop_multipart_create_charges_one_put_qps) { - RateLimiterConfigGuard guard; - config::enable_s3_rate_limiter = true; - auto& manager = S3RateLimiterManager::instance(); - manager.qps_limiter(S3RateLimitType::PUT)->reset(0, 0, 1); - manager.bytes_limiter(S3RateLimitType::PUT)->reset(0, 0, 0); - - auto fake = std::make_shared(); - // Azure implements create_multipart_upload as a provider-side no-op. - fake->create_multipart_upload_provider_calls_per_logical_call = 0; - RateLimitedObjStorageClient client(fake); - ObjectStoragePathOptions opts {.bucket = "b", .key = "k"}; - - EXPECT_EQ(0, client.create_multipart_upload(opts).resp.status.code); - EXPECT_EQ(1, fake->create_multipart_upload_calls); - EXPECT_EQ(0, fake->create_multipart_upload_provider_calls); - - auto resp = client.create_multipart_upload(opts); - EXPECT_EQ(ErrorCode::EXCEEDED_LIMIT, resp.resp.status.code); - EXPECT_EQ(0, resp.resp.http_code); - EXPECT_EQ(1, fake->create_multipart_upload_calls); - EXPECT_EQ(0, fake->create_multipart_upload_provider_calls); -} - -TEST(RateLimitedObjStorageClientTest, presigned_url_bypasses_rate_limiters) { - RateLimiterConfigGuard guard; - config::enable_s3_rate_limiter = true; - auto& manager = S3RateLimiterManager::instance(); - auto* get_qps = manager.qps_limiter(S3RateLimitType::GET); - auto* put_qps = manager.qps_limiter(S3RateLimitType::PUT); - get_qps->reset(0, 0, 1); - put_qps->reset(0, 0, 1); - EXPECT_EQ(0, get_qps->add(1)); - EXPECT_EQ(0, put_qps->add(1)); - - auto fake = std::make_shared(); - RateLimitedObjStorageClient client(fake); - ObjectStoragePathOptions opts {.bucket = "b", .key = "k"}; - - EXPECT_EQ("presigned", client.generate_presigned_url(opts, 60, S3ClientConf {})); - EXPECT_EQ(1, fake->calls); -} - -} // namespace doris::io diff --git a/be/test/io/fs/s3_file_writer_test.cpp b/be/test/io/fs/s3_file_writer_test.cpp index 3937d6e38561fe..392367fd3ca2c3 100644 --- a/be/test/io/fs/s3_file_writer_test.cpp +++ b/be/test/io/fs/s3_file_writer_test.cpp @@ -48,6 +48,7 @@ #include "common/config.h" #include "common/status.h" +#include "cpp/client/s3_obj_storage_backend.h" #include "cpp/sync_point.h" #include "io/fs/file_reader.h" #include "io/fs/file_system.h" @@ -55,7 +56,6 @@ #include "io/fs/local_file_system.h" #include "io/fs/s3_file_bufferpool.h" #include "io/fs/s3_file_system.h" -#include "io/fs/s3_obj_storage_client.h" #include "io/io_common.h" #include "runtime/exec_env.h" #include "storage/index/index_file_writer.h" @@ -269,7 +269,7 @@ static auto test_mock_callbacks = std::array { pair->first = mock_client->head_object(req); }}, MockCallback {"s3_client_factory::create", [](auto&& outcome) { - auto pair = try_any_cast_ret>( + auto pair = try_any_cast_ret>( outcome); pair->second = true; }}}; @@ -1048,8 +1048,8 @@ TEST_F(S3FileWriterTest, multi_part_complete_error_3) { sp->set_call_back("S3FileWriter::_complete:3", [](auto&& outcome) { auto pair = try_any_cast_ret(outcome); pair->second = true; - pair->first = io::ObjectStorageResponse { - .status = convert_to_obj_response(Status::IOError("inject error"))}; + pair->first = + io::ObjectStorageResponse {.status = {TStatusCode::INTERNAL_ERROR, "inject error"}}; }); Defer defer {[&]() { sp->clear_call_back("S3FileWriter::_complete:3"); }}; auto client = s3_fs->client_holder(); @@ -1089,10 +1089,10 @@ namespace io { /** * This class is for boundary test */ -class SimpleMockObjStorageClient : public io::ObjStorageClient { +class SimpleMockObjStorageBackend : public io::ObjStorageBackend { public: - SimpleMockObjStorageClient() = default; - ~SimpleMockObjStorageClient() override = default; + SimpleMockObjStorageBackend() = default; + ~SimpleMockObjStorageBackend() override = default; ObjectStorageResponse default_response {ObjectStorageResponse::OK()}; ObjectStorageUploadResponse default_upload_response {.resp = ObjectStorageResponse::OK(), @@ -1173,14 +1173,11 @@ class SimpleMockObjStorageClient : public io::ObjStorageClient { return default_response; } - ObjectStorageResponse list_objects(const ObjectStoragePathOptions& opts, - std::vector* files) override { + ObjectStorageListPage list_objects(const ObjectStoragePathOptions& opts, + std::string_view /*continuation_token*/) override { std::lock_guard lock(_mutex); last_opts = opts; - if (files) { - *files = default_file_list; - } - return default_response; + return {.resp = default_response}; } ObjectStorageResponse delete_objects(const ObjectStoragePathOptions& opts, @@ -1197,15 +1194,8 @@ class SimpleMockObjStorageClient : public io::ObjStorageClient { return default_response; } - ObjectStorageResponse delete_objects_recursively( - const ObjectStoragePathOptions& opts) override { - std::lock_guard lock(_mutex); - last_opts = opts; - return default_response; - } - std::string generate_presigned_url(const ObjectStoragePathOptions& opts, - int64_t expiration_secs, const S3ClientConf& conf) override { + int64_t expiration_secs) override { std::lock_guard lock(_mutex); last_opts = opts; last_expiration_secs = expiration_secs; @@ -1293,7 +1283,7 @@ class SimpleMockObjStorageClient : public io::ObjStorageClient { * Create a mock S3 client and a S3FileWriter. * @return A tuple containing the mock S3 client and the S3FileWriter. */ -std::tuple, std::shared_ptr> +std::tuple, std::shared_ptr> create_s3_client(const std::string& path) { doris::io::FileWriterOptions opts; io::FileWriterPtr file_writer; @@ -1301,8 +1291,8 @@ create_s3_client(const std::string& path) { EXPECT_TRUE(st.ok()) << st; std::shared_ptr s3_file_writer(static_cast(file_writer.release())); auto holder = std::make_shared(S3ClientConf {}); - auto mock_client = std::make_shared(); - holder->_client = mock_client; + auto mock_client = std::make_shared(); + holder->_client = std::make_shared(mock_client); s3_file_writer->_obj_client = holder; return {mock_client, s3_file_writer}; } @@ -1399,7 +1389,7 @@ TEST_F(S3FileWriterTest, write_buffer_boundary) { sp->clear_all_call_backs(); // s3_file_writer is the interface to write to s3 - // mock_client is a SimpleMockObjStorageClient for testing, it holds the data in memory + // mock_client is a SimpleMockObjStorageBackend for testing, it holds the data in memory // we check the data in mock_client to make sure s3_file_writer is working as expected auto test = [](char magic_char, size_t data_size, const std::string& filename) { std::string content = generate_test_string(magic_char, data_size); @@ -1504,8 +1494,8 @@ TEST_F(S3FileWriterTest, test_empty_file) { auto st = s3_fs->create_file("test_empty_file.idx", &file_writer, &opts); EXPECT_TRUE(st.ok()) << st; auto holder = std::make_shared(S3ClientConf {}); - auto mock_client = std::make_shared(); - holder->_client = mock_client; + auto mock_client = std::make_shared(); + holder->_client = std::make_shared(mock_client); dynamic_cast(file_writer.get())->_obj_client = holder; auto fs = io::global_local_filesystem(); std::string index_path = "/tmp/empty_index_file_test"; diff --git a/be/test/io/fs/s3_obj_storage_client_role_test.cpp b/be/test/io/fs/s3_obj_storage_client_role_test.cpp index 4646de690df047..15d46c59bc8a57 100644 --- a/be/test/io/fs/s3_obj_storage_client_role_test.cpp +++ b/be/test/io/fs/s3_obj_storage_client_role_test.cpp @@ -17,14 +17,15 @@ #include -#include "io/fs/obj_storage_client.h" +#include "cpp/client/obj_storage_client.h" +#include "io/fs/file_system.h" #include "util/s3_util.h" namespace doris { class S3ObjStorageClientRoleTest : public testing::Test { protected: - static std::shared_ptr obj_storage_client; + static std::shared_ptr obj_storage_client; static std::string bucket; static std::string prefix; @@ -48,20 +49,20 @@ class S3ObjStorageClientRoleTest : public testing::Test { S3ObjStorageClientRoleTest::prefix = std::getenv("AWS_PREFIX"); } - S3ObjStorageClientRoleTest::obj_storage_client = S3ClientFactory::instance().create( + auto client_result = S3ClientFactory::instance().create( {.endpoint = endpoint, .region = region, .ak = "", .sk = "", .token = "", .bucket = bucket, - .provider = io::ObjStorageType::AWS, + .provider = ObjStorageType::AWS, .use_virtual_addressing = false, .cred_provider_type = CredProviderType::InstanceProfile, .role_arn = role_arn, .external_id = external_id}); - - ASSERT_TRUE(S3ObjStorageClientRoleTest::obj_storage_client != nullptr); + ASSERT_TRUE(client_result.has_value()) << client_result.error(); + S3ObjStorageClientRoleTest::obj_storage_client = std::move(client_result).value(); } void SetUp() override { @@ -71,7 +72,7 @@ class S3ObjStorageClientRoleTest : public testing::Test { } }; -std::shared_ptr S3ObjStorageClientRoleTest::obj_storage_client = nullptr; +std::shared_ptr S3ObjStorageClientRoleTest::obj_storage_client = nullptr; std::string S3ObjStorageClientRoleTest::bucket; std::string S3ObjStorageClientRoleTest::prefix; @@ -85,10 +86,16 @@ TEST_F(S3ObjStorageClientRoleTest, put_list_delete_object) { std::vector files; // clang-format off - response = S3ObjStorageClientRoleTest::obj_storage_client->list_objects({.bucket = bucket, - .prefix = prefix + "S3ObjStorageClientRoleTest/put_list_delete_object",}, &files); + ObjectListIterator iter(S3ObjStorageClientRoleTest::obj_storage_client, {.bucket = bucket, + .key = prefix + "S3ObjStorageClientRoleTest/put_list_delete_object"}); // clang-format on - EXPECT_EQ(response.status.code, ErrorCode::OK); + for (auto obj = iter.next(); obj.results_.has_value(); obj = iter.next()) { + EXPECT_EQ(obj.resp.status.code, ErrorCode::OK); + files.push_back({.file_name = obj.results_->file_path, + .file_size = obj.results_->size, + .is_file = true}); + } + EXPECT_TRUE(iter.is_valid()); EXPECT_EQ(files.size(), 1); files.clear(); @@ -98,10 +105,16 @@ TEST_F(S3ObjStorageClientRoleTest, put_list_delete_object) { EXPECT_EQ(response.status.code, ErrorCode::OK); // clang-format off - response = S3ObjStorageClientRoleTest::obj_storage_client->list_objects({.bucket = bucket, - .prefix = prefix + "S3ObjStorageClientRoleTest/put_list_delete_object",}, &files); + iter = ObjectListIterator(S3ObjStorageClientRoleTest::obj_storage_client, {.bucket = bucket, + .key = prefix + "S3ObjStorageClientRoleTest/put_list_delete_object"}); // clang-format on - EXPECT_EQ(response.status.code, ErrorCode::OK); + for (auto obj = iter.next(); obj.results_.has_value(); obj = iter.next()) { + EXPECT_EQ(obj.resp.status.code, ErrorCode::OK); + files.push_back({.file_name = obj.results_->file_path, + .file_size = obj.results_->size, + .is_file = true}); + } + EXPECT_TRUE(iter.is_valid()); EXPECT_EQ(files.size(), 0); } @@ -120,23 +133,35 @@ TEST_F(S3ObjStorageClientRoleTest, delete_objects_recursively) { std::vector files; // clang-format off - auto response = S3ObjStorageClientRoleTest::obj_storage_client->list_objects({.bucket = bucket, - .prefix = prefix + "S3ObjStorageClientRoleTest/delete_objects_recursively",}, &files); + ObjectListIterator iter(S3ObjStorageClientRoleTest::obj_storage_client, {.bucket = bucket, + .key = prefix + "S3ObjStorageClientRoleTest/delete_objects_recursively"}); // clang-format on - EXPECT_EQ(response.status.code, ErrorCode::OK); + for (auto obj = iter.next(); obj.results_.has_value(); obj = iter.next()) { + EXPECT_EQ(obj.resp.status.code, ErrorCode::OK); + files.push_back({.file_name = obj.results_->file_path, + .file_size = obj.results_->size, + .is_file = true}); + } + EXPECT_TRUE(iter.is_valid()); EXPECT_EQ(files.size(), 22); files.clear(); - response = S3ObjStorageClientRoleTest::obj_storage_client->delete_objects_recursively( + auto response = S3ObjStorageClientRoleTest::obj_storage_client->delete_objects_recursively( {.bucket = bucket, .prefix = prefix + "S3ObjStorageClientRoleTest/delete_objects_recursively"}); EXPECT_EQ(response.status.code, ErrorCode::OK); // clang-format off - response = S3ObjStorageClientRoleTest::obj_storage_client->list_objects({.bucket = bucket, - .prefix = prefix + "S3ObjStorageClientRoleTest/delete_objects_recursively",}, &files); + iter = ObjectListIterator(S3ObjStorageClientRoleTest::obj_storage_client, {.bucket = bucket, + .key = prefix + "S3ObjStorageClientRoleTest/delete_objects_recursively"}); // clang-format on - EXPECT_EQ(response.status.code, ErrorCode::OK); + for (auto obj = iter.next(); obj.results_.has_value(); obj = iter.next()) { + EXPECT_EQ(obj.resp.status.code, ErrorCode::OK); + files.push_back({.file_name = obj.results_->file_path, + .file_size = obj.results_->size, + .is_file = true}); + } + EXPECT_TRUE(iter.is_valid()); EXPECT_EQ(files.size(), 0); } @@ -151,7 +176,7 @@ TEST_F(S3ObjStorageClientRoleTest, multipart_upload) { std::string body = "S3ObjStorageClientRoleTest::multipart_upload"; body.resize(5 * 1024 * 1024); - std::vector completed_parts; + std::vector completed_parts; response = S3ObjStorageClientRoleTest::obj_storage_client->upload_part( {.bucket = bucket, @@ -160,8 +185,9 @@ TEST_F(S3ObjStorageClientRoleTest, multipart_upload) { body, 1); EXPECT_EQ(response.resp.status.code, ErrorCode::OK); - doris::io::ObjectCompleteMultiPart completed_part { - 1, response.etag.has_value() ? std::move(response.etag.value()) : ""}; + ObjectCompleteMultiPart completed_part { + .part_num = 1, + .etag = response.etag.has_value() ? std::move(response.etag.value()) : ""}; completed_parts.emplace_back(std::move(completed_part)); @@ -172,8 +198,9 @@ TEST_F(S3ObjStorageClientRoleTest, multipart_upload) { body, 2); EXPECT_EQ(response.resp.status.code, ErrorCode::OK); - doris::io::ObjectCompleteMultiPart completed_part2 { - 2, response.etag.has_value() ? std::move(response.etag.value()) : ""}; + ObjectCompleteMultiPart completed_part2 { + .part_num = 2, + .etag = response.etag.has_value() ? std::move(response.etag.value()) : ""}; completed_parts.emplace_back(std::move(completed_part2)); auto response2 = S3ObjStorageClientRoleTest::obj_storage_client->complete_multipart_upload( @@ -185,4 +212,4 @@ TEST_F(S3ObjStorageClientRoleTest, multipart_upload) { EXPECT_EQ(response2.status.code, ErrorCode::OK); } -} // namespace doris \ No newline at end of file +} // namespace doris diff --git a/be/test/io/fs/s3_obj_storage_client_test.cpp b/be/test/io/fs/s3_obj_storage_client_test.cpp index 97b44b8fa0cba4..976789578122f9 100644 --- a/be/test/io/fs/s3_obj_storage_client_test.cpp +++ b/be/test/io/fs/s3_obj_storage_client_test.cpp @@ -17,14 +17,15 @@ #include -#include "io/fs/obj_storage_client.h" +#include "cpp/client/obj_storage_client.h" +#include "io/fs/file_system.h" #include "util/s3_util.h" namespace doris { class S3ObjStorageClientTest : public testing::Test { protected: - static std::shared_ptr obj_storage_client; + static std::shared_ptr obj_storage_client; static std::string bucket; static void SetUpTestSuite() { @@ -39,20 +40,20 @@ class S3ObjStorageClientTest : public testing::Test { S3ObjStorageClientTest::bucket = std::getenv("AWS_BUCKET"); - S3ObjStorageClientTest::obj_storage_client = S3ClientFactory::instance().create({ + auto client_result = S3ClientFactory::instance().create({ .endpoint = endpoint, .region = "dummy-region", .ak = access_key, .sk = secret_key, .token = "", .bucket = bucket, - .provider = io::ObjStorageType::AWS, + .provider = ObjStorageType::AWS, .use_virtual_addressing = false, .role_arn = "", .external_id = "", }); - - ASSERT_TRUE(S3ObjStorageClientTest::obj_storage_client != nullptr); + ASSERT_TRUE(client_result.has_value()) << client_result.error(); + S3ObjStorageClientTest::obj_storage_client = std::move(client_result).value(); } void SetUp() override { @@ -62,7 +63,7 @@ class S3ObjStorageClientTest : public testing::Test { } }; -std::shared_ptr S3ObjStorageClientTest::obj_storage_client = nullptr; +std::shared_ptr S3ObjStorageClientTest::obj_storage_client = nullptr; std::string S3ObjStorageClientTest::bucket; TEST_F(S3ObjStorageClientTest, put_list_delete_object) { @@ -75,10 +76,16 @@ TEST_F(S3ObjStorageClientTest, put_list_delete_object) { std::vector files; // clang-format off - response = S3ObjStorageClientTest::obj_storage_client->list_objects({.bucket = bucket, - .prefix = "S3ObjStorageClientTest/put_list_delete_object",}, &files); + ObjectListIterator iter(S3ObjStorageClientTest::obj_storage_client, {.bucket = bucket, + .key = "S3ObjStorageClientTest/put_list_delete_object"}); // clang-format on - EXPECT_EQ(response.status.code, ErrorCode::OK); + for (auto obj = iter.next(); obj.results_.has_value(); obj = iter.next()) { + EXPECT_EQ(obj.resp.status.code, ErrorCode::OK); + files.push_back({.file_name = obj.results_->file_path, + .file_size = obj.results_->size, + .is_file = true}); + } + EXPECT_TRUE(iter.is_valid()); EXPECT_EQ(files.size(), 1); files.clear(); @@ -87,10 +94,16 @@ TEST_F(S3ObjStorageClientTest, put_list_delete_object) { EXPECT_EQ(response.status.code, ErrorCode::OK); // clang-format off - response = S3ObjStorageClientTest::obj_storage_client->list_objects({.bucket = bucket, - .prefix = "S3ObjStorageClientTest/put_list_delete_object",}, &files); + iter = ObjectListIterator(S3ObjStorageClientTest::obj_storage_client, {.bucket = bucket, + .key = "S3ObjStorageClientTest/put_list_delete_object"}); // clang-format on - EXPECT_EQ(response.status.code, ErrorCode::OK); + for (auto obj = iter.next(); obj.results_.has_value(); obj = iter.next()) { + EXPECT_EQ(obj.resp.status.code, ErrorCode::OK); + files.push_back({.file_name = obj.results_->file_path, + .file_size = obj.results_->size, + .is_file = true}); + } + EXPECT_TRUE(iter.is_valid()); EXPECT_EQ(files.size(), 0); } @@ -108,22 +121,34 @@ TEST_F(S3ObjStorageClientTest, delete_objects_recursively) { std::vector files; // clang-format off - auto response = S3ObjStorageClientTest::obj_storage_client->list_objects({.bucket = bucket, - .prefix = "S3ObjStorageClientTest/delete_objects_recursively",}, &files); + ObjectListIterator iter(S3ObjStorageClientTest::obj_storage_client, {.bucket = bucket, + .key = "S3ObjStorageClientTest/delete_objects_recursively",}); // clang-format on - EXPECT_EQ(response.status.code, ErrorCode::OK); + for (auto obj = iter.next(); obj.results_.has_value(); obj = iter.next()) { + EXPECT_EQ(obj.resp.status.code, ErrorCode::OK); + files.push_back({.file_name = obj.results_->file_path, + .file_size = obj.results_->size, + .is_file = true}); + } + EXPECT_TRUE(iter.is_valid()); EXPECT_EQ(files.size(), 22); files.clear(); - response = S3ObjStorageClientTest::obj_storage_client->delete_objects_recursively( + auto response = S3ObjStorageClientTest::obj_storage_client->delete_objects_recursively( {.bucket = bucket, .prefix = "S3ObjStorageClientTest/delete_objects_recursively"}); EXPECT_EQ(response.status.code, ErrorCode::OK); // clang-format off - response = S3ObjStorageClientTest::obj_storage_client->list_objects({.bucket = bucket, - .prefix = "S3ObjStorageClientTest/delete_objects_recursively",}, &files); + iter = ObjectListIterator(S3ObjStorageClientTest::obj_storage_client, {.bucket = bucket, + .key = "S3ObjStorageClientTest/delete_objects_recursively"}); // clang-format on - EXPECT_EQ(response.status.code, ErrorCode::OK); + for (auto obj = iter.next(); obj.results_.has_value(); obj = iter.next()) { + EXPECT_EQ(obj.resp.status.code, ErrorCode::OK); + files.push_back({.file_name = obj.results_->file_path, + .file_size = obj.results_->size, + .is_file = true}); + } + EXPECT_TRUE(iter.is_valid()); EXPECT_EQ(files.size(), 0); } @@ -138,7 +163,7 @@ TEST_F(S3ObjStorageClientTest, multipart_upload) { std::string body = "S3ObjStorageClientTest::multipart_upload"; body.resize(5 * 1024 * 1024); - std::vector completed_parts; + std::vector completed_parts; response = S3ObjStorageClientTest::obj_storage_client->upload_part( {.bucket = bucket, @@ -147,8 +172,9 @@ TEST_F(S3ObjStorageClientTest, multipart_upload) { body, 1); EXPECT_EQ(response.resp.status.code, ErrorCode::OK); - doris::io::ObjectCompleteMultiPart completed_part { - 1, response.etag.has_value() ? std::move(response.etag.value()) : ""}; + ObjectCompleteMultiPart completed_part { + .part_num = 1, + .etag = response.etag.has_value() ? std::move(response.etag.value()) : ""}; completed_parts.emplace_back(std::move(completed_part)); @@ -159,8 +185,9 @@ TEST_F(S3ObjStorageClientTest, multipart_upload) { body, 2); EXPECT_EQ(response.resp.status.code, ErrorCode::OK); - doris::io::ObjectCompleteMultiPart completed_part2 { - 2, response.etag.has_value() ? std::move(response.etag.value()) : ""}; + ObjectCompleteMultiPart completed_part2 { + .part_num = 2, + .etag = response.etag.has_value() ? std::move(response.etag.value()) : ""}; completed_parts.emplace_back(std::move(completed_part2)); auto response2 = S3ObjStorageClientTest::obj_storage_client->complete_multipart_upload( @@ -172,4 +199,4 @@ TEST_F(S3ObjStorageClientTest, multipart_upload) { EXPECT_EQ(response2.status.code, ErrorCode::OK); } -} // namespace doris \ No newline at end of file +} // namespace doris diff --git a/be/test/io/fs/s3_obj_stroage_client_mock_test.cpp b/be/test/io/fs/s3_obj_stroage_client_mock_test.cpp index 1a00bcb8642f0c..ca3d5a9989dff1 100644 --- a/be/test/io/fs/s3_obj_stroage_client_mock_test.cpp +++ b/be/test/io/fs/s3_obj_stroage_client_mock_test.cpp @@ -21,29 +21,16 @@ #include #include +#include "cpp/client/obj_storage_client.h" +#include "cpp/client/s3_obj_storage_backend.h" #include "gmock/gmock.h" -#include "io/fs/rate_limited_obj_storage_client.h" -#include "io/fs/s3_obj_storage_client.h" -#include "util/s3_rate_limiter_manager.h" +#include "io/fs/file_system.h" #include "util/s3_util.h" #include "util/string_util.h" using namespace Aws::S3::Model; namespace doris::io { -namespace { - -struct RateLimiterConfigGuard { - bool enable = config::enable_s3_rate_limiter; - - ~RateLimiterConfigGuard() { - config::enable_s3_rate_limiter = enable; - S3RateLimiterManager::instance().refresh(); - } -}; - -} // namespace - class MockS3Client : public Aws::S3::S3Client { public: MockS3Client() {}; @@ -52,6 +39,20 @@ class MockS3Client : public Aws::S3::S3Client { (const Aws::S3::Model::ListObjectsV2Request& request), (const, override)); }; +class CountingGetRateLimitPolicy final : public ObjStorageRateLimitPolicy { +public: + explicit CountingGetRateLimitPolicy(size_t* request_count) : request_count_(request_count) {} + + ObjStorageRateLimitToken acquire(ObjStorageRequestType type, size_t) const override { + EXPECT_EQ(type, ObjStorageRequestType::GET); + ++*request_count_; + return {}; + } + +private: + size_t* request_count_; +}; + class S3ObjStorageClientMockTest : public testing::Test { static void SetUpTestSuite() { S3ClientFactory::instance(); }; static void TearDownTestSuite() {}; @@ -63,10 +64,10 @@ class S3ObjStorageClientMockTest : public testing::Test { Aws::SDKOptions S3ObjStorageClientMockTest::options {}; TEST_F(S3ObjStorageClientMockTest, list_objects_compatibility) { - // If storage only supports ListObjectsV1, s3_obj_storage_client.list_objects + // If storage only supports ListObjectsV1, s3_obj_storage_backend.list_objects // should return an error. auto mock_s3_client = std::make_shared(); - S3ObjStorageClient s3_obj_storage_client(mock_s3_client); + S3ObjStorageBackend s3_obj_storage_backend(mock_s3_client); std::vector files; @@ -75,11 +76,11 @@ TEST_F(S3ObjStorageClientMockTest, list_objects_compatibility) { EXPECT_CALL(*mock_s3_client, ListObjectsV2(testing::_)) .WillOnce(testing::Return(ListObjectsV2Outcome(result))); - auto response = s3_obj_storage_client.list_objects( - {.bucket = "dummy-bucket", .prefix = "S3ObjStorageClientMockTest/list_objects_test"}, - &files); + auto page = s3_obj_storage_backend.list_objects( + {.bucket = "dummy-bucket", .key = "S3ObjStorageClientMockTest/list_objects_test"}, {}); - EXPECT_EQ(response.status.code, ErrorCode::INTERNAL_ERROR); + EXPECT_TRUE(page.objects.empty()); + EXPECT_EQ(page.resp.status.code, ErrorCode::INTERNAL_ERROR); files.clear(); } @@ -96,16 +97,13 @@ ListObjectsV2Result CreatePageResult(const std::string& nextToken, return result; } -TEST_F(S3ObjStorageClientMockTest, list_objects_pagination_charges_one_get_qps) { - RateLimiterConfigGuard guard; - config::enable_s3_rate_limiter = true; - auto& manager = S3RateLimiterManager::instance(); - manager.qps_limiter(S3RateLimitType::GET)->reset(0, 0, 1); - manager.bytes_limiter(S3RateLimitType::GET)->reset(0, 0, 0); - +TEST_F(S3ObjStorageClientMockTest, list_objects_with_pagination) { auto mock_s3_client = std::make_shared(); - auto s3_obj_storage_client = std::make_shared(mock_s3_client); - RateLimitedObjStorageClient rate_limited_client(s3_obj_storage_client); + size_t get_request_count = 0; + auto backend = std::make_shared(mock_s3_client); + ObjStorageClient obj_storage_client( + std::move(backend), std::make_shared(&get_request_count)); + std::string prefix = "S3ObjStorageClientMockTest/list_objects_with_pagination/"; std::vector> pages = { {"key1", "key2"}, // page1 @@ -113,6 +111,12 @@ TEST_F(S3ObjStorageClientMockTest, list_objects_pagination_charges_one_get_qps) {"key5"} // page3 }; + for (auto& page : pages) { + for (auto& key : page) { + key = prefix + key; + } + } + EXPECT_CALL(*mock_s3_client, ListObjectsV2(testing::_)) .WillOnce([&](const ListObjectsV2Request& req) { // page1：no ContinuationToken @@ -132,19 +136,25 @@ TEST_F(S3ObjStorageClientMockTest, list_objects_pagination_charges_one_get_qps) }); std::vector files; - const ObjectStoragePathOptions opts { - .bucket = "dummy-bucket", - .prefix = "S3ObjStorageClientMockTest/list_objects_with_pagination"}; - auto response = rate_limited_client.list_objects(opts, &files); + std::string continuation_token; + bool has_more = true; + while (has_more) { + auto page = obj_storage_client.list_objects( + {.bucket = "dummy-bucket", + .key = "S3ObjStorageClientMockTest/list_objects_with_pagination"}, + continuation_token); + EXPECT_EQ(page.resp.status.code, ErrorCode::OK); + for (const auto& object : page.objects) { + files.push_back( + {.file_name = object.file_path, .file_size = object.size, .is_file = true}); + } + continuation_token = std::move(page.continuation_token); + has_more = page.has_more; + } - EXPECT_EQ(response.status.code, ErrorCode::OK); EXPECT_EQ(files.size(), 5); - - // The first logical list used one GET token despite issuing three provider requests. - // A second logical list is rejected before it reaches the provider. - response = rate_limited_client.list_objects(opts, &files); - EXPECT_EQ(ErrorCode::EXCEEDED_LIMIT, response.status.code); - EXPECT_EQ(0, response.http_code); + EXPECT_EQ(get_request_count, pages.size()); + files.clear(); } TEST_F(S3ObjStorageClientMockTest, test_ca_cert) { diff --git a/be/test/io/s3_client_factory_test.cpp b/be/test/io/s3_client_factory_test.cpp index 53787e7150ec8f..a2c30fa7f608e2 100644 --- a/be/test/io/s3_client_factory_test.cpp +++ b/be/test/io/s3_client_factory_test.cpp @@ -19,6 +19,7 @@ #include #include #include +#include #include #include @@ -27,10 +28,13 @@ #include #include "cloud/config.h" +#include "common/config.h" #include "cpp/aws_common.h" +#include "cpp/client/s3_obj_storage_backend.h" #include "cpp/custom_aws_credentials_provider_chain.h" -#include "io/fs/rate_limited_obj_storage_client.h" -#include "io/fs/s3_obj_storage_client.h" +#include "cpp/sync_point.h" +#include "io/fs/s3_file_system.h" +#include "util/s3_rate_limiter_manager.h" #include "util/s3_uri.h" #include "util/s3_util.h" @@ -45,6 +49,21 @@ class S3ClientFactoryTest : public testing::Test { namespace { +S3ClientConf make_factory_conf(std::string endpoint, bool is_internal_bucket) { + S3ClientConf conf; + conf.endpoint = std::move(endpoint); + conf.region = "us-east-1"; + conf.cred_provider_type = CredProviderType::Anonymous; + conf.is_internal_bucket = is_internal_bucket; + return conf; +} + +S3ClientConf make_hash_collision_conf(std::string endpoint, bool is_internal_bucket) { + auto conf = make_factory_conf(std::move(endpoint), is_internal_bucket); + conf.use_virtual_addressing = !is_internal_bucket; + return conf; +} + class CloudModeConfigGuard { public: explicit CloudModeConfigGuard(bool cloud_mode) @@ -63,40 +82,58 @@ class CloudModeConfigGuard { std::string _cloud_unique_id; }; -S3ClientConf make_factory_conf(std::string endpoint, bool is_internal_bucket) { - S3ClientConf conf; - conf.endpoint = std::move(endpoint); - conf.region = "us-east-1"; - conf.cred_provider_type = CredProviderType::Anonymous; - conf.is_internal_bucket = is_internal_bucket; - return conf; -} +class RateLimiterConfigGuard { +public: + RateLimiterConfigGuard() + : _enabled(config::enable_s3_rate_limiter), + _qps_max_speed(_qps()->get_max_speed()), + _qps_max_burst(_qps()->get_max_burst()), + _qps_limit(_qps()->get_limit()), + _bytes_max_speed(_bytes()->get_max_speed()), + _bytes_max_burst(_bytes()->get_max_burst()), + _bytes_limit(_bytes()->get_limit()) {} + + ~RateLimiterConfigGuard() { + config::enable_s3_rate_limiter = _enabled; + _qps()->reset(_qps_max_speed, _qps_max_burst, _qps_limit); + _bytes()->reset(_bytes_max_speed, _bytes_max_burst, _bytes_limit); + } -S3ClientConf make_hash_collision_conf(std::string endpoint, bool is_internal_bucket) { - auto conf = make_factory_conf(std::move(endpoint), is_internal_bucket); - conf.use_virtual_addressing = !is_internal_bucket; - return conf; -} +private: + static S3RateLimiterHolder* _qps() { + return S3RateLimiterManager::instance().qps_limiter(S3RateLimitType::GET); + } + static S3RateLimiterHolder* _bytes() { + return S3RateLimiterManager::instance().bytes_limiter(S3RateLimitType::GET); + } -} // namespace + bool _enabled; + size_t _qps_max_speed; + size_t _qps_max_burst; + size_t _qps_limit; + size_t _bytes_max_speed; + size_t _bytes_max_burst; + size_t _bytes_limit; +}; -TEST_F(S3ClientFactoryTest, WrapsAllClientsInNonCloudMode) { - CloudModeConfigGuard guard(false); - auto& factory = S3ClientFactory::instance(); +class SyncPointProcessingGuard { +public: + SyncPointProcessingGuard() : _was_enabled(SyncPoint::get_instance()->get_enable()) { + SyncPoint::get_instance()->enable_processing(); + } + ~SyncPointProcessingGuard() { + if (!_was_enabled) { + SyncPoint::get_instance()->disable_processing(); + } + } - auto external_client = - factory.create(make_factory_conf("non-cloud-external-rate-limit.example.com", false)); - auto internal_client = - factory.create(make_factory_conf("non-cloud-internal-rate-limit.example.com", true)); +private: + bool _was_enabled; +}; - ASSERT_NE(external_client, nullptr); - ASSERT_NE(internal_client, nullptr); - EXPECT_NE(std::dynamic_pointer_cast(external_client), nullptr); - EXPECT_NE(std::dynamic_pointer_cast(internal_client), nullptr); -} +} // namespace -TEST_F(S3ClientFactoryTest, WrapsOnlyInternalClientsInCloudModeAndDistinguishesHashCollisions) { - CloudModeConfigGuard guard(true); +TEST_F(S3ClientFactoryTest, DistinguishesHashCollisions) { auto external_conf = make_hash_collision_conf("cloud-rate-limit-hash-collision.example.com", false); auto internal_conf = @@ -105,16 +142,20 @@ TEST_F(S3ClientFactoryTest, WrapsOnlyInternalClientsInCloudModeAndDistinguishesH ASSERT_NE(external_conf, internal_conf); auto& factory = S3ClientFactory::instance(); - auto external_client = factory.create(external_conf); - auto internal_client = factory.create(internal_conf); + auto external_result = factory.create(external_conf); + auto internal_result = factory.create(internal_conf); + ASSERT_TRUE(external_result.has_value()) << external_result.error(); + ASSERT_TRUE(internal_result.has_value()) << internal_result.error(); + auto external_client = std::move(external_result).value(); + auto internal_client = std::move(internal_result).value(); - ASSERT_NE(external_client, nullptr); - ASSERT_NE(internal_client, nullptr); - EXPECT_EQ(std::dynamic_pointer_cast(external_client), nullptr); - EXPECT_NE(std::dynamic_pointer_cast(internal_client), nullptr); EXPECT_NE(external_client, internal_client); - EXPECT_EQ(factory.create(external_conf), external_client); - EXPECT_EQ(factory.create(internal_conf), internal_client); + auto cached_external = factory.create(external_conf); + auto cached_internal = factory.create(internal_conf); + ASSERT_TRUE(cached_external.has_value()) << cached_external.error(); + ASSERT_TRUE(cached_internal.has_value()) << cached_internal.error(); + EXPECT_EQ(cached_external.value(), external_client); + EXPECT_EQ(cached_internal.value(), internal_client); } TEST_F(S3ClientFactoryTest, ObjClientHolderResetDistinguishesHashCollisions) { @@ -124,10 +165,12 @@ TEST_F(S3ClientFactoryTest, ObjClientHolderResetDistinguishesHashCollisions) { make_hash_collision_conf("s3-client-holder-hash-collision.example.com", true); ASSERT_EQ(external_conf.get_hash(), internal_conf.get_hash()); - auto external_client = - std::make_shared(std::shared_ptr {}); - auto internal_client = - std::make_shared(std::shared_ptr {}); + auto external_backend = + std::make_shared(std::shared_ptr {}); + auto internal_backend = + std::make_shared(std::shared_ptr {}); + auto external_client = std::make_shared(external_backend); + auto internal_client = std::make_shared(internal_backend); int create_count = 0; S3ClientFactory::instance().set_client_creator_for_test( [&](const S3ClientConf& conf) -> std::shared_ptr { @@ -146,6 +189,55 @@ TEST_F(S3ClientFactoryTest, ObjClientHolderResetDistinguishesHashCollisions) { EXPECT_EQ(holder.s3_client_conf(), internal_conf); } +TEST_F(S3ClientFactoryTest, SelectsRateLimiterByDeploymentAndBucketType) { + RateLimiterConfigGuard rate_limiter_guard; + SyncPointProcessingGuard sync_point_guard; + auto* sync_point = SyncPoint::get_instance(); + SyncPoint::CallbackGuard create_backend_callback; + sync_point->set_call_back( + "s3_client_factory::create", + [](auto&& args) { + auto result = try_any_cast_ret>(args); + result->second = true; + }, + &create_backend_callback); + SyncPoint::CallbackGuard head_object_callback; + sync_point->set_call_back( + "s3_file_system::head_object", + [](auto&& args) { + auto result = try_any_cast_ret(args); + result->first = + Aws::S3::Model::HeadObjectOutcome(Aws::S3::Model::HeadObjectResult {}); + result->second = true; + }, + &head_object_callback); + + config::enable_s3_rate_limiter = true; + auto check_selection = [&](bool cloud_mode, bool internal_bucket, bool expect_limited, + std::string endpoint) { + CloudModeConfigGuard cloud_mode_guard(cloud_mode); + auto& manager = S3RateLimiterManager::instance(); + manager.qps_limiter(S3RateLimitType::GET)->reset(0, 0, 1); + manager.bytes_limiter(S3RateLimitType::GET)->reset(0, 0, 0); + + auto result = S3ClientFactory::instance().create( + make_factory_conf(std::move(endpoint), internal_bucket)); + ASSERT_TRUE(result.has_value()) << result.error(); + auto client = std::move(result).value(); + EXPECT_TRUE(client->head_object({.bucket = "bucket", .key = "key"}).resp.ok()); + auto second = client->head_object({.bucket = "bucket", .key = "key"}); + if (expect_limited) { + EXPECT_EQ(second.resp.status.code, static_cast(ErrorCode::EXCEEDED_LIMIT)); + } else { + EXPECT_TRUE(second.resp.ok()); + } + }; + + check_selection(false, false, true, "non-cloud-external-rate-limit.example.com"); + check_selection(true, true, true, "cloud-internal-rate-limit.example.com"); + check_selection(true, false, false, "cloud-external-no-rate-limit.example.com"); +} + TEST_F(S3ClientFactoryTest, AwsCredentialsProvider) { S3ClientFactory& factory = S3ClientFactory::instance(); S3ClientConf anonymous_conf; @@ -166,20 +258,20 @@ TEST_F(S3ClientFactoryTest, AwsCredentialsProvider) { config::aws_credentials_provider_version = "v2"; { - auto provider_v2 = factory.get_aws_credentials_provider(anonymous_conf); + auto provider_v2 = factory.create_aws_credentials_provider(anonymous_conf).provider; auto custom_chain_v2 = std::dynamic_pointer_cast(provider_v2); ASSERT_NE(custom_chain_v2, nullptr); } { - auto provider_v2 = factory.get_aws_credentials_provider(ak_sk_conf); + auto provider_v2 = factory.create_aws_credentials_provider(ak_sk_conf).provider; auto custom_chain_v2 = std::dynamic_pointer_cast(provider_v2); ASSERT_NE(custom_chain_v2, nullptr); } { - auto provider_v2 = factory.get_aws_credentials_provider(role_conf1); + auto provider_v2 = factory.create_aws_credentials_provider(role_conf1).provider; auto instance_profile_v2 = std::dynamic_pointer_cast( provider_v2); @@ -187,14 +279,14 @@ TEST_F(S3ClientFactoryTest, AwsCredentialsProvider) { } { - auto provider_v2 = factory.get_aws_credentials_provider(role_conf2); + auto provider_v2 = factory.create_aws_credentials_provider(role_conf2).provider; auto custom_chain_v2 = std::dynamic_pointer_cast(provider_v2); ASSERT_NE(custom_chain_v2, nullptr); } { - auto provider_v2 = factory.get_aws_credentials_provider(web_identity_conf); + auto provider_v2 = factory.create_aws_credentials_provider(web_identity_conf).provider; auto web_identity_v2 = std::dynamic_pointer_cast( provider_v2); @@ -203,21 +295,21 @@ TEST_F(S3ClientFactoryTest, AwsCredentialsProvider) { config::aws_credentials_provider_version = "v1"; { - auto provider_v1 = factory.get_aws_credentials_provider(anonymous_conf); + auto provider_v1 = factory.create_aws_credentials_provider(anonymous_conf).provider; auto default_chain_v1 = std::dynamic_pointer_cast(provider_v1); ASSERT_NE(default_chain_v1, nullptr); } { - auto provider_v1 = factory.get_aws_credentials_provider(ak_sk_conf); + auto provider_v1 = factory.create_aws_credentials_provider(ak_sk_conf).provider; auto default_chain_v1 = std::dynamic_pointer_cast(provider_v1); ASSERT_NE(default_chain_v1, nullptr); } { - auto provider_v1 = factory.get_aws_credentials_provider(role_conf1); + auto provider_v1 = factory.create_aws_credentials_provider(role_conf1).provider; auto default_chain_v1 = std::dynamic_pointer_cast( provider_v1); @@ -225,7 +317,7 @@ TEST_F(S3ClientFactoryTest, AwsCredentialsProvider) { } { - auto provider_v1 = factory.get_aws_credentials_provider(role_conf2); + auto provider_v1 = factory.create_aws_credentials_provider(role_conf2).provider; auto default_chain_v1 = std::dynamic_pointer_cast(provider_v1); ASSERT_NE(default_chain_v1, nullptr); @@ -380,25 +472,25 @@ TEST_F(S3ClientFactoryTest, AwsCredentialsProviderV2ProviderTypeWithoutRoleArn) S3ClientConf default_conf; default_conf.cred_provider_type = CredProviderType::Default; - auto provider = factory.get_aws_credentials_provider(default_conf); + auto provider = factory.create_aws_credentials_provider(default_conf).provider; ASSERT_NE(std::dynamic_pointer_cast(provider), nullptr); S3ClientConf env_conf; env_conf.cred_provider_type = CredProviderType::Env; - provider = factory.get_aws_credentials_provider(env_conf); + provider = factory.create_aws_credentials_provider(env_conf).provider; ASSERT_NE(std::dynamic_pointer_cast(provider), nullptr); S3ClientConf sys_conf; sys_conf.cred_provider_type = CredProviderType::SystemProperties; - provider = factory.get_aws_credentials_provider(sys_conf); + provider = factory.create_aws_credentials_provider(sys_conf).provider; ASSERT_NE( std::dynamic_pointer_cast(provider), nullptr); S3ClientConf web_identity_conf; web_identity_conf.cred_provider_type = CredProviderType::WebIdentity; - provider = factory.get_aws_credentials_provider(web_identity_conf); + provider = factory.create_aws_credentials_provider(web_identity_conf).provider; ASSERT_NE(std::dynamic_pointer_cast( provider), nullptr); @@ -409,7 +501,7 @@ TEST_F(S3ClientFactoryTest, AwsCredentialsProviderV2ProviderTypeWithoutRoleArn) } S3ClientConf container_conf; container_conf.cred_provider_type = CredProviderType::Container; - provider = factory.get_aws_credentials_provider(container_conf); + provider = factory.create_aws_credentials_provider(container_conf).provider; ASSERT_NE(std::dynamic_pointer_cast(provider), nullptr); if (old_container_uri == nullptr) { unsetenv("AWS_CONTAINER_CREDENTIALS_RELATIVE_URI"); @@ -417,13 +509,13 @@ TEST_F(S3ClientFactoryTest, AwsCredentialsProviderV2ProviderTypeWithoutRoleArn) S3ClientConf instance_profile_conf; instance_profile_conf.cred_provider_type = CredProviderType::InstanceProfile; - provider = factory.get_aws_credentials_provider(instance_profile_conf); + provider = factory.create_aws_credentials_provider(instance_profile_conf).provider; ASSERT_NE(std::dynamic_pointer_cast(provider), nullptr); S3ClientConf anonymous_conf; anonymous_conf.cred_provider_type = CredProviderType::Anonymous; - provider = factory.get_aws_credentials_provider(anonymous_conf); + provider = factory.create_aws_credentials_provider(anonymous_conf).provider; ASSERT_NE(std::dynamic_pointer_cast(provider), nullptr); } @@ -436,7 +528,7 @@ TEST_F(S3ClientFactoryTest, AwsCredentialsProviderV2WithRoleArnAlwaysAssumeRole) CredProviderType::Default, CredProviderType::Env, CredProviderType::SystemProperties, CredProviderType::WebIdentity, CredProviderType::Container, CredProviderType::InstanceProfile, - CredProviderType::Anonymous, + CredProviderType::Anonymous, CredProviderType::Simple, }; for (auto provider_type : provider_types) { @@ -444,12 +536,25 @@ TEST_F(S3ClientFactoryTest, AwsCredentialsProviderV2WithRoleArnAlwaysAssumeRole) conf.cred_provider_type = provider_type; conf.role_arn = "arn:aws:iam::123456789012:role/test-role"; conf.external_id = "external-id"; - auto provider = factory.get_aws_credentials_provider(conf); + auto provider = factory.create_aws_credentials_provider(conf).provider; ASSERT_NE(std::dynamic_pointer_cast(provider), nullptr); } } +TEST_F(S3ClientFactoryTest, AwsCredentialsProviderV2SimpleWithoutAkSkUsesDefaultChain) { + S3ClientFactory& factory = S3ClientFactory::instance(); + config::aws_credentials_provider_version = "v2"; + + S3ClientConf conf; + conf.cred_provider_type = CredProviderType::Simple; + auto result = factory.create_aws_credentials_provider(conf); + + ASSERT_TRUE(result); + EXPECT_NE(std::dynamic_pointer_cast(result.provider), + nullptr); +} + TEST_F(S3ClientFactoryTest, AwsCredentialsProviderAkSkTakePrecedenceOverRoleArn) { S3ClientFactory& factory = S3ClientFactory::instance(); S3ClientConf conf; @@ -460,12 +565,12 @@ TEST_F(S3ClientFactoryTest, AwsCredentialsProviderAkSkTakePrecedenceOverRoleArn) conf.cred_provider_type = CredProviderType::InstanceProfile; config::aws_credentials_provider_version = "v2"; - auto provider_v2 = factory.get_aws_credentials_provider(conf); + auto provider_v2 = factory.create_aws_credentials_provider(conf).provider; ASSERT_NE(std::dynamic_pointer_cast(provider_v2), nullptr); config::aws_credentials_provider_version = "v1"; - auto provider_v1 = factory.get_aws_credentials_provider(conf); + auto provider_v1 = factory.create_aws_credentials_provider(conf).provider; ASSERT_NE(std::dynamic_pointer_cast(provider_v1), nullptr); @@ -479,11 +584,31 @@ TEST_F(S3ClientFactoryTest, AwsCredentialsProviderV1RoleArnDefaultFallback) { S3ClientConf conf; conf.cred_provider_type = CredProviderType::Default; conf.role_arn = "arn:aws:iam::123456789012:role/test-role"; - auto provider = factory.get_aws_credentials_provider(conf); + auto provider = factory.create_aws_credentials_provider(conf).provider; ASSERT_NE(std::dynamic_pointer_cast(provider), nullptr); config::aws_credentials_provider_version = "v2"; } +TEST_F(S3ClientFactoryTest, AwsCredentialsProviderV1PartialCredentialsUseDefaultChain) { + S3ClientFactory& factory = S3ClientFactory::instance(); + config::aws_credentials_provider_version = "v1"; + + for (bool provide_access_key : {false, true}) { + S3ClientConf conf; + conf.cred_provider_type = CredProviderType::Default; + conf.role_arn = "arn:aws:iam::123456789012:role/test-role"; + conf.ak = provide_access_key ? "ak" : ""; + conf.sk = provide_access_key ? "" : "sk"; + + auto provider = factory.create_aws_credentials_provider(conf).provider; + EXPECT_NE( + std::dynamic_pointer_cast(provider), + nullptr); + } + + config::aws_credentials_provider_version = "v2"; +} + } // namespace doris diff --git a/be/test/storage/cloud_file_cache_write_index_only_test.cpp b/be/test/storage/cloud_file_cache_write_index_only_test.cpp index c330b93fee489b..7815844492cb37 100644 --- a/be/test/storage/cloud_file_cache_write_index_only_test.cpp +++ b/be/test/storage/cloud_file_cache_write_index_only_test.cpp @@ -29,13 +29,13 @@ #include "cloud/config.h" #include "common/config.h" #include "core/block/block.h" +#include "cpp/client/s3_obj_storage_backend.h" #include "cpp/sync_point.h" #include "io/cache/block_file_cache_factory.h" #include "io/fs/file_writer.h" #include "io/fs/local_file_system.h" #include "io/fs/s3_file_system.h" #include "io/fs/s3_file_writer.h" -#include "io/fs/s3_obj_storage_client.h" #include "io/io_common.h" #include "runtime/exec_env.h" #include "storage/index/inverted/inverted_index_writer.h" @@ -337,7 +337,7 @@ class CloudFileCacheWriteIndexOnlyTest : public testing::Test { sp->set_call_back( "s3_client_factory::create", [](auto&& args) { - auto* ret = try_any_cast_ret>(args); + auto* ret = try_any_cast_ret>(args); ret->second = true; }, s3_client_guard); diff --git a/be/test/storage/rowset/beta_rowset_test.cpp b/be/test/storage/rowset/beta_rowset_test.cpp index 07d135702e8e25..1e9795dedd92c1 100644 --- a/be/test/storage/rowset/beta_rowset_test.cpp +++ b/be/test/storage/rowset/beta_rowset_test.cpp @@ -40,12 +40,12 @@ #include "common/config.h" #include "common/status.h" +#include "cpp/client/s3_obj_storage_backend.h" #include "cpp/sync_point.h" #include "gtest/gtest_pred_impl.h" #include "io/fs/file_system.h" #include "io/fs/local_file_system.h" #include "io/fs/s3_file_system.h" -#include "io/fs/s3_obj_storage_client.h" #include "json2pb/json_to_pb.h" #include "runtime/exec_env.h" #include "storage/data_dir.h" @@ -313,7 +313,8 @@ TEST_F(BetaRowsetTest, ReadTest) { aws_cred, aws_config, Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never, true); - client.reset(new io::S3ObjStorageClient(std::move(s3_client))); + client = std::make_shared( + std::make_shared(std::move(s3_client))); rowset.rowset_meta()->set_num_segments(1); rowset.rowset_meta()->set_remote_storage_resource(storage_resource); @@ -327,7 +328,7 @@ TEST_F(BetaRowsetTest, ReadTest) { { Aws::Auth::AWSCredentials aws_cred("ak", "sk"); Aws::Client::ClientConfiguration aws_config; - client.reset(new io::S3ObjStorageClient( + client = std::make_shared(std::make_shared( std::make_shared(S3ClientMockGetError()))); rowset.rowset_meta()->set_num_segments(1); @@ -342,7 +343,7 @@ TEST_F(BetaRowsetTest, ReadTest) { { Aws::Auth::AWSCredentials aws_cred("ak", "sk"); Aws::Client::ClientConfiguration aws_config; - client.reset(new io::S3ObjStorageClient( + client = std::make_shared(std::make_shared( std::make_shared(S3ClientMockGetErrorData()))); rowset.rowset_meta()->set_num_segments(1); diff --git a/cloud/src/recycler/CMakeLists.txt b/cloud/src/recycler/CMakeLists.txt index 3721e23faabdf7..d8206878fcbc8a 100644 --- a/cloud/src/recycler/CMakeLists.txt +++ b/cloud/src/recycler/CMakeLists.txt @@ -29,10 +29,6 @@ if (NOT ENABLE_HDFS_STORAGE_VAULT) list(REMOVE_ITEM SRC_LIST ${CMAKE_CURRENT_SOURCE_DIR}/hdfs_accessor.cpp) endif() -if(BUILD_AZURE STREQUAL "OFF") - list(REMOVE_ITEM SRC_LIST "${CMAKE_CURRENT_SOURCE_DIR}/azure_obj_client.cpp") -endif() - if(BUILD_CHECK_META STREQUAL "OFF") list(REMOVE_ITEM SRC_LIST ${CMAKE_CURRENT_SOURCE_DIR}/meta_checker.cpp) endif () diff --git a/cloud/src/recycler/azure_obj_client.cpp b/cloud/src/recycler/azure_obj_client.cpp deleted file mode 100644 index 5390c1c5f99608..00000000000000 --- a/cloud/src/recycler/azure_obj_client.cpp +++ /dev/null @@ -1,341 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "recycler/azure_obj_client.h" - -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "common/config.h" -#include "common/logging.h" -#include "common/stopwatch.h" -#include "cpp/obj_retry_strategy.h" -#include "cpp/sync_point.h" -#include "cpp/token_bucket_rate_limiter.h" -#include "recycler/s3_accessor.h" -#include "recycler/util.h" - -using namespace Azure::Storage::Blobs; - -namespace doris::cloud { - -template -auto s3_rate_limit(S3RateLimitType op, Func callback) -> decltype(callback()) { - using T = decltype(callback()); - // Fault injection for testing rate limit handling in recycler - if (config::enable_s3_rate_limit_inject && op == S3RateLimitType::PUT) { - if (rand() % 100 < config::s3_rate_limit_inject_probility) { - throw std::runtime_error("Azure exceeds request limit"); - } - } - if (!config::enable_s3_rate_limiter) { - return callback(); - } - auto sleep_duration = - doris::apply_s3_rate_limit(op, AccessorRateLimiter::instance().rate_limiter(op), - config::s3_rate_limiter_log_interval); - if (sleep_duration < 0) { - throw std::runtime_error("Azure exceeds request limit"); - } - return callback(); -} - -template -auto s3_get_rate_limit(Func callback) -> decltype(callback()) { - return s3_rate_limit(S3RateLimitType::GET, std::move(callback)); -} - -template -auto s3_put_rate_limit(Func callback) -> decltype(callback()) { - return s3_rate_limit(S3RateLimitType::PUT, std::move(callback)); -} - -static constexpr size_t BlobBatchMaxOperations = 256; -static constexpr char BlobNotFound[] = "BlobNotFound"; - -template -ObjectStorageResponse do_azure_client_call(Func f, std::string_view url, std::string_view key) { - try { - f(); - } catch (Azure::Core::RequestFailedException& e) { - doris::record_object_request_failed(static_cast(e.StatusCode)); - auto msg = fmt::format( - "Azure request failed because {}, http_code: {}, request_id: {}, url: {}, " - "key: {}", - e.Message, static_cast(e.StatusCode), e.RequestId, url, key); - LOG_WARNING(msg); - return {-1, std::move(msg)}; - } catch (std::exception& e) { - auto msg = fmt::format("Azure request failed because {}, url: {}, key: {}", e.what(), url, - key); - LOG_WARNING(msg); - return {-1, std::move(msg)}; - } - return {}; -} - -static const Azure::DateTime SystemClockEpoch {1970, 1, 1}; - -class AzureListIterator final : public ObjectListIterator { -public: - AzureListIterator(std::shared_ptr client, std::string prefix) - : client_(std::move(client)), req_({.Prefix = std::move(prefix)}) { - TEST_SYNC_POINT_CALLBACK("AzureListIterator", &req_); - } - - ~AzureListIterator() override = default; - - bool is_valid() override { return is_valid_; } - - bool has_next() override { - if (!is_valid_) { - return false; - } - - if (!results_.empty()) { - return true; - } - - if (!has_more_) { - return false; - } - - ListBlobsPagedResponse resp; - auto obj_resp = do_azure_client_call( - [&]() { - resp = s3_get_rate_limit([&]() { - SCOPED_BVAR_LATENCY(s3_bvar::s3_list_latency); - return client_->ListBlobs(req_); - }); - }, - client_->GetUrl(), req_.Prefix.Value()); - if (obj_resp.ret != 0) { - is_valid_ = false; - return false; - } - - has_more_ = resp.NextPageToken.HasValue(); - DCHECK(!(has_more_ && resp.Blobs.empty())) << has_more_ << ' ' << resp.Blobs.empty(); - req_.ContinuationToken = std::move(resp.NextPageToken); - results_.reserve(resp.Blobs.size()); - for (auto&& item : std::ranges::reverse_view(resp.Blobs)) { - DCHECK(item.Name.starts_with(*req_.Prefix)) << item.Name << ' ' << *req_.Prefix; - results_.emplace_back(ObjectMeta { - .key = std::move(item.Name), - .size = item.BlobSize, - // `Azure::DateTime` adds the offset of `SystemClockEpoch` to the given Unix timestamp, - // so here we need to subtract this offset to obtain the Unix timestamp of the mtime. - // https://github.com/Azure/azure-sdk-for-cpp/blob/azure-core_1.12.0/sdk/core/azure-core/inc/azure/core/datetime.hpp#L129 - .mtime_s = duration_cast(item.Details.LastModified - - SystemClockEpoch) - .count()}); - } - - return !results_.empty(); - } - - std::optional next() override { - std::optional res; - if (!has_next()) { - return res; - } - - res = std::move(results_.back()); - results_.pop_back(); - return res; - } - -private: - std::shared_ptr client_; - ListBlobsOptions req_; - std::vector results_; - bool is_valid_ {true}; - bool has_more_ {true}; -}; - -AzureObjClient::~AzureObjClient() = default; - -ObjectStorageResponse AzureObjClient::put_object(ObjectStoragePathRef path, - std::string_view stream) { - auto client = client_->GetBlockBlobClient(path.key); - return do_azure_client_call( - [&]() { - s3_put_rate_limit([&]() { - SCOPED_BVAR_LATENCY(s3_bvar::s3_put_latency); - return client.UploadFrom(reinterpret_cast(stream.data()), - stream.size()); - }); - }, - client_->GetUrl(), path.key); -} - -ObjectStorageResponse AzureObjClient::head_object(ObjectStoragePathRef path, ObjectMeta* res) { - Models::BlobProperties properties {}; - bool not_found = false; - - auto resp = do_azure_client_call( - [&]() { - try { - properties = s3_get_rate_limit([&]() { - SCOPED_BVAR_LATENCY(s3_bvar::s3_head_latency); - return client_->GetBlockBlobClient(path.key).GetProperties().Value; - }); - } catch (Azure::Storage::StorageException& e) { - if (e.StatusCode == Azure::Core::Http::HttpStatusCode::NotFound) { - not_found = true; - return; - } - throw; - } - }, - client_->GetUrl(), path.key); - - if (not_found) { - return {1}; - } - - if (resp.ret != 0) { - return resp; - } - - res->key = path.key; - res->mtime_s = properties.LastModified.time_since_epoch().count(); - res->size = properties.BlobSize; - return {0}; -} - -std::unique_ptr AzureObjClient::list_objects(ObjectStoragePathRef path) { - return std::make_unique(client_, path.key); -} - -// As Azure's doc said, the batch size is 256 -// You can find out the num in https://learn.microsoft.com/en-us/rest/api/storageservices/blob-batch?tabs=microsoft-entra-id -// > Each batch request supports a maximum of 256 subrequests. -ObjectStorageResponse AzureObjClient::delete_objects(const std::string& bucket, - std::vector keys, - ObjClientOptions option) { - if (keys.empty()) { - return {0}; - } - - // TODO(ByteYue) : use range to adate this code when compiler is ready - // auto chunkedView = objs | std::views::chunk(BlobBatchMaxOperations); - auto begin = std::begin(keys); - auto end = std::end(keys); - - while (begin != end) { - auto batch = client_->CreateBatch(); - auto chunk_end = begin; - size_t batch_size = BlobBatchMaxOperations; - TEST_SYNC_POINT_CALLBACK("AzureObjClient::delete_objects", &batch_size); - std::advance(chunk_end, - std::min(batch_size, static_cast(std::distance(begin, end)))); - std::vector> deferred_resps; - deferred_resps.reserve(std::distance(begin, chunk_end)); - for (auto it = begin; it != chunk_end; ++it) { - deferred_resps.emplace_back(batch.DeleteBlob(*it)); - } - auto resp = do_azure_client_call( - [&]() { - s3_put_rate_limit([&]() { - SCOPED_BVAR_LATENCY(s3_bvar::s3_delete_objects_latency); - return client_->SubmitBatch(batch); - }); - }, - client_->GetUrl(), *begin); - if (resp.ret != 0) { - return resp; - } - for (auto&& defer : deferred_resps) { - try { - auto r = defer.GetResponse(); - if (!r.Value.Deleted) { - LOG_INFO("Azure batch delete failed, url {}", client_->GetUrl()); - return {-1}; - } - } catch (Azure::Storage::StorageException& e) { - if (Azure::Core::Http::HttpStatusCode::NotFound == e.StatusCode && - 0 == strcmp(e.ErrorCode.c_str(), BlobNotFound)) { - continue; - } - doris::record_object_request_failed(static_cast(e.StatusCode)); - auto msg = fmt::format( - "Azure request failed because {}, http code {}, request id {}, url {}", - e.Message, static_cast(e.StatusCode), e.RequestId, client_->GetUrl()); - LOG_WARNING(msg); - return {-1, std::move(msg)}; - } - } - - begin = chunk_end; - } - - return {0}; -} - -ObjectStorageResponse AzureObjClient::delete_object(ObjectStoragePathRef path) { - return do_azure_client_call( - [&]() { - if (auto r = s3_put_rate_limit([&]() { - SCOPED_BVAR_LATENCY(s3_bvar::s3_delete_object_latency); - return client_->DeleteBlob(path.key); - }); - !r.Value.Deleted) { - throw std::runtime_error("Delete azure blob failed"); - } - }, - client_->GetUrl(), path.key); -} - -ObjectStorageResponse AzureObjClient::delete_objects_recursively(ObjectStoragePathRef path, - ObjClientOptions option, - int64_t expiration_time) { - return delete_objects_recursively_(path, option, expiration_time, BlobBatchMaxOperations); -} - -ObjectStorageResponse AzureObjClient::get_life_cycle(const std::string& bucket, - int64_t* expiration_days) { - // TODO(plat1ko) - *expiration_days = INT64_MAX; - return {0}; -} - -ObjectStorageResponse AzureObjClient::check_versioning(const std::string& bucket) { - // TODO(plat1ko) - return {0}; -} - -ObjectStorageResponse AzureObjClient::abort_multipart_upload(ObjectStoragePathRef path, - const std::string& upload_id) { - // delete uncommitted blobs - // https://learn.microsoft.com/en-us/rest/api/storageservices/delete-blob?tabs=microsoft-entra-id#remarks - return delete_object(path); -} - -} // namespace doris::cloud diff --git a/cloud/src/recycler/azure_obj_client.h b/cloud/src/recycler/azure_obj_client.h deleted file mode 100644 index bdf9b4eda752cc..00000000000000 --- a/cloud/src/recycler/azure_obj_client.h +++ /dev/null @@ -1,62 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include - -#include "recycler/obj_storage_client.h" - -namespace Azure::Storage::Blobs { -class BlobContainerClient; -} // namespace Azure::Storage::Blobs - -namespace doris::cloud { -class AzureObjClient final : public ObjStorageClient { -public: - AzureObjClient(std::shared_ptr client) - : client_(std::move(client)) {} - ~AzureObjClient() override; - - ObjectStorageResponse put_object(ObjectStoragePathRef path, std::string_view stream) override; - - ObjectStorageResponse head_object(ObjectStoragePathRef path, ObjectMeta* res) override; - - std::unique_ptr list_objects(ObjectStoragePathRef path) override; - - ObjectStorageResponse delete_objects(const std::string& bucket, std::vector keys, - ObjClientOptions option) override; - - ObjectStorageResponse delete_object(ObjectStoragePathRef path) override; - - ObjectStorageResponse delete_objects_recursively(ObjectStoragePathRef path, - ObjClientOptions option, - int64_t expiration_time = 0) override; - - ObjectStorageResponse get_life_cycle(const std::string& bucket, - int64_t* expiration_days) override; - - ObjectStorageResponse check_versioning(const std::string& bucket) override; - - ObjectStorageResponse abort_multipart_upload(ObjectStoragePathRef path, - const std::string& upload_id) override; - -private: - std::shared_ptr client_; -}; - -} // namespace doris::cloud diff --git a/cloud/src/recycler/obj_storage_client.cpp b/cloud/src/recycler/obj_storage_client.cpp deleted file mode 100644 index f1fc52f2c281be..00000000000000 --- a/cloud/src/recycler/obj_storage_client.cpp +++ /dev/null @@ -1,154 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "recycler/obj_storage_client.h" - -#include - -#include "common/config.h" -#include "cpp/sync_point.h" -#include "recycler/sync_executor.h" - -using namespace std::chrono; - -namespace doris::cloud { - -ObjectStorageResponse ObjStorageClient::delete_objects_recursively_(ObjectStoragePathRef path, - const ObjClientOptions& option, - int64_t expired_time, - size_t batch_size) { - TEST_SYNC_POINT_CALLBACK("ObjStorageClient::delete_objects_recursively_", &batch_size); - auto list_iter = list_objects(path); - ObjectStorageResponse ret; - size_t num_deleted = 0; - int error_count = 0; - size_t batch_count = 0; - auto start_time = steady_clock::now(); - - // Read max tasks per batch from config, validate to prevent overflow - int32_t config_val = config::recycler_max_tasks_per_batch; - size_t max_tasks_per_batch = 1000; // default value - if (config_val > 0) { - max_tasks_per_batch = static_cast(config_val); - } else { - LOG(WARNING) << "recycler_max_tasks_per_batch=" << config_val - << " is not positive, using default 1000"; - } - - while (true) { - // Create a new SyncExecutor for each batch - // Note: cancel lambda only takes effect within the current batch - SyncExecutor batch_executor( - option.executor, fmt::format("delete batch under {}/{}", path.bucket, path.key), - [](const int& r) { return r != 0; }); - - std::vector keys; - size_t tasks_in_batch = 0; - bool has_more = true; - - // Collect tasks until reaching batch limit or no more files - while (tasks_in_batch < max_tasks_per_batch && has_more) { - auto obj = list_iter->next(); - if (!obj.has_value()) { - has_more = false; - break; - } - if (expired_time > 0 && obj->mtime_s > expired_time) { - continue; - } - - num_deleted++; - keys.emplace_back(std::move(obj->key)); - - // Submit a delete task when we have batch_size keys - if (keys.size() >= batch_size) { - batch_executor.add([this, &path, k = std::move(keys), option]() mutable { - return delete_objects(path.bucket, std::move(k), option).ret; - }); - keys.clear(); - tasks_in_batch++; - } - } - - // Handle remaining keys (less than batch_size) - if (!keys.empty()) { - batch_executor.add([this, &path, k = std::move(keys), option]() mutable { - return delete_objects(path.bucket, std::move(k), option).ret; - }); - tasks_in_batch++; - } - - // Before exiting on empty batch, check if listing is valid - // Avoid silently treating listing failure as success - if (tasks_in_batch == 0) { - if (!list_iter->is_valid()) { - LOG(WARNING) << "list_iter invalid with no tasks collected"; - ret = {-1}; - } - break; - } - - // Wait for current batch to complete - bool finished = true; - std::vector rets = batch_executor.when_all(&finished); - batch_count++; - - for (int r : rets) { - if (r != 0) { - error_count++; - } - } - - // Log batch progress for monitoring long-running delete tasks - auto batch_elapsed = duration_cast(steady_clock::now() - start_time).count(); - LOG(INFO) << "delete objects under " << path.bucket << "/" << path.key << " batch " - << batch_count << " completed" - << ", tasks_in_batch=" << tasks_in_batch << ", total_deleted=" << num_deleted - << ", elapsed=" << batch_elapsed << " ms"; - - // Check finished status: false means stop_token triggered, task timeout, or task invalid - if (!finished) { - LOG(WARNING) << "batch execution did not finish normally, stopping"; - ret = {-1}; - break; - } - - // Check if list_iter is still valid (network errors, etc.) - if (!list_iter->is_valid()) { - LOG(WARNING) << "list_iter became invalid during iteration"; - ret = {-1}; - break; - } - - // batch_executor goes out of scope, resources are automatically released - } - - if (error_count > 0) { - LOG(WARNING) << "delete_objects_recursively completed with " << error_count << " errors"; - ret = {-1}; - } - - auto elapsed = duration_cast(steady_clock::now() - start_time).count(); - LOG(INFO) << "delete objects under " << path.bucket << "/" << path.key - << " finished, ret=" << ret.ret << ", total_batches=" << batch_count - << ", num_deleted=" << num_deleted << ", error_count=" << error_count - << ", cost=" << elapsed << " ms"; - - return ret; -} - -} // namespace doris::cloud diff --git a/cloud/src/recycler/obj_storage_client.h b/cloud/src/recycler/obj_storage_client.h deleted file mode 100644 index 358dfba4943fd7..00000000000000 --- a/cloud/src/recycler/obj_storage_client.h +++ /dev/null @@ -1,115 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include -#include -#include -#include - -namespace doris::cloud { - -struct ObjectStoragePathRef { - const std::string& bucket; - const std::string& key; -}; - -struct ObjectStorageResponse { - enum Code : int { - UNDEFINED = -1, - OK = 0, - NOT_FOUND = 1, - }; - - ObjectStorageResponse(int r = OK, std::string msg = "") : ret(r), error_msg(std::move(msg)) {} - // clang-format off - int ret {OK}; // To unify the error handle logic with BE, we'd better use the same error code as BE - // clang-format on - std::string error_msg; -}; - -struct ObjectMeta { - std::string key; - int64_t size {0}; - int64_t mtime_s {0}; -}; - -class ObjectListIterator { -public: - virtual ~ObjectListIterator() = default; - virtual bool is_valid() = 0; - virtual bool has_next() = 0; - virtual std::optional next() = 0; -}; - -class SimpleThreadPool; -struct ObjClientOptions { - bool prefetch {true}; - std::shared_ptr executor; -}; - -class ObjStorageClient { -public: - ObjStorageClient() = default; - virtual ~ObjStorageClient() = default; - - ObjStorageClient(const ObjStorageClient&) = delete; - ObjStorageClient& operator=(const ObjStorageClient&) = delete; - - virtual ObjectStorageResponse put_object(ObjectStoragePathRef path, - std::string_view stream) = 0; - - // If it exists, it will return the corresponding object meta - virtual ObjectStorageResponse head_object(ObjectStoragePathRef path, ObjectMeta* res) = 0; - - // According to the passed bucket and prefix, it traverses and retrieves all files under the prefix, and returns the name and file size of all files. - // **Attention**: The ObjectMeta contains the full key in object storage - virtual std::unique_ptr list_objects(ObjectStoragePathRef path) = 0; - - // According to the bucket and prefix specified by the user, it performs batch deletion based on the object names in the object array. - virtual ObjectStorageResponse delete_objects(const std::string& bucket, - std::vector keys, - ObjClientOptions option) = 0; - - // Delete the file named key in the object storage bucket. - virtual ObjectStorageResponse delete_object(ObjectStoragePathRef path) = 0; - - // According to the prefix, recursively delete all objects under the prefix. - // If `expiration_time` > 0, only delete objects with mtime earlier than `expiration_time`. - virtual ObjectStorageResponse delete_objects_recursively(ObjectStoragePathRef path, - ObjClientOptions option, - int64_t expiration_time = 0) = 0; - - // Get the objects' expiration time on the bucket - virtual ObjectStorageResponse get_life_cycle(const std::string& bucket, - int64_t* expiration_days) = 0; - - // Check if the objects' versioning is on or off - // returns 0 when versioning is on, otherwise versioning is off or check failed - virtual ObjectStorageResponse check_versioning(const std::string& bucket) = 0; - - virtual ObjectStorageResponse abort_multipart_upload(ObjectStoragePathRef path, - const std::string& upload_id) = 0; - -protected: - ObjectStorageResponse delete_objects_recursively_(ObjectStoragePathRef path, - const ObjClientOptions& option, - int64_t expiration_time, size_t batch_size); -}; - -} // namespace doris::cloud diff --git a/cloud/src/recycler/recycler.cpp b/cloud/src/recycler/recycler.cpp index c2b0e6b7c176eb..b96a25deb13158 100644 --- a/cloud/src/recycler/recycler.cpp +++ b/cloud/src/recycler/recycler.cpp @@ -717,6 +717,7 @@ int InstanceRecycler::init_storage_vault_accessors() { << "but HDFS storage vaults were detected"; #endif } else if (vault.has_obj_info()) { + // TODO: Propagate object storage session tokens to Recycler in a follow-up PR. auto s3_conf = S3Conf::from_obj_store_info(vault.obj_info()); if (!s3_conf) { LOG(WARNING) << "failed to init object accessor, invalid conf, instance_id=" diff --git a/cloud/src/recycler/s3_accessor.cpp b/cloud/src/recycler/s3_accessor.cpp index 0c3713d92d1d2f..dcfa7430872f57 100644 --- a/cloud/src/recycler/s3_accessor.cpp +++ b/cloud/src/recycler/s3_accessor.cpp @@ -20,16 +20,12 @@ #include #include #include -#include #include -#include -#include #include -#include -#include #include #include +#include #ifdef USE_AZURE #include @@ -46,32 +42,21 @@ #include "common/simple_thread_pool.h" #include "common/string_util.h" #include "common/util.h" +#include "cpp/client/auth/aws_credential_factory.h" +#ifdef USE_AZURE +#include "cpp/client/auth/azure_auth_factory.h" +#include "cpp/client/azure_obj_storage_backend.h" +#endif #include "cpp/aws_logger.h" -#include "cpp/custom_aws_credentials_provider_chain.h" +#include "cpp/client/s3_obj_storage_backend.h" #include "cpp/obj_retry_strategy.h" #include "cpp/sync_point.h" #include "cpp/token_bucket_rate_limiter.h" #include "cpp/util.h" -#ifdef USE_AZURE -#include "recycler/azure_obj_client.h" -#endif -#include "recycler/obj_storage_client.h" -#include "recycler/s3_obj_client.h" #include "recycler/storage_vault_accessor.h" +#include "recycler/sync_executor.h" namespace doris::cloud { -namespace s3_bvar { -bvar::LatencyRecorder s3_get_latency("s3_get"); -bvar::LatencyRecorder s3_put_latency("s3_put"); -bvar::LatencyRecorder s3_delete_object_latency("s3_delete_object"); -bvar::LatencyRecorder s3_delete_objects_latency("s3_delete_objects"); -bvar::LatencyRecorder s3_head_latency("s3_head"); -bvar::LatencyRecorder s3_multi_part_upload_latency("s3_multi_part_upload"); -bvar::LatencyRecorder s3_list_latency("s3_list"); -bvar::LatencyRecorder s3_list_object_versions_latency("s3_list_object_versions"); -bvar::LatencyRecorder s3_get_bucket_version_latency("s3_get_bucket_version"); -bvar::LatencyRecorder s3_copy_object_latency("s3_copy_object"); -}; // namespace s3_bvar AccessorRateLimiter::AccessorRateLimiter() : _rate_limiters({std::make_unique( @@ -109,6 +94,32 @@ int reset_s3_rate_limiter(S3RateLimitType type, size_t max_speed, size_t max_bur return AccessorRateLimiter::instance().rate_limiter(type)->reset(max_speed, max_burst, limit); } +class RecyclerObjStorageRateLimitPolicy final : public ObjStorageRateLimitPolicy { +public: + ObjStorageRateLimitToken acquire(ObjStorageRequestType type, size_t) const override { + const auto limiter_type = + type == ObjStorageRequestType::GET ? S3RateLimitType::GET : S3RateLimitType::PUT; + if (config::enable_s3_rate_limit_inject && limiter_type == S3RateLimitType::PUT && + rand() % 100 < config::s3_rate_limit_inject_probility) { + return ObjStorageRateLimitToken { + .resp = ObjectStorageResponse::rate_limit( + "object storage PUT request rejected by recycler fault injection"), + }; + } + if (config::enable_s3_rate_limiter && + doris::apply_s3_rate_limit(limiter_type, + AccessorRateLimiter::instance().rate_limiter(limiter_type), + config::s3_rate_limiter_log_interval) < 0) { + return ObjStorageRateLimitToken { + .resp = ObjectStorageResponse::rate_limit(fmt::format( + "object storage {} request rejected by recycler rate limiter", + to_string(limiter_type))), + }; + } + return ObjStorageRateLimitToken {}; + } +}; + S3Environment::S3Environment() { LOG(INFO) << "Initializing S3 environment"; aws_options_ = Aws::SDKOptions {}; @@ -158,25 +169,27 @@ S3Environment::~S3Environment() { class S3ListIterator final : public ListIterator { public: - S3ListIterator(std::unique_ptr iter, size_t prefix_length) - : iter_(std::move(iter)), prefix_length_(prefix_length) {} + S3ListIterator(std::shared_ptr client, ObjectStoragePathOptions opts, + size_t prefix_length) + : iter_(std::move(client), std::move(opts)), prefix_length_(prefix_length) {} ~S3ListIterator() override = default; - bool is_valid() override { return iter_->is_valid(); } + bool is_valid() override { return iter_.is_valid(); } - bool has_next() override { return iter_->has_next(); } + bool has_next() override { return iter_.has_next().ok(); } std::optional next() override { - std::optional ret; - if (auto obj = iter_->next(); obj.has_value()) { - ret = FileMeta { - .path = get_relative_path(obj->key), - .size = obj->size, - .mtime_s = obj->mtime_s, - }; + auto result = iter_.next(); + if (!result.results_.has_value()) { + return std::nullopt; } - return ret; + auto& obj = *result.results_; + return FileMeta { + .path = get_relative_path(obj.file_path), + .size = obj.size, + .mtime_s = obj.mtime_s, + }; } private: @@ -184,7 +197,7 @@ class S3ListIterator final : public ListIterator { return key.substr(prefix_length_); } - std::unique_ptr iter_; + ObjectListIterator iter_; size_t prefix_length_; }; @@ -229,7 +242,6 @@ std::optional S3Conf::from_obj_store_info(const ObjectStoreInfoPB& obj_i s3_conf.sk = obj_info.sk(); } } - if (obj_info.has_cred_provider_type()) { s3_conf.cred_provider_type = cred_provider_type_from_pb(obj_info.cred_provider_type()); } @@ -281,97 +293,72 @@ int S3Accessor::create(S3Conf conf, std::shared_ptr* accessor) { static std::shared_ptr worker_pool; -std::shared_ptr S3Accessor::_get_aws_credentials_provider_v1( - const S3Conf& s3_conf) { - if (!s3_conf.ak.empty() && !s3_conf.sk.empty()) { - Aws::Auth::AWSCredentials aws_cred(s3_conf.ak, s3_conf.sk); - DCHECK(!aws_cred.IsExpiredOrEmpty()); - return std::make_shared(std::move(aws_cred)); - } - - if (s3_conf.cred_provider_type == CredProviderType::InstanceProfile) { - if (s3_conf.role_arn.empty()) { - return std::make_shared(); +RecursiveDeleteOptions S3Accessor::make_recursive_delete_options( + int64_t expiration_time, std::shared_ptr pool) { + RecursiveDeleteOptions options { + .expiration_time = expiration_time, + .max_tasks_per_batch = + config::recycler_max_tasks_per_batch > 0 + ? static_cast(config::recycler_max_tasks_per_batch) + : 1000, + }; + struct ExecutorState { + explicit ExecutorState(std::shared_ptr pool_) : pool(std::move(pool_)) { + reset(); } - Aws::Client::ClientConfiguration clientConfiguration = - S3Environment::getClientConfiguration(); - if (_ca_cert_file_path.empty()) { - _ca_cert_file_path = - get_valid_ca_cert_path(doris::cloud::split(config::ca_cert_file_paths, ';')); + void reset() { + executor = std::make_unique>( + pool, "delete object storage batches", + [](const ObjectStorageResponse& response) { return !response.ok(); }); } - if (!_ca_cert_file_path.empty()) { - clientConfiguration.caFile = _ca_cert_file_path; - } - - auto stsClient = std::make_shared( - std::make_shared(), - clientConfiguration); - - return std::make_shared( - s3_conf.role_arn, Aws::String(), s3_conf.external_id, - Aws::Auth::DEFAULT_CREDS_LOAD_FREQ_SECONDS, stsClient); - } - return std::make_shared(); -} -std::shared_ptr S3Accessor::_create_credentials_provider( - CredProviderType type) { - switch (type) { - case CredProviderType::Env: - return std::make_shared(); - case CredProviderType::SystemProperties: - return std::make_shared(); - case CredProviderType::WebIdentity: - return std::make_shared(); - case CredProviderType::Container: - return std::make_shared( - Aws::Environment::GetEnv("AWS_CONTAINER_CREDENTIALS_RELATIVE_URI").c_str()); - case CredProviderType::InstanceProfile: - return std::make_shared(); - case CredProviderType::Anonymous: - return std::make_shared(); - case CredProviderType::Default: - default: - return std::make_shared(); - } -} - -std::shared_ptr S3Accessor::_get_aws_credentials_provider_v2( - const S3Conf& s3_conf) { - if (!s3_conf.ak.empty() && !s3_conf.sk.empty()) { - Aws::Auth::AWSCredentials aws_cred(s3_conf.ak, s3_conf.sk); - DCHECK(!aws_cred.IsExpiredOrEmpty()); - return std::make_shared(std::move(aws_cred)); - } - - if (!s3_conf.role_arn.empty()) { - Aws::Client::ClientConfiguration clientConfiguration = - S3Environment::getClientConfiguration(); - if (_ca_cert_file_path.empty()) { - _ca_cert_file_path = - get_valid_ca_cert_path(doris::cloud::split(config::ca_cert_file_paths, ';')); + std::shared_ptr pool; + std::unique_ptr> executor; + }; + auto state = std::make_shared(std::move(pool)); + options.executor.submit = [state](ObjStorageDeleteTask task) { + state->executor->add(std::move(task)); + return ObjectStorageResponse::OK(); + }; + options.executor.wait = [state]() { + bool finished = false; + auto responses = state->executor->when_all(&finished); + state->reset(); + if (!finished) { + return ObjectStorageResponse { + .status = {TStatusCode::INTERNAL_ERROR, + "object storage batch deletion did not finish"}, + .http_code = 0, + }; } - if (!_ca_cert_file_path.empty()) { - clientConfiguration.caFile = _ca_cert_file_path; + for (auto& response : responses) { + if (!response.ok()) { + return response; + } } - - auto stsClient = std::make_shared( - _create_credentials_provider(s3_conf.cred_provider_type), clientConfiguration); - - return std::make_shared( - s3_conf.role_arn, Aws::String(), s3_conf.external_id, - Aws::Auth::DEFAULT_CREDS_LOAD_FREQ_SECONDS, stsClient); - } - return _create_credentials_provider(s3_conf.cred_provider_type); + return ObjectStorageResponse::OK(); + }; + return options; } -std::shared_ptr S3Accessor::get_aws_credentials_provider( - const S3Conf& s3_conf) { - if (config::aws_credentials_provider_version == "v2") { - return _get_aws_credentials_provider_v2(s3_conf); +AwsCredentialResult S3Accessor::create_aws_credentials_provider(const S3Conf& s3_conf) { + auto sts_config = S3Environment::getClientConfiguration(); + if (!_ca_cert_file_path.empty()) { + sts_config.caFile = _ca_cert_file_path; } - return _get_aws_credentials_provider_v1(s3_conf); + return AwsCredentialFactory::create({ + .version = config::aws_credentials_provider_version == "v2" + ? AwsCredentialProviderVersion::V2 + : AwsCredentialProviderVersion::V1, + .access_key = s3_conf.ak, + .secret_key = s3_conf.sk, + .provider_type = s3_conf.cred_provider_type, + .role_arn = s3_conf.role_arn, + .external_id = s3_conf.external_id, + .empty_credentials = EmptyCredentialsBehavior::DEFAULT_CHAIN, + .sts_client_config = std::move(sts_config), + }); } int S3Accessor::init() { @@ -388,8 +375,6 @@ int S3Accessor::init() { #ifdef USE_AZURE Azure::Storage::Blobs::BlobClientOptions options; options.Retry.MaxRetries = config::max_s3_client_retry; - auto cred = - std::make_shared(conf_.ak, conf_.sk); uri_ = fmt::format("{}/{}", conf_.endpoint, conf_.bucket); if (uri_.find("://") == std::string::npos) { uri_ = "https://" + uri_; @@ -400,11 +385,29 @@ int S3Accessor::init() { // All policies in the PerRetryPolicies are downstream of the RetryPolicy. // Therefore, the policy can record retries after the RetryPolicy has handled the response. options.PerRetryPolicies.emplace_back(std::make_unique()); - auto container_client = std::make_shared( - uri_, cred, std::move(options)); + auto built = AzureAuthFactory::create(uri_, + { + .type = AzureCredentialType::SHARED_KEY, + .account_name = conf_.ak, + .account_key = conf_.sk, + }, + std::move(options)); + if (!built) { + LOG_WARNING("failed to create Azure client").tag("error", built.error); + return -1; + } // uri format for debug: ${scheme}://${ak}.blob.core.windows.net/${bucket}/${prefix} uri_ = normalize_http_uri(uri_ + '/' + conf_.prefix); - obj_client_ = std::make_shared(std::move(container_client)); + auto backend = + std::make_shared(std::move(built.container_client), + ObjectClientConfig { + .endpoint = conf_.endpoint, + .ak = conf_.ak, + .sk = conf_.sk, + }, + std::move(built.shared_key_credential)); + obj_client_ = std::make_shared( + std::move(backend), std::make_shared()); return 0; #else LOG_FATAL("BE is not compiled with azure support, export BUILD_AZURE=ON before building"); @@ -445,11 +448,23 @@ int S3Accessor::init() { // curl error 28 on slow/large S3 DeleteObjects (OVH cold vault). aws_config.requestTimeoutMs = 30000; aws_config.connectTimeoutMs = 5000; + auto credentials = create_aws_credentials_provider(conf_); + if (!credentials) { + LOG(WARNING) << "failed to create AWS credential provider: " << credentials.error; + return -1; + } auto s3_client = std::make_shared( - get_aws_credentials_provider(conf_), std::move(aws_config), + std::move(credentials.provider), std::move(aws_config), Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never, conf_.use_virtual_addressing /* useVirtualAddressing */); - obj_client_ = std::make_shared(std::move(s3_client), conf_.endpoint); + auto backend = std::make_shared(std::move(s3_client), + ObjectClientConfig { + .endpoint = conf_.endpoint, + .ak = conf_.ak, + .sk = conf_.sk, + }); + obj_client_ = std::make_shared( + std::move(backend), std::make_shared()); return 0; } } @@ -458,9 +473,13 @@ int S3Accessor::init() { int S3Accessor::delete_prefix_impl(const std::string& path_prefix, int64_t expiration_time) { LOG_INFO("delete prefix").tag("uri", to_uri(path_prefix)); return obj_client_ - ->delete_objects_recursively({.bucket = conf_.bucket, .key = get_key(path_prefix)}, - {.executor = worker_pool}, expiration_time) - .ret; + ->delete_objects_recursively( + { + .bucket = conf_.bucket, + .prefix = get_key(path_prefix), + }, + make_recursive_delete_options(expiration_time, worker_pool)) + .status.code; } int S3Accessor::delete_prefix(const std::string& path_prefix, int64_t expiration_time) { @@ -501,29 +520,30 @@ int S3Accessor::delete_files(const std::vector& paths) { keys.emplace_back(get_key(path)); } - return obj_client_->delete_objects(conf_.bucket, std::move(keys), {.executor = worker_pool}) - .ret; + return obj_client_->delete_objects({.bucket = conf_.bucket}, std::move(keys)).status.code; } int S3Accessor::delete_file(const std::string& path) { LOG_INFO("delete file").tag("uri", to_uri(path)); - int ret = obj_client_->delete_object({.bucket = conf_.bucket, .key = get_key(path)}).ret; - static_assert(ObjectStorageResponse::OK == 0); - if (ret == ObjectStorageResponse::OK || ret == ObjectStorageResponse::NOT_FOUND) { + int ret = + obj_client_->delete_object({.bucket = conf_.bucket, .key = get_key(path)}).status.code; + static_assert(ObjectStorageStatus::OK == 0); + if (ret == ObjectStorageStatus::OK || ret == ObjectStorageStatus::NOT_FOUND) { return 0; } return ret; } int S3Accessor::put_file(const std::string& path, const std::string& content) { - return obj_client_->put_object({.bucket = conf_.bucket, .key = get_key(path)}, content).ret; + return obj_client_->put_object({.bucket = conf_.bucket, .key = get_key(path)}, content) + .status.code; } int S3Accessor::list_prefix(const std::string& path_prefix, std::unique_ptr* res) { - size_t prefix_length = conf_.prefix.empty() ? 0 : conf_.prefix.length() + 1; *res = std::make_unique( - obj_client_->list_objects({.bucket = conf_.bucket, .key = get_key(path_prefix)}), - prefix_length); + obj_client_, + ObjectStoragePathOptions {.bucket = conf_.bucket, .prefix = get_key(path_prefix)}, + conf_.prefix.empty() ? 0 : conf_.prefix.length() + 1); return 0; } @@ -543,8 +563,14 @@ int S3Accessor::list_all(std::unique_ptr* res) { } int S3Accessor::exists(const std::string& path) { - ObjectMeta obj_meta; - return obj_client_->head_object({.bucket = conf_.bucket, .key = get_key(path)}, &obj_meta).ret; + auto response = obj_client_->head_object({.bucket = conf_.bucket, .key = get_key(path)}).resp; + if (response.ok()) { + return 0; + } + if (response.status.code == ObjectStorageStatus::NOT_FOUND) { + return 1; + } + return -1; } int S3Accessor::abort_multipart_upload(const std::string& path, const std::string& upload_id) { @@ -552,9 +578,9 @@ int S3Accessor::abort_multipart_upload(const std::string& path, const std::strin int ret = obj_client_ ->abort_multipart_upload({.bucket = conf_.bucket, .key = get_key(path)}, upload_id) - .ret; - static_assert(ObjectStorageResponse::OK == 0); - if (ret == ObjectStorageResponse::OK || ret == ObjectStorageResponse::NOT_FOUND) { + .status.code; + static_assert(ObjectStorageStatus::OK == 0); + if (ret == ObjectStorageStatus::OK || ret == ObjectStorageStatus::NOT_FOUND) { return 0; } LOG_WARNING("fail abort multipart upload") @@ -565,11 +591,11 @@ int S3Accessor::abort_multipart_upload(const std::string& path, const std::strin } int S3Accessor::get_life_cycle(int64_t* expiration_days) { - return obj_client_->get_life_cycle(conf_.bucket, expiration_days).ret; + return obj_client_->get_life_cycle(conf_.bucket, expiration_days).status.code; } int S3Accessor::check_versioning() { - return obj_client_->check_versioning(conf_.bucket).ret; + return obj_client_->check_versioning(conf_.bucket).status.code; } int GcsAccessor::delete_prefix_impl(const std::string& path_prefix, int64_t expiration_time) { @@ -580,8 +606,17 @@ int GcsAccessor::delete_prefix_impl(const std::string& path_prefix, int64_t expi int skip = 0; int64_t del_nonexisted = 0; int del = 0; - auto iter = obj_client_->list_objects({conf_.bucket, get_key(path_prefix)}); - for (auto obj = iter->next(); obj.has_value(); obj = iter->next()) { + ObjectListIterator iter(obj_client_, {.bucket = conf_.bucket, .prefix = get_key(path_prefix)}); + for (;;) { + auto result = iter.next(); + if (!result.results_.has_value()) { + if (result.resp.status.code != ObjectStorageStatus::NOT_FOUND && + result.resp.status.code != ObjectStorageStatus::OK) { + ret = result.resp.status.code; + } + break; + } + auto& obj = *result.results_; if (!(++cnt % 100)) { LOG_INFO("loop delete prefix") .tag("uri", to_uri(path_prefix)) @@ -590,17 +625,18 @@ int GcsAccessor::delete_prefix_impl(const std::string& path_prefix, int64_t expi .tag("del_nonexisted", del_nonexisted) .tag("skipped", skip); } - if (expiration_time > 0 && obj->mtime_s > expiration_time) { + if (expiration_time > 0 && obj.mtime_s > expiration_time) { skip++; continue; } del++; // FIXME(plat1ko): Delete objects by batch with genuine GCS client - int del_ret = obj_client_->delete_object({conf_.bucket, obj->key}).ret; - del_nonexisted += (del_ret == ObjectStorageResponse::NOT_FOUND); - static_assert(ObjectStorageResponse::OK == 0); - if (del_ret != ObjectStorageResponse::OK && del_ret != ObjectStorageResponse::NOT_FOUND) { + int del_ret = obj_client_->delete_object({.bucket = conf_.bucket, .key = obj.file_path}) + .status.code; + del_nonexisted += (del_ret == ObjectStorageStatus::NOT_FOUND); + static_assert(ObjectStorageStatus::OK == 0); + if (del_ret != ObjectStorageStatus::OK && del_ret != ObjectStorageStatus::NOT_FOUND) { ret = del_ret; } } @@ -612,7 +648,7 @@ int GcsAccessor::delete_prefix_impl(const std::string& path_prefix, int64_t expi .tag("del_nonexisted", del_nonexisted) .tag("skipped", skip); - if (!iter->is_valid()) { + if (!iter.is_valid()) { return -1; } diff --git a/cloud/src/recycler/s3_accessor.h b/cloud/src/recycler/s3_accessor.h index 28d12c7f9808db..aac4def278612c 100644 --- a/cloud/src/recycler/s3_accessor.h +++ b/cloud/src/recycler/s3_accessor.h @@ -18,14 +18,14 @@ #pragma once #include -#include #include #include #include #include "cpp/aws_common.h" -#include "recycler/obj_storage_client.h" +#include "cpp/client/auth/aws_credential_factory.h" +#include "cpp/client/obj_storage_client.h" #include "recycler/storage_vault_accessor.h" namespace Aws::S3 { @@ -41,19 +41,6 @@ namespace cloud { class ObjectStoreInfoPB; class SimpleThreadPool; -namespace s3_bvar { -extern bvar::LatencyRecorder s3_get_latency; -extern bvar::LatencyRecorder s3_put_latency; -extern bvar::LatencyRecorder s3_delete_object_latency; -extern bvar::LatencyRecorder s3_delete_objects_latency; -extern bvar::LatencyRecorder s3_head_latency; -extern bvar::LatencyRecorder s3_multi_part_upload_latency; -extern bvar::LatencyRecorder s3_list_latency; -extern bvar::LatencyRecorder s3_list_object_versions_latency; -extern bvar::LatencyRecorder s3_get_bucket_version_latency; -extern bvar::LatencyRecorder s3_copy_object_latency; -}; // namespace s3_bvar - class S3Environment { public: S3Environment(const S3Environment&) = delete; @@ -153,21 +140,14 @@ class S3Accessor : public StorageVaultAccessor { int check_versioning(); protected: + static RecursiveDeleteOptions make_recursive_delete_options( + int64_t expiration_time, std::shared_ptr pool); + int list_prefix(const std::string& path_prefix, std::unique_ptr* res); virtual int delete_prefix_impl(const std::string& path_prefix, int64_t expiration_time = 0); - std::shared_ptr _get_aws_credentials_provider_v1( - const S3Conf& s3_conf); - - std::shared_ptr _get_aws_credentials_provider_v2( - const S3Conf& s3_conf); - - std::shared_ptr _create_credentials_provider( - CredProviderType type); - - std::shared_ptr get_aws_credentials_provider( - const S3Conf& s3_conf); + AwsCredentialResult create_aws_credentials_provider(const S3Conf& s3_conf); std::string get_key(const std::string& relative_path) const; std::string to_uri(const std::string& relative_path) const; diff --git a/cloud/src/recycler/s3_obj_client.cpp b/cloud/src/recycler/s3_obj_client.cpp deleted file mode 100644 index 3f299d0fef03e5..00000000000000 --- a/cloud/src/recycler/s3_obj_client.cpp +++ /dev/null @@ -1,449 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "recycler/s3_obj_client.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -#include "common/config.h" -#include "common/logging.h" -#include "common/stopwatch.h" -#include "cpp/obj_retry_strategy.h" -#include "cpp/sync_point.h" -#include "cpp/token_bucket_rate_limiter.h" -#include "recycler/s3_accessor.h" -#include "recycler/util.h" - -namespace doris::cloud { - -[[maybe_unused]] static Aws::Client::AWSError s3_error_factory() { - return {Aws::S3::S3Errors::INTERNAL_FAILURE, "exceeds limit", "exceeds limit", false}; -} - -void record_s3_request_failed(const Aws::S3::S3Error& error) { - doris::record_object_request_failed(static_cast(error.GetResponseCode())); -} - -template -auto s3_rate_limit(S3RateLimitType op, Func callback) -> decltype(callback()) { - using T = decltype(callback()); - // Fault injection for testing rate limit handling in recycler - if (config::enable_s3_rate_limit_inject && op == S3RateLimitType::PUT) { - if (rand() % 100 < config::s3_rate_limit_inject_probility) { - return T(s3_error_factory()); - } - } - if (!config::enable_s3_rate_limiter) { - return callback(); - } - auto sleep_duration = - doris::apply_s3_rate_limit(op, AccessorRateLimiter::instance().rate_limiter(op), - config::s3_rate_limiter_log_interval); - if (sleep_duration < 0) { - return T(s3_error_factory()); - } - return callback(); -} - -template -auto s3_get_rate_limit(Func callback) -> decltype(callback()) { - return s3_rate_limit(S3RateLimitType::GET, std::move(callback)); -} - -template -auto s3_put_rate_limit(Func callback) -> decltype(callback()) { - return s3_rate_limit(S3RateLimitType::PUT, std::move(callback)); -} - -class S3ObjListIterator final : public ObjectListIterator { -public: - S3ObjListIterator(std::shared_ptr client, std::string bucket, - std::string prefix, std::string endpoint) - : client_(std::move(client)), endpoint_(std::move(endpoint)) { - req_.WithBucket(std::move(bucket)).WithPrefix(std::move(prefix)); - TEST_SYNC_POINT_CALLBACK("S3ObjListIterator", &req_); - } - - ~S3ObjListIterator() override = default; - - bool is_valid() override { return is_valid_; } - - bool has_next() override { - if (!is_valid_) { - return false; - } - - if (!results_.empty()) { - return true; - } - - if (!has_more_) { - return false; - } - - auto outcome = s3_get_rate_limit([&]() { - SCOPED_BVAR_LATENCY(s3_bvar::s3_list_latency); - return client_->ListObjectsV2(req_); - }); - - const auto& request_id = outcome.IsSuccess() ? outcome.GetResult().GetRequestId() - : outcome.GetError().GetRequestId(); - if (!outcome.IsSuccess()) { - // Treat NoSuchKey as empty response for compatibility with some S3-compatible storage providers - // e.g. TOS by ByteDance Cloud (Volcano Engine) - if (outcome.GetError().GetErrorType() == Aws::S3::S3Errors::NO_SUCH_KEY) { - LOG_INFO("NoSuchKey error when listing objects, treat as empty response") - .tag("endpoint", endpoint_) - .tag("bucket", req_.GetBucket()) - .tag("prefix", req_.GetPrefix()) - .tag("request_id", request_id); - has_more_ = false; - return false; - } - - record_s3_request_failed(outcome.GetError()); - LOG_WARNING("failed to list objects") - .tag("endpoint", endpoint_) - .tag("bucket", req_.GetBucket()) - .tag("prefix", req_.GetPrefix()) - .tag("responseCode", static_cast(outcome.GetError().GetResponseCode())) - .tag("error", outcome.GetError().GetMessage()) - .tag("request_id", request_id); - is_valid_ = false; - return false; - } - - if (outcome.GetResult().GetIsTruncated() && - outcome.GetResult().GetNextContinuationToken().empty()) { - LOG_WARNING("failed to list objects, isTruncated but no continuation token") - .tag("endpoint", endpoint_) - .tag("bucket", req_.GetBucket()) - .tag("prefix", req_.GetPrefix()) - .tag("request_id", request_id); - - is_valid_ = false; - return false; - } - - has_more_ = outcome.GetResult().GetIsTruncated(); - req_.SetContinuationToken(std::move( - const_cast(outcome.GetResult().GetNextContinuationToken()))); - - auto&& content = outcome.GetResult().GetContents(); - // clang-format off - DCHECK(!(has_more_ && req_.GetContinuationToken().empty())) - << "has_more=" << has_more_ - << " token=" << req_.GetContinuationToken() - << " request_id=" << request_id; - // clang-format on - if (has_more_ && req_.GetContinuationToken().empty()) { - LOG(ERROR) << "it is impossible to have more results but no continuation token"; - has_more_ = false; - } - if (has_more_ && content.empty()) { - LOG(INFO) << "Empty page with more results (possible concurrent deletion), continuing" - << " request_id=" << request_id; - return has_next(); - } - - results_.reserve(content.size()); - for (auto&& obj : std::ranges::reverse_view(content)) { - DCHECK(obj.GetKey().starts_with(req_.GetPrefix())) - << obj.GetKey() << ' ' << req_.GetPrefix(); - results_.emplace_back( - ObjectMeta {.key = std::move(const_cast(obj.GetKey())), - .size = obj.GetSize(), - .mtime_s = obj.GetLastModified().Seconds()}); - } - - return !results_.empty(); - } - - std::optional next() override { - std::optional res; - if (!has_next()) { - return res; - } - - res = std::move(results_.back()); - results_.pop_back(); - return res; - } - -private: - std::shared_ptr client_; - Aws::S3::Model::ListObjectsV2Request req_; - std::vector results_; - std::string endpoint_; - bool is_valid_ {true}; - bool has_more_ {true}; -}; - -static constexpr size_t MaxDeleteBatch = 1000; - -S3ObjClient::~S3ObjClient() = default; - -ObjectStorageResponse S3ObjClient::put_object(ObjectStoragePathRef path, std::string_view stream) { - Aws::S3::Model::PutObjectRequest request; - request.WithBucket(path.bucket).WithKey(path.key); - auto input = Aws::MakeShared("S3Accessor"); - *input << stream; - request.SetBody(input); - auto outcome = s3_put_rate_limit([&]() { - SCOPED_BVAR_LATENCY(s3_bvar::s3_put_latency); - return s3_client_->PutObject(request); - }); - if (!outcome.IsSuccess()) { - record_s3_request_failed(outcome.GetError()); - LOG_WARNING("failed to put object") - .tag("endpoint", endpoint_) - .tag("bucket", path.bucket) - .tag("key", path.key) - .tag("responseCode", static_cast(outcome.GetError().GetResponseCode())) - .tag("error", outcome.GetError().GetMessage()) - .tag("request_id", outcome.GetError().GetRequestId()); - return -1; - } - return 0; -} - -ObjectStorageResponse S3ObjClient::head_object(ObjectStoragePathRef path, ObjectMeta* res) { - Aws::S3::Model::HeadObjectRequest request; - request.WithBucket(path.bucket).WithKey(path.key); - auto outcome = s3_get_rate_limit([&]() { - SCOPED_BVAR_LATENCY(s3_bvar::s3_head_latency); - return s3_client_->HeadObject(request); - }); - if (outcome.IsSuccess()) { - res->key = path.key; - res->size = outcome.GetResult().GetContentLength(); - res->mtime_s = outcome.GetResult().GetLastModified().Seconds(); - return 0; - } else if (outcome.GetError().GetResponseCode() == Aws::Http::HttpResponseCode::NOT_FOUND) { - return 1; - } else { - record_s3_request_failed(outcome.GetError()); - LOG_WARNING("failed to head object") - .tag("endpoint", endpoint_) - .tag("bucket", path.bucket) - .tag("key", path.key) - .tag("responseCode", static_cast(outcome.GetError().GetResponseCode())) - .tag("error", outcome.GetError().GetMessage()) - .tag("request_id", outcome.GetError().GetRequestId()); - return -1; - } -} - -std::unique_ptr S3ObjClient::list_objects(ObjectStoragePathRef path) { - return std::make_unique(s3_client_, path.bucket, path.key, endpoint_); -} - -ObjectStorageResponse S3ObjClient::delete_objects(const std::string& bucket, - std::vector keys, - ObjClientOptions option) { - if (keys.empty()) { - return {0}; - } - - Aws::S3::Model::DeleteObjectsRequest delete_request; - delete_request.SetBucket(bucket); - - auto issue_delete = [&bucket, &delete_request, - this](std::vector objects) -> int { - if (objects.size() == 1) { - return delete_object({.bucket = bucket, .key = objects[0].GetKey()}).ret; - } - - Aws::S3::Model::Delete del; - del.WithObjects(std::move(objects)).SetQuiet(true); - delete_request.SetDelete(std::move(del)); - auto delete_outcome = s3_put_rate_limit([&]() { - SCOPED_BVAR_LATENCY(s3_bvar::s3_delete_objects_latency); - return s3_client_->DeleteObjects(delete_request); - }); - if (!delete_outcome.IsSuccess()) { - record_s3_request_failed(delete_outcome.GetError()); - LOG_WARNING("failed to delete objects") - .tag("endpoint", endpoint_) - .tag("bucket", bucket) - .tag("key[0]", delete_request.GetDelete().GetObjects().front().GetKey()) - .tag("responseCode", - static_cast(delete_outcome.GetError().GetResponseCode())) - .tag("error", delete_outcome.GetError().GetMessage()) - .tag("request_id", delete_outcome.GetError().GetRequestId()); - return -1; - } - - return 0; - }; - - int ret = 0; - // `DeleteObjectsRequest` can only contain 1000 keys at most. - std::vector objects; - - size_t delete_batch_size = MaxDeleteBatch; - TEST_INJECTION_POINT_CALLBACK("S3ObjClient::delete_objects", &delete_batch_size); - - // std::views::chunk(1000) - for (auto&& key : keys) { - objects.emplace_back().SetKey(std::move(key)); - if (objects.size() < delete_batch_size) { - continue; - } - - ret = issue_delete(std::move(objects)); - if (ret != 0) { - return {ret}; - } - } - - if (!objects.empty()) { - ret = issue_delete(std::move(objects)); - } - - return {ret}; -} - -ObjectStorageResponse S3ObjClient::delete_object(ObjectStoragePathRef path) { - Aws::S3::Model::DeleteObjectRequest request; - request.WithBucket(path.bucket).WithKey(path.key); - auto outcome = s3_put_rate_limit([&]() { - SCOPED_BVAR_LATENCY(s3_bvar::s3_delete_object_latency); - return s3_client_->DeleteObject(request); - }); - TEST_SYNC_POINT_CALLBACK("S3ObjClient::delete_object", &outcome); - if (!outcome.IsSuccess()) { - if (outcome.GetError().GetResponseCode() == Aws::Http::HttpResponseCode::NOT_FOUND) { - return {ObjectStorageResponse::NOT_FOUND, outcome.GetError().GetMessage()}; - } - record_s3_request_failed(outcome.GetError()); - LOG_WARNING("failed to delete object") - .tag("endpoint", endpoint_) - .tag("bucket", path.bucket) - .tag("key", path.key) - .tag("responseCode", static_cast(outcome.GetError().GetResponseCode())) - .tag("error", outcome.GetError().GetMessage()) - .tag("exception", outcome.GetError().GetExceptionName()) - .tag("request_id", outcome.GetError().GetRequestId()); - return {ObjectStorageResponse::UNDEFINED, outcome.GetError().GetMessage()}; - } - return {ObjectStorageResponse::OK}; -} - -ObjectStorageResponse S3ObjClient::delete_objects_recursively(ObjectStoragePathRef path, - ObjClientOptions option, - int64_t expiration_time) { - return delete_objects_recursively_(path, option, expiration_time, MaxDeleteBatch); -} - -ObjectStorageResponse S3ObjClient::get_life_cycle(const std::string& bucket, - int64_t* expiration_days) { - Aws::S3::Model::GetBucketLifecycleConfigurationRequest request; - request.SetBucket(bucket); - - auto outcome = s3_get_rate_limit( - [&]() { return s3_client_->GetBucketLifecycleConfiguration(request); }); - bool has_lifecycle = false; - if (outcome.IsSuccess()) { - const auto& rules = outcome.GetResult().GetRules(); - for (const auto& rule : rules) { - if (rule.NoncurrentVersionExpirationHasBeenSet()) { - has_lifecycle = true; - *expiration_days = rule.GetNoncurrentVersionExpiration().GetNoncurrentDays(); - } - } - } else { - record_s3_request_failed(outcome.GetError()); - LOG_WARNING("Err for check interval: failed to get bucket lifecycle") - .tag("endpoint", endpoint_) - .tag("bucket", bucket) - .tag("responseCode", static_cast(outcome.GetError().GetResponseCode())) - .tag("error", outcome.GetError().GetMessage()) - .tag("request_id", outcome.GetError().GetRequestId()); - return -1; - } - - if (!has_lifecycle) { - LOG_WARNING("Err for check interval: bucket doesn't have lifecycle configuration") - .tag("endpoint", endpoint_) - .tag("bucket", bucket); - return -1; - } - return 0; -} - -ObjectStorageResponse S3ObjClient::check_versioning(const std::string& bucket) { - Aws::S3::Model::GetBucketVersioningRequest request; - request.SetBucket(bucket); - auto outcome = s3_get_rate_limit([&]() { return s3_client_->GetBucketVersioning(request); }); - - if (outcome.IsSuccess()) { - const auto& versioning_configuration = outcome.GetResult().GetStatus(); - if (versioning_configuration != Aws::S3::Model::BucketVersioningStatus::Enabled) { - LOG_WARNING("Err for check interval: bucket doesn't enable bucket versioning") - .tag("endpoint", endpoint_) - .tag("bucket", bucket); - return -1; - } - } else { - record_s3_request_failed(outcome.GetError()); - LOG_WARNING("Err for check interval: failed to get status of bucket versioning") - .tag("endpoint", endpoint_) - .tag("bucket", bucket) - .tag("responseCode", static_cast(outcome.GetError().GetResponseCode())) - .tag("error", outcome.GetError().GetMessage()) - .tag("request_id", outcome.GetError().GetRequestId()); - return -1; - } - return 0; -} - -ObjectStorageResponse S3ObjClient::abort_multipart_upload(ObjectStoragePathRef path, - const std::string& upload_id) { - Aws::S3::Model::AbortMultipartUploadRequest request; - request.WithBucket(path.bucket).WithKey(path.key).WithUploadId(upload_id); - auto outcome = s3_put_rate_limit([&]() { return s3_client_->AbortMultipartUpload(request); }); - if (!outcome.IsSuccess()) { - if (outcome.GetError().GetResponseCode() == Aws::Http::HttpResponseCode::NOT_FOUND) { - return {ObjectStorageResponse::OK}; - } - record_s3_request_failed(outcome.GetError()); - LOG_WARNING("failed to abort multipart upload") - .tag("endpoint", endpoint_) - .tag("bucket", path.bucket) - .tag("key", path.key) - .tag("upload_id", upload_id) - .tag("responseCode", static_cast(outcome.GetError().GetResponseCode())) - .tag("error", outcome.GetError().GetMessage()) - .tag("exception", outcome.GetError().GetExceptionName()) - .tag("request_id", outcome.GetError().GetRequestId()); - return {ObjectStorageResponse::UNDEFINED, outcome.GetError().GetMessage()}; - } - return {ObjectStorageResponse::OK}; -} - -} // namespace doris::cloud diff --git a/cloud/src/recycler/s3_obj_client.h b/cloud/src/recycler/s3_obj_client.h deleted file mode 100644 index e53564b6c9f708..00000000000000 --- a/cloud/src/recycler/s3_obj_client.h +++ /dev/null @@ -1,64 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include - -#include "recycler/obj_storage_client.h" - -namespace Aws::S3 { -class S3Client; -} // namespace Aws::S3 - -namespace doris::cloud { - -class S3ObjClient final : public ObjStorageClient { -public: - S3ObjClient(std::shared_ptr client, std::string endpoint) - : s3_client_(std::move(client)), endpoint_(std::move(endpoint)) {} - ~S3ObjClient() override; - - ObjectStorageResponse put_object(ObjectStoragePathRef path, std::string_view stream) override; - - ObjectStorageResponse head_object(ObjectStoragePathRef path, ObjectMeta* res) override; - - std::unique_ptr list_objects(ObjectStoragePathRef path) override; - - ObjectStorageResponse delete_objects(const std::string& bucket, std::vector keys, - ObjClientOptions option) override; - - ObjectStorageResponse delete_object(ObjectStoragePathRef path) override; - - ObjectStorageResponse delete_objects_recursively(ObjectStoragePathRef path, - ObjClientOptions option, - int64_t expiration_time = 0) override; - - ObjectStorageResponse get_life_cycle(const std::string& bucket, - int64_t* expiration_days) override; - - ObjectStorageResponse check_versioning(const std::string& bucket) override; - - ObjectStorageResponse abort_multipart_upload(ObjectStoragePathRef path, - const std::string& upload_id) override; - -private: - std::shared_ptr s3_client_; - std::string endpoint_; -}; - -} // namespace doris::cloud \ No newline at end of file diff --git a/cloud/test/recycler_batch_delete_test.cpp b/cloud/test/recycler_batch_delete_test.cpp index ffb47c8745c859..7b382d8c30055d 100644 --- a/cloud/test/recycler_batch_delete_test.cpp +++ b/cloud/test/recycler_batch_delete_test.cpp @@ -19,385 +19,297 @@ #include #include -#include +#include #include +#include #include #include "common/config.h" -#include "common/logging.h" #include "common/simple_thread_pool.h" -#include "recycler/obj_storage_client.h" +#include "cpp/client/obj_storage_client.h" +#include "recycler/s3_accessor.h" -using namespace doris; +namespace doris { +namespace { -namespace doris::cloud { - -// Mock ObjectListIterator for testing -class MockObjectListIterator : public ObjectListIterator { +class MockObjStorageBackend final : public ObjStorageBackend { public: - MockObjectListIterator(std::vector objects, int fail_after = -1) - : objects_(std::move(objects)), fail_after_(fail_after) {} - - bool is_valid() override { return is_valid_; } - - bool has_next() override { - if (!is_valid_) return false; - return current_index_ < objects_.size(); + MockObjStorageBackend(std::vector objects, size_t batch_size, + int iterator_fail_after = -1) + : objects_(std::move(objects)), + batch_size_(batch_size), + iterator_fail_after_(iterator_fail_after) {} + + ObjectStorageUploadResponse create_multipart_upload(const ObjectStoragePathOptions&) override { + return {.resp = ObjectStorageResponse::OK()}; } - - std::optional next() override { - if (!is_valid_ || current_index_ >= objects_.size()) { - return std::nullopt; - } - - // Simulate iterator becoming invalid after certain number of calls - if (fail_after_ >= 0 && static_cast(current_index_) >= fail_after_) { - is_valid_ = false; - return std::nullopt; - } - - return objects_[current_index_++]; + ObjectStorageResponse put_object(const ObjectStoragePathOptions&, std::string_view) override { + return ObjectStorageResponse::OK(); } - - void set_invalid() { is_valid_ = false; } - -private: - std::vector objects_; - size_t current_index_ = 0; - bool is_valid_ = true; - int fail_after_ = -1; // -1 means never fail -}; - -// Mock ObjStorageClient for testing delete_objects_recursively_ -class MockObjStorageClient : public ObjStorageClient { -public: - MockObjStorageClient(std::vector objects, int iterator_fail_after = -1) - : objects_(std::move(objects)), iterator_fail_after_(iterator_fail_after) {} - - ObjectStorageResponse put_object(ObjectStoragePathRef path, std::string_view stream) override { - return {0}; + ObjectStorageUploadResponse upload_part(const ObjectStoragePathOptions&, std::string_view, + int) override { + return {.resp = ObjectStorageResponse::OK()}; } - - ObjectStorageResponse head_object(ObjectStoragePathRef path, ObjectMeta* res) override { - return {0}; + ObjectStorageResponse complete_multipart_upload( + const ObjectStoragePathOptions&, const std::vector&) override { + return ObjectStorageResponse::OK(); } - - std::unique_ptr list_objects(ObjectStoragePathRef path) override { - return std::make_unique(objects_, iterator_fail_after_); + ObjectStorageHeadResponse head_object(const ObjectStoragePathOptions&) override { + return {.resp = ObjectStorageResponse::OK()}; } - - ObjectStorageResponse delete_objects(const std::string& bucket, std::vector keys, - ObjClientOptions option) override { - delete_calls_++; - total_keys_deleted_ += keys.size(); - - // Simulate delete failure if configured - if (fail_delete_after_ >= 0 && delete_calls_ > fail_delete_after_) { - return {-1, "simulated delete failure"}; + ObjectStorageResponse get_object(const ObjectStoragePathOptions&, void*, size_t, size_t, + size_t*) override { + return ObjectStorageResponse::OK(); + } + ObjectStorageListPage list_objects(const ObjectStoragePathOptions&, + std::string_view continuation_token) override { + ++list_calls_; + const size_t index = + continuation_token.empty() + ? 0 + : static_cast(std::stoull(std::string(continuation_token))); + if (iterator_fail_after_ >= 0 && index >= static_cast(iterator_fail_after_)) { + return {.resp = {.status = {TStatusCode::INTERNAL_ERROR, "simulated list failure"}}}; } - - return {0}; + ObjectStorageListPage page {.resp = ObjectStorageResponse::OK()}; + if (index < objects_.size()) { + page.objects.emplace_back(objects_[index]); + page.has_more = index + 1 < objects_.size(); + if (page.has_more) { + page.continuation_token = std::to_string(index + 1); + } + } + return page; } - - ObjectStorageResponse delete_object(ObjectStoragePathRef path) override { return {0}; } - - ObjectStorageResponse delete_objects_recursively(ObjectStoragePathRef path, - ObjClientOptions option, - int64_t expiration_time = 0) override { - return delete_objects_recursively_(path, option, expiration_time, 1000); + ObjectStorageResponse delete_objects(const ObjectStoragePathOptions&, + std::vector keys) override { + const int call = delete_calls_.fetch_add(1); + if (fail_delete_after_.load() >= 0 && call >= fail_delete_after_.load()) { + return {.status = {TStatusCode::INTERNAL_ERROR, "simulated delete failure"}}; + } + std::lock_guard lock(deleted_keys_mutex_); + deleted_keys_.insert(deleted_keys_.end(), keys.begin(), keys.end()); + return ObjectStorageResponse::OK(); } - - ObjectStorageResponse get_life_cycle(const std::string& bucket, - int64_t* expiration_days) override { - return {0}; + ObjectStorageResponse delete_object(const ObjectStoragePathOptions&) override { + return ObjectStorageResponse::OK(); } - - ObjectStorageResponse check_versioning(const std::string& bucket) override { return {0}; } - - ObjectStorageResponse abort_multipart_upload(ObjectStoragePathRef path, - const std::string& upload_id) override { - return {0}; + std::string generate_presigned_url(const ObjectStoragePathOptions&, int64_t) override { + return {}; + } + ObjStorageCapabilities capabilities() const override { + return {.max_delete_batch = batch_size_}; } - // Test helper methods - int get_delete_calls() const { return delete_calls_; } - size_t get_total_keys_deleted() const { return total_keys_deleted_; } - void set_fail_delete_after(int n) { fail_delete_after_ = n; } + int delete_calls() const { return delete_calls_.load(); } + int list_calls() const { return list_calls_.load(); } + const std::vector& deleted_keys() const { return deleted_keys_; } + void fail_delete() { fail_delete_after_ = 0; } private: std::vector objects_; - int iterator_fail_after_ = -1; + size_t batch_size_; + int iterator_fail_after_; + std::atomic list_calls_ {0}; std::atomic delete_calls_ {0}; - std::atomic total_keys_deleted_ {0}; - int fail_delete_after_ = -1; // -1 means never fail + std::atomic fail_delete_after_ {-1}; + std::mutex deleted_keys_mutex_; + std::vector deleted_keys_; }; -class RecyclerBatchDeleteTest : public testing::Test { -protected: - void SetUp() override { - thread_pool_ = std::make_shared(4); - thread_pool_->start(); - } +class TestS3Accessor final : public cloud::S3Accessor { +public: + using cloud::S3Accessor::make_recursive_delete_options; +}; - void TearDown() override { - if (thread_pool_) { - thread_pool_->stop(); - } +class ScopedMaxTasksPerBatch { +public: + explicit ScopedMaxTasksPerBatch(int32_t value) + : original_(cloud::config::recycler_max_tasks_per_batch) { + cloud::config::recycler_max_tasks_per_batch = value; } + ~ScopedMaxTasksPerBatch() { cloud::config::recycler_max_tasks_per_batch = original_; } + +private: + int32_t original_; +}; - std::vector generate_objects(size_t count) { - std::vector objects; - objects.reserve(count); - for (size_t i = 0; i < count; ++i) { - objects.push_back(ObjectMeta { - .key = "test_key_" + std::to_string(i), - .size = 100, - .mtime_s = 0, - }); +class CountingRateLimitPolicy final : public ObjStorageRateLimitPolicy { +public: + CountingRateLimitPolicy(size_t* get_requests, size_t* put_requests) + : get_requests_(get_requests), put_requests_(put_requests) {} + + ObjStorageRateLimitToken acquire(ObjStorageRequestType type, size_t) const override { + if (type == ObjStorageRequestType::GET) { + ++*get_requests_; + } else { + ++*put_requests_; } - return objects; + return {}; } - std::shared_ptr thread_pool_; +private: + size_t* get_requests_; + size_t* put_requests_; }; -// Test 1: Basic batch processing with multiple batches -TEST_F(RecyclerBatchDeleteTest, MultipleBatches) { - // Save original config and set small batch size for testing - int32_t original_config = config::recycler_max_tasks_per_batch; - config::recycler_max_tasks_per_batch = 3; // 3 tasks per batch - - // Create 10 objects, with batch_size=2 (keys per task), max_tasks_per_batch=3 - // Expected: 10 objects / 2 keys per task = 5 tasks - // 5 tasks / 3 tasks per batch = 2 batches (3 tasks + 2 tasks) - auto objects = generate_objects(10); - MockObjStorageClient client(objects); - - ObjClientOptions options; - options.executor = thread_pool_; - - // Use batch_size=2 to create more tasks - auto response = client.delete_objects_recursively_( - {.bucket = "test_bucket", .key = "test_prefix"}, options, 0, 2); - - EXPECT_EQ(response.ret, 0); - EXPECT_EQ(client.get_delete_calls(), 5); // 10 objects / 2 = 5 delete calls - EXPECT_EQ(client.get_total_keys_deleted(), 10); // All 10 keys deleted - - // Restore config - config::recycler_max_tasks_per_batch = original_config; -} - -// Test 2: Iterator becomes invalid during iteration -TEST_F(RecyclerBatchDeleteTest, IteratorInvalidMidway) { - int32_t original_config = config::recycler_max_tasks_per_batch; - config::recycler_max_tasks_per_batch = 100; - - // Create 20 objects but iterator fails after 10 - auto objects = generate_objects(20); - MockObjStorageClient client(objects, 10); // fail_after=10 - - ObjClientOptions options; - options.executor = thread_pool_; - - auto response = client.delete_objects_recursively_( - {.bucket = "test_bucket", .key = "test_prefix"}, options, 0, 5); - - // Should return error because iterator became invalid - EXPECT_EQ(response.ret, -1); - // Should have processed some objects before failure - EXPECT_GT(client.get_total_keys_deleted(), 0); - EXPECT_LT(client.get_total_keys_deleted(), 20); - - config::recycler_max_tasks_per_batch = original_config; +std::vector make_objects(size_t count) { + std::vector objects; + for (size_t i = 0; i < count; ++i) { + objects.push_back({ + .file_path = "test_key_" + std::to_string(i), + .size = 100, + .mtime_s = static_cast(i), + }); + } + return objects; } -// Test 3: Delete operation fails (triggers cancel) -TEST_F(RecyclerBatchDeleteTest, DeleteFailureTriggersCancel) { - int32_t original_config = config::recycler_max_tasks_per_batch; - config::recycler_max_tasks_per_batch = 10; - - auto objects = generate_objects(30); - MockObjStorageClient client(objects); - client.set_fail_delete_after(2); // Fail after 2 successful deletes - - ObjClientOptions options; - options.executor = thread_pool_; - - auto response = client.delete_objects_recursively_( - {.bucket = "test_bucket", .key = "test_prefix"}, options, 0, 5); - - // Should return error because delete failed - EXPECT_EQ(response.ret, -1); +TEST(RecyclerBatchDeleteTest, UsesProviderBatchCapability) { + auto backend = std::make_shared(make_objects(10), 3); + ObjStorageClient client(backend); + auto response = client.delete_objects_recursively({.bucket = "bucket", .prefix = "test_key_"}); - config::recycler_max_tasks_per_batch = original_config; + EXPECT_TRUE(response.ok()); + EXPECT_EQ(backend->delete_calls(), 4); + EXPECT_EQ(backend->deleted_keys().size(), 10); } -// Test 4: Empty object list -TEST_F(RecyclerBatchDeleteTest, EmptyObjectList) { - int32_t original_config = config::recycler_max_tasks_per_batch; - config::recycler_max_tasks_per_batch = 100; +TEST(RecyclerBatchDeleteTest, CountsEveryDeleteObjectsBatch) { + auto backend = std::make_shared(std::vector {}, 3); + size_t get_requests = 0; + size_t put_requests = 0; + ObjStorageClient client( + backend, std::make_shared(&get_requests, &put_requests)); - std::vector empty_objects; - MockObjStorageClient client(empty_objects); + std::vector keys(7, "key"); + auto response = client.delete_objects({.bucket = "bucket"}, std::move(keys)); - ObjClientOptions options; - options.executor = thread_pool_; - - auto response = client.delete_objects_recursively_( - {.bucket = "test_bucket", .key = "test_prefix"}, options, 0, 1000); - - EXPECT_EQ(response.ret, 0); - EXPECT_EQ(client.get_delete_calls(), 0); - EXPECT_EQ(client.get_total_keys_deleted(), 0); - - config::recycler_max_tasks_per_batch = original_config; + EXPECT_TRUE(response.ok()); + EXPECT_EQ(get_requests, 0); + EXPECT_EQ(put_requests, 3); + EXPECT_EQ(backend->delete_calls(), 3); } -// Test 5: Objects less than batch_size -TEST_F(RecyclerBatchDeleteTest, ObjectsLessThanBatchSize) { - int32_t original_config = config::recycler_max_tasks_per_batch; - config::recycler_max_tasks_per_batch = 100; - - auto objects = generate_objects(5); - MockObjStorageClient client(objects); - - ObjClientOptions options; - options.executor = thread_pool_; - - // batch_size=1000, but only 5 objects - auto response = client.delete_objects_recursively_( - {.bucket = "test_bucket", .key = "test_prefix"}, options, 0, 1000); +TEST(RecyclerBatchDeleteTest, CountsEveryRecursiveListAndDeleteRequest) { + auto backend = std::make_shared(make_objects(5), 2); + size_t get_requests = 0; + size_t put_requests = 0; + ObjStorageClient client( + backend, std::make_shared(&get_requests, &put_requests)); - EXPECT_EQ(response.ret, 0); - EXPECT_EQ(client.get_delete_calls(), 1); // All 5 keys in one delete call - EXPECT_EQ(client.get_total_keys_deleted(), 5); + auto response = client.delete_objects_recursively({.bucket = "bucket", .prefix = "test_key_"}); - config::recycler_max_tasks_per_batch = original_config; + EXPECT_TRUE(response.ok()); + EXPECT_EQ(get_requests, 5); + EXPECT_EQ(put_requests, 3); + EXPECT_EQ(backend->delete_calls(), 3); } -// Test 6: Exact batch boundary -TEST_F(RecyclerBatchDeleteTest, ExactBatchBoundary) { - int32_t original_config = config::recycler_max_tasks_per_batch; - config::recycler_max_tasks_per_batch = 2; // 2 tasks per batch - - // 8 objects with batch_size=2 = 4 tasks - // 4 tasks with max_tasks_per_batch=2 = exactly 2 batches - auto objects = generate_objects(8); - MockObjStorageClient client(objects); - - ObjClientOptions options; - options.executor = thread_pool_; - - auto response = client.delete_objects_recursively_( - {.bucket = "test_bucket", .key = "test_prefix"}, options, 0, 2); - - EXPECT_EQ(response.ret, 0); - EXPECT_EQ(client.get_delete_calls(), 4); // 8 / 2 = 4 tasks - EXPECT_EQ(client.get_total_keys_deleted(), 8); - - config::recycler_max_tasks_per_batch = original_config; +TEST(RecyclerBatchDeleteTest, ProductionExecutorRunsMultipleTaskBatches) { + ScopedMaxTasksPerBatch max_tasks_per_batch(2); + auto pool = std::make_shared(4, "recursive_delete_test"); + ASSERT_EQ(pool->start(), 0); + + auto options = TestS3Accessor::make_recursive_delete_options(0, pool); + size_t executor_batches = 0; + auto production_wait = std::move(options.executor.wait); + options.executor.wait = [&executor_batches, + production_wait = std::move(production_wait)]() mutable { + ++executor_batches; + return production_wait(); + }; + + auto backend = std::make_shared(make_objects(10), 2); + ObjStorageClient client(backend); + auto response = + client.delete_objects_recursively({.bucket = "bucket", .prefix = "test_key_"}, options); + + EXPECT_TRUE(response.ok()); + EXPECT_EQ(executor_batches, 3); + EXPECT_EQ(backend->delete_calls(), 5); + EXPECT_EQ(backend->deleted_keys().size(), 10); + EXPECT_EQ(pool->stop(), 0); } -// Test 7: Invalid config value (negative) -TEST_F(RecyclerBatchDeleteTest, InvalidConfigNegative) { - int32_t original_config = config::recycler_max_tasks_per_batch; - config::recycler_max_tasks_per_batch = -1; // Invalid negative value - - auto objects = generate_objects(10); - MockObjStorageClient client(objects); - - ObjClientOptions options; - options.executor = thread_pool_; - - // Should use default value 1000 and still work - auto response = client.delete_objects_recursively_( - {.bucket = "test_bucket", .key = "test_prefix"}, options, 0, 5); - - EXPECT_EQ(response.ret, 0); - EXPECT_EQ(client.get_total_keys_deleted(), 10); - - config::recycler_max_tasks_per_batch = original_config; +TEST(RecyclerBatchDeleteTest, StreamsDeleteTasksWhileListing) { + auto backend = std::make_shared(make_objects(5), 1); + ObjStorageClient client(backend); + std::vector list_calls_at_submit; + RecursiveDeleteOptions options {.max_tasks_per_batch = 1000}; + options.executor.submit = [backend, &list_calls_at_submit](ObjStorageDeleteTask task) { + list_calls_at_submit.push_back(backend->list_calls()); + return task(); + }; + options.executor.wait = [] { return ObjectStorageResponse::OK(); }; + + auto response = + client.delete_objects_recursively({.bucket = "bucket", .prefix = "test_key_"}, options); + + EXPECT_TRUE(response.ok()); + ASSERT_EQ(list_calls_at_submit.size(), 5); + EXPECT_EQ(list_calls_at_submit.front(), 1); + EXPECT_EQ(backend->list_calls(), 5); + EXPECT_EQ(backend->deleted_keys().size(), 5); } -// Test 8: Invalid config value (zero) -TEST_F(RecyclerBatchDeleteTest, InvalidConfigZero) { - int32_t original_config = config::recycler_max_tasks_per_batch; - config::recycler_max_tasks_per_batch = 0; // Invalid zero value - - auto objects = generate_objects(10); - MockObjStorageClient client(objects); - - ObjClientOptions options; - options.executor = thread_pool_; - - // Should use default value 1000 and still work - auto response = client.delete_objects_recursively_( - {.bucket = "test_bucket", .key = "test_prefix"}, options, 0, 5); - - EXPECT_EQ(response.ret, 0); - EXPECT_EQ(client.get_total_keys_deleted(), 10); - - config::recycler_max_tasks_per_batch = original_config; +TEST(RecyclerBatchDeleteTest, ProductionExecutorPropagatesCancellation) { + ScopedMaxTasksPerBatch max_tasks_per_batch(3); + auto pool = std::make_shared(1, "recursive_delete_failure_test"); + ASSERT_EQ(pool->start(), 0); + + auto backend = std::make_shared(make_objects(6), 1); + backend->fail_delete(); + ObjStorageClient client(backend); + auto response = client.delete_objects_recursively( + {.bucket = "bucket", .prefix = "test_key_"}, + TestS3Accessor::make_recursive_delete_options(0, pool)); + + EXPECT_FALSE(response.ok()); + EXPECT_EQ(response.status.msg, "object storage batch deletion did not finish"); + EXPECT_EQ(backend->delete_calls(), 1); + EXPECT_EQ(pool->stop(), 0); } -// Test 9: Expiration time filtering -TEST_F(RecyclerBatchDeleteTest, ExpirationTimeFiltering) { - int32_t original_config = config::recycler_max_tasks_per_batch; - config::recycler_max_tasks_per_batch = 100; - - std::vector objects; - // Create 10 objects: 5 with old mtime (should be deleted), 5 with new mtime (should be kept) - for (int i = 0; i < 5; ++i) { - objects.push_back(ObjectMeta { - .key = "old_key_" + std::to_string(i), - .size = 100, - .mtime_s = 100, // Old timestamp - }); +TEST(RecyclerBatchDeleteTest, InvalidMaxTasksPerBatchUsesDefault) { + auto pool = std::make_shared(1, "recursive_delete_config_test"); + { + ScopedMaxTasksPerBatch max_tasks_per_batch(0); + auto options = TestS3Accessor::make_recursive_delete_options(0, pool); + EXPECT_EQ(options.max_tasks_per_batch, 1000); } - for (int i = 0; i < 5; ++i) { - objects.push_back(ObjectMeta { - .key = "new_key_" + std::to_string(i), - .size = 100, - .mtime_s = 1000, // New timestamp - }); + { + ScopedMaxTasksPerBatch max_tasks_per_batch(-1); + auto options = TestS3Accessor::make_recursive_delete_options(0, pool); + EXPECT_EQ(options.max_tasks_per_batch, 1000); } - - MockObjStorageClient client(objects); - - ObjClientOptions options; - options.executor = thread_pool_; - - // Set expiration_time=500, so only objects with mtime_s <= 500 should be deleted - auto response = client.delete_objects_recursively_( - {.bucket = "test_bucket", .key = "test_prefix"}, options, 500, 1000); - - EXPECT_EQ(response.ret, 0); - EXPECT_EQ(client.get_total_keys_deleted(), 5); // Only old objects deleted - - config::recycler_max_tasks_per_batch = original_config; } -// Test 10: Iterator invalid at start (empty batch scenario) -TEST_F(RecyclerBatchDeleteTest, IteratorInvalidAtStart) { - int32_t original_config = config::recycler_max_tasks_per_batch; - config::recycler_max_tasks_per_batch = 100; - - // Iterator fails immediately (fail_after=0) - auto objects = generate_objects(10); - MockObjStorageClient client(objects, 0); +TEST(RecyclerBatchDeleteTest, FiltersByExpirationTime) { + auto backend = std::make_shared(make_objects(10), 1000); + ObjStorageClient client(backend); + auto response = client.delete_objects_recursively({.bucket = "bucket", .prefix = "test_key_"}, + {.expiration_time = 4}); - ObjClientOptions options; - options.executor = thread_pool_; - - auto response = client.delete_objects_recursively_( - {.bucket = "test_bucket", .key = "test_prefix"}, options, 0, 5); - - // Should return error because iterator was invalid from the start - EXPECT_EQ(response.ret, -1); - EXPECT_EQ(client.get_delete_calls(), 0); + EXPECT_TRUE(response.ok()); + ASSERT_EQ(backend->deleted_keys().size(), 5); + EXPECT_EQ(backend->deleted_keys().back(), "test_key_4"); +} - config::recycler_max_tasks_per_batch = original_config; +TEST(RecyclerBatchDeleteTest, PropagatesListAndDeleteFailures) { + auto list_failure_backend = std::make_shared(make_objects(10), 3, 2); + ObjStorageClient list_failure(list_failure_backend); + EXPECT_FALSE( + list_failure.delete_objects_recursively({.bucket = "bucket", .prefix = "test_key_"}) + .ok()); + + auto delete_failure_backend = std::make_shared(make_objects(3), 3); + delete_failure_backend->fail_delete(); + ObjStorageClient delete_failure(delete_failure_backend); + EXPECT_FALSE( + delete_failure.delete_objects_recursively({.bucket = "bucket", .prefix = "test_key_"}) + .ok()); } -} // namespace doris::cloud +} // namespace +} // namespace doris diff --git a/cloud/test/s3_accessor_mock_test.cpp b/cloud/test/s3_accessor_mock_test.cpp index 5f1e1cc299c0ac..c2f6c4133676f6 100644 --- a/cloud/test/s3_accessor_mock_test.cpp +++ b/cloud/test/s3_accessor_mock_test.cpp @@ -25,8 +25,8 @@ #include "common/config.h" #include "common/logging.h" +#include "cpp/client/s3_obj_storage_backend.h" #include "cpp/sync_point.h" -#include "recycler/s3_obj_client.h" using namespace doris; using namespace Aws::S3::Model; @@ -66,21 +66,21 @@ class MockS3Client : public Aws::S3::S3Client { }; TEST_F(S3AccessorMockTest, list_objects_compatibility) { - // If storage only supports ListObjectsV1, s3_obj_storage_client.list_objects + // If storage only supports ListObjectsV1, s3_obj_storage_backend.list_objects // should return an error. auto mock_s3_client = std::make_shared(); - S3ObjClient s3_obj_client(mock_s3_client, "dummy-endpoint"); + S3ObjStorageBackend s3_obj_client(mock_s3_client, {.endpoint = "dummy-endpoint"}); ListObjectsV2Result result; result.SetIsTruncated(true); EXPECT_CALL(*mock_s3_client, ListObjectsV2(testing::_)) .WillOnce(testing::Return(ListObjectsV2Outcome(result))); - auto response = s3_obj_client.list_objects( - {.bucket = "dummy-bucket", .key = "S3AccessorMockTest/list_objects_compatibility"}); + auto page = s3_obj_client.list_objects( + {.bucket = "dummy-bucket", .key = "S3AccessorMockTest/list_objects_compatibility"}, {}); - EXPECT_FALSE(response->has_next()); - EXPECT_FALSE(response->is_valid()); + EXPECT_FALSE(page.resp.ok()); + EXPECT_TRUE(page.objects.empty()); } } // namespace doris::cloud diff --git a/cloud/test/s3_accessor_test.cpp b/cloud/test/s3_accessor_test.cpp index d63baa2a7b5834..207e5ebe5b9a3f 100644 --- a/cloud/test/s3_accessor_test.cpp +++ b/cloud/test/s3_accessor_test.cpp @@ -219,7 +219,7 @@ TEST_F(S3AccessorTest, s3) { auto* sp = SyncPoint::get_instance(); std::vector guards; sp->set_call_back( - "S3ObjListIterator", + "S3ObjStorageBackend::list_objects", [](auto&& args) { auto* req = try_any_cast(args[0]); req->SetMaxKeys(7); @@ -261,7 +261,7 @@ TEST_F(S3AccessorTest, azure) { auto* sp = SyncPoint::get_instance(); std::vector guards; sp->set_call_back( - "AzureListIterator", + "AzureObjStorageBackend::list_objects", [](auto&& args) { auto* req = try_any_cast(args[0]); req->PageSizeHint = 7; @@ -303,7 +303,7 @@ TEST_F(S3AccessorTest, gcs) { auto* sp = SyncPoint::get_instance(); std::vector guards; sp->set_call_back( - "S3ObjListIterator", + "S3ObjStorageBackend::list_objects", [](auto&& args) { auto* req = try_any_cast(args[0]); req->SetMaxKeys(7); @@ -449,7 +449,7 @@ TEST_F(S3AccessorRoleTest, s3) { auto* sp = SyncPoint::get_instance(); std::vector guards; sp->set_call_back( - "S3ObjListIterator", + "S3ObjStorageBackend::list_objects", [](auto&& args) { auto* req = try_any_cast(args[0]); req->SetMaxKeys(7); diff --git a/common/cpp/CMakeLists.txt b/common/cpp/CMakeLists.txt index 81a600e57bf84d..008b8fb70204f9 100644 --- a/common/cpp/CMakeLists.txt +++ b/common/cpp/CMakeLists.txt @@ -19,4 +19,9 @@ set(LIBRARY_OUTPUT_PATH "${BUILD_DIR}/src/common_cpp") file(GLOB_RECURSE SRC_FILES CONFIGURE_DEPENDS *.cpp) +if(NOT BUILD_AZURE STREQUAL "ON") + list(REMOVE_ITEM SRC_FILES + "${CMAKE_CURRENT_SOURCE_DIR}/client/azure_obj_storage_backend.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/client/auth/azure_auth_factory.cpp") +endif() add_library(CommonCPP STATIC ${SRC_FILES}) diff --git a/common/cpp/client/auth/aws_credential_factory.cpp b/common/cpp/client/auth/aws_credential_factory.cpp new file mode 100644 index 00000000000000..56cb68ab562f47 --- /dev/null +++ b/common/cpp/client/auth/aws_credential_factory.cpp @@ -0,0 +1,110 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "aws_credential_factory.h" + +#include +#include +#include +#include +#include +#include +#include + +#include "cpp/custom_aws_credentials_provider_chain.h" + +namespace doris { +namespace { + +using Provider = Aws::Auth::AWSCredentialsProvider; + +std::shared_ptr create_v2_base_provider(CredProviderType type) { + switch (type) { + case CredProviderType::Env: + return std::make_shared(); + case CredProviderType::SystemProperties: + return std::make_shared(); + case CredProviderType::WebIdentity: + return std::make_shared(); + case CredProviderType::Container: + return std::make_shared( + Aws::Environment::GetEnv("AWS_CONTAINER_CREDENTIALS_RELATIVE_URI").c_str()); + case CredProviderType::Anonymous: + return std::make_shared(); + case CredProviderType::Default: + case CredProviderType::Simple: + return std::make_shared(); + case CredProviderType::InstanceProfile: + return std::make_shared(); + } + return nullptr; +} + +AwsCredentialResult assume_role(const AwsCredentialOptions& options, + std::shared_ptr base_provider) { + auto sts_client = + std::make_shared(base_provider, options.sts_client_config); + return { + .provider = std::make_shared( + options.role_arn, Aws::String(), options.external_id, + Aws::Auth::DEFAULT_CREDS_LOAD_FREQ_SECONDS, std::move(sts_client)), + }; +} + +} // namespace + +AwsCredentialResult AwsCredentialFactory::create(const AwsCredentialOptions& options) { + const bool has_access_key = !options.access_key.empty(); + const bool has_secret_key = !options.secret_key.empty(); + + if (has_access_key && has_secret_key) { + Aws::Auth::AWSCredentials credentials(options.access_key, options.secret_key); + if (!options.session_token.empty()) { + credentials.SetSessionToken(options.session_token); + } + return { + .provider = std::make_shared( + std::move(credentials)), + }; + } + + if (options.version == AwsCredentialProviderVersion::V1) { + if (options.provider_type == CredProviderType::InstanceProfile) { + auto base = std::make_shared(); + return options.role_arn.empty() ? AwsCredentialResult {.provider = std::move(base)} + : assume_role(options, std::move(base)); + } + if (!has_access_key && !has_secret_key && + options.empty_credentials == EmptyCredentialsBehavior::ANONYMOUS) { + return { + .provider = std::make_shared(), + }; + } + return { + .provider = std::make_shared(), + }; + } + + auto base = create_v2_base_provider(options.provider_type); + if (base == nullptr) { + return {.error = "simple credential provider requires access key and secret key"}; + } + return options.role_arn.empty() ? AwsCredentialResult {.provider = std::move(base)} + : assume_role(options, std::move(base)); +} + +} // namespace doris diff --git a/common/cpp/client/auth/aws_credential_factory.h b/common/cpp/client/auth/aws_credential_factory.h new file mode 100644 index 00000000000000..f5d528fcfd0e51 --- /dev/null +++ b/common/cpp/client/auth/aws_credential_factory.h @@ -0,0 +1,67 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include +#include + +#include "cpp/aws_common.h" + +namespace Aws::Auth { +class AWSCredentialsProvider; +} + +namespace doris { + +enum class AwsCredentialProviderVersion { + V1, + V2, +}; + +enum class EmptyCredentialsBehavior { + ANONYMOUS, + DEFAULT_CHAIN, +}; + +struct AwsCredentialOptions { + AwsCredentialProviderVersion version = AwsCredentialProviderVersion::V1; + std::string access_key; + std::string secret_key; + std::string session_token; + CredProviderType provider_type = CredProviderType::Default; + std::string role_arn; + std::string external_id; + EmptyCredentialsBehavior empty_credentials = EmptyCredentialsBehavior::DEFAULT_CHAIN; + Aws::Client::ClientConfiguration sts_client_config; +}; + +struct AwsCredentialResult { + std::shared_ptr provider {}; + std::string error {}; + + explicit operator bool() const { return provider != nullptr; } +}; + +class AwsCredentialFactory { +public: + static AwsCredentialResult create(const AwsCredentialOptions& options); +}; + +} // namespace doris diff --git a/common/cpp/client/auth/azure_auth_factory.cpp b/common/cpp/client/auth/azure_auth_factory.cpp new file mode 100644 index 00000000000000..57bfc7da438bad --- /dev/null +++ b/common/cpp/client/auth/azure_auth_factory.cpp @@ -0,0 +1,44 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "azure_auth_factory.h" + +#include + +namespace doris { + +AzureClientBuildResult AzureAuthFactory::create( + std::string_view container_url, const AzureCredentialOptions& credential, + Azure::Storage::Blobs::BlobClientOptions client_options) { + if (credential.type != AzureCredentialType::SHARED_KEY) { + return {.error = "unsupported Azure credential type"}; + } + if (credential.account_name.empty() || credential.account_key.empty()) { + return {.error = "Azure shared-key credentials require account name and account key"}; + } + + auto shared_key = std::make_shared( + credential.account_name, credential.account_key); + auto client = std::make_shared( + std::string(container_url), shared_key, std::move(client_options)); + return { + .container_client = std::move(client), + .shared_key_credential = std::move(shared_key), + }; +} + +} // namespace doris diff --git a/common/cpp/client/auth/azure_auth_factory.h b/common/cpp/client/auth/azure_auth_factory.h new file mode 100644 index 00000000000000..f41e6b8aefc198 --- /dev/null +++ b/common/cpp/client/auth/azure_auth_factory.h @@ -0,0 +1,56 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include + +namespace Azure::Storage { +class StorageSharedKeyCredential; +} + +namespace doris { + +enum class AzureCredentialType { + SHARED_KEY, +}; + +struct AzureCredentialOptions { + AzureCredentialType type = AzureCredentialType::SHARED_KEY; + std::string account_name; + std::string account_key; +}; + +struct AzureClientBuildResult { + std::shared_ptr container_client {}; + std::shared_ptr shared_key_credential {}; + std::string error {}; + + explicit operator bool() const { return container_client != nullptr; } +}; + +class AzureAuthFactory { +public: + static AzureClientBuildResult create(std::string_view container_url, + const AzureCredentialOptions& credential, + Azure::Storage::Blobs::BlobClientOptions client_options); +}; + +} // namespace doris diff --git a/common/cpp/client/azure_obj_storage_backend.cpp b/common/cpp/client/azure_obj_storage_backend.cpp new file mode 100644 index 00000000000000..d32662612972d4 --- /dev/null +++ b/common/cpp/client/azure_obj_storage_backend.cpp @@ -0,0 +1,508 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "azure_obj_storage_backend.h" + +#include +#include + +#include "cpp/obj_retry_strategy.h" + +using namespace Azure::Storage::Blobs; + +namespace { +std::string wrap_object_storage_path_msg(const doris::ObjectStoragePathOptions& opts) { + return fmt::format("bucket {}, key {}, prefix {}, path {}", opts.bucket, opts.key, opts.prefix, + opts.path.native()); +} + +std::string to_lower_ascii(std::string_view input) { + std::string lowered(input); + std::transform(lowered.begin(), lowered.end(), lowered.begin(), + [](unsigned char ch) { return static_cast(std::tolower(ch)); }); + return lowered; +} + +template +T to_endian(T value) { + if constexpr (std::endian::native == target) { + return value; // No swap needed + } else { + static_assert(std::endian::native == std::endian::big || + std::endian::native == std::endian::little, + "Unsupported endianness"); + return byte_swap(value); + } +} + +inline void encode_fixed32_le(uint8_t* buf, uint32_t val) { + val = to_endian(val); + memcpy(buf, &val, sizeof(val)); +} + +auto base64_encode_part_num(int part_num) { + uint8_t buf[4]; + encode_fixed32_le(buf, static_cast(part_num)); + return Aws::Utils::HashingUtils::Base64Encode({buf, sizeof(buf)}); +} + +constexpr char SAS_TOKEN_URL_TEMPLATE[] = "{}/{}/{}{}"; +constexpr char BlobNotFound[] = "BlobNotFound"; +} // namespace + +namespace doris { + +// As Azure's doc said, the batch size is 256 +// You can find out the num in https://learn.microsoft.com/en-us/rest/api/storageservices/blob-batch?tabs=microsoft-entra-id +// > Each batch request supports a maximum of 256 subrequests. +constexpr size_t BlobBatchMaxOperations = 256; + +bool is_azure_tls_ca_error_message(std::string_view message) { + std::string lower = to_lower_ascii(message); + return lower.find("ssl ca cert") != std::string::npos || + lower.find("peer failed verification") != std::string::npos || + lower.find("unable to get local issuer certificate") != std::string::npos || + lower.find("problem with the ssl ca cert") != std::string::npos; +} + +std::string build_azure_tls_debug_suffix(std::string_view error_message, + std::string_view tls_debug_context) { + if (tls_debug_context.empty() || !is_azure_tls_ca_error_message(error_message)) { + return ""; + } + return fmt::format(", {}", tls_debug_context); +} + +static ObjectStorageResponse make_azure_std_exception_response(const std::exception& e, + const ObjectStoragePathOptions& opts, + std::string_view tls_debug_context) { + auto msg = fmt::format("Azure request failed because {}, path msg {}{}", e.what(), + wrap_object_storage_path_msg(opts), + build_azure_tls_debug_suffix(e.what(), tls_debug_context)); + LOG(WARNING) << msg; + return {.status = ObjectStorageStatus {TStatusCode::INTERNAL_ERROR, std::move(msg)}, + .http_code = 0, + .request_id = ""}; +} + +template +ObjectStorageResponse do_azure_client_call(Func f, const ObjectStoragePathOptions& opts, + std::string_view tls_debug_context) { + try { + f(); + } catch (Azure::Core::RequestFailedException& e) { + doris::record_object_request_failed(static_cast(e.StatusCode)); + auto msg = fmt::format( + "Azure request failed because {}, error msg {}, http code {}, path msg {}{}", + e.what(), e.Message, static_cast(e.StatusCode), + wrap_object_storage_path_msg(opts), + build_azure_tls_debug_suffix(fmt::format("{} {}", e.what(), e.Message), + tls_debug_context)); + LOG(WARNING) << msg; + return {.status = ObjectStorageStatus {TStatusCode::INTERNAL_ERROR, std::move(msg)}, + .http_code = static_cast(e.StatusCode), + .request_id = std::move(e.RequestId)}; + } catch (const std::exception& e) { + return make_azure_std_exception_response(e, opts, tls_debug_context); + } + return ObjectStorageResponse::OK(); +} + +struct AzureBatchDeleter { + AzureBatchDeleter(BlobContainerClient* client, const ObjectStoragePathOptions& opts, + std::string_view tls_debug_context) + : _client(client), + _batch(client->CreateBatch()), + _opts(opts), + _tls_debug_context(tls_debug_context) {} + // Submit one blob to be deleted in `AzureBatchDeleter::execute` + void delete_blob(const std::string& blob_name) { + deferred_resps.emplace_back(_batch.DeleteBlob(blob_name)); + } + ObjectStorageResponse execute() { + if (deferred_resps.empty()) { + return ObjectStorageResponse::OK(); + } + auto resp = do_azure_client_call( + [&]() { + client_bvar::ScopedLatency scoped_latency( + client_bvar::s3_delete_objects_latency); + _client->SubmitBatch(_batch); + }, + _opts, _tls_debug_context); + if (resp.status.code != TStatusCode::OK) { + return resp; + } + + for (auto&& defer_response : deferred_resps) { + try { + auto r = defer_response.GetResponse(); + if (!r.Value.Deleted) { + auto msg = fmt::format("Azure batch delete failed, path msg {}", + wrap_object_storage_path_msg(_opts)); + LOG(WARNING) << msg; + return {.status = ObjectStorageStatus {TStatusCode::INTERNAL_ERROR, + std::move(msg)}, + .http_code = 0, + .request_id = ""}; + } + } catch (Azure::Core::RequestFailedException& e) { + if (Azure::Core::Http::HttpStatusCode::NotFound == e.StatusCode && + 0 == strcmp(e.ErrorCode.c_str(), BlobNotFound)) { + continue; + } + doris::record_object_request_failed(static_cast(e.StatusCode)); + auto msg = fmt::format( + "Azure request failed because {}, error msg {}, http code {}, path msg " + "{}{}", + e.what(), e.Message, static_cast(e.StatusCode), + wrap_object_storage_path_msg(_opts), + build_azure_tls_debug_suffix(fmt::format("{} {}", e.what(), e.Message), + _tls_debug_context)); + LOG(WARNING) << msg; + return {.status = ObjectStorageStatus {TStatusCode::INTERNAL_ERROR, std::move(msg)}, + .http_code = static_cast(e.StatusCode), + .request_id = std::move(e.RequestId)}; + } + } + + return ObjectStorageResponse::OK(); + } + +private: + BlobContainerClient* _client; + BlobContainerBatch _batch; + const ObjectStoragePathOptions& _opts; + std::string_view _tls_debug_context; + std::vector> deferred_resps; +}; + +// Azure would do nothing +ObjectStorageUploadResponse AzureObjStorageBackend::create_multipart_upload( + const ObjectStoragePathOptions& opts) { + return ObjectStorageUploadResponse { + .resp = ObjectStorageResponse::OK(), + }; +} + +ObjectStorageResponse AzureObjStorageBackend::put_object(const ObjectStoragePathOptions& opts, + std::string_view stream) { + auto client = _client->GetBlockBlobClient(opts.key); + return do_azure_client_call( + [&]() { + client_bvar::ScopedLatency scoped_latency(client_bvar::s3_put_latency); + client.UploadFrom(reinterpret_cast(stream.data()), stream.size()); + }, + opts, _config.tls_debug_context); +} + +ObjectStorageUploadResponse AzureObjStorageBackend::upload_part( + const ObjectStoragePathOptions& opts, std::string_view stream, int part_num) { + auto client = _client->GetBlockBlobClient(opts.key); + try { + Azure::Core::IO::MemoryBodyStream memory_body( + reinterpret_cast(stream.data()), stream.size()); + // The blockId must be base64 encoded + client_bvar::ScopedLatency scoped_latency(client_bvar::s3_multi_part_upload_latency); + client.StageBlock(base64_encode_part_num(part_num), memory_body); + } catch (Azure::Core::RequestFailedException& e) { + record_object_request_failed(static_cast(e.StatusCode)); + auto tls_debug_suffix = build_azure_tls_debug_suffix( + fmt::format("{} {}", e.what(), e.Message), _config.tls_debug_context); + auto msg = fmt::format( + "Azure request failed because {}, error msg {}, http code {}, path msg {}{}", + e.what(), e.Message, static_cast(e.StatusCode), + wrap_object_storage_path_msg(opts), tls_debug_suffix); + LOG(WARNING) << msg; + // clang-format off + return { + .resp = { + .status = ObjectStorageStatus {TStatusCode::INTERNAL_ERROR, std::move(msg)}, + .http_code = static_cast(e.StatusCode), + .request_id = std::move(e.RequestId), + }, + }; + // clang-format on + } catch (const std::exception& e) { + return {.resp = make_azure_std_exception_response(e, opts, _config.tls_debug_context)}; + } + return ObjectStorageUploadResponse {.resp = ObjectStorageResponse::OK()}; +} + +ObjectStorageResponse AzureObjStorageBackend::complete_multipart_upload( + const ObjectStoragePathOptions& opts, + const std::vector& completed_parts) { + auto client = _client->GetBlockBlobClient(opts.key); + std::vector string_block_ids; + std::ranges::transform( + completed_parts, std::back_inserter(string_block_ids), + [](const ObjectCompleteMultiPart& i) { return base64_encode_part_num(i.part_num); }); + return do_azure_client_call( + [&]() { + client_bvar::ScopedLatency scoped_latency( + client_bvar::s3_multi_part_upload_latency); + client.CommitBlockList(string_block_ids); + }, + opts, _config.tls_debug_context); +} + +ObjectStorageHeadResponse AzureObjStorageBackend::head_object( + const ObjectStoragePathOptions& opts) { + try { + Models::BlobProperties properties = [&]() { + client_bvar::ScopedLatency scoped_latency(client_bvar::s3_head_latency); + return _client->GetBlockBlobClient(opts.key).GetProperties().Value; + }(); + return {.resp = ObjectStorageResponse::OK(), .file_size = properties.BlobSize}; + } catch (Azure::Core::RequestFailedException& e) { + if (e.StatusCode == Azure::Core::Http::HttpStatusCode::NotFound) { + return ObjectStorageHeadResponse { + .resp = {.status = ObjectStorageStatus {TStatusCode::NOT_FOUND, ""}, + .http_code = static_cast(e.StatusCode), + .request_id = std::move(e.RequestId)}, + }; + } + record_object_request_failed(static_cast(e.StatusCode)); + auto tls_debug_suffix = build_azure_tls_debug_suffix( + fmt::format("{} {}", e.what(), e.Message), _config.tls_debug_context); + auto msg = fmt::format( + "Azure request failed because {}, error msg {}, http code {}, path msg {}{}", + e.what(), e.Message, static_cast(e.StatusCode), + wrap_object_storage_path_msg(opts), tls_debug_suffix); + LOG(WARNING) << msg << ", request_id=" << e.RequestId; + return ObjectStorageHeadResponse { + .resp = {.status = + ObjectStorageStatus {TStatusCode::INTERNAL_ERROR, std::move(msg)}, + .http_code = static_cast(e.StatusCode), + .request_id = std::move(e.RequestId)}, + }; + } catch (const std::exception& e) { + return {.resp = make_azure_std_exception_response(e, opts, _config.tls_debug_context)}; + } +} + +ObjectStorageResponse AzureObjStorageBackend::get_object(const ObjectStoragePathOptions& opts, + void* buffer, size_t offset, + size_t bytes_read, size_t* size_return) { + auto client = _client->GetBlockBlobClient(opts.key); + return do_azure_client_call( + [&]() { + DownloadBlobToOptions download_opts; + Azure::Core::Http::HttpRange range {.Offset = static_cast(offset), + .Length = bytes_read}; + download_opts.Range = range; + client_bvar::ScopedLatency scoped_latency(client_bvar::s3_get_latency); + auto resp = client.DownloadTo(reinterpret_cast(buffer), bytes_read, + download_opts); + *size_return = resp.Value.ContentRange.Length.Value(); + }, + opts, _config.tls_debug_context); +} + +ObjectStorageListPage AzureObjStorageBackend::list_objects(const ObjectStoragePathOptions& opts, + std::string_view continuation_token) { + const auto& prefix = opts.prefix.empty() ? opts.key : opts.prefix; + ListBlobsOptions request; + request.Prefix = prefix; + request.PageSizeHint = OBJECT_LIST_PAGE_SIZE; + if (!continuation_token.empty()) { + request.ContinuationToken = std::string(continuation_token); + } + TEST_SYNC_POINT_CALLBACK("AzureObjStorageBackend::list_objects", &request); + + try { + auto response = [&]() { + client_bvar::ScopedLatency scoped_latency(client_bvar::s3_list_latency); + return _client->ListBlobs(request); + }(); + const bool has_more = response.NextPageToken.HasValue(); + auto next_token = has_more ? response.NextPageToken.Value() : std::string {}; + if (has_more && next_token.empty()) { + return { + .resp = {.status = {TStatusCode::INTERNAL_ERROR, + "Azure list response has an empty continuation token"}, + .http_code = 0}, + }; + } + ObjectStorageListPage page {.resp = ObjectStorageResponse::OK(), + .continuation_token = std::move(next_token), + .has_more = has_more}; + page.objects.reserve(response.Blobs.size()); + for (auto&& item : response.Blobs) { + DCHECK(item.Name.starts_with(*request.Prefix)) << item.Name << ' ' << *request.Prefix; + page.objects.emplace_back(ObjectMeta { + .file_path = std::move(item.Name), + .size = item.BlobSize, + // `Azure::DateTime` adds the offset of `SystemClockEpoch` to the given Unix timestamp, + // so here we need to subtract this offset to obtain the Unix timestamp of the mtime. + // https://github.com/Azure/azure-sdk-for-cpp/blob/azure-core_1.12.0/sdk/core/azure-core/inc/azure/core/datetime.hpp#L129 + .mtime_s = duration_cast(item.Details.LastModified - + SystemClockEpoch) + .count()}); + } + return page; + } catch (Azure::Core::RequestFailedException& e) { + record_object_request_failed(static_cast(e.StatusCode)); + auto tls_debug_suffix = build_azure_tls_debug_suffix( + fmt::format("{} {}", e.what(), e.Message), _config.tls_debug_context); + LOG(WARNING) << fmt::format("Azure request failed because {}, url: {}, prefix: {}{}", + e.what(), _client->GetUrl(), request.Prefix.Value(), + tls_debug_suffix); + return { + .resp = {.status = {e.StatusCode == Azure::Core::Http::HttpStatusCode::NotFound + ? TStatusCode::NOT_FOUND + : TStatusCode::INTERNAL_ERROR, + e.Message + tls_debug_suffix}, + .http_code = static_cast(e.StatusCode), + .request_id = std::move(e.RequestId)}, + }; + } catch (std::exception& e) { + LOG(WARNING) << fmt::format("Azure request failed because {}, url: {}, prefix: {}", + e.what(), _client->GetUrl(), request.Prefix.Value()); + return { + .resp = {.status = {TStatusCode::INTERNAL_ERROR, e.what()}, + .http_code = 0, + .request_id = ""}, + }; + } +} + +// As Azure's doc said, the batch size is 256 +// You can find out the num in https://learn.microsoft.com/en-us/rest/api/storageservices/blob-batch?tabs=microsoft-entra-id +// > Each batch request supports a maximum of 256 subrequests. +ObjectStorageResponse AzureObjStorageBackend::delete_objects(const ObjectStoragePathOptions& opts, + std::vector objs) { + // TODO(ByteYue) : use range to adate this code when compiler is ready + // auto chunkedView = objs | std::views::chunk(BlobBatchMaxOperations); + auto begin = std::begin(objs); + auto end = std::end(objs); + + while (begin != end) { + auto deleter = AzureBatchDeleter(_client.get(), opts, _config.tls_debug_context); + auto chunk_end = begin; + size_t batch_size = BlobBatchMaxOperations; + TEST_SYNC_POINT_CALLBACK("AzureObjClient::delete_objects", &batch_size); + TEST_SYNC_POINT_CALLBACK("AzureObjStorageClient::delete_objects", &batch_size); + batch_size = std::max(1, batch_size); + std::advance(chunk_end, + std::min(batch_size, static_cast(std::distance(begin, end)))); + + std::ranges::for_each(std::ranges::subrange(begin, chunk_end), + [&](const std::string& obj) { deleter.delete_blob(obj); }); + begin = chunk_end; + if (auto resp = deleter.execute(); resp.status.code != TStatusCode::OK) { + return resp; + } + } + return ObjectStorageResponse::OK(); +} + +ObjectStorageResponse AzureObjStorageBackend::delete_object(const ObjectStoragePathOptions& opts) { + try { + auto resp = [&]() { + client_bvar::ScopedLatency scoped_latency(client_bvar::s3_delete_object_latency); + return _client->DeleteBlob(opts.key); + }(); + if (!resp.Value.Deleted) { + return { + .status = + ObjectStorageStatus {TStatusCode::IO_ERROR, "Delete azure blob failed"}, + .http_code = 0, + .request_id = "", + }; + } + return ObjectStorageResponse::OK(); + } catch (Azure::Core::RequestFailedException& e) { + if (e.StatusCode == Azure::Core::Http::HttpStatusCode::NotFound && + e.ErrorCode == BlobNotFound) { + return ObjectStorageResponse::OK(); + } + record_object_request_failed(static_cast(e.StatusCode)); + auto tls_debug_suffix = build_azure_tls_debug_suffix( + fmt::format("{} {}", e.what(), e.Message), _config.tls_debug_context); + auto msg = fmt::format( + "Azure request failed because {}, error msg {}, http code {}, path msg {}{}", + e.what(), e.Message, static_cast(e.StatusCode), + wrap_object_storage_path_msg(opts), tls_debug_suffix); + LOG(WARNING) << msg; + return { + .status = ObjectStorageStatus {TStatusCode::INTERNAL_ERROR, std::move(msg)}, + .http_code = static_cast(e.StatusCode), + .request_id = std::move(e.RequestId), + }; + } catch (std::exception& e) { + auto msg = fmt::format("Azure request failed because {}, path msg {}{}", e.what(), + wrap_object_storage_path_msg(opts), + build_azure_tls_debug_suffix(e.what(), _config.tls_debug_context)); + LOG(WARNING) << msg; + return { + .status = ObjectStorageStatus {TStatusCode::INTERNAL_ERROR, std::move(msg)}, + .http_code = 0, + .request_id = "", + }; + } +} + +std::string AzureObjStorageBackend::generate_presigned_url(const ObjectStoragePathOptions& opts, + int64_t expiration_secs) { + Azure::Storage::Sas::BlobSasBuilder sas_builder; + sas_builder.ExpiresOn = + std::chrono::system_clock::now() + std::chrono::seconds(expiration_secs); + sas_builder.BlobContainerName = opts.bucket; + sas_builder.BlobName = opts.key; + sas_builder.Resource = Azure::Storage::Sas::BlobSasResource::Blob; + sas_builder.Protocol = Azure::Storage::Sas::SasProtocol::HttpsOnly; + sas_builder.SetPermissions(Azure::Storage::Sas::BlobSasPermissions::Read); + + auto credential = _credential; + if (credential == nullptr) { + credential = std::make_shared(_config.ak, + _config.sk); + } + std::string sasToken = sas_builder.GenerateSasToken(*credential); + + std::string endpoint = _config.endpoint; + // TODO: config to force use global endpoint + if (false) { + endpoint = fmt::format("https://{}.blob.core.windows.net", _config.ak); + } + auto sasURL = fmt::format(SAS_TOKEN_URL_TEMPLATE, endpoint, opts.bucket, opts.key, sasToken); + if (sasURL.find("://") == std::string::npos) { + sasURL = "https://" + sasURL; + } + return sasURL; +} + +ObjectStorageResponse AzureObjStorageBackend::get_life_cycle(const std::string& /*bucket*/, + int64_t* expiration_days) { + // TODO(plat1ko) + *expiration_days = INT64_MAX; + return ObjectStorageResponse::OK(); +} + +ObjectStorageResponse AzureObjStorageBackend::check_versioning(const std::string& /*bucket*/) { + // TODO(plat1ko) + return ObjectStorageResponse::OK(); +} + +ObjectStorageResponse AzureObjStorageBackend::abort_multipart_upload( + const ObjectStoragePathOptions& opts, const std::string& upload_id) { + // delete uncommitted blobs + // https://learn.microsoft.com/en-us/rest/api/storageservices/delete-blob?tabs=microsoft-entra-id#remarks + return delete_object(opts); +} +} // namespace doris diff --git a/common/cpp/client/azure_obj_storage_backend.h b/common/cpp/client/azure_obj_storage_backend.h new file mode 100644 index 00000000000000..14f626110d4bd3 --- /dev/null +++ b/common/cpp/client/azure_obj_storage_backend.h @@ -0,0 +1,121 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#ifdef USE_AZURE +#include +#include +#endif +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "client_bvar.h" +#include "cpp/obj_retry_strategy.h" +#include "cpp/util.h" +#include "obj_storage_client.h" +#include "s3_common.h" + +namespace Azure::Storage::Blobs { +class BlobContainerClient; +} // namespace Azure::Storage::Blobs + +namespace doris { + +using namespace Azure::Storage::Blobs; + +static const Azure::DateTime SystemClockEpoch {1970, 1, 1}; + +bool is_azure_tls_ca_error_message(std::string_view message); +std::string build_azure_tls_debug_suffix(std::string_view error_message, + std::string_view tls_debug_context); + +class ObjClientHolder; + +class AzureObjStorageBackend final : public ObjStorageBackend { +public: + AzureObjStorageBackend( + std::shared_ptr client, + ObjectClientConfig config, + std::shared_ptr credential = nullptr) + : _config(std::move(config)), + _client(std::move(client)), + _credential(std::move(credential)) {} + ~AzureObjStorageBackend() override = default; + ObjectStorageUploadResponse create_multipart_upload( + const ObjectStoragePathOptions& opts) override; + ObjectStorageResponse put_object(const ObjectStoragePathOptions& opts, + std::string_view stream) override; + ObjectStorageUploadResponse upload_part(const ObjectStoragePathOptions& opts, std::string_view, + int partNum) override; + ObjectStorageResponse complete_multipart_upload( + const ObjectStoragePathOptions& opts, + const std::vector& completed_parts) override; + ObjectStorageHeadResponse head_object(const ObjectStoragePathOptions& opts) override; + ObjectStorageResponse get_object(const ObjectStoragePathOptions& opts, void* buffer, + size_t offset, size_t bytes_read, + size_t* size_return) override; + ObjectStorageListPage list_objects(const ObjectStoragePathOptions& path, + std::string_view continuation_token) override; + ObjectStorageResponse delete_objects(const ObjectStoragePathOptions& opts, + std::vector objs) override; + ObjectStorageResponse delete_object(const ObjectStoragePathOptions& opts) override; + std::string generate_presigned_url(const ObjectStoragePathOptions& opts, + int64_t expiration_secs) override; + ObjectStorageResponse get_life_cycle(const std::string& bucket, + int64_t* expiration_days) override; + + ObjectStorageResponse check_versioning(const std::string& bucket) override; + + ObjectStorageResponse abort_multipart_upload(const ObjectStoragePathOptions& opts, + const std::string& upload_id) override; + ObjStorageCapabilities capabilities() const override { return {.max_delete_batch = 256}; } + +private: + ObjectClientConfig _config; + std::shared_ptr _client; + std::shared_ptr _credential; +}; + +} // namespace doris + +namespace doris::io { +using ::doris::AzureObjStorageBackend; +using ::doris::build_azure_tls_debug_suffix; +using ::doris::is_azure_tls_ca_error_message; +} // namespace doris::io diff --git a/common/cpp/client/client_bvar.cpp b/common/cpp/client/client_bvar.cpp new file mode 100644 index 00000000000000..d98ecbf9baee1e --- /dev/null +++ b/common/cpp/client/client_bvar.cpp @@ -0,0 +1,37 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "client_bvar.h" + +#include + +namespace doris { + +namespace client_bvar { +bvar::LatencyRecorder s3_get_latency("s3_get"); +bvar::LatencyRecorder s3_put_latency("s3_put"); +bvar::LatencyRecorder s3_delete_object_latency("s3_delete_object"); +bvar::LatencyRecorder s3_delete_objects_latency("s3_delete_objects"); +bvar::LatencyRecorder s3_head_latency("s3_head"); +bvar::LatencyRecorder s3_multi_part_upload_latency("s3_multi_part_upload"); +bvar::LatencyRecorder s3_list_latency("s3_list"); +bvar::LatencyRecorder s3_list_object_versions_latency("s3_list_object_versions"); +bvar::LatencyRecorder s3_get_bucket_version_latency("s3_get_bucket_version"); +bvar::LatencyRecorder s3_copy_object_latency("s3_copy_object"); + +} // namespace client_bvar +} // namespace doris diff --git a/common/cpp/client/client_bvar.h b/common/cpp/client/client_bvar.h new file mode 100644 index 00000000000000..dfbcae8f692972 --- /dev/null +++ b/common/cpp/client/client_bvar.h @@ -0,0 +1,58 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include + +namespace doris { + +namespace client_bvar { + +class ScopedLatency { +public: + explicit ScopedLatency(bvar::LatencyRecorder& recorder) + : recorder_(recorder), start_(std::chrono::steady_clock::now()) {} + ~ScopedLatency() { + recorder_ << std::chrono::duration_cast( + std::chrono::steady_clock::now() - start_) + .count(); + } + + ScopedLatency(const ScopedLatency&) = delete; + ScopedLatency& operator=(const ScopedLatency&) = delete; + +private: + bvar::LatencyRecorder& recorder_; + std::chrono::steady_clock::time_point start_; +}; + +extern bvar::LatencyRecorder s3_get_latency; +extern bvar::LatencyRecorder s3_put_latency; +extern bvar::LatencyRecorder s3_delete_object_latency; +extern bvar::LatencyRecorder s3_delete_objects_latency; +extern bvar::LatencyRecorder s3_head_latency; +extern bvar::LatencyRecorder s3_multi_part_upload_latency; +extern bvar::LatencyRecorder s3_list_latency; +extern bvar::LatencyRecorder s3_list_object_versions_latency; +extern bvar::LatencyRecorder s3_get_bucket_version_latency; +extern bvar::LatencyRecorder s3_copy_object_latency; +} // namespace client_bvar + +} // namespace doris diff --git a/common/cpp/client/obj_storage_client.cpp b/common/cpp/client/obj_storage_client.cpp new file mode 100644 index 00000000000000..fe72e6f7a6469f --- /dev/null +++ b/common/cpp/client/obj_storage_client.cpp @@ -0,0 +1,304 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "obj_storage_client.h" + +#include + +#include +#include + +namespace doris { +namespace { + +ObjStorageRateLimitToken acquire_rate_limit( + const std::shared_ptr& policy, ObjStorageRequestType type, + size_t estimated_bytes = 0) { + if (!policy) { + return {}; + } + return policy->acquire(type, estimated_bytes); +} + +} // namespace + +ObjStorageRateLimitToken ObjStorageClient::acquire(ObjStorageRequestType type, + size_t estimated_bytes) const { + return acquire_rate_limit(rate_limit_policy_, type, estimated_bytes); +} + +ObjectStorageUploadResponse ObjStorageClient::create_multipart_upload( + const ObjectStoragePathOptions& opts) { + auto rate_limit = acquire(ObjStorageRequestType::PUT); + if (!rate_limit.resp.ok()) { + return {.resp = std::move(rate_limit.resp)}; + } + return backend_->create_multipart_upload(opts); +} + +ObjectStorageResponse ObjStorageClient::put_object(const ObjectStoragePathOptions& opts, + std::string_view stream) { + auto rate_limit = acquire(ObjStorageRequestType::PUT, stream.size()); + if (!rate_limit.resp.ok()) { + return rate_limit.resp; + } + return backend_->put_object(opts, stream); +} + +ObjectStorageUploadResponse ObjStorageClient::upload_part(const ObjectStoragePathOptions& opts, + std::string_view stream, int part_num) { + auto rate_limit = acquire(ObjStorageRequestType::PUT, stream.size()); + if (!rate_limit.resp.ok()) { + return {.resp = std::move(rate_limit.resp)}; + } + return backend_->upload_part(opts, stream, part_num); +} + +ObjectStorageResponse ObjStorageClient::complete_multipart_upload( + const ObjectStoragePathOptions& opts, + const std::vector& completed_parts) { + auto rate_limit = acquire(ObjStorageRequestType::PUT); + if (!rate_limit.resp.ok()) { + return rate_limit.resp; + } + return backend_->complete_multipart_upload(opts, completed_parts); +} + +ObjectStorageHeadResponse ObjStorageClient::head_object(const ObjectStoragePathOptions& opts) { + auto rate_limit = acquire(ObjStorageRequestType::GET); + if (!rate_limit.resp.ok()) { + return {.resp = std::move(rate_limit.resp)}; + } + return backend_->head_object(opts); +} + +ObjectStorageResponse ObjStorageClient::get_object(const ObjectStoragePathOptions& opts, + void* buffer, size_t offset, size_t bytes_read, + size_t* size_return) { + auto rate_limit = acquire(ObjStorageRequestType::GET, bytes_read); + if (!rate_limit.resp.ok()) { + return rate_limit.resp; + } + auto response = backend_->get_object(opts, buffer, offset, bytes_read, size_return); + if (response.ok()) { + rate_limit.settle_bytes(*size_return); + } + return response; +} + +ObjectStorageListPage ObjStorageClient::list_objects(const ObjectStoragePathOptions& opts, + std::string_view continuation_token) { + auto rate_limit = acquire(ObjStorageRequestType::GET); + if (!rate_limit.resp.ok()) { + return {.resp = std::move(rate_limit.resp)}; + } + return backend_->list_objects(opts, continuation_token); +} + +ObjectStorageResponse ObjectListIterator::has_next() { + if (!is_valid_) { + return { + .status = {TStatusCode::INTERNAL_ERROR, "Iterator is invalid"}, + .http_code = 0, + }; + } + while (next_index_ == objects_.size()) { + if (!has_more_) { + return { + .status = {TStatusCode::END_OF_FILE, "No more results"}, + .http_code = 200, + }; + } + auto page = client_->list_objects(opts_, continuation_token_); + if (!page.resp.ok()) { + is_valid_ = false; + return page.resp; + } + objects_ = std::move(page.objects); + next_index_ = 0; + continuation_token_ = std::move(page.continuation_token); + has_more_ = page.has_more; + } + return ObjectStorageResponse::OK(); +} + +ObjectStorageListResponse ObjectListIterator::next() { + auto response = has_next(); + if (response.status.code == ObjectStorageStatus::END_OF_FILE) { + return {.resp = ObjectStorageResponse::OK(), .results_ = {}}; + } + if (!response.ok()) { + return {.resp = std::move(response), .results_ = {}}; + } + return { + .resp = ObjectStorageResponse::OK(), + .results_ = std::move(objects_[next_index_++]), + }; +} + +ObjectStorageResponse ObjStorageClient::delete_objects(const ObjectStoragePathOptions& opts, + std::vector objs) { + const auto max_batch_size = std::max(1, backend_->capabilities().max_delete_batch); + for (size_t begin = 0; begin < objs.size(); begin += max_batch_size) { + const auto end = std::min(begin + max_batch_size, objs.size()); + auto rate_limit = acquire(ObjStorageRequestType::PUT); + if (!rate_limit.resp.ok()) { + return rate_limit.resp; + } + std::vector batch(std::make_move_iterator(objs.begin() + begin), + std::make_move_iterator(objs.begin() + end)); + auto response = backend_->delete_objects(opts, std::move(batch)); + if (!response.ok()) { + return response; + } + } + return ObjectStorageResponse::OK(); +} + +ObjectStorageResponse ObjStorageClient::delete_object(const ObjectStoragePathOptions& opts) { + auto rate_limit = acquire(ObjStorageRequestType::PUT); + if (!rate_limit.resp.ok()) { + return rate_limit.resp; + } + return backend_->delete_object(opts); +} + +ObjectStorageResponse ObjStorageClient::delete_objects_recursively( + const ObjectStoragePathOptions& opts, const RecursiveDeleteOptions& options) { + auto list_opts = opts; + if (list_opts.prefix.empty()) { + list_opts.prefix = list_opts.key; + } + auto delete_batch_size = std::max(1, backend_->capabilities().max_delete_batch); + TEST_SYNC_POINT_CALLBACK("ObjStorageClient::delete_objects_recursively_", &delete_batch_size); + delete_batch_size = std::max(1, delete_batch_size); + const auto max_tasks_per_batch = std::max(1, options.max_tasks_per_batch); + std::vector keys; + keys.reserve(delete_batch_size); + size_t pending_tasks = 0; + + auto wait_for_tasks = [&]() { + if (pending_tasks == 0) { + return ObjectStorageResponse::OK(); + } + pending_tasks = 0; + return options.executor ? options.executor.wait() : ObjectStorageResponse::OK(); + }; + auto submit_delete_task = [&]() { + ObjStorageDeleteTask task = [backend = backend_, rate_limit_policy = rate_limit_policy_, + bucket = opts.bucket, batch = std::move(keys)]() mutable { + auto rate_limit = acquire_rate_limit(rate_limit_policy, ObjStorageRequestType::PUT); + if (!rate_limit.resp.ok()) { + return rate_limit.resp; + } + return backend->delete_objects(ObjectStoragePathOptions {.bucket = std::move(bucket)}, + std::move(batch)); + }; + keys.clear(); + keys.reserve(delete_batch_size); + + ObjectStorageResponse response; + if (options.executor) { + response = options.executor.submit(std::move(task)); + } else { + response = task(); + } + if (!response.ok()) { + auto wait_response = wait_for_tasks(); + if (!wait_response.ok()) { + return wait_response; + } + return response; + } + ++pending_tasks; + return pending_tasks == max_tasks_per_batch ? wait_for_tasks() + : ObjectStorageResponse::OK(); + }; + + std::string continuation_token; + bool has_more = true; + while (has_more) { + auto page = list_objects(list_opts, continuation_token); + if (!page.resp.ok()) { + if (!keys.empty()) { + auto submit_response = submit_delete_task(); + if (!submit_response.ok()) { + return submit_response; + } + } + auto delete_response = wait_for_tasks(); + if (!delete_response.ok()) { + return delete_response; + } + return page.resp; + } + continuation_token = std::move(page.continuation_token); + has_more = page.has_more; + for (auto& object : page.objects) { + if (options.expiration_time > 0 && object.mtime_s > options.expiration_time) { + continue; + } + keys.emplace_back(std::move(object.file_path)); + if (keys.size() == delete_batch_size) { + auto response = submit_delete_task(); + if (!response.ok()) { + return response; + } + } + } + } + if (!keys.empty()) { + auto response = submit_delete_task(); + if (!response.ok()) { + return response; + } + } + return wait_for_tasks(); +} + +std::string ObjStorageClient::generate_presigned_url(const ObjectStoragePathOptions& opts, + int64_t expiration_secs) { + return backend_->generate_presigned_url(opts, expiration_secs); +} + +ObjectStorageResponse ObjStorageClient::get_life_cycle(const std::string& bucket, + int64_t* expiration_days) { + auto rate_limit = acquire(ObjStorageRequestType::GET); + if (!rate_limit.resp.ok()) { + return rate_limit.resp; + } + return backend_->get_life_cycle(bucket, expiration_days); +} + +ObjectStorageResponse ObjStorageClient::check_versioning(const std::string& bucket) { + auto rate_limit = acquire(ObjStorageRequestType::GET); + if (!rate_limit.resp.ok()) { + return rate_limit.resp; + } + return backend_->check_versioning(bucket); +} + +ObjectStorageResponse ObjStorageClient::abort_multipart_upload(const ObjectStoragePathOptions& opts, + const std::string& upload_id) { + auto rate_limit = acquire(ObjStorageRequestType::PUT); + if (!rate_limit.resp.ok()) { + return rate_limit.resp; + } + return backend_->abort_multipart_upload(opts, upload_id); +} + +} // namespace doris diff --git a/common/cpp/client/obj_storage_client.h b/common/cpp/client/obj_storage_client.h new file mode 100644 index 00000000000000..bfecd59cdcbe59 --- /dev/null +++ b/common/cpp/client/obj_storage_client.h @@ -0,0 +1,348 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace doris { +// Names are in lexico order. +enum class ObjStorageType : uint8_t { + UNKNOWN = 0, + AWS = 1, + AZURE = 2, + BOS = 3, + COS = 4, + OSS = 5, + OBS = 6, + GCP = 7, + TOS = 8, +}; + +/// eg: +/// s3://bucket1/path/to/file.txt +/// path: s3://bucket1/path/to/file.txt +/// bucket: bucket1 +/// key: path/to/file.txt +struct ObjectStoragePathOptions { + std::filesystem::path path = ""; + std::string bucket {}; // blob container in azure + std::string key {}; // blob name in azure + std::string prefix {}; // for list and recursive delete + std::optional upload_id = std::nullopt; // only used for S3 upload +}; + +struct ObjectClientConfig { + std::string endpoint {}; + std::string ak {}; + std::string sk {}; + std::string tls_debug_context {}; +}; + +struct ObjectMeta { + std::string file_path {}; + int64_t size {0}; + int64_t mtime_s {0}; +}; + +struct ObjectCompleteMultiPart { + int part_num = 0; + std::string etag {}; +}; + +struct ObjectStorageStatus { + enum Code : int { + UNDEFINED = -1, + OK = TStatusCode::OK, + NOT_FOUND = TStatusCode::NOT_FOUND, + END_OF_FILE = TStatusCode::END_OF_FILE, + RATE_LIMIT = TStatusCode::LIMIT_REACH, + }; + + ObjectStorageStatus(int r = OK, std::string msg = "") : code(r), msg(std::move(msg)) {} + // clang-format off + int code {OK}; // To unify the error handle logic with BE, we'd better use the same error code as BE + // clang-format on + std::string msg; +}; + +// We only store error code along with err_msg instead of Status to unify BE and recycler's error handle logic +struct ObjectStorageResponse { + ObjectStorageStatus status {0, ""}; + int http_code {200}; + std::string request_id {}; + static ObjectStorageResponse OK() { + // clang-format off + return { + .status = ObjectStorageStatus{0, ""}, + .http_code = 200, + .request_id = "" + }; + // clang-format on + } + + static ObjectStorageResponse rate_limit(std::string message) { + return { + .status = ObjectStorageStatus {ObjectStorageStatus::RATE_LIMIT, std::move(message)}, + .http_code = 429, + }; + } + + bool ok() const { return status.code == ObjectStorageStatus::OK; } +}; + +enum class ObjStorageRequestType { + GET, + PUT, +}; + +inline constexpr int32_t OBJECT_LIST_PAGE_SIZE = 1000; + +// One admission result for one object-storage backend request. `settle` is used by +// byte-aware limiters to refund a short read after the call completes. +struct ObjStorageRateLimitToken { + ObjectStorageResponse resp = ObjectStorageResponse::OK(); + std::function settle {}; + + void settle_bytes(size_t actual_bytes) const { + if (settle) { + settle(actual_bytes); + } + } +}; + +class ObjStorageRateLimitPolicy { +public: + virtual ~ObjStorageRateLimitPolicy() = default; + virtual ObjStorageRateLimitToken acquire(ObjStorageRequestType type, + size_t estimated_bytes) const = 0; +}; + +struct ObjectStorageUploadResponse { + ObjectStorageResponse resp = ObjectStorageResponse::OK(); + std::optional upload_id = std::nullopt; + std::optional etag = std::nullopt; +}; + +struct ObjectStorageHeadResponse { + ObjectStorageResponse resp = ObjectStorageResponse::OK(); + long long file_size {0}; +}; + +struct ObjectStorageListResponse { + ObjectStorageResponse resp = ObjectStorageResponse::OK(); + std::optional results_ = std::nullopt; +}; + +struct ObjectStorageListPage { + ObjectStorageResponse resp = ObjectStorageResponse::OK(); + std::vector objects {}; + std::string continuation_token {}; + bool has_more = false; +}; + +struct ObjStorageCapabilities { + size_t max_delete_batch = 1; +}; + +using ObjStorageDeleteTask = std::function; + +// A streaming executor for recursive deletion. submit() must enqueue the task immediately so the +// producer is subject to the executor's queue backpressure while it continues listing. wait() +// completes the current synchronization batch and prepares the executor for the next one. +struct ObjStorageDeleteExecutor { + std::function submit {}; + std::function wait {}; + + explicit operator bool() const { return submit && wait; } +}; + +struct RecursiveDeleteOptions { + int64_t expiration_time = 0; + size_t max_tasks_per_batch = 1; + ObjStorageDeleteExecutor executor {}; +}; + +class ObjStorageBackend { +public: + virtual ~ObjStorageBackend() = default; + // Create a multi-part upload request. On AWS-compatible systems, it will return an upload ID, but not on Azure. + // The input parameters should include the bucket and key for the object storage. + virtual ObjectStorageUploadResponse create_multipart_upload( + const ObjectStoragePathOptions& opts) = 0; + // To directly upload a piece of data to object storage and generate a user-visible file. + // You need to clearly specify the bucket and key + virtual ObjectStorageResponse put_object(const ObjectStoragePathOptions& opts, + std::string_view stream) = 0; + // To upload a part of a large file to object storage as a temporary file, which is not visible to the user + // The temporary file's ID is the value of the part_num passed in + // You need to specify the bucket and key along with the upload_id if it's AWS-compatible system + // For the same bucket and key, as well as the same part_num, it will directly replace the original temporary file. + virtual ObjectStorageUploadResponse upload_part(const ObjectStoragePathOptions& opts, + std::string_view stream, int part_num) = 0; + // To combine the previously uploaded multiple file parts into a complete file, the file name is the name of the key passed in. + // If it is an AWS-compatible system, the upload_id needs to be included. + // After a successful execution, the large file can be accessed in the object storage + virtual ObjectStorageResponse complete_multipart_upload( + const ObjectStoragePathOptions& opts, + const std::vector& completed_parts) = 0; + // According to the passed bucket and key, it will access whether the corresponding file exists in the object storage. + // If it exists, it will return the corresponding file size + virtual ObjectStorageHeadResponse head_object(const ObjectStoragePathOptions& opts) = 0; + // According to the bucket and key, it finds the corresponding file in the object storage + // and starting from the offset, it reads bytes_read bytes into the buffer, with size_return recording the actual number of bytes read + virtual ObjectStorageResponse get_object(const ObjectStoragePathOptions& opts, void* buffer, + size_t offset, size_t bytes_read, + size_t* size_return) = 0; + // Return at most one page of objects. One call corresponds to exactly one backend request. + // **Notice**: The files returned by this function contain the full key in object storage. + virtual ObjectStorageListPage list_objects(const ObjectStoragePathOptions& path, + std::string_view continuation_token) = 0; + + // According to the bucket and prefix specified by the user, it performs batch deletion based on the object names in the object array. + virtual ObjectStorageResponse delete_objects(const ObjectStoragePathOptions& opts, + std::vector objs) = 0; + // Delete the file named key in the object storage bucket. + virtual ObjectStorageResponse delete_object(const ObjectStoragePathOptions& opts) = 0; + virtual ObjStorageCapabilities capabilities() const { return {}; } + // Return a presigned URL for users to access the object + virtual std::string generate_presigned_url(const ObjectStoragePathOptions& opts, + int64_t expiration_secs) = 0; + + // Get the objects' expiration time on the bucket + virtual ObjectStorageResponse get_life_cycle(const std::string& /*bucket*/, + int64_t* /*expiration_days*/) { + return {.status = {TStatusCode::NOT_IMPLEMENTED_ERROR, + "object storage lifecycle is not supported"}, + .http_code = 0}; + } + + // Check if the objects' versioning is on or off + // returns 0 when versioning is on, otherwise versioning is off or check failed + virtual ObjectStorageResponse check_versioning(const std::string& /*bucket*/) { + return {.status = {TStatusCode::NOT_IMPLEMENTED_ERROR, + "object storage versioning is not supported"}, + .http_code = 0}; + } + + virtual ObjectStorageResponse abort_multipart_upload(const ObjectStoragePathOptions& /*path*/, + const std::string& /*upload_id*/) { + return {.status = {TStatusCode::NOT_IMPLEMENTED_ERROR, + "aborting multipart uploads is not supported"}, + .http_code = 0}; + } +}; + +// The only object-storage interface exposed to upper layers. It combines a backend implementation +// with an optional runtime policy, so backends cannot accidentally bypass rate limiting. +class ObjStorageClient final { +public: + explicit ObjStorageClient( + std::shared_ptr backend, + std::shared_ptr rate_limit_policy = nullptr) + : backend_(std::move(backend)), rate_limit_policy_(std::move(rate_limit_policy)) {} + + ObjectStorageUploadResponse create_multipart_upload(const ObjectStoragePathOptions& opts); + ObjectStorageResponse put_object(const ObjectStoragePathOptions& opts, std::string_view stream); + ObjectStorageUploadResponse upload_part(const ObjectStoragePathOptions& opts, + std::string_view stream, int part_num); + ObjectStorageResponse complete_multipart_upload( + const ObjectStoragePathOptions& opts, + const std::vector& completed_parts); + ObjectStorageHeadResponse head_object(const ObjectStoragePathOptions& opts); + ObjectStorageResponse get_object(const ObjectStoragePathOptions& opts, void* buffer, + size_t offset, size_t bytes_read, size_t* size_return); + ObjectStorageListPage list_objects(const ObjectStoragePathOptions& opts, + std::string_view continuation_token = {}); + ObjectStorageResponse delete_objects(const ObjectStoragePathOptions& opts, + std::vector objs); + ObjectStorageResponse delete_object(const ObjectStoragePathOptions& opts); + ObjectStorageResponse delete_objects_recursively( + const ObjectStoragePathOptions& opts, + const RecursiveDeleteOptions& options = RecursiveDeleteOptions {}); + ObjStorageCapabilities capabilities() const { return backend_->capabilities(); } + std::string generate_presigned_url(const ObjectStoragePathOptions& opts, + int64_t expiration_secs); + ObjectStorageResponse get_life_cycle(const std::string& bucket, int64_t* expiration_days); + ObjectStorageResponse check_versioning(const std::string& bucket); + ObjectStorageResponse abort_multipart_upload(const ObjectStoragePathOptions& opts, + const std::string& upload_id); + +private: + ObjStorageRateLimitToken acquire(ObjStorageRequestType type, size_t estimated_bytes = 0) const; + + std::shared_ptr backend_; + std::shared_ptr rate_limit_policy_; +}; + +// A client-side iterator above ObjStorageClient. It requests one fixed-size page at a time, so +// every ObjStorageClient::list_objects call maps to one rate-limit admission and one SDK request. +class ObjectListIterator { +public: + ObjectListIterator(std::shared_ptr client, ObjectStoragePathOptions opts) + : client_(std::move(client)), opts_(std::move(opts)) {} + + bool is_valid() const { return is_valid_; } + ObjectStorageResponse has_next(); + ObjectStorageListResponse next(); + +private: + std::shared_ptr client_; + ObjectStoragePathOptions opts_; + std::vector objects_; + size_t next_index_ = 0; + std::string continuation_token_; + bool has_more_ = true; + bool is_valid_ = true; +}; +} // namespace doris + +// Keep the BE namespace spelling source-compatible while the implementation is +// shared with Recycler in `doris`. +namespace doris::io { +using ::doris::ObjStorageCapabilities; +using ::doris::ObjStorageClient; +using ::doris::ObjStorageDeleteExecutor; +using ::doris::ObjStorageDeleteTask; +using ::doris::ObjStorageBackend; +using ::doris::ObjStorageRateLimitPolicy; +using ::doris::ObjStorageRateLimitToken; +using ::doris::ObjStorageRequestType; +using ::doris::ObjStorageType; +using ::doris::ObjectClientConfig; +using ::doris::ObjectCompleteMultiPart; +using ::doris::ObjectListIterator; +using ::doris::ObjectMeta; +using ::doris::ObjectStorageHeadResponse; +using ::doris::ObjectStorageListPage; +using ::doris::ObjectStorageListResponse; +using ::doris::ObjectStoragePathOptions; +using ::doris::ObjectStorageResponse; +using ::doris::ObjectStorageStatus; +using ::doris::ObjectStorageUploadResponse; +using ::doris::RecursiveDeleteOptions; +} // namespace doris::io diff --git a/be/src/io/fs/s3_common.h b/common/cpp/client/s3_common.h similarity index 100% rename from be/src/io/fs/s3_common.h rename to common/cpp/client/s3_common.h diff --git a/common/cpp/client/s3_obj_storage_backend.cpp b/common/cpp/client/s3_obj_storage_backend.cpp new file mode 100644 index 00000000000000..02ff510eb7fa71 --- /dev/null +++ b/common/cpp/client/s3_obj_storage_backend.cpp @@ -0,0 +1,611 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "s3_obj_storage_backend.h" + +#include +#include + +#include +#include + +#include "client_bvar.h" +#include "cpp/obj_retry_strategy.h" + +namespace Aws::S3::Model { +class DeleteObjectRequest; +} // namespace Aws::S3::Model + +using Aws::S3::Model::CompletedPart; +using Aws::S3::Model::CompletedMultipartUpload; +using Aws::S3::Model::CompleteMultipartUploadRequest; +using Aws::S3::Model::CreateMultipartUploadRequest; +using Aws::S3::Model::UploadPartRequest; +using Aws::S3::Model::UploadPartOutcome; + +namespace doris { +using namespace Aws::S3::Model; +namespace { + +constexpr int64_t S3_REQUEST_THRESHOLD_MS = 5000; + +int64_t elapsed_time_milliseconds(std::chrono::steady_clock::time_point start) { + return std::chrono::duration_cast(std::chrono::steady_clock::now() - + start) + .count(); +} + +void record_s3_request_failed(const Aws::S3::S3Error& error) { + record_object_request_failed(static_cast(error.GetResponseCode())); +} + +std::string object_identity(const ObjectStoragePathOptions& opts) { + return opts.path.empty() ? opts.key : opts.path.native(); +} + +std::string s3_error_message(const Aws::S3::S3Error& error, std::string_view message) { + return fmt::format("{}: {} {} code={}, type={}, request_id={}", message, + error.GetExceptionName(), error.GetMessage(), + static_cast(error.GetResponseCode()), + static_cast(error.GetErrorType()), error.GetRequestId()); +} + +} // namespace + +ObjectStorageStatus s3fs_error(const Aws::S3::S3Error& err, std::string_view msg) { + using namespace Aws::Http; + switch (err.GetResponseCode()) { + case HttpResponseCode::NOT_FOUND: + return {TStatusCode::NOT_FOUND, s3_error_message(err, msg)}; + case HttpResponseCode::FORBIDDEN: + // TODO: no permission and other 4xx errors should be handled separately + return {TStatusCode::NOT_AUTHORIZED, s3_error_message(err, msg)}; + case HttpResponseCode::REQUEST_NOT_MADE: + return {-1, s3_error_message(err, msg)}; + default: + return {TStatusCode::INTERNAL_ERROR, s3_error_message(err, msg)}; + } +} + +ObjectStorageUploadResponse S3ObjStorageBackend::create_multipart_upload( + const ObjectStoragePathOptions& opts) { + CreateMultipartUploadRequest request; + request.WithBucket(opts.bucket).WithKey(opts.key); + request.SetContentType("application/octet-stream"); + + const auto start = std::chrono::steady_clock::now(); + auto outcome = SYNC_POINT_HOOK_RETURN_VALUE( + [&]() { + client_bvar::ScopedLatency scoped_latency( + client_bvar::s3_multi_part_upload_latency); + return _client->CreateMultipartUpload(request); + }(), + "s3_file_writer::create_multi_part_upload", std::cref(request).get()); + SYNC_POINT_CALLBACK("s3_file_writer::_open", &outcome); + const auto elapsed_ms = elapsed_time_milliseconds(start); + + const auto& request_id = outcome.IsSuccess() ? outcome.GetResult().GetRequestId() + : outcome.GetError().GetRequestId(); + + LOG_IF(INFO, elapsed_ms > S3_REQUEST_THRESHOLD_MS) + << "CreateMultipartUpload cost=" << elapsed_ms << "ms" + << ", request_id=" << request_id << ", bucket=" << opts.bucket << ", key=" << opts.key; + + if (!outcome.IsSuccess()) { + record_s3_request_failed(outcome.GetError()); + auto st = s3fs_error(outcome.GetError(), fmt::format("failed to CreateMultipartUpload: {} ", + opts.path.native())); + LOG(WARNING) << st.code << " request_id=" << request_id; + return ObjectStorageUploadResponse { + .resp = {.status = st, + .http_code = static_cast(outcome.GetError().GetResponseCode()), + .request_id = outcome.GetError().GetRequestId()}, + }; + } + + return ObjectStorageUploadResponse {.resp = ObjectStorageResponse::OK(), + .upload_id {outcome.GetResult().GetUploadId()}}; +} + +ObjectStorageResponse S3ObjStorageBackend::put_object(const ObjectStoragePathOptions& opts, + std::string_view stream) { + Aws::S3::Model::PutObjectRequest request; + request.WithBucket(opts.bucket).WithKey(opts.key); + auto string_view_stream = std::make_shared(stream.data(), stream.size()); + Aws::Utils::ByteBuffer part_md5(Aws::Utils::HashingUtils::CalculateMD5(*string_view_stream)); + request.SetContentMD5(Aws::Utils::HashingUtils::Base64Encode(part_md5)); + request.SetBody(string_view_stream); + request.SetContentLength(stream.size()); + request.SetContentType("application/octet-stream"); + + const auto start = std::chrono::steady_clock::now(); + auto outcome = SYNC_POINT_HOOK_RETURN_VALUE( + [&]() { + client_bvar::ScopedLatency scoped_latency(client_bvar::s3_put_latency); + return _client->PutObject(request); + }(), + "s3_file_writer::put_object", std::cref(request).get(), &stream); + const auto elapsed_ms = elapsed_time_milliseconds(start); + + const auto& request_id = outcome.IsSuccess() ? outcome.GetResult().GetRequestId() + : outcome.GetError().GetRequestId(); + + if (!outcome.IsSuccess()) { + record_s3_request_failed(outcome.GetError()); + auto st = s3fs_error(outcome.GetError(), + fmt::format("failed to put object: {}", opts.path.native())); + LOG(WARNING) << st.code << ", request_id=" << request_id; + return ObjectStorageResponse { + .status = st, + .http_code = static_cast(outcome.GetError().GetResponseCode()), + .request_id = outcome.GetError().GetRequestId()}; + } + + LOG_IF(INFO, elapsed_ms > S3_REQUEST_THRESHOLD_MS) + << "PutObject cost=" << elapsed_ms << "ms" + << ", request_id=" << request_id << ", bucket=" << opts.bucket << ", key=" << opts.key; + return ObjectStorageResponse::OK(); +} + +ObjectStorageUploadResponse S3ObjStorageBackend::upload_part(const ObjectStoragePathOptions& opts, + std::string_view stream, + int part_num) { + UploadPartRequest request; + request.WithBucket(opts.bucket) + .WithKey(opts.key) + .WithPartNumber(part_num) + .WithUploadId(*opts.upload_id); + auto string_view_stream = std::make_shared(stream.data(), stream.size()); + + request.SetBody(string_view_stream); + + Aws::Utils::ByteBuffer part_md5(Aws::Utils::HashingUtils::CalculateMD5(*string_view_stream)); + request.SetContentMD5(Aws::Utils::HashingUtils::Base64Encode(part_md5)); + + request.SetContentLength(stream.size()); + request.SetContentType("application/octet-stream"); + + const auto start = std::chrono::steady_clock::now(); + auto outcome = SYNC_POINT_HOOK_RETURN_VALUE( + [&]() { + client_bvar::ScopedLatency scoped_latency( + client_bvar::s3_multi_part_upload_latency); + + return _client->UploadPart(request); + }(), + "s3_file_writer::upload_part", std::cref(request).get(), &stream); + const auto elapsed_ms = elapsed_time_milliseconds(start); + + const auto& request_id = outcome.IsSuccess() ? outcome.GetResult().GetRequestId() + : outcome.GetError().GetRequestId(); + + TEST_SYNC_POINT_CALLBACK("S3FileWriter::_upload_one_part", &outcome); + if (!outcome.IsSuccess()) { + record_s3_request_failed(outcome.GetError()); + auto st = s3fs_error(outcome.GetError(), + fmt::format("failed to UploadPart: {}, part_num {}, upload_id={}", + opts.path.native(), part_num, *opts.upload_id)); + + LOG(WARNING) << st.code << ", request_id=" << request_id; + return ObjectStorageUploadResponse { + .resp = {.status = st, + .http_code = static_cast(outcome.GetError().GetResponseCode()), + .request_id = outcome.GetError().GetRequestId()}}; + } + LOG_IF(INFO, elapsed_ms > S3_REQUEST_THRESHOLD_MS) + << "UploadPart cost=" << elapsed_ms << "ms" + << ", request_id=" << request_id << ", bucket=" << opts.bucket << ", key=" << opts.key + << ", part_num=" << part_num << ", upload_id=" << *opts.upload_id; + return ObjectStorageUploadResponse {.resp = ObjectStorageResponse::OK(), + .etag = outcome.GetResult().GetETag()}; +} + +ObjectStorageResponse S3ObjStorageBackend::complete_multipart_upload( + const ObjectStoragePathOptions& opts, + const std::vector& completed_parts) { + CompleteMultipartUploadRequest request; + request.WithBucket(opts.bucket).WithKey(opts.key).WithUploadId(*opts.upload_id); + + CompletedMultipartUpload completed_upload; + std::vector complete_parts; + std::ranges::transform(completed_parts, std::back_inserter(complete_parts), + [](const ObjectCompleteMultiPart& part_ptr) { + CompletedPart part; + part.SetPartNumber(part_ptr.part_num); + part.SetETag(part_ptr.etag); + return part; + }); + completed_upload.SetParts(std::move(complete_parts)); + request.WithMultipartUpload(completed_upload); + + TEST_SYNC_POINT_RETURN_WITH_VALUE("S3FileWriter::_complete:3", ObjectStorageResponse(), this); + + const auto start = std::chrono::steady_clock::now(); + auto outcome = SYNC_POINT_HOOK_RETURN_VALUE( + [&]() { + client_bvar::ScopedLatency scoped_latency( + client_bvar::s3_multi_part_upload_latency); + return _client->CompleteMultipartUpload(request); + }(), + "s3_file_writer::complete_multi_part", std::cref(request).get()); + const auto elapsed_ms = elapsed_time_milliseconds(start); + + const auto& request_id = outcome.IsSuccess() ? outcome.GetResult().GetRequestId() + : outcome.GetError().GetRequestId(); + + if (!outcome.IsSuccess()) { + record_s3_request_failed(outcome.GetError()); + auto st = s3fs_error(outcome.GetError(), + fmt::format("failed to CompleteMultipartUpload: {}, upload_id={}", + opts.path.native(), *opts.upload_id)); + LOG(WARNING) << st.code << ", request_id=" << request_id; + return {.status = st, + .http_code = static_cast(outcome.GetError().GetResponseCode()), + .request_id = outcome.GetError().GetRequestId()}; + } + + LOG_IF(INFO, elapsed_ms > S3_REQUEST_THRESHOLD_MS) + << "CompleteMultipartUpload cost=" << elapsed_ms << "ms" + << ", request_id=" << request_id << ", bucket=" << opts.bucket << ", key=" << opts.key + << ", upload_id=" << *opts.upload_id; + return ObjectStorageResponse::OK(); +} + +ObjectStorageHeadResponse S3ObjStorageBackend::head_object(const ObjectStoragePathOptions& opts) { + Aws::S3::Model::HeadObjectRequest request; + request.WithBucket(opts.bucket).WithKey(opts.key); + + auto outcome = SYNC_POINT_HOOK_RETURN_VALUE( + [&]() { + client_bvar::ScopedLatency scoped_latency(client_bvar::s3_head_latency); + return _client->HeadObject(request); + }(), + "s3_file_system::head_object", std::ref(request).get()); + + if (outcome.IsSuccess()) { + return {.resp = ObjectStorageResponse::OK(), + .file_size = outcome.GetResult().GetContentLength()}; + } else if (outcome.GetError().GetResponseCode() == Aws::Http::HttpResponseCode::NOT_FOUND) { + return {.resp = {.status = TStatusCode::NOT_FOUND}, .file_size = 0}; + } else { + record_s3_request_failed(outcome.GetError()); + LOG(WARNING) << "failed to head object" + << "bucket " << opts.bucket << " key " << opts.key << " responseCode " + << outcome.GetError() << " error " << outcome.GetError().GetMessage() + << " request_id " << outcome.GetError().GetRequestId(); + return {.resp = {.status = s3fs_error( + outcome.GetError(), + fmt::format("failed to head object: {}", object_identity(opts))), + .http_code = static_cast(outcome.GetError().GetResponseCode()), + .request_id = outcome.GetError().GetRequestId()}, + .file_size = -1}; + } +} + +ObjectStorageResponse S3ObjStorageBackend::get_object(const ObjectStoragePathOptions& opts, + void* buffer, size_t offset, + size_t bytes_read, size_t* size_return) { + Aws::S3::Model::GetObjectRequest request; + request.WithBucket(opts.bucket).WithKey(opts.key); + request.SetRange(fmt::format("bytes={}-{}", offset, offset + bytes_read - 1)); + request.SetResponseStreamFactory(AwsWriteableStreamFactory(buffer, bytes_read)); + + auto outcome = [&]() { + client_bvar::ScopedLatency scoped_latency(client_bvar::s3_get_latency); + return _client->GetObject(request); + }(); + if (!outcome.IsSuccess()) { + record_s3_request_failed(outcome.GetError()); + return ObjectStorageResponse { + .status = s3fs_error(outcome.GetError(), fmt::format("failed to get object: {}", + object_identity(opts))), + .http_code = static_cast(outcome.GetError().GetResponseCode()), + .request_id = outcome.GetError().GetRequestId(), + }; + } + *size_return = outcome.GetResult().GetContentLength(); + SYNC_POINT_CALLBACK("s3_obj_storage_client::get_object", size_return); + if (*size_return != bytes_read) { + const auto& request_id = outcome.GetResult().GetRequestId(); + return ObjectStorageResponse { + .status = {TStatusCode::INTERNAL_ERROR, + fmt::format("incomplete read from {}, expect {}, got {}, request_id={}", + object_identity(opts), bytes_read, *size_return, + request_id)}, + .request_id = request_id}; + } + return ObjectStorageResponse::OK(); +} + +ObjectStorageListPage S3ObjStorageBackend::list_objects(const ObjectStoragePathOptions& opts, + std::string_view continuation_token) { + const auto& prefix = opts.prefix.empty() ? opts.key : opts.prefix; + Aws::S3::Model::ListObjectsV2Request request; + request.WithBucket(opts.bucket).WithPrefix(prefix).WithMaxKeys(OBJECT_LIST_PAGE_SIZE); + if (!continuation_token.empty()) { + request.SetContinuationToken(std::string(continuation_token)); + } + TEST_SYNC_POINT_CALLBACK("S3ObjStorageBackend::list_objects", &request); + + auto outcome = [&]() { + client_bvar::ScopedLatency scoped_latency(client_bvar::s3_list_latency); + return _client->ListObjectsV2(request); + }(); + + const auto& request_id = outcome.IsSuccess() ? outcome.GetResult().GetRequestId() + : outcome.GetError().GetRequestId(); + if (!outcome.IsSuccess()) { + // Some S3-compatible providers (for example TOS) return NoSuchKey instead of an empty page + // when a prefix does not exist. + if (outcome.GetError().GetErrorType() == Aws::S3::S3Errors::NO_SUCH_KEY) { + LOG(INFO) << fmt::format( + "NoSuchKey when listing objects, treat as empty response, endpoint: {}, " + "bucket: {}, prefix: {}, request_id: {}", + _config.endpoint, request.GetBucket(), request.GetPrefix(), request_id); + return {.resp = ObjectStorageResponse::OK()}; + } + record_object_request_failed(static_cast(outcome.GetError().GetResponseCode())); + const auto status = s3fs_error(outcome.GetError(), + fmt::format("failed to list objects: {}, prefix: {}", + request.GetBucket(), request.GetPrefix())); + LOG(WARNING) << fmt::format( + "failed to list objects, endpoint: {}, bucket: {}, prefix: {}, responseCode: {}, " + "error: {}, request_id: {}", + _config.endpoint, request.GetBucket(), request.GetPrefix(), + static_cast(outcome.GetError().GetResponseCode()), + outcome.GetError().GetMessage(), request_id); + return { + .resp = {.status = status, + .http_code = static_cast(outcome.GetError().GetResponseCode()), + .request_id = request_id}, + }; + } + + const auto& result = outcome.GetResult(); + if (result.GetIsTruncated() && result.GetNextContinuationToken().empty()) { + LOG(WARNING) << fmt::format( + "failed to list objects, isTruncated but no continuation token, endpoint: {}, " + "bucket: {}, prefix: {}, request_id: {}", + _config.endpoint, request.GetBucket(), request.GetPrefix(), request_id); + return { + .resp = {.status = {TStatusCode::INTERNAL_ERROR, + fmt::format("failed to list objects: {}, prefix: {}", + request.GetBucket(), request.GetPrefix())}, + .http_code = 0, + .request_id = request_id}, + }; + } + + ObjectStorageListPage page { + .resp = ObjectStorageResponse::OK(), + .continuation_token = result.GetNextContinuationToken(), + .has_more = result.GetIsTruncated(), + }; + const auto& content = result.GetContents(); + page.objects.reserve(content.size()); + for (const auto& obj : content) { + DCHECK(obj.GetKey().starts_with(request.GetPrefix())) + << obj.GetKey() << ' ' << request.GetPrefix(); + page.objects.emplace_back(ObjectMeta {.file_path = obj.GetKey(), + .size = obj.GetSize(), + .mtime_s = obj.GetLastModified().Seconds()}); + } + return page; +} + +ObjectStorageResponse S3ObjStorageBackend::delete_objects(const ObjectStoragePathOptions& opts, + std::vector objs) { + size_t max_delete_batch = 1000; + TEST_SYNC_POINT_CALLBACK("S3ObjClient::delete_objects", &max_delete_batch); + TEST_SYNC_POINT_CALLBACK("S3ObjStorageClient::delete_objects", &max_delete_batch); + max_delete_batch = std::max(1, max_delete_batch); + for (size_t begin = 0; begin < objs.size(); begin += max_delete_batch) { + const size_t end = std::min(begin + max_delete_batch, objs.size()); + if (end - begin == 1) { + auto single_opts = opts; + single_opts.key = std::move(objs[begin]); + auto resp = delete_object(single_opts); + if (!resp.ok()) { + return resp; + } + continue; + } + + Aws::S3::Model::DeleteObjectsRequest delete_request; + delete_request.SetBucket(opts.bucket); + Aws::S3::Model::Delete del; + Aws::Vector objects; + objects.reserve(end - begin); + for (size_t i = begin; i < end; ++i) { + Aws::S3::Model::ObjectIdentifier object; + object.SetKey(std::move(objs[i])); + objects.emplace_back(std::move(object)); + } + del.WithObjects(std::move(objects)).SetQuiet(true); + delete_request.SetDelete(std::move(del)); + + auto delete_outcome = [&]() { + client_bvar::ScopedLatency scoped_latency(client_bvar::s3_delete_objects_latency); + return _client->DeleteObjects(delete_request); + }(); + SYNC_POINT_CALLBACK("s3_obj_storage_client::delete_objects", &delete_outcome); + SYNC_POINT_CALLBACK("s3_obj_storage_client::delete_objects_recursively", &delete_outcome); + if (!delete_outcome.IsSuccess()) { + record_s3_request_failed(delete_outcome.GetError()); + LOG(WARNING) << fmt::format( + "failed to delete objects, endpoint: {}, bucket: {}, key: {}, responseCode: " + "{}, error: {}, request_id: {}", + _config.endpoint, opts.bucket, + delete_request.GetDelete().GetObjects().front().GetKey(), + static_cast(delete_outcome.GetError().GetResponseCode()), + delete_outcome.GetError().GetMessage(), + delete_outcome.GetError().GetRequestId()); + return ObjectStorageResponse { + .status = s3fs_error(delete_outcome.GetError(), + fmt::format("failed to delete dir {}", opts.key)), + .http_code = static_cast(delete_outcome.GetError().GetResponseCode()), + .request_id = delete_outcome.GetError().GetRequestId()}; + } + if (!delete_outcome.GetResult().GetErrors().empty()) { + const auto& error = delete_outcome.GetResult().GetErrors().front(); + LOG(WARNING) << fmt::format( + "failed to delete object in batch, endpoint: {}, bucket: {}, key: {}, error " + "code: {}, error: {}, request_id: {}", + _config.endpoint, opts.bucket, error.GetKey(), error.GetCode(), + error.GetMessage(), delete_outcome.GetResult().GetRequestId()); + return ObjectStorageResponse { + .status = {TStatusCode::INTERNAL_ERROR, + fmt::format("failed to delete object {}: {}, request_id={}", + error.GetKey(), error.GetMessage(), + delete_outcome.GetResult().GetRequestId())}, + .request_id = delete_outcome.GetResult().GetRequestId()}; + } + } + return ObjectStorageResponse::OK(); +} + +ObjectStorageResponse S3ObjStorageBackend::delete_object(const ObjectStoragePathOptions& opts) { + Aws::S3::Model::DeleteObjectRequest request; + request.WithBucket(opts.bucket).WithKey(opts.key); + + auto outcome = [&]() { + client_bvar::ScopedLatency scoped_latency(client_bvar::s3_delete_object_latency); + + return _client->DeleteObject(request); + }(); + TEST_SYNC_POINT_CALLBACK("S3ObjClient::delete_object", &outcome); + TEST_SYNC_POINT_CALLBACK("S3ObjStorageClient::delete_object", &outcome); + if (outcome.IsSuccess() || + outcome.GetError().GetResponseCode() == Aws::Http::HttpResponseCode::NOT_FOUND) { + return ObjectStorageResponse::OK(); + } + record_s3_request_failed(outcome.GetError()); + LOG(WARNING) << fmt::format( + "failed to delete object, endpoint: {}, bucket: {}, key: {}, responseCode: {}, " + "error: {}, request_id: {}", + _config.endpoint, opts.bucket, opts.key, + static_cast(outcome.GetError().GetResponseCode()), outcome.GetError().GetMessage(), + outcome.GetError().GetRequestId()); + return ObjectStorageResponse { + .status = s3fs_error(outcome.GetError(), + fmt::format("failed to delete object {}", opts.key)), + .http_code = static_cast(outcome.GetError().GetResponseCode()), + .request_id = outcome.GetError().GetRequestId()}; +} + +std::string S3ObjStorageBackend::generate_presigned_url(const ObjectStoragePathOptions& opts, + int64_t expiration_secs) { + return _client->GeneratePresignedUrl(opts.bucket, opts.key, Aws::Http::HttpMethod::HTTP_GET, + expiration_secs); +} + +ObjectStorageResponse S3ObjStorageBackend::check_versioning(const std::string& bucket) { + Aws::S3::Model::GetBucketVersioningRequest request; + request.SetBucket(bucket); + + auto outcome = _client->GetBucketVersioning(request); + + if (outcome.IsSuccess()) { + const auto& versioning_configuration = outcome.GetResult().GetStatus(); + if (versioning_configuration != Aws::S3::Model::BucketVersioningStatus::Enabled) { + LOG(WARNING) << "Err for check interval: bucket doesn't enable bucket versioning" + << " endpoint=" << _config.endpoint << " bucket=" << bucket; + return ObjectStorageResponse { + .status = {TStatusCode::INTERNAL_ERROR, + fmt::format("bucket versioning is not enabled: {}", bucket)}}; + } + } else { + record_s3_request_failed(outcome.GetError()); + LOG(WARNING) << "Err for check interval: failed to get status of bucket versioning" + << " endpoint=" << _config.endpoint << " bucket=" << bucket + << " responseCode=" << static_cast(outcome.GetError().GetResponseCode()) + << " error=" << outcome.GetError().GetMessage() + << " request_id=" << outcome.GetError().GetRequestId(); + return ObjectStorageResponse { + .status = {-1}, + .http_code = static_cast(outcome.GetError().GetResponseCode()), + .request_id = outcome.GetError().GetRequestId()}; + } + return ObjectStorageResponse::OK(); +} + +ObjectStorageResponse S3ObjStorageBackend::abort_multipart_upload( + const ObjectStoragePathOptions& opts, const std::string& upload_id) { + Aws::S3::Model::AbortMultipartUploadRequest request; + request.WithBucket(opts.bucket).WithKey(opts.key).WithUploadId(upload_id); + + auto outcome = _client->AbortMultipartUpload(request); + if (!outcome.IsSuccess()) { + LOG(WARNING) << "failed to abort multipart upload" + << " endpoint=" << _config.endpoint << " bucket=" << opts.bucket + << " key=" << opts.key << " upload_id=" << upload_id + << " responseCode=" << static_cast(outcome.GetError().GetResponseCode()) + << " error=" << outcome.GetError().GetMessage() + << " request_id=" << outcome.GetError().GetRequestId(); + if (outcome.GetError().GetResponseCode() == Aws::Http::HttpResponseCode::NOT_FOUND) { + return ObjectStorageResponse::OK(); + } + record_s3_request_failed(outcome.GetError()); + return ObjectStorageResponse { + .status = {TStatusCode::INTERNAL_ERROR, + fmt::format("failed to abort multipart upload: {}, upload_id={}", + opts.path.native(), upload_id)}, + .http_code = static_cast(outcome.GetError().GetResponseCode()), + .request_id = outcome.GetError().GetRequestId(), + }; + } + return ObjectStorageResponse::OK(); +} + +ObjectStorageResponse S3ObjStorageBackend::get_life_cycle(const std::string& bucket, + int64_t* expiration_days) { + Aws::S3::Model::GetBucketLifecycleConfigurationRequest request; + request.SetBucket(bucket); + + auto outcome = _client->GetBucketLifecycleConfiguration(request); + bool has_lifecycle = false; + if (outcome.IsSuccess()) { + const auto& rules = outcome.GetResult().GetRules(); + for (const auto& rule : rules) { + if (rule.NoncurrentVersionExpirationHasBeenSet()) { + has_lifecycle = true; + *expiration_days = rule.GetNoncurrentVersionExpiration().GetNoncurrentDays(); + } + } + } else { + record_s3_request_failed(outcome.GetError()); + LOG(WARNING) << "Err for check interval: failed to get bucket lifecycle" + << " endpoint=" << _config.endpoint << " bucket=" << bucket + << " responseCode=" << static_cast(outcome.GetError().GetResponseCode()) + << " error=" << outcome.GetError().GetMessage() + << " request_id=" << outcome.GetError().GetRequestId(); + return ObjectStorageResponse { + .status = s3fs_error(outcome.GetError(), + fmt::format("failed to get lift cycle: {}", bucket)), + .http_code = static_cast(outcome.GetError().GetResponseCode()), + .request_id = outcome.GetError().GetRequestId()}; + } + + if (!has_lifecycle) { + LOG(WARNING) << "Err for check interval: bucket doesn't have lifecycle configuration" + << " endpoint=" << _config.endpoint << " bucket=" << bucket; + return ObjectStorageResponse {.status = {-1}}; + } + return ObjectStorageResponse::OK(); +} + +} // namespace doris diff --git a/common/cpp/client/s3_obj_storage_backend.h b/common/cpp/client/s3_obj_storage_backend.h new file mode 100644 index 00000000000000..048893db84b121 --- /dev/null +++ b/common/cpp/client/s3_obj_storage_backend.h @@ -0,0 +1,133 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "client_bvar.h" +#include "cpp/obj_retry_strategy.h" +#include "cpp/sync_point.h" +#include "obj_storage_client.h" +#include "s3_common.h" + +namespace Aws::S3 { +class S3Client; +namespace Model { +class CompletedPart; +} +} // namespace Aws::S3 + +namespace doris { + +ObjectStorageStatus s3fs_error(const Aws::S3::S3Error& err, std::string_view msg); + +class S3ObjStorageBackend final : public ObjStorageBackend { +public: + S3ObjStorageBackend(std::shared_ptr client, ObjectClientConfig config = {}) + : _config(std::move(config)), _client(std::move(client)) {} + ~S3ObjStorageBackend() override = default; + ObjectStorageUploadResponse create_multipart_upload( + const ObjectStoragePathOptions& opts) override; + ObjectStorageResponse put_object(const ObjectStoragePathOptions& opts, + std::string_view stream) override; + ObjectStorageUploadResponse upload_part(const ObjectStoragePathOptions& opts, std::string_view, + int partNum) override; + ObjectStorageResponse complete_multipart_upload( + const ObjectStoragePathOptions& opts, + const std::vector& completed_parts) override; + ObjectStorageHeadResponse head_object(const ObjectStoragePathOptions& opts) override; + ObjectStorageResponse get_object(const ObjectStoragePathOptions& opts, void* buffer, + size_t offset, size_t bytes_read, + size_t* size_return) override; + ObjectStorageListPage list_objects(const ObjectStoragePathOptions& opts, + std::string_view continuation_token) override; + ObjectStorageResponse delete_objects(const ObjectStoragePathOptions& opts, + std::vector objs) override; + ObjectStorageResponse delete_object(const ObjectStoragePathOptions& opts) override; + std::string generate_presigned_url(const ObjectStoragePathOptions& opts, + int64_t expiration_secs) override; + ObjectStorageResponse get_life_cycle(const std::string& bucket, + int64_t* expiration_days) override; + + ObjectStorageResponse check_versioning(const std::string& bucket) override; + + ObjectStorageResponse abort_multipart_upload(const ObjectStoragePathOptions& opts, + const std::string& upload_id) override; + ObjStorageCapabilities capabilities() const override { return {.max_delete_batch = 1000}; } + +private: + ObjectClientConfig _config; + std::shared_ptr _client; +}; + +} // namespace doris + +namespace doris::io { +using ::doris::S3ObjStorageBackend; +} // namespace doris::io