From 859e24d16004162416354da7729ce0a8e282c490 Mon Sep 17 00:00:00 2001 From: Jinming Hu Date: Fri, 30 Oct 2026 11:38:04 +1100 Subject: [PATCH 1/6] Hide public layout APIs while preserving managed download routing Move Blob layout discovery into private implementation code and remove the public Data Lake wrappers and single-shot layout endpoint options. Retain locality-aware managed downloads and update focused tests and unreleased changelogs. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- sdk/storage/azure-storage-blobs/CHANGELOG.md | 8 +- .../inc/azure/storage/blobs/blob_client.hpp | 13 +- .../inc/azure/storage/blobs/blob_options.hpp | 29 --- .../azure/storage/blobs/blob_responses.hpp | 53 ----- .../azure-storage-blobs/src/blob_client.cpp | 47 ++-- .../src/blob_responses.cpp | 5 +- .../src/private/data_locality.cpp | 2 +- .../src/private/data_locality.hpp | 43 +++- .../test/ut/data_locality_test.cpp | 47 ++-- .../azure-storage-files-datalake/CHANGELOG.md | 8 +- .../files/datalake/datalake_file_client.hpp | 19 +- .../files/datalake/datalake_options.hpp | 27 --- .../files/datalake/datalake_responses.hpp | 41 ---- .../src/datalake_file_client.cpp | 36 +--- .../src/datalake_responses.cpp | 8 - .../test/ut/data_locality_test.cpp | 204 +++--------------- 16 files changed, 147 insertions(+), 443 deletions(-) diff --git a/sdk/storage/azure-storage-blobs/CHANGELOG.md b/sdk/storage/azure-storage-blobs/CHANGELOG.md index d9059e77f6..95fd1b89e4 100644 --- a/sdk/storage/azure-storage-blobs/CHANGELOG.md +++ b/sdk/storage/azure-storage-blobs/CHANGELOG.md @@ -5,11 +5,7 @@ ### Features Added - Added support for persisted session authentication for eligible blob download requests. -- Added `BlobClient::GetLayout` and related options and response types for retrieving locality - endpoints, byte ranges, and blob properties. Ranges are exposed directly through - `Models::BlobLayoutInfo::Ranges` as a vector of `Models::BlobLayoutRange`. -- Added `DownloadBlobOptions::LayoutEndpoint` for one-shot download routing and - `DownloadBlobToOptions::LayoutAwareRouting` (`Auto`, `Disabled`, `Enabled`; default `Disabled`) for +- Added `DownloadBlobToOptions::LayoutAwareRouting` (`Auto`, `Disabled`, `Enabled`; default `Disabled`) for managed downloads. - Added `Models::DownloadHint` and `Models::DownloadBlobDetails::DownloadHint`. @@ -19,6 +15,8 @@ ### Other Changes +- Kept layout discovery private and removed explicit single-shot download endpoint selection while + retaining locality-aware managed downloads. - Changed the default response format for `ListBlobs` and `ListBlobsByHierarchy` from XML to Apache Arrow. Set `ListBlobsOptions::ResponseFormat` to `StorageResponseFormat::Xml` to continue requesting XML responses. diff --git a/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_client.hpp b/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_client.hpp index ec3511e048..ac542df500 100644 --- a/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_client.hpp +++ b/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_client.hpp @@ -24,6 +24,7 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { namespace Azure { namespace Storage { namespace Blobs { namespace _detail { + class BlobLayoutClient; AZ_STORAGE_BLOBS_DLLEXPORT extern const Azure::Core::Context::Key DataLakeInteroperabilityExtraOptionsKey; } // namespace _detail @@ -425,17 +426,6 @@ namespace Azure { namespace Storage { namespace Blobs { const GetAccountInfoOptions& options = GetAccountInfoOptions(), const Azure::Core::Context& context = Azure::Core::Context()) const; - /** - * @brief Gets the data locality layout for this blob. - * - * @param options Optional parameters to execute this function. - * @param context Context for cancelling long running operations. - * @return A paged response containing the blob's data locality layout. - */ - BlobLayoutPagedResponse GetLayout( - const GetBlobLayoutOptions& options = GetBlobLayoutOptions(), - const Azure::Core::Context& context = Azure::Core::Context()) const; - protected: /** @brief Blob Url */ Azure::Core::Url m_blobUrl; @@ -461,5 +451,6 @@ namespace Azure { namespace Storage { namespace Blobs { friend class BlobLeaseClient; friend class BlobServiceBatch; friend class BlobContainerBatch; + friend class _detail::BlobLayoutClient; }; }}} // namespace Azure::Storage::Blobs diff --git a/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_options.hpp b/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_options.hpp index 923b6fd4ab..5c79059a7a 100644 --- a/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_options.hpp +++ b/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_options.hpp @@ -932,14 +932,6 @@ namespace Azure { namespace Storage { namespace Blobs { * @brief Optional. Configures whether to do content validation for blob downloads. */ Azure::Nullable ValidationOptions; - - /** - * @brief Optional. The layout endpoint to use for this download. - * - * When set, the request is sent to this endpoint while preserving the client endpoint in the - * Host header. - */ - std::string LayoutEndpoint; }; /** @@ -1957,27 +1949,6 @@ namespace Azure { namespace Storage { namespace Blobs { { }; - /** - * @brief Optional parameters for #Azure::Storage::Blobs::BlobClient::GetLayout. - */ - struct GetBlobLayoutOptions final - { - /** - * @brief The range of bytes for which to retrieve the layout. - */ - Azure::Nullable Range; - - /** - * @brief The continuation token for retrieving the next page. - */ - Azure::Nullable ContinuationToken; - - /** - * @brief Access conditions for the operation. - */ - BlobAccessConditions AccessConditions; - }; - namespace _detail { inline std::string TagsToString(const std::map& tags) { diff --git a/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_responses.hpp b/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_responses.hpp index 4521c85d1e..de54ac6a48 100644 --- a/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_responses.hpp +++ b/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_responses.hpp @@ -29,38 +29,6 @@ namespace Azure { namespace Storage { namespace Models { - /** - * @brief Describes a range in a blob's data locality layout. - */ - struct BlobLayoutRange final - { - /** - * @brief The range of bytes served by the endpoint. - */ - Azure::Core::Http::HttpRange Range; - - /** - * @brief The host and port of the endpoint serving this range. - */ - std::string Endpoint; - }; - - /** - * @brief A page of blob layout information and the blob properties returned with it. - */ - struct BlobLayoutInfo final - { - /** - * @brief The ranges in this page of the blob's data locality layout. - */ - std::vector Ranges; - - /** - * @brief The blob properties returned with the layout. - */ - BlobProperties Properties; - }; - /** * @brief Response type for #Azure::Storage::Blobs::BlobClient::DownloadTo. */ @@ -514,27 +482,6 @@ namespace Azure { namespace Storage { friend class Azure::Core::PagedResponse; }; - /** - * @brief Response type for #Azure::Storage::Blobs::BlobClient::GetLayout. - */ - class BlobLayoutPagedResponse final - : public Azure::Core::PagedResponse { - public: - /** - * @brief The layout and blob properties in this page. - */ - Models::BlobLayoutInfo Layout; - - private: - void OnNextPage(const Azure::Core::Context& context); - - std::shared_ptr m_blobClient; - GetBlobLayoutOptions m_operationOptions; - - friend class Blobs::BlobClient; - friend class Azure::Core::PagedResponse; - }; - namespace _detail { void ParseListBlobsResult(Models::_detail::ListBlobsResult& result); } // namespace _detail diff --git a/sdk/storage/azure-storage-blobs/src/blob_client.cpp b/sdk/storage/azure-storage-blobs/src/blob_client.cpp index afcda50efd..ad36f3149b 100644 --- a/sdk/storage/azure-storage-blobs/src/blob_client.cpp +++ b/sdk/storage/azure-storage-blobs/src/blob_client.cpp @@ -112,10 +112,10 @@ namespace Azure { namespace Storage { namespace Blobs { blobClient = std::move(blobClient)]() { try { - GetBlobLayoutOptions options; + _detail::GetBlobLayoutOptions options; options.Range = range; options.AccessConditions.IfMatch = eTag; - auto page = blobClient.GetLayout(options, context); + auto page = _detail::BlobLayoutClient::GetLayout(blobClient, options, context); _detail::DataLocalityLayout layout; for (; page.HasPage(); page.MoveToNextPage(context)) { @@ -140,7 +140,7 @@ namespace Azure { namespace Storage { namespace Blobs { std::sort( layout.Ranges.begin(), layout.Ranges.end(), - [](const Models::BlobLayoutRange& lhs, const Models::BlobLayoutRange& rhs) { + [](const _detail::BlobLayoutRange& lhs, const _detail::BlobLayoutRange& rhs) { return lhs.Range.Offset < rhs.Range.Offset; }); return layout; @@ -348,9 +348,8 @@ namespace Azure { namespace Storage { namespace Blobs { } std::string dataLocalityEndpoint; - const auto downloadContext = !options.LayoutEndpoint.empty() - ? _internal::WithDataLocalityEndpoint(context, options.LayoutEndpoint) - : context.TryGetValue(_internal::DataLocalityEndpointKey, dataLocalityEndpoint) + const auto downloadContext + = context.TryGetValue(_internal::DataLocalityEndpointKey, dataLocalityEndpoint) ? context : _internal::WithReplicaStatus(context); auto downloadResponse = _detail::BlobClient::Download( @@ -488,9 +487,10 @@ namespace Azure { namespace Storage { namespace Blobs { return downloadResponse; } - BlobLayoutPagedResponse BlobClient::GetLayout( + _detail::BlobLayoutPagedResponse _detail::BlobLayoutClient::GetLayout( + const Blobs::BlobClient& blobClient, const GetBlobLayoutOptions& options, - const Azure::Core::Context& context) const + const Azure::Core::Context& context) { _detail::BlobClient::GetBlobLayoutOptions protocolLayerOptions; protocolLayerOptions.Marker = options.ContinuationToken; @@ -510,17 +510,18 @@ namespace Azure { namespace Storage { namespace Blobs { protocolLayerOptions.IfUnmodifiedSince = options.AccessConditions.IfUnmodifiedSince; protocolLayerOptions.IfMatch = options.AccessConditions.IfMatch; protocolLayerOptions.IfNoneMatch = options.AccessConditions.IfNoneMatch; - if (m_clientConfiguration.CustomerProvidedKey.HasValue()) + if (blobClient.m_clientConfiguration.CustomerProvidedKey.HasValue()) { - protocolLayerOptions.EncryptionKey = m_clientConfiguration.CustomerProvidedKey.Value().Key; + protocolLayerOptions.EncryptionKey + = blobClient.m_clientConfiguration.CustomerProvidedKey.Value().Key; protocolLayerOptions.EncryptionKeySha256 - = m_clientConfiguration.CustomerProvidedKey.Value().KeyHash; + = blobClient.m_clientConfiguration.CustomerProvidedKey.Value().KeyHash; protocolLayerOptions.EncryptionAlgorithm - = m_clientConfiguration.CustomerProvidedKey.Value().Algorithm.ToString(); + = blobClient.m_clientConfiguration.CustomerProvidedKey.Value().Algorithm.ToString(); } - auto response - = _detail::BlobClient::GetLayout(*m_pipeline, m_blobUrl, protocolLayerOptions, context); + auto response = _detail::BlobClient::GetLayout( + *blobClient.m_pipeline, blobClient.m_blobUrl, protocolLayerOptions, context); BlobLayoutPagedResponse pagedResponse; std::map endpoints; for (const auto& endpoint : response.Value.Endpoints) @@ -544,7 +545,7 @@ namespace Azure { namespace Storage { namespace Blobs { { throw Azure::Core::RequestFailedException("Invalid blob layout range."); } - Models::BlobLayoutRange layoutRange; + BlobLayoutRange layoutRange; layoutRange.Range.Offset = range.Start; layoutRange.Range.Length = static_cast(rangeLength); layoutRange.Endpoint = endpoint->second; @@ -557,7 +558,7 @@ namespace Azure { namespace Storage { namespace Blobs { } CompleteBlobPropertiesFromResponse(pagedResponse.Layout.Properties, *response.RawResponse); - pagedResponse.m_blobClient = std::make_shared(*this); + pagedResponse.m_blobClient = std::make_shared(blobClient); pagedResponse.m_operationOptions = options; pagedResponse.CurrentPageToken = options.ContinuationToken.ValueOr(std::string()); pagedResponse.NextPageToken = response.Value.NextMarker; @@ -654,9 +655,10 @@ namespace Azure { namespace Storage { namespace Blobs { chunkOptions.Range.Value().Length = length; chunkOptions.AccessConditions.IfMatch = eTag; chunkOptions.ValidationOptions = options.ValidationOptions; - const auto chunkContext = dataLocalityState ? _internal::WithDataLocalityEndpoint( - context, dataLocalityState->GetEndpoint(offset, length)) - : context; + const auto chunkContext = dataLocalityState + ? _internal::WithDataLocalityEndpoint( + context, dataLocalityState->GetEndpoint(offset, length)) + : context; auto chunk = Download(chunkOptions, chunkContext); int64_t bytesRead = chunk.Value.BodyStream->ReadToCount( buffer + (offset - firstChunkOffset), @@ -824,9 +826,10 @@ namespace Azure { namespace Storage { namespace Blobs { chunkOptions.Range.Value().Length = length; chunkOptions.AccessConditions.IfMatch = eTag; chunkOptions.ValidationOptions = options.ValidationOptions; - const auto chunkContext = dataLocalityState ? _internal::WithDataLocalityEndpoint( - context, dataLocalityState->GetEndpoint(offset, length)) - : context; + const auto chunkContext = dataLocalityState + ? _internal::WithDataLocalityEndpoint( + context, dataLocalityState->GetEndpoint(offset, length)) + : context; auto chunk = Download(chunkOptions, chunkContext); bodyStreamToFile( *(chunk.Value.BodyStream), diff --git a/sdk/storage/azure-storage-blobs/src/blob_responses.cpp b/sdk/storage/azure-storage-blobs/src/blob_responses.cpp index 3f7279681d..077ce0c135 100644 --- a/sdk/storage/azure-storage-blobs/src/blob_responses.cpp +++ b/sdk/storage/azure-storage-blobs/src/blob_responses.cpp @@ -7,14 +7,15 @@ #include "azure/storage/blobs/blob_container_client.hpp" #include "azure/storage/blobs/blob_service_client.hpp" #include "azure/storage/blobs/page_blob_client.hpp" +#include "private/data_locality.hpp" namespace Azure { namespace Storage { namespace Blobs { - void BlobLayoutPagedResponse::OnNextPage(const Azure::Core::Context& context) + void _detail::BlobLayoutPagedResponse::OnNextPage(const Azure::Core::Context& context) { m_operationOptions.ContinuationToken = NextPageToken; m_operationOptions.AccessConditions.IfMatch = Layout.Properties.ETag; - *this = m_blobClient->GetLayout(m_operationOptions, context); + *this = BlobLayoutClient::GetLayout(*m_blobClient, m_operationOptions, context); } std::unique_ptr StartBlobCopyOperation::PollInternal( diff --git a/sdk/storage/azure-storage-blobs/src/private/data_locality.cpp b/sdk/storage/azure-storage-blobs/src/private/data_locality.cpp index 698bcf0b22..dff056d632 100644 --- a/sdk/storage/azure-storage-blobs/src/private/data_locality.cpp +++ b/sdk/storage/azure-storage-blobs/src/private/data_locality.cpp @@ -28,7 +28,7 @@ namespace Azure { namespace Storage { namespace Blobs { namespace _detail { layout.Ranges.begin(), layout.Ranges.end(), offset, - [](const Models::BlobLayoutRange& value, int64_t target) { + [](const BlobLayoutRange& value, int64_t target) { return value.Range.Offset <= target && value.Range.Length.Value() <= target - value.Range.Offset; }); diff --git a/sdk/storage/azure-storage-blobs/src/private/data_locality.hpp b/sdk/storage/azure-storage-blobs/src/private/data_locality.hpp index 4871ca9df6..0695bbd80c 100644 --- a/sdk/storage/azure-storage-blobs/src/private/data_locality.hpp +++ b/sdk/storage/azure-storage-blobs/src/private/data_locality.hpp @@ -19,9 +19,50 @@ namespace Azure { namespace Storage { namespace Blobs { namespace _detail { constexpr std::chrono::minutes DataLocalityLayoutValidity{5}; constexpr std::chrono::minutes DataLocalityLayoutExtendedExpiry{5}; + struct BlobLayoutRange final + { + Azure::Core::Http::HttpRange Range; + std::string Endpoint; + }; + + struct BlobLayoutInfo final + { + std::vector Ranges; + Models::BlobProperties Properties; + }; + + struct GetBlobLayoutOptions final + { + Azure::Nullable Range; + Azure::Nullable ContinuationToken; + BlobAccessConditions AccessConditions; + }; + + class BlobLayoutPagedResponse final : public Azure::Core::PagedResponse { + public: + BlobLayoutInfo Layout; + + private: + void OnNextPage(const Azure::Core::Context& context); + + std::shared_ptr m_blobClient; + GetBlobLayoutOptions m_operationOptions; + + friend class BlobLayoutClient; + friend class Azure::Core::PagedResponse; + }; + + class BlobLayoutClient final { + public: + static BlobLayoutPagedResponse GetLayout( + const Blobs::BlobClient& blobClient, + const GetBlobLayoutOptions& options = GetBlobLayoutOptions(), + const Azure::Core::Context& context = Azure::Core::Context()); + }; + struct DataLocalityLayout final { - std::vector Ranges; + std::vector Ranges; int64_t BlobSize = 0; Azure::ETag ETag; }; diff --git a/sdk/storage/azure-storage-blobs/test/ut/data_locality_test.cpp b/sdk/storage/azure-storage-blobs/test/ut/data_locality_test.cpp index 2e5e7c15ed..19edf315ca 100644 --- a/sdk/storage/azure-storage-blobs/test/ut/data_locality_test.cpp +++ b/sdk/storage/azure-storage-blobs/test/ut/data_locality_test.cpp @@ -72,12 +72,13 @@ namespace Azure { namespace Storage { namespace Test { { std::lock_guard lock(m_state->Mutex); - m_state->Requests.push_back(LocalityRequest{ - request.GetUrl().GetHost(), - hostHeader == headers.end() ? std::string() : hostHeader->second, - ifMatchHeader == headers.end() ? std::string() : ifMatchHeader->second, - rangeHeader == headers.end() ? std::string() : rangeHeader->second, - isLayout}); + m_state->Requests.push_back( + LocalityRequest{ + request.GetUrl().GetHost(), + hostHeader == headers.end() ? std::string() : hostHeader->second, + ifMatchHeader == headers.end() ? std::string() : ifMatchHeader->second, + rangeHeader == headers.end() ? std::string() : rangeHeader->second, + isLayout}); } if (isLayout) @@ -164,9 +165,10 @@ namespace Azure { namespace Storage { namespace Test { rangeHeader == headers.end() ? Core::Http::HttpStatusCode::Ok : Core::Http::HttpStatusCode::PartialContent, "OK"); - response->SetBodyStream(std::make_unique( - reinterpret_cast(m_data->data() + offset), - static_cast(length))); + response->SetBodyStream( + std::make_unique( + reinterpret_cast(m_data->data() + offset), + static_cast(length))); response->SetHeader("content-length", std::to_string(length)); if (rangeHeader != headers.end()) { @@ -223,12 +225,12 @@ namespace Azure { namespace Storage { namespace Test { return options; } - Blobs::Models::BlobLayoutRange MakeLayoutRange( + Blobs::_detail::BlobLayoutRange MakeLayoutRange( int64_t offset, int64_t length, std::string endpoint) { - Blobs::Models::BlobLayoutRange range; + Blobs::_detail::BlobLayoutRange range; range.Range.Offset = offset; range.Range.Length = length; range.Endpoint = std::move(endpoint); @@ -243,12 +245,11 @@ namespace Azure { namespace Storage { namespace Test { } } // namespace - class DataLocalityRecordedTest : public StorageTest { - }; + class DataLocalityRecordedTest : public StorageTest {}; TEST(DataLocalityTest, SelectsIdealEndpoint) { - using Range = Blobs::Models::BlobLayoutRange; + using Range = Blobs::_detail::BlobLayoutRange; struct TestCase final { const char* Name; @@ -337,7 +338,7 @@ namespace Azure { namespace Storage { namespace Test { auto data = std::make_shared(1024 * 1024, 'n'); auto client = CreateLocalityClient(state, data); - auto page = client.GetLayout(); + auto page = Blobs::_detail::BlobLayoutClient::GetLayout(client); EXPECT_TRUE(page.Layout.Ranges.empty()); EXPECT_EQ(page.Layout.Properties.ETag.ToString(), "\"locality-etag\""); @@ -355,9 +356,9 @@ namespace Azure { namespace Storage { namespace Test { "data-locality-multi-endpoint.bin", clientOptions); - std::vector ranges; + std::vector ranges; std::set endpoints; - auto page = blobClient.GetLayout(); + auto page = Blobs::_detail::BlobLayoutClient::GetLayout(blobClient); EXPECT_EQ(page.Layout.Properties.BlobSize, 64 * 1024 * 1024); EXPECT_EQ(page.Layout.Properties.BlobType, Blobs::Models::BlobType::BlockBlob); EXPECT_EQ(page.Layout.Properties.HttpHeaders.ContentType, "application/octet-stream"); @@ -416,7 +417,7 @@ namespace Azure { namespace Storage { namespace Test { for (bool bounded : {true, false}) { SCOPED_TRACE(bounded); - Blobs::GetBlobLayoutOptions options; + Blobs::_detail::GetBlobLayoutOptions options; options.Range = Core::Http::HttpRange(); options.Range.Value().Offset = bounded ? 8 * 1024 * 1024 - 2 : blobSize - 16; if (bounded) @@ -426,7 +427,8 @@ namespace Azure { namespace Storage { namespace Test { const auto offset = options.Range.Value().Offset; bool coversOffset = false; size_t rangeCount = 0; - for (auto page = client.GetLayout(options); page.HasPage(); page.MoveToNextPage()) + for (auto page = Blobs::_detail::BlobLayoutClient::GetLayout(client, options); page.HasPage(); + page.MoveToNextPage()) { EXPECT_EQ(page.Layout.Properties.BlobSize, blobSize); EXPECT_GT(page.Layout.Properties.BlobSize, options.Range.Value().Length.ValueOr(16)); @@ -775,14 +777,13 @@ namespace Azure { namespace Storage { namespace Test { EXPECT_EQ(state->Requests[2].IfMatch, "\"locality-etag\""); } - TEST(DataLocalityTest, RoutesOneShotDownloadToLayoutEndpoint) + TEST(DataLocalityTest, OneShotDownloadUsesClientEndpoint) { auto state = std::make_shared(); auto data = std::make_shared(4 * 1024, 'o'); auto client = CreateLocalityClient(state, data); Blobs::DownloadBlobOptions options; - options.LayoutEndpoint = "locality0.test:443"; auto response = client.Download(options); EXPECT_TRUE(response.Value.Details.DownloadHint.HasValue()); @@ -791,8 +792,8 @@ namespace Azure { namespace Storage { namespace Test { std::lock_guard lock(state->Mutex); ASSERT_EQ(state->Requests.size(), 1U); EXPECT_FALSE(state->Requests.front().IsLayout); - EXPECT_EQ(state->Requests.front().Host, "locality0.test"); - EXPECT_EQ(state->Requests.front().HostHeader, "primary.test"); + EXPECT_EQ(state->Requests.front().Host, "primary.test"); + EXPECT_TRUE(state->Requests.front().HostHeader.empty()); } TEST(DataLocalityTest, FallsBackOnUnsupportedLayout) diff --git a/sdk/storage/azure-storage-files-datalake/CHANGELOG.md b/sdk/storage/azure-storage-files-datalake/CHANGELOG.md index b33a93452f..4df178ac32 100644 --- a/sdk/storage/azure-storage-files-datalake/CHANGELOG.md +++ b/sdk/storage/azure-storage-files-datalake/CHANGELOG.md @@ -5,11 +5,7 @@ ### Features Added - Added support for persisted session authentication for eligible file download requests. -- Added `DataLakeFileClient::GetLayout` and related options and response types for retrieving - locality endpoints, byte ranges, and file properties. Ranges are exposed directly through - `Models::FileLayoutInfo::Ranges` as a vector of `Models::FileLayoutRange`. -- Added `DownloadFileOptions::LayoutEndpoint` for one-shot download routing and - `DownloadFileToOptions::LayoutAwareRouting` (`Auto`, `Disabled`, `Enabled`; default `Disabled`) for +- Added `DownloadFileToOptions::LayoutAwareRouting` (`Auto`, `Disabled`, `Enabled`; default `Disabled`) for managed downloads. - Added the `Models::DownloadHint` alias and `Models::DownloadFileDetails::DownloadHint`. @@ -22,6 +18,8 @@ ### Other Changes +- Removed public layout discovery and explicit single-shot download endpoint selection while retaining + locality-aware managed downloads. - Changed the default initial transfer size for partitioned uploads and downloads from 256 MiB to 4 MiB, matching the default subsequent chunk size. Explicitly configured `InitialChunkSize` and `SingleUploadThreshold` values are still respected. diff --git a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_file_client.hpp b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_file_client.hpp index e3af80df50..e04d891bc0 100644 --- a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_file_client.hpp +++ b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_file_client.hpp @@ -189,17 +189,6 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { const DownloadFileOptions& options = DownloadFileOptions(), const Azure::Core::Context& context = Azure::Core::Context()) const; - /** - * @brief Gets the data locality layout for this file. - * @param options Optional parameters to execute this function. - * @param context Context for cancelling long running operations. - * @return A paged response containing the file's data locality layout. - * @remark This request is sent to the blob endpoint. - */ - FileLayoutPagedResponse GetLayout( - const GetFileLayoutOptions& options = GetFileLayoutOptions(), - const Azure::Core::Context& context = Azure::Core::Context()) const; - /** * @brief Creates a new file, or updates the content of an existing file. Updating * an existing file overwrites any existing metadata on the file. @@ -299,10 +288,10 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { std::shared_ptr pipeline, _detail::DatalakeClientConfiguration clientConfiguration) : DataLakePathClient( - std::move(fileUrl), - std::move(blobClient), - pipeline, - std::move(clientConfiguration)) + std::move(fileUrl), + std::move(blobClient), + pipeline, + std::move(clientConfiguration)) { } diff --git a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_options.hpp b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_options.hpp index 266eb0b768..ee231c28cb 100644 --- a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_options.hpp +++ b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_options.hpp @@ -890,27 +890,6 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { Nullable IncludeUserPrincipalName; }; - /** - * @brief Optional parameters for #Azure::Storage::Files::DataLake::DataLakeFileClient::GetLayout. - */ - struct GetFileLayoutOptions final - { - /** - * @brief The range of bytes for which to retrieve the layout. - */ - Azure::Nullable Range; - - /** - * @brief The continuation token for retrieving the next page. - */ - Azure::Nullable ContinuationToken; - - /** - * @brief Access conditions for the operation. - */ - PathAccessConditions AccessConditions; - }; - /** * @brief Optional parameters for #Azure::Storage::Files::DataLake::DataLakeFileClient::Download. * @remark Some optional parameter is mandatory in certain combination. @@ -949,12 +928,6 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { * @brief Optional. Configures whether to do content validation for file downloads. */ Azure::Nullable ValidationOptions; - - /** - * @brief The locality endpoint to use for this download. An empty value uses the normal - * endpoint. - */ - std::string LayoutEndpoint; }; /** diff --git a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_responses.hpp b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_responses.hpp index d0660cf788..7cdd9f8dcd 100644 --- a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_responses.hpp +++ b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_responses.hpp @@ -463,26 +463,6 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { Azure::Nullable Permissions; }; - /** - * @brief A file range and its resolved locality endpoint. - */ - using FileLayoutRange = Blobs::Models::BlobLayoutRange; - - /** - * @brief A page of file layout information and the file properties returned with it. - */ - struct FileLayoutInfo final - { - /** - * @brief The ranges and locality endpoints in this page. - */ - std::vector Ranges; - /** - * @brief The file properties returned with this page. - */ - PathProperties Properties; - }; - /** * @brief The access control list of a path. */ @@ -823,27 +803,6 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { } // namespace Models - /** - * @brief Response type for - * #Azure::Storage::Files::DataLake::DataLakeFileClient::GetLayout. - */ - class FileLayoutPagedResponse final : public Azure::Core::PagedResponse { - public: - /** - * @brief The layout and file properties in this page. - */ - Models::FileLayoutInfo Layout; - - private: - void OnNextPage(const Azure::Core::Context& context); - - std::shared_ptr m_fileClient; - GetFileLayoutOptions m_operationOptions; - - friend class DataLakeFileClient; - friend class Azure::Core::PagedResponse; - }; - /** * @brief Response type for * #Azure::Storage::Files::DataLake::DataLakeServiceClient::ListFileSystems. diff --git a/sdk/storage/azure-storage-files-datalake/src/datalake_file_client.cpp b/sdk/storage/azure-storage-files-datalake/src/datalake_file_client.cpp index 6b957c633b..1a7a63b8be 100644 --- a/sdk/storage/azure-storage-files-datalake/src/datalake_file_client.cpp +++ b/sdk/storage/azure-storage-files-datalake/src/datalake_file_client.cpp @@ -204,7 +204,6 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { blobOptions.Range = options.Range; blobOptions.RangeHashAlgorithm = options.RangeHashAlgorithm; blobOptions.Range = options.Range; - blobOptions.LayoutEndpoint = options.LayoutEndpoint; blobOptions.AccessConditions.IfMatch = options.AccessConditions.IfMatch; blobOptions.AccessConditions.IfNoneMatch = options.AccessConditions.IfNoneMatch; blobOptions.AccessConditions.IfModifiedSince = options.AccessConditions.IfModifiedSince; @@ -218,10 +217,11 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { } auto response = m_blobClient.Download( blobOptions, - options.IncludeUserPrincipalName.HasValue() ? context.WithValue( - Blobs::_detail::DataLakeInteroperabilityExtraOptionsKey, - options.IncludeUserPrincipalName.Value()) - : context); + options.IncludeUserPrincipalName.HasValue() + ? context.WithValue( + Blobs::_detail::DataLakeInteroperabilityExtraOptionsKey, + options.IncludeUserPrincipalName.Value()) + : context); Models::DownloadFileResult ret; ret.Body = std::move(response.Value.BodyStream); ret.Details.HttpHeaders = std::move(response.Value.Details.HttpHeaders); @@ -285,32 +285,6 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { std::move(ret), std::move(response.RawResponse)); } - FileLayoutPagedResponse DataLakeFileClient::GetLayout( - const GetFileLayoutOptions& options, - const Azure::Core::Context& context) const - { - Blobs::GetBlobLayoutOptions blobOptions; - blobOptions.Range = options.Range; - blobOptions.ContinuationToken = options.ContinuationToken; - blobOptions.AccessConditions.IfMatch = options.AccessConditions.IfMatch; - blobOptions.AccessConditions.IfNoneMatch = options.AccessConditions.IfNoneMatch; - blobOptions.AccessConditions.IfModifiedSince = options.AccessConditions.IfModifiedSince; - blobOptions.AccessConditions.IfUnmodifiedSince = options.AccessConditions.IfUnmodifiedSince; - blobOptions.AccessConditions.LeaseId = options.AccessConditions.LeaseId; - auto response = m_blobClient.GetLayout(blobOptions, context); - - FileLayoutPagedResponse result; - result.Layout.Ranges = std::move(response.Layout.Ranges); - result.Layout.Properties = _detail::PathPropertiesFromBlobProperties( - std::move(response.Layout.Properties), *response.RawResponse); - result.CurrentPageToken = std::move(response.CurrentPageToken); - result.NextPageToken = std::move(response.NextPageToken); - result.RawResponse = std::move(response.RawResponse); - result.m_fileClient = std::make_shared(*this); - result.m_operationOptions = options; - return result; - } - Azure::Response DataLakeFileClient::UploadFrom( const std::string& fileName, const UploadFileFromOptions& options, diff --git a/sdk/storage/azure-storage-files-datalake/src/datalake_responses.cpp b/sdk/storage/azure-storage-files-datalake/src/datalake_responses.cpp index 3652ac5891..4ad9003d78 100644 --- a/sdk/storage/azure-storage-files-datalake/src/datalake_responses.cpp +++ b/sdk/storage/azure-storage-files-datalake/src/datalake_responses.cpp @@ -10,14 +10,6 @@ #include "private/datalake_utilities.hpp" namespace Azure { namespace Storage { namespace Files { namespace DataLake { - void FileLayoutPagedResponse::OnNextPage(const Azure::Core::Context& context) - { - auto options = m_operationOptions; - options.ContinuationToken = NextPageToken; - options.AccessConditions.IfMatch = Layout.Properties.ETag; - *this = m_fileClient->GetLayout(options, context); - } - namespace Models { Acl Acl::FromString(const std::string& aclString) diff --git a/sdk/storage/azure-storage-files-datalake/test/ut/data_locality_test.cpp b/sdk/storage/azure-storage-files-datalake/test/ut/data_locality_test.cpp index 899b4191e3..cfb27763b4 100644 --- a/sdk/storage/azure-storage-files-datalake/test/ut/data_locality_test.cpp +++ b/sdk/storage/azure-storage-files-datalake/test/ut/data_locality_test.cpp @@ -8,7 +8,6 @@ #include #include #include -#include #include #include @@ -58,21 +57,21 @@ namespace Azure { namespace Storage { namespace Test { }; const bool isLayout = query.count("comp") && query.at("comp") == "layout"; const auto marker = query.count("marker") ? query.at("marker") : std::string(); - m_state->Requests.push_back(LocalityRequest{ - request.GetUrl().GetHost(), - header("host"), - header("x-ms-range"), - header("if-match"), - marker, - isLayout}); + m_state->Requests.push_back( + LocalityRequest{ + request.GetUrl().GetHost(), + header("host"), + header("x-ms-range"), + header("if-match"), + marker, + isLayout}); auto response = std::make_unique( 1, 1, - isLayout - ? (m_state->NoLayout ? Core::Http::HttpStatusCode::NoContent - : Core::Http::HttpStatusCode::Ok) - : Core::Http::HttpStatusCode::PartialContent, + isLayout ? (m_state->NoLayout ? Core::Http::HttpStatusCode::NoContent + : Core::Http::HttpStatusCode::Ok) + : Core::Http::HttpStatusCode::PartialContent, "OK"); response->SetHeader("etag", "\"layout-etag\""); response->SetHeader("last-modified", "Thu, 23 Aug 2001 07:00:00 GMT"); @@ -113,9 +112,10 @@ namespace Azure { namespace Storage { namespace Test { response->SetHeader( "content-range", "bytes " + std::to_string(offset) + "-" + std::to_string(end) + "/12"); response->SetHeader("x-ms-download-hint", "layout"); - response->SetBodyStream(std::make_unique( - reinterpret_cast(data.data() + offset), - static_cast(end - offset + 1))); + response->SetBodyStream( + std::make_unique( + reinterpret_cast(data.data() + offset), + static_cast(end - offset + 1))); return response; } @@ -148,162 +148,29 @@ namespace Azure { namespace Storage { namespace Test { } } // namespace - class DataLakeLocalityRecordedTest : public StorageTest { - }; - - TEST_F(DataLakeLocalityRecordedTest, GetLayoutAndDownloadHint_PLAYBACKONLY_) - { - if (!m_testContext.IsPlaybackMode()) - { - GTEST_SKIP() << "Reuses the Blob multi-endpoint recording."; - } - m_testContext.RecordingPath = std::string(AZURE_TEST_RECORDING_DIR) - + "/../../../azure-storage-blobs/test/ut/recordings"; - m_testContext.RenameTest( - "DataLocalityRecordedTest", "GetLayoutMultipleEndpoints_PLAYBACKONLY_"); - auto options = InitStorageClientOptions(); - auto client = DataLake::DataLakeFileClient( - "https://" + StandardStorageAccountName() - + ".dfs.core.windows.net/data-locality-test/data-locality-multi-endpoint.bin", - GetTestCredential(), - options); - auto page = client.GetLayout(); - EXPECT_EQ(page.Layout.Properties.FileSize, 64 * 1024 * 1024); - EXPECT_EQ(page.Layout.Properties.HttpHeaders.ContentType, "application/octet-stream"); - ASSERT_TRUE(page.Layout.Properties.IsServerEncrypted.HasValue()); - EXPECT_TRUE(page.Layout.Properties.IsServerEncrypted.Value()); - EXPECT_TRUE(page.Layout.Properties.ETag.HasValue()); - EXPECT_FALSE(page.Layout.Properties.IsDirectory); - std::set endpoints; - std::vector ranges; - for (; page.HasPage(); page.MoveToNextPage()) - { - for (const auto& range : page.Layout.Ranges) - { - ASSERT_TRUE(range.Range.Length.HasValue()); - EXPECT_GT(range.Range.Length.Value(), 0); - EXPECT_FALSE(range.Endpoint.empty()); - ranges.push_back(range); - endpoints.insert(range.Endpoint); - } - } - EXPECT_GE(endpoints.size(), 2U); - ASSERT_GT(ranges.size(), 1U); - DataLake::DownloadFileOptions downloadOptions; - downloadOptions.Range = Core::Http::HttpRange(); - downloadOptions.Range.Value().Offset = ranges[1].Range.Offset - 2; - downloadOptions.Range.Value().Length = 5; - auto response = client.Download(downloadOptions); - EXPECT_EQ(response.Value.Body->ReadToEnd().size(), 5U); - EXPECT_EQ(response.Value.FileSize, 64 * 1024 * 1024); - ASSERT_TRUE(response.Value.Details.DownloadHint.HasValue()); - EXPECT_EQ(response.Value.Details.DownloadHint.Value(), Blobs::Models::DownloadHint::Layout); - } - - TEST_F(DataLakeLocalityRecordedTest, GetLayoutRangeReturnsFullFileSize_PLAYBACKONLY_) - { - if (!m_testContext.IsPlaybackMode()) - { - GTEST_SKIP() << "Reuses the Blob ranged layout recording."; - } - m_testContext.RecordingPath = std::string(AZURE_TEST_RECORDING_DIR) - + "/../../../azure-storage-blobs/test/ut/recordings"; - m_testContext.RenameTest( - "DataLocalityRecordedTest", "GetLayoutRangeReturnsFullBlobSize_PLAYBACKONLY_"); - auto clientOptions = InitStorageClientOptions(); - auto client = DataLake::DataLakeFileClient( - "https://" + StandardStorageAccountName() - + ".dfs.core.windows.net/data-locality-test/data-locality-multi-endpoint.bin", - GetTestCredential(), - clientOptions); - constexpr int64_t fileSize = 64 * 1024 * 1024; - for (bool bounded : {true, false}) - { - SCOPED_TRACE(bounded); - DataLake::GetFileLayoutOptions options; - options.Range = Core::Http::HttpRange(); - options.Range.Value().Offset = bounded ? 8 * 1024 * 1024 - 2 : fileSize - 16; - if (bounded) - { - options.Range.Value().Length = 5; - } - const auto offset = options.Range.Value().Offset; - bool coversOffset = false; - size_t rangeCount = 0; - for (auto page = client.GetLayout(options); page.HasPage(); page.MoveToNextPage()) - { - EXPECT_EQ(page.Layout.Properties.FileSize, fileSize); - EXPECT_GT(page.Layout.Properties.FileSize, options.Range.Value().Length.ValueOr(16)); - EXPECT_TRUE(page.Layout.Properties.ETag.HasValue()); - for (const auto& range : page.Layout.Ranges) - { - ASSERT_TRUE(range.Range.Length.HasValue()); - ASSERT_GT(range.Range.Length.Value(), 0); - ASSERT_GE(range.Range.Offset, 0); - ASSERT_LE(range.Range.Offset, fileSize); - ASSERT_LE(range.Range.Length.Value(), fileSize - range.Range.Offset); - EXPECT_FALSE(range.Endpoint.empty()); - coversOffset = coversOffset - || (range.Range.Offset <= offset - && offset - range.Range.Offset < range.Range.Length.Value()); - ++rangeCount; - } - } - EXPECT_GT(rangeCount, 0U); - EXPECT_TRUE(coversOffset); - } - } - - TEST(DataLakeLocalityTest, GetLayoutMapsPropertiesAndPages) - { - auto state = std::make_shared(); - auto client = CreateClient(state, 0); - DataLake::GetFileLayoutOptions options; - options.Range = Core::Http::HttpRange(); - options.Range.Value().Offset = 2; - options.Range.Value().Length = 10; - options.AccessConditions.IfMatch = ETag("\"layout-etag\""); - auto page = client.GetLayout(options); - EXPECT_EQ(page.CurrentPageToken, ""); - EXPECT_EQ(page.NextPageToken.Value(), "page2"); - ASSERT_EQ(page.Layout.Ranges.size(), 1U); - EXPECT_EQ(page.Layout.Ranges[0].Endpoint, "locality0.test:443"); - EXPECT_EQ(page.Layout.Properties.FileSize, 12); - EXPECT_EQ(page.Layout.Properties.Metadata.at("layout"), "metadata"); - EXPECT_EQ(page.Layout.Properties.Owner.Value(), "owner"); - EXPECT_EQ(page.Layout.Properties.Group.Value(), "group"); - EXPECT_EQ(page.Layout.Properties.Permissions.Value(), "rw-r-----"); - EXPECT_EQ(page.Layout.Properties.EncryptionContext.Value(), "context"); - ASSERT_TRUE(page.Layout.Properties.Acls.HasValue()); - EXPECT_EQ(page.Layout.Properties.Acls.Value().size(), 3U); - page.MoveToNextPage(); - EXPECT_EQ(page.CurrentPageToken, "page2"); - ASSERT_EQ(page.Layout.Ranges.size(), 1U); - EXPECT_EQ(page.Layout.Ranges[0].Range.Offset, 6); - EXPECT_EQ(page.Layout.Ranges[0].Range.Length.Value(), 6); - EXPECT_EQ(page.Layout.Ranges[0].Endpoint, "locality1.test:443"); - page.MoveToNextPage(); - EXPECT_FALSE(page.HasPage()); - ASSERT_EQ(state->Requests.size(), 2U); - EXPECT_EQ(state->Requests[0].Host, "account.blob.core.windows.net"); - EXPECT_EQ(state->Requests[1].Range, "bytes=2-11"); - EXPECT_EQ(state->Requests[1].IfMatch, "\"layout-etag\""); - EXPECT_EQ(state->Requests[1].Marker, "page2"); - } - - TEST(DataLakeLocalityTest, GetLayoutNoContent) + TEST(DataLakeLocalityTest, ManagedDownloadFallsBackWithoutLayout) { auto state = std::make_shared(); state->NoLayout = true; - auto page = CreateClient(state, 0).GetLayout(); - EXPECT_TRUE(page.Layout.Ranges.empty()); - EXPECT_EQ(page.Layout.Properties.FileSize, 12); - EXPECT_EQ(page.RawResponse->GetStatusCode(), Core::Http::HttpStatusCode::NoContent); - page.MoveToNextPage(); - EXPECT_FALSE(page.HasPage()); + DataLake::DownloadFileToOptions options; + options.Range = Core::Http::HttpRange(); + options.Range.Value().Length = 12; + options.LayoutAwareRouting = DataLake::LayoutAwareRouting::Enabled; + options.TransferOptions.InitialChunkSize = 1; + options.TransferOptions.ChunkSize = 6; + options.TransferOptions.Concurrency = 1; + std::vector content(12); + CreateClient(state, 0).DownloadTo(content.data(), content.size(), options); + EXPECT_EQ(std::string(content.begin(), content.end()), "abcdefghijkl"); + ASSERT_EQ(state->Requests.size(), 4U); + EXPECT_TRUE(state->Requests[1].IsLayout); + for (const auto& request : state->Requests) + { + EXPECT_EQ(request.Host, "account.blob.core.windows.net"); + } } - TEST(DataLakeLocalityTest, RoutesOneShotDownloadForAllClientOrigins) + TEST(DataLakeLocalityTest, OneShotDownloadUsesClientEndpointForAllClientOrigins) { for (int origin = 0; origin != 4; ++origin) { @@ -314,12 +181,11 @@ namespace Azure { namespace Storage { namespace Test { options.Range = Core::Http::HttpRange(); options.Range.Value().Offset = 6; options.Range.Value().Length = 1; - options.LayoutEndpoint = "locality1.test:443"; auto response = client.Download(options); EXPECT_EQ(response.Value.Body->ReadToEnd(), std::vector{'g'}); ASSERT_EQ(state->Requests.size(), 1U); - EXPECT_EQ(state->Requests[0].Host, "locality1.test"); - EXPECT_EQ(state->Requests[0].HostHeader, "account.blob.core.windows.net"); + EXPECT_EQ(state->Requests[0].Host, "account.blob.core.windows.net"); + EXPECT_TRUE(state->Requests[0].HostHeader.empty()); ASSERT_TRUE(response.Value.Details.DownloadHint.HasValue()); EXPECT_EQ(response.Value.Details.DownloadHint.Value(), Blobs::Models::DownloadHint::Layout); } From 10dff20d24bba92cd7572188521be38a3e30b9a0 Mon Sep 17 00:00:00 2001 From: Jinming Hu Date: Fri, 30 Oct 2026 11:45:30 +1100 Subject: [PATCH 2/6] Remove changelog notes for unshipped layout API removals Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- sdk/storage/azure-storage-blobs/CHANGELOG.md | 2 -- sdk/storage/azure-storage-files-datalake/CHANGELOG.md | 2 -- 2 files changed, 4 deletions(-) diff --git a/sdk/storage/azure-storage-blobs/CHANGELOG.md b/sdk/storage/azure-storage-blobs/CHANGELOG.md index 95fd1b89e4..bc163fd414 100644 --- a/sdk/storage/azure-storage-blobs/CHANGELOG.md +++ b/sdk/storage/azure-storage-blobs/CHANGELOG.md @@ -15,8 +15,6 @@ ### Other Changes -- Kept layout discovery private and removed explicit single-shot download endpoint selection while - retaining locality-aware managed downloads. - Changed the default response format for `ListBlobs` and `ListBlobsByHierarchy` from XML to Apache Arrow. Set `ListBlobsOptions::ResponseFormat` to `StorageResponseFormat::Xml` to continue requesting XML responses. diff --git a/sdk/storage/azure-storage-files-datalake/CHANGELOG.md b/sdk/storage/azure-storage-files-datalake/CHANGELOG.md index 4df178ac32..8191034a84 100644 --- a/sdk/storage/azure-storage-files-datalake/CHANGELOG.md +++ b/sdk/storage/azure-storage-files-datalake/CHANGELOG.md @@ -18,8 +18,6 @@ ### Other Changes -- Removed public layout discovery and explicit single-shot download endpoint selection while retaining - locality-aware managed downloads. - Changed the default initial transfer size for partitioned uploads and downloads from 256 MiB to 4 MiB, matching the default subsequent chunk size. Explicitly configured `InitialChunkSize` and `SingleUploadThreshold` values are still respected. From 3835f5a6476abe11054df23fb2d902f4facf6b0d Mon Sep 17 00:00:00 2001 From: Jinming Hu Date: Fri, 30 Oct 2026 11:54:00 +1100 Subject: [PATCH 3/6] Keep layout APIs private on the existing storage clients Move layout options, models and paged responses into _detail namespaces in their original headers. Keep GetLayout private on BlobClient and DataLakeFileClient, remove the separate layout accessor, and retain managed download routing. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../inc/azure/storage/blobs/blob_client.hpp | 13 +- .../inc/azure/storage/blobs/blob_options.hpp | 21 +++ .../azure/storage/blobs/blob_responses.hpp | 55 ++++++++ .../azure-storage-blobs/src/blob_client.cpp | 131 +++++++++--------- .../src/blob_responses.cpp | 3 +- .../src/private/data_locality.cpp | 2 +- .../src/private/data_locality.hpp | 43 +----- .../test/ut/data_locality_test.cpp | 130 +---------------- .../files/datalake/datalake_file_client.hpp | 5 + .../files/datalake/datalake_options.hpp | 23 +++ .../files/datalake/datalake_responses.hpp | 45 ++++++ .../src/datalake_file_client.cpp | 26 ++++ .../src/datalake_responses.cpp | 8 ++ 13 files changed, 265 insertions(+), 240 deletions(-) diff --git a/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_client.hpp b/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_client.hpp index ac542df500..e9d0c611ea 100644 --- a/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_client.hpp +++ b/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_client.hpp @@ -24,7 +24,7 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { namespace Azure { namespace Storage { namespace Blobs { namespace _detail { - class BlobLayoutClient; + class DataLocalityLayoutState; AZ_STORAGE_BLOBS_DLLEXPORT extern const Azure::Core::Context::Key DataLakeInteroperabilityExtraOptionsKey; } // namespace _detail @@ -435,6 +435,15 @@ namespace Azure { namespace Storage { namespace Blobs { _detail::BlobClientConfiguration m_clientConfiguration; private: + _detail::BlobLayoutPagedResponse GetLayout( + const _detail::GetBlobLayoutOptions& options = _detail::GetBlobLayoutOptions(), + const Azure::Core::Context& context = Azure::Core::Context()) const; + + std::unique_ptr<_detail::DataLocalityLayoutState> CreateDataLocalityLayoutState( + Azure::Nullable range, + Azure::ETag eTag, + Azure::Core::Context context) const; + explicit BlobClient( Azure::Core::Url blobUrl, std::shared_ptr pipeline, @@ -451,6 +460,6 @@ namespace Azure { namespace Storage { namespace Blobs { friend class BlobLeaseClient; friend class BlobServiceBatch; friend class BlobContainerBatch; - friend class _detail::BlobLayoutClient; + friend class _detail::BlobLayoutPagedResponse; }; }}} // namespace Azure::Storage::Blobs diff --git a/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_options.hpp b/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_options.hpp index 5c79059a7a..bc6a9825a2 100644 --- a/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_options.hpp +++ b/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_options.hpp @@ -1950,6 +1950,27 @@ namespace Azure { namespace Storage { namespace Blobs { }; namespace _detail { + /** + * @brief Optional parameters for BlobClient::GetLayout. + */ + struct GetBlobLayoutOptions final + { + /** + * @brief The range of bytes for which to retrieve the layout. + */ + Azure::Nullable Range; + + /** + * @brief The continuation token for retrieving the next page. + */ + Azure::Nullable ContinuationToken; + + /** + * @brief Access conditions for the operation. + */ + BlobAccessConditions AccessConditions; + }; + inline std::string TagsToString(const std::map& tags) { return std::accumulate( diff --git a/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_responses.hpp b/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_responses.hpp index de54ac6a48..9bbc9240a5 100644 --- a/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_responses.hpp +++ b/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_responses.hpp @@ -29,6 +29,40 @@ namespace Azure { namespace Storage { namespace Models { + namespace _detail { + /** + * @brief Describes a range in a blob's data locality layout. + */ + struct BlobLayoutRange final + { + /** + * @brief The range of bytes served by the endpoint. + */ + Azure::Core::Http::HttpRange Range; + + /** + * @brief The host and port of the endpoint serving this range. + */ + std::string Endpoint; + }; + + /** + * @brief A page of blob layout information and the blob properties returned with it. + */ + struct BlobLayoutInfo final + { + /** + * @brief The ranges in this page of the blob's data locality layout. + */ + std::vector Ranges; + + /** + * @brief The blob properties returned with the layout. + */ + BlobProperties Properties; + }; + } // namespace _detail + /** * @brief Response type for #Azure::Storage::Blobs::BlobClient::DownloadTo. */ @@ -483,6 +517,27 @@ namespace Azure { namespace Storage { }; namespace _detail { + /** + * @brief Response type for BlobClient::GetLayout. + */ + class BlobLayoutPagedResponse final + : public Azure::Core::PagedResponse { + public: + /** + * @brief The layout and blob properties in this page. + */ + Models::_detail::BlobLayoutInfo Layout; + + private: + void OnNextPage(const Azure::Core::Context& context); + + std::shared_ptr m_blobClient; + GetBlobLayoutOptions m_operationOptions; + + friend class Blobs::BlobClient; + friend class Azure::Core::PagedResponse; + }; + void ParseListBlobsResult(Models::_detail::ListBlobsResult& result); } // namespace _detail diff --git a/sdk/storage/azure-storage-blobs/src/blob_client.cpp b/sdk/storage/azure-storage-blobs/src/blob_client.cpp index ad36f3149b..4ddb334562 100644 --- a/sdk/storage/azure-storage-blobs/src/blob_client.cpp +++ b/sdk/storage/azure-storage-blobs/src/blob_client.cpp @@ -100,65 +100,66 @@ namespace Azure { namespace Storage { namespace Blobs { } } - std::unique_ptr<_detail::DataLocalityLayoutState> CreateDataLocalityLayoutState( - BlobClient blobClient, - Azure::Nullable range, - Azure::ETag eTag, - Azure::Core::Context context) - { - auto refresher = [range = std::move(range), - eTag = std::move(eTag), - context = std::move(context), - blobClient = std::move(blobClient)]() { - try + } // namespace + + std::unique_ptr<_detail::DataLocalityLayoutState> BlobClient::CreateDataLocalityLayoutState( + Azure::Nullable range, + Azure::ETag eTag, + Azure::Core::Context context) const + { + auto refresher = [range = std::move(range), + eTag = std::move(eTag), + context = std::move(context), + blobClient = *this]() { + try + { + _detail::GetBlobLayoutOptions options; + options.Range = range; + options.AccessConditions.IfMatch = eTag; + auto page = blobClient.GetLayout(options, context); + _detail::DataLocalityLayout layout; + for (; page.HasPage(); page.MoveToNextPage(context)) { - _detail::GetBlobLayoutOptions options; - options.Range = range; - options.AccessConditions.IfMatch = eTag; - auto page = _detail::BlobLayoutClient::GetLayout(blobClient, options, context); - _detail::DataLocalityLayout layout; - for (; page.HasPage(); page.MoveToNextPage(context)) + if (!layout.ETag.HasValue()) { - if (!layout.ETag.HasValue()) - { - layout.ETag = page.Layout.Properties.ETag; - } - if (layout.BlobSize == 0) - { - layout.BlobSize = page.Layout.Properties.BlobSize; - } - for (auto& layoutRange : page.Layout.Ranges) + layout.ETag = page.Layout.Properties.ETag; + } + if (layout.BlobSize == 0) + { + layout.BlobSize = page.Layout.Properties.BlobSize; + } + for (auto& layoutRange : page.Layout.Ranges) + { + if (layoutRange.Endpoint.empty() || layoutRange.Range.Offset < 0 + || !layoutRange.Range.Length.HasValue() || layoutRange.Range.Length.Value() <= 0) { - if (layoutRange.Endpoint.empty() || layoutRange.Range.Offset < 0 - || !layoutRange.Range.Length.HasValue() || layoutRange.Range.Length.Value() <= 0) - { - throw Azure::Core::RequestFailedException("Invalid blob layout range."); - } - layout.Ranges.push_back(std::move(layoutRange)); + throw Azure::Core::RequestFailedException("Invalid blob layout range."); } + layout.Ranges.push_back(std::move(layoutRange)); } - std::sort( - layout.Ranges.begin(), - layout.Ranges.end(), - [](const _detail::BlobLayoutRange& lhs, const _detail::BlobLayoutRange& rhs) { - return lhs.Range.Offset < rhs.Range.Offset; - }); - return layout; } - catch (const StorageException& exception) + std::sort( + layout.Ranges.begin(), + layout.Ranges.end(), + [](const Models::_detail::BlobLayoutRange& lhs, + const Models::_detail::BlobLayoutRange& rhs) { + return lhs.Range.Offset < rhs.Range.Offset; + }); + return layout; + } + catch (const StorageException& exception) + { + if (exception.StatusCode == Core::Http::HttpStatusCode::BadRequest + || static_cast(exception.StatusCode) >= 500) { - if (exception.StatusCode == Core::Http::HttpStatusCode::BadRequest - || static_cast(exception.StatusCode) >= 500) - { - return _detail::DataLocalityLayout(); - } - throw; + return _detail::DataLocalityLayout(); } - }; + throw; + } + }; - return std::make_unique<_detail::DataLocalityLayoutState>(std::move(refresher)); - } - } // namespace + return std::make_unique<_detail::DataLocalityLayoutState>(std::move(refresher)); + } BlobClient BlobClient::CreateFromConnectionString( const std::string& connectionString, @@ -487,10 +488,9 @@ namespace Azure { namespace Storage { namespace Blobs { return downloadResponse; } - _detail::BlobLayoutPagedResponse _detail::BlobLayoutClient::GetLayout( - const Blobs::BlobClient& blobClient, - const GetBlobLayoutOptions& options, - const Azure::Core::Context& context) + _detail::BlobLayoutPagedResponse BlobClient::GetLayout( + const _detail::GetBlobLayoutOptions& options, + const Azure::Core::Context& context) const { _detail::BlobClient::GetBlobLayoutOptions protocolLayerOptions; protocolLayerOptions.Marker = options.ContinuationToken; @@ -510,19 +510,18 @@ namespace Azure { namespace Storage { namespace Blobs { protocolLayerOptions.IfUnmodifiedSince = options.AccessConditions.IfUnmodifiedSince; protocolLayerOptions.IfMatch = options.AccessConditions.IfMatch; protocolLayerOptions.IfNoneMatch = options.AccessConditions.IfNoneMatch; - if (blobClient.m_clientConfiguration.CustomerProvidedKey.HasValue()) + if (m_clientConfiguration.CustomerProvidedKey.HasValue()) { - protocolLayerOptions.EncryptionKey - = blobClient.m_clientConfiguration.CustomerProvidedKey.Value().Key; + protocolLayerOptions.EncryptionKey = m_clientConfiguration.CustomerProvidedKey.Value().Key; protocolLayerOptions.EncryptionKeySha256 - = blobClient.m_clientConfiguration.CustomerProvidedKey.Value().KeyHash; + = m_clientConfiguration.CustomerProvidedKey.Value().KeyHash; protocolLayerOptions.EncryptionAlgorithm - = blobClient.m_clientConfiguration.CustomerProvidedKey.Value().Algorithm.ToString(); + = m_clientConfiguration.CustomerProvidedKey.Value().Algorithm.ToString(); } - auto response = _detail::BlobClient::GetLayout( - *blobClient.m_pipeline, blobClient.m_blobUrl, protocolLayerOptions, context); - BlobLayoutPagedResponse pagedResponse; + auto response + = _detail::BlobClient::GetLayout(*m_pipeline, m_blobUrl, protocolLayerOptions, context); + _detail::BlobLayoutPagedResponse pagedResponse; std::map endpoints; for (const auto& endpoint : response.Value.Endpoints) { @@ -545,7 +544,7 @@ namespace Azure { namespace Storage { namespace Blobs { { throw Azure::Core::RequestFailedException("Invalid blob layout range."); } - BlobLayoutRange layoutRange; + Models::_detail::BlobLayoutRange layoutRange; layoutRange.Range.Offset = range.Start; layoutRange.Range.Length = static_cast(rangeLength); layoutRange.Endpoint = endpoint->second; @@ -558,7 +557,7 @@ namespace Azure { namespace Storage { namespace Blobs { } CompleteBlobPropertiesFromResponse(pagedResponse.Layout.Properties, *response.RawResponse); - pagedResponse.m_blobClient = std::make_shared(blobClient); + pagedResponse.m_blobClient = std::make_shared(*this); pagedResponse.m_operationOptions = options; pagedResponse.CurrentPageToken = options.ContinuationToken.ValueOr(std::string()); pagedResponse.NextPageToken = response.Value.NextMarker; @@ -627,7 +626,7 @@ namespace Azure { namespace Storage { namespace Blobs { Core::Http::HttpRange remainingRange; remainingRange.Offset = remainingOffset; remainingRange.Length = remainingSize; - dataLocalityState = CreateDataLocalityLayoutState(*this, remainingRange, eTag, context); + dataLocalityState = CreateDataLocalityLayoutState(remainingRange, eTag, context); dataLocalityState->WaitForInitialLayout(); if (!dataLocalityState->HasLayout()) { @@ -775,7 +774,7 @@ namespace Azure { namespace Storage { namespace Blobs { Core::Http::HttpRange remainingRange; remainingRange.Offset = remainingOffset; remainingRange.Length = remainingSize; - dataLocalityState = CreateDataLocalityLayoutState(*this, remainingRange, eTag, context); + dataLocalityState = CreateDataLocalityLayoutState(remainingRange, eTag, context); dataLocalityState->WaitForInitialLayout(); if (!dataLocalityState->HasLayout()) { diff --git a/sdk/storage/azure-storage-blobs/src/blob_responses.cpp b/sdk/storage/azure-storage-blobs/src/blob_responses.cpp index 077ce0c135..f28508c6ad 100644 --- a/sdk/storage/azure-storage-blobs/src/blob_responses.cpp +++ b/sdk/storage/azure-storage-blobs/src/blob_responses.cpp @@ -7,7 +7,6 @@ #include "azure/storage/blobs/blob_container_client.hpp" #include "azure/storage/blobs/blob_service_client.hpp" #include "azure/storage/blobs/page_blob_client.hpp" -#include "private/data_locality.hpp" namespace Azure { namespace Storage { namespace Blobs { @@ -15,7 +14,7 @@ namespace Azure { namespace Storage { namespace Blobs { { m_operationOptions.ContinuationToken = NextPageToken; m_operationOptions.AccessConditions.IfMatch = Layout.Properties.ETag; - *this = BlobLayoutClient::GetLayout(*m_blobClient, m_operationOptions, context); + *this = m_blobClient->GetLayout(m_operationOptions, context); } std::unique_ptr StartBlobCopyOperation::PollInternal( diff --git a/sdk/storage/azure-storage-blobs/src/private/data_locality.cpp b/sdk/storage/azure-storage-blobs/src/private/data_locality.cpp index dff056d632..fccd0a4611 100644 --- a/sdk/storage/azure-storage-blobs/src/private/data_locality.cpp +++ b/sdk/storage/azure-storage-blobs/src/private/data_locality.cpp @@ -28,7 +28,7 @@ namespace Azure { namespace Storage { namespace Blobs { namespace _detail { layout.Ranges.begin(), layout.Ranges.end(), offset, - [](const BlobLayoutRange& value, int64_t target) { + [](const Models::_detail::BlobLayoutRange& value, int64_t target) { return value.Range.Offset <= target && value.Range.Length.Value() <= target - value.Range.Offset; }); diff --git a/sdk/storage/azure-storage-blobs/src/private/data_locality.hpp b/sdk/storage/azure-storage-blobs/src/private/data_locality.hpp index 0695bbd80c..e58937f956 100644 --- a/sdk/storage/azure-storage-blobs/src/private/data_locality.hpp +++ b/sdk/storage/azure-storage-blobs/src/private/data_locality.hpp @@ -19,50 +19,9 @@ namespace Azure { namespace Storage { namespace Blobs { namespace _detail { constexpr std::chrono::minutes DataLocalityLayoutValidity{5}; constexpr std::chrono::minutes DataLocalityLayoutExtendedExpiry{5}; - struct BlobLayoutRange final - { - Azure::Core::Http::HttpRange Range; - std::string Endpoint; - }; - - struct BlobLayoutInfo final - { - std::vector Ranges; - Models::BlobProperties Properties; - }; - - struct GetBlobLayoutOptions final - { - Azure::Nullable Range; - Azure::Nullable ContinuationToken; - BlobAccessConditions AccessConditions; - }; - - class BlobLayoutPagedResponse final : public Azure::Core::PagedResponse { - public: - BlobLayoutInfo Layout; - - private: - void OnNextPage(const Azure::Core::Context& context); - - std::shared_ptr m_blobClient; - GetBlobLayoutOptions m_operationOptions; - - friend class BlobLayoutClient; - friend class Azure::Core::PagedResponse; - }; - - class BlobLayoutClient final { - public: - static BlobLayoutPagedResponse GetLayout( - const Blobs::BlobClient& blobClient, - const GetBlobLayoutOptions& options = GetBlobLayoutOptions(), - const Azure::Core::Context& context = Azure::Core::Context()); - }; - struct DataLocalityLayout final { - std::vector Ranges; + std::vector Ranges; int64_t BlobSize = 0; Azure::ETag ETag; }; diff --git a/sdk/storage/azure-storage-blobs/test/ut/data_locality_test.cpp b/sdk/storage/azure-storage-blobs/test/ut/data_locality_test.cpp index 19edf315ca..eeff8db104 100644 --- a/sdk/storage/azure-storage-blobs/test/ut/data_locality_test.cpp +++ b/sdk/storage/azure-storage-blobs/test/ut/data_locality_test.cpp @@ -13,7 +13,6 @@ #include #include #include -#include #include #include @@ -225,12 +224,12 @@ namespace Azure { namespace Storage { namespace Test { return options; } - Blobs::_detail::BlobLayoutRange MakeLayoutRange( + Blobs::Models::_detail::BlobLayoutRange MakeLayoutRange( int64_t offset, int64_t length, std::string endpoint) { - Blobs::_detail::BlobLayoutRange range; + Blobs::Models::_detail::BlobLayoutRange range; range.Range.Offset = offset; range.Range.Length = length; range.Endpoint = std::move(endpoint); @@ -245,11 +244,9 @@ namespace Azure { namespace Storage { namespace Test { } } // namespace - class DataLocalityRecordedTest : public StorageTest {}; - TEST(DataLocalityTest, SelectsIdealEndpoint) { - using Range = Blobs::_detail::BlobLayoutRange; + using Range = Blobs::Models::_detail::BlobLayoutRange; struct TestCase final { const char* Name; @@ -331,127 +328,6 @@ namespace Azure { namespace Storage { namespace Test { EXPECT_EQ(refreshCount, 1); } - TEST(DataLocalityTest, GetLayoutNoContentReturnsBlobProperties) - { - auto state = std::make_shared(); - state->LayoutStatus = Core::Http::HttpStatusCode::NoContent; - auto data = std::make_shared(1024 * 1024, 'n'); - auto client = CreateLocalityClient(state, data); - - auto page = Blobs::_detail::BlobLayoutClient::GetLayout(client); - - EXPECT_TRUE(page.Layout.Ranges.empty()); - EXPECT_EQ(page.Layout.Properties.ETag.ToString(), "\"locality-etag\""); - EXPECT_EQ(page.Layout.Properties.BlobSize, static_cast(data->size())); - EXPECT_EQ(page.Layout.Properties.Metadata.at("layout"), "metadata"); - EXPECT_EQ(page.RawResponse->GetStatusCode(), Core::Http::HttpStatusCode::NoContent); - } - - TEST_F(DataLocalityRecordedTest, GetLayoutMultipleEndpoints_PLAYBACKONLY_) - { - auto clientOptions = InitStorageClientOptions(); - auto blobClient = Blobs::BlobClient::CreateFromConnectionString( - StandardStorageConnectionString(), - "data-locality-test", - "data-locality-multi-endpoint.bin", - clientOptions); - - std::vector ranges; - std::set endpoints; - auto page = Blobs::_detail::BlobLayoutClient::GetLayout(blobClient); - EXPECT_EQ(page.Layout.Properties.BlobSize, 64 * 1024 * 1024); - EXPECT_EQ(page.Layout.Properties.BlobType, Blobs::Models::BlobType::BlockBlob); - EXPECT_EQ(page.Layout.Properties.HttpHeaders.ContentType, "application/octet-stream"); - EXPECT_TRUE(page.Layout.Properties.IsServerEncrypted); - for (; page.HasPage(); page.MoveToNextPage()) - { - for (const auto& range : page.Layout.Ranges) - { - EXPECT_GE(range.Range.Offset, 0); - ASSERT_TRUE(range.Range.Length.HasValue()); - EXPECT_GT(range.Range.Length.Value(), 0); - EXPECT_FALSE(range.Endpoint.empty()); - ranges.push_back(range); - endpoints.insert(range.Endpoint); - } - } - - if (m_testContext.IsLiveMode() && endpoints.size() < 2) - { - GTEST_SKIP() << "The configured live account does not return a multi-endpoint layout."; - } - ASSERT_GT(ranges.size(), 1U); - EXPECT_GE(endpoints.size(), 2U); - - const auto differentEndpoint - = std::adjacent_find(ranges.begin(), ranges.end(), [](auto const& left, auto const& right) { - return left.Endpoint != right.Endpoint; - }); - ASSERT_NE(differentEndpoint, ranges.end()); - const auto nextRange = differentEndpoint + 1; - ASSERT_GE(nextRange->Range.Offset, 2); - - constexpr int64_t downloadLength = 5; - const int64_t downloadOffset = nextRange->Range.Offset - 2; - Blobs::DownloadBlobOptions baselineOptions; - baselineOptions.Range = Core::Http::HttpRange(); - baselineOptions.Range.Value().Offset = downloadOffset; - baselineOptions.Range.Value().Length = downloadLength; - auto downloadResponse = blobClient.Download(baselineOptions); - auto content = downloadResponse.Value.BodyStream->ReadToEnd(); - ASSERT_EQ(content.size(), static_cast(downloadLength)); - ASSERT_TRUE(downloadResponse.Value.Details.DownloadHint.HasValue()); - EXPECT_EQ( - downloadResponse.Value.Details.DownloadHint.Value(), Blobs::Models::DownloadHint::Layout); - } - - TEST_F(DataLocalityRecordedTest, GetLayoutRangeReturnsFullBlobSize_PLAYBACKONLY_) - { - auto clientOptions = InitStorageClientOptions(); - auto client = Blobs::BlobClient::CreateFromConnectionString( - StandardStorageConnectionString(), - "data-locality-test", - "data-locality-multi-endpoint.bin", - clientOptions); - constexpr int64_t blobSize = 64 * 1024 * 1024; - for (bool bounded : {true, false}) - { - SCOPED_TRACE(bounded); - Blobs::_detail::GetBlobLayoutOptions options; - options.Range = Core::Http::HttpRange(); - options.Range.Value().Offset = bounded ? 8 * 1024 * 1024 - 2 : blobSize - 16; - if (bounded) - { - options.Range.Value().Length = 5; - } - const auto offset = options.Range.Value().Offset; - bool coversOffset = false; - size_t rangeCount = 0; - for (auto page = Blobs::_detail::BlobLayoutClient::GetLayout(client, options); page.HasPage(); - page.MoveToNextPage()) - { - EXPECT_EQ(page.Layout.Properties.BlobSize, blobSize); - EXPECT_GT(page.Layout.Properties.BlobSize, options.Range.Value().Length.ValueOr(16)); - EXPECT_TRUE(page.Layout.Properties.ETag.HasValue()); - for (const auto& range : page.Layout.Ranges) - { - ASSERT_TRUE(range.Range.Length.HasValue()); - ASSERT_GT(range.Range.Length.Value(), 0); - ASSERT_GE(range.Range.Offset, 0); - ASSERT_LE(range.Range.Offset, blobSize); - ASSERT_LE(range.Range.Length.Value(), blobSize - range.Range.Offset); - EXPECT_FALSE(range.Endpoint.empty()); - coversOffset = coversOffset - || (range.Range.Offset <= offset - && offset - range.Range.Offset < range.Range.Length.Value()); - ++rangeCount; - } - } - EXPECT_GT(rangeCount, 0U); - EXPECT_TRUE(coversOffset); - } - } - TEST(DataLocalityTest, RoutesDownloadChunks) { auto state = std::make_shared(); diff --git a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_file_client.hpp b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_file_client.hpp index e04d891bc0..5aa8de62c9 100644 --- a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_file_client.hpp +++ b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_file_client.hpp @@ -282,6 +282,10 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { const Azure::Core::Context& context = Azure::Core::Context()) const; private: + _detail::FileLayoutPagedResponse GetLayout( + const _detail::GetFileLayoutOptions& options = _detail::GetFileLayoutOptions(), + const Azure::Core::Context& context = Azure::Core::Context()) const; + explicit DataLakeFileClient( Azure::Core::Url fileUrl, Blobs::BlobClient blobClient, @@ -297,5 +301,6 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { friend class DataLakeFileSystemClient; friend class DataLakeDirectoryClient; + friend class _detail::FileLayoutPagedResponse; }; }}}} // namespace Azure::Storage::Files::DataLake diff --git a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_options.hpp b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_options.hpp index ee231c28cb..269f60fec2 100644 --- a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_options.hpp +++ b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_options.hpp @@ -890,6 +890,29 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { Nullable IncludeUserPrincipalName; }; + namespace _detail { + /** + * @brief Optional parameters for DataLakeFileClient::GetLayout. + */ + struct GetFileLayoutOptions final + { + /** + * @brief The range of bytes for which to retrieve the layout. + */ + Azure::Nullable Range; + + /** + * @brief The continuation token for retrieving the next page. + */ + Azure::Nullable ContinuationToken; + + /** + * @brief Access conditions for the operation. + */ + PathAccessConditions AccessConditions; + }; + } // namespace _detail + /** * @brief Optional parameters for #Azure::Storage::Files::DataLake::DataLakeFileClient::Download. * @remark Some optional parameter is mandatory in certain combination. diff --git a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_responses.hpp b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_responses.hpp index 7cdd9f8dcd..ba29da9f75 100644 --- a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_responses.hpp +++ b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_responses.hpp @@ -463,6 +463,28 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { Azure::Nullable Permissions; }; + namespace _detail { + /** + * @brief A file range and its resolved locality endpoint. + */ + using FileLayoutRange = Blobs::Models::_detail::BlobLayoutRange; + + /** + * @brief A page of file layout information and the file properties returned with it. + */ + struct FileLayoutInfo final + { + /** + * @brief The ranges and locality endpoints in this page. + */ + std::vector Ranges; + /** + * @brief The file properties returned with this page. + */ + PathProperties Properties; + }; + } // namespace _detail + /** * @brief The access control list of a path. */ @@ -803,6 +825,29 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { } // namespace Models + namespace _detail { + /** + * @brief Response type for DataLakeFileClient::GetLayout. + */ + class FileLayoutPagedResponse final + : public Azure::Core::PagedResponse { + public: + /** + * @brief The layout and file properties in this page. + */ + Models::_detail::FileLayoutInfo Layout; + + private: + void OnNextPage(const Azure::Core::Context& context); + + std::shared_ptr m_fileClient; + GetFileLayoutOptions m_operationOptions; + + friend class DataLake::DataLakeFileClient; + friend class Azure::Core::PagedResponse; + }; + } // namespace _detail + /** * @brief Response type for * #Azure::Storage::Files::DataLake::DataLakeServiceClient::ListFileSystems. diff --git a/sdk/storage/azure-storage-files-datalake/src/datalake_file_client.cpp b/sdk/storage/azure-storage-files-datalake/src/datalake_file_client.cpp index 1a7a63b8be..b52db9ab47 100644 --- a/sdk/storage/azure-storage-files-datalake/src/datalake_file_client.cpp +++ b/sdk/storage/azure-storage-files-datalake/src/datalake_file_client.cpp @@ -285,6 +285,32 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { std::move(ret), std::move(response.RawResponse)); } + _detail::FileLayoutPagedResponse DataLakeFileClient::GetLayout( + const _detail::GetFileLayoutOptions& options, + const Azure::Core::Context& context) const + { + Blobs::_detail::GetBlobLayoutOptions blobOptions; + blobOptions.Range = options.Range; + blobOptions.ContinuationToken = options.ContinuationToken; + blobOptions.AccessConditions.IfMatch = options.AccessConditions.IfMatch; + blobOptions.AccessConditions.IfNoneMatch = options.AccessConditions.IfNoneMatch; + blobOptions.AccessConditions.IfModifiedSince = options.AccessConditions.IfModifiedSince; + blobOptions.AccessConditions.IfUnmodifiedSince = options.AccessConditions.IfUnmodifiedSince; + blobOptions.AccessConditions.LeaseId = options.AccessConditions.LeaseId; + auto response = m_blobClient.GetLayout(blobOptions, context); + + _detail::FileLayoutPagedResponse result; + result.Layout.Ranges = std::move(response.Layout.Ranges); + result.Layout.Properties = _detail::PathPropertiesFromBlobProperties( + std::move(response.Layout.Properties), *response.RawResponse); + result.CurrentPageToken = std::move(response.CurrentPageToken); + result.NextPageToken = std::move(response.NextPageToken); + result.RawResponse = std::move(response.RawResponse); + result.m_fileClient = std::make_shared(*this); + result.m_operationOptions = options; + return result; + } + Azure::Response DataLakeFileClient::UploadFrom( const std::string& fileName, const UploadFileFromOptions& options, diff --git a/sdk/storage/azure-storage-files-datalake/src/datalake_responses.cpp b/sdk/storage/azure-storage-files-datalake/src/datalake_responses.cpp index 4ad9003d78..bde58f9822 100644 --- a/sdk/storage/azure-storage-files-datalake/src/datalake_responses.cpp +++ b/sdk/storage/azure-storage-files-datalake/src/datalake_responses.cpp @@ -10,6 +10,14 @@ #include "private/datalake_utilities.hpp" namespace Azure { namespace Storage { namespace Files { namespace DataLake { + void _detail::FileLayoutPagedResponse::OnNextPage(const Azure::Core::Context& context) + { + auto options = m_operationOptions; + options.ContinuationToken = NextPageToken; + options.AccessConditions.IfMatch = Layout.Properties.ETag; + *this = m_fileClient->GetLayout(options, context); + } + namespace Models { Acl Acl::FromString(const std::string& aclString) From 4285ee7057e4945b436938bed3951a8d3b20c44d Mon Sep 17 00:00:00 2001 From: Jinming Hu Date: Fri, 30 Oct 2026 13:22:37 +1100 Subject: [PATCH 4/6] Use a friend function for data locality layout state creation Keep CreateDataLocalityLayoutState as a free function in _detail and grant it access to private GetLayout without introducing another client member or accessor class. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../inc/azure/storage/blobs/blob_client.hpp | 19 ++++++++++++++----- .../azure-storage-blobs/src/blob_client.cpp | 13 ++++++++----- 2 files changed, 22 insertions(+), 10 deletions(-) diff --git a/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_client.hpp b/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_client.hpp index e9d0c611ea..e9a1363cfa 100644 --- a/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_client.hpp +++ b/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_client.hpp @@ -23,8 +23,17 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { namespace Azure { namespace Storage { namespace Blobs { + class BlobClient; + namespace _detail { class DataLocalityLayoutState; + + std::unique_ptr CreateDataLocalityLayoutState( + Blobs::BlobClient blobClient, + Azure::Nullable range, + Azure::ETag eTag, + Azure::Core::Context context); + AZ_STORAGE_BLOBS_DLLEXPORT extern const Azure::Core::Context::Key DataLakeInteroperabilityExtraOptionsKey; } // namespace _detail @@ -439,11 +448,6 @@ namespace Azure { namespace Storage { namespace Blobs { const _detail::GetBlobLayoutOptions& options = _detail::GetBlobLayoutOptions(), const Azure::Core::Context& context = Azure::Core::Context()) const; - std::unique_ptr<_detail::DataLocalityLayoutState> CreateDataLocalityLayoutState( - Azure::Nullable range, - Azure::ETag eTag, - Azure::Core::Context context) const; - explicit BlobClient( Azure::Core::Url blobUrl, std::shared_ptr pipeline, @@ -461,5 +465,10 @@ namespace Azure { namespace Storage { namespace Blobs { friend class BlobServiceBatch; friend class BlobContainerBatch; friend class _detail::BlobLayoutPagedResponse; + friend std::unique_ptr<_detail::DataLocalityLayoutState> _detail::CreateDataLocalityLayoutState( + Blobs::BlobClient blobClient, + Azure::Nullable range, + Azure::ETag eTag, + Azure::Core::Context context); }; }}} // namespace Azure::Storage::Blobs diff --git a/sdk/storage/azure-storage-blobs/src/blob_client.cpp b/sdk/storage/azure-storage-blobs/src/blob_client.cpp index 4ddb334562..6622d68019 100644 --- a/sdk/storage/azure-storage-blobs/src/blob_client.cpp +++ b/sdk/storage/azure-storage-blobs/src/blob_client.cpp @@ -102,15 +102,16 @@ namespace Azure { namespace Storage { namespace Blobs { } // namespace - std::unique_ptr<_detail::DataLocalityLayoutState> BlobClient::CreateDataLocalityLayoutState( + std::unique_ptr<_detail::DataLocalityLayoutState> _detail::CreateDataLocalityLayoutState( + Blobs::BlobClient blobClient, Azure::Nullable range, Azure::ETag eTag, - Azure::Core::Context context) const + Azure::Core::Context context) { auto refresher = [range = std::move(range), eTag = std::move(eTag), context = std::move(context), - blobClient = *this]() { + blobClient = std::move(blobClient)]() { try { _detail::GetBlobLayoutOptions options; @@ -626,7 +627,8 @@ namespace Azure { namespace Storage { namespace Blobs { Core::Http::HttpRange remainingRange; remainingRange.Offset = remainingOffset; remainingRange.Length = remainingSize; - dataLocalityState = CreateDataLocalityLayoutState(remainingRange, eTag, context); + dataLocalityState + = _detail::CreateDataLocalityLayoutState(*this, remainingRange, eTag, context); dataLocalityState->WaitForInitialLayout(); if (!dataLocalityState->HasLayout()) { @@ -774,7 +776,8 @@ namespace Azure { namespace Storage { namespace Blobs { Core::Http::HttpRange remainingRange; remainingRange.Offset = remainingOffset; remainingRange.Length = remainingSize; - dataLocalityState = CreateDataLocalityLayoutState(remainingRange, eTag, context); + dataLocalityState + = _detail::CreateDataLocalityLayoutState(*this, remainingRange, eTag, context); dataLocalityState->WaitForInitialLayout(); if (!dataLocalityState->HasLayout()) { From 7780b24d9341aaff83dc7178d64c2334dda7c4f9 Mon Sep 17 00:00:00 2001 From: Jinming Hu Date: Fri, 30 Oct 2026 13:26:50 +1100 Subject: [PATCH 5/6] Create layout state through the existing private state class Move the factory signature into the private data locality header and replace the long friend-function declaration on BlobClient with friendship for DataLocalityLayoutState. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../inc/azure/storage/blobs/blob_client.hpp | 15 +-------------- .../azure-storage-blobs/src/blob_client.cpp | 6 +++--- .../src/private/data_locality.hpp | 7 +++++++ 3 files changed, 11 insertions(+), 17 deletions(-) diff --git a/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_client.hpp b/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_client.hpp index e9a1363cfa..cff2526293 100644 --- a/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_client.hpp +++ b/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_client.hpp @@ -23,17 +23,8 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { namespace Azure { namespace Storage { namespace Blobs { - class BlobClient; - namespace _detail { class DataLocalityLayoutState; - - std::unique_ptr CreateDataLocalityLayoutState( - Blobs::BlobClient blobClient, - Azure::Nullable range, - Azure::ETag eTag, - Azure::Core::Context context); - AZ_STORAGE_BLOBS_DLLEXPORT extern const Azure::Core::Context::Key DataLakeInteroperabilityExtraOptionsKey; } // namespace _detail @@ -465,10 +456,6 @@ namespace Azure { namespace Storage { namespace Blobs { friend class BlobServiceBatch; friend class BlobContainerBatch; friend class _detail::BlobLayoutPagedResponse; - friend std::unique_ptr<_detail::DataLocalityLayoutState> _detail::CreateDataLocalityLayoutState( - Blobs::BlobClient blobClient, - Azure::Nullable range, - Azure::ETag eTag, - Azure::Core::Context context); + friend class _detail::DataLocalityLayoutState; }; }}} // namespace Azure::Storage::Blobs diff --git a/sdk/storage/azure-storage-blobs/src/blob_client.cpp b/sdk/storage/azure-storage-blobs/src/blob_client.cpp index 6622d68019..d744302a3a 100644 --- a/sdk/storage/azure-storage-blobs/src/blob_client.cpp +++ b/sdk/storage/azure-storage-blobs/src/blob_client.cpp @@ -102,7 +102,7 @@ namespace Azure { namespace Storage { namespace Blobs { } // namespace - std::unique_ptr<_detail::DataLocalityLayoutState> _detail::CreateDataLocalityLayoutState( + std::unique_ptr<_detail::DataLocalityLayoutState> _detail::DataLocalityLayoutState::Create( Blobs::BlobClient blobClient, Azure::Nullable range, Azure::ETag eTag, @@ -628,7 +628,7 @@ namespace Azure { namespace Storage { namespace Blobs { remainingRange.Offset = remainingOffset; remainingRange.Length = remainingSize; dataLocalityState - = _detail::CreateDataLocalityLayoutState(*this, remainingRange, eTag, context); + = _detail::DataLocalityLayoutState::Create(*this, remainingRange, eTag, context); dataLocalityState->WaitForInitialLayout(); if (!dataLocalityState->HasLayout()) { @@ -777,7 +777,7 @@ namespace Azure { namespace Storage { namespace Blobs { remainingRange.Offset = remainingOffset; remainingRange.Length = remainingSize; dataLocalityState - = _detail::CreateDataLocalityLayoutState(*this, remainingRange, eTag, context); + = _detail::DataLocalityLayoutState::Create(*this, remainingRange, eTag, context); dataLocalityState->WaitForInitialLayout(); if (!dataLocalityState->HasLayout()) { diff --git a/sdk/storage/azure-storage-blobs/src/private/data_locality.hpp b/sdk/storage/azure-storage-blobs/src/private/data_locality.hpp index e58937f956..f113cd94b9 100644 --- a/sdk/storage/azure-storage-blobs/src/private/data_locality.hpp +++ b/sdk/storage/azure-storage-blobs/src/private/data_locality.hpp @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -33,6 +34,12 @@ namespace Azure { namespace Storage { namespace Blobs { namespace _detail { class DataLocalityLayoutState final { public: + static std::unique_ptr Create( + Blobs::BlobClient blobClient, + Azure::Nullable range, + Azure::ETag eTag, + Azure::Core::Context context); + explicit DataLocalityLayoutState(std::function refresh); ~DataLocalityLayoutState() { WaitForRefreshTask(); } From 68083a2c52257a32b525c38075f1cdb74fb1e4ca Mon Sep 17 00:00:00 2001 From: Jinming Hu Date: Fri, 30 Oct 2026 13:46:09 +1100 Subject: [PATCH 6/6] Remove the redundant Data Lake layout forwarding API Remove DataLakeFileClient GetLayout and its layout-specific options, models and paged response. Managed downloads continue using the Blob implementation directly. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../files/datalake/datalake_file_client.hpp | 5 --- .../files/datalake/datalake_options.hpp | 23 ---------- .../files/datalake/datalake_responses.hpp | 45 ------------------- .../src/datalake_file_client.cpp | 26 ----------- .../src/datalake_responses.cpp | 9 ---- 5 files changed, 108 deletions(-) diff --git a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_file_client.hpp b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_file_client.hpp index 5aa8de62c9..e04d891bc0 100644 --- a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_file_client.hpp +++ b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_file_client.hpp @@ -282,10 +282,6 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { const Azure::Core::Context& context = Azure::Core::Context()) const; private: - _detail::FileLayoutPagedResponse GetLayout( - const _detail::GetFileLayoutOptions& options = _detail::GetFileLayoutOptions(), - const Azure::Core::Context& context = Azure::Core::Context()) const; - explicit DataLakeFileClient( Azure::Core::Url fileUrl, Blobs::BlobClient blobClient, @@ -301,6 +297,5 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { friend class DataLakeFileSystemClient; friend class DataLakeDirectoryClient; - friend class _detail::FileLayoutPagedResponse; }; }}}} // namespace Azure::Storage::Files::DataLake diff --git a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_options.hpp b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_options.hpp index 269f60fec2..ee231c28cb 100644 --- a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_options.hpp +++ b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_options.hpp @@ -890,29 +890,6 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { Nullable IncludeUserPrincipalName; }; - namespace _detail { - /** - * @brief Optional parameters for DataLakeFileClient::GetLayout. - */ - struct GetFileLayoutOptions final - { - /** - * @brief The range of bytes for which to retrieve the layout. - */ - Azure::Nullable Range; - - /** - * @brief The continuation token for retrieving the next page. - */ - Azure::Nullable ContinuationToken; - - /** - * @brief Access conditions for the operation. - */ - PathAccessConditions AccessConditions; - }; - } // namespace _detail - /** * @brief Optional parameters for #Azure::Storage::Files::DataLake::DataLakeFileClient::Download. * @remark Some optional parameter is mandatory in certain combination. diff --git a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_responses.hpp b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_responses.hpp index ba29da9f75..7cdd9f8dcd 100644 --- a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_responses.hpp +++ b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_responses.hpp @@ -463,28 +463,6 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { Azure::Nullable Permissions; }; - namespace _detail { - /** - * @brief A file range and its resolved locality endpoint. - */ - using FileLayoutRange = Blobs::Models::_detail::BlobLayoutRange; - - /** - * @brief A page of file layout information and the file properties returned with it. - */ - struct FileLayoutInfo final - { - /** - * @brief The ranges and locality endpoints in this page. - */ - std::vector Ranges; - /** - * @brief The file properties returned with this page. - */ - PathProperties Properties; - }; - } // namespace _detail - /** * @brief The access control list of a path. */ @@ -825,29 +803,6 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { } // namespace Models - namespace _detail { - /** - * @brief Response type for DataLakeFileClient::GetLayout. - */ - class FileLayoutPagedResponse final - : public Azure::Core::PagedResponse { - public: - /** - * @brief The layout and file properties in this page. - */ - Models::_detail::FileLayoutInfo Layout; - - private: - void OnNextPage(const Azure::Core::Context& context); - - std::shared_ptr m_fileClient; - GetFileLayoutOptions m_operationOptions; - - friend class DataLake::DataLakeFileClient; - friend class Azure::Core::PagedResponse; - }; - } // namespace _detail - /** * @brief Response type for * #Azure::Storage::Files::DataLake::DataLakeServiceClient::ListFileSystems. diff --git a/sdk/storage/azure-storage-files-datalake/src/datalake_file_client.cpp b/sdk/storage/azure-storage-files-datalake/src/datalake_file_client.cpp index b52db9ab47..1a7a63b8be 100644 --- a/sdk/storage/azure-storage-files-datalake/src/datalake_file_client.cpp +++ b/sdk/storage/azure-storage-files-datalake/src/datalake_file_client.cpp @@ -285,32 +285,6 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { std::move(ret), std::move(response.RawResponse)); } - _detail::FileLayoutPagedResponse DataLakeFileClient::GetLayout( - const _detail::GetFileLayoutOptions& options, - const Azure::Core::Context& context) const - { - Blobs::_detail::GetBlobLayoutOptions blobOptions; - blobOptions.Range = options.Range; - blobOptions.ContinuationToken = options.ContinuationToken; - blobOptions.AccessConditions.IfMatch = options.AccessConditions.IfMatch; - blobOptions.AccessConditions.IfNoneMatch = options.AccessConditions.IfNoneMatch; - blobOptions.AccessConditions.IfModifiedSince = options.AccessConditions.IfModifiedSince; - blobOptions.AccessConditions.IfUnmodifiedSince = options.AccessConditions.IfUnmodifiedSince; - blobOptions.AccessConditions.LeaseId = options.AccessConditions.LeaseId; - auto response = m_blobClient.GetLayout(blobOptions, context); - - _detail::FileLayoutPagedResponse result; - result.Layout.Ranges = std::move(response.Layout.Ranges); - result.Layout.Properties = _detail::PathPropertiesFromBlobProperties( - std::move(response.Layout.Properties), *response.RawResponse); - result.CurrentPageToken = std::move(response.CurrentPageToken); - result.NextPageToken = std::move(response.NextPageToken); - result.RawResponse = std::move(response.RawResponse); - result.m_fileClient = std::make_shared(*this); - result.m_operationOptions = options; - return result; - } - Azure::Response DataLakeFileClient::UploadFrom( const std::string& fileName, const UploadFileFromOptions& options, diff --git a/sdk/storage/azure-storage-files-datalake/src/datalake_responses.cpp b/sdk/storage/azure-storage-files-datalake/src/datalake_responses.cpp index bde58f9822..c6fa0aa1c5 100644 --- a/sdk/storage/azure-storage-files-datalake/src/datalake_responses.cpp +++ b/sdk/storage/azure-storage-files-datalake/src/datalake_responses.cpp @@ -4,20 +4,11 @@ #include "azure/storage/files/datalake/datalake_responses.hpp" #include "azure/storage/files/datalake/datalake_directory_client.hpp" -#include "azure/storage/files/datalake/datalake_file_client.hpp" #include "azure/storage/files/datalake/datalake_path_client.hpp" #include "azure/storage/files/datalake/datalake_service_client.hpp" #include "private/datalake_utilities.hpp" namespace Azure { namespace Storage { namespace Files { namespace DataLake { - void _detail::FileLayoutPagedResponse::OnNextPage(const Azure::Core::Context& context) - { - auto options = m_operationOptions; - options.ContinuationToken = NextPageToken; - options.AccessConditions.IfMatch = Layout.Properties.ETag; - *this = m_fileClient->GetLayout(options, context); - } - namespace Models { Acl Acl::FromString(const std::string& aclString)