diff --git a/sdk/storage/azure-storage-blobs/CHANGELOG.md b/sdk/storage/azure-storage-blobs/CHANGELOG.md index d9059e77f6..bc163fd414 100644 --- a/sdk/storage/azure-storage-blobs/CHANGELOG.md +++ b/sdk/storage/azure-storage-blobs/CHANGELOG.md @@ -5,11 +5,7 @@ ### Features Added - Added support for persisted session authentication for eligible blob download requests. -- Added `BlobClient::GetLayout` and related options and response types for retrieving locality - endpoints, byte ranges, and blob properties. Ranges are exposed directly through - `Models::BlobLayoutInfo::Ranges` as a vector of `Models::BlobLayoutRange`. -- Added `DownloadBlobOptions::LayoutEndpoint` for one-shot download routing and - `DownloadBlobToOptions::LayoutAwareRouting` (`Auto`, `Disabled`, `Enabled`; default `Disabled`) for +- Added `DownloadBlobToOptions::LayoutAwareRouting` (`Auto`, `Disabled`, `Enabled`; default `Disabled`) for managed downloads. - Added `Models::DownloadHint` and `Models::DownloadBlobDetails::DownloadHint`. diff --git a/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_client.hpp b/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_client.hpp index ec3511e048..cff2526293 100644 --- a/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_client.hpp +++ b/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_client.hpp @@ -24,6 +24,7 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { namespace Azure { namespace Storage { namespace Blobs { namespace _detail { + class DataLocalityLayoutState; AZ_STORAGE_BLOBS_DLLEXPORT extern const Azure::Core::Context::Key DataLakeInteroperabilityExtraOptionsKey; } // namespace _detail @@ -425,17 +426,6 @@ namespace Azure { namespace Storage { namespace Blobs { const GetAccountInfoOptions& options = GetAccountInfoOptions(), const Azure::Core::Context& context = Azure::Core::Context()) const; - /** - * @brief Gets the data locality layout for this blob. - * - * @param options Optional parameters to execute this function. - * @param context Context for cancelling long running operations. - * @return A paged response containing the blob's data locality layout. - */ - BlobLayoutPagedResponse GetLayout( - const GetBlobLayoutOptions& options = GetBlobLayoutOptions(), - const Azure::Core::Context& context = Azure::Core::Context()) const; - protected: /** @brief Blob Url */ Azure::Core::Url m_blobUrl; @@ -445,6 +435,10 @@ namespace Azure { namespace Storage { namespace Blobs { _detail::BlobClientConfiguration m_clientConfiguration; private: + _detail::BlobLayoutPagedResponse GetLayout( + const _detail::GetBlobLayoutOptions& options = _detail::GetBlobLayoutOptions(), + const Azure::Core::Context& context = Azure::Core::Context()) const; + explicit BlobClient( Azure::Core::Url blobUrl, std::shared_ptr pipeline, @@ -461,5 +455,7 @@ namespace Azure { namespace Storage { namespace Blobs { friend class BlobLeaseClient; friend class BlobServiceBatch; friend class BlobContainerBatch; + friend class _detail::BlobLayoutPagedResponse; + friend class _detail::DataLocalityLayoutState; }; }}} // namespace Azure::Storage::Blobs diff --git a/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_options.hpp b/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_options.hpp index 923b6fd4ab..bc6a9825a2 100644 --- a/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_options.hpp +++ b/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_options.hpp @@ -932,14 +932,6 @@ namespace Azure { namespace Storage { namespace Blobs { * @brief Optional. Configures whether to do content validation for blob downloads. */ Azure::Nullable ValidationOptions; - - /** - * @brief Optional. The layout endpoint to use for this download. - * - * When set, the request is sent to this endpoint while preserving the client endpoint in the - * Host header. - */ - std::string LayoutEndpoint; }; /** @@ -1957,28 +1949,28 @@ namespace Azure { namespace Storage { namespace Blobs { { }; - /** - * @brief Optional parameters for #Azure::Storage::Blobs::BlobClient::GetLayout. - */ - struct GetBlobLayoutOptions final - { + namespace _detail { /** - * @brief The range of bytes for which to retrieve the layout. + * @brief Optional parameters for BlobClient::GetLayout. */ - Azure::Nullable Range; + struct GetBlobLayoutOptions final + { + /** + * @brief The range of bytes for which to retrieve the layout. + */ + Azure::Nullable Range; - /** - * @brief The continuation token for retrieving the next page. - */ - Azure::Nullable ContinuationToken; + /** + * @brief The continuation token for retrieving the next page. + */ + Azure::Nullable ContinuationToken; - /** - * @brief Access conditions for the operation. - */ - BlobAccessConditions AccessConditions; - }; + /** + * @brief Access conditions for the operation. + */ + BlobAccessConditions AccessConditions; + }; - namespace _detail { inline std::string TagsToString(const std::map& tags) { return std::accumulate( diff --git a/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_responses.hpp b/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_responses.hpp index 4521c85d1e..9bbc9240a5 100644 --- a/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_responses.hpp +++ b/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_responses.hpp @@ -29,37 +29,39 @@ namespace Azure { namespace Storage { namespace Models { - /** - * @brief Describes a range in a blob's data locality layout. - */ - struct BlobLayoutRange final - { + namespace _detail { /** - * @brief The range of bytes served by the endpoint. + * @brief Describes a range in a blob's data locality layout. */ - Azure::Core::Http::HttpRange Range; + struct BlobLayoutRange final + { + /** + * @brief The range of bytes served by the endpoint. + */ + Azure::Core::Http::HttpRange Range; + + /** + * @brief The host and port of the endpoint serving this range. + */ + std::string Endpoint; + }; /** - * @brief The host and port of the endpoint serving this range. + * @brief A page of blob layout information and the blob properties returned with it. */ - std::string Endpoint; - }; + struct BlobLayoutInfo final + { + /** + * @brief The ranges in this page of the blob's data locality layout. + */ + std::vector Ranges; - /** - * @brief A page of blob layout information and the blob properties returned with it. - */ - struct BlobLayoutInfo final - { - /** - * @brief The ranges in this page of the blob's data locality layout. - */ - std::vector Ranges; - - /** - * @brief The blob properties returned with the layout. - */ - BlobProperties Properties; - }; + /** + * @brief The blob properties returned with the layout. + */ + BlobProperties Properties; + }; + } // namespace _detail /** * @brief Response type for #Azure::Storage::Blobs::BlobClient::DownloadTo. @@ -514,28 +516,28 @@ namespace Azure { namespace Storage { friend class Azure::Core::PagedResponse; }; - /** - * @brief Response type for #Azure::Storage::Blobs::BlobClient::GetLayout. - */ - class BlobLayoutPagedResponse final - : public Azure::Core::PagedResponse { - public: + namespace _detail { /** - * @brief The layout and blob properties in this page. + * @brief Response type for BlobClient::GetLayout. */ - Models::BlobLayoutInfo Layout; + class BlobLayoutPagedResponse final + : public Azure::Core::PagedResponse { + public: + /** + * @brief The layout and blob properties in this page. + */ + Models::_detail::BlobLayoutInfo Layout; - private: - void OnNextPage(const Azure::Core::Context& context); + private: + void OnNextPage(const Azure::Core::Context& context); - std::shared_ptr m_blobClient; - GetBlobLayoutOptions m_operationOptions; + std::shared_ptr m_blobClient; + GetBlobLayoutOptions m_operationOptions; - friend class Blobs::BlobClient; - friend class Azure::Core::PagedResponse; - }; + friend class Blobs::BlobClient; + friend class Azure::Core::PagedResponse; + }; - namespace _detail { void ParseListBlobsResult(Models::_detail::ListBlobsResult& result); } // namespace _detail diff --git a/sdk/storage/azure-storage-blobs/src/blob_client.cpp b/sdk/storage/azure-storage-blobs/src/blob_client.cpp index afcda50efd..d744302a3a 100644 --- a/sdk/storage/azure-storage-blobs/src/blob_client.cpp +++ b/sdk/storage/azure-storage-blobs/src/blob_client.cpp @@ -100,65 +100,67 @@ namespace Azure { namespace Storage { namespace Blobs { } } - std::unique_ptr<_detail::DataLocalityLayoutState> CreateDataLocalityLayoutState( - BlobClient blobClient, - Azure::Nullable range, - Azure::ETag eTag, - Azure::Core::Context context) - { - auto refresher = [range = std::move(range), - eTag = std::move(eTag), - context = std::move(context), - blobClient = std::move(blobClient)]() { - try + } // namespace + + std::unique_ptr<_detail::DataLocalityLayoutState> _detail::DataLocalityLayoutState::Create( + Blobs::BlobClient blobClient, + Azure::Nullable range, + Azure::ETag eTag, + Azure::Core::Context context) + { + auto refresher = [range = std::move(range), + eTag = std::move(eTag), + context = std::move(context), + blobClient = std::move(blobClient)]() { + try + { + _detail::GetBlobLayoutOptions options; + options.Range = range; + options.AccessConditions.IfMatch = eTag; + auto page = blobClient.GetLayout(options, context); + _detail::DataLocalityLayout layout; + for (; page.HasPage(); page.MoveToNextPage(context)) { - GetBlobLayoutOptions options; - options.Range = range; - options.AccessConditions.IfMatch = eTag; - auto page = blobClient.GetLayout(options, context); - _detail::DataLocalityLayout layout; - for (; page.HasPage(); page.MoveToNextPage(context)) + if (!layout.ETag.HasValue()) { - if (!layout.ETag.HasValue()) - { - layout.ETag = page.Layout.Properties.ETag; - } - if (layout.BlobSize == 0) - { - layout.BlobSize = page.Layout.Properties.BlobSize; - } - for (auto& layoutRange : page.Layout.Ranges) + layout.ETag = page.Layout.Properties.ETag; + } + if (layout.BlobSize == 0) + { + layout.BlobSize = page.Layout.Properties.BlobSize; + } + for (auto& layoutRange : page.Layout.Ranges) + { + if (layoutRange.Endpoint.empty() || layoutRange.Range.Offset < 0 + || !layoutRange.Range.Length.HasValue() || layoutRange.Range.Length.Value() <= 0) { - if (layoutRange.Endpoint.empty() || layoutRange.Range.Offset < 0 - || !layoutRange.Range.Length.HasValue() || layoutRange.Range.Length.Value() <= 0) - { - throw Azure::Core::RequestFailedException("Invalid blob layout range."); - } - layout.Ranges.push_back(std::move(layoutRange)); + throw Azure::Core::RequestFailedException("Invalid blob layout range."); } + layout.Ranges.push_back(std::move(layoutRange)); } - std::sort( - layout.Ranges.begin(), - layout.Ranges.end(), - [](const Models::BlobLayoutRange& lhs, const Models::BlobLayoutRange& rhs) { - return lhs.Range.Offset < rhs.Range.Offset; - }); - return layout; } - catch (const StorageException& exception) + std::sort( + layout.Ranges.begin(), + layout.Ranges.end(), + [](const Models::_detail::BlobLayoutRange& lhs, + const Models::_detail::BlobLayoutRange& rhs) { + return lhs.Range.Offset < rhs.Range.Offset; + }); + return layout; + } + catch (const StorageException& exception) + { + if (exception.StatusCode == Core::Http::HttpStatusCode::BadRequest + || static_cast(exception.StatusCode) >= 500) { - if (exception.StatusCode == Core::Http::HttpStatusCode::BadRequest - || static_cast(exception.StatusCode) >= 500) - { - return _detail::DataLocalityLayout(); - } - throw; + return _detail::DataLocalityLayout(); } - }; + throw; + } + }; - return std::make_unique<_detail::DataLocalityLayoutState>(std::move(refresher)); - } - } // namespace + return std::make_unique<_detail::DataLocalityLayoutState>(std::move(refresher)); + } BlobClient BlobClient::CreateFromConnectionString( const std::string& connectionString, @@ -348,9 +350,8 @@ namespace Azure { namespace Storage { namespace Blobs { } std::string dataLocalityEndpoint; - const auto downloadContext = !options.LayoutEndpoint.empty() - ? _internal::WithDataLocalityEndpoint(context, options.LayoutEndpoint) - : context.TryGetValue(_internal::DataLocalityEndpointKey, dataLocalityEndpoint) + const auto downloadContext + = context.TryGetValue(_internal::DataLocalityEndpointKey, dataLocalityEndpoint) ? context : _internal::WithReplicaStatus(context); auto downloadResponse = _detail::BlobClient::Download( @@ -488,8 +489,8 @@ namespace Azure { namespace Storage { namespace Blobs { return downloadResponse; } - BlobLayoutPagedResponse BlobClient::GetLayout( - const GetBlobLayoutOptions& options, + _detail::BlobLayoutPagedResponse BlobClient::GetLayout( + const _detail::GetBlobLayoutOptions& options, const Azure::Core::Context& context) const { _detail::BlobClient::GetBlobLayoutOptions protocolLayerOptions; @@ -521,7 +522,7 @@ namespace Azure { namespace Storage { namespace Blobs { auto response = _detail::BlobClient::GetLayout(*m_pipeline, m_blobUrl, protocolLayerOptions, context); - BlobLayoutPagedResponse pagedResponse; + _detail::BlobLayoutPagedResponse pagedResponse; std::map endpoints; for (const auto& endpoint : response.Value.Endpoints) { @@ -544,7 +545,7 @@ namespace Azure { namespace Storage { namespace Blobs { { throw Azure::Core::RequestFailedException("Invalid blob layout range."); } - Models::BlobLayoutRange layoutRange; + Models::_detail::BlobLayoutRange layoutRange; layoutRange.Range.Offset = range.Start; layoutRange.Range.Length = static_cast(rangeLength); layoutRange.Endpoint = endpoint->second; @@ -626,7 +627,8 @@ namespace Azure { namespace Storage { namespace Blobs { Core::Http::HttpRange remainingRange; remainingRange.Offset = remainingOffset; remainingRange.Length = remainingSize; - dataLocalityState = CreateDataLocalityLayoutState(*this, remainingRange, eTag, context); + dataLocalityState + = _detail::DataLocalityLayoutState::Create(*this, remainingRange, eTag, context); dataLocalityState->WaitForInitialLayout(); if (!dataLocalityState->HasLayout()) { @@ -654,9 +656,10 @@ namespace Azure { namespace Storage { namespace Blobs { chunkOptions.Range.Value().Length = length; chunkOptions.AccessConditions.IfMatch = eTag; chunkOptions.ValidationOptions = options.ValidationOptions; - const auto chunkContext = dataLocalityState ? _internal::WithDataLocalityEndpoint( - context, dataLocalityState->GetEndpoint(offset, length)) - : context; + const auto chunkContext = dataLocalityState + ? _internal::WithDataLocalityEndpoint( + context, dataLocalityState->GetEndpoint(offset, length)) + : context; auto chunk = Download(chunkOptions, chunkContext); int64_t bytesRead = chunk.Value.BodyStream->ReadToCount( buffer + (offset - firstChunkOffset), @@ -773,7 +776,8 @@ namespace Azure { namespace Storage { namespace Blobs { Core::Http::HttpRange remainingRange; remainingRange.Offset = remainingOffset; remainingRange.Length = remainingSize; - dataLocalityState = CreateDataLocalityLayoutState(*this, remainingRange, eTag, context); + dataLocalityState + = _detail::DataLocalityLayoutState::Create(*this, remainingRange, eTag, context); dataLocalityState->WaitForInitialLayout(); if (!dataLocalityState->HasLayout()) { @@ -824,9 +828,10 @@ namespace Azure { namespace Storage { namespace Blobs { chunkOptions.Range.Value().Length = length; chunkOptions.AccessConditions.IfMatch = eTag; chunkOptions.ValidationOptions = options.ValidationOptions; - const auto chunkContext = dataLocalityState ? _internal::WithDataLocalityEndpoint( - context, dataLocalityState->GetEndpoint(offset, length)) - : context; + const auto chunkContext = dataLocalityState + ? _internal::WithDataLocalityEndpoint( + context, dataLocalityState->GetEndpoint(offset, length)) + : context; auto chunk = Download(chunkOptions, chunkContext); bodyStreamToFile( *(chunk.Value.BodyStream), diff --git a/sdk/storage/azure-storage-blobs/src/blob_responses.cpp b/sdk/storage/azure-storage-blobs/src/blob_responses.cpp index 3f7279681d..f28508c6ad 100644 --- a/sdk/storage/azure-storage-blobs/src/blob_responses.cpp +++ b/sdk/storage/azure-storage-blobs/src/blob_responses.cpp @@ -10,7 +10,7 @@ namespace Azure { namespace Storage { namespace Blobs { - void BlobLayoutPagedResponse::OnNextPage(const Azure::Core::Context& context) + void _detail::BlobLayoutPagedResponse::OnNextPage(const Azure::Core::Context& context) { m_operationOptions.ContinuationToken = NextPageToken; m_operationOptions.AccessConditions.IfMatch = Layout.Properties.ETag; diff --git a/sdk/storage/azure-storage-blobs/src/private/data_locality.cpp b/sdk/storage/azure-storage-blobs/src/private/data_locality.cpp index 698bcf0b22..fccd0a4611 100644 --- a/sdk/storage/azure-storage-blobs/src/private/data_locality.cpp +++ b/sdk/storage/azure-storage-blobs/src/private/data_locality.cpp @@ -28,7 +28,7 @@ namespace Azure { namespace Storage { namespace Blobs { namespace _detail { layout.Ranges.begin(), layout.Ranges.end(), offset, - [](const Models::BlobLayoutRange& value, int64_t target) { + [](const Models::_detail::BlobLayoutRange& value, int64_t target) { return value.Range.Offset <= target && value.Range.Length.Value() <= target - value.Range.Offset; }); diff --git a/sdk/storage/azure-storage-blobs/src/private/data_locality.hpp b/sdk/storage/azure-storage-blobs/src/private/data_locality.hpp index 4871ca9df6..f113cd94b9 100644 --- a/sdk/storage/azure-storage-blobs/src/private/data_locality.hpp +++ b/sdk/storage/azure-storage-blobs/src/private/data_locality.hpp @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -21,7 +22,7 @@ namespace Azure { namespace Storage { namespace Blobs { namespace _detail { struct DataLocalityLayout final { - std::vector Ranges; + std::vector Ranges; int64_t BlobSize = 0; Azure::ETag ETag; }; @@ -33,6 +34,12 @@ namespace Azure { namespace Storage { namespace Blobs { namespace _detail { class DataLocalityLayoutState final { public: + static std::unique_ptr Create( + Blobs::BlobClient blobClient, + Azure::Nullable range, + Azure::ETag eTag, + Azure::Core::Context context); + explicit DataLocalityLayoutState(std::function refresh); ~DataLocalityLayoutState() { WaitForRefreshTask(); } diff --git a/sdk/storage/azure-storage-blobs/test/ut/data_locality_test.cpp b/sdk/storage/azure-storage-blobs/test/ut/data_locality_test.cpp index 2e5e7c15ed..eeff8db104 100644 --- a/sdk/storage/azure-storage-blobs/test/ut/data_locality_test.cpp +++ b/sdk/storage/azure-storage-blobs/test/ut/data_locality_test.cpp @@ -13,7 +13,6 @@ #include #include #include -#include #include #include @@ -72,12 +71,13 @@ namespace Azure { namespace Storage { namespace Test { { std::lock_guard lock(m_state->Mutex); - m_state->Requests.push_back(LocalityRequest{ - request.GetUrl().GetHost(), - hostHeader == headers.end() ? std::string() : hostHeader->second, - ifMatchHeader == headers.end() ? std::string() : ifMatchHeader->second, - rangeHeader == headers.end() ? std::string() : rangeHeader->second, - isLayout}); + m_state->Requests.push_back( + LocalityRequest{ + request.GetUrl().GetHost(), + hostHeader == headers.end() ? std::string() : hostHeader->second, + ifMatchHeader == headers.end() ? std::string() : ifMatchHeader->second, + rangeHeader == headers.end() ? std::string() : rangeHeader->second, + isLayout}); } if (isLayout) @@ -164,9 +164,10 @@ namespace Azure { namespace Storage { namespace Test { rangeHeader == headers.end() ? Core::Http::HttpStatusCode::Ok : Core::Http::HttpStatusCode::PartialContent, "OK"); - response->SetBodyStream(std::make_unique( - reinterpret_cast(m_data->data() + offset), - static_cast(length))); + response->SetBodyStream( + std::make_unique( + reinterpret_cast(m_data->data() + offset), + static_cast(length))); response->SetHeader("content-length", std::to_string(length)); if (rangeHeader != headers.end()) { @@ -223,12 +224,12 @@ namespace Azure { namespace Storage { namespace Test { return options; } - Blobs::Models::BlobLayoutRange MakeLayoutRange( + Blobs::Models::_detail::BlobLayoutRange MakeLayoutRange( int64_t offset, int64_t length, std::string endpoint) { - Blobs::Models::BlobLayoutRange range; + Blobs::Models::_detail::BlobLayoutRange range; range.Range.Offset = offset; range.Range.Length = length; range.Endpoint = std::move(endpoint); @@ -243,12 +244,9 @@ namespace Azure { namespace Storage { namespace Test { } } // namespace - class DataLocalityRecordedTest : public StorageTest { - }; - TEST(DataLocalityTest, SelectsIdealEndpoint) { - using Range = Blobs::Models::BlobLayoutRange; + using Range = Blobs::Models::_detail::BlobLayoutRange; struct TestCase final { const char* Name; @@ -330,126 +328,6 @@ namespace Azure { namespace Storage { namespace Test { EXPECT_EQ(refreshCount, 1); } - TEST(DataLocalityTest, GetLayoutNoContentReturnsBlobProperties) - { - auto state = std::make_shared(); - state->LayoutStatus = Core::Http::HttpStatusCode::NoContent; - auto data = std::make_shared(1024 * 1024, 'n'); - auto client = CreateLocalityClient(state, data); - - auto page = client.GetLayout(); - - EXPECT_TRUE(page.Layout.Ranges.empty()); - EXPECT_EQ(page.Layout.Properties.ETag.ToString(), "\"locality-etag\""); - EXPECT_EQ(page.Layout.Properties.BlobSize, static_cast(data->size())); - EXPECT_EQ(page.Layout.Properties.Metadata.at("layout"), "metadata"); - EXPECT_EQ(page.RawResponse->GetStatusCode(), Core::Http::HttpStatusCode::NoContent); - } - - TEST_F(DataLocalityRecordedTest, GetLayoutMultipleEndpoints_PLAYBACKONLY_) - { - auto clientOptions = InitStorageClientOptions(); - auto blobClient = Blobs::BlobClient::CreateFromConnectionString( - StandardStorageConnectionString(), - "data-locality-test", - "data-locality-multi-endpoint.bin", - clientOptions); - - std::vector ranges; - std::set endpoints; - auto page = blobClient.GetLayout(); - EXPECT_EQ(page.Layout.Properties.BlobSize, 64 * 1024 * 1024); - EXPECT_EQ(page.Layout.Properties.BlobType, Blobs::Models::BlobType::BlockBlob); - EXPECT_EQ(page.Layout.Properties.HttpHeaders.ContentType, "application/octet-stream"); - EXPECT_TRUE(page.Layout.Properties.IsServerEncrypted); - for (; page.HasPage(); page.MoveToNextPage()) - { - for (const auto& range : page.Layout.Ranges) - { - EXPECT_GE(range.Range.Offset, 0); - ASSERT_TRUE(range.Range.Length.HasValue()); - EXPECT_GT(range.Range.Length.Value(), 0); - EXPECT_FALSE(range.Endpoint.empty()); - ranges.push_back(range); - endpoints.insert(range.Endpoint); - } - } - - if (m_testContext.IsLiveMode() && endpoints.size() < 2) - { - GTEST_SKIP() << "The configured live account does not return a multi-endpoint layout."; - } - ASSERT_GT(ranges.size(), 1U); - EXPECT_GE(endpoints.size(), 2U); - - const auto differentEndpoint - = std::adjacent_find(ranges.begin(), ranges.end(), [](auto const& left, auto const& right) { - return left.Endpoint != right.Endpoint; - }); - ASSERT_NE(differentEndpoint, ranges.end()); - const auto nextRange = differentEndpoint + 1; - ASSERT_GE(nextRange->Range.Offset, 2); - - constexpr int64_t downloadLength = 5; - const int64_t downloadOffset = nextRange->Range.Offset - 2; - Blobs::DownloadBlobOptions baselineOptions; - baselineOptions.Range = Core::Http::HttpRange(); - baselineOptions.Range.Value().Offset = downloadOffset; - baselineOptions.Range.Value().Length = downloadLength; - auto downloadResponse = blobClient.Download(baselineOptions); - auto content = downloadResponse.Value.BodyStream->ReadToEnd(); - ASSERT_EQ(content.size(), static_cast(downloadLength)); - ASSERT_TRUE(downloadResponse.Value.Details.DownloadHint.HasValue()); - EXPECT_EQ( - downloadResponse.Value.Details.DownloadHint.Value(), Blobs::Models::DownloadHint::Layout); - } - - TEST_F(DataLocalityRecordedTest, GetLayoutRangeReturnsFullBlobSize_PLAYBACKONLY_) - { - auto clientOptions = InitStorageClientOptions(); - auto client = Blobs::BlobClient::CreateFromConnectionString( - StandardStorageConnectionString(), - "data-locality-test", - "data-locality-multi-endpoint.bin", - clientOptions); - constexpr int64_t blobSize = 64 * 1024 * 1024; - for (bool bounded : {true, false}) - { - SCOPED_TRACE(bounded); - Blobs::GetBlobLayoutOptions options; - options.Range = Core::Http::HttpRange(); - options.Range.Value().Offset = bounded ? 8 * 1024 * 1024 - 2 : blobSize - 16; - if (bounded) - { - options.Range.Value().Length = 5; - } - const auto offset = options.Range.Value().Offset; - bool coversOffset = false; - size_t rangeCount = 0; - for (auto page = client.GetLayout(options); page.HasPage(); page.MoveToNextPage()) - { - EXPECT_EQ(page.Layout.Properties.BlobSize, blobSize); - EXPECT_GT(page.Layout.Properties.BlobSize, options.Range.Value().Length.ValueOr(16)); - EXPECT_TRUE(page.Layout.Properties.ETag.HasValue()); - for (const auto& range : page.Layout.Ranges) - { - ASSERT_TRUE(range.Range.Length.HasValue()); - ASSERT_GT(range.Range.Length.Value(), 0); - ASSERT_GE(range.Range.Offset, 0); - ASSERT_LE(range.Range.Offset, blobSize); - ASSERT_LE(range.Range.Length.Value(), blobSize - range.Range.Offset); - EXPECT_FALSE(range.Endpoint.empty()); - coversOffset = coversOffset - || (range.Range.Offset <= offset - && offset - range.Range.Offset < range.Range.Length.Value()); - ++rangeCount; - } - } - EXPECT_GT(rangeCount, 0U); - EXPECT_TRUE(coversOffset); - } - } - TEST(DataLocalityTest, RoutesDownloadChunks) { auto state = std::make_shared(); @@ -775,14 +653,13 @@ namespace Azure { namespace Storage { namespace Test { EXPECT_EQ(state->Requests[2].IfMatch, "\"locality-etag\""); } - TEST(DataLocalityTest, RoutesOneShotDownloadToLayoutEndpoint) + TEST(DataLocalityTest, OneShotDownloadUsesClientEndpoint) { auto state = std::make_shared(); auto data = std::make_shared(4 * 1024, 'o'); auto client = CreateLocalityClient(state, data); Blobs::DownloadBlobOptions options; - options.LayoutEndpoint = "locality0.test:443"; auto response = client.Download(options); EXPECT_TRUE(response.Value.Details.DownloadHint.HasValue()); @@ -791,8 +668,8 @@ namespace Azure { namespace Storage { namespace Test { std::lock_guard lock(state->Mutex); ASSERT_EQ(state->Requests.size(), 1U); EXPECT_FALSE(state->Requests.front().IsLayout); - EXPECT_EQ(state->Requests.front().Host, "locality0.test"); - EXPECT_EQ(state->Requests.front().HostHeader, "primary.test"); + EXPECT_EQ(state->Requests.front().Host, "primary.test"); + EXPECT_TRUE(state->Requests.front().HostHeader.empty()); } TEST(DataLocalityTest, FallsBackOnUnsupportedLayout) diff --git a/sdk/storage/azure-storage-files-datalake/CHANGELOG.md b/sdk/storage/azure-storage-files-datalake/CHANGELOG.md index b33a93452f..8191034a84 100644 --- a/sdk/storage/azure-storage-files-datalake/CHANGELOG.md +++ b/sdk/storage/azure-storage-files-datalake/CHANGELOG.md @@ -5,11 +5,7 @@ ### Features Added - Added support for persisted session authentication for eligible file download requests. -- Added `DataLakeFileClient::GetLayout` and related options and response types for retrieving - locality endpoints, byte ranges, and file properties. Ranges are exposed directly through - `Models::FileLayoutInfo::Ranges` as a vector of `Models::FileLayoutRange`. -- Added `DownloadFileOptions::LayoutEndpoint` for one-shot download routing and - `DownloadFileToOptions::LayoutAwareRouting` (`Auto`, `Disabled`, `Enabled`; default `Disabled`) for +- Added `DownloadFileToOptions::LayoutAwareRouting` (`Auto`, `Disabled`, `Enabled`; default `Disabled`) for managed downloads. - Added the `Models::DownloadHint` alias and `Models::DownloadFileDetails::DownloadHint`. diff --git a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_file_client.hpp b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_file_client.hpp index e3af80df50..e04d891bc0 100644 --- a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_file_client.hpp +++ b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_file_client.hpp @@ -189,17 +189,6 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { const DownloadFileOptions& options = DownloadFileOptions(), const Azure::Core::Context& context = Azure::Core::Context()) const; - /** - * @brief Gets the data locality layout for this file. - * @param options Optional parameters to execute this function. - * @param context Context for cancelling long running operations. - * @return A paged response containing the file's data locality layout. - * @remark This request is sent to the blob endpoint. - */ - FileLayoutPagedResponse GetLayout( - const GetFileLayoutOptions& options = GetFileLayoutOptions(), - const Azure::Core::Context& context = Azure::Core::Context()) const; - /** * @brief Creates a new file, or updates the content of an existing file. Updating * an existing file overwrites any existing metadata on the file. @@ -299,10 +288,10 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { std::shared_ptr pipeline, _detail::DatalakeClientConfiguration clientConfiguration) : DataLakePathClient( - std::move(fileUrl), - std::move(blobClient), - pipeline, - std::move(clientConfiguration)) + std::move(fileUrl), + std::move(blobClient), + pipeline, + std::move(clientConfiguration)) { } diff --git a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_options.hpp b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_options.hpp index 266eb0b768..ee231c28cb 100644 --- a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_options.hpp +++ b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_options.hpp @@ -890,27 +890,6 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { Nullable IncludeUserPrincipalName; }; - /** - * @brief Optional parameters for #Azure::Storage::Files::DataLake::DataLakeFileClient::GetLayout. - */ - struct GetFileLayoutOptions final - { - /** - * @brief The range of bytes for which to retrieve the layout. - */ - Azure::Nullable Range; - - /** - * @brief The continuation token for retrieving the next page. - */ - Azure::Nullable ContinuationToken; - - /** - * @brief Access conditions for the operation. - */ - PathAccessConditions AccessConditions; - }; - /** * @brief Optional parameters for #Azure::Storage::Files::DataLake::DataLakeFileClient::Download. * @remark Some optional parameter is mandatory in certain combination. @@ -949,12 +928,6 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { * @brief Optional. Configures whether to do content validation for file downloads. */ Azure::Nullable ValidationOptions; - - /** - * @brief The locality endpoint to use for this download. An empty value uses the normal - * endpoint. - */ - std::string LayoutEndpoint; }; /** diff --git a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_responses.hpp b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_responses.hpp index d0660cf788..7cdd9f8dcd 100644 --- a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_responses.hpp +++ b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_responses.hpp @@ -463,26 +463,6 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { Azure::Nullable Permissions; }; - /** - * @brief A file range and its resolved locality endpoint. - */ - using FileLayoutRange = Blobs::Models::BlobLayoutRange; - - /** - * @brief A page of file layout information and the file properties returned with it. - */ - struct FileLayoutInfo final - { - /** - * @brief The ranges and locality endpoints in this page. - */ - std::vector Ranges; - /** - * @brief The file properties returned with this page. - */ - PathProperties Properties; - }; - /** * @brief The access control list of a path. */ @@ -823,27 +803,6 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { } // namespace Models - /** - * @brief Response type for - * #Azure::Storage::Files::DataLake::DataLakeFileClient::GetLayout. - */ - class FileLayoutPagedResponse final : public Azure::Core::PagedResponse { - public: - /** - * @brief The layout and file properties in this page. - */ - Models::FileLayoutInfo Layout; - - private: - void OnNextPage(const Azure::Core::Context& context); - - std::shared_ptr m_fileClient; - GetFileLayoutOptions m_operationOptions; - - friend class DataLakeFileClient; - friend class Azure::Core::PagedResponse; - }; - /** * @brief Response type for * #Azure::Storage::Files::DataLake::DataLakeServiceClient::ListFileSystems. diff --git a/sdk/storage/azure-storage-files-datalake/src/datalake_file_client.cpp b/sdk/storage/azure-storage-files-datalake/src/datalake_file_client.cpp index 6b957c633b..1a7a63b8be 100644 --- a/sdk/storage/azure-storage-files-datalake/src/datalake_file_client.cpp +++ b/sdk/storage/azure-storage-files-datalake/src/datalake_file_client.cpp @@ -204,7 +204,6 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { blobOptions.Range = options.Range; blobOptions.RangeHashAlgorithm = options.RangeHashAlgorithm; blobOptions.Range = options.Range; - blobOptions.LayoutEndpoint = options.LayoutEndpoint; blobOptions.AccessConditions.IfMatch = options.AccessConditions.IfMatch; blobOptions.AccessConditions.IfNoneMatch = options.AccessConditions.IfNoneMatch; blobOptions.AccessConditions.IfModifiedSince = options.AccessConditions.IfModifiedSince; @@ -218,10 +217,11 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { } auto response = m_blobClient.Download( blobOptions, - options.IncludeUserPrincipalName.HasValue() ? context.WithValue( - Blobs::_detail::DataLakeInteroperabilityExtraOptionsKey, - options.IncludeUserPrincipalName.Value()) - : context); + options.IncludeUserPrincipalName.HasValue() + ? context.WithValue( + Blobs::_detail::DataLakeInteroperabilityExtraOptionsKey, + options.IncludeUserPrincipalName.Value()) + : context); Models::DownloadFileResult ret; ret.Body = std::move(response.Value.BodyStream); ret.Details.HttpHeaders = std::move(response.Value.Details.HttpHeaders); @@ -285,32 +285,6 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { std::move(ret), std::move(response.RawResponse)); } - FileLayoutPagedResponse DataLakeFileClient::GetLayout( - const GetFileLayoutOptions& options, - const Azure::Core::Context& context) const - { - Blobs::GetBlobLayoutOptions blobOptions; - blobOptions.Range = options.Range; - blobOptions.ContinuationToken = options.ContinuationToken; - blobOptions.AccessConditions.IfMatch = options.AccessConditions.IfMatch; - blobOptions.AccessConditions.IfNoneMatch = options.AccessConditions.IfNoneMatch; - blobOptions.AccessConditions.IfModifiedSince = options.AccessConditions.IfModifiedSince; - blobOptions.AccessConditions.IfUnmodifiedSince = options.AccessConditions.IfUnmodifiedSince; - blobOptions.AccessConditions.LeaseId = options.AccessConditions.LeaseId; - auto response = m_blobClient.GetLayout(blobOptions, context); - - FileLayoutPagedResponse result; - result.Layout.Ranges = std::move(response.Layout.Ranges); - result.Layout.Properties = _detail::PathPropertiesFromBlobProperties( - std::move(response.Layout.Properties), *response.RawResponse); - result.CurrentPageToken = std::move(response.CurrentPageToken); - result.NextPageToken = std::move(response.NextPageToken); - result.RawResponse = std::move(response.RawResponse); - result.m_fileClient = std::make_shared(*this); - result.m_operationOptions = options; - return result; - } - Azure::Response DataLakeFileClient::UploadFrom( const std::string& fileName, const UploadFileFromOptions& options, diff --git a/sdk/storage/azure-storage-files-datalake/src/datalake_responses.cpp b/sdk/storage/azure-storage-files-datalake/src/datalake_responses.cpp index 3652ac5891..c6fa0aa1c5 100644 --- a/sdk/storage/azure-storage-files-datalake/src/datalake_responses.cpp +++ b/sdk/storage/azure-storage-files-datalake/src/datalake_responses.cpp @@ -4,20 +4,11 @@ #include "azure/storage/files/datalake/datalake_responses.hpp" #include "azure/storage/files/datalake/datalake_directory_client.hpp" -#include "azure/storage/files/datalake/datalake_file_client.hpp" #include "azure/storage/files/datalake/datalake_path_client.hpp" #include "azure/storage/files/datalake/datalake_service_client.hpp" #include "private/datalake_utilities.hpp" namespace Azure { namespace Storage { namespace Files { namespace DataLake { - void FileLayoutPagedResponse::OnNextPage(const Azure::Core::Context& context) - { - auto options = m_operationOptions; - options.ContinuationToken = NextPageToken; - options.AccessConditions.IfMatch = Layout.Properties.ETag; - *this = m_fileClient->GetLayout(options, context); - } - namespace Models { Acl Acl::FromString(const std::string& aclString) diff --git a/sdk/storage/azure-storage-files-datalake/test/ut/data_locality_test.cpp b/sdk/storage/azure-storage-files-datalake/test/ut/data_locality_test.cpp index 899b4191e3..cfb27763b4 100644 --- a/sdk/storage/azure-storage-files-datalake/test/ut/data_locality_test.cpp +++ b/sdk/storage/azure-storage-files-datalake/test/ut/data_locality_test.cpp @@ -8,7 +8,6 @@ #include #include #include -#include #include #include @@ -58,21 +57,21 @@ namespace Azure { namespace Storage { namespace Test { }; const bool isLayout = query.count("comp") && query.at("comp") == "layout"; const auto marker = query.count("marker") ? query.at("marker") : std::string(); - m_state->Requests.push_back(LocalityRequest{ - request.GetUrl().GetHost(), - header("host"), - header("x-ms-range"), - header("if-match"), - marker, - isLayout}); + m_state->Requests.push_back( + LocalityRequest{ + request.GetUrl().GetHost(), + header("host"), + header("x-ms-range"), + header("if-match"), + marker, + isLayout}); auto response = std::make_unique( 1, 1, - isLayout - ? (m_state->NoLayout ? Core::Http::HttpStatusCode::NoContent - : Core::Http::HttpStatusCode::Ok) - : Core::Http::HttpStatusCode::PartialContent, + isLayout ? (m_state->NoLayout ? Core::Http::HttpStatusCode::NoContent + : Core::Http::HttpStatusCode::Ok) + : Core::Http::HttpStatusCode::PartialContent, "OK"); response->SetHeader("etag", "\"layout-etag\""); response->SetHeader("last-modified", "Thu, 23 Aug 2001 07:00:00 GMT"); @@ -113,9 +112,10 @@ namespace Azure { namespace Storage { namespace Test { response->SetHeader( "content-range", "bytes " + std::to_string(offset) + "-" + std::to_string(end) + "/12"); response->SetHeader("x-ms-download-hint", "layout"); - response->SetBodyStream(std::make_unique( - reinterpret_cast(data.data() + offset), - static_cast(end - offset + 1))); + response->SetBodyStream( + std::make_unique( + reinterpret_cast(data.data() + offset), + static_cast(end - offset + 1))); return response; } @@ -148,162 +148,29 @@ namespace Azure { namespace Storage { namespace Test { } } // namespace - class DataLakeLocalityRecordedTest : public StorageTest { - }; - - TEST_F(DataLakeLocalityRecordedTest, GetLayoutAndDownloadHint_PLAYBACKONLY_) - { - if (!m_testContext.IsPlaybackMode()) - { - GTEST_SKIP() << "Reuses the Blob multi-endpoint recording."; - } - m_testContext.RecordingPath = std::string(AZURE_TEST_RECORDING_DIR) - + "/../../../azure-storage-blobs/test/ut/recordings"; - m_testContext.RenameTest( - "DataLocalityRecordedTest", "GetLayoutMultipleEndpoints_PLAYBACKONLY_"); - auto options = InitStorageClientOptions(); - auto client = DataLake::DataLakeFileClient( - "https://" + StandardStorageAccountName() - + ".dfs.core.windows.net/data-locality-test/data-locality-multi-endpoint.bin", - GetTestCredential(), - options); - auto page = client.GetLayout(); - EXPECT_EQ(page.Layout.Properties.FileSize, 64 * 1024 * 1024); - EXPECT_EQ(page.Layout.Properties.HttpHeaders.ContentType, "application/octet-stream"); - ASSERT_TRUE(page.Layout.Properties.IsServerEncrypted.HasValue()); - EXPECT_TRUE(page.Layout.Properties.IsServerEncrypted.Value()); - EXPECT_TRUE(page.Layout.Properties.ETag.HasValue()); - EXPECT_FALSE(page.Layout.Properties.IsDirectory); - std::set endpoints; - std::vector ranges; - for (; page.HasPage(); page.MoveToNextPage()) - { - for (const auto& range : page.Layout.Ranges) - { - ASSERT_TRUE(range.Range.Length.HasValue()); - EXPECT_GT(range.Range.Length.Value(), 0); - EXPECT_FALSE(range.Endpoint.empty()); - ranges.push_back(range); - endpoints.insert(range.Endpoint); - } - } - EXPECT_GE(endpoints.size(), 2U); - ASSERT_GT(ranges.size(), 1U); - DataLake::DownloadFileOptions downloadOptions; - downloadOptions.Range = Core::Http::HttpRange(); - downloadOptions.Range.Value().Offset = ranges[1].Range.Offset - 2; - downloadOptions.Range.Value().Length = 5; - auto response = client.Download(downloadOptions); - EXPECT_EQ(response.Value.Body->ReadToEnd().size(), 5U); - EXPECT_EQ(response.Value.FileSize, 64 * 1024 * 1024); - ASSERT_TRUE(response.Value.Details.DownloadHint.HasValue()); - EXPECT_EQ(response.Value.Details.DownloadHint.Value(), Blobs::Models::DownloadHint::Layout); - } - - TEST_F(DataLakeLocalityRecordedTest, GetLayoutRangeReturnsFullFileSize_PLAYBACKONLY_) - { - if (!m_testContext.IsPlaybackMode()) - { - GTEST_SKIP() << "Reuses the Blob ranged layout recording."; - } - m_testContext.RecordingPath = std::string(AZURE_TEST_RECORDING_DIR) - + "/../../../azure-storage-blobs/test/ut/recordings"; - m_testContext.RenameTest( - "DataLocalityRecordedTest", "GetLayoutRangeReturnsFullBlobSize_PLAYBACKONLY_"); - auto clientOptions = InitStorageClientOptions(); - auto client = DataLake::DataLakeFileClient( - "https://" + StandardStorageAccountName() - + ".dfs.core.windows.net/data-locality-test/data-locality-multi-endpoint.bin", - GetTestCredential(), - clientOptions); - constexpr int64_t fileSize = 64 * 1024 * 1024; - for (bool bounded : {true, false}) - { - SCOPED_TRACE(bounded); - DataLake::GetFileLayoutOptions options; - options.Range = Core::Http::HttpRange(); - options.Range.Value().Offset = bounded ? 8 * 1024 * 1024 - 2 : fileSize - 16; - if (bounded) - { - options.Range.Value().Length = 5; - } - const auto offset = options.Range.Value().Offset; - bool coversOffset = false; - size_t rangeCount = 0; - for (auto page = client.GetLayout(options); page.HasPage(); page.MoveToNextPage()) - { - EXPECT_EQ(page.Layout.Properties.FileSize, fileSize); - EXPECT_GT(page.Layout.Properties.FileSize, options.Range.Value().Length.ValueOr(16)); - EXPECT_TRUE(page.Layout.Properties.ETag.HasValue()); - for (const auto& range : page.Layout.Ranges) - { - ASSERT_TRUE(range.Range.Length.HasValue()); - ASSERT_GT(range.Range.Length.Value(), 0); - ASSERT_GE(range.Range.Offset, 0); - ASSERT_LE(range.Range.Offset, fileSize); - ASSERT_LE(range.Range.Length.Value(), fileSize - range.Range.Offset); - EXPECT_FALSE(range.Endpoint.empty()); - coversOffset = coversOffset - || (range.Range.Offset <= offset - && offset - range.Range.Offset < range.Range.Length.Value()); - ++rangeCount; - } - } - EXPECT_GT(rangeCount, 0U); - EXPECT_TRUE(coversOffset); - } - } - - TEST(DataLakeLocalityTest, GetLayoutMapsPropertiesAndPages) - { - auto state = std::make_shared(); - auto client = CreateClient(state, 0); - DataLake::GetFileLayoutOptions options; - options.Range = Core::Http::HttpRange(); - options.Range.Value().Offset = 2; - options.Range.Value().Length = 10; - options.AccessConditions.IfMatch = ETag("\"layout-etag\""); - auto page = client.GetLayout(options); - EXPECT_EQ(page.CurrentPageToken, ""); - EXPECT_EQ(page.NextPageToken.Value(), "page2"); - ASSERT_EQ(page.Layout.Ranges.size(), 1U); - EXPECT_EQ(page.Layout.Ranges[0].Endpoint, "locality0.test:443"); - EXPECT_EQ(page.Layout.Properties.FileSize, 12); - EXPECT_EQ(page.Layout.Properties.Metadata.at("layout"), "metadata"); - EXPECT_EQ(page.Layout.Properties.Owner.Value(), "owner"); - EXPECT_EQ(page.Layout.Properties.Group.Value(), "group"); - EXPECT_EQ(page.Layout.Properties.Permissions.Value(), "rw-r-----"); - EXPECT_EQ(page.Layout.Properties.EncryptionContext.Value(), "context"); - ASSERT_TRUE(page.Layout.Properties.Acls.HasValue()); - EXPECT_EQ(page.Layout.Properties.Acls.Value().size(), 3U); - page.MoveToNextPage(); - EXPECT_EQ(page.CurrentPageToken, "page2"); - ASSERT_EQ(page.Layout.Ranges.size(), 1U); - EXPECT_EQ(page.Layout.Ranges[0].Range.Offset, 6); - EXPECT_EQ(page.Layout.Ranges[0].Range.Length.Value(), 6); - EXPECT_EQ(page.Layout.Ranges[0].Endpoint, "locality1.test:443"); - page.MoveToNextPage(); - EXPECT_FALSE(page.HasPage()); - ASSERT_EQ(state->Requests.size(), 2U); - EXPECT_EQ(state->Requests[0].Host, "account.blob.core.windows.net"); - EXPECT_EQ(state->Requests[1].Range, "bytes=2-11"); - EXPECT_EQ(state->Requests[1].IfMatch, "\"layout-etag\""); - EXPECT_EQ(state->Requests[1].Marker, "page2"); - } - - TEST(DataLakeLocalityTest, GetLayoutNoContent) + TEST(DataLakeLocalityTest, ManagedDownloadFallsBackWithoutLayout) { auto state = std::make_shared(); state->NoLayout = true; - auto page = CreateClient(state, 0).GetLayout(); - EXPECT_TRUE(page.Layout.Ranges.empty()); - EXPECT_EQ(page.Layout.Properties.FileSize, 12); - EXPECT_EQ(page.RawResponse->GetStatusCode(), Core::Http::HttpStatusCode::NoContent); - page.MoveToNextPage(); - EXPECT_FALSE(page.HasPage()); + DataLake::DownloadFileToOptions options; + options.Range = Core::Http::HttpRange(); + options.Range.Value().Length = 12; + options.LayoutAwareRouting = DataLake::LayoutAwareRouting::Enabled; + options.TransferOptions.InitialChunkSize = 1; + options.TransferOptions.ChunkSize = 6; + options.TransferOptions.Concurrency = 1; + std::vector content(12); + CreateClient(state, 0).DownloadTo(content.data(), content.size(), options); + EXPECT_EQ(std::string(content.begin(), content.end()), "abcdefghijkl"); + ASSERT_EQ(state->Requests.size(), 4U); + EXPECT_TRUE(state->Requests[1].IsLayout); + for (const auto& request : state->Requests) + { + EXPECT_EQ(request.Host, "account.blob.core.windows.net"); + } } - TEST(DataLakeLocalityTest, RoutesOneShotDownloadForAllClientOrigins) + TEST(DataLakeLocalityTest, OneShotDownloadUsesClientEndpointForAllClientOrigins) { for (int origin = 0; origin != 4; ++origin) { @@ -314,12 +181,11 @@ namespace Azure { namespace Storage { namespace Test { options.Range = Core::Http::HttpRange(); options.Range.Value().Offset = 6; options.Range.Value().Length = 1; - options.LayoutEndpoint = "locality1.test:443"; auto response = client.Download(options); EXPECT_EQ(response.Value.Body->ReadToEnd(), std::vector{'g'}); ASSERT_EQ(state->Requests.size(), 1U); - EXPECT_EQ(state->Requests[0].Host, "locality1.test"); - EXPECT_EQ(state->Requests[0].HostHeader, "account.blob.core.windows.net"); + EXPECT_EQ(state->Requests[0].Host, "account.blob.core.windows.net"); + EXPECT_TRUE(state->Requests[0].HostHeader.empty()); ASSERT_TRUE(response.Value.Details.DownloadHint.HasValue()); EXPECT_EQ(response.Value.Details.DownloadHint.Value(), Blobs::Models::DownloadHint::Layout); }