From b9e2b1507a24d81770580bc5bcad0b36cc009235 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Tue, 11 Aug 2026 19:54:06 -0400 Subject: [PATCH 01/14] Send agentless exposures directly to EVP --- .../communication/BackendApiFactory.java | 56 +++++--- .../datadog/communication/EvpProxyApi.java | 3 +- .../communication/HttpResponseException.java | 18 +++ .../communication/EvpProxyApiTest.java | 65 ++++++++++ .../feature-flagging-lib/build.gradle.kts | 2 + .../AgentlessExposureBackendApi.java | 66 ++++++++++ .../ExposureBackendApiFactory.java | 72 +++++++++++ .../featureflag/ExposureWriterImpl.java | 24 ++-- .../AgentlessExposureBackendApiTest.java | 122 ++++++++++++++++++ .../ExposureBackendApiFactoryTest.java | 103 +++++++++++++++ .../featureflag/ExposureWriterTests.java | 49 ++++++- 11 files changed, 546 insertions(+), 34 deletions(-) create mode 100644 communication/src/main/java/datadog/communication/HttpResponseException.java create mode 100644 communication/src/test/java/datadog/communication/EvpProxyApiTest.java create mode 100644 products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/AgentlessExposureBackendApi.java create mode 100644 products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/ExposureBackendApiFactory.java create mode 100644 products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessExposureBackendApiTest.java create mode 100644 products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/ExposureBackendApiFactoryTest.java diff --git a/communication/src/main/java/datadog/communication/BackendApiFactory.java b/communication/src/main/java/datadog/communication/BackendApiFactory.java index 3ce78b88c22..5b7d92b29a0 100644 --- a/communication/src/main/java/datadog/communication/BackendApiFactory.java +++ b/communication/src/main/java/datadog/communication/BackendApiFactory.java @@ -24,25 +24,39 @@ public BackendApiFactory(Config config, SharedCommunicationObjects sharedCommuni } public @Nullable BackendApi createBackendApi(Intake intake) { - HttpRetryPolicy.Factory retryPolicyFactory = new HttpRetryPolicy.Factory(5, 100, 2.0, true); - if (intake.isAgentlessEnabled(config)) { - HttpUrl agentlessUrl = HttpUrl.get(intake.getAgentlessUrl(config)); - String apiKey = config.getApiKey(); - if (apiKey == null || apiKey.isEmpty()) { - throw new FatalAgentMisconfigurationError( - "Agentless mode is enabled and api key is not set. Please set application key"); - } - String traceId = config.getIdGenerationStrategy().generateTraceId().toString(); - return new IntakeApi( - agentlessUrl, - apiKey, - traceId, - retryPolicyFactory, - sharedCommunicationObjects.getIntakeHttpClient(), - true); + return createDirectIntakeApi(intake); } + BackendApi backendApi = createEvpProxyApi(intake); + if (backendApi == null) { + log.warn( + "Cannot create backend API client since agentless mode is disabled, " + + "and agent does not support EVP proxy"); + } + return backendApi; + } + + /** Creates an authenticated API client that sends data directly to a Datadog intake. */ + public BackendApi createDirectIntakeApi(Intake intake) { + HttpUrl agentlessUrl = HttpUrl.get(intake.getAgentlessUrl(config)); + String apiKey = config.getApiKey(); + if (apiKey == null || apiKey.isEmpty()) { + throw new FatalAgentMisconfigurationError( + "Agentless mode is enabled and api key is not set. Please set application key"); + } + String traceId = config.getIdGenerationStrategy().generateTraceId().toString(); + return new IntakeApi( + agentlessUrl, + apiKey, + traceId, + retryPolicyFactory(), + sharedCommunicationObjects.getIntakeHttpClient(), + true); + } + + /** Creates an API client that sends data through a compatible local EVP proxy. */ + public @Nullable BackendApi createEvpProxyApi(Intake intake) { DDAgentFeaturesDiscovery featuresDiscovery = sharedCommunicationObjects.featuresDiscovery(config); featuresDiscovery.discoverIfOutdated(); @@ -55,14 +69,14 @@ public BackendApiFactory(Config config, SharedCommunicationObjects sharedCommuni traceId, evpProxyUrl, subdomain, - retryPolicyFactory, + retryPolicyFactory(), sharedCommunicationObjects.agentHttpClient, true); } - - log.warn( - "Cannot create backend API client since agentless mode is disabled, " - + "and agent does not support EVP proxy"); return null; } + + private static HttpRetryPolicy.Factory retryPolicyFactory() { + return new HttpRetryPolicy.Factory(5, 100, 2.0, true); + } } diff --git a/communication/src/main/java/datadog/communication/EvpProxyApi.java b/communication/src/main/java/datadog/communication/EvpProxyApi.java index 83037ab9663..49f0285aac1 100644 --- a/communication/src/main/java/datadog/communication/EvpProxyApi.java +++ b/communication/src/main/java/datadog/communication/EvpProxyApi.java @@ -95,7 +95,8 @@ public T post( return responseParser.apply(responseBodyStream); } else { - throw new IOException( + throw new HttpResponseException( + response.code(), "Request to " + uri + " returned error response " diff --git a/communication/src/main/java/datadog/communication/HttpResponseException.java b/communication/src/main/java/datadog/communication/HttpResponseException.java new file mode 100644 index 00000000000..ac9b62cdb2d --- /dev/null +++ b/communication/src/main/java/datadog/communication/HttpResponseException.java @@ -0,0 +1,18 @@ +package datadog.communication; + +import java.io.IOException; + +/** An HTTP request failed with a non-success response. */ +public final class HttpResponseException extends IOException { + + private final int statusCode; + + public HttpResponseException(final int statusCode, final String message) { + super(message); + this.statusCode = statusCode; + } + + public int getStatusCode() { + return statusCode; + } +} diff --git a/communication/src/test/java/datadog/communication/EvpProxyApiTest.java b/communication/src/test/java/datadog/communication/EvpProxyApiTest.java new file mode 100644 index 00000000000..14c6962bf8e --- /dev/null +++ b/communication/src/test/java/datadog/communication/EvpProxyApiTest.java @@ -0,0 +1,65 @@ +package datadog.communication; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import datadog.communication.http.HttpRetryPolicy; +import java.io.IOException; +import okhttp3.MediaType; +import okhttp3.OkHttpClient; +import okhttp3.RequestBody; +import okhttp3.mockwebserver.MockResponse; +import okhttp3.mockwebserver.MockWebServer; +import okhttp3.mockwebserver.RecordedRequest; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +class EvpProxyApiTest { + + private MockWebServer server; + private OkHttpClient client; + + @BeforeEach + void setUp() throws IOException { + server = new MockWebServer(); + server.start(); + client = new OkHttpClient.Builder().build(); + } + + @AfterEach + void tearDown() throws IOException { + client.dispatcher().executorService().shutdownNow(); + client.connectionPool().evictAll(); + server.shutdown(); + } + + @Test + void reportsHttpStatusForRejectedRequest() throws Exception { + server.enqueue(new MockResponse().setResponseCode(404).setBody("not found")); + final EvpProxyApi api = + new EvpProxyApi( + "123", + server.url("/evp_proxy/v4/"), + "event-platform-intake", + HttpRetryPolicy.Factory.NEVER_RETRY, + client, + false); + + final HttpResponseException exception = + assertThrows( + HttpResponseException.class, + () -> + api.post( + "exposures", + RequestBody.create(MediaType.parse("application/json"), "{}"), + stream -> null, + null, + false)); + + assertEquals(404, exception.getStatusCode()); + final RecordedRequest request = server.takeRequest(); + assertEquals("/evp_proxy/v4/api/v2/exposures", request.getPath()); + assertEquals("event-platform-intake", request.getHeader("X-Datadog-EVP-Subdomain")); + } +} diff --git a/products/feature-flagging/feature-flagging-lib/build.gradle.kts b/products/feature-flagging/feature-flagging-lib/build.gradle.kts index 425e217e822..266966ca894 100644 --- a/products/feature-flagging/feature-flagging-lib/build.gradle.kts +++ b/products/feature-flagging/feature-flagging-lib/build.gradle.kts @@ -20,6 +20,7 @@ dependencies { api(project(":communication")) implementation(project(":internal-api")) api(project(":products:feature-flagging:feature-flagging-bootstrap")) + compileOnly(project(":products:feature-flagging:feature-flagging-config")) implementation(project(":utils:logging-utils")) api(project(":utils:queue-utils")) @@ -29,6 +30,7 @@ dependencies { testImplementation(libs.bundles.junit5) testImplementation(libs.bundles.mockito) + testImplementation(project(":products:feature-flagging:feature-flagging-config")) testImplementation(project(":utils:test-utils")) testImplementation(project(":dd-java-agent:testing")) } diff --git a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/AgentlessExposureBackendApi.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/AgentlessExposureBackendApi.java new file mode 100644 index 00000000000..97c798f0e1d --- /dev/null +++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/AgentlessExposureBackendApi.java @@ -0,0 +1,66 @@ +package com.datadog.featureflag; + +import datadog.communication.BackendApi; +import datadog.communication.HttpResponseException; +import datadog.communication.http.OkHttpUtils; +import datadog.communication.util.IOThrowingFunction; +import java.io.IOException; +import java.io.InputStream; +import java.net.ConnectException; +import javax.annotation.Nullable; +import okhttp3.RequestBody; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** Sends exposures through a local EVP proxy, with a safe direct intake fallback. */ +final class AgentlessExposureBackendApi implements BackendApi { + + private static final Logger LOGGER = LoggerFactory.getLogger(AgentlessExposureBackendApi.class); + + private final BackendApi localApi; + private final BackendApi directApi; + private volatile BackendApi activeApi; + + AgentlessExposureBackendApi(final BackendApi localApi, final BackendApi directApi) { + this.localApi = localApi; + this.directApi = directApi; + this.activeApi = localApi; + } + + @Override + public T post( + final String uri, + final RequestBody requestBody, + final IOThrowingFunction responseParser, + @Nullable final OkHttpUtils.CustomListener requestListener, + final boolean requestCompression) + throws IOException { + final BackendApi selectedApi = activeApi; + try { + return selectedApi.post( + uri, requestBody, responseParser, requestListener, requestCompression); + } catch (final IOException exception) { + if (selectedApi != localApi || !isDefinitiveRejection(exception)) { + throw exception; + } + + if (activeApi == localApi) { + LOGGER.debug( + "Switching Feature Flagging exposure delivery from the local EVP proxy to direct intake"); + activeApi = directApi; + } + return directApi.post(uri, requestBody, responseParser, requestListener, requestCompression); + } + } + + private static boolean isDefinitiveRejection(final IOException exception) { + if (exception instanceof ConnectException) { + return true; + } + if (exception instanceof HttpResponseException) { + final int statusCode = ((HttpResponseException) exception).getStatusCode(); + return statusCode == 403 || statusCode == 404 || statusCode == 405; + } + return false; + } +} diff --git a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/ExposureBackendApiFactory.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/ExposureBackendApiFactory.java new file mode 100644 index 00000000000..f0eb1fa0e3d --- /dev/null +++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/ExposureBackendApiFactory.java @@ -0,0 +1,72 @@ +package com.datadog.featureflag; + +import static datadog.trace.api.featureflag.config.FeatureFlaggingConfig.CONFIGURATION_SOURCE_AGENTLESS; + +import datadog.communication.BackendApi; +import datadog.communication.BackendApiFactory; +import datadog.communication.ddagent.SharedCommunicationObjects; +import datadog.trace.api.Config; +import datadog.trace.api.intake.Intake; +import javax.annotation.Nullable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** Selects the transport for Feature Flagging exposure events. */ +final class ExposureBackendApiFactory { + + private static final Logger LOGGER = LoggerFactory.getLogger(ExposureBackendApiFactory.class); + + private final Config config; + private final BackendApiFactory backendApiFactory; + + ExposureBackendApiFactory( + final Config config, final SharedCommunicationObjects sharedCommunicationObjects) { + this(config, new BackendApiFactory(config, sharedCommunicationObjects)); + } + + ExposureBackendApiFactory(final Config config, final BackendApiFactory backendApiFactory) { + this.config = config; + this.backendApiFactory = backendApiFactory; + } + + @Nullable + BackendApi create() { + final BackendApi localApi = backendApiFactory.createEvpProxyApi(Intake.EVENT_PLATFORM); + if (!CONFIGURATION_SOURCE_AGENTLESS.equals(config.getFeatureFlaggingConfigurationSource())) { + if (localApi == null) { + LOGGER.warn( + "Feature Flagging exposure delivery is disabled because the local Agent does not support the EVP proxy"); + } + return localApi; + } + + final BackendApi directApi = createDirectApi(); + if (localApi != null && directApi != null) { + return new AgentlessExposureBackendApi(localApi, directApi); + } + if (localApi != null) { + return localApi; + } + if (directApi != null) { + return directApi; + } + + LOGGER.warn( + "Feature Flagging exposure delivery is disabled because no compatible local EVP proxy or direct intake credentials are available"); + return null; + } + + @Nullable + private BackendApi createDirectApi() { + final String apiKey = config.getApiKey(); + if (apiKey == null || apiKey.isEmpty()) { + return null; + } + try { + return backendApiFactory.createDirectIntakeApi(Intake.EVENT_PLATFORM); + } catch (final IllegalArgumentException exception) { + LOGGER.debug("Cannot configure direct Feature Flagging exposure delivery", exception); + return null; + } + } +} diff --git a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/ExposureWriterImpl.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/ExposureWriterImpl.java index 9932a20256b..0d8d967ed56 100644 --- a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/ExposureWriterImpl.java +++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/ExposureWriterImpl.java @@ -10,13 +10,11 @@ import datadog.common.queue.MessagePassingBlockingQueue; import datadog.common.queue.Queues; import datadog.communication.BackendApi; -import datadog.communication.BackendApiFactory; import datadog.communication.ddagent.SharedCommunicationObjects; import datadog.trace.api.Config; import datadog.trace.api.featureflag.FeatureFlaggingGateway; import datadog.trace.api.featureflag.exposure.ExposureEvent; import datadog.trace.api.featureflag.exposure.ExposuresRequest; -import datadog.trace.api.intake.Intake; import datadog.trace.api.internal.VisibleForTesting; import java.util.ArrayList; import java.util.HashMap; @@ -47,6 +45,15 @@ public ExposureWriterImpl(final SharedCommunicationObjects sco, final Config con final TimeUnit timeUnit, final SharedCommunicationObjects sco, final Config config) { + this(capacity, flushInterval, timeUnit, new ExposureBackendApiFactory(config, sco), config); + } + + ExposureWriterImpl( + final int capacity, + final long flushInterval, + final TimeUnit timeUnit, + final ExposureBackendApiFactory backendApiFactory, + final Config config) { this.queue = Queues.mpscBlockingConsumerArrayQueue(capacity); final Map context = new HashMap<>(4); context.put("service", config.getServiceName() == null ? "unknown" : config.getServiceName()); @@ -58,12 +65,7 @@ public ExposureWriterImpl(final SharedCommunicationObjects sco, final Config con } final ExposureSerializingHandler serializer = new ExposureSerializingHandler( - new BackendApiFactory(config, sco), - queue, - flushInterval, - timeUnit, - context, - this::close); + backendApiFactory, queue, flushInterval, timeUnit, context, this::close); this.serializerThread = newAgentThread(FEATURE_FLAG_EXPOSURE_PROCESSOR, serializer); } @@ -102,7 +104,7 @@ private static class ExposureSerializingHandler implements Runnable { private long lastTicks; private final JsonAdapter jsonAdapter; - private final BackendApiFactory backendApiFactory; + private final ExposureBackendApiFactory backendApiFactory; private BackendApi evp; private final Map context; @@ -112,7 +114,7 @@ private static class ExposureSerializingHandler implements Runnable { private final Runnable errorCallback; public ExposureSerializingHandler( - final BackendApiFactory backendApiFactory, + final ExposureBackendApiFactory backendApiFactory, final MessagePassingBlockingQueue queue, final long flushInterval, final TimeUnit timeUnit, @@ -134,7 +136,7 @@ public ExposureSerializingHandler( @Override public void run() { - evp = backendApiFactory.createBackendApi(Intake.EVENT_PLATFORM); + evp = backendApiFactory.create(); if (evp == null) { errorCallback.run(); throw new IllegalArgumentException("EVP Proxy not available"); diff --git a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessExposureBackendApiTest.java b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessExposureBackendApiTest.java new file mode 100644 index 00000000000..9455634ec45 --- /dev/null +++ b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessExposureBackendApiTest.java @@ -0,0 +1,122 @@ +package com.datadog.featureflag; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import datadog.communication.BackendApi; +import datadog.communication.HttpResponseException; +import datadog.communication.http.OkHttpUtils; +import datadog.communication.util.IOThrowingFunction; +import java.io.IOException; +import java.io.InputStream; +import java.net.ConnectException; +import java.net.SocketException; +import java.net.SocketTimeoutException; +import java.util.ArrayList; +import java.util.List; +import javax.annotation.Nullable; +import okhttp3.MediaType; +import okhttp3.RequestBody; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +class AgentlessExposureBackendApiTest { + + @ParameterizedTest + @ValueSource(ints = {403, 404, 405}) + void replaysRejectedBatchDirectlyAndKeepsDirectRoute(final int statusCode) throws Exception { + final RecordingBackendApi local = + new RecordingBackendApi(new HttpResponseException(statusCode, "rejected")); + final RecordingBackendApi direct = new RecordingBackendApi(); + final AgentlessExposureBackendApi api = new AgentlessExposureBackendApi(local, direct); + final RequestBody firstBody = requestBody("first"); + final RequestBody secondBody = requestBody("second"); + + api.post("exposures", firstBody, stream -> null, null, false); + api.post("exposures", secondBody, stream -> null, null, false); + + assertEquals(1, local.calls); + assertEquals(2, direct.calls); + assertSame(firstBody, local.requestBodies.get(0)); + assertSame(firstBody, direct.requestBodies.get(0)); + assertSame(secondBody, direct.requestBodies.get(1)); + } + + @Test + void fallsBackAfterConnectionRefusal() throws Exception { + final RecordingBackendApi local = + new RecordingBackendApi(new ConnectException("connection refused")); + final RecordingBackendApi direct = new RecordingBackendApi(); + final AgentlessExposureBackendApi api = new AgentlessExposureBackendApi(local, direct); + + api.post("exposures", requestBody("exposure"), stream -> null, null, false); + + assertEquals(1, local.calls); + assertEquals(1, direct.calls); + } + + @ParameterizedTest + @ValueSource(ints = {429, 500}) + void doesNotReplayAmbiguousHttpFailure(final int statusCode) { + assertNoDirectReplay(new HttpResponseException(statusCode, "ambiguous")); + } + + @Test + void doesNotReplayTimeout() { + assertNoDirectReplay(new SocketTimeoutException("timed out")); + } + + @Test + void doesNotReplayConnectionReset() { + assertNoDirectReplay(new SocketException("connection reset")); + } + + private static void assertNoDirectReplay(final IOException failure) { + final RecordingBackendApi local = new RecordingBackendApi(failure); + final RecordingBackendApi direct = new RecordingBackendApi(); + final AgentlessExposureBackendApi api = new AgentlessExposureBackendApi(local, direct); + + assertThrows( + IOException.class, + () -> api.post("exposures", requestBody("exposure"), stream -> null, null, false)); + + assertEquals(1, local.calls); + assertEquals(0, direct.calls); + } + + private static RequestBody requestBody(final String value) { + return RequestBody.create(MediaType.parse("application/json"), value); + } + + private static final class RecordingBackendApi implements BackendApi { + private final IOException failure; + private final List requestBodies = new ArrayList<>(); + private int calls; + + private RecordingBackendApi() { + this(null); + } + + private RecordingBackendApi(@Nullable final IOException failure) { + this.failure = failure; + } + + @Override + public T post( + final String uri, + final RequestBody requestBody, + final IOThrowingFunction responseParser, + @Nullable final OkHttpUtils.CustomListener requestListener, + final boolean requestCompression) + throws IOException { + calls++; + requestBodies.add(requestBody); + if (failure != null) { + throw failure; + } + return null; + } + } +} diff --git a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/ExposureBackendApiFactoryTest.java b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/ExposureBackendApiFactoryTest.java new file mode 100644 index 00000000000..602a3219ef3 --- /dev/null +++ b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/ExposureBackendApiFactoryTest.java @@ -0,0 +1,103 @@ +package com.datadog.featureflag; + +import static datadog.trace.api.featureflag.config.FeatureFlaggingConfig.CONFIGURATION_SOURCE_AGENTLESS; +import static datadog.trace.api.featureflag.config.FeatureFlaggingConfig.CONFIGURATION_SOURCE_REMOTE_CONFIG; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import datadog.communication.BackendApi; +import datadog.communication.BackendApiFactory; +import datadog.trace.api.Config; +import datadog.trace.api.intake.Intake; +import org.junit.jupiter.api.Test; + +class ExposureBackendApiFactoryTest { + + @Test + void remoteConfigUsesOnlyLocalEvpProxy() { + final Config config = config(CONFIGURATION_SOURCE_REMOTE_CONFIG, "api-key"); + final BackendApiFactory backendApiFactory = mock(BackendApiFactory.class); + final BackendApi localApi = mock(BackendApi.class); + when(backendApiFactory.createEvpProxyApi(Intake.EVENT_PLATFORM)).thenReturn(localApi); + + final BackendApi selected = new ExposureBackendApiFactory(config, backendApiFactory).create(); + + assertSame(localApi, selected); + verify(backendApiFactory, never()).createDirectIntakeApi(Intake.EVENT_PLATFORM); + } + + @Test + void agentlessPrefersLocalEvpProxyWithDirectFallback() { + final Config config = config(CONFIGURATION_SOURCE_AGENTLESS, "api-key"); + final BackendApiFactory backendApiFactory = mock(BackendApiFactory.class); + when(backendApiFactory.createEvpProxyApi(Intake.EVENT_PLATFORM)) + .thenReturn(mock(BackendApi.class)); + when(backendApiFactory.createDirectIntakeApi(Intake.EVENT_PLATFORM)) + .thenReturn(mock(BackendApi.class)); + + final BackendApi selected = new ExposureBackendApiFactory(config, backendApiFactory).create(); + + assertInstanceOf(AgentlessExposureBackendApi.class, selected); + } + + @Test + void agentlessUsesDirectIntakeWhenLocalEvpProxyIsUnavailable() { + final Config config = config(CONFIGURATION_SOURCE_AGENTLESS, "api-key"); + final BackendApiFactory backendApiFactory = mock(BackendApiFactory.class); + final BackendApi directApi = mock(BackendApi.class); + when(backendApiFactory.createDirectIntakeApi(Intake.EVENT_PLATFORM)).thenReturn(directApi); + + final BackendApi selected = new ExposureBackendApiFactory(config, backendApiFactory).create(); + + assertSame(directApi, selected); + } + + @Test + void agentlessUsesLocalEvpProxyWhenApiKeyIsUnavailable() { + final Config config = config(CONFIGURATION_SOURCE_AGENTLESS, null); + final BackendApiFactory backendApiFactory = mock(BackendApiFactory.class); + final BackendApi localApi = mock(BackendApi.class); + when(backendApiFactory.createEvpProxyApi(Intake.EVENT_PLATFORM)).thenReturn(localApi); + + final BackendApi selected = new ExposureBackendApiFactory(config, backendApiFactory).create(); + + assertSame(localApi, selected); + verify(backendApiFactory, never()).createDirectIntakeApi(Intake.EVENT_PLATFORM); + } + + @Test + void agentlessDisablesDeliveryWhenNoRouteIsAvailable() { + final Config config = config(CONFIGURATION_SOURCE_AGENTLESS, null); + final BackendApiFactory backendApiFactory = mock(BackendApiFactory.class); + + final BackendApi selected = new ExposureBackendApiFactory(config, backendApiFactory).create(); + + assertNull(selected); + } + + @Test + void agentlessKeepsLocalRouteWhenDirectUrlIsInvalid() { + final Config config = config(CONFIGURATION_SOURCE_AGENTLESS, "api-key"); + final BackendApiFactory backendApiFactory = mock(BackendApiFactory.class); + final BackendApi localApi = mock(BackendApi.class); + when(backendApiFactory.createEvpProxyApi(Intake.EVENT_PLATFORM)).thenReturn(localApi); + when(backendApiFactory.createDirectIntakeApi(Intake.EVENT_PLATFORM)) + .thenThrow(new IllegalArgumentException("invalid URL")); + + final BackendApi selected = new ExposureBackendApiFactory(config, backendApiFactory).create(); + + assertSame(localApi, selected); + } + + private static Config config(final String source, final String apiKey) { + final Config config = mock(Config.class); + when(config.getFeatureFlaggingConfigurationSource()).thenReturn(source); + when(config.getApiKey()).thenReturn(apiKey); + return config; + } +} diff --git a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/ExposureWriterTests.java b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/ExposureWriterTests.java index 76b9e2602d8..cfd65cceb2c 100644 --- a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/ExposureWriterTests.java +++ b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/ExposureWriterTests.java @@ -1,5 +1,6 @@ package com.datadog.featureflag; +import static datadog.trace.api.featureflag.config.FeatureFlaggingConfig.CONFIGURATION_SOURCE_AGENTLESS; import static java.util.Collections.singletonList; import static java.util.Collections.singletonMap; import static java.util.concurrent.TimeUnit.MILLISECONDS; @@ -13,8 +14,11 @@ import com.squareup.moshi.JsonAdapter; import com.squareup.moshi.Moshi; +import datadog.communication.BackendApiFactory; +import datadog.communication.IntakeApi; import datadog.communication.ddagent.DDAgentFeaturesDiscovery; import datadog.communication.ddagent.SharedCommunicationObjects; +import datadog.communication.http.HttpRetryPolicy; import datadog.trace.agent.test.server.http.JavaTestHttpServer; import datadog.trace.agent.test.server.http.JavaTestHttpServer.HandlerApi; import datadog.trace.api.Config; @@ -57,6 +61,8 @@ class ExposureWriterTests { private static final String EXPOSURES_ENDPOINT = "/evp_proxy/api/v2/exposures"; + private static final String DIRECT_EXPOSURES_ENDPOINT = "/api/v2/exposures"; + private static final String API_KEY = "test-api-key"; private static final double TIMEOUT_SECONDS = 5; private final PollingConditions poll = new PollingConditions(TIMEOUT_SECONDS); @@ -75,7 +81,11 @@ void setUp() { JavaTestHttpServer.httpServer( s -> s.handlers( - h -> h.prefix(EXPOSURES_ENDPOINT, api -> handleExposureRequest(api, adapter)))); + h -> { + h.prefix(EXPOSURES_ENDPOINT, api -> handleExposureRequest(api, adapter)); + h.prefix( + DIRECT_EXPOSURES_ENDPOINT, api -> handleExposureRequest(api, adapter)); + })); sharedCommunicationObjects = sharedCommunicationObjects(true); } @@ -131,6 +141,43 @@ void testExposureEventWrites(String service, String env, String version) throws } } + @Test + void testAgentlessExposureEventWritesDirectlyWithApiKey() throws Exception { + Config config = mockConfig("test-service"); + when(config.getFeatureFlaggingConfigurationSource()).thenReturn(CONFIGURATION_SOURCE_AGENTLESS); + when(config.getApiKey()).thenReturn(API_KEY); + BackendApiFactory backendApiFactory = mock(BackendApiFactory.class); + IntakeApi directApi = + new IntakeApi( + HttpUrl.get(server.getAddress()).resolve("/api/v2/"), + API_KEY, + "123", + HttpRetryPolicy.Factory.NEVER_RETRY, + new OkHttpClient.Builder().build(), + false); + when(backendApiFactory.createDirectIntakeApi(datadog.trace.api.intake.Intake.EVENT_PLATFORM)) + .thenReturn(directApi); + ExposureBackendApiFactory exposureBackendApiFactory = + new ExposureBackendApiFactory(config, backendApiFactory); + List exposures = buildExposures(5); + + try (ExposureWriterImpl writer = + new ExposureWriterImpl(1 << 4, 100, MILLISECONDS, exposureBackendApiFactory, config)) { + writer.init(); + for (ExposureEvent exposure : exposures) { + writer.accept(exposure); + } + + poll.eventually( + () -> { + assertEquals(DIRECT_EXPOSURES_ENDPOINT, server.getLastRequest().getPath()); + assertEquals(API_KEY, server.getLastRequest().getHeader("dd-api-key")); + assertNull(server.getLastRequest().getHeader("X-Datadog-EVP-Subdomain")); + assertExposures(allExposures(), exposures); + }); + } + } + @Test void testLruCache() throws Exception { Config config = mockConfig("test-service"); From 085ebc53fbfbbabcb90a841806929bfdf71557d0 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Tue, 11 Aug 2026 20:21:14 -0400 Subject: [PATCH 02/14] Prepare exposure delivery before agentless activation --- .../featureflag/FeatureFlaggingSystem.java | 47 ++++++++++++++++++- .../FeatureFlaggingSystemTest.java | 6 ++- 2 files changed, 49 insertions(+), 4 deletions(-) diff --git a/products/feature-flagging/feature-flagging-agent/src/main/java/com/datadog/featureflag/FeatureFlaggingSystem.java b/products/feature-flagging/feature-flagging-agent/src/main/java/com/datadog/featureflag/FeatureFlaggingSystem.java index 91b32ee1d64..2795c743b2e 100644 --- a/products/feature-flagging/feature-flagging-agent/src/main/java/com/datadog/featureflag/FeatureFlaggingSystem.java +++ b/products/feature-flagging/feature-flagging-agent/src/main/java/com/datadog/featureflag/FeatureFlaggingSystem.java @@ -41,6 +41,12 @@ public static synchronized void start(final SharedCommunicationObjects sco) { } if (CONFIGURATION_SOURCE_AGENTLESS.equals(config.getFeatureFlaggingConfigurationSource())) { + try { + initializeExposureWriter(sco, config); + } catch (final RuntimeException | Error e) { + STARTED = false; + throw e; + } final FeatureFlaggingGateway.ActivationListener activationListener = () -> activateAgentless(sco, config); ACTIVATION_LISTENER = activationListener; @@ -79,8 +85,13 @@ private static void initializeSystem(final SharedCommunicationObjects sco, final LOGGER.debug("Feature Flagging system disabled by unsupported configuration source"); return; } - final ExposureWriter exposureWriter = new ExposureWriterImpl(sco, config); - initialize(configService, exposureWriter); + final ExposureWriter exposureWriter = EXPOSURE_WRITER; + if (exposureWriter == null) { + final ExposureWriter newExposureWriter = new ExposureWriterImpl(sco, config); + initialize(configService, newExposureWriter); + } else { + initializeConfigurationSource(configService, exposureWriter); + } // APM span enrichment: agent-side listener for flag-evaluation seam events. Uses the process- // wide singleton so a subsystem restart reuses the one already-registered trace interceptor @@ -93,6 +104,34 @@ private static void initializeSystem(final SharedCommunicationObjects sco, final LOGGER.debug("Feature Flagging system started"); } + private static void initializeExposureWriter( + final SharedCommunicationObjects sco, final Config config) { + final ExposureWriter exposureWriter = new ExposureWriterImpl(sco, config); + try { + exposureWriter.init(); + EXPOSURE_WRITER = exposureWriter; + } catch (final RuntimeException | Error e) { + exposureWriter.close(); + throw e; + } + } + + private static void initializeConfigurationSource( + final ConfigurationSourceService configService, final ExposureWriter exposureWriter) { + try { + configService.init(); + CONFIG_SERVICE = configService; + } catch (final RuntimeException | Error e) { + EXPOSURE_WRITER = null; + try { + exposureWriter.close(); + } finally { + configService.close(); + } + throw e; + } + } + static void initialize( final ConfigurationSourceService configService, final ExposureWriter exposureWriter) { try { @@ -167,4 +206,8 @@ public static synchronized void stop() { static boolean isAwaitingApplicationActivation() { return ACTIVATION_LISTENER != null; } + + static boolean isExposureWriterStarted() { + return EXPOSURE_WRITER != null; + } } diff --git a/products/feature-flagging/feature-flagging-agent/src/test/java/com/datadog/featureflag/FeatureFlaggingSystemTest.java b/products/feature-flagging/feature-flagging-agent/src/test/java/com/datadog/featureflag/FeatureFlaggingSystemTest.java index d408d91da4e..8d18f752e8c 100644 --- a/products/feature-flagging/feature-flagging-agent/src/test/java/com/datadog/featureflag/FeatureFlaggingSystemTest.java +++ b/products/feature-flagging/feature-flagging-agent/src/test/java/com/datadog/featureflag/FeatureFlaggingSystemTest.java @@ -38,7 +38,7 @@ class FeatureFlaggingSystemTest { @WithConfig( key = FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_BASE_URL, value = "http://127.0.0.1:1") - void agentlessStartWaitsForApplicationProviderActivation() { + void agentlessStartPreparesExposureDeliveryAndWaitsForApplicationProviderActivation() { SharedCommunicationObjects sharedCommunicationObjects = sharedCommunicationObjects(); clearInvocations(sharedCommunicationObjects); @@ -46,7 +46,7 @@ void agentlessStartWaitsForApplicationProviderActivation() { FeatureFlaggingSystem.start(sharedCommunicationObjects); assertTrue(FeatureFlaggingSystem.isAwaitingApplicationActivation()); - verifyNoInteractions(sharedCommunicationObjects); + assertTrue(FeatureFlaggingSystem.isExposureWriterStarted()); FeatureFlaggingGateway.activate(); @@ -56,6 +56,7 @@ void agentlessStartWaitsForApplicationProviderActivation() { } assertFalse(FeatureFlaggingSystem.isAwaitingApplicationActivation()); + assertFalse(FeatureFlaggingSystem.isExposureWriterStarted()); } @Test @@ -72,6 +73,7 @@ void agentlessStopRemovesPendingApplicationProviderActivation() { assertTrue(FeatureFlaggingSystem.isAwaitingApplicationActivation()); FeatureFlaggingSystem.stop(); + clearInvocations(sharedCommunicationObjects); FeatureFlaggingGateway.activate(); assertFalse(FeatureFlaggingSystem.isAwaitingApplicationActivation()); From 9992388df3e12fb18b062b9897cb68cf36e53bdc Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Tue, 11 Aug 2026 20:33:21 -0400 Subject: [PATCH 03/14] Assert lazy agentless configuration startup --- .../java/com/datadog/featureflag/FeatureFlaggingSystem.java | 4 ++++ .../com/datadog/featureflag/FeatureFlaggingSystemTest.java | 3 +++ 2 files changed, 7 insertions(+) diff --git a/products/feature-flagging/feature-flagging-agent/src/main/java/com/datadog/featureflag/FeatureFlaggingSystem.java b/products/feature-flagging/feature-flagging-agent/src/main/java/com/datadog/featureflag/FeatureFlaggingSystem.java index 2795c743b2e..0d1e9147af5 100644 --- a/products/feature-flagging/feature-flagging-agent/src/main/java/com/datadog/featureflag/FeatureFlaggingSystem.java +++ b/products/feature-flagging/feature-flagging-agent/src/main/java/com/datadog/featureflag/FeatureFlaggingSystem.java @@ -210,4 +210,8 @@ static boolean isAwaitingApplicationActivation() { static boolean isExposureWriterStarted() { return EXPOSURE_WRITER != null; } + + static boolean isConfigurationSourceStarted() { + return CONFIG_SERVICE != null; + } } diff --git a/products/feature-flagging/feature-flagging-agent/src/test/java/com/datadog/featureflag/FeatureFlaggingSystemTest.java b/products/feature-flagging/feature-flagging-agent/src/test/java/com/datadog/featureflag/FeatureFlaggingSystemTest.java index 8d18f752e8c..40b1f59136c 100644 --- a/products/feature-flagging/feature-flagging-agent/src/test/java/com/datadog/featureflag/FeatureFlaggingSystemTest.java +++ b/products/feature-flagging/feature-flagging-agent/src/test/java/com/datadog/featureflag/FeatureFlaggingSystemTest.java @@ -47,16 +47,19 @@ void agentlessStartPreparesExposureDeliveryAndWaitsForApplicationProviderActivat assertTrue(FeatureFlaggingSystem.isAwaitingApplicationActivation()); assertTrue(FeatureFlaggingSystem.isExposureWriterStarted()); + assertFalse(FeatureFlaggingSystem.isConfigurationSourceStarted()); FeatureFlaggingGateway.activate(); assertFalse(FeatureFlaggingSystem.isAwaitingApplicationActivation()); + assertTrue(FeatureFlaggingSystem.isConfigurationSourceStarted()); } finally { FeatureFlaggingSystem.stop(); } assertFalse(FeatureFlaggingSystem.isAwaitingApplicationActivation()); assertFalse(FeatureFlaggingSystem.isExposureWriterStarted()); + assertFalse(FeatureFlaggingSystem.isConfigurationSourceStarted()); } @Test From 37165945d5324cef95a58254d39e52eda7035047 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Tue, 11 Aug 2026 20:52:57 -0400 Subject: [PATCH 04/14] log message Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- .../src/main/java/datadog/communication/BackendApiFactory.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/communication/src/main/java/datadog/communication/BackendApiFactory.java b/communication/src/main/java/datadog/communication/BackendApiFactory.java index 5b7d92b29a0..0f2a9c9fac1 100644 --- a/communication/src/main/java/datadog/communication/BackendApiFactory.java +++ b/communication/src/main/java/datadog/communication/BackendApiFactory.java @@ -43,7 +43,7 @@ public BackendApi createDirectIntakeApi(Intake intake) { String apiKey = config.getApiKey(); if (apiKey == null || apiKey.isEmpty()) { throw new FatalAgentMisconfigurationError( - "Agentless mode is enabled and api key is not set. Please set application key"); + "Agentless mode is enabled and API key is not set. Please set DD_API_KEY"); } String traceId = config.getIdGenerationStrategy().generateTraceId().toString(); return new IntakeApi( From 44c48964769f483cf6aa833eacb5808ff45bb9ec Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Thu, 13 Aug 2026 11:06:38 -0400 Subject: [PATCH 05/14] feat(openfeature): add direct flagevaluation fallback --- ...va => AgentlessFeatureFlagBackendApi.java} | 15 ++-- .../ExposureBackendApiFactory.java | 72 --------------- .../featureflag/ExposureWriterImpl.java | 11 ++- .../FeatureFlagBackendApiFactory.java | 90 +++++++++++++++++++ .../featureflag/FlagEvaluationWriterImpl.java | 82 ++++++++++++++++- ...> AgentlessFeatureFlagBackendApiTest.java} | 19 ++-- .../featureflag/ExposureWriterTests.java | 7 +- ... => FeatureFlagBackendApiFactoryTest.java} | 47 ++++++---- .../FlagEvaluationWriterImplTest.java | 59 ++++++++++++ 9 files changed, 290 insertions(+), 112 deletions(-) rename products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/{AgentlessExposureBackendApi.java => AgentlessFeatureFlagBackendApi.java} (76%) delete mode 100644 products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/ExposureBackendApiFactory.java create mode 100644 products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FeatureFlagBackendApiFactory.java rename products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/{AgentlessExposureBackendApiTest.java => AgentlessFeatureFlagBackendApiTest.java} (82%) rename products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/{ExposureBackendApiFactoryTest.java => FeatureFlagBackendApiFactoryTest.java} (73%) diff --git a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/AgentlessExposureBackendApi.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/AgentlessFeatureFlagBackendApi.java similarity index 76% rename from products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/AgentlessExposureBackendApi.java rename to products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/AgentlessFeatureFlagBackendApi.java index 97c798f0e1d..663f99d49a5 100644 --- a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/AgentlessExposureBackendApi.java +++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/AgentlessFeatureFlagBackendApi.java @@ -12,18 +12,22 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -/** Sends exposures through a local EVP proxy, with a safe direct intake fallback. */ -final class AgentlessExposureBackendApi implements BackendApi { +/** Sends Feature Flag events through a local EVP proxy, with a safe direct intake fallback. */ +final class AgentlessFeatureFlagBackendApi implements BackendApi { - private static final Logger LOGGER = LoggerFactory.getLogger(AgentlessExposureBackendApi.class); + private static final Logger LOGGER = + LoggerFactory.getLogger(AgentlessFeatureFlagBackendApi.class); private final BackendApi localApi; private final BackendApi directApi; + private final String eventType; private volatile BackendApi activeApi; - AgentlessExposureBackendApi(final BackendApi localApi, final BackendApi directApi) { + AgentlessFeatureFlagBackendApi( + final BackendApi localApi, final BackendApi directApi, final String eventType) { this.localApi = localApi; this.directApi = directApi; + this.eventType = eventType; this.activeApi = localApi; } @@ -46,7 +50,8 @@ public T post( if (activeApi == localApi) { LOGGER.debug( - "Switching Feature Flagging exposure delivery from the local EVP proxy to direct intake"); + "Switching Feature Flagging {} delivery from the local EVP proxy to direct intake", + eventType); activeApi = directApi; } return directApi.post(uri, requestBody, responseParser, requestListener, requestCompression); diff --git a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/ExposureBackendApiFactory.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/ExposureBackendApiFactory.java deleted file mode 100644 index f0eb1fa0e3d..00000000000 --- a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/ExposureBackendApiFactory.java +++ /dev/null @@ -1,72 +0,0 @@ -package com.datadog.featureflag; - -import static datadog.trace.api.featureflag.config.FeatureFlaggingConfig.CONFIGURATION_SOURCE_AGENTLESS; - -import datadog.communication.BackendApi; -import datadog.communication.BackendApiFactory; -import datadog.communication.ddagent.SharedCommunicationObjects; -import datadog.trace.api.Config; -import datadog.trace.api.intake.Intake; -import javax.annotation.Nullable; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** Selects the transport for Feature Flagging exposure events. */ -final class ExposureBackendApiFactory { - - private static final Logger LOGGER = LoggerFactory.getLogger(ExposureBackendApiFactory.class); - - private final Config config; - private final BackendApiFactory backendApiFactory; - - ExposureBackendApiFactory( - final Config config, final SharedCommunicationObjects sharedCommunicationObjects) { - this(config, new BackendApiFactory(config, sharedCommunicationObjects)); - } - - ExposureBackendApiFactory(final Config config, final BackendApiFactory backendApiFactory) { - this.config = config; - this.backendApiFactory = backendApiFactory; - } - - @Nullable - BackendApi create() { - final BackendApi localApi = backendApiFactory.createEvpProxyApi(Intake.EVENT_PLATFORM); - if (!CONFIGURATION_SOURCE_AGENTLESS.equals(config.getFeatureFlaggingConfigurationSource())) { - if (localApi == null) { - LOGGER.warn( - "Feature Flagging exposure delivery is disabled because the local Agent does not support the EVP proxy"); - } - return localApi; - } - - final BackendApi directApi = createDirectApi(); - if (localApi != null && directApi != null) { - return new AgentlessExposureBackendApi(localApi, directApi); - } - if (localApi != null) { - return localApi; - } - if (directApi != null) { - return directApi; - } - - LOGGER.warn( - "Feature Flagging exposure delivery is disabled because no compatible local EVP proxy or direct intake credentials are available"); - return null; - } - - @Nullable - private BackendApi createDirectApi() { - final String apiKey = config.getApiKey(); - if (apiKey == null || apiKey.isEmpty()) { - return null; - } - try { - return backendApiFactory.createDirectIntakeApi(Intake.EVENT_PLATFORM); - } catch (final IllegalArgumentException exception) { - LOGGER.debug("Cannot configure direct Feature Flagging exposure delivery", exception); - return null; - } - } -} diff --git a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/ExposureWriterImpl.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/ExposureWriterImpl.java index 928c2da6681..7b8ca052feb 100644 --- a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/ExposureWriterImpl.java +++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/ExposureWriterImpl.java @@ -41,14 +41,19 @@ public ExposureWriterImpl(final SharedCommunicationObjects sco, final Config con final TimeUnit timeUnit, final SharedCommunicationObjects sco, final Config config) { - this(capacity, flushInterval, timeUnit, new ExposureBackendApiFactory(config, sco), config); + this( + capacity, + flushInterval, + timeUnit, + new FeatureFlagBackendApiFactory(config, sco, "exposure", true), + config); } ExposureWriterImpl( final int capacity, final long flushInterval, final TimeUnit timeUnit, - final ExposureBackendApiFactory backendApiFactory, + final FeatureFlagBackendApiFactory backendApiFactory, final Config config) { this.queue = Queues.mpscBlockingConsumerArrayQueue(capacity); final ExposureSerializingHandler serializer = @@ -104,7 +109,7 @@ private static class ExposureSerializingHandler implements Runnable { private final Runnable errorCallback; ExposureSerializingHandler( - final ExposureBackendApiFactory backendApiFactory, + final FeatureFlagBackendApiFactory backendApiFactory, final MessagePassingBlockingQueue queue, final long flushInterval, final TimeUnit timeUnit, diff --git a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FeatureFlagBackendApiFactory.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FeatureFlagBackendApiFactory.java new file mode 100644 index 00000000000..5c15198570e --- /dev/null +++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FeatureFlagBackendApiFactory.java @@ -0,0 +1,90 @@ +package com.datadog.featureflag; + +import static datadog.trace.api.featureflag.config.FeatureFlaggingConfig.CONFIGURATION_SOURCE_AGENTLESS; + +import datadog.communication.BackendApi; +import datadog.communication.BackendApiFactory; +import datadog.communication.ddagent.SharedCommunicationObjects; +import datadog.trace.api.Config; +import datadog.trace.api.intake.Intake; +import javax.annotation.Nullable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** Selects the transport for Feature Flagging events. */ +final class FeatureFlagBackendApiFactory { + + private static final Logger LOGGER = LoggerFactory.getLogger(FeatureFlagBackendApiFactory.class); + + private final Config config; + private final BackendApiFactory backendApiFactory; + private final String eventType; + private final boolean responseCompression; + + FeatureFlagBackendApiFactory( + final Config config, + final SharedCommunicationObjects sharedCommunicationObjects, + final String eventType, + final boolean responseCompression) { + this( + config, + new BackendApiFactory(config, sharedCommunicationObjects), + eventType, + responseCompression); + } + + FeatureFlagBackendApiFactory( + final Config config, + final BackendApiFactory backendApiFactory, + final String eventType, + final boolean responseCompression) { + this.config = config; + this.backendApiFactory = backendApiFactory; + this.eventType = eventType; + this.responseCompression = responseCompression; + } + + @Nullable + BackendApi create() { + final BackendApi localApi = + backendApiFactory.createEvpProxyApi(Intake.EVENT_PLATFORM, responseCompression); + if (!CONFIGURATION_SOURCE_AGENTLESS.equals(config.getFeatureFlaggingConfigurationSource())) { + if (localApi == null) { + LOGGER.warn( + "Feature Flagging {} delivery is disabled because the local Agent does not support the EVP proxy", + eventType); + } + return localApi; + } + + final BackendApi directApi = createDirectApi(); + if (localApi != null && directApi != null) { + return new AgentlessFeatureFlagBackendApi(localApi, directApi, eventType); + } + if (localApi != null) { + return localApi; + } + if (directApi != null) { + return directApi; + } + + LOGGER.warn( + "Feature Flagging {} delivery is disabled because no compatible local EVP proxy or direct intake credentials are available", + eventType); + return null; + } + + @Nullable + private BackendApi createDirectApi() { + final String apiKey = config.getApiKey(); + if (apiKey == null || apiKey.isEmpty()) { + return null; + } + try { + return backendApiFactory.createDirectIntakeApi(Intake.EVENT_PLATFORM, responseCompression); + } catch (final IllegalArgumentException exception) { + LOGGER.debug("Cannot configure direct Feature Flagging {} delivery", eventType, exception); + return null; + } + } +} diff --git a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FlagEvaluationWriterImpl.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FlagEvaluationWriterImpl.java index e15666aa10a..64884b78b8e 100644 --- a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FlagEvaluationWriterImpl.java +++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FlagEvaluationWriterImpl.java @@ -6,6 +6,7 @@ import datadog.common.queue.MessagePassingBlockingQueue; import datadog.common.queue.Queues; +import datadog.communication.BackendApi; import datadog.communication.BackendApiFactory; import datadog.communication.EvpProxy; import datadog.communication.ddagent.SharedCommunicationObjects; @@ -13,6 +14,7 @@ import datadog.trace.api.featureflag.FeatureFlaggingGateway; import datadog.trace.api.featureflag.flagevaluation.FlagEvalEvent; import datadog.trace.api.featureflag.flagevaluation.FlagEvaluationWriter; +import datadog.trace.api.intake.Intake; import datadog.trace.api.telemetry.CoreMetricCollector; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import java.util.ArrayList; @@ -23,6 +25,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Supplier; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -108,7 +111,12 @@ public FlagEvaluationWriterImpl(final SharedCommunicationObjects sco, final Conf final TimeUnit timeUnit, final SharedCommunicationObjects sco, final Config config) { - this(capacity, flushInterval, timeUnit, new BackendApiFactory(config, sco), config); + this( + capacity, + flushInterval, + timeUnit, + new FeatureFlagBackendApiFactory(config, sco, "flag evaluation", false), + config); } /** Package-private constructor allowing a BackendApiFactory to be injected for tests. */ @@ -118,10 +126,33 @@ public FlagEvaluationWriterImpl(final SharedCommunicationObjects sco, final Conf final TimeUnit timeUnit, final BackendApiFactory backendApiFactory, final Config config) { + this( + capacity, + flushInterval, + timeUnit, + () -> backendApiFactory.createBackendApi(Intake.EVENT_PLATFORM, false), + config); + } + + FlagEvaluationWriterImpl( + final int capacity, + final long flushInterval, + final TimeUnit timeUnit, + final FeatureFlagBackendApiFactory backendApiFactory, + final Config config) { + this(capacity, flushInterval, timeUnit, backendApiFactory::create, config); + } + + private FlagEvaluationWriterImpl( + final int capacity, + final long flushInterval, + final TimeUnit timeUnit, + final Supplier backendApiSupplier, + final Config config) { this.queue = Queues.mpscBlockingConsumerArrayQueue(capacity); this.serializer = new FlagEvaluationSerializingHandler( - backendApiFactory, + backendApiSupplier, queue, flushInterval, timeUnit, @@ -318,7 +349,7 @@ static class FlagEvaluationSerializingHandler implements Runnable { final ConcurrentHashMap contextTruncatedCounts, final Runnable errorCallback) { this( - backendApiFactory, + () -> backendApiFactory.createBackendApi(Intake.EVENT_PLATFORM, false), queue, flushInterval, timeUnit, @@ -339,10 +370,53 @@ static class FlagEvaluationSerializingHandler implements Runnable { final ConcurrentHashMap contextTruncatedCounts, final Runnable errorCallback, final int payloadSizeLimitBytes) { + this( + () -> backendApiFactory.createBackendApi(Intake.EVENT_PLATFORM, false), + queue, + flushInterval, + timeUnit, + context, + droppedQueueOverflow, + contextTruncatedCounts, + errorCallback, + payloadSizeLimitBytes); + } + + FlagEvaluationSerializingHandler( + final Supplier backendApiSupplier, + final MessagePassingBlockingQueue queue, + final long flushInterval, + final TimeUnit timeUnit, + final Map context, + final AtomicLong droppedQueueOverflow, + final ConcurrentHashMap contextTruncatedCounts, + final Runnable errorCallback) { + this( + backendApiSupplier, + queue, + flushInterval, + timeUnit, + context, + droppedQueueOverflow, + contextTruncatedCounts, + errorCallback, + FLAG_EVALUATION_PAYLOAD_SIZE_LIMIT_BYTES); + } + + FlagEvaluationSerializingHandler( + final Supplier backendApiSupplier, + final MessagePassingBlockingQueue queue, + final long flushInterval, + final TimeUnit timeUnit, + final Map context, + final AtomicLong droppedQueueOverflow, + final ConcurrentHashMap contextTruncatedCounts, + final Runnable errorCallback, + final int payloadSizeLimitBytes) { this.queue = queue; this.evpPublisher = new FeatureFlagEvpPublisher<>( - backendApiFactory, FlagEvaluationPayloads.FlagEvaluationsRequest.class, false); + backendApiSupplier, FlagEvaluationPayloads.FlagEvaluationsRequest.class); this.context = context; this.droppedQueueOverflow = droppedQueueOverflow; this.contextTruncatedCounts = contextTruncatedCounts; diff --git a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessExposureBackendApiTest.java b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessFeatureFlagBackendApiTest.java similarity index 82% rename from products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessExposureBackendApiTest.java rename to products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessFeatureFlagBackendApiTest.java index 9455634ec45..0a2403ec25d 100644 --- a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessExposureBackendApiTest.java +++ b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessFeatureFlagBackendApiTest.java @@ -22,7 +22,7 @@ import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; -class AgentlessExposureBackendApiTest { +class AgentlessFeatureFlagBackendApiTest { @ParameterizedTest @ValueSource(ints = {403, 404, 405}) @@ -30,12 +30,13 @@ void replaysRejectedBatchDirectlyAndKeepsDirectRoute(final int statusCode) throw final RecordingBackendApi local = new RecordingBackendApi(new HttpResponseException(statusCode, "rejected")); final RecordingBackendApi direct = new RecordingBackendApi(); - final AgentlessExposureBackendApi api = new AgentlessExposureBackendApi(local, direct); + final AgentlessFeatureFlagBackendApi api = + new AgentlessFeatureFlagBackendApi(local, direct, "flag evaluation"); final RequestBody firstBody = requestBody("first"); final RequestBody secondBody = requestBody("second"); - api.post("exposures", firstBody, stream -> null, null, false); - api.post("exposures", secondBody, stream -> null, null, false); + api.post("flagevaluation", firstBody, stream -> null, null, false); + api.post("flagevaluation", secondBody, stream -> null, null, false); assertEquals(1, local.calls); assertEquals(2, direct.calls); @@ -49,9 +50,10 @@ void fallsBackAfterConnectionRefusal() throws Exception { final RecordingBackendApi local = new RecordingBackendApi(new ConnectException("connection refused")); final RecordingBackendApi direct = new RecordingBackendApi(); - final AgentlessExposureBackendApi api = new AgentlessExposureBackendApi(local, direct); + final AgentlessFeatureFlagBackendApi api = + new AgentlessFeatureFlagBackendApi(local, direct, "flag evaluation"); - api.post("exposures", requestBody("exposure"), stream -> null, null, false); + api.post("flagevaluation", requestBody("evaluation"), stream -> null, null, false); assertEquals(1, local.calls); assertEquals(1, direct.calls); @@ -76,11 +78,12 @@ void doesNotReplayConnectionReset() { private static void assertNoDirectReplay(final IOException failure) { final RecordingBackendApi local = new RecordingBackendApi(failure); final RecordingBackendApi direct = new RecordingBackendApi(); - final AgentlessExposureBackendApi api = new AgentlessExposureBackendApi(local, direct); + final AgentlessFeatureFlagBackendApi api = + new AgentlessFeatureFlagBackendApi(local, direct, "flag evaluation"); assertThrows( IOException.class, - () -> api.post("exposures", requestBody("exposure"), stream -> null, null, false)); + () -> api.post("flagevaluation", requestBody("evaluation"), stream -> null, null, false)); assertEquals(1, local.calls); assertEquals(0, direct.calls); diff --git a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/ExposureWriterTests.java b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/ExposureWriterTests.java index cfd65cceb2c..cc2eb5bb5c5 100644 --- a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/ExposureWriterTests.java +++ b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/ExposureWriterTests.java @@ -155,10 +155,11 @@ void testAgentlessExposureEventWritesDirectlyWithApiKey() throws Exception { HttpRetryPolicy.Factory.NEVER_RETRY, new OkHttpClient.Builder().build(), false); - when(backendApiFactory.createDirectIntakeApi(datadog.trace.api.intake.Intake.EVENT_PLATFORM)) + when(backendApiFactory.createDirectIntakeApi( + datadog.trace.api.intake.Intake.EVENT_PLATFORM, true)) .thenReturn(directApi); - ExposureBackendApiFactory exposureBackendApiFactory = - new ExposureBackendApiFactory(config, backendApiFactory); + FeatureFlagBackendApiFactory exposureBackendApiFactory = + new FeatureFlagBackendApiFactory(config, backendApiFactory, "exposure", true); List exposures = buildExposures(5); try (ExposureWriterImpl writer = diff --git a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/ExposureBackendApiFactoryTest.java b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FeatureFlagBackendApiFactoryTest.java similarity index 73% rename from products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/ExposureBackendApiFactoryTest.java rename to products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FeatureFlagBackendApiFactoryTest.java index 602a3219ef3..b5248fcfbda 100644 --- a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/ExposureBackendApiFactoryTest.java +++ b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FeatureFlagBackendApiFactoryTest.java @@ -16,33 +16,37 @@ import datadog.trace.api.intake.Intake; import org.junit.jupiter.api.Test; -class ExposureBackendApiFactoryTest { +class FeatureFlagBackendApiFactoryTest { @Test void remoteConfigUsesOnlyLocalEvpProxy() { final Config config = config(CONFIGURATION_SOURCE_REMOTE_CONFIG, "api-key"); final BackendApiFactory backendApiFactory = mock(BackendApiFactory.class); final BackendApi localApi = mock(BackendApi.class); - when(backendApiFactory.createEvpProxyApi(Intake.EVENT_PLATFORM)).thenReturn(localApi); + when(backendApiFactory.createEvpProxyApi(Intake.EVENT_PLATFORM, false)).thenReturn(localApi); - final BackendApi selected = new ExposureBackendApiFactory(config, backendApiFactory).create(); + final BackendApi selected = + new FeatureFlagBackendApiFactory(config, backendApiFactory, "flag evaluation", false) + .create(); assertSame(localApi, selected); - verify(backendApiFactory, never()).createDirectIntakeApi(Intake.EVENT_PLATFORM); + verify(backendApiFactory, never()).createDirectIntakeApi(Intake.EVENT_PLATFORM, false); } @Test void agentlessPrefersLocalEvpProxyWithDirectFallback() { final Config config = config(CONFIGURATION_SOURCE_AGENTLESS, "api-key"); final BackendApiFactory backendApiFactory = mock(BackendApiFactory.class); - when(backendApiFactory.createEvpProxyApi(Intake.EVENT_PLATFORM)) + when(backendApiFactory.createEvpProxyApi(Intake.EVENT_PLATFORM, false)) .thenReturn(mock(BackendApi.class)); - when(backendApiFactory.createDirectIntakeApi(Intake.EVENT_PLATFORM)) + when(backendApiFactory.createDirectIntakeApi(Intake.EVENT_PLATFORM, false)) .thenReturn(mock(BackendApi.class)); - final BackendApi selected = new ExposureBackendApiFactory(config, backendApiFactory).create(); + final BackendApi selected = + new FeatureFlagBackendApiFactory(config, backendApiFactory, "flag evaluation", false) + .create(); - assertInstanceOf(AgentlessExposureBackendApi.class, selected); + assertInstanceOf(AgentlessFeatureFlagBackendApi.class, selected); } @Test @@ -50,9 +54,12 @@ void agentlessUsesDirectIntakeWhenLocalEvpProxyIsUnavailable() { final Config config = config(CONFIGURATION_SOURCE_AGENTLESS, "api-key"); final BackendApiFactory backendApiFactory = mock(BackendApiFactory.class); final BackendApi directApi = mock(BackendApi.class); - when(backendApiFactory.createDirectIntakeApi(Intake.EVENT_PLATFORM)).thenReturn(directApi); + when(backendApiFactory.createDirectIntakeApi(Intake.EVENT_PLATFORM, false)) + .thenReturn(directApi); - final BackendApi selected = new ExposureBackendApiFactory(config, backendApiFactory).create(); + final BackendApi selected = + new FeatureFlagBackendApiFactory(config, backendApiFactory, "flag evaluation", false) + .create(); assertSame(directApi, selected); } @@ -62,12 +69,14 @@ void agentlessUsesLocalEvpProxyWhenApiKeyIsUnavailable() { final Config config = config(CONFIGURATION_SOURCE_AGENTLESS, null); final BackendApiFactory backendApiFactory = mock(BackendApiFactory.class); final BackendApi localApi = mock(BackendApi.class); - when(backendApiFactory.createEvpProxyApi(Intake.EVENT_PLATFORM)).thenReturn(localApi); + when(backendApiFactory.createEvpProxyApi(Intake.EVENT_PLATFORM, false)).thenReturn(localApi); - final BackendApi selected = new ExposureBackendApiFactory(config, backendApiFactory).create(); + final BackendApi selected = + new FeatureFlagBackendApiFactory(config, backendApiFactory, "flag evaluation", false) + .create(); assertSame(localApi, selected); - verify(backendApiFactory, never()).createDirectIntakeApi(Intake.EVENT_PLATFORM); + verify(backendApiFactory, never()).createDirectIntakeApi(Intake.EVENT_PLATFORM, false); } @Test @@ -75,7 +84,9 @@ void agentlessDisablesDeliveryWhenNoRouteIsAvailable() { final Config config = config(CONFIGURATION_SOURCE_AGENTLESS, null); final BackendApiFactory backendApiFactory = mock(BackendApiFactory.class); - final BackendApi selected = new ExposureBackendApiFactory(config, backendApiFactory).create(); + final BackendApi selected = + new FeatureFlagBackendApiFactory(config, backendApiFactory, "flag evaluation", false) + .create(); assertNull(selected); } @@ -85,11 +96,13 @@ void agentlessKeepsLocalRouteWhenDirectUrlIsInvalid() { final Config config = config(CONFIGURATION_SOURCE_AGENTLESS, "api-key"); final BackendApiFactory backendApiFactory = mock(BackendApiFactory.class); final BackendApi localApi = mock(BackendApi.class); - when(backendApiFactory.createEvpProxyApi(Intake.EVENT_PLATFORM)).thenReturn(localApi); - when(backendApiFactory.createDirectIntakeApi(Intake.EVENT_PLATFORM)) + when(backendApiFactory.createEvpProxyApi(Intake.EVENT_PLATFORM, false)).thenReturn(localApi); + when(backendApiFactory.createDirectIntakeApi(Intake.EVENT_PLATFORM, false)) .thenThrow(new IllegalArgumentException("invalid URL")); - final BackendApi selected = new ExposureBackendApiFactory(config, backendApiFactory).create(); + final BackendApi selected = + new FeatureFlagBackendApiFactory(config, backendApiFactory, "flag evaluation", false) + .create(); assertSame(localApi, selected); } diff --git a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationWriterImplTest.java b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationWriterImplTest.java index b354fdbb6c4..a395162a8b8 100644 --- a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationWriterImplTest.java +++ b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationWriterImplTest.java @@ -11,6 +11,7 @@ import static com.datadog.featureflag.FlagEvaluationTestSupport.metricSum; import static com.datadog.featureflag.FlagEvaluationTestSupport.repeat; import static com.datadog.featureflag.FlagEvaluationTestSupport.simpleEvent; +import static datadog.trace.api.featureflag.config.FeatureFlaggingConfig.CONFIGURATION_SOURCE_AGENTLESS; import static java.util.Collections.emptyMap; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -31,13 +32,18 @@ import datadog.common.queue.Queues; import datadog.communication.BackendApi; import datadog.communication.BackendApiFactory; +import datadog.communication.IntakeApi; import datadog.communication.ddagent.SharedCommunicationObjects; +import datadog.communication.http.HttpRetryPolicy; +import datadog.trace.agent.test.server.http.JavaTestHttpServer; +import datadog.trace.api.Config; import datadog.trace.api.featureflag.FeatureFlaggingGateway; import datadog.trace.api.featureflag.flagevaluation.FlagEvalEvent; import datadog.trace.api.featureflag.ufc.v1.ServerConfiguration; import datadog.trace.api.intake.Intake; import datadog.trace.api.telemetry.CoreMetricCollector; import datadog.trace.api.telemetry.MetricCollector; +import datadog.trace.test.util.PollingConditions; import java.io.IOException; import java.lang.reflect.Field; import java.util.Collection; @@ -46,6 +52,8 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; +import okhttp3.HttpUrl; +import okhttp3.OkHttpClient; import okhttp3.RequestBody; import okio.Buffer; import org.junit.jupiter.api.AfterEach; @@ -54,6 +62,10 @@ class FlagEvaluationWriterImplTest { + private static final String DIRECT_FLAG_EVALUATION_ENDPOINT = "/api/v2/flagevaluation"; + private static final String API_KEY = "test-api-key"; + private static final double TIMEOUT_SECONDS = 5; + @BeforeEach void clearCoreMetricsBefore() { clearCoreMetrics(); @@ -650,6 +662,53 @@ void scoConstructorCreatesUsableWriter() { writer.close(); } + @Test + void agentlessWritesFlagEvaluationsDirectlyWhenLocalProxyIsUnavailable() throws Exception { + try (JavaTestHttpServer server = + JavaTestHttpServer.httpServer( + s -> + s.handlers( + h -> + h.prefix( + DIRECT_FLAG_EVALUATION_ENDPOINT, + api -> api.getResponse().status(200).send("OK"))))) { + final Config config = cfg(); + when(config.getFeatureFlaggingConfigurationSource()) + .thenReturn(CONFIGURATION_SOURCE_AGENTLESS); + when(config.getApiKey()).thenReturn(API_KEY); + final BackendApiFactory backendApiFactory = mock(BackendApiFactory.class); + final IntakeApi directApi = + new IntakeApi( + HttpUrl.get(server.getAddress()).resolve("/api/v2/"), + API_KEY, + "123", + HttpRetryPolicy.Factory.NEVER_RETRY, + new OkHttpClient.Builder().build(), + false); + when(backendApiFactory.createDirectIntakeApi(Intake.EVENT_PLATFORM, false)) + .thenReturn(directApi); + final FeatureFlagBackendApiFactory featureFlagBackendApiFactory = + new FeatureFlagBackendApiFactory(config, backendApiFactory, "flag evaluation", false); + final PollingConditions poll = new PollingConditions(TIMEOUT_SECONDS); + + try (FlagEvaluationWriterImpl writer = + new FlagEvaluationWriterImpl( + 16, 1, TimeUnit.MILLISECONDS, featureFlagBackendApiFactory, config)) { + writer.startForTest(); + writer.enqueue(simpleEvent("direct-flag", "on")); + + poll.eventually( + () -> { + assertNotNull(server.getLastRequest()); + assertEquals(DIRECT_FLAG_EVALUATION_ENDPOINT, server.getLastRequest().getPath()); + assertEquals(API_KEY, server.getLastRequest().getHeader("dd-api-key")); + assertNull(server.getLastRequest().getHeader("X-Datadog-EVP-Subdomain")); + assertTrue(server.getLastRequest().getBody().length > 0); + }); + } + } + } + @Test void countContextTruncatedAccumulatesPerReason() { final BackendApi mockEvp = mock(BackendApi.class); From 59165392a0da76690b305e2c8facd656b14698a6 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Thu, 13 Aug 2026 11:18:33 -0400 Subject: [PATCH 06/14] test(openfeature): cover both direct EVP signals --- .../AgentlessFeatureFlagBackendApiTest.java | 38 ++++++++++++++++--- .../FeatureFlagBackendApiFactoryTest.java | 25 ++++++++++++ 2 files changed, 58 insertions(+), 5 deletions(-) diff --git a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessFeatureFlagBackendApiTest.java b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessFeatureFlagBackendApiTest.java index 0a2403ec25d..23226697fe7 100644 --- a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessFeatureFlagBackendApiTest.java +++ b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessFeatureFlagBackendApiTest.java @@ -15,11 +15,14 @@ import java.net.SocketTimeoutException; import java.util.ArrayList; import java.util.List; +import java.util.stream.Stream; import javax.annotation.Nullable; import okhttp3.MediaType; import okhttp3.RequestBody; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; import org.junit.jupiter.params.provider.ValueSource; class AgentlessFeatureFlagBackendApiTest { @@ -45,20 +48,40 @@ void replaysRejectedBatchDirectlyAndKeepsDirectRoute(final int statusCode) throw assertSame(secondBody, direct.requestBodies.get(1)); } - @Test - void fallsBackAfterConnectionRefusal() throws Exception { + @ParameterizedTest + @MethodSource("featureFlagRoutes") + void fallsBackAfterConnectionRefusal(final String route, final String eventType) + throws Exception { final RecordingBackendApi local = new RecordingBackendApi(new ConnectException("connection refused")); final RecordingBackendApi direct = new RecordingBackendApi(); final AgentlessFeatureFlagBackendApi api = - new AgentlessFeatureFlagBackendApi(local, direct, "flag evaluation"); + new AgentlessFeatureFlagBackendApi(local, direct, eventType); - api.post("flagevaluation", requestBody("evaluation"), stream -> null, null, false); + api.post(route, requestBody(eventType), stream -> null, null, false); assertEquals(1, local.calls); assertEquals(1, direct.calls); } + @Test + void doesNotReturnToLocalRouteAfterSwitchingToDirectIntake() throws Exception { + final RecordingBackendApi local = + new RecordingBackendApi(new ConnectException("connection refused")); + final RecordingBackendApi direct = new RecordingBackendApi(); + final AgentlessFeatureFlagBackendApi api = + new AgentlessFeatureFlagBackendApi(local, direct, "exposure"); + + api.post("exposures", requestBody("first"), stream -> null, null, false); + direct.failure = new IOException("direct intake failed"); + + assertThrows( + IOException.class, + () -> api.post("exposures", requestBody("second"), stream -> null, null, false)); + assertEquals(1, local.calls); + assertEquals(2, direct.calls); + } + @ParameterizedTest @ValueSource(ints = {429, 500}) void doesNotReplayAmbiguousHttpFailure(final int statusCode) { @@ -93,8 +116,13 @@ private static RequestBody requestBody(final String value) { return RequestBody.create(MediaType.parse("application/json"), value); } + private static Stream featureFlagRoutes() { + return Stream.of( + Arguments.of("exposures", "exposure"), Arguments.of("flagevaluation", "flag evaluation")); + } + private static final class RecordingBackendApi implements BackendApi { - private final IOException failure; + private IOException failure; private final List requestBodies = new ArrayList<>(); private int calls; diff --git a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FeatureFlagBackendApiFactoryTest.java b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FeatureFlagBackendApiFactoryTest.java index b5248fcfbda..e9fc9962eb1 100644 --- a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FeatureFlagBackendApiFactoryTest.java +++ b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FeatureFlagBackendApiFactoryTest.java @@ -33,6 +33,19 @@ void remoteConfigUsesOnlyLocalEvpProxy() { verify(backendApiFactory, never()).createDirectIntakeApi(Intake.EVENT_PLATFORM, false); } + @Test + void remoteConfigDisablesDeliveryWhenLocalEvpProxyIsUnavailable() { + final Config config = config(CONFIGURATION_SOURCE_REMOTE_CONFIG, "api-key"); + final BackendApiFactory backendApiFactory = mock(BackendApiFactory.class); + + final BackendApi selected = + new FeatureFlagBackendApiFactory(config, backendApiFactory, "exposure", true).create(); + + assertNull(selected); + verify(backendApiFactory).createEvpProxyApi(Intake.EVENT_PLATFORM, true); + verify(backendApiFactory, never()).createDirectIntakeApi(Intake.EVENT_PLATFORM, true); + } + @Test void agentlessPrefersLocalEvpProxyWithDirectFallback() { final Config config = config(CONFIGURATION_SOURCE_AGENTLESS, "api-key"); @@ -91,6 +104,18 @@ void agentlessDisablesDeliveryWhenNoRouteIsAvailable() { assertNull(selected); } + @Test + void agentlessDisablesDeliveryWhenApiKeyIsEmpty() { + final Config config = config(CONFIGURATION_SOURCE_AGENTLESS, ""); + final BackendApiFactory backendApiFactory = mock(BackendApiFactory.class); + + final BackendApi selected = + new FeatureFlagBackendApiFactory(config, backendApiFactory, "exposure", true).create(); + + assertNull(selected); + verify(backendApiFactory, never()).createDirectIntakeApi(Intake.EVENT_PLATFORM, true); + } + @Test void agentlessKeepsLocalRouteWhenDirectUrlIsInvalid() { final Config config = config(CONFIGURATION_SOURCE_AGENTLESS, "api-key"); From 9ed502c5c1d5794299a57fc817ba8aea315a112f Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Thu, 13 Aug 2026 16:51:56 -0400 Subject: [PATCH 07/14] Defer direct exposure intake fallback --- .../AgentlessExposureBackendApi.java | 39 +++++++++++++-- .../ExposureBackendApiFactory.java | 17 ++++--- .../AgentlessExposureBackendApiTest.java | 48 +++++++++++++++++-- .../ExposureBackendApiFactoryTest.java | 18 ++++++- 4 files changed, 106 insertions(+), 16 deletions(-) diff --git a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/AgentlessExposureBackendApi.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/AgentlessExposureBackendApi.java index 97c798f0e1d..28f046832ab 100644 --- a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/AgentlessExposureBackendApi.java +++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/AgentlessExposureBackendApi.java @@ -7,6 +7,7 @@ import java.io.IOException; import java.io.InputStream; import java.net.ConnectException; +import java.util.function.Supplier; import javax.annotation.Nullable; import okhttp3.RequestBody; import org.slf4j.Logger; @@ -18,12 +19,14 @@ final class AgentlessExposureBackendApi implements BackendApi { private static final Logger LOGGER = LoggerFactory.getLogger(AgentlessExposureBackendApi.class); private final BackendApi localApi; - private final BackendApi directApi; + private final Supplier directApiSupplier; private volatile BackendApi activeApi; + private volatile boolean directApiCreationAttempted; - AgentlessExposureBackendApi(final BackendApi localApi, final BackendApi directApi) { + AgentlessExposureBackendApi( + final BackendApi localApi, final Supplier directApiSupplier) { this.localApi = localApi; - this.directApi = directApi; + this.directApiSupplier = directApiSupplier; this.activeApi = localApi; } @@ -44,12 +47,38 @@ public T post( throw exception; } - if (activeApi == localApi) { + final BackendApi directApi = getOrCreateDirectApi(); + if (directApi == null) { + throw exception; + } + return directApi.post(uri, requestBody, responseParser, requestListener, requestCompression); + } + } + + @Nullable + private BackendApi getOrCreateDirectApi() { + final BackendApi selectedApi = activeApi; + if (selectedApi != localApi) { + return selectedApi; + } + + synchronized (this) { + final BackendApi currentApi = activeApi; + if (currentApi != localApi) { + return currentApi; + } + if (directApiCreationAttempted) { + return null; + } + + final BackendApi directApi = directApiSupplier.get(); + if (directApi != null) { LOGGER.debug( "Switching Feature Flagging exposure delivery from the local EVP proxy to direct intake"); activeApi = directApi; } - return directApi.post(uri, requestBody, responseParser, requestListener, requestCompression); + directApiCreationAttempted = true; + return directApi; } } diff --git a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/ExposureBackendApiFactory.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/ExposureBackendApiFactory.java index f0eb1fa0e3d..8381cbec078 100644 --- a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/ExposureBackendApiFactory.java +++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/ExposureBackendApiFactory.java @@ -40,13 +40,14 @@ BackendApi create() { return localApi; } - final BackendApi directApi = createDirectApi(); - if (localApi != null && directApi != null) { - return new AgentlessExposureBackendApi(localApi, directApi); - } if (localApi != null) { + if (hasDirectCredentials()) { + return new AgentlessExposureBackendApi(localApi, this::createDirectApi); + } return localApi; } + + final BackendApi directApi = createDirectApi(); if (directApi != null) { return directApi; } @@ -56,10 +57,14 @@ BackendApi create() { return null; } + private boolean hasDirectCredentials() { + final String apiKey = config.getApiKey(); + return apiKey != null && !apiKey.isEmpty(); + } + @Nullable private BackendApi createDirectApi() { - final String apiKey = config.getApiKey(); - if (apiKey == null || apiKey.isEmpty()) { + if (!hasDirectCredentials()) { return null; } try { diff --git a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessExposureBackendApiTest.java b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessExposureBackendApiTest.java index 9455634ec45..c8c4f48165d 100644 --- a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessExposureBackendApiTest.java +++ b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessExposureBackendApiTest.java @@ -15,6 +15,7 @@ import java.net.SocketTimeoutException; import java.util.ArrayList; import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; import javax.annotation.Nullable; import okhttp3.MediaType; import okhttp3.RequestBody; @@ -30,13 +31,22 @@ void replaysRejectedBatchDirectlyAndKeepsDirectRoute(final int statusCode) throw final RecordingBackendApi local = new RecordingBackendApi(new HttpResponseException(statusCode, "rejected")); final RecordingBackendApi direct = new RecordingBackendApi(); - final AgentlessExposureBackendApi api = new AgentlessExposureBackendApi(local, direct); + final AtomicInteger directApiCreations = new AtomicInteger(); + final AgentlessExposureBackendApi api = + new AgentlessExposureBackendApi( + local, + () -> { + directApiCreations.incrementAndGet(); + return direct; + }); final RequestBody firstBody = requestBody("first"); final RequestBody secondBody = requestBody("second"); + assertEquals(0, directApiCreations.get()); api.post("exposures", firstBody, stream -> null, null, false); api.post("exposures", secondBody, stream -> null, null, false); + assertEquals(1, directApiCreations.get()); assertEquals(1, local.calls); assertEquals(2, direct.calls); assertSame(firstBody, local.requestBodies.get(0)); @@ -49,7 +59,7 @@ void fallsBackAfterConnectionRefusal() throws Exception { final RecordingBackendApi local = new RecordingBackendApi(new ConnectException("connection refused")); final RecordingBackendApi direct = new RecordingBackendApi(); - final AgentlessExposureBackendApi api = new AgentlessExposureBackendApi(local, direct); + final AgentlessExposureBackendApi api = new AgentlessExposureBackendApi(local, () -> direct); api.post("exposures", requestBody("exposure"), stream -> null, null, false); @@ -73,10 +83,41 @@ void doesNotReplayConnectionReset() { assertNoDirectReplay(new SocketException("connection reset")); } + @Test + void doesNotRetryDirectApiCreationWhenFallbackIsUnavailable() { + final RecordingBackendApi local = + new RecordingBackendApi(new HttpResponseException(404, "rejected")); + final AtomicInteger directApiCreations = new AtomicInteger(); + final AgentlessExposureBackendApi api = + new AgentlessExposureBackendApi( + local, + () -> { + directApiCreations.incrementAndGet(); + return null; + }); + + assertThrows( + HttpResponseException.class, + () -> api.post("exposures", requestBody("first"), stream -> null, null, false)); + assertThrows( + HttpResponseException.class, + () -> api.post("exposures", requestBody("second"), stream -> null, null, false)); + + assertEquals(2, local.calls); + assertEquals(1, directApiCreations.get()); + } + private static void assertNoDirectReplay(final IOException failure) { final RecordingBackendApi local = new RecordingBackendApi(failure); final RecordingBackendApi direct = new RecordingBackendApi(); - final AgentlessExposureBackendApi api = new AgentlessExposureBackendApi(local, direct); + final AtomicInteger directApiCreations = new AtomicInteger(); + final AgentlessExposureBackendApi api = + new AgentlessExposureBackendApi( + local, + () -> { + directApiCreations.incrementAndGet(); + return direct; + }); assertThrows( IOException.class, @@ -84,6 +125,7 @@ private static void assertNoDirectReplay(final IOException failure) { assertEquals(1, local.calls); assertEquals(0, direct.calls); + assertEquals(0, directApiCreations.get()); } private static RequestBody requestBody(final String value) { diff --git a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/ExposureBackendApiFactoryTest.java b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/ExposureBackendApiFactoryTest.java index 602a3219ef3..ac77a02ef92 100644 --- a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/ExposureBackendApiFactoryTest.java +++ b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/ExposureBackendApiFactoryTest.java @@ -43,6 +43,7 @@ void agentlessPrefersLocalEvpProxyWithDirectFallback() { final BackendApi selected = new ExposureBackendApiFactory(config, backendApiFactory).create(); assertInstanceOf(AgentlessExposureBackendApi.class, selected); + verify(backendApiFactory, never()).createDirectIntakeApi(Intake.EVENT_PLATFORM); } @Test @@ -81,7 +82,7 @@ void agentlessDisablesDeliveryWhenNoRouteIsAvailable() { } @Test - void agentlessKeepsLocalRouteWhenDirectUrlIsInvalid() { + void agentlessDoesNotValidateDirectUrlWhileLocalRouteIsAvailable() { final Config config = config(CONFIGURATION_SOURCE_AGENTLESS, "api-key"); final BackendApiFactory backendApiFactory = mock(BackendApiFactory.class); final BackendApi localApi = mock(BackendApi.class); @@ -91,7 +92,20 @@ void agentlessKeepsLocalRouteWhenDirectUrlIsInvalid() { final BackendApi selected = new ExposureBackendApiFactory(config, backendApiFactory).create(); - assertSame(localApi, selected); + assertInstanceOf(AgentlessExposureBackendApi.class, selected); + verify(backendApiFactory, never()).createDirectIntakeApi(Intake.EVENT_PLATFORM); + } + + @Test + void agentlessDisablesDeliveryWhenDirectUrlIsInvalidAndLocalRouteIsUnavailable() { + final Config config = config(CONFIGURATION_SOURCE_AGENTLESS, "api-key"); + final BackendApiFactory backendApiFactory = mock(BackendApiFactory.class); + when(backendApiFactory.createDirectIntakeApi(Intake.EVENT_PLATFORM)) + .thenThrow(new IllegalArgumentException("invalid URL")); + + final BackendApi selected = new ExposureBackendApiFactory(config, backendApiFactory).create(); + + assertNull(selected); } private static Config config(final String source, final String apiKey) { From a83643118af6b66318dd6a16c04bd99e037592ab Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Thu, 13 Aug 2026 19:23:02 -0400 Subject: [PATCH 08/14] test(openfeature): cover direct exposure fallback branches --- .../AgentlessExposureBackendApiTest.java | 19 +++++++++++++++- .../ExposureBackendApiFactoryTest.java | 22 +++++++++++++++++++ 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessExposureBackendApiTest.java b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessExposureBackendApiTest.java index c8c4f48165d..de34668f0f5 100644 --- a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessExposureBackendApiTest.java +++ b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessExposureBackendApiTest.java @@ -67,6 +67,23 @@ void fallsBackAfterConnectionRefusal() throws Exception { assertEquals(1, direct.calls); } + @Test + void doesNotReturnToLocalRouteAfterSwitchingToDirectIntake() throws Exception { + final RecordingBackendApi local = + new RecordingBackendApi(new ConnectException("connection refused")); + final RecordingBackendApi direct = new RecordingBackendApi(); + final AgentlessExposureBackendApi api = new AgentlessExposureBackendApi(local, () -> direct); + + api.post("exposures", requestBody("first"), stream -> null, null, false); + direct.failure = new IOException("direct intake failed"); + + assertThrows( + IOException.class, + () -> api.post("exposures", requestBody("second"), stream -> null, null, false)); + assertEquals(1, local.calls); + assertEquals(2, direct.calls); + } + @ParameterizedTest @ValueSource(ints = {429, 500}) void doesNotReplayAmbiguousHttpFailure(final int statusCode) { @@ -133,7 +150,7 @@ private static RequestBody requestBody(final String value) { } private static final class RecordingBackendApi implements BackendApi { - private final IOException failure; + private IOException failure; private final List requestBodies = new ArrayList<>(); private int calls; diff --git a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/ExposureBackendApiFactoryTest.java b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/ExposureBackendApiFactoryTest.java index ac77a02ef92..2f966924f44 100644 --- a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/ExposureBackendApiFactoryTest.java +++ b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/ExposureBackendApiFactoryTest.java @@ -31,6 +31,17 @@ void remoteConfigUsesOnlyLocalEvpProxy() { verify(backendApiFactory, never()).createDirectIntakeApi(Intake.EVENT_PLATFORM); } + @Test + void remoteConfigDisablesDeliveryWhenLocalEvpProxyIsUnavailable() { + final Config config = config(CONFIGURATION_SOURCE_REMOTE_CONFIG, "api-key"); + final BackendApiFactory backendApiFactory = mock(BackendApiFactory.class); + + final BackendApi selected = new ExposureBackendApiFactory(config, backendApiFactory).create(); + + assertNull(selected); + verify(backendApiFactory, never()).createDirectIntakeApi(Intake.EVENT_PLATFORM); + } + @Test void agentlessPrefersLocalEvpProxyWithDirectFallback() { final Config config = config(CONFIGURATION_SOURCE_AGENTLESS, "api-key"); @@ -81,6 +92,17 @@ void agentlessDisablesDeliveryWhenNoRouteIsAvailable() { assertNull(selected); } + @Test + void agentlessDisablesDeliveryWhenApiKeyIsEmpty() { + final Config config = config(CONFIGURATION_SOURCE_AGENTLESS, ""); + final BackendApiFactory backendApiFactory = mock(BackendApiFactory.class); + + final BackendApi selected = new ExposureBackendApiFactory(config, backendApiFactory).create(); + + assertNull(selected); + verify(backendApiFactory, never()).createDirectIntakeApi(Intake.EVENT_PLATFORM); + } + @Test void agentlessDoesNotValidateDirectUrlWhileLocalRouteIsAvailable() { final Config config = config(CONFIGURATION_SOURCE_AGENTLESS, "api-key"); From 855b295fdf186f82b680d409f3c4306a3dd5aa81 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Thu, 13 Aug 2026 21:28:12 -0400 Subject: [PATCH 09/14] fix(openfeature): clarify feature flag transport policy --- .../java/datadog/communication/IntakeApi.java | 8 ++- .../datadog/communication/IntakeApiTest.java | 65 +++++++++++++++++++ .../AgentlessFeatureFlagBackendApi.java | 14 ++-- .../featureflag/ExposureWriterImpl.java | 2 +- .../FeatureFlagBackendApiFactory.java | 42 ++++++------ .../featureflag/FeatureFlagEventType.java | 26 ++++++++ .../featureflag/FlagEvaluationWriterImpl.java | 7 +- .../featureflag/ExposureWriterTests.java | 2 +- .../FeatureFlagBackendApiFactoryTest.java | 43 ++++++------ .../FlagEvaluationWriterImplTest.java | 3 +- 10 files changed, 150 insertions(+), 62 deletions(-) create mode 100644 communication/src/test/java/datadog/communication/IntakeApiTest.java create mode 100644 products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FeatureFlagEventType.java diff --git a/communication/src/main/java/datadog/communication/IntakeApi.java b/communication/src/main/java/datadog/communication/IntakeApi.java index 3f1fe9df67f..1a6f3f91bc7 100644 --- a/communication/src/main/java/datadog/communication/IntakeApi.java +++ b/communication/src/main/java/datadog/communication/IntakeApi.java @@ -25,6 +25,7 @@ public class IntakeApi implements BackendApi { private static final String ACCEPT_ENCODING_HEADER = "Accept-Encoding"; private static final String CONTENT_ENCODING_HEADER = "Content-Encoding"; private static final String GZIP_ENCODING = "gzip"; + private static final String IDENTITY_ENCODING = "identity"; private final String apiKey; private final String traceId; @@ -73,9 +74,10 @@ public T post( requestBuilder.addHeader(CONTENT_ENCODING_HEADER, GZIP_ENCODING); } - if (responseCompression) { - requestBuilder.addHeader(ACCEPT_ENCODING_HEADER, GZIP_ENCODING); - } + // OkHttp adds Accept-Encoding: gzip when this header is absent. Always set the header so a + // caller can disable response compression on the wire. + requestBuilder.addHeader( + ACCEPT_ENCODING_HEADER, responseCompression ? GZIP_ENCODING : IDENTITY_ENCODING); Request request = requestBuilder.build(); try (okhttp3.Response response = diff --git a/communication/src/test/java/datadog/communication/IntakeApiTest.java b/communication/src/test/java/datadog/communication/IntakeApiTest.java new file mode 100644 index 00000000000..326cf21ca84 --- /dev/null +++ b/communication/src/test/java/datadog/communication/IntakeApiTest.java @@ -0,0 +1,65 @@ +package datadog.communication; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import datadog.communication.http.HttpRetryPolicy; +import java.io.IOException; +import okhttp3.MediaType; +import okhttp3.OkHttpClient; +import okhttp3.RequestBody; +import okhttp3.mockwebserver.MockResponse; +import okhttp3.mockwebserver.MockWebServer; +import okhttp3.mockwebserver.RecordedRequest; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +class IntakeApiTest { + + private static final MediaType JSON = MediaType.parse("application/json"); + + private MockWebServer server; + private OkHttpClient client; + + @BeforeEach + void setUp() throws IOException { + server = new MockWebServer(); + server.start(); + client = new OkHttpClient.Builder().build(); + } + + @AfterEach + void tearDown() throws IOException { + client.dispatcher().executorService().shutdownNow(); + client.connectionPool().evictAll(); + server.shutdown(); + } + + @Test + void requestsGzipResponseCompressionWhenEnabled() throws Exception { + assertEquals("gzip", postAndReadAcceptEncoding(true)); + } + + @Test + void requestsIdentityResponseEncodingWhenCompressionIsDisabled() throws Exception { + assertEquals("identity", postAndReadAcceptEncoding(false)); + } + + private String postAndReadAcceptEncoding(final boolean responseCompression) throws Exception { + server.enqueue(new MockResponse().setResponseCode(200).setBody("{}")); + final IntakeApi api = + new IntakeApi( + server.url("/api/v2/"), + "api-key", + "123", + HttpRetryPolicy.Factory.NEVER_RETRY, + client, + responseCompression); + + api.post("flagevaluation", RequestBody.create(JSON, "{}"), responseBody -> null, null, false); + + final RecordedRequest request = server.takeRequest(); + assertEquals("/api/v2/flagevaluation", request.getPath()); + return request.getHeader("Accept-Encoding"); + } +} diff --git a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/AgentlessFeatureFlagBackendApi.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/AgentlessFeatureFlagBackendApi.java index 10327eee328..769ebfd1dd1 100644 --- a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/AgentlessFeatureFlagBackendApi.java +++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/AgentlessFeatureFlagBackendApi.java @@ -19,20 +19,20 @@ final class AgentlessFeatureFlagBackendApi implements BackendApi { private static final Logger LOGGER = LoggerFactory.getLogger(AgentlessFeatureFlagBackendApi.class); - private final BackendApi localApi; + private final BackendApi proxyApi; private final Supplier directApiSupplier; private final String eventType; private volatile BackendApi activeApi; private volatile boolean directApiCreationAttempted; AgentlessFeatureFlagBackendApi( - final BackendApi localApi, + final BackendApi proxyApi, final Supplier directApiSupplier, final String eventType) { - this.localApi = localApi; + this.proxyApi = proxyApi; this.directApiSupplier = directApiSupplier; this.eventType = eventType; - this.activeApi = localApi; + this.activeApi = proxyApi; } @Override @@ -48,7 +48,7 @@ public T post( return selectedApi.post( uri, requestBody, responseParser, requestListener, requestCompression); } catch (final IOException exception) { - if (selectedApi != localApi || !isDefinitiveRejection(exception)) { + if (selectedApi != proxyApi || !isDefinitiveRejection(exception)) { throw exception; } @@ -63,13 +63,13 @@ public T post( @Nullable private BackendApi getOrCreateDirectApi() { final BackendApi selectedApi = activeApi; - if (selectedApi != localApi) { + if (selectedApi != proxyApi) { return selectedApi; } synchronized (this) { final BackendApi currentApi = activeApi; - if (currentApi != localApi) { + if (currentApi != proxyApi) { return currentApi; } if (directApiCreationAttempted) { diff --git a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/ExposureWriterImpl.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/ExposureWriterImpl.java index 7b8ca052feb..017616ed070 100644 --- a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/ExposureWriterImpl.java +++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/ExposureWriterImpl.java @@ -45,7 +45,7 @@ public ExposureWriterImpl(final SharedCommunicationObjects sco, final Config con capacity, flushInterval, timeUnit, - new FeatureFlagBackendApiFactory(config, sco, "exposure", true), + new FeatureFlagBackendApiFactory(config, sco, FeatureFlagEventType.EXPOSURE), config); } diff --git a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FeatureFlagBackendApiFactory.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FeatureFlagBackendApiFactory.java index 0081cac3343..1727d156915 100644 --- a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FeatureFlagBackendApiFactory.java +++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FeatureFlagBackendApiFactory.java @@ -18,50 +18,44 @@ final class FeatureFlagBackendApiFactory { private final Config config; private final BackendApiFactory backendApiFactory; - private final String eventType; - private final boolean responseCompression; + private final FeatureFlagEventType eventType; FeatureFlagBackendApiFactory( final Config config, final SharedCommunicationObjects sharedCommunicationObjects, - final String eventType, - final boolean responseCompression) { - this( - config, - new BackendApiFactory(config, sharedCommunicationObjects), - eventType, - responseCompression); + final FeatureFlagEventType eventType) { + this(config, new BackendApiFactory(config, sharedCommunicationObjects), eventType); } FeatureFlagBackendApiFactory( final Config config, final BackendApiFactory backendApiFactory, - final String eventType, - final boolean responseCompression) { + final FeatureFlagEventType eventType) { this.config = config; this.backendApiFactory = backendApiFactory; this.eventType = eventType; - this.responseCompression = responseCompression; } @Nullable BackendApi create() { - final BackendApi localApi = - backendApiFactory.createEvpProxyApi(Intake.EVENT_PLATFORM, responseCompression); + final BackendApi proxyApi = + backendApiFactory.createEvpProxyApi( + Intake.EVENT_PLATFORM, eventType.responseCompressionEnabled()); if (!CONFIGURATION_SOURCE_AGENTLESS.equals(config.getFeatureFlaggingConfigurationSource())) { - if (localApi == null) { + if (proxyApi == null) { LOGGER.warn( "Feature Flagging {} delivery is disabled because the local Agent does not support the EVP proxy", - eventType); + eventType.logName()); } - return localApi; + return proxyApi; } - if (localApi != null) { + if (proxyApi != null) { if (hasDirectCredentials()) { - return new AgentlessFeatureFlagBackendApi(localApi, this::createDirectApi, eventType); + return new AgentlessFeatureFlagBackendApi( + proxyApi, this::createDirectApi, eventType.logName()); } - return localApi; + return proxyApi; } final BackendApi directApi = createDirectApi(); @@ -71,7 +65,7 @@ BackendApi create() { LOGGER.warn( "Feature Flagging {} delivery is disabled because no compatible local EVP proxy or direct intake credentials are available", - eventType); + eventType.logName()); return null; } @@ -86,9 +80,11 @@ private BackendApi createDirectApi() { return null; } try { - return backendApiFactory.createDirectIntakeApi(Intake.EVENT_PLATFORM, responseCompression); + return backendApiFactory.createDirectIntakeApi( + Intake.EVENT_PLATFORM, eventType.responseCompressionEnabled()); } catch (final IllegalArgumentException exception) { - LOGGER.debug("Cannot configure direct Feature Flagging {} delivery", eventType, exception); + LOGGER.debug( + "Cannot configure direct Feature Flagging {} delivery", eventType.logName(), exception); return null; } } diff --git a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FeatureFlagEventType.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FeatureFlagEventType.java new file mode 100644 index 00000000000..dee8b424fa3 --- /dev/null +++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FeatureFlagEventType.java @@ -0,0 +1,26 @@ +package com.datadog.featureflag; + +/** Defines event-specific transport behavior for Feature Flag delivery. */ +enum FeatureFlagEventType { + // Keep the established exposure transport behavior for compatibility. + EXPOSURE("exposure", true), + + // Flag evaluation writers ignore successful response bodies, so gzip negotiation adds no value. + FLAG_EVALUATION("flag evaluation", false); + + private final String logName; + private final boolean responseCompressionEnabled; + + FeatureFlagEventType(final String logName, final boolean responseCompressionEnabled) { + this.logName = logName; + this.responseCompressionEnabled = responseCompressionEnabled; + } + + String logName() { + return logName; + } + + boolean responseCompressionEnabled() { + return responseCompressionEnabled; + } +} diff --git a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FlagEvaluationWriterImpl.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FlagEvaluationWriterImpl.java index 64884b78b8e..1bc203022ac 100644 --- a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FlagEvaluationWriterImpl.java +++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FlagEvaluationWriterImpl.java @@ -115,7 +115,7 @@ public FlagEvaluationWriterImpl(final SharedCommunicationObjects sco, final Conf capacity, flushInterval, timeUnit, - new FeatureFlagBackendApiFactory(config, sco, "flag evaluation", false), + new FeatureFlagBackendApiFactory(config, sco, FeatureFlagEventType.FLAG_EVALUATION), config); } @@ -130,7 +130,10 @@ public FlagEvaluationWriterImpl(final SharedCommunicationObjects sco, final Conf capacity, flushInterval, timeUnit, - () -> backendApiFactory.createBackendApi(Intake.EVENT_PLATFORM, false), + () -> + backendApiFactory.createBackendApi( + Intake.EVENT_PLATFORM, + FeatureFlagEventType.FLAG_EVALUATION.responseCompressionEnabled()), config); } diff --git a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/ExposureWriterTests.java b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/ExposureWriterTests.java index cc2eb5bb5c5..78e4a72ebba 100644 --- a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/ExposureWriterTests.java +++ b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/ExposureWriterTests.java @@ -159,7 +159,7 @@ void testAgentlessExposureEventWritesDirectlyWithApiKey() throws Exception { datadog.trace.api.intake.Intake.EVENT_PLATFORM, true)) .thenReturn(directApi); FeatureFlagBackendApiFactory exposureBackendApiFactory = - new FeatureFlagBackendApiFactory(config, backendApiFactory, "exposure", true); + new FeatureFlagBackendApiFactory(config, backendApiFactory, FeatureFlagEventType.EXPOSURE); List exposures = buildExposures(5); try (ExposureWriterImpl writer = diff --git a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FeatureFlagBackendApiFactoryTest.java b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FeatureFlagBackendApiFactoryTest.java index c498caff27b..ddd873a1e9d 100644 --- a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FeatureFlagBackendApiFactoryTest.java +++ b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FeatureFlagBackendApiFactoryTest.java @@ -1,5 +1,7 @@ package com.datadog.featureflag; +import static com.datadog.featureflag.FeatureFlagEventType.EXPOSURE; +import static com.datadog.featureflag.FeatureFlagEventType.FLAG_EVALUATION; import static datadog.trace.api.featureflag.config.FeatureFlaggingConfig.CONFIGURATION_SOURCE_AGENTLESS; import static datadog.trace.api.featureflag.config.FeatureFlaggingConfig.CONFIGURATION_SOURCE_REMOTE_CONFIG; import static org.junit.jupiter.api.Assertions.assertInstanceOf; @@ -22,14 +24,13 @@ class FeatureFlagBackendApiFactoryTest { void remoteConfigUsesOnlyLocalEvpProxy() { final Config config = config(CONFIGURATION_SOURCE_REMOTE_CONFIG, "api-key"); final BackendApiFactory backendApiFactory = mock(BackendApiFactory.class); - final BackendApi localApi = mock(BackendApi.class); - when(backendApiFactory.createEvpProxyApi(Intake.EVENT_PLATFORM, false)).thenReturn(localApi); + final BackendApi proxyApi = mock(BackendApi.class); + when(backendApiFactory.createEvpProxyApi(Intake.EVENT_PLATFORM, false)).thenReturn(proxyApi); final BackendApi selected = - new FeatureFlagBackendApiFactory(config, backendApiFactory, "flag evaluation", false) - .create(); + new FeatureFlagBackendApiFactory(config, backendApiFactory, FLAG_EVALUATION).create(); - assertSame(localApi, selected); + assertSame(proxyApi, selected); verify(backendApiFactory, never()).createDirectIntakeApi(Intake.EVENT_PLATFORM, false); } @@ -39,7 +40,7 @@ void remoteConfigDisablesDeliveryWhenLocalEvpProxyIsUnavailable() { final BackendApiFactory backendApiFactory = mock(BackendApiFactory.class); final BackendApi selected = - new FeatureFlagBackendApiFactory(config, backendApiFactory, "exposure", true).create(); + new FeatureFlagBackendApiFactory(config, backendApiFactory, EXPOSURE).create(); assertNull(selected); verify(backendApiFactory).createEvpProxyApi(Intake.EVENT_PLATFORM, true); @@ -56,8 +57,7 @@ void agentlessPrefersLocalEvpProxyWithDirectFallback() { .thenReturn(mock(BackendApi.class)); final BackendApi selected = - new FeatureFlagBackendApiFactory(config, backendApiFactory, "flag evaluation", false) - .create(); + new FeatureFlagBackendApiFactory(config, backendApiFactory, FLAG_EVALUATION).create(); assertInstanceOf(AgentlessFeatureFlagBackendApi.class, selected); verify(backendApiFactory, never()).createDirectIntakeApi(Intake.EVENT_PLATFORM, false); @@ -72,8 +72,7 @@ void agentlessUsesDirectIntakeWhenLocalEvpProxyIsUnavailable() { .thenReturn(directApi); final BackendApi selected = - new FeatureFlagBackendApiFactory(config, backendApiFactory, "flag evaluation", false) - .create(); + new FeatureFlagBackendApiFactory(config, backendApiFactory, FLAG_EVALUATION).create(); assertSame(directApi, selected); } @@ -82,14 +81,13 @@ void agentlessUsesDirectIntakeWhenLocalEvpProxyIsUnavailable() { void agentlessUsesLocalEvpProxyWhenApiKeyIsUnavailable() { final Config config = config(CONFIGURATION_SOURCE_AGENTLESS, null); final BackendApiFactory backendApiFactory = mock(BackendApiFactory.class); - final BackendApi localApi = mock(BackendApi.class); - when(backendApiFactory.createEvpProxyApi(Intake.EVENT_PLATFORM, false)).thenReturn(localApi); + final BackendApi proxyApi = mock(BackendApi.class); + when(backendApiFactory.createEvpProxyApi(Intake.EVENT_PLATFORM, false)).thenReturn(proxyApi); final BackendApi selected = - new FeatureFlagBackendApiFactory(config, backendApiFactory, "flag evaluation", false) - .create(); + new FeatureFlagBackendApiFactory(config, backendApiFactory, FLAG_EVALUATION).create(); - assertSame(localApi, selected); + assertSame(proxyApi, selected); verify(backendApiFactory, never()).createDirectIntakeApi(Intake.EVENT_PLATFORM, false); } @@ -99,8 +97,7 @@ void agentlessDisablesDeliveryWhenNoRouteIsAvailable() { final BackendApiFactory backendApiFactory = mock(BackendApiFactory.class); final BackendApi selected = - new FeatureFlagBackendApiFactory(config, backendApiFactory, "flag evaluation", false) - .create(); + new FeatureFlagBackendApiFactory(config, backendApiFactory, FLAG_EVALUATION).create(); assertNull(selected); } @@ -111,7 +108,7 @@ void agentlessDisablesDeliveryWhenApiKeyIsEmpty() { final BackendApiFactory backendApiFactory = mock(BackendApiFactory.class); final BackendApi selected = - new FeatureFlagBackendApiFactory(config, backendApiFactory, "exposure", true).create(); + new FeatureFlagBackendApiFactory(config, backendApiFactory, EXPOSURE).create(); assertNull(selected); verify(backendApiFactory, never()).createDirectIntakeApi(Intake.EVENT_PLATFORM, true); @@ -121,14 +118,13 @@ void agentlessDisablesDeliveryWhenApiKeyIsEmpty() { void agentlessDoesNotValidateDirectUrlWhileLocalRouteIsAvailable() { final Config config = config(CONFIGURATION_SOURCE_AGENTLESS, "api-key"); final BackendApiFactory backendApiFactory = mock(BackendApiFactory.class); - final BackendApi localApi = mock(BackendApi.class); - when(backendApiFactory.createEvpProxyApi(Intake.EVENT_PLATFORM, false)).thenReturn(localApi); + final BackendApi proxyApi = mock(BackendApi.class); + when(backendApiFactory.createEvpProxyApi(Intake.EVENT_PLATFORM, false)).thenReturn(proxyApi); when(backendApiFactory.createDirectIntakeApi(Intake.EVENT_PLATFORM, false)) .thenThrow(new IllegalArgumentException("invalid URL")); final BackendApi selected = - new FeatureFlagBackendApiFactory(config, backendApiFactory, "flag evaluation", false) - .create(); + new FeatureFlagBackendApiFactory(config, backendApiFactory, FLAG_EVALUATION).create(); assertInstanceOf(AgentlessFeatureFlagBackendApi.class, selected); verify(backendApiFactory, never()).createDirectIntakeApi(Intake.EVENT_PLATFORM, false); @@ -142,8 +138,7 @@ void agentlessDisablesDeliveryWhenDirectUrlIsInvalidAndLocalRouteIsUnavailable() .thenThrow(new IllegalArgumentException("invalid URL")); final BackendApi selected = - new FeatureFlagBackendApiFactory(config, backendApiFactory, "flag evaluation", false) - .create(); + new FeatureFlagBackendApiFactory(config, backendApiFactory, FLAG_EVALUATION).create(); assertNull(selected); } diff --git a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationWriterImplTest.java b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationWriterImplTest.java index a395162a8b8..5d366c8b689 100644 --- a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationWriterImplTest.java +++ b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationWriterImplTest.java @@ -688,7 +688,8 @@ void agentlessWritesFlagEvaluationsDirectlyWhenLocalProxyIsUnavailable() throws when(backendApiFactory.createDirectIntakeApi(Intake.EVENT_PLATFORM, false)) .thenReturn(directApi); final FeatureFlagBackendApiFactory featureFlagBackendApiFactory = - new FeatureFlagBackendApiFactory(config, backendApiFactory, "flag evaluation", false); + new FeatureFlagBackendApiFactory( + config, backendApiFactory, FeatureFlagEventType.FLAG_EVALUATION); final PollingConditions poll = new PollingConditions(TIMEOUT_SECONDS); try (FlagEvaluationWriterImpl writer = From 3e668a120a616771669716d84e910703917254ea Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Thu, 13 Aug 2026 21:47:06 -0400 Subject: [PATCH 10/14] fix(feature-flags): prevent lost agentless activation --- .../featureflag/FeatureFlaggingSystem.java | 30 +++++++--- .../FeatureFlaggingSystemTest.java | 59 +++++++++++++++++++ 2 files changed, 80 insertions(+), 9 deletions(-) diff --git a/products/feature-flagging/feature-flagging-agent/src/main/java/com/datadog/featureflag/FeatureFlaggingSystem.java b/products/feature-flagging/feature-flagging-agent/src/main/java/com/datadog/featureflag/FeatureFlaggingSystem.java index aaa8e4a7aba..b3fd4a062f2 100644 --- a/products/feature-flagging/feature-flagging-agent/src/main/java/com/datadog/featureflag/FeatureFlaggingSystem.java +++ b/products/feature-flagging/feature-flagging-agent/src/main/java/com/datadog/featureflag/FeatureFlaggingSystem.java @@ -14,6 +14,11 @@ public class FeatureFlaggingSystem { + @FunctionalInterface + interface ExposureWriterFactory { + ExposureWriter create(SharedCommunicationObjects sco, Config config); + } + private static final Logger LOGGER = LoggerFactory.getLogger(FeatureFlaggingSystem.class); private static volatile ConfigurationSourceService CONFIG_SERVICE; @@ -25,11 +30,16 @@ public class FeatureFlaggingSystem { private FeatureFlaggingSystem() {} + public static void start(final SharedCommunicationObjects sco) { + start(sco, ExposureWriterImpl::new); + } + @SuppressFBWarnings( value = "USO_UNSAFE_STATIC_METHOD_SYNCHRONIZATION", justification = "Agent-internal class; Class object does not escape to app code and lock only guards the subsystem lifecycle.") - public static synchronized void start(final SharedCommunicationObjects sco) { + static synchronized void start( + final SharedCommunicationObjects sco, final ExposureWriterFactory exposureWriterFactory) { if (STARTED) { LOGGER.debug("Feature Flagging system already started"); return; @@ -44,16 +54,16 @@ public static synchronized void start(final SharedCommunicationObjects sco) { } if (CONFIGURATION_SOURCE_AGENTLESS.equals(config.getFeatureFlaggingConfigurationSource())) { - try { - initializeExposureWriter(sco, config); - } catch (final RuntimeException | Error e) { - STARTED = false; - throw e; - } final FeatureFlaggingGateway.ActivationListener activationListener = () -> activateAgentless(sco, config); ACTIVATION_LISTENER = activationListener; FeatureFlaggingGateway.addActivationListener(activationListener); + try { + initializeExposureWriter(sco, config, exposureWriterFactory); + } catch (final RuntimeException | Error e) { + stop(); + throw e; + } LOGGER.debug("Feature Flagging system awaiting application provider activation"); return; } @@ -134,8 +144,10 @@ private static void initializeSystem(final SharedCommunicationObjects sco, final } private static void initializeExposureWriter( - final SharedCommunicationObjects sco, final Config config) { - final ExposureWriter exposureWriter = new ExposureWriterImpl(sco, config); + final SharedCommunicationObjects sco, + final Config config, + final ExposureWriterFactory exposureWriterFactory) { + final ExposureWriter exposureWriter = exposureWriterFactory.create(sco, config); try { exposureWriter.init(); EXPOSURE_WRITER = exposureWriter; diff --git a/products/feature-flagging/feature-flagging-agent/src/test/java/com/datadog/featureflag/FeatureFlaggingSystemTest.java b/products/feature-flagging/feature-flagging-agent/src/test/java/com/datadog/featureflag/FeatureFlaggingSystemTest.java index dd85b6c6700..f4f47274e93 100644 --- a/products/feature-flagging/feature-flagging-agent/src/test/java/com/datadog/featureflag/FeatureFlaggingSystemTest.java +++ b/products/feature-flagging/feature-flagging-agent/src/test/java/com/datadog/featureflag/FeatureFlaggingSystemTest.java @@ -14,6 +14,7 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.clearInvocations; +import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; @@ -32,6 +33,11 @@ import datadog.trace.api.featureflag.flagevaluation.FlagEvaluationWriter; import datadog.trace.test.junit.utils.config.WithConfig; import datadog.trace.test.util.PollingConditions; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import okhttp3.HttpUrl; import okhttp3.OkHttpClient; @@ -78,6 +84,59 @@ void agentlessStartPreparesExposureDeliveryAndWaitsForApplicationProviderActivat assertFalse(FeatureFlaggingSystem.isConfigurationSourceStarted()); } + @Test + @WithConfig(key = FEATURE_FLAGS_CONFIGURATION_SOURCE, value = "agentless") + @WithConfig(key = FeatureFlaggingConfig.FLAGGING_EVALUATION_COUNTS_ENABLED, value = "false") + @WithConfig( + key = FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_BASE_URL, + value = "http://127.0.0.1:1") + void agentlessActivationDuringEarlyWriterInitializationIsNotLost() throws Exception { + final SharedCommunicationObjects sharedCommunicationObjects = sharedCommunicationObjects(); + final ExposureWriter exposureWriter = mock(ExposureWriter.class); + final CountDownLatch writerInitializationStarted = new CountDownLatch(1); + final CountDownLatch activationAttempted = new CountDownLatch(1); + final CountDownLatch finishWriterInitialization = new CountDownLatch(1); + final ExecutorService executor = Executors.newFixedThreadPool(2); + doAnswer( + ignored -> { + writerInitializationStarted.countDown(); + assertTrue(finishWriterInitialization.await(5, TimeUnit.SECONDS)); + return null; + }) + .when(exposureWriter) + .init(); + + try { + final Future start = + executor.submit( + () -> + FeatureFlaggingSystem.start( + sharedCommunicationObjects, (ignoredSco, ignoredConfig) -> exposureWriter)); + + assertTrue(writerInitializationStarted.await(5, TimeUnit.SECONDS)); + assertTrue(FeatureFlaggingSystem.isAwaitingApplicationActivation()); + + final Future activation = + executor.submit( + () -> { + activationAttempted.countDown(); + FeatureFlaggingGateway.activate(); + }); + assertTrue(activationAttempted.await(5, TimeUnit.SECONDS)); + + finishWriterInitialization.countDown(); + start.get(5, TimeUnit.SECONDS); + + new PollingConditions(TIMEOUT_SECONDS) + .eventually(() -> assertFalse(FeatureFlaggingSystem.isAwaitingApplicationActivation())); + verify(exposureWriter).init(); + activation.cancel(true); + } finally { + finishWriterInitialization.countDown(); + executor.shutdownNow(); + } + } + @Test @WithConfig(key = FEATURE_FLAGS_CONFIGURATION_SOURCE, value = "agentless") @WithConfig(key = API_KEY, value = "") From 3a108403cfd9bb836617624aed12b50040ad0741 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Thu, 13 Aug 2026 22:02:38 -0400 Subject: [PATCH 11/14] test(feature-flags): cover agentless startup rollback --- .../FeatureFlaggingSystemTest.java | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/products/feature-flagging/feature-flagging-agent/src/test/java/com/datadog/featureflag/FeatureFlaggingSystemTest.java b/products/feature-flagging/feature-flagging-agent/src/test/java/com/datadog/featureflag/FeatureFlaggingSystemTest.java index f4f47274e93..c7ebfa71f01 100644 --- a/products/feature-flagging/feature-flagging-agent/src/test/java/com/datadog/featureflag/FeatureFlaggingSystemTest.java +++ b/products/feature-flagging/feature-flagging-agent/src/test/java/com/datadog/featureflag/FeatureFlaggingSystemTest.java @@ -9,6 +9,7 @@ import static org.junit.jupiter.api.Assertions.assertInstanceOf; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; @@ -137,6 +138,38 @@ void agentlessActivationDuringEarlyWriterInitializationIsNotLost() throws Except } } + @Test + @WithConfig(key = FEATURE_FLAGS_CONFIGURATION_SOURCE, value = "agentless") + @WithConfig(key = FeatureFlaggingConfig.FLAGGING_EVALUATION_COUNTS_ENABLED, value = "false") + void agentlessExposureWriterInitializationFailureCleansUpAndAllowsRetry() { + final SharedCommunicationObjects sharedCommunicationObjects = sharedCommunicationObjects(); + final ExposureWriter failedWriter = mock(ExposureWriter.class); + final IllegalStateException initializationFailure = + new IllegalStateException("writer initialization failed"); + doThrow(initializationFailure).when(failedWriter).init(); + + final IllegalStateException thrown = + assertThrows( + IllegalStateException.class, + () -> + FeatureFlaggingSystem.start( + sharedCommunicationObjects, (ignoredSco, ignoredConfig) -> failedWriter)); + + assertSame(initializationFailure, thrown); + assertFalse(FeatureFlaggingSystem.isAwaitingApplicationActivation()); + assertFalse(FeatureFlaggingSystem.isExposureWriterStarted()); + assertFalse(FeatureFlaggingSystem.isConfigurationSourceStarted()); + verify(failedWriter).close(); + + final ExposureWriter retryWriter = mock(ExposureWriter.class); + FeatureFlaggingSystem.start( + sharedCommunicationObjects, (ignoredSco, ignoredConfig) -> retryWriter); + + assertTrue(FeatureFlaggingSystem.isAwaitingApplicationActivation()); + assertTrue(FeatureFlaggingSystem.isExposureWriterStarted()); + verify(retryWriter).init(); + } + @Test @WithConfig(key = FEATURE_FLAGS_CONFIGURATION_SOURCE, value = "agentless") @WithConfig(key = API_KEY, value = "") From 614ec21d2e601acceed4e590eeca07399cfd21c3 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Fri, 14 Aug 2026 00:18:56 -0400 Subject: [PATCH 12/14] fix(feature-flags): remove obsolete SpotBugs suppression --- .../java/com/datadog/featureflag/FeatureFlaggingSystem.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/products/feature-flagging/feature-flagging-agent/src/main/java/com/datadog/featureflag/FeatureFlaggingSystem.java b/products/feature-flagging/feature-flagging-agent/src/main/java/com/datadog/featureflag/FeatureFlaggingSystem.java index b3fd4a062f2..9759e90feed 100644 --- a/products/feature-flagging/feature-flagging-agent/src/main/java/com/datadog/featureflag/FeatureFlaggingSystem.java +++ b/products/feature-flagging/feature-flagging-agent/src/main/java/com/datadog/featureflag/FeatureFlaggingSystem.java @@ -34,10 +34,6 @@ public static void start(final SharedCommunicationObjects sco) { start(sco, ExposureWriterImpl::new); } - @SuppressFBWarnings( - value = "USO_UNSAFE_STATIC_METHOD_SYNCHRONIZATION", - justification = - "Agent-internal class; Class object does not escape to app code and lock only guards the subsystem lifecycle.") static synchronized void start( final SharedCommunicationObjects sco, final ExposureWriterFactory exposureWriterFactory) { if (STARTED) { From 041bc1056cce13b8a2e449a4879b13a49a54e5b7 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Tue, 18 Aug 2026 12:16:18 -0700 Subject: [PATCH 13/14] test(openfeature): close direct intake client --- .../FlagEvaluationWriterImplTest.java | 64 ++++++++++--------- 1 file changed, 35 insertions(+), 29 deletions(-) diff --git a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationWriterImplTest.java b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationWriterImplTest.java index 5d366c8b689..010f5ea5568 100644 --- a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationWriterImplTest.java +++ b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationWriterImplTest.java @@ -677,35 +677,41 @@ void agentlessWritesFlagEvaluationsDirectlyWhenLocalProxyIsUnavailable() throws .thenReturn(CONFIGURATION_SOURCE_AGENTLESS); when(config.getApiKey()).thenReturn(API_KEY); final BackendApiFactory backendApiFactory = mock(BackendApiFactory.class); - final IntakeApi directApi = - new IntakeApi( - HttpUrl.get(server.getAddress()).resolve("/api/v2/"), - API_KEY, - "123", - HttpRetryPolicy.Factory.NEVER_RETRY, - new OkHttpClient.Builder().build(), - false); - when(backendApiFactory.createDirectIntakeApi(Intake.EVENT_PLATFORM, false)) - .thenReturn(directApi); - final FeatureFlagBackendApiFactory featureFlagBackendApiFactory = - new FeatureFlagBackendApiFactory( - config, backendApiFactory, FeatureFlagEventType.FLAG_EVALUATION); - final PollingConditions poll = new PollingConditions(TIMEOUT_SECONDS); - - try (FlagEvaluationWriterImpl writer = - new FlagEvaluationWriterImpl( - 16, 1, TimeUnit.MILLISECONDS, featureFlagBackendApiFactory, config)) { - writer.startForTest(); - writer.enqueue(simpleEvent("direct-flag", "on")); - - poll.eventually( - () -> { - assertNotNull(server.getLastRequest()); - assertEquals(DIRECT_FLAG_EVALUATION_ENDPOINT, server.getLastRequest().getPath()); - assertEquals(API_KEY, server.getLastRequest().getHeader("dd-api-key")); - assertNull(server.getLastRequest().getHeader("X-Datadog-EVP-Subdomain")); - assertTrue(server.getLastRequest().getBody().length > 0); - }); + final OkHttpClient client = new OkHttpClient.Builder().build(); + try { + final IntakeApi directApi = + new IntakeApi( + HttpUrl.get(server.getAddress()).resolve("/api/v2/"), + API_KEY, + "123", + HttpRetryPolicy.Factory.NEVER_RETRY, + client, + false); + when(backendApiFactory.createDirectIntakeApi(Intake.EVENT_PLATFORM, false)) + .thenReturn(directApi); + final FeatureFlagBackendApiFactory featureFlagBackendApiFactory = + new FeatureFlagBackendApiFactory( + config, backendApiFactory, FeatureFlagEventType.FLAG_EVALUATION); + final PollingConditions poll = new PollingConditions(TIMEOUT_SECONDS); + + try (FlagEvaluationWriterImpl writer = + new FlagEvaluationWriterImpl( + 16, 1, TimeUnit.MILLISECONDS, featureFlagBackendApiFactory, config)) { + writer.startForTest(); + writer.enqueue(simpleEvent("direct-flag", "on")); + + poll.eventually( + () -> { + assertNotNull(server.getLastRequest()); + assertEquals(DIRECT_FLAG_EVALUATION_ENDPOINT, server.getLastRequest().getPath()); + assertEquals(API_KEY, server.getLastRequest().getHeader("dd-api-key")); + assertNull(server.getLastRequest().getHeader("X-Datadog-EVP-Subdomain")); + assertTrue(server.getLastRequest().getBody().length > 0); + }); + } + } finally { + client.dispatcher().executorService().shutdownNow(); + client.connectionPool().evictAll(); } } } From 9b8d7589aa5f75dcfebeef3b9a98fda2840df4ce Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Tue, 18 Aug 2026 13:10:09 -0700 Subject: [PATCH 14/14] fix(openfeature): prevent duplicate feature flag events --- .../communication/BackendApiFactory.java | 10 +- .../communication/BackendApiFactoryTest.java | 34 +++++ ...gEvaluationEnqueueContentionBenchmark.java | 9 +- .../FlagEvaluationHotPathBenchmark.java | 13 +- .../featureflag/ExposureWriterImpl.java | 5 +- .../FeatureFlagBackendApiFactory.java | 15 +- .../featureflag/FlagEvaluationWriterImpl.java | 134 +++--------------- .../featureflag/ExposureWriterTests.java | 64 +++++++++ .../FeatureFlagBackendApiFactoryTest.java | 10 +- .../FlagEvaluationTestSupport.java | 11 +- .../FlagEvaluationWriterImplTest.java | 49 ++++--- 11 files changed, 205 insertions(+), 149 deletions(-) diff --git a/communication/src/main/java/datadog/communication/BackendApiFactory.java b/communication/src/main/java/datadog/communication/BackendApiFactory.java index e48974605e3..2ac0447fc7d 100644 --- a/communication/src/main/java/datadog/communication/BackendApiFactory.java +++ b/communication/src/main/java/datadog/communication/BackendApiFactory.java @@ -64,13 +64,19 @@ public BackendApi createDirectIntakeApi(Intake intake, boolean responseCompressi responseCompression); } - /** Creates an API client that sends data through a compatible local EVP proxy. */ + /** Creates an API client that uses the specified retry policy with a compatible local proxy. */ public @Nullable BackendApi createEvpProxyApi(Intake intake) { return createEvpProxyApi(intake, true); } /** Creates an API client that sends data through a compatible local EVP proxy. */ public @Nullable BackendApi createEvpProxyApi(Intake intake, boolean responseCompression) { + return createEvpProxyApi(intake, responseCompression, retryPolicyFactory()); + } + + /** Creates an API client that sends data through a compatible local EVP proxy. */ + public @Nullable BackendApi createEvpProxyApi( + Intake intake, boolean responseCompression, HttpRetryPolicy.Factory retryPolicyFactory) { DDAgentFeaturesDiscovery featuresDiscovery = sharedCommunicationObjects.featuresDiscovery(config); featuresDiscovery.discoverIfOutdated(); @@ -91,7 +97,7 @@ public BackendApi createDirectIntakeApi(Intake intake, boolean responseCompressi traceId, evpProxyUrl, subdomain, - retryPolicyFactory(), + retryPolicyFactory, sharedCommunicationObjects.agentHttpClient, responseCompression); } diff --git a/communication/src/test/java/datadog/communication/BackendApiFactoryTest.java b/communication/src/test/java/datadog/communication/BackendApiFactoryTest.java index 7aa33de7742..726c34a7f73 100644 --- a/communication/src/test/java/datadog/communication/BackendApiFactoryTest.java +++ b/communication/src/test/java/datadog/communication/BackendApiFactoryTest.java @@ -4,9 +4,11 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; import datadog.communication.ddagent.DDAgentFeaturesDiscovery; import datadog.communication.ddagent.SharedCommunicationObjects; +import datadog.communication.http.HttpRetryPolicy; import datadog.metrics.api.Monitoring; import datadog.trace.api.Config; import datadog.trace.api.ProtocolVersion; @@ -61,6 +63,38 @@ void advertisedEvpProxyEndpointSupportsDisabledResponseCompression() throws Exce } } + @Test + void explicitNoRetryProxyPolicyDoesNotReplayAmbiguousFailure() throws Exception { + final MockWebServer agent = new MockWebServer(); + agent.enqueue(new MockResponse().setResponseCode(500).setBody("ambiguous")); + agent.enqueue(new MockResponse().setResponseCode(200).setBody("{}")); + agent.start(); + try { + final FakeFeaturesDiscovery discovery = new FakeFeaturesDiscovery(V4_EVP_PROXY_ENDPOINT); + final BackendApiFactory factory = + new BackendApiFactory( + Config.get(), sharedCommunicationObjects(discovery, agent.url("/"))); + final BackendApi api = + factory.createEvpProxyApi( + Intake.EVENT_PLATFORM, false, HttpRetryPolicy.Factory.NEVER_RETRY); + + assertNotNull(api); + assertThrows( + HttpResponseException.class, + () -> + api.post( + "flagevaluation", + RequestBody.create(JSON, "{}".getBytes(StandardCharsets.UTF_8)), + stream -> null, + null, + false)); + + assertEquals(1, agent.getRequestCount()); + } finally { + agent.shutdown(); + } + } + private static SharedCommunicationObjects sharedCommunicationObjects( final DDAgentFeaturesDiscovery discovery, final HttpUrl agentUrl) { final TestSharedCommunicationObjects sco = new TestSharedCommunicationObjects(discovery); diff --git a/products/feature-flagging/feature-flagging-lib/src/jmh/java/com/datadog/featureflag/FlagEvaluationEnqueueContentionBenchmark.java b/products/feature-flagging/feature-flagging-lib/src/jmh/java/com/datadog/featureflag/FlagEvaluationEnqueueContentionBenchmark.java index 8da76d4a88e..bbec5d65229 100644 --- a/products/feature-flagging/feature-flagging-lib/src/jmh/java/com/datadog/featureflag/FlagEvaluationEnqueueContentionBenchmark.java +++ b/products/feature-flagging/feature-flagging-lib/src/jmh/java/com/datadog/featureflag/FlagEvaluationEnqueueContentionBenchmark.java @@ -7,6 +7,7 @@ import datadog.trace.api.Config; import datadog.trace.api.featureflag.FeatureFlaggingGateway; import datadog.trace.api.featureflag.flagevaluation.FlagEvalEvent; +import datadog.trace.api.intake.Intake; import de.thetaphi.forbiddenapis.SuppressForbidden; import java.util.HashMap; import java.util.Map; @@ -90,7 +91,13 @@ public void setUp() { final Config config = Config.get(); final BackendApiFactory factory = new BackendApiFactory(config, null); // Capacity well above what the batch-draining consumer should ever let build up. - writer = new FlagEvaluationWriterImpl(1 << 20, Long.MAX_VALUE, NANOSECONDS, factory, config); + writer = + new FlagEvaluationWriterImpl( + 1 << 20, + Long.MAX_VALUE, + NANOSECONDS, + () -> factory.createBackendApi(Intake.EVENT_PLATFORM, false), + config); } /** diff --git a/products/feature-flagging/feature-flagging-lib/src/jmh/java/com/datadog/featureflag/FlagEvaluationHotPathBenchmark.java b/products/feature-flagging/feature-flagging-lib/src/jmh/java/com/datadog/featureflag/FlagEvaluationHotPathBenchmark.java index e22d05bee07..fbd467b88d2 100644 --- a/products/feature-flagging/feature-flagging-lib/src/jmh/java/com/datadog/featureflag/FlagEvaluationHotPathBenchmark.java +++ b/products/feature-flagging/feature-flagging-lib/src/jmh/java/com/datadog/featureflag/FlagEvaluationHotPathBenchmark.java @@ -6,6 +6,7 @@ import datadog.communication.BackendApiFactory; import datadog.trace.api.Config; import datadog.trace.api.featureflag.flagevaluation.FlagEvalEvent; +import datadog.trace.api.intake.Intake; import java.util.HashMap; import java.util.Map; import org.openjdk.jmh.annotations.Benchmark; @@ -79,10 +80,18 @@ public void setUp() { final BackendApiFactory factory = new BackendApiFactory(config, null); final Map ddContext = new HashMap<>(); ddContext.put("service", "bench-service"); - handler = FlagEvaluationWriterImpl.createHandlerForTest(factory, ddContext); + handler = + FlagEvaluationWriterImpl.createHandlerForTest( + () -> factory.createBackendApi(Intake.EVENT_PLATFORM, false), ddContext); // Capacity large enough that the benchmark never overflows within a measurement window. - writer = new FlagEvaluationWriterImpl(1 << 20, Long.MAX_VALUE, NANOSECONDS, factory, config); + writer = + new FlagEvaluationWriterImpl( + 1 << 20, + Long.MAX_VALUE, + NANOSECONDS, + () -> factory.createBackendApi(Intake.EVENT_PLATFORM, false), + config); } /** diff --git a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/ExposureWriterImpl.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/ExposureWriterImpl.java index 017616ed070..fcd50e5dc34 100644 --- a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/ExposureWriterImpl.java +++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/ExposureWriterImpl.java @@ -187,9 +187,12 @@ protected void flushIfNecessary() { } try { evpPublisher.post(EXPOSURES_ROUTE, payload); - this.buffer.clear(); } catch (Exception e) { LOGGER.debug("Could not submit exposures", e); + } finally { + // Best-effort delivery must not retry an ambiguously accepted batch. A later definitive + // proxy rejection could otherwise replay the same exposures through direct intake. + this.buffer.clear(); } } } diff --git a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FeatureFlagBackendApiFactory.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FeatureFlagBackendApiFactory.java index 1727d156915..0dbf9c74254 100644 --- a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FeatureFlagBackendApiFactory.java +++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FeatureFlagBackendApiFactory.java @@ -5,6 +5,7 @@ import datadog.communication.BackendApi; import datadog.communication.BackendApiFactory; import datadog.communication.ddagent.SharedCommunicationObjects; +import datadog.communication.http.HttpRetryPolicy; import datadog.trace.api.Config; import datadog.trace.api.intake.Intake; import javax.annotation.Nullable; @@ -38,9 +39,17 @@ final class FeatureFlagBackendApiFactory { @Nullable BackendApi create() { + final boolean directFallbackAvailable = + CONFIGURATION_SOURCE_AGENTLESS.equals(config.getFeatureFlaggingConfigurationSource()) + && hasDirectCredentials(); final BackendApi proxyApi = - backendApiFactory.createEvpProxyApi( - Intake.EVENT_PLATFORM, eventType.responseCompressionEnabled()); + directFallbackAvailable + ? backendApiFactory.createEvpProxyApi( + Intake.EVENT_PLATFORM, + eventType.responseCompressionEnabled(), + HttpRetryPolicy.Factory.NEVER_RETRY) + : backendApiFactory.createEvpProxyApi( + Intake.EVENT_PLATFORM, eventType.responseCompressionEnabled()); if (!CONFIGURATION_SOURCE_AGENTLESS.equals(config.getFeatureFlaggingConfigurationSource())) { if (proxyApi == null) { LOGGER.warn( @@ -51,7 +60,7 @@ BackendApi create() { } if (proxyApi != null) { - if (hasDirectCredentials()) { + if (directFallbackAvailable) { return new AgentlessFeatureFlagBackendApi( proxyApi, this::createDirectApi, eventType.logName()); } diff --git a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FlagEvaluationWriterImpl.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FlagEvaluationWriterImpl.java index 1bc203022ac..2da8ffe0256 100644 --- a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FlagEvaluationWriterImpl.java +++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FlagEvaluationWriterImpl.java @@ -7,14 +7,12 @@ import datadog.common.queue.MessagePassingBlockingQueue; import datadog.common.queue.Queues; import datadog.communication.BackendApi; -import datadog.communication.BackendApiFactory; import datadog.communication.EvpProxy; import datadog.communication.ddagent.SharedCommunicationObjects; import datadog.trace.api.Config; import datadog.trace.api.featureflag.FeatureFlaggingGateway; import datadog.trace.api.featureflag.flagevaluation.FlagEvalEvent; import datadog.trace.api.featureflag.flagevaluation.FlagEvaluationWriter; -import datadog.trace.api.intake.Intake; import datadog.trace.api.telemetry.CoreMetricCollector; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import java.util.ArrayList; @@ -33,8 +31,8 @@ * EVP flagevaluation writer for Java. * *

Uses the same EVP publisher path as ExposureWriterImpl, with two-tier aggregation replacing - * the single-exposure buffer. Routes to the Agent-advertised EVP proxy endpoint for - * /api/v2/flagevaluation. + * the single-exposure buffer. Uses a local EVP proxy when available. Agentless mode can use direct + * intake when no compatible local route is available. * *

Two-tier aggregation contract: Full key: (flagKey, variant, allocationKey, runtimeDefault, * errorMessage, targetingKey, canonical-context-key). Degraded key: (flagKey, variant, @@ -102,51 +100,15 @@ private static void countMetric(final String metricName, final long value, final new ConcurrentHashMap<>(); public FlagEvaluationWriterImpl(final SharedCommunicationObjects sco, final Config config) { - this(DEFAULT_CAPACITY, FLUSH_INTERVAL_SECONDS, SECONDS, sco, config); - } - - FlagEvaluationWriterImpl( - final int capacity, - final long flushInterval, - final TimeUnit timeUnit, - final SharedCommunicationObjects sco, - final Config config) { - this( - capacity, - flushInterval, - timeUnit, - new FeatureFlagBackendApiFactory(config, sco, FeatureFlagEventType.FLAG_EVALUATION), - config); - } - - /** Package-private constructor allowing a BackendApiFactory to be injected for tests. */ - FlagEvaluationWriterImpl( - final int capacity, - final long flushInterval, - final TimeUnit timeUnit, - final BackendApiFactory backendApiFactory, - final Config config) { this( - capacity, - flushInterval, - timeUnit, - () -> - backendApiFactory.createBackendApi( - Intake.EVENT_PLATFORM, - FeatureFlagEventType.FLAG_EVALUATION.responseCompressionEnabled()), + DEFAULT_CAPACITY, + FLUSH_INTERVAL_SECONDS, + SECONDS, + new FeatureFlagBackendApiFactory(config, sco, FeatureFlagEventType.FLAG_EVALUATION)::create, config); } FlagEvaluationWriterImpl( - final int capacity, - final long flushInterval, - final TimeUnit timeUnit, - final FeatureFlagBackendApiFactory backendApiFactory, - final Config config) { - this(capacity, flushInterval, timeUnit, backendApiFactory::create, config); - } - - private FlagEvaluationWriterImpl( final int capacity, final long flushInterval, final TimeUnit timeUnit, @@ -162,7 +124,8 @@ private FlagEvaluationWriterImpl( FeatureFlagEvpContext.from(config), droppedQueueOverflow, contextTruncatedCounts, - this::close); + this::close, + FLAG_EVALUATION_PAYLOAD_SIZE_LIMIT_BYTES); this.serializerThread = newAgentThread(FEATURE_FLAG_EVALUATION_PROCESSOR, serializer); } @@ -342,70 +305,6 @@ static class FlagEvaluationSerializingHandler implements Runnable { private final AtomicBoolean shutdownRequested = new AtomicBoolean(false); private final CountDownLatch finalFlushDone = new CountDownLatch(1); - FlagEvaluationSerializingHandler( - final BackendApiFactory backendApiFactory, - final MessagePassingBlockingQueue queue, - final long flushInterval, - final TimeUnit timeUnit, - final Map context, - final AtomicLong droppedQueueOverflow, - final ConcurrentHashMap contextTruncatedCounts, - final Runnable errorCallback) { - this( - () -> backendApiFactory.createBackendApi(Intake.EVENT_PLATFORM, false), - queue, - flushInterval, - timeUnit, - context, - droppedQueueOverflow, - contextTruncatedCounts, - errorCallback, - FLAG_EVALUATION_PAYLOAD_SIZE_LIMIT_BYTES); - } - - FlagEvaluationSerializingHandler( - final BackendApiFactory backendApiFactory, - final MessagePassingBlockingQueue queue, - final long flushInterval, - final TimeUnit timeUnit, - final Map context, - final AtomicLong droppedQueueOverflow, - final ConcurrentHashMap contextTruncatedCounts, - final Runnable errorCallback, - final int payloadSizeLimitBytes) { - this( - () -> backendApiFactory.createBackendApi(Intake.EVENT_PLATFORM, false), - queue, - flushInterval, - timeUnit, - context, - droppedQueueOverflow, - contextTruncatedCounts, - errorCallback, - payloadSizeLimitBytes); - } - - FlagEvaluationSerializingHandler( - final Supplier backendApiSupplier, - final MessagePassingBlockingQueue queue, - final long flushInterval, - final TimeUnit timeUnit, - final Map context, - final AtomicLong droppedQueueOverflow, - final ConcurrentHashMap contextTruncatedCounts, - final Runnable errorCallback) { - this( - backendApiSupplier, - queue, - flushInterval, - timeUnit, - context, - droppedQueueOverflow, - contextTruncatedCounts, - errorCallback, - FLAG_EVALUATION_PAYLOAD_SIZE_LIMIT_BYTES); - } - FlagEvaluationSerializingHandler( final Supplier backendApiSupplier, final MessagePassingBlockingQueue queue, @@ -622,16 +521,17 @@ private boolean shouldFlush() { */ static class SerializingHandlerForTest extends FlagEvaluationSerializingHandler { - SerializingHandlerForTest(final BackendApiFactory factory, final Map context) { - this(factory, context, FLAG_EVALUATION_PAYLOAD_SIZE_LIMIT_BYTES); + SerializingHandlerForTest( + final Supplier backendApiSupplier, final Map context) { + this(backendApiSupplier, context, FLAG_EVALUATION_PAYLOAD_SIZE_LIMIT_BYTES); } SerializingHandlerForTest( - final BackendApiFactory factory, + final Supplier backendApiSupplier, final Map context, final int payloadSizeLimitBytes) { super( - factory, + backendApiSupplier, Queues.mpscBlockingConsumerArrayQueue(DEFAULT_CAPACITY), Long.MAX_VALUE, // effectively never auto-flush TimeUnit.NANOSECONDS, @@ -695,14 +595,14 @@ int fullTierSizeForTest() { /** Factory method for test use - creates a SerializingHandlerForTest. */ static SerializingHandlerForTest createHandlerForTest( - final BackendApiFactory factory, final Map context) { - return new SerializingHandlerForTest(factory, context); + final Supplier backendApiSupplier, final Map context) { + return new SerializingHandlerForTest(backendApiSupplier, context); } static SerializingHandlerForTest createHandlerForTest( - final BackendApiFactory factory, + final Supplier backendApiSupplier, final Map context, final int payloadSizeLimitBytes) { - return new SerializingHandlerForTest(factory, context, payloadSizeLimitBytes); + return new SerializingHandlerForTest(backendApiSupplier, context, payloadSizeLimitBytes); } } diff --git a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/ExposureWriterTests.java b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/ExposureWriterTests.java index 78e4a72ebba..daaf213ebfb 100644 --- a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/ExposureWriterTests.java +++ b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/ExposureWriterTests.java @@ -9,11 +9,17 @@ import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import com.squareup.moshi.JsonAdapter; import com.squareup.moshi.Moshi; +import datadog.communication.BackendApi; import datadog.communication.BackendApiFactory; import datadog.communication.IntakeApi; import datadog.communication.ddagent.DDAgentFeaturesDiscovery; @@ -30,8 +36,11 @@ import datadog.trace.api.featureflag.exposure.Flag; import datadog.trace.api.featureflag.exposure.Subject; import datadog.trace.api.featureflag.exposure.Variant; +import datadog.trace.api.intake.Intake; import datadog.trace.test.util.PollingConditions; import java.io.ByteArrayInputStream; +import java.net.ConnectException; +import java.net.SocketTimeoutException; import java.util.ArrayList; import java.util.Collections; import java.util.Iterator; @@ -50,12 +59,15 @@ import java.util.concurrent.Future; import okhttp3.HttpUrl; import okhttp3.OkHttpClient; +import okhttp3.RequestBody; +import okio.Buffer; import okio.Okio; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; +import org.mockito.ArgumentCaptor; import org.tabletest.junit.TableTest; class ExposureWriterTests { @@ -288,6 +300,58 @@ void testSerializationFailureDoesNotPoisonFollowingExposures() throws Exception } } + @Test + void testAmbiguousExposureBatchIsNotRetriedOrReplayedDirectly() throws Exception { + final Config config = mockConfig("test-service"); + when(config.getFeatureFlaggingConfigurationSource()).thenReturn(CONFIGURATION_SOURCE_AGENTLESS); + when(config.getApiKey()).thenReturn(API_KEY); + final BackendApiFactory backendApiFactory = mock(BackendApiFactory.class); + final BackendApi proxyApi = mock(BackendApi.class); + final BackendApi directApi = mock(BackendApi.class); + when(backendApiFactory.createEvpProxyApi( + Intake.EVENT_PLATFORM, true, HttpRetryPolicy.Factory.NEVER_RETRY)) + .thenReturn(proxyApi); + when(backendApiFactory.createDirectIntakeApi(Intake.EVENT_PLATFORM, true)) + .thenReturn(directApi); + when(proxyApi.post(eq("exposures"), any(RequestBody.class), any(), any(), eq(false))) + .thenThrow(new SocketTimeoutException("ambiguous timeout")) + .thenThrow(new ConnectException("definitive refusal")); + final FeatureFlagBackendApiFactory featureFlagBackendApiFactory = + new FeatureFlagBackendApiFactory(config, backendApiFactory, FeatureFlagEventType.EXPOSURE); + final List exposures = buildExposures(2); + + try (ExposureWriterImpl writer = + new ExposureWriterImpl(1 << 4, 100, MILLISECONDS, featureFlagBackendApiFactory, config)) { + writer.init(); + writer.accept(exposures.get(0)); + + poll.eventually( + () -> + verify(proxyApi) + .post(eq("exposures"), any(RequestBody.class), any(), any(), eq(false))); + MILLISECONDS.sleep(300); + verify(proxyApi, times(1)) + .post(eq("exposures"), any(RequestBody.class), any(), any(), eq(false)); + verify(directApi, never()) + .post(eq("exposures"), any(RequestBody.class), any(), any(), eq(false)); + + writer.accept(exposures.get(1)); + poll.eventually( + () -> + verify(directApi) + .post(eq("exposures"), any(RequestBody.class), any(), any(), eq(false))); + + final ArgumentCaptor directBody = ArgumentCaptor.forClass(RequestBody.class); + verify(directApi).post(eq("exposures"), directBody.capture(), any(), any(), eq(false)); + final Buffer buffer = new Buffer(); + directBody.getValue().writeTo(buffer); + final ExposuresRequest directRequest = + new Moshi.Builder().build().adapter(ExposuresRequest.class).fromJson(buffer.readUtf8()); + assertNotNull(directRequest); + assertExposures(directRequest.exposures, singletonList(exposures.get(1))); + } + } + @Test void testWriterStopsReceivingExposuresIfEvpProxyIsNotAvailable() throws Exception { SharedCommunicationObjects sharedCommunicationObjects = sharedCommunicationObjects(false); diff --git a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FeatureFlagBackendApiFactoryTest.java b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FeatureFlagBackendApiFactoryTest.java index ddd873a1e9d..8b117742715 100644 --- a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FeatureFlagBackendApiFactoryTest.java +++ b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FeatureFlagBackendApiFactoryTest.java @@ -14,6 +14,7 @@ import datadog.communication.BackendApi; import datadog.communication.BackendApiFactory; +import datadog.communication.http.HttpRetryPolicy; import datadog.trace.api.Config; import datadog.trace.api.intake.Intake; import org.junit.jupiter.api.Test; @@ -51,7 +52,8 @@ void remoteConfigDisablesDeliveryWhenLocalEvpProxyIsUnavailable() { void agentlessPrefersLocalEvpProxyWithDirectFallback() { final Config config = config(CONFIGURATION_SOURCE_AGENTLESS, "api-key"); final BackendApiFactory backendApiFactory = mock(BackendApiFactory.class); - when(backendApiFactory.createEvpProxyApi(Intake.EVENT_PLATFORM, false)) + when(backendApiFactory.createEvpProxyApi( + Intake.EVENT_PLATFORM, false, HttpRetryPolicy.Factory.NEVER_RETRY)) .thenReturn(mock(BackendApi.class)); when(backendApiFactory.createDirectIntakeApi(Intake.EVENT_PLATFORM, false)) .thenReturn(mock(BackendApi.class)); @@ -60,6 +62,8 @@ void agentlessPrefersLocalEvpProxyWithDirectFallback() { new FeatureFlagBackendApiFactory(config, backendApiFactory, FLAG_EVALUATION).create(); assertInstanceOf(AgentlessFeatureFlagBackendApi.class, selected); + verify(backendApiFactory) + .createEvpProxyApi(Intake.EVENT_PLATFORM, false, HttpRetryPolicy.Factory.NEVER_RETRY); verify(backendApiFactory, never()).createDirectIntakeApi(Intake.EVENT_PLATFORM, false); } @@ -119,7 +123,9 @@ void agentlessDoesNotValidateDirectUrlWhileLocalRouteIsAvailable() { final Config config = config(CONFIGURATION_SOURCE_AGENTLESS, "api-key"); final BackendApiFactory backendApiFactory = mock(BackendApiFactory.class); final BackendApi proxyApi = mock(BackendApi.class); - when(backendApiFactory.createEvpProxyApi(Intake.EVENT_PLATFORM, false)).thenReturn(proxyApi); + when(backendApiFactory.createEvpProxyApi( + Intake.EVENT_PLATFORM, false, HttpRetryPolicy.Factory.NEVER_RETRY)) + .thenReturn(proxyApi); when(backendApiFactory.createDirectIntakeApi(Intake.EVENT_PLATFORM, false)) .thenThrow(new IllegalArgumentException("invalid URL")); diff --git a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationTestSupport.java b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationTestSupport.java index 4a65a81a8bf..502f0e76c69 100644 --- a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationTestSupport.java +++ b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationTestSupport.java @@ -16,6 +16,7 @@ import datadog.communication.BackendApi; import datadog.communication.BackendApiFactory; import datadog.trace.api.featureflag.flagevaluation.FlagEvalEvent; +import datadog.trace.api.intake.Intake; import datadog.trace.api.telemetry.CoreMetricCollector; import datadog.trace.api.telemetry.MetricCollector; import java.lang.reflect.Type; @@ -25,6 +26,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.function.Supplier; import okhttp3.RequestBody; import okio.Buffer; @@ -41,6 +43,10 @@ final class FlagEvaluationTestSupport { private FlagEvaluationTestSupport() {} + static Supplier backendApiSupplier(final BackendApiFactory factory) { + return () -> factory.createBackendApi(Intake.EVENT_PLATFORM, false); + } + static void clearCoreMetrics() { CoreMetricCollector.getInstance().drain(); } @@ -98,7 +104,7 @@ static TestWriterSetup buildTestWriter(final BackendApi mockEvp) { context.put("service", "test-service"); final FlagEvaluationWriterImpl.SerializingHandlerForTest handler = - FlagEvaluationWriterImpl.createHandlerForTest(factory, context); + FlagEvaluationWriterImpl.createHandlerForTest(backendApiSupplier(factory), context); return new TestWriterSetup(handler, mockEvp, factory); } @@ -112,7 +118,8 @@ static TestWriterSetup buildTestWriter( context.put("service", "test-service"); final FlagEvaluationWriterImpl.SerializingHandlerForTest handler = - FlagEvaluationWriterImpl.createHandlerForTest(factory, context, payloadSizeLimitBytes); + FlagEvaluationWriterImpl.createHandlerForTest( + backendApiSupplier(factory), context, payloadSizeLimitBytes); return new TestWriterSetup(handler, mockEvp, factory); } diff --git a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationWriterImplTest.java b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationWriterImplTest.java index 010f5ea5568..ba3d03ec6d5 100644 --- a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationWriterImplTest.java +++ b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationWriterImplTest.java @@ -1,6 +1,7 @@ package com.datadog.featureflag; import static com.datadog.featureflag.FlagEvaluationTestSupport.JSON_MAP; +import static com.datadog.featureflag.FlagEvaluationTestSupport.backendApiSupplier; import static com.datadog.featureflag.FlagEvaluationTestSupport.buildTestWriter; import static com.datadog.featureflag.FlagEvaluationTestSupport.cfg; import static com.datadog.featureflag.FlagEvaluationTestSupport.clearCoreMetrics; @@ -105,7 +106,8 @@ void startRegistersWriterAndCloseDeregistersIt() { final BackendApiFactory factory = mock(BackendApiFactory.class); when(factory.createBackendApi(any(), anyBoolean())).thenReturn(mockEvp); final FlagEvaluationWriterImpl writer = - new FlagEvaluationWriterImpl(16, Long.MAX_VALUE, TimeUnit.NANOSECONDS, factory, cfg()); + new FlagEvaluationWriterImpl( + 16, Long.MAX_VALUE, TimeUnit.NANOSECONDS, backendApiSupplier(factory), cfg()); writer.start(); assertEquals(writer, FeatureFlaggingGateway.getFlagEvalWriter()); @@ -123,7 +125,7 @@ void queueOverflowIncrementsObservableDropCounter() { final BackendApiFactory factory = mock(BackendApiFactory.class); when(factory.createBackendApi(any(), anyBoolean())).thenReturn(mockEvp); final FlagEvaluationWriterImpl writer = - new FlagEvaluationWriterImpl(2, 10L, TimeUnit.SECONDS, factory, cfg()); + new FlagEvaluationWriterImpl(2, 10L, TimeUnit.SECONDS, backendApiSupplier(factory), cfg()); for (int i = 0; i < 100; i++) { writer.enqueue(simpleEvent("of-flag", "on")); @@ -148,7 +150,8 @@ void enqueueAfterCloseIsDroppedAndCounted() { final BackendApiFactory factory = mock(BackendApiFactory.class); when(factory.createBackendApi(any(), anyBoolean())).thenReturn(mockEvp); final FlagEvaluationWriterImpl writer = - new FlagEvaluationWriterImpl(16, Long.MAX_VALUE, TimeUnit.NANOSECONDS, factory, cfg()); + new FlagEvaluationWriterImpl( + 16, Long.MAX_VALUE, TimeUnit.NANOSECONDS, backendApiSupplier(factory), cfg()); writer.close(); writer.enqueue(simpleEvent("closed-flag", "on")); @@ -172,7 +175,8 @@ void enqueueDisabledDropsAndCountsAsClosedDrop() { final BackendApiFactory factory = mock(BackendApiFactory.class); when(factory.createBackendApi(any(), anyBoolean())).thenReturn(mockEvp); final FlagEvaluationWriterImpl writer = - new FlagEvaluationWriterImpl(16, Long.MAX_VALUE, TimeUnit.NANOSECONDS, factory, cfg()); + new FlagEvaluationWriterImpl( + 16, Long.MAX_VALUE, TimeUnit.NANOSECONDS, backendApiSupplier(factory), cfg()); FeatureFlaggingGateway.setFlagEvaluationEnqueueEnabled(false); writer.enqueue(simpleEvent("disabled-flag", "on")); @@ -194,7 +198,8 @@ void closeSweepsAndCountsEventsLeftInTheQueue() { final BackendApiFactory factory = mock(BackendApiFactory.class); when(factory.createBackendApi(any(), anyBoolean())).thenReturn(mockEvp); final FlagEvaluationWriterImpl writer = - new FlagEvaluationWriterImpl(16, Long.MAX_VALUE, TimeUnit.NANOSECONDS, factory, cfg()); + new FlagEvaluationWriterImpl( + 16, Long.MAX_VALUE, TimeUnit.NANOSECONDS, backendApiSupplier(factory), cfg()); // The worker is never started, so nothing drains these; close() must account for them rather // than leave them silently stranded. Stands in for the narrow window where a lock-free @@ -220,7 +225,8 @@ void enqueueIgnoresNullEvent() { final BackendApiFactory factory = mock(BackendApiFactory.class); when(factory.createBackendApi(any(), anyBoolean())).thenReturn(mockEvp); final FlagEvaluationWriterImpl writer = - new FlagEvaluationWriterImpl(16, Long.MAX_VALUE, TimeUnit.NANOSECONDS, factory, cfg()); + new FlagEvaluationWriterImpl( + 16, Long.MAX_VALUE, TimeUnit.NANOSECONDS, backendApiSupplier(factory), cfg()); writer.enqueue(null); @@ -234,7 +240,8 @@ void enqueueDoesNotAggregateOnTheCallingThread() { final BackendApiFactory factory = mock(BackendApiFactory.class); when(factory.createBackendApi(any(), anyBoolean())).thenReturn(mockEvp); final FlagEvaluationWriterImpl writer = - new FlagEvaluationWriterImpl(16, Long.MAX_VALUE, TimeUnit.NANOSECONDS, factory, cfg()); + new FlagEvaluationWriterImpl( + 16, Long.MAX_VALUE, TimeUnit.NANOSECONDS, backendApiSupplier(factory), cfg()); writer.enqueue(simpleEvent("g2-flag", "on")); writer.enqueue(simpleEvent("g2-flag", "on")); @@ -247,7 +254,7 @@ void enqueueDoesNotAggregateOnTheCallingThread() { void handlerRunFailsFastWhenEvpProxyIsUnavailable() { final BackendApiFactory factory = mock(BackendApiFactory.class); final FlagEvaluationWriterImpl.SerializingHandlerForTest handler = - FlagEvaluationWriterImpl.createHandlerForTest(factory, context()); + FlagEvaluationWriterImpl.createHandlerForTest(backendApiSupplier(factory), context()); assertThrows(IllegalArgumentException.class, handler::run); } @@ -270,14 +277,15 @@ void flushIfNecessaryDoesNotReturnEarlyWhenOnlyQueueDropsArePending() { final AtomicLong queueDrops = new AtomicLong(1); final FlagEvaluationWriterImpl.FlagEvaluationSerializingHandler handler = new FlagEvaluationWriterImpl.FlagEvaluationSerializingHandler( - mock(BackendApiFactory.class), - Queues.mpscBlockingConsumerArrayQueue(16), + () -> null, + Queues.mpscBlockingConsumerArrayQueue(16), Long.MAX_VALUE, TimeUnit.NANOSECONDS, context(), queueDrops, new java.util.concurrent.ConcurrentHashMap<>(), - () -> {}); + () -> {}, + FlagEvaluationWriterImpl.FLAG_EVALUATION_PAYLOAD_SIZE_LIMIT_BYTES); handler.flushIfNecessary(); @@ -300,14 +308,15 @@ void workerHandlesEmptyPolls() throws Exception { }); final FlagEvaluationWriterImpl.FlagEvaluationSerializingHandler handler = new FlagEvaluationWriterImpl.FlagEvaluationSerializingHandler( - factory, + backendApiSupplier(factory), queue, Long.MAX_VALUE, TimeUnit.NANOSECONDS, context(), new AtomicLong(0), new java.util.concurrent.ConcurrentHashMap<>(), - () -> {}); + () -> {}, + FlagEvaluationWriterImpl.FLAG_EVALUATION_PAYLOAD_SIZE_LIMIT_BYTES); handler.run(); @@ -379,7 +388,7 @@ void finalFlushRunsWithoutTheInterruptFlagSet() throws Exception { when(factory.createBackendApi(any(), anyBoolean())).thenReturn(mockEvp); final FlagEvaluationWriterImpl writer = new FlagEvaluationWriterImpl( - 64, TimeUnit.DAYS.toSeconds(1), TimeUnit.SECONDS, factory, cfg()); + 64, TimeUnit.DAYS.toSeconds(1), TimeUnit.SECONDS, backendApiSupplier(factory), cfg()); writer.startForTest(); writer.enqueue(simpleEvent("interrupt-flag", "on")); @@ -408,7 +417,7 @@ void closeDrainsAndFinalFlushesQueuedEvents() throws Exception { when(factory.createBackendApi(any(), anyBoolean())).thenReturn(mockEvp); final FlagEvaluationWriterImpl writer = new FlagEvaluationWriterImpl( - 64, TimeUnit.DAYS.toSeconds(1), TimeUnit.SECONDS, factory, cfg()); + 64, TimeUnit.DAYS.toSeconds(1), TimeUnit.SECONDS, backendApiSupplier(factory), cfg()); writer.startForTest(); writer.enqueue(simpleEvent("shutdown-flag", "on")); @@ -428,7 +437,8 @@ void continuousTrafficFlushesWithoutWaitingForIdle() throws Exception { final BackendApiFactory factory = mock(BackendApiFactory.class); when(factory.createBackendApi(any(), anyBoolean())).thenReturn(mockEvp); final FlagEvaluationWriterImpl writer = - new FlagEvaluationWriterImpl(1 << 12, 1, TimeUnit.MILLISECONDS, factory, cfg()); + new FlagEvaluationWriterImpl( + 1 << 12, 1, TimeUnit.MILLISECONDS, backendApiSupplier(factory), cfg()); writer.startForTest(); boolean posted = false; @@ -696,7 +706,7 @@ void agentlessWritesFlagEvaluationsDirectlyWhenLocalProxyIsUnavailable() throws try (FlagEvaluationWriterImpl writer = new FlagEvaluationWriterImpl( - 16, 1, TimeUnit.MILLISECONDS, featureFlagBackendApiFactory, config)) { + 16, 1, TimeUnit.MILLISECONDS, featureFlagBackendApiFactory::create, config)) { writer.startForTest(); writer.enqueue(simpleEvent("direct-flag", "on")); @@ -722,7 +732,8 @@ void countContextTruncatedAccumulatesPerReason() { final BackendApiFactory factory = mock(BackendApiFactory.class); when(factory.createBackendApi(any(), anyBoolean())).thenReturn(mockEvp); final FlagEvaluationWriterImpl writer = - new FlagEvaluationWriterImpl(16, Long.MAX_VALUE, TimeUnit.NANOSECONDS, factory, cfg()); + new FlagEvaluationWriterImpl( + 16, Long.MAX_VALUE, TimeUnit.NANOSECONDS, backendApiSupplier(factory), cfg()); writer.countContextTruncated("field_count"); writer.countContextTruncated("field_count"); @@ -753,7 +764,7 @@ void hasCapacityForEnqueueReflectsQueueSaturationAndCountsPreQueueOverflow() { final int capacity = 2; final FlagEvaluationWriterImpl writer = new FlagEvaluationWriterImpl( - capacity, Long.MAX_VALUE, TimeUnit.NANOSECONDS, factory, cfg()); + capacity, Long.MAX_VALUE, TimeUnit.NANOSECONDS, backendApiSupplier(factory), cfg()); assertTrue(writer.hasCapacityForEnqueue());