Skip to content

feat: DH-23262, DH-23265: Iceberg 1.11.0 compatibility #8331

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
devinrsmith merged 3 commits into from
Aug 7, 2026
Merged

feat: DH-23262, DH-23265: Iceberg 1.11.0 compatibility #8331

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
8e68d29
feat: DH-23262, DH-23265: Iceberg 1.11.0 compatibility
devinrsmith Aug 6, 2026
2c6476d
bump to 1.10.2
devinrsmith Aug 6, 2026
dcb1f5d
empty commit
devinrsmith Aug 7, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion extensions/iceberg/s3/src/main/java/io/deephaven/iceberg/util/DeephavenAwsClientFactory.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,9 @@ public S3AsyncClient s3Async() {
return S3AsyncClient.builder()
.applyMutation(asyncHttpClientProperties::applyAsyncHttpClientConfigurations)
.applyMutation(awsClientProperties::applyClientRegionConfiguration)
.applyMutation(awsClientProperties::applyClientCredentialConfigurations)
.applyMutation(awsClientProperties::applyLegacyMd5Plugin)
.applyMutation(
b -> s3FileIOProperties.applyCredentialConfigurations(awsClientProperties, b))
.applyMutation(s3FileIOProperties::applyEndpointConfigurations)
.build();
}
Expand Down
44 changes: 0 additions & 44 deletions ...berg/s3/src/main/java/io/deephaven/iceberg/util/DeephavenS3ClientCredentialsProvider.java
View file
Open in desktop

This file was deleted.

18 changes: 1 addition & 17 deletions extensions/iceberg/s3/src/main/java/io/deephaven/iceberg/util/IcebergToolsS3.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,8 @@
package io.deephaven.iceberg.util;

import com.google.common.base.Strings;
import io.deephaven.extensions.s3.S3InstructionsBasedAwsClientFactory;
import io.deephaven.extensions.s3.S3Instructions;
import io.deephaven.extensions.s3.S3InstructionsBasedAwsClientFactory;
import io.deephaven.util.reference.CleanupReferenceProcessor;
import org.apache.hadoop.conf.Configuration;
import org.apache.iceberg.CatalogProperties;
Expand All @@ -19,7 +19,6 @@
import org.apache.iceberg.rest.RESTCatalog;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import org.jetbrains.annotations.VisibleForTesting;

import java.util.HashMap;
import java.util.Map;
Expand All @@ -29,15 +28,6 @@
*/
public final class IcebergToolsS3 {

@VisibleForTesting
static final String CLIENT_CREDENTIALS_PROVIDER_ACCESS_KEY_ID =
String.format("%s.%s", AwsClientProperties.CLIENT_CREDENTIALS_PROVIDER, S3FileIOProperties.ACCESS_KEY_ID);

@VisibleForTesting
static final String CLIENT_CREDENTIALS_PROVIDER_SECRET_ACCESS_KEY =
String.format("%s.%s", AwsClientProperties.CLIENT_CREDENTIALS_PROVIDER,
S3FileIOProperties.SECRET_ACCESS_KEY);

/**
* Create an Iceberg catalog adapter for a REST catalog backed by S3 storage. If {@code null} is provided for a
* value, the system defaults will be used.
Expand Down Expand Up @@ -70,12 +60,6 @@ public static IcebergCatalogAdapter createS3Rest(
if (!Strings.isNullOrEmpty(accessKeyId) && !Strings.isNullOrEmpty(secretAccessKey)) {
properties.put(S3FileIOProperties.ACCESS_KEY_ID, accessKeyId);
properties.put(S3FileIOProperties.SECRET_ACCESS_KEY, secretAccessKey);

// Use the Deephaven specific S3ClientCredentialsProvider
properties.put(AwsClientProperties.CLIENT_CREDENTIALS_PROVIDER,
DeephavenS3ClientCredentialsProvider.class.getName());
properties.put(CLIENT_CREDENTIALS_PROVIDER_ACCESS_KEY_ID, accessKeyId);
properties.put(CLIENT_CREDENTIALS_PROVIDER_SECRET_ACCESS_KEY, secretAccessKey);
}
if (!Strings.isNullOrEmpty(region)) {
properties.put(AwsClientProperties.CLIENT_REGION, region);
Expand Down
11 changes: 0 additions & 11 deletions extensions/iceberg/s3/src/test/java/io/deephaven/iceberg/util/IcebergToolsTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@
import org.apache.iceberg.PartitionSpec;
import org.apache.iceberg.Schema;
import org.apache.iceberg.Snapshot;
import org.apache.iceberg.aws.AwsClientProperties;
import org.apache.iceberg.aws.s3.S3FileIO;
import org.apache.iceberg.aws.s3.S3FileIOProperties;
import org.apache.iceberg.catalog.Namespace;
Expand Down Expand Up @@ -51,10 +50,6 @@
import static io.deephaven.iceberg.util.IcebergCatalogAdapter.NAMESPACE_DEFINITION;
import static io.deephaven.iceberg.util.IcebergCatalogAdapter.TABLES_DEFINITION;
import static io.deephaven.iceberg.util.IcebergTableAdapter.SNAPSHOT_DEFINITION;
import static io.deephaven.iceberg.util.IcebergToolsS3.CLIENT_CREDENTIALS_PROVIDER_ACCESS_KEY_ID;
import static io.deephaven.iceberg.util.IcebergToolsS3.CLIENT_CREDENTIALS_PROVIDER_SECRET_ACCESS_KEY;
import static org.apache.iceberg.aws.s3.S3FileIOProperties.ACCESS_KEY_ID;
import static org.apache.iceberg.aws.s3.S3FileIOProperties.SECRET_ACCESS_KEY;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.failBecauseExceptionWasNotThrown;

Expand Down Expand Up @@ -242,12 +237,6 @@ private static S3FileIO createS3FileIO(final Map<String, String> properties) {
// TODO (DH-19253): Add support for S3CrtAsyncClient
newProperties.put(S3FileIOProperties.S3_CRT_ENABLED, "false");

// Set the client credentials provider
newProperties.put(AwsClientProperties.CLIENT_CREDENTIALS_PROVIDER,
DeephavenS3ClientCredentialsProvider.class.getName());
newProperties.put(CLIENT_CREDENTIALS_PROVIDER_ACCESS_KEY_ID, newProperties.get(ACCESS_KEY_ID));
newProperties.put(CLIENT_CREDENTIALS_PROVIDER_SECRET_ACCESS_KEY, newProperties.get(SECRET_ACCESS_KEY));

fileIO.initialize(newProperties);
return fileIO;
}
Expand Down
51 changes: 0 additions & 51 deletions extensions/iceberg/src/main/java/io/deephaven/iceberg/util/InjectAWSProperties.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,12 +3,10 @@
//
package io.deephaven.iceberg.util;

import org.apache.iceberg.CatalogProperties;
import org.jetbrains.annotations.NotNull;

import java.util.LinkedHashMap;
import java.util.Map;
import java.util.Set;

/**
* Static helper that injects Deephaven-specific AWS/S3 settings into the property map passed to
Expand All @@ -17,62 +15,21 @@
*/
class InjectAWSProperties {

/** -- Duplicated from AwsClientProperties -- **/
private static final String CLIENT_CREDENTIALS_PROVIDER = "client.credentials-provider";
private static final String CLIENT_CREDENTIAL_PROVIDER_PREFIX = "client.credentials-provider.";
private static final String REFRESH_CREDENTIALS_ENABLED = "client.refresh-credentials-enabled";
private static final String REFRESH_CREDENTIALS_ENDPOINT = "client.refresh-credentials-endpoint";

/** -- Duplicated from S3FileIOProperties -- **/
private static final String S3_CRT_ENABLED = "s3.crt.enabled";
private static final String ACCESS_KEY_ID = "s3.access-key-id";
private static final String SECRET_ACCESS_KEY = "s3.secret-access-key";
private static final String SESSION_TOKEN = "s3.session-token";
private static final String REMOTE_SIGNING_ENABLED = "s3.remote-signing-enabled";

/** -- Duplicated from VendedCredentialsProvider -- **/
private static final String URI = "credentials.uri";

/** -- Duplicated from AwsProperties -- **/
private static final String CLIENT_FACTORY = "client.factory";

/**
* The following properties are forwarded to the credentials provider, if not set already. This consists of all
* properties that could be accessed in DeephavenS3ClientCredentialsProvider.resolveCredentials
*/
private static final Set<String> CREDENTIALS_PROVIDER_PROPERTIES_TO_FORWARD = Set.of(
REFRESH_CREDENTIALS_ENABLED,
REFRESH_CREDENTIALS_ENDPOINT,
ACCESS_KEY_ID,
SECRET_ACCESS_KEY,
SESSION_TOKEN,
REMOTE_SIGNING_ENABLED,
URI,
CatalogProperties.URI);


/** -- Deephaven defaults -- **/
// Same as DeephavenS3ClientCredentialsProvider.class.getName()
private static final String CLIENT_CREDENTIALS_PROVIDER_DEFAULT =
"io.deephaven.iceberg.util.DeephavenS3ClientCredentialsProvider";

// Same as DeephavenAwsClientFactory.class.getName()
private static final String DEFAULT_CLIENT_FACTORY = "io.deephaven.iceberg.util.DeephavenAwsClientFactory";

// TODO (DH-19253): Add support for S3CrtAsyncClient
private static final String S3_CRT_ENABLED_DEFAULT = "false";

private static void injectCredentialsProperties(
@NotNull final Map<String, String> properties,
@NotNull final String key) {
if (properties.containsKey(key)) {
final String injectedKey = CLIENT_CREDENTIAL_PROVIDER_PREFIX + key;
if (!properties.containsKey(injectedKey)) {
properties.put(injectedKey, properties.get(key));
}
}
}

/**
* Creates a new map with Deephaven-specific properties injected. The input map is not modified.
*/
Expand All @@ -82,14 +39,6 @@ static Map<String, String> injectDeephavenProperties(@NotNull final Map<String,
// TODO (DH-19253): Add support for S3CrtAsyncClient
updatedProperties.putIfAbsent(S3_CRT_ENABLED, S3_CRT_ENABLED_DEFAULT);

// TODO (DH-19508): Remove this once Iceberg fix for #13131 is released
if (!updatedProperties.containsKey(CLIENT_CREDENTIALS_PROVIDER)) {
updatedProperties.put(CLIENT_CREDENTIALS_PROVIDER, CLIENT_CREDENTIALS_PROVIDER_DEFAULT);
for (final String key : CREDENTIALS_PROVIDER_PROPERTIES_TO_FORWARD) {
injectCredentialsProperties(updatedProperties, key);
}
}

if (!updatedProperties.containsKey(CLIENT_FACTORY)) {
updatedProperties.put(CLIENT_FACTORY, DEFAULT_CLIENT_FACTORY);
}
Expand Down
36 changes: 1 addition & 35 deletions ...nsions/iceberg/src/test/java/io/deephaven/iceberg/util/InjectDeephavenPropertiesTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@
//
package io.deephaven.iceberg.util;

import org.apache.iceberg.CatalogProperties;
import org.junit.jupiter.api.Test;

import java.util.HashMap;
Expand All @@ -16,8 +15,6 @@
*/
class InjectDeephavenPropertiesTest {

private static final String DEFAULT_PROVIDER = "io.deephaven.iceberg.util.DeephavenS3ClientCredentialsProvider";

@Test
void defaultsInjectionTest() {
final Map<String, String> props = new HashMap<>();
Expand All @@ -28,41 +25,10 @@ void defaultsInjectionTest() {

// Returned map must contain defaults
assertThat(updated)
.containsEntry("s3.crt.enabled", "false")
.containsEntry("client.credentials-provider", DEFAULT_PROVIDER);
.containsEntry("s3.crt.enabled", "false");
assertThat(updated.keySet()).noneMatch(k -> k.startsWith("client.credentials-provider.s3."));
}

@Test
void forwardsInjectionTest() {
final Map<String, String> props = new HashMap<>(Map.of(
"s3.access-key-id", "AK",
"s3.secret-access-key", "SK",
"s3.session-token", "TOKEN",
"s3.remote-signing-enabled", "true",
"client.refresh-credentials-enabled", "false",
"client.refresh-credentials-endpoint", "http://example.com/creds",
"credentials.uri", "http://example.com/role",
CatalogProperties.URI, "s3://bucket/catalog"));

final Map<String, String> updated = IcebergTools.injectDeephavenProperties(props);

// Deephaven defaults and forwarded counterparts were added
assertThat(updated).containsAllEntriesOf(props);
assertThat(updated)
.containsEntry("client.credentials-provider", DEFAULT_PROVIDER)
.containsEntry("client.credentials-provider.s3.access-key-id", "AK")
.containsEntry("client.credentials-provider.s3.secret-access-key", "SK")
.containsEntry("client.credentials-provider.s3.session-token", "TOKEN")
.containsEntry("client.credentials-provider.s3.remote-signing-enabled", "true")
.containsEntry("client.credentials-provider.client.refresh-credentials-enabled", "false")
.containsEntry("client.credentials-provider.client.refresh-credentials-endpoint",
"http://example.com/creds")
.containsEntry("client.credentials-provider.credentials.uri", "http://example.com/role")
.containsEntry("client.credentials-provider." + CatalogProperties.URI,
"s3://bucket/catalog");
}

@Test
void doesNotOverrideproviderTest() {
final Map<String, String> props = new HashMap<>(Map.of(
Expand Down
2 changes: 1 addition & 1 deletion gradle/libs.versions.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ gwt = "2.13.1"
gwtJetty = "9.4.44.v20210927"
hadoop = "3.4.3"
hdrhistogram = "2.2.2"
iceberg = "1.10.1"
iceberg = "1.10.2"
immutables = "2.12.2"
jackson = "2.21.2"
jakarta-servlet = "6.0.0"
Expand Down
Loading