diff --git a/clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseEnum.java b/clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseEnum.java index 8f6855920..f213cd74f 100644 --- a/clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseEnum.java +++ b/clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseEnum.java @@ -39,17 +39,66 @@ protected ClickHouseEnum(Collection params) { names = new String[size]; values = new int[size]; + boolean[] hasValue = new boolean[size]; int i = 0; for (String p : params) { - int index = p.lastIndexOf('='); - if (index > 0) { - names[i] = ClickHouseUtils.unescape(p.substring(0, index)); - values[i] = Integer.parseInt(p.substring(index + 1)); + int nameEnd = quotedNameEnd(p); + if (nameEnd == p.length()) { + names[i] = ClickHouseUtils.unescape(p); } else { - throw new IllegalArgumentException("Invalid enum entry: " + p); + int index = nameEnd > 0 ? nameEnd : p.lastIndexOf('='); + if (index > 0 && p.charAt(index) == '=') { + names[i] = ClickHouseUtils.unescape(p.substring(0, index)); + values[i] = Integer.parseInt(p.substring(index + 1)); + hasValue[i] = true; + } else { + throw new IllegalArgumentException("Invalid enum entry: " + p); + } } i++; } + + // The server gives an entry without a value the number base + position. The base is the value of the + // first entry, or 1 when the first entry has no value. All entries after the first one need a value, or + // none of them. + int valuesAfterFirst = 0; + for (i = 1; i < size; i++) { + if (hasValue[i]) { + valuesAfterFirst++; + } + } + if (valuesAfterFirst != 0 && valuesAfterFirst != size - 1) { + throw new IllegalArgumentException( + "Either all or none of the enum entries after the first one must have a value, got " + params); + } + int base = size > 0 && hasValue[0] ? values[0] : 1; + for (i = 0; i < size; i++) { + if (!hasValue[i]) { + values[i] = base + i; + } + } + } + + private static int quotedNameEnd(String entry) { + int len = entry.length(); + if (len == 0 || !ClickHouseUtils.isQuote(entry.charAt(0))) { + return -1; + } + + char quote = entry.charAt(0); + for (int i = 1; i < len; i++) { + char ch = entry.charAt(i); + if (ch == '\\') { + i++; + } else if (ch == quote) { + if (i + 1 < len && entry.charAt(i + 1) == quote) { + i++; + } else { + return i + 1; + } + } + } + return -1; } public ClickHouseEnum(String[] names, int[] values) { diff --git a/clickhouse-data/src/test/java/com/clickhouse/data/ClickHouseColumnTest.java b/clickhouse-data/src/test/java/com/clickhouse/data/ClickHouseColumnTest.java index 1f707e320..3fe9e0003 100644 --- a/clickhouse-data/src/test/java/com/clickhouse/data/ClickHouseColumnTest.java +++ b/clickhouse-data/src/test/java/com/clickhouse/data/ClickHouseColumnTest.java @@ -302,6 +302,51 @@ public void testEnumWidth(String typeName, ClickHouseDataType expectedType) { Assert.assertEquals(variant.getNestedColumns().get(1).getDataType(), expectedType); } + @DataProvider(name = "enumImplicitValuesProvider") + private Object[][] getEnumImplicitValues() { + return new Object[][] { + { "Enum8('a', 'b')", ClickHouseDataType.Enum8, new String[] { "a", "b" }, new int[] { 1, 2 } }, + { "Enum16('a', 'b')", ClickHouseDataType.Enum16, new String[] { "a", "b" }, new int[] { 1, 2 } }, + { "Enum('a', 'b')", ClickHouseDataType.Enum8, new String[] { "a", "b" }, new int[] { 1, 2 } }, + { "Enum8('a' = 5, 'b', 'c')", ClickHouseDataType.Enum8, new String[] { "a", "b", "c" }, + new int[] { 5, 6, 7 } }, + { "Enum8('a' = -3, 'b')", ClickHouseDataType.Enum8, new String[] { "a", "b" }, new int[] { -3, -2 } }, + { "Enum8('a', 'b' = 5, 'c' = 9)", ClickHouseDataType.Enum8, new String[] { "a", "b", "c" }, + new int[] { 1, 5, 9 } }, + { "Enum('a' = 127, 'b')", ClickHouseDataType.Enum16, new String[] { "a", "b" }, + new int[] { 127, 128 } }, + { "Enum8('x=y', 'b')", ClickHouseDataType.Enum8, new String[] { "x=y", "b" }, new int[] { 1, 2 } }, + { "Enum8('it''s', 'b')", ClickHouseDataType.Enum8, new String[] { "it's", "b" }, new int[] { 1, 2 } }, + }; + } + + @Test(dataProvider = "enumImplicitValuesProvider", groups = { "unit" }) + public void testEnumImplicitValues(String typeName, ClickHouseDataType expectedType, String[] names, + int[] values) { + ClickHouseColumn column = ClickHouseColumn.of("e", typeName); + Assert.assertEquals(column.getDataType(), expectedType); + Assert.assertEquals(column.getEnumConstants().getNames(), names); + Assert.assertEquals(column.getEnumConstants().getValues(), values); + + ClickHouseColumn tuple = ClickHouseColumn.of("e", "Tuple(Int32, Nullable(" + typeName + "), Float64)"); + Assert.assertEquals(tuple.getNestedColumns().get(1).getEnumConstants(), column.getEnumConstants()); + Assert.assertEquals(tuple.getNestedColumns().get(2).getDataType(), ClickHouseDataType.Float64); + } + + @DataProvider(name = "enumMixedValuesProvider") + private Object[][] getEnumMixedValues() { + return new Object[][] { + { "Enum8('a' = 5, 'b', 'c' = 10, 'd')" }, + { "Enum8('a' = 1, 'b' = 2, 'c')" }, + { "Enum8('a', 'b' = 5, 'c')" }, + }; + } + + @Test(dataProvider = "enumMixedValuesProvider", groups = { "unit" }) + public void testEnumMixedValues(String typeName) { + Assert.assertThrows(IllegalArgumentException.class, () -> ClickHouseColumn.of("e", typeName)); + } + @Test(groups = { "unit" }) public void testBareEnumWidthFromDataType() { Assert.assertEquals(ClickHouseColumn.of("e", ClickHouseDataType.Enum, false, false, "'a'=1").getDataType(), diff --git a/client-v2/src/main/java/com/clickhouse/client/api/Client.java b/client-v2/src/main/java/com/clickhouse/client/api/Client.java index 0d843d3c0..3ff12f1e1 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/Client.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/Client.java @@ -1167,10 +1167,10 @@ public Builder setClientName(String clientName) { * Sets the client name and the client metadata, which identify the application to the server. See * {@link #setClientName(String)} for the client name. The client adds the metadata to the comment group of the * {@code User-Agent} header, in the iteration order of {@code metadata}, and then its own {@code lv}, - * {@code os} and {@code hc} (HTTP client) entries if the metadata does not set these keys. For example, the - * client name {@code my-app/1.0} and the metadata {@code fv=flink/2.0.0} and {@code api=table} give - * {@code my-app/1.0 clickhouse-java-v2/ (fv:flink/2.0.0; api:table; lv:java/; - * os:; hc:ahc_)}. + * {@code os}, {@code arch} (CPU architecture) and {@code hc} (HTTP client) entries if the metadata does not set + * these keys. For example, the client name {@code my-app/1.0} and the metadata {@code fv=flink/2.0.0} and + * {@code api=table} give {@code my-app/1.0 clickhouse-java-v2/ (fv:flink/2.0.0; api:table; + * lv:java/; os:; arch:; hc:ahc_)}. *

* Keys and values are trimmed, and entries with a {@code null} or empty key or value are ignored. Keys and * values must be printable ASCII, a key must not contain spaces or any of {@code :;(),=}, and a value must not diff --git a/client-v2/src/main/java/com/clickhouse/client/api/internal/WorkloadIdentitySupport.java b/client-v2/src/main/java/com/clickhouse/client/api/internal/WorkloadIdentitySupport.java index 29106c527..5cbcd9b1b 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/internal/WorkloadIdentitySupport.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/internal/WorkloadIdentitySupport.java @@ -15,12 +15,13 @@ * Builds the identification of the client and of the application that uses it. The identification is the value of * the {@code User-Agent} header, in the standard ClickHouse client format: * {@code clickhouse-java-v2/ (:; ...; lv:java/; os:; - * hc:ahc_)}. + * arch:; hc:ahc_)}. *

* The application part is the client name ({@link ClientConfigProperties#CLIENT_NAME}), or the custom * {@code User-Agent} header if the application set one. The key/value pairs are the client metadata - * ({@link ClientConfigProperties#CLIENT_ID_METADATA}). The {@code hc} entry is the HTTP client that sends the requests: - * {@code ahc_} (Apache HttpClient) and its version. + * ({@link ClientConfigProperties#CLIENT_ID_METADATA}). The {@code arch} entry is the CPU architecture that the JVM + * reports. The {@code hc} entry is the HTTP client that sends the requests: {@code ahc_} (Apache HttpClient) and its + * version. *

* The client metadata is validated when {@link Client.Builder#build()} parses the configuration. {@link Client} * creates one instance and updates it when the client name changes. The instance composes the value for the client @@ -35,6 +36,8 @@ public final class WorkloadIdentitySupport { private static final String METADATA_KEY_OS = "os"; + private static final String METADATA_KEY_ARCHITECTURE = "arch"; + private static final String METADATA_KEY_HTTP_CLIENT = "hc"; /** @@ -147,8 +150,8 @@ private static boolean isClientMetadataText(String str, String forbiddenChars) { /** * Composes the client part: {@code clickhouse-java-v2/ (:; ...)}. The key/value pairs are the - * client metadata, if it is set, then {@code lv:java/}, {@code os:} and - * {@code hc:ahc_} if the client metadata does not set these keys (ignoring case). The + * client metadata, if it is set, then {@code lv:java/}, {@code os:}, {@code arch:} + * and {@code hc:ahc_} if the client metadata does not set these keys (ignoring case). The * client sends requests over HTTP with Apache HttpClient ({@code ahc}). */ private static String composeClientPart(Map configuration) { @@ -157,6 +160,7 @@ private static String composeClientPart(Map configuration) { ClientUtils.putIfAbsentIgnoreCase(metadata, METADATA_KEY_LANGUAGE_VERSION, "java/" + System.getProperty("java.version")); ClientUtils.putIfAbsentIgnoreCase(metadata, METADATA_KEY_OS, System.getProperty("os.name")); + ClientUtils.putIfAbsentIgnoreCase(metadata, METADATA_KEY_ARCHITECTURE, System.getProperty("os.arch")); ClientUtils.putIfAbsentIgnoreCase(metadata, METADATA_KEY_HTTP_CLIENT, "ahc_" + Versions.APACHE_HTTP_CLIENT); StringJoiner comment = new StringJoiner("; ", Client.CLIENT_USER_AGENT + Client.clientVersion + " (", ")"); diff --git a/client-v2/src/test/java/com/clickhouse/client/HttpTransportTests.java b/client-v2/src/test/java/com/clickhouse/client/HttpTransportTests.java index ffb5730fb..acfeb423c 100644 --- a/client-v2/src/test/java/com/clickhouse/client/HttpTransportTests.java +++ b/client-v2/src/test/java/com/clickhouse/client/HttpTransportTests.java @@ -1271,9 +1271,9 @@ public void testUserAgentHasCompleteProductName(String clientName, Pattern userA @DataProvider(name = "testUserAgentHasCompleteProductName_dataProvider") public static Object[][] testUserAgentHasCompleteProductName_dataProvider() { return new Object[][] { - { "", Pattern.compile("clickhouse-java-v2/[^ ]+ \\(lv:java/[^;]+; os:[^;()]+; hc:ahc_[\\d.]+\\)") }, - { "test-client/1.0", Pattern.compile("test-client/1.0 clickhouse-java-v2/[^ ]+ \\(lv:java/[^;]+; os:[^;()]+; hc:ahc_[\\d.]+\\)")}, - { "test-client/", Pattern.compile("test-client/ clickhouse-java-v2/[^ ]+ \\(lv:java/[^;]+; os:[^;()]+; hc:ahc_[\\d.]+\\)")}}; + { "", Pattern.compile("clickhouse-java-v2/[^ ]+ \\(lv:java/[^;]+; os:[^;()]+; arch:[^;()]+; hc:ahc_[\\d.]+\\)") }, + { "test-client/1.0", Pattern.compile("test-client/1.0 clickhouse-java-v2/[^ ]+ \\(lv:java/[^;]+; os:[^;()]+; arch:[^;()]+; hc:ahc_[\\d.]+\\)")}, + { "test-client/", Pattern.compile("test-client/ clickhouse-java-v2/[^ ]+ \\(lv:java/[^;]+; os:[^;()]+; arch:[^;()]+; hc:ahc_[\\d.]+\\)")}}; } @Test(dataProvider = "testClientNameDataProvider") diff --git a/client-v2/src/test/java/com/clickhouse/client/api/UserAgentUnitTest.java b/client-v2/src/test/java/com/clickhouse/client/api/UserAgentUnitTest.java index 723d825ae..e940e57dd 100644 --- a/client-v2/src/test/java/com/clickhouse/client/api/UserAgentUnitTest.java +++ b/client-v2/src/test/java/com/clickhouse/client/api/UserAgentUnitTest.java @@ -58,6 +58,7 @@ public static Object[][] userAgentData() { Map osMetadata = new LinkedHashMap<>(); osMetadata.put("os", "Android 14"); osMetadata.put("api", "table"); + osMetadata.put("arch", "arm64-v8a"); Map withIgnoredEntries = new LinkedHashMap<>(); withIgnoredEntries.put("api", ""); @@ -103,7 +104,8 @@ public static Object[][] userAgentData() { {(Consumer) b -> b.setClientName(" my-app/1.0 "), userAgent("my-app/1.0", null)}, {(Consumer) b -> b.setClientName("my-app/1.0", osMetadata), - "my-app/1.0 clickhouse-java-v2/" + Client.clientVersion + " (os:Android 14; api:table; lv:java/" + "my-app/1.0 clickhouse-java-v2/" + Client.clientVersion + + " (os:Android 14; api:table; arch:arm64-v8a; lv:java/" + System.getProperty("java.version") + "; hc:ahc_" + Versions.APACHE_HTTP_CLIENT + ")"}, }; @@ -165,6 +167,7 @@ private static String userAgent(String application, String metadata) { + " (" + (metadata == null ? "" : metadata + "; ") + "lv:java/" + System.getProperty("java.version") + "; os:" + System.getProperty("os.name") + + "; arch:" + System.getProperty("os.arch") + "; hc:ahc_" + Versions.APACHE_HTTP_CLIENT + ")"; } } diff --git a/client-v2/src/test/java/com/clickhouse/client/api/internal/HttpAPIClientHelperTest.java b/client-v2/src/test/java/com/clickhouse/client/api/internal/HttpAPIClientHelperTest.java index fd3dfb000..138777bc8 100644 --- a/client-v2/src/test/java/com/clickhouse/client/api/internal/HttpAPIClientHelperTest.java +++ b/client-v2/src/test/java/com/clickhouse/client/api/internal/HttpAPIClientHelperTest.java @@ -595,7 +595,8 @@ private static String userAgentHeader(HttpAPIClientHelper helper, String clientN private static String clientPart() { return "clickhouse-java-v2/" + Client.clientVersion + " (lv:java/" + System.getProperty("java.version") - + "; os:" + System.getProperty("os.name") + "; hc:ahc_" + Versions.APACHE_HTTP_CLIENT + ")"; + + "; os:" + System.getProperty("os.name") + "; arch:" + System.getProperty("os.arch") + "; hc:ahc_" + + Versions.APACHE_HTTP_CLIENT + ")"; } private static String headerValue(HttpPost req, String name) { diff --git a/client-v2/src/test/java/com/clickhouse/client/api/internal/WorkloadIdentitySupportTest.java b/client-v2/src/test/java/com/clickhouse/client/api/internal/WorkloadIdentitySupportTest.java index b3861e960..b48300458 100644 --- a/client-v2/src/test/java/com/clickhouse/client/api/internal/WorkloadIdentitySupportTest.java +++ b/client-v2/src/test/java/com/clickhouse/client/api/internal/WorkloadIdentitySupportTest.java @@ -22,6 +22,8 @@ public class WorkloadIdentitySupportTest { private static final String OS = "os:" + System.getProperty("os.name"); + private static final String ARCHITECTURE = "arch:" + System.getProperty("os.arch"); + private static final String HTTP_CLIENT = "hc:ahc_" + Versions.APACHE_HTTP_CLIENT; @Test(dataProvider = "requestUserAgents") @@ -102,12 +104,15 @@ public void testClientMetadataReplacesClientEntries(String metadata, String expe @DataProvider(name = "clientEntriesMetadata") public static Object[][] clientEntriesMetadata() { return new Object[][]{ - {"os=Android 14", "os:Android 14; " + LANGUAGE_VERSION + "; " + HTTP_CLIENT}, - {"api=table,lv=kotlin/2.0", "api:table; lv:kotlin/2.0; " + OS + "; " + HTTP_CLIENT}, - {"hc=okhttp_4.12", "hc:okhttp_4.12; " + LANGUAGE_VERSION + "; " + OS}, - {"OS=Android 14,LV=kotlin/2.0", "OS:Android 14; LV:kotlin/2.0; " + HTTP_CLIENT}, - {"os=Android 14,hc=okhttp_4.12,api=table,lv=kotlin/2.0", - "os:Android 14; hc:okhttp_4.12; api:table; lv:kotlin/2.0"}, + {"os=Android 14", "os:Android 14; " + LANGUAGE_VERSION + "; " + ARCHITECTURE + "; " + HTTP_CLIENT}, + {"api=table,lv=kotlin/2.0", + "api:table; lv:kotlin/2.0; " + OS + "; " + ARCHITECTURE + "; " + HTTP_CLIENT}, + {"hc=okhttp_4.12", "hc:okhttp_4.12; " + LANGUAGE_VERSION + "; " + OS + "; " + ARCHITECTURE}, + {"arch=arm64-v8a", "arch:arm64-v8a; " + LANGUAGE_VERSION + "; " + OS + "; " + HTTP_CLIENT}, + {"OS=Android 14,LV=kotlin/2.0,ARCH=arm64-v8a", + "OS:Android 14; LV:kotlin/2.0; ARCH:arm64-v8a; " + HTTP_CLIENT}, + {"os=Android 14,hc=okhttp_4.12,api=table,arch=arm64-v8a,lv=kotlin/2.0", + "os:Android 14; hc:okhttp_4.12; api:table; arch:arm64-v8a; lv:kotlin/2.0"}, }; } @@ -146,6 +151,7 @@ private static String userAgent(String application) { private static String userAgent(String application, String metadata) { return (application == null ? "" : application + " ") + "clickhouse-java-v2/" + Client.clientVersion + " (" - + (metadata == null ? "" : metadata + "; ") + LANGUAGE_VERSION + "; " + OS + "; " + HTTP_CLIENT + ")"; + + (metadata == null ? "" : metadata + "; ") + LANGUAGE_VERSION + "; " + OS + "; " + ARCHITECTURE + "; " + + HTTP_CLIENT + ")"; } } diff --git a/client-v2/src/test/java/com/clickhouse/client/datatypes/RowBinaryFormatWriterTest.java b/client-v2/src/test/java/com/clickhouse/client/datatypes/RowBinaryFormatWriterTest.java index 9f46b0f0c..186614eb7 100644 --- a/client-v2/src/test/java/com/clickhouse/client/datatypes/RowBinaryFormatWriterTest.java +++ b/client-v2/src/test/java/com/clickhouse/client/datatypes/RowBinaryFormatWriterTest.java @@ -304,6 +304,10 @@ private Object[][] enumTypesInSchema() { {"ENUM('a' = -128, 'b' = 127)", "b"}, {"Variant(Bool, Enum('a' = 1, 'b' = 1000))", "b"}, {"Nullable(Enum('a' = 1, 'b' = 1000))", "a"}, + {"Enum8('a', 'b')", "b"}, + {"Enum8('a' = 5, 'b', 'c')", "c"}, + {"Enum('a' = 127, 'b')", "b"}, + {"Nullable(Enum8('x=y', 'b'))", "x=y"}, }; } diff --git a/docs/clickhouse-docs/client.mdx b/docs/clickhouse-docs/client.mdx index f2527dda6..658f0e608 100644 --- a/docs/clickhouse-docs/client.mdx +++ b/docs/clickhouse-docs/client.mdx @@ -264,7 +264,7 @@ Configuration is defined during client creation. See `com.clickhouse.client.api. | `allowBinaryReaderToReuseBuffers(boolean reuse)` | `reuse` - flag to enable/disable | If enabled, reader will use pre-allocated buffers to do numbers transcoding. Reduces GC pressure for numeric data. | - | - | | `columnToMethodMatchingStrategy(ColumnToMethodMatchingStrategy strategy)` | `strategy` - matching strategy implementation | Sets custom strategy to be used for matching DTO class fields and DB columns when registering DTO. | `none` | `none` | | `setClientName(String clientName)` | `clientName` - application name string | Sets the client name, which identifies the application. Will be passed as the first part of the `User-Agent` header. Use it instead of setting the `User-Agent` header. Does not change the client metadata. | - | `client_name` | -| `setClientName(String clientName, Map metadata)` | `clientName` - application name string
`metadata` - key/value pairs | Sets the client name and the client metadata. The metadata is added to the comment group of the `User-Agent` header, in the iteration order of the map, then the `lv`, `os` and `hc` entries if the map does not set these keys. Keys and values must be printable ASCII, a key must not contain spaces or any of `:;(),=`, and a value must not contain any of `;()`. As a property, the value is written as `key1=value1,key2=value2` (escape a `,` or `\` in a value with `\`). Client-level only: not read from operation settings. | - | `client_id_metadata` | +| `setClientName(String clientName, Map metadata)` | `clientName` - application name string
`metadata` - key/value pairs | Sets the client name and the client metadata. The metadata is added to the comment group of the `User-Agent` header, in the iteration order of the map, then the `lv`, `os`, `arch` and `hc` entries if the map does not set these keys. Keys and values must be printable ASCII, a key must not contain spaces or any of `:;(),=`, and a value must not contain any of `;()`. As a property, the value is written as `key1=value1,key2=value2` (escape a `,` or `\` in a value with `\`). Client-level only: not read from operation settings. | - | `client_id_metadata` | | `registerClientMetrics(Object registry, String name)` | `registry` - Micrometer registry instance
`name` - metrics group name | Registers sensors with Micrometer (https://micrometer.io/) registry instance. | - | - | | `setServerVersion(String version)` | `version` - server version string | Sets server version to avoid version detection. | - | `server_version` | | `typeHintMapping(Map typeHintMapping)` | `typeHintMapping` - map of type hints | Sets type hint mapping for ClickHouse types. For example, to make multidimensional arrays be present as Java containers. | - | `type_hint_mapping` | @@ -278,8 +278,8 @@ Configuration is defined during client creation. See `com.clickhouse.client.api. There are two fields in a query log that identify application originated a request: `client_name` and `http_user_agent`. Native TCP protocol uses `client_name` to identify application. HTTP protocol uses `http_user_agent` to identify application. Client builder has method `setClientName` correct values for both protocols. -The field `http_user_agent` is set according to the standard ClickHouse client format: `/ clickhouse-java-v2/ (:; ...; lv:java/; os:; hc:ahc_)`. -What is set by `setClientName` method comes first. Key/value pairs set by `setClientName(String, Map)` are added to the comment group, then `lv`, `os` and `hc` if the pairs do not set these keys. +The field `http_user_agent` is set according to the standard ClickHouse client format: `/ clickhouse-java-v2/ (:; ...; lv:java/; os:; arch:; hc:ahc_)`. +What is set by `setClientName` method comes first. Key/value pairs set by `setClientName(String, Map)` are added to the comment group, then `lv`, `os`, `arch` and `hc` if the pairs do not set these keys. For example: ```java showLineNumbers @@ -301,7 +301,7 @@ public Client createClientWithMetadata() { ``` will result in the following `http_user_agent` value: ``` -my-app-01/1.0 clickhouse-java-v2/0.12.0 (fv:flink/2.0.0; api:table; lv:java/17.0.17; os:Linux; hc:ahc_5.6.4) +my-app-01/1.0 clickhouse-java-v2/0.12.0 (fv:flink/2.0.0; api:table; lv:java/17.0.17; os:Linux; arch:amd64; hc:ahc_5.6.4) ``` Use `setClientName` to identify the application. Do not set the `User-Agent` header for this: a custom `User-Agent` header replaces diff --git a/docs/clickhouse-docs/jdbc.mdx b/docs/clickhouse-docs/jdbc.mdx index 8e97ad506..d6685ac41 100644 --- a/docs/clickhouse-docs/jdbc.mdx +++ b/docs/clickhouse-docs/jdbc.mdx @@ -147,11 +147,11 @@ conn.setClientInfo(com.clickhouse.jdbc.ClientInfoProperties.APPLICATION_NAME.get Both ways will result in the following `http_user_agent` value in the query log: ``` -my-app-01/1.0 jdbc-v2/0.12.0 clickhouse-java-v2/0.12.0 (lv:java/17.0.17; os:Linux; hc:ahc_5.6.4) +my-app-01/1.0 jdbc-v2/0.12.0 clickhouse-java-v2/0.12.0 (lv:java/17.0.17; os:Linux; arch:amd64; hc:ahc_5.6.4) ``` To add key/value pairs to the comment group of the header, set the `client_id_metadata` property. For example, -`client_id_metadata=fv=flink/2.0.0,api=table` gives `(fv:flink/2.0.0; api:table; lv:java/17.0.17; os:Linux; hc:ahc_5.6.4)`. +`client_id_metadata=fv=flink/2.0.0,api=table` gives `(fv:flink/2.0.0; api:table; lv:java/17.0.17; os:Linux; arch:amd64; hc:ahc_5.6.4)`. **Note:** We recommend using `app_name/version` format for `client_name` property because it helps to identify the application in the query log. Use the client name to identify the application. Do not set a custom `User-Agent` HTTP header for this: it replaces the application part of the header, including `jdbc-v2/`. diff --git a/docs/features.md b/docs/features.md index fc23cc010..53001a34a 100644 --- a/docs/features.md +++ b/docs/features.md @@ -40,7 +40,7 @@ This document lists stable, user-visible behavior in `client-v2` and `jdbc-v2` t - Span recording (tracing SPI): `Client.Builder.setSpanRecorder(SpanRecorder)` registers a backend-agnostic recorder (package `com.clickhouse.client.api.observability`) that observes client operations. Every operation - a query, a command or an insert, including the `ping` and `getTableSchema` calls, which run a query - starts one operation span, and every transport request made for it, including each retry, starts a child request span. `SpanRecorder` and `Span` are plain interfaces; an implementation extends the `DefaultSpanRecorder` base class and overrides only what it wants to record, so a recorder keeps working when the client starts a kind of span it does not know about. The registered recorder is the first thing the client calls, and it is called with everything the client knows about the operation - its settings object, the statement, the target table, the batch size, the endpoint, the metrics of the completed operation and the failure - so an implementation is free to record whatever it needs and in whatever form. Deriving the standard span names and attribute values from those structures is done by `SpanSupport`, which a recorder implementation calls if it wants them; using it is opt-in, and its methods may be overridden to report other values. A recorder opts in through `DefaultSpanRecorder#getSpanSupport()` (or `SpanSupport.DEFAULT`). Span names and attribute keys follow the OpenTelemetry semantic conventions for database and HTTP client spans, the keys are defined by the `SpanAttribute` enum, and the values are derived by `SpanSupport`, so every recorder that uses it reports the same information. An operation span is named `query `, or `insert .` for an insert, and carries `db.system.name`, `db.namespace`, `clickhouse.query_id`, `db.query.text` (query/command), `db.query.parameter.` (parameterized query), `db.operation.name` (insert), `db.collection.name` (insert), `db.operation.batch.size` (POJO insert), `server.address`/`server.port` of the first configured endpoint (each attempt reports its own on the request span), the metrics of the completed operation on success, and `error.type` plus `db.response.status_code` on failure. Success is reported per operation kind, because the metrics that describe a read are not the ones that describe a write: `SpanRecorder#recordQuerySuccess` is called for a read operation and `SpanRecorder#recordInsertSuccess` for an insert, and each records only the metrics of its own kind - a query reports `db.response.returned_rows`, `clickhouse.response.read_rows` and `clickhouse.response.read_bytes`, an insert reports `clickhouse.response.written_rows` and `clickhouse.response.written_bytes`, and both report `clickhouse.query_id`. These values come from the server's progress summary, which is not always available (for example a query reports them with `QuerySettings#waitEndOfQuery(true)`); a metric the server did not report is left out. The same distinction is on the metrics themselves: `OperationMetrics#getOperationType()` returns `OperationType.QUERY` or `OperationType.INSERT`. It reports the kind of the call the application made, not the kind of work the server did, so a command that writes - `INSERT INTO ... SELECT` run through `execute` - is reported as a query. A request span is named `POST` and carries `http.request.method`, the `server.address`/`server.port` of that attempt, `http.response.status_code` once a response is received, and `error.type`/`db.response.status_code` when the attempt fails. An operation span is started on the calling thread, so it joins the caller's ambient trace even when `async_operations` runs the operation on the client's executor, and it is ended when the operation returns its response to the caller - so it covers sending the request and receiving the response head, not streaming the response body afterwards. Each span of an operation that started is ended exactly once, also when the operation failed; an operation that could not be started at all - a closed client, for example - does not report a span. When no recorder is registered nothing is recorded and no span-related work is done. - Metrics recording (metrics SPI): `Client.Builder.setMetricsRecorder(MetricsRecorder)` registers a backend-agnostic recorder (package `com.clickhouse.client.api.observability`) that receives the metrics of client operations, so an application can export them to any metrics backend. It follows the same pattern as the span SPI: `MetricsRecorder` is a plain interface, an implementation extends the `DefaultMetricsRecorder` base class and overrides only what it wants to record, so a recorder keeps working when the client starts reporting an event it does not know about, and the reusable `MetricsSupport` class derives the standard values from the same structures (opt-in through `DefaultMetricsRecorder#getMetricsSupport()`, and overridable). Every completed operation reports exactly one event - `recordQuerySuccess`/`recordInsertSuccess` or `recordQueryFailure`/`recordInsertFailure` - so counting those events gives the number of operations by outcome, and every retried attempt reports `recordQueryRetry`/`recordInsertRetry`, so an operation that succeeds on its third attempt reports two retries. The durations of a successful operation are read from its `OperationMetrics`; a failed operation has no metrics, so the client measures its duration itself and passes it in. Metric names, units and attribute keys follow the OpenTelemetry semantic conventions for database clients where a convention exists and are placed under `clickhouse.` where it does not: the `MetricName` enum defines `db.client.operation.duration` (unit `s`), `clickhouse.client.operation.serialization.duration` (unit `s`), `clickhouse.client.operation.count` and `clickhouse.client.operation.retries`, each with the unit and description an exporter registers its instrument with. Durations are reported in **seconds**, while `ClientMetrics` values are milliseconds - `MetricsSupport` converts them and returns `MetricsSupport.DURATION_UNKNOWN` for a duration the client did not measure (the serialization duration is measured for a POJO insert only), so a recorder never reports a made-up value. The attribute keys are the `MetricAttribute` enum - `db.system.name`, `db.namespace`, `db.operation.name` (`query` or `insert`), `db.collection.name` (insert), and `error.type` plus `db.response.status_code` on a failure. That is deliberately a smaller set than `SpanAttribute`, because an attribute of a metric becomes a time series: the statement text, the query id and the statement parameters stay on spans. A successful operation has no `error.type` attribute, so the successful time series is the one without it. When no recorder is registered nothing is recorded and no metrics-related work is done. The recorder is registered per client instance and must be thread-safe. - Configuration surface: Supports arbitrary client options, cookies, custom headers, server-setting prefixes, client naming, query id suppliers, and buffer sizing. -- `User-Agent` header and client metadata: Every request sends a `User-Agent` header in the standard ClickHouse client format, ` clickhouse-java-v2/ (:; ...; lv:java/; os:; hc:ahc_)`, where `` is a custom `User-Agent` header value if one is set and the client name otherwise, with leading and trailing whitespace removed (omitted when both are blank). A blank client name or `User-Agent` value set for one operation counts as not set: the request uses the value of the client configuration. The client name is the documented way to identify the application; a custom `User-Agent` header replaces it. The `hc` entry is the HTTP client: `ahc_` (Apache HttpClient) and the version of the library that the client is built with (also in the shaded jar). `Client.Builder.setClientName(String, Map)` (or the `client_id_metadata` property, written as `key1=value1,key2=value2`, where a `,` or `\` in a value is escaped with `\`) adds key/value pairs to the comment group in the given order, then `lv`, `os` and `hc` if the metadata does not set these keys, ignoring case (so an application can replace them); keys and values are trimmed and entries with a `null` or empty key or value are ignored. Keys and values must be printable ASCII, a key must not contain spaces or any of `:;(),=`, and a value must not contain any of `;()`: an invalid entry is rejected with `ClientMisconfigurationException` when the client is built. Client metadata is a client-level setting: it is not read from operation settings, and it is kept when the client name changes (`Client.Builder.setClientName(String)`, `Client#updateClientName`). The client logs a warning when the `User-Agent` value of the client configuration is longer than 200 characters. +- `User-Agent` header and client metadata: Every request sends a `User-Agent` header in the standard ClickHouse client format, ` clickhouse-java-v2/ (:; ...; lv:java/; os:; arch:; hc:ahc_)`, where `` is a custom `User-Agent` header value if one is set and the client name otherwise, with leading and trailing whitespace removed (omitted when both are blank). A blank client name or `User-Agent` value set for one operation counts as not set: the request uses the value of the client configuration. The client name is the documented way to identify the application; a custom `User-Agent` header replaces it. The `arch` entry is the CPU architecture that the JVM reports (`os.arch`, for example `amd64` or `aarch64`). The `hc` entry is the HTTP client: `ahc_` (Apache HttpClient) and the version of the library that the client is built with (also in the shaded jar). `Client.Builder.setClientName(String, Map)` (or the `client_id_metadata` property, written as `key1=value1,key2=value2`, where a `,` or `\` in a value is escaped with `\`) adds key/value pairs to the comment group in the given order, then `lv`, `os`, `arch` and `hc` if the metadata does not set these keys, ignoring case (so an application can replace them); keys and values are trimmed and entries with a `null` or empty key or value are ignored. Keys and values must be printable ASCII, a key must not contain spaces or any of `:;(),=`, and a value must not contain any of `;()`: an invalid entry is rejected with `ClientMisconfigurationException` when the client is built. Client metadata is a client-level setting: it is not read from operation settings, and it is kept when the client name changes (`Client.Builder.setClientName(String)`, `Client#updateClientName`). The client logs a warning when the `User-Agent` value of the client configuration is longer than 200 characters. - SQL helpers: Includes SQL quoting and temporal formatting helpers used by callers building SQL text safely. Compatibility-sensitive traits: diff --git a/docs/integration-client.md b/docs/integration-client.md index 7bda9298a..8e2640f85 100644 --- a/docs/integration-client.md +++ b/docs/integration-client.md @@ -219,13 +219,13 @@ public void updateClientWorkload(Client client, String workloadOrTenant) { #### How it is observed on the server (`User-Agent` header) -The Java Client communicates over HTTP and passes the client name as the leading segment of the HTTP `User-Agent` header. The client automatically appends the client library version and a comment group with the Java version (`lv`), the operating system (`os`), and the HTTP client (`hc`: `ahc_` and the Apache HttpClient version), in the standard ClickHouse client format: +The Java Client communicates over HTTP and passes the client name as the leading segment of the HTTP `User-Agent` header. The client automatically appends the client library version and a comment group with the Java version (`lv`), the operating system (`os`), the CPU architecture (`arch`), and the HTTP client (`hc`: `ahc_` and the Apache HttpClient version), in the standard ClickHouse client format: ```text -order-service/1.2.0 clickhouse-java-v2/0.12.0 (lv:java/17.0.2; os:Linux; hc:ahc_5.6.4) +order-service/1.2.0 clickhouse-java-v2/0.12.0 (lv:java/17.0.2; os:Linux; arch:amd64; hc:ahc_5.6.4) ``` -To add key/value pairs to the comment group (for example, the version of a framework that the application runs in), use `setClientName(String, Map)`. The pairs are added in the iteration order of the map, then `lv`, `os` and `hc` if the map does not set these keys (set them to replace the values of the client): +To add key/value pairs to the comment group (for example, the version of a framework that the application runs in), use `setClientName(String, Map)`. The pairs are added in the iteration order of the map, then `lv`, `os`, `arch` and `hc` if the map does not set these keys (set them to replace the values of the client): ```java import com.clickhouse.client.api.Client; @@ -248,7 +248,7 @@ public Client createClientWithMetadata() { ``` ```text -Flink-ClickHouse-Sink/0.2.0 clickhouse-java-v2/0.12.0 (fv:flink/2.0.0; api:table; lv:java/17.0.2; os:Linux; hc:ahc_5.6.4) +Flink-ClickHouse-Sink/0.2.0 clickhouse-java-v2/0.12.0 (fv:flink/2.0.0; api:table; lv:java/17.0.2; os:Linux; arch:amd64; hc:ahc_5.6.4) ``` The same pairs can be set with the `client_id_metadata` property, written as `fv=flink/2.0.0,api=table` (escape a `,` or `\` in a value with `\`). Keys and values must be printable ASCII, a key must not contain spaces or any of `:;(),=`, and a value must not contain any of `;()`. `setClientName(String)` and `client.updateClientName(...)` do not change the metadata. diff --git a/docs/integration-jdbc.md b/docs/integration-jdbc.md index 477a2210d..889bff903 100644 --- a/docs/integration-jdbc.md +++ b/docs/integration-jdbc.md @@ -214,13 +214,13 @@ public void executeWorkloadTask(Connection conn, String taskName) throws SQLExce #### How it is observed on the server (`User-Agent` header) -The JDBC driver communicates over HTTP and passes the client name as the leading segment of the HTTP `User-Agent` header. The driver automatically appends the JDBC driver version, the client version, and a comment group with detected frameworks (e.g. Apache Spark, as `fw`), the Java version (`lv`), the operating system (`os`), and the HTTP client (`hc`: `ahc_` and the Apache HttpClient version): +The JDBC driver communicates over HTTP and passes the client name as the leading segment of the HTTP `User-Agent` header. The driver automatically appends the JDBC driver version, the client version, and a comment group with detected frameworks (e.g. Apache Spark, as `fw`), the Java version (`lv`), the operating system (`os`), the CPU architecture (`arch`), and the HTTP client (`hc`: `ahc_` and the Apache HttpClient version): ```text -order-service/1.2.0 jdbc-v2/0.12.0 clickhouse-java-v2/0.12.0 (lv:java/17.0.2; os:Linux; hc:ahc_5.6.4) +order-service/1.2.0 jdbc-v2/0.12.0 clickhouse-java-v2/0.12.0 (lv:java/17.0.2; os:Linux; arch:amd64; hc:ahc_5.6.4) ``` -To add key/value pairs to the comment group of the header, set the `client_id_metadata` property (for example, `client_id_metadata=fv=flink/2.0.0,api=table` gives `(fv:flink/2.0.0; api:table; lv:java/17.0.2; os:Linux; hc:ahc_5.6.4)`). +To add key/value pairs to the comment group of the header, set the `client_id_metadata` property (for example, `client_id_metadata=fv=flink/2.0.0,api=table` gives `(fv:flink/2.0.0; api:table; lv:java/17.0.2; os:Linux; arch:amd64; hc:ahc_5.6.4)`). Always identify the application with the client name (`client_name` or `ApplicationName`). Do not set a custom `User-Agent` HTTP header for this: it replaces the application part of the header, including `jdbc-v2/`. diff --git a/history/latest/2458.md b/history/latest/2458.md new file mode 100644 index 000000000..956df8900 --- /dev/null +++ b/history/latest/2458.md @@ -0,0 +1,4 @@ +- **[client-v2, jdbc-v2]** The `User-Agent` header now has the CPU architecture that the JVM reports (`os.arch`) as + the `arch` entry, after `os`: `(lv:java/17.0.2; os:Linux; arch:amd64; hc:ahc_5.6.4)`. Client metadata can set + `arch` to replace the value of the client. + (https://github.com/ClickHouse/clickhouse-java/issues/2458) diff --git a/history/latest/3344.md b/history/latest/3344.md new file mode 100644 index 000000000..c6346d2d8 --- /dev/null +++ b/history/latest/3344.md @@ -0,0 +1,9 @@ +- **[client-v2, jdbc-v2]** Fixed Enum type names that leave out the values, such as `Enum8('a', 'b')` or + `Enum8('a' = 5, 'b')`, in a type name that the application writes. This covers a `TableSchema` for + `Client#newBinaryFormatReader(response, schema)`, `RowBinaryFormatWriter`, `Client#register(Class, schema)`, + `Connection#createArrayOf` and `Connection#createStruct`. They failed with `Invalid enum entry`, and a quoted name + that contains `=` failed with `NumberFormatException`. Now the client numbers these entries the same way as the + server. An entry without a value gets the value of the first entry plus its position, and counting starts at 1 when + the first entry has no value. A list where only some of the entries after the first one have a value is rejected, + as on the server. Type names from the server always have explicit values and are read as before. + (https://github.com/ClickHouse/clickhouse-java/issues/3344) diff --git a/jdbc-v2/src/test/java/com/clickhouse/jdbc/ConnectionTest.java b/jdbc-v2/src/test/java/com/clickhouse/jdbc/ConnectionTest.java index 6df60e8a4..d36e0691f 100644 --- a/jdbc-v2/src/test/java/com/clickhouse/jdbc/ConnectionTest.java +++ b/jdbc-v2/src/test/java/com/clickhouse/jdbc/ConnectionTest.java @@ -426,8 +426,8 @@ public void testClientMetadataInUserAgent(boolean setInUrl) throws Exception { } String expectedSuffix = "clickhouse-java-v2/" + Client.clientVersion + " (fv:flink/2.0.0; api:table; lv:java/" - + System.getProperty("java.version") + "; os:" + System.getProperty("os.name") + "; hc:ahc_" - + Versions.APACHE_HTTP_CLIENT + ")"; + + System.getProperty("java.version") + "; os:" + System.getProperty("os.name") + "; arch:" + + System.getProperty("os.arch") + "; hc:ahc_" + Versions.APACHE_HTTP_CLIENT + ")"; try (Connection localConnection = new ConnectionImpl(url, info); Statement stmt = localConnection.createStatement()) { String userAgent = executeQueryAndVerifyUserAgent(stmt, null);