From 42e3f049d409d41f0459b7e53f6f1611ff1a59a4 Mon Sep 17 00:00:00 2001 From: Bryan Keller Date: Tue, 11 Feb 2025 08:13:03 -0800 Subject: [PATCH 1/4] Core: Adjust Jackson settings for large metadata --- .../java/org/apache/iceberg/rest/RESTObjectMapper.java | 7 ++++++- core/src/main/java/org/apache/iceberg/util/JsonUtil.java | 7 ++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/org/apache/iceberg/rest/RESTObjectMapper.java b/core/src/main/java/org/apache/iceberg/rest/RESTObjectMapper.java index 36ed10916ab1..cf437e11c99b 100644 --- a/core/src/main/java/org/apache/iceberg/rest/RESTObjectMapper.java +++ b/core/src/main/java/org/apache/iceberg/rest/RESTObjectMapper.java @@ -21,12 +21,17 @@ import com.fasterxml.jackson.annotation.JsonAutoDetect; import com.fasterxml.jackson.annotation.PropertyAccessor; import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonFactoryBuilder; import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.PropertyNamingStrategies; class RESTObjectMapper { - private static final JsonFactory FACTORY = new JsonFactory(); + private static final JsonFactory FACTORY = + new JsonFactoryBuilder() + .configure(JsonFactory.Feature.INTERN_FIELD_NAMES, false) + .configure(JsonFactory.Feature.FAIL_ON_SYMBOL_HASH_OVERFLOW, false) + .build(); private static final ObjectMapper MAPPER = new ObjectMapper(FACTORY); private static volatile boolean isInitialized = false; diff --git a/core/src/main/java/org/apache/iceberg/util/JsonUtil.java b/core/src/main/java/org/apache/iceberg/util/JsonUtil.java index 972b89c56717..37d48f5a02c5 100644 --- a/core/src/main/java/org/apache/iceberg/util/JsonUtil.java +++ b/core/src/main/java/org/apache/iceberg/util/JsonUtil.java @@ -19,6 +19,7 @@ package org.apache.iceberg.util; import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonFactoryBuilder; import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; @@ -43,7 +44,11 @@ public class JsonUtil { private JsonUtil() {} - private static final JsonFactory FACTORY = new JsonFactory(); + private static final JsonFactory FACTORY = + new JsonFactoryBuilder() + .configure(JsonFactory.Feature.INTERN_FIELD_NAMES, false) + .configure(JsonFactory.Feature.FAIL_ON_SYMBOL_HASH_OVERFLOW, false) + .build(); private static final ObjectMapper MAPPER = new ObjectMapper(FACTORY); public static JsonFactory factory() { From ceb6a17cbc74634c7fa9536cee0cb330467d7d1f Mon Sep 17 00:00:00 2001 From: Bryan Keller Date: Tue, 11 Feb 2025 09:37:43 -0800 Subject: [PATCH 2/4] fix test classpath issue --- mr/build.gradle | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/mr/build.gradle b/mr/build.gradle index 557b46c682a1..f41c5410a93f 100644 --- a/mr/build.gradle +++ b/mr/build.gradle @@ -51,8 +51,12 @@ project(':iceberg-mr') { exclude group: 'org.apache.hive', module: 'hive-storage-api' } - testImplementation libs.calcite.core - testImplementation libs.calcite.druid + testImplementation(libs.calcite.core) { + exclude group: 'org.apache.calcite.avatica', module: 'avatica' + } + testImplementation(libs.calcite.druid) { + exclude group: 'org.apache.calcite.avatica', module: 'avatica' + } testImplementation project(path: ':iceberg-data', configuration: 'testArtifacts') testImplementation project(path: ':iceberg-api', configuration: 'testArtifacts') @@ -60,7 +64,6 @@ project(':iceberg-mr') { testImplementation project(path: ':iceberg-hive-metastore', configuration: 'testArtifacts') testImplementation libs.avro.avro - testImplementation libs.calcite.core testImplementation libs.kryo.shaded testImplementation platform(libs.jackson.bom) testImplementation libs.jackson.annotations From 3c5d4387792293a03a8cbdc19215e9f00dffb46a Mon Sep 17 00:00:00 2001 From: Bryan Keller Date: Tue, 11 Feb 2025 11:26:23 -0800 Subject: [PATCH 3/4] Disable canonicalization instead --- .../main/java/org/apache/iceberg/rest/RESTObjectMapper.java | 3 +-- core/src/main/java/org/apache/iceberg/util/JsonUtil.java | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/org/apache/iceberg/rest/RESTObjectMapper.java b/core/src/main/java/org/apache/iceberg/rest/RESTObjectMapper.java index cf437e11c99b..42595a783a73 100644 --- a/core/src/main/java/org/apache/iceberg/rest/RESTObjectMapper.java +++ b/core/src/main/java/org/apache/iceberg/rest/RESTObjectMapper.java @@ -29,8 +29,7 @@ class RESTObjectMapper { private static final JsonFactory FACTORY = new JsonFactoryBuilder() - .configure(JsonFactory.Feature.INTERN_FIELD_NAMES, false) - .configure(JsonFactory.Feature.FAIL_ON_SYMBOL_HASH_OVERFLOW, false) + .configure(JsonFactory.Feature.CANONICALIZE_FIELD_NAMES, false) .build(); private static final ObjectMapper MAPPER = new ObjectMapper(FACTORY); private static volatile boolean isInitialized = false; diff --git a/core/src/main/java/org/apache/iceberg/util/JsonUtil.java b/core/src/main/java/org/apache/iceberg/util/JsonUtil.java index 37d48f5a02c5..81c2d23ea6d6 100644 --- a/core/src/main/java/org/apache/iceberg/util/JsonUtil.java +++ b/core/src/main/java/org/apache/iceberg/util/JsonUtil.java @@ -46,8 +46,7 @@ private JsonUtil() {} private static final JsonFactory FACTORY = new JsonFactoryBuilder() - .configure(JsonFactory.Feature.INTERN_FIELD_NAMES, false) - .configure(JsonFactory.Feature.FAIL_ON_SYMBOL_HASH_OVERFLOW, false) + .configure(JsonFactory.Feature.CANONICALIZE_FIELD_NAMES, false) .build(); private static final ObjectMapper MAPPER = new ObjectMapper(FACTORY); From 60533745d9539a0c69eff37182924545abe7f47e Mon Sep 17 00:00:00 2001 From: Bryan Keller Date: Tue, 11 Feb 2025 12:41:01 -0800 Subject: [PATCH 4/4] Revert "Disable canonicalization instead" This reverts commit 3c5d4387792293a03a8cbdc19215e9f00dffb46a. --- .../main/java/org/apache/iceberg/rest/RESTObjectMapper.java | 3 ++- core/src/main/java/org/apache/iceberg/util/JsonUtil.java | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/org/apache/iceberg/rest/RESTObjectMapper.java b/core/src/main/java/org/apache/iceberg/rest/RESTObjectMapper.java index 42595a783a73..cf437e11c99b 100644 --- a/core/src/main/java/org/apache/iceberg/rest/RESTObjectMapper.java +++ b/core/src/main/java/org/apache/iceberg/rest/RESTObjectMapper.java @@ -29,7 +29,8 @@ class RESTObjectMapper { private static final JsonFactory FACTORY = new JsonFactoryBuilder() - .configure(JsonFactory.Feature.CANONICALIZE_FIELD_NAMES, false) + .configure(JsonFactory.Feature.INTERN_FIELD_NAMES, false) + .configure(JsonFactory.Feature.FAIL_ON_SYMBOL_HASH_OVERFLOW, false) .build(); private static final ObjectMapper MAPPER = new ObjectMapper(FACTORY); private static volatile boolean isInitialized = false; diff --git a/core/src/main/java/org/apache/iceberg/util/JsonUtil.java b/core/src/main/java/org/apache/iceberg/util/JsonUtil.java index 81c2d23ea6d6..37d48f5a02c5 100644 --- a/core/src/main/java/org/apache/iceberg/util/JsonUtil.java +++ b/core/src/main/java/org/apache/iceberg/util/JsonUtil.java @@ -46,7 +46,8 @@ private JsonUtil() {} private static final JsonFactory FACTORY = new JsonFactoryBuilder() - .configure(JsonFactory.Feature.CANONICALIZE_FIELD_NAMES, false) + .configure(JsonFactory.Feature.INTERN_FIELD_NAMES, false) + .configure(JsonFactory.Feature.FAIL_ON_SYMBOL_HASH_OVERFLOW, false) .build(); private static final ObjectMapper MAPPER = new ObjectMapper(FACTORY);