Skip to content

API, Core: Support default values in UpdateSchema #12211

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
rdblue merged 10 commits into from
Feb 13, 2025
Merged

API, Core: Support default values in UpdateSchema #12211

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
0823872
API, Core: Support default values in UpdateSchema.
rdblue Feb 9, 2025
77b6790
Apply spotless.
rdblue Feb 9, 2025
2e68565
API: Add default implementations to fix revapi.
rdblue Feb 10, 2025
9e5810f
Fix rename detection.
rdblue Feb 10, 2025
585c763
Remove stale tests from Flink.
rdblue Feb 11, 2025
c955bfe
Core: Fix default value handling when conversion fails.
rdblue Feb 11, 2025
60ec570
Remove updateColumn(name, doc, defaultValue).
rdblue Feb 11, 2025
4d9f1ea
Use Literal instead of Object in UpdateSchema.
rdblue Feb 13, 2025
514f7d7
Fix checkstyle and tests.
rdblue Feb 13, 2025
01862f0
Fix errorprone.
rdblue Feb 13, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
288 changes: 264 additions & 24 deletions api/src/main/java/org/apache/iceberg/UpdateSchema.java
View file
Open in desktop

Large diffs are not rendered by default.

4 changes: 4 additions & 0 deletions api/src/main/java/org/apache/iceberg/expressions/Expressions.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
import org.apache.iceberg.relocated.com.google.common.collect.Lists;
import org.apache.iceberg.transforms.Transform;
import org.apache.iceberg.transforms.Transforms;
import org.apache.iceberg.util.NaNUtil;

/** Factory methods for creating {@link Expression expressions}. */
public class Expressions {
Expand Down Expand Up @@ -245,6 +246,9 @@ public static <T> UnboundPredicate<T> predicate(Operation op, String name, Liter
&& op != Operation.NOT_NAN,
"Cannot create %s predicate inclusive a value",
op);
Preconditions.checkArgument(
!NaNUtil.isNaN(lit.value()),
"Invalid expression literal: NaN, use isNaN or notNaN instead");
return new UnboundPredicate<T>(op, ref(name), lit);
}

Expand Down
79 changes: 67 additions & 12 deletions api/src/main/java/org/apache/iceberg/types/Types.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
import java.util.regex.Pattern;
import org.apache.iceberg.Schema;
import org.apache.iceberg.expressions.Expressions;
import org.apache.iceberg.expressions.Literal;
import org.apache.iceberg.relocated.com.google.common.base.Joiner;
import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
Expand Down Expand Up @@ -559,14 +560,20 @@ public static Builder optional(String name) {
return new Builder(true, name);
}

public static Builder builder() {
return new Builder();
}

public static class Builder {
private final boolean isOptional;
private final String name;
private boolean isOptional = true;
private String name = null;

@rdblue rdblue Feb 9, 2025

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

These changes enable the builder to handle all updates, even to whether the field is optional and the field name. That way the builder is always used to copy instead of needing to support specific fields when changing the name.

private Integer id = null;
private Type type = null;
private String doc = null;
private Object initialDefault = null;
private Object writeDefault = null;
private Literal<?> initialDefault = null;
private Literal<?> writeDefault = null;

private Builder() {}

private Builder(boolean isFieldOptional, String fieldName) {
isOptional = isFieldOptional;
Expand All @@ -583,6 +590,26 @@ private Builder(NestedField toCopy) {
this.writeDefault = toCopy.writeDefault;
}

public Builder asRequired() {
this.isOptional = false;
return this;
}

public Builder asOptional() {
this.isOptional = true;
return this;
}

public Builder isOptional(boolean fieldIsOptional) {
this.isOptional = fieldIsOptional;
return this;
}

public Builder withName(String fieldName) {
this.name = fieldName;
Comment thread
Fokko marked this conversation as resolved.
return this;
}

public Builder withId(int fieldId) {
id = fieldId;
return this;
Expand All @@ -598,12 +625,32 @@ public Builder withDoc(String fieldDoc) {
return this;
}

/**
* Set the initial default using an Object.
*
* @deprecated will be removed in 2.0.0; use {@link #withInitialDefault(Literal)} instead.
*/
@Deprecated
public Builder withInitialDefault(Object fieldInitialDefault) {
return withInitialDefault(Expressions.lit(fieldInitialDefault));
}

public Builder withInitialDefault(Literal<?> fieldInitialDefault) {
initialDefault = fieldInitialDefault;
return this;
}

/**
* Set the write default using an Object.
*
* @deprecated will be removed in 2.0.0; use {@link #withWriteDefault(Literal)} instead.
*/
@Deprecated
public Builder withWriteDefault(Object fieldWriteDefault) {
return withWriteDefault(Expressions.lit(fieldWriteDefault));
}

public Builder withWriteDefault(Literal<?> fieldWriteDefault) {
writeDefault = fieldWriteDefault;
return this;
}
Expand All @@ -620,17 +667,17 @@ public NestedField build() {
private final String name;
private final Type type;
private final String doc;
private final Object initialDefault;
private final Object writeDefault;
private final Literal<?> initialDefault;
private final Literal<?> writeDefault;

private NestedField(
boolean isOptional,
int id,
String name,
Type type,
String doc,
Object initialDefault,
Object writeDefault) {
Literal<?> initialDefault,
Literal<?> writeDefault) {
Preconditions.checkNotNull(name, "Name cannot be null");
Preconditions.checkNotNull(type, "Type cannot be null");
this.isOptional = isOptional;
Expand All @@ -642,12 +689,12 @@ private NestedField(
this.writeDefault = castDefault(writeDefault, type);
}

private static Object castDefault(Object defaultValue, Type type) {
private static Literal<?> castDefault(Literal<?> defaultValue, Type type) {
if (type.isNestedType() && defaultValue != null) {
throw new IllegalArgumentException(
String.format("Invalid default value for %s: %s (must be null)", type, defaultValue));
} else if (defaultValue != null) {
return Expressions.lit(defaultValue).to(type).value();
return defaultValue.to(type);
}

return null;
Expand Down Expand Up @@ -699,14 +746,22 @@ public String doc() {
return doc;
}

public Object initialDefault() {
public Literal<?> initialDefaultLiteral() {
return initialDefault;
}

public Object writeDefault() {
public Object initialDefault() {
return initialDefault != null ? initialDefault.value() : null;
}

public Literal<?> writeDefaultLiteral() {
return writeDefault;
}

public Object writeDefault() {
return writeDefault != null ? writeDefault.value() : null;
}

@Override
public String toString() {
return String.format(
Expand Down
10 changes: 6 additions & 4 deletions core/src/main/java/org/apache/iceberg/SchemaParser.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,8 @@
import java.util.Iterator;
import java.util.List;
import java.util.Set;
import org.apache.iceberg.expressions.Expressions;
import org.apache.iceberg.expressions.Literal;
import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
import org.apache.iceberg.relocated.com.google.common.collect.Lists;
import org.apache.iceberg.types.Type;
Expand Down Expand Up @@ -197,9 +199,9 @@ private static Type typeFromJson(JsonNode json) {
throw new IllegalArgumentException("Cannot parse type from json: " + json);
}

private static Object defaultFromJson(String defaultField, Type type, JsonNode json) {
private static Literal<?> defaultFromJson(String defaultField, Type type, JsonNode json) {
if (json.has(defaultField)) {
return SingleValueParser.fromJson(type, json.get(defaultField));
return Expressions.lit(SingleValueParser.fromJson(type, json.get(defaultField)));
}

return null;
Expand Down Expand Up @@ -229,8 +231,8 @@ private static Types.StructType structFromJson(JsonNode json) {
String name = JsonUtil.getString(NAME, field);
Type type = typeFromJson(JsonUtil.get(TYPE, field));

Object initialDefault = defaultFromJson(INITIAL_DEFAULT, type, field);
Object writeDefault = defaultFromJson(WRITE_DEFAULT, type, field);
Literal<?> initialDefault = defaultFromJson(INITIAL_DEFAULT, type, field);
Literal<?> writeDefault = defaultFromJson(WRITE_DEFAULT, type, field);

String doc = JsonUtil.getStringOrNull(DOC, field);
boolean isRequired = JsonUtil.getBool(REQUIRED, field);
Expand Down
Loading