From 3c0f72161f93cd2b44dfc0a69d45758a7bd10b16 Mon Sep 17 00:00:00 2001 From: Anton Okolnychyi Date: Wed, 18 Feb 2026 21:09:06 -0800 Subject: [PATCH] Spark 4.1: Use table IDs in scan equals/hashCode --- .../apache/iceberg/spark/source/SparkBatchQueryScan.java | 2 ++ .../apache/iceberg/spark/source/SparkChangelogScan.java | 4 +++- .../iceberg/spark/source/SparkCopyOnWriteScan.java | 8 +++++++- .../org/apache/iceberg/spark/source/SparkStagedScan.java | 9 ++++++++- 4 files changed, 20 insertions(+), 3 deletions(-) diff --git a/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/source/SparkBatchQueryScan.java b/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/source/SparkBatchQueryScan.java index 0ec77d9d0637..0db1df3aeb40 100644 --- a/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/source/SparkBatchQueryScan.java +++ b/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/source/SparkBatchQueryScan.java @@ -253,6 +253,7 @@ public boolean equals(Object o) { SparkBatchQueryScan that = (SparkBatchQueryScan) o; return table().name().equals(that.table().name()) + && Objects.equals(table().uuid(), that.table().uuid()) && Objects.equals(branch(), that.branch()) && readSchema().equals(that.readSchema()) // compare Spark schemas to ignore field ids && filtersDesc().equals(that.filtersDesc()) @@ -268,6 +269,7 @@ && runtimeFiltersDesc().equals(that.runtimeFiltersDesc()) public int hashCode() { return Objects.hash( table().name(), + table().uuid(), branch(), readSchema(), filtersDesc(), diff --git a/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/source/SparkChangelogScan.java b/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/source/SparkChangelogScan.java index eb4659f3ebc5..c6c5edf45c47 100644 --- a/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/source/SparkChangelogScan.java +++ b/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/source/SparkChangelogScan.java @@ -148,6 +148,7 @@ public boolean equals(Object o) { SparkChangelogScan that = (SparkChangelogScan) o; return table.name().equals(that.table.name()) + && Objects.equals(table.uuid(), that.table.uuid()) && readSchema().equals(that.readSchema()) // compare Spark schemas to ignore field IDs && filtersDesc().equals(that.filtersDesc()) && Objects.equals(startSnapshotId, that.startSnapshotId) @@ -156,7 +157,8 @@ && filtersDesc().equals(that.filtersDesc()) @Override public int hashCode() { - return Objects.hash(table.name(), readSchema(), filtersDesc(), startSnapshotId, endSnapshotId); + return Objects.hash( + table.name(), table.uuid(), readSchema(), filtersDesc(), startSnapshotId, endSnapshotId); } private String filtersDesc() { diff --git a/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/source/SparkCopyOnWriteScan.java b/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/source/SparkCopyOnWriteScan.java index 38664ce8bbfd..a0b531b3a5b8 100644 --- a/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/source/SparkCopyOnWriteScan.java +++ b/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/source/SparkCopyOnWriteScan.java @@ -161,6 +161,7 @@ public boolean equals(Object o) { SparkCopyOnWriteScan that = (SparkCopyOnWriteScan) o; return table().name().equals(that.table().name()) + && Objects.equals(table().uuid(), that.table().uuid()) && readSchema().equals(that.readSchema()) // compare Spark schemas to ignore field ids && filtersDesc().equals(that.filtersDesc()) && Objects.equals(snapshotId(), that.snapshotId()) @@ -170,7 +171,12 @@ && filtersDesc().equals(that.filtersDesc()) @Override public int hashCode() { return Objects.hash( - table().name(), readSchema(), filtersDesc(), snapshotId(), filteredLocations); + table().name(), + table().uuid(), + readSchema(), + filtersDesc(), + snapshotId(), + filteredLocations); } @Override diff --git a/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/source/SparkStagedScan.java b/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/source/SparkStagedScan.java index 435c2cbd15b5..db12ccb23f9b 100644 --- a/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/source/SparkStagedScan.java +++ b/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/source/SparkStagedScan.java @@ -91,6 +91,7 @@ public boolean equals(Object other) { SparkStagedScan that = (SparkStagedScan) other; return table().name().equals(that.table().name()) + && Objects.equals(table().uuid(), that.table().uuid()) && Objects.equals(taskSetId, that.taskSetId) && readSchema().equals(that.readSchema()) && splitSize == that.splitSize @@ -101,7 +102,13 @@ && readSchema().equals(that.readSchema()) @Override public int hashCode() { return Objects.hash( - table().name(), taskSetId, readSchema(), splitSize, splitLookback, openFileCost); + table().name(), + table().uuid(), + taskSetId, + readSchema(), + splitSize, + splitLookback, + openFileCost); } @Override