diff --git a/spark/src/main/scala/org/apache/comet/serde/operator/CometIcebergNativeWrite.scala b/spark/src/main/scala/org/apache/comet/serde/operator/CometIcebergNativeWrite.scala index 5867bbb3f79..b6fc9fd62bf 100644 --- a/spark/src/main/scala/org/apache/comet/serde/operator/CometIcebergNativeWrite.scala +++ b/spark/src/main/scala/org/apache/comet/serde/operator/CometIcebergNativeWrite.scala @@ -89,6 +89,16 @@ object CometIcebergNativeWrite extends CometOperatorSerde[IcebergWriteExec] with private val ParquetWritePropertyPrefix = "write.parquet." private val ParquetMrPropertyPrefix = "parquet." + // Hadoop-side `parquet.*` keys that iceberg-java's writer never consumes, so seeing them + // in the session Hadoop configuration does not indicate the native writer would diverge. + // `parquet.hadoop.vectored.io.enabled` is a reader-side vectored-IO knob declared by + // parquet-hadoop as `ParquetInputFormat.HADOOP_VECTORED_IO_ENABLED` (default `true` in + // parquet-hadoop 1.16+) and only consulted by parquet-mr's Hadoop reader path. Keep it + // out of the writer-compatibility gate so that environments which seed it into the + // session Hadoop configuration do not silently disable native Iceberg writes. + private val IgnoredHadoopParquetConfKeys: Set[String] = Set( + "parquet.hadoop.vectored.io.enabled") + private lazy val vettedParquetWriteKeys: Set[String] = Set( PropertyKeys.ParquetCompressionCodec, PropertyKeys.ParquetCompressionLevel, @@ -271,7 +281,8 @@ object CometIcebergNativeWrite extends CometOperatorSerde[IcebergWriteExec] with private val requireNoParquetHadoopConfOverrides: TriggerRule = ctx => ctx.hadoopConf.asScala .map(_.getKey) - .find(_.startsWith(ParquetMrPropertyPrefix)) + .filter(_.startsWith(ParquetMrPropertyPrefix)) + .find(k => !IgnoredHadoopParquetConfKeys.contains(k)) .map(k => s"Hadoop configuration sets $k (reaches iceberg-java's writer but not native)") private val requireSupportedStorageScheme: TriggerRule = ctx => diff --git a/spark/src/test/scala/org/apache/comet/CometIcebergWriteDetectionSuite.scala b/spark/src/test/scala/org/apache/comet/CometIcebergWriteDetectionSuite.scala index 71da5160f79..264b8d26171 100644 --- a/spark/src/test/scala/org/apache/comet/CometIcebergWriteDetectionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometIcebergWriteDetectionSuite.scala @@ -357,6 +357,20 @@ class CometIcebergWriteDetectionSuite extends CometTestBase with CometIcebergTes } } + // parquet.hadoop.vectored.io.enabled is a reader-side vectored-IO knob declared by + // parquet-hadoop (ParquetInputFormat.HADOOP_VECTORED_IO_ENABLED, default true in + // parquet-hadoop 1.16+). iceberg-java's writer never consumes it, so it must not + // disable native Iceberg writes when it happens to be present in the session + // Hadoop configuration. + test("Compatible when only parquet.hadoop.vectored.io.enabled is set in Hadoop configuration") { + withDetectionCatalog { dir => + createTable(dir, "vectored_io_only", partitionSpec = "") + withSQLConf("parquet.hadoop.vectored.io.enabled" -> "true") { + assertSupportLevelIs[Compatible]("vectored_io_only") + } + } + } + test("fall-back: io-impl set") { withDetectionCatalog { dir => createTable(