From 481dd3f52e20c4ea83c94427c625912118ce08db Mon Sep 17 00:00:00 2001 From: Minh Vu Date: Sun, 25 Oct 2026 12:22:07 +0100 Subject: [PATCH 1/4] refactor(table)!: name path-based data file additions explicitly Signed-off-by: Minh Vu --- catalog/rest/rest_integration_test.go | 6 +- .../rest/scan_planning_integration_test.go | 2 +- catalog/sql/sql_integration_test.go | 2 +- catalog/sql/sql_test.go | 2 +- table/addfiles_merge_regression_test.go | 20 ++--- table/arrow_utils.go | 2 +- table/arrow_utils_test.go | 4 +- table/branch_planning_test.go | 8 +- table/compaction/analyze_test.go | 4 +- table/compaction/cdc_stress_test.go | 4 +- .../compaction/pos_delete_collect_ext_test.go | 4 +- table/data_file_meta.go | 2 +- table/dv_rewrite_test.go | 2 +- table/equality_delete_commit_test.go | 10 +-- table/equality_delete_reader_test.go | 16 ++-- table/equality_delete_writer_test.go | 2 +- table/internal/parquet_files.go | 2 +- .../orphan_cleanup_integration_test.go | 2 +- ...anifest_list_partitions_regression_test.go | 10 +-- table/mor_delete_conflict_test.go | 2 +- table/no_replay_removals_test.go | 6 +- table/occ_scenario_test.go | 8 +- table/read_perf_deletes_test.go | 4 +- table/replace_files_test.go | 14 +-- table/rewrite_data_files_test.go | 36 ++++---- table/rewrite_files_test.go | 20 ++--- table/rewrite_manifests_test.go | 6 +- table/row_delta_test.go | 18 ++-- table/snapshots_datafiles_internal_test.go | 2 +- table/table_test.go | 90 +++++++++---------- table/transaction.go | 16 ++-- table/transaction_empty_append_test.go | 10 +-- table/transaction_internal_test.go | 8 +- table/transaction_test.go | 2 +- table/variant_shredded_test.go | 16 ++-- table/write_file_io_test.go | 2 +- website/src/api.md | 2 +- 37 files changed, 185 insertions(+), 181 deletions(-) diff --git a/catalog/rest/rest_integration_test.go b/catalog/rest/rest_integration_test.go index 35dd2bd5b0..3ddd5d891d 100644 --- a/catalog/rest/rest_integration_test.go +++ b/catalog/rest/rest_integration_test.go @@ -385,7 +385,7 @@ func (s *RestIntegrationSuite) TestWriteCommitTable() { defer mustFS(s.T(), tbl).Remove(pqfile) txn := tbl.NewTransaction() - s.Require().NoError(txn.AddFiles(s.ctx, []string{pqfile}, nil, false)) + s.Require().NoError(txn.AddDataFilesByPath(s.ctx, []string{pqfile}, nil, false)) updated, err := txn.Commit(s.ctx) s.Require().NoError(err) @@ -465,10 +465,10 @@ func (s *RestIntegrationSuite) TestMultiTableCommit() { // Build transactions that add data files to each table. tx1 := tbl1.NewTransaction() - s.Require().NoError(tx1.AddFiles(s.ctx, []string{pq1}, nil, false)) + s.Require().NoError(tx1.AddDataFilesByPath(s.ctx, []string{pq1}, nil, false)) tx2 := tbl2.NewTransaction() - s.Require().NoError(tx2.AddFiles(s.ctx, []string{pq2}, nil, false)) + s.Require().NoError(tx2.AddDataFilesByPath(s.ctx, []string{pq2}, nil, false)) // Commit both atomically via MultiTableTransaction. mtx, err := catalog.NewMultiTableTransaction(s.cat) diff --git a/catalog/rest/scan_planning_integration_test.go b/catalog/rest/scan_planning_integration_test.go index 6d679c623a..2e43f4d534 100644 --- a/catalog/rest/scan_planning_integration_test.go +++ b/catalog/rest/scan_planning_integration_test.go @@ -268,7 +268,7 @@ func (s *RestIntegrationSuite) createScanPlanningTable(cat *rest.Catalog, ident s.T().Cleanup(func() { s.Require().NoError(mustFS(s.T(), tbl).Remove(dataPath)) }) txn := tbl.NewTransaction() - s.Require().NoError(txn.AddFiles(s.ctx, []string{dataPath}, nil, false)) + s.Require().NoError(txn.AddDataFilesByPath(s.ctx, []string{dataPath}, nil, false)) updated, err := txn.Commit(s.ctx) s.Require().NoError(err) s.Require().NotNil(updated.CurrentSnapshot()) diff --git a/catalog/sql/sql_integration_test.go b/catalog/sql/sql_integration_test.go index 9f8a279325..11032ab4dd 100644 --- a/catalog/sql/sql_integration_test.go +++ b/catalog/sql/sql_integration_test.go @@ -297,7 +297,7 @@ func (s *SQLIntegrationSuite) TestWriteCommitTable() { }(fs, pqfile) txn := tbl.NewTransaction() - s.Require().NoError(txn.AddFiles(s.ctx, []string{pqfile}, nil, false)) + s.Require().NoError(txn.AddDataFilesByPath(s.ctx, []string{pqfile}, nil, false)) updated, err := txn.Commit(s.ctx) s.Require().NoError(err) diff --git a/catalog/sql/sql_test.go b/catalog/sql/sql_test.go index 889c9609d4..a45131a578 100644 --- a/catalog/sql/sql_test.go +++ b/catalog/sql/sql_test.go @@ -2524,7 +2524,7 @@ func (s *SqliteCatalogTestSuite) TestCommitTable() { s.EqualValues(0, tbl.Metadata().CurrentSchema().ID) tx := tbl.NewTransaction() - s.Require().NoError(tx.AddFiles(ctx, []string{pqfile}, nil, false)) + s.Require().NoError(tx.AddDataFilesByPath(ctx, []string{pqfile}, nil, false)) updated, err := tx.Commit(ctx) s.Require().NoError(err) diff --git a/table/addfiles_merge_regression_test.go b/table/addfiles_merge_regression_test.go index cac2a25804..0e4ee3cf15 100644 --- a/table/addfiles_merge_regression_test.go +++ b/table/addfiles_merge_regression_test.go @@ -17,14 +17,14 @@ package table_test -// Regression test for: AddFiles() ignoring commit.manifest-merge.enabled. +// Regression test for: AddDataFilesByPath() ignoring commit.manifest-merge.enabled. // -// Root cause: transaction.go hardcoded .fastAppend() in AddFiles(), bypassing +// Root cause: transaction.go hardcoded .fastAppend() in AddDataFilesByPath(), bypassing // appendSnapshotProducer() which reads ManifestMergeEnabledKey. // // Java reference: BaseTable.newAppend() always uses MergeAppend (merge default=true). // AddDataFiles() and Append() in Go both correctly call appendSnapshotProducer(). -// Only AddFiles() was missing the call. +// Only AddDataFilesByPath() was missing the call. // // Fix: change line 780 in transaction.go from: // @@ -71,10 +71,10 @@ func (c *mergeCatalog) CommitTable(_ context.Context, _ table.Identifier, _ []ta return meta, "", nil } -// TestAddFilesRespectsMergeEnabled verifies that AddFiles() respects the +// TestAddFilesRespectsMergeEnabled verifies that AddDataFilesByPath() respects the // commit.manifest-merge.enabled table property. // -// Before fix: 3 sequential AddFiles() commits produce 3 manifests because +// Before fix: 3 sequential AddDataFilesByPath() commits produce 3 manifests because // fastAppend() is hardcoded and the merge property is ignored. // // After fix: with minCountToMerge=2, the 3rd commit triggers a merge and @@ -131,7 +131,7 @@ func TestAddFilesRespectsMergeEnabled(t *testing.T) { cat, ) - // Perform 3 separate AddFiles commits — simulating the committer's 1-per-batch pattern. + // Perform 3 separate AddDataFilesByPath commits — simulating the committer's 1-per-batch pattern. // minCountToMerge=2 means the 3rd commit should trigger a merge. const numCommits = 3 for i := range numCommits { @@ -139,8 +139,8 @@ func TestAddFilesRespectsMergeEnabled(t *testing.T) { writeParquet(filePath) txn := tbl.NewTransaction() - require.NoError(t, txn.AddFiles(ctx, []string{filePath}, nil, false), - "AddFiles commit %d failed", i+1) + require.NoError(t, txn.AddDataFilesByPath(ctx, []string{filePath}, nil, false), + "AddDataFilesByPath commit %d failed", i+1) tbl, err = txn.Commit(ctx) require.NoError(t, err, "Commit %d failed", i+1) @@ -155,8 +155,8 @@ func TestAddFilesRespectsMergeEnabled(t *testing.T) { // After fix: merge fires on commit 3 (accumulated 3 > minCount 2) → 1 merged manifest. // Before fix: 3 manifests (fastAppend carries all forward without merging). assert.Equal(t, 1, len(manifests), - "expected 1 merged manifest after %d AddFiles commits with manifest merge enabled "+ + "expected 1 merged manifest after %d AddDataFilesByPath commits with manifest merge enabled "+ "(minCountToMerge=2), got %d manifests — "+ - "AddFiles() is not respecting commit.manifest-merge.enabled", + "AddDataFilesByPath() is not respecting commit.manifest-merge.enabled", numCommits, len(manifests)) } diff --git a/table/arrow_utils.go b/table/arrow_utils.go index 9173bfc150..0c4752667f 100644 --- a/table/arrow_utils.go +++ b/table/arrow_utils.go @@ -1881,7 +1881,7 @@ func filesToDataFiles(ctx context.Context, fileIO iceio.IO, meta *MetadataBuilde } }() - // Pass 0 so AddFiles does not claim sort_order_id on files + // Pass 0 so AddDataFilesByPath does not claim sort_order_id on files // it did not write (#1184). Callers that know the file's // sort layout use FileToDataFile with an explicit id. dataFiles[i] = fileToDataFile(ctx, fileIO, filePath, currentSchema, currentSpec, 0, meta.props) diff --git a/table/arrow_utils_test.go b/table/arrow_utils_test.go index 7d94abd1db..621d963ad0 100644 --- a/table/arrow_utils_test.go +++ b/table/arrow_utils_test.go @@ -3497,7 +3497,7 @@ func TestGeoTypeParquetRoundTrip(t *testing.T) { // stored ARROW:schema, as Java/Spark writers produce. geoarrow-go maps the // Parquet GEOMETRY logical type to geoarrow.wkb on read, so the file projects // onto a matching geometry column. A binary table column rejects it, both on -// read and in the FileToDataFile (AddFiles) schema check: geo types are +// read and in the FileToDataFile (AddDataFilesByPath) schema check: geo types are // deliberately not promotable to or from binary (see iceberg.PromoteType and // https://github.com/apache/iceberg-go/issues/2193), and the binary subtests // assert that rejection. @@ -3603,7 +3603,7 @@ func TestGeoParquetReadWithoutStoredSchema(t *testing.T) { assert.ErrorContains(t, err, "cannot promote geometry(srid:4326) to binary") }) - t.Run("AddFiles schema check rejects binary column", func(t *testing.T) { + t.Run("AddDataFilesByPath schema check rejects binary column", func(t *testing.T) { binarySchema := iceberg.NewSchema(0, iceberg.NestedField{ID: 1, Name: "geom", Type: iceberg.PrimitiveTypes.Binary, Required: false}) _, err := table.FileToDataFile(context.Background(), iceio.LocalFS{}, path, binarySchema, *iceberg.UnpartitionedSpec, 0, nil) require.Error(t, err) diff --git a/table/branch_planning_test.go b/table/branch_planning_test.go index cb8667ac8d..868d8c3620 100644 --- a/table/branch_planning_test.go +++ b/table/branch_planning_test.go @@ -430,7 +430,7 @@ func TestBranchAddDataFilesDuplicateCheckUsesBranchHead(t *testing.T) { }) } -// AddFiles runs the same check from paths, through its own snapshot lookup. +// AddDataFilesByPath runs the same check from paths, through its own snapshot lookup. func TestBranchAddFilesDuplicateCheckUsesBranchHead(t *testing.T) { t.Run("rejects a file the branch already references", func(t *testing.T) { tbl := seedDivergentBranch(t, newBranchPlanningTable(t, nil)) @@ -439,7 +439,7 @@ func TestBranchAddFilesDuplicateCheckUsesBranchHead(t *testing.T) { txn, err := tbl.NewTransactionOnBranchWithError(branchPlanningRef) require.NoError(t, err) - err = txn.AddFiles(t.Context(), []string{branchOnly}, nil, false) + err = txn.AddDataFilesByPath(t.Context(), []string{branchOnly}, nil, false) require.ErrorContains(t, err, "cannot add files that are already referenced by table") assert.Contains(t, err.Error(), branchOnly) }) @@ -451,7 +451,7 @@ func TestBranchAddFilesDuplicateCheckUsesBranchHead(t *testing.T) { txn, err := tbl.NewTransactionOnBranchWithError(branchPlanningRef) require.NoError(t, err) - require.NoError(t, txn.AddFiles(t.Context(), []string{mainOnly}, nil, false)) + require.NoError(t, txn.AddDataFilesByPath(t.Context(), []string{mainOnly}, nil, false)) tbl, err = txn.Commit(t.Context()) require.NoError(t, err) @@ -602,7 +602,7 @@ func TestBranchRowDeltaResolvesRemovedDeletesOnBranchHead(t *testing.T) { writeParquetFile(t, dataPath, arrowSchema, `[{"id":1},{"id":2},{"id":3},{"id":4},{"id":5}]`) txn := tbl.NewTransaction() - require.NoError(t, txn.AddFiles(t.Context(), []string{dataPath}, nil, false)) + require.NoError(t, txn.AddDataFilesByPath(t.Context(), []string{dataPath}, nil, false)) tbl, err = txn.Commit(t.Context()) require.NoError(t, err) diff --git a/table/compaction/analyze_test.go b/table/compaction/analyze_test.go index 2b51582c3f..7d0b52e10c 100644 --- a/table/compaction/analyze_test.go +++ b/table/compaction/analyze_test.go @@ -113,7 +113,7 @@ func TestAnalyze_SmallFiles(t *testing.T) { fmt.Sprintf(`[{"id": %d, "data": "row-%d"}]`, i+1, i+1)) tx := tbl.NewTransaction() - require.NoError(t, tx.AddFiles(t.Context(), []string{dataPath}, nil, false)) + require.NoError(t, tx.AddDataFilesByPath(t.Context(), []string{dataPath}, nil, false)) tbl, err = tx.Commit(t.Context()) require.NoError(t, err) } @@ -147,7 +147,7 @@ func TestAnalyze_AllOptimal(t *testing.T) { writeTestParquetFile(t, dataPath, arrowSc, `[{"id": 1, "data": "hello"}]`) tx := tbl.NewTransaction() - require.NoError(t, tx.AddFiles(t.Context(), []string{dataPath}, nil, false)) + require.NoError(t, tx.AddDataFilesByPath(t.Context(), []string{dataPath}, nil, false)) tbl, err = tx.Commit(t.Context()) require.NoError(t, err) diff --git a/table/compaction/cdc_stress_test.go b/table/compaction/cdc_stress_test.go index bec98d9019..78012de95f 100644 --- a/table/compaction/cdc_stress_test.go +++ b/table/compaction/cdc_stress_test.go @@ -83,7 +83,7 @@ func TestCDCStress(t *testing.T) { seedPath := tbl.Location() + "/data/seed.parquet" writeParquet(t, seedPath, arrowSc, generateRowsJSON(baselineRows)) tx := tbl.NewTransaction() - require.NoError(t, tx.AddFiles(t.Context(), []string{seedPath}, nil, false)) + require.NoError(t, tx.AddDataFilesByPath(t.Context(), []string{seedPath}, nil, false)) tbl, err = tx.Commit(t.Context()) require.NoError(t, err) @@ -99,7 +99,7 @@ func TestCDCStress(t *testing.T) { fmt.Sprintf(`[{"id": %d, "data": "cdc-%d"}]`, extraID, cycle)) tx := tbl.NewTransaction() - require.NoError(t, tx.AddFiles(t.Context(), []string{dataPath}, nil, false)) + require.NoError(t, tx.AddDataFilesByPath(t.Context(), []string{dataPath}, nil, false)) tbl, err = tx.Commit(t.Context()) require.NoError(t, err) diff --git a/table/compaction/pos_delete_collect_ext_test.go b/table/compaction/pos_delete_collect_ext_test.go index f757e99b4c..eb83d97576 100644 --- a/table/compaction/pos_delete_collect_ext_test.go +++ b/table/compaction/pos_delete_collect_ext_test.go @@ -49,7 +49,7 @@ func TestCollectDeadPositionDeletes(t *testing.T) { p := tbl.Location() + fmt.Sprintf("/data/d-%d.parquet", i) writeParquet(t, p, arrowSc, fmt.Sprintf(`[{"id": %d, "data": "r%d"}]`, i+1, i+1)) tx := tbl.NewTransaction() - require.NoError(t, tx.AddFiles(ctx, []string{p}, nil, false)) + require.NoError(t, tx.AddDataFilesByPath(ctx, []string{p}, nil, false)) tbl, err = tx.Commit(ctx) require.NoError(t, err) } @@ -160,7 +160,7 @@ func TestDeadPositionDeleteRoundTrip(t *testing.T) { writeParquet(t, dataPaths[i], arrowSc, fmt.Sprintf(`[{"id": %d, "data": "r%d"}]`, i+1, i+1)) } tx := tbl.NewTransaction() - require.NoError(t, tx.AddFiles(ctx, dataPaths, nil, false)) + require.NoError(t, tx.AddDataFilesByPath(ctx, dataPaths, nil, false)) tbl, err = tx.Commit(ctx) require.NoError(t, err) diff --git a/table/data_file_meta.go b/table/data_file_meta.go index aabc710f0e..b04bca0af8 100644 --- a/table/data_file_meta.go +++ b/table/data_file_meta.go @@ -293,7 +293,7 @@ func DataFileFromMetadata(args DataFileArgs) (iceberg.DataFile, error) { statsPlan, pathMapping, tblutils.VariantFieldIDsFromSchema(args.Schema), - // Metadata-only path: no file is read here, so no arrow schema; AddFiles recomputes variant bounds in fileToDataFile. + // Metadata-only path: no file is read here, so no arrow schema; AddDataFilesByPath recomputes variant bounds in fileToDataFile. nil, ) if stats == nil { diff --git a/table/dv_rewrite_test.go b/table/dv_rewrite_test.go index 028f461b6d..f18ae5d029 100644 --- a/table/dv_rewrite_test.go +++ b/table/dv_rewrite_test.go @@ -239,7 +239,7 @@ func TestRewriteFiles_DataSequenceNumberKeepsDVApplicableToReplacement(t *testin oldDataPath := tbl.Location() + "/data/old-data.parquet" writeParquetFile(t, oldDataPath, arrowSc, `[{"id":1,"data":"a"},{"id":2,"data":"b"}]`) tx := tbl.NewTransaction() - require.NoError(t, tx.AddFiles(t.Context(), []string{oldDataPath}, nil, false)) + require.NoError(t, tx.AddDataFilesByPath(t.Context(), []string{oldDataPath}, nil, false)) tbl, err = tx.Commit(t.Context()) require.NoError(t, err) diff --git a/table/equality_delete_commit_test.go b/table/equality_delete_commit_test.go index fbedd93386..76e06c52f0 100644 --- a/table/equality_delete_commit_test.go +++ b/table/equality_delete_commit_test.go @@ -61,7 +61,7 @@ func TestCommitEqualityDeletesUnpartitioned(t *testing.T) { ]`) tx := tbl.NewTransaction() - require.NoError(t, tx.AddFiles(t.Context(), []string{dataPath}, nil, false)) + require.NoError(t, tx.AddDataFilesByPath(t.Context(), []string{dataPath}, nil, false)) tbl, err = tx.Commit(t.Context()) require.NoError(t, err) assertRowCount(t, tbl, 5) @@ -190,7 +190,7 @@ func TestCommitEqualityDeletesConcurrentAppendConflict(t *testing.T) { {"id": 3, "data": "c"} ]`) tx := tbl.NewTransaction() - require.NoError(t, tx.AddFiles(t.Context(), []string{dataPath}, nil, false)) + require.NoError(t, tx.AddDataFilesByPath(t.Context(), []string{dataPath}, nil, false)) tbl, err = tx.Commit(t.Context()) require.NoError(t, err) @@ -205,7 +205,7 @@ func TestCommitEqualityDeletesConcurrentAppendConflict(t *testing.T) { peerPath := tbl.Location() + "/data/peer.parquet" writeParquetFile(t, peerPath, arrowSc, `[{"id": 4, "data": "d"}]`) peerTxn := tbl.NewTransaction() - require.NoError(t, peerTxn.AddFiles(t.Context(), []string{peerPath}, nil, false)) + require.NoError(t, peerTxn.AddDataFilesByPath(t.Context(), []string{peerPath}, nil, false)) _, err = peerTxn.Commit(t.Context()) require.NoError(t, err) @@ -241,7 +241,7 @@ func TestCommitEqualityDeletesSnapshotIsolationAllowsConcurrentAppend(t *testing {"id": 2, "data": "b"} ]`) tx := tbl.NewTransaction() - require.NoError(t, tx.AddFiles(t.Context(), []string{dataPath}, nil, false)) + require.NoError(t, tx.AddDataFilesByPath(t.Context(), []string{dataPath}, nil, false)) tbl, err = tx.Commit(t.Context()) require.NoError(t, err) @@ -253,7 +253,7 @@ func TestCommitEqualityDeletesSnapshotIsolationAllowsConcurrentAppend(t *testing peerPath := tbl.Location() + "/data/peer.parquet" writeParquetFile(t, peerPath, arrowSc, `[{"id": 3, "data": "c"}]`) peerTxn := tbl.NewTransaction() - require.NoError(t, peerTxn.AddFiles(t.Context(), []string{peerPath}, nil, false)) + require.NoError(t, peerTxn.AddDataFilesByPath(t.Context(), []string{peerPath}, nil, false)) _, err = peerTxn.Commit(t.Context()) require.NoError(t, err) diff --git a/table/equality_delete_reader_test.go b/table/equality_delete_reader_test.go index 412dde78da..e2a064e0aa 100644 --- a/table/equality_delete_reader_test.go +++ b/table/equality_delete_reader_test.go @@ -97,7 +97,7 @@ func TestEqualityDeleteReadRoundTrip(t *testing.T) { ]`) tx := tbl.NewTransaction() - require.NoError(t, tx.AddFiles(t.Context(), []string{dataPath}, nil, false)) + require.NoError(t, tx.AddDataFilesByPath(t.Context(), []string{dataPath}, nil, false)) tbl, err = tx.Commit(t.Context()) require.NoError(t, err) assertRowCount(t, tbl, 5) @@ -197,7 +197,7 @@ func TestEqualityDeleteReadPrunesNonOverlappingDeleteFiles(t *testing.T) { ]`) tx := tbl.NewTransaction() - require.NoError(t, tx.AddFiles(ctx, dataPaths, nil, false)) + require.NoError(t, tx.AddDataFilesByPath(ctx, dataPaths, nil, false)) tbl, err = tx.Commit(ctx) require.NoError(t, err) @@ -276,7 +276,7 @@ func TestEqualityDeleteReadResolvesRenamedDataColumnByFieldID(t *testing.T) { ]`) tx := tbl.NewTransaction() - require.NoError(t, tx.AddFiles(t.Context(), []string{dataPath}, nil, false)) + require.NoError(t, tx.AddDataFilesByPath(t.Context(), []string{dataPath}, nil, false)) tbl, err = tx.Commit(t.Context()) require.NoError(t, err) @@ -400,7 +400,7 @@ func TestEqualityDeleteReadResolvesLiteralDottedColumnName(t *testing.T) { ]`) tx := tbl.NewTransaction() - require.NoError(t, tx.AddFiles(t.Context(), []string{dataPath}, nil, false)) + require.NoError(t, tx.AddDataFilesByPath(t.Context(), []string{dataPath}, nil, false)) tbl, err := tx.Commit(t.Context()) require.NoError(t, err) @@ -447,7 +447,7 @@ func TestEqualityDeleteReadRejectsAmbiguousColumns(t *testing.T) { dataPath := tbl.Location() + "/data/data.parquet" writeParquetFile(t, dataPath, arrowSc, `[{"id": 1, "data": "one"}]`) tx := tbl.NewTransaction() - require.NoError(t, tx.AddFiles(t.Context(), []string{dataPath}, nil, false)) + require.NoError(t, tx.AddDataFilesByPath(t.Context(), []string{dataPath}, nil, false)) tbl, err = tx.Commit(t.Context()) require.NoError(t, err) @@ -709,7 +709,7 @@ func TestEqualityDeleteMultiColumnKey(t *testing.T) { ]`) tx := tbl.NewTransaction() - require.NoError(t, tx.AddFiles(t.Context(), []string{dataPath}, nil, false)) + require.NoError(t, tx.AddDataFilesByPath(t.Context(), []string{dataPath}, nil, false)) tbl, err = tx.Commit(t.Context()) require.NoError(t, err) @@ -837,7 +837,7 @@ func TestEqualityDeleteNullableFastPathKeys(t *testing.T) { writeParquetFile(t, dataPath, arrowSc, tt.dataJSON) tx := tbl.NewTransaction() - require.NoError(t, tx.AddFiles(t.Context(), []string{dataPath}, nil, false)) + require.NoError(t, tx.AddDataFilesByPath(t.Context(), []string{dataPath}, nil, false)) tbl, err = tx.Commit(t.Context()) require.NoError(t, err) @@ -928,7 +928,7 @@ func TestEqualityDeleteNullableCompositeFastPathKey(t *testing.T) { writeParquetFile(t, dataPath, arrowSc, tt.dataJSON) tx := tbl.NewTransaction() - require.NoError(t, tx.AddFiles(t.Context(), []string{dataPath}, nil, false)) + require.NoError(t, tx.AddDataFilesByPath(t.Context(), []string{dataPath}, nil, false)) tbl, err = tx.Commit(t.Context()) require.NoError(t, err) diff --git a/table/equality_delete_writer_test.go b/table/equality_delete_writer_test.go index cb0537944b..cc781ccb7a 100644 --- a/table/equality_delete_writer_test.go +++ b/table/equality_delete_writer_test.go @@ -522,7 +522,7 @@ func TestWriteEqualityDeleteFilesCommitViaRowDelta(t *testing.T) { ]`) tx := tbl.NewTransaction() - require.NoError(t, tx.AddFiles(t.Context(), []string{dataPath}, nil, false)) + require.NoError(t, tx.AddDataFilesByPath(t.Context(), []string{dataPath}, nil, false)) tbl, err = tx.Commit(t.Context()) require.NoError(t, err) assertRowCount(t, tbl, 3) diff --git a/table/internal/parquet_files.go b/table/internal/parquet_files.go index eb714a07de..810c14a664 100644 --- a/table/internal/parquet_files.go +++ b/table/internal/parquet_files.go @@ -537,7 +537,7 @@ type geoColumn struct { // order. They are not collected here, so their null counts are not recovered // either: the manifest omits them, which costs pruning but never gives a wrong // answer. The same applies to -// geo files imported via DataFileStatsFromMeta alone (e.g. AddFiles). +// geo files imported via DataFileStatsFromMeta alone (e.g. AddDataFilesByPath). // TODO(#2158): compute geo bounds and null counts for geo columns nested in // structs/lists/maps. func collectGeoColumns(sc *arrow.Schema, colMapping map[string]int) []geoColumn { diff --git a/table/maintenance/orphan_cleanup_integration_test.go b/table/maintenance/orphan_cleanup_integration_test.go index 268adcdda2..fa0ac95c96 100644 --- a/table/maintenance/orphan_cleanup_integration_test.go +++ b/table/maintenance/orphan_cleanup_integration_test.go @@ -157,7 +157,7 @@ func (s *OrphanCleanupIntegrationSuite) createTableWithData() *table.Table { // Commit the data file to the table txn := tbl.NewTransaction() - s.Require().NoError(txn.AddFiles(s.ctx, []string{dataFile}, nil, false)) + s.Require().NoError(txn.AddDataFilesByPath(s.ctx, []string{dataFile}, nil, false)) updatedTable, err := txn.Commit(s.ctx) s.Require().NoError(err) diff --git a/table/manifest_list_partitions_regression_test.go b/table/manifest_list_partitions_regression_test.go index 998a7727ce..0180996e1b 100644 --- a/table/manifest_list_partitions_regression_test.go +++ b/table/manifest_list_partitions_regression_test.go @@ -17,13 +17,13 @@ package table_test -// End-to-end regression test for issue #1309: AddFiles() after a prior Delete() +// End-to-end regression test for issue #1309: AddDataFilesByPath() after a prior Delete() // produces a manifest list whose `partitions` field serializes as Avro null, // which Redshift Spectrum rejects ("Wrong type in Avro file. Field: partitions. // Expected: 12 (union). Got: 7 (null)"). // // The trigger is the inheritance path: Delete() uses the mergeOverwrite producer -// and the following AddFiles() uses fastAppend, which inherits the delete +// and the following AddDataFilesByPath() uses fastAppend, which inherits the delete // snapshot's manifests unchanged. When the manifest list is rewritten, an // inherited manifest whose `partitions` summaries are present-but-empty (an // unpartitioned table has zero field summaries) round-trips as a non-nil pointer @@ -149,13 +149,13 @@ func TestManifestListPartitionsPresentAfterDeleteThenAddFiles(t *testing.T) { tbl, err = tbl.Delete(ctx, iceberg.EqualTo(iceberg.Reference("id"), int64(2)), nil) require.NoError(t, err) - // Step 3 — AddFiles (fastAppend) inherits the delete snapshot's manifests. + // Step 3 — AddDataFilesByPath (fastAppend) inherits the delete snapshot's manifests. // This is the commit whose manifest list #1309 reports as unreadable. extraPath := location + "/data-extra.parquet" writeInt64Parquet(t, fs, extraPath, arrowSchema, []int64{4, 5}) txn := tbl.NewTransaction() - require.NoError(t, txn.AddFiles(ctx, []string{extraPath}, nil, false)) + require.NoError(t, txn.AddDataFilesByPath(ctx, []string{extraPath}, nil, false)) tbl, err = txn.Commit(ctx) require.NoError(t, err) @@ -187,7 +187,7 @@ func TestManifestListPartitionsPresentAfterDeleteThenAddFiles(t *testing.T) { "iceberg-go must read the appended rows back with their real values") } -// writeInt64Parquet writes a single-column ("id") Parquet file so AddFiles has a +// writeInt64Parquet writes a single-column ("id") Parquet file so AddDataFilesByPath has a // real file to ingest. func writeInt64Parquet(t *testing.T, fs iceio.LocalFS, path string, arrowSchema *arrow.Schema, ids []int64) { t.Helper() diff --git a/table/mor_delete_conflict_test.go b/table/mor_delete_conflict_test.go index 84778c39bf..17adaa5193 100644 --- a/table/mor_delete_conflict_test.go +++ b/table/mor_delete_conflict_test.go @@ -140,7 +140,7 @@ func TestMergeOnReadDeleteOfDataFileAddedInSameTransaction(t *testing.T) { writeRowDeltaDataFile(t, tbl, path, 11, 12) txn := tbl.NewTransaction() - require.NoError(t, txn.AddFiles(ctx, []string{path}, nil, false)) + require.NoError(t, txn.AddDataFilesByPath(ctx, []string{path}, nil, false)) require.NoError(t, txn.Delete(ctx, iceberg.EqualTo(iceberg.Reference("id"), int64(11)), nil)) tbl, err := txn.Commit(ctx) require.NoError(t, err) diff --git a/table/no_replay_removals_test.go b/table/no_replay_removals_test.go index 34e3b181b8..36dbf2fd6b 100644 --- a/table/no_replay_removals_test.go +++ b/table/no_replay_removals_test.go @@ -153,7 +153,7 @@ func TestReplaceFilesWithRemovalsFailsInsteadOfReplaying(t *testing.T) { writeParquetFile(t, dataPath, arrowSc, `[{"id": 1, "data": "alpha"}, {"id": 2, "data": "beta"}, {"id": 3, "data": "gamma"}]`) tx := tbl.NewTransaction() - require.NoError(t, tx.AddFiles(t.Context(), []string{dataPath}, nil, false)) + require.NoError(t, tx.AddDataFilesByPath(t.Context(), []string{dataPath}, nil, false)) tbl, err = tx.Commit(t.Context()) require.NoError(t, err) @@ -250,7 +250,7 @@ func TestReplaceFilesDataOnlyStillReplays(t *testing.T) { writeParquetFile(t, dataPath, arrowSc, `[{"id": 1, "data": "alpha"}, {"id": 2, "data": "beta"}]`) tx := tbl.NewTransaction() - require.NoError(t, tx.AddFiles(t.Context(), []string{dataPath}, nil, false)) + require.NoError(t, tx.AddDataFilesByPath(t.Context(), []string{dataPath}, nil, false)) // Overwrites validate added data files under the default // serializable isolation; relax to snapshot so the peer's append is // not rejected as a semantic conflict and the replay itself is @@ -285,7 +285,7 @@ func TestReplaceFilesDataOnlyStillReplays(t *testing.T) { peerPath := tbl.Location() + "/data/data-peer.parquet" writeParquetFile(t, peerPath, arrowSc, `[{"id": 100, "data": "peer"}]`) peerTxn := tbl.NewTransaction() - require.NoError(t, peerTxn.AddFiles(t.Context(), []string{peerPath}, nil, false)) + require.NoError(t, peerTxn.AddDataFilesByPath(t.Context(), []string{peerPath}, nil, false)) _, err = peerTxn.Commit(t.Context()) require.NoError(t, err) diff --git a/table/occ_scenario_test.go b/table/occ_scenario_test.go index 452924a33e..ff3b68a286 100644 --- a/table/occ_scenario_test.go +++ b/table/occ_scenario_test.go @@ -497,7 +497,7 @@ func (s *OCCScenarioTestSuite) TestReplaceFilesNoResurrectionAfterConflict() { `[{"id":1,"val":"a"},{"id":2,"val":"b"},{"id":3,"val":"c"}]`) tx := baseTbl.NewTransaction() - s.Require().NoError(tx.AddFiles(s.ctx, []string{file0Path}, nil, false)) + s.Require().NoError(tx.AddDataFilesByPath(s.ctx, []string{file0Path}, nil, false)) _, err = tx.Commit(s.ctx) s.Require().NoError(err) metaBase := cat.current @@ -511,7 +511,7 @@ func (s *OCCScenarioTestSuite) TestReplaceFilesNoResurrectionAfterConflict() { writeParquetFile(s.T(), peerPath, arrowSc, `[{"id":4,"val":"d"}]`) ptx := peerTbl.NewTransaction() - s.Require().NoError(ptx.AddFiles(s.ctx, []string{peerPath}, nil, false)) + s.Require().NoError(ptx.AddDataFilesByPath(s.ctx, []string{peerPath}, nil, false)) _, err = ptx.Commit(s.ctx) s.Require().NoError(err) metaPeer := peerCat.current @@ -589,7 +589,7 @@ func (s *OCCScenarioTestSuite) TestReplaceFilesNoResurrectionAfterTwoConflicts() `[{"id":1,"val":"a"},{"id":2,"val":"b"},{"id":3,"val":"c"}]`) tx := baseTbl.NewTransaction() - s.Require().NoError(tx.AddFiles(s.ctx, []string{file0Path}, nil, false)) + s.Require().NoError(tx.AddDataFilesByPath(s.ctx, []string{file0Path}, nil, false)) _, err = tx.Commit(s.ctx) s.Require().NoError(err) metaBase := cat.current @@ -607,7 +607,7 @@ func (s *OCCScenarioTestSuite) TestReplaceFilesNoResurrectionAfterTwoConflicts() peer++ ptx := peerTbl.NewTransaction() - s.Require().NoError(ptx.AddFiles(s.ctx, []string{peerPath}, nil, false)) + s.Require().NoError(ptx.AddDataFilesByPath(s.ctx, []string{peerPath}, nil, false)) _, err := ptx.Commit(s.ctx) s.Require().NoError(err) diff --git a/table/read_perf_deletes_test.go b/table/read_perf_deletes_test.go index 350edeed12..f586e82200 100644 --- a/table/read_perf_deletes_test.go +++ b/table/read_perf_deletes_test.go @@ -61,7 +61,7 @@ func TestReadPerfUnderDeletePressure(t *testing.T) { writeParquetFile(t, dataPath, arrowSc, generateRowsJSON(numRows)) tx := tbl.NewTransaction() - require.NoError(t, tx.AddFiles(t.Context(), []string{dataPath}, nil, false)) + require.NoError(t, tx.AddDataFilesByPath(t.Context(), []string{dataPath}, nil, false)) tbl, err = tx.Commit(t.Context()) require.NoError(t, err) @@ -210,7 +210,7 @@ func setupTableWithDeletes(tb testing.TB, numRows, numDeletes int) *table.Table ctx := context.Background() tx := tbl.NewTransaction() - require.NoError(tb, tx.AddFiles(ctx, []string{dataPath}, nil, false)) + require.NoError(tb, tx.AddDataFilesByPath(ctx, []string{dataPath}, nil, false)) tbl, err = tx.Commit(ctx) require.NoError(tb, err) diff --git a/table/replace_files_test.go b/table/replace_files_test.go index a8e2e62fc8..172de05adb 100644 --- a/table/replace_files_test.go +++ b/table/replace_files_test.go @@ -123,7 +123,7 @@ func TestReplaceFiles_DataAndDeleteFiles(t *testing.T) { ]`) tx := tbl.NewTransaction() - require.NoError(t, tx.AddFiles(t.Context(), []string{dataPath}, nil, false)) + require.NoError(t, tx.AddDataFilesByPath(t.Context(), []string{dataPath}, nil, false)) tbl, err = tx.Commit(t.Context()) require.NoError(t, err) assertRowCount(t, tbl, 3) @@ -164,7 +164,7 @@ func TestReplaceFiles_DataAndDeleteFiles(t *testing.T) { {"id": 3, "data": "gamma"} ]`) - // Build new DataFile directly (not via AddFiles which would commit it) + // Build new DataFile directly (not via AddDataFilesByPath which would commit it) newDataFileBuilder, err := iceberg.NewDataFileBuilder( *iceberg.UnpartitionedSpec, iceberg.EntryContentData, newDataPath, iceberg.ParquetFile, nil, nil, nil, 2, 512) @@ -200,7 +200,7 @@ func TestReplaceFilesWithDeleteFilesPreservesDataSequenceNumber(t *testing.T) { dataPath := tbl.Location() + "/data/data.parquet" writeParquetFile(t, dataPath, arrowSc, `[{"id":1,"data":"a"},{"id":2,"data":"b"}]`) tx := tbl.NewTransaction() - require.NoError(t, tx.AddFiles(t.Context(), []string{dataPath}, nil, false)) + require.NoError(t, tx.AddDataFilesByPath(t.Context(), []string{dataPath}, nil, false)) tbl, err = tx.Commit(t.Context()) require.NoError(t, err) @@ -497,7 +497,7 @@ func TestReplaceFilesWithDeleteFilesRejectsPartialPositionDeleteToDVRewrite(t *t dataPath := tbl.Location() + "/data/data.parquet" writeParquetFile(t, dataPath, arrowSc, `[{"id":1,"data":"a"},{"id":2,"data":"b"}]`) tx := tbl.NewTransaction() - require.NoError(t, tx.AddFiles(t.Context(), []string{dataPath}, nil, false)) + require.NoError(t, tx.AddDataFilesByPath(t.Context(), []string{dataPath}, nil, false)) tbl, err = tx.Commit(t.Context()) require.NoError(t, err) @@ -802,7 +802,7 @@ func TestReplaceFiles_DelegatesToReplaceDataFilesWhenNoDeleteFiles(t *testing.T) writeParquetFile(t, dataPath, arrowSc, `[{"id": 1, "data": "hello"}]`) tx := tbl.NewTransaction() - require.NoError(t, tx.AddFiles(t.Context(), []string{dataPath}, nil, false)) + require.NoError(t, tx.AddDataFilesByPath(t.Context(), []string{dataPath}, nil, false)) tbl, err = tx.Commit(t.Context()) require.NoError(t, err) @@ -842,7 +842,7 @@ func TestReplaceFiles_ValidationErrors(t *testing.T) { writeParquetFile(t, dataPath, arrowSc, `[{"id": 1, "data": "hello"}]`) tx := tbl.NewTransaction() - require.NoError(t, tx.AddFiles(t.Context(), []string{dataPath}, nil, false)) + require.NoError(t, tx.AddDataFilesByPath(t.Context(), []string{dataPath}, nil, false)) tbl, err = tx.Commit(t.Context()) require.NoError(t, err) @@ -968,7 +968,7 @@ func seedDataFilesWithPositionDeletes(t *testing.T) (*table.Table, []table.FileS } tx := tbl.NewTransaction() - require.NoError(t, tx.AddFiles(t.Context(), dataPaths, nil, false)) + require.NoError(t, tx.AddDataFilesByPath(t.Context(), dataPaths, nil, false)) tbl, err = tx.Commit(t.Context()) require.NoError(t, err) diff --git a/table/rewrite_data_files_test.go b/table/rewrite_data_files_test.go index d9bc9dc6d8..df1498494d 100644 --- a/table/rewrite_data_files_test.go +++ b/table/rewrite_data_files_test.go @@ -195,7 +195,7 @@ func TestRewriteDataFiles_SmallFiles(t *testing.T) { fmt.Sprintf(`[{"id": %d, "data": "row-%d"}]`, i+1, i+1)) tx := tbl.NewTransaction() - require.NoError(t, tx.AddFiles(t.Context(), []string{dataPath}, nil, false)) + require.NoError(t, tx.AddDataFilesByPath(t.Context(), []string{dataPath}, nil, false)) tbl, err = tx.Commit(t.Context()) require.NoError(t, err) } @@ -241,7 +241,7 @@ func TestRewriteDataFiles_WithPositionDeletes(t *testing.T) { `[{"id": %d, "data": "a"}, {"id": %d, "data": "b"}]`, i*2+1, i*2+2)) tx := tbl.NewTransaction() - require.NoError(t, tx.AddFiles(t.Context(), []string{dataPath}, nil, false)) + require.NoError(t, tx.AddDataFilesByPath(t.Context(), []string{dataPath}, nil, false)) tbl, err = tx.Commit(t.Context()) require.NoError(t, err) } @@ -327,7 +327,7 @@ func TestExecuteCompactionGroup_TargetFileSizeForwarded(t *testing.T) { writeParquetFile(t, dataPath, arrowSc, fmt.Sprintf(`[{"id": %d, "data": "row-%d"}]`, i+1, i+1)) tx := tbl.NewTransaction() - require.NoError(t, tx.AddFiles(t.Context(), []string{dataPath}, nil, false)) + require.NoError(t, tx.AddDataFilesByPath(t.Context(), []string{dataPath}, nil, false)) tbl, err = tx.Commit(t.Context()) require.NoError(t, err) } @@ -388,7 +388,7 @@ func TestExecuteCompactionGroup_ParquetRowGroupLimitForwarded(t *testing.T) { dataPath := tbl.Location() + "/data/rg-limit.parquet" writeParquetFile(t, dataPath, arrowSc, rows.String()) tx := tbl.NewTransaction() - require.NoError(t, tx.AddFiles(t.Context(), []string{dataPath}, nil, false)) + require.NoError(t, tx.AddDataFilesByPath(t.Context(), []string{dataPath}, nil, false)) tbl, err = tx.Commit(t.Context()) require.NoError(t, err) @@ -443,7 +443,7 @@ func TestExecuteCompactionGroup_ScanConcurrencyForwarded(t *testing.T) { writeParquetFile(t, dataPath, arrowSc, fmt.Sprintf(`[{"id": %d, "data": "row-%d"}]`, i+1, i+1)) tx := tbl.NewTransaction() - require.NoError(t, tx.AddFiles(t.Context(), []string{dataPath}, nil, false)) + require.NoError(t, tx.AddDataFilesByPath(t.Context(), []string{dataPath}, nil, false)) tbl, err = tx.Commit(t.Context()) require.NoError(t, err) } @@ -484,7 +484,7 @@ func TestRewriteDataFiles_GroupOptionsForwarded(t *testing.T) { writeParquetFile(t, dataPath, arrowSc, fmt.Sprintf(`[{"id": %d, "data": "row-%d"}]`, i+1, i+1)) tx := tbl.NewTransaction() - require.NoError(t, tx.AddFiles(t.Context(), []string{dataPath}, nil, false)) + require.NoError(t, tx.AddDataFilesByPath(t.Context(), []string{dataPath}, nil, false)) tbl, err = tx.Commit(t.Context()) require.NoError(t, err) } @@ -549,7 +549,7 @@ func TestRewriteDataFiles_PartialProgress(t *testing.T) { fmt.Sprintf(`[{"id": %d, "data": "row"}]`, i+1)) tx := tbl.NewTransaction() - require.NoError(t, tx.AddFiles(t.Context(), []string{dataPath}, nil, false)) + require.NoError(t, tx.AddDataFilesByPath(t.Context(), []string{dataPath}, nil, false)) tbl, err = tx.Commit(t.Context()) require.NoError(t, err) } @@ -594,7 +594,7 @@ func TestRewriteDataFiles_PartialProgressMaxCommitsProcessesAllGroups(t *testing writeParquetFile(t, dataPath, arrowSc, fmt.Sprintf(`[{"id": %d, "data": "row"}]`, i+1)) tx := tbl.NewTransaction() - require.NoError(t, tx.AddFiles(t.Context(), []string{dataPath}, nil, false)) + require.NoError(t, tx.AddDataFilesByPath(t.Context(), []string{dataPath}, nil, false)) tbl, err = tx.Commit(t.Context()) require.NoError(t, err) } @@ -634,7 +634,7 @@ func TestRewriteDataFiles_PartialProgressRejectsCommittedParentReuse(t *testing. dataPath := tbl.Location() + "/data/reuse.parquet" writeParquetFile(t, dataPath, arrowSc, `[{"id": 1, "data": "row"}]`) tx := tbl.NewTransaction() - require.NoError(t, tx.AddFiles(t.Context(), []string{dataPath}, nil, false)) + require.NoError(t, tx.AddDataFilesByPath(t.Context(), []string{dataPath}, nil, false)) tbl, err = tx.Commit(t.Context()) require.NoError(t, err) @@ -679,7 +679,7 @@ func TestRewriteDataFiles_PartialProgressStopsOnUnknownCommitState(t *testing.T) writeParquetFile(t, dataPath, arrowSc, fmt.Sprintf(`[{"id": %d, "data": "row"}, {"id": %d, "data": "row"}]`, i*2+1, i*2+2)) tx := tbl.NewTransaction() - require.NoError(t, tx.AddFiles(t.Context(), []string{dataPath}, nil, false)) + require.NoError(t, tx.AddDataFilesByPath(t.Context(), []string{dataPath}, nil, false)) tbl, err = tx.Commit(t.Context()) require.NoError(t, err) } @@ -718,7 +718,7 @@ func TestRewriteDataFiles_PartialProgressRetainsSharedPositionDeleteAfterFailure `[{"id": %d, "data": "a"}, {"id": %d, "data": "b"}]`, i*2+1, i*2+2)) tx := tbl.NewTransaction() - require.NoError(t, tx.AddFiles(t.Context(), []string{dataPath}, nil, false)) + require.NoError(t, tx.AddDataFilesByPath(t.Context(), []string{dataPath}, nil, false)) tbl, err = tx.Commit(t.Context()) require.NoError(t, err) } @@ -802,7 +802,7 @@ func TestRewriteDataFiles_PartialProgressCleansOutputsBeforeCommit(t *testing.T) `[{"id": %d, "data": "row"}]`, i+1)) tx := tbl.NewTransaction() - require.NoError(t, tx.AddFiles(t.Context(), []string{dataPath}, nil, false)) + require.NoError(t, tx.AddDataFilesByPath(t.Context(), []string{dataPath}, nil, false)) tbl, err = tx.Commit(t.Context()) require.NoError(t, err) } @@ -985,7 +985,7 @@ func TestRewriteDataFiles_ContextCancellation(t *testing.T) { fmt.Sprintf(`[{"id": %d, "data": "row"}]`, i+1)) tx := tbl.NewTransaction() - require.NoError(t, tx.AddFiles(t.Context(), []string{dataPath}, nil, false)) + require.NoError(t, tx.AddDataFilesByPath(t.Context(), []string{dataPath}, nil, false)) tbl, err = tx.Commit(t.Context()) require.NoError(t, err) } @@ -1023,7 +1023,7 @@ func TestRewriteDataFiles_DeadEqualityDeletesDropped(t *testing.T) { `[{"id": %d, "data": "a"}, {"id": %d, "data": "b"}]`, i*2+1, i*2+2)) tx := tbl.NewTransaction() - require.NoError(t, tx.AddFiles(t.Context(), []string{dataPath}, nil, false)) + require.NoError(t, tx.AddDataFilesByPath(t.Context(), []string{dataPath}, nil, false)) tbl, err = tx.Commit(t.Context()) require.NoError(t, err) } @@ -1108,7 +1108,7 @@ func TestRewriteDataFiles_PartialRewritePreservesEqDelete(t *testing.T) { `[{"id": %d, "data": "a"}]`, i+1)) tx := tbl.NewTransaction() - require.NoError(t, tx.AddFiles(t.Context(), []string{dataPath}, nil, false)) + require.NoError(t, tx.AddDataFilesByPath(t.Context(), []string{dataPath}, nil, false)) tbl, err = tx.Commit(t.Context()) require.NoError(t, err) } @@ -1120,7 +1120,7 @@ func TestRewriteDataFiles_PartialRewritePreservesEqDelete(t *testing.T) { preserveDataPath := tbl.Location() + "/data/preserve.parquet" writeParquetFile(t, preserveDataPath, arrowSc, `[{"id": 50, "data": "preserved"}]`) tx := tbl.NewTransaction() - require.NoError(t, tx.AddFiles(t.Context(), []string{preserveDataPath}, nil, false)) + require.NoError(t, tx.AddDataFilesByPath(t.Context(), []string{preserveDataPath}, nil, false)) tbl, err = tx.Commit(t.Context()) require.NoError(t, err) @@ -1193,7 +1193,7 @@ func TestRewriteDataFiles_PreserveDeadEqualityDeletesOptOut(t *testing.T) { `[{"id": %d, "data": "a"}]`, i+1)) tx := tbl.NewTransaction() - require.NoError(t, tx.AddFiles(t.Context(), []string{dataPath}, nil, false)) + require.NoError(t, tx.AddDataFilesByPath(t.Context(), []string{dataPath}, nil, false)) tbl, err = tx.Commit(t.Context()) require.NoError(t, err) } @@ -1299,7 +1299,7 @@ func TestRewriteDataFiles_PartialProgressPreservesEqDeletes(t *testing.T) { `[{"id": %d, "data": "a"}]`, i+1)) tx := tbl.NewTransaction() - require.NoError(t, tx.AddFiles(t.Context(), []string{dataPath}, nil, false)) + require.NoError(t, tx.AddDataFilesByPath(t.Context(), []string{dataPath}, nil, false)) tbl, err = tx.Commit(t.Context()) require.NoError(t, err) } diff --git a/table/rewrite_files_test.go b/table/rewrite_files_test.go index 5b6d21d5d9..fb4189f4cd 100644 --- a/table/rewrite_files_test.go +++ b/table/rewrite_files_test.go @@ -268,7 +268,7 @@ func TestRewriteFiles_Commit_SingleShot(t *testing.T) { writeParquetFile(t, dataPath, arrowSc, fmt.Sprintf(`[{"id": %d, "data": "seed"}]`, i+1)) seedTx := tbl.NewTransaction() - require.NoError(t, seedTx.AddFiles(t.Context(), []string{dataPath}, nil, false)) + require.NoError(t, seedTx.AddDataFilesByPath(t.Context(), []string{dataPath}, nil, false)) tbl, err = seedTx.Commit(t.Context()) require.NoError(t, err) } @@ -311,7 +311,7 @@ func TestRewriteFiles_PreservesIndependentPositionDeleteSequences(t *testing.T) writeParquetFile(t, dataPath, arrowSc, `[{"id": 1, "data": "a"}, {"id": 2, "data": "b"}]`) tx := tbl.NewTransaction() - require.NoError(t, tx.AddFiles(t.Context(), []string{dataPath}, nil, false)) + require.NoError(t, tx.AddDataFilesByPath(t.Context(), []string{dataPath}, nil, false)) tbl, err = tx.Commit(t.Context()) require.NoError(t, err) @@ -427,7 +427,7 @@ func TestRewriteFiles_DeleteFile_RoutesByContentType(t *testing.T) { dataPath := tbl.Location() + "/data/seed.parquet" writeParquetFile(t, dataPath, arrowSc, `[{"id": 1, "data": "seed"}]`) seedTx := tbl.NewTransaction() - require.NoError(t, seedTx.AddFiles(t.Context(), []string{dataPath}, nil, false)) + require.NoError(t, seedTx.AddDataFilesByPath(t.Context(), []string{dataPath}, nil, false)) tbl, err = seedTx.Commit(t.Context()) require.NoError(t, err) @@ -462,7 +462,7 @@ func TestRewriteFiles_DeleteFile_RoutesByContentType(t *testing.T) { // tombstone handling: a rewrite that removes a data file leaves a // DELETED manifest entry for its path on the branch head. That entry // records a removal — the path is no longer referenced by the table — -// so a subsequent AddFiles for the same path must not report a false +// so a subsequent AddDataFilesByPath for the same path must not report a false // "already referenced" conflict. func TestAddFiles_ReAddsPathDeletedByRewrite(t *testing.T) { tbl := newRewriteTestTable(t) @@ -472,7 +472,7 @@ func TestAddFiles_ReAddsPathDeletedByRewrite(t *testing.T) { dataPath := tbl.Location() + "/data/seed.parquet" writeParquetFile(t, dataPath, arrowSc, `[{"id": 1, "data": "seed"}]`) seedTx := tbl.NewTransaction() - require.NoError(t, seedTx.AddFiles(t.Context(), []string{dataPath}, nil, false)) + require.NoError(t, seedTx.AddDataFilesByPath(t.Context(), []string{dataPath}, nil, false)) tbl, err = seedTx.Commit(t.Context()) require.NoError(t, err) @@ -488,7 +488,7 @@ func TestAddFiles_ReAddsPathDeletedByRewrite(t *testing.T) { require.NoError(t, err) readdTx := tbl.NewTransaction() - require.NoError(t, readdTx.AddFiles(t.Context(), []string{dataPath}, nil, false), + require.NoError(t, readdTx.AddDataFilesByPath(t.Context(), []string{dataPath}, nil, false), "a path whose only head entry is a DELETED tombstone is not referenced by the table") tbl, err = readdTx.Commit(t.Context()) require.NoError(t, err) @@ -557,7 +557,7 @@ func TestRewriteFiles_DistributedEquivalence(t *testing.T) { writeParquetFile(t, dataPath, arrowSc, fmt.Sprintf(`[{"id": %d, "data": "row-%d"}]`, i+1, i+1)) tx := tbl.NewTransaction() - require.NoError(t, tx.AddFiles(t.Context(), []string{dataPath}, nil, false)) + require.NoError(t, tx.AddDataFilesByPath(t.Context(), []string{dataPath}, nil, false)) tbl, err = tx.Commit(t.Context()) require.NoError(t, err) } @@ -631,7 +631,7 @@ func TestRewriteFiles_DropsSafePositionDeletes(t *testing.T) { writeParquetFile(t, dataPath, arrowSc, fmt.Sprintf( `[{"id": %d, "data": "a"}, {"id": %d, "data": "b"}]`, i*2+1, i*2+2)) tx := tbl.NewTransaction() - require.NoError(t, tx.AddFiles(t.Context(), []string{dataPath}, nil, false)) + require.NoError(t, tx.AddDataFilesByPath(t.Context(), []string{dataPath}, nil, false)) tbl, err = tx.Commit(t.Context()) require.NoError(t, err) } @@ -735,7 +735,7 @@ func TestRewriteFiles_RejectsConcurrentEqDelete(t *testing.T) { writeParquetFile(t, dataPath, arrowSc, fmt.Sprintf(`[{"id": %d, "data": "row-%d"}]`, i+1, i+1)) tx := tbl.NewTransaction() - require.NoError(t, tx.AddFiles(t.Context(), []string{dataPath}, nil, false)) + require.NoError(t, tx.AddDataFilesByPath(t.Context(), []string{dataPath}, nil, false)) tbl, err = tx.Commit(t.Context()) require.NoError(t, err) } @@ -865,7 +865,7 @@ func TestRewriteFiles_RejectsConcurrentPartitionScopedPosDelete(t *testing.T) { writeParquetFile(t, dataPath, arrowSc, fmt.Sprintf(`[{"id": %d, "data": "row-%d"}]`, i+1, i+1)) tx := tbl.NewTransaction() - require.NoError(t, tx.AddFiles(t.Context(), []string{dataPath}, nil, false)) + require.NoError(t, tx.AddDataFilesByPath(t.Context(), []string{dataPath}, nil, false)) tbl, err = tx.Commit(t.Context()) require.NoError(t, err) } diff --git a/table/rewrite_manifests_test.go b/table/rewrite_manifests_test.go index b5b205c11e..7c1e448368 100644 --- a/table/rewrite_manifests_test.go +++ b/table/rewrite_manifests_test.go @@ -102,7 +102,7 @@ func appendSeparateManifests(t *testing.T, ctx context.Context, tbl *table.Table writeOneRowParquet(t, fs, filePath) txn := tbl.NewTransaction() - require.NoError(t, txn.AddFiles(ctx, []string{filePath}, nil, false)) + require.NoError(t, txn.AddDataFilesByPath(ctx, []string{filePath}, nil, false)) tbl, err = txn.Commit(ctx) require.NoError(t, err) } @@ -662,7 +662,7 @@ func TestRewriteManifestsSpecIDFilter(t *testing.T) { specOneFile := dir + "/spec1-0.parquet" writeOneRowParquet(t, fs, specOneFile) txn = tbl.NewTransaction() - require.NoError(t, txn.AddFiles(ctx, []string{specOneFile}, nil, false)) + require.NoError(t, txn.AddDataFilesByPath(ctx, []string{specOneFile}, nil, false)) tbl, err = txn.Commit(ctx) require.NoError(t, err) @@ -906,7 +906,7 @@ func appendOne(t *testing.T, ctx context.Context, tbl *table.Table, fs iceio.Wri writeOneRowParquet(t, fs, path) txn := tbl.NewTransaction() - require.NoError(t, txn.AddFiles(ctx, []string{path}, nil, false)) + require.NoError(t, txn.AddDataFilesByPath(ctx, []string{path}, nil, false)) out, err := txn.Commit(ctx) require.NoError(t, err) diff --git a/table/row_delta_test.go b/table/row_delta_test.go index 86d025758f..4ea7805807 100644 --- a/table/row_delta_test.go +++ b/table/row_delta_test.go @@ -573,7 +573,7 @@ func TestRowDeltaIntegrationPosDeleteRoundTrip(t *testing.T) { ]`) tx := tbl.NewTransaction() - err = tx.AddFiles(t.Context(), []string{dataPath}, nil, false) + err = tx.AddDataFilesByPath(t.Context(), []string{dataPath}, nil, false) require.NoError(t, err) tbl, err = tx.Commit(t.Context()) @@ -747,7 +747,7 @@ func newTableWithLiveDV(t *testing.T) (*table.Table, string, string, iceberg.Dat ]`) tx := tbl.NewTransaction() - require.NoError(t, tx.AddFiles(t.Context(), []string{dataPath}, nil, false)) + require.NoError(t, tx.AddDataFilesByPath(t.Context(), []string{dataPath}, nil, false)) tbl, err = tx.Commit(t.Context()) require.NoError(t, err) @@ -780,7 +780,7 @@ func newTableWithSharedPuffinDVs(t *testing.T) (*table.Table, string, string, st writeParquetFile(t, pathB, arrowSc, `[{"id": 3, "data": "gamma"}, {"id": 4, "data": "delta"}]`) tx := tbl.NewTransaction() - require.NoError(t, tx.AddFiles(t.Context(), []string{pathA, pathB}, nil, false)) + require.NoError(t, tx.AddDataFilesByPath(t.Context(), []string{pathA, pathB}, nil, false)) tbl, err = tx.Commit(t.Context()) require.NoError(t, err) @@ -859,7 +859,7 @@ func TestRowDeltaDVSupersessionSingleSnapshot(t *testing.T) { ]`) tx := tbl.NewTransaction() - require.NoError(t, tx.AddFiles(t.Context(), []string{dataPath}, nil, false)) + require.NoError(t, tx.AddDataFilesByPath(t.Context(), []string{dataPath}, nil, false)) tbl, err = tx.Commit(t.Context()) require.NoError(t, err) assertRowCount(t, tbl, 5) @@ -1359,7 +1359,7 @@ func TestRowDeltaRemoveDeletesStaleReference(t *testing.T) { writeParquetFile(t, pathB, arrowSc, `[{"id": 3, "data": "gamma"}, {"id": 4, "data": "delta"}]`) tx := tbl.NewTransaction() - require.NoError(t, tx.AddFiles(t.Context(), []string{pathA, pathB}, nil, false)) + require.NoError(t, tx.AddDataFilesByPath(t.Context(), []string{pathA, pathB}, nil, false)) tbl, err = tx.Commit(t.Context()) require.NoError(t, err) @@ -1551,7 +1551,7 @@ func TestRowDeltaRemoveDeletesUnknownFile(t *testing.T) { writeParquetFile(t, realDataPath, arrowSc, `[{"id": 1, "data": "alpha"}]`) tx := tbl.NewTransaction() - require.NoError(t, tx.AddFiles(t.Context(), []string{realDataPath}, nil, false)) + require.NoError(t, tx.AddDataFilesByPath(t.Context(), []string{realDataPath}, nil, false)) tbl, err = tx.Commit(t.Context()) require.NoError(t, err) @@ -1771,7 +1771,7 @@ func TestRowDeltaFastAppendRejectsDuplicateAddedDVs(t *testing.T) { ]`) tx := tbl.NewTransaction() - require.NoError(t, tx.AddFiles(t.Context(), []string{dataPath}, nil, false)) + require.NoError(t, tx.AddDataFilesByPath(t.Context(), []string{dataPath}, nil, false)) tbl, err = tx.Commit(t.Context()) require.NoError(t, err) @@ -1827,7 +1827,7 @@ func TestRowDeltaRejectsConcurrentFirstDVOnRetry(t *testing.T) { ]`) tx := baseTbl.NewTransaction() - require.NoError(t, tx.AddFiles(t.Context(), []string{dataPath}, nil, false)) + require.NoError(t, tx.AddDataFilesByPath(t.Context(), []string{dataPath}, nil, false)) _, err = tx.Commit(t.Context()) require.NoError(t, err) metaBase := baseCat.current @@ -1903,7 +1903,7 @@ func seedRowDeltaDataFile(t *testing.T, tbl *table.Table, path string, ids ...in writeRowDeltaDataFile(t, tbl, path, ids...) tx := tbl.NewTransaction() - require.NoError(t, tx.AddFiles(t.Context(), []string{path}, nil, false)) + require.NoError(t, tx.AddDataFilesByPath(t.Context(), []string{path}, nil, false)) tbl, err := tx.Commit(t.Context()) require.NoError(t, err) diff --git a/table/snapshots_datafiles_internal_test.go b/table/snapshots_datafiles_internal_test.go index fdd8a84328..1dd7c6103b 100644 --- a/table/snapshots_datafiles_internal_test.go +++ b/table/snapshots_datafiles_internal_test.go @@ -93,7 +93,7 @@ func writeSnapshotTestManifestList(t *testing.T, fs iceio.WriteFileIO, dir strin // Why: a manifest entry with status DELETED is a tombstone recording a // removal, not a file reachable from the snapshot. Yielding it would make // existence and duplicate checks (ReplaceFiles' belong-to-table check, -// AddFiles' duplicate walk, classifyFilesForDeletions) treat a file +// AddDataFilesByPath' duplicate walk, classifyFilesForDeletions) treat a file // deleted by the snapshot as still live. ADDED and EXISTING entries are // both live and must keep flowing through. // Condition: a snapshot whose data and delete manifests each carry one diff --git a/table/table_test.go b/table/table_test.go index c00672a4ea..4ca4112986 100644 --- a/table/table_test.go +++ b/table/table_test.go @@ -556,7 +556,7 @@ func (t *TableWritingTestSuite) TestAddFilesUnpartitioned() { } tx := tbl.NewTransaction() - t.Require().NoError(tx.AddFiles(t.ctx, files, nil, false)) + t.Require().NoError(tx.AddDataFilesByPath(t.ctx, files, nil, false)) stagedTbl, err := tx.StagedTable() t.Require().NoError(err) @@ -605,7 +605,7 @@ func (t *TableWritingTestSuite) TestAddFilesFileNotFound() { files = append(files, t.location+"/unpartitioned_file_not_found/unknown.parquet") tx := tbl.NewTransaction() - err := tx.AddFiles(t.ctx, files, nil, false) + err := tx.AddDataFilesByPath(t.ctx, files, nil, false) t.Error(err) t.ErrorIs(err, fs.ErrNotExist) } @@ -624,7 +624,7 @@ func (t *TableWritingTestSuite) TestAddFilesUnpartitionedHasFieldIDs() { } tx := tbl.NewTransaction() - t.Require().NoError(tx.AddFiles(t.ctx, files, nil, false)) + t.Require().NoError(tx.AddDataFilesByPath(t.ctx, files, nil, false)) stagedTbl, err := tx.StagedTable() t.Require().NoError(err) @@ -679,7 +679,7 @@ func (t *TableWritingTestSuite) TestAddFilesFieldIDsWithDifferentNames() { } tx := tbl.NewTransaction() - t.Require().NoError(tx.AddFiles(t.ctx, files, nil, false)) + t.Require().NoError(tx.AddDataFilesByPath(t.ctx, files, nil, false)) stagedTbl, err := tx.StagedTable() t.Require().NoError(err) @@ -735,7 +735,7 @@ func (t *TableWritingTestSuite) TestAddFilesFailsSchemaMismatch() { files := []string{filePath} tx := tbl.NewTransaction() - err = tx.AddFiles(t.ctx, files, nil, false) + err = tx.AddDataFilesByPath(t.ctx, files, nil, false) t.Error(err) t.EqualError(err, `error encountered during file conversion: invalid schema: mismatch in fields: Table Field Requested Field @@ -775,7 +775,7 @@ func (t *TableWritingTestSuite) TestAddFilesPartitionedTable() { } tx := tbl.NewTransaction() - t.Require().NoError(tx.AddFiles(t.ctx, files, nil, false)) + t.Require().NoError(tx.AddDataFilesByPath(t.ctx, files, nil, false)) stagedTbl, err := tx.StagedTable() t.Require().NoError(err) @@ -832,7 +832,7 @@ func (t *TableWritingTestSuite) TestAddFilesToBucketPartitionedTableFails() { } tx := tbl.NewTransaction() - err := tx.AddFiles(t.ctx, files, nil, false) + err := tx.AddDataFilesByPath(t.ctx, files, nil, false) t.Error(err) t.ErrorContains(err, "cannot infer partition value from parquet metadata for a non-linear partition field: baz_bucket_3 with transform bucket[3]") } @@ -860,7 +860,7 @@ func (t *TableWritingTestSuite) TestAddFilesToPartitionedTableFailsLowerAndUpper } tx := tbl.NewTransaction() - err := tx.AddFiles(t.ctx, files, nil, false) + err := tx.AddDataFilesByPath(t.ctx, files, nil, false) t.Error(err) t.ErrorContains(err, "cannot infer partition value from parquet metadata as there is more than one value for partition field: baz. (low: 123, high: 124)") } @@ -898,7 +898,7 @@ func (t *TableWritingTestSuite) TestAddFilesWithLargeAndRegular() { t.writeParquet(mustFS(t.T(), tbl).(iceio.WriteFileIO), filePathLarge, arrTableLarge) tx := tbl.NewTransaction() - t.Require().NoError(tx.AddFiles(t.ctx, []string{filePath, filePathLarge}, nil, false)) + t.Require().NoError(tx.AddDataFilesByPath(t.ctx, []string{filePath, filePathLarge}, nil, false)) scan, err := tx.Scan(table.WithOptions(iceberg.Properties{ table.ScanOptionArrowUseLargeTypes: "true", @@ -921,7 +921,7 @@ func (t *TableWritingTestSuite) TestScanWithOptionsAliasing() { t.writeParquet(mustFS(t.T(), tbl).(iceio.WriteFileIO), filePath, t.arrTbl) tx := tbl.NewTransaction() - t.Require().NoError(tx.AddFiles(t.ctx, []string{filePath}, nil, false)) + t.Require().NoError(tx.AddDataFilesByPath(t.ctx, []string{filePath}, nil, false)) opts := iceberg.Properties{ table.ScanOptionArrowUseLargeTypes: "false", @@ -946,7 +946,7 @@ func (t *TableWritingTestSuite) TestAddFilesValidUpcast() { t.writeParquet(mustFS(t.T(), tbl).(iceio.WriteFileIO), filePath, t.arrTablePromotedTypes) tx := tbl.NewTransaction() - t.Require().NoError(tx.AddFiles(t.ctx, []string{filePath}, nil, false)) + t.Require().NoError(tx.AddDataFilesByPath(t.ctx, []string{filePath}, nil, false)) scan, err := tx.Scan() t.Require().NoError(err) @@ -994,7 +994,7 @@ func (t *TableWritingTestSuite) TestAddFilesSubsetOfSchema() { t.writeParquet(mustFS(t.T(), tbl).(iceio.WriteFileIO), filePath, withoutCol) tx := tbl.NewTransaction() - t.Require().NoError(tx.AddFiles(t.ctx, []string{filePath}, nil, false)) + t.Require().NoError(tx.AddDataFilesByPath(t.ctx, []string{filePath}, nil, false)) scan, err := tx.Scan() t.Require().NoError(err) @@ -1029,9 +1029,9 @@ func (t *TableWritingTestSuite) TestAddFilesDuplicateFilesInFilePaths() { t.writeParquet(mustFS(t.T(), tbl).(iceio.WriteFileIO), filePath, t.arrTbl) tx := tbl.NewTransaction() - err := tx.AddFiles(t.ctx, []string{filePath, filePath}, nil, false) + err := tx.AddDataFilesByPath(t.ctx, []string{filePath, filePath}, nil, false) t.Error(err) - t.ErrorContains(err, "file paths must be unique for AddFiles") + t.ErrorContains(err, "file paths must be unique for AddDataFilesByPath") } func (t *TableWritingTestSuite) TestAddFilesReferencedByCurrentSnapshot() { @@ -1049,9 +1049,9 @@ func (t *TableWritingTestSuite) TestAddFilesReferencedByCurrentSnapshot() { } tx := tbl.NewTransaction() - t.Require().NoError(tx.AddFiles(t.ctx, files, nil, false)) + t.Require().NoError(tx.AddDataFilesByPath(t.ctx, files, nil, false)) - err := tx.AddFiles(t.ctx, files[len(files)-1:], nil, false) + err := tx.AddDataFilesByPath(t.ctx, files[len(files)-1:], nil, false) t.Error(err) t.ErrorContains(err, "cannot add files that are already referenced by table, files:") } @@ -1071,9 +1071,9 @@ func (t *TableWritingTestSuite) TestAddFilesReferencedCurrentSnapshotIgnoreDupli } tx := tbl.NewTransaction() - t.Require().NoError(tx.AddFiles(t.ctx, files, nil, false)) + t.Require().NoError(tx.AddDataFilesByPath(t.ctx, files, nil, false)) - t.Require().NoError(tx.AddFiles(t.ctx, files[len(files)-1:], nil, true)) + t.Require().NoError(tx.AddDataFilesByPath(t.ctx, files[len(files)-1:], nil, true)) staged, err := tx.StagedTable() t.Require().NoError(err) @@ -1139,7 +1139,7 @@ func (t *TableWritingTestSuite) TestReplaceDataFilesByPath() { ) for i := range 5 { tx := tbl.NewTransaction() - t.Require().NoError(tx.AddFiles(ctx, files[i:i+1], nil, false)) + t.Require().NoError(tx.AddDataFilesByPath(ctx, files[i:i+1], nil, false)) tbl, err = tx.Commit(ctx) t.Require().NoError(err) } @@ -1438,7 +1438,7 @@ func (t *TableWritingTestSuite) TestReplaceDataFiles() { } tx := tbl.NewTransaction() - t.Require().NoError(tx.AddFiles(t.ctx, files, nil, false)) + t.Require().NoError(tx.AddDataFilesByPath(t.ctx, files, nil, false)) tbl, err := tx.Commit(t.ctx) t.Require().NoError(err) @@ -1521,7 +1521,7 @@ func (t *TableWritingTestSuite) TestReplaceDataFilesValidatesPartitionSpecID() { t.writeParquet(mustFS(t.T(), tbl).(iceio.WriteFileIO), existingPath, t.arrTbl) tx := tbl.NewTransaction() - t.Require().NoError(tx.AddFiles(t.ctx, []string{existingPath}, nil, false)) + t.Require().NoError(tx.AddDataFilesByPath(t.ctx, []string{existingPath}, nil, false)) tbl, err := tx.Commit(t.ctx) t.Require().NoError(err) @@ -1551,7 +1551,7 @@ func (t *TableWritingTestSuite) TestReplaceDataFilesValidatesPartitionData() { t.writeParquet(mustFS(t.T(), tbl).(iceio.WriteFileIO), existingPath, t.arrTbl) tx := tbl.NewTransaction() - t.Require().NoError(tx.AddFiles(t.ctx, []string{existingPath}, nil, false)) + t.Require().NoError(tx.AddDataFilesByPath(t.ctx, []string{existingPath}, nil, false)) tbl, err := tx.Commit(t.ctx) t.Require().NoError(err) @@ -1686,7 +1686,7 @@ func (t *TableWritingTestSuite) TestReplaceDataFilesDuplicateAddPaths() { t.writeParquet(mustFS(t.T(), tbl).(iceio.WriteFileIO), existingPath, t.arrTbl) tx := tbl.NewTransaction() - t.Require().NoError(tx.AddFiles(t.ctx, []string{existingPath}, nil, false)) + t.Require().NoError(tx.AddDataFilesByPath(t.ctx, []string{existingPath}, nil, false)) tbl, err := tx.Commit(t.ctx) t.Require().NoError(err) @@ -1708,7 +1708,7 @@ func (t *TableWritingTestSuite) TestReplaceDataFilesDuplicateDeletePaths() { t.writeParquet(mustFS(t.T(), tbl).(iceio.WriteFileIO), existingPath, t.arrTbl) tx := tbl.NewTransaction() - t.Require().NoError(tx.AddFiles(t.ctx, []string{existingPath}, nil, false)) + t.Require().NoError(tx.AddDataFilesByPath(t.ctx, []string{existingPath}, nil, false)) tbl, err := tx.Commit(t.ctx) t.Require().NoError(err) @@ -1732,7 +1732,7 @@ func (t *TableWritingTestSuite) TestReplaceDataFilesAddAlreadyReferenced() { t.writeParquet(mustFS(t.T(), tbl).(iceio.WriteFileIO), existingPath2, t.arrTbl) tx := tbl.NewTransaction() - t.Require().NoError(tx.AddFiles(t.ctx, []string{existingPath1, existingPath2}, nil, false)) + t.Require().NoError(tx.AddDataFilesByPath(t.ctx, []string{existingPath1, existingPath2}, nil, false)) tbl, err := tx.Commit(t.ctx) t.Require().NoError(err) @@ -1753,7 +1753,7 @@ func (t *TableWritingTestSuite) TestReplaceDataFilesDeleteNotInTable() { t.writeParquet(mustFS(t.T(), tbl).(iceio.WriteFileIO), existingPath, t.arrTbl) tx := tbl.NewTransaction() - t.Require().NoError(tx.AddFiles(t.ctx, []string{existingPath}, nil, false)) + t.Require().NoError(tx.AddDataFilesByPath(t.ctx, []string{existingPath}, nil, false)) tbl, err := tx.Commit(t.ctx) t.Require().NoError(err) @@ -1776,7 +1776,7 @@ func (t *TableWritingTestSuite) TestReplaceDataFilesNilDeleteFile() { t.writeParquet(mustFS(t.T(), tbl).(iceio.WriteFileIO), existingPath, t.arrTbl) tx := tbl.NewTransaction() - t.Require().NoError(tx.AddFiles(t.ctx, []string{existingPath}, nil, false)) + t.Require().NoError(tx.AddDataFilesByPath(t.ctx, []string{existingPath}, nil, false)) tbl, err := tx.Commit(t.ctx) t.Require().NoError(err) @@ -1797,7 +1797,7 @@ func (t *TableWritingTestSuite) TestReplaceDataFilesNilAddFile() { t.writeParquet(mustFS(t.T(), tbl).(iceio.WriteFileIO), existingPath, t.arrTbl) tx := tbl.NewTransaction() - t.Require().NoError(tx.AddFiles(t.ctx, []string{existingPath}, nil, false)) + t.Require().NoError(tx.AddDataFilesByPath(t.ctx, []string{existingPath}, nil, false)) tbl, err := tx.Commit(t.ctx) t.Require().NoError(err) @@ -1832,7 +1832,7 @@ func (t *TableWritingTestSuite) TestReplaceDataFilesInvalidAddContentType() { t.writeParquet(mustFS(t.T(), tbl).(iceio.WriteFileIO), existingPath, t.arrTbl) tx := tbl.NewTransaction() - t.Require().NoError(tx.AddFiles(t.ctx, []string{existingPath}, nil, false)) + t.Require().NoError(tx.AddDataFilesByPath(t.ctx, []string{existingPath}, nil, false)) tbl, err := tx.Commit(t.ctx) t.Require().NoError(err) @@ -1893,7 +1893,7 @@ func (t *TableWritingTestSuite) TestReplaceDataFilesByPathDuplicateDeletePaths() t.writeParquet(mustFS(t.T(), tbl).(iceio.WriteFileIO), existingPath, t.arrTbl) tx := tbl.NewTransaction() - t.Require().NoError(tx.AddFiles(t.ctx, []string{existingPath}, nil, false)) + t.Require().NoError(tx.AddDataFilesByPath(t.ctx, []string{existingPath}, nil, false)) tbl, err := tx.Commit(t.ctx) t.Require().NoError(err) @@ -1914,7 +1914,7 @@ func (t *TableWritingTestSuite) TestReplaceDataFilesByPathDuplicateAddPaths() { t.writeParquet(mustFS(t.T(), tbl).(iceio.WriteFileIO), existingPath, t.arrTbl) tx := tbl.NewTransaction() - t.Require().NoError(tx.AddFiles(t.ctx, []string{existingPath}, nil, false)) + t.Require().NoError(tx.AddDataFilesByPath(t.ctx, []string{existingPath}, nil, false)) tbl, err := tx.Commit(t.ctx) t.Require().NoError(err) @@ -1937,7 +1937,7 @@ func (t *TableWritingTestSuite) TestReplaceDataFilesByPathAddAlreadyReferenced() t.writeParquet(mustFS(t.T(), tbl).(iceio.WriteFileIO), existingPath2, t.arrTbl) tx := tbl.NewTransaction() - t.Require().NoError(tx.AddFiles(t.ctx, []string{existingPath1, existingPath2}, nil, false)) + t.Require().NoError(tx.AddDataFilesByPath(t.ctx, []string{existingPath1, existingPath2}, nil, false)) tbl, err := tx.Commit(t.ctx) t.Require().NoError(err) @@ -1955,7 +1955,7 @@ func (t *TableWritingTestSuite) TestReplaceDataFilesByPathDeleteNotInTable() { t.writeParquet(mustFS(t.T(), tbl).(iceio.WriteFileIO), existingPath, t.arrTbl) tx := tbl.NewTransaction() - t.Require().NoError(tx.AddFiles(t.ctx, []string{existingPath}, nil, false)) + t.Require().NoError(tx.AddDataFilesByPath(t.ctx, []string{existingPath}, nil, false)) tbl, err := tx.Commit(t.ctx) t.Require().NoError(err) @@ -2013,7 +2013,7 @@ func (t *TableWritingTestSuite) TestExpireSnapshots() { for i := range 5 { tx := tbl.NewTransaction() - t.Require().NoError(tx.AddFiles(ctx, files[i:i+1], nil, false)) + t.Require().NoError(tx.AddDataFilesByPath(ctx, files[i:i+1], nil, false)) tbl, err = tx.Commit(ctx) t.Require().NoError(err) } @@ -2064,7 +2064,7 @@ func (t *TableWritingTestSuite) TestExpireSnapshotsNoOpWhenNothingToExpire() { // Create 3 snapshots for i := range 3 { tx := tbl.NewTransaction() - t.Require().NoError(tx.AddFiles(ctx, files[i:i+1], nil, false)) + t.Require().NoError(tx.AddDataFilesByPath(ctx, files[i:i+1], nil, false)) tbl, err = tx.Commit(ctx) t.Require().NoError(err) } @@ -2130,7 +2130,7 @@ func (t *TableWritingTestSuite) TestExpireSnapshotsUsesTableProperties() { for i := range 5 { tx := tbl.NewTransaction() - t.Require().NoError(tx.AddFiles(ctx, files[i:i+1], nil, false)) + t.Require().NoError(tx.AddDataFilesByPath(ctx, files[i:i+1], nil, false)) tbl, err = tx.Commit(ctx) t.Require().NoError(err) } @@ -2182,7 +2182,7 @@ func (t *TableWritingTestSuite) TestExpireSnapshotsWithMissingParent() { // Create 5 snapshots, each one with a parent pointing to the previous for i := range 5 { tx := tbl.NewTransaction() - t.Require().NoError(tx.AddFiles(ctx, files[i:i+1], nil, false)) + t.Require().NoError(tx.AddDataFilesByPath(ctx, files[i:i+1], nil, false)) tbl, err = tx.Commit(ctx) t.Require().NoError(err) } @@ -2302,7 +2302,7 @@ func (t *TableWritingTestSuite) TestExpireSnapshotsRejectsOnRefRollback() { // Create 5 snapshots for i := range 5 { tx := tbl.NewTransaction() - t.Require().NoError(tx.AddFiles(ctx, files[i:i+1], nil, false)) + t.Require().NoError(tx.AddDataFilesByPath(ctx, files[i:i+1], nil, false)) tbl, err = tx.Commit(ctx) t.Require().NoError(err) } @@ -2372,7 +2372,7 @@ func (t *TableWritingTestSuite) TestExpireSnapshotsRejectsOnRefUpdate() { // Create 3 snapshots for i := range 3 { tx := tbl.NewTransaction() - t.Require().NoError(tx.AddFiles(ctx, files[i:i+1], nil, false)) + t.Require().NoError(tx.AddDataFilesByPath(ctx, files[i:i+1], nil, false)) tbl, err = tx.Commit(ctx) t.Require().NoError(err) } @@ -3152,13 +3152,13 @@ func (t *TableWritingTestSuite) TestDeleteOldMetadataLogsErrorOnFileNotFound() { // transaction 1 to create metadata file tx := tbl.NewTransaction() - tx.AddFiles(ctx, files[0:1], nil, false) + tx.AddDataFilesByPath(ctx, files[0:1], nil, false) tbl_new, err := tx.Commit(ctx) t.Require().NoError(err) // transaction 2 to add files tx_new := tbl_new.NewTransaction() - tx_new.AddFiles(ctx, files[1:2], nil, false) + tx_new.AddDataFilesByPath(ctx, files[1:2], nil, false) _, err = tx_new.Commit(ctx) t.Require().NoError(err) @@ -3205,7 +3205,7 @@ func (t *TableWritingTestSuite) TestDeleteOldMetadataNoErrorLogsOnFileFound() { // transaction 1 to create metadata file tx := tbl.NewTransaction() - tx.AddFiles(ctx, files[0:1], nil, false) + tx.AddDataFilesByPath(ctx, files[0:1], nil, false) tbl_new, err := tx.Commit(ctx) t.Require().NoError(err) @@ -3217,7 +3217,7 @@ func (t *TableWritingTestSuite) TestDeleteOldMetadataNoErrorLogsOnFileFound() { // transaction 2 to add files tx_new := tbl_new.NewTransaction() - tx_new.AddFiles(ctx, files[1:2], nil, false) + tx_new.AddDataFilesByPath(ctx, files[1:2], nil, false) _, err = tx_new.Commit(ctx) t.Require().NoError(err) @@ -3602,7 +3602,7 @@ func TestWritesToUnknownTransformSpecFail(t *testing.T) { location := filepath.ToSlash(strings.ReplaceAll(t.TempDir(), "#", "")) ctx := context.Background() - t.Run("AddFiles", func(t *testing.T) { + t.Run("AddDataFilesByPath", func(t *testing.T) { tbl := newTable(t, location, formatVersion, unknownSpec) arrTable, err := array.TableFromJSON(memory.DefaultAllocator, arrSchema, []string{ `[{"foo": true, "bar": "bar_string", "baz": 1, "qux": "2024-03-07"}]`, @@ -3615,7 +3615,7 @@ func TestWritesToUnknownTransformSpecFail(t *testing.T) { require.NoError(t, err) require.NoError(t, pqarrow.WriteTable(arrTable, fo, arrTable.NumRows(), nil, pqarrow.DefaultWriterProps())) - err = tbl.NewTransaction().AddFiles(ctx, []string{filePath}, nil, false) + err = tbl.NewTransaction().AddDataFilesByPath(ctx, []string{filePath}, nil, false) require.ErrorIs(t, err, iceberg.ErrInvalidTransform) assert.ErrorContains(t, err, "custom_transform[42]") }) diff --git a/table/transaction.go b/table/transaction.go index e8369e6f93..4be79c41c4 100644 --- a/table/transaction.go +++ b/table/transaction.go @@ -947,7 +947,7 @@ func (t *Transaction) ReplaceDataFilesByPath(ctx context.Context, filesToDelete, } if len(filesToDelete) == 0 { if len(filesToAdd) > 0 { - return t.AddFiles(ctx, filesToAdd, snapshotProps, false) + return t.AddDataFilesByPath(ctx, filesToAdd, snapshotProps, false) } } @@ -1558,7 +1558,7 @@ func (t *Transaction) ensureNameMapping() error { // This is useful for clients who have already constructed DataFile objects with metadata, // avoiding the need to read files to extract schema and statistics. // -// Unlike AddFiles, this method does not read files from storage. It validates only metadata +// Unlike AddDataFilesByPath, this method does not read files from storage. It validates only metadata // that can be checked without opening files (for example spec-id and partition field IDs). // // Each DataFile may target any partition spec registered in the table metadata, @@ -2230,15 +2230,15 @@ func (t *Transaction) replaceFiles(ctx context.Context, dataFilesToDelete, dataF return nil } -type AddFilesOption func(addFilesOp *addFilesOperation) +type AddDataFilesByPathOption func(addFilesOp *addFilesOperation) type addFilesOperation struct { concurrency int } -// WithAddFilesConcurrency overwrites the default concurrency for add files operation. +// WithAddDataFilesByPathConcurrency overrides the concurrency used by AddDataFilesByPath. // Default: runtime.GOMAXPROCS(0) -func WithAddFilesConcurrency(concurrency int) AddFilesOption { +func WithAddDataFilesByPathConcurrency(concurrency int) AddDataFilesByPathOption { return func(op *addFilesOperation) { if concurrency > 0 { op.concurrency = concurrency @@ -2246,7 +2246,9 @@ func WithAddFilesConcurrency(concurrency int) AddFilesOption { } } -func (t *Transaction) AddFiles(ctx context.Context, filePaths []string, snapshotProps iceberg.Properties, ignoreDuplicates bool, opts ...AddFilesOption) error { +// AddDataFilesByPath adds data files at the provided paths to the table. +// Unlike AddDataFiles, it opens each file to read its schema and statistics. +func (t *Transaction) AddDataFilesByPath(ctx context.Context, filePaths []string, snapshotProps iceberg.Properties, ignoreDuplicates bool, opts ...AddDataFilesByPathOption) error { meta, err := t.txnMeta() if err != nil { return err @@ -2261,7 +2263,7 @@ func (t *Transaction) AddFiles(ctx context.Context, filePaths []string, snapshot set := make(map[string]struct{}, len(filePaths)) for _, filePath := range filePaths { if _, ok := set[filePath]; ok { - return errors.New("file paths must be unique for AddFiles") + return errors.New("file paths must be unique for AddDataFilesByPath") } set[filePath] = struct{}{} } diff --git a/table/transaction_empty_append_test.go b/table/transaction_empty_append_test.go index edf5227ec9..b0907b2bd4 100644 --- a/table/transaction_empty_append_test.go +++ b/table/transaction_empty_append_test.go @@ -34,10 +34,10 @@ import ( // such commits for metadata-only bookkeeping (e.g. recording ingestion // progress in snapshot or table properties) that must still serialize // through the producer's AssertRefSnapshotID requirement, which only -// engages when a snapshot moves the head. AddFiles with an empty path +// engages when a snapshot moves the head. AddDataFilesByPath with an empty path // list must keep committing an empty append rather than degrading to a // silent no-op. -// Condition: AddFiles with no paths on an empty table, then again on +// Condition: AddDataFilesByPath with no paths on an empty table, then again on // the resulting table after real data was added. // Assertion: each commit creates a new "append" snapshot chained to its // parent, the branch head moves, and existing data files stay reachable. @@ -62,7 +62,7 @@ func TestAddFilesEmptyCommitsEmptySnapshot(t *testing.T) { // Empty append on an empty table creates the branch head. tx := tbl.NewTransaction() - require.NoError(t, tx.AddFiles(t.Context(), nil, nil, false)) + require.NoError(t, tx.AddDataFilesByPath(t.Context(), nil, nil, false)) tbl, err = tx.Commit(t.Context()) require.NoError(t, err) @@ -80,14 +80,14 @@ func TestAddFilesEmptyCommitsEmptySnapshot(t *testing.T) { writeParquetFile(t, dataPath, arrowSc, `[{"id": 1, "data": "alpha"}]`) txData := tbl.NewTransaction() - require.NoError(t, txData.AddFiles(t.Context(), []string{dataPath}, nil, false)) + require.NoError(t, txData.AddDataFilesByPath(t.Context(), []string{dataPath}, nil, false)) tbl, err = txData.Commit(t.Context()) require.NoError(t, err) withData := tbl.CurrentSnapshot() require.NotNil(t, withData) txEmpty := tbl.NewTransaction() - require.NoError(t, txEmpty.AddFiles(t.Context(), nil, nil, false)) + require.NoError(t, txEmpty.AddDataFilesByPath(t.Context(), nil, nil, false)) tbl, err = txEmpty.Commit(t.Context()) require.NoError(t, err) diff --git a/table/transaction_internal_test.go b/table/transaction_internal_test.go index 0b09ed8354..bf0414dc46 100644 --- a/table/transaction_internal_test.go +++ b/table/transaction_internal_test.go @@ -1173,7 +1173,9 @@ func TestTransactionEntryPointsRejectNilAndBrokenTransaction(t *testing.T) { {"AppendTable", func(txn *Transaction) error { return txn.AppendTable(ctx, nil, 0, nil) }}, {"Append", func(txn *Transaction) error { return txn.Append(ctx, nil, nil) }}, {"AddDataFiles", func(txn *Transaction) error { return txn.AddDataFiles(ctx, []iceberg.DataFile{dataFile}, nil) }}, - {"AddFiles", func(txn *Transaction) error { return txn.AddFiles(ctx, []string{"file://a.parquet"}, nil, false) }}, + {"AddDataFilesByPath", func(txn *Transaction) error { + return txn.AddDataFilesByPath(ctx, []string{"file://a.parquet"}, nil, false) + }}, {"ReplaceDataFilesByPath", func(txn *Transaction) error { return txn.ReplaceDataFilesByPath(ctx, []string{"a"}, []string{"b"}, nil) }}, @@ -2169,7 +2171,7 @@ func TestTableCommitWithSnapshotProducerPinsBaseHead(t *testing.T) { tbl, _, _ := newProducerAssertRefTable(t) seed := tbl.NewTransaction() - require.NoError(t, seed.AddFiles(t.Context(), nil, nil, false)) + require.NoError(t, seed.AddDataFilesByPath(t.Context(), nil, nil, false)) tbl, err := seed.Commit(t.Context()) require.NoError(t, err) @@ -2177,7 +2179,7 @@ func TestTableCommitWithSnapshotProducerPinsBaseHead(t *testing.T) { require.NotNil(t, baseHead) tx := tbl.NewTransaction() - require.NoError(t, tx.AddFiles(t.Context(), nil, nil, false)) + require.NoError(t, tx.AddDataFilesByPath(t.Context(), nil, nil, false)) tc, err := tx.TableCommit() require.NoError(t, err) diff --git a/table/transaction_test.go b/table/transaction_test.go index ad08e47c3d..f9328dc4ab 100644 --- a/table/transaction_test.go +++ b/table/transaction_test.go @@ -155,7 +155,7 @@ func (s *SparkIntegrationTestSuite) TestAddFile() { } tx := tbl.NewTransaction() - err = tx.AddFiles(s.ctx, []string{filename}, nil, false) + err = tx.AddDataFilesByPath(s.ctx, []string{filename}, nil, false) s.Require().NoError(err) tbl, err = tx.Commit(s.ctx) diff --git a/table/variant_shredded_test.go b/table/variant_shredded_test.go index babc68060b..d1bc9c2313 100644 --- a/table/variant_shredded_test.go +++ b/table/variant_shredded_test.go @@ -72,7 +72,7 @@ func TestShreddedVariantTableScan(t *testing.T) { ) tx := tbl.NewTransaction() - require.NoError(t, tx.AddFiles(t.Context(), []string{dataPath}, nil, false)) + require.NoError(t, tx.AddDataFilesByPath(t.Context(), []string{dataPath}, nil, false)) tbl, err = tx.Commit(t.Context()) require.NoError(t, err) @@ -185,7 +185,7 @@ func TestShreddedVariantScanNulls(t *testing.T) { &rowDeltaCatalog{metadata: meta}) tx := tbl.NewTransaction() - require.NoError(t, tx.AddFiles(t.Context(), []string{dataPath}, nil, false)) + require.NoError(t, tx.AddDataFilesByPath(t.Context(), []string{dataPath}, nil, false)) tbl, err = tx.Commit(t.Context()) require.NoError(t, err) @@ -427,7 +427,7 @@ func scanShreddedFixture(t *testing.T, fixture string) arrow.Table { return scanned } -// scanShreddedFixtureErr returns the error from AddFiles or the scan instead of +// scanShreddedFixtureErr returns the error from AddDataFilesByPath or the scan instead of // failing, for the rejection cases. func scanShreddedFixtureErr(t *testing.T, fixture string) error { t.Helper() @@ -467,7 +467,7 @@ func scanShreddedFixtureRaw(t *testing.T, fixture string) (arrow.Table, error) { &rowDeltaCatalog{metadata: meta}) tx := tbl.NewTransaction() - if err := tx.AddFiles(t.Context(), []string{dataPath}, nil, false); err != nil { + if err := tx.AddDataFilesByPath(t.Context(), []string{dataPath}, nil, false); err != nil { return nil, err } tbl, err = tx.Commit(t.Context()) @@ -573,7 +573,7 @@ func writeFullyShreddedVariantFile(t *testing.T, path string, fieldID, nRows int writeVariantColumnParquet(t, path, fieldID, shreddedType, bldr) } -// TestShreddedVariantAddFilesBounds pins that AddFiles (which opens the file and +// TestShreddedVariantAddFilesBounds pins that AddDataFilesByPath (which opens the file and // recomputes) emits correct shredded variant child bounds under the parent field id. func TestShreddedVariantAddFilesBounds(t *testing.T) { location := filepath.ToSlash(t.TempDir()) @@ -591,7 +591,7 @@ func TestShreddedVariantAddFilesBounds(t *testing.T) { func(ctx context.Context) (iceio.IO, error) { return iceio.LocalFS{}, nil }, &rowDeltaCatalog{metadata: meta}) tx := tbl.NewTransaction() - require.NoError(t, tx.AddFiles(t.Context(), []string{dataPath}, nil, false)) + require.NoError(t, tx.AddDataFilesByPath(t.Context(), []string{dataPath}, nil, false)) tbl, err = tx.Commit(t.Context()) require.NoError(t, err) @@ -614,8 +614,8 @@ func TestShreddedVariantAddFilesBounds(t *testing.T) { return append(append([]byte{}, v.Metadata().Bytes()...), v.Bytes()...) } - assert.Equal(t, buildObj(0, "row-A"), tasks[0].File.LowerBoundValues()[1], "AddFiles lower bound") - assert.Equal(t, buildObj(nData-1, "row-E"), tasks[0].File.UpperBoundValues()[1], "AddFiles upper bound") + assert.Equal(t, buildObj(0, "row-A"), tasks[0].File.LowerBoundValues()[1], "AddDataFilesByPath lower bound") + assert.Equal(t, buildObj(nData-1, "row-E"), tasks[0].File.UpperBoundValues()[1], "AddDataFilesByPath upper bound") } // setupVariantScanTable writes a shredded variant table from rows and returns a closure reading the "id" column of a filtered scan. diff --git a/table/write_file_io_test.go b/table/write_file_io_test.go index 13e3af85b6..76dc39ff91 100644 --- a/table/write_file_io_test.go +++ b/table/write_file_io_test.go @@ -158,7 +158,7 @@ func TestWritePathsReturnErrorForReadOnlyIO(t *testing.T) { { name: "add files", run: func(ctx context.Context, tbl *Table) error { - return tbl.NewTransaction().AddFiles(ctx, []string{ + return tbl.NewTransaction().AddDataFilesByPath(ctx, []string{ "file:///tmp/read-only-write/add-files.parquet", }, nil, false) }, diff --git a/website/src/api.md b/website/src/api.md index f8df5d69dc..ae92e0adcb 100644 --- a/website/src/api.md +++ b/website/src/api.md @@ -353,7 +353,7 @@ When you already have data files (e.g. produced by another writer), register the ```go txn := tbl.NewTransaction() -err := txn.AddFiles(ctx, []string{ +err := txn.AddDataFilesByPath(ctx, []string{ "s3://my-bucket/sales/orders/data/file-1.parquet", "s3://my-bucket/sales/orders/data/file-2.parquet", }, nil /* snapshot props */, false /* ignoreDuplicates */) From 3b5bc473c5ae8fe5d2a2e0fd145924fb38be01f2 Mon Sep 17 00:00:00 2001 From: Minh Vu Date: Sun, 25 Oct 2026 12:32:16 +0100 Subject: [PATCH 2/4] docs(release): note v1 API compatibility change --- website/src/releases.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/website/src/releases.md b/website/src/releases.md index 074e1453fc..e4c3bb9c97 100644 --- a/website/src/releases.md +++ b/website/src/releases.md @@ -42,6 +42,8 @@ Per-release notes (highlights, breaking changes, contributors) are published on ### Unreleased compatibility notes +- **Path-based data-file addition renamed.** Replace `Transaction.AddFiles` with `AddDataFilesByPath`, `AddFilesOption` with `AddDataFilesByPathOption`, and `WithAddFilesConcurrency` with `WithAddDataFilesByPathConcurrency`. Behavior is unchanged. + - **View representation field uses the Go SQL initialism.** Replace `view.Representation.Sql` with `view.Representation.SQL`. Serialized view metadata still uses the `sql` JSON field. - **Orphan cleanup results now expose only `OrphanFiles`.** Replace `result.OrphanFileLocations` with `result.OrphanFiles` (each entry has `Path` and `SizeBytes`). `OrphanCleanupPlan.Files()` still provides path-only lists during planning. - **Catalog identifier helper renamed.** `catalog.TableNameFromIdent` is removed. Use `catalog.ObjectNameFromIdent` for tables, views, and functions. From 6ccc7f84c68437a57f94b3f44cd8032d8d4cbce7 Mon Sep 17 00:00:00 2001 From: Minh Vu Date: Sun, 25 Oct 2026 15:43:16 +0100 Subject: [PATCH 3/4] chore: sync release notes with main to resolve conflicts --- website/src/releases.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/website/src/releases.md b/website/src/releases.md index e4c3bb9c97..e0d398b749 100644 --- a/website/src/releases.md +++ b/website/src/releases.md @@ -42,8 +42,7 @@ Per-release notes (highlights, breaking changes, contributors) are published on ### Unreleased compatibility notes -- **Path-based data-file addition renamed.** Replace `Transaction.AddFiles` with `AddDataFilesByPath`, `AddFilesOption` with `AddDataFilesByPathOption`, and `WithAddFilesConcurrency` with `WithAddDataFilesByPathConcurrency`. Behavior is unchanged. - +- **Unify statistics blob types with Puffin.** Replace `table.BlobType` with `puffin.BlobType` when constructing `table.BlobMetadata`. Replace `table.BlobTypeApacheDatasketchesThetaV1` with `puffin.BlobTypeDataSketchesTheta` and `table.BlobTypeDeletionVectorV1` with `puffin.BlobTypeDeletionVector`. Use `puffin.BlobType.IsKnown()` to identify the types defined by the Puffin specification; the redundant `table.BlobType.IsValid()` method is removed. Unknown non-empty blob type names still round-trip through table metadata. - **View representation field uses the Go SQL initialism.** Replace `view.Representation.Sql` with `view.Representation.SQL`. Serialized view metadata still uses the `sql` JSON field. - **Orphan cleanup results now expose only `OrphanFiles`.** Replace `result.OrphanFileLocations` with `result.OrphanFiles` (each entry has `Path` and `SizeBytes`). `OrphanCleanupPlan.Files()` still provides path-only lists during planning. - **Catalog identifier helper renamed.** `catalog.TableNameFromIdent` is removed. Use `catalog.ObjectNameFromIdent` for tables, views, and functions. From 345948afeff3d98de006259915916434b7da6f1c Mon Sep 17 00:00:00 2001 From: Matt Topol Date: Mon, 26 Oct 2026 13:54:12 -0400 Subject: [PATCH 4/4] docs(release): move compatibility note to a follow-up PR The note for this change lands in a follow-up release-notes PR so the pre-v1 renames can merge in parallel. --- website/src/releases.md | 1 - 1 file changed, 1 deletion(-) diff --git a/website/src/releases.md b/website/src/releases.md index 9638cf1eff..723138fd8a 100644 --- a/website/src/releases.md +++ b/website/src/releases.md @@ -42,7 +42,6 @@ Per-release notes (highlights, breaking changes, contributors) are published on ### Unreleased compatibility notes -- **Path-based data-file addition renamed.** Replace `Transaction.AddFiles` with `AddDataFilesByPath`, `AddFilesOption` with `AddDataFilesByPathOption`, and `WithAddFilesConcurrency` with `WithAddDataFilesByPathConcurrency`. Behavior is unchanged. - **Filesystem adapter construction consolidated.** Replace `io.FSPreProcName(fsys, fn)` with `io.FS(fsys, io.WithNamePreprocessor(fn))`. Plain `io.FS(fsys)` remains supported. - **Table purge is required on catalogs.** The `catalog.Catalog` interface now embeds `catalog.PurgeableCatalog`; every custom catalog implementation must provide `PurgeTable(ctx, identifier) error`. The narrow `catalog.PurgeableTable` interface is renamed to `catalog.PurgeableCatalog`. Built-in purge behavior is unchanged. - **Unify statistics blob types with Puffin.** Replace `table.BlobType` with `puffin.BlobType` when constructing `table.BlobMetadata`. Replace `table.BlobTypeApacheDatasketchesThetaV1` with `puffin.BlobTypeDataSketchesTheta` and `table.BlobTypeDeletionVectorV1` with `puffin.BlobTypeDeletionVector`. Use `puffin.BlobType.IsKnown()` to identify the types defined by the Puffin specification; the redundant `table.BlobType.IsValid()` method is removed. Unknown non-empty blob type names still round-trip through table metadata.