Repository navigation
[SPARK-57152][SDP] Implement SCD2 Batch Processor; Find Affected Aux/Target Table Rows #56283
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -26,7 +26,7 @@ import org.apache.spark.sql.types.{DataType, StructField, StructType} | |
| import org.apache.spark.util.ArrayImplicits._ | ||
|
|
||
| /** | ||
| * Per-microbatch processor for SCD Type 2 AutoCDC flows, complying with the specified | ||
| * [[changeArgs]] configuration. | ||
| * | ||
| * @param changeArgs The CDC flow configuration. | ||
|
|
@@ -37,6 +37,18 @@ case class Scd2BatchProcessor( | |
| changeArgs: ChangeArgs, | ||
| resolvedSequencingType: DataType) { | ||
|
|
||
| /** | ||
| * Backtick-quoted key column names. Use when the name flows through an expression parser | ||
| * (e.g., [[F.col]]), which interprets dotted names as struct-field accesses. | ||
| */ | ||
| private lazy val keysQuoted: Seq[String] = changeArgs.keys.map(_.quoted) | ||
|
|
||
| /** | ||
| * Raw key column names. Use when the name is matched literally against a schema field | ||
| * (e.g., DataFrame `.join(other, usingColumns)`), where backticks are NOT stripped. | ||
| */ | ||
| private lazy val keysRaw: Seq[String] = changeArgs.keys.map(_.name) | ||
|
|
||
| /** | ||
| * Reconcile a CDC microbatch into the canonical form the auxiliary- and target-table merges | ||
| * consume. | ||
|
|
@@ -50,14 +62,17 @@ case class Scd2BatchProcessor( | |
| * whether across microbatches or within the same microbatch, is the responsibility of | ||
| * downstream reconciliation - not preprocessing. | ||
| * | ||
| * @param microbatchDf | ||
| * the incoming CDC microbatch. | ||
| * @return | ||
| * a dataframe that retains every input row 1:1 - no rows added, dropped, reordered, or | ||
| * merged - with the following schema, in column order: | ||
| * 1. The user columns of `microbatchDf` that survive [[ChangeArgs.columnSelection]], in | ||
| * the order they appeared in the input. | ||
| * 2. [[startAtColName]], populated with the sequence value of the row. | ||
| * 3. [[endAtColName]], populated with the sequence value of the row IFF it's a delete | ||
| * event, null otherwise. | ||
| * 4. [[cdcMetadataColName]], conforming to [[cdcMetadataColSchema]]. | ||
| */ | ||
| private[autocdc] def preprocessMicrobatch(microbatchDf: DataFrame): DataFrame = { | ||
| microbatchDf | ||
|
|
@@ -108,7 +123,7 @@ case class Scd2BatchProcessor( | |
| private def extendMicrobatchRowsWithCdcMetadata(microbatchDf: DataFrame): DataFrame = { | ||
| microbatchDf.withColumn( | ||
| colName = AutoCdcReservedNames.cdcMetadataColName, | ||
| col = Scd2BatchProcessor.constructCdcMetadataCol( | ||
| recordStartAt = changeArgs.sequencing, | ||
| sequencingType = resolvedSequencingType | ||
| ) | ||
|
|
@@ -159,6 +174,172 @@ case class Scd2BatchProcessor( | |
| microbatch.select(finalColumnsToSelect: _*) | ||
| } | ||
|
|
||
| /** | ||
| * For each key in the preprocessed microbatch, compute the earliest [[recordStartAtFieldName]] | ||
| * across the key's events. | ||
| * | ||
| * @param preprocessedBatchDf | ||
| * a validated and preprocessed microbatch as produced by [[preprocessMicrobatch]] - in | ||
| * particular, non-null key columns and a non-null [[recordStartAtFieldName]] on every row. | ||
| * @return | ||
| * a dataframe containing one row per distinct key, with the key columns | ||
| * ([[ChangeArgs.keys]]) and a [[minSequenceColName]] column carrying the min | ||
| * [[recordStartAtFieldName]] across that key's records in the microbatch. | ||
| */ | ||
| private[autocdc] def computeMinimumSequencePerKey(preprocessedBatchDf: DataFrame): DataFrame = { | ||
| val recordStartAt = | ||
| Scd2BatchProcessor.recordStartAtOf(F.col(AutoCdcReservedNames.cdcMetadataColName)) | ||
| preprocessedBatchDf | ||
| .groupBy(keysQuoted.map(F.col): _*) | ||
| .agg(F.min(recordStartAt).alias(Scd2BatchProcessor.minSequenceColName)) | ||
| } | ||
|
|
||
| /** | ||
| * Find the auxiliary-table rows whose state matters for reconciling the microbatch. | ||
| * | ||
| * @param rawAuxiliaryTableDf | ||
| * the auxiliary table in its native schema, which is expected to contain | ||
| * [[deletedByBatchIdColName]] in addition to all of the columns in the target table. | ||
| * @param perKeyMinimumSequenceInMicrobatchDf | ||
| * one row per distinct key as produced by [[computeMinimumSequencePerKey]], representing | ||
| * the minimum sequence for that key in the microbatch. | ||
| * @param batchId | ||
| * the underlying Spark streaming query's batchId, used to scope aux-row visibility for | ||
| * replay-stability across retries of the same microbatch. | ||
| * @return | ||
| * a dataframe containing all the affected aux rows, with the aux-only | ||
| * [[deletedByBatchIdColName]] column dropped so the result is union-compatible with | ||
| * preprocessed microbatch rows and target-table rows downstream. | ||
| */ | ||
| private[autocdc] def findAffectedRowsFromAuxiliaryTable( | ||
| rawAuxiliaryTableDf: DataFrame, | ||
| perKeyMinimumSequenceInMicrobatchDf: DataFrame, | ||
| batchId: Long | ||
| ): DataFrame = { | ||
| val auxTableRecordStartAtField = Scd2BatchProcessor.recordStartAtOf( | ||
| F.col(AutoCdcReservedNames.cdcMetadataColName) | ||
| ) | ||
| val auxTableDeletedByBatchIdCol = F.col(Scd2BatchProcessor.deletedByBatchIdColName) | ||
|
|
||
| val reducedAuxiliaryTableDf = rawAuxiliaryTableDf | ||
| .filter( | ||
| // [[deletedByBatchIdColName]] carries the batchId whose MERGE logically deleted the | ||
| // row, or null on live aux rows. Rows deleted by other batches are excluded - those | ||
| // decisions are final. Rows deleted by THIS batch are re-included - this can only | ||
| // happen when the current execution is a retry of a partially-failed prior attempt of | ||
| // the same microbatch. | ||
| auxTableDeletedByBatchIdCol.isNull || | ||
| auxTableDeletedByBatchIdCol === F.lit(batchId) | ||
| ) | ||
| // Drop the aux-only idempotency column so the output schema matches target-table rows | ||
| // and preprocessed-microbatch rows (which share the same canonical SCD2 row schema). | ||
| .drop(Scd2BatchProcessor.deletedByBatchIdColName) | ||
|
|
||
| val perKeyMinimumSequenceInMicrobatchCol = F.col(Scd2BatchProcessor.minSequenceColName) | ||
|
|
||
| // Per key, identify the sequence value associated with the anchor row in the aux table. | ||
| // | ||
| // The anchor row is the aux row with the largest [[recordStartAtFieldName]] strictly less | ||
| // than the min sequence in the incoming microbatch for that key. The reconciler needs this | ||
| // "left context" in two cases: | ||
| // (1) Incoming no-op upsert: without the anchor, it would look like a new run head, when in | ||
| // reality it's a part of an existing no-op run/head. | ||
| // (2) Incoming state-changing upsert that bisects two aux no-ops: the anchor surfaces | ||
| // the before-half so both halves can be promoted to target. (The after-half is | ||
| // picked up by the >= minSeq branch.) | ||
| // | ||
| // Because no-op upserts are stored only in the aux table, the anchor concept only exists when | ||
| // pulling in rows from the aux table, and is not relevant for the target table. | ||
| // | ||
| // Keys with no aux row strictly before the min sequence have no anchor; their affected set | ||
| // reduces to "all aux rows at or after the min sequence." | ||
| // | ||
| // The shape of this DataFrame is: [key1, key2, ... keyN, anchorSequence] | ||
| val perKeyAnchorSequenceDf = reducedAuxiliaryTableDf | ||
| // The number of rows in [[perKeyMinimumSequenceInMicrobatchDf]] is bounded by the | ||
| // number of unique keys in the microbatch, which should typically be small. The | ||
| // auxiliary table should generally also be small, containing only no-op upsert runs | ||
| // and tombstones per key. Therefore this join should be cheap, and broadcast joinable. | ||
| .join(perKeyMinimumSequenceInMicrobatchDf, keysRaw) | ||
| .filter(auxTableRecordStartAtField < perKeyMinimumSequenceInMicrobatchCol) | ||
| .groupBy(keysQuoted.map(F.col): _*) | ||
| .agg( | ||
| F.max(auxTableRecordStartAtField).as(Scd2BatchProcessor.anchorSequenceColName) | ||
| ) | ||
| val anchorSequenceCol = F.col(Scd2BatchProcessor.anchorSequenceColName) | ||
| val auxRowIsAnchorRow = auxTableRecordStartAtField === anchorSequenceCol | ||
|
|
||
| // Now that we have the minimum sequence in the microbatch and the sequence of the anchor row, | ||
| // we have enough information to compute the full set of auxiliary rows that may affect or | ||
| // be affected by the microbatch. Membership here is a conservative superset: every row that | ||
| // could possibly participate in reconciliation is included, but downstream reconciliation | ||
| // determines the actual outcome per row. | ||
| val auxRowIsAtOrAfterMinSequenceInMicrobatch = | ||
| auxTableRecordStartAtField >= perKeyMinimumSequenceInMicrobatchCol | ||
|
|
||
| val auxRowAffectsMicrobatch = auxRowIsAtOrAfterMinSequenceInMicrobatch || auxRowIsAnchorRow | ||
|
|
||
| val affectedRowsFromAuxiliaryTable = reducedAuxiliaryTableDf | ||
| // Per row, project the minimum microbatch sequence and anchor sequence for that row's key | ||
| // set onto the row, so the affected-row predicate can be evaluated in a single filter. | ||
| .join(perKeyMinimumSequenceInMicrobatchDf, keysRaw) | ||
| .join( | ||
| perKeyAnchorSequenceDf, | ||
| keysRaw, | ||
| joinType = "left" | ||
| ) | ||
| .filter(auxRowAffectsMicrobatch) | ||
| .drop(perKeyMinimumSequenceInMicrobatchCol, anchorSequenceCol) | ||
|
|
||
| affectedRowsFromAuxiliaryTable | ||
| } | ||
|
|
||
| /** | ||
| * Find the target-table rows whose state matters for reconciling the microbatch. | ||
| * | ||
| * @param targetTableDf | ||
| * the target table in its native schema. | ||
| * @param perKeyMinimumSequenceInMicrobatchDf | ||
| * one row per distinct key as produced by [[computeMinimumSequencePerKey]], representing | ||
| * the minimum sequence for that key in the microbatch. | ||
| * @return | ||
| * a dataframe containing the affected target rows, with all columns passed-through. | ||
| */ | ||
| private[autocdc] def findAffectedRowsFromTargetTable( | ||
| targetTableDf: DataFrame, | ||
| perKeyMinimumSequenceInMicrobatchDf: DataFrame | ||
| ): DataFrame = { | ||
| val targetEndAtCol = F.col(Scd2BatchProcessor.endAtColName) | ||
| val perKeyMinimumSequenceInMicrobatchCol = F.col(Scd2BatchProcessor.minSequenceColName) | ||
|
|
||
| // Per key, identify all the rows in the target table that may be affected by the | ||
| // incoming microbatch. | ||
| // | ||
| // Unlike the auxiliary table, the target table holds visible rows only: no hidden open | ||
| // no-op upsert rows, no tombstones. Visible rows for a given key form a non-overlapping | ||
| // interval partition over the sequencing axis, and at most one row has a null [[endAtColName]] | ||
| // (the currently active row per key). | ||
| // | ||
| // Hence we can simply grab all rows that were active at some point after the min sequencing | ||
| // per key, which can be determined entirely by the row's [[endAtColName]]. | ||
| val isCurrentlyActiveRow = targetEndAtCol.isNull | ||
|
|
||
| // `>=` (rather than strict `>`) additionally pulls in the row that closes exactly at the | ||
|
Contributor
There was a problem hiding this comment. I think it's possible to have multiple rows here given the duplicate discussion from the prior PR if multiple updates at exactly
Contributor
Author
There was a problem hiding this comment. So this should actually never actually be possible by construction if we implement SCD2 correctly E2E. In SCD2 the target table should never have records that overlap by [startAt, endAt) intervals for any given key. The input here should be the raw target table exactly as-is, which means there can never be rows with duplicate A large part of how we make this target-table invariant true by construction will come in following PRs, ex. we do deduplication post-decomposition here.
Contributor
There was a problem hiding this comment. Ok. I'm a bit torn on this, because ideally it'd be best to avoid these action at a distance assumptions, but I don't see a way to achieve that without adding extra pointless (and potentially costly) structure to the query just to check for what's ultimately an edge case of an edge case. I suppose ultimately the fuzz testing would catch any violation of this; when we have more of the implementation, I would also consider adding a test for ingestion against a target table that does have overlapping records just to capture the behavior in case something crazy happens.
Contributor
Author
There was a problem hiding this comment. I actually agree, I think we should explicitly validate any required invariants at runtime, if we cannot make them true by construction at compile time. In fact in the next PR, I add validation post-decomposition to assert all the rows are well-formed as per recordStartAt/startAt/endAt expected invariants. I suspect we'll have this discussion there too on whether the validation <=> complexity tradeoff is worth it, personally I think it is. I would be happy to add similar validation when we pull in aux/target table rows to validate each row conforms to the expected shapes. There could be some middle ground too where validations are gated by some test/experimental build only flag. |
||
| // smallest incoming sequence: the consecutive left neighbor of that incoming event. This | ||
| // provides "left context" for the smallest event, analogous to the anchor row in | ||
| // [[findAffectedRowsFromAuxiliaryTable]]. It may need to be demoted from a target run | ||
| // boundary to an aux no-op continuation if the incoming event at minSeq turns out to | ||
| // extend an earlier run. | ||
| val rowEndsAfterMinimumSequence = targetEndAtCol >= perKeyMinimumSequenceInMicrobatchCol | ||
| val rowMayBeAffected = isCurrentlyActiveRow || rowEndsAfterMinimumSequence | ||
|
|
||
| val affectedRowsFromTargetTable = targetTableDf | ||
| .join(perKeyMinimumSequenceInMicrobatchDf, keysRaw) | ||
| .filter(rowMayBeAffected) | ||
| .drop(perKeyMinimumSequenceInMicrobatchCol) | ||
|
|
||
| affectedRowsFromTargetTable | ||
| } | ||
| } | ||
|
|
||
| /** | ||
|
|
@@ -204,6 +385,9 @@ case class Scd2BatchProcessor( | |
| * latest row values for that run. The target table in its entirety represents the SCD2 | ||
| * representation of the CDC flow's source table. | ||
| * | ||
| * Lifetime invariant: As per the SCD2 contract, the target table should never have overlapping | ||
| * rows by [startAt, endAt) intervals. | ||
| * | ||
| * ------------- | ||
| * Concept: aux table. | ||
| * | ||
|
|
@@ -248,11 +432,24 @@ case class Scd2BatchProcessor( | |
| */ | ||
| object Scd2BatchProcessor { | ||
| /** | ||
| * CDC metadata column field that represents the exact time (sequence) of the CDC event that | ||
| * produced this row. Null only for synthetic decomposition tails. | ||
| */ | ||
| private[autocdc] val recordStartAtFieldName: String = "__RECORD_START_AT" | ||
|
|
||
| /** | ||
| * Aux-table only column that holds the microbatch id by which a row was logically | ||
| * deleted (null if the row is still live). Future microbatches must treat any row with a | ||
| * non-null value here, other than the current batch's id, as deleted, and may safely | ||
| * physically reap them since prior microbatches commit before the next one starts. | ||
| * | ||
| * Logically-deleted rows exist as a concept on the auxiliary table to provide | ||
| * idempotency, should a microbatch fail between a MERGE executed against the auxiliary | ||
| * table and the MERGE executed against the target table. | ||
| */ | ||
| private[autocdc] val deletedByBatchIdColName: String = | ||
| s"${AutoCdcReservedNames.prefix}deleted_by_batch_id" | ||
|
|
||
| /** | ||
| * What this column represents depends on which AutoCDC artifact table it is read from. | ||
| * | ||
|
|
@@ -284,9 +481,9 @@ object Scd2BatchProcessor { | |
| private[autocdc] val endAtColName: String = "__END_AT" | ||
|
|
||
| /** | ||
| * Column names reserved by AutoCDC that will be projected onto the microbatch and | ||
| * eventually persisted in the target table. If the user's source dataframe contains any of | ||
| * these columns, SCD2 reconciliation will fail. | ||
| * | ||
| * TODO(SPARK-57251): validate at [[AutoCdcMergeFlow]] construction time that the source | ||
| * schema and column selection do not collide with these reserved names, so we fail fast | ||
|
|
@@ -299,9 +496,27 @@ object Scd2BatchProcessor { | |
| ) | ||
|
|
||
| /** | ||
| * Name of temporary column projected onto microbatch to compute the min sequencing value per | ||
| * key within the microbatch. | ||
| */ | ||
| private[autocdc] val minSequenceColName: String = s"${AutoCdcReservedNames.prefix}min_sequence" | ||
|
|
||
| /** | ||
| * Name of the temporary column used to identify the sequence associated with the anchor | ||
| * row found in the auxiliary table for the incoming microbatch. Since sequences must be unique | ||
| * amongst all rows for a key (or risk undefined behavior), this sequence value uniquely | ||
| * identifies an exact row in the aux. | ||
| */ | ||
| private val anchorSequenceColName: String = s"${AutoCdcReservedNames.prefix}anchor_sequence" | ||
|
|
||
| /** Project the [[recordStartAtFieldName]] out of an SCD2 CDC metadata column. */ | ||
| private def recordStartAtOf(cdcMetadataCol: Column): Column = | ||
| cdcMetadataCol.getField(recordStartAtFieldName) | ||
|
|
||
| /** | ||
| * Schema of the CDC metadata struct column for SCD2 rows. | ||
| */ | ||
| private[pipelines] def cdcMetadataColSchema(sequencingType: DataType): StructType = | ||
| StructType( | ||
| Seq( | ||
| // The sequence value of the originating CDC event for this row. Nullable because | ||
|
|
@@ -312,20 +527,19 @@ object Scd2BatchProcessor { | |
| ) | ||
|
|
||
| /** | ||
| * Construct the CDC metadata struct column for SCD2 rows, following the exact schema and | ||
| * field ordering defined by [[cdcMetadataColSchema]]. | ||
| */ | ||
| private def constructCdcMetadataCol( | ||
| recordStartAt: Column, | ||
| sequencingType: DataType | ||
| ): Column = { | ||
| val cdcMetadataFieldsInOrder = cdcMetadataColSchema(sequencingType).fields.map { field => | ||
| val value = field.name match { | ||
| case `recordStartAtFieldName` => recordStartAt | ||
| case other => | ||
| throw SparkException.internalError( | ||
| s"Unable to construct SCD2 CDC metadata column due to unknown `${other}` field." | ||
| ) | ||
| } | ||
| value.cast(field.dataType).as(field.name) | ||
|
|
||
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
An aside on my intentions with these scaladocs here and going forward:
I want each individual, unit-testable, function in SCD2 microbatch processor to have a well defined contracts; it should be clear exactly what dataframe each function expects and what dataframe each function returns, including the schema and logical invariants of each.
I'm a big believer in correctness by construction, I would love for there to be a way to encode this information into scala types which are both self-documenting and compile-time verifiable.
But we only actually know the user's key/data columns at runtime, so we can't leverage the typed
DatasetAPI in a meaningful way here.At best we could introduce case-class wrappers around the
DataFramethat communicate intent/invariants of the wrappedDataFramevia the name. But that can't enforce someone doesn't incorrectly construct a wrapped DataFrame (ex. one the violates the declared invariants), so it doesn't buy us much more than just documenting like this.