From 9728e962b7f6f5d0e41c5552e61f3722684a4eee Mon Sep 17 00:00:00 2001 From: sduvvuri26 <42085381+sduvvuri26@users.noreply.github.com> Date: Thu, 30 Jul 2026 14:40:17 +0530 Subject: [PATCH] [v3-3-test] Docs: clarify Databricks retry_delay uses exponential backoff (#68622) (cherry picked from commit ee8bb6d26499af27340ba025c8d4113559f0071a) Co-authored-by: sduvvuri26 <42085381+sduvvuri26@users.noreply.github.com> --- .../docs/operators/repos_create.rst | 2 +- .../docs/operators/repos_delete.rst | 2 +- .../docs/operators/repos_update.rst | 2 +- .../providers/databricks/hooks/databricks.py | 5 +-- .../databricks/hooks/databricks_base.py | 5 +-- .../databricks/operators/databricks.py | 33 ++++++++++++------- .../databricks/operators/databricks_repos.py | 18 ++++++---- .../databricks/triggers/databricks.py | 8 +++-- 8 files changed, 49 insertions(+), 26 deletions(-) diff --git a/providers/databricks/docs/operators/repos_create.rst b/providers/databricks/docs/operators/repos_create.rst index fbbcca940c302..adc76a4f01862 100644 --- a/providers/databricks/docs/operators/repos_create.rst +++ b/providers/databricks/docs/operators/repos_create.rst @@ -53,7 +53,7 @@ To use this operator you need to provide at least ``git_url`` parameter. * - databricks_retry_limit: integer - amount of times retry if the Databricks backend is unreachable. * - databricks_retry_delay: decimal - - number of seconds to wait between retries. + - minimum wait in seconds between retryable attempts (exponential backoff by default). Examples -------- diff --git a/providers/databricks/docs/operators/repos_delete.rst b/providers/databricks/docs/operators/repos_delete.rst index 09d8880c344d2..cd58435ac03a3 100644 --- a/providers/databricks/docs/operators/repos_delete.rst +++ b/providers/databricks/docs/operators/repos_delete.rst @@ -45,7 +45,7 @@ To use this operator you need to provide either ``repo_path`` or ``repo_id``. * - databricks_retry_limit: integer - amount of times retry if the Databricks backend is unreachable. * - databricks_retry_delay: decimal - - number of seconds to wait between retries. + - minimum wait in seconds between retryable attempts (exponential backoff by default). Examples -------- diff --git a/providers/databricks/docs/operators/repos_update.rst b/providers/databricks/docs/operators/repos_update.rst index 5af97e9751ce4..43a0402547f64 100644 --- a/providers/databricks/docs/operators/repos_update.rst +++ b/providers/databricks/docs/operators/repos_update.rst @@ -50,7 +50,7 @@ To use this operator you need to provide either ``branch`` or ``tag`` and either * - databricks_retry_limit: integer - amount of times retry if the Databricks backend is unreachable. * - databricks_retry_delay: decimal - - number of seconds to wait between retries. + - minimum wait in seconds between retryable attempts (exponential backoff by default). Examples -------- diff --git a/providers/databricks/src/airflow/providers/databricks/hooks/databricks.py b/providers/databricks/src/airflow/providers/databricks/hooks/databricks.py index 7bba763455235..fa8141c989b4d 100644 --- a/providers/databricks/src/airflow/providers/databricks/hooks/databricks.py +++ b/providers/databricks/src/airflow/providers/databricks/hooks/databricks.py @@ -274,8 +274,9 @@ class DatabricksHook(BaseDatabricksHook): will wait before timing-out. :param retry_limit: The number of times to retry the connection in case of service outages. - :param retry_delay: The number of seconds to wait between retries (it - might be a floating point number). + :param retry_delay: Minimum wait in seconds between retryable attempts when using the + default retry strategy. The wait uses exponential backoff (doubling after each + failure, capped at ``2 ** retry_limit`` seconds). May be a floating point number. :param retry_args: An optional dictionary with arguments passed to ``tenacity.Retrying`` class. """ diff --git a/providers/databricks/src/airflow/providers/databricks/hooks/databricks_base.py b/providers/databricks/src/airflow/providers/databricks/hooks/databricks_base.py index 7ab51fc7dfa12..f254bc24cf955 100644 --- a/providers/databricks/src/airflow/providers/databricks/hooks/databricks_base.py +++ b/providers/databricks/src/airflow/providers/databricks/hooks/databricks_base.py @@ -96,8 +96,9 @@ class BaseDatabricksHook(BaseHook): will wait before timing-out. :param retry_limit: The number of times to retry the connection in case of service outages. - :param retry_delay: The number of seconds to wait between retries (it - might be a floating point number). + :param retry_delay: Minimum wait in seconds between retryable attempts when using the + default retry strategy. The wait uses exponential backoff (doubling after each + failure, capped at ``2 ** retry_limit`` seconds). May be a floating point number. :param retry_args: An optional dictionary with arguments passed to ``tenacity.Retrying`` class. :param caller: The name of the operator that is calling the hook. """ diff --git a/providers/databricks/src/airflow/providers/databricks/operators/databricks.py b/providers/databricks/src/airflow/providers/databricks/operators/databricks.py index 4bae6b28f0aa1..25cbae3321c6d 100644 --- a/providers/databricks/src/airflow/providers/databricks/operators/databricks.py +++ b/providers/databricks/src/airflow/providers/databricks/operators/databricks.py @@ -406,8 +406,10 @@ class DatabricksCreateJobsOperator(BaseOperator): this run. By default the operator will poll every 30 seconds. :param databricks_retry_limit: Amount of times retry if the Databricks backend is unreachable. Its value must be greater than or equal to 1. - :param databricks_retry_delay: Number of seconds to wait between retries (it - might be a floating point number). + :param databricks_retry_delay: Minimum wait in seconds between retryable attempts when + using the default retry strategy. The wait uses exponential backoff (doubling after + each failure, capped at ``2 ** databricks_retry_limit`` seconds). May be a floating + point number. :param databricks_retry_args: An optional dictionary with arguments passed to ``tenacity.Retrying`` class. .. note:: @@ -642,8 +644,10 @@ class DatabricksSubmitRunOperator(BaseOperator): this run. By default the operator will poll every 30 seconds. :param databricks_retry_limit: Amount of times retry if the Databricks backend is unreachable. Its value must be greater than or equal to 1. - :param databricks_retry_delay: Number of seconds to wait between retries (it - might be a floating point number). + :param databricks_retry_delay: Minimum wait in seconds between retryable attempts when + using the default retry strategy. The wait uses exponential backoff (doubling after + each failure, capped at ``2 ** databricks_retry_limit`` seconds). May be a floating + point number. :param databricks_retry_args: An optional dictionary with arguments passed to ``tenacity.Retrying`` class. :param do_xcom_push: Whether we should push run_id and run_page_url to xcom. :param git_source: Optional specification of a remote git repository from which @@ -1058,8 +1062,10 @@ class DatabricksRunNowOperator(BaseOperator): this run. By default, the operator will poll every 30 seconds. :param databricks_retry_limit: Amount of times retry if the Databricks backend is unreachable. Its value must be greater than or equal to 1. - :param databricks_retry_delay: Number of seconds to wait between retries (it - might be a floating point number). + :param databricks_retry_delay: Minimum wait in seconds between retryable attempts when + using the default retry strategy. The wait uses exponential backoff (doubling after + each failure, capped at ``2 ** databricks_retry_limit`` seconds). May be a floating + point number. :param databricks_retry_args: An optional dictionary with arguments passed to ``tenacity.Retrying`` class. :param do_xcom_push: Whether we should push run_id and run_page_url to xcom. :param wait_for_termination: if we should wait for termination of the job run. ``True`` by default. @@ -1291,8 +1297,10 @@ class DatabricksSQLStatementsOperator(DatabricksSQLStatementsMixin, BaseOperator this statement. By default the operator will poll every 30 seconds. :param databricks_retry_limit: Amount of times retry if the Databricks backend is unreachable. Its value must be greater than or equal to 1. - :param databricks_retry_delay: Number of seconds to wait between retries (it - might be a floating point number). + :param databricks_retry_delay: Minimum wait in seconds between retryable attempts when + using the default retry strategy. The wait uses exponential backoff (doubling after + each failure, capped at ``2 ** databricks_retry_limit`` seconds). May be a floating + point number. :param databricks_retry_args: An optional dictionary with arguments passed to ``tenacity.Retrying`` class. :param do_xcom_push: Whether we should push statement_id to xcom.: :param timeout: The timeout for the Airflow task executing the SQL statement. By default a value of 3600 seconds is used. @@ -1449,7 +1457,8 @@ class DatabricksTaskBaseOperator(BaseOperator, ABC): :param databricks_task_key: An optional task_key used to refer to the task by Databricks API. By default this will be set to the hash of ``dag_id + task_id``. :param databricks_retry_args: An optional dictionary with arguments passed to ``tenacity.Retrying`` class. - :param databricks_retry_delay: Number of seconds to wait between retries. + :param databricks_retry_delay: Minimum wait in seconds between retryable attempts when + using the default retry strategy (exponential backoff). May be a floating point number. :param databricks_retry_limit: Amount of times to retry if the Databricks backend is unreachable. :param deferrable: Whether to run the operator in the deferrable mode. :param existing_cluster_id: ID for existing cluster on which to run this task. @@ -1783,7 +1792,8 @@ class DatabricksNotebookOperator(DatabricksTaskBaseOperator): https://docs.databricks.com/dev-tools/api/latest/jobs.html#operation/JobsCreate :param databricks_conn_id: The name of the Airflow connection to use. :param databricks_retry_args: An optional dictionary with arguments passed to ``tenacity.Retrying`` class. - :param databricks_retry_delay: Number of seconds to wait between retries. + :param databricks_retry_delay: Minimum wait in seconds between retryable attempts when + using the default retry strategy (exponential backoff). May be a floating point number. :param databricks_retry_limit: Amount of times to retry if the Databricks backend is unreachable. :param deferrable: Whether to run the operator in the deferrable mode. :param existing_cluster_id: ID for existing cluster on which to run this task. @@ -1930,7 +1940,8 @@ class DatabricksTaskOperator(DatabricksTaskBaseOperator): :param task_config: The configuration of the task to be run on Databricks. :param databricks_conn_id: The name of the Airflow connection to use. :param databricks_retry_args: An optional dictionary with arguments passed to ``tenacity.Retrying`` class. - :param databricks_retry_delay: Number of seconds to wait between retries. + :param databricks_retry_delay: Minimum wait in seconds between retryable attempts when + using the default retry strategy (exponential backoff). May be a floating point number. :param databricks_retry_limit: Amount of times to retry if the Databricks backend is unreachable. :param deferrable: Whether to run the operator in the deferrable mode. :param existing_cluster_id: ID for existing cluster on which to run this task. diff --git a/providers/databricks/src/airflow/providers/databricks/operators/databricks_repos.py b/providers/databricks/src/airflow/providers/databricks/operators/databricks_repos.py index cb513ea8873ba..1abb742efbc47 100644 --- a/providers/databricks/src/airflow/providers/databricks/operators/databricks_repos.py +++ b/providers/databricks/src/airflow/providers/databricks/operators/databricks_repos.py @@ -51,8 +51,10 @@ class DatabricksReposCreateOperator(BaseOperator): connection and create the key ``host`` and leave the ``host`` field empty. (templated) :param databricks_retry_limit: Amount of times retry if the Databricks backend is unreachable. Its value must be greater than or equal to 1. - :param databricks_retry_delay: Number of seconds to wait between retries (it - might be a floating point number). + :param databricks_retry_delay: Minimum wait in seconds between retryable attempts when + using the default retry strategy. The wait uses exponential backoff (doubling after + each failure, capped at ``2 ** databricks_retry_limit`` seconds). May be a floating + point number. """ # Used in airflow.models.BaseOperator @@ -201,8 +203,10 @@ class DatabricksReposUpdateOperator(BaseOperator): connection and create the key ``host`` and leave the ``host`` field empty. (templated) :param databricks_retry_limit: Amount of times retry if the Databricks backend is unreachable. Its value must be greater than or equal to 1. - :param databricks_retry_delay: Number of seconds to wait between retries (it - might be a floating point number). + :param databricks_retry_delay: Minimum wait in seconds between retryable attempts when + using the default retry strategy. The wait uses exponential backoff (doubling after + each failure, capped at ``2 ** databricks_retry_limit`` seconds). May be a floating + point number. """ # Used in airflow.models.BaseOperator @@ -275,8 +279,10 @@ class DatabricksReposDeleteOperator(BaseOperator): connection and create the key ``host`` and leave the ``host`` field empty. (templated) :param databricks_retry_limit: Amount of times retry if the Databricks backend is unreachable. Its value must be greater than or equal to 1. - :param databricks_retry_delay: Number of seconds to wait between retries (it - might be a floating point number). + :param databricks_retry_delay: Minimum wait in seconds between retryable attempts when + using the default retry strategy. The wait uses exponential backoff (doubling after + each failure, capped at ``2 ** databricks_retry_limit`` seconds). May be a floating + point number. """ # Used in airflow.models.BaseOperator diff --git a/providers/databricks/src/airflow/providers/databricks/triggers/databricks.py b/providers/databricks/src/airflow/providers/databricks/triggers/databricks.py index 2bb626b911418..57d489550f200 100644 --- a/providers/databricks/src/airflow/providers/databricks/triggers/databricks.py +++ b/providers/databricks/src/airflow/providers/databricks/triggers/databricks.py @@ -36,7 +36,9 @@ class DatabricksExecutionTrigger(BaseTrigger): :param polling_period_seconds: Controls the rate of the poll for the result of this run. By default, the trigger will poll every 30 seconds. :param retry_limit: The number of times to retry the connection in case of service outages. - :param retry_delay: The number of seconds to wait between retries. + :param retry_delay: Minimum wait in seconds between retryable attempts when using the + default retry strategy. The wait uses exponential backoff (doubling after each + failure, capped at ``2 ** retry_limit`` seconds). May be a floating point number. :param retry_args: An optional dictionary with arguments passed to ``tenacity.Retrying`` class. :param run_page_url: The run page url. :param repair_run: Repair the databricks run in case of failure. @@ -138,7 +140,9 @@ class DatabricksSQLStatementExecutionTrigger(BaseTrigger): :param polling_period_seconds: Controls the rate of the poll for the result of this run. By default, the trigger will poll every 30 seconds. :param retry_limit: The number of times to retry the connection in case of service outages. - :param retry_delay: The number of seconds to wait between retries. + :param retry_delay: Minimum wait in seconds between retryable attempts when using the + default retry strategy. The wait uses exponential backoff (doubling after each + failure, capped at ``2 ** retry_limit`` seconds). May be a floating point number. :param retry_args: An optional dictionary with arguments passed to ``tenacity.Retrying`` class. :param caller: The name of the operator that is calling the hook. """