diff --git a/providers/databricks/docs/operators/repos_create.rst b/providers/databricks/docs/operators/repos_create.rst index fbbcca940c302..adc76a4f01862 100644 --- a/providers/databricks/docs/operators/repos_create.rst +++ b/providers/databricks/docs/operators/repos_create.rst @@ -53,7 +53,7 @@ To use this operator you need to provide at least ``git_url`` parameter. * - databricks_retry_limit: integer - amount of times retry if the Databricks backend is unreachable. * - databricks_retry_delay: decimal - - number of seconds to wait between retries. + - minimum wait in seconds between retryable attempts (exponential backoff by default). Examples -------- diff --git a/providers/databricks/docs/operators/repos_delete.rst b/providers/databricks/docs/operators/repos_delete.rst index 09d8880c344d2..cd58435ac03a3 100644 --- a/providers/databricks/docs/operators/repos_delete.rst +++ b/providers/databricks/docs/operators/repos_delete.rst @@ -45,7 +45,7 @@ To use this operator you need to provide either ``repo_path`` or ``repo_id``. * - databricks_retry_limit: integer - amount of times retry if the Databricks backend is unreachable. * - databricks_retry_delay: decimal - - number of seconds to wait between retries. + - minimum wait in seconds between retryable attempts (exponential backoff by default). Examples -------- diff --git a/providers/databricks/docs/operators/repos_update.rst b/providers/databricks/docs/operators/repos_update.rst index 5af97e9751ce4..43a0402547f64 100644 --- a/providers/databricks/docs/operators/repos_update.rst +++ b/providers/databricks/docs/operators/repos_update.rst @@ -50,7 +50,7 @@ To use this operator you need to provide either ``branch`` or ``tag`` and either * - databricks_retry_limit: integer - amount of times retry if the Databricks backend is unreachable. * - databricks_retry_delay: decimal - - number of seconds to wait between retries. + - minimum wait in seconds between retryable attempts (exponential backoff by default). Examples -------- diff --git a/providers/databricks/src/airflow/providers/databricks/hooks/databricks.py b/providers/databricks/src/airflow/providers/databricks/hooks/databricks.py index 7bba763455235..fa8141c989b4d 100644 --- a/providers/databricks/src/airflow/providers/databricks/hooks/databricks.py +++ b/providers/databricks/src/airflow/providers/databricks/hooks/databricks.py @@ -274,8 +274,9 @@ class DatabricksHook(BaseDatabricksHook): will wait before timing-out. :param retry_limit: The number of times to retry the connection in case of service outages. - :param retry_delay: The number of seconds to wait between retries (it - might be a floating point number). + :param retry_delay: Minimum wait in seconds between retryable attempts when using the + default retry strategy. The wait uses exponential backoff (doubling after each + failure, capped at ``2 ** retry_limit`` seconds). May be a floating point number. :param retry_args: An optional dictionary with arguments passed to ``tenacity.Retrying`` class. """ diff --git a/providers/databricks/src/airflow/providers/databricks/hooks/databricks_base.py b/providers/databricks/src/airflow/providers/databricks/hooks/databricks_base.py index 7ab51fc7dfa12..f254bc24cf955 100644 --- a/providers/databricks/src/airflow/providers/databricks/hooks/databricks_base.py +++ b/providers/databricks/src/airflow/providers/databricks/hooks/databricks_base.py @@ -96,8 +96,9 @@ class BaseDatabricksHook(BaseHook): will wait before timing-out. :param retry_limit: The number of times to retry the connection in case of service outages. - :param retry_delay: The number of seconds to wait between retries (it - might be a floating point number). + :param retry_delay: Minimum wait in seconds between retryable attempts when using the + default retry strategy. The wait uses exponential backoff (doubling after each + failure, capped at ``2 ** retry_limit`` seconds). May be a floating point number. :param retry_args: An optional dictionary with arguments passed to ``tenacity.Retrying`` class. :param caller: The name of the operator that is calling the hook. """ diff --git a/providers/databricks/src/airflow/providers/databricks/operators/databricks.py b/providers/databricks/src/airflow/providers/databricks/operators/databricks.py index 9898993d4147e..1f143f9141044 100644 --- a/providers/databricks/src/airflow/providers/databricks/operators/databricks.py +++ b/providers/databricks/src/airflow/providers/databricks/operators/databricks.py @@ -340,8 +340,10 @@ class DatabricksCreateJobsOperator(BaseOperator): this run. By default the operator will poll every 30 seconds. :param databricks_retry_limit: Amount of times retry if the Databricks backend is unreachable. Its value must be greater than or equal to 1. - :param databricks_retry_delay: Number of seconds to wait between retries (it - might be a floating point number). + :param databricks_retry_delay: Minimum wait in seconds between retryable attempts when + using the default retry strategy. The wait uses exponential backoff (doubling after + each failure, capped at ``2 ** databricks_retry_limit`` seconds). May be a floating + point number. :param databricks_retry_args: An optional dictionary with arguments passed to ``tenacity.Retrying`` class. .. note:: @@ -550,8 +552,10 @@ class DatabricksSubmitRunOperator(BaseOperator): this run. By default the operator will poll every 30 seconds. :param databricks_retry_limit: Amount of times retry if the Databricks backend is unreachable. Its value must be greater than or equal to 1. - :param databricks_retry_delay: Number of seconds to wait between retries (it - might be a floating point number). + :param databricks_retry_delay: Minimum wait in seconds between retryable attempts when + using the default retry strategy. The wait uses exponential backoff (doubling after + each failure, capped at ``2 ** databricks_retry_limit`` seconds). May be a floating + point number. :param databricks_retry_args: An optional dictionary with arguments passed to ``tenacity.Retrying`` class. :param do_xcom_push: Whether we should push run_id and run_page_url to xcom. :param git_source: Optional specification of a remote git repository from which @@ -879,8 +883,10 @@ class DatabricksRunNowOperator(BaseOperator): this run. By default, the operator will poll every 30 seconds. :param databricks_retry_limit: Amount of times retry if the Databricks backend is unreachable. Its value must be greater than or equal to 1. - :param databricks_retry_delay: Number of seconds to wait between retries (it - might be a floating point number). + :param databricks_retry_delay: Minimum wait in seconds between retryable attempts when + using the default retry strategy. The wait uses exponential backoff (doubling after + each failure, capped at ``2 ** databricks_retry_limit`` seconds). May be a floating + point number. :param databricks_retry_args: An optional dictionary with arguments passed to ``tenacity.Retrying`` class. :param do_xcom_push: Whether we should push run_id and run_page_url to xcom. :param wait_for_termination: if we should wait for termination of the job run. ``True`` by default. @@ -1078,8 +1084,10 @@ class DatabricksSQLStatementsOperator(DatabricksSQLStatementsMixin, BaseOperator this statement. By default the operator will poll every 30 seconds. :param databricks_retry_limit: Amount of times retry if the Databricks backend is unreachable. Its value must be greater than or equal to 1. - :param databricks_retry_delay: Number of seconds to wait between retries (it - might be a floating point number). + :param databricks_retry_delay: Minimum wait in seconds between retryable attempts when + using the default retry strategy. The wait uses exponential backoff (doubling after + each failure, capped at ``2 ** databricks_retry_limit`` seconds). May be a floating + point number. :param databricks_retry_args: An optional dictionary with arguments passed to ``tenacity.Retrying`` class. :param do_xcom_push: Whether we should push statement_id to xcom.: :param timeout: The timeout for the Airflow task executing the SQL statement. By default a value of 3600 seconds is used. @@ -1236,7 +1244,8 @@ class DatabricksTaskBaseOperator(BaseOperator, ABC): :param databricks_task_key: An optional task_key used to refer to the task by Databricks API. By default this will be set to the hash of ``dag_id + task_id``. :param databricks_retry_args: An optional dictionary with arguments passed to ``tenacity.Retrying`` class. - :param databricks_retry_delay: Number of seconds to wait between retries. + :param databricks_retry_delay: Minimum wait in seconds between retryable attempts when + using the default retry strategy (exponential backoff). May be a floating point number. :param databricks_retry_limit: Amount of times to retry if the Databricks backend is unreachable. :param deferrable: Whether to run the operator in the deferrable mode. :param existing_cluster_id: ID for existing cluster on which to run this task. @@ -1570,7 +1579,8 @@ class DatabricksNotebookOperator(DatabricksTaskBaseOperator): https://docs.databricks.com/dev-tools/api/latest/jobs.html#operation/JobsCreate :param databricks_conn_id: The name of the Airflow connection to use. :param databricks_retry_args: An optional dictionary with arguments passed to ``tenacity.Retrying`` class. - :param databricks_retry_delay: Number of seconds to wait between retries. + :param databricks_retry_delay: Minimum wait in seconds between retryable attempts when + using the default retry strategy (exponential backoff). May be a floating point number. :param databricks_retry_limit: Amount of times to retry if the Databricks backend is unreachable. :param deferrable: Whether to run the operator in the deferrable mode. :param existing_cluster_id: ID for existing cluster on which to run this task. @@ -1717,7 +1727,8 @@ class DatabricksTaskOperator(DatabricksTaskBaseOperator): :param task_config: The configuration of the task to be run on Databricks. :param databricks_conn_id: The name of the Airflow connection to use. :param databricks_retry_args: An optional dictionary with arguments passed to ``tenacity.Retrying`` class. - :param databricks_retry_delay: Number of seconds to wait between retries. + :param databricks_retry_delay: Minimum wait in seconds between retryable attempts when + using the default retry strategy (exponential backoff). May be a floating point number. :param databricks_retry_limit: Amount of times to retry if the Databricks backend is unreachable. :param deferrable: Whether to run the operator in the deferrable mode. :param existing_cluster_id: ID for existing cluster on which to run this task. diff --git a/providers/databricks/src/airflow/providers/databricks/operators/databricks_repos.py b/providers/databricks/src/airflow/providers/databricks/operators/databricks_repos.py index cb513ea8873ba..1abb742efbc47 100644 --- a/providers/databricks/src/airflow/providers/databricks/operators/databricks_repos.py +++ b/providers/databricks/src/airflow/providers/databricks/operators/databricks_repos.py @@ -51,8 +51,10 @@ class DatabricksReposCreateOperator(BaseOperator): connection and create the key ``host`` and leave the ``host`` field empty. (templated) :param databricks_retry_limit: Amount of times retry if the Databricks backend is unreachable. Its value must be greater than or equal to 1. - :param databricks_retry_delay: Number of seconds to wait between retries (it - might be a floating point number). + :param databricks_retry_delay: Minimum wait in seconds between retryable attempts when + using the default retry strategy. The wait uses exponential backoff (doubling after + each failure, capped at ``2 ** databricks_retry_limit`` seconds). May be a floating + point number. """ # Used in airflow.models.BaseOperator @@ -201,8 +203,10 @@ class DatabricksReposUpdateOperator(BaseOperator): connection and create the key ``host`` and leave the ``host`` field empty. (templated) :param databricks_retry_limit: Amount of times retry if the Databricks backend is unreachable. Its value must be greater than or equal to 1. - :param databricks_retry_delay: Number of seconds to wait between retries (it - might be a floating point number). + :param databricks_retry_delay: Minimum wait in seconds between retryable attempts when + using the default retry strategy. The wait uses exponential backoff (doubling after + each failure, capped at ``2 ** databricks_retry_limit`` seconds). May be a floating + point number. """ # Used in airflow.models.BaseOperator @@ -275,8 +279,10 @@ class DatabricksReposDeleteOperator(BaseOperator): connection and create the key ``host`` and leave the ``host`` field empty. (templated) :param databricks_retry_limit: Amount of times retry if the Databricks backend is unreachable. Its value must be greater than or equal to 1. - :param databricks_retry_delay: Number of seconds to wait between retries (it - might be a floating point number). + :param databricks_retry_delay: Minimum wait in seconds between retryable attempts when + using the default retry strategy. The wait uses exponential backoff (doubling after + each failure, capped at ``2 ** databricks_retry_limit`` seconds). May be a floating + point number. """ # Used in airflow.models.BaseOperator diff --git a/providers/databricks/src/airflow/providers/databricks/triggers/databricks.py b/providers/databricks/src/airflow/providers/databricks/triggers/databricks.py index 2bb626b911418..57d489550f200 100644 --- a/providers/databricks/src/airflow/providers/databricks/triggers/databricks.py +++ b/providers/databricks/src/airflow/providers/databricks/triggers/databricks.py @@ -36,7 +36,9 @@ class DatabricksExecutionTrigger(BaseTrigger): :param polling_period_seconds: Controls the rate of the poll for the result of this run. By default, the trigger will poll every 30 seconds. :param retry_limit: The number of times to retry the connection in case of service outages. - :param retry_delay: The number of seconds to wait between retries. + :param retry_delay: Minimum wait in seconds between retryable attempts when using the + default retry strategy. The wait uses exponential backoff (doubling after each + failure, capped at ``2 ** retry_limit`` seconds). May be a floating point number. :param retry_args: An optional dictionary with arguments passed to ``tenacity.Retrying`` class. :param run_page_url: The run page url. :param repair_run: Repair the databricks run in case of failure. @@ -138,7 +140,9 @@ class DatabricksSQLStatementExecutionTrigger(BaseTrigger): :param polling_period_seconds: Controls the rate of the poll for the result of this run. By default, the trigger will poll every 30 seconds. :param retry_limit: The number of times to retry the connection in case of service outages. - :param retry_delay: The number of seconds to wait between retries. + :param retry_delay: Minimum wait in seconds between retryable attempts when using the + default retry strategy. The wait uses exponential backoff (doubling after each + failure, capped at ``2 ** retry_limit`` seconds). May be a floating point number. :param retry_args: An optional dictionary with arguments passed to ``tenacity.Retrying`` class. :param caller: The name of the operator that is calling the hook. """