diff --git a/docs/build b/docs/build index f69786b61b483..1756b4634fe4d 100755 --- a/docs/build +++ b/docs/build @@ -110,7 +110,7 @@ def display_errors_summary() -> None: print("=" * 50) -def check_file_not_contains(file_path: str, pattern: str, message: str) -> None: +def assert_file_not_contains(file_path: str, pattern: str, message: str) -> None: with open(file_path, "rb", 0) as doc_file: pattern_compiled = re.compile(pattern) @@ -144,7 +144,7 @@ def find_modules(deprecated_only: bool = False) -> Set[str]: return modules_names -def check_operators_and_hooks_ref() -> None: +def check_class_links_in_operators_and_hooks_ref() -> None: with open("operators-and-hooks-ref.rst") as ref_file: content = ref_file.read() current_modules_in_file = set(re.findall(r":mod:`(.+?)`", content)) @@ -163,20 +163,64 @@ def check_operators_and_hooks_ref() -> None: file_path="operators-and-hooks-ref.rst", line_no=0, message=( - f"You should try to keep the list of operators and hooks up to date.\n" + f"New module detected." + f"Please add them to the list of operators and hooks - `operators-and-hooks-ref.rst` " + f"file.\n" f"\n" - f"Missing modules:\n " + f"New modules:\n" f"{module_text_list}" ), ) ) +def check_guide_links_in_operators_and_hooks_ref() -> None: + all_guides = glob("howto/operator/**/*.rst", recursive=True) + # Remove extension + all_guides = ( + guide.rpartition(".")[0] + for guide in all_guides + if "_partials" not in guide + ) + # Remove partials and index + all_guides = ( + guide + for guide in all_guides + if "/_partials/" not in guide and not guide.endswith("index") + ) + + with open("operators-and-hooks-ref.rst") as ref_file: + content = ref_file.read() + + missing_guides = [ + guide + for guide in all_guides + if guide not in content + ] + if missing_guides: + guide_text_list = "\n".join(f":doc:`How to use <{guide}>`" for guide in missing_guides) + + build_errors.append( + DocBuildError( + file_path="operators-and-hooks-ref.rst", + line_no=0, + message=( + f"New guide detected. " + f"Please add them to the list of operators and hooks - `operators-and-hooks-ref.rst` " + f"file.\n" + f"You can copy the relevant parts of the link from the section below:\n" + f"\n" + f"{guide_text_list}" + ), + ) + ) + + def check_exampleinclude_for_example_dags(): all_docs_files = glob("**/*rst", recursive=True) for doc_file in all_docs_files: - check_file_not_contains( + assert_file_not_contains( file_path=doc_file, pattern=r"literalinclude::.+example_dags", message=( @@ -276,7 +320,8 @@ print("Current working directory: ", os.getcwd()) prepare_directories() clean_files() -check_operators_and_hooks_ref() +check_class_links_in_operators_and_hooks_ref() +check_guide_links_in_operators_and_hooks_ref() check_exampleinclude_for_example_dags() if build_errors: diff --git a/docs/howto/operator/external.rst b/docs/howto/operator/external_task_sensor.rst similarity index 100% rename from docs/howto/operator/external.rst rename to docs/howto/operator/external_task_sensor.rst diff --git a/docs/howto/operator/index.rst b/docs/howto/operator/index.rst index 9343f28ca6d08..e8c4bf57e284a 100644 --- a/docs/howto/operator/index.rst +++ b/docs/howto/operator/index.rst @@ -39,5 +39,5 @@ determine what actually executes when your DAG runs. kubernetes papermill python - external + external_task_sensor yandexcloud diff --git a/docs/operators-and-hooks-ref.rst b/docs/operators-and-hooks-ref.rst index c6de4160bf283..2d0fe39fa0452 100644 --- a/docs/operators-and-hooks-ref.rst +++ b/docs/operators-and-hooks-ref.rst @@ -29,28 +29,75 @@ Fundamentals **Base:** -* :mod:`airflow.hooks.base_hook` -* :mod:`airflow.hooks.dbapi_hook` -* :mod:`airflow.models.baseoperator` -* :mod:`airflow.sensors.base_sensor_operator` +.. list-table:: + :header-rows: 1 + + * - Module + - Guides + + * - :mod:`airflow.hooks.base_hook` + - + + * - :mod:`airflow.hooks.dbapi_hook` + - + + * - :mod:`airflow.models.baseoperator` + - + + * - :mod:`airflow.sensors.base_sensor_operator` + - **Operators:** -* :mod:`airflow.operators.branch_operator` -* :mod:`airflow.operators.check_operator` -* :mod:`airflow.operators.dagrun_operator` -* :mod:`airflow.operators.dummy_operator` -* :mod:`airflow.operators.generic_transfer` -* :mod:`airflow.operators.latest_only_operator` -* :mod:`airflow.operators.subdag_operator` +.. list-table:: + :header-rows: 1 + + * - Operators + - Guides + + * - :mod:`airflow.operators.branch_operator` + - + + * - :mod:`airflow.operators.check_operator` + - + + * - :mod:`airflow.operators.dagrun_operator` + - + + * - :mod:`airflow.operators.dummy_operator` + - + + * - :mod:`airflow.operators.generic_transfer` + - + + * - :mod:`airflow.operators.latest_only_operator` + - + + * - :mod:`airflow.operators.subdag_operator` + - **Sensors:** -* :mod:`airflow.sensors.weekday_sensor` -* :mod:`airflow.sensors.external_task_sensor` -* :mod:`airflow.sensors.sql_sensor` -* :mod:`airflow.sensors.time_delta_sensor` -* :mod:`airflow.sensors.time_sensor` +.. list-table:: + :header-rows: 1 + + * - Sensors + - Guides + + * - :mod:`airflow.sensors.weekday_sensor` + - + + * - :mod:`airflow.sensors.external_task_sensor` + - :doc:`How to use ` + + * - :mod:`airflow.sensors.sql_sensor` + - + + * - :mod:`airflow.sensors.time_delta_sensor` + - + + * - :mod:`airflow.sensors.time_sensor` + - .. _Apache: @@ -72,8 +119,8 @@ Foundation. * - Service name - Guides - Hook - - Operators - - Sensors + - Operator + - Sensor * - `Apache Cassandra `__ - @@ -158,7 +205,7 @@ Foundation. * - Source - Destination - Guide - - Operators + - Operator * - `Amazon Simple Storage Service (S3) `_ - `Apache Hive `__ @@ -228,8 +275,8 @@ These integrations allow you to perform various operations within the Microsoft * - Service name - Hook - - Operators - - Sensors + - Operator + - Sensor * - `Azure Batch `__ - :mod:`airflow.providers.microsoft.azure.hooks.azure_batch` @@ -280,7 +327,7 @@ These integrations allow you to copy data from/to Microsoft Azure. * - Source - Destination - Guide - - Operators + - Operator * - `Azure Data Lake Storage `__ - `Google Cloud Storage (GCS) `__ @@ -318,8 +365,8 @@ These integrations allow you to perform various operations within the Amazon Web * - Service name - Guide - Hook - - Operators - - Sensors + - Operator + - Sensor * - `AWS Batch `__ - @@ -371,7 +418,7 @@ These integrations allow you to perform various operations within the Amazon Web - * - `Amazon EC2 `__ - - + - :doc:`How to use ` - - :mod:`airflow.providers.amazon.aws.operators.ecs` - @@ -448,7 +495,7 @@ These integrations allow you to copy data from/to Amazon Web Services. * - Source - Destination - Guide - - Operators + - Operator * - .. _integration:AWS-Discovery-ref: @@ -571,8 +618,8 @@ These integrations allow you to perform various operations within the Google Clo * - Service name - Guide - Hook - - Operators - - Sensors + - Operator + - Sensor * - `AutoML `__ @@ -624,7 +671,7 @@ These integrations allow you to perform various operations within the Google Clo - * - `Datacatalog `__ - - + - :doc:`How to use ` - :mod:`airflow.providers.google.cloud.hooks.datacatalog` - :mod:`airflow.providers.google.cloud.operators.datacatalog` - @@ -778,7 +825,7 @@ These integrations allow you to copy data from/to Google Cloud Platform. * - Source - Destination - Guide - - Operators + - Operator * - .. _integration:GCP-Discovery-ref: @@ -893,12 +940,12 @@ These integrations allow you to copy data from/to Google Cloud Platform. * - `Google Spreadsheet `__ - `Google Cloud Storage (GCS) `__ - - + - :doc:`How to use ` - :mod:`airflow.providers.google.cloud.operators.sheets_to_gcs` * - `Google Cloud Storage (GCS) `__ - `Google Spreadsheet `__ - - + - :doc:`How to use ` - :mod:`airflow.providers.google.suite.operators.gcs_to_sheets` .. _integration:GCP-Discovery: @@ -917,8 +964,8 @@ Other operators and hooks :header-rows: 1 * - Guide - - Operators - - Hooks + - Operator + - Hook * - :doc:`How to use ` - :mod:`airflow.providers.google.cloud.operators.translate_speech` @@ -939,8 +986,8 @@ Google Marketing Platform * - Source - Destination - Guide - - Operators - - Sensors + - Operator + - Sensor * - `Analytics360 `__ - :doc:`How to use ` @@ -975,7 +1022,7 @@ Other Google operators and hooks * - Service name - Guide - Hook - - Operators + - Operator * - `Google Ads `__ - :doc:`How to use ` @@ -1027,8 +1074,8 @@ These integrations allow you to perform various operations within the Yandex.Clo * - Service name - Guide - Hook - - Operators - - Sensors + - Operator + - Sensor * - `Base Classes `__ - :doc:`How to use ` @@ -1059,8 +1106,8 @@ These integrations allow you to perform various operations within various servic * - Service name - Guide - Hook - - Operators - - Sensors + - Operator + - Sensor * - `Atlassian Jira `__ - @@ -1180,11 +1227,11 @@ These integrations allow you to perform various operations within various servic * - Source - Destination - Guide - - Operators + - Operator * - `Google Cloud Storage (GCS) `__ - `Google Drive `__ - - + - :doc:`How to use ` - :mod:`airflow.providers.google.suite.operators.gcs_to_gdrive` * - `Vertica `__ @@ -1213,8 +1260,8 @@ These integrations allow you to perform various operations using various softwar * - Service name - Guide - Hook - - Operators - - Sensors + - Operator + - Sensor * - `Celery `__ - @@ -1277,9 +1324,9 @@ These integrations allow you to perform various operations using various softwar * - `MySQL `__ - - :mod:`airflow.providers.mysql.operators.mysql` + - - :mod:`airflow.providers.mysql.hooks.mysql` - - :mod:`airflow.providers.mssql.operators.mysql` + - :mod:`airflow.providers.mysql.operators.mysql` - * - `OpenFaaS `__ @@ -1314,7 +1361,7 @@ These integrations allow you to perform various operations using various softwar * - `Python `__ - - - + - :doc:`How to use ` - :mod:`airflow.operators.python` - :mod:`airflow.sensors.python` @@ -1355,7 +1402,7 @@ These integrations allow you to copy data. * - Source - Destination - Guide - - Operators + - Operator * - `Apache Hive `__ - `Samba `__ @@ -1443,9 +1490,9 @@ communication protocols or interface. * - Service name - Guide - - Hook - - Operators - - Sensors + - Hooks + - Operator + - Sensor * - `File Transfer Protocol (FTP) `__ - @@ -1460,7 +1507,7 @@ communication protocols or interface. - :mod:`airflow.sensors.filesystem` * - `Hypertext Transfer Protocol (HTTP) `__ - - + - :doc:`How to use ` - :mod:`airflow.providers.http.hooks.http` - :mod:`airflow.providers.http.operators.http` - :mod:`airflow.providers.http.sensors.http` @@ -1518,7 +1565,7 @@ These integrations allow you to copy data. * - Source - Destination - Guide - - Operators + - Operator * - `Amazon Simple Storage Service (S3) `_ - `SSH File Transfer Protocol (SFTP) `__