diff --git a/generated/provider_dependencies.json b/generated/provider_dependencies.json index 3883ff3b2df43..a4854099c473b 100644 --- a/generated/provider_dependencies.json +++ b/generated/provider_dependencies.json @@ -1096,6 +1096,7 @@ "deps": [ "apache-airflow>=2.9.0", "ipykernel>=6.29.4", + "nbconvert>=7.16.1", "pandas>=2.1.2,<2.2", "papermill[all]>=2.6.0", "scrapbook[all]>=0.5.0" diff --git a/providers/papermill/README.rst b/providers/papermill/README.rst index ac64d7af4b19d..7a5c53672b21f 100644 --- a/providers/papermill/README.rst +++ b/providers/papermill/README.rst @@ -58,6 +58,7 @@ PIP package Version required ``scrapbook[all]`` ``>=0.5.0`` ``ipykernel`` ``>=6.29.4`` ``pandas`` ``>=2.1.2,<2.2`` +``nbconvert`` ``>=7.16.1`` ================== ================== Cross provider package dependencies diff --git a/providers/papermill/pyproject.toml b/providers/papermill/pyproject.toml index 278973df9f6c0..6ce789e140f25 100644 --- a/providers/papermill/pyproject.toml +++ b/providers/papermill/pyproject.toml @@ -62,6 +62,7 @@ dependencies = [ "scrapbook[all]>=0.5.0", "ipykernel>=6.29.4", "pandas>=2.1.2,<2.2", + "nbconvert>=7.16.1", ] # The optional dependencies should be modified in place in the generated file diff --git a/providers/papermill/src/airflow/providers/papermill/get_provider_info.py b/providers/papermill/src/airflow/providers/papermill/get_provider_info.py index ce492ddafe83a..e30a7167256fd 100644 --- a/providers/papermill/src/airflow/providers/papermill/get_provider_info.py +++ b/providers/papermill/src/airflow/providers/papermill/get_provider_info.py @@ -89,6 +89,7 @@ def get_provider_info(): "scrapbook[all]>=0.5.0", "ipykernel>=6.29.4", "pandas>=2.1.2,<2.2", + "nbconvert>=7.16.1", ], "optional-dependencies": {"common.compat": ["apache-airflow-providers-common-compat"]}, "devel-dependencies": [], diff --git a/providers/papermill/src/airflow/providers/papermill/operators/papermill.py b/providers/papermill/src/airflow/providers/papermill/operators/papermill.py index ddb7a92541231..44f6db12e9c92 100644 --- a/providers/papermill/src/airflow/providers/papermill/operators/papermill.py +++ b/providers/papermill/src/airflow/providers/papermill/operators/papermill.py @@ -17,6 +17,7 @@ # under the License. from __future__ import annotations +import subprocess from collections.abc import Collection, Sequence from functools import cached_property from typing import TYPE_CHECKING, ClassVar @@ -68,6 +69,8 @@ class PapermillOperator(BaseOperator): "kernel_name", "language_name", "kernel_conn_id", + "nbconvert", + "nbconvert_args", ) def __init__( @@ -79,6 +82,8 @@ def __init__( kernel_name: str | None = None, language_name: str | None = None, kernel_conn_id: str | None = None, + nbconvert: bool = False, + nbconvert_args: list[str] | None = None, **kwargs, ) -> None: super().__init__(**kwargs) @@ -95,6 +100,8 @@ def __init__( self.kernel_name = kernel_name self.language_name = language_name self.kernel_conn_id = kernel_conn_id + self.nbconvert = nbconvert + self.nbconvert_args = nbconvert_args def execute(self, context: Context): if not isinstance(self.input_nb, NoteBook): @@ -132,6 +139,29 @@ def execute(self, context: Context): **remote_kernel_kwargs, ) + # Convert the executed notebook to HTML using nbconvert + if self.nbconvert: + nbconvert_args = self.nbconvert_args or [] + if not isinstance(nbconvert_args, list): + raise ValueError("nbconvert_args must be a list") + + # Build the nbconvert command + command = [ + "jupyter", + "nbconvert", + "--to", + "html", + "--log-level", + "WARN", + self.output_nb.url, + ] + nbconvert_args + try: + subprocess.run(command, check=True) + self.log.info("Output HTML: %s", self.output_nb.url.replace(".ipynb", ".html")) + except subprocess.CalledProcessError as e: + self.log.error("nbconvert failed with output:\n%s", e.stdout) + raise + @cached_property def hook(self) -> KernelHook | None: """Get valid hook."""