add nbconvert to papermill operator

generated/provider_dependencies.json

@@ -1096,6 +1096,7 @@
     "deps": [
       "apache-airflow>=2.9.0",
       "ipykernel>=6.29.4",
+      "nbconvert>=7.16.1",
       "pandas>=2.1.2,<2.2",
       "papermill[all]>=2.6.0",
       "scrapbook[all]>=0.5.0"

providers/papermill/README.rst

@@ -58,6 +58,7 @@ PIP package         Version required
 ``scrapbook[all]``  ``>=0.5.0``
 ``ipykernel``       ``>=6.29.4``
 ``pandas``          ``>=2.1.2,<2.2``
+``nbconvert``       ``>=7.16.1``
 ==================  ==================
 
 Cross provider package dependencies

providers/papermill/pyproject.toml

@@ -62,6 +62,7 @@ dependencies = [
     "scrapbook[all]>=0.5.0",
     "ipykernel>=6.29.4",
     "pandas>=2.1.2,<2.2",
+    "nbconvert>=7.16.1",
 ]
 
 # The optional dependencies should be modified in place in the generated file

providers/papermill/src/airflow/providers/papermill/get_provider_info.py

@@ -89,6 +89,7 @@ def get_provider_info():
             "scrapbook[all]>=0.5.0",
             "ipykernel>=6.29.4",
             "pandas>=2.1.2,<2.2",
+            "nbconvert>=7.16.1",
         ],
         "optional-dependencies": {"common.compat": ["apache-airflow-providers-common-compat"]},
         "devel-dependencies": [],

providers/papermill/src/airflow/providers/papermill/operators/papermill.py

@@ -17,6 +17,7 @@
 # under the License.
 from __future__ import annotations
 
+import subprocess
 from collections.abc import Collection, Sequence
 from functools import cached_property
 from typing import TYPE_CHECKING, ClassVar
@@ -68,6 +69,8 @@ class PapermillOperator(BaseOperator):
         "kernel_name",
         "language_name",
         "kernel_conn_id",
+        "nbconvert",
+        "nbconvert_args",
     )
 
     def __init__(
@@ -79,6 +82,8 @@ def __init__(
         kernel_name: str | None = None,
         language_name: str | None = None,
         kernel_conn_id: str | None = None,
+        nbconvert: bool = False,
+        nbconvert_args: list[str] | None = None,
         **kwargs,
     ) -> None:
         super().__init__(**kwargs)
@@ -95,6 +100,8 @@ def __init__(
         self.kernel_name = kernel_name
         self.language_name = language_name
         self.kernel_conn_id = kernel_conn_id
+        self.nbconvert = nbconvert
+        self.nbconvert_args = nbconvert_args
 
     def execute(self, context: Context):
         if not isinstance(self.input_nb, NoteBook):
@@ -132,6 +139,29 @@ def execute(self, context: Context):
             **remote_kernel_kwargs,
         )
 
+        # Convert the executed notebook to HTML using nbconvert
+        if self.nbconvert:
+            nbconvert_args = self.nbconvert_args or []
+            if not isinstance(nbconvert_args, list):
+                raise ValueError("nbconvert_args must be a list")
+
+            # Build the nbconvert command
+            command = [
+                "jupyter",
+                "nbconvert",
+                "--to",
+                "html",
+                "--log-level",
+                "WARN",
+                self.output_nb.url,
+            ] + nbconvert_args
+            try:
+                subprocess.run(command, check=True)
+                self.log.info("Output HTML: %s", self.output_nb.url.replace(".ipynb", ".html"))
+            except subprocess.CalledProcessError as e:
+                self.log.error("nbconvert failed with output:\n%s", e.stdout)
+                raise
+
     @cached_property
     def hook(self) -> KernelHook | None:
         """Get valid hook."""