From 1281ac176901eee1e9b500f34ee811362172dff3 Mon Sep 17 00:00:00 2001 From: Matthieu Vilatte Date: Sat, 12 Nov 2022 23:06:13 +0100 Subject: [PATCH 1/2] add cloud_run_operator + tests --- .../providers/google/cloud/hooks/cloud_run.py | 60 +++++++++++++++++ .../google/cloud/operators/cloud_run.py | 64 +++++++++++++++++++ .../cloud/utils/credentials_provider.py | 58 +++++++++++++---- .../google/common/hooks/base_google.py | 34 +++++++--- airflow/providers/google/provider.yaml | 12 ++++ .../operators/cloud/cloud_run.rst | 60 +++++++++++++++++ generated/provider_dependencies.json | 2 + .../google/cloud/hooks/test_cloud_run.py | 58 +++++++++++++++++ .../google/cloud/operators/test_cloud_run.py | 45 +++++++++++++ .../cloud/utils/test_credentials_provider.py | 15 +++++ .../google/cloud/cloud_run/__init__.py | 16 +++++ .../cloud/cloud_run/example_cloud_run.py | 49 ++++++++++++++ 12 files changed, 452 insertions(+), 21 deletions(-) create mode 100644 airflow/providers/google/cloud/hooks/cloud_run.py create mode 100644 airflow/providers/google/cloud/operators/cloud_run.py create mode 100644 docs/apache-airflow-providers-google/operators/cloud/cloud_run.rst create mode 100644 tests/providers/google/cloud/hooks/test_cloud_run.py create mode 100644 tests/providers/google/cloud/operators/test_cloud_run.py create mode 100644 tests/system/providers/google/cloud/cloud_run/__init__.py create mode 100644 tests/system/providers/google/cloud/cloud_run/example_cloud_run.py diff --git a/airflow/providers/google/cloud/hooks/cloud_run.py b/airflow/providers/google/cloud/hooks/cloud_run.py new file mode 100644 index 0000000000000..3bf6810287cc0 --- /dev/null +++ b/airflow/providers/google/cloud/hooks/cloud_run.py @@ -0,0 +1,60 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""This module contains a Google Cloud Run Hook.""" +from __future__ import annotations + +from typing import Sequence + +import google.auth.transport.requests + +from airflow.hooks.base import BaseHook +from airflow.providers.google.common.hooks.base_google import GoogleBaseHook + + +class CloudRunHook(GoogleBaseHook): + """Hook for Google Cloud Run.""" + + def __init__( + self, + gcp_conn_id: str = "google_cloud_default", + cloud_run_conn_id: str = "http_default", + delegate_to: str | None = None, + impersonation_chain: str | Sequence[str] | None = None, + ) -> None: + super().__init__( + gcp_conn_id=gcp_conn_id, delegate_to=delegate_to, impersonation_chain=impersonation_chain + ) + self.cloud_run_conn_id = cloud_run_conn_id + + def get_conn(self) -> dict: + """ + Retrieves HTTP authentication header allowing + authenticated Google Cloud Run call. + :return: Authentication header + :rtype: dict + """ + http_connection = BaseHook.get_connection(self.cloud_run_conn_id) + credentials = self.get_id_token_credentials(target_audience=http_connection.host) + auth_req = google.auth.transport.requests.Request() + credentials.refresh(auth_req) + + authentication_header = { + "Authorization": f"Bearer {credentials.token}", + "Content-Type": "application/json", + } + return authentication_header diff --git a/airflow/providers/google/cloud/operators/cloud_run.py b/airflow/providers/google/cloud/operators/cloud_run.py new file mode 100644 index 0000000000000..5fe51536312ed --- /dev/null +++ b/airflow/providers/google/cloud/operators/cloud_run.py @@ -0,0 +1,64 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""This module contains a Google CloudRun operator.""" +from __future__ import annotations + +from typing import Sequence + +from airflow.providers.google.cloud.hooks.cloud_run import CloudRunHook +from airflow.providers.http.operators.http import SimpleHttpOperator + + +class CloudRunOperator(SimpleHttpOperator): + """ + Performs an authenticated call against CloudRun + + Under the hood, this operator use SimpleHttpOperator. + Check the documentation of SimpleHttpOperator for extra options. + + This operator use the GCP connection to get a token and add it to http request Header. + + .. seealso:: + For more information on how to use this operator, take a look at the guide: + :ref:`howto/operator:CloudRunOperator` + + :param http_conn_id: The :ref:`CloudRun http connection` to run + the operator against + :param gcp_conn_id: (Optional) The connection ID used to connect to Google Cloud. + """ + + template_fields: Sequence[str] = ("gcp_conn_id",) + + def __init__( + self, + *, + gcp_conn_id: str = "google_cloud_default", + **kwargs, + ) -> None: + super().__init__(**kwargs) + self.gcp_conn_id = gcp_conn_id + + def execute(self, context): + cloud_run = CloudRunHook( + gcp_conn_id=self.gcp_conn_id, + cloud_run_conn_id=self.http_conn_id, + ) + + authentication_header = cloud_run.get_conn() + self.headers = {**authentication_header, **self.headers} + return super().execute(context) diff --git a/airflow/providers/google/cloud/utils/credentials_provider.py b/airflow/providers/google/cloud/utils/credentials_provider.py index f10dd3d4946c2..cccb543cab0b7 100644 --- a/airflow/providers/google/cloud/utils/credentials_provider.py +++ b/airflow/providers/google/cloud/utils/credentials_provider.py @@ -33,6 +33,7 @@ import google.oauth2.service_account from google.auth import impersonated_credentials from google.auth.environment_vars import CREDENTIALS, LEGACY_PROJECT, PROJECT +from google.oauth2 import service_account from airflow.exceptions import AirflowException from airflow.providers.google.cloud._internal_client.secret_manager_client import _SecretManagerClient @@ -199,10 +200,12 @@ def __init__( key_secret_name: str | None = None, key_secret_project_id: str | None = None, scopes: Collection[str] | None = None, + target_audience: str | None = None, delegate_to: str | None = None, disable_logging: bool = False, target_principal: str | None = None, delegates: Sequence[str] | None = None, + is_id_token_credentials: bool = False, ) -> None: super().__init__() key_options = [key_path, key_secret_name, keyfile_dict] @@ -216,10 +219,12 @@ def __init__( self.key_secret_name = key_secret_name self.key_secret_project_id = key_secret_project_id self.scopes = scopes + self.target_audience = target_audience self.delegate_to = delegate_to self.disable_logging = disable_logging self.target_principal = target_principal self.delegates = delegates + self.is_id_token_credentials = is_id_token_credentials def get_credentials_and_project(self) -> tuple[google.auth.credentials.Credentials, str]: """ @@ -263,10 +268,7 @@ def _get_credentials_using_keyfile_dict(self): # Depending on how the JSON was formatted, it may contain # escaped newlines. Convert those to actual newlines. self.keyfile_dict["private_key"] = self.keyfile_dict["private_key"].replace("\\n", "\n") - credentials = google.oauth2.service_account.Credentials.from_service_account_info( - self.keyfile_dict, scopes=self.scopes - ) - project_id = credentials.project_id + credentials, project_id = self._get_credentials_using_info(self.keyfile_dict) return credentials, project_id def _get_credentials_using_key_path(self): @@ -277,10 +279,7 @@ def _get_credentials_using_key_path(self): raise AirflowException("Unrecognised extension for key file.") self._log_debug("Getting connection using JSON key file %s", self.key_path) - credentials = google.oauth2.service_account.Credentials.from_service_account_file( - self.key_path, scopes=self.scopes - ) - project_id = credentials.project_id + credentials, project_id = self._get_credentials_using_file(self.key_path) return credentials, project_id def _get_credentials_using_key_secret_name(self): @@ -305,10 +304,7 @@ def _get_credentials_using_key_secret_name(self): except json.decoder.JSONDecodeError: raise AirflowException("Key data read from GCP Secret Manager is not valid JSON.") - credentials = google.oauth2.service_account.Credentials.from_service_account_info( - keyfile_dict, scopes=self.scopes - ) - project_id = credentials.project_id + credentials, project_id = self._get_credentials_using_info(keyfile_dict) return credentials, project_id def _get_credentials_using_adc(self): @@ -318,6 +314,34 @@ def _get_credentials_using_adc(self): credentials, project_id = google.auth.default(scopes=self.scopes) return credentials, project_id + def _get_credentials_using_file(self, file): + if self.is_id_token_credentials: + credentials = service_account.IDTokenCredentials.from_service_account_file( + file, + target_audience=self.target_audience, + ) + project_id = None + else: + credentials = google.oauth2.service_account.Credentials.from_service_account_file( + file, scopes=self.scopes + ) + project_id = credentials.project_id + return credentials, project_id + + def _get_credentials_using_info(self, info): + if self.is_id_token_credentials: + credentials = service_account.IDTokenCredentials.from_service_account_info( + info, + target_audience=self.target_audience, + ) + project_id = None + else: + credentials = google.oauth2.service_account.Credentials.from_service_account_info( + info, scopes=self.scopes + ) + project_id = credentials.project_id + return credentials, project_id + def _log_info(self, *args, **kwargs) -> None: if not self.disable_logging: self.log.info(*args, **kwargs) @@ -332,6 +356,16 @@ def get_credentials_and_project_id(*args, **kwargs) -> tuple[google.auth.credent return _CredentialProvider(*args, **kwargs).get_credentials_and_project() +def get_id_token_credentials(*args, **kwargs) -> tuple[google.auth.credentials.Credentials]: + """Returns the ID Token type Credentials object for Google API.""" + kwargs = {**kwargs, "is_id_token_credentials": True} + id_token_credentials, _ = _CredentialProvider( + *args, + **kwargs, + ).get_credentials_and_project() + return id_token_credentials + + def _get_scopes(scopes: str | None = None) -> Sequence[str]: """ Parse a comma-separated string containing OAuth2 scopes if `scopes` is provided. diff --git a/airflow/providers/google/common/hooks/base_google.py b/airflow/providers/google/common/hooks/base_google.py index cfad2db0a5e73..395b190a588d8 100644 --- a/airflow/providers/google/common/hooks/base_google.py +++ b/airflow/providers/google/common/hooks/base_google.py @@ -52,6 +52,7 @@ _get_scopes, _get_target_principal_and_delegates, get_credentials_and_project_id, + get_id_token_credentials, ) from airflow.providers.google.common.consts import CLIENT_INFO from airflow.utils.process_utils import patch_environ @@ -232,11 +233,7 @@ def __init__( self._cached_credentials: google.auth.credentials.Credentials | None = None self._cached_project_id: str | None = None - def get_credentials_and_project_id(self) -> tuple[google.auth.credentials.Credentials, str | None]: - """Returns the Credentials object for Google API and the associated project_id""" - if self._cached_credentials is not None: - return self._cached_credentials, self._cached_project_id - + def get_connection_info(self): key_path: str | None = self._get_field("key_path", None) try: keyfile_dict: str | None = self._get_field("keyfile_dict", None) @@ -247,14 +244,24 @@ def get_credentials_and_project_id(self) -> tuple[google.auth.credentials.Creden raise AirflowException("Invalid key JSON.") key_secret_name: str | None = self._get_field("key_secret_name", None) key_secret_project_id: str | None = self._get_field("key_secret_project_id", None) + return { + "key_path": key_path, + "keyfile_dict": keyfile_dict_json, + "key_secret_name": key_secret_name, + "key_secret_project_id": key_secret_project_id, + } + + def get_credentials_and_project_id(self) -> tuple[google.auth.credentials.Credentials, str | None]: + """Returns the Credentials object for Google API and the associated project_id""" + if self._cached_credentials is not None: + return self._cached_credentials, self._cached_project_id + + connection_info = self.get_connection_info() target_principal, delegates = _get_target_principal_and_delegates(self.impersonation_chain) credentials, project_id = get_credentials_and_project_id( - key_path=key_path, - keyfile_dict=keyfile_dict_json, - key_secret_name=key_secret_name, - key_secret_project_id=key_secret_project_id, + **connection_info, scopes=self.scopes, delegate_to=self.delegate_to, target_principal=target_principal, @@ -270,6 +277,15 @@ def get_credentials_and_project_id(self) -> tuple[google.auth.credentials.Creden return credentials, project_id + def get_id_token_credentials(self, target_audience: str = "") -> google.auth.credentials.Credentials: + connection_info = self.get_connection_info() + return get_id_token_credentials( + **connection_info, + scopes=self.scopes, + delegate_to=self.delegate_to, + target_audience=target_audience, + ) + def get_credentials(self) -> google.auth.credentials.Credentials: """Returns the Credentials object for Google API""" credentials, _ = self.get_credentials_and_project_id() diff --git a/airflow/providers/google/provider.yaml b/airflow/providers/google/provider.yaml index 7c3efe2d5a656..889cf1297dbf0 100644 --- a/airflow/providers/google/provider.yaml +++ b/airflow/providers/google/provider.yaml @@ -58,6 +58,7 @@ versions: dependencies: - apache-airflow>=2.3.0 - apache-airflow-providers-common-sql>=1.3.1 + - apache-airflow-providers-http>=4.0.0 # Google has very clear rules on what dependencies should be used. All the limits below # follow strict guidelines of Google Libraries as quoted here: # While this issue is open, dependents of google-api-core, google-cloud-core. and google-auth @@ -210,6 +211,11 @@ integrations: - /docs/apache-airflow-providers-google/operators/cloud/cloud_memorystore_memcached.rst logo: /integration-logos/gcp/Cloud-Memorystore.png tags: [gcp] + - integration-name: Google Cloud Run + external-doc-url: https://cloud.google.com/run/ + how-to-guide: + - /docs/apache-airflow-providers-google/operators/cloud/cloud_run.rst + tags: [gcp] - integration-name: Google Cloud OS Login external-doc-url: https://cloud.google.com/compute/docs/oslogin/ logo: /integration-logos/gcp/Google-Cloud-Generic.png @@ -466,6 +472,9 @@ operators: - integration-name: Google Cloud Memorystore python-modules: - airflow.providers.google.cloud.operators.cloud_memorystore + - integration-name: Google Cloud Run + python-modules: + - airflow.providers.google.cloud.operators.cloud_run - integration-name: Google Cloud SQL python-modules: - airflow.providers.google.cloud.operators.cloud_sql @@ -676,6 +685,9 @@ hooks: - integration-name: Google Cloud Memorystore python-modules: - airflow.providers.google.cloud.hooks.cloud_memorystore + - integration-name: Google Cloud Run + python-modules: + - airflow.providers.google.cloud.hooks.cloud_run - integration-name: Google Cloud SQL python-modules: - airflow.providers.google.cloud.hooks.cloud_sql diff --git a/docs/apache-airflow-providers-google/operators/cloud/cloud_run.rst b/docs/apache-airflow-providers-google/operators/cloud/cloud_run.rst new file mode 100644 index 0000000000000..f815d0676c068 --- /dev/null +++ b/docs/apache-airflow-providers-google/operators/cloud/cloud_run.rst @@ -0,0 +1,60 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +Google CloudRun Operator +=============================== + +`Cloud Run `__ is Google's fully managed, +Container as a Service (CaaS) Solution. +It is a serverless compute platform service which allows users to +deploy code written in any programming language, with an https endpoint. + +Airflow provides operators to make authenticated call against deployed Cloud Run. + +Prerequisite Tasks +^^^^^^^^^^^^^^^^^^ + +.. include:: ../_partials/prerequisite_tasks.rst + +Cloud Run call +^^^^^^^^^^^^^^^ + +.. _howto/operator:CloudRunOperator: + +Send Authenticated HTTP request +"""""""""""""" + +To send an authenticated HTTP request against a Cloud Run you can use +:class:`~airflow.providers.google.cloud.operators.cloud_run.CloudRunOperator`. + +.. exampleinclude:: /../../tests/system/providers/google/cloud/bigquery/example_cloud_run.py + :language: python + :dedent: 4 + :start-after: [START howto_operator_cloud_run_call] + :end-before: [END howto_operator_cloud_run_call] + +This operator extends `SimpleHttpOperator `__. +See the associated doc for extra arguments you can use (header, data). + +Reference +^^^^^^^^^ + +For further information, look at: + +* `Client Library Documentation `__ +* `Product Documentation `__ diff --git a/generated/provider_dependencies.json b/generated/provider_dependencies.json index 178af7b20d095..d781a84a0a8b7 100644 --- a/generated/provider_dependencies.json +++ b/generated/provider_dependencies.json @@ -301,6 +301,7 @@ "deps": [ "PyOpenSSL", "apache-airflow-providers-common-sql>=1.3.1", + "apache-airflow-providers-http>=4.0.0", "apache-airflow>=2.3.0", "asgiref>=3.5.2", "gcloud-aio-bigquery>=6.1.2", @@ -360,6 +361,7 @@ "cncf.kubernetes", "common.sql", "facebook", + "http", "microsoft.azure", "microsoft.mssql", "mysql", diff --git a/tests/providers/google/cloud/hooks/test_cloud_run.py b/tests/providers/google/cloud/hooks/test_cloud_run.py new file mode 100644 index 0000000000000..8bd70b7b6f8f3 --- /dev/null +++ b/tests/providers/google/cloud/hooks/test_cloud_run.py @@ -0,0 +1,58 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from unittest import mock + +from airflow.providers.google.cloud.hooks.cloud_run import CloudRunHook + +CLOUD_RUN_CONN_ID = "test-cloud-run-conn" +GCP_CONN_ID = "test-conn" +TOKEN = "TOKEN" +AUTHENTICATION_HEADER = { + "Authorization": f"Bearer {TOKEN}", + "Content-Type": "application/json", +} +CREDENTIALS = { + "token": TOKEN, +} + + +class MockedCredentials: + def __init__(self): + self.token = TOKEN + + def refresh(self, *args): + pass + + +class TestCloudRunHook: + @mock.patch("airflow.providers.google.cloud.hooks.cloud_run.GoogleBaseHook.get_id_token_credentials") + @mock.patch("airflow.providers.google.cloud.hooks.cloud_run.BaseHook.get_connection") + @mock.patch("airflow.providers.google.cloud.hooks.cloud_run.google.auth.transport.requests.Request") + def test_get_conn(self, mock_auth_request, mock_base_hook_get_conn, mock_google_get_credentials): + mock_google_get_credentials.return_value = MockedCredentials() + hook = CloudRunHook( + gcp_conn_id=GCP_CONN_ID, + cloud_run_conn_id=CLOUD_RUN_CONN_ID, + ) + authentication_header = hook.get_conn() + mock_base_hook_get_conn.assert_called_with(CLOUD_RUN_CONN_ID) + mock_auth_request.assert_called_once() + + assert authentication_header == AUTHENTICATION_HEADER diff --git a/tests/providers/google/cloud/operators/test_cloud_run.py b/tests/providers/google/cloud/operators/test_cloud_run.py new file mode 100644 index 0000000000000..d6df5908ca2e5 --- /dev/null +++ b/tests/providers/google/cloud/operators/test_cloud_run.py @@ -0,0 +1,45 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from unittest import mock +from unittest.mock import MagicMock + +from airflow.providers.google.cloud.operators.cloud_run import CloudRunOperator + +TASK_ID = "task-id" +HTTP_CONN_ID = "test-http-conn" +GCP_CONN_ID = "test-conn" +AUTHENTICATION_HEADER = {"Authorisation": "Bearer TOKEN"} + + +class TestCloudRunOperator: + @mock.patch("airflow.providers.google.cloud.operators.cloud_run.CloudRunHook") + @mock.patch("airflow.providers.google.cloud.operators.cloud_run.SimpleHttpOperator.execute") + def test_execute(self, mock_http_execute, mock_cloudrun_hook): + mock_cloudrun_hook.return_value.get_conn.return_value = AUTHENTICATION_HEADER + op = CloudRunOperator( + task_id=TASK_ID, + gcp_conn_id=GCP_CONN_ID, + http_conn_id=HTTP_CONN_ID, + ) + op.execute(context=MagicMock()) + + mock_cloudrun_hook.return_value.get_conn.assert_called_once() + mock_http_execute.assert_called_once() + assert op.headers == AUTHENTICATION_HEADER diff --git a/tests/providers/google/cloud/utils/test_credentials_provider.py b/tests/providers/google/cloud/utils/test_credentials_provider.py index d618f653d976b..174962c1055db 100644 --- a/tests/providers/google/cloud/utils/test_credentials_provider.py +++ b/tests/providers/google/cloud/utils/test_credentials_provider.py @@ -40,6 +40,7 @@ _get_target_principal_and_delegates, build_gcp_conn, get_credentials_and_project_id, + get_id_token_credentials, provide_gcp_conn_and_credentials, provide_gcp_connection, provide_gcp_credentials, @@ -283,6 +284,20 @@ def test_get_credentials_and_project_id_with_service_account_info(self, mock_fro "connection using JSON Dict" ] == cm.output + @mock.patch( + "google.oauth2.service_account.IDTokenCredentials.from_service_account_info", + ) + def test_get_id_token_credentials_with_service_account_info(self, mock_from_service_account_info): + service_account = {"private_key": "PRIVATE_KEY"} + with self.assertLogs(level="DEBUG", logger=CRED_PROVIDER_LOGGER_NAME) as cm: + result = get_id_token_credentials(keyfile_dict=service_account) + mock_from_service_account_info.assert_called_once_with(service_account, target_audience=None) + assert mock_from_service_account_info.return_value == result + assert [ + "DEBUG:airflow.providers.google.cloud.utils.credentials_provider._CredentialProvider:Getting " + "connection using JSON Dict" + ] == cm.output + @mock.patch("google.auth.default", return_value=("CREDENTIALS", "PROJECT_ID")) @mock.patch("google.oauth2.service_account.Credentials.from_service_account_info") @mock.patch("airflow.providers.google.cloud.utils.credentials_provider._SecretManagerClient") diff --git a/tests/system/providers/google/cloud/cloud_run/__init__.py b/tests/system/providers/google/cloud/cloud_run/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/tests/system/providers/google/cloud/cloud_run/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/tests/system/providers/google/cloud/cloud_run/example_cloud_run.py b/tests/system/providers/google/cloud/cloud_run/example_cloud_run.py new file mode 100644 index 0000000000000..c2709142218d5 --- /dev/null +++ b/tests/system/providers/google/cloud/cloud_run/example_cloud_run.py @@ -0,0 +1,49 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Example Airflow DAG for Google Cloud Run service testing cloud run call. +""" +from __future__ import annotations + +import os +from datetime import datetime + +from airflow import models +from airflow.providers.google.cloud.operators.cloud_run import CloudRunOperator + +ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID") +DAG_ID = "cloud_run" + + +with models.DAG( + DAG_ID, + schedule="@once", + start_date=datetime(2021, 1, 1), + catchup=False, + tags=["example", "cloud_run"], +) as dag: + # [START howto_operator_cloud_run_call] + create_dataset = CloudRunOperator( + task_id="cloud_run_call", http_conn_id="cloud_run_http_conn", gcp_conn_id="gcp_conn", endpoint="" + ) + # [END howto_operator_cloud_run_call] + +from tests.system.utils import get_test_run # noqa: E402 + +# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +test_run = get_test_run(dag) From 45ce866b35f824d1ee1208ce1d624203570832bc Mon Sep 17 00:00:00 2001 From: Matthieu Vilatte Date: Fri, 18 Nov 2022 14:26:27 +0100 Subject: [PATCH 2/2] fix build doc --- .../operators/cloud/cloud_run.rst | 8 ++++---- docs/spelling_wordlist.txt | 1 + .../providers/google/cloud/cloud_run/example_cloud_run.py | 8 ++++++-- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/docs/apache-airflow-providers-google/operators/cloud/cloud_run.rst b/docs/apache-airflow-providers-google/operators/cloud/cloud_run.rst index f815d0676c068..2f038d7b79462 100644 --- a/docs/apache-airflow-providers-google/operators/cloud/cloud_run.rst +++ b/docs/apache-airflow-providers-google/operators/cloud/cloud_run.rst @@ -17,7 +17,7 @@ under the License. Google CloudRun Operator -=============================== +======================== `Cloud Run `__ is Google's fully managed, Container as a Service (CaaS) Solution. @@ -32,17 +32,17 @@ Prerequisite Tasks .. include:: ../_partials/prerequisite_tasks.rst Cloud Run call -^^^^^^^^^^^^^^^ +^^^^^^^^^^^^^^ .. _howto/operator:CloudRunOperator: Send Authenticated HTTP request -"""""""""""""" +""""""""""""""""""""""""""""""" To send an authenticated HTTP request against a Cloud Run you can use :class:`~airflow.providers.google.cloud.operators.cloud_run.CloudRunOperator`. -.. exampleinclude:: /../../tests/system/providers/google/cloud/bigquery/example_cloud_run.py +.. exampleinclude:: /../../tests/system/providers/google/cloud/cloud_run/example_cloud_run.py :language: python :dedent: 4 :start-after: [START howto_operator_cloud_run_call] diff --git a/docs/spelling_wordlist.txt b/docs/spelling_wordlist.txt index eaa0553ed6d1d..468a832783413 100644 --- a/docs/spelling_wordlist.txt +++ b/docs/spelling_wordlist.txt @@ -181,6 +181,7 @@ bugfixes buildType burstable bytestring +CaaS cacert callables Cancelled diff --git a/tests/system/providers/google/cloud/cloud_run/example_cloud_run.py b/tests/system/providers/google/cloud/cloud_run/example_cloud_run.py index c2709142218d5..a13c197113dde 100644 --- a/tests/system/providers/google/cloud/cloud_run/example_cloud_run.py +++ b/tests/system/providers/google/cloud/cloud_run/example_cloud_run.py @@ -38,8 +38,12 @@ tags=["example", "cloud_run"], ) as dag: # [START howto_operator_cloud_run_call] - create_dataset = CloudRunOperator( - task_id="cloud_run_call", http_conn_id="cloud_run_http_conn", gcp_conn_id="gcp_conn", endpoint="" + call_cloud_run = CloudRunOperator( + task_id="cloud_run_call", + http_conn_id="cloud_run_http_conn", + gcp_conn_id="gcp_conn", + method="GET", + endpoint="/my-endpoint", ) # [END howto_operator_cloud_run_call]