feat: support self-signed jwt in requests and urllib3 transports (#679)

Author: busunkim96

google/auth/transport/requests.py

@@ -45,6 +45,7 @@
 from google.auth import exceptions
 from google.auth import transport
 import google.auth.transport._mtls_helper
+from google.oauth2 import service_account
 
 _LOGGER = logging.getLogger(__name__)
 
@@ -313,6 +314,9 @@ def my_cert_callback():
             refreshing credentials. If not passed,
             an instance of :class:`~google.auth.transport.requests.Request`
             is created.
+        default_host (Optional[str]): A host like "pubsub.googleapis.com".
+            This is used when a self-signed JWT is created from service
+            account credentials.
     """
 
     def __init__(
@@ -322,13 +326,15 @@ def __init__(
         max_refresh_attempts=transport.DEFAULT_MAX_REFRESH_ATTEMPTS,
         refresh_timeout=None,
         auth_request=None,
+        default_host=None,
     ):
         super(AuthorizedSession, self).__init__()
         self.credentials = credentials
         self._refresh_status_codes = refresh_status_codes
         self._max_refresh_attempts = max_refresh_attempts
         self._refresh_timeout = refresh_timeout
         self._is_mtls = False
+        self._default_host = default_host
 
         if auth_request is None:
             auth_request_session = requests.Session()
@@ -347,6 +353,17 @@ def __init__(
         # credentials.refresh).
         self._auth_request = auth_request
 
+        # https://google.aip.dev/auth/4111
+        # Attempt to use self-signed JWTs when a service account is used.
+        # A default host must be explicitly provided.
+        if (
+            isinstance(self.credentials, service_account.Credentials)
+            and self._default_host
+        ):
+            self.credentials._create_self_signed_jwt(
+                "https://{}/".format(self._default_host)
+            )
+
     def configure_mtls_channel(self, client_cert_callback=None):
         """Configure the client certificate and key for SSL connection.
 

google/auth/transport/urllib3.py

@@ -49,6 +49,7 @@
 from google.auth import environment_vars
 from google.auth import exceptions
 from google.auth import transport
+from google.oauth2 import service_account
 
 _LOGGER = logging.getLogger(__name__)
 
@@ -262,6 +263,9 @@ def my_cert_callback():
             retried.
         max_refresh_attempts (int): The maximum number of times to attempt to
             refresh the credentials and retry the request.
+        default_host (Optional[str]): A host like "pubsub.googleapis.com".
+            This is used when a self-signed JWT is created from service
+            account credentials.
     """
 
     def __init__(
@@ -270,6 +274,7 @@ def __init__(
         http=None,
         refresh_status_codes=transport.DEFAULT_REFRESH_STATUS_CODES,
         max_refresh_attempts=transport.DEFAULT_MAX_REFRESH_ATTEMPTS,
+        default_host=None,
     ):
         if http is None:
             self.http = _make_default_http()
@@ -281,10 +286,22 @@ def __init__(
         self.credentials = credentials
         self._refresh_status_codes = refresh_status_codes
         self._max_refresh_attempts = max_refresh_attempts
+        self._default_host = default_host
         # Request instance used by internal methods (for example,
         # credentials.refresh).
         self._request = Request(self.http)
 
+        # https://google.aip.dev/auth/4111
+        # Attempt to use self-signed JWTs when a service account is used.
+        # A default host must be explicitly provided.
+        if (
+            isinstance(self.credentials, service_account.Credentials)
+            and self._default_host
+        ):
+            self.credentials._create_self_signed_jwt(
+                "https://{}/".format(self._default_host)
+            )
+
         super(AuthorizedHttp, self).__init__()
 
     def configure_mtls_channel(self, client_cert_callback=None):

system_tests/noxfile.py

@@ -293,13 +293,30 @@ def grpc(session):
     session.run("pytest", "system_tests_sync/test_grpc.py")
 
 
+@nox.session(python=PYTHON_VERSIONS_SYNC)
+def requests(session):
+    session.install(LIBRARY_DIR)
+    session.install(*TEST_DEPENDENCIES_SYNC)
+    session.env[EXPLICIT_CREDENTIALS_ENV] = SERVICE_ACCOUNT_FILE
+    session.run("pytest", "system_tests_sync/test_requests.py")
+
+
+@nox.session(python=PYTHON_VERSIONS_SYNC)
+def urllib3(session):
+    session.install(LIBRARY_DIR)
+    session.install(*TEST_DEPENDENCIES_SYNC)
+    session.env[EXPLICIT_CREDENTIALS_ENV] = SERVICE_ACCOUNT_FILE
+    session.run("pytest", "system_tests_sync/test_urllib3.py")
+
+
 @nox.session(python=PYTHON_VERSIONS_SYNC)
 def mtls_http(session):
     session.install(LIBRARY_DIR)
     session.install(*TEST_DEPENDENCIES_SYNC, "pyopenssl")
     session.env[EXPLICIT_CREDENTIALS_ENV] = SERVICE_ACCOUNT_FILE
     session.run("pytest", "system_tests_sync/test_mtls_http.py")
 
+
 #ASYNC SYSTEM TESTS
 
 @nox.session(python=PYTHON_VERSIONS_ASYNC)

system_tests/system_tests_sync/test_grpc.py

@@ -57,8 +57,9 @@ def test_grpc_request_with_regular_credentials_and_self_signed_jwt(http_request)
     list_topics_iter = client.list_topics(project="projects/{}".format(project_id))
     list(list_topics_iter)
 
-    # Check that self-signed JWT was created
+    # Check that self-signed JWT was created and is being used
     assert credentials._jwt_credentials is not None
+    assert credentials._jwt_credentials.token == credentials.token
 
 
 def test_grpc_request_with_jwt_credentials():

system_tests/system_tests_sync/test_requests.py

@@ -0,0 +1,40 @@
+# Copyright 2021 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import google.auth
+import google.auth.credentials
+import google.auth.transport.requests
+from google.oauth2 import service_account
+
+
+def test_authorized_session_with_service_account_and_self_signed_jwt():
+    credentials, project_id = google.auth.default()
+
+    credentials = credentials.with_scopes(
+        scopes=[],
+        default_scopes=["https://www.googleapis.com/auth/pubsub"],
+    )
+
+    session = google.auth.transport.requests.AuthorizedSession(
+        credentials=credentials, default_host="pubsub.googleapis.com"
+    )
+
+    # List Pub/Sub Topics through the REST API
+    # https://cloud.google.com/pubsub/docs/reference/rest/v1/projects.topics/list
+    response = session.get("https://pubsub.googleapis.com/v1/projects/{}/topics".format(project_id))
+    response.raise_for_status()
+
+    # Check that self-signed JWT was created and is being used
+    assert credentials._jwt_credentials is not None
+    assert credentials._jwt_credentials.token == credentials.token

system_tests/system_tests_sync/test_urllib3.py

@@ -0,0 +1,44 @@
+# Copyright 2021 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import google.auth
+import google.auth.credentials
+import google.auth.transport.requests
+from google.oauth2 import service_account
+
+
+def test_authorized_session_with_service_account_and_self_signed_jwt():
+    credentials, project_id = google.auth.default()
+
+    credentials = credentials.with_scopes(
+        scopes=[],
+        default_scopes=["https://www.googleapis.com/auth/pubsub"],
+    )
+
+    http = google.auth.transport.urllib3.AuthorizedHttp(
+        credentials=credentials, default_host="pubsub.googleapis.com"
+    )
+
+    # List Pub/Sub Topics through the REST API
+    # https://cloud.google.com/pubsub/docs/reference/rest/v1/projects.topics/list
+    response = http.urlopen(
+        method="GET",
+        url="https://pubsub.googleapis.com/v1/projects/{}/topics".format(project_id)
+    )
+
+    assert response.status == 200
+
+    # Check that self-signed JWT was created and is being used
+    assert credentials._jwt_credentials is not None
+    assert credentials._jwt_credentials.token == credentials.token

tests/transport/test_requests.py

@@ -30,6 +30,7 @@
 import google.auth.credentials
 import google.auth.transport._mtls_helper
 import google.auth.transport.requests
+from google.oauth2 import service_account
 from tests.transport import compliance
 
 
@@ -372,6 +373,25 @@ def test_request_timeout_w_refresh_timeout_timeout_error(self, frozen_time):
                 "GET", self.TEST_URL, timeout=60, max_allowed_time=2.9
             )
 
+    def test_authorized_session_without_default_host(self):
+        credentials = mock.create_autospec(service_account.Credentials)
+
+        authed_session = google.auth.transport.requests.AuthorizedSession(credentials)
+
+        authed_session.credentials._create_self_signed_jwt.assert_not_called()
+
+    def test_authorized_session_with_default_host(self):
+        default_host = "pubsub.googleapis.com"
+        credentials = mock.create_autospec(service_account.Credentials)
+
+        authed_session = google.auth.transport.requests.AuthorizedSession(
+            credentials, default_host=default_host
+        )
+
+        authed_session.credentials._create_self_signed_jwt.assert_called_once_with(
+            "https://{}/".format(default_host)
+        )
+
     def test_configure_mtls_channel_with_callback(self):
         mock_callback = mock.Mock()
         mock_callback.return_value = (

tests/transport/test_urllib3.py

@@ -26,6 +26,7 @@
 import google.auth.credentials
 import google.auth.transport._mtls_helper
 import google.auth.transport.urllib3
+from google.oauth2 import service_account
 from tests.transport import compliance
 
 
@@ -158,6 +159,25 @@ def test_urlopen_refresh(self):
             ("GET", self.TEST_URL, None, {"authorization": "token1"}, {}),
         ]
 
+    def test_urlopen_no_default_host(self):
+        credentials = mock.create_autospec(service_account.Credentials)
+
+        authed_http = google.auth.transport.urllib3.AuthorizedHttp(credentials)
+
+        authed_http.credentials._create_self_signed_jwt.assert_not_called()
+
+    def test_urlopen_with_default_host(self):
+        default_host = "pubsub.googleapis.com"
+        credentials = mock.create_autospec(service_account.Credentials)
+
+        authed_http = google.auth.transport.urllib3.AuthorizedHttp(
+            credentials, default_host=default_host
+        )
+
+        authed_http.credentials._create_self_signed_jwt.assert_called_once_with(
+            "https://{}/".format(default_host)
+        )
+
     def test_proxies(self):
         http = mock.create_autospec(urllib3.PoolManager)
         authed_http = google.auth.transport.urllib3.AuthorizedHttp(None, http=http)