test: run Django tests against Spanner emulator

.github/workflows/django_tests_against_emulator.yml

@@ -0,0 +1,33 @@
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+name: Run Django backend tests against emulator
+jobs:
+  system-tests:
+    runs-on: ubuntu-latest
+
+    services:
+      emulator:
+        image: gcr.io/cloud-spanner-emulator/emulator:latest
+        ports:
+          - 9010:9010
+          - 9020:9020
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+      - name: Setup Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.8
+      - name: Run system tests
+        run: sh django_test_suite.sh
+        env:
+          SPANNER_EMULATOR_HOST: localhost:9010
+          GOOGLE_CLOUD_PROJECT: emulator-test-project
+          GOOGLE_CLOUD_TESTS_CREATE_SPANNER_INSTANCE: true
+          DJANGO_WORKER_INDEX: 0
+          DJANGO_WORKER_COUNT: 1
+          SPANNER_TEST_INSTANCE: google-cloud-django-backend-tests

create_test_instance.py

@@ -0,0 +1,26 @@
+# Copyright 2016 Google LLC All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+
+from google.cloud.spanner_v1 import Client
+
+
+client = Client(
+    project=os.getenv("GOOGLE_CLOUD_PROJECT", "emulator-test-project")
+)
+
+instance = client.instance("google-cloud-django-backend-tests")
+created_op = instance.create()
+created_op.result(30)  # block until completion

django_spanner/base.py

@@ -4,6 +4,8 @@
 # license that can be found in the LICENSE file or at
 # https://developers.google.com/open-source/licenses/bsd
 
+import os
+
 from django.db.backends.base.base import BaseDatabaseWrapper
 from google.cloud import spanner_dbapi as Database, spanner_v1 as spanner
 
@@ -103,7 +105,9 @@ class DatabaseWrapper(BaseDatabaseWrapper):
 
     @property
     def instance(self):
-        return spanner.Client().instance(self.settings_dict["INSTANCE"])
+        return spanner.Client(
+            project=os.environ["GOOGLE_CLOUD_PROJECT"]
+        ).instance(self.settings_dict["INSTANCE"])
 
     @property
     def _nodb_connection(self):
@@ -113,7 +117,7 @@ def _nodb_connection(self):
 
     def get_connection_params(self):
         return {
-            "project": self.settings_dict["PROJECT"],
+            "project": os.environ["GOOGLE_CLOUD_PROJECT"],
             "instance_id": self.settings_dict["INSTANCE"],
             "database_id": self.settings_dict["NAME"],
             "user_agent": "django_spanner/2.2.0a1",

django_test_suite.sh

@@ -5,7 +5,26 @@
 # license that can be found in the LICENSE file.
 
 # exit when any command fails
-set -e
+set -x pipefail
+
+# Disable buffering, so that the logs stream through.
+export PYTHONUNBUFFERED=1
+
+# Export essential environment variables for Django tests.
+export RUNNING_SPANNER_BACKEND_TESTS=1
+
+pip3 install .
+# Create a unique DJANGO_TESTS_DIR per worker to avoid
+# any clashes with configured tests by other workers.
+export DJANGO_TESTS_DIR="django_tests_$DJANGO_WORKER_INDEX"
+mkdir -p $DJANGO_TESTS_DIR && git clone --depth 1 --single-branch --branch spanner-2.2.x https://github.com/timgraham/django.git $DJANGO_TESTS_DIR/django
+
+# Install dependencies for Django tests.
+sudo apt-get update
+apt-get install -y libffi-dev libjpeg-dev zlib1g-dev libmemcached-dev
+cd $DJANGO_TESTS_DIR/django && pip3 install -e . && pip3 install -r tests/requirements/py3.txt; cd ../../
+
+python3 create_test_instance.py
 
 # If no SPANNER_TEST_DB is set, generate a unique one
 # so that we can have multiple tests running without
@@ -15,7 +34,9 @@ TEST_DBNAME=${SPANNER_TEST_DB:-$(python3 -c 'import os, time; print(chr(ord("a")
 TEST_DBNAME_OTHER="$TEST_DBNAME-ot"
 TEST_APPS=${DJANGO_TEST_APPS:-basic}
 INSTANCE=${SPANNER_TEST_INSTANCE:-django-tests}
-PROJECT=${PROJECT_ID:-appdev-soda-spanner-staging}
+PROJECT=${PROJECT_ID}
+SPANNER_EMULATOR_HOST=${SPANNER_EMULATOR_HOST}
+GOOGLE_CLOUD_PROJECT=${GOOGLE_CLOUD_PROJECT}
 SETTINGS_FILE="$TEST_DBNAME-settings"
 TESTS_DIR=${DJANGO_TESTS_DIR:-django_tests}
 
@@ -42,21 +63,11 @@ PASSWORD_HASHERS = [
 !
 }
 
-setup_emulator_if_needed() {
-    if [[ $SPANNER_EMULATOR_HOST != "" ]]
-    then
-        echo "Running the emulator at: $SPANNER_EMULATOR_HOST"
-        ./emulator_main --host_port "$SPANNER_EMULATOR_HOST" &
-        SPANNER_INSTANCE=$INSTANCE python3 .kokoro/ensure_instance_exists.py
-    fi
-}
-
 run_django_tests() {
     cd $TESTS_DIR/django/tests
     create_settings
     echo -e "\033[32mRunning Django tests: $TEST_APPS\033[00m"
-    python3 runtests.py $TEST_APPS --verbosity=2 --noinput --settings $SETTINGS_FILE
+    python3 runtests.py $TEST_APPS --verbosity=3 --noinput --settings $SETTINGS_FILE
 }
 
-setup_emulator_if_needed
 run_django_tests