From 0a265279b95a7c8bea71727b17afd57280d37d0e Mon Sep 17 00:00:00 2001 From: stojdza Date: Thu, 2 Apr 2020 17:20:44 +0200 Subject: [PATCH 01/10] Add script for canary tests --- package.json | 1 + 1 file changed, 1 insertion(+) diff --git a/package.json b/package.json index c684589a76..2c6fd44342 100644 --- a/package.json +++ b/package.json @@ -102,6 +102,7 @@ "test:inside-gb": "cross-env NODE_ENV=test jest --verbose --config jest_gb.config.js", "test:debug": "cross-env NODE_ENV=test node --inspect-brk node_modules/.bin/jest --runInBand --verbose --config jest.config.js", "device-tests": "cross-env NODE_ENV=test jest --no-cache --maxWorkers=3 --reporters=default --reporters=jest-junit --verbose --config jest_ui.config.js", + "device-tests-canary": "cross-env NODE_ENV=test jest --no-cache --maxWorkers=2 --testNamePattern=@canary --reporters=default --reporters=jest-junit --verbose --config jest_ui.config.js", "device-tests:local": "cross-env NODE_ENV=test jest --runInBand --reporters=default --reporters=jest-junit --detectOpenHandles --verbose --config jest_ui.config.js", "device-tests:debug": "cross-env NODE_ENV=test node $NODE_DEBUG_OPTION --inspect-brk node_modules/jest/bin/jest --runInBand --reporters=default --reporters=jest-junit --detectOpenHandles --verbose --config jest_ui.config.js", "test:e2e": "yarn test:e2e:android && yarn test:e2e:ios", From 1e73fada05f278f4bc6ef122becab9fd6e0d8691 Mon Sep 17 00:00:00 2001 From: stojdza Date: Thu, 2 Apr 2020 17:21:39 +0200 Subject: [PATCH 02/10] Run canary tests on every PR --- .circleci/config.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index f492208834..3cb39b1cc8 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -84,7 +84,7 @@ jobs: curl -u "$SAUCE_USERNAME:$SAUCE_ACCESS_KEY" -X POST -H "Content-Type: application/octet-stream" https://saucelabs.com/rest/v1/storage/automattic/Gutenberg-$SAUCE_FILENAME.apk?overwrite=true --data-binary @./android/app/build/outputs/apk/debug/app-debug.apk - run: name: Run Device Tests - command: yarn device-tests + command: yarn device-tests-canary environment: JEST_JUNIT_OUTPUT: "reports/test-results/android-test-results.xml" - store_test_results: @@ -143,7 +143,7 @@ jobs: - run: name: Run Device Tests command: | - yarn device-tests + yarn device-tests-canary environment: JEST_JUNIT_OUTPUT: "reports/test-results/ios-test-results.xml" - store_test_results: @@ -172,6 +172,6 @@ workflows: platform: android check-tests: true - ios-device-checks: - name: Test iOS on Device + name: Test iOS on Device - Canaries - android-device-checks: - name: Test Android on Device + name: Test Android on Device - Canaries From 60d8aaf677187a4866ded8db1887a6e289cc2187 Mon Sep 17 00:00:00 2001 From: stojdza Date: Thu, 2 Apr 2020 17:22:47 +0200 Subject: [PATCH 03/10] Mark 3 tests as canaries --- __device-tests__/gutenberg-editor-heading.test.js | 2 +- __device-tests__/gutenberg-editor-image.test.js | 2 +- __device-tests__/gutenberg-editor-lists.test.js | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/__device-tests__/gutenberg-editor-heading.test.js b/__device-tests__/gutenberg-editor-heading.test.js index c3d62168c4..c9a5d4f5c9 100644 --- a/__device-tests__/gutenberg-editor-heading.test.js +++ b/__device-tests__/gutenberg-editor-heading.test.js @@ -16,7 +16,7 @@ import testData from './helpers/test-data'; jasmine.DEFAULT_TIMEOUT_INTERVAL = 1000000; -describe( 'Gutenberg Editor tests', () => { +describe( 'Gutenberg Editor tests @canary', () => { let driver; let editorPage; let allPassed = true; diff --git a/__device-tests__/gutenberg-editor-image.test.js b/__device-tests__/gutenberg-editor-image.test.js index e504807156..98b9784604 100644 --- a/__device-tests__/gutenberg-editor-image.test.js +++ b/__device-tests__/gutenberg-editor-image.test.js @@ -18,7 +18,7 @@ import testData from './helpers/test-data'; jasmine.DEFAULT_TIMEOUT_INTERVAL = 1000000; -describe( 'Gutenberg Editor Image Block tests', () => { +describe( 'Gutenberg Editor Image Block tests @canary', () => { let driver; let editorPage; let allPassed = true; diff --git a/__device-tests__/gutenberg-editor-lists.test.js b/__device-tests__/gutenberg-editor-lists.test.js index 5fd4b52e03..b3ab4e7bcf 100644 --- a/__device-tests__/gutenberg-editor-lists.test.js +++ b/__device-tests__/gutenberg-editor-lists.test.js @@ -16,7 +16,7 @@ import testData from './helpers/test-data'; jasmine.DEFAULT_TIMEOUT_INTERVAL = 1000000; -describe( 'Gutenberg Editor tests for List block', () => { +describe( 'Gutenberg Editor tests for List block @canary', () => { let driver; let editorPage; let allPassed = true; From 7c428527fa3110193b012d900d11c5691a211aa3 Mon Sep 17 00:00:00 2001 From: stojdza Date: Thu, 2 Apr 2020 17:27:49 +0200 Subject: [PATCH 04/10] Remove "Canaries" from check names --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 3cb39b1cc8..0c5a58ad38 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -172,6 +172,6 @@ workflows: platform: android check-tests: true - ios-device-checks: - name: Test iOS on Device - Canaries + name: Test iOS on Device - android-device-checks: - name: Test Android on Device - Canaries + name: Test Android on Device From 4ce3f629528341a41cd467c63268410411728f0c Mon Sep 17 00:00:00 2001 From: stojdza Date: Fri, 3 Apr 2020 14:32:16 +0200 Subject: [PATCH 05/10] Revert "Remove "Canaries" from check names" This reverts commit 7c428527fa3110193b012d900d11c5691a211aa3. --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 0c5a58ad38..3cb39b1cc8 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -172,6 +172,6 @@ workflows: platform: android check-tests: true - ios-device-checks: - name: Test iOS on Device + name: Test iOS on Device - Canaries - android-device-checks: - name: Test Android on Device + name: Test Android on Device - Canaries From e36b431171c1daa0e1fad60b2ae931d82634b485 Mon Sep 17 00:00:00 2001 From: stojdza Date: Mon, 6 Apr 2020 18:49:34 +0200 Subject: [PATCH 06/10] Add scheduled full test suite --- .circleci/config.yml | 118 +++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 114 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 3cb39b1cc8..81d2d7a338 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -59,7 +59,7 @@ jobs: JEST_JUNIT_OUTPUT: "reports/test-results/android-test-results.xml" - store_test_results: path: ./reports/test-results - android-device-checks: + android-device-checks-canary: docker: - image: circleci/android:api-29-node steps: @@ -89,7 +89,7 @@ jobs: JEST_JUNIT_OUTPUT: "reports/test-results/android-test-results.xml" - store_test_results: path: ./reports/test-results - ios-device-checks: + ios-device-checks-canary: macos: xcode: "11.2.1" steps: @@ -156,6 +156,103 @@ jobs: key: ios-build-cache-{{ checksum "ios-checksums.txt" }} paths: - ios/build/gutenberg/Build/Products/Release-iphonesimulator/GutenbergDemo.app + android-device-checks-full: + docker: + - image: circleci/android:api-29-node + steps: + - checkout + - run: + name: Checkout Gutenberg + command: git submodule update --init --recursive + - yarn-install + - add-jest-reporter-dir + - run: + name: Set Environment Variables + command: | + echo 'export TEST_RN_PLATFORM=android' >> $BASH_ENV + echo 'export TEST_ENV=sauce' >> $BASH_ENV + - run: + name: Bundle Android and Generate debug .apk file for testing + command: yarn test:e2e:build-app:android + - run: + name: Upload apk to sauce labs + command: | + source bin/sauce-pre-upload.sh + curl -u "$SAUCE_USERNAME:$SAUCE_ACCESS_KEY" -X POST -H "Content-Type: application/octet-stream" https://saucelabs.com/rest/v1/storage/automattic/Gutenberg-$SAUCE_FILENAME.apk?overwrite=true --data-binary @./android/app/build/outputs/apk/debug/app-debug.apk + - run: + name: Run Device Tests + command: yarn device-tests-canary + environment: + JEST_JUNIT_OUTPUT: "reports/test-results/android-test-results.xml" + - store_test_results: + path: ./reports/test-results + ios-device-checks-full: + macos: + xcode: "11.2.1" + steps: + - checkout + - checkout-gutenberg + - yarn-install + - add-jest-reporter-dir + - run: + name: Set Environment Variables + command: | + echo 'export TEST_RN_PLATFORM=ios' >> $BASH_ENV + echo 'export TEST_ENV=sauce' >> $BASH_ENV + - run: + name: Prepare build cache key + command: find yarn.lock ios react-native-aztec/ios react-native-gutenberg-bridge/ios -type f -print0 | sort -z | xargs -0 shasum | tee ios-checksums.txt + - restore_cache: + name: Restore Build Cache + keys: + - ios-build-cache-{{ checksum "ios-checksums.txt" }} + - restore_cache: + name: Restore Dependencies Cache + keys: + - dependencies-v3-{{ checksum "ios/Gemfile.lock" }}-{{ checksum "ios/Podfile.lock" }}-{{ + checksum "yarn.lock" }} + - dependencies-v3-{{ checksum "ios/Gemfile.lock" }}-{{ checksum "ios/Podfile.lock" }} + - dependencies-v3-{{ checksum "ios/Gemfile.lock" }} + - dependencies-v3- + - run: + name: Build (if needed) + command: test -e ios/build/gutenberg/Build/Products/Release-iphonesimulator/GutenbergDemo.app || SKIP_BUNDLING=true yarn test:e2e:build-app:ios + - save_cache: + name: Save Dependencies Cache + key: dependencies-v3-{{ checksum "ios/Gemfile.lock" }}-{{ checksum "ios/Podfile.lock" }}-{{ + checksum "yarn.lock" }} + paths: + - ios/Pods + - ~/Library/Caches/CocoaPods + - ~/.cocoapods/repos/trunk + - ios/vendor + - run: + name: Bundle iOS + command: yarn test:e2e:bundle:ios + - run: + name: Generate .app file for testing + command: WORK_DIR=$(pwd) && cd ./ios/build/gutenberg/Build/Products/Release-iphonesimulator && zip -r $WORK_DIR/ios/GutenbergDemo.app.zip GutenbergDemo.app + - run: + name: Upload .app to sauce labs + command: | + source bin/sauce-pre-upload.sh + curl -u "$SAUCE_USERNAME:$SAUCE_ACCESS_KEY" -X POST -H "Content-Type: application/octet-stream" https://saucelabs.com/rest/v1/storage/automattic/Gutenberg-$SAUCE_FILENAME.app.zip?overwrite=true --data-binary @./ios/GutenbergDemo.app.zip + - run: + name: Run Device Tests + command: | + yarn device-tests-canary + environment: + JEST_JUNIT_OUTPUT: "reports/test-results/ios-test-results.xml" + - store_test_results: + path: ./reports/test-results + - run: + name: Prepare build cache + command: rm ios/build/gutenberg/Build/Products/Release-iphonesimulator/GutenbergDemo.app/main.jsbundle + - save_cache: + name: Save Build Cache + key: ios-build-cache-{{ checksum "ios-checksums.txt" }} + paths: + - ios/build/gutenberg/Build/Products/Release-iphonesimulator/GutenbergDemo.app workflows: gutenberg-mobile: @@ -171,7 +268,20 @@ workflows: name: Test Android platform: android check-tests: true - - ios-device-checks: + - ios-device-checks-canary: name: Test iOS on Device - Canaries - - android-device-checks: + - android-device-checks-canary: name: Test Android on Device - Canaries + + ui-tests-full-scheduled: + jobs: + - ios-device-checks-full: + name: Test iOS on Device - Scheduled + - android-device-checks-full: + name: Test iOS on Device - Scheduled + triggers: + - schedule: + cron: '1 1,13 * * *' + filters: + branches: + only: develop From 2325290b4a60df642d1c3db0c83e30ac791eebcc Mon Sep 17 00:00:00 2001 From: stojdza Date: Mon, 6 Apr 2020 19:27:39 +0200 Subject: [PATCH 07/10] fix indentations --- .circleci/config.yml | 194 +++++++++++++++++++++---------------------- 1 file changed, 97 insertions(+), 97 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 81d2d7a338..f27475dbc5 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -156,103 +156,103 @@ jobs: key: ios-build-cache-{{ checksum "ios-checksums.txt" }} paths: - ios/build/gutenberg/Build/Products/Release-iphonesimulator/GutenbergDemo.app - android-device-checks-full: - docker: - - image: circleci/android:api-29-node - steps: - - checkout - - run: - name: Checkout Gutenberg - command: git submodule update --init --recursive - - yarn-install - - add-jest-reporter-dir - - run: - name: Set Environment Variables - command: | - echo 'export TEST_RN_PLATFORM=android' >> $BASH_ENV - echo 'export TEST_ENV=sauce' >> $BASH_ENV - - run: - name: Bundle Android and Generate debug .apk file for testing - command: yarn test:e2e:build-app:android - - run: - name: Upload apk to sauce labs - command: | - source bin/sauce-pre-upload.sh - curl -u "$SAUCE_USERNAME:$SAUCE_ACCESS_KEY" -X POST -H "Content-Type: application/octet-stream" https://saucelabs.com/rest/v1/storage/automattic/Gutenberg-$SAUCE_FILENAME.apk?overwrite=true --data-binary @./android/app/build/outputs/apk/debug/app-debug.apk - - run: - name: Run Device Tests - command: yarn device-tests-canary - environment: - JEST_JUNIT_OUTPUT: "reports/test-results/android-test-results.xml" - - store_test_results: - path: ./reports/test-results - ios-device-checks-full: - macos: - xcode: "11.2.1" - steps: - - checkout - - checkout-gutenberg - - yarn-install - - add-jest-reporter-dir - - run: - name: Set Environment Variables - command: | - echo 'export TEST_RN_PLATFORM=ios' >> $BASH_ENV - echo 'export TEST_ENV=sauce' >> $BASH_ENV - - run: - name: Prepare build cache key - command: find yarn.lock ios react-native-aztec/ios react-native-gutenberg-bridge/ios -type f -print0 | sort -z | xargs -0 shasum | tee ios-checksums.txt - - restore_cache: - name: Restore Build Cache - keys: - - ios-build-cache-{{ checksum "ios-checksums.txt" }} - - restore_cache: - name: Restore Dependencies Cache - keys: - - dependencies-v3-{{ checksum "ios/Gemfile.lock" }}-{{ checksum "ios/Podfile.lock" }}-{{ - checksum "yarn.lock" }} - - dependencies-v3-{{ checksum "ios/Gemfile.lock" }}-{{ checksum "ios/Podfile.lock" }} - - dependencies-v3-{{ checksum "ios/Gemfile.lock" }} - - dependencies-v3- - - run: - name: Build (if needed) - command: test -e ios/build/gutenberg/Build/Products/Release-iphonesimulator/GutenbergDemo.app || SKIP_BUNDLING=true yarn test:e2e:build-app:ios - - save_cache: - name: Save Dependencies Cache - key: dependencies-v3-{{ checksum "ios/Gemfile.lock" }}-{{ checksum "ios/Podfile.lock" }}-{{ - checksum "yarn.lock" }} - paths: - - ios/Pods - - ~/Library/Caches/CocoaPods - - ~/.cocoapods/repos/trunk - - ios/vendor - - run: - name: Bundle iOS - command: yarn test:e2e:bundle:ios - - run: - name: Generate .app file for testing - command: WORK_DIR=$(pwd) && cd ./ios/build/gutenberg/Build/Products/Release-iphonesimulator && zip -r $WORK_DIR/ios/GutenbergDemo.app.zip GutenbergDemo.app - - run: - name: Upload .app to sauce labs - command: | - source bin/sauce-pre-upload.sh - curl -u "$SAUCE_USERNAME:$SAUCE_ACCESS_KEY" -X POST -H "Content-Type: application/octet-stream" https://saucelabs.com/rest/v1/storage/automattic/Gutenberg-$SAUCE_FILENAME.app.zip?overwrite=true --data-binary @./ios/GutenbergDemo.app.zip - - run: - name: Run Device Tests - command: | - yarn device-tests-canary - environment: - JEST_JUNIT_OUTPUT: "reports/test-results/ios-test-results.xml" - - store_test_results: - path: ./reports/test-results - - run: - name: Prepare build cache - command: rm ios/build/gutenberg/Build/Products/Release-iphonesimulator/GutenbergDemo.app/main.jsbundle - - save_cache: - name: Save Build Cache - key: ios-build-cache-{{ checksum "ios-checksums.txt" }} - paths: - - ios/build/gutenberg/Build/Products/Release-iphonesimulator/GutenbergDemo.app + android-device-checks-full: + docker: + - image: circleci/android:api-29-node + steps: + - checkout + - run: + name: Checkout Gutenberg + command: git submodule update --init --recursive + - yarn-install + - add-jest-reporter-dir + - run: + name: Set Environment Variables + command: | + echo 'export TEST_RN_PLATFORM=android' >> $BASH_ENV + echo 'export TEST_ENV=sauce' >> $BASH_ENV + - run: + name: Bundle Android and Generate debug .apk file for testing + command: yarn test:e2e:build-app:android + - run: + name: Upload apk to sauce labs + command: | + source bin/sauce-pre-upload.sh + curl -u "$SAUCE_USERNAME:$SAUCE_ACCESS_KEY" -X POST -H "Content-Type: application/octet-stream" https://saucelabs.com/rest/v1/storage/automattic/Gutenberg-$SAUCE_FILENAME.apk?overwrite=true --data-binary @./android/app/build/outputs/apk/debug/app-debug.apk + - run: + name: Run Device Tests + command: yarn device-tests-canary + environment: + JEST_JUNIT_OUTPUT: "reports/test-results/android-test-results.xml" + - store_test_results: + path: ./reports/test-results + ios-device-checks-full: + macos: + xcode: "11.2.1" + steps: + - checkout + - checkout-gutenberg + - yarn-install + - add-jest-reporter-dir + - run: + name: Set Environment Variables + command: | + echo 'export TEST_RN_PLATFORM=ios' >> $BASH_ENV + echo 'export TEST_ENV=sauce' >> $BASH_ENV + - run: + name: Prepare build cache key + command: find yarn.lock ios react-native-aztec/ios react-native-gutenberg-bridge/ios -type f -print0 | sort -z | xargs -0 shasum | tee ios-checksums.txt + - restore_cache: + name: Restore Build Cache + keys: + - ios-build-cache-{{ checksum "ios-checksums.txt" }} + - restore_cache: + name: Restore Dependencies Cache + keys: + - dependencies-v3-{{ checksum "ios/Gemfile.lock" }}-{{ checksum "ios/Podfile.lock" }}-{{ + checksum "yarn.lock" }} + - dependencies-v3-{{ checksum "ios/Gemfile.lock" }}-{{ checksum "ios/Podfile.lock" }} + - dependencies-v3-{{ checksum "ios/Gemfile.lock" }} + - dependencies-v3- + - run: + name: Build (if needed) + command: test -e ios/build/gutenberg/Build/Products/Release-iphonesimulator/GutenbergDemo.app || SKIP_BUNDLING=true yarn test:e2e:build-app:ios + - save_cache: + name: Save Dependencies Cache + key: dependencies-v3-{{ checksum "ios/Gemfile.lock" }}-{{ checksum "ios/Podfile.lock" }}-{{ + checksum "yarn.lock" }} + paths: + - ios/Pods + - ~/Library/Caches/CocoaPods + - ~/.cocoapods/repos/trunk + - ios/vendor + - run: + name: Bundle iOS + command: yarn test:e2e:bundle:ios + - run: + name: Generate .app file for testing + command: WORK_DIR=$(pwd) && cd ./ios/build/gutenberg/Build/Products/Release-iphonesimulator && zip -r $WORK_DIR/ios/GutenbergDemo.app.zip GutenbergDemo.app + - run: + name: Upload .app to sauce labs + command: | + source bin/sauce-pre-upload.sh + curl -u "$SAUCE_USERNAME:$SAUCE_ACCESS_KEY" -X POST -H "Content-Type: application/octet-stream" https://saucelabs.com/rest/v1/storage/automattic/Gutenberg-$SAUCE_FILENAME.app.zip?overwrite=true --data-binary @./ios/GutenbergDemo.app.zip + - run: + name: Run Device Tests + command: | + yarn device-tests-canary + environment: + JEST_JUNIT_OUTPUT: "reports/test-results/ios-test-results.xml" + - store_test_results: + path: ./reports/test-results + - run: + name: Prepare build cache + command: rm ios/build/gutenberg/Build/Products/Release-iphonesimulator/GutenbergDemo.app/main.jsbundle + - save_cache: + name: Save Build Cache + key: ios-build-cache-{{ checksum "ios-checksums.txt" }} + paths: + - ios/build/gutenberg/Build/Products/Release-iphonesimulator/GutenbergDemo.app workflows: gutenberg-mobile: From cbe9c8425978801ebeb3346eaab01b1ca85e4a23 Mon Sep 17 00:00:00 2001 From: stojdza Date: Mon, 6 Apr 2020 19:40:49 +0200 Subject: [PATCH 08/10] =?UTF-8?q?=C2=A7Remove=20changes=20from=20Canary=20?= =?UTF-8?q?branch?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .circleci/config.yml | 117 ++---------------- .../gutenberg-editor-heading.test.js | 2 +- .../gutenberg-editor-image.test.js | 2 +- .../gutenberg-editor-lists.test.js | 2 +- package.json | 1 - 5 files changed, 13 insertions(+), 111 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index f27475dbc5..b0537c5a14 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -59,7 +59,7 @@ jobs: JEST_JUNIT_OUTPUT: "reports/test-results/android-test-results.xml" - store_test_results: path: ./reports/test-results - android-device-checks-canary: + android-device-checks: docker: - image: circleci/android:api-29-node steps: @@ -84,12 +84,12 @@ jobs: curl -u "$SAUCE_USERNAME:$SAUCE_ACCESS_KEY" -X POST -H "Content-Type: application/octet-stream" https://saucelabs.com/rest/v1/storage/automattic/Gutenberg-$SAUCE_FILENAME.apk?overwrite=true --data-binary @./android/app/build/outputs/apk/debug/app-debug.apk - run: name: Run Device Tests - command: yarn device-tests-canary + command: yarn device-tests environment: JEST_JUNIT_OUTPUT: "reports/test-results/android-test-results.xml" - store_test_results: path: ./reports/test-results - ios-device-checks-canary: + ios-device-checks: macos: xcode: "11.2.1" steps: @@ -143,7 +143,7 @@ jobs: - run: name: Run Device Tests command: | - yarn device-tests-canary + yarn device-tests environment: JEST_JUNIT_OUTPUT: "reports/test-results/ios-test-results.xml" - store_test_results: @@ -156,103 +156,6 @@ jobs: key: ios-build-cache-{{ checksum "ios-checksums.txt" }} paths: - ios/build/gutenberg/Build/Products/Release-iphonesimulator/GutenbergDemo.app - android-device-checks-full: - docker: - - image: circleci/android:api-29-node - steps: - - checkout - - run: - name: Checkout Gutenberg - command: git submodule update --init --recursive - - yarn-install - - add-jest-reporter-dir - - run: - name: Set Environment Variables - command: | - echo 'export TEST_RN_PLATFORM=android' >> $BASH_ENV - echo 'export TEST_ENV=sauce' >> $BASH_ENV - - run: - name: Bundle Android and Generate debug .apk file for testing - command: yarn test:e2e:build-app:android - - run: - name: Upload apk to sauce labs - command: | - source bin/sauce-pre-upload.sh - curl -u "$SAUCE_USERNAME:$SAUCE_ACCESS_KEY" -X POST -H "Content-Type: application/octet-stream" https://saucelabs.com/rest/v1/storage/automattic/Gutenberg-$SAUCE_FILENAME.apk?overwrite=true --data-binary @./android/app/build/outputs/apk/debug/app-debug.apk - - run: - name: Run Device Tests - command: yarn device-tests-canary - environment: - JEST_JUNIT_OUTPUT: "reports/test-results/android-test-results.xml" - - store_test_results: - path: ./reports/test-results - ios-device-checks-full: - macos: - xcode: "11.2.1" - steps: - - checkout - - checkout-gutenberg - - yarn-install - - add-jest-reporter-dir - - run: - name: Set Environment Variables - command: | - echo 'export TEST_RN_PLATFORM=ios' >> $BASH_ENV - echo 'export TEST_ENV=sauce' >> $BASH_ENV - - run: - name: Prepare build cache key - command: find yarn.lock ios react-native-aztec/ios react-native-gutenberg-bridge/ios -type f -print0 | sort -z | xargs -0 shasum | tee ios-checksums.txt - - restore_cache: - name: Restore Build Cache - keys: - - ios-build-cache-{{ checksum "ios-checksums.txt" }} - - restore_cache: - name: Restore Dependencies Cache - keys: - - dependencies-v3-{{ checksum "ios/Gemfile.lock" }}-{{ checksum "ios/Podfile.lock" }}-{{ - checksum "yarn.lock" }} - - dependencies-v3-{{ checksum "ios/Gemfile.lock" }}-{{ checksum "ios/Podfile.lock" }} - - dependencies-v3-{{ checksum "ios/Gemfile.lock" }} - - dependencies-v3- - - run: - name: Build (if needed) - command: test -e ios/build/gutenberg/Build/Products/Release-iphonesimulator/GutenbergDemo.app || SKIP_BUNDLING=true yarn test:e2e:build-app:ios - - save_cache: - name: Save Dependencies Cache - key: dependencies-v3-{{ checksum "ios/Gemfile.lock" }}-{{ checksum "ios/Podfile.lock" }}-{{ - checksum "yarn.lock" }} - paths: - - ios/Pods - - ~/Library/Caches/CocoaPods - - ~/.cocoapods/repos/trunk - - ios/vendor - - run: - name: Bundle iOS - command: yarn test:e2e:bundle:ios - - run: - name: Generate .app file for testing - command: WORK_DIR=$(pwd) && cd ./ios/build/gutenberg/Build/Products/Release-iphonesimulator && zip -r $WORK_DIR/ios/GutenbergDemo.app.zip GutenbergDemo.app - - run: - name: Upload .app to sauce labs - command: | - source bin/sauce-pre-upload.sh - curl -u "$SAUCE_USERNAME:$SAUCE_ACCESS_KEY" -X POST -H "Content-Type: application/octet-stream" https://saucelabs.com/rest/v1/storage/automattic/Gutenberg-$SAUCE_FILENAME.app.zip?overwrite=true --data-binary @./ios/GutenbergDemo.app.zip - - run: - name: Run Device Tests - command: | - yarn device-tests-canary - environment: - JEST_JUNIT_OUTPUT: "reports/test-results/ios-test-results.xml" - - store_test_results: - path: ./reports/test-results - - run: - name: Prepare build cache - command: rm ios/build/gutenberg/Build/Products/Release-iphonesimulator/GutenbergDemo.app/main.jsbundle - - save_cache: - name: Save Build Cache - key: ios-build-cache-{{ checksum "ios-checksums.txt" }} - paths: - - ios/build/gutenberg/Build/Products/Release-iphonesimulator/GutenbergDemo.app workflows: gutenberg-mobile: @@ -268,16 +171,16 @@ workflows: name: Test Android platform: android check-tests: true - - ios-device-checks-canary: - name: Test iOS on Device - Canaries - - android-device-checks-canary: - name: Test Android on Device - Canaries + - ios-device-checks: + name: Test iOS on Device + - android-device-checks: + name: Test Android on Device ui-tests-full-scheduled: jobs: - - ios-device-checks-full: + - ios-device-checks: name: Test iOS on Device - Scheduled - - android-device-checks-full: + - android-device-checks: name: Test iOS on Device - Scheduled triggers: - schedule: diff --git a/__device-tests__/gutenberg-editor-heading.test.js b/__device-tests__/gutenberg-editor-heading.test.js index c9a5d4f5c9..c3d62168c4 100644 --- a/__device-tests__/gutenberg-editor-heading.test.js +++ b/__device-tests__/gutenberg-editor-heading.test.js @@ -16,7 +16,7 @@ import testData from './helpers/test-data'; jasmine.DEFAULT_TIMEOUT_INTERVAL = 1000000; -describe( 'Gutenberg Editor tests @canary', () => { +describe( 'Gutenberg Editor tests', () => { let driver; let editorPage; let allPassed = true; diff --git a/__device-tests__/gutenberg-editor-image.test.js b/__device-tests__/gutenberg-editor-image.test.js index 98b9784604..e504807156 100644 --- a/__device-tests__/gutenberg-editor-image.test.js +++ b/__device-tests__/gutenberg-editor-image.test.js @@ -18,7 +18,7 @@ import testData from './helpers/test-data'; jasmine.DEFAULT_TIMEOUT_INTERVAL = 1000000; -describe( 'Gutenberg Editor Image Block tests @canary', () => { +describe( 'Gutenberg Editor Image Block tests', () => { let driver; let editorPage; let allPassed = true; diff --git a/__device-tests__/gutenberg-editor-lists.test.js b/__device-tests__/gutenberg-editor-lists.test.js index b3ab4e7bcf..5fd4b52e03 100644 --- a/__device-tests__/gutenberg-editor-lists.test.js +++ b/__device-tests__/gutenberg-editor-lists.test.js @@ -16,7 +16,7 @@ import testData from './helpers/test-data'; jasmine.DEFAULT_TIMEOUT_INTERVAL = 1000000; -describe( 'Gutenberg Editor tests for List block @canary', () => { +describe( 'Gutenberg Editor tests for List block', () => { let driver; let editorPage; let allPassed = true; diff --git a/package.json b/package.json index f4f0bd003e..7a935d3085 100644 --- a/package.json +++ b/package.json @@ -102,7 +102,6 @@ "test:inside-gb": "cross-env NODE_ENV=test jest --verbose --config jest_gb.config.js", "test:debug": "cross-env NODE_ENV=test node --inspect-brk node_modules/.bin/jest --runInBand --verbose --config jest.config.js", "device-tests": "cross-env NODE_ENV=test jest --no-cache --maxWorkers=3 --reporters=default --reporters=jest-junit --verbose --config jest_ui.config.js", - "device-tests-canary": "cross-env NODE_ENV=test jest --no-cache --maxWorkers=2 --testNamePattern=@canary --reporters=default --reporters=jest-junit --verbose --config jest_ui.config.js", "device-tests:local": "cross-env NODE_ENV=test jest --runInBand --reporters=default --reporters=jest-junit --detectOpenHandles --verbose --config jest_ui.config.js", "device-tests:debug": "cross-env NODE_ENV=test node $NODE_DEBUG_OPTION --inspect-brk node_modules/jest/bin/jest --runInBand --reporters=default --reporters=jest-junit --detectOpenHandles --verbose --config jest_ui.config.js", "test:e2e": "yarn test:e2e:android && yarn test:e2e:ios", From f298e997ce1eaf1d7e19e0e151bfc10b5a84edd2 Mon Sep 17 00:00:00 2001 From: stojdza Date: Wed, 8 Apr 2020 11:19:20 +0200 Subject: [PATCH 09/10] Fix obvious copy/paste mistake --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b0537c5a14..42dcf04f27 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -181,7 +181,7 @@ workflows: - ios-device-checks: name: Test iOS on Device - Scheduled - android-device-checks: - name: Test iOS on Device - Scheduled + name: Test Android on Device - Scheduled triggers: - schedule: cron: '1 1,13 * * *' From c387ae16896cc5ce87f3dcf76b541843cd7edc1d Mon Sep 17 00:00:00 2001 From: rachelmcr Date: Fri, 17 Apr 2020 13:27:36 +0100 Subject: [PATCH 10/10] Add Slack failure message for scheduled runs --- .circleci/config.yml | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 42dcf04f27..1a8e1bd5c9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,5 +1,8 @@ version: 2.1 +orbs: + slack: circleci/slack@3.4.2 + commands: yarn-install: steps: @@ -60,6 +63,11 @@ jobs: - store_test_results: path: ./reports/test-results android-device-checks: + parameters: + post-to-slack: + description: Post to Slack when tests fail. SLACK_WEBHOOK ENV variable must be set. + type: boolean + default: false docker: - image: circleci/android:api-29-node steps: @@ -89,7 +97,20 @@ jobs: JEST_JUNIT_OUTPUT: "reports/test-results/android-test-results.xml" - store_test_results: path: ./reports/test-results + - when: + condition: << parameters.post-to-slack >> + steps: + - slack/status: + fail_only: true + include_job_number_field: false + include_project_field: false + failure_message: ':red_circle: Scheduled tests failed on Android device!' ios-device-checks: + parameters: + post-to-slack: + description: Post to Slack when tests fail. SLACK_WEBHOOK ENV variable must be set. + type: boolean + default: false macos: xcode: "11.2.1" steps: @@ -156,6 +177,14 @@ jobs: key: ios-build-cache-{{ checksum "ios-checksums.txt" }} paths: - ios/build/gutenberg/Build/Products/Release-iphonesimulator/GutenbergDemo.app + - when: + condition: << parameters.post-to-slack >> + steps: + - slack/status: + fail_only: true + include_job_number_field: false + include_project_field: false + failure_message: ':red_circle: Scheduled tests failed on iOS device!' workflows: gutenberg-mobile: @@ -180,8 +209,10 @@ workflows: jobs: - ios-device-checks: name: Test iOS on Device - Scheduled + post-to-slack: true - android-device-checks: name: Test Android on Device - Scheduled + post-to-slack: true triggers: - schedule: cron: '1 1,13 * * *'