From 2afe844abdb19742cb6f25b5b7dbcf2eb8b558c1 Mon Sep 17 00:00:00 2001 From: Gleb Bahmutov Date: Thu, 18 Feb 2021 07:49:25 -0500 Subject: [PATCH 01/20] feat: rework test phases to make onSuccess default BREAKING CHANGE: now the onSuccess step is the default testing step, running the Cypress tests against the preview or production deploy. Another change is the default browser is Chromium instead of Electron. --- README.md | 13 ++++++++++--- manifest.yml | 42 ++++++++++++++++++++++++++++++------------ 2 files changed, 40 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 003442e3..8b100a04 100644 --- a/README.md +++ b/README.md @@ -177,9 +177,9 @@ package = "netlify-plugin-cypress" See [cypress-example-kitchensink](https://github.com/cypress-io/cypress-example-kitchensink) for instance. -### Chromium +### browser -By default all tests run using built-in Electron browser. If you want to use Chromium: +By default all tests run using Chromium browser. If you want to use Electron: ```toml [build] @@ -195,7 +195,8 @@ publish = "build" [[plugins]] package = "netlify-plugin-cypress" [plugins.inputs] - browser = "chromium" + # allowed values: electron, chromium + browser = "electron" ``` ### testing SPA routes @@ -294,6 +295,12 @@ Name | Description [bahmutov/eleventyone](https://github.com/bahmutov/eleventyone) | Example used in [Test Sites Deployed To Netlify Using netlify-plugin-cypress](https://glebbahmutov.com/blog/test-netlify/) tutorial [gatsby-starter-portfolio-cara](https://github.com/bahmutov/gatsby-starter-portfolio-cara) | A Gatsby site example +## Major upgrades + +### v1 to v2 + +- The default browser has been switched to Chromium. If you want to use the built-in Electron use an explicit option [browser](#browser) + ## Debugging Set environment variable `DEBUG=netlify-plugin-cypress` to see the debug logs. To see even more information, set `DEBUG=netlify-plugin-cypress,netlify-plugin-cypress:verbose` diff --git a/manifest.yml b/manifest.yml index f462bf50..2e3e0613 100644 --- a/manifest.yml +++ b/manifest.yml @@ -1,26 +1,44 @@ name: netlify-plugin-cypress inputs: - # these settings apply during postBuild step - # when we are testing the site served from the distribution folder + # these settings apply during onSuccess step + - name: enable + description: Run tests against the preview or production deploy + default: true + - name: record + description: Record test results to Cypress Dashboard default: false + - name: spec + description: | + Run just the given spec or spec pattern, + equivalent to "cypress run --spec ..." + - name: group + description: | + If recording to Cypress Dashboard, + pass the group name with "cypress run --record --group ..." + - name: tag - - name: spa - # by default run the tests - - name: skip - default: false - # by default the tests run in Electron - # but because of the dependency we download Chromium - # so you can set "browser = electron" + description: | + If recording to Cypress Dashboard, + pass the tag with "cypress run --record --tag ..." + + # Cypress comes with built-in Electron browser + # and this NPM package installs Chromium browser - name: browser - default: electron + description: Allowed values are chromium, electron + default: chromium # tells the plugin how to start the server using custom command # and waiting for an url, record to the dashboard, tag, etc # see README "testing the site before build" - name: preBuild + description: Run tests before building the site - # you can control how the plugin runs the tests after deploy - - name: onSuccess + # tells the plugin to start a static server during postBuild + # and test just the built static site. + # see README "testing the site after build" + # NOTE: does not execute Netlify API redirects or functions + - name: postBuild + description: Run tests against the built static site From 3b1fd8267a3b9c66511bb6f5832847d67ae97211 Mon Sep 17 00:00:00 2001 From: Gleb Bahmutov Date: Tue, 2 Mar 2021 13:48:49 -0500 Subject: [PATCH 02/20] update code to make the routing test pass --- README.md | 6 +++ manifest.yml | 15 +++---- package-lock.json | 13 ++++-- package.json | 3 +- src/index.js | 82 +++++++++++++++++++------------------- tests/routing/README.md | 7 +++- tests/routing/netlify.toml | 11 ++++- 7 files changed, 83 insertions(+), 54 deletions(-) diff --git a/README.md b/README.md index 8b100a04..e5f87bbd 100644 --- a/README.md +++ b/README.md @@ -339,6 +339,12 @@ Set environment variable `DEBUG=netlify-plugin-cypress` to see the debug logs. T Switch to using Chromium browser that seems to be a bit more reliable. Use browser = "chromium" setting. +## Changelog + +### v1 to v2 + +- We have changed the default testing phase. In v1 the tests executed after building the site by default. In v2 the tests run against the deployed URL by default, and you need to enable the testing during `preBuild` or `postBuild` steps. + ## License This project is licensed under the terms of the [MIT license](LICENSE.md). diff --git a/manifest.yml b/manifest.yml index 2e3e0613..7bb50678 100644 --- a/manifest.yml +++ b/manifest.yml @@ -1,10 +1,17 @@ name: netlify-plugin-cypress inputs: - # these settings apply during onSuccess step + # by default the Cypress tests run against the deployed URL + # and these settings apply during the "onSuccess" step - name: enable description: Run tests against the preview or production deploy default: true + # Cypress comes with built-in Electron browser + # and this NPM package installs Chromium browser + - name: browser + description: Allowed values are chromium, electron + default: chromium + - name: record description: Record test results to Cypress Dashboard default: false @@ -24,12 +31,6 @@ inputs: If recording to Cypress Dashboard, pass the tag with "cypress run --record --tag ..." - # Cypress comes with built-in Electron browser - # and this NPM package installs Chromium browser - - name: browser - description: Allowed values are chromium, electron - default: chromium - # tells the plugin how to start the server using custom command # and waiting for an url, record to the dashboard, tag, etc # see README "testing the site before build" diff --git a/package-lock.json b/package-lock.json index 1ec73e69..75b69aac 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10750,6 +10750,12 @@ "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", "dev": true }, + "ramda": { + "version": "0.26.1", + "resolved": "https://registry.npmjs.org/ramda/-/ramda-0.26.1.tgz", + "integrity": "sha512-hLWjpy7EnsDBb0p+Z3B7rPi3GDeRG5ZtiI33kJhTt+ORCd38AbAIjB/9zRIUoeTbE/AVX5ZkU7m6bznsvrf8eQ==", + "dev": true + }, "shebang-command": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", @@ -27076,10 +27082,9 @@ } }, "ramda": { - "version": "0.26.1", - "resolved": "https://registry.npmjs.org/ramda/-/ramda-0.26.1.tgz", - "integrity": "sha512-hLWjpy7EnsDBb0p+Z3B7rPi3GDeRG5ZtiI33kJhTt+ORCd38AbAIjB/9zRIUoeTbE/AVX5ZkU7m6bznsvrf8eQ==", - "dev": true + "version": "0.27.1", + "resolved": "https://registry.npmjs.org/ramda/-/ramda-0.27.1.tgz", + "integrity": "sha512-PgIdVpn5y5Yns8vqb8FzBUEYn98V3xcPgawAkkgj0YJ0qDsnHCiNmZYfOGMgOvoB0eWFLpYbhxUR3mxfDIMvpw==" }, "random-bytes": { "version": "1.0.0", diff --git a/package.json b/package.json index c3ec3d7f..56411b55 100644 --- a/package.json +++ b/package.json @@ -24,7 +24,8 @@ "debug": "4.1.1", "got": "10.7.0", "local-web-server": "^4.2.1", - "puppeteer": "^7.0.1" + "puppeteer": "^7.0.1", + "ramda": "0.27.1" }, "repository": { "type": "git", diff --git a/src/index.js b/src/index.js index dfb5efdb..ba9957cf 100644 --- a/src/index.js +++ b/src/index.js @@ -1,5 +1,6 @@ // @ts-check const { stripIndent } = require('common-tags') +const R = require('ramda') const debug = require('debug')('netlify-plugin-cypress') const debugVerbose = require('debug')('netlify-plugin-cypress:verbose') const { ping, getBrowserPath, serveFolder } = require('./utils') @@ -278,21 +279,26 @@ const hasRecordKey = () => typeof process.env.CYPRESS_RECORD_KEY === 'string' module.exports = { onPreBuild: async (arg) => { + // we need to install everything to be ready await install(arg) await cypressVerify(arg) await cypressInfo(arg) - debug('cypress plugin preBuild inputs %o', arg.inputs) - const preBuildInputs = arg.inputs && arg.inputs.preBuild - if (!preBuildInputs) { - debug('there are no preBuild inputs') + const { inputs, utils } = arg + + const preBuildInputs = inputs.preBuild || {} + debug('preBuild inputs %o', preBuildInputs) + + const enablePreBuildTests = Boolean(preBuildInputs.enable) + if (!enablePreBuildTests) { + debug('Skipping preBuild tests') return } - const browser = arg.inputs.browser || DEFAULT_BROWSER + const browser = preBuildInputs.browser || DEFAULT_BROWSER - const closeServer = startServerMaybe(arg.utils.run, preBuildInputs) - await waitOnMaybe(arg.utils.build, preBuildInputs) + const closeServer = startServerMaybe(utils.run, preBuildInputs) + await waitOnMaybe(utils.build, preBuildInputs) const baseUrl = preBuildInputs['wait-on'] const record = hasRecordKey() && Boolean(preBuildInputs.record) @@ -323,47 +329,49 @@ module.exports = { closeServer() } - const errorCallback = arg.utils.build.failBuild.bind(arg.utils.build) - const summaryCallback = arg.utils.status.show.bind(arg.utils.status) + const errorCallback = utils.build.failBuild.bind(utils.build) + const summaryCallback = utils.status.show.bind(utils.status) processCypressResults(results, errorCallback, summaryCallback) }, - onPostBuild: async (arg) => { - debugVerbose('postBuild arg %o', arg) - debug('cypress plugin postBuild inputs %o', arg.inputs) + onPostBuild: async ({ inputs, constants, utils }) => { + debugVerbose('===postBuild===') - const skipTests = Boolean(arg.inputs.skip) - if (skipTests) { - console.log('Skipping tests because skip=true') + const postBuildInputs = inputs.postBuild || {} + debug('cypress plugin postBuild inputs %o', postBuildInputs) + + const enablePostBuildTests = Boolean(postBuildInputs.enable) + if (!enablePostBuildTests) { + debug('Skipping postBuild tests') return } - const fullPublishFolder = arg.constants.PUBLISH_DIR + const fullPublishFolder = constants.PUBLISH_DIR debug('folder to publish is "%s"', fullPublishFolder) - const browser = arg.inputs.browser || DEFAULT_BROWSER + const browser = postBuildInputs.browser || DEFAULT_BROWSER // only if the user wants to record the tests and has set the record key // then we should attempt recording - const record = hasRecordKey() && Boolean(arg.inputs.record) + const record = hasRecordKey() && Boolean(postBuildInputs.record) - const spec = arg.inputs.spec + const spec = postBuildInputs.spec let group let tag if (record) { - group = arg.inputs.group || 'postBuild' + group = postBuildInputs.group || 'postBuild' - if (arg.inputs.tag) { - tag = arg.inputs.tag + if (postBuildInputs.tag) { + tag = postBuildInputs.tag } else { tag = process.env.CONTEXT } } - const spa = arg.inputs.spa + const spa = postBuildInputs.spa - const errorCallback = arg.utils.build.failBuild.bind(arg.utils.build) - const summaryCallback = arg.utils.status.show.bind(arg.utils.status) + const errorCallback = utils.build.failBuild.bind(utils.build) + const summaryCallback = utils.status.show.bind(utils.status) await postBuild({ fullPublishFolder, @@ -382,11 +390,12 @@ module.exports = { * Executes after successful Netlify deployment. * @param {any} arg */ - onSuccess: async (arg) => { - debugVerbose('onSuccess arg %o', arg) + onSuccess: async ({ utils, inputs, constants }) => { + debugVerbose('onSuccess arg %o', { utils, inputs, constants }) - const { utils, inputs, constants } = arg - debug('onSuccess inputs %o', inputs) + // extract test run parameters + const onSuccessInputs = R.omit(['preBuild', 'postBuild'], inputs || {}) + debug('onSuccess inputs %o', onSuccessInputs) const isLocal = constants.IS_LOCAL const siteName = process.env.SITE_NAME @@ -397,16 +406,9 @@ module.exports = { isLocal, }) - // extract test run parameters - const onSuccessInputs = inputs.onSuccess - if (!onSuccessInputs) { - debug('no onSuccess inputs, skipping testing the deployed url') - return - } - - const enableTests = Boolean(onSuccessInputs.enable) - if (!enableTests) { - console.log('Skipping tests because enable=false') + const enableOnSuccessTests = Boolean(onSuccessInputs.enable) + if (!enableOnSuccessTests) { + debug('Skipping onSuccess tests') return } @@ -419,7 +421,7 @@ module.exports = { return errorCallback('Missing DEPLOY_PRIME_URL') } - const browser = arg.inputs.browser || DEFAULT_BROWSER + const browser = onSuccessInputs.browser || DEFAULT_BROWSER // only if the user wants to record the tests and has set the record key // then we should attempt recording diff --git a/tests/routing/README.md b/tests/routing/README.md index 673ff714..491da7fa 100644 --- a/tests/routing/README.md +++ b/tests/routing/README.md @@ -5,6 +5,11 @@ This example tests the redirect fallback that makes client-side routing defined ![Test screenshot](images/routing.png) +These tests run: +- [x] before the build +- [x] after the build +- [ ] on deploy success + ## Local build -Use command `npm run netlify:build` to try the plugin locally +Use command `DEBUG=netlify-plugin-cypress npm run netlify:build` to try the plugin locally diff --git a/tests/routing/netlify.toml b/tests/routing/netlify.toml index 7916d428..d997eb7d 100644 --- a/tests/routing/netlify.toml +++ b/tests/routing/netlify.toml @@ -8,6 +8,8 @@ publish = "build" CYPRESS_CACHE_FOLDER = "./node_modules/CypressBinary" # set TERM variable for terminal output TERM = "xterm" +# do not print too many progress messages +CI = "1" [[plugins]] # local Cypress plugin will test our site after it is built @@ -17,15 +19,22 @@ TERM = "xterm" # run Cypress tests once on the site before it is built # and then after building the static folder + [plugins.inputs] + # these settings apply to the default step onSuccess + # in this case, we do not want to run any tests after deploy + enable = false + # let's run tests against development server # before building it (and testing the built site) [plugins.inputs.preBuild] + enable = true start = 'npm start' wait-on = 'http://localhost:3000' wait-on-timeout = '45' # seconds # and test the built site after - [plugins.inputs] + [plugins.inputs.postBuild] + enable = true # must allow our test server to redirect unknown routes to "/" # so that client-side routing can correctly route them # can be set to true or "index.html" (or similar fallback filename in the built folder) From fdaf721186f4620811d8c77f37ec791c770b1eae Mon Sep 17 00:00:00 2001 From: Gleb Bahmutov Date: Tue, 2 Mar 2021 14:02:11 -0500 Subject: [PATCH 03/20] set up pre-release channel --- package.json | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/package.json b/package.json index 56411b55..6c8d6e36 100644 --- a/package.json +++ b/package.json @@ -50,5 +50,14 @@ "react-scripts": "3.4.1", "semantic-release": "^17.0.4", "serve": "11.3.0" + }, + "release": { + "branches": [ + "main", + { + "name": "prepare-v2", + "prerelease": true + } + ] } } From 99978734dcede5c9a70112f3ecf20ba2fc449430 Mon Sep 17 00:00:00 2001 From: Gleb Bahmutov Date: Tue, 2 Mar 2021 14:07:01 -0500 Subject: [PATCH 04/20] update test-prebuild --- tests/test-prebuild-only/README.md | 10 ++++++++++ tests/test-prebuild-only/netlify.toml | 12 ++++++++---- tests/test-prebuild-only/package.json | 8 ++++++++ 3 files changed, 26 insertions(+), 4 deletions(-) create mode 100644 tests/test-prebuild-only/README.md create mode 100644 tests/test-prebuild-only/package.json diff --git a/tests/test-prebuild-only/README.md b/tests/test-prebuild-only/README.md new file mode 100644 index 00000000..f79d8d6a --- /dev/null +++ b/tests/test-prebuild-only/README.md @@ -0,0 +1,10 @@ +# run pre-build tests only + +These tests run: +- [x] before the build +- [ ] after the build +- [ ] on deploy success + +## Local build + +Use command `DEBUG=netlify-plugin-cypress npm run netlify:build` to try the plugin locally diff --git a/tests/test-prebuild-only/netlify.toml b/tests/test-prebuild-only/netlify.toml index c4db823d..8ca7e3b7 100644 --- a/tests/test-prebuild-only/netlify.toml +++ b/tests/test-prebuild-only/netlify.toml @@ -8,6 +8,8 @@ publish = "public" CYPRESS_CACHE_FOLDER = "./node_modules/CypressBinary" # set TERM variable for terminal output TERM = "xterm" +# do not print too many progress messages +CI = "1" [[plugins]] # local Cypress plugin will test our site after it is built @@ -17,12 +19,14 @@ TERM = "xterm" # run Cypress tests once on the site before it is built # and do not run the tests after it was built + [plugins.inputs] + # these settings apply to the default step onSuccess + # in this case, we do not want to run any tests after deploy + enable = false + # let's run tests against development server [plugins.inputs.preBuild] + enable = true start = 'npx serve public' wait-on = 'http://localhost:5000' wait-on-timeout = '30' # seconds - - # and skip tests after building it - [plugins.inputs] - skip = true diff --git a/tests/test-prebuild-only/package.json b/tests/test-prebuild-only/package.json new file mode 100644 index 00000000..1fd9eb09 --- /dev/null +++ b/tests/test-prebuild-only/package.json @@ -0,0 +1,8 @@ +{ + "name": "prebuild-only-example", + "version": "0.1.0", + "private": true, + "scripts": { + "netlify:build": "../../node_modules/.bin/netlify build" + } +} From d58c8211b9e3b6908b59b77c4f68b145d96dae09 Mon Sep 17 00:00:00 2001 From: Gleb Bahmutov Date: Tue, 2 Mar 2021 14:16:11 -0500 Subject: [PATCH 05/20] update netlify dev --- circle.yml | 4 ++-- tests/test-netlify-dev/README.md | 10 ++++++++++ tests/test-netlify-dev/netlify.toml | 7 +++++-- tests/test-netlify-dev/package.json | 8 ++++++++ 4 files changed, 25 insertions(+), 4 deletions(-) create mode 100644 tests/test-netlify-dev/README.md create mode 100644 tests/test-netlify-dev/package.json diff --git a/circle.yml b/circle.yml index f5b18e5a..75fdb904 100644 --- a/circle.yml +++ b/circle.yml @@ -84,7 +84,7 @@ jobs: at: ~/ - run: name: Netlify Build 🏗 - command: npx netlify build + command: npm run netlify:build working_directory: tests/test-prebuild-only environment: DEBUG: netlify-plugin-cypress @@ -110,7 +110,7 @@ jobs: at: ~/ - run: name: Netlify Build 🏗 - command: npx netlify build + command: npm run netlify:build working_directory: tests/test-netlify-dev environment: DEBUG: netlify-plugin-cypress diff --git a/tests/test-netlify-dev/README.md b/tests/test-netlify-dev/README.md new file mode 100644 index 00000000..7025154a --- /dev/null +++ b/tests/test-netlify-dev/README.md @@ -0,0 +1,10 @@ +# use Netlify dev command + +These tests run: +- [x] before the build +- [ ] after the build +- [ ] on deploy success + +## Local build + +Use command `DEBUG=netlify-plugin-cypress npm run netlify:build` to try the plugin locally diff --git a/tests/test-netlify-dev/netlify.toml b/tests/test-netlify-dev/netlify.toml index 7ec4fbf0..ddb426d9 100644 --- a/tests/test-netlify-dev/netlify.toml +++ b/tests/test-netlify-dev/netlify.toml @@ -12,6 +12,8 @@ functions = "src/functions" CYPRESS_CACHE_FOLDER = "./node_modules/CypressBinary" # set TERM variable for terminal output TERM = "xterm" +# do not print too many progress messages +CI = "1" [dev] command = "npx serve public" @@ -32,9 +34,10 @@ TERM = "xterm" # and Netlify functions # let's run tests against Netlify Dev server [plugins.inputs.preBuild] + enable = true start = 'npx netlify dev' wait-on = 'http://localhost:8888' - # and skip tests after building it + # do no run tests after deploy [plugins.inputs] - skip = true + enable = false diff --git a/tests/test-netlify-dev/package.json b/tests/test-netlify-dev/package.json new file mode 100644 index 00000000..1fd9eb09 --- /dev/null +++ b/tests/test-netlify-dev/package.json @@ -0,0 +1,8 @@ +{ + "name": "prebuild-only-example", + "version": "0.1.0", + "private": true, + "scripts": { + "netlify:build": "../../node_modules/.bin/netlify build" + } +} From def24d53d334f18ccf231b7d6708418893937ef7 Mon Sep 17 00:00:00 2001 From: Gleb Bahmutov Date: Tue, 2 Mar 2021 15:30:56 -0500 Subject: [PATCH 06/20] update netlify toml --- netlify.toml | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/netlify.toml b/netlify.toml index 6ae29701..8d51e493 100644 --- a/netlify.toml +++ b/netlify.toml @@ -10,14 +10,15 @@ publish = "public" [[plugins]] # local Cypress plugin will test our site after it is built package = "." + + # run tests after deploying to Netlify [plugins.inputs] - # browser = "electron" - # spec = 'cypress/integration/spec.js' - # [plugins.inputs.preBuild] - # start = 'npm start' - # wait-on = 'http://localhost:5000' - # wait-on-timeout = '3' # seconds - [plugins.inputs.onSuccess] - enable = true record = true group = 'deployed' + + # run tests after building the site + [plugins.inputs.postBuild] + enable = true + record = true + group = 'postBuild' + From 2e4de9a62f94d3e681ba2275931e5274302bbc92 Mon Sep 17 00:00:00 2001 From: Gleb Bahmutov Date: Tue, 2 Mar 2021 15:42:14 -0500 Subject: [PATCH 07/20] update use-chromium --- tests/use-chromium/README.md | 6 ++++++ tests/use-chromium/netlify.toml | 7 +++++++ 2 files changed, 13 insertions(+) create mode 100644 tests/use-chromium/README.md diff --git a/tests/use-chromium/README.md b/tests/use-chromium/README.md new file mode 100644 index 00000000..c37f0eb4 --- /dev/null +++ b/tests/use-chromium/README.md @@ -0,0 +1,6 @@ +# use Chromium browser + +These tests run: +- [ ] before the build +- [x] after the build +- [ ] on deploy success diff --git a/tests/use-chromium/netlify.toml b/tests/use-chromium/netlify.toml index 30b29026..97707cc2 100644 --- a/tests/use-chromium/netlify.toml +++ b/tests/use-chromium/netlify.toml @@ -6,5 +6,12 @@ publish = "public" # local Cypress plugin will test our site after it is built # in production, please use: package = "netlify-plugin-cypress" package = "../../" + [plugins.inputs] + # do not run tests after deploy + enable = false + + # only run tests postBuild before deploy + [plugins.inputs.postBuild] + enable = true browser = "chromium" From 2a36fb8559a5c261e4c3f560ca1e64803e2aabb0 Mon Sep 17 00:00:00 2001 From: Gleb Bahmutov Date: Tue, 2 Mar 2021 15:43:49 -0500 Subject: [PATCH 08/20] update basic test --- tests/basic/README.md | 6 ++++++ tests/basic/netlify.toml | 8 ++++++++ 2 files changed, 14 insertions(+) create mode 100644 tests/basic/README.md diff --git a/tests/basic/README.md b/tests/basic/README.md new file mode 100644 index 00000000..cd0201d8 --- /dev/null +++ b/tests/basic/README.md @@ -0,0 +1,6 @@ +# basic test + +These tests run: +- [ ] before the build +- [x] after the build +- [ ] on deploy success diff --git a/tests/basic/netlify.toml b/tests/basic/netlify.toml index 13b880f2..529ad1ab 100644 --- a/tests/basic/netlify.toml +++ b/tests/basic/netlify.toml @@ -6,3 +6,11 @@ publish = "public" # local Cypress plugin will test our site after it is built # in production, please use: package = "netlify-plugin-cypress" package = "../../" + + # do not run tests after deploy (testing) + [plugins.inputs] + enable = false + + # run tests postBuild + [plugins.inputs.postBuild] + enable = true From 30cfd627e933f59797a3698498373379743b887f Mon Sep 17 00:00:00 2001 From: Gleb Bahmutov Date: Tue, 2 Mar 2021 15:45:16 -0500 Subject: [PATCH 09/20] update recommended test --- tests/recommended/README.md | 6 ++++++ tests/recommended/netlify.toml | 12 ++++++++++++ 2 files changed, 18 insertions(+) create mode 100644 tests/recommended/README.md diff --git a/tests/recommended/README.md b/tests/recommended/README.md new file mode 100644 index 00000000..926ccf9c --- /dev/null +++ b/tests/recommended/README.md @@ -0,0 +1,6 @@ +# recommended test + +These tests run: +- [ ] before the build +- [x] after the build +- [ ] on deploy success diff --git a/tests/recommended/netlify.toml b/tests/recommended/netlify.toml index c23731f1..028f14b1 100644 --- a/tests/recommended/netlify.toml +++ b/tests/recommended/netlify.toml @@ -13,3 +13,15 @@ TERM = "xterm" # local Cypress plugin will test our site after it is built # in production, please use: package = "netlify-plugin-cypress" package = "../../" + + # this is a testing situation + # in the real system we recommend running tests post deploy + # by default and enable preBuild and postBuild if needed + + # do not run tests after deploy (testing) + [plugins.inputs] + enable = false + + # run tests postBuild + [plugins.inputs.postBuild] + enable = true From facb0e8b99174bcc4e113779489929cd5dd2c4c7 Mon Sep 17 00:00:00 2001 From: Gleb Bahmutov Date: Tue, 2 Mar 2021 15:46:35 -0500 Subject: [PATCH 10/20] update recording test --- tests/recording/README.md | 6 ++++++ tests/recording/netlify.toml | 6 ++++++ 2 files changed, 12 insertions(+) create mode 100644 tests/recording/README.md diff --git a/tests/recording/README.md b/tests/recording/README.md new file mode 100644 index 00000000..926ccf9c --- /dev/null +++ b/tests/recording/README.md @@ -0,0 +1,6 @@ +# recommended test + +These tests run: +- [ ] before the build +- [x] after the build +- [ ] on deploy success diff --git a/tests/recording/netlify.toml b/tests/recording/netlify.toml index 6e7218bf..a799aad8 100644 --- a/tests/recording/netlify.toml +++ b/tests/recording/netlify.toml @@ -13,7 +13,13 @@ TERM = "xterm" # local Cypress plugin will test our site after it is built # in production, please use: package = "netlify-plugin-cypress" package = "../../" + + # do not run tests after deploy [plugins.inputs] + enable = false + + # run and record tests post build + [plugins.inputs.postBuild] record = true group = "test once after build" tag = "recommended" From 6b3392688804008181a2e1367984db60f12b52ca Mon Sep 17 00:00:00 2001 From: Gleb Bahmutov Date: Tue, 2 Mar 2021 15:52:34 -0500 Subject: [PATCH 11/20] update html test --- tests/html-pages/README.md | 5 +++++ tests/html-pages/netlify.toml | 8 ++++++++ tests/recording/netlify.toml | 1 + 3 files changed, 14 insertions(+) diff --git a/tests/html-pages/README.md b/tests/html-pages/README.md index d21a97b7..ff0adba8 100644 --- a/tests/html-pages/README.md +++ b/tests/html-pages/README.md @@ -9,3 +9,8 @@ Test locally with ``` $ DEBUG=netlify-plugin-cypress ../../node_modules/.bin/netlify build ``` + +These tests run: +- [ ] before the build +- [x] after the build +- [ ] on deploy success diff --git a/tests/html-pages/netlify.toml b/tests/html-pages/netlify.toml index 13b880f2..0bda095e 100644 --- a/tests/html-pages/netlify.toml +++ b/tests/html-pages/netlify.toml @@ -6,3 +6,11 @@ publish = "public" # local Cypress plugin will test our site after it is built # in production, please use: package = "netlify-plugin-cypress" package = "../../" + + # do not run tests after deploy + [plugins.inputs] + enable = false + + # run and record tests post build + [plugins.inputs.postBuild] + enable = true diff --git a/tests/recording/netlify.toml b/tests/recording/netlify.toml index a799aad8..83478d09 100644 --- a/tests/recording/netlify.toml +++ b/tests/recording/netlify.toml @@ -20,6 +20,7 @@ TERM = "xterm" # run and record tests post build [plugins.inputs.postBuild] + enable = true record = true group = "test once after build" tag = "recommended" From 5ada7e9cd3e9087c93bb5263f3e7c9016afbdae2 Mon Sep 17 00:00:00 2001 From: Gleb Bahmutov Date: Tue, 2 Mar 2021 16:04:04 -0500 Subject: [PATCH 12/20] add enable --- tests/test-twice/README.md | 15 +++++++++++++++ tests/test-twice/netlify.toml | 9 +++++++-- 2 files changed, 22 insertions(+), 2 deletions(-) create mode 100644 tests/test-twice/README.md diff --git a/tests/test-twice/README.md b/tests/test-twice/README.md new file mode 100644 index 00000000..491da7fa --- /dev/null +++ b/tests/test-twice/README.md @@ -0,0 +1,15 @@ +# routing example +> Client-side routing with fallback + +This example tests the redirect fallback that makes client-side routing defined in [src/App.js](src/App.js) work. See [netlify.toml](netlify.toml) + +![Test screenshot](images/routing.png) + +These tests run: +- [x] before the build +- [x] after the build +- [ ] on deploy success + +## Local build + +Use command `DEBUG=netlify-plugin-cypress npm run netlify:build` to try the plugin locally diff --git a/tests/test-twice/netlify.toml b/tests/test-twice/netlify.toml index 82701ef0..c55d8bc8 100644 --- a/tests/test-twice/netlify.toml +++ b/tests/test-twice/netlify.toml @@ -17,9 +17,14 @@ TERM = "xterm" # run Cypress tests once on the site before it is built # and then after building the static folder + # do not run tests after deploy + [plugins.inputs] + enable = false + # let's run tests against development server # before building it (and testing the built site) [plugins.inputs.preBuild] + enable = true start = 'npx serve public' wait-on = 'http://localhost:5000' wait-on-timeout = '30' # seconds @@ -27,8 +32,8 @@ TERM = "xterm" group = "test twice: 1 - before build" tag = "test twice" - # and test after - [plugins.inputs] + # and test after build + [plugins.inputs.postBuild] record = true group = "test twice: 2 - after build" tag = "test twice" From 13cc485cfe03cafdc05c3b0141dcb20ff3a1e6cd Mon Sep 17 00:00:00 2001 From: Gleb Bahmutov Date: Tue, 2 Mar 2021 16:10:09 -0500 Subject: [PATCH 13/20] enable release job for all branches --- circle.yml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/circle.yml b/circle.yml index 75fdb904..7bf59618 100644 --- a/circle.yml +++ b/circle.yml @@ -175,6 +175,8 @@ workflows: requires: - cypress/install - release: + # run the release job on all branches + # since we might want to release a beta version requires: - build - 'basic test' @@ -186,8 +188,3 @@ workflows: - test-using-chromium - test-netlify-dev - 'routing' - filters: - branches: - only: - - master - - beta From 7e1e80559e93b89eef3914c1fa9ef0d0f1c1c10f Mon Sep 17 00:00:00 2001 From: Gleb Bahmutov Date: Tue, 2 Mar 2021 16:10:19 -0500 Subject: [PATCH 14/20] feat: enable post-deploy tests by default BREAKING CHANGE: enable onSuccess hook, and disable other hooks. From daa34222709131411243d529bfec7f0a13108d22 Mon Sep 17 00:00:00 2001 From: Gleb Bahmutov Date: Tue, 2 Mar 2021 16:29:09 -0500 Subject: [PATCH 15/20] blank circleci variables --- circle.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/circle.yml b/circle.yml index 7bf59618..d8da3cc2 100644 --- a/circle.yml +++ b/circle.yml @@ -22,7 +22,16 @@ jobs: - node/with-cache: steps: - run: npm ci - - run: npx semantic-release + # allow CircleCI to release beta versions + # from pull request build jobs + - run: + name: Semantic release 🚀 + command: npx semantic-release + # by tricking Circle and removing the environment variables + environment: + CIRCLE_PR_NUMBER: + CIRCLE_PULL_REQUEST: + CI_PULL_REQUEST: 'basic test': executor: cypress/base-12-14-0 From 3d538d4c7ae56d82b9566561676cbf620e950589 Mon Sep 17 00:00:00 2001 From: Gleb Bahmutov Date: Tue, 2 Mar 2021 16:36:26 -0500 Subject: [PATCH 16/20] hmm release --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 6c8d6e36..90cc1d23 100644 --- a/package.json +++ b/package.json @@ -53,7 +53,7 @@ }, "release": { "branches": [ - "main", + { "name": "main" }, { "name": "prepare-v2", "prerelease": true From 8a6a31311cdb4bd7506a4559574cdbceb3ede79e Mon Sep 17 00:00:00 2001 From: Gleb Bahmutov Date: Tue, 2 Mar 2021 16:46:56 -0500 Subject: [PATCH 17/20] set master branch name --- circle.yml | 23 ++++++++++++++--------- package.json | 2 +- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/circle.yml b/circle.yml index d8da3cc2..635c83b7 100644 --- a/circle.yml +++ b/circle.yml @@ -17,6 +17,10 @@ jobs: executor: name: node/default tag: '12' + environment: + # since we do not need Cypress to publish the NPM package + # we can skip the binary download + CYPRESS_INSTALL_BINARY: 0 steps: - checkout - node/with-cache: @@ -188,12 +192,13 @@ workflows: # since we might want to release a beta version requires: - build - - 'basic test' - - 'html-pages' - - 'recommended test' - - 'recording test' - - 'test-twice' - - 'test-prebuild-only' - - test-using-chromium - - test-netlify-dev - - 'routing' + # temporary while publishing pre-release + # - 'basic test' + # - 'html-pages' + # - 'recommended test' + # - 'recording test' + # - 'test-twice' + # - 'test-prebuild-only' + # - test-using-chromium + # - test-netlify-dev + # - 'routing' diff --git a/package.json b/package.json index 90cc1d23..0cfc465f 100644 --- a/package.json +++ b/package.json @@ -53,7 +53,7 @@ }, "release": { "branches": [ - { "name": "main" }, + { "name": "master" }, { "name": "prepare-v2", "prerelease": true From a4a31a2e52f4ec1e5a97e66fd13cc83099268b83 Mon Sep 17 00:00:00 2001 From: Gleb Bahmutov Date: Tue, 2 Mar 2021 16:52:04 -0500 Subject: [PATCH 18/20] feat: chromium should be the default browser --- src/index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/index.js b/src/index.js index ba9957cf..93c9f26e 100644 --- a/src/index.js +++ b/src/index.js @@ -6,7 +6,7 @@ const debugVerbose = require('debug')('netlify-plugin-cypress:verbose') const { ping, getBrowserPath, serveFolder } = require('./utils') const PLUGIN_NAME = 'netlify-plugin-cypress' -const DEFAULT_BROWSER = 'electron' +const DEFAULT_BROWSER = 'chromium' function startServerMaybe(run, options = {}) { const startCommand = options.start From e03b1da10866a0f9b6b0e1f3feb99f00fddc5ce4 Mon Sep 17 00:00:00 2001 From: Gleb Bahmutov Date: Wed, 3 Mar 2021 11:43:02 -0500 Subject: [PATCH 19/20] update readme file --- README.md | 93 +++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 60 insertions(+), 33 deletions(-) diff --git a/README.md b/README.md index e5f87bbd..9b460c32 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,6 @@ # netlify-plugin-cypress [![CircleCI](https://circleci.com/gh/cypress-io/netlify-plugin-cypress/tree/master.svg?style=svg&circle-token=9cbb587a5a0ae4ce28b011dd03d10d66de906708)](https://circleci.com/gh/cypress-io/netlify-plugin-cypress/tree/master) [![renovate-app badge][renovate-badge]][renovate-app] [![netlify-plugin-cypress](https://img.shields.io/endpoint?url=https://dashboard.cypress.io/badge/simple/ixroqc/master&style=flat&logo=cypress)](https://dashboard.cypress.io/projects/ixroqc/runs) [![Netlify Status](https://api.netlify.com/api/v1/badges/76892baf-2ad8-4642-b283-f2135963ff51/deploy-status)](https://app.netlify.com/sites/sad-lumiere-6a00a5/deploys) -> Runs Cypress end-to-end tests after Netlify builds the site but before it is deployed - -**Note:** currently the built site is served statically and tested _without proxying redirects_. +> Runs Cypress end-to-end tests on Netlify Build ## Install and use @@ -26,7 +24,7 @@ This plugin installs [via Puppeteer](https://github.com/puppeteer/puppeteer) Chr ## How does it work -When Netlify Build runs, it "knows" the output folder name and calls the `netlify-plugin-cypress` after the build has finished with that folder. Then the plugin runs Cypress tests using its [NPM module API](https://on.cypress.io/module-api). If the tests pass, the plugin finishes and the Netlify deploy starts. +When Netlify Build runs, it calls the plugin `netlify-plugin-cypress` before and after the build, and after the deployment. The plugin runs the Cypress tests using its [NPM module API](https://on.cypress.io/module-api) against the local folder or against the deployed URL. ## Examples @@ -36,7 +34,7 @@ Here is the most basic [Netlify config file](https://docs.netlify.com/configure- ```toml [[plugins]] - # local Cypress plugin will test our site after it is built + # runs Cypress tests against the deployed URL package = "netlify-plugin-cypress" ``` @@ -61,7 +59,7 @@ CYPRESS_CACHE_FOLDER = "./node_modules/CypressBinary" TERM = "xterm" [[plugins]] -# local Cypress plugin will test our site after it is built +# runs Cypress tests against the deployed URL package = "netlify-plugin-cypress" ``` @@ -77,22 +75,15 @@ publish = "build" # ...remaining configuration... ``` -### testing deployed url - -After successful deployment you can run tests against the `DEPLOY_PRIME_URL` provided by the Netlify system. +### tutorial -```toml -[[plugins]] -package = "netlify-plugin-cypress" - [plugins.inputs.onSuccess] - enable = true -``` +Read the full tutorial at [Test Sites Deployed To Netlify Using netlify-plugin-cypress](https://glebbahmutov.com/blog/test-netlify/). -The following parameters can be used with "onSuccess" tests: `record`, `group`, `tag`, `spec`. +**Note:** if any tests against the deployed URL fail, the Netlify build still considers it a success. Thus if you want to have a test check against the deploy, install [Cypress GitHub App](https://on.cypress.io/github-integration). The app will provide its own failing status check in this case. -Read the full tutorial [Test Sites Deployed To Netlify Using netlify-plugin-cypress](https://glebbahmutov.com/blog/test-netlify/). +### options -**Note:** if any tests against the deployed URL fail, the Netlify build still considers it a success. Thus if you want to have a test check against the deploy, install [Cypress GitHub App](https://on.cypress.io/github-integration). The app will provide its own failing status check in this case. +You can control the browser, the specs to run, record tests on Cypress Dashboard, etc, see [manifest.yml](./manifest.yml) file. ### recording @@ -110,7 +101,7 @@ publish = "build" TERM = "xterm" [[plugins]] -# local Cypress plugin will test our site after it is built +# runs Cypress tests against the deployed URL package = "netlify-plugin-cypress" [plugins.inputs] record = true @@ -132,7 +123,7 @@ You can change the group name for the recorded run using `group` parameter ```toml [[plugins]] -# local Cypress plugin will test our site after it is built +# runs Cypress tests against the deployed URL package = "netlify-plugin-cypress" [plugins.inputs] record = true @@ -145,7 +136,7 @@ You can give recorded run [tags](https://on.cypress.io/module-api#cypress-run) u ```toml [[plugins]] -# local Cypress plugin will test our site after it is built +# runs Cypress tests against the deployed URL package = "netlify-plugin-cypress" [plugins.inputs] record = true @@ -169,7 +160,7 @@ publish = "build" TERM = "xterm" [[plugins]] -# local Cypress plugin will test our site after it is built +# runs Cypress tests against the deployed URL package = "netlify-plugin-cypress" [plugins.inputs] spec = "cypress/integration/smoke*.js" @@ -179,7 +170,7 @@ See [cypress-example-kitchensink](https://github.com/cypress-io/cypress-example- ### browser -By default all tests run using Chromium browser. If you want to use Electron: +By default all tests run using the Chromium browser. If you want to use Electron: ```toml [build] @@ -201,7 +192,7 @@ package = "netlify-plugin-cypress" ### testing SPA routes -SPAs need catch-all redirect setup to make non-root paths accesssible by tests. You can enable this with `spa` parameter. +SPAs need catch-all redirect setup to make non-root paths accessible by tests. You can enable this with `spa` parameter. ``` [[plugins]] @@ -216,7 +207,7 @@ See [lws-spa](https://github.com/lwsjs/spa) for more options and [tests/routing] ### testing the site before build -By default this plugin tests static site _after build_. But maybe you want to run end-to-end tests against the _local development server_. You can start local server, wait for it to respond and then run Cypress tests by passing parameters to this plugin. Here is a sample config file +By default this plugin tests static site _after deploy_. But maybe you want to run end-to-end tests against the _local development server_. You can start the local server, wait for it to respond and then run Cypress tests by passing parameters to this plugin. Here is a sample config file ```toml [[plugins]] @@ -229,10 +220,42 @@ By default this plugin tests static site _after build_. But maybe you want to ru wait-on-timeout = '30' # seconds ``` -Parameters you can place into `preBuild` inputs: `start`, `wait-on`, `wait-on-timeout`, `spec`, `record`, `group`, and `tag`. If there is `preBuild` and `postBuild` testing with different tags, the first one wins :) +Parameters you can place into `preBuild` inputs: `start`, `wait-on`, `wait-on-timeout`, `spec`, `record`, `group`, and `tag`. See [netlify-plugin-prebuild-example](https://github.com/cypress-io/netlify-plugin-prebuild-example) repo +### testing the site after build + +By default this plugin tests static site _after deploy_. But maybe you want to run end-to-end tests locally after building the static site. Cypress includes a local static server for this case. Here is a sample config file + +```toml +[[plugins]] + package = "netlify-plugin-cypress" + # let's run tests against the built site + [plugins.inputs.postBuild] + enable = true +``` + +Parameters you can place into `postBuild` inputs: `spec`, `record`, `group`, `tag`, and `spa`. + +#### The SPA parameter + +If your site requires all unknown URLs to redirect back to the index page, use the `spa` parameter + +```toml +[[plugins]] + package = "netlify-plugin-cypress" + # let's run tests against the built site + [plugins.inputs.postBuild] + enable = true + # must allow our test server to redirect unknown routes to "/" + # so that client-side routing can correctly route them + # can be set to true or "index.html" (or similar fallback filename in the built folder) + spa = true +``` + +See [the routing example](./tests/routing/netlify.toml). + ### using Netlify CLI Even better when testing the prebuilt site is to run the [Netlify CLI](https://cli.netlify.com/) to make sure the local API redirects and Netlify functions work in addition to the web site. Add `netlify-cli` as a dev dependency and start it during testing. @@ -254,13 +277,17 @@ For more, see [tests/test-netlify-dev](./tests/test-netlify-dev) example and rea ### skipping tests -If you are testing the site before building it, you probably want to skip testing it after the build. See [tests/test-prebuild-only](./tests/test-prebuild-only/netlify.toml): +If you are testing the site before building it and want to skip testing the deployed URL ```toml [[plugins]] package = "netlify-plugin-cypress" + # do not test the deployed URL [plugins.inputs] - skip = true + enable = false + # test the local site + [plugins.inputs.preBuild] + enable = true ``` ### parallelization @@ -300,6 +327,7 @@ Name | Description ### v1 to v2 - The default browser has been switched to Chromium. If you want to use the built-in Electron use an explicit option [browser](#browser) +- We have changed the default testing phase. In v1 the tests executed after building the site by default. In v2 the tests run against the deployed URL by default, and you need to enable the testing during `preBuild` or `postBuild` steps. ## Debugging @@ -339,11 +367,10 @@ Set environment variable `DEBUG=netlify-plugin-cypress` to see the debug logs. T Switch to using Chromium browser that seems to be a bit more reliable. Use browser = "chromium" setting. -## Changelog - -### v1 to v2 - -- We have changed the default testing phase. In v1 the tests executed after building the site by default. In v2 the tests run against the deployed URL by default, and you need to enable the testing during `preBuild` or `postBuild` steps. +
+ You want to skip Puppeteer download + If you do not plan on using Chromium to run the tests, if you want to use the built-in Electron browser, you can save time by skipping the Puppeteer download. Set the environment variable PUPPETEER_SKIP_DOWNLOAD = 1 on your CI. +
## License From e4194aba1d214849aba1a67241efb6fb9a7af8c0 Mon Sep 17 00:00:00 2001 From: Gleb Bahmutov Date: Wed, 3 Mar 2021 11:53:41 -0500 Subject: [PATCH 20/20] format --- package.json | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/package.json b/package.json index f6f83e68..6b3f4364 100644 --- a/package.json +++ b/package.json @@ -53,7 +53,9 @@ }, "release": { "branches": [ - { "name": "master" }, + { + "name": "master" + }, { "name": "prepare-v2", "prerelease": true