// Usage: // // Run all services: // npm run test:services // // Run some services: // npm run test:services -- --only=service1,service2,service3 // // Alternatively, pass a newline-separated list of services to stdin. // echo "service1\nservice2\nservice3" | npm run test:services -- --stdin // // Run tests but skip tests which intercept requests: // SKIP_INTERCEPTED=TRUE npm run test:services -- // // Run tests on a given instance: // SKIP_INTERCEPTED=TRUE TESTED_SERVER_URL=https://test.shields.io npm run test:services -- // // Service tests are run in CI in two cases: scheduled builds and pull // requests. The scheduled builds run _all_ the service tests, whereas the // pull requests run service tests designated in the PR title. In this way, // affected services can be proven working during code review without needing // to run all the slow (and likely flaky) service tests. // // Example pull request titles: // // - [Travis] Fix timeout issues // - [Travis Sonar] Support user token authentication // - [CRAN CPAN CTAN] Add test coverage // // The pull request script test:services:pr is split into two parts. First the // :prepare script infers the pull request context, fetches the PR title, and // writes the list of affected services to a file. Then the :run script reads // the list of affected services and runs the appropriate tests. // // There are three reasons to separate these two steps into separate processes // and build stages: // // 1. Generating the list of services to test is necessarily asynchronous, and // in Mocha, exclusive tests (`it.only` and `describe.only`) can only be // applied synchronously. In other words, if you try to add exclusive tests // in an asynchronous callback, all the tests will run. This is true even // when using `_mocha --delay`, as we are. Undoubtedly this could be fixed, // though it's not worth it. The problem is obscure and therefore low // for Mocha, which is quite backlogged. There is an easy workaround, which // is to generate the list of services to test in a separate process. // 2. Executing these two steps of the test runner separately makes the process // easier to reason about and much easier to debug on a dev machine. // 3. Getting "pipefail" to work cross platform with an npm script seems tricky. // Relying on npm scripts is safer. Using "pre" makes it impossible to run // the second step without the first. 'use strict' const minimist = require('minimist') const envFlag = require('node-env-flag') const readAllStdinSync = require('read-all-stdin-sync') const { createTestServer } = require('../server/in-process-server-test-helpers') const Runner = require('./runner') require('../unhandled-rejection.spec') let baseUrl, server if (process.env.TESTED_SERVER_URL) { baseUrl = process.env.TESTED_SERVER_URL } else { const port = 1111 baseUrl = 'http://localhost:1111' before('Start running the server', function() { server = createTestServer({ port }) server.start() }) after('Shut down the server', async function() { if (server) { await server.stop() } }) } const skipIntercepted = envFlag(process.env.SKIP_INTERCEPTED, false) const runner = new Runner({ baseUrl, skipIntercepted }) runner.prepare() // The server's request cache causes side effects between tests. if (!process.env.TESTED_SERVER_URL) { runner.beforeEach = () => { server.reset() } } const args = minimist(process.argv.slice(3)) const stdinOption = args.stdin const onlyOption = args.only let onlyServices if (stdinOption && onlyOption) { console.error('Do not use --only with --stdin') } else if (stdinOption) { const allStdin = readAllStdinSync().trim() onlyServices = allStdin ? allStdin.split('\n') : [] } else if (onlyOption) { onlyServices = onlyOption.split(',') } if (typeof onlyServices === 'undefined' || onlyServices.includes('*****')) { console.info('Running all service tests.') } else if (onlyServices.length === 0) { console.info('No service tests to run. Exiting.') process.exit(0) } else { console.info( `Running tests for ${onlyServices.length} services: ${onlyServices.join( ', ' )}.\n` ) runner.only(onlyServices) } runner.toss() // Invoke run() asynchronously, because Mocha will not start otherwise. process.nextTick(run)