fix [Sonar] legacy API tests (#4251)

* tests: fix sonar legacy API tests * docs: add inline documentation on sonar mocked tests * docs: add more inline sonar docs
2019-10-26 17:23:04 -05:00
parent 14d9d99aab
commit b05fe5f62c
9 changed files with 465 additions and 40 deletions
--- a/services/sonar/sonar-base.js
+++ b/services/sonar/sonar-base.js
@@ -2,7 +2,19 @@

 const Joi = require('@hapi/joi')
 const { isLegacyVersion } = require('./sonar-helpers')
-const { BaseJsonService } = require('..')
+const { BaseJsonService, NotFound } = require('..')
+
+// It is possible to see HTTP 404 response codes and HTTP 200 responses
+// with empty arrays of metric values, with both the legacy (pre v5.3) and modern APIs.
+//
+// 404 responses can occur with non-existent component keys, as well as unknown/unsupported metrics.
+//
+// 200 responses with empty arrays can occur when the metric key is valid, but the data
+// is unavailable for the specified component, for example using the metric key `tests` with a
+// component that is not capturing test results.
+// It can also happen when using an older/deprecated
+// metric key with a newer version of Sonar, for example using the metric key
+// `public_documented_api_density` with SonarQube v7.x or higher

 const modernSchema = Joi.object({
  component: Joi.object({
@@ -14,8 +26,9 @@ const modernSchema = Joi.object({
            Joi.number().min(0),
            Joi.allow('OK', 'ERROR')
          ).required(),
-        }).required()
+        })
      )
+      .min(0)
      .required(),
  }).required(),
 }).required()
@@ -31,7 +44,7 @@ const legacySchema = Joi.array()
              Joi.number().min(0),
              Joi.allow('OK', 'ERROR')
            ).required(),
-          }).required()
+          })
        )
        .required(),
    }).required()
@@ -83,12 +96,22 @@ module.exports = class SonarBase extends BaseJsonService {
    const metrics = {}

    if (useLegacyApi) {
-      json[0].msr.forEach(measure => {
+      const [{ msr: measures }] = json
+      if (!measures.length) {
+        throw new NotFound({ prettyMessage: 'metric not found' })
+      }
+      measures.forEach(measure => {
        // Most values are numeric, but not all of them.
        metrics[measure.key] = parseInt(measure.val) || measure.val
      })
    } else {
-      json.component.measures.forEach(measure => {
+      const {
+        component: { measures },
+      } = json
+      if (!measures.length) {
+        throw new NotFound({ prettyMessage: 'metric not found' })
+      }
+      measures.forEach(measure => {
        // Most values are numeric, but not all of them.
        metrics[measure.metric] = parseInt(measure.value) || measure.value
      })
--- a/services/sonar/sonar-coverage.tester.js
+++ b/services/sonar/sonar-coverage.tester.js
@@ -3,10 +3,14 @@
 const t = (module.exports = require('../tester').createServiceTester())
 const { isIntegerPercentage } = require('../test-validators')

+// The service tests targeting the legacy SonarQube API are mocked
+// because of the lack of publicly accessible, self-hosted, legacy SonarQube instances
+// See https://github.com/badges/shields/issues/4221#issuecomment-546611598 for more details
+// This is an uncommon scenario Shields has to support for Sonar, and should not be used as a model
+// for other service tests.
+
 t.create('Coverage')
-  .get(
-    '/org.ow2.petals%3Apetals-se-ase.json?server=http://sonar.petalslink.com'
-  )
+  .get('/swellaby%3Aletra.json?server=https://sonarcloud.io')
  .expectBadge({
    label: 'coverage',
    message: isIntegerPercentage,
@@ -16,7 +20,27 @@ t.create('Coverage (legacy API supported)')
  .get(
    '/org.ow2.petals%3Apetals-se-ase.json?server=http://sonar.petalslink.com&sonarVersion=4.2'
  )
+  .intercept(nock =>
+    nock('http://sonar.petalslink.com/api')
+      .get('/resources')
+      .query({
+        resource: 'org.ow2.petals:petals-se-ase',
+        depth: 0,
+        metrics: 'coverage',
+        includeTrends: true,
+      })
+      .reply(200, [
+        {
+          msr: [
+            {
+              key: 'coverage',
+              val: 83,
+            },
+          ],
+        },
+      ])
+  )
  .expectBadge({
    label: 'coverage',
-    message: isIntegerPercentage,
+    message: '83%',
  })
--- a/services/sonar/sonar-documented-api-density.tester.js
+++ b/services/sonar/sonar-documented-api-density.tester.js
@@ -1,22 +1,78 @@
 'use strict'

-const { isIntegerPercentage } = require('../test-validators')
 const t = (module.exports = require('../tester').createServiceTester())

-t.create('Documented API Density')
+// The service tests targeting the legacy SonarQube API are mocked
+// because of the lack of publicly accessible, self-hosted, legacy SonarQube instances
+// See https://github.com/badges/shields/issues/4221#issuecomment-546611598 for more details
+// This is an uncommon scenario Shields has to support for Sonar, and should not be used as a model
+// for other service tests.
+
+// This metric was deprecated in SonarQube 6.2 and dropped in SonarQube 7.x+
+// https://docs.sonarqube.org/6.7/MetricDefinitions.html#src-11634682_MetricDefinitions-Documentation
+// https://docs.sonarqube.org/7.0/MetricDefinitions.html
+// https://sonarcloud.io/api/measures/component?componentKey=org.sonarsource.sonarqube:sonarqube&metricKeys=public_documented_api_density
+t.create('Documented API Density (not found)')
  .get(
-    '/org.ow2.petals%3Apetals-se-ase.json?server=http://sonar.petalslink.com'
+    '/org.sonarsource.sonarqube%3Asonarqube.json?server=https://sonarcloud.io'
  )
  .expectBadge({
    label: 'public documented api density',
-    message: isIntegerPercentage,
+    message: 'metric not found',
+  })
+
+t.create('Documented API Density')
+  .get(
+    '/org.ow2.petals%3Apetals-se-ase.json?server=http://sonar.somewhatold.com&sonarVersion=6.1'
+  )
+  .intercept(nock =>
+    nock('http://sonar.somewhatold.com/api')
+      .get('/measures/component')
+      .query({
+        componentKey: 'org.ow2.petals:petals-se-ase',
+        metricKeys: 'public_documented_api_density',
+      })
+      .reply(200, {
+        component: {
+          measures: [
+            {
+              metric: 'public_documented_api_density',
+              value: 91,
+            },
+          ],
+        },
+      })
+  )
+  .expectBadge({
+    label: 'public documented api density',
+    message: '91%',
  })

 t.create('Documented API Density (legacy API supported)')
  .get(
    '/org.ow2.petals%3Apetals-se-ase.json?server=http://sonar.petalslink.com&sonarVersion=4.2'
  )
+  .intercept(nock =>
+    nock('http://sonar.petalslink.com/api')
+      .get('/resources')
+      .query({
+        resource: 'org.ow2.petals:petals-se-ase',
+        depth: 0,
+        metrics: 'public_documented_api_density',
+        includeTrends: true,
+      })
+      .reply(200, [
+        {
+          msr: [
+            {
+              key: 'public_documented_api_density',
+              val: 79,
+            },
+          ],
+        },
+      ])
+  )
  .expectBadge({
    label: 'public documented api density',
-    message: isIntegerPercentage,
+    message: '79%',
  })
--- a/services/sonar/sonar-fortify-rating.tester.js
+++ b/services/sonar/sonar-fortify-rating.tester.js
@@ -2,7 +2,13 @@

 const t = (module.exports = require('../tester').createServiceTester())

-// The below tests are using a mocked API response because
+// The service tests targeting the legacy SonarQube API are mocked
+// because of the lack of publicly accessible, self-hosted, legacy SonarQube instances
+// See https://github.com/badges/shields/issues/4221#issuecomment-546611598 for more details
+// This is an uncommon scenario Shields has to support for Sonar, and should not be used as a model
+// for other service tests.
+
+// The below tests are all using a mocked API response because
 // neither SonarCloud.io nor any known public SonarQube deployments
 // have the Fortify plugin installed and in use, so there are no
 // available live endpoints to hit.
@@ -75,3 +81,27 @@ t.create('Fortify Security Rating (nonexistent component)')
    label: 'fortify-security-rating',
    message: 'component or metric not found, or legacy API not supported',
  })
+
+t.create('Fortify Security Rating (legacy API metric not found)')
+  .get(
+    '/org.ow2.petals%3Apetals-se-ase.json?server=http://sonar.petalslink.com&sonarVersion=4.2'
+  )
+  .intercept(nock =>
+    nock('http://sonar.petalslink.com/api')
+      .get('/resources')
+      .query({
+        resource: 'org.ow2.petals:petals-se-ase',
+        depth: 0,
+        metrics: 'fortify-security-rating',
+        includeTrends: true,
+      })
+      .reply(200, [
+        {
+          msr: [],
+        },
+      ])
+  )
+  .expectBadge({
+    label: 'fortify-security-rating',
+    message: 'metric not found',
+  })
--- a/services/sonar/sonar-generic.tester.js
+++ b/services/sonar/sonar-generic.tester.js
@@ -4,6 +4,7 @@ const { isMetric } = require('../test-validators')
 const t = (module.exports = require('../tester').createServiceTester())

 t.create('Security Rating')
+  .timeout(10000)
  .get(
    '/security_rating/com.luckybox:luckybox.json?server=https://sonarcloud.io'
  )
--- a/services/sonar/sonar-quality-gate.tester.js
+++ b/services/sonar/sonar-quality-gate.tester.js
@@ -5,6 +5,12 @@ const t = (module.exports = require('../tester').createServiceTester())

 const isQualityGateStatus = Joi.allow('passed', 'failed')

+// The service tests targeting the legacy SonarQube API are mocked
+// because of the lack of publicly accessible, self-hosted, legacy SonarQube instances
+// See https://github.com/badges/shields/issues/4221#issuecomment-546611598 for more details
+// This is an uncommon scenario Shields has to support for Sonar, and should not be used as a model
+// for other service tests.
+
 t.create('Quality Gate')
  .get(
    '/quality_gate/swellaby%3Aazdo-shellcheck.json?server=https://sonarcloud.io'
@@ -18,7 +24,27 @@ t.create('Quality Gate (Alert Status)')
  .get(
    '/alert_status/org.ow2.petals%3Apetals-se-ase.json?server=http://sonar.petalslink.com&sonarVersion=4.2'
  )
+  .intercept(nock =>
+    nock('http://sonar.petalslink.com/api')
+      .get('/resources')
+      .query({
+        resource: 'org.ow2.petals:petals-se-ase',
+        depth: 0,
+        metrics: 'alert_status',
+        includeTrends: true,
+      })
+      .reply(200, [
+        {
+          msr: [
+            {
+              key: 'alert_status',
+              val: 'OK',
+            },
+          ],
+        },
+      ])
+  )
  .expectBadge({
    label: 'quality gate',
-    message: isQualityGateStatus,
+    message: 'passed',
  })
--- a/services/sonar/sonar-tech-debt.tester.js
+++ b/services/sonar/sonar-tech-debt.tester.js
@@ -1,22 +1,48 @@
 'use strict'

-const { isIntegerPercentage } = require('../test-validators')
+const { isPercentage } = require('../test-validators')
 const t = (module.exports = require('../tester').createServiceTester())

+// The service tests targeting the legacy SonarQube API are mocked
+// because of the lack of publicly accessible, self-hosted, legacy SonarQube instances
+// See https://github.com/badges/shields/issues/4221#issuecomment-546611598 for more details
+// This is an uncommon scenario Shields has to support for Sonar, and should not be used as a model
+// for other service tests.
+
 t.create('Tech Debt')
  .get(
-    '/tech_debt/org.ow2.petals%3Apetals-se-ase.json?server=http://sonar.petalslink.com'
+    '/tech_debt/org.sonarsource.sonarqube%3Asonarqube.json?server=https://sonarcloud.io'
  )
  .expectBadge({
    label: 'tech debt',
-    message: isIntegerPercentage,
+    message: isPercentage,
  })

 t.create('Tech Debt (legacy API supported)')
  .get(
    '/tech_debt/org.ow2.petals%3Apetals-se-ase.json?server=http://sonar.petalslink.com&sonarVersion=4.2'
  )
+  .intercept(nock =>
+    nock('http://sonar.petalslink.com/api')
+      .get('/resources')
+      .query({
+        resource: 'org.ow2.petals:petals-se-ase',
+        depth: 0,
+        metrics: 'sqale_debt_ratio',
+        includeTrends: true,
+      })
+      .reply(200, [
+        {
+          msr: [
+            {
+              key: 'sqale_debt_ratio',
+              val: '7',
+            },
+          ],
+        },
+      ])
+  )
  .expectBadge({
    label: 'tech debt',
-    message: isIntegerPercentage,
+    message: '7%',
  })
--- a/services/sonar/sonar-tests.tester.js
+++ b/services/sonar/sonar-tests.tester.js
@@ -21,6 +21,12 @@ const isMetricAllowZero = Joi.alternatives(
    .required()
 )

+// The service tests targeting the legacy SonarQube API are mocked
+// because of the lack of publicly accessible, self-hosted, legacy SonarQube instances
+// See https://github.com/badges/shields/issues/4221#issuecomment-546611598 for more details
+// This is an uncommon scenario Shields has to support for Sonar, and should not be used as a model
+// for other service tests.
+
 t.create('Tests')
  .timeout(10000)
  .get(
@@ -32,13 +38,40 @@ t.create('Tests')
  })

 t.create('Tests (legacy API supported)')
-  .timeout(10000)
  .get(
    '/tests/org.ow2.petals%3Apetals-se-ase.json?server=http://sonar.petalslink.com&sonarVersion=4.2'
  )
+  .intercept(nock =>
+    nock('http://sonar.petalslink.com/api')
+      .get('/resources')
+      .query({
+        resource: 'org.ow2.petals:petals-se-ase',
+        depth: 0,
+        metrics: 'tests,test_failures,skipped_tests',
+        includeTrends: true,
+      })
+      .reply(200, [
+        {
+          msr: [
+            {
+              key: 'tests',
+              val: '71',
+            },
+            {
+              key: 'test_failures',
+              val: '2',
+            },
+            {
+              key: 'skipped_tests',
+              val: '1',
+            },
+          ],
+        },
+      ])
+  )
  .expectBadge({
    label: 'tests',
-    message: isDefaultTestTotals,
+    message: '68 passed, 2 failed, 1 skipped',
  })

 t.create('Tests with compact message')
@@ -90,13 +123,32 @@ t.create('Total Test Count')
  })

 t.create('Total Test Count (legacy API supported)')
-  .timeout(10000)
  .get(
    '/total_tests/org.ow2.petals%3Apetals-se-ase.json?server=http://sonar.petalslink.com&sonarVersion=4.2'
  )
+  .intercept(nock =>
+    nock('http://sonar.petalslink.com/api')
+      .get('/resources')
+      .query({
+        resource: 'org.ow2.petals:petals-se-ase',
+        depth: 0,
+        metrics: 'tests',
+        includeTrends: true,
+      })
+      .reply(200, [
+        {
+          msr: [
+            {
+              key: 'tests',
+              val: '132',
+            },
+          ],
+        },
+      ])
+  )
  .expectBadge({
    label: 'total tests',
-    message: isMetric,
+    message: '132',
  })

 t.create('Test Failures Count')
@@ -110,13 +162,32 @@ t.create('Test Failures Count')
  })

 t.create('Test Failures Count (legacy API supported)')
-  .timeout(10000)
  .get(
    '/test_failures/org.ow2.petals%3Apetals-se-ase.json?server=http://sonar.petalslink.com&sonarVersion=4.2'
  )
+  .intercept(nock =>
+    nock('http://sonar.petalslink.com/api')
+      .get('/resources')
+      .query({
+        resource: 'org.ow2.petals:petals-se-ase',
+        depth: 0,
+        metrics: 'test_failures',
+        includeTrends: true,
+      })
+      .reply(200, [
+        {
+          msr: [
+            {
+              key: 'test_failures',
+              val: '2',
+            },
+          ],
+        },
+      ])
+  )
  .expectBadge({
    label: 'test failures',
-    message: isMetricAllowZero,
+    message: '2',
  })

 t.create('Test Errors Count')
@@ -130,13 +201,32 @@ t.create('Test Errors Count')
  })

 t.create('Test Errors Count (legacy API supported)')
-  .timeout(10000)
  .get(
    '/test_errors/org.ow2.petals%3Apetals-se-ase.json?server=http://sonar.petalslink.com&sonarVersion=4.2'
  )
+  .intercept(nock =>
+    nock('http://sonar.petalslink.com/api')
+      .get('/resources')
+      .query({
+        resource: 'org.ow2.petals:petals-se-ase',
+        depth: 0,
+        metrics: 'test_errors',
+        includeTrends: true,
+      })
+      .reply(200, [
+        {
+          msr: [
+            {
+              key: 'test_errors',
+              val: '3',
+            },
+          ],
+        },
+      ])
+  )
  .expectBadge({
    label: 'test errors',
-    message: isMetricAllowZero,
+    message: '3',
  })

 t.create('Skipped Tests Count')
@@ -150,13 +240,32 @@ t.create('Skipped Tests Count')
  })

 t.create('Skipped Tests Count (legacy API supported)')
-  .timeout(10000)
  .get(
    '/skipped_tests/org.ow2.petals%3Apetals-se-ase.json?server=http://sonar.petalslink.com&sonarVersion=4.2'
  )
+  .intercept(nock =>
+    nock('http://sonar.petalslink.com/api')
+      .get('/resources')
+      .query({
+        resource: 'org.ow2.petals:petals-se-ase',
+        depth: 0,
+        metrics: 'skipped_tests',
+        includeTrends: true,
+      })
+      .reply(200, [
+        {
+          msr: [
+            {
+              key: 'skipped_tests',
+              val: '1',
+            },
+          ],
+        },
+      ])
+  )
  .expectBadge({
    label: 'skipped tests',
-    message: isMetricAllowZero,
+    message: '1',
  })

 t.create('Test Success Rate')
@@ -170,11 +279,30 @@ t.create('Test Success Rate')
  })

 t.create('Test Success Rate (legacy API supported)')
-  .timeout(10000)
  .get(
    '/test_success_density/org.ow2.petals%3Apetals-se-ase.json?server=http://sonar.petalslink.com&sonarVersion=4.2'
  )
+  .intercept(nock =>
+    nock('http://sonar.petalslink.com/api')
+      .get('/resources')
+      .query({
+        resource: 'org.ow2.petals:petals-se-ase',
+        depth: 0,
+        metrics: 'test_success_density',
+        includeTrends: true,
+      })
+      .reply(200, [
+        {
+          msr: [
+            {
+              key: 'test_success_density',
+              val: '97',
+            },
+          ],
+        },
+      ])
+  )
  .expectBadge({
    label: 'tests',
-    message: isIntegerPercentage,
+    message: '97%',
  })
--- a/services/sonar/sonar-violations.tester.js
+++ b/services/sonar/sonar-violations.tester.js
@@ -10,9 +10,16 @@ const isViolationsLongFormMetric = Joi.alternatives(
  )
 )

+// The service tests targeting the legacy SonarQube API are mocked
+// because of the lack of publicly accessible, self-hosted, legacy SonarQube instances
+// See https://github.com/badges/shields/issues/4221#issuecomment-546611598 for more details
+// This is an uncommon scenario Shields has to support for Sonar, and should not be used as a model
+// for other service tests.
+
 t.create('Violations')
+  .timeout(10000)
  .get(
-    '/violations/org.ow2.petals%3Apetals-se-ase.json?server=http://sonar.petalslink.com'
+    '/violations/org.sonarsource.sonarqube%3Asonarqube.json?server=https://sonarcloud.io'
  )
  .expectBadge({
    label: 'violations',
@@ -23,14 +30,35 @@ t.create('Violations (legacy API supported)')
  .get(
    '/violations/org.ow2.petals%3Apetals-se-ase.json?server=http://sonar.petalslink.com&sonarVersion=4.2'
  )
+  .intercept(nock =>
+    nock('http://sonar.petalslink.com/api')
+      .get('/resources')
+      .query({
+        resource: 'org.ow2.petals:petals-se-ase',
+        depth: 0,
+        metrics: 'violations',
+        includeTrends: true,
+      })
+      .reply(200, [
+        {
+          msr: [
+            {
+              key: 'violations',
+              val: '7',
+            },
+          ],
+        },
+      ])
+  )
  .expectBadge({
    label: 'violations',
-    message: isMetric,
+    message: '7',
  })

 t.create('Violations Long Format')
+  .timeout(10000)
  .get(
-    '/violations/org.ow2.petals%3Apetals-se-ase.json?server=http://sonar.petalslink.com&format=long'
+    '/violations/org.sonarsource.sonarqube%3Asonarqube.json?server=https://sonarcloud.io&format=long'
  )
  .expectBadge({
    label: 'violations',
@@ -41,14 +69,56 @@ t.create('Violations Long Format (legacy API supported)')
  .get(
    '/violations/org.ow2.petals%3Apetals-se-ase.json?server=http://sonar.petalslink.com&sonarVersion=4.2&format=long'
  )
+  .intercept(nock =>
+    nock('http://sonar.petalslink.com/api')
+      .get('/resources')
+      .query({
+        resource: 'org.ow2.petals:petals-se-ase',
+        depth: 0,
+        metrics:
+          'violations,blocker_violations,critical_violations,major_violations,minor_violations,info_violations',
+        includeTrends: true,
+      })
+      .reply(200, [
+        {
+          msr: [
+            {
+              key: 'violations',
+              val: '10',
+            },
+            {
+              key: 'blocker_violations',
+              val: '1',
+            },
+            {
+              key: 'critical_violations',
+              val: '0',
+            },
+            {
+              key: 'major_violations',
+              val: '2',
+            },
+            {
+              key: 'minor_violations',
+              val: '0',
+            },
+            {
+              key: 'info_violations',
+              val: '7',
+            },
+          ],
+        },
+      ])
+  )
  .expectBadge({
    label: 'violations',
-    message: isViolationsLongFormMetric,
+    message: '1 blocker, 2 major, 7 info',
  })

 t.create('Blocker Violations')
+  .timeout(10000)
  .get(
-    '/blocker_violations/org.ow2.petals%3Apetals-se-ase.json?server=http://sonar.petalslink.com'
+    '/blocker_violations/org.sonarsource.sonarqube%3Asonarqube.json?server=https://sonarcloud.io'
  )
  .expectBadge({
    label: 'blocker violations',
@@ -59,14 +129,35 @@ t.create('Blocker Violations (legacy API supported)')
  .get(
    '/blocker_violations/org.ow2.petals%3Apetals-se-ase.json?server=http://sonar.petalslink.com&sonarVersion=4.2'
  )
+  .intercept(nock =>
+    nock('http://sonar.petalslink.com/api')
+      .get('/resources')
+      .query({
+        resource: 'org.ow2.petals:petals-se-ase',
+        depth: 0,
+        metrics: 'blocker_violations',
+        includeTrends: true,
+      })
+      .reply(200, [
+        {
+          msr: [
+            {
+              key: 'blocker_violations',
+              val: '1',
+            },
+          ],
+        },
+      ])
+  )
  .expectBadge({
    label: 'blocker violations',
-    message: isMetric,
+    message: '1',
  })

 t.create('Critical Violations')
+  .timeout(10000)
  .get(
-    '/critical_violations/org.ow2.petals%3Apetals-se-ase.json?server=http://sonar.petalslink.com'
+    '/critical_violations/org.sonarsource.sonarqube%3Asonarqube.json?server=https://sonarcloud.io'
  )
  .expectBadge({
    label: 'critical violations',
@@ -77,7 +168,27 @@ t.create('Critical Violations (legacy API supported)')
  .get(
    '/critical_violations/org.ow2.petals%3Apetals-se-ase.json?server=http://sonar.petalslink.com&sonarVersion=4.2'
  )
+  .intercept(nock =>
+    nock('http://sonar.petalslink.com/api')
+      .get('/resources')
+      .query({
+        resource: 'org.ow2.petals:petals-se-ase',
+        depth: 0,
+        metrics: 'critical_violations',
+        includeTrends: true,
+      })
+      .reply(200, [
+        {
+          msr: [
+            {
+              key: 'critical_violations',
+              val: '2',
+            },
+          ],
+        },
+      ])
+  )
  .expectBadge({
    label: 'critical violations',
-    message: isMetric,
+    message: '2',
  })