Switch [OpenCollective] badges to use GraphQL and auth (#9387)

* [OpenCollective] update opencollective to api v2 (#9346) * update opencollective to api v2 * fix tests * fix: do not filter by accountType for opencollective/all * remove 404 * remove required in schema * cnt -> count * keep by-tier code as-is --------- Co-authored-by: repo-ranger[bot] <39074581+repo-ranger[bot]@users.noreply.github.com> * allow calling OpenCollective api with an auth token * add test for opencollective auth * cache OpenCollective badges for longer --------- Co-authored-by: xxchan <xxchan22f@gmail.com> Co-authored-by: repo-ranger[bot] <39074581+repo-ranger[bot]@users.noreply.github.com>
2023-08-20 19:24:38 +01:00
parent 692829f91c
commit 8f76982e1c
12 changed files with 229 additions and 241 deletions
--- a/config/custom-environment-variables.yml
+++ b/config/custom-environment-variables.yml
@@ -95,6 +95,7 @@ private:
  obs_user: 'OBS_USER'
  obs_pass: 'OBS_PASS'
  redis_url: 'REDIS_URL'
+  opencollective_token: 'OPENCOLLECTIVE_TOKEN'
  postgres_url: 'POSTGRES_URL'
  sentry_dsn: 'SENTRY_DSN'
  sl_insight_userUuid: 'SL_INSIGHT_USER_UUID'
--- a/core/server/server.js
+++ b/core/server/server.js
@@ -184,6 +184,7 @@ const privateConfigSchema = Joi.object({
  obs_user: Joi.string(),
  obs_pass: Joi.string(),
  redis_url: Joi.string().uri({ scheme: ['redis', 'rediss'] }),
+  opencollective_token: Joi.string(),
  postgres_url: Joi.string().uri({ scheme: 'postgresql' }),
  sentry_dsn: Joi.string(),
  sl_insight_userUuid: Joi.string(),
--- a/doc/server-secrets.md
+++ b/doc/server-secrets.md
@@ -231,7 +231,7 @@ installation access to private npm packages

 [npm token]: https://docs.npmjs.com/getting-started/working_with_tokens

-## Open Build Service
+### Open Build Service

 - `OBS_USER` (yml: `private.obs_user`)
 - `OBS_PASS` (yml: `private.obs_user`)
@@ -246,6 +246,14 @@ they can only be scoped to execute specific actions on a POST request. This
 means however, that an actual account is required to read the build status
 of a package.

+### OpenCollective
+
+- `OPENCOLLECTIVE_TOKEN` (yml: `opencollective_token`)
+
+OpenCollective's GraphQL API only allows 10 reqs/minute for anonymous users.
+An [API token](https://graphql-docs-v2.opencollective.com/access)
+can be provided to access a higher rate limit of 100 reqs/minute.
+
 ### SymfonyInsight (formerly Sensiolabs)

 - `SL_INSIGHT_USER_UUID` (yml: `private.sl_insight_userUuid`)
--- a/services/opencollective/opencollective-all.service.js
+++ b/services/opencollective/opencollective-all.service.js
@@ -16,12 +16,18 @@ export default class OpencollectiveAll extends OpencollectiveBase {
    },
  }

+  static _cacheLength = 900
+
  static defaultBadgeData = {
    label: 'backers and sponsors',
  }

  async handle({ collective }) {
-    const { backersCount } = await this.fetchCollectiveInfo(collective)
+    const data = await this.fetchCollectiveInfo({
+      collective,
+      accountType: [],
+    })
+    const backersCount = this.getCount(data)
    return this.constructor.render(backersCount)
  }
 }
--- a/services/opencollective/opencollective-all.tester.js
+++ b/services/opencollective/opencollective-all.tester.js
@@ -2,25 +2,6 @@ import { nonNegativeInteger } from '../validators.js'
 import { createServiceTester } from '../tester.js'
 export const t = await createServiceTester()

-t.create('renders correctly')
-  .get('/shields.json')
-  .intercept(nock =>
-    nock('https://opencollective.com/').get('/shields.json').reply(200, {
-      slug: 'shields',
-      currency: 'USD',
-      image:
-        'https://opencollective-production.s3-us-west-1.amazonaws.com/44dcbb90-1ee9-11e8-a4c3-7bb1885c0b6e.png',
-      balance: 105494,
-      yearlyIncome: 157371,
-      backersCount: 35,
-      contributorsCount: 276,
-    }),
-  )
-  .expectBadge({
-    label: 'backers and sponsors',
-    message: '35',
-    color: 'brightgreen',
-  })
 t.create('gets amount of backers and sponsors')
  .get('/shields.json')
  .expectBadge({
@@ -28,23 +9,10 @@ t.create('gets amount of backers and sponsors')
    message: nonNegativeInteger,
  })

-t.create('renders not found correctly')
-  .get('/nonexistent-collective.json')
-  .intercept(nock =>
-    nock('https://opencollective.com/')
-      .get('/nonexistent-collective.json')
-      .reply(404, 'Not found'),
-  )
-  .expectBadge({
-    label: 'backers and sponsors',
-    message: 'collective not found',
-    color: 'red',
-  })
-
 t.create('handles not found correctly')
  .get('/nonexistent-collective.json')
  .expectBadge({
    label: 'backers and sponsors',
-    message: 'collective not found',
-    color: 'red',
+    message: 'No collective found with slug nonexistent-collective',
+    color: 'lightgrey',
  })
--- a/services/opencollective/opencollective-backers.service.js
+++ b/services/opencollective/opencollective-backers.service.js
@@ -16,15 +16,19 @@ export default class OpencollectiveBackers extends OpencollectiveBase {
    },
  }

+  static _cacheLength = 900
+
  static defaultBadgeData = {
    label: 'backers',
  }

  async handle({ collective }) {
-    const { backersCount } = await this.fetchCollectiveBackersCount(
+    const data = await this.fetchCollectiveInfo({
      collective,
-      { userType: 'users' },
-    )
+      accountType: ['INDIVIDUAL'],
+    })
+    const backersCount = this.getCount(data)
+
    return this.constructor.render(backersCount)
  }
 }
--- a/services/opencollective/opencollective-backers.tester.js
+++ b/services/opencollective/opencollective-backers.tester.js
@@ -2,80 +2,6 @@ import { nonNegativeInteger } from '../validators.js'
 import { createServiceTester } from '../tester.js'
 export const t = await createServiceTester()

-t.create('renders correctly')
-  .get('/shields.json')
-  .intercept(nock =>
-    nock('https://opencollective.com/')
-      .get('/shields/members/users.json')
-      .reply(200, [
-        { MemberId: 8685, type: 'USER', role: 'ADMIN' },
-        { MemberId: 8686, type: 'USER', role: 'ADMIN' },
-        { MemberId: 8682, type: 'USER', role: 'ADMIN' },
-        { MemberId: 10305, type: 'USER', role: 'BACKER', tier: 'backer' },
-        { MemberId: 10396, type: 'USER', role: 'BACKER', tier: 'backer' },
-        { MemberId: 10733, type: 'USER', role: 'BACKER' },
-        { MemberId: 8684, type: 'USER', role: 'ADMIN' },
-        { MemberId: 10741, type: 'USER', role: 'BACKER' },
-        {
-          MemberId: 10756,
-          type: 'USER',
-          role: 'BACKER',
-          tier: 'monthly backer',
-        },
-        { MemberId: 11578, type: 'USER', role: 'CONTRIBUTOR' },
-        { MemberId: 13459, type: 'USER', role: 'CONTRIBUTOR' },
-        {
-          MemberId: 13507,
-          type: 'USER',
-          role: 'BACKER',
-          tier: 'monthly backer',
-        },
-        { MemberId: 13512, type: 'USER', role: 'BACKER' },
-        { MemberId: 13513, type: 'USER', role: 'FUNDRAISER' },
-        { MemberId: 13984, type: 'USER', role: 'BACKER', tier: 'backer' },
-        { MemberId: 14916, type: 'USER', role: 'BACKER' },
-        {
-          MemberId: 16326,
-          type: 'USER',
-          role: 'BACKER',
-          tier: 'monthly backer',
-        },
-        { MemberId: 18252, type: 'USER', role: 'BACKER', tier: 'backer' },
-        { MemberId: 17631, type: 'USER', role: 'BACKER', tier: 'backer' },
-        {
-          MemberId: 16420,
-          type: 'USER',
-          role: 'BACKER',
-          tier: 'monthly backer',
-        },
-        { MemberId: 17186, type: 'USER', role: 'BACKER', tier: 'backer' },
-        { MemberId: 18791, type: 'USER', role: 'BACKER', tier: 'backer' },
-        {
-          MemberId: 19279,
-          type: 'USER',
-          role: 'BACKER',
-          tier: 'monthly backer',
-        },
-        { MemberId: 19863, type: 'USER', role: 'BACKER', tier: 'backer' },
-        { MemberId: 21451, type: 'USER', role: 'BACKER', tier: 'backer' },
-        { MemberId: 22718, type: 'USER', role: 'BACKER' },
-        { MemberId: 23561, type: 'USER', role: 'BACKER', tier: 'backer' },
-        { MemberId: 25092, type: 'USER', role: 'CONTRIBUTOR' },
-        { MemberId: 24473, type: 'USER', role: 'BACKER', tier: 'backer' },
-        { MemberId: 25439, type: 'USER', role: 'BACKER', tier: 'backer' },
-        { MemberId: 24483, type: 'USER', role: 'BACKER', tier: 'backer' },
-        { MemberId: 25090, type: 'USER', role: 'CONTRIBUTOR' },
-        { MemberId: 26404, type: 'USER', role: 'BACKER', tier: 'backer' },
-        { MemberId: 27026, type: 'USER', role: 'BACKER', tier: 'backer' },
-        { MemberId: 27132, type: 'USER', role: 'CONTRIBUTOR' },
-      ]),
-  )
-  .expectBadge({
-    label: 'backers',
-    message: '25',
-    color: 'brightgreen',
-  })
-
 t.create('gets amount of backers').get('/shields.json').expectBadge({
  label: 'backers',
  message: nonNegativeInteger,
@@ -85,6 +11,6 @@ t.create('handles not found correctly')
  .get('/nonexistent-collective.json')
  .expectBadge({
    label: 'backers',
-    message: 'collective not found',
-    color: 'red',
+    message: 'No collective found with slug nonexistent-collective',
+    color: 'lightgrey',
  })
--- a/services/opencollective/opencollective-base.js
+++ b/services/opencollective/opencollective-base.js
@@ -1,28 +1,38 @@
+import gql from 'graphql-tag'
 import Joi from 'joi'
+import { BaseGraphqlService } from '../index.js'
 import { nonNegativeInteger } from '../validators.js'
-import { BaseJsonService } from '../index.js'
 import { metric } from '../text-formatters.js'

-// https://developer.opencollective.com/#/api/collectives?id=get-info
-const collectiveDetailsSchema = Joi.object().keys({
-  slug: Joi.string().required(),
-  backersCount: nonNegativeInteger,
-})
+const schema = Joi.object({
+  data: Joi.object({
+    account: Joi.object({
+      name: Joi.string(),
+      slug: Joi.string(),
+      members: Joi.object({
+        totalCount: nonNegativeInteger,
+        nodes: Joi.array().items(
+          Joi.object({
+            tier: Joi.object({
+              legacyId: Joi.number(),
+              name: Joi.string(),
+            }).allow(null),
+          }),
+        ),
+      }).required(),
+    }).required(),
+  }).required(),
+}).required()

-// https://developer.opencollective.com/#/api/collectives?id=get-members
-function buildMembersArraySchema({ userType, tierRequired }) {
-  const keys = {
-    MemberId: Joi.number().required(),
-    type: userType || Joi.string().required(),
-    role: Joi.string().required(),
-  }
-  if (tierRequired) keys.tier = Joi.string().required()
-  return Joi.array().items(Joi.object().keys(keys))
-}
-
-export default class OpencollectiveBase extends BaseJsonService {
+export default class OpencollectiveBase extends BaseGraphqlService {
  static category = 'funding'

+  static auth = {
+    passKey: 'opencollective_token',
+    authorizedOrigins: ['https://api.opencollective.com'],
+    isRequired: false,
+  }
+
  static buildRoute(base, withTierId) {
    return {
      base: `opencollective${base ? `/${base}` : ''}`,
@@ -38,45 +48,51 @@ export default class OpencollectiveBase extends BaseJsonService {
    }
  }

-  async fetchCollectiveInfo(collective) {
-    return this._requestJson({
-      schema: collectiveDetailsSchema,
-      // https://developer.opencollective.com/#/api/collectives?id=get-info
-      url: `https://opencollective.com/${collective}.json`,
-      httpErrors: {
-        404: 'collective not found',
-      },
-    })
+  async fetchCollectiveInfo({ collective, accountType }) {
+    return this._requestGraphql(
+      this.authHelper.withQueryStringAuth(
+        { passKey: 'personalToken' },
+        {
+          schema,
+          url: 'https://api.opencollective.com/graphql/v2',
+          query: gql`
+            query account($slug: String, $accountType: [AccountType]) {
+              account(slug: $slug) {
+                name
+                slug
+                members(accountType: $accountType, role: BACKER) {
+                  totalCount
+                  nodes {
+                    tier {
+                      legacyId
+                      name
+                    }
+                  }
+                }
+              }
+            }
+          `,
+          variables: {
+            slug: collective,
+            accountType,
+          },
+          options: {
+            headers: { 'content-type': 'application/json' },
+          },
+        },
+      ),
+    )
  }

-  async fetchCollectiveBackersCount(collective, { userType, tierId }) {
-    const schema = buildMembersArraySchema({
-      userType:
-        userType === 'users'
-          ? 'USER'
-          : userType === 'organizations'
-          ? 'ORGANIZATION'
-          : undefined,
-      tierRequired: tierId,
-    })
-    const members = await this._requestJson({
-      schema,
-      // https://developer.opencollective.com/#/api/collectives?id=get-members
-      // https://developer.opencollective.com/#/api/collectives?id=get-members-per-tier
-      url: `https://opencollective.com/${collective}/members/${
-        userType || 'all'
-      }.json${tierId ? `?TierId=${tierId}` : ''}`,
-      httpErrors: {
-        404: 'collective not found',
+  getCount(data) {
+    const {
+      data: {
+        account: {
+          members: { totalCount },
+        },
      },
-    })
+    } = data

-    const result = {
-      backersCount: members.filter(member => member.role === 'BACKER').length,
-    }
-    // Find the title of the tier
-    if (tierId && members.length > 0)
-      result.tier = members.map(member => member.tier)[0]
-    return result
+    return totalCount
  }
 }
--- a/services/opencollective/opencollective-base.spec.js
+++ b/services/opencollective/opencollective-base.spec.js
@@ -0,0 +1,37 @@
+import { expect } from 'chai'
+import nock from 'nock'
+import { cleanUpNockAfterEach, defaultContext } from '../test-helpers.js'
+import OpencollectiveBase from './opencollective-base.js'
+
+class DummyOpencollectiveService extends OpencollectiveBase {
+  static route = this.buildRoute('dummy')
+
+  async handle({ collective }) {
+    const data = await this.fetchCollectiveInfo({
+      collective,
+      accountType: [],
+    })
+    return this.constructor.render(this.getCount(data))
+  }
+}
+
+describe('OpencollectiveBase', function () {
+  describe('auth', function () {
+    cleanUpNockAfterEach()
+
+    const config = { private: { opencollective_token: 'fake-token' } }
+
+    it('sends the auth information as configured', async function () {
+      const scope = nock('https://api.opencollective.com')
+        .post('/graphql/v2')
+        .query({ personalToken: 'fake-token' })
+        .reply(200, { data: { account: { members: { totalCount: 1 } } } })
+
+      expect(
+        await DummyOpencollectiveService.invoke(defaultContext, config, {}),
+      ).to.deep.equal({ color: 'brightgreen', label: undefined, message: '1' })
+
+      scope.done()
+    })
+  })
+})
--- a/services/opencollective/opencollective-by-tier.service.js
+++ b/services/opencollective/opencollective-by-tier.service.js
@@ -1,9 +1,91 @@
-import OpencollectiveBase from './opencollective-base.js'
+import Joi from 'joi'
+import { nonNegativeInteger } from '../validators.js'
+import { BaseJsonService } from '../index.js'
+import { metric } from '../text-formatters.js'

 const documentation = `<h3>How to get the tierId</h3>
 <p>According to <a target="_blank" href="https://developer.opencollective.com/#/api/collectives?id=get-members-per-tier">open collectives documentation</a>, you can find the tierId by looking at the URL after clicking on a Tier Card on the collective page. (e.g. tierId for https://opencollective.com/shields/order/2988 is 2988)</p>`

-export default class OpencollectiveByTier extends OpencollectiveBase {
+// https://developer.opencollective.com/#/api/collectives?id=get-info
+const collectiveDetailsSchema = Joi.object().keys({
+  slug: Joi.string().required(),
+  backersCount: nonNegativeInteger,
+})
+
+// https://developer.opencollective.com/#/api/collectives?id=get-members
+function buildMembersArraySchema({ userType, tierRequired }) {
+  const keys = {
+    MemberId: Joi.number().required(),
+    type: userType || Joi.string().required(),
+    role: Joi.string().required(),
+  }
+  if (tierRequired) keys.tier = Joi.string().required()
+  return Joi.array().items(Joi.object().keys(keys))
+}
+
+class OpencollectiveBaseJson extends BaseJsonService {
+  static category = 'funding'
+
+  static buildRoute(base, withTierId) {
+    return {
+      base: `opencollective${base ? `/${base}` : ''}`,
+      pattern: `:collective${withTierId ? '/:tierId' : ''}`,
+    }
+  }
+
+  static render(backersCount, label) {
+    return {
+      label,
+      message: metric(backersCount),
+      color: backersCount > 0 ? 'brightgreen' : 'lightgrey',
+    }
+  }
+
+  async fetchCollectiveInfo(collective) {
+    return this._requestJson({
+      schema: collectiveDetailsSchema,
+      // https://developer.opencollective.com/#/api/collectives?id=get-info
+      url: `https://opencollective.com/${collective}.json`,
+      httpErrors: {
+        404: 'collective not found',
+      },
+    })
+  }
+
+  async fetchCollectiveBackersCount(collective, { userType, tierId }) {
+    const schema = buildMembersArraySchema({
+      userType:
+        userType === 'users'
+          ? 'USER'
+          : userType === 'organizations'
+          ? 'ORGANIZATION'
+          : undefined,
+      tierRequired: tierId,
+    })
+    const members = await this._requestJson({
+      schema,
+      // https://developer.opencollective.com/#/api/collectives?id=get-members
+      // https://developer.opencollective.com/#/api/collectives?id=get-members-per-tier
+      url: `https://opencollective.com/${collective}/members/${
+        userType || 'all'
+      }.json${tierId ? `?TierId=${tierId}` : ''}`,
+      httpErrors: {
+        404: 'collective not found',
+      },
+    })
+
+    const result = {
+      backersCount: members.filter(member => member.role === 'BACKER').length,
+    }
+    // Find the title of the tier
+    if (tierId && members.length > 0)
+      result.tier = members.map(member => member.tier)[0]
+    return result
+  }
+}
+
+// TODO: 1. pagination is needed. 2. use new graphql api instead of legacy rest api
+export default class OpencollectiveByTier extends OpencollectiveBaseJson {
  static route = this.buildRoute('tier', true)

  static examples = [
--- a/services/opencollective/opencollective-sponsors.service.js
+++ b/services/opencollective/opencollective-sponsors.service.js
@@ -16,15 +16,18 @@ export default class OpencollectiveSponsors extends OpencollectiveBase {
    },
  }

+  static _cacheLength = 900
+
  static defaultBadgeData = {
    label: 'sponsors',
  }

  async handle({ collective }) {
-    const { backersCount } = await this.fetchCollectiveBackersCount(
+    const data = await this.fetchCollectiveInfo({
      collective,
-      { userType: 'organizations' },
-    )
+      accountType: ['ORGANIZATION'],
+    })
+    const backersCount = this.getCount(data)
    return this.constructor.render(backersCount)
  }
 }
--- a/services/opencollective/opencollective-sponsors.tester.js
+++ b/services/opencollective/opencollective-sponsors.tester.js
@@ -2,80 +2,16 @@ import { nonNegativeInteger } from '../validators.js'
 import { createServiceTester } from '../tester.js'
 export const t = await createServiceTester()

-t.create('renders correctly')
-  .get('/shields.json')
-  .intercept(nock =>
-    nock('https://opencollective.com/')
-      .get('/shields/members/organizations.json')
-      .reply(200, [
-        { MemberId: 8683, type: 'ORGANIZATION', role: 'HOST' },
-        {
-          MemberId: 13484,
-          type: 'ORGANIZATION',
-          role: 'BACKER',
-          tier: 'backer',
-        },
-        { MemberId: 13508, type: 'ORGANIZATION', role: 'FUNDRAISER' },
-        { MemberId: 15987, type: 'ORGANIZATION', role: 'BACKER' },
-        {
-          MemberId: 16561,
-          type: 'ORGANIZATION',
-          role: 'BACKER',
-          tier: 'sponsor',
-        },
-        {
-          MemberId: 16469,
-          type: 'ORGANIZATION',
-          role: 'BACKER',
-          tier: 'sponsor',
-        },
-        {
-          MemberId: 18162,
-          type: 'ORGANIZATION',
-          role: 'BACKER',
-          tier: 'sponsor',
-        },
-        {
-          MemberId: 21023,
-          type: 'ORGANIZATION',
-          role: 'BACKER',
-          tier: 'sponsor',
-        },
-        {
-          MemberId: 21482,
-          type: 'ORGANIZATION',
-          role: 'BACKER',
-          tier: 'monthly backer',
-        },
-        {
-          MemberId: 26367,
-          type: 'ORGANIZATION',
-          role: 'BACKER',
-          tier: 'monthly backer',
-        },
-        { MemberId: 27531, type: 'ORGANIZATION', role: 'BACKER' },
-        {
-          MemberId: 29443,
-          type: 'ORGANIZATION',
-          role: 'BACKER',
-          tier: 'monthly backer',
-        },
-      ]),
-  )
-  .expectBadge({
-    label: 'sponsors',
-    message: '10',
-    color: 'brightgreen',
-  })
 t.create('gets amount of sponsors').get('/shields.json').expectBadge({
  label: 'sponsors',
  message: nonNegativeInteger,
+  color: 'brightgreen',
 })

 t.create('handles not found correctly')
  .get('/nonexistent-collective.json')
  .expectBadge({
    label: 'sponsors',
-    message: 'collective not found',
-    color: 'red',
+    message: 'No collective found with slug nonexistent-collective',
+    color: 'lightgrey',
  })