Improve [F-droid] support to .yml metadata format (#2377)

closes: #2083

services/f-droid/f-droid.service.js

@@ -1,20 +1,78 @@
 'use strict'
 
+const Joi = require('joi')
+const yaml = require('js-yaml')
 const BaseService = require('../base')
 const { addv: versionText } = require('../../lib/text-formatters')
 const { version: versionColor } = require('../../lib/color-formatters')
 const { InvalidResponse } = require('../errors')
 
 module.exports = class FDroid extends BaseService {
-  async fetch({ appId }) {
-    // currently, we only use the txt format. There are few apps using the yml format.
-    const url = `https://gitlab.com/fdroid/fdroiddata/raw/master/metadata/${appId}.txt`
-    const { buffer } = await this._request({
-      url,
+  static render({ version }) {
+    return {
+      message: versionText(version),
+      color: versionColor(version),
+    }
+  }
+
+  async handle({ appId }, queryParams) {
+    const constructor = this.constructor
+    const { metadata_format: format } = constructor.validateParams(queryParams)
+    const url = `https://gitlab.com/fdroid/fdroiddata/raw/master/metadata/${appId}`
+    const fetchOpts = {
       options: {},
       errorMessages: {
         404: 'app not found',
       },
+    }
+    const fetch = format === 'yml' ? this.fetchYaml : this.fetchText
+    let result
+
+    try {
+      // currently, we only use the txt format to the initial fetch because
+      // there are more apps with that format but yml is now the standard format
+      // on f-droid, so if txt is not found we look for yml as the fallback
+      result = await fetch.call(this, url, fetchOpts)
+    } catch (error) {
+      if (format) {
+        // if the format was specified it doesn't make the fallback request
+        throw error
+      }
+      result = await this.fetchYaml(url, fetchOpts)
+    }
+
+    return constructor.render(result)
+  }
+
+  async fetchYaml(url, options) {
+    const { buffer } = await this._request({
+      url: `${url}.yml`,
+      ...options,
+    })
+
+    // we assume the yaml layout as provided here:
+    // https://gitlab.com/fdroid/fdroiddata/raw/master/metadata/org.dystopia.email.yml
+    try {
+      const { CurrentVersion: version } = yaml.safeLoad(
+        buffer.toString(),
+        'utf8'
+      )
+      if (!version) {
+        throw new Error('could not find version on website')
+      }
+      return { version }
+    } catch (error) {
+      throw new InvalidResponse({
+        prettyMessage: 'invalid response',
+        underlyingError: error,
+      })
+    }
+  }
+
+  async fetchText(url, options) {
+    const { buffer } = await this._request({
+      url: `${url}.txt`,
+      ...options,
     })
     const metadata = buffer.toString()
     // we assume the layout as provided here:
@@ -25,6 +83,7 @@ module.exports = class FDroid extends BaseService {
     const lastVersion = metadata.substring(
       positionOfCurrentVersionAtEndOfTheFile
     )
+
     const match = lastVersion.match(/^Current Version:\s*(.*?)\s*$/m)
     if (!match) {
       throw new InvalidResponse({
@@ -35,16 +94,12 @@ module.exports = class FDroid extends BaseService {
     return { version: match[1] }
   }
 
-  static render({ version }) {
-    return {
-      message: versionText(version),
-      color: versionColor(version),
-    }
-  }
+  static validateParams(queryParams) {
+    const queryParamsSchema = Joi.object({
+      metadata_format: Joi.string().valid(['yml', 'txt']),
+    }).required()
 
-  async handle({ appId }) {
-    const result = await this.fetch({ appId })
-    return this.constructor.render(result)
+    return this._validateQueryParams(queryParams, queryParamsSchema)
   }
 
   // Metadata
@@ -61,6 +116,7 @@ module.exports = class FDroid extends BaseService {
       base: 'f-droid/v',
       format: '(.+)',
       capture: ['appId'],
+      queryParams: ['metadata_format'],
     }
   }
 
@@ -73,6 +129,14 @@ module.exports = class FDroid extends BaseService {
         staticExample: this.render({ version: '1.0' }),
         keywords: ['fdroid', 'android', 'app'],
       },
+      {
+        title: 'F-Droid (explicit metadata format)',
+        exampleUrl: 'org.dystopia.email',
+        pattern: ':appId',
+        queryParams: { metadata_format: 'yml' },
+        staticExample: this.render({ version: '1.2.1' }),
+        keywords: ['fdroid', 'android', 'app'],
+      },
     ]
   }
 }

services/f-droid/f-droid.tester.js

@@ -6,87 +6,193 @@ const t = new ServiceTester({ id: 'f-droid', title: 'F-Droid' })
 const Joi = require('joi')
 module.exports = t
 
-const testString =
-  'Categories:System\n' +
-  'License:MIT\n' +
-  'Web Site:https://github.com/axxapy/apkExtractor/blob/HEAD/README.md\n' +
-  'Source Code:https://github.com/axxapy/apkExtractor\n' +
-  'Issue Tracker:https://github.com/axxapy/apkExtractor/issues\n' +
-  '\n' +
-  'Auto Name:Apk Extractor\n' +
-  'Summary:Get APK files from installed apps\n' +
-  'Description:\n' +
-  'Extract APKs from your device, even if installed from the Playstore. Root access\n' +
-  'is required for paid apps.\n' +
-  '\n' +
-  '* Fast and easy to use.\n' +
-  '* Extracts almost all applications, includes system applications.\n' +
-  '* ROOT access only required for extracting paid apps.\n' +
-  "* Apk's will be saved in /sdcard/Download/Eimon/.\n" +
-  '* Provided Search option to search applications.\n' +
-  '* Compatible with latest version of Android 6.0\n' +
-  '* Saved apk format : AppPackageName.apk.\n' +
-  'Current Version:1.8\n' +
-  '.\n' +
-  '\n' +
-  'Repo Type:git\n' +
-  'Repo:https://github.com/axxapy/apkExtractor\n' +
-  '\n' +
-  'Build:1.0,1\n' +
-  '    commit=9b3b62c3ceda74b17eaa22c9e4f893aac10c4442\n' +
-  '    gradle=yes\n' +
-  '\n' +
-  'Build:1.1,2\n' +
-  '    commit=1.1\n' +
-  '    gradle=yes\n' +
-  '\n' +
-  'Build:1.2,3\n' +
-  '    disable=lintVitalRelease fails\n' +
-  '    commit=1.2\n' +
-  '    gradle=yes\n' +
-  '\n' +
-  'Build:1.3,4\n' +
-  '    commit=1.3\n' +
-  '    gradle=yes\n' +
-  '\n' +
-  'Build:1.4,5\n' +
-  '    commit=1.4\n' +
-  '    gradle=yes\n' +
-  '\n' +
-  'Auto Update Mode:Version %v\n' +
-  'Update Check Mode:Tags\n' +
-  'Current Version:1.4\n' +
-  'Current Version Code:5\n'
-const base = 'https://gitlab.com'
-const path = '/fdroid/fdroiddata/raw/master/metadata/axp.tool.apkextractor.txt'
+const testString = `
+Categories:System
+License:MIT
+Web Site:https://github.com/axxapy/apkExtractor/blob/HEAD/README.md
+Source Code:https://github.com/axxapy/apkExtractor
+Issue Tracker:https://github.com/axxapy/apkExtractor/issues
 
-t.create('Package is found')
+Auto Name:Apk Extractor
+Summary:Get APK files from installed apps
+Description:
+Extract APKs from your device, even if installed from the Playstore. Root access
+is required for paid apps.
+
+* Fast and easy to use.
+* Extracts almost all applications, includes system applications.
+* ROOT access only required for extracting paid apps.
+* Apks will be saved in /sdcard/Download/Eimon/.
+* Provided Search option to search applications.
+* Compatible with latest version of Android 6.0
+* Saved apk format : AppPackageName.apk.
+Current Version:1.8
+
+Repo Type:git
+Repo:https://github.com/axxapy/apkExtractor
+
+Build:1.0,1
+    commit=9b3b62c3ceda74b17eaa22c9e4f893aac10c4442
+    gradle=yes
+
+Build:1.1,2
+    commit=1.1
+    gradle=yes
+
+Build:1.2,3
+    disable=lintVitalRelease fails
+    commit=1.2
+    gradle=yes
+
+Build:1.3,4
+    commit=1.3
+    gradle=yes
+
+Build:1.4,5
+    commit=1.4
+    gradle=yes
+
+Auto Update Mode:Version %v
+Update Check Mode:Tags
+Current Version:1.4
+Current Version Code:5
+`
+const testYmlString = `
+Categories: System
+License: MIT
+WebSite: https://github.com/axxapy/apkExtractor/blob/HEAD/README.md
+SourceCode: https://github.com/axxapy/apkExtractor
+IssueTracker: https://github.com/axxapy/apkExtractor/issues
+
+AutoName: Apk Extractor
+Summary: Get APK files from installed apps
+Description: |-
+    Extract APKs from your device, even if installed from the Playstore. Root access
+    is required for paid apps.
+
+    * Fast and easy to use.
+    * Extracts almost all applications, includes system applications.
+    * ROOT access only required for extracting paid apps.
+    * Apk's will be saved in /sdcard/Download/Eimon/.
+    * Provided Search option to search applications.
+    * Compatible with latest version of Android 6.0
+    * Saved apk format : AppPackageName.apk.
+
+RepoType: git
+Repo: https://github.com/axxapy/apkExtractor
+
+Builds:
+  - versionName: '1.2'
+    versionCode: 32
+    commit: '0.32'
+    subdir: app
+    gradle:
+      - yes
+
+  - versionName: '1.4'
+    versionCode: 33
+    commit: '5'
+    subdir: app
+    gradle:
+      - yes
+
+AutoUpdateMode: Version %v
+UpdateCheckMode: Tags
+CurrentVersion: 1.4
+CurrentVersionCode: 33
+`
+const base = 'https://gitlab.com'
+const path = '/fdroid/fdroiddata/raw/master/metadata/axp.tool.apkextractor'
+
+t.create('Package is found with default metadata format')
   .get('/v/axp.tool.apkextractor.json')
   .intercept(nock =>
     nock(base)
-      .get(path)
+      .get(`${path}.txt`)
       .reply(200, testString)
   )
   .expectJSON({ name: 'f-droid', value: 'v1.4' })
 
+t.create('Package is found with fallback yml matadata format')
+  .get('/v/axp.tool.apkextractor.json')
+  .intercept(nock =>
+    nock(base)
+      .get(`${path}.txt`)
+      .reply(404)
+  )
+  .intercept(nock =>
+    nock(base)
+      .get(`${path}.yml`)
+      .reply(200, testYmlString)
+  )
+  .expectJSON({ name: 'f-droid', value: 'v1.4' })
+
+t.create('Package is found with yml matadata format')
+  .get('/v/axp.tool.apkextractor.json?metadata_format=yml')
+  .intercept(nock =>
+    nock(base)
+      .get(`${path}.yml`)
+      .reply(200, testYmlString)
+  )
+  .expectJSON({ name: 'f-droid', value: 'v1.4' })
+
+t.create('Package is not found with "metadata_format" query parameter')
+  .get('/v/axp.tool.apkextractor.json?metadata_format=yml')
+  .intercept(nock =>
+    nock(base)
+      .get(`${path}.yml`)
+      .reply(404)
+  )
+  .expectJSON({ name: 'f-droid', value: 'app not found' })
+
+t.create('Package is found yml matadata format with missing "CurrentVersion"')
+  .get('/v/axp.tool.apkextractor.json?metadata_format=yml')
+  .intercept(nock =>
+    nock(base)
+      .get(`${path}.yml`)
+      .reply(200, 'Categories: System')
+  )
+  .expectJSON({ name: 'f-droid', value: 'invalid response' })
+
+t.create('Package is found with bad yml matadata format')
+  .get('/v/axp.tool.apkextractor.json?metadata_format=yml')
+  .intercept(nock =>
+    nock(base)
+      .get(`${path}.yml`)
+      .reply(200, '.CurrentVersion: 1.4')
+  )
+  .expectJSON({ name: 'f-droid', value: 'invalid response' })
+
 t.create('Package is not found')
   .get('/v/axp.tool.apkextractor.json')
   .intercept(nock =>
     nock(base)
-      .get(path)
-      .reply(404, testString)
+      .get(`${path}.txt`)
+      .reply(404)
+  )
+  .intercept(nock =>
+    nock(base)
+      .get(`${path}.yml`)
+      .reply(404)
   )
   .expectJSON({ name: 'f-droid', value: 'app not found' })
 
 t.create('The api changed')
-  .get('/v/axp.tool.apkextractor.json')
+  .get('/v/axp.tool.apkextractor.json?metadata_format=yml')
   .intercept(nock =>
     nock(base)
-      .get(path)
+      .get(`${path}.yml`)
       .reply(200, '')
   )
   .expectJSON({ name: 'f-droid', value: 'invalid response' })
 
+t.create('Package is not found due invalid metadata format')
+  .get('/v/axp.tool.apkextractor.json?metadata_format=xml')
+  .expectJSON({
+    name: 'f-droid',
+    value: 'invalid query parameter: metadata_format',
+  })
+
 /* If this test fails, either the API has changed or the app was deleted. */
 t.create('The real api did not change')
   .get('/v/org.thosp.yourlocalweather.json')