[core] Refactor Tools and add get-feed Tool (#105)

This commit refactors the existing tools, by moving the tools logic to a new `tools.ts` file, so that the main `cmd.ts` file remains clear. Besides that we also add a new tool `get-feed` which can be used to run the `getFeed` function from the command line. The function is called with a source and returns the generated source and items, as they are saved in the database by the `add-source-v1` Supabase edge function.
2026-03-11 17:47:47 -05:00 · 2023-12-17 18:04:53 +01:00
parent 08e9170a80
commit 1cb58e1e0f
6 changed files with 171 additions and 117 deletions
--- a/supabase/functions/_cmd/cmd.ts
+++ b/supabase/functions/_cmd/cmd.ts
@@ -1,8 +1,7 @@
-import { generateKey } from '../_shared/utils/encrypt.ts';
 import { log } from '../_shared/utils/log.ts';
 import { runScheduler } from './scheduler/scheduler.ts';
 import { runWorker } from './worker/worker.ts';
-import { generateAppleSecretKey } from './tools/tools.ts';
+import { runTools } from './tools/tools.ts';

 /**
 * Next to the Supabase Edge functions we also have to create an command which
@@ -26,47 +25,12 @@ const main = (args: string[]) => {
      log('error', 'Worker crashed', { error: err.toString() });
      Deno.exit(1);
    });
-  } else if (
-    args.length === 2 && args[0] === 'tools' &&
-    args[1] === 'generate-key'
-  ) {
-    /**
-     * The "tools generate-key" command can be invoked via the following
-     * command:
-     *   deno run --allow-net --allow-env --import-map=./supabase/functions/import_map.json ./supabase/functions/_cmd/cmd.ts tools generate-key
-     */
-    generateKey().then((data) => {
-      log('info', 'Encryption key was generated', {
-        key: data.rawKey,
-        iv: data.iv,
-      });
+  } else if (args.length >= 2 && args[0] === 'tools') {
+    log('info', 'Start tools...');
+    runTools(args).then(() => {
      Deno.exit(0);
    }).catch((err) => {
-      log('error', 'Failed to generate encryption key', {
-        error: err.toString(),
-      });
-      Deno.exit(1);
-    });
-  } else if (
-    args.length === 6 && args[0] === 'tools' &&
-    args[1] === 'generate-apple-secret-key'
-  ) {
-    /**
-     * The "tools generate-key" command can be invoked via the following
-     * command:
-     *   deno run --allow-env --allow-read --import-map=./supabase/functions/import_map.json ./supabase/functions/_cmd/cmd.ts tools generate-apple-secret-key <KEY-ID> <TEAM-ID> <SERVICE-ID> <FILE>
-     */
-    generateAppleSecretKey(args[2], args[3], args[4], args[5]).then((data) => {
-      log('info', 'Encryption key was generated', {
-        kid: data.kid,
-        exp: new Date(data.exp * 1000).toString(),
-        jwt: data.jwt,
-      });
-      Deno.exit(0);
-    }).catch((err) => {
-      log('error', 'Failed to generate encryption key', {
-        error: err.toString(),
-      });
+      log('error', 'Tools crashed', { error: err.toString() });
      Deno.exit(1);
    });
  } else {
--- a/supabase/functions/_cmd/tools/apple-secret-key.ts
+++ b/supabase/functions/_cmd/tools/apple-secret-key.ts
@@ -0,0 +1,79 @@
+const base64URL = (value: string) => {
+  return globalThis.btoa(value).replace(/[=]/g, '').replace(/[+]/g, '-')
+    .replace(/[\/]/g, '_');
+};
+
+const stringToArrayBuffer = (value: string): ArrayBuffer => {
+  const buf = new ArrayBuffer(value.length);
+  const bufView = new Uint8Array(buf);
+  for (let i = 0; i < value.length; i++) {
+    bufView[i] = value.charCodeAt(i);
+  }
+  return buf;
+};
+
+const arrayBufferToString = (buf: ArrayBuffer): string => {
+  return String.fromCharCode(...new Uint8Array(buf));
+};
+
+export const generateAppleSecretKey = async (
+  kid: string,
+  iss: string,
+  sub: string,
+  file: string,
+): Promise<{ kid: string; jwt: string; exp: number }> => {
+  const contents = await Deno.readTextFile(file);
+
+  if (
+    !contents.match(/^\s*-+BEGIN PRIVATE KEY-+[^-]+-+END PRIVATE KEY-+\s*$/i)
+  ) {
+    throw new Error(
+      `Chosen file does not appear to be a PEM encoded PKCS8 private key file.`,
+    );
+  }
+
+  // remove PEM headers and spaces
+  const pkcs8 = stringToArrayBuffer(
+    globalThis.atob(contents.replace(/-+[^-]+-+/g, '').replace(/\s+/g, '')),
+  );
+
+  const privateKey = await globalThis.crypto.subtle.importKey(
+    'pkcs8',
+    pkcs8,
+    {
+      name: 'ECDSA',
+      namedCurve: 'P-256',
+    },
+    true,
+    ['sign'],
+  );
+
+  const iat = Math.floor(Date.now() / 1000);
+  const exp = iat + 180 * 24 * 60 * 60;
+
+  const jwt = [
+    base64URL(JSON.stringify({ typ: 'JWT', kid, alg: 'ES256' })),
+    base64URL(
+      JSON.stringify({
+        iss,
+        sub,
+        iat,
+        exp,
+        aud: 'https://appleid.apple.com',
+      }),
+    ),
+  ];
+
+  const signature = await globalThis.crypto.subtle.sign(
+    {
+      name: 'ECDSA',
+      hash: 'SHA-256',
+    },
+    privateKey,
+    stringToArrayBuffer(jwt.join('.')),
+  );
+
+  jwt.push(base64URL(arrayBufferToString(signature)));
+
+  return { kid, jwt: jwt.join('.'), exp };
+};
--- a/supabase/functions/_cmd/tools/tools.ts
+++ b/supabase/functions/_cmd/tools/tools.ts
@@ -1,79 +1,80 @@
-const base64URL = (value: string) => {
-  return globalThis.btoa(value).replace(/[=]/g, '').replace(/[+]/g, '-')
-    .replace(/[\/]/g, '_');
-};
+import { createClient } from '@supabase/supabase-js';

-const stringToArrayBuffer = (value: string): ArrayBuffer => {
-  const buf = new ArrayBuffer(value.length);
-  const bufView = new Uint8Array(buf);
-  for (let i = 0; i < value.length; i++) {
-    bufView[i] = value.charCodeAt(i);
-  }
-  return buf;
-};
-
-const arrayBufferToString = (buf: ArrayBuffer): string => {
-  return String.fromCharCode(...new Uint8Array(buf));
-};
-
-export const generateAppleSecretKey = async (
-  kid: string,
-  iss: string,
-  sub: string,
-  file: string,
-): Promise<{ kid: string; jwt: string; exp: number }> => {
-  const contents = await Deno.readTextFile(file);
+import { generateKey } from '../../_shared/utils/encrypt.ts';
+import { log } from '../../_shared/utils/log.ts';
+import { generateAppleSecretKey } from './apple-secret-key.ts';
+import { getFeed } from '../../_shared/feed/feed.ts';

+export const runTools = async (args: string[]): Promise<void> => {
+  /**
+   * The "tools generate-key" command can be invoked via the following command:
+   *   deno run --no-lock --allow-net --allow-env --import-map=./supabase/functions/import_map.json ./supabase/functions/_cmd/cmd.ts tools generate-key
+   */
  if (
-    !contents.match(/^\s*-+BEGIN PRIVATE KEY-+[^-]+-+END PRIVATE KEY-+\s*$/i)
+    args.length === 2 && args[0] === 'tools' &&
+    args[1] === 'generate-key'
  ) {
-    throw new Error(
-      `Chosen file does not appear to be a PEM encoded PKCS8 private key file.`,
-    );
+    const data = await generateKey();
+    log('info', 'Encryption key was generated', {
+      key: data.rawKey,
+      iv: data.iv,
+    });
+    return;
  }

-  // remove PEM headers and spaces
-  const pkcs8 = stringToArrayBuffer(
-    globalThis.atob(contents.replace(/-+[^-]+-+/g, '').replace(/\s+/g, '')),
-  );
+  /**
+   * The "tools generate-apple-secret-key" command can be invoked via the
+   * following command:
+   *   deno run --no-lock --allow-env --allow-read --import-map=./supabase/functions/import_map.json ./supabase/functions/_cmd/cmd.ts tools generate-apple-secret-key <KEY-ID> <TEAM-ID> <SERVICE-ID> <FILE>
+   */
+  if (
+    args.length === 6 && args[0] === 'tools' &&
+    args[1] === 'generate-apple-secret-key'
+  ) {
+    const data = await generateAppleSecretKey(
+      args[2],
+      args[3],
+      args[4],
+      args[5],
+    );
+    log('info', 'Encryption key was generated', {
+      kid: data.kid,
+      exp: new Date(data.exp * 1000).toString(),
+      jwt: data.jwt,
+    });
+    return;
+  }

-  const privateKey = await globalThis.crypto.subtle.importKey(
-    'pkcs8',
-    pkcs8,
-    {
-      name: 'ECDSA',
-      namedCurve: 'P-256',
-    },
-    true,
-    ['sign'],
-  );
-
-  const iat = Math.floor(Date.now() / 1000);
-  const exp = iat + 180 * 24 * 60 * 60;
-
-  const jwt = [
-    base64URL(JSON.stringify({ typ: 'JWT', kid, alg: 'ES256' })),
-    base64URL(
-      JSON.stringify({
-        iss,
-        sub,
-        iat,
-        exp,
-        aud: 'https://appleid.apple.com',
-      }),
-    ),
-  ];
-
-  const signature = await globalThis.crypto.subtle.sign(
-    {
-      name: 'ECDSA',
-      hash: 'SHA-256',
-    },
-    privateKey,
-    stringToArrayBuffer(jwt.join('.')),
-  );
-
-  jwt.push(base64URL(arrayBufferToString(signature)));
-
-  return { kid, jwt: jwt.join('.'), exp };
+  /**
+   * The "tools get-feed" command can be invoked via the following command:
+   *   deno run --no-lock --allow-env --allow-read --allow-net --import-map=./supabase/functions/import_map.json ./supabase/functions/_cmd/cmd.ts tools get-feed <SOURCE>
+   *
+   * The command gets a source and the items for a provided source. The provided
+   * source must contain the type and options to get a feed. All other required
+   * properties for a source are added by the function.
+   *
+   * Example:
+   *   deno run --no-lock --allow-env --allow-read --allow-net --import-map=./supabase/functions/import_map.json ./supabase/functions/_cmd/cmd.ts tools get-feed '{"type": "reddit", "options": {"reddit": "/r/kubernetes"}}'
+   */
+  if (
+    args.length === 3 && args[0] === 'tools' &&
+    args[1] === 'get-feed'
+  ) {
+    const { source, items } = await getFeed(
+      createClient('http://localhost:54321', 'test123'),
+      undefined,
+      { id: '', tier: 'free', createdAt: 0, updatedAt: 0 },
+      {
+        ...JSON.parse(args[2]),
+        id: '',
+        columnId: 'mycolumn',
+        userId: 'myuser',
+      },
+    );
+    log('info', 'Add source', {
+      source: source,
+      items: items,
+    });
+    return;
+  }
 };
--- a/supabase/functions/_shared/feed/nitter_test.ts
+++ b/supabase/functions/_shared/feed/nitter_test.ts
@@ -304,7 +304,6 @@ Deno.test('getNitterFeed - Tag', async () => {

  assertSpyCall(fetchWithTimeoutSpy, 0, {
    args: ['https://nitter.net/search/rss?f=tweets&q=kubernetes', {
-      headers: undefined,
      method: 'get',
    }, 5000],
    returned: new Promise((resolve) => {
@@ -424,7 +423,6 @@ Deno.test('getNitterFeed - User', async () => {

  assertSpyCall(fetchWithTimeoutSpy, 0, {
    args: ['https://nitter.net/rico_berger/rss', {
-      headers: undefined,
      method: 'get',
    }, 5000],
    returned: new Promise((resolve) => {
--- a/supabase/functions/_shared/feed/utils/getAndParseFeed.ts
+++ b/supabase/functions/_shared/feed/utils/getAndParseFeed.ts
@@ -32,7 +32,7 @@ export const getAndParseFeed = async (
    } else {
      utils.log('error', 'Failed to get feed', {
        source: source,
-        error: err,
+        error: err.toString(),
      });
      throw new feedutils.FeedGetAndParseError('Failed to get feed');
    }
@@ -55,8 +55,8 @@ const _parseFeed = async (
      requestUrl: requestUrl,
      responseStatus: response.status,
      responseBody: xml,
-      responseHeaders: response.headers,
-      error: err,
+      responseHeaders: Object.fromEntries(response.headers.entries()),
+      error: err.toString(),
    });
    throw new feedutils.FeedGetAndParseError('Failed to parse feed');
  }