From 9e59439226734b1d34d800fc2c5e7d893fb01259 Mon Sep 17 00:00:00 2001 From: Rico Berger Date: Tue, 12 Dec 2023 18:50:29 +0100 Subject: [PATCH] [core] Add Tests for Sources (#98) This commit adds tests for all available sources. This commit also fixes the parsing of Atom feeds for the RSS source, where the `dc:date` field must be used for the `publishedAt` field. --- .github/workflows/continuous-integration.yaml | 2 +- supabase/functions/_shared/feed/googlenews.ts | 11 +- .../functions/_shared/feed/googlenews_test.ts | 272 ++++ supabase/functions/_shared/feed/lemmy.ts | 7 +- supabase/functions/_shared/feed/lemmy_test.ts | 511 ++++++ supabase/functions/_shared/feed/mastodon.ts | 11 +- .../functions/_shared/feed/mastodon_test.ts | 316 ++++ supabase/functions/_shared/feed/medium.ts | 105 +- .../functions/_shared/feed/medium_test.ts | 369 ++++- supabase/functions/_shared/feed/nitter.ts | 13 +- .../functions/_shared/feed/nitter_test.ts | 457 ++++++ supabase/functions/_shared/feed/pinterest.ts | 58 +- .../functions/_shared/feed/pinterest_test.ts | 188 +++ supabase/functions/_shared/feed/podcast.ts | 15 +- .../functions/_shared/feed/podcast_test.ts | 504 ++++++ supabase/functions/_shared/feed/reddit.ts | 7 +- .../functions/_shared/feed/reddit_test.ts | 228 +++ supabase/functions/_shared/feed/rss.ts | 40 +- supabase/functions/_shared/feed/rss_test.ts | 1370 +++++++++++++++++ .../functions/_shared/feed/stackoverflow.ts | 15 +- .../_shared/feed/stackoverflow_test.ts | 203 +++ supabase/functions/_shared/feed/tumblr.ts | 7 +- .../functions/_shared/feed/tumblr_test.ts | 178 +++ .../functions/_shared/feed/utils/index.ts | 12 + supabase/functions/_shared/feed/utils/test.ts | 32 + supabase/functions/_shared/feed/youtube.ts | 22 +- .../functions/_shared/feed/youtube_test.ts | 364 +++++ supabase/functions/_shared/utils/index.ts | 7 + supabase/functions/import_map.json | 1 + 29 files changed, 5172 insertions(+), 153 deletions(-) create mode 100644 supabase/functions/_shared/feed/googlenews_test.ts create mode 100644 supabase/functions/_shared/feed/lemmy_test.ts create mode 100644 supabase/functions/_shared/feed/mastodon_test.ts create mode 100644 supabase/functions/_shared/feed/nitter_test.ts create mode 100644 supabase/functions/_shared/feed/pinterest_test.ts create mode 100644 supabase/functions/_shared/feed/podcast_test.ts create mode 100644 supabase/functions/_shared/feed/reddit_test.ts create mode 100644 supabase/functions/_shared/feed/rss_test.ts create mode 100644 supabase/functions/_shared/feed/stackoverflow_test.ts create mode 100644 supabase/functions/_shared/feed/tumblr_test.ts create mode 100644 supabase/functions/_shared/feed/utils/index.ts create mode 100644 supabase/functions/_shared/feed/utils/test.ts create mode 100644 supabase/functions/_shared/feed/youtube_test.ts create mode 100644 supabase/functions/_shared/utils/index.ts diff --git a/.github/workflows/continuous-integration.yaml b/.github/workflows/continuous-integration.yaml index 32b9717..84ff549 100644 --- a/.github/workflows/continuous-integration.yaml +++ b/.github/workflows/continuous-integration.yaml @@ -50,4 +50,4 @@ jobs: - name: Test run: | - deno test --import-map=supabase/functions/import_map.json supabase/functions + deno test --allow-env --import-map=supabase/functions/import_map.json supabase/functions diff --git a/supabase/functions/_shared/feed/googlenews.ts b/supabase/functions/_shared/feed/googlenews.ts index 22e177c..9f60bbe 100644 --- a/supabase/functions/_shared/feed/googlenews.ts +++ b/supabase/functions/_shared/feed/googlenews.ts @@ -7,10 +7,9 @@ import { Redis } from 'redis'; import { ISource } from '../models/source.ts'; import { IItem } from '../models/item.ts'; -import { getFavicon } from './utils/getFavicon.ts'; import { IProfile } from '../models/profile.ts'; -import { fetchWithTimeout } from '../utils/fetchWithTimeout.ts'; -import { log } from '../utils/log.ts'; +import { utils } from '../utils/index.ts'; +import { feedutils } from './utils/index.ts'; export const getGooglenewsFeed = async ( _supabaseClient: SupabaseClient, @@ -63,11 +62,11 @@ export const getGooglenewsFeed = async ( * Get the RSS for the provided `googlenews` url and parse it. If a feed * doesn't contains an item we return an error. */ - const response = await fetchWithTimeout(source.options.googlenews.url, { + const response = await utils.fetchWithTimeout(source.options.googlenews.url, { method: 'get', }, 5000); const xml = await response.text(); - log('debug', 'Add source', { + utils.log('debug', 'Add source', { sourceType: 'googlenews', requestUrl: source.options.googlenews.url, responseStatus: response.status, @@ -222,7 +221,7 @@ const getMedia = async ( return cachedMediaURL; } - const favicon = await getFavicon(entry.source?.url); + const favicon = await feedutils.getFavicon(entry.source?.url); if (favicon && favicon.url.startsWith('https://')) { await redisClient.set(cacheKey, favicon.url); return favicon.url; diff --git a/supabase/functions/_shared/feed/googlenews_test.ts b/supabase/functions/_shared/feed/googlenews_test.ts new file mode 100644 index 0000000..7456942 --- /dev/null +++ b/supabase/functions/_shared/feed/googlenews_test.ts @@ -0,0 +1,272 @@ +import { createClient } from '@supabase/supabase-js'; +import { + assertSpyCall, + assertSpyCalls, + returnsNext, + stub, +} from 'std/testing/mock'; + +import { ISource } from '../models/source.ts'; +import { IProfile } from '../models/profile.ts'; +import { getGooglenewsFeed } from './googlenews.ts'; +import { utils } from '../utils/index.ts'; +import { feedutils } from './utils/index.ts'; + +const supabaseClient = createClient('http://localhost:54321', 'test123'); +const mockProfile: IProfile = { + id: '', + tier: 'free', + createdAt: 0, + updatedAt: 0, +}; +const mockSource: ISource = { + id: '', + columnId: 'mycolumn', + userId: 'myuser', + type: 'medium', + title: '', +}; + +const responseUrl = ` + + + NFE/5.0 + Schlagzeilen - Aktuell - Google News + https://news.google.com/topics/CAAqJggKIiBDQkFTRWdvSUwyMHZNRFZxYUdjU0FtUmxHZ0pFUlNnQVAB?hl=de &gl=DE &ceid=DE:de + de + news-webmaster@google.com + 2023 Google Inc. + Wed, 06 Dec 2023 20:10:12 GMT + Google News + + GDL ruft von Donnerstagabend an zu eintägigem Streik auf - tagesschau.de + https://news.google.com/rss/articles/CBMiOGh0dHBzOi8vd3d3LnRhZ2Vzc2NoYXUuZGUvd2lydHNjaGFmdC9nZGwtc3RyZWlrLTE5MC5odG1s0gEA?oc=5 + CBMiOGh0dHBzOi8vd3d3LnRhZ2Vzc2NoYXUuZGUvd2lydHNjaGFmdC9nZGwtc3RyZWlrLTE5MC5odG1s0gEA + Wed, 06 Dec 2023 19:20:00 GMT + <ol ><li ><a href="https://news.google.com/rss/articles/CBMiOGh0dHBzOi8vd3d3LnRhZ2Vzc2NoYXUuZGUvd2lydHNjaGFmdC9nZGwtc3RyZWlrLTE5MC5odG1s0gEA?oc=5" target="_blank">GDL ruft von Donnerstagabend an zu eintägigem Streik auf </a >&nbsp;&nbsp;<font color="#6f6f6f">tagesschau.de </font ></li ><li ><a href="https://news.google.com/rss/articles/CBMidWh0dHBzOi8vd3d3LmJpbGQuZGUvcG9saXRpay9pbmxhbmQvcG9saXRpay1pbmxhbmQvZGV1dHNjaGUtYmFobi1tb3JnZW4tbmFlY2hzdGVyLWxva2Z1ZWhyZXItc3RyZWlrLTg2MzQ4NjM0LmJpbGQuaHRtbNIBAA?oc=5" target="_blank">Deutsche Bahn: Morgen nächster Lokführer-Streik! | Politik </a >&nbsp;&nbsp;<font color="#6f6f6f">BILD </font ></li ><li ><a href="https://news.google.com/rss/articles/CBMihAFodHRwczovL3d3dy5uZHIuZGUvbmFjaHJpY2h0ZW4vc2NobGVzd2lnLWhvbHN0ZWluL2t1cnpuYWNocmljaHRlbi9TY2hsZXN3aWctSG9sc3RlaW4tYWt0dWVsbC1OYWNocmljaHRlbi1pbS1VZWJlcmJsaWNrLG5ld3MzNTU2Lmh0bWzSAQA?oc=5" target="_blank">Schleswig-Holstein aktuell: Nachrichten im Überblick </a >&nbsp;&nbsp;<font color="#6f6f6f">NDR.de </font ></li ><li ><a href="https://news.google.com/rss/articles/CBMiiAFodHRwczovL3d3dy5sci1vbmxpbmUuZGUvbGF1c2l0ei9jb3R0YnVzL3JlMi1jb3R0YnVzLWJlcmxpbi1zdHJlY2tlLXdpcmQtd2llZGVyLWdlc3BlcnJ0LV8td2FzLWZhaHJnYWVzdGUtYmVhY2h0ZW4tbXVlc3Nlbi03MjQ0MjYwOS5odG1s0gEA?oc=5" target="_blank">Bahnstreik und Sperrung - Nichts geht mehr beim RE2 Cottbus – Berlin </a >&nbsp;&nbsp;<font color="#6f6f6f">Lausitzer Rundschau </font ></li ><li ><a href="https://news.google.com/rss/articles/CBMingFodHRwczovL3d3dy5zcGllZ2VsLmRlL3dpcnRzY2hhZnQvc2VydmljZS9kZXV0c2NoZS1iYWhuLWdkbC1sb2tmdWVocmVyLWRlci1iYWhuLXN0cmVpa2VuLWFiLWRvbm5lcnN0YWctYmlzLWZyZWl0YWdhYmVuZC1hLTJhN2Q1OWUyLTBmOTUtNDVhYi04OTFkLTI4NjcxMGI1MWI1MdIBAA?oc=5" target="_blank">Deutsche Bahn/GDL: Lokführer der Bahn streiken ab Donnerstag bis Freitagabend </a >&nbsp;&nbsp;<font color="#6f6f6f">DER SPIEGEL </font ></li ></ol > + tagesschau.de + + + Lassen die USA die Ukraine im Stich?: US-Finanzministerin spricht von „katastrophaler Situation“ - Tagesspiegel + https://news.google.com/rss/articles/CBMilgFodHRwczovL3d3dy50YWdlc3NwaWVnZWwuZGUvaW50ZXJuYXRpb25hbGVzL2xhc3Nlbi1kaWUtdXNhLWRpZS11a3JhaW5lLWltLXN0aWNoLXVzLWZpbmFuem1pbmlzdGVyaW4tc3ByaWNodC12b24ta2F0YXN0cm9waGFsZXItc2l0dWF0aW9uLTEwODg3Nzg3Lmh0bWzSAQA?oc=5 + CBMilgFodHRwczovL3d3dy50YWdlc3NwaWVnZWwuZGUvaW50ZXJuYXRpb25hbGVzL2xhc3Nlbi1kaWUtdXNhLWRpZS11a3JhaW5lLWltLXN0aWNoLXVzLWZpbmFuem1pbmlzdGVyaW4tc3ByaWNodC12b24ta2F0YXN0cm9waGFsZXItc2l0dWF0aW9uLTEwODg3Nzg3Lmh0bWzSAQA + Wed, 06 Dec 2023 15:38:00 GMT + <ol ><li ><a href="https://news.google.com/rss/articles/CBMilgFodHRwczovL3d3dy50YWdlc3NwaWVnZWwuZGUvaW50ZXJuYXRpb25hbGVzL2xhc3Nlbi1kaWUtdXNhLWRpZS11a3JhaW5lLWltLXN0aWNoLXVzLWZpbmFuem1pbmlzdGVyaW4tc3ByaWNodC12b24ta2F0YXN0cm9waGFsZXItc2l0dWF0aW9uLTEwODg3Nzg3Lmh0bWzSAQA?oc=5" target="_blank">Lassen die USA die Ukraine im Stich?: US-Finanzministerin spricht von „katastrophaler Situation“</a >&nbsp;&nbsp;<font color="#6f6f6f">Tagesspiegel </font ></li ><li ><a href="https://news.google.com/rss/articles/CCAiC0lETWpHLXRoZ3ZNmAEB?oc=5" target="_blank">Rüstungskonferenz in Washington: Ukraine-Hilfen der USA laufen aus </a >&nbsp;&nbsp;<font color="#6f6f6f">tagesschau </font ></li ><li ><a href="https://news.google.com/rss/articles/CBMiiwFodHRwczovL3d3dy5iaWxkLmRlL3BvbGl0aWsvYXVzbGFuZC9wb2xpdGlrLWF1c2xhbmQvc2VsZW5za3lqLXNhZ3QtYXVmdHJpdHQtdm9yLXVzLXNlbmF0b3Jlbi1hYi1ldHdhcy1kYXp3aXNjaGVuZ2Vrb21tZW4tODYzMzg3OTIuYmlsZC5odG1s0gEA?oc=5" target="_blank">Selenskyj sagt Auftritt vor US-Senatoren ab: „Etwas dazwischengekommen“</a >&nbsp;&nbsp;<font color="#6f6f6f">BILD </font ></li ><li ><a href="https://news.google.com/rss/articles/CBMic2h0dHBzOi8vd3d3LnQtb25saW5lLmRlL25hY2hyaWNodGVuL2F1c2xhbmQvdXNhL3VzLXdhaGwvaWRfMTAwMjk2MDQ4L3VzYS1oYWJlbi1rZWluLWdlbGQtbWVoci1mdWVyLWRpZS11a3JhaW5lLmh0bWzSAQA?oc=5" target="_blank">USA haben kein Geld mehr für die Ukraine </a >&nbsp;&nbsp;<font color="#6f6f6f">t-online </font ></li ><li ><a href="https://news.google.com/rss/articles/CBMibGh0dHBzOi8vd3d3Lm1vcmdlbnBvc3QuZGUvcG9saXRpay9hcnRpY2xlMjQwNzUyNTI2L1VrcmFpbmUtS3JpZWctQW1lcmlrYS1kYXJmLW5pY2h0LXZvbi1kZXItRmFobmUtZ2VoZW4uaHRtbNIBAA?oc=5" target="_blank">Ukraine-Krieg: Putins wichtigste Unterstützer sitzen in Washington </a >&nbsp;&nbsp;<font color="#6f6f6f">Berliner Morgenpost </font ></li ></ol > + Tagesspiegel + + +`; + +const responseSearch = ` + + + NFE/5.0 + "Chemnitz" - Google News + https://news.google.com/search?q=Chemnitz &hl=de &gl=DE &ceid=DE:de + de + news-webmaster@google.com + 2023 Google Inc. + Wed, 06 Dec 2023 20:19:34 GMT + Google News + + Chemnitz: Preisschock bei "eins energie"! Fernwärme wird 75 Prozent teurer - TAG24 + https://news.google.com/rss/articles/CBMibGh0dHBzOi8vd3d3LnRhZzI0LmRlL2NoZW1uaXR6L2xva2FsZXMvcHJlaXNzY2hvY2stYmVpLWVpbnMtZW5lcmdpZS1mZXJud2Flcm1lLXdpcmQtNzUtcHJvemVudC10ZXVyZXItMzAzMjA5MdIBcGh0dHBzOi8vd3d3LnRhZzI0LmRlL2FtcC9jaGVtbml0ei9sb2thbGVzL3ByZWlzc2Nob2NrLWJlaS1laW5zLWVuZXJnaWUtZmVybndhZXJtZS13aXJkLTc1LXByb3plbnQtdGV1cmVyLTMwMzIwOTE?oc=5 + CBMibGh0dHBzOi8vd3d3LnRhZzI0LmRlL2NoZW1uaXR6L2xva2FsZXMvcHJlaXNzY2hvY2stYmVpLWVpbnMtZW5lcmdpZS1mZXJud2Flcm1lLXdpcmQtNzUtcHJvemVudC10ZXVyZXItMzAzMjA5MdIBcGh0dHBzOi8vd3d3LnRhZzI0LmRlL2FtcC9jaGVtbml0ei9sb2thbGVzL3ByZWlzc2Nob2NrLWJlaS1laW5zLWVuZXJnaWUtZmVybndhZXJtZS13aXJkLTc1LXByb3plbnQtdGV1cmVyLTMwMzIwOTE + Wed, 06 Dec 2023 04:30:00 GMT + <a href="https://news.google.com/rss/articles/CBMibGh0dHBzOi8vd3d3LnRhZzI0LmRlL2NoZW1uaXR6L2xva2FsZXMvcHJlaXNzY2hvY2stYmVpLWVpbnMtZW5lcmdpZS1mZXJud2Flcm1lLXdpcmQtNzUtcHJvemVudC10ZXVyZXItMzAzMjA5MdIBcGh0dHBzOi8vd3d3LnRhZzI0LmRlL2FtcC9jaGVtbml0ei9sb2thbGVzL3ByZWlzc2Nob2NrLWJlaS1laW5zLWVuZXJnaWUtZmVybndhZXJtZS13aXJkLTc1LXByb3plbnQtdGV1cmVyLTMwMzIwOTE?oc=5" target="_blank">Chemnitz: Preisschock bei "eins energie"! Fernwärme wird 75 Prozent teurer </a >&nbsp;&nbsp;<font color="#6f6f6f">TAG24 </font > + TAG24 + + + Ehemaliges Chemnitzer Straßenbahndepot wird zu Garagencampus - MDR + https://news.google.com/rss/articles/CBMiN2h0dHBzOi8vd3d3Lm1kci5kZS92aWRlby9tZHItdmlkZW9zL2EvdmlkZW8tNzc5OTUyLmh0bWzSAQA?oc=5 + CBMiN2h0dHBzOi8vd3d3Lm1kci5kZS92aWRlby9tZHItdmlkZW9zL2EvdmlkZW8tNzc5OTUyLmh0bWzSAQA + Wed, 06 Dec 2023 19:24:28 GMT + <a href="https://news.google.com/rss/articles/CBMiN2h0dHBzOi8vd3d3Lm1kci5kZS92aWRlby9tZHItdmlkZW9zL2EvdmlkZW8tNzc5OTUyLmh0bWzSAQA?oc=5" target="_blank">Ehemaliges Chemnitzer Straßenbahndepot wird zu Garagencampus </a >&nbsp;&nbsp;<font color="#6f6f6f">MDR </font > + MDR + + +`; + +Deno.test('getGooglenewsFeed - Url', async () => { + const fetchWithTimeoutSpy = stub( + utils, + 'fetchWithTimeout', + returnsNext([ + new Promise((resolve) => { + resolve(new Response(responseUrl, { status: 200 })); + }), + ]), + ); + + try { + const { source, items } = await getGooglenewsFeed( + supabaseClient, + undefined, + mockProfile, + { + ...mockSource, + options: { + googlenews: { + type: 'url', + url: + 'https://news.google.com/topics/CAAqJggKIiBDQkFTRWdvSUwyMHZNRFZxYUdjU0FtUmxHZ0pFUlNnQVAB?hl=de&gl=DE&ceid=DE%3Ade', + }, + }, + }, + ); + feedutils.assertEqualsSource(source, { + 'id': 'googlenews-myuser-mycolumn-8c1368ef1bc9e52356bffba3ce60cd48', + 'columnId': 'mycolumn', + 'userId': 'myuser', + 'type': 'googlenews', + 'title': 'Schlagzeilen - Aktuell - Google News', + 'options': { + 'googlenews': { + 'type': 'url', + 'url': + 'https://news.google.com/rss/topics/CAAqJggKIiBDQkFTRWdvSUwyMHZNRFZxYUdjU0FtUmxHZ0pFUlNnQVAB?hl=de&gl=DE&ceid=DE%3Ade', + }, + }, + 'link': + 'https://news.google.com/topics/CAAqJggKIiBDQkFTRWdvSUwyMHZNRFZxYUdjU0FtUmxHZ0pFUlNnQVAB?hl=de &gl=DE &ceid=DE:de', + }); + feedutils.assertEqualsItems(items, [{ + 'id': + 'googlenews-myuser-mycolumn-8c1368ef1bc9e52356bffba3ce60cd48-b8e1de5555c562ab270e8398ee948d16', + 'userId': 'myuser', + 'columnId': 'mycolumn', + 'sourceId': 'googlenews-myuser-mycolumn-8c1368ef1bc9e52356bffba3ce60cd48', + 'title': + 'GDL ruft von Donnerstagabend an zu eintägigem Streik auf - tagesschau.de', + 'link': + 'https://news.google.com/rss/articles/CBMiOGh0dHBzOi8vd3d3LnRhZ2Vzc2NoYXUuZGUvd2lydHNjaGFmdC9nZGwtc3RyZWlrLTE5MC5odG1s0gEA?oc=5', + 'description': + '
  1. GDL ruft von Donnerstagabend an zu eintägigem Streik auf   tagesschau.de
  2. Deutsche Bahn: Morgen nächster Lokführer-Streik! | Politik   BILD
  3. Schleswig-Holstein aktuell: Nachrichten im Überblick   NDR.de
  4. Bahnstreik und Sperrung - Nichts geht mehr beim RE2 Cottbus – Berlin   Lausitzer Rundschau
  5. Deutsche Bahn/GDL: Lokführer der Bahn streiken ab Donnerstag bis Freitagabend   DER SPIEGEL
', + 'author': 'tagesschau.de', + 'publishedAt': 1701890400, + }, { + 'id': + 'googlenews-myuser-mycolumn-8c1368ef1bc9e52356bffba3ce60cd48-196ae0209c5e61dd3b746eda4f7e7eef', + 'userId': 'myuser', + 'columnId': 'mycolumn', + 'sourceId': 'googlenews-myuser-mycolumn-8c1368ef1bc9e52356bffba3ce60cd48', + 'title': + 'Lassen die USA die Ukraine im Stich?: US-Finanzministerin spricht von „katastrophaler Situation“ - Tagesspiegel', + 'link': + 'https://news.google.com/rss/articles/CBMilgFodHRwczovL3d3dy50YWdlc3NwaWVnZWwuZGUvaW50ZXJuYXRpb25hbGVzL2xhc3Nlbi1kaWUtdXNhLWRpZS11a3JhaW5lLWltLXN0aWNoLXVzLWZpbmFuem1pbmlzdGVyaW4tc3ByaWNodC12b24ta2F0YXN0cm9waGFsZXItc2l0dWF0aW9uLTEwODg3Nzg3Lmh0bWzSAQA?oc=5', + 'description': + '
  1. Lassen die USA die Ukraine im Stich?: US-Finanzministerin spricht von „katastrophaler Situation“  Tagesspiegel
  2. Rüstungskonferenz in Washington: Ukraine-Hilfen der USA laufen aus   tagesschau
  3. Selenskyj sagt Auftritt vor US-Senatoren ab: „Etwas dazwischengekommen“  BILD
  4. USA haben kein Geld mehr für die Ukraine   t-online
  5. Ukraine-Krieg: Putins wichtigste Unterstützer sitzen in Washington   Berliner Morgenpost
', + 'author': 'Tagesspiegel', + 'publishedAt': 1701877080, + }]); + } finally { + fetchWithTimeoutSpy.restore(); + } + + assertSpyCall(fetchWithTimeoutSpy, 0, { + args: [ + 'https://news.google.com/rss/topics/CAAqJggKIiBDQkFTRWdvSUwyMHZNRFZxYUdjU0FtUmxHZ0pFUlNnQVAB?hl=de&gl=DE&ceid=DE%3Ade', + { method: 'get' }, + 5000, + ], + returned: new Promise((resolve) => { + resolve(new Response(responseUrl, { status: 200 })); + }), + }); + assertSpyCalls(fetchWithTimeoutSpy, 1); +}); + +Deno.test('getGooglenewsFeed - Search', async () => { + const fetchWithTimeoutSpy = stub( + utils, + 'fetchWithTimeout', + returnsNext([ + new Promise((resolve) => { + resolve(new Response(responseSearch, { status: 200 })); + }), + ]), + ); + + try { + const { source, items } = await getGooglenewsFeed( + supabaseClient, + undefined, + mockProfile, + { + ...mockSource, + options: { + googlenews: { + type: 'search', + search: 'Chemnitz', + ceid: 'DE:de', + gl: 'DE', + hl: 'de', + }, + }, + }, + ); + feedutils.assertEqualsSource(source, { + 'id': 'googlenews-myuser-mycolumn-5ef472fe226393772d05c92261df68e1', + 'columnId': 'mycolumn', + 'userId': 'myuser', + 'type': 'googlenews', + 'title': '"Chemnitz" - Google News', + 'options': { + 'googlenews': { + 'type': 'search', + 'search': 'Chemnitz', + 'ceid': 'DE:de', + 'gl': 'DE', + 'hl': 'de', + 'url': + 'https://news.google.com/rss/search?q=Chemnitz&hl=de&gl=DE&ceid=DE:de', + }, + }, + 'link': + 'https://news.google.com/search?q=Chemnitz &hl=de &gl=DE &ceid=DE:de', + }); + feedutils.assertEqualsItems(items, [{ + 'id': + 'googlenews-myuser-mycolumn-5ef472fe226393772d05c92261df68e1-d834d987207ffda8946e25df15adea75', + 'userId': 'myuser', + 'columnId': 'mycolumn', + 'sourceId': 'googlenews-myuser-mycolumn-5ef472fe226393772d05c92261df68e1', + 'title': + 'Chemnitz: Preisschock bei "eins energie"! Fernwärme wird 75 Prozent teurer - TAG24', + 'link': + 'https://news.google.com/rss/articles/CBMibGh0dHBzOi8vd3d3LnRhZzI0LmRlL2NoZW1uaXR6L2xva2FsZXMvcHJlaXNzY2hvY2stYmVpLWVpbnMtZW5lcmdpZS1mZXJud2Flcm1lLXdpcmQtNzUtcHJvemVudC10ZXVyZXItMzAzMjA5MdIBcGh0dHBzOi8vd3d3LnRhZzI0LmRlL2FtcC9jaGVtbml0ei9sb2thbGVzL3ByZWlzc2Nob2NrLWJlaS1laW5zLWVuZXJnaWUtZmVybndhZXJtZS13aXJkLTc1LXByb3plbnQtdGV1cmVyLTMwMzIwOTE?oc=5', + 'description': + 'Chemnitz: Preisschock bei "eins energie"! Fernwärme wird 75 Prozent teurer   TAG24 ', + 'author': 'TAG24', + 'publishedAt': 1701837000, + }, { + 'id': + 'googlenews-myuser-mycolumn-5ef472fe226393772d05c92261df68e1-f9fdb0be0cc37b302c47ad155e25ae7a', + 'userId': 'myuser', + 'columnId': 'mycolumn', + 'sourceId': 'googlenews-myuser-mycolumn-5ef472fe226393772d05c92261df68e1', + 'title': + 'Ehemaliges Chemnitzer Straßenbahndepot wird zu Garagencampus - MDR', + 'link': + 'https://news.google.com/rss/articles/CBMiN2h0dHBzOi8vd3d3Lm1kci5kZS92aWRlby9tZHItdmlkZW9zL2EvdmlkZW8tNzc5OTUyLmh0bWzSAQA?oc=5', + 'description': + 'Ehemaliges Chemnitzer Straßenbahndepot wird zu Garagencampus   MDR ', + 'author': 'MDR', + 'publishedAt': 1701890668, + }]); + } finally { + fetchWithTimeoutSpy.restore(); + } + + assertSpyCall(fetchWithTimeoutSpy, 0, { + args: [ + 'https://news.google.com/rss/search?q=Chemnitz&hl=de&gl=DE&ceid=DE:de', + { method: 'get' }, + 5000, + ], + returned: new Promise((resolve) => { + resolve(new Response(responseSearch, { status: 200 })); + }), + }); + assertSpyCalls(fetchWithTimeoutSpy, 1); +}); diff --git a/supabase/functions/_shared/feed/lemmy.ts b/supabase/functions/_shared/feed/lemmy.ts index e575647..1de35b5 100644 --- a/supabase/functions/_shared/feed/lemmy.ts +++ b/supabase/functions/_shared/feed/lemmy.ts @@ -8,8 +8,7 @@ import { unescape } from 'lodash'; import { IItem } from '../models/item.ts'; import { ISource } from '../models/source.ts'; import { IProfile } from '../models/profile.ts'; -import { fetchWithTimeout } from '../utils/fetchWithTimeout.ts'; -import { log } from '../utils/log.ts'; +import { utils } from '../utils/index.ts'; /** * `instances` contains a list of known Lemmy instances. This list is used to @@ -149,11 +148,11 @@ export const getLemmyFeed = async ( * Get the RSS for the provided `lemmy` url and parse it. If a feed doesn't * contains an item we return an error. */ - const response = await fetchWithTimeout(source.options.lemmy, { + const response = await utils.fetchWithTimeout(source.options.lemmy, { method: 'get', }, 5000); const xml = await response.text(); - log('debug', 'Add source', { + utils.log('debug', 'Add source', { sourceType: 'lemmy', requestUrl: source.options.lemmy, responseStatus: response.status, diff --git a/supabase/functions/_shared/feed/lemmy_test.ts b/supabase/functions/_shared/feed/lemmy_test.ts new file mode 100644 index 0000000..623b4cf --- /dev/null +++ b/supabase/functions/_shared/feed/lemmy_test.ts @@ -0,0 +1,511 @@ +import { assertEquals } from 'std/assert'; +import { createClient } from '@supabase/supabase-js'; +import { + assertSpyCall, + assertSpyCalls, + returnsNext, + stub, +} from 'std/testing/mock'; + +import { ISource } from '../models/source.ts'; +import { IProfile } from '../models/profile.ts'; +import { getLemmyFeed, isLemmyUrl } from './lemmy.ts'; +import { utils } from '../utils/index.ts'; +import { feedutils } from './utils/index.ts'; + +const supabaseClient = createClient('http://localhost:54321', 'test123'); +const mockProfile: IProfile = { + id: '', + tier: 'free', + createdAt: 0, + updatedAt: 0, +}; +const mockSource: ISource = { + id: '', + columnId: 'mycolumn', + userId: 'myuser', + type: 'medium', + title: '', +}; + +const responseUser = ` + + + Lemmy.World - lwCET + https://lemmy.world/u/lwCET + + + [Weekly Community Spotlights] c/forgottenweapons and aneurysmposting@sopuli.xyz + https://lemmy.world/pictrs/image/d22b2935-f27b-49f0-81e7-c81c21088467.png + lwCET to communityspotlight
180 points | 19 comments
https://lemmy.world/pictrs/image/d22b2935-f27b-49f0-81e7-c81c21088467.png

Hello World,

+

This week’s Community Spotlights are:

+

LW Community: Forgotten Weapons (!forgottenweapons@lemmy.world) - A community dedicated to discussion around historical arms, mechanically unique arms, and Ian McCollum’s Forgotten Weapons content.
+Mod(s): @FireTower@lemmy.world

+
+

Fediverse Community: Aneurysm Posting (!aneurysmposting@sopuli.xyz) - For shitposting by people who can smell burnt toast.
+Mod(s): @PinkyCoyote@sopuli.xyz

+
+

How to submit a community you would like to see spotlighted

+

Comment on any Weekly Spotlight post or suggest a community on our Discord server in the community-spotlight channel. You can also send a message to the Community Team with your suggestions.

]]> + https://lemmy.world/post/9209829 + https://lemmy.world/post/9209829 + Wed, 06 Dec 2023 11:32:19 +0000 + https://lemmy.world/u/lwCET + + + LW Holiday Logos + https://lemmy.world/pictrs/image/f3189f30-f8c8-4c4f-b957-e3a7bfd1c784.png + lwCET to lemmyworld
361 points | 47 comments
https://lemmy.world/pictrs/image/f3189f30-f8c8-4c4f-b957-e3a7bfd1c784.png

Hello World,

+

You may have seen the LW holiday themed logos we have used for Halloween and Thanksgiving. LW’s users represent many countries around the world and we want to celebrate holidays and other special days that are local to you, but our team is fairly small and we aren’t aware of a lot of the local customs out there. So we’re asking you what you would like to see represented in a LW themed logo. What are some holidays/special days in your area and how do you celebrate them? And not just major holidays, we would like to celebrate festivals, days of remembrance, and other special days.

+

Please, comment below your suggestions and ideas on how we could represent them in the LW logo.

+

EDIT: Mostly looking for events throughout the year. What’s left of 2023 is already in work. Thanks!

]]> + https://lemmy.world/post/9036399 + https://lemmy.world/post/9036399 + Sat, 02 Dec 2023 05:39:05 +0000 + https://lemmy.world/u/lwCET + + +`; + +const responseCommunity = ` + + + Lemmy.World - lemmyworld + https://lemmy.world/c/lemmyworld + This Community is intended for **posts about the Lemmy.world server** by the admins. + +For support with issues at Lemmy.world, go to [the Lemmy.world Support community](https://lemmy.world/c/support). + +## Support e-mail +Any support requests are best sent to info@lemmy.world e-mail. + +## Donations +If you would like to make a donation to support the cost of running this platform, please do so at the mastodon. world donation URLs: + +- https://opencollective.com/mastodonworld +- https://patreon.com/mastodonworld + + LW Holiday Logos + https://lemmy.world/pictrs/image/f3189f30-f8c8-4c4f-b957-e3a7bfd1c784.png + lwCET to lemmyworld
361 points | 47 comments
https://lemmy.world/pictrs/image/f3189f30-f8c8-4c4f-b957-e3a7bfd1c784.png

Hello World,

+

You may have seen the LW holiday themed logos we have used for Halloween and Thanksgiving. LW’s users represent many countries around the world and we want to celebrate holidays and other special days that are local to you, but our team is fairly small and we aren’t aware of a lot of the local customs out there. So we’re asking you what you would like to see represented in a LW themed logo. What are some holidays/special days in your area and how do you celebrate them? And not just major holidays, we would like to celebrate festivals, days of remembrance, and other special days.

+

Please, comment below your suggestions and ideas on how we could represent them in the LW logo.

+

EDIT: Mostly looking for events throughout the year. What’s left of 2023 is already in work. Thanks!

]]> + https://lemmy.world/post/9036399 + https://lemmy.world/post/9036399 + Sat, 02 Dec 2023 05:39:05 +0000 + https://lemmy.world/u/lwCET + + + Lemmy.World Junior Cloud Engineer + https://lemmy.world/post/8054956 + lwadmin to lemmyworld
501 points | 2 comments

Hello World!

+

Lemmy.World is looking for new engineers to help with our growing community. Volunteers will assist our existing infrastructure team with monitoring, maintenance and automation development tasks. They will report to our head of infrastructure.

+

We are looking for junior admins for this role. You will learn a modern cloud infra stack, including Terraform, DataDog, CloudFlare and ma

+

Keep in mind that while this is a volunteer gig, we would ask you to be able to commit to at least 5-10 hours a week. We also understand this is a hobby and that family and work comes first.

+

Applicants must be okay with providing their CV, LinkedIn profile; along with sitting for a video interview.

+

We are an international team that works from both North America EST time (-4) and Europe CEST (+2) so we would ask that candidates be flexible with their availability.

+

To learn more and begin your application process, click here. This is not a paid position.

]]> + https://lemmy.world/post/8054956 + https://lemmy.world/post/8054956 + Fri, 10 Nov 2023 09:48:21 +0000 + https://lemmy.world/u/lwadmin + + +`; + +const responseCommunityIIC = ` + + + Lemmy.World - idiotsincars + https://lemmy.world/c/idiotsincars + This ~~subreddit~~ community is devoted to the lovable idiots who do hilarious, idiot things in their idiot cars (or trucks, motorcycles, tractors, or other vehicle). We honor them with gifs, videos, images, and laughter. + + 9/30/23 Don't be this guy + https://i.imgur.com/rGcxspg.mp4 + SuperSleuth to idiotsincars
97 points | 4 comments
https://i.imgur.com/rGcxspg.mp4

use this link if video doesn’t load

]]> + https://lemmy.world/post/6063706 + https://lemmy.world/post/6063706 + Sun, 01 Oct 2023 00:42:07 +0000 + https://lemm.ee/u/SuperSleuth + + + Dash Cam Owners Australia September 2023 On the Road Compilation + https://youtu.be/6Xr6tsMCDzs?si=apq7rpNvByYnpXN2 + BestTestInTheWest to idiotsincars
22 points | 2 comments
https://youtu.be/6Xr6tsMCDzs?si=apq7rpNvByYnpXN2]]> + https://lemmy.world/post/5679506 + https://lemmy.world/post/5679506 + Sun, 24 Sep 2023 22:50:29 +0000 + https://lemmy.world/u/BestTestInTheWest + + + I merge now! + https://files.catbox.moe/7ino16.mp4 + Overstuff9499 to idiotsincars
13 points | 7 comments
https://files.catbox.moe/7ino16.mp4

i guess i should read their minds.

]]> + https://lemmy.world/post/4072199 + https://lemmy.world/post/4072199 + Tue, 29 Aug 2023 13:56:40 +0000 + https://lemm.ee/u/Overstuff9499 + + + Dash Cam Owners Australia August 2023 On the Road Compilation + https://youtu.be/TtWnAIcU6Cs?si=RQ9VJGcLF4xR0xsx + BestTestInTheWest to idiotsincars
26 points | 1 comments
https://youtu.be/TtWnAIcU6Cs?si=RQ9VJGcLF4xR0xsx

Dash Cam Owners Australia August 2023 On the Road Compilation

]]> + https://lemmy.world/post/3965818 + https://lemmy.world/post/3965818 + Sun, 27 Aug 2023 19:53:07 +0000 + https://lemmy.world/u/BestTestInTheWest + + + Car playing PacMan with the double yellow line in a dangerous curve + https://i.imgur.com/gHFeqCi.mp4 + LazaroFilm to idiotsincars
45 points | 1 comments
https://i.imgur.com/gHFeqCi.mp4

And they honk back‽

]]> + https://lemmy.world/post/3303420 + https://lemmy.world/post/3303420 + Wed, 16 Aug 2023 22:26:18 +0000 + https://lemmy.world/u/LazaroFilm + + + Cop car forgot how stop signs work + https://i.imgur.com/hsaoKWm.mp4 + LazaroFilm to idiotsincars
49 points | 4 comments
https://i.imgur.com/hsaoKWm.mp4]]> + https://lemmy.world/post/3303348 + https://lemmy.world/post/3303348 + Wed, 16 Aug 2023 22:18:38 +0000 + https://lemmy.world/u/LazaroFilm + + + Honey, I forgot the KeÅŸkek + https://i.imgur.com/lXKg8Gn.mp4 + SuperSleuth to idiotsincars
108 points | 5 comments
https://i.imgur.com/lXKg8Gn.mp4

A seven car pileup during a wedding convoy in Denizli, Turkey. Click here if you can’t see the video.

]]> + https://lemmy.world/post/2916600 + https://lemmy.world/post/2916600 + Wed, 09 Aug 2023 13:20:04 +0000 + https://lemm.ee/u/SuperSleuth + + + BMW unexpectedly using blinkers before brake checking cargo truck [YT original in post] + http://regna.nu/7u70dz.gif + Regna to idiotsincars
91 points | 11 comments
http://regna.nu/7u70dz.gif

Edit: Changed link for the pic to another site

+

The guy behind Trucker Dashcam // Sweden is one of the nicest truckers I’ve ever heard of. He mainly does dashcam compilations nowadays, and includes videos from friends and subscibers as well.

+]]> + https://lemmy.world/post/2370743 + https://lemmy.world/post/2370743 + Sun, 30 Jul 2023 10:19:11 +0000 + https://lemmy.world/u/Regna + + +`; + +Deno.test('isLemmyUrl', () => { + assertEquals(isLemmyUrl('https://lemmy.world/c/lemmyworld'), true); + assertEquals(isLemmyUrl('https://www.google.de/'), false); +}); + +Deno.test('getLemmyFeed - User', async () => { + const fetchWithTimeoutSpy = stub( + utils, + 'fetchWithTimeout', + returnsNext([ + new Promise((resolve) => { + resolve(new Response(responseUser, { status: 200 })); + }), + ]), + ); + + try { + const { source, items } = await getLemmyFeed( + supabaseClient, + undefined, + mockProfile, + { ...mockSource, options: { lemmy: 'https://lemmy.world/u/lwCET' } }, + ); + feedutils.assertEqualsSource(source, { + 'id': 'lemmy-myuser-mycolumn-9b51f0368938451bfbd740fad833b7a4', + 'columnId': 'mycolumn', + 'userId': 'myuser', + 'type': 'lemmy', + 'title': 'Lemmy.World - lwCET', + 'options': { 'lemmy': 'https://lemmy.world/feeds/u/lwCET.xml?sort=New' }, + 'link': 'https://lemmy.world/u/lwCET', + }); + feedutils.assertEqualsItems(items, [{ + 'id': + 'lemmy-myuser-mycolumn-9b51f0368938451bfbd740fad833b7a4-1f643a920e7a0d3766558a52ddb28f96', + 'userId': 'myuser', + 'columnId': 'mycolumn', + 'sourceId': 'lemmy-myuser-mycolumn-9b51f0368938451bfbd740fad833b7a4', + 'title': + '[Weekly Community Spotlights] c/forgottenweapons and aneurysmposting@sopuli.xyz', + 'link': 'https://lemmy.world/post/9209829', + 'media': + 'https://lemmy.world/pictrs/image/d22b2935-f27b-49f0-81e7-c81c21088467.png', + 'description': + 'submitted by lwCET to communityspotlight
180 points | 19 comments
https://lemmy.world/pictrs/image/d22b2935-f27b-49f0-81e7-c81c21088467.png

Hello World,

\n

This week’s Community Spotlights are:

\n

LW Community: Forgotten Weapons (!forgottenweapons@lemmy.world) - A community dedicated to discussion around historical arms, mechanically unique arms, and Ian McCollum’s Forgotten Weapons content.
\nMod(s): @FireTower@lemmy.world

\n
\n

Fediverse Community: Aneurysm Posting (!aneurysmposting@sopuli.xyz) - For shitposting by people who can smell burnt toast.
\nMod(s): @PinkyCoyote@sopuli.xyz

\n
\n

How to submit a community you would like to see spotlighted

\n

Comment on any Weekly Spotlight post or suggest a community on our Discord server in the community-spotlight channel. You can also send a message to the Community Team with your suggestions.

', + 'author': 'https://lemmy.world/u/lwCET', + 'publishedAt': 1701862339, + }, { + 'id': + 'lemmy-myuser-mycolumn-9b51f0368938451bfbd740fad833b7a4-86c6108b553905422e6268948f22ba54', + 'userId': 'myuser', + 'columnId': 'mycolumn', + 'sourceId': 'lemmy-myuser-mycolumn-9b51f0368938451bfbd740fad833b7a4', + 'title': 'LW Holiday Logos', + 'link': 'https://lemmy.world/post/9036399', + 'media': + 'https://lemmy.world/pictrs/image/f3189f30-f8c8-4c4f-b957-e3a7bfd1c784.png', + 'description': + 'submitted by lwCET to lemmyworld
361 points | 47 comments
https://lemmy.world/pictrs/image/f3189f30-f8c8-4c4f-b957-e3a7bfd1c784.png

Hello World,

\n

You may have seen the LW holiday themed logos we have used for Halloween and Thanksgiving. LW’s users represent many countries around the world and we want to celebrate holidays and other special days that are local to you, but our team is fairly small and we aren’t aware of a lot of the local customs out there. So we’re asking you what you would like to see represented in a LW themed logo. What are some holidays/special days in your area and how do you celebrate them? And not just major holidays, we would like to celebrate festivals, days of remembrance, and other special days.

\n

Please, comment below your suggestions and ideas on how we could represent them in the LW logo.

\n

EDIT: Mostly looking for events throughout the year. What’s left of 2023 is already in work. Thanks!

', + 'author': 'https://lemmy.world/u/lwCET', + 'publishedAt': 1701495545, + }]); + } finally { + fetchWithTimeoutSpy.restore(); + } + + assertSpyCall(fetchWithTimeoutSpy, 0, { + args: [ + 'https://lemmy.world/feeds/u/lwCET.xml?sort=New', + { method: 'get' }, + 5000, + ], + returned: new Promise((resolve) => { + resolve(new Response(responseUser, { status: 200 })); + }), + }); + assertSpyCalls(fetchWithTimeoutSpy, 1); +}); + +Deno.test('getLemmyFeed - Community', async () => { + const fetchWithTimeoutSpy = stub( + utils, + 'fetchWithTimeout', + returnsNext([ + new Promise((resolve) => { + resolve(new Response(responseCommunity, { status: 200 })); + }), + ]), + ); + + try { + const { source, items } = await getLemmyFeed( + supabaseClient, + undefined, + mockProfile, + { ...mockSource, options: { lemmy: 'https://lemmy.world/c/lemmyworld' } }, + ); + feedutils.assertEqualsSource(source, { + 'id': 'lemmy-myuser-mycolumn-8024684791f06e280f6fbd7217099f42', + 'columnId': 'mycolumn', + 'userId': 'myuser', + 'type': 'lemmy', + 'title': 'Lemmy.World - lemmyworld', + 'options': { + 'lemmy': 'https://lemmy.world/feeds/c/lemmyworld.xml?sort=New', + }, + 'link': 'https://lemmy.world/c/lemmyworld', + }); + feedutils.assertEqualsItems(items, [{ + 'id': + 'lemmy-myuser-mycolumn-8024684791f06e280f6fbd7217099f42-86c6108b553905422e6268948f22ba54', + 'userId': 'myuser', + 'columnId': 'mycolumn', + 'sourceId': 'lemmy-myuser-mycolumn-8024684791f06e280f6fbd7217099f42', + 'title': 'LW Holiday Logos', + 'link': 'https://lemmy.world/post/9036399', + 'media': + 'https://lemmy.world/pictrs/image/f3189f30-f8c8-4c4f-b957-e3a7bfd1c784.png', + 'description': + 'submitted by lwCET to lemmyworld
361 points | 47 comments
https://lemmy.world/pictrs/image/f3189f30-f8c8-4c4f-b957-e3a7bfd1c784.png

Hello World,

\n

You may have seen the LW holiday themed logos we have used for Halloween and Thanksgiving. LW’s users represent many countries around the world and we want to celebrate holidays and other special days that are local to you, but our team is fairly small and we aren’t aware of a lot of the local customs out there. So we’re asking you what you would like to see represented in a LW themed logo. What are some holidays/special days in your area and how do you celebrate them? And not just major holidays, we would like to celebrate festivals, days of remembrance, and other special days.

\n

Please, comment below your suggestions and ideas on how we could represent them in the LW logo.

\n

EDIT: Mostly looking for events throughout the year. What’s left of 2023 is already in work. Thanks!

', + 'author': 'https://lemmy.world/u/lwCET', + 'publishedAt': 1701495545, + }, { + 'id': + 'lemmy-myuser-mycolumn-8024684791f06e280f6fbd7217099f42-797a3b386c90d913c61e1c831cba6f46', + 'userId': 'myuser', + 'columnId': 'mycolumn', + 'sourceId': 'lemmy-myuser-mycolumn-8024684791f06e280f6fbd7217099f42', + 'title': 'Lemmy.World Junior Cloud Engineer', + 'link': 'https://lemmy.world/post/8054956', + 'description': + 'submitted by lwadmin to lemmyworld
501 points | 2 comments

Hello World!

\n

Lemmy.World is looking for new engineers to help with our growing community. Volunteers will assist our existing infrastructure team with monitoring, maintenance and automation development tasks. They will report to our head of infrastructure.

\n

We are looking for junior admins for this role. You will learn a modern cloud infra stack, including Terraform, DataDog, CloudFlare and ma

\n

Keep in mind that while this is a volunteer gig, we would ask you to be able to commit to at least 5-10 hours a week. We also understand this is a hobby and that family and work comes first.

\n

Applicants must be okay with providing their CV, LinkedIn profile; along with sitting for a video interview.

\n

We are an international team that works from both North America EST time (-4) and Europe CEST (+2) so we would ask that candidates be flexible with their availability.

\n

To learn more and begin your application process, click here. This is not a paid position.

', + 'author': 'https://lemmy.world/u/lwadmin', + 'publishedAt': 1699609701, + }]); + } finally { + fetchWithTimeoutSpy.restore(); + } + + assertSpyCall(fetchWithTimeoutSpy, 0, { + args: [ + 'https://lemmy.world/feeds/c/lemmyworld.xml?sort=New', + { method: 'get' }, + 5000, + ], + returned: new Promise((resolve) => { + resolve(new Response(responseCommunity, { status: 200 })); + }), + }); + assertSpyCalls(fetchWithTimeoutSpy, 1); +}); + +Deno.test('getLemmyFeed - Community - IIC', async () => { + const fetchWithTimeoutSpy = stub( + utils, + 'fetchWithTimeout', + returnsNext([ + new Promise((resolve) => { + resolve(new Response(responseCommunityIIC, { status: 200 })); + }), + ]), + ); + + try { + const { source, items } = await getLemmyFeed( + supabaseClient, + undefined, + mockProfile, + { + ...mockSource, + options: { lemmy: 'https://lemmy.world/feeds/c/idiotsincars.xml' }, + }, + ); + feedutils.assertEqualsSource(source, { + 'id': 'lemmy-myuser-mycolumn-e5cb4d4594ce7d19987fd42ca1dd837f', + 'columnId': 'mycolumn', + 'userId': 'myuser', + 'type': 'lemmy', + 'title': 'Lemmy.World - idiotsincars', + 'options': { + 'lemmy': 'https://lemmy.world/feeds/c/idiotsincars.xml?sort=New', + }, + 'link': 'https://lemmy.world/c/idiotsincars', + }); + feedutils.assertEqualsItems(items, [{ + 'id': + 'lemmy-myuser-mycolumn-e5cb4d4594ce7d19987fd42ca1dd837f-33d0c935222609e6d7afe3d3054affec', + 'userId': 'myuser', + 'columnId': 'mycolumn', + 'sourceId': 'lemmy-myuser-mycolumn-e5cb4d4594ce7d19987fd42ca1dd837f', + 'title': '9/30/23 Don\'t be this guy', + 'link': 'https://lemmy.world/post/6063706', + 'media': 'https://i.imgur.com/rGcxspg.mp4', + 'description': + 'submitted by SuperSleuth to idiotsincars
97 points | 4 comments
https://i.imgur.com/rGcxspg.mp4

use this link if video doesn’t load

', + 'author': 'https://lemm.ee/u/SuperSleuth', + 'publishedAt': 1696120927, + }, { + 'id': + 'lemmy-myuser-mycolumn-e5cb4d4594ce7d19987fd42ca1dd837f-44104b8b3612c665abe8b93a2cdd2ce0', + 'userId': 'myuser', + 'columnId': 'mycolumn', + 'sourceId': 'lemmy-myuser-mycolumn-e5cb4d4594ce7d19987fd42ca1dd837f', + 'title': + 'Dash Cam Owners Australia September 2023 On the Road Compilation', + 'link': 'https://lemmy.world/post/5679506', + 'media': 'https://youtu.be/6Xr6tsMCDzs?si=apq7rpNvByYnpXN2', + 'description': + 'submitted by BestTestInTheWest to idiotsincars
22 points | 2 comments
https://youtu.be/6Xr6tsMCDzs?si=apq7rpNvByYnpXN2', + 'author': 'https://lemmy.world/u/BestTestInTheWest', + 'publishedAt': 1695595829, + }, { + 'id': + 'lemmy-myuser-mycolumn-e5cb4d4594ce7d19987fd42ca1dd837f-d36d08de3466c62c4feb90c491e68113', + 'userId': 'myuser', + 'columnId': 'mycolumn', + 'sourceId': 'lemmy-myuser-mycolumn-e5cb4d4594ce7d19987fd42ca1dd837f', + 'title': 'I merge now!', + 'link': 'https://lemmy.world/post/4072199', + 'media': 'https://files.catbox.moe/7ino16.mp4', + 'description': + 'submitted by Overstuff9499 to idiotsincars
13 points | 7 comments
https://files.catbox.moe/7ino16.mp4

i guess i should read their minds.

', + 'author': 'https://lemm.ee/u/Overstuff9499', + 'publishedAt': 1693317400, + }, { + 'id': + 'lemmy-myuser-mycolumn-e5cb4d4594ce7d19987fd42ca1dd837f-325c767708c578205330addd91232fc8', + 'userId': 'myuser', + 'columnId': 'mycolumn', + 'sourceId': 'lemmy-myuser-mycolumn-e5cb4d4594ce7d19987fd42ca1dd837f', + 'title': 'Dash Cam Owners Australia August 2023 On the Road Compilation', + 'link': 'https://lemmy.world/post/3965818', + 'media': 'https://youtu.be/TtWnAIcU6Cs?si=RQ9VJGcLF4xR0xsx', + 'description': + 'submitted by BestTestInTheWest to idiotsincars
26 points | 1 comments
https://youtu.be/TtWnAIcU6Cs?si=RQ9VJGcLF4xR0xsx

Dash Cam Owners Australia August 2023 On the Road Compilation

', + 'author': 'https://lemmy.world/u/BestTestInTheWest', + 'publishedAt': 1693165987, + }, { + 'id': + 'lemmy-myuser-mycolumn-e5cb4d4594ce7d19987fd42ca1dd837f-6cea754fc3ac4867b66309151e8ef5eb', + 'userId': 'myuser', + 'columnId': 'mycolumn', + 'sourceId': 'lemmy-myuser-mycolumn-e5cb4d4594ce7d19987fd42ca1dd837f', + 'title': + 'Car playing PacMan with the double yellow line in a dangerous curve', + 'link': 'https://lemmy.world/post/3303420', + 'media': 'https://i.imgur.com/gHFeqCi.mp4', + 'description': + 'submitted by LazaroFilm to idiotsincars
45 points | 1 comments
https://i.imgur.com/gHFeqCi.mp4

And they honk back‽

', + 'author': 'https://lemmy.world/u/LazaroFilm', + 'publishedAt': 1692224778, + }, { + 'id': + 'lemmy-myuser-mycolumn-e5cb4d4594ce7d19987fd42ca1dd837f-08bb1f161a61a21a47457171e25c3fd1', + 'userId': 'myuser', + 'columnId': 'mycolumn', + 'sourceId': 'lemmy-myuser-mycolumn-e5cb4d4594ce7d19987fd42ca1dd837f', + 'title': 'Cop car forgot how stop signs work', + 'link': 'https://lemmy.world/post/3303348', + 'media': 'https://i.imgur.com/hsaoKWm.mp4', + 'description': + 'submitted by LazaroFilm to idiotsincars
49 points | 4 comments
https://i.imgur.com/hsaoKWm.mp4', + 'author': 'https://lemmy.world/u/LazaroFilm', + 'publishedAt': 1692224318, + }, { + 'id': + 'lemmy-myuser-mycolumn-e5cb4d4594ce7d19987fd42ca1dd837f-a567cf7498d0506a2f8a954354aa95c0', + 'userId': 'myuser', + 'columnId': 'mycolumn', + 'sourceId': 'lemmy-myuser-mycolumn-e5cb4d4594ce7d19987fd42ca1dd837f', + 'title': 'Honey, I forgot the KeÅŸkek', + 'link': 'https://lemmy.world/post/2916600', + 'media': 'https://i.imgur.com/lXKg8Gn.mp4', + 'description': + 'submitted by SuperSleuth to idiotsincars
108 points | 5 comments
https://i.imgur.com/lXKg8Gn.mp4

A seven car pileup during a wedding convoy in Denizli, Turkey. Click here if you can’t see the video.

', + 'author': 'https://lemm.ee/u/SuperSleuth', + 'publishedAt': 1691587204, + }, { + 'id': + 'lemmy-myuser-mycolumn-e5cb4d4594ce7d19987fd42ca1dd837f-ad7fbbbe7e47d9ca72aded5c66ab5bde', + 'userId': 'myuser', + 'columnId': 'mycolumn', + 'sourceId': 'lemmy-myuser-mycolumn-e5cb4d4594ce7d19987fd42ca1dd837f', + 'title': + 'BMW unexpectedly using blinkers before brake checking cargo truck [YT original in post]', + 'link': 'https://lemmy.world/post/2370743', + 'media': 'http://regna.nu/7u70dz.gif', + 'description': + 'submitted by Regna to idiotsincars
91 points | 11 comments
http://regna.nu/7u70dz.gif

Edit: Changed link for the pic to another site

\n

The guy behind Trucker Dashcam // Sweden is one of the nicest truckers I’ve ever heard of. He mainly does dashcam compilations nowadays, and includes videos from friends and subscibers as well.

\n', + 'author': 'https://lemmy.world/u/Regna', + 'publishedAt': 1690712351, + }]); + } finally { + fetchWithTimeoutSpy.restore(); + } + + assertSpyCall(fetchWithTimeoutSpy, 0, { + args: [ + 'https://lemmy.world/feeds/c/idiotsincars.xml?sort=New', + { method: 'get' }, + 5000, + ], + returned: new Promise((resolve) => { + resolve(new Response(responseCommunityIIC, { status: 200 })); + }), + }); + assertSpyCalls(fetchWithTimeoutSpy, 1); +}); diff --git a/supabase/functions/_shared/feed/mastodon.ts b/supabase/functions/_shared/feed/mastodon.ts index fabd24e..9bf1765 100644 --- a/supabase/functions/_shared/feed/mastodon.ts +++ b/supabase/functions/_shared/feed/mastodon.ts @@ -7,10 +7,9 @@ import { unescape } from 'lodash'; import { IItem } from '../models/item.ts'; import { ISource } from '../models/source.ts'; -import { uploadSourceIcon } from './utils/uploadFile.ts'; +import { feedutils } from './utils/index.ts'; import { IProfile } from '../models/profile.ts'; -import { fetchWithTimeout } from '../utils/fetchWithTimeout.ts'; -import { log } from '../utils/log.ts'; +import { utils } from '../utils/index.ts'; export const getMastodonFeed = async ( supabaseClient: SupabaseClient, @@ -43,13 +42,13 @@ export const getMastodonFeed = async ( /** * Get the RSS for the provided Mastodon username, hashtag or url. */ - const response = await fetchWithTimeout( + const response = await utils.fetchWithTimeout( source.options.mastodon, { method: 'get' }, 5000, ); const xml = await response.text(); - log('debug', 'Add source', { + utils.log('debug', 'Add source', { sourceType: 'mastodon', requestUrl: source.options.mastodon, responseStatus: response.status, @@ -88,7 +87,7 @@ export const getMastodonFeed = async ( */ if (!source.icon && feed.image?.url) { source.icon = feed.image.url; - source.icon = await uploadSourceIcon(supabaseClient, source); + source.icon = await feedutils.uploadSourceIcon(supabaseClient, source); } /** diff --git a/supabase/functions/_shared/feed/mastodon_test.ts b/supabase/functions/_shared/feed/mastodon_test.ts new file mode 100644 index 0000000..38da9fe --- /dev/null +++ b/supabase/functions/_shared/feed/mastodon_test.ts @@ -0,0 +1,316 @@ +import { createClient } from '@supabase/supabase-js'; +import { + assertSpyCall, + assertSpyCalls, + returnsNext, + stub, +} from 'std/testing/mock'; + +import { ISource } from '../models/source.ts'; +import { IProfile } from '../models/profile.ts'; +import { getMastodonFeed } from './mastodon.ts'; +import { utils } from '../utils/index.ts'; +import { feedutils } from './utils/index.ts'; + +const supabaseClient = createClient('http://localhost:54321', 'test123'); +const mockProfile: IProfile = { + id: '', + tier: 'free', + createdAt: 0, + updatedAt: 0, +}; +const mockSource: ISource = { + id: '', + columnId: 'mycolumn', + userId: 'myuser', + type: 'medium', + title: '', +}; + +const responseTag = ` + + + #kubernetes + Public posts tagged #kubernetes + https://hachyderm.io/tags/kubernetes + Fri, 08 Dec 2023 12:52:54 +0000 + Mastodon v4.2.1 + + https://ioc.exchange/@jon404/111544891736153138 + https://ioc.exchange/@jon404/111544891736153138 + Fri, 08 Dec 2023 12:52:54 +0000 + <p>Shameless blog series plug</p><hr><p>If you're in to <a href="https://ioc.exchange/tags/kubernetes" class="mention hashtag" rel="nofollow noopener noreferrer" target="_blank">#<span>kubernetes</span></a> and <a href="https://ioc.exchange/tags/homelab" class="mention hashtag" rel="nofollow noopener noreferrer" target="_blank">#<span>homelab</span></a> setups, I've started a blog series at LQ.org that will cover the <a href="https://ioc.exchange/tags/virtualization" class="mention hashtag" rel="nofollow noopener noreferrer" target="_blank">#<span>virtualization</span></a>, network integration, and <a href="https://ioc.exchange/tags/selfhosted" class="mention hashtag" rel="nofollow noopener noreferrer" target="_blank">#<span>selfhosted</span></a> apps I'll be running in my on-prem mini k8s cluster. </p><p>I'm planning on running Alpine/Xen on refurbished desktop machines, with Debian VMs/k8s-1.28 and MetalLB/Calico for integrating into my <a href="https://ioc.exchange/tags/OpenBSD" class="mention hashtag" rel="nofollow noopener noreferrer" target="_blank">#<span>OpenBSD</span></a> / <a href="https://ioc.exchange/tags/BGP" class="mention hashtag" rel="nofollow noopener noreferrer" target="_blank">#<span>BGP</span></a> / <a href="https://ioc.exchange/tags/OSPF" class="mention hashtag" rel="nofollow noopener noreferrer" target="_blank">#<span>OSPF</span></a> network backbone.</p><p><a href="https://www.linuxquestions.org/questions/blog/rocket357-328529/on-premise-kubernetes-part-1-5-39090/" rel="nofollow noopener noreferrer" translate="no" target="_blank"><span class="invisible">https://www.</span><span class="ellipsis">linuxquestions.org/questions/b</span><span class="invisible">log/rocket357-328529/on-premise-kubernetes-part-1-5-39090/</span></a></p> + kubernetes + homelab + selfhosted + openbsd + bgp + ospf + virtualization + + + https://botsin.space/@k8s_releases/111544452566443481 + https://botsin.space/@k8s_releases/111544452566443481 + Fri, 08 Dec 2023 11:01:13 +0000 + <p>New Kubernetes Release Candidate Release</p><p>✨ Kubernetes v1.29.0-rc.2 ✨</p><p><a href="https://github.com/kubernetes/kubernetes/releases/tag/v1.29.0-rc.2" rel="nofollow noopener noreferrer" target="_blank"><span class="invisible">https://</span><span class="ellipsis">github.com/kubernetes/kubernet</span><span class="invisible">es/releases/tag/v1.29.0-rc.2</span></a></p><p><a href="https://botsin.space/tags/Kubernetes" class="mention hashtag" rel="nofollow noopener noreferrer" target="_blank">#<span>Kubernetes</span></a> <a href="https://botsin.space/tags/k8s" class="mention hashtag" rel="nofollow noopener noreferrer" target="_blank">#<span>k8s</span></a> <a href="https://botsin.space/tags/kube" class="mention hashtag" rel="nofollow noopener noreferrer" target="_blank">#<span>kube</span></a></p> + kubernetes + k8s + kube + + + https://vmst.io/@ErikBussink/111544298911638567 + https://vmst.io/@ErikBussink/111544298911638567 + Fri, 08 Dec 2023 10:22:08 +0000 + <p>Plan for the weekend<br>- Deploy the add-on TPM2 keys on 4 supermicro servers and enable the encryption service on my <a href="https://vmst.io/tags/vSphere" class="mention hashtag" rel="nofollow noopener noreferrer" target="_blank">#<span>vSphere</span></a> cluster<br>- Attach some Tigo solar optimizer on some solar panels<br>- deploy a new set of NSX ALB load-balancer VM for integration in <a href="https://vmst.io/tags/NSX" class="mention hashtag" rel="nofollow noopener noreferrer" target="_blank">#<span>NSX</span></a> for <a href="https://vmst.io/tags/kubernetes" class="mention hashtag" rel="nofollow noopener noreferrer" target="_blank">#<span>kubernetes</span></a> <br><a href="https://vmst.io/tags/HomeDC" class="mention hashtag" rel="nofollow noopener noreferrer" target="_blank">#<span>HomeDC</span></a> <a href="https://vmst.io/tags/homelab" class="mention hashtag" rel="nofollow noopener noreferrer" target="_blank">#<span>homelab</span></a></p> + vSphere + nsx + kubernetes + homedc + homelab + + + https://fosstodon.org/@imdciangot/111544228725688234 + https://fosstodon.org/@imdciangot/111544228725688234 + Fri, 08 Dec 2023 10:04:17 +0000 + <p>🚀 ALERT: Seamlessly Access HPC Resources, possible addictive 🚀</p><p>The Interlink project aims to bridge the gap between Kubernetes and HPC environments, enabling seamless access to world-class HPC centers, all within the familiar context of cloud-based computing interfaces.</p><p>Learn how with this first demo, and join the SciGeeks crew for more!</p><p><a href="https://youtu.be/-djIQGPvYdI?si=wBPHbz3iu7A3qOSx" rel="nofollow noopener noreferrer" translate="no" target="_blank"><span class="invisible">https://</span><span class="ellipsis">youtu.be/-djIQGPvYdI?si=wBPHbz</span><span class="invisible">3iu7A3qOSx</span></a></p><p>Super early stage of this adventure, give feedback!</p><p><a href="https://fosstodon.org/tags/kubernetes" class="mention hashtag" rel="nofollow noopener noreferrer" target="_blank">#<span>kubernetes</span></a> <a href="https://fosstodon.org/tags/HPC" class="mention hashtag" rel="nofollow noopener noreferrer" target="_blank">#<span>HPC</span></a> <a href="https://fosstodon.org/tags/DistributedComputing" class="mention hashtag" rel="nofollow noopener noreferrer" target="_blank">#<span>DistributedComputing</span></a> <a href="https://fosstodon.org/tags/Research" class="mention hashtag" rel="nofollow noopener noreferrer" target="_blank">#<span>Research</span></a> <a href="https://fosstodon.org/tags/ai" class="mention hashtag" rel="nofollow noopener noreferrer" target="_blank">#<span>ai</span></a> <a href="https://fosstodon.org/tags/digitaltwins" class="mention hashtag" rel="nofollow noopener noreferrer" target="_blank">#<span>digitaltwins</span></a></p> + + nonadult + + kubernetes + hpc + distributedcomputing + research + ai + digitaltwins + + +`; + +const responseUser = ` + + + Rico Berger + Public posts from @ricoberger@hachyderm.io + https://hachyderm.io/@ricoberger + + https://media.hachyderm.io/accounts/avatars/109/773/619/675/865/785/original/bf731ded4166a661.png + Rico Berger + https://hachyderm.io/@ricoberger + + Sun, 29 Jan 2023 17:56:17 +0000 + https://media.hachyderm.io/accounts/avatars/109/773/619/675/865/785/original/bf731ded4166a661.png + Mastodon v4.2.1 + + https://hachyderm.io/@ricoberger/109773781555026547 + https://hachyderm.io/@ricoberger/109773781555026547 + Sun, 29 Jan 2023 17:56:17 +0000 + <p>🎉🎉🎉 kubenav v4 the <a href="https://hachyderm.io/tags/kubernetes" class="mention hashtag" rel="tag">#<span>kubernetes</span></a> dashboard for iOS and Android is now available <a href="https://kubenav.io" target="_blank" rel="nofollow noopener noreferrer" translate="no"><span class="invisible">https://</span><span class="">kubenav.io</span><span class="invisible"></span></a> 🥳🥳🥳</p> + + nonadult + + + nonadult + + + nonadult + + + nonadult + + kubernetes + + +`; + +Deno.test('getMastodonFeed - Tag', async () => { + const fetchWithTimeoutSpy = stub( + utils, + 'fetchWithTimeout', + returnsNext([ + new Promise((resolve) => { + resolve(new Response(responseTag, { status: 200 })); + }), + ]), + ); + + try { + const { source, items } = await getMastodonFeed( + supabaseClient, + undefined, + mockProfile, + { + ...mockSource, + options: { mastodon: 'https://hachyderm.io/tags/Kubernetes' }, + }, + ); + feedutils.assertEqualsSource(source, { + 'id': 'mastodon-myuser-mycolumn-91151e10882e4fff432ff509b8c6b027', + 'columnId': 'mycolumn', + 'userId': 'myuser', + 'type': 'mastodon', + 'title': '#kubernetes', + 'options': { 'mastodon': 'https://hachyderm.io/tags/Kubernetes.rss' }, + 'link': 'https://hachyderm.io/tags/kubernetes', + }); + feedutils.assertEqualsItems(items, [{ + 'id': + 'mastodon-myuser-mycolumn-91151e10882e4fff432ff509b8c6b027-5abb9cbadfb77c35a46b5c21fa96622e', + 'userId': 'myuser', + 'columnId': 'mycolumn', + 'sourceId': 'mastodon-myuser-mycolumn-91151e10882e4fff432ff509b8c6b027', + 'title': '', + 'link': 'https://ioc.exchange/@jon404/111544891736153138', + 'description': + '

Shameless blog series plug

If you\'re in to #kubernetes and #homelab setups, I\'ve started a blog series at LQ.org that will cover the #virtualization, network integration, and #selfhosted apps I\'ll be running in my on-prem mini k8s cluster.

I\'m planning on running Alpine/Xen on refurbished desktop machines, with Debian VMs/k8s-1.28 and MetalLB/Calico for integrating into my #OpenBSD / #BGP / #OSPF network backbone.

linuxquestions.org/questions/b

', + 'author': '@jon404@ioc.exchange', + 'publishedAt': 1702039974, + }, { + 'id': + 'mastodon-myuser-mycolumn-91151e10882e4fff432ff509b8c6b027-0ef142642e2dfd8cf186543d81e2b89c', + 'userId': 'myuser', + 'columnId': 'mycolumn', + 'sourceId': 'mastodon-myuser-mycolumn-91151e10882e4fff432ff509b8c6b027', + 'title': '', + 'link': 'https://botsin.space/@k8s_releases/111544452566443481', + 'description': + '

New Kubernetes Release Candidate Release

✨ Kubernetes v1.29.0-rc.2 ✨

github.com/kubernetes/kubernet

#Kubernetes #k8s #kube

', + 'author': '@k8s_releases@botsin.space', + 'publishedAt': 1702033273, + }, { + 'id': + 'mastodon-myuser-mycolumn-91151e10882e4fff432ff509b8c6b027-ab07d25df9ede52a07a36c203261bef4', + 'userId': 'myuser', + 'columnId': 'mycolumn', + 'sourceId': 'mastodon-myuser-mycolumn-91151e10882e4fff432ff509b8c6b027', + 'title': '', + 'link': 'https://vmst.io/@ErikBussink/111544298911638567', + 'description': + '

Plan for the weekend
- Deploy the add-on TPM2 keys on 4 supermicro servers and enable the encryption service on my #vSphere cluster
- Attach some Tigo solar optimizer on some solar panels
- deploy a new set of NSX ALB load-balancer VM for integration in #NSX for #kubernetes
#HomeDC #homelab

', + 'author': '@ErikBussink@vmst.io', + 'publishedAt': 1702030928, + }, { + 'id': + 'mastodon-myuser-mycolumn-91151e10882e4fff432ff509b8c6b027-5c1055543a76a190ae210057583b2225', + 'userId': 'myuser', + 'columnId': 'mycolumn', + 'sourceId': 'mastodon-myuser-mycolumn-91151e10882e4fff432ff509b8c6b027', + 'title': '', + 'link': 'https://fosstodon.org/@imdciangot/111544228725688234', + 'options': { + 'media': [ + 'https://media.hachyderm.io/cache/media_attachments/files/111/544/228/748/711/741/original/84994e14c0b7384e.png', + ], + }, + 'description': + '

🚀 ALERT: Seamlessly Access HPC Resources, possible addictive 🚀

The Interlink project aims to bridge the gap between Kubernetes and HPC environments, enabling seamless access to world-class HPC centers, all within the familiar context of cloud-based computing interfaces.

Learn how with this first demo, and join the SciGeeks crew for more!

youtu.be/-djIQGPvYdI?si=wBPHbz

Super early stage of this adventure, give feedback!

#kubernetes #HPC #DistributedComputing #Research #ai #digitaltwins

', + 'author': '@imdciangot@fosstodon.org', + 'publishedAt': 1702029857, + }]); + } finally { + fetchWithTimeoutSpy.restore(); + } + + assertSpyCall(fetchWithTimeoutSpy, 0, { + args: ['https://hachyderm.io/tags/Kubernetes.rss', { method: 'get' }, 5000], + returned: new Promise((resolve) => { + resolve(new Response(responseTag, { status: 200 })); + }), + }); + assertSpyCalls(fetchWithTimeoutSpy, 1); +}); + +Deno.test('getMastodonFeed - User', async () => { + const fetchWithTimeoutSpy = stub( + utils, + 'fetchWithTimeout', + returnsNext([ + new Promise((resolve) => { + resolve(new Response(responseUser, { status: 200 })); + }), + ]), + ); + + const uploadSourceIconSpy = stub( + feedutils, + 'uploadSourceIcon', + returnsNext([ + new Promise((resolve) => { + resolve( + 'https://media.hachyderm.io/accounts/avatars/109/773/619/675/865/785/original/bf731ded4166a661.png', + ); + }), + ]), + ); + + try { + const { source, items } = await getMastodonFeed( + supabaseClient, + undefined, + mockProfile, + { ...mockSource, options: { mastodon: '@ricoberger@hachyderm.io' } }, + ); + feedutils.assertEqualsSource(source, { + 'id': 'mastodon-myuser-mycolumn-5673bdae9d06a0744e93d647fe5cef2e', + 'columnId': 'mycolumn', + 'userId': 'myuser', + 'type': 'mastodon', + 'title': 'Rico Berger', + 'options': { 'mastodon': 'https://hachyderm.io/@ricoberger.rss' }, + 'link': 'https://hachyderm.io/@ricoberger', + 'icon': + 'https://media.hachyderm.io/accounts/avatars/109/773/619/675/865/785/original/bf731ded4166a661.png', + }); + feedutils.assertEqualsItems(items, [{ + 'id': + 'mastodon-myuser-mycolumn-5673bdae9d06a0744e93d647fe5cef2e-576a014e46fbb2feae01d3143d1ec565', + 'userId': 'myuser', + 'columnId': 'mycolumn', + 'sourceId': 'mastodon-myuser-mycolumn-5673bdae9d06a0744e93d647fe5cef2e', + 'title': '', + 'link': 'https://hachyderm.io/@ricoberger/109773781555026547', + 'options': { + 'media': [ + 'https://media.hachyderm.io/media_attachments/files/109/773/779/869/674/363/original/ac40bbbaa3710aa1.png', + 'https://media.hachyderm.io/media_attachments/files/109/773/779/875/654/377/original/c0147e5b7f00c319.png', + 'https://media.hachyderm.io/media_attachments/files/109/773/779/881/805/268/original/352c2b12ba3611ef.png', + 'https://media.hachyderm.io/media_attachments/files/109/773/779/882/749/096/original/ce2283bdeb25b07f.png', + ], + }, + 'description': + '

🎉🎉🎉 kubenav v4 the dashboard for iOS and Android is now available kubenav.io 🥳🥳🥳

', + 'author': '@ricoberger@hachyderm.io', + 'publishedAt': 1675014977, + }]); + } finally { + fetchWithTimeoutSpy.restore(); + uploadSourceIconSpy.restore(); + } + + assertSpyCall(fetchWithTimeoutSpy, 0, { + args: ['https://hachyderm.io/@ricoberger.rss', { method: 'get' }, 5000], + returned: new Promise((resolve) => { + resolve(new Response(responseUser, { status: 200 })); + }), + }); + assertSpyCall(uploadSourceIconSpy, 0, { + args: [ + supabaseClient, + { + 'id': 'mastodon-myuser-mycolumn-5673bdae9d06a0744e93d647fe5cef2e', + 'columnId': 'mycolumn', + 'userId': 'myuser', + 'type': 'mastodon', + 'title': 'Rico Berger', + 'options': { 'mastodon': 'https://hachyderm.io/@ricoberger.rss' }, + 'link': 'https://hachyderm.io/@ricoberger', + 'icon': + 'https://media.hachyderm.io/accounts/avatars/109/773/619/675/865/785/original/bf731ded4166a661.png', + }, + ], + returned: new Promise((resolve) => { + resolve(undefined); + }), + }); + assertSpyCalls(fetchWithTimeoutSpy, 1); + assertSpyCalls(uploadSourceIconSpy, 1); +}); diff --git a/supabase/functions/_shared/feed/medium.ts b/supabase/functions/_shared/feed/medium.ts index 240c323..f0fc511 100644 --- a/supabase/functions/_shared/feed/medium.ts +++ b/supabase/functions/_shared/feed/medium.ts @@ -7,11 +7,54 @@ import { unescape } from 'lodash'; import { IItem } from '../models/item.ts'; import { ISource } from '../models/source.ts'; -import { Favicon, getFavicon } from './utils/getFavicon.ts'; -import { uploadSourceIcon } from './utils/uploadFile.ts'; +import { Favicon, feedutils } from './utils/index.ts'; import { IProfile } from '../models/profile.ts'; -import { fetchWithTimeout } from '../utils/fetchWithTimeout.ts'; -import { log } from '../utils/log.ts'; +import { utils } from '../utils/index.ts'; + +/** + * `faviconFilter` is a filter function for the favicons. It filters out all the + * favicons which are not hosted on the Medium CDN. + */ +export const faviconFilter = (favicons: Favicon[]): Favicon[] => { + return favicons.filter((favicon) => { + return favicon.url.startsWith('https://cdn-images'); + }); +}; + +/** + * `parseMediumOption` parses the provided `medium` option and returns a valid + * Medium feed url. The `medium` option can be a Medium url, a Medium tag or a + * Medium username. If the provided option is not valid we throw an error. + */ +export const parseMediumOption = (input?: string): string => { + if (input) { + if (input.length > 1 && input[0] === '#') { + return `https://medium.com/feed/tag/${input.slice(1)}`; + } else if (input.length > 1 && input[0] === '@') { + return `https://medium.com/feed/${input}`; + } else { + const parsedUrl = new URL(input); + const parsedHostname = parsedUrl.hostname.split('.'); + if ( + parsedHostname.length === 2 && parsedHostname[0] === 'medium' && + parsedHostname[1] === 'com' + ) { + return `https://medium.com/feed/${ + input.replace('https://medium.com/', '').replace('feed/', '') + }`; + } else if ( + parsedHostname.length === 3 && parsedHostname[1] === 'medium' && + parsedHostname[2] === 'com' + ) { + return `https://${parsedHostname[0]}.medium.com/feed`; + } else { + throw new Error('Invalid source options'); + } + } + } else { + throw new Error('Invalid source options'); + } +}; /** * `isMediumUrl` checks if the provided `url` is a valid Medium url. A url is @@ -28,51 +71,19 @@ export const getMediumFeed = async ( _profile: IProfile, source: ISource, ): Promise<{ source: ISource; items: IItem[] }> => { - /** - * Since the `medium` option supports multiple input format we need to - * normalize it to a valid Medium feed url. If this is not possible we - * consider the provided option as invalid. - */ - if (source.options?.medium) { - const input = source.options.medium; - if (input.length > 1 && input[0] === '#') { - source.options.medium = `https://medium.com/feed/tag/${input.slice(1)}`; - } else if (input.length > 1 && input[0] === '@') { - source.options.medium = `https://medium.com/feed/${input}`; - } else { - const parsedUrl = new URL(input); - const parsedHostname = parsedUrl.hostname.split('.'); - if ( - parsedHostname.length === 2 && parsedHostname[0] === 'medium' && - parsedHostname[1] === 'com' - ) { - source.options.medium = `https://medium.com/feed/${ - input.replace('https://medium.com/', '').replace('feed/', '') - }`; - } else if ( - parsedHostname.length === 3 && parsedHostname[1] === 'medium' && - parsedHostname[2] === 'com' - ) { - source.options.medium = `https://${parsedHostname[0]}.medium.com/feed`; - } else { - throw new Error('Invalid source options'); - } - } - } else { - throw new Error('Invalid source options'); - } + const parsedMediumOption = parseMediumOption(source.options?.medium); /** * Get the RSS for the provided `medium` url and parse it. If a feed doesn't * contains an item we return an error. */ - const response = await fetchWithTimeout(source.options.medium, { + const response = await utils.fetchWithTimeout(parsedMediumOption, { method: 'get', }, 5000); const xml = await response.text(); - log('debug', 'Add source', { + utils.log('debug', 'Add source', { sourceType: 'medium', - requestUrl: source.options.medium, + requestUrl: parsedMediumOption, responseStatus: response.status, }); const feed = await parseFeed(xml); @@ -87,18 +98,11 @@ export const getMediumFeed = async ( * to try to get the favicon when the source is created the first time. */ if (source.id === '' && feed.links.length > 0) { - const favicon = await getFavicon( - feed.links[0], - (favicons: Favicon[]): Favicon[] => { - return favicons.filter((favicon) => { - return favicon.url.startsWith('https://cdn-images'); - }); - }, - ); + const favicon = await feedutils.getFavicon(feed.links[0], faviconFilter); if (favicon && favicon.url.startsWith('https://')) { source.icon = favicon.url; - source.icon = await uploadSourceIcon(supabaseClient, source); + source.icon = await feedutils.uploadSourceIcon(supabaseClient, source); } } @@ -111,11 +115,12 @@ export const getMediumFeed = async ( source.id = generateSourceId( source.userId, source.columnId, - source.options.medium, + parsedMediumOption, ); } source.type = 'medium'; source.title = feed.title.value; + source.options = { medium: parsedMediumOption }; if (feed.links.length > 0) { source.link = feed.links[0]; } diff --git a/supabase/functions/_shared/feed/medium_test.ts b/supabase/functions/_shared/feed/medium_test.ts index 0e1759a..d3100c4 100644 --- a/supabase/functions/_shared/feed/medium_test.ts +++ b/supabase/functions/_shared/feed/medium_test.ts @@ -1,5 +1,136 @@ import { assertEquals } from 'std/assert'; -import { isMediumUrl } from './medium.ts'; +import { createClient } from '@supabase/supabase-js'; +import { + assertSpyCall, + assertSpyCalls, + returnsNext, + stub, +} from 'std/testing/mock'; + +import { ISource } from '../models/source.ts'; +import { IProfile } from '../models/profile.ts'; +import { + faviconFilter, + getMediumFeed, + isMediumUrl, + parseMediumOption, +} from './medium.ts'; +import { utils } from '../utils/index.ts'; +import { feedutils } from './utils/index.ts'; + +const supabaseClient = createClient('http://localhost:54321', 'test123'); +const mockProfile: IProfile = { + id: '', + tier: 'free', + createdAt: 0, + updatedAt: 0, +}; +const mockSource: ISource = { + id: '', + columnId: 'mycolumn', + userId: 'myuser', + type: 'medium', + title: '', +}; + +const responseTag = ` + + + <![CDATA[Kubernetes on Medium]]> + + https://medium.com/tag/kubernetes/latest?source=rss------kubernetes-5 + + https://cdn-images-1.medium.com/proxy/1*TGH72Nnw24QL3iV9IOm4VA.png + Kubernetes on Medium + https://medium.com/tag/kubernetes/latest?source=rss------kubernetes-5 + + Medium + Tue, 05 Dec 2023 18:26:48 GMT + + + + + <![CDATA[Securing the Cloud: A Comprehensive Guide to Building a Kubernetes Threat Model for Enhanced…]]> +

In the ever-evolving landscape of cloud computing and container orchestration, Kubernetes has emerged as a dominant force, providing a…

Continue reading on Medium »

]]> + https://blackcatdev.medium.com/securing-the-cloud-a-comprehensive-guide-to-building-a-kubernetes-threat-model-for-enhanced-e695e24db604?source=rss------kubernetes-5 + https://medium.com/p/e695e24db604 + + + + + + + Tue, 05 Dec 2023 18:18:40 GMT + 2023-12-05T18:18:40.249Z + + + <![CDATA[Fortifying Kubernetes Deployments: A Comprehensive Guide to Securing CI/CD Pipelines]]> +

# Securing CI/CD Pipelines for Kubernetes Deployments

Continue reading on Medium »

]]> + https://blackcatdev.medium.com/fortifying-kubernetes-deployments-a-comprehensive-guide-to-securing-ci-cd-pipelines-096aacb27637?source=rss------kubernetes-5 + https://medium.com/p/096aacb27637 + + + + + + + Tue, 05 Dec 2023 18:17:03 GMT + 2023-12-05T18:17:03.957Z + + +`; + +const responseUser = ` + + + <![CDATA[Stories by Yuri Shkuro on Medium]]> + + https://medium.com/@YuriShkuro?source=rss-cff2e4ba6058------2 + + https://cdn-images-1.medium.com/fit/c/150/150/0*4Ljs60hv_8hpF_mj.jpg + Stories by Yuri Shkuro on Medium + https://medium.com/@YuriShkuro?source=rss-cff2e4ba6058------2 + + Medium + Tue, 05 Dec 2023 20:19:35 GMT + + + + + <![CDATA[Experiment: Migrating OpenTracing-based application in Go to use the OpenTelemetry SDK]]> + https://medium.com/jaegertracing/experiment-migrating-opentracing-based-application-in-go-to-use-the-opentelemetry-sdk-29b09fe2fbc4?source=rss-cff2e4ba6058------2 + https://medium.com/p/29b09fe2fbc4 + + + + + + Thu, 09 Feb 2023 05:44:06 GMT + 2023-02-09T05:46:24.711Z + TL;DR: This post explains how Jaeger’s 🚗 HotROD 🚗 app was migrated to the OpenTelemetry SDK.

Jaeger’s HotROD demo has been around for a few years. It was written with OpenTracing-based instrumentation, including a couple of OSS libraries for HTTP and gRPC middleware, and used Jaeger’s native SDK for Go, jaeger-client-go. The latter was deprecated in 2022, so we had a choice to either convert all of the HotROD app’s instrumentation to OpenTelemetry, or try the OpenTracing-bridge, which is a required part of every OpenTelemetry API / SDK. The bridge is an adapter layer that wraps an OpenTelemetry Tracer in a facade to makes it look like the OpenTracing Tracer. This way we can use the OpenTelemetry SDK in an application like HotROD that only understands the OpenTracing API.

I wanted to try the bridge solution, to minimize the code changes in the application. It is not the most efficient way, since an adapter layer incurs some performance overhead, but for a demo app it seemed like a reasonable trade-off.

The code can be found in the Jaeger repository (at specific commit hash).

Setup

First, we need to initialize the OpenTelemetry SDK and create an OpenTracing Bridge. Fortunately, I did not have to start from scratch, because there was an earlier pull request #3390 by @rbroggi, which I picked up to make certain improvements. Initialization happens in pkg/tracing/init.go :

In the beginning, this is a pretty vanilla OpenTelemetry SDK initialization. We create an exporter using a helper function (more on it below), and build a TracerProvider. The compliant OpenTelemetry instrumentation would use this provider to create named Tracer objects as needed, usually with distinct names reflecting the instrumentation library or the application component. However, the OpenTracing API did not have the concept of named tracers, its Tracer as a singleton, so here we create a Tracer with a blank name (in line 23) and pass it to the bridge factory that wraps it and returns an OpenTracing Tracer.

Side note: in a better-organized code there would also be some sort of close/shutdown function returned so that the caller of tracing.Init could gracefully shutdown the tracer, e.g. to flush the span buffers when stopping the application.

The original PR used the Jaeger exporter that lets the SDK export data in the Jaeger’s native data format. However, last year we extended Jaeger to accept OpenTelemetry’s OTLP format directly, so I decided to add a bit of flexibility and make the choice of the exporter configurable:

Broken Traces

At this point things should have started to work. However, the resulting traces looked like this:

Trace with many spans, but all coming from a single service frontend.
Another part of the workflow captured as a different trace. It looks like there are two services here, but in fact the HotROD app simulates the sql and redis database services, it’s not actually making RPC calls to them.

Instead of one trace per request we are getting several disjoined traces. This is where @rbroggi’s PR got stuck. After some debugging I came to realize that the SDK defaults to a no-op propagation, so no trace context was sent in RPC requests between services, resulting in multiple disjoined traces for the same workflow. It was easy to fix, but it felt like an unnecessary friction in using the OpenTelemetry SDK. I also added the Baggage propagator, which we will discuss later.

Since the Init() function is called many times by different services in the HotROD app, I only set the propagator once using sync.Once.

After this change, the traces looked better, more colorful, so I committed the change.

A better-looking trace after “fixing” the propagation.

Traces Still Broken

However, I should’ve paid better attention. Notice the lone span in the middle called /driver.DriverService/FindNearest. Let’s take a closer look:

A client span trying to make a gRPC request to service driver.

This span is a client-side of a gRPC request from frontend service to driver service. The latter is missing from the trace! This was a different issue with the context propagation. There was an error returned when trying to inject the context into the request headers. The instrumentation actually logged the error back into the client span, which we can see in the Logs section: Invalid Inject/Extract carrier. Unfortunately, it was difficult to spot this error without opening up the span, because the RPC itself was successful, and the instrumentation was correct in not setting the error=true span tag, which would’ve shown in the Jaeger UI as a red icon.

After a bit more digging I found the issue, which was due to a bug in the OpenTelemetry SDK’s bridge implementation. You can read about it in the following GitHub issue.

[opentracing] OT Bridge does not work with OT gRPC instrumentation · Issue #3678 · open-telemetry/opentelemetry-go

As of this writing, the fix if still waiting to be merged, so as a workaround I made a branch of opentracing-contrib/go-grpc and changed it to use TextMap propagation instead of HTTPHeaders, which by chance happened to work with the bridge code.

With these fixes, we were back to the “classic” HotROD traces.

Full HotROD trace, as the AI overlords intended.

RPC Metrics

I was ready to call it a day, but there was one piece missing. The original Jaeger SDK initialization code had one extra feature — it was enabling the collection of RPC metrics from spans (supported only by the Go SDK in Jaeger). My original blog post, Take OpenTracing for a HotROD ride, had a discussion about it, so it was a shame to lose this during this upgrade. If I were upgrading to the OpenTelemetry instrumentation as well, it might have contained a metrics-oriented instrumentation, although it would somewhat miss the point of the blog post that tracing instrumentation is already sufficient in this case. Another possibility is to generate metrics from spans using a special processor in the OpenTelemetry Collector, but using the Collector is not part of the HotROD demo setup.

The OpenTelemetry SDK has the notion of span processors, an abstract API invoked on all finished spans. It is similar to how the RPCMetricsObserver was implemented in the jaeger-client-go, so I did what any scrappy engineer would do — copy & paste the code from jaeger-client-go directly into the HotROD code and adopt it to implement otel.SpanProcessor. And voilà:

$ curl http://127.0.0.1:8083/debug/vars | grep '"requests.endpoint_HTTP'
"requests.endpoint_HTTP_GET_/.error_false": 3,
"requests.endpoint_HTTP_GET_/.error_true": 0,
"requests.endpoint_HTTP_GET_/config.error_false": 4,
"requests.endpoint_HTTP_GET_/config.error_true": 0,
"requests.endpoint_HTTP_GET_/customer.error_false": 4,
"requests.endpoint_HTTP_GET_/customer.error_true": 0,
"requests.endpoint_HTTP_GET_/debug/vars.error_false": 5,
"requests.endpoint_HTTP_GET_/debug/vars.error_true": 0,
"requests.endpoint_HTTP_GET_/dispatch.error_false": 4,
"requests.endpoint_HTTP_GET_/dispatch.error_true": 0,
"requests.endpoint_HTTP_GET_/route.error_false": 40,
"requests.endpoint_HTTP_GET_/route.error_true": 0,

Baggage

As I was looking through the metrics in HotROD, I realized there was another area I neglected. These sections in the expvar output were not supposed to be empty:

$ curl http://127.0.0.1:8083/debug/vars | grep route.calc.by
"route.calc.by.customer.sec": {},
"route.calc.by.session.sec": {}

These measures require baggage to work. The term “baggage” was introduced in the academia (Jonathan Mace et al., SOSP 2015 Best Paper Award). It refers to a general-purpose context propagation mechanism, which can be used to carry both the tracing context and any other contextual metadata across the distributed workflow execution. The HotROD app demonstrates a number of capabilities that require baggage propagation, and they were all completely broken after upgrading to OpenTelemetry SDK 😭.

The first thing that broke was propagation of baggage from the web UI. HotROD does not start the trace in the browser, only in the backend. The Jaeger SDK had a feature that allowed it to accept baggage from the incoming request even when there was no incoming tracing context. Internally the Jaeger SDK achieved this by returning an “invalid” SpanContext from the Extract method where the trace ID / span ID were blank, but the baggage was present. Digging through the OpenTracing Bridge code I found that it returns an error in this case. This could probably be fixed there, but I decided to add a workaround directly to HotROD where I used the OpenTelemetry’s Baggage propagator to extract the baggage from the request manually and then copy it into the span.

I trimmed down the code example above a bit to only show relevant parts. The otelBaggageExtractor function creates a middleware that manually extracts the baggage into the current Context. Then the instrumentation library nethttp is given a span observer (invoked after the server span is created) which copies the baggage from the context into the span. This functionality is only needed at the root span, because once the trace context is propagated through the workflow, the Bridge correctly propagates the baggage as well (remember that I registered Baggage propagator as a global propagator in the Init function, as shown in the earlier code snippet). I was actually pleasantly surprised that the maintainers were able to achieve that, because the OpenTracing API operates purely on Span objects, not on the Context, while in OpenTelemetry the baggage is carried in the Context, a lower logical level.

One other small change I had to make was to change the web UI to use the baggage header (per W3C standard), instead of the jaeger-baggage header that was recognized by the Jaeger SDK.

Strictly speaking, these were all the changes I had to make to the HotROD code to make the baggage work. Yet, it didn’t work. Some baggage values were correctly propagated, but others were missing. After more digging I found several places where it was silently dropped on the floor because of some (misplaced, in my opinion) validations in the baggage and the bridge/opentracing packages in the OpenTelemetry SDK. The ticket below explains the issue in more details.

Baggage not working with OpenTracing Bridge · Issue #3685 · open-telemetry/opentelemetry-go

Running against a patched version of OpenTelemetry SDK yielded the desired behavior and the baggage-reliant functionality was restored. I was getting performance metrics grouped by baggage values:

$ curl http://127.0.0.1:8083/debug/vars | grep route.calc.by
"route.calc.by.customer.sec": {
  "Amazing Coffee Roasters": 0.9080000000000004, 
  "Japanese Desserts": 1.0490000000000002, 
  "Rachel's Floral Designs": 1.0090000000000003, 
  "Trom Chocolatier": 1.0000000000000004
},
"route.calc.by.session.sec": {
  "2885": 1.4760000000000002, 
  "5161": 2.4899999999999993
}

And the mutex instrumentation was able to capture IDs of multiple transactions in the queue (see the original blog post for explanation of this one):

Logs show a transactions blocked on three other transactions.

Summary

Overall, the migration required fairly minimal amount of changes to the code, mostly because I chose to reuse the existing OpenTracing instrumentation and only swap the SDK from Jaeger to OpenTelemetry. The most friction with the migration was due to a couple of bugs in the OpenTelemetry Bridge code (and likely one place in the baggage package). This only leads me to believe that the baggage functionality is not yet widely used, especially when someone uses the OpenTracing instrumentation with a bridge to OpenTelemetry, so it is likely I just ran into a bunch of the early adopter issues.

At this point I am interested in taking the next step and doing a full migration of HotROD to OpenTelemetry (or help reviewing if someone wants to volunteer!) It could make a complementary Part 2 to this post to describe how that goes.

There is also a possible Part 3 involving a no-less interesting migration to the OpenTelemetry Metrics. Right now all of the Jaeger code base is using an internal abstraction for metrics backed by the Prometheus SDK.

Stay tuned.

Experiment: Migrating OpenTracing-based application in Go to use the OpenTelemetry SDK was originally published in JaegerTracing on Medium, where people are continuing the conversation by highlighting and responding to this story.

]]> + + + <![CDATA[Better alignment with OpenTelemetry by focusing on OTLP]]> + https://medium.com/jaegertracing/better-alignment-with-opentelemetry-by-focusing-on-otlp-f3688939073f?source=rss-cff2e4ba6058------2 + https://medium.com/p/f3688939073f + + + Thu, 03 Nov 2022 18:12:55 GMT + 2022-11-03T18:12:55.688Z + TL;DR: proposal (and a survey) to deprecate native Jaeger exporters in OpenTelemetry SDKs in favor of OTLP exporters.

Photo by Miquel Parera on Unsplash

This is a re-post from the OpenTelemetry blog article.

By Jason Plumb (Splunk) | Thursday, November 03, 2022

Back in May of 2022, the Jaeger project announced native support for the OpenTelemetry Protocol (OTLP). This followed a generous deprecation cycle for the Jaeger client libraries across many languages. With these changes, OpenTelemetry users are now able to send traces into Jaeger with industry-standard OTLP, and the Jaeger client library repositories have been finally archived.

We intend to deprecate Jaeger exporters from OpenTelemetry in the near future, and are looking for your feedback to determine the length of the deprecation phase. The best way to provide feedback is by filling out a 4-question survey or commenting on the existing draft pull request.

OpenTelemetry Support

This interoperability is a wonderful victory both for Jaeger users and for OpenTelemetry users. However, we’re not done yet. The OpenTelemetry specification still requires support for Jaeger client exporters across languages.

This causes challenges for both Jaeger users and OpenTelemetry maintainers:

  1. Confusing Choices: Currently, users are faced with a choice of exporter (Jaeger or OTLP), and this can be a source of confusion. A user might be inclined, when exporting telemetry to Jaeger, to simply choose the Jaeger exporter because the name matches (even though Jaeger now actively encourages the use of OTLP).
    If we can eliminate this potentially confusing choice, we can improve the user experience and continue standardizing on a single interoperable protocol. We love it when things “just work” out of the box!
  2. Maintenance and duplication: Because the Jaeger client libraries are now archived, they will not receive updates (including security patches). To continue properly supporting Jaeger client exporters, OpenTelemetry authors would be required to re-implement some of the functionality it had previously leveraged from the Jaeger clients.
    Now that Jaeger supports OTLP, this feels like a step backwards: It results in an increased maintenance burden with very little benefit.

User Impact

The proposal is to deprecate the following exporters from OpenTelemetry in favor of using native OTLP into Jaeger:

  • Jaeger Thrift over HTTP
  • Jaeger Protobuf via gRPC
  • Jaeger Thrift over UDP

In addition to application configuration changes, there could be other architectural considerations. HTTP and gRPC should be straightforward replacements, although it may require exposing ports 4317 and 4318 if they are not already accessible.

Thrift over UDP implies the use of the Jaeger Agent. Users with this deployment configuration will need to make a slightly more complicated change, typically one of the following:

  1. Direct ingest. Applications will change from using Thrift+UDP to sending OTLP traces directly to their jaeger-collector instance. This may also have sampling implications.
  2. Replacing the Jaeger Agent with a sidecar OpenTelemetry Collector instance. This could have sampling implications and requires changes to your infrastructure deployment code.

Intent to Deprecate — We’d Like Your Feedback!

In order to better support users and the interop between OpenTelemetry and Jaeger, we intend to deprecate and eventually remove support for Jaeger client exporters / Jaeger native data format in OpenTelemetry.

We would like your feedback! We want to hear from users who could be impacted by this change. To better make a data-informed decision, we have put together a short 4-question survey.

Your input will help us to choose how long to deprecate before removal.

A draft PR has been created in the specification to support this deprecation. If would like to contribute and provide feedback, visit the link above and add some comments. We want to hear from you.

Better alignment with OpenTelemetry by focusing on OTLP was originally published in JaegerTracing on Medium, where people are continuing the conversation by highlighting and responding to this story.

]]> + + + <![CDATA[TEMPLE: Six Pillars of Observability]]> + https://medium.com/@YuriShkuro/temple-six-pillars-of-observability-4ac3e3deb402?source=rss-cff2e4ba6058------2 + https://medium.com/p/4ac3e3deb402 + + + Mon, 19 Sep 2022 01:51:28 GMT + 2022-10-05T01:29:48.705Z + A temple in Italy with six front pillars
Valley of the Temples, Agrigento, AG, Italy. Photo by Dario Crisafulli on Unsplash.

In the past few years, much has been talked and written about the “three pillars of observability”: metrics, logs, and traces. A Google search for the phrase brings up over 7,000 results, with almost every observability vendor having a blog post or an e-book on the topic. Recently, the term MELT started showing up that adds “events” to the mix as a distinct telemetry signal. In this post, I want to show that there are even more distinct types and introduce TEMPLE, which stands for traces, events, metrics, profiles, logs, and exceptions. I call them six pillars of observability, on one hand to make fun of the previous terms and acronyms, but also to make the case that these signals serve distinct use cases for observability of cloud-native systems. If I fail at the latter and you don’t buy my arguments, at least the TEMPLE acronym works much better with “pillars” 😉.

Attribution: One of my colleagues at Meta, Henry Bond, started using the acronym TEMPL in the internal documents. I added “Exceptions”, for completeness, and ended up with TEMPLE.

Six Pillars Explained

I will try to illustrate why I think these six telemetry types deserve to be considered as separate. It does not mean some of them cannot be supported by the same backend, but they differ in the following aspects:

  • How each telemetry type is produced
  • Which unique storage requirements they impose
  • How they are used in the user workflows

Even though the TEMPLE acronym implies a certain ordering of the signals, I do not ascribe any meaning to that other than to make up a pleasant word. For better continuity of the explanation, I will go through them in a different order.

Also, naming is hard. I will point out how, surprisingly, most of the terms we use in this space are ambiguous, and the boundaries between telemetry types are not as strict as they appear to be.

Metrics, the original pillar. Numerical measurements with attributes, which are easily aggregatable both spacially (along the attribute dimensions) and temporally (combining values into less discrete time intervals). Metrics aggregates remain highly accurate, which makes them great for monitoring, but aggregations lose the original level of details, which makes metrics not as good for troubleshooting & investigations.

In the context of cloud native applications, metrics usually refer to operational metrics of the software. Business metrics are actually a different category that is better captured via structured logs.

Logs, the ancient pillar (if you question which came first, “ancient” or “original”, take it up with Marvel). Logs are a confusing category, ranging from arbitrary printf-like statements (aka unstructured logs) to highly structured and even schematized events. When structured logs are schematized (cf. Schema-first Application Telemetry), they are often sent to different tables in the warehouse and used for analytics, including business analytics. Schema-free structured logs are what Honeycomb calls “arbitrarily-wide events”. It’s a bit of a misnomer, because each individual log record is not “arbitrarily wide”, in fact it usually has a completely static shape that will not change in the given call site, but when we mix these log records from different call sites we can expect all kinds of shapes, making the resulting destination an “arbitrarily wide” table. Most modern logging backends can ingest structured logs and allow search and analytics on these arbitrary dimensions.

Many logs are generated in response to a service processing specific input requests, i.e. they are request-scoped. We, as an industry, haven’t really figured out how best to deal with request-scoped logs. On one hand, they look like other logs and can be stored and analyzed in a similar fashion. On the other hand, distributed tracing is specifically designed as request-scoped logging, so these logs could be much more useful when viewed in the context of a distributed trace. I met engineers whose teams chose to use tracing APIs exclusively to capture logs, so that those can be always visualized in the context of traces. A common approach to solve this is to capture trace ID as a field in the logs and build cross-correlation between logging and tracing tools.

Traces, the “new cool kid on the block” pillar. The term tracing is quite overloaded (just look at Linux Tracing documentation). In the context of cloud native observability, tracing usually refers to distributed tracing, or a special form of structured logs that are request-scoped, or more generally workflow-centric. In contrast to plain structured logging, tracing captures not only the log records, but also causality between them, and once those records are stitched into a trace they represent the trajectory of a single request (or a workflow) through a distributed system end-to-end.

Tracing opens up a realm of possibilities for reasoning about a system:

  • Unique monitoring capabilities, e.g., an end-to-end latency of messaging workflows, which is difficult to observe with any other telemetry.
  • Debugging capabilities, in particular, root cause isolation. Traces may not always tell you why a system is misbehaving, but they often can narrow down which of the thousands of distributed components is at fault.
  • Resource usage attribution by different caller profiles or business lines.

Events, the misunderstood pillar. This is perhaps the worst-named category of telemetry signals because, strictly speaking, pretty much all telemetry is “events”. What people usually mean by this category is change events, i.e., events that are external to the observed system that cause some changes in that system. The most common examples are: deployments of application code (and the corresponding code commits), configuration changes, experiments, DR-related traffic drains, perhaps auto-scaling events, etc.

There is no practical bound to what could be considered an event that affects the system. For instance, in the early days of Uber, Halloween was the night of the highest user traffic, such that SREs would spend the whole night in the “war room”, monitoring the system and firefighting (Big Bang Theory flashback: Howard: “guidance system for drunk people”, Raj: “they already had that, it’s called Uber”). As the business became more global, the impact of Halloween as US-centric holiday became less pronounced on the system traffic, but one can easily see how holidays, or some other public events like sports or concerts, can become factors that affect the behavior of a system and might be useful to show to the operators as part of the system’s observability.

One could reasonably ask: why can’t we treat events simply as structured logs? As far as the data shape in the storage, there is indeed not much difference. However, logs usually require less rigor from the backends capturing them: some level of data loss may be acceptable, and the pipelines are often set up to down-sample or throttle the logs. For example, if a bug is causing an application to log a certain error message, it’s likely that we’ll have many similar logs, so it’s not critical to guarantee that every one of them is stored. This is very different from the handling of change events, which should be all stored reliably, because if we miss the record about that one single code deployment that caused the issue and needs to be rolled back, our outage investigation might take much longer. Similarly, when querying for logs, it’s usually sufficient to find some samples of a pattern, or to get aggregate statistics, there is not much emphasis on finding a very specific individual log record. But with change events, we’re precisely looking for very specific instances. Finally, change events are usually produced in much fewer volumes than logs. These differences in the requirements often lead to different designs and trade-offs in the logs and events backends.

Profiles, the geek pillar. Profiles are another category of telemetry that is tricky to define, although you would know it when you see one. Profiles are just being introduced as a new signal to the OpenTelemetry scope in the OTEP-212, and even that document had a bit of a tough time defining what a profile is. Its latest definition is “a collection of stack traces with some metric associated with each stack trace, typically representing the number of times that stack trace was encountered”. Mouthful.

Many engineers encounter profiling tasks at some point, but from my experience most of them do not have to deal with profiles very often, unless they specialize in performance and efficiency optimizations. Profiling tools, as a result, tend to be somewhat esoteric, focusing on power users.

Profiles, unlike most other telemetry types, almost never require explicit instrumentation, instead relying on deeper integration with the runtimes to capture the call stacks. They often generate very large amounts of data, requiring specially designed backends.

Exceptions, the forgotten pillar. Finally, let’s not forget the exceptions. Remember the “Stacktrace or GTFO” comics? When I first came across it, I was working at an investment bank developing trading systems, and we had zero observability into the system running in production (we could get access to logs on the hosts, but only by going through a special permissions escalation process, because … lawyers). So the comics resonated with me a lot at the time. But years later, my attitude changed to “couldn’t he just use Sentry or something?”

My first experience with Sentry was by accident. We just integrated Jaeger SDK into a Python framework that was widely used at Uber. Next morning I am getting a UBN ticket (“UnBreakNow”, i.e., high urgency) that says they are getting errors in production and the stacktrace points to Jaeger SDK code. But instead of a stacktrace the ticket had a link to Sentry, which was the open source system Uber deployed to capture and aggregate exceptions. I was blown away by the amount of information captured by Raven (Sentry’s SDK) besides the stacktrace itself. The most useful was the ability to inspect values of all local variables at every frame of the stack. That immediately revealed the root cause, which had to do with the handling of utf-8 strings.

Exceptions are, strictly speaking, a specialized form of structured logs, although you may need much more structure than the typical structured logging API allows (like nested collections, etc.) The processing pipelines for exceptions are also pretty specialized: they often involve symbolication, fingerprinting, stacks pruning, and deduplication. Finally, the UI for viewing this data is also highly customized to this data source. All these factors lead me to conclude that exceptions should really be treated as an independent telemetry type.

Pillars are not Observability

Now that we covered the six pillars, it’s worth remembering that pillars do not guarantee observability, which is defined, perhaps counterintuitively, as the ability to understand the system (‘s internal state) from its outputs, not just to observe the outputs. These pillars are just different types of telemetry that can be produced, the raw data. To be effective in investigations, the observability platform needs to be able to combine these signals into solutions for specific workflows. Even with free-form investigations, where you are “on your own” because all the guided investigation workflows failed, the platform can provide many features to assist you, such as understanding the metadata of the telemetry and allowing cross-telemetry correlations, or automation of insights and pattern recognition. Pillars are what you build upon, not the end goal.

Takeaways

  1. Stop saying “three pillars”. There are more than three.
  2. Start saying TEMPLE, if you must name them.
  3. Don’t take it seriously. The boundaries are diffuse.
  4. Pillars ≠ observability, they are just data.
]]> + + +`; Deno.test('isMediumUrl', () => { assertEquals(isMediumUrl('https://acceldataio.medium.com'), true); @@ -8,3 +139,239 @@ Deno.test('isMediumUrl', () => { assertEquals(isMediumUrl('https://medium.com/@YuriShkuro'), true); assertEquals(isMediumUrl('https://www.google.de/'), false); }); + +Deno.test('parseMediumOption', () => { + assertEquals( + parseMediumOption('#kubernetes'), + 'https://medium.com/feed/tag/kubernetes', + ); + assertEquals( + parseMediumOption('@YuriShkuro'), + 'https://medium.com/feed/@YuriShkuro', + ); + assertEquals( + parseMediumOption('https://medium.com/feed/tag/kubernetes'), + 'https://medium.com/feed/tag/kubernetes', + ); + assertEquals( + parseMediumOption('https://medium.com/feed/@YuriShkuro'), + 'https://medium.com/feed/@YuriShkuro', + ); + assertEquals( + parseMediumOption('https://acceldataio.medium.com'), + 'https://acceldataio.medium.com/feed', + ); + assertEquals( + parseMediumOption('https://acceldataio.medium.com/feed'), + 'https://acceldataio.medium.com/feed', + ); + assertEquals( + parseMediumOption('https://medium.com/jaegertracing'), + 'https://medium.com/feed/jaegertracing', + ); + assertEquals( + parseMediumOption('https://medium.com/feed/jaegertracing'), + 'https://medium.com/feed/jaegertracing', + ); +}); + +Deno.test('getMediumFeed - Tag', async () => { + const fetchWithTimeoutSpy = stub( + utils, + 'fetchWithTimeout', + returnsNext([ + new Promise((resolve) => { + resolve(new Response(responseTag, { status: 200 })); + }), + ]), + ); + + const getFaviconSpy = stub( + feedutils, + 'getFavicon', + returnsNext([ + new Promise((resolve) => { + resolve(undefined); + }), + ]), + ); + + try { + const { source, items } = await getMediumFeed( + supabaseClient, + undefined, + mockProfile, + { ...mockSource, options: { medium: '#kubernetes' } }, + ); + feedutils.assertEqualsSource(source, { + id: 'medium-myuser-mycolumn-40f28b0a56743a117745ac7dfd785111', + columnId: 'mycolumn', + userId: 'myuser', + type: 'medium', + title: 'Kubernetes on Medium', + options: { 'medium': 'https://medium.com/feed/tag/kubernetes' }, + link: + 'https://medium.com/tag/kubernetes/latest?source=rss------kubernetes-5', + }); + feedutils.assertEqualsItems(items, [{ + id: + 'medium-myuser-mycolumn-40f28b0a56743a117745ac7dfd785111-6286c43a67f72950c40bd4f537b66ed4', + userId: 'myuser', + columnId: 'mycolumn', + sourceId: 'medium-myuser-mycolumn-40f28b0a56743a117745ac7dfd785111', + title: + 'Securing the Cloud: A Comprehensive Guide to Building a Kubernetes Threat Model for Enhanced…', + link: + 'https://blackcatdev.medium.com/securing-the-cloud-a-comprehensive-guide-to-building-a-kubernetes-threat-model-for-enhanced-e695e24db604?source=rss------kubernetes-5', + media: 'https://cdn-images-1.medium.com/max/2600/0*HDPbjcSA7DITNbFp', + description: + '

In the ever-evolving landscape of cloud computing and container orchestration, Kubernetes has emerged as a dominant force, providing a…

Continue reading on Medium »

', + author: 'BlackCatDev', + publishedAt: 1701800320, + }, { + id: + 'medium-myuser-mycolumn-40f28b0a56743a117745ac7dfd785111-3ee71eda8d5f6978bb8b813f6ba34608', + userId: 'myuser', + columnId: 'mycolumn', + sourceId: 'medium-myuser-mycolumn-40f28b0a56743a117745ac7dfd785111', + title: + 'Fortifying Kubernetes Deployments: A Comprehensive Guide to Securing CI/CD Pipelines', + link: + 'https://blackcatdev.medium.com/fortifying-kubernetes-deployments-a-comprehensive-guide-to-securing-ci-cd-pipelines-096aacb27637?source=rss------kubernetes-5', + media: 'https://cdn-images-1.medium.com/max/2600/0*XZum3wouH1vlsKV9', + description: + '

# Securing CI/CD Pipelines for Kubernetes Deployments

Continue reading on Medium »

', + author: 'BlackCatDev', + publishedAt: 1701800223, + }]); + } finally { + fetchWithTimeoutSpy.restore(); + getFaviconSpy.restore(); + } + + assertSpyCall(fetchWithTimeoutSpy, 0, { + args: ['https://medium.com/feed/tag/kubernetes', { method: 'get' }, 5000], + returned: new Promise((resolve) => { + resolve(new Response(responseTag, { status: 200 })); + }), + }); + assertSpyCall(getFaviconSpy, 0, { + args: [ + 'https://medium.com/tag/kubernetes/latest?source=rss------kubernetes-5', + faviconFilter, + ], + returned: new Promise((resolve) => { + resolve(undefined); + }), + }); + assertSpyCalls(fetchWithTimeoutSpy, 1); + assertSpyCalls(getFaviconSpy, 1); +}); + +Deno.test('getMediumFeed - User', async () => { + const fetchWithTimeoutSpy = stub( + utils, + 'fetchWithTimeout', + returnsNext([ + new Promise((resolve) => { + resolve(new Response(responseUser, { status: 200 })); + }), + ]), + ); + + const getFaviconSpy = stub( + feedutils, + 'getFavicon', + returnsNext([ + new Promise((resolve) => { + resolve(undefined); + }), + ]), + ); + + try { + const { source, items } = await getMediumFeed( + supabaseClient, + undefined, + mockProfile, + { ...mockSource, options: { medium: '@YuriShkuro' } }, + ); + feedutils.assertEqualsSource(source, { + id: 'medium-myuser-mycolumn-77107c9209365c6c6c601a9f018a05f4', + columnId: 'mycolumn', + userId: 'myuser', + type: 'medium', + title: 'Stories by Yuri Shkuro on Medium', + options: { medium: 'https://medium.com/feed/@YuriShkuro' }, + link: 'https://medium.com/@YuriShkuro?source=rss-cff2e4ba6058------2', + }); + feedutils.assertEqualsItems(items, [{ + id: + 'medium-myuser-mycolumn-77107c9209365c6c6c601a9f018a05f4-7219a04f5c706f5e6c9e4af2f1060fe4', + userId: 'myuser', + columnId: 'mycolumn', + sourceId: 'medium-myuser-mycolumn-77107c9209365c6c6c601a9f018a05f4', + title: + 'Experiment: Migrating OpenTracing-based application in Go to use the OpenTelemetry SDK', + link: + 'https://medium.com/jaegertracing/experiment-migrating-opentracing-based-application-in-go-to-use-the-opentelemetry-sdk-29b09fe2fbc4?source=rss-cff2e4ba6058------2', + media: + 'https://cdn-images-1.medium.com/max/1024/1*b-RsMFPGcTtpxKaL2oRMEg.png', + description: + '

TL;DR: This post explains how Jaeger’s 🚗 HotROD 🚗 app was migrated to the OpenTelemetry SDK.

Jaeger’s HotROD demo has been around for a few years. It was written with OpenTracing-based instrumentation, including a couple of OSS libraries for HTTP and gRPC middleware, and used Jaeger’s native SDK for Go, jaeger-client-go. The latter was deprecated in 2022, so we had a choice to either convert all of the HotROD app’s instrumentation to OpenTelemetry, or try the OpenTracing-bridge, which is a required part of every OpenTelemetry API / SDK. The bridge is an adapter layer that wraps an OpenTelemetry Tracer in a facade to makes it look like the OpenTracing Tracer. This way we can use the OpenTelemetry SDK in an application like HotROD that only understands the OpenTracing API.

I wanted to try the bridge solution, to minimize the code changes in the application. It is not the most efficient way, since an adapter layer incurs some performance overhead, but for a demo app it seemed like a reasonable trade-off.

The code can be found in the Jaeger repository (at specific commit hash).

Setup

First, we need to initialize the OpenTelemetry SDK and create an OpenTracing Bridge. Fortunately, I did not have to start from scratch, because there was an earlier pull request #3390 by @rbroggi, which I picked up to make certain improvements. Initialization happens in pkg/tracing/init.go :

In the beginning, this is a pretty vanilla OpenTelemetry SDK initialization. We create an exporter using a helper function (more on it below), and build a TracerProvider. The compliant OpenTelemetry instrumentation would use this provider to create named Tracer objects as needed, usually with distinct names reflecting the instrumentation library or the application component. However, the OpenTracing API did not have the concept of named tracers, its Tracer as a singleton, so here we create a Tracer with a blank name (in line 23) and pass it to the bridge factory that wraps it and returns an OpenTracing Tracer.

Side note: in a better-organized code there would also be some sort of close/shutdown function returned so that the caller of tracing.Init could gracefully shutdown the tracer, e.g. to flush the span buffers when stopping the application.

The original PR used the Jaeger exporter that lets the SDK export data in the Jaeger’s native data format. However, last year we extended Jaeger to accept OpenTelemetry’s OTLP format directly, so I decided to add a bit of flexibility and make the choice of the exporter configurable:

Broken Traces

At this point things should have started to work. However, the resulting traces looked like this:

Trace with many spans, but all coming from a single service frontend.
Another part of the workflow captured as a different trace. It looks like there are two services here, but in fact the HotROD app simulates the sql and redis database services, it’s not actually making RPC calls to them.

Instead of one trace per request we are getting several disjoined traces. This is where @rbroggi’s PR got stuck. After some debugging I came to realize that the SDK defaults to a no-op propagation, so no trace context was sent in RPC requests between services, resulting in multiple disjoined traces for the same workflow. It was easy to fix, but it felt like an unnecessary friction in using the OpenTelemetry SDK. I also added the Baggage propagator, which we will discuss later.

Since the Init() function is called many times by different services in the HotROD app, I only set the propagator once using sync.Once.

After this change, the traces looked better, more colorful, so I committed the change.

A better-looking trace after “fixing” the propagation.

Traces Still Broken

However, I should’ve paid better attention. Notice the lone span in the middle called /driver.DriverService/FindNearest. Let’s take a closer look:

A client span trying to make a gRPC request to service driver.

This span is a client-side of a gRPC request from frontend service to driver service. The latter is missing from the trace! This was a different issue with the context propagation. There was an error returned when trying to inject the context into the request headers. The instrumentation actually logged the error back into the client span, which we can see in the Logs section: Invalid Inject/Extract carrier. Unfortunately, it was difficult to spot this error without opening up the span, because the RPC itself was successful, and the instrumentation was correct in not setting the error=true span tag, which would’ve shown in the Jaeger UI as a red icon.

After a bit more digging I found the issue, which was due to a bug in the OpenTelemetry SDK’s bridge implementation. You can read about it in the following GitHub issue.

[opentracing] OT Bridge does not work with OT gRPC instrumentation · Issue #3678 · open-telemetry/opentelemetry-go

As of this writing, the fix if still waiting to be merged, so as a workaround I made a branch of opentracing-contrib/go-grpc and changed it to use TextMap propagation instead of HTTPHeaders, which by chance happened to work with the bridge code.

With these fixes, we were back to the “classic” HotROD traces.

Full HotROD trace, as the AI overlords intended.

RPC Metrics

I was ready to call it a day, but there was one piece missing. The original Jaeger SDK initialization code had one extra feature — it was enabling the collection of RPC metrics from spans (supported only by the Go SDK in Jaeger). My original blog post, Take OpenTracing for a HotROD ride, had a discussion about it, so it was a shame to lose this during this upgrade. If I were upgrading to the OpenTelemetry instrumentation as well, it might have contained a metrics-oriented instrumentation, although it would somewhat miss the point of the blog post that tracing instrumentation is already sufficient in this case. Another possibility is to generate metrics from spans using a special processor in the OpenTelemetry Collector, but using the Collector is not part of the HotROD demo setup.

The OpenTelemetry SDK has the notion of span processors, an abstract API invoked on all finished spans. It is similar to how the RPCMetricsObserver was implemented in the jaeger-client-go, so I did what any scrappy engineer would do — copy & paste the code from jaeger-client-go directly into the HotROD code and adopt it to implement otel.SpanProcessor. And voilà:

$ curl http://127.0.0.1:8083/debug/vars | grep \'"requests.endpoint_HTTP\'
"requests.endpoint_HTTP_GET_/.error_false": 3,
"requests.endpoint_HTTP_GET_/.error_true": 0,
"requests.endpoint_HTTP_GET_/config.error_false": 4,
"requests.endpoint_HTTP_GET_/config.error_true": 0,
"requests.endpoint_HTTP_GET_/customer.error_false": 4,
"requests.endpoint_HTTP_GET_/customer.error_true": 0,
"requests.endpoint_HTTP_GET_/debug/vars.error_false": 5,
"requests.endpoint_HTTP_GET_/debug/vars.error_true": 0,
"requests.endpoint_HTTP_GET_/dispatch.error_false": 4,
"requests.endpoint_HTTP_GET_/dispatch.error_true": 0,
"requests.endpoint_HTTP_GET_/route.error_false": 40,
"requests.endpoint_HTTP_GET_/route.error_true": 0,

Baggage

As I was looking through the metrics in HotROD, I realized there was another area I neglected. These sections in the expvar output were not supposed to be empty:

$ curl http://127.0.0.1:8083/debug/vars | grep route.calc.by
"route.calc.by.customer.sec": {},
"route.calc.by.session.sec": {}

These measures require baggage to work. The term “baggage” was introduced in the academia (Jonathan Mace et al., SOSP 2015 Best Paper Award). It refers to a general-purpose context propagation mechanism, which can be used to carry both the tracing context and any other contextual metadata across the distributed workflow execution. The HotROD app demonstrates a number of capabilities that require baggage propagation, and they were all completely broken after upgrading to OpenTelemetry SDK 😭.

The first thing that broke was propagation of baggage from the web UI. HotROD does not start the trace in the browser, only in the backend. The Jaeger SDK had a feature that allowed it to accept baggage from the incoming request even when there was no incoming tracing context. Internally the Jaeger SDK achieved this by returning an “invalid” SpanContext from the Extract method where the trace ID / span ID were blank, but the baggage was present. Digging through the OpenTracing Bridge code I found that it returns an error in this case. This could probably be fixed there, but I decided to add a workaround directly to HotROD where I used the OpenTelemetry’s Baggage propagator to extract the baggage from the request manually and then copy it into the span.

I trimmed down the code example above a bit to only show relevant parts. The otelBaggageExtractor function creates a middleware that manually extracts the baggage into the current Context. Then the instrumentation library nethttp is given a span observer (invoked after the server span is created) which copies the baggage from the context into the span. This functionality is only needed at the root span, because once the trace context is propagated through the workflow, the Bridge correctly propagates the baggage as well (remember that I registered Baggage propagator as a global propagator in the Init function, as shown in the earlier code snippet). I was actually pleasantly surprised that the maintainers were able to achieve that, because the OpenTracing API operates purely on Span objects, not on the Context, while in OpenTelemetry the baggage is carried in the Context, a lower logical level.

One other small change I had to make was to change the web UI to use the baggage header (per W3C standard), instead of the jaeger-baggage header that was recognized by the Jaeger SDK.

Strictly speaking, these were all the changes I had to make to the HotROD code to make the baggage work. Yet, it didn’t work. Some baggage values were correctly propagated, but others were missing. After more digging I found several places where it was silently dropped on the floor because of some (misplaced, in my opinion) validations in the baggage and the bridge/opentracing packages in the OpenTelemetry SDK. The ticket below explains the issue in more details.

Baggage not working with OpenTracing Bridge · Issue #3685 · open-telemetry/opentelemetry-go

Running against a patched version of OpenTelemetry SDK yielded the desired behavior and the baggage-reliant functionality was restored. I was getting performance metrics grouped by baggage values:

$ curl http://127.0.0.1:8083/debug/vars | grep route.calc.by
"route.calc.by.customer.sec": {
  "Amazing Coffee Roasters": 0.9080000000000004, 
  "Japanese Desserts": 1.0490000000000002, 
  "Rachel\'s Floral Designs": 1.0090000000000003, 
  "Trom Chocolatier": 1.0000000000000004
},
"route.calc.by.session.sec": {
  "2885": 1.4760000000000002, 
  "5161": 2.4899999999999993
}

And the mutex instrumentation was able to capture IDs of multiple transactions in the queue (see the original blog post for explanation of this one):

Logs show a transactions blocked on three other transactions.

Summary

Overall, the migration required fairly minimal amount of changes to the code, mostly because I chose to reuse the existing OpenTracing instrumentation and only swap the SDK from Jaeger to OpenTelemetry. The most friction with the migration was due to a couple of bugs in the OpenTelemetry Bridge code (and likely one place in the baggage package). This only leads me to believe that the baggage functionality is not yet widely used, especially when someone uses the OpenTracing instrumentation with a bridge to OpenTelemetry, so it is likely I just ran into a bunch of the early adopter issues.

At this point I am interested in taking the next step and doing a full migration of HotROD to OpenTelemetry (or help reviewing if someone wants to volunteer!) It could make a complementary Part 2 to this post to describe how that goes.

There is also a possible Part 3 involving a no-less interesting migration to the OpenTelemetry Metrics. Right now all of the Jaeger code base is using an internal abstraction for metrics backed by the Prometheus SDK.

Stay tuned.

Experiment: Migrating OpenTracing-based application in Go to use the OpenTelemetry SDK was originally published in JaegerTracing on Medium, where people are continuing the conversation by highlighting and responding to this story.

', + author: 'Yuri Shkuro', + publishedAt: 1675921446, + }, { + id: + 'medium-myuser-mycolumn-77107c9209365c6c6c601a9f018a05f4-c4bb87fb4dbeee1a531afbe4fc8b5433', + userId: 'myuser', + columnId: 'mycolumn', + sourceId: 'medium-myuser-mycolumn-77107c9209365c6c6c601a9f018a05f4', + title: 'Better alignment with OpenTelemetry by focusing on OTLP', + link: + 'https://medium.com/jaegertracing/better-alignment-with-opentelemetry-by-focusing-on-otlp-f3688939073f?source=rss-cff2e4ba6058------2', + media: + 'https://cdn-images-1.medium.com/max/1024/1*iFJFYZsdPvaFuwaAoZ1HRQ.jpeg', + description: + '

TL;DR: proposal (and a survey) to deprecate native Jaeger exporters in OpenTelemetry SDKs in favor of OTLP exporters.

Photo by Miquel Parera on Unsplash

This is a re-post from the OpenTelemetry blog article.

By Jason Plumb (Splunk) | Thursday, November 03, 2022

Back in May of 2022, the Jaeger project announced native support for the OpenTelemetry Protocol (OTLP). This followed a generous deprecation cycle for the Jaeger client libraries across many languages. With these changes, OpenTelemetry users are now able to send traces into Jaeger with industry-standard OTLP, and the Jaeger client library repositories have been finally archived.

We intend to deprecate Jaeger exporters from OpenTelemetry in the near future, and are looking for your feedback to determine the length of the deprecation phase. The best way to provide feedback is by filling out a 4-question survey or commenting on the existing draft pull request.

OpenTelemetry Support

This interoperability is a wonderful victory both for Jaeger users and for OpenTelemetry users. However, we’re not done yet. The OpenTelemetry specification still requires support for Jaeger client exporters across languages.

This causes challenges for both Jaeger users and OpenTelemetry maintainers:

  1. Confusing Choices: Currently, users are faced with a choice of exporter (Jaeger or OTLP), and this can be a source of confusion. A user might be inclined, when exporting telemetry to Jaeger, to simply choose the Jaeger exporter because the name matches (even though Jaeger now actively encourages the use of OTLP).
    If we can eliminate this potentially confusing choice, we can improve the user experience and continue standardizing on a single interoperable protocol. We love it when things “just work” out of the box!
  2. Maintenance and duplication: Because the Jaeger client libraries are now archived, they will not receive updates (including security patches). To continue properly supporting Jaeger client exporters, OpenTelemetry authors would be required to re-implement some of the functionality it had previously leveraged from the Jaeger clients.
    Now that Jaeger supports OTLP, this feels like a step backwards: It results in an increased maintenance burden with very little benefit.

User Impact

The proposal is to deprecate the following exporters from OpenTelemetry in favor of using native OTLP into Jaeger:

In addition to application configuration changes, there could be other architectural considerations. HTTP and gRPC should be straightforward replacements, although it may require exposing ports 4317 and 4318 if they are not already accessible.

Thrift over UDP implies the use of the Jaeger Agent. Users with this deployment configuration will need to make a slightly more complicated change, typically one of the following:

  1. Direct ingest. Applications will change from using Thrift+UDP to sending OTLP traces directly to their jaeger-collector instance. This may also have sampling implications.
  2. Replacing the Jaeger Agent with a sidecar OpenTelemetry Collector instance. This could have sampling implications and requires changes to your infrastructure deployment code.

Intent to Deprecate — We’d Like Your Feedback!

In order to better support users and the interop between OpenTelemetry and Jaeger, we intend to deprecate and eventually remove support for Jaeger client exporters / Jaeger native data format in OpenTelemetry.

We would like your feedback! We want to hear from users who could be impacted by this change. To better make a data-informed decision, we have put together a short 4-question survey.

Your input will help us to choose how long to deprecate before removal.

A draft PR has been created in the specification to support this deprecation. If would like to contribute and provide feedback, visit the link above and add some comments. We want to hear from you.

Better alignment with OpenTelemetry by focusing on OTLP was originally published in JaegerTracing on Medium, where people are continuing the conversation by highlighting and responding to this story.

', + author: 'Yuri Shkuro', + publishedAt: 1667499175, + }, { + id: + 'medium-myuser-mycolumn-77107c9209365c6c6c601a9f018a05f4-1c4fe0b61cf2c0ce653194b1c8ce3554', + userId: 'myuser', + columnId: 'mycolumn', + sourceId: 'medium-myuser-mycolumn-77107c9209365c6c6c601a9f018a05f4', + title: 'TEMPLE: Six Pillars of Observability', + link: + 'https://medium.com/@YuriShkuro/temple-six-pillars-of-observability-4ac3e3deb402?source=rss-cff2e4ba6058------2', + media: + 'https://cdn-images-1.medium.com/max/1024/1*1yZ4WP2IDrpFNpuf7tKkhQ.png', + description: + '
A temple in Italy with six front pillars
Valley of the Temples, Agrigento, AG, Italy. Photo by Dario Crisafulli on Unsplash.

In the past few years, much has been talked and written about the “three pillars of observability”: metrics, logs, and traces. A Google search for the phrase brings up over 7,000 results, with almost every observability vendor having a blog post or an e-book on the topic. Recently, the term MELT started showing up that adds “events” to the mix as a distinct telemetry signal. In this post, I want to show that there are even more distinct types and introduce TEMPLE, which stands for traces, events, metrics, profiles, logs, and exceptions. I call them six pillars of observability, on one hand to make fun of the previous terms and acronyms, but also to make the case that these signals serve distinct use cases for observability of cloud-native systems. If I fail at the latter and you don’t buy my arguments, at least the TEMPLE acronym works much better with “pillars” 😉.

Attribution: One of my colleagues at Meta, Henry Bond, started using the acronym TEMPL in the internal documents. I added “Exceptions”, for completeness, and ended up with TEMPLE.

Six Pillars Explained

I will try to illustrate why I think these six telemetry types deserve to be considered as separate. It does not mean some of them cannot be supported by the same backend, but they differ in the following aspects:

Even though the TEMPLE acronym implies a certain ordering of the signals, I do not ascribe any meaning to that other than to make up a pleasant word. For better continuity of the explanation, I will go through them in a different order.

Also, naming is hard. I will point out how, surprisingly, most of the terms we use in this space are ambiguous, and the boundaries between telemetry types are not as strict as they appear to be.

Metrics, the original pillar. Numerical measurements with attributes, which are easily aggregatable both spacially (along the attribute dimensions) and temporally (combining values into less discrete time intervals). Metrics aggregates remain highly accurate, which makes them great for monitoring, but aggregations lose the original level of details, which makes metrics not as good for troubleshooting & investigations.

In the context of cloud native applications, metrics usually refer to operational metrics of the software. Business metrics are actually a different category that is better captured via structured logs.

Logs, the ancient pillar (if you question which came first, “ancient” or “original”, take it up with Marvel). Logs are a confusing category, ranging from arbitrary printf-like statements (aka unstructured logs) to highly structured and even schematized events. When structured logs are schematized (cf. Schema-first Application Telemetry), they are often sent to different tables in the warehouse and used for analytics, including business analytics. Schema-free structured logs are what Honeycomb calls “arbitrarily-wide events”. It’s a bit of a misnomer, because each individual log record is not “arbitrarily wide”, in fact it usually has a completely static shape that will not change in the given call site, but when we mix these log records from different call sites we can expect all kinds of shapes, making the resulting destination an “arbitrarily wide” table. Most modern logging backends can ingest structured logs and allow search and analytics on these arbitrary dimensions.

Many logs are generated in response to a service processing specific input requests, i.e. they are request-scoped. We, as an industry, haven’t really figured out how best to deal with request-scoped logs. On one hand, they look like other logs and can be stored and analyzed in a similar fashion. On the other hand, distributed tracing is specifically designed as request-scoped logging, so these logs could be much more useful when viewed in the context of a distributed trace. I met engineers whose teams chose to use tracing APIs exclusively to capture logs, so that those can be always visualized in the context of traces. A common approach to solve this is to capture trace ID as a field in the logs and build cross-correlation between logging and tracing tools.

Traces, the “new cool kid on the block” pillar. The term tracing is quite overloaded (just look at Linux Tracing documentation). In the context of cloud native observability, tracing usually refers to distributed tracing, or a special form of structured logs that are request-scoped, or more generally workflow-centric. In contrast to plain structured logging, tracing captures not only the log records, but also causality between them, and once those records are stitched into a trace they represent the trajectory of a single request (or a workflow) through a distributed system end-to-end.

Tracing opens up a realm of possibilities for reasoning about a system:

Events, the misunderstood pillar. This is perhaps the worst-named category of telemetry signals because, strictly speaking, pretty much all telemetry is “events”. What people usually mean by this category is change events, i.e., events that are external to the observed system that cause some changes in that system. The most common examples are: deployments of application code (and the corresponding code commits), configuration changes, experiments, DR-related traffic drains, perhaps auto-scaling events, etc.

There is no practical bound to what could be considered an event that affects the system. For instance, in the early days of Uber, Halloween was the night of the highest user traffic, such that SREs would spend the whole night in the “war room”, monitoring the system and firefighting (Big Bang Theory flashback: Howard: “guidance system for drunk people”, Raj: “they already had that, it’s called Uber”). As the business became more global, the impact of Halloween as US-centric holiday became less pronounced on the system traffic, but one can easily see how holidays, or some other public events like sports or concerts, can become factors that affect the behavior of a system and might be useful to show to the operators as part of the system’s observability.

One could reasonably ask: why can’t we treat events simply as structured logs? As far as the data shape in the storage, there is indeed not much difference. However, logs usually require less rigor from the backends capturing them: some level of data loss may be acceptable, and the pipelines are often set up to down-sample or throttle the logs. For example, if a bug is causing an application to log a certain error message, it’s likely that we’ll have many similar logs, so it’s not critical to guarantee that every one of them is stored. This is very different from the handling of change events, which should be all stored reliably, because if we miss the record about that one single code deployment that caused the issue and needs to be rolled back, our outage investigation might take much longer. Similarly, when querying for logs, it’s usually sufficient to find some samples of a pattern, or to get aggregate statistics, there is not much emphasis on finding a very specific individual log record. But with change events, we’re precisely looking for very specific instances. Finally, change events are usually produced in much fewer volumes than logs. These differences in the requirements often lead to different designs and trade-offs in the logs and events backends.

Profiles, the geek pillar. Profiles are another category of telemetry that is tricky to define, although you would know it when you see one. Profiles are just being introduced as a new signal to the OpenTelemetry scope in the OTEP-212, and even that document had a bit of a tough time defining what a profile is. Its latest definition is “a collection of stack traces with some metric associated with each stack trace, typically representing the number of times that stack trace was encountered”. Mouthful.

Many engineers encounter profiling tasks at some point, but from my experience most of them do not have to deal with profiles very often, unless they specialize in performance and efficiency optimizations. Profiling tools, as a result, tend to be somewhat esoteric, focusing on power users.

Profiles, unlike most other telemetry types, almost never require explicit instrumentation, instead relying on deeper integration with the runtimes to capture the call stacks. They often generate very large amounts of data, requiring specially designed backends.

Exceptions, the forgotten pillar. Finally, let’s not forget the exceptions. Remember the “Stacktrace or GTFO” comics? When I first came across it, I was working at an investment bank developing trading systems, and we had zero observability into the system running in production (we could get access to logs on the hosts, but only by going through a special permissions escalation process, because … lawyers). So the comics resonated with me a lot at the time. But years later, my attitude changed to “couldn’t he just use Sentry or something?”

My first experience with Sentry was by accident. We just integrated Jaeger SDK into a Python framework that was widely used at Uber. Next morning I am getting a UBN ticket (“UnBreakNow”, i.e., high urgency) that says they are getting errors in production and the stacktrace points to Jaeger SDK code. But instead of a stacktrace the ticket had a link to Sentry, which was the open source system Uber deployed to capture and aggregate exceptions. I was blown away by the amount of information captured by Raven (Sentry’s SDK) besides the stacktrace itself. The most useful was the ability to inspect values of all local variables at every frame of the stack. That immediately revealed the root cause, which had to do with the handling of utf-8 strings.

Exceptions are, strictly speaking, a specialized form of structured logs, although you may need much more structure than the typical structured logging API allows (like nested collections, etc.) The processing pipelines for exceptions are also pretty specialized: they often involve symbolication, fingerprinting, stacks pruning, and deduplication. Finally, the UI for viewing this data is also highly customized to this data source. All these factors lead me to conclude that exceptions should really be treated as an independent telemetry type.

Pillars are not Observability

Now that we covered the six pillars, it’s worth remembering that pillars do not guarantee observability, which is defined, perhaps counterintuitively, as the ability to understand the system (‘s internal state) from its outputs, not just to observe the outputs. These pillars are just different types of telemetry that can be produced, the raw data. To be effective in investigations, the observability platform needs to be able to combine these signals into solutions for specific workflows. Even with free-form investigations, where you are “on your own” because all the guided investigation workflows failed, the platform can provide many features to assist you, such as understanding the metadata of the telemetry and allowing cross-telemetry correlations, or automation of insights and pattern recognition. Pillars are what you build upon, not the end goal.

Takeaways

  1. Stop saying “three pillars”. There are more than three.
  2. Start saying TEMPLE, if you must name them.
  3. Don’t take it seriously. The boundaries are diffuse.
  4. Pillars ≠ observability, they are just data.
', + author: 'Yuri Shkuro', + publishedAt: 1663552288, + }]); + } finally { + fetchWithTimeoutSpy.restore(); + getFaviconSpy.restore(); + } + + assertSpyCall(fetchWithTimeoutSpy, 0, { + args: ['https://medium.com/feed/@YuriShkuro', { method: 'get' }, 5000], + returned: new Promise((resolve) => { + resolve(new Response(responseUser, { status: 200 })); + }), + }); + assertSpyCall(getFaviconSpy, 0, { + args: [ + 'https://medium.com/@YuriShkuro?source=rss-cff2e4ba6058------2', + faviconFilter, + ], + returned: new Promise((resolve) => { + resolve(undefined); + }), + }); + assertSpyCalls(fetchWithTimeoutSpy, 1); + assertSpyCalls(getFaviconSpy, 1); +}); diff --git a/supabase/functions/_shared/feed/nitter.ts b/supabase/functions/_shared/feed/nitter.ts index ff95cdb..a842cae 100644 --- a/supabase/functions/_shared/feed/nitter.ts +++ b/supabase/functions/_shared/feed/nitter.ts @@ -7,14 +7,13 @@ import { unescape } from 'lodash'; import { IItem } from '../models/item.ts'; import { ISource } from '../models/source.ts'; -import { uploadSourceIcon } from './utils/uploadFile.ts'; +import { feedutils } from './utils/index.ts'; import { IProfile } from '../models/profile.ts'; -import { fetchWithTimeout } from '../utils/fetchWithTimeout.ts'; import { FEEDDECK_SOURCE_NITTER_BASIC_AUTH, FEEDDECK_SOURCE_NITTER_INSTANCE, } from '../utils/constants.ts'; -import { log } from '../utils/log.ts'; +import { utils } from '../utils/index.ts'; export const getNitterFeed = async ( supabaseClient: SupabaseClient, @@ -32,7 +31,7 @@ export const getNitterFeed = async ( * Get the RSS for the provided `nitter` username or search term. If a feed * doesn't contains an item we return an error. */ - const response = await fetchWithTimeout( + const response = await utils.fetchWithTimeout( nitterOptions.feedUrl, { headers: nitterOptions.isCustomInstance ? undefined : { @@ -43,7 +42,7 @@ export const getNitterFeed = async ( 5000, ); const xml = await response.text(); - log('debug', 'Add source', { + utils.log('debug', 'Add source', { sourceType: 'nitter', requestUrl: nitterOptions.feedUrl, responseStatus: response.status, @@ -80,7 +79,7 @@ export const getNitterFeed = async ( */ if (!source.icon && nitterOptions.isUsername && feed.image?.url) { source.icon = feed.image.url; - source.icon = await uploadSourceIcon(supabaseClient, source); + source.icon = await feedutils.uploadSourceIcon(supabaseClient, source); } /** @@ -178,7 +177,7 @@ const skipEntry = ( * instance or a username or search term, where we have to use our own Nitter * instance. */ -const parseNitterOptions = ( +export const parseNitterOptions = ( options: string, ): { feedUrl: string; diff --git a/supabase/functions/_shared/feed/nitter_test.ts b/supabase/functions/_shared/feed/nitter_test.ts new file mode 100644 index 0000000..5fd90ae --- /dev/null +++ b/supabase/functions/_shared/feed/nitter_test.ts @@ -0,0 +1,457 @@ +import { assertEquals } from 'std/assert'; +import { createClient } from '@supabase/supabase-js'; +import { + assertSpyCall, + assertSpyCalls, + returnsNext, + stub, +} from 'std/testing/mock'; + +import { ISource } from '../models/source.ts'; +import { IProfile } from '../models/profile.ts'; +import { getNitterFeed, parseNitterOptions } from './nitter.ts'; +import { utils } from '../utils/index.ts'; +import { feedutils } from './utils/index.ts'; + +const supabaseClient = createClient('http://localhost:54321', 'test123'); +const mockProfile: IProfile = { + id: '', + tier: 'free', + createdAt: 0, + updatedAt: 0, +}; +const mockSource: ISource = { + id: '', + columnId: 'mycolumn', + userId: 'myuser', + type: 'medium', + title: '', +}; + +const responseTag = ` + + + + Search results for "kubernetes" + http://nitter.feeddeck.app/search + Twitter feed for: Search "kubernetes". Generated by nitter.feeddeck.app + en-us + 40 + + RT by @lunacyyan: Kubernetes V1.27 : Safeguarding Pod with MemoryThrottlingFactor + +Read more: https://faun.pub/kubernetes-v1-27-safeguarding-pod-with-memorythrottlingfactor-cfbccde10de + @CloudIslamabad + Kubernetes V1.27 : Safeguarding Pod with MemoryThrottlingFactor
+
+Read more: faun.pub/kubernetes-v1-27-sa…

]]> + Sat, 09 Dec 2023 17:42:53 GMT + http://nitter.feeddeck.app/CloudIslamabad/status/1733542748857483570#m + http://nitter.feeddeck.app/CloudIslamabad/status/1733542748857483570#m + + + RT by @mochizuki875: お疲れ様です！ +明日はCNDT2023のCommunity LTにて、16:10からKubernetes Meetup Noviceが登壇させて頂きます！ + +ぜひ遊びに来ていただけると嬉しいです！！ +よろしくお願いいたします！ +https://event.cloudnativedays.jp/cndt2023/community_lt + +#k8snovice #CNDT2023 + @URyo_0213 + お疲れ様です！
+明日はCNDT2023のCommunity LTにて、16:10からKubernetes Meetup Noviceが登壇させて頂きます！
+
+ぜひ遊びに来ていただけると嬉しいです！！
+よろしくお願いいたします！
+event.cloudnativedays.jp/cnd…
+
+#k8snovice #CNDT2023

]]> + Sun, 10 Dec 2023 11:22:30 GMT + http://nitter.feeddeck.app/URyo_0213/status/1733809409393099181#m + http://nitter.feeddeck.app/URyo_0213/status/1733809409393099181#m + + + Did you pass the Kubernetes certified application developer (CKAD) exam on your first attempt? + +#Kubernetes #devops #cka + @AbdelVA + Did you pass the Kubernetes certified application developer (CKAD) exam on your first attempt?
+
+#Kubernetes #devops #cka

]]> + Sun, 10 Dec 2023 11:42:45 GMT + http://nitter.feeddeck.app/AbdelVA/status/1733814503551197653#m + http://nitter.feeddeck.app/AbdelVA/status/1733814503551197653#m + + + RT by @eliyyov: Your app, +in a container, +in a pod, +in Kubernetes, +as a YAML file +with Helm Charts +deployed with ArgoCD + @memenetes + Your app,
+in a container,
+in a pod,
+in Kubernetes,
+as a YAML file
+with Helm Charts
+deployed with ArgoCD

+]]> + Thu, 30 Nov 2023 17:01:23 GMT + http://nitter.feeddeck.app/memenetes/status/1730270811548701061#m + http://nitter.feeddeck.app/memenetes/status/1730270811548701061#m + + +`; + +const responseUser = ` + + + + Rico Berger / @rico_berger + http://nitter.feeddeck.app/rico_berger + Twitter feed for: @rico_berger. Generated by nitter.feeddeck.app + en-us + 40 + + Rico Berger / @rico_berger + http://nitter.feeddeck.app/rico_berger + http://nitter.feeddeck.app/pic/pbs.twimg.com%2Fprofile_images%2F1076066289511206912%2Fi8mug5EL_400x400.jpg + 128 + 128 + + + RT by @rico_berger: Blog: Gateway API v1.0: GA Release - https://kubernetes.io/blog/2023/10/31/gateway-api-ga/ #Kubernetes + @K8sContributors + Blog: Gateway API v1.0: GA Release - kubernetes.io/blog/2023/10/3… #Kubernetes

+]]> + Tue, 31 Oct 2023 18:42:00 GMT + http://nitter.feeddeck.app/K8sContributors/status/1719424500591210559#m + http://nitter.feeddeck.app/K8sContributors/status/1719424500591210559#m + + + RT by @rico_berger: We are excited to announce the launch of OpenTofu, an open source alternative to Terraform's widely used infrastructure as code provisioning tool. + +Read the announcement: +https://hubs.la/Q022JfPQ0 +#opensource #opentofu #ossummit + @linuxfoundation + We are excited to announce the launch of OpenTofu, an open source alternative to Terraform's widely used infrastructure as code provisioning tool.
+
+Read the announcement:
+hubs.la/Q022JfPQ0
+#opensource #opentofu #ossummit

+]]> + Wed, 20 Sep 2023 07:00:00 GMT + http://nitter.feeddeck.app/linuxfoundation/status/1704389933526299129#m + http://nitter.feeddeck.app/linuxfoundation/status/1704389933526299129#m + + + RT by @rico_berger: 🎉🎉🎉 kubenav v4 is finally available and can be downloaded from the Apple App Store (https://apps.apple.com/us/app/kubenav/id1494512160) or Google Play (https://play.google.com/store/apps/details?id=io.kubenav.kubenav) 🥳🥳🥳 #Kubernetes + @kubenav + 🎉🎉🎉 kubenav v4 is finally available and can be downloaded from the Apple App Store (apps.apple.com/us/app/kubena…) or Google Play (play.google.com/store/apps/d…) 🥳🥳🥳 #Kubernetes

+ + + +]]> + Wed, 25 Jan 2023 17:29:00 GMT + http://nitter.feeddeck.app/kubenav/status/1618299915129720832#m + http://nitter.feeddeck.app/kubenav/status/1618299915129720832#m + + +`; + +Deno.test('parseNitterOptions', () => { + assertEquals( + parseNitterOptions('https://nitter.net/rico_berger/rss'), + { + feedUrl: 'https://nitter.net/rico_berger/rss', + sourceTitle: '@rico_berger', + isUsername: true, + isCustomInstance: true, + }, + ); + assertEquals( + parseNitterOptions('https://nitter.net/search/rss?f=tweets&q=kubernetes'), + { + feedUrl: 'https://nitter.net/search/rss?f=tweets&q=kubernetes', + sourceTitle: 'kubernetes', + isUsername: false, + isCustomInstance: true, + }, + ); + assertEquals( + parseNitterOptions('@rico_berger'), + { + feedUrl: '/rico_berger/rss', + sourceTitle: '@rico_berger', + isUsername: true, + isCustomInstance: false, + }, + ); + assertEquals( + parseNitterOptions('kubernetes'), + { + feedUrl: '/search/rss?f=tweets&q=kubernetes', + sourceTitle: 'kubernetes', + isUsername: false, + isCustomInstance: false, + }, + ); +}); + +Deno.test('getNitterFeed - Tag', async () => { + const fetchWithTimeoutSpy = stub( + utils, + 'fetchWithTimeout', + returnsNext([ + new Promise((resolve) => { + resolve(new Response(responseTag, { status: 200 })); + }), + ]), + ); + + try { + const { source, items } = await getNitterFeed( + supabaseClient, + undefined, + mockProfile, + { + ...mockSource, + options: { + nitter: 'https://nitter.net/search/rss?f=tweets&q=kubernetes', + }, + }, + ); + feedutils.assertEqualsSource(source, { + 'id': 'nitter-myuser-mycolumn-14a83e961dc175e20f36e70373cbae6e', + 'columnId': 'mycolumn', + 'userId': 'myuser', + 'type': 'nitter', + 'title': 'kubernetes', + 'options': { + 'nitter': 'https://nitter.net/search/rss?f=tweets&q=kubernetes', + }, + 'link': 'http://nitter.feeddeck.app/search', + }); + feedutils.assertEqualsItems(items, [{ + 'id': + 'nitter-myuser-mycolumn-14a83e961dc175e20f36e70373cbae6e-e15978ba81b685189dfb89c07aa969db', + 'userId': 'myuser', + 'columnId': 'mycolumn', + 'sourceId': 'nitter-myuser-mycolumn-14a83e961dc175e20f36e70373cbae6e', + 'title': + 'RT by @lunacyyan: Kubernetes V1.27 : Safeguarding Pod with MemoryThrottlingFactor\n\nRead more: https://faun.pub/kubernetes-v1-27-safeguarding-pod-with-memorythrottlingfactor-cfbccde10de', + 'link': + 'http://nitter.feeddeck.app/CloudIslamabad/status/1733542748857483570#m', + 'description': + '

Kubernetes V1.27 : Safeguarding Pod with MemoryThrottlingFactor
\n
\nRead more: faun.pub/kubernetes-v1-27-sa…

', + 'author': '@CloudIslamabad', + 'publishedAt': 1702143773, + }, { + 'id': + 'nitter-myuser-mycolumn-14a83e961dc175e20f36e70373cbae6e-34df7ecc085e3e91133b6912c0775e03', + 'userId': 'myuser', + 'columnId': 'mycolumn', + 'sourceId': 'nitter-myuser-mycolumn-14a83e961dc175e20f36e70373cbae6e', + 'title': + 'RT by @mochizuki875: お疲れ様です！\n明日はCNDT2023のCommunity LTにて、16:10からKubernetes Meetup Noviceが登壇させて頂きます！\n\nぜひ遊びに来ていただけると嬉しいです！！\nよろしくお願いいたします！\nhttps://event.cloudnativedays.jp/cndt2023/community_lt\n\n#k8snovice #CNDT2023', + 'link': + 'http://nitter.feeddeck.app/URyo_0213/status/1733809409393099181#m', + 'description': + '

お疲れ様です！
\n明日はCNDT2023のCommunity LTにて、16:10からKubernetes Meetup Noviceが登壇させて頂きます！
\n
\nぜひ遊びに来ていただけると嬉しいです！！
\nよろしくお願いいたします！
\nevent.cloudnativedays.jp/cnd…
\n
\n#k8snovice #CNDT2023

', + 'author': '@URyo_0213', + 'publishedAt': 1702207350, + }, { + 'id': + 'nitter-myuser-mycolumn-14a83e961dc175e20f36e70373cbae6e-475ee6bb2ed19e551d0547545ba4e5a0', + 'userId': 'myuser', + 'columnId': 'mycolumn', + 'sourceId': 'nitter-myuser-mycolumn-14a83e961dc175e20f36e70373cbae6e', + 'title': + 'Did you pass the Kubernetes certified application developer (CKAD) exam on your first attempt?\n\n#Kubernetes #devops #cka', + 'link': 'http://nitter.feeddeck.app/AbdelVA/status/1733814503551197653#m', + 'description': + '

Did you pass the Kubernetes certified application developer (CKAD) exam on your first attempt?
\n
\n#Kubernetes #devops #cka

', + 'author': '@AbdelVA', + 'publishedAt': 1702208565, + }, { + 'id': + 'nitter-myuser-mycolumn-14a83e961dc175e20f36e70373cbae6e-d07f5752845c01a8288c393109770f3c', + 'userId': 'myuser', + 'columnId': 'mycolumn', + 'sourceId': 'nitter-myuser-mycolumn-14a83e961dc175e20f36e70373cbae6e', + 'title': + 'RT by @eliyyov: Your app,\nin a container,\nin a pod,\nin Kubernetes,\nas a YAML file\nwith Helm Charts\ndeployed with ArgoCD', + 'link': + 'http://nitter.feeddeck.app/memenetes/status/1730270811548701061#m', + 'options': { + 'media': [ + 'https://nitter.feeddeck.app/pic/ext_tw_video_thumb%2F1730270796612788224%2Fpu%2Fimg%2Fn2of8rx2u0oXtVAw.jpg', + ], + }, + 'description': + '

Your app,
\nin a container,
\nin a pod,
\nin Kubernetes,
\nas a YAML file
\nwith Helm Charts
\ndeployed with ArgoCD

\n', + 'author': '@memenetes', + 'publishedAt': 1701363683, + }]); + } finally { + fetchWithTimeoutSpy.restore(); + } + + assertSpyCall(fetchWithTimeoutSpy, 0, { + args: ['https://nitter.net/search/rss?f=tweets&q=kubernetes', { + headers: undefined, + method: 'get', + }, 5000], + returned: new Promise((resolve) => { + resolve(new Response(responseTag, { status: 200 })); + }), + }); + assertSpyCalls(fetchWithTimeoutSpy, 1); +}); + +Deno.test('getNitterFeed - User', async () => { + const fetchWithTimeoutSpy = stub( + utils, + 'fetchWithTimeout', + returnsNext([ + new Promise((resolve) => { + resolve(new Response(responseUser, { status: 200 })); + }), + ]), + ); + const uploadSourceIconSpy = stub( + feedutils, + 'uploadSourceIcon', + returnsNext([ + new Promise((resolve) => { + resolve( + 'https://media.hachyderm.io/accounts/avatars/109/773/619/675/865/785/original/bf731ded4166a661.png', + ); + }), + ]), + ); + + try { + const { source, items } = await getNitterFeed( + supabaseClient, + undefined, + mockProfile, + { + ...mockSource, + options: { nitter: 'https://nitter.net/rico_berger/rss' }, + }, + ); + feedutils.assertEqualsSource(source, { + 'id': 'nitter-myuser-mycolumn-2f69b5c79645e7868dbefdee825bbb90', + 'columnId': 'mycolumn', + 'userId': 'myuser', + 'type': 'nitter', + 'title': '@rico_berger', + 'options': { 'nitter': 'https://nitter.net/rico_berger/rss' }, + 'link': 'http://nitter.feeddeck.app/rico_berger', + 'icon': + 'https://media.hachyderm.io/accounts/avatars/109/773/619/675/865/785/original/bf731ded4166a661.png', + }); + feedutils.assertEqualsItems(items, [{ + 'id': + 'nitter-myuser-mycolumn-2f69b5c79645e7868dbefdee825bbb90-ed3ded7ee38f30e9c7e5e9f181ed96c2', + 'userId': 'myuser', + 'columnId': 'mycolumn', + 'sourceId': 'nitter-myuser-mycolumn-2f69b5c79645e7868dbefdee825bbb90', + 'title': + 'RT by @rico_berger: Blog: Gateway API v1.0: GA Release - https://kubernetes.io/blog/2023/10/31/gateway-api-ga/ #Kubernetes', + 'link': + 'http://nitter.feeddeck.app/K8sContributors/status/1719424500591210559#m', + 'options': { + 'media': [ + 'https://nitter.feeddeck.app/pic/card_img%2F1729729773448970240%2F7PcfwkrY%3Fformat%3Dpng%26name%3D420x420_2', + ], + }, + 'description': + '

Blog: Gateway API v1.0: GA Release - kubernetes.io/blog/2023/10/3… #Kubernetes

\n', + 'author': '@K8sContributors', + 'publishedAt': 1698777720, + }, { + 'id': + 'nitter-myuser-mycolumn-2f69b5c79645e7868dbefdee825bbb90-d185c4fa2b5b22eb0ea0cfd5de56802a', + 'userId': 'myuser', + 'columnId': 'mycolumn', + 'sourceId': 'nitter-myuser-mycolumn-2f69b5c79645e7868dbefdee825bbb90', + 'title': + 'RT by @rico_berger: We are excited to announce the launch of OpenTofu, an open source alternative to Terraform\'s widely used infrastructure as code provisioning tool.\n\nRead the announcement:\nhttps://hubs.la/Q022JfPQ0\n#opensource #opentofu #ossummit', + 'link': + 'http://nitter.feeddeck.app/linuxfoundation/status/1704389933526299129#m', + 'options': { + 'media': [ + 'https://nitter.feeddeck.app/pic/media%2FF6c1nOIWIAAOSAy.jpg', + ], + }, + 'description': + '

We are excited to announce the launch of OpenTofu, an open source alternative to Terraform\'s widely used infrastructure as code provisioning tool.
\n
\nRead the announcement:
\nhubs.la/Q022JfPQ0
\n#opensource #opentofu #ossummit

\n', + 'author': '@linuxfoundation', + 'publishedAt': 1695193200, + }, { + 'id': + 'nitter-myuser-mycolumn-2f69b5c79645e7868dbefdee825bbb90-708a757b055af5aa824ef6c361a4f30c', + 'userId': 'myuser', + 'columnId': 'mycolumn', + 'sourceId': 'nitter-myuser-mycolumn-2f69b5c79645e7868dbefdee825bbb90', + 'title': + 'RT by @rico_berger: 🎉🎉🎉 kubenav v4 is finally available and can be downloaded from the Apple App Store (https://apps.apple.com/us/app/kubenav/id1494512160) or Google Play (https://play.google.com/store/apps/details?id=io.kubenav.kubenav) 🥳🥳🥳 #Kubernetes', + 'link': 'http://nitter.feeddeck.app/kubenav/status/1618299915129720832#m', + 'options': { + 'media': [ + 'https://nitter.feeddeck.app/pic/media%2FFnVbCgSXEBAg-am.jpg', + 'https://nitter.feeddeck.app/pic/media%2FFnVbCgOXEAk3St9.jpg', + 'https://nitter.feeddeck.app/pic/media%2FFnVbCgOXEAYOZFK.jpg', + 'https://nitter.feeddeck.app/pic/media%2FFnVbH9hXEAEsd4I.jpg', + ], + }, + 'description': + '

🎉🎉🎉 kubenav v4 is finally available and can be downloaded from the Apple App Store (apps.apple.com/us/app/kubena…) or Google Play (play.google.com/store/apps/d…) 🥳🥳🥳 #Kubernetes

\n\n\n\n', + 'author': '@kubenav', + 'publishedAt': 1674667740, + }]); + } finally { + fetchWithTimeoutSpy.restore(); + uploadSourceIconSpy.restore(); + } + + assertSpyCall(fetchWithTimeoutSpy, 0, { + args: ['https://nitter.net/rico_berger/rss', { + headers: undefined, + method: 'get', + }, 5000], + returned: new Promise((resolve) => { + resolve(new Response(responseUser, { status: 200 })); + }), + }); + assertSpyCall(uploadSourceIconSpy, 0, { + args: [ + supabaseClient, + { + 'id': 'nitter-myuser-mycolumn-2f69b5c79645e7868dbefdee825bbb90', + 'columnId': 'mycolumn', + 'userId': 'myuser', + 'type': 'nitter', + 'title': '@rico_berger', + 'options': { 'nitter': 'https://nitter.net/rico_berger/rss' }, + 'link': 'http://nitter.feeddeck.app/rico_berger', + 'icon': + 'https://media.hachyderm.io/accounts/avatars/109/773/619/675/865/785/original/bf731ded4166a661.png', + }, + ], + returned: new Promise((resolve) => { + resolve( + 'https://media.hachyderm.io/accounts/avatars/109/773/619/675/865/785/original/bf731ded4166a661.png', + ); + }), + }); + assertSpyCalls(fetchWithTimeoutSpy, 1); + assertSpyCalls(uploadSourceIconSpy, 1); +}); diff --git a/supabase/functions/_shared/feed/pinterest.ts b/supabase/functions/_shared/feed/pinterest.ts index 5d834a8..6a15e83 100644 --- a/supabase/functions/_shared/feed/pinterest.ts +++ b/supabase/functions/_shared/feed/pinterest.ts @@ -8,8 +8,7 @@ import { unescape } from 'lodash'; import { IItem } from '../models/item.ts'; import { ISource } from '../models/source.ts'; import { IProfile } from '../models/profile.ts'; -import { fetchWithTimeout } from '../utils/fetchWithTimeout.ts'; -import { log } from '../utils/log.ts'; +import { utils } from '../utils/index.ts'; const pinterestUrls = [ 'pinterest.at', @@ -55,19 +54,12 @@ export const isPinterestUrl = (url: string): boolean => { return false; }; -export const getPinterestFeed = async ( - _supabaseClient: SupabaseClient, - _redisClient: Redis | undefined, - _profile: IProfile, - source: ISource, -): Promise<{ source: ISource; items: IItem[] }> => { - /** - * Since the `pinterest` option supports multiple input format we need to - * normalize it to a valid Pinterest feed url. If this is not possible we - * consider the provided option as invalid. - */ - if (source.options?.pinterest) { - const input = source.options.pinterest; +/** + * `parsePinterestOption` parses the provided `input` and returns a valid + * Pinterest RSS feed url. + */ +export const parsePinterestOption = (input?: string): string => { + if (input) { /** * If the input starts with `@` we assume that a username or board was * provided in the form of `@username` or `@username/board`. We then use @@ -75,13 +67,9 @@ export const getPinterestFeed = async ( */ if (input.length > 1 && input[0] === '@') { if (input.includes('/')) { - source.options.pinterest = `https://www.pinterest.com/${ - input.substring(1) - }.rss`; + return `https://www.pinterest.com/${input.substring(1)}.rss`; } else { - source.options.pinterest = `https://www.pinterest.com/${ - input.substring(1) - }/feed.rss`; + return `https://www.pinterest.com/${input.substring(1)}/feed.rss`; } } else { /** @@ -101,18 +89,16 @@ export const getPinterestFeed = async ( pinterestDotComUrl.endsWith('.rss') || pinterestDotComUrl.endsWith('/feed.rss') ) { - source.options.pinterest = pinterestDotComUrl; + return pinterestDotComUrl; } else { const urlParameters = pinterestDotComUrl.replace( 'https://www.pinterest.com/', '', ).replace(/\/$/, ''); if (urlParameters.includes('/')) { - source.options.pinterest = - `https://www.pinterest.com/${urlParameters}.rss`; + return `https://www.pinterest.com/${urlParameters}.rss`; } else { - source.options.pinterest = - `https://www.pinterest.com/${urlParameters}/feed.rss`; + return `https://www.pinterest.com/${urlParameters}/feed.rss`; } } } else { @@ -122,18 +108,27 @@ export const getPinterestFeed = async ( } else { throw new Error('Invalid source options'); } +}; + +export const getPinterestFeed = async ( + _supabaseClient: SupabaseClient, + _redisClient: Redis | undefined, + _profile: IProfile, + source: ISource, +): Promise<{ source: ISource; items: IItem[] }> => { + const parsedPinterestOption = parsePinterestOption(source.options?.pinterest); /** * Get the RSS for the provided `pinterest` url and parse it. If a feed * doesn't contains an item we return an error. */ - const response = await fetchWithTimeout(source.options.pinterest, { + const response = await utils.fetchWithTimeout(parsedPinterestOption, { method: 'get', }, 5000); const xml = await response.text(); - log('debug', 'Add source', { + utils.log('debug', 'Add source', { sourceType: 'pinterest', - requestUrl: source.options.pinterest, + requestUrl: parsedPinterestOption, responseStatus: response.status, }); const feed = await parseFeed(xml); @@ -151,11 +146,12 @@ export const getPinterestFeed = async ( source.id = generateSourceId( source.userId, source.columnId, - source.options.pinterest, + parsedPinterestOption, ); } source.type = 'pinterest'; source.title = feed.title.value; + source.options = { pinterest: parsedPinterestOption }; if (feed.links.length > 0) { source.link = feed.links[0]; } @@ -199,7 +195,7 @@ export const getPinterestFeed = async ( media: getMedia(entry), description: getItemDescription(entry), author: `@${ - source.options.pinterest.replace('https://www.pinterest.com/', '') + parsedPinterestOption.replace('https://www.pinterest.com/', '') .replace('.rss', '').replace('/feed.rss', '').split('/')[0] }`, publishedAt: Math.floor(entry.published!.getTime() / 1000), diff --git a/supabase/functions/_shared/feed/pinterest_test.ts b/supabase/functions/_shared/feed/pinterest_test.ts new file mode 100644 index 0000000..5edd2b4 --- /dev/null +++ b/supabase/functions/_shared/feed/pinterest_test.ts @@ -0,0 +1,188 @@ +import { assertEquals, assertThrows } from 'std/assert'; +import { createClient } from '@supabase/supabase-js'; +import { + assertSpyCall, + assertSpyCalls, + returnsNext, + stub, +} from 'std/testing/mock'; + +import { ISource } from '../models/source.ts'; +import { IProfile } from '../models/profile.ts'; +import { + getPinterestFeed, + isPinterestUrl, + parsePinterestOption, +} from './pinterest.ts'; +import { utils } from '../utils/index.ts'; +import { feedutils } from './utils/index.ts'; + +const supabaseClient = createClient('http://localhost:54321', 'test123'); +const mockProfile: IProfile = { + id: '', + tier: 'free', + createdAt: 0, + updatedAt: 0, +}; +const mockSource: ISource = { + id: '', + columnId: 'mycolumn', + userId: 'myuser', + type: 'medium', + title: '', +}; + +const responseUser = ` + + + Pinterest Deutschland + https://www.pinterest.com/pinterestde/ + Pinterest ist deine App voller Ideen zum Entdecken, Selbermachen, Ausprobieren. + + en-us + Sun, 10 Dec 2023 16:33:00 GMT + + Willy Wonka und die Schokoladenfabril Warner Kinofilm Photo Edit | Photshop #DezemberChallenge + https://www.pinterest.de/pin/458452437083858830/ + <a href="https://www.pinterest.de/pin/458452437083858830/"><img src="https://i.pinimg.com/236x/38/ec/d5/38ecd5a70f26b7aed5a03d5d3d202a90.jpg"></a>Willy Wonka und die Schokoladenfabril Warner Kinofilm Photo Edit | Photshop #DezemberChallenge + Thu, 07 Dec 2023 17:28:35 GMT + https://www.pinterest.de/pin/458452437083858830/ + + + + <link>https://www.pinterest.de/pin/458452437083858826/</link> + <description><a href="https://www.pinterest.de/pin/458452437083858826/"><img src="https://i.pinimg.com/236x/bd/fe/c1/bdfec1ee5d5342c250460e7065cb037f.jpg"></a></description> + <pubDate>Thu, 07 Dec 2023 17:28:01 GMT</pubDate> + <guid>https://www.pinterest.de/pin/458452437083858826/</guid> + </item> + <item> + <title /> + <link>https://www.pinterest.de/pin/458452437083858815/</link> + <description><a href="https://www.pinterest.de/pin/458452437083858815/"><img src="https://i.pinimg.com/236x/bb/b6/8b/bbb68be1c4069ba9c1facdcf66063849.jpg"></a></description> + <pubDate>Thu, 07 Dec 2023 17:26:56 GMT</pubDate> + <guid>https://www.pinterest.de/pin/458452437083858815/</guid> + </item> + </channel> +</rss>`; + +Deno.test('isPinterestUrl', () => { + assertEquals( + isPinterestUrl('https://www.pinterest.de/pinterestde/essen-und-trinken/'), + true, + ); + assertEquals(isPinterestUrl('https://www.google.de/'), false); +}); + +Deno.test('parsePinterestOption', () => { + assertEquals( + parsePinterestOption('https://www.pinterest.de/pinterestde'), + 'https://www.pinterest.com/pinterestde/feed.rss', + ); + assertEquals( + parsePinterestOption('https://www.pinterest.de/aurelianstoian/math/'), + 'https://www.pinterest.com/aurelianstoian/math.rss', + ); + assertEquals( + parsePinterestOption('@pinterestde'), + 'https://www.pinterest.com/pinterestde/feed.rss', + ); + assertEquals( + parsePinterestOption('@aurelianstoian/math'), + 'https://www.pinterest.com/aurelianstoian/math.rss', + ); + assertThrows(() => parsePinterestOption(undefined)); + assertThrows(() => parsePinterestOption('')); +}); + +Deno.test('getPinterestFeed - User', async () => { + const fetchWithTimeoutSpy = stub( + utils, + 'fetchWithTimeout', + returnsNext([ + new Promise((resolve) => { + resolve(new Response(responseUser, { status: 200 })); + }), + ]), + ); + + try { + const { source, items } = await getPinterestFeed( + supabaseClient, + undefined, + mockProfile, + { + ...mockSource, + options: { pinterest: 'https://www.pinterest.de/pinterestde' }, + }, + ); + feedutils.assertEqualsSource(source, { + 'id': 'pinterest-myuser-mycolumn-18a73e1c3eb363440dbd64cfa9dfd1ab', + 'columnId': 'mycolumn', + 'userId': 'myuser', + 'type': 'pinterest', + 'title': 'Pinterest Deutschland', + 'options': { + 'pinterest': 'https://www.pinterest.com/pinterestde/feed.rss', + }, + 'link': 'https://www.pinterest.com/pinterestde/', + }); + feedutils.assertEqualsItems(items, [{ + 'id': + 'pinterest-myuser-mycolumn-18a73e1c3eb363440dbd64cfa9dfd1ab-b943f6a54297d4cab0bb47da53a3966a', + 'userId': 'myuser', + 'columnId': 'mycolumn', + 'sourceId': 'pinterest-myuser-mycolumn-18a73e1c3eb363440dbd64cfa9dfd1ab', + 'title': + 'Willy Wonka und die Schokoladenfabril Warner Kinofilm Photo Edit | Photshop #DezemberChallenge', + 'link': 'https://www.pinterest.de/pin/458452437083858830/', + 'media': + 'https://i.pinimg.com/236x/38/ec/d5/38ecd5a70f26b7aed5a03d5d3d202a90.jpg', + 'description': + '<a href="https://www.pinterest.de/pin/458452437083858830/"><img src="https://i.pinimg.com/236x/38/ec/d5/38ecd5a70f26b7aed5a03d5d3d202a90.jpg"></a>Willy Wonka und die Schokoladenfabril Warner Kinofilm Photo Edit | Photshop #DezemberChallenge', + 'author': '@pinterestde', + 'publishedAt': 1701970115, + }, { + 'id': + 'pinterest-myuser-mycolumn-18a73e1c3eb363440dbd64cfa9dfd1ab-3bf66585db7320d3182510e8a691350c', + 'userId': 'myuser', + 'columnId': 'mycolumn', + 'sourceId': 'pinterest-myuser-mycolumn-18a73e1c3eb363440dbd64cfa9dfd1ab', + 'title': '', + 'link': 'https://www.pinterest.de/pin/458452437083858826/', + 'media': + 'https://i.pinimg.com/236x/bd/fe/c1/bdfec1ee5d5342c250460e7065cb037f.jpg', + 'description': + '<a href="https://www.pinterest.de/pin/458452437083858826/"><img src="https://i.pinimg.com/236x/bd/fe/c1/bdfec1ee5d5342c250460e7065cb037f.jpg"></a>', + 'author': '@pinterestde', + 'publishedAt': 1701970081, + }, { + 'id': + 'pinterest-myuser-mycolumn-18a73e1c3eb363440dbd64cfa9dfd1ab-b604f0bd41e5f641231a99b9606854a3', + 'userId': 'myuser', + 'columnId': 'mycolumn', + 'sourceId': 'pinterest-myuser-mycolumn-18a73e1c3eb363440dbd64cfa9dfd1ab', + 'title': '', + 'link': 'https://www.pinterest.de/pin/458452437083858815/', + 'media': + 'https://i.pinimg.com/236x/bb/b6/8b/bbb68be1c4069ba9c1facdcf66063849.jpg', + 'description': + '<a href="https://www.pinterest.de/pin/458452437083858815/"><img src="https://i.pinimg.com/236x/bb/b6/8b/bbb68be1c4069ba9c1facdcf66063849.jpg"></a>', + 'author': '@pinterestde', + 'publishedAt': 1701970016, + }]); + } finally { + fetchWithTimeoutSpy.restore(); + } + + assertSpyCall(fetchWithTimeoutSpy, 0, { + args: [ + 'https://www.pinterest.com/pinterestde/feed.rss', + { method: 'get' }, + 5000, + ], + returned: new Promise((resolve) => { + resolve(new Response(responseUser, { status: 200 })); + }), + }); + assertSpyCalls(fetchWithTimeoutSpy, 1); +}); diff --git a/supabase/functions/_shared/feed/podcast.ts b/supabase/functions/_shared/feed/podcast.ts index 3e3a0de..739af73 100644 --- a/supabase/functions/_shared/feed/podcast.ts +++ b/supabase/functions/_shared/feed/podcast.ts @@ -7,10 +7,9 @@ import { unescape } from 'lodash'; import { ISource } from '../models/source.ts'; import { IItem } from '../models/item.ts'; -import { uploadSourceIcon } from './utils/uploadFile.ts'; +import { feedutils } from './utils/index.ts'; import { IProfile } from '../models/profile.ts'; -import { fetchWithTimeout } from '../utils/fetchWithTimeout.ts'; -import { log } from '../utils/log.ts'; +import { utils } from '../utils/index.ts'; export const getPodcastFeed = async ( supabaseClient: SupabaseClient, @@ -38,11 +37,11 @@ export const getPodcastFeed = async ( * Get the RSS for the provided `podcast` url and parse it. If a feed doesn't * contains an item we return an error. */ - const response = await fetchWithTimeout(source.options.podcast, { + const response = await utils.fetchWithTimeout(source.options.podcast, { method: 'get', }, 5000); const xml = await response.text(); - log('debug', 'Add source', { + utils.log('debug', 'Add source', { sourceType: 'podcast', requestUrl: source.options.podcast, responseStatus: response.status, @@ -75,12 +74,12 @@ export const getPodcastFeed = async ( if (!source.icon) { if (feed.image?.url) { source.icon = feed.image.url; - source.icon = await uploadSourceIcon(supabaseClient, source); + source.icon = await feedutils.uploadSourceIcon(supabaseClient, source); // deno-lint-ignore no-explicit-any } else if ((feed as any)['itunes:image']?.href) { // deno-lint-ignore no-explicit-any source.icon = (feed as any)['itunes:image'].href; - source.icon = await uploadSourceIcon(supabaseClient, source); + source.icon = await feedutils.uploadSourceIcon(supabaseClient, source); } } @@ -173,7 +172,7 @@ const skipEntry = ( * Podcast id. */ const getRSSFeedFromApplePodcast = async (id: string): Promise<string> => { - const resp = await fetchWithTimeout( + const resp = await utils.fetchWithTimeout( `https://itunes.apple.com/lookup?id=${id}&entity=podcast`, { method: 'get' }, 5000, diff --git a/supabase/functions/_shared/feed/podcast_test.ts b/supabase/functions/_shared/feed/podcast_test.ts new file mode 100644 index 0000000..11be30e --- /dev/null +++ b/supabase/functions/_shared/feed/podcast_test.ts @@ -0,0 +1,504 @@ +import { createClient } from '@supabase/supabase-js'; +import { + assertSpyCall, + assertSpyCalls, + returnsNext, + stub, +} from 'std/testing/mock'; + +import { ISource } from '../models/source.ts'; +import { IProfile } from '../models/profile.ts'; +import { getPodcastFeed } from './podcast.ts'; +import { utils } from '../utils/index.ts'; +import { feedutils } from './utils/index.ts'; +import { assertEqualsItems, assertEqualsSource } from './utils/test.ts'; + +const supabaseClient = createClient('http://localhost:54321', 'test123'); +const mockProfile: IProfile = { + id: '', + tier: 'free', + createdAt: 0, + updatedAt: 0, +}; +const mockSource: ISource = { + id: '', + columnId: 'mycolumn', + userId: 'myuser', + type: 'medium', + title: '', +}; + +const responsePodcastRSS = `<?xml version="1.0" encoding="UTF-8"?> +<rss xmlns:atom="http://www.w3.org/2005/Atom" xmlns:cc="http://web.resource.org/cc/" xmlns:content="http://purl.org/rss/1.0/modules/content/" xmlns:googleplay="http://www.google.com/schemas/play-podcasts/1.0" xmlns:itunes="http://www.itunes.com/dtds/podcast-1.0.dtd" xmlns:media="http://search.yahoo.com/mrss/" xmlns:podcast="https://podcastindex.org/namespace/1.0" xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" version="2.0"> + <channel> + <atom:link href="https://feeds.libsyn.com/419861/rss" rel="self" type="application/rss+xml" /> + <title>Kubernetes Podcast from Google + Tue, 05 Dec 2023 23:37:00 +0000 + Wed, 06 Dec 2023 09:03:41 +0000 + Libsyn WebEngine 2.0 + https://kubernetespodcast.com + en-us + + https://kubernetespodcast.com + kubernetespodcast@google.com (kubernetespodcast@google.com) + + + https://static.libsyn.com/p/assets/7/2/d/d/72ddd143dc0d42c316c3140a3186d450/Kubernetes-Podcast-Logo_1400x1400.png + Kubernetes Podcast from Google + + + Abdel Sghiouar, Kaslin Fields + + + + + + false + + + kubernetespodcast@google.com + + + episodic + + https://feeds.libsyn.com/419861/rss + + no + + KubeCon NA 2023 + KubeCon NA 2023 + Tue, 05 Dec 2023 23:37:00 +0000 + + + + This episode Kaslin went to KubeCon North America In Chicago. She spoke to folks on the ground, asked them about their impressions of the conference, and collected a bunch of cool responses.

Do you have something cool to share? Some questions? Let us know:

- web: kubernetespodcast.com

- mail: kubernetespodcast@google.com

- twitter: @kubernetespod

News of the week

Google researchers discover 'Reptar,’ a new CPU vulnerability

Reptar by Tavis Ormandy

Tim Hockin: Kubernetes Needs a Complexity Budget

Kubernetes' Tim Hockin on a decade of dominance and the future of AI in open source

Keynote: A Vision for Vision - Kubernetes in Its Second Decade - Tim Hockin

Open and Secure: A Manual for Practicing Thread Modeling to Assess and Fortify Open Source and Security

Announcing our latest book release: a comprehensive security guide to assess and fortify open source security

Links from the interview

CNCF LLM Starter Pack

Crossplane

Web Assembly

Intro to Kubernetes Gateway API

Links from the post-interview chat

SIG ContribEx Comms Team Rap by Bart Farrell

]]> + This episode Kaslin went to KubeCon North America In Chicago. She spoke to folks on the ground, asked them about their impressions of the conference, and collected a bunch of cool responses.

Do you have something cool to share? Some questions? Let us know:

- web: kubernetespodcast.com

- mail: kubernetespodcast@google.com

- twitter: @kubernetespod

News of the week

Google researchers discover 'Reptar,’ a new CPU vulnerability

Reptar by Tavis Ormandy

Tim Hockin: Kubernetes Needs a Complexity Budget

Kubernetes' Tim Hockin on a decade of dominance and the future of AI in open source

Keynote: A Vision for Vision - Kubernetes in Its Second Decade - Tim Hockin

Open and Secure: A Manual for Practicing Thread Modeling to Assess and Fortify Open Source and Security

Announcing our latest book release: a comprehensive security guide to assess and fortify open source security

Links from the interview

CNCF LLM Starter Pack

Crossplane

Web Assembly

Intro to Kubernetes Gateway API

Links from the post-interview chat

SIG ContribEx Comms Team Rap by Bart Farrell

]]> + + 54:53 + false + + + 214 + full + + + Kubernetes Pen Testing, with Jesper Larsson + Kubernetes Pen Testing, with Jesper Larsson + Wed, 29 Nov 2023 00:18:00 +0000 + + + + Jesper Larsson is a Freelance PenTester. Jesper works with a hacker community called Cure53. Co-organizes SecurityFest in Gothenburg, Sweden. Hosts Säkerhetspodcasten or The Security Podcast. Jesper is also a Star on Hackad, a Swedish TV Series about hacking.

Do you have something cool to share? Some questions? Let us know:

- web: kubernetespodcast.com

- mail: kubernetespodcast@google.com

- twitter: @kubernetespod

News of the week

Kubernetes Removals, Deprecations, and Major Changes in Kubernetes 1.29

Introducing SIG etcd

etcd, with Marek Siarkowicz and Wenjia Zhang (The Kubernetes Podcast from Google)

WebAssembly (WASM) and OpenShift: A Powerful Duo for Modern Applications

Linux Foundation Events

Pass the torch in ContribEx #7603

Links from the interview

Cure53 Hacker Community

Säkerhetspodcasten

Hackad TV Show on IMDB

SecurityFest Gothenburg

Falco by Sysdig

Wolfi by Chainguard

The Untold Story of NotPetya, the Most Devastating Cyberattack in History

Links from the post-interview chat

The Untold Story of NotPetya, the Most Devastating Cyberattack in History

]]> + Jesper Larsson is a Freelance PenTester. Jesper works with a hacker community called Cure53. Co-organizes SecurityFest in Gothenburg, Sweden. Hosts Säkerhetspodcasten or The Security Podcast. Jesper is also a Star on Hackad, a Swedish TV Series about hacking.

Do you have something cool to share? Some questions? Let us know:

- web: kubernetespodcast.com

- mail: kubernetespodcast@google.com

- twitter: @kubernetespod

News of the week

Kubernetes Removals, Deprecations, and Major Changes in Kubernetes 1.29

Introducing SIG etcd

etcd, with Marek Siarkowicz and Wenjia Zhang (The Kubernetes Podcast from Google)

WebAssembly (WASM) and OpenShift: A Powerful Duo for Modern Applications

Linux Foundation Events

Pass the torch in ContribEx #7603

Links from the interview

Cure53 Hacker Community

Säkerhetspodcasten

Hackad TV Show on IMDB

SecurityFest Gothenburg

Falco by Sysdig

Wolfi by Chainguard

The Untold Story of NotPetya, the Most Devastating Cyberattack in History

Links from the post-interview chat

The Untold Story of NotPetya, the Most Devastating Cyberattack in History

]]> + + 51:13 + false + + + 213 + full + + +`; + +Deno.test('getPodcastFeed - RSS', async () => { + const fetchWithTimeoutSpy = stub( + utils, + 'fetchWithTimeout', + returnsNext([ + new Promise((resolve) => { + resolve(new Response(responsePodcastRSS, { status: 200 })); + }), + ]), + ); + + const uploadSourceIconSpy = stub( + feedutils, + 'uploadSourceIcon', + returnsNext([ + new Promise((resolve) => { + resolve( + 'https://static.libsyn.com/p/assets/7/2/d/d/72ddd143dc0d42c316c3140a3186d450/Kubernetes-Podcast-Logo_1400x1400.png', + ); + }), + ]), + ); + + try { + const { source, items } = await getPodcastFeed( + supabaseClient, + undefined, + mockProfile, + { + ...mockSource, + options: { podcast: 'https://kubernetespodcast.com/feeds/audio.xml' }, + }, + ); + assertEqualsSource(source, { + 'id': 'podcast-myuser-mycolumn-9d151d96e51e542b848a39982f685eef', + 'columnId': 'mycolumn', + 'userId': 'myuser', + 'type': 'podcast', + 'title': 'Kubernetes Podcast from Google', + 'options': { 'podcast': 'https://kubernetespodcast.com/feeds/audio.xml' }, + 'link': 'https://kubernetespodcast.com', + 'icon': + 'https://static.libsyn.com/p/assets/7/2/d/d/72ddd143dc0d42c316c3140a3186d450/Kubernetes-Podcast-Logo_1400x1400.png', + }); + assertEqualsItems(items, [{ + 'id': + 'podcast-myuser-mycolumn-9d151d96e51e542b848a39982f685eef-b7986c0276dcd01cdce685b148530a99', + 'userId': 'myuser', + 'columnId': 'mycolumn', + 'sourceId': 'podcast-myuser-mycolumn-9d151d96e51e542b848a39982f685eef', + 'title': 'KubeCon NA 2023', + 'link': 'http://sites.libsyn.com/419861/kubecon-na-2023', + 'media': + 'https://traffic.libsyn.com/secure/e780d51f-f115-44a6-8252-aed9216bb521/KPOD214.mp3?dest-id=3486674', + 'description': + '

This episode Kaslin went to KubeCon North America In Chicago. She spoke to folks on the ground, asked them about their impressions of the conference, and collected a bunch of cool responses.

Do you have something cool to share? Some questions? Let us know:

- web: kubernetespodcast.com

- mail: kubernetespodcast@google.com

- twitter: @kubernetespod

News of the week

Google researchers discover \'Reptar,’ a new CPU vulnerability

Reptar by Tavis Ormandy

Tim Hockin: Kubernetes Needs a Complexity Budget

Kubernetes\' Tim Hockin on a decade of dominance and the future of AI in open source

Keynote: A Vision for Vision - Kubernetes in Its Second Decade - Tim Hockin

Open and Secure: A Manual for Practicing Thread Modeling to Assess and Fortify Open Source and Security

Announcing our latest book release: a comprehensive security guide to assess and fortify open source security

Links from the interview

CNCF LLM Starter Pack

Crossplane

Web Assembly

Intro to Kubernetes Gateway API

Links from the post-interview chat

SIG ContribEx Comms Team Rap by Bart Farrell

', + 'publishedAt': 1701819420, + }, { + 'id': + 'podcast-myuser-mycolumn-9d151d96e51e542b848a39982f685eef-4c7c98567f8fbe488d28b0cd032f7a72', + 'userId': 'myuser', + 'columnId': 'mycolumn', + 'sourceId': 'podcast-myuser-mycolumn-9d151d96e51e542b848a39982f685eef', + 'title': 'Kubernetes Pen Testing, with Jesper Larsson', + 'link': + 'http://sites.libsyn.com/419861/kubernetes-pen-testing-with-jesper-larsson', + 'media': + 'https://traffic.libsyn.com/secure/e780d51f-f115-44a6-8252-aed9216bb521/KPOD213.mp3?dest-id=3486674', + 'description': + '

Jesper Larsson is a Freelance PenTester. Jesper works with a hacker community called Cure53. Co-organizes SecurityFest in Gothenburg, Sweden. Hosts Säkerhetspodcasten or The Security Podcast. Jesper is also a Star on Hackad, a Swedish TV Series about hacking.

Do you have something cool to share? Some questions? Let us know:

- web: kubernetespodcast.com

- mail: kubernetespodcast@google.com

- twitter: @kubernetespod

News of the week

Kubernetes Removals, Deprecations, and Major Changes in Kubernetes 1.29

Introducing SIG etcd

etcd, with Marek Siarkowicz and Wenjia Zhang (The Kubernetes Podcast from Google)

WebAssembly (WASM) and OpenShift: A Powerful Duo for Modern Applications

Linux Foundation Events

Pass the torch in ContribEx #7603

Links from the interview

Cure53 Hacker Community

Säkerhetspodcasten

Hackad TV Show on IMDB

SecurityFest Gothenburg

Falco by Sysdig

Wolfi by Chainguard

The Untold Story of NotPetya, the Most Devastating Cyberattack in History

Links from the post-interview chat

The Untold Story of NotPetya, the Most Devastating Cyberattack in History

', + 'publishedAt': 1701217080, + }]); + } finally { + fetchWithTimeoutSpy.restore(); + uploadSourceIconSpy.restore(); + } + + assertSpyCall(fetchWithTimeoutSpy, 0, { + args: [ + 'https://kubernetespodcast.com/feeds/audio.xml', + { method: 'get' }, + 5000, + ], + returned: new Promise((resolve) => { + resolve(new Response(responsePodcastRSS, { status: 200 })); + }), + }); + assertSpyCall(uploadSourceIconSpy, 0, { + args: [ + supabaseClient, + { + 'id': 'podcast-myuser-mycolumn-9d151d96e51e542b848a39982f685eef', + 'columnId': 'mycolumn', + 'userId': 'myuser', + 'type': 'podcast', + 'title': 'Kubernetes Podcast from Google', + 'options': { + 'podcast': 'https://kubernetespodcast.com/feeds/audio.xml', + }, + 'link': 'https://kubernetespodcast.com', + 'icon': + 'https://static.libsyn.com/p/assets/7/2/d/d/72ddd143dc0d42c316c3140a3186d450/Kubernetes-Podcast-Logo_1400x1400.png', + }, + ], + returned: new Promise((resolve) => { + resolve( + 'https://static.libsyn.com/p/assets/7/2/d/d/72ddd143dc0d42c316c3140a3186d450/Kubernetes-Podcast-Logo_1400x1400.png', + ); + }), + }); + assertSpyCalls(fetchWithTimeoutSpy, 1); + assertSpyCalls(uploadSourceIconSpy, 1); +}); + +const responsePodcastApple = `{ + "resultCount":1, + "results": [ +{"wrapperType":"track", "kind":"podcast", "collectionId":1120964487, "trackId":1120964487, "artistName":"Changelog Media", "collectionName":"Go Time: Golang, Software Engineering", "trackName":"Go Time: Golang, Software Engineering", "collectionCensoredName":"Go Time: Golang, Software Engineering", "trackCensoredName":"Go Time: Golang, Software Engineering", "collectionViewUrl":"https://podcasts.apple.com/us/podcast/go-time-golang-software-engineering/id1120964487?uo=4", "feedUrl":"https://changelog.com/gotime/feed", "trackViewUrl":"https://podcasts.apple.com/us/podcast/go-time-golang-software-engineering/id1120964487?uo=4", "artworkUrl30":"https://is1-ssl.mzstatic.com/image/thumb/Podcasts113/v4/0d/25/86/0d258649-4cfe-2750-6316-dffd9dbe3b8d/mza_5183216603141582042.png/30x30bb.jpg", "artworkUrl60":"https://is1-ssl.mzstatic.com/image/thumb/Podcasts113/v4/0d/25/86/0d258649-4cfe-2750-6316-dffd9dbe3b8d/mza_5183216603141582042.png/60x60bb.jpg", "artworkUrl100":"https://is1-ssl.mzstatic.com/image/thumb/Podcasts113/v4/0d/25/86/0d258649-4cfe-2750-6316-dffd9dbe3b8d/mza_5183216603141582042.png/100x100bb.jpg", "collectionPrice":0.00, "trackPrice":0.00, "collectionHdPrice":0, "releaseDate":"2023-11-08T13:15:00Z", "collectionExplicitness":"notExplicit", "trackExplicitness":"cleaned", "trackCount":304, "trackTimeMillis":5264, "country":"USA", "currency":"USD", "primaryGenreName":"Technology", "contentAdvisoryRating":"Clean", "artworkUrl600":"https://is1-ssl.mzstatic.com/image/thumb/Podcasts113/v4/0d/25/86/0d258649-4cfe-2750-6316-dffd9dbe3b8d/mza_5183216603141582042.png/600x600bb.jpg", "genreIds":["1318", "26", "1304", "1499"], "genres":["Technology", "Podcasts", "Education", "How To"]}] +}`; +const responsePodcastAppleRSS = ` + + + Go Time: Golang, Software Engineering + All rights reserved + https://changelog.com/gotime + + + en-us + Your source for diverse discussions from around the Go community. This show records LIVE every Tuesday at 3pm US Eastern. Join the Golang community and chat with us during the show in the #gotimefm channel of Gophers slack. Panelists include Mat Ryer, Jon Calhoun, Natalie Pistunovich, Johnny Boursiquot, Angelica Hill, Kris Brandow, and Ian Lopshire. We discuss cloud infrastructure, distributed systems, microservices, Kubernetes, Docker… oh and also Go! Some people search for GoTime or GoTimeFM and can’t find the show, so now the strings GoTime and GoTimeFM are in our description too. + Changelog Media + Your source for diverse discussions from around the Go community. This show records LIVE every Tuesday at 3pm US Eastern. Join the Golang community and chat with us during the show in the #gotimefm channel of Gophers slack. Panelists include Mat Ryer, Jon Calhoun, Natalie Pistunovich, Johnny Boursiquot, Angelica Hill, Kris Brandow, and Ian Lopshire. We discuss cloud infrastructure, distributed systems, microservices, Kubernetes, Docker… oh and also Go! Some people search for GoTime or GoTimeFM and can’t find the show, so now the strings GoTime and GoTimeFM are in our description too. + no + + + Changelog Media + + go, golang, open source, software, development, devops, architecture, docker, kubernetes + + + + + Support our work by joining Changelog++ + Mat Ryer + Jon Calhoun + Natalie Pistunovich + Johnny Boursiquot + Angelica Hill + Kris Brandow + Ian Lopshire + + Event-driven systems &architecture + https://changelog.com/gotime/297 + changelog.com/2/2126 + Tue, 14 Nov 2023 22:05:00 +0000 + + Event-driven systems may not be the go-to solution for everyone because of the challenges they can add. While the system reacting to events published in other parts of the system seem elegant, some of the complexities they bring can be challenging. However, they do offer durability, autonomy &flexibility. In this episode, we’ll define event-driven architecture, discuss the problems it solves, challenges it poses &potential solutions. + Event-driven systems may not be the go-to solution for everyone because of the challenges they can add. While the system reacting to events published in other parts of the system seem elegant, some of the complexities they bring can be challenging. However, they do offer durability, autonomy & flexibility.

+

In this episode, we’ll define event-driven architecture, discuss the problems it solves, challenges it poses & potential solutions.

+ +

Leave us a comment

+ +

Changelog++ members save 1 minute on this episode because they made the ads disappear. Join today!

+ +

Sponsors:

+

    +
  • Fastly – Our bandwidth partner. Fastly powers fast, secure, and scalable digital experiences. Move beyond your content delivery network to their powerful edge cloud platform. Learn more at fastly.com +
  • Fly.io – The home of Changelog.com — Deploy your apps and databases close to your users. In minutes you can run your Ruby, Go, Node, Deno, Python, or Elixir app (and databases!) all over the world. No ops required. Learn more at fly.io/changelog and check out the speedrun in their docs. +
  • Typesense – Lightning fast, globally distributed Search-as-a-Service that runs in memory. You literally can’t get any faster! +
    • +

+ + +

Featuring:

+

+ +

Show Notes:

+

+

+

Something missing or broken? PRs welcome!

]]> + full + + 1:05:24 + no + go, golang, open source, software, development, devops, architecture, docker, kubernetes + with Chris Richardson, Indu Alagarsamy &Viktor Stanchev + Event-driven systems may not be the go-to solution for everyone because of the challenges they can add. While the system reacting to events published in other parts of the system seem elegant, some of the complexities they bring can be challenging. However, they do offer durability, autonomy &flexibility. In this episode, we’ll define event-driven architecture, discuss the problems it solves, challenges it poses &potential solutions. + Changelog Media + Changelog Media + Angelica Hill + Chris Richardson + Indu Alagarsamy + Viktor Stanchev + + + + + Principles of simplicity + https://changelog.com/gotime/296 + changelog.com/2/2255 + Wed, 08 Nov 2023 13:15:00 +0000 + + Rob Pike says, “Simplicity is the art of hiding complexity.” If that’s true, what is simplicity in the context of writing software in Go? Is it even something we should strive for? Can software be too simple? Ian &Kris discuss with return guest sam boyer. + Rob Pike says, “Simplicity is the art of hiding complexity.” If that’s true, what is simplicity in the context of writing software in Go? Is it even something we should strive for? Can software be too simple? Ian & Kris discuss with return guest sam boyer.

+ +

Leave us a comment

+ +

Changelog++ members save 2 minutes on this episode because they made the ads disappear. Join today!

+ +

Sponsors:

+

    +
  • Changelog News – A podcast+newsletter combo that’s brief, entertaining & always on-point. Subscribe today. +
  • Fastly – Our bandwidth partner. Fastly powers fast, secure, and scalable digital experiences. Move beyond your content delivery network to their powerful edge cloud platform. Learn more at fastly.com +
  • Fly.io – The home of Changelog.com — Deploy your apps and databases close to your users. In minutes you can run your Ruby, Go, Node, Deno, Python, or Elixir app (and databases!) all over the world. No ops required. Learn more at fly.io/changelog and check out the speedrun in their docs. +
    • +

+ + +

Featuring:

+

+ +

Show Notes:

+

+

+

Something missing or broken? PRs welcome!

]]> + full + + 1:27:44 + no + go, golang, open source, software, development, devops, architecture, docker, kubernetes + with sam boyer + Rob Pike says, “Simplicity is the art of hiding complexity.” If that’s true, what is simplicity in the context of writing software in Go? Is it even something we should strive for? Can software be too simple? Ian &Kris discuss with return guest sam boyer. + Changelog Media + Changelog Media + Ian Lopshire + Kris Brandow + sam boyer + + + + +`; + +Deno.test('getPodcastFeed - Apple', async () => { + const fetchWithTimeoutSpy = stub( + utils, + 'fetchWithTimeout', + returnsNext([ + new Promise((resolve) => { + resolve(new Response(responsePodcastApple, { status: 200 })); + }), + new Promise((resolve) => { + resolve(new Response(responsePodcastAppleRSS, { status: 200 })); + }), + ]), + ); + + const uploadSourceIconSpy = stub( + feedutils, + 'uploadSourceIcon', + returnsNext([ + new Promise((resolve) => { + resolve( + 'https://cdn.changelog.com/uploads/covers/go-time-original.png?v=63725770357', + ); + }), + ]), + ); + + try { + const { source, items } = await getPodcastFeed( + supabaseClient, + undefined, + mockProfile, + { + ...mockSource, + options: { + podcast: + 'https://podcasts.apple.com/de/podcast/go-time-golang-software-engineering/id1120964487', + }, + }, + ); + assertEqualsSource(source, { + 'id': 'podcast-myuser-mycolumn-aad37b7b4ebb1f79286d7b9e24bb4163', + 'columnId': 'mycolumn', + 'userId': 'myuser', + 'type': 'podcast', + 'title': 'Go Time: Golang, Software Engineering', + 'options': { 'podcast': 'https://changelog.com/gotime/feed' }, + 'link': 'https://changelog.com/gotime', + 'icon': + 'https://cdn.changelog.com/uploads/covers/go-time-original.png?v=63725770357', + }); + assertEqualsItems(items, [{ + 'id': + 'podcast-myuser-mycolumn-aad37b7b4ebb1f79286d7b9e24bb4163-04c1e4407a4e70983deb0950f6afc8b2', + 'userId': 'myuser', + 'columnId': 'mycolumn', + 'sourceId': 'podcast-myuser-mycolumn-aad37b7b4ebb1f79286d7b9e24bb4163', + 'title': 'Event-driven systems &architecture', + 'link': 'https://changelog.com/gotime/297', + 'media': + 'https://op3.dev/e/https://cdn.changelog.com/uploads/gotime/297/go-time-297.mp3', + 'description': + 'Event-driven systems may not be the go-to solution for everyone because of the challenges they can add. While the system reacting to events published in other parts of the system seem elegant, some of the complexities they bring can be challenging. However, they do offer durability, autonomy &flexibility. In this episode, we’ll define event-driven architecture, discuss the problems it solves, challenges it poses &potential solutions.', + 'author': 'Changelog Media', + 'publishedAt': 1699999500, + }, { + 'id': + 'podcast-myuser-mycolumn-aad37b7b4ebb1f79286d7b9e24bb4163-2eb33370a9f9245ca2b1fc4491983383', + 'userId': 'myuser', + 'columnId': 'mycolumn', + 'sourceId': 'podcast-myuser-mycolumn-aad37b7b4ebb1f79286d7b9e24bb4163', + 'title': 'Principles of simplicity', + 'link': 'https://changelog.com/gotime/296', + 'media': + 'https://op3.dev/e/https://cdn.changelog.com/uploads/gotime/296/go-time-296.mp3', + 'description': + 'Rob Pike says, “Simplicity is the art of hiding complexity.” If that’s true, what is simplicity in the context of writing software in Go? Is it even something we should strive for? Can software be too simple? Ian &Kris discuss with return guest sam boyer.', + 'author': 'Changelog Media', + 'publishedAt': 1699449300, + }]); + } finally { + fetchWithTimeoutSpy.restore(); + uploadSourceIconSpy.restore(); + } + + assertSpyCall(fetchWithTimeoutSpy, 0, { + args: [ + 'https://itunes.apple.com/lookup?id=1120964487&entity=podcast', + { method: 'get' }, + 5000, + ], + returned: new Promise((resolve) => { + resolve(new Response(responsePodcastApple, { status: 200 })); + }), + }); + assertSpyCall(fetchWithTimeoutSpy, 1, { + args: [ + 'https://changelog.com/gotime/feed', + { method: 'get' }, + 5000, + ], + returned: new Promise((resolve) => { + resolve(new Response(responsePodcastAppleRSS, { status: 200 })); + }), + }); + assertSpyCall(uploadSourceIconSpy, 0, { + args: [ + supabaseClient, + { + 'id': 'podcast-myuser-mycolumn-aad37b7b4ebb1f79286d7b9e24bb4163', + 'columnId': 'mycolumn', + 'userId': 'myuser', + 'type': 'podcast', + 'title': 'Go Time: Golang, Software Engineering', + 'options': { 'podcast': 'https://changelog.com/gotime/feed' }, + 'link': 'https://changelog.com/gotime', + 'icon': + 'https://cdn.changelog.com/uploads/covers/go-time-original.png?v=63725770357', + }, + ], + returned: new Promise((resolve) => { + resolve( + 'https://cdn.changelog.com/uploads/covers/go-time-original.png?v=63725770357', + ); + }), + }); + assertSpyCalls(fetchWithTimeoutSpy, 2); + assertSpyCalls(uploadSourceIconSpy, 1); +}); diff --git a/supabase/functions/_shared/feed/reddit.ts b/supabase/functions/_shared/feed/reddit.ts index 4912330..15e2acd 100644 --- a/supabase/functions/_shared/feed/reddit.ts +++ b/supabase/functions/_shared/feed/reddit.ts @@ -8,8 +8,7 @@ import { unescape } from 'lodash'; import { IItem } from '../models/item.ts'; import { ISource } from '../models/source.ts'; import { IProfile } from '../models/profile.ts'; -import { fetchWithTimeout } from '../utils/fetchWithTimeout.ts'; -import { log } from '../utils/log.ts'; +import { utils } from '../utils/index.ts'; /** * `isRedditUrl` checks if the provided `url` is a valid Reddit url. A url is @@ -46,11 +45,11 @@ export const getRedditFeed = async ( * Get the RSS for the provided `youtube` url and parse it. If a feed doesn't * contains an item we return an error. */ - const response = await fetchWithTimeout(source.options.reddit, { + const response = await utils.fetchWithTimeout(source.options.reddit, { method: 'get', }, 5000); const xml = await response.text(); - log('debug', 'Add source', { + utils.log('debug', 'Add source', { sourceType: 'reddit', requestUrl: source.options.reddit, responseStatus: response.status, diff --git a/supabase/functions/_shared/feed/reddit_test.ts b/supabase/functions/_shared/feed/reddit_test.ts new file mode 100644 index 0000000..e5d0df4 --- /dev/null +++ b/supabase/functions/_shared/feed/reddit_test.ts @@ -0,0 +1,228 @@ +import { assertEquals } from 'std/assert'; +import { createClient } from '@supabase/supabase-js'; +import { + assertSpyCall, + assertSpyCalls, + returnsNext, + stub, +} from 'std/testing/mock'; + +import { ISource } from '../models/source.ts'; +import { IProfile } from '../models/profile.ts'; +import { getRedditFeed, isRedditUrl } from './reddit.ts'; +import { utils } from '../utils/index.ts'; +import { feedutils } from './utils/index.ts'; + +const supabaseClient = createClient('http://localhost:54321', 'test123'); +const mockProfile: IProfile = { + id: '', + tier: 'free', + createdAt: 0, + updatedAt: 0, +}; +const mockSource: ISource = { + id: '', + columnId: 'mycolumn', + userId: 'myuser', + type: 'medium', + title: '', +}; + +const responseSubreddit = ` + + + 2023-12-10T17:06:31+00:00 + https://www.redditstatic.com/icon.png/ + /r/kubernetes.rss + + + Kubernetes discussion, news, support, and link sharing. + Kubernetes + + + /u/gctaylor + https://www.reddit.com/user/gctaylor + + + <!-- SC_OFF --><div class="md"><p>This monthly post can be used to share Kubernetes-related job openings within <strong>your</strong> company. Please include:</p> <ul> <li>Name of the company</li> <li>Location requirements (or lack thereof)</li> <li>At least one of: a link to a job posting/application page or contact details<br/></li> </ul> <p>If you are interested in a job, please contact the poster directly. </p> <p>Common reasons for comment removal:</p> <ul> <li>Not meeting the above requirements</li> <li>Recruiter post / recruiter listings</li> <li>Negative, inflammatory, or abrasive tone</li> </ul> </div><!-- SC_ON --> &#32; submitted by &#32; <a href="https://www.reddit.com/user/gctaylor"> /u/gctaylor </a> <br/> <span><a href="https://www.reddit.com/r/kubernetes/comments/18895rv/monthly_who_is_hiring/">[link]</a></span> &#32; <span><a href="https://www.reddit.com/r/kubernetes/comments/18895rv/monthly_who_is_hiring/">[Kommentare]</a></span> + t3_18895rv + + 2023-12-01T11:00:17+00:00 + 2023-12-01T11:00:17+00:00 + Monthly: Who is hiring? + + + + /u/gctaylor + https://www.reddit.com/user/gctaylor + + + <!-- SC_OFF --><div class="md"><p>Got something working? Figure something out? Make progress that you are excited about? Share here!</p> </div><!-- SC_ON --> &#32; submitted by &#32; <a href="https://www.reddit.com/user/gctaylor"> /u/gctaylor </a> <br/> <span><a href="https://www.reddit.com/r/kubernetes/comments/18dkeyn/weekly_share_your_victories_thread/">[link]</a></span> &#32; <span><a href="https://www.reddit.com/r/kubernetes/comments/18dkeyn/weekly_share_your_victories_thread/">[Kommentare]</a></span> + t3_18dkeyn + + 2023-12-08T11:00:13+00:00 + 2023-12-08T11:00:13+00:00 + Weekly: Share your victories thread + + + + /u/Lanky-Ad4698 + https://www.reddit.com/user/Lanky-Ad4698 + + + <!-- SC_OFF --><div class="md"><p>My environment plan:</p> <p>Local: KinD</p> <p>Dev: Hetzner Single VPS</p> <p>Prod: Hetzner Multiple Servers</p> <p>What is the best way to deploy K8s on a single VPS to save money in dev environment? Control plane, worker nodes all on single VPS. The goal is to have the dev environment as similar to Prod (portable), but want to save money on cheap single VPS.</p> <p>I plan on self managing K8s. I know somebody in the comments is just going to be like, just do managed K8s. On that budget mode and want to learn. I really don’t think self managing K8s is that bad and only considered scary because most people just jump straight to managed immediately. I mean I will possibly do managed on PRD, but then Dev and PRD not portable.</p> <p>In terms of stateful Pods, I want dev to have all that in K8s. But PRD most likely will be managed database and session store. Unless having state full things in K8s isn’t that bad. But from what I’ve read nobody likes keeping state full things in K8s. You can kind of see my problem, making dev cheap makes it not as portable to PRD.</p> <p>Yes I’m aware that single VPS K8s is not HA, but that’s not a problem for Dev environment.</p> <p>I see so many tools for self managed K8s, and idk what is the way. Kops? Kubespray?</p> </div><!-- SC_ON --> &#32; submitted by &#32; <a href="https://www.reddit.com/user/Lanky-Ad4698"> /u/Lanky-Ad4698 </a> <br/> <span><a href="https://www.reddit.com/r/kubernetes/comments/18f3d7o/best_way_to_deploy_k8s_to_single_vps_for_dev/">[link]</a></span> &#32; <span><a href="https://www.reddit.com/r/kubernetes/comments/18f3d7o/best_way_to_deploy_k8s_to_single_vps_for_dev/">[Kommentare]</a></span> + t3_18f3d7o + + 2023-12-10T13:13:56+00:00 + 2023-12-10T13:13:56+00:00 + Best way to deploy K8s to single VPS for dev environment + + + + /u/abexami + https://www.reddit.com/user/abexami + + + <!-- SC_OFF --><div class="md"><p>In our company, we utilize VMWare VSphere as our virtualization solution and a NetApp SAN for storage. The NetApp SAN is connected to VCenter, which is running in FC (Fibre Channel) mode. We have chosen not to support iSCSI or any TCP/IP bound protocol.</p> <p>Currently, we have a production-ready Kubernetes cluster running on nodes from VCenter, which supports our stateless applications. However, we encountered a challenge when we wanted to migrate our stateful workloads (databases, object storages, etc.) to K8S. We are in need of a resilient solution to provide PersistentVolumes in our cluster, and we prefer not to use hostpath. Therefore, we require a CSI plugin that can provide dynamic volumes.</p> <p>After exploring different options, it appears that the NetApp&#39;s CSI plugin (Trident) is not yet production-ready and does not support FC mode. This information is based on the documentation and an issue raised on the Trident GitHub repository.</p> <p>There is a CSI plugin compatible with FC SAN for Dell products, but it seems to be specific to Dell.</p> <p>Overall, we didn&#39;t find a proper way to connect the NetApp SAN directly to K8S and went for another solution and we reached VMWare CNS (Cloud Native Storage) plugin for VSphere, conceptually, it can do the job for us (in conjunction with VSphere CSI and/or VSphere CPI), but it seems that it only support vSAN as storage backend and not the SAN luns (I&#39;m not sure, it was not clear enough in the docs).</p> <p>I have two (or maybe three questions now):</p> <ol> <li>Is there any CSI plugin for NetApp SAN FC mode to directly use it in K8S?</li> <li>Is it possible to connect the CNS directly to SAN and to the VSphere CSI in K8S?</li> <li>If the answer to neither of the above is yes, what can we do to provide K8S storages with our existing hardware?</li> </ol> </div><!-- SC_ON --> &#32; submitted by &#32; <a href="https://www.reddit.com/user/abexami"> /u/abexami </a> <br/> <span><a href="https://www.reddit.com/r/kubernetes/comments/18f5b8v/how_to_use_netapp_san_storage_to_provide/">[link]</a></span> &#32; <span><a href="https://www.reddit.com/r/kubernetes/comments/18f5b8v/how_to_use_netapp_san_storage_to_provide/">[Kommentare]</a></span> + t3_18f5b8v + + 2023-12-10T14:56:01+00:00 + 2023-12-10T14:56:01+00:00 + How to use NetApp SAN storage to provide Kubernetes Persistent Volumes? + + + + /u/ekayan + https://www.reddit.com/user/ekayan + + + <!-- SC_OFF --><div class="md"><p>Wanted some opinion on CPU limits in the K8s world.</p> <p>I understand CPU is a compressible resource.</p> <p>&#x200B;</p> <p>There are some school of thoughts which advocate NOT setting limits on CPU and let pods overuse when they need.</p> <ul> <li>advocated in - Kubernetes Patterns, 2nd Edition book - <a href="https://learning.oreilly.com/library/view/kubernetes-patterns-2nd/9781098131678/">link</a></li> <li>an year write up here - <a href="https://home.robusta.dev/blog/stop-using-cpu-limits">link</a></li> <li>some old discussion here with POC-- <a href="https://www.reddit.com/r/kubernetes/comments/ulx54i/k8s_without_cpu_limits_we_put_it_on_the_lab_to/">link</a></li> </ul> <p>&#x200B;</p> <p>However, on the documentation, I see that the containers without limits could use all the resouces on the worker node.</p> <p><a href="https://kubernetes.io/docs/tasks/configure-pod-container/assign-cpu-resource/#if-you-do-not-specify-a-cpu-limit">https://kubernetes.io/docs/tasks/configure-pod-container/assign-cpu-resource/#if-you-do-not-specify-a-cpu-limit</a></p> <pre><code>The Container has no upper bound on the CPU resources it can use. The Container could use all of the CPU resources available on the Node where it is running. </code></pre> <p>&#x200B;</p> <p>My question is :</p> <ul> <li>If I don&#39;t set CPU limits, will the containers use all the CPU resources on the worker node ONLY if the worker node has the free/unused resource available?</li> <li>in the extreme scenario if CPU resources are exhausted, will all pods will get proportionally their cut &quot;according to the CPU requests you set.&quot; ?</li> </ul> <p>Just being cautious if other containers in the worker node will suffer if I remove the CPU limits.</p> <p>&#x200B;</p> <p>Thanks much for any thoughts in advance.</p> </div><!-- SC_ON --> &#32; submitted by &#32; <a href="https://www.reddit.com/user/ekayan"> /u/ekayan </a> <br/> <span><a href="https://www.reddit.com/r/kubernetes/comments/18exirq/request_for_opinion_cpu_limits_in_the_k8s_world/">[link]</a></span> &#32; <span><a href="https://www.reddit.com/r/kubernetes/comments/18exirq/request_for_opinion_cpu_limits_in_the_k8s_world/">[Kommentare]</a></span> + t3_18exirq + + 2023-12-10T06:40:08+00:00 + 2023-12-10T06:40:08+00:00 + [Request for opinion] : CPU limits in the K8s world + +`; + +Deno.test('isRedditUrl', () => { + assertEquals( + isRedditUrl('https://www.reddit.com/r/kubernetes/'), + true, + ); + assertEquals(isRedditUrl('https://www.google.de/'), false); +}); + +Deno.test('getRedditFeed', async () => { + const fetchWithTimeoutSpy = stub( + utils, + 'fetchWithTimeout', + returnsNext([ + new Promise((resolve) => { + resolve(new Response(responseSubreddit, { status: 200 })); + }), + ]), + ); + + try { + const { source, items } = await getRedditFeed( + supabaseClient, + undefined, + mockProfile, + { + ...mockSource, + options: { reddit: '/r/kubernetes' }, + }, + ); + feedutils.assertEqualsSource(source, { + 'id': 'reddit-myuser-mycolumn-87e62a33042b3fdf4eac36ae57d55fc8', + 'columnId': 'mycolumn', + 'userId': 'myuser', + 'type': 'reddit', + 'title': 'Kubernetes', + 'options': { 'reddit': 'https://www.reddit.com/r/kubernetes.rss' }, + 'link': 'https://www.reddit.com/r/kubernetes.rss', + }); + feedutils.assertEqualsItems(items, [{ + 'id': + 'reddit-myuser-mycolumn-87e62a33042b3fdf4eac36ae57d55fc8-109de6824b3a6446882072dce0d4539d', + 'userId': 'myuser', + 'columnId': 'mycolumn', + 'sourceId': 'reddit-myuser-mycolumn-87e62a33042b3fdf4eac36ae57d55fc8', + 'title': 'Monthly: Who is hiring?', + 'link': + 'https://www.reddit.com/r/kubernetes/comments/18895rv/monthly_who_is_hiring/', + 'description': + '

This monthly post can be used to share Kubernetes-related job openings within your company. Please include:

If you are interested in a job, please contact the poster directly.

Common reasons for comment removal:

submitted by /u/gctaylor
[link] [Kommentare]', + 'author': '/u/gctaylor', + 'publishedAt': 1701428417, + }, { + 'id': + 'reddit-myuser-mycolumn-87e62a33042b3fdf4eac36ae57d55fc8-eb77579ebc7a3ef77471fe91fb4feecc', + 'userId': 'myuser', + 'columnId': 'mycolumn', + 'sourceId': 'reddit-myuser-mycolumn-87e62a33042b3fdf4eac36ae57d55fc8', + 'title': 'Weekly: Share your victories thread', + 'link': + 'https://www.reddit.com/r/kubernetes/comments/18dkeyn/weekly_share_your_victories_thread/', + 'description': + '

Got something working? Figure something out? Make progress that you are excited about? Share here!

submitted by /u/gctaylor
[link] [Kommentare]', + 'author': '/u/gctaylor', + 'publishedAt': 1702033213, + }, { + 'id': + 'reddit-myuser-mycolumn-87e62a33042b3fdf4eac36ae57d55fc8-081da9534f66b5a2f9f345747197319d', + 'userId': 'myuser', + 'columnId': 'mycolumn', + 'sourceId': 'reddit-myuser-mycolumn-87e62a33042b3fdf4eac36ae57d55fc8', + 'title': 'Best way to deploy K8s to single VPS for dev environment', + 'link': + 'https://www.reddit.com/r/kubernetes/comments/18f3d7o/best_way_to_deploy_k8s_to_single_vps_for_dev/', + 'description': + '

My environment plan:

Local: KinD

Dev: Hetzner Single VPS

Prod: Hetzner Multiple Servers

What is the best way to deploy K8s on a single VPS to save money in dev environment? Control plane, worker nodes all on single VPS. The goal is to have the dev environment as similar to Prod (portable), but want to save money on cheap single VPS.

I plan on self managing K8s. I know somebody in the comments is just going to be like, just do managed K8s. On that budget mode and want to learn. I really don’t think self managing K8s is that bad and only considered scary because most people just jump straight to managed immediately. I mean I will possibly do managed on PRD, but then Dev and PRD not portable.

In terms of stateful Pods, I want dev to have all that in K8s. But PRD most likely will be managed database and session store. Unless having state full things in K8s isn’t that bad. But from what I’ve read nobody likes keeping state full things in K8s. You can kind of see my problem, making dev cheap makes it not as portable to PRD.

Yes I’m aware that single VPS K8s is not HA, but that’s not a problem for Dev environment.

I see so many tools for self managed K8s, and idk what is the way. Kops? Kubespray?

submitted by /u/Lanky-Ad4698
[link] [Kommentare]', + 'author': '/u/Lanky-Ad4698', + 'publishedAt': 1702214036, + }, { + 'id': + 'reddit-myuser-mycolumn-87e62a33042b3fdf4eac36ae57d55fc8-ea9508517c2f840daf415352c2c2eaf1', + 'userId': 'myuser', + 'columnId': 'mycolumn', + 'sourceId': 'reddit-myuser-mycolumn-87e62a33042b3fdf4eac36ae57d55fc8', + 'title': + 'How to use NetApp SAN storage to provide Kubernetes Persistent Volumes?', + 'link': + 'https://www.reddit.com/r/kubernetes/comments/18f5b8v/how_to_use_netapp_san_storage_to_provide/', + 'description': + '

In our company, we utilize VMWare VSphere as our virtualization solution and a NetApp SAN for storage. The NetApp SAN is connected to VCenter, which is running in FC (Fibre Channel) mode. We have chosen not to support iSCSI or any TCP/IP bound protocol.

Currently, we have a production-ready Kubernetes cluster running on nodes from VCenter, which supports our stateless applications. However, we encountered a challenge when we wanted to migrate our stateful workloads (databases, object storages, etc.) to K8S. We are in need of a resilient solution to provide PersistentVolumes in our cluster, and we prefer not to use hostpath. Therefore, we require a CSI plugin that can provide dynamic volumes.

After exploring different options, it appears that the NetApp\'s CSI plugin (Trident) is not yet production-ready and does not support FC mode. This information is based on the documentation and an issue raised on the Trident GitHub repository.

There is a CSI plugin compatible with FC SAN for Dell products, but it seems to be specific to Dell.

Overall, we didn\'t find a proper way to connect the NetApp SAN directly to K8S and went for another solution and we reached VMWare CNS (Cloud Native Storage) plugin for VSphere, conceptually, it can do the job for us (in conjunction with VSphere CSI and/or VSphere CPI), but it seems that it only support vSAN as storage backend and not the SAN luns (I\'m not sure, it was not clear enough in the docs).

I have two (or maybe three questions now):

  1. Is there any CSI plugin for NetApp SAN FC mode to directly use it in K8S?
  2. Is it possible to connect the CNS directly to SAN and to the VSphere CSI in K8S?
  3. If the answer to neither of the above is yes, what can we do to provide K8S storages with our existing hardware?
submitted by /u/abexami
[link] [Kommentare]', + 'author': '/u/abexami', + 'publishedAt': 1702220161, + }, { + 'id': + 'reddit-myuser-mycolumn-87e62a33042b3fdf4eac36ae57d55fc8-a0c951151b5cfea0d6f9281c791ade02', + 'userId': 'myuser', + 'columnId': 'mycolumn', + 'sourceId': 'reddit-myuser-mycolumn-87e62a33042b3fdf4eac36ae57d55fc8', + 'title': '[Request for opinion] : CPU limits in the K8s world', + 'link': + 'https://www.reddit.com/r/kubernetes/comments/18exirq/request_for_opinion_cpu_limits_in_the_k8s_world/', + 'description': + '

Wanted some opinion on CPU limits in the K8s world.

I understand CPU is a compressible resource.

​

There are some school of thoughts which advocate NOT setting limits on CPU and let pods overuse when they need.

​

However, on the documentation, I see that the containers without limits could use all the resouces on the worker node.

https://kubernetes.io/docs/tasks/configure-pod-container/assign-cpu-resource/#if-you-do-not-specify-a-cpu-limit

The Container has no upper bound on the CPU resources it can use. The Container could use all of the CPU resources available on the Node where it is running.

​

My question is :

Just being cautious if other containers in the worker node will suffer if I remove the CPU limits.

​

Thanks much for any thoughts in advance.

submitted by /u/ekayan
[link] [Kommentare]', + 'author': '/u/ekayan', + 'publishedAt': 1702190408, + }]); + } finally { + fetchWithTimeoutSpy.restore(); + } + + assertSpyCall(fetchWithTimeoutSpy, 0, { + args: [ + 'https://www.reddit.com/r/kubernetes.rss', + { method: 'get' }, + 5000, + ], + returned: new Promise((resolve) => { + resolve(new Response(responseSubreddit, { status: 200 })); + }), + }); + assertSpyCalls(fetchWithTimeoutSpy, 1); +}); diff --git a/supabase/functions/_shared/feed/rss.ts b/supabase/functions/_shared/feed/rss.ts index f4ef72d..d844dd0 100644 --- a/supabase/functions/_shared/feed/rss.ts +++ b/supabase/functions/_shared/feed/rss.ts @@ -8,11 +8,9 @@ import * as cheerio from 'cheerio'; import { IItem } from '../models/item.ts'; import { ISource } from '../models/source.ts'; -import { getFavicon } from './utils/getFavicon.ts'; -import { uploadSourceIcon } from './utils/uploadFile.ts'; +import { feedutils } from './utils/index.ts'; import { IProfile } from '../models/profile.ts'; -import { fetchWithTimeout } from '../utils/fetchWithTimeout.ts'; -import { log } from '../utils/log.ts'; +import { utils } from '../utils/index.ts'; export const getRSSFeed = async ( supabaseClient: SupabaseClient, @@ -31,7 +29,7 @@ export const getRSSFeed = async ( let feed = await getFeed(source); if (!feed) { - log( + utils.log( 'debug', 'Failed to get RSS feed, try to get RSS feed from website', { requestUrl: source.options.rss }, @@ -86,7 +84,7 @@ export const getRSSFeed = async ( */ if (!source.icon) { if (source.link) { - const favicon = await getFavicon(source.link); + const favicon = await feedutils.getFavicon(source.link); if (favicon && favicon.url.startsWith('https://')) { source.icon = favicon.url; } @@ -100,7 +98,7 @@ export const getRSSFeed = async ( } } - source.icon = await uploadSourceIcon(supabaseClient, source); + source.icon = await feedutils.uploadSourceIcon(supabaseClient, source); } /** @@ -144,6 +142,8 @@ export const getRSSFeed = async ( ? Math.floor(entry.published.getTime() / 1000) : entry.updated ? Math.floor(entry.updated.getTime() / 1000) + : entry['dc:date'] + ? getDCDateTimestamp(entry['dc:date']) : Math.floor(new Date().getTime() / 1000), }); } @@ -158,13 +158,13 @@ export const getRSSFeed = async ( */ const getFeed = async (source: ISource): Promise => { try { - const response = await fetchWithTimeout( + const response = await utils.fetchWithTimeout( source.options!.rss!, { method: 'get' }, 5000, ); const xml = await response.text(); - log('debug', 'Add source', { + utils.log('debug', 'Add source', { sourceType: 'rss', requestUrl: source.options!.rss!, responseStatus: response.status, @@ -194,7 +194,7 @@ const getFeedFromWebsite = async ( source: ISource, ): Promise => { try { - const response = await fetchWithTimeout( + const response = await utils.fetchWithTimeout( source.options!.rss!, { method: 'get' }, 5000, @@ -236,7 +236,7 @@ const skipEntry = ( if ( !entry.title?.value || (entry.links.length === 0 || !entry.links[0].href) || - (!entry.published && !entry.updated) + (!entry.published && !entry.updated && !entry['dc:date']) ) { return true; } @@ -251,11 +251,29 @@ const skipEntry = ( Math.floor(entry.updated.getTime() / 1000) <= (sourceUpdatedAt - 10) ) { return true; + } else if ( + entry['dc:date'] && + getDCDateTimestamp(entry['dc:date']) <= (sourceUpdatedAt - 10) + ) { + return true; } return false; }; +/** + * `getDCDateTimestamp` is a helper function to get the timestamp of a `dc:date` + * tag. The `dc:date` tag can either be a `Date` object or an object with a + * `value` property which is a `Date` object. + */ +const getDCDateTimestamp = (dcdate: Date | { value: Date }): number => { + if (dcdate instanceof Date) { + return Math.floor(dcdate.getTime() / 1000); + } else { + return Math.floor(dcdate.value.getTime() / 1000); + } +}; + /** * `generateSourceId` generates a unique source id based on the user id, column * id and the link of the RSS feed. We use the MD5 algorithm for the link to diff --git a/supabase/functions/_shared/feed/rss_test.ts b/supabase/functions/_shared/feed/rss_test.ts new file mode 100644 index 0000000..bbecb41 --- /dev/null +++ b/supabase/functions/_shared/feed/rss_test.ts @@ -0,0 +1,1370 @@ +import { createClient } from '@supabase/supabase-js'; +import { + assertSpyCall, + assertSpyCalls, + returnsNext, + stub, +} from 'std/testing/mock'; + +import { ISource } from '../models/source.ts'; +import { IProfile } from '../models/profile.ts'; +import { getRSSFeed } from './rss.ts'; +import { utils } from '../utils/index.ts'; +import { feedutils } from './utils/index.ts'; +import { assertEqualsItems, assertEqualsSource } from './utils/test.ts'; + +const supabaseClient = createClient('http://localhost:54321', 'test123'); +const mockProfile: IProfile = { + id: '', + tier: 'free', + createdAt: 0, + updatedAt: 0, +}; +const mockSource: ISource = { + id: '', + columnId: 'mycolumn', + userId: 'myuser', + type: 'medium', + title: '', +}; + +const responseTagesschauWebsiteHTML = ` + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + tagesschau.de - die erste Adresse für Nachrichten und Information | tagesschau.de + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +`; +const responseTagesschauWebsiteRSS = ` + + + tagesschau.de - die erste Adresse für Nachrichten und Information + https://www.tagesschau.de/ + Die aktuellen Beiträge der Seite https://www.tagesschau.de/ + de + ARD-aktuell / tagesschau.de + Tue, 12 Dec 2023 15:46:52 +0100 + Tue, 12 Dec 2023 15:46:52 +0100 + http://blogs.law.harvard.edu/tech/rss + 90 + tagesschau.de + de + ARD-aktuell / tagesschau.de + 2023-12-12T14:44:56Z + tagesschau.de + + Haushaltskrise: Um wie viele Milliarden es wirklich geht + https://www.tagesschau.de/wirtschaft/haushalt-volumen-100.html + Die Spitzen der Ampelkoalition suchen Auswege aus der Haushaltskrise. Die Wirtschaft drängt, einige Vorhaben stehen auf der Kippe. Was macht die Gespräche so schwierig - und um wie viel Geld geht es genau? Von H.-J. Vieweger. + Tue, 12 Dec 2023 15:20:21 +0100 + 2371e076-f0bd-4e39-8fcf-3e6cf3308eff + 2023-12-12T14:20:21Z + Christian Lindner, Robert Habeck und Olaf Scholz im Bundestag | dpa

Die Spitzen der Ampelkoalition suchen Auswege aus der Haushaltskrise. Die Wirtschaft drängt, einige Vorhaben stehen auf der Kippe. Was macht die Gespräche so schwierig - und um wie viel Geld geht es genau? Von H.-J. Vieweger.[mehr]

]]> + + + Verhandlungen über Haushalt gehen weiter + https://www.tagesschau.de/inland/innenpolitik/haushalt-verhandlungen-fortsetzung-100.html + Sie tagen wieder: Seit dem Vormittag sitzen die Spitzen der Ampel zusammen, um eine Lösung für den Haushalt 2024 zu finden. Vertreter von Wirtschaft und Gewerkschaften dringen auf eine schnelle Einigung. + Tue, 12 Dec 2023 12:20:01 +0100 + 7b616ae4-0bd3-46cc-92b5-86c91235bfc3 + 2023-12-12T11:20:01Z + Robert Habeck (M, Bündnis 90/Die Grünen), Bundesminister für Wirtschaft und Klimaschutz, kommt ins Bundeskanzleramt | dpa

Sie tagen wieder: Seit dem Vormittag sitzen die Spitzen der Ampel zusammen, um eine Lösung für den Haushalt 2024 zu finden. Vertreter von Wirtschaft und Gewerkschaften dringen auf eine schnelle Einigung.[mehr]

]]> + + + Ringen um Abschlusserklärung: Die dicken Bretter der Klimakonferenz + https://www.tagesschau.de/ausland/asien/cop28-knackpunkte-100.html + Auf der Klimakonferenz ist es den Teilnehmern bislang nicht gelungen, sich auf einen gemeinsamen Beschluss zu verständigen. Der größte Knackpunkt ist der Ausstieg aus fossilen Energien. Doch es gibt weitere. Von M. Polansky. + Tue, 12 Dec 2023 13:24:39 +0100 + 43ab2bf0-e01d-48d8-ad12-cd4de07583ff + 2023-12-12T12:24:39Z + Verkleidete Klimaaktivisten fordern auf der Klimakonferenz in Dubai den Ausstieg aus fossilen Energien. | EPA

Auf der Klimakonferenz ist es den Teilnehmern bislang nicht gelungen, sich auf einen gemeinsamen Beschluss zu verständigen. Der größte Knackpunkt ist der Ausstieg aus fossilen Energien. Doch es gibt weitere. Von M. Polansky.[mehr]

]]> + + +`; + +Deno.test('getRSSFeed - Tagesschau Website', async () => { + const fetchWithTimeoutSpy = stub( + utils, + 'fetchWithTimeout', + returnsNext([ + new Promise((resolve) => { + resolve(new Response(responseTagesschauWebsiteHTML, { status: 200 })); + }), + new Promise((resolve) => { + resolve(new Response(responseTagesschauWebsiteHTML, { status: 200 })); + }), + new Promise((resolve) => { + resolve(new Response(responseTagesschauWebsiteRSS, { status: 200 })); + }), + ]), + ); + + const getFaviconSpy = stub( + feedutils, + 'getFavicon', + returnsNext([ + new Promise((resolve) => { + resolve({ + url: + 'https://www.tagesschau.de/resources/assets/image/favicon/apple-icon-152x152.png', + size: 20251, + extension: 'png', + }); + }), + ]), + ); + + const uploadSourceIconSpy = stub( + feedutils, + 'uploadSourceIcon', + returnsNext([ + new Promise((resolve) => { + resolve( + 'https://www.tagesschau.de/resources/assets/image/favicon/apple-icon-152x152.png', + ); + }), + ]), + ); + + try { + const { source, items } = await getRSSFeed( + supabaseClient, + undefined, + mockProfile, + { + ...mockSource, + options: { + rss: 'https://www.tagesschau.de', + }, + }, + ); + assertEqualsSource(source, { + 'id': 'rss-myuser-mycolumn-790bbd13d8bff02d80672419ea0709b5', + 'columnId': 'mycolumn', + 'userId': 'myuser', + 'type': 'rss', + 'title': + 'tagesschau.de - die erste Adresse für Nachrichten und Information', + 'options': { 'rss': 'https://www.tagesschau.de/index~rss2.xml' }, + 'link': 'https://www.tagesschau.de/', + 'icon': + 'https://www.tagesschau.de/resources/assets/image/favicon/apple-icon-152x152.png', + }); + assertEqualsItems(items, [{ + 'id': + 'rss-myuser-mycolumn-790bbd13d8bff02d80672419ea0709b5-0937a33dc3b35c2982ebb30ca389f6f8', + 'userId': 'myuser', + 'columnId': 'mycolumn', + 'sourceId': 'rss-myuser-mycolumn-790bbd13d8bff02d80672419ea0709b5', + 'title': 'Haushaltskrise: Um wie viele Milliarden es wirklich geht', + 'link': 'https://www.tagesschau.de/wirtschaft/haushalt-volumen-100.html', + 'media': + 'https://images.tagesschau.de/image/2d5406e9-671d-4110-8302-de9b0a8b4832/AAABjBuUz7s/AAABibBxqrQ/16x9-1280/lindner-habeck-scholz-102.jpg', + 'description': + 'Die Spitzen der Ampelkoalition suchen Auswege aus der Haushaltskrise. Die Wirtschaft drängt, einige Vorhaben stehen auf der Kippe. Was macht die Gespräche so schwierig - und um wie viel Geld geht es genau? Von H.-J. Vieweger.', + 'publishedAt': 1702390821, + }, { + 'id': + 'rss-myuser-mycolumn-790bbd13d8bff02d80672419ea0709b5-73a9b28a98ee50d9d79c0b7b03fae3a8', + 'userId': 'myuser', + 'columnId': 'mycolumn', + 'sourceId': 'rss-myuser-mycolumn-790bbd13d8bff02d80672419ea0709b5', + 'title': 'Verhandlungen über Haushalt gehen weiter', + 'link': + 'https://www.tagesschau.de/inland/innenpolitik/haushalt-verhandlungen-fortsetzung-100.html', + 'media': + 'https://images.tagesschau.de/image/aff9abd4-3b6c-497a-9ff9-be59a588de60/AAABjF3BVcA/AAABibBxqrQ/16x9-1280/habeck-482.jpg', + 'description': + 'Sie tagen wieder: Seit dem Vormittag sitzen die Spitzen der Ampel zusammen, um eine Lösung für den Haushalt 2024 zu finden. Vertreter von Wirtschaft und Gewerkschaften dringen auf eine schnelle Einigung.', + 'publishedAt': 1702380001, + }, { + 'id': + 'rss-myuser-mycolumn-790bbd13d8bff02d80672419ea0709b5-fbdf4ad3cc0a0edfbde09afe55844dde', + 'userId': 'myuser', + 'columnId': 'mycolumn', + 'sourceId': 'rss-myuser-mycolumn-790bbd13d8bff02d80672419ea0709b5', + 'title': + 'Ringen um Abschlusserklärung: Die dicken Bretter der Klimakonferenz', + 'link': + 'https://www.tagesschau.de/ausland/asien/cop28-knackpunkte-100.html', + 'media': + 'https://images.tagesschau.de/image/7fa64dfe-5ca7-4845-8dac-5cdeaa1e4c1d/AAABjF3uH6c/AAABibBxqrQ/16x9-1280/cop-aktivisten-102.jpg', + 'description': + 'Auf der Klimakonferenz ist es den Teilnehmern bislang nicht gelungen, sich auf einen gemeinsamen Beschluss zu verständigen. Der größte Knackpunkt ist der Ausstieg aus fossilen Energien. Doch es gibt weitere. Von M. Polansky.', + 'publishedAt': 1702383879, + }]); + } finally { + fetchWithTimeoutSpy.restore(); + getFaviconSpy.restore(); + uploadSourceIconSpy.restore(); + } + + assertSpyCall(fetchWithTimeoutSpy, 0, { + args: [ + 'https://www.tagesschau.de', + { method: 'get' }, + 5000, + ], + returned: new Promise((resolve) => { + resolve(new Response(responseTagesschauWebsiteHTML, { status: 200 })); + }), + }); + assertSpyCall(fetchWithTimeoutSpy, 1, { + args: [ + 'https://www.tagesschau.de', + { method: 'get' }, + 5000, + ], + returned: new Promise((resolve) => { + resolve(new Response(responseTagesschauWebsiteHTML, { status: 200 })); + }), + }); + assertSpyCall(fetchWithTimeoutSpy, 2, { + args: [ + 'https://www.tagesschau.de/index~rss2.xml', + { method: 'get' }, + 5000, + ], + returned: new Promise((resolve) => { + resolve(new Response(responseTagesschauWebsiteRSS, { status: 200 })); + }), + }); + assertSpyCall(getFaviconSpy, 0, { + args: ['https://www.tagesschau.de/'], + returned: new Promise((resolve) => { + resolve({ + url: + 'https://www.tagesschau.de/resources/assets/image/favicon/apple-icon-152x152.png', + size: 20251, + extension: 'png', + }); + }), + }); + assertSpyCall(uploadSourceIconSpy, 0, { + args: [ + supabaseClient, + { + 'id': 'rss-myuser-mycolumn-790bbd13d8bff02d80672419ea0709b5', + 'columnId': 'mycolumn', + 'userId': 'myuser', + 'type': 'rss', + 'title': + 'tagesschau.de - die erste Adresse für Nachrichten und Information', + 'options': { 'rss': 'https://www.tagesschau.de/index~rss2.xml' }, + 'link': 'https://www.tagesschau.de/', + 'icon': + 'https://www.tagesschau.de/resources/assets/image/favicon/apple-icon-152x152.png', + }, + ], + returned: new Promise((resolve) => { + resolve( + 'https://www.tagesschau.de/resources/assets/image/favicon/apple-icon-152x152.png', + ); + }), + }); + assertSpyCalls(fetchWithTimeoutSpy, 3); + assertSpyCalls(getFaviconSpy, 1); + assertSpyCalls(uploadSourceIconSpy, 1); +}); + +const responseTagesschauAtom = ` + + tagesschau.de - die erste Adresse für Nachrichten und Information + + + tagesschau.de + + Die aktuellen Beiträge der Seite https://www.tagesschau.de/ + ARD-aktuell / tagesschau.de + 2023-12-12T16:10:24Z + de + ARD-aktuell / tagesschau.de + + Haushaltskrise: Um wie viele Milliarden es wirklich geht + + 2371e076-f0bd-4e39-8fcf-3e6cf3308eff + Die Spitzen der Ampelkoalition suchen Auswege aus der Haushaltskrise. Die Wirtschaft drängt, einige Vorhaben stehen auf der Kippe. Was macht die Gespräche so schwierig - und um wie viel Geld geht es genau? Von H.-J. Vieweger. + <![CDATA[<p ><a href="https://www.tagesschau.de/wirtschaft/haushalt-volumen-100.html"><img src="https://images.tagesschau.de/image/2d5406e9-671d-4110-8302-de9b0a8b4832/AAABjBuUz7s/AAABibBxqrQ/16x9-1280/lindner-habeck-scholz-102.jpg" alt="Christian Lindner, Robert Habeck und Olaf Scholz im Bundestag | dpa" /></a ><br/><br/>Die Spitzen der Ampelkoalition suchen Auswege aus der Haushaltskrise. Die Wirtschaft drängt, einige Vorhaben stehen auf der Kippe. Was macht die Gespräche so schwierig - und um wie viel Geld geht es genau? <em >Von H.-J. Vieweger.</em >[<a href="https://www.tagesschau.de/wirtschaft/haushalt-volumen-100.html">mehr </a >]</p >]]> + 2023-12-12T14:20:21Z + + + Verhandlungen über Haushalt gehen weiter + + 7b616ae4-0bd3-46cc-92b5-86c91235bfc3 + Sie tagen wieder: Seit dem Vormittag sitzen die Spitzen der Ampel zusammen, um eine Lösung für den Haushalt 2024 zu finden. Vertreter von Wirtschaft und Gewerkschaften dringen auf eine schnelle Einigung. + <![CDATA[<p ><a href="https://www.tagesschau.de/inland/innenpolitik/haushalt-verhandlungen-fortsetzung-100.html"><img src="https://images.tagesschau.de/image/aff9abd4-3b6c-497a-9ff9-be59a588de60/AAABjF3BVcA/AAABibBxqrQ/16x9-1280/habeck-482.jpg" alt="Robert Habeck (M, Bündnis 90/Die Grünen), Bundesminister für Wirtschaft und Klimaschutz, kommt ins Bundeskanzleramt | dpa" /></a ><br/><br/>Sie tagen wieder: Seit dem Vormittag sitzen die Spitzen der Ampel zusammen, um eine Lösung für den Haushalt 2024 zu finden. Vertreter von Wirtschaft und Gewerkschaften dringen auf eine schnelle Einigung.[<a href="https://www.tagesschau.de/inland/innenpolitik/haushalt-verhandlungen-fortsetzung-100.html">mehr </a >]</p >]]> + 2023-12-12T11:20:01Z + +`; + +Deno.test('getRSSFeed - Tagesschau Atom', async () => { + const fetchWithTimeoutSpy = stub( + utils, + 'fetchWithTimeout', + returnsNext([ + new Promise((resolve) => { + resolve(new Response(responseTagesschauAtom, { status: 200 })); + }), + ]), + ); + + const getFaviconSpy = stub( + feedutils, + 'getFavicon', + returnsNext([ + new Promise((resolve) => { + resolve(undefined); + }), + ]), + ); + + const uploadSourceIconSpy = stub( + feedutils, + 'uploadSourceIcon', + returnsNext([ + new Promise((resolve) => { + resolve(undefined); + }), + ]), + ); + + try { + const { source, items } = await getRSSFeed( + supabaseClient, + undefined, + mockProfile, + { + ...mockSource, + options: { + rss: 'https://www.tagesschau.de/index~atom.xml', + }, + }, + ); + assertEqualsSource(source, { + 'id': 'rss-myuser-mycolumn-8465a4b4e81845fd534f45a3ef63ae7f', + 'columnId': 'mycolumn', + 'userId': 'myuser', + 'type': 'rss', + 'title': + 'tagesschau.de - die erste Adresse für Nachrichten und Information', + 'options': { 'rss': 'https://www.tagesschau.de/index~atom.xml' }, + 'link': 'https://www.tagesschau.de/', + }); + assertEqualsItems(items, [{ + 'id': + 'rss-myuser-mycolumn-8465a4b4e81845fd534f45a3ef63ae7f-0937a33dc3b35c2982ebb30ca389f6f8', + 'userId': 'myuser', + 'columnId': 'mycolumn', + 'sourceId': 'rss-myuser-mycolumn-8465a4b4e81845fd534f45a3ef63ae7f', + 'title': 'Haushaltskrise: Um wie viele Milliarden es wirklich geht', + 'link': 'https://www.tagesschau.de/wirtschaft/haushalt-volumen-100.html', + 'media': + 'https://images.tagesschau.de/image/2d5406e9-671d-4110-8302-de9b0a8b4832/AAABjBuUz7s/AAABibBxqrQ/16x9-1280/lindner-habeck-scholz-102.jpg', + 'description': + 'Die Spitzen der Ampelkoalition suchen Auswege aus der Haushaltskrise. Die Wirtschaft drängt, einige Vorhaben stehen auf der Kippe. Was macht die Gespräche so schwierig - und um wie viel Geld geht es genau? Von H.-J. Vieweger.', + 'publishedAt': 1702390821, + }, { + 'id': + 'rss-myuser-mycolumn-8465a4b4e81845fd534f45a3ef63ae7f-73a9b28a98ee50d9d79c0b7b03fae3a8', + 'userId': 'myuser', + 'columnId': 'mycolumn', + 'sourceId': 'rss-myuser-mycolumn-8465a4b4e81845fd534f45a3ef63ae7f', + 'title': 'Verhandlungen über Haushalt gehen weiter', + 'link': + 'https://www.tagesschau.de/inland/innenpolitik/haushalt-verhandlungen-fortsetzung-100.html', + 'media': + 'https://images.tagesschau.de/image/aff9abd4-3b6c-497a-9ff9-be59a588de60/AAABjF3BVcA/AAABibBxqrQ/16x9-1280/habeck-482.jpg', + 'description': + 'Sie tagen wieder: Seit dem Vormittag sitzen die Spitzen der Ampel zusammen, um eine Lösung für den Haushalt 2024 zu finden. Vertreter von Wirtschaft und Gewerkschaften dringen auf eine schnelle Einigung.', + 'publishedAt': 1702380001, + }]); + } finally { + fetchWithTimeoutSpy.restore(); + getFaviconSpy.restore(); + uploadSourceIconSpy.restore(); + } + + assertSpyCall(fetchWithTimeoutSpy, 0, { + args: [ + 'https://www.tagesschau.de/index~atom.xml', + { method: 'get' }, + 5000, + ], + returned: new Promise((resolve) => { + resolve(new Response(responseTagesschauAtom, { status: 200 })); + }), + }); + assertSpyCall(getFaviconSpy, 0, { + args: ['https://www.tagesschau.de/'], + returned: new Promise((resolve) => { + resolve(undefined); + }), + }); + assertSpyCall(uploadSourceIconSpy, 0, { + args: [ + supabaseClient, + { + 'id': 'rss-myuser-mycolumn-8465a4b4e81845fd534f45a3ef63ae7f', + 'columnId': 'mycolumn', + 'userId': 'myuser', + 'type': 'rss', + 'title': + 'tagesschau.de - die erste Adresse für Nachrichten und Information', + 'options': { 'rss': 'https://www.tagesschau.de/index~atom.xml' }, + 'link': 'https://www.tagesschau.de/', + icon: undefined, + }, + ], + returned: new Promise((resolve) => { + resolve(undefined); + }), + }); + assertSpyCalls(fetchWithTimeoutSpy, 1); + assertSpyCalls(getFaviconSpy, 1); + assertSpyCalls(uploadSourceIconSpy, 1); +}); + +const responseTagesschauRDF = ` + + + tagesschau.de - die erste Adresse für Nachrichten und Information + https://www.tagesschau.de/ + Die aktuellen Beiträge der Seite https://www.tagesschau.de/ + de + ARD-aktuell / tagesschau.de + Tue, 12 Dec 2023 17:18:00 +0100 + Tue, 12 Dec 2023 17:18:00 +0100 + 90 + + tagesschau.de + de + ARD-aktuell / tagesschau.de + 2023-12-12T16:14:28+00:00 + tagesschau.de + hourly + 1 + 2023-12-12T16:18:00+00:00 + + + + + + + + + tagesschau | ARD-aktuell + https://images.tagesschau.de/image/89045d82-5cd5-46ad-8f91-73911add30ee/AAABh3YLLz0/AAABibBx4co/original/tagesschau-logo-100.jpg + https://tagesschau.de + + + Haushaltskrise: Um wie viele Milliarden es wirklich geht + https://www.tagesschau.de/wirtschaft/haushalt-volumen-100.html + Die Spitzen der Ampelkoalition suchen Auswege aus der Haushaltskrise. Die Wirtschaft drängt, einige Vorhaben stehen auf der Kippe. Was macht die Gespräche so schwierig - und um wie viel Geld geht es genau? Von H.-J. Vieweger. + Tue, 12 Dec 2023 15:20:21 +0100 + 2371e076-f0bd-4e39-8fcf-3e6cf3308eff + 2023-12-12T14:20:21Z + Christian Lindner, Robert Habeck und Olaf Scholz im Bundestag | dpa

Die Spitzen der Ampelkoalition suchen Auswege aus der Haushaltskrise. Die Wirtschaft drängt, einige Vorhaben stehen auf der Kippe. Was macht die Gespräche so schwierig - und um wie viel Geld geht es genau? Von H.-J. Vieweger.[mehr]

]]> + text/xml + ARD-aktuell / tagesschau.de + de + tagesschau.de + haushalt-volumen-100 + + all + + + + Verhandlungen über Haushalt gehen weiter + https://www.tagesschau.de/inland/innenpolitik/haushalt-verhandlungen-fortsetzung-100.html + Sie tagen wieder: Seit dem Vormittag sitzen die Spitzen der Ampel zusammen, um eine Lösung für den Haushalt 2024 zu finden. Vertreter von Wirtschaft und Gewerkschaften dringen auf eine schnelle Einigung. + Tue, 12 Dec 2023 12:20:01 +0100 + 7b616ae4-0bd3-46cc-92b5-86c91235bfc3 + 2023-12-12T11:20:01Z + Robert Habeck (M, Bündnis 90/Die Grünen), Bundesminister für Wirtschaft und Klimaschutz, kommt ins Bundeskanzleramt | dpa

Sie tagen wieder: Seit dem Vormittag sitzen die Spitzen der Ampel zusammen, um eine Lösung für den Haushalt 2024 zu finden. Vertreter von Wirtschaft und Gewerkschaften dringen auf eine schnelle Einigung.[mehr]

]]> + text/xml + ARD-aktuell / tagesschau.de + de + tagesschau.de + haushalt-verhandlungen-fortsetzung-100 + + all + + +`; + +Deno.test('getRSSFeed - Tagesschau RDF', async () => { + const fetchWithTimeoutSpy = stub( + utils, + 'fetchWithTimeout', + returnsNext([ + new Promise((resolve) => { + resolve(new Response(responseTagesschauRDF, { status: 200 })); + }), + ]), + ); + + const getFaviconSpy = stub( + feedutils, + 'getFavicon', + returnsNext([ + new Promise((resolve) => { + resolve(undefined); + }), + ]), + ); + + const uploadSourceIconSpy = stub( + feedutils, + 'uploadSourceIcon', + returnsNext([ + new Promise((resolve) => { + resolve(undefined); + }), + ]), + ); + + try { + const { source, items } = await getRSSFeed( + supabaseClient, + undefined, + mockProfile, + { + ...mockSource, + options: { + rss: 'https://www.tagesschau.de/index~rdf.xml', + }, + }, + ); + assertEqualsSource(source, { + 'id': 'rss-myuser-mycolumn-315eca5c9ed1af9968989241a5b7de09', + 'columnId': 'mycolumn', + 'userId': 'myuser', + 'type': 'rss', + 'title': + 'tagesschau.de - die erste Adresse für Nachrichten und Information', + 'options': { 'rss': 'https://www.tagesschau.de/index~rdf.xml' }, + 'link': 'https://www.tagesschau.de/', + }); + assertEqualsItems(items, [{ + 'id': + 'rss-myuser-mycolumn-315eca5c9ed1af9968989241a5b7de09-d1a49b5818b63c6c2d4cbabb45086a8c', + 'userId': 'myuser', + 'columnId': 'mycolumn', + 'sourceId': 'rss-myuser-mycolumn-315eca5c9ed1af9968989241a5b7de09', + 'title': 'Haushaltskrise: Um wie viele Milliarden es wirklich geht', + 'link': 'https://www.tagesschau.de/wirtschaft/haushalt-volumen-100.html', + 'description': + 'Die Spitzen der Ampelkoalition suchen Auswege aus der Haushaltskrise. Die Wirtschaft drängt, einige Vorhaben stehen auf der Kippe. Was macht die Gespräche so schwierig - und um wie viel Geld geht es genau? Von H.-J. Vieweger.', + 'publishedAt': 1702390821, + }, { + 'id': + 'rss-myuser-mycolumn-315eca5c9ed1af9968989241a5b7de09-c2f8e98ba67224f9e65c3623e7989fd6', + 'userId': 'myuser', + 'columnId': 'mycolumn', + 'sourceId': 'rss-myuser-mycolumn-315eca5c9ed1af9968989241a5b7de09', + 'title': 'Verhandlungen über Haushalt gehen weiter', + 'link': + 'https://www.tagesschau.de/inland/innenpolitik/haushalt-verhandlungen-fortsetzung-100.html', + 'description': + 'Sie tagen wieder: Seit dem Vormittag sitzen die Spitzen der Ampel zusammen, um eine Lösung für den Haushalt 2024 zu finden. Vertreter von Wirtschaft und Gewerkschaften dringen auf eine schnelle Einigung.', + 'publishedAt': 1702380001, + }]); + } finally { + fetchWithTimeoutSpy.restore(); + getFaviconSpy.restore(); + uploadSourceIconSpy.restore(); + } + + assertSpyCall(fetchWithTimeoutSpy, 0, { + args: [ + 'https://www.tagesschau.de/index~rdf.xml', + { method: 'get' }, + 5000, + ], + returned: new Promise((resolve) => { + resolve(new Response(responseTagesschauRDF, { status: 200 })); + }), + }); + assertSpyCall(getFaviconSpy, 0, { + args: ['https://www.tagesschau.de/'], + returned: new Promise((resolve) => { + resolve(undefined); + }), + }); + assertSpyCall(uploadSourceIconSpy, 0, { + args: [ + supabaseClient, + { + 'id': 'rss-myuser-mycolumn-315eca5c9ed1af9968989241a5b7de09', + 'columnId': 'mycolumn', + 'userId': 'myuser', + 'type': 'rss', + 'title': + 'tagesschau.de - die erste Adresse für Nachrichten und Information', + 'options': { 'rss': 'https://www.tagesschau.de/index~rdf.xml' }, + 'link': 'https://www.tagesschau.de/', + icon: undefined, + }, + ], + returned: new Promise((resolve) => { + resolve(undefined); + }), + }); + assertSpyCalls(fetchWithTimeoutSpy, 1); + assertSpyCalls(getFaviconSpy, 1); + assertSpyCalls(uploadSourceIconSpy, 1); +}); + +const responseNYT = ` + + + NYT >World News + https://www.nytimes.com/section/world + + + en-us + Copyright 2023 The New York Times Company + Tue, 12 Dec 2023 17:20:39 +0000 + Tue, 12 Dec 2023 17:18:21 +0000 + + NYT >World News + https://static01.nyt.com/images/misc/NYT_logo_rss_250x40.png + https://www.nytimes.com/section/world + + + Israel-Hamas War: Houthis Strike Commercial Ship in Red Sea, Fanning Fears of Wider War + https://www.nytimes.com/live/2023/12/12/world/israel-hamas-gaza-war-news + https://www.nytimes.com/live/2023/12/12/world/israel-hamas-gaza-war-news + + A missile hit a tanker in one of the first successful strikes on a ship by the Iranian-backed Houthi militia in Yemen, which has vowed to target vessels in protest of Israel’s assault on Gaza. + The New York Times + Tue, 12 Dec 2023 17:17:11 +0000 + + Khaled Abdullah/Reuters + Protesting in Sanaa, Yemen, this month to show support for Palestinians in Gaza. + + + Russia-Ukraine War: As Zelensky Pleads for Aid, Republicans Demand Border Concessions From Biden + https://www.nytimes.com/live/2023/12/12/us/zelensky-biden-visit + https://www.nytimes.com/live/2023/12/12/us/zelensky-biden-visit + + Ukraine’s president is in Washington with an urgent request for more help to fight Russia, but Republicans in both chambers say they won’t act without a border deal. Mr. Zelensky will meet soon with President Biden. + The New York Times + Tue, 12 Dec 2023 17:17:11 +0000 + + Kent Nishimura for The New York Times + President Volodymyr Zelensky of Ukraine arriving at the Capitol on Tuesday, where he earned a show of support from Senators Chuck Schumer and Mitch McConnell. + + +`; + +Deno.test('getRSSFeed - NYT', async () => { + const fetchWithTimeoutSpy = stub( + utils, + 'fetchWithTimeout', + returnsNext([ + new Promise((resolve) => { + resolve(new Response(responseNYT, { status: 200 })); + }), + ]), + ); + + const getFaviconSpy = stub( + feedutils, + 'getFavicon', + returnsNext([ + new Promise((resolve) => { + resolve(undefined); + }), + ]), + ); + + const uploadSourceIconSpy = stub( + feedutils, + 'uploadSourceIcon', + returnsNext([ + new Promise((resolve) => { + resolve(undefined); + }), + ]), + ); + + try { + const { source, items } = await getRSSFeed( + supabaseClient, + undefined, + mockProfile, + { + ...mockSource, + options: { + rss: 'https://rss.nytimes.com/services/xml/rss/nyt/World.xml', + }, + }, + ); + assertEqualsSource(source, { + 'id': 'rss-myuser-mycolumn-befdaecfac50335eaa1a93512d673fb6', + 'columnId': 'mycolumn', + 'userId': 'myuser', + 'type': 'rss', + 'title': 'NYT >World News', + 'options': { + 'rss': 'https://rss.nytimes.com/services/xml/rss/nyt/World.xml', + }, + 'link': 'https://www.nytimes.com/section/world', + }); + assertEqualsItems(items, [{ + 'id': + 'rss-myuser-mycolumn-befdaecfac50335eaa1a93512d673fb6-d22cc21707b95a8e85b4c7bd88b712e5', + 'userId': 'myuser', + 'columnId': 'mycolumn', + 'sourceId': 'rss-myuser-mycolumn-befdaecfac50335eaa1a93512d673fb6', + 'title': + 'Israel-Hamas War: Houthis Strike Commercial Ship in Red Sea, Fanning Fears of Wider War', + 'link': + 'https://www.nytimes.com/live/2023/12/12/world/israel-hamas-gaza-war-news', + 'media': + 'https://static01.nyt.com/images/2023/12/12/multimedia/12israel-hamas-header-sub-cthq/12israel-hamas-header-sub-cthq-mediumSquareAt3X.jpg', + 'description': + 'A missile hit a tanker in one of the first successful strikes on a ship by the Iranian-backed Houthi militia in Yemen, which has vowed to target vessels in protest of Israel’s assault on Gaza.', + 'author': 'The New York Times', + 'publishedAt': 1702401431, + }, { + 'id': + 'rss-myuser-mycolumn-befdaecfac50335eaa1a93512d673fb6-92c1a2023e17f7793d796690ff6ef250', + 'userId': 'myuser', + 'columnId': 'mycolumn', + 'sourceId': 'rss-myuser-mycolumn-befdaecfac50335eaa1a93512d673fb6', + 'title': + 'Russia-Ukraine War: As Zelensky Pleads for Aid, Republicans Demand Border Concessions From Biden', + 'link': 'https://www.nytimes.com/live/2023/12/12/us/zelensky-biden-visit', + 'media': + 'https://static01.nyt.com/images/2023/12/12/multimedia/12zelensky-dc-blog/12zelensky-dc-hp-mediumSquareAt3X-v2.jpg', + 'description': + 'Ukraine’s president is in Washington with an urgent request for more help to fight Russia, but Republicans in both chambers say they won’t act without a border deal. Mr. Zelensky will meet soon with President Biden.', + 'author': 'The New York Times', + 'publishedAt': 1702401431, + }]); + } finally { + fetchWithTimeoutSpy.restore(); + getFaviconSpy.restore(); + uploadSourceIconSpy.restore(); + } + + assertSpyCall(fetchWithTimeoutSpy, 0, { + args: [ + 'https://rss.nytimes.com/services/xml/rss/nyt/World.xml', + { method: 'get' }, + 5000, + ], + returned: new Promise((resolve) => { + resolve(new Response(responseNYT, { status: 200 })); + }), + }); + assertSpyCall(getFaviconSpy, 0, { + args: ['https://www.nytimes.com/section/world'], + returned: new Promise((resolve) => { + resolve(undefined); + }), + }); + assertSpyCall(uploadSourceIconSpy, 0, { + args: [ + supabaseClient, + { + 'id': 'rss-myuser-mycolumn-befdaecfac50335eaa1a93512d673fb6', + 'columnId': 'mycolumn', + 'userId': 'myuser', + 'type': 'rss', + 'title': 'NYT >World News', + 'options': { + 'rss': 'https://rss.nytimes.com/services/xml/rss/nyt/World.xml', + }, + 'link': 'https://www.nytimes.com/section/world', + icon: undefined, + }, + ], + returned: new Promise((resolve) => { + resolve(undefined); + }), + }); + assertSpyCalls(fetchWithTimeoutSpy, 1); + assertSpyCalls(getFaviconSpy, 1); + assertSpyCalls(uploadSourceIconSpy, 1); +}); + +const responseCNN = ` + + + <![CDATA[CNN.com - RSS Channel]]> + + http://www.cnn.com + + http://i2.cdn.turner.com/cnn/2015/images/09/24/cnn.digital.png + CNN.com - RSS Channel + http://www.cnn.com + + coredev-bumblebee + Tue, 12 Dec 2023 16:50:57 GMT + Tue, 12 Dec 2023 16:50:57 GMT + + + 10 + + <![CDATA[Zelensky visits Washington in push for more Ukraine aid]]> + + https://www.cnn.com/politics/live-news/zelensky-biden-visit-12-12-23/index.html + https://www.cnn.com/politics/live-news/zelensky-biden-visit-12-12-23/index.html + Tue, 12 Dec 2023 16:11:05 GMT + + + + + + + + + + + + + + + + <![CDATA[Humanitarian crisis worsens in Gaza as Israel-Hamas war intensifies]]> + + https://www.cnn.com/middleeast/live-news/israel-hamas-war-gaza-news-12-12-23/index.html + https://www.cnn.com/middleeast/live-news/israel-hamas-war-gaza-news-12-12-23/index.html + Tue, 12 Dec 2023 16:01:14 GMT + + + + + + + + + + + + + + + +`; + +Deno.test('getRSSFeed - CNN', async () => { + const fetchWithTimeoutSpy = stub( + utils, + 'fetchWithTimeout', + returnsNext([ + new Promise((resolve) => { + resolve(new Response(responseCNN, { status: 200 })); + }), + ]), + ); + + const getFaviconSpy = stub( + feedutils, + 'getFavicon', + returnsNext([ + new Promise((resolve) => { + resolve(undefined); + }), + ]), + ); + + const uploadSourceIconSpy = stub( + feedutils, + 'uploadSourceIcon', + returnsNext([ + new Promise((resolve) => { + resolve(undefined); + }), + ]), + ); + + try { + const { source, items } = await getRSSFeed( + supabaseClient, + undefined, + mockProfile, + { + ...mockSource, + options: { + rss: 'http://rss.cnn.com/rss/cnn_latest.rss', + }, + }, + ); + assertEqualsSource(source, { + 'id': 'rss-myuser-mycolumn-27c792d08caef396bf1e3ce488f6b4ea', + 'columnId': 'mycolumn', + 'userId': 'myuser', + 'type': 'rss', + 'title': 'CNN.com - RSS Channel', + 'options': { 'rss': 'http://rss.cnn.com/rss/cnn_latest.rss' }, + 'link': 'http://www.cnn.com', + }); + assertEqualsItems(items, [{ + 'id': + 'rss-myuser-mycolumn-27c792d08caef396bf1e3ce488f6b4ea-6577cd61df49e5dd769eb44e0be07ca5', + 'userId': 'myuser', + 'columnId': 'mycolumn', + 'sourceId': 'rss-myuser-mycolumn-27c792d08caef396bf1e3ce488f6b4ea', + 'title': 'Zelensky visits Washington in push for more Ukraine aid', + 'link': + 'https://www.cnn.com/politics/live-news/zelensky-biden-visit-12-12-23/index.html', + 'media': + 'https://cdn.cnn.com/cnnnext/dam/assets/231212094255-01-zelensky-dc-visit-121223-super-169.jpg', + 'description': + 'Ukrainian President Volodymyr Zelensky is meeting with President Joe Biden at the White House and lawmakers on Capitol Hill Tuesday as discussions on a Ukraine aid deal remain stalled. Follow for the latest live news updates.', + 'publishedAt': 1702397465, + }, { + 'id': + 'rss-myuser-mycolumn-27c792d08caef396bf1e3ce488f6b4ea-45e3785b62227cf241b472cd51107ae2', + 'userId': 'myuser', + 'columnId': 'mycolumn', + 'sourceId': 'rss-myuser-mycolumn-27c792d08caef396bf1e3ce488f6b4ea', + 'title': + 'Humanitarian crisis worsens in Gaza as Israel-Hamas war intensifies', + 'link': + 'https://www.cnn.com/middleeast/live-news/israel-hamas-war-gaza-news-12-12-23/index.html', + 'media': + 'https://cdn.cnn.com/cnnnext/dam/assets/231211100814-gaza-skyline-5t-121123-super-169.jpeg', + 'description': + 'The United Nations General Assembly will resume its emergency session Tuesday on Gaza, days after the United States vetoed a Security Council resolution calling for a ceasefire. Follow for live updates.', + 'publishedAt': 1702396874, + }]); + } finally { + fetchWithTimeoutSpy.restore(); + getFaviconSpy.restore(); + uploadSourceIconSpy.restore(); + } + + assertSpyCall(fetchWithTimeoutSpy, 0, { + args: [ + 'http://rss.cnn.com/rss/cnn_latest.rss', + { method: 'get' }, + 5000, + ], + returned: new Promise((resolve) => { + resolve(new Response(responseCNN, { status: 200 })); + }), + }); + assertSpyCall(getFaviconSpy, 0, { + args: ['http://www.cnn.com'], + returned: new Promise((resolve) => { + resolve(undefined); + }), + }); + assertSpyCall(uploadSourceIconSpy, 0, { + args: [ + supabaseClient, + { + 'id': 'rss-myuser-mycolumn-27c792d08caef396bf1e3ce488f6b4ea', + 'columnId': 'mycolumn', + 'userId': 'myuser', + 'type': 'rss', + 'title': 'CNN.com - RSS Channel', + 'options': { 'rss': 'http://rss.cnn.com/rss/cnn_latest.rss' }, + 'link': 'http://www.cnn.com', + icon: undefined, + }, + ], + returned: new Promise((resolve) => { + resolve(undefined); + }), + }); + assertSpyCalls(fetchWithTimeoutSpy, 1); + assertSpyCalls(getFaviconSpy, 1); + assertSpyCalls(uploadSourceIconSpy, 1); +}); + +const responseNTV = ` + + + + n-tv.de - Startseite + https://www.n-tv.de/ + n-tv.de, Nachrichten seriös, schnell und kompetent. Artikel und Videos aus Politik, Wirtschaft, Börse, Sport und aller Welt. + de-de + n-tv Nachrichtenfernsehen GmbH. RSS Meldungen dürfen nur unverändert wiedergegeben und ausschließlich online verwendet werden. Die eingeräumten Nutzungsrechte beinhalten ausdrücklich nicht das Recht zur Weitergabe an Dritte. Insbesondere ist es nicht gestattet, die Daten auf öffentlichen Screens oder zum Download anzubieten - weder kostenlos noch kostenpflichtig. + Tue, 12 Dec 2023 18:32:01 +0100 + 5 + + https://www.n-tv.de/resources/02291043/adaptive/images/head_logo.png + n-tv.de - Startseite + https://www.n-tv.de/ + + + Tinte oder Laser: Das sind aktuell die besten Drucker + Für einen guten Drucker im Homeoffice kann man 200 Euro, aber auch 500 Euro ausgeben, ein hoher Preis garantiert aber keine hohe Qualität. Welches Gerät das richtige ist, entscheiden individuelle Prioritäten und wie intensiv es eingesetzt wird. Stiftung Warentest hilft bei der Kaufentscheidung. + https://www.n-tv.de/technik/Das-sind-aktuell-die-besten-Drucker-article24595055.html + https://www.n-tv.de/technik/Das-sind-aktuell-die-besten-Drucker-article24595055.html + Technik + Tue, 12 Dec 2023 18:21:58 +0100 + Für einen guten Drucker im Homeoffice kann man 200 Euro, aber auch 500 Euro ausgeben, ein hoher Preis garantiert aber keine hohe Qualität. Welches Gerät das richtige ist, entscheiden individuelle Prioritäten und wie intensiv es eingesetzt wird. Stiftung Warentest hilft bei der Kaufentscheidung.]]> + + + + Auch andere Konzerne betroffen: Hacker legen ukrainischen Mobilfunkanbieter lahm + Am Morgen wird der größte Mobilfunkanbieter der Ukraine, Kyivstar, nach eigenen Angaben Opfer eines Hackerangriffs. Landesweit fallen Verbindungen aus. Die Polizei sei eingeschaltet worden, teilt der Konzern mit. Auch andere Unternehmen sind betroffen. + https://www.n-tv.de/politik/Hacker-legen-ukrainischen-Mobilfunkanbieter-lahm-article24594969.html + https://www.n-tv.de/politik/Hacker-legen-ukrainischen-Mobilfunkanbieter-lahm-article24594969.html + Politik + Tue, 12 Dec 2023 18:13:28 +0100 + Am Morgen wird der größte Mobilfunkanbieter der Ukraine, Kyivstar, nach eigenen Angaben Opfer eines Hackerangriffs. Landesweit fallen Verbindungen aus. Die Polizei sei eingeschaltet worden, teilt der Konzern mit. Auch andere Unternehmen sind betroffen.]]> + + + +`; + +Deno.test('getRSSFeed - NTV', async () => { + const fetchWithTimeoutSpy = stub( + utils, + 'fetchWithTimeout', + returnsNext([ + new Promise((resolve) => { + resolve(new Response(responseNTV, { status: 200 })); + }), + ]), + ); + + const getFaviconSpy = stub( + feedutils, + 'getFavicon', + returnsNext([ + new Promise((resolve) => { + resolve(undefined); + }), + ]), + ); + + const uploadSourceIconSpy = stub( + feedutils, + 'uploadSourceIcon', + returnsNext([ + new Promise((resolve) => { + resolve(undefined); + }), + ]), + ); + + try { + const { source, items } = await getRSSFeed( + supabaseClient, + undefined, + mockProfile, + { + ...mockSource, + options: { + rss: 'https://www.n-tv.de/rss', + }, + }, + ); + assertEqualsSource(source, { + 'id': 'rss-myuser-mycolumn-390a57640c2613653000729c6fef53ae', + 'columnId': 'mycolumn', + 'userId': 'myuser', + 'type': 'rss', + 'title': 'n-tv.de - Startseite', + 'options': { 'rss': 'https://www.n-tv.de/rss' }, + 'link': 'https://www.n-tv.de/', + }); + assertEqualsItems(items, [{ + 'id': + 'rss-myuser-mycolumn-390a57640c2613653000729c6fef53ae-b959f7366439bfcc1381b9a5f9fcf636', + 'userId': 'myuser', + 'columnId': 'mycolumn', + 'sourceId': 'rss-myuser-mycolumn-390a57640c2613653000729c6fef53ae', + 'title': 'Tinte oder Laser: Das sind aktuell die besten Drucker', + 'link': + 'https://www.n-tv.de/technik/Das-sind-aktuell-die-besten-Drucker-article24595055.html', + 'media': + 'https://bilder2.n-tv.de/img/incoming/crop24595613/7928676886-cImg_4_3-w250/Drucker.jpg', + 'description': + 'Für einen guten Drucker im Homeoffice kann man 200 Euro, aber auch 500 Euro ausgeben, ein hoher Preis garantiert aber keine hohe Qualität. Welches Gerät das richtige ist, entscheiden individuelle Prioritäten und wie intensiv es eingesetzt wird. Stiftung Warentest hilft bei der Kaufentscheidung.', + 'publishedAt': 1702401718, + }, { + 'id': + 'rss-myuser-mycolumn-390a57640c2613653000729c6fef53ae-36e1c97d7833bb69e6e53704e866e3ba', + 'userId': 'myuser', + 'columnId': 'mycolumn', + 'sourceId': 'rss-myuser-mycolumn-390a57640c2613653000729c6fef53ae', + 'title': + 'Auch andere Konzerne betroffen: Hacker legen ukrainischen Mobilfunkanbieter lahm', + 'link': + 'https://www.n-tv.de/politik/Hacker-legen-ukrainischen-Mobilfunkanbieter-lahm-article24594969.html', + 'media': + 'https://bilder2.n-tv.de/img/incoming/crop24595013/3588678578-cImg_4_3-w250/imago0241272335h.jpg', + 'description': + 'Am Morgen wird der größte Mobilfunkanbieter der Ukraine, Kyivstar, nach eigenen Angaben Opfer eines Hackerangriffs. Landesweit fallen Verbindungen aus. Die Polizei sei eingeschaltet worden, teilt der Konzern mit. Auch andere Unternehmen sind betroffen.', + 'publishedAt': 1702401208, + }]); + } finally { + fetchWithTimeoutSpy.restore(); + getFaviconSpy.restore(); + uploadSourceIconSpy.restore(); + } + + assertSpyCall(fetchWithTimeoutSpy, 0, { + args: [ + 'https://www.n-tv.de/rss', + { method: 'get' }, + 5000, + ], + returned: new Promise((resolve) => { + resolve(new Response(responseNTV, { status: 200 })); + }), + }); + assertSpyCall(getFaviconSpy, 0, { + args: ['https://www.n-tv.de/'], + returned: new Promise((resolve) => { + resolve(undefined); + }), + }); + assertSpyCall(uploadSourceIconSpy, 0, { + args: [ + supabaseClient, + { + 'id': 'rss-myuser-mycolumn-390a57640c2613653000729c6fef53ae', + 'columnId': 'mycolumn', + 'userId': 'myuser', + 'type': 'rss', + 'title': 'n-tv.de - Startseite', + 'options': { 'rss': 'https://www.n-tv.de/rss' }, + 'link': 'https://www.n-tv.de/', + icon: undefined, + }, + ], + returned: new Promise((resolve) => { + resolve(undefined); + }), + }); + assertSpyCalls(fetchWithTimeoutSpy, 1); + assertSpyCalls(getFaviconSpy, 1); + assertSpyCalls(uploadSourceIconSpy, 1); +}); + +const responseFP = ` + + Freie Presse - Chemnitz + + + Freie Presse + + 2023-12-12T18:06:34Z + tag:www.freiepresse.de,2023-12-12:/rss/rss_chemnitz.php + + + https://www.freiepresse.de/chemnitz/nach-streit-im-chemnitzer-kuechwald-buehnenverein-werner-haas-gruendet-neues-theater-artikel13168401 + 2023-12-12T18:00:00Z + Nach Streit im Chemnitzer Küchwald-Bühnenverein: Werner Haas gründet neues Theater + <img src="https://www.freiepresse.de/DYNIMG/72/08/13067208_W740C2040x1360o0x0.jpg" />Es ist nicht lange still geblieben um Werner Haas. Der Regisseur und Chorleiter trennte sich im Sommer im Streit vom Verein zur Belebung der Küchwaldbühne. Er hatte den Verein gegründet und leitete die Laientheatergruppe, die fast jährlich auf der Freilichtbühne spielte. Für 2023 hatte sich der Vereinsvorstand aber mehrheitlich gegen eine Premiere entschieden. Haas habe kein tragfähiges Konzept da... + Andreas Seidel/Archiv + + + + + https://www.freiepresse.de/chemnitz/oberflaechliche-ermittlungen-fehlende-technik-im-gerichtssaal-prozessauftakt-zu-2018er-aufarbeitung-wirft-fragen-auf-artikel13168461 + 2023-12-12T17:00:00Z + Oberflächliche Ermittlungen, fehlende Technik im Gerichtssaal: Prozessauftakt zu 2018er-Aufarbeitung wirft Fragen auf + <img src="https://www.freiepresse.de/DYNIMG/73/46/13067346_W740C2040x1360o0x0.jpg" />Der Beginn der Verhandlung war auf 9 Uhr am Montag angesetzt, als die Vernehmung der ersten Zeugin begann, war es 13 Uhr. Sie sollte unter anderem den Weg skizzieren, den die mutmaßlichen Täter am Tattag zu den Tatorten genommen hatten. Vier Angeklagte mit je einem Vertreter auf der einen Seite, Staatsanwalt und drei Nebenkläger-Vertreter auf der anderen. Der Vorsitzende Richter Jürgen Zöllner lie... + Harry Haertel + + +`; + +Deno.test('getRSSFeed - FP', async () => { + const fetchWithTimeoutSpy = stub( + utils, + 'fetchWithTimeout', + returnsNext([ + new Promise((resolve) => { + resolve(new Response(responseFP, { status: 200 })); + }), + ]), + ); + + const getFaviconSpy = stub( + feedutils, + 'getFavicon', + returnsNext([ + new Promise((resolve) => { + resolve(undefined); + }), + ]), + ); + + const uploadSourceIconSpy = stub( + feedutils, + 'uploadSourceIcon', + returnsNext([ + new Promise((resolve) => { + resolve(undefined); + }), + ]), + ); + + try { + const { source, items } = await getRSSFeed( + supabaseClient, + undefined, + mockProfile, + { + ...mockSource, + options: { + rss: 'https://www.freiepresse.de/rss/rss_chemnitz.php', + }, + }, + ); + assertEqualsSource(source, { + 'id': 'rss-myuser-mycolumn-3d20715877c48f9b58c13809c5abc600', + 'columnId': 'mycolumn', + 'userId': 'myuser', + 'type': 'rss', + 'title': 'Freie Presse - Chemnitz', + 'options': { 'rss': 'https://www.freiepresse.de/rss/rss_chemnitz.php' }, + 'link': 'https://www.freiepresse.de/rss/rss_chemnitz.php', + }); + assertEqualsItems(items, [{ + 'id': + 'rss-myuser-mycolumn-3d20715877c48f9b58c13809c5abc600-39bfc1acb8b30883dce00bb6167b766d', + 'userId': 'myuser', + 'columnId': 'mycolumn', + 'sourceId': 'rss-myuser-mycolumn-3d20715877c48f9b58c13809c5abc600', + 'title': + 'Nach Streit im Chemnitzer Küchwald-Bühnenverein: Werner Haas gründet neues Theater', + 'link': + 'https://www.freiepresse.de/chemnitz/nach-streit-im-chemnitzer-kuechwald-buehnenverein-werner-haas-gruendet-neues-theater-artikel13168401', + 'media': + 'https://www.freiepresse.de/DYNIMG/72/08/13067208_W740C2040x1360o0x0.jpg', + 'description': + 'Es ist nicht lange still geblieben um Werner Haas. Der Regisseur und Chorleiter trennte sich im Sommer im Streit vom Verein zur Belebung der Küchwaldbühne. Er hatte den Verein gegründet und leitete die Laientheatergruppe, die fast jährlich auf der Freilichtbühne spielte. Für 2023 hatte sich der Vereinsvorstand aber mehrheitlich gegen eine Premiere entschieden. Haas habe kein tragfähiges Konzept da...', + 'publishedAt': 1702404000, + }, { + 'id': + 'rss-myuser-mycolumn-3d20715877c48f9b58c13809c5abc600-cab31961af28e14e6b601b7e68bcd678', + 'userId': 'myuser', + 'columnId': 'mycolumn', + 'sourceId': 'rss-myuser-mycolumn-3d20715877c48f9b58c13809c5abc600', + 'title': + 'Oberflächliche Ermittlungen, fehlende Technik im Gerichtssaal: Prozessauftakt zu 2018er-Aufarbeitung wirft Fragen auf', + 'link': + 'https://www.freiepresse.de/chemnitz/oberflaechliche-ermittlungen-fehlende-technik-im-gerichtssaal-prozessauftakt-zu-2018er-aufarbeitung-wirft-fragen-auf-artikel13168461', + 'media': + 'https://www.freiepresse.de/DYNIMG/73/46/13067346_W740C2040x1360o0x0.jpg', + 'description': + 'Der Beginn der Verhandlung war auf 9 Uhr am Montag angesetzt, als die Vernehmung der ersten Zeugin begann, war es 13 Uhr. Sie sollte unter anderem den Weg skizzieren, den die mutmaßlichen Täter am Tattag zu den Tatorten genommen hatten. Vier Angeklagte mit je einem Vertreter auf der einen Seite, Staatsanwalt und drei Nebenkläger-Vertreter auf der anderen. Der Vorsitzende Richter Jürgen Zöllner lie...', + 'publishedAt': 1702400400, + }]); + } finally { + fetchWithTimeoutSpy.restore(); + getFaviconSpy.restore(); + uploadSourceIconSpy.restore(); + } + + assertSpyCall(fetchWithTimeoutSpy, 0, { + args: [ + 'https://www.freiepresse.de/rss/rss_chemnitz.php', + { method: 'get' }, + 5000, + ], + returned: new Promise((resolve) => { + resolve(new Response(responseFP, { status: 200 })); + }), + }); + assertSpyCall(getFaviconSpy, 0, { + args: ['https://www.freiepresse.de/rss/rss_chemnitz.php'], + returned: new Promise((resolve) => { + resolve(undefined); + }), + }); + assertSpyCall(uploadSourceIconSpy, 0, { + args: [ + supabaseClient, + { + 'id': 'rss-myuser-mycolumn-3d20715877c48f9b58c13809c5abc600', + 'columnId': 'mycolumn', + 'userId': 'myuser', + 'type': 'rss', + 'title': 'Freie Presse - Chemnitz', + 'options': { 'rss': 'https://www.freiepresse.de/rss/rss_chemnitz.php' }, + 'link': 'https://www.freiepresse.de/rss/rss_chemnitz.php', + icon: undefined, + }, + ], + returned: new Promise((resolve) => { + resolve(undefined); + }), + }); + assertSpyCalls(fetchWithTimeoutSpy, 1); + assertSpyCalls(getFaviconSpy, 1); + assertSpyCalls(uploadSourceIconSpy, 1); +}); diff --git a/supabase/functions/_shared/feed/stackoverflow.ts b/supabase/functions/_shared/feed/stackoverflow.ts index 73e7f75..5385446 100644 --- a/supabase/functions/_shared/feed/stackoverflow.ts +++ b/supabase/functions/_shared/feed/stackoverflow.ts @@ -8,8 +8,7 @@ import { unescape } from 'lodash'; import { ISource } from '../models/source.ts'; import { IItem } from '../models/item.ts'; import { IProfile } from '../models/profile.ts'; -import { fetchWithTimeout } from '../utils/fetchWithTimeout.ts'; -import { log } from '../utils/log.ts'; +import { utils } from '../utils/index.ts'; export const getStackoverflowFeed = async ( _supabaseClient: SupabaseClient, @@ -34,11 +33,15 @@ export const getStackoverflowFeed = async ( * Get the RSS for the provided `stackoverflow` url and parse it. If a feed * doesn't contains an item we return an error. */ - const response = await fetchWithTimeout(source.options.stackoverflow.url, { - method: 'get', - }, 5000); + const response = await utils.fetchWithTimeout( + source.options.stackoverflow.url, + { + method: 'get', + }, + 5000, + ); const xml = await response.text(); - log('debug', 'Add source', { + utils.log('debug', 'Add source', { sourceType: 'stackoverflow', requestUrl: source.options.stackoverflow.url, responseStatus: response.status, diff --git a/supabase/functions/_shared/feed/stackoverflow_test.ts b/supabase/functions/_shared/feed/stackoverflow_test.ts new file mode 100644 index 0000000..52060fd --- /dev/null +++ b/supabase/functions/_shared/feed/stackoverflow_test.ts @@ -0,0 +1,203 @@ +import { createClient } from '@supabase/supabase-js'; +import { + assertSpyCall, + assertSpyCalls, + returnsNext, + stub, +} from 'std/testing/mock'; + +import { ISource } from '../models/source.ts'; +import { IProfile } from '../models/profile.ts'; +import { getStackoverflowFeed } from './stackoverflow.ts'; +import { utils } from '../utils/index.ts'; +import { feedutils } from './utils/index.ts'; + +const supabaseClient = createClient('http://localhost:54321', 'test123'); +const mockProfile: IProfile = { + id: '', + tier: 'free', + createdAt: 0, + updatedAt: 0, +}; +const mockSource: ISource = { + id: '', + columnId: 'mycolumn', + userId: 'myuser', + type: 'medium', + title: '', +}; + +const responseTag = ` + + Newest questions tagged kubernetes - Stack Overflow + + + most recent 30 from stackoverflow.com + 2023-12-10T17:21:25Z + https://stackoverflow.com/feeds/tag?tagnames=kubernetes&sort=newest + https://creativecommons.org/licenses/by-sa/4.0/rdf + + https://stackoverflow.com/q/77635559 + 0 + Kubernetes Ingress behind Cloud Load Balancer + + + + + + Dominik.W + https://stackoverflow.com/users/23076575 + + + 2023-12-10T16:39:44Z + 2023-12-10T16:39:44Z + <p>When using an Ingress Controller in Kubernetes the Ingress service is usually exposed via Load Balancer. Now I’m trining to understand on how this exactly works. +As I understand it the Ingress Controller is just running as an Pod like any other app and gets exposed via the Load Balancer. +When configuring the external load balancer what target do I set, the Worker nodes or the master nodes, or does this even matter because I use a Service and then it’s automatically internally Load balanced?</p> +<p>I try to get this Right so I can setup a Kubernetes Cluster in the Hetzner Cloud, because it has no managed service I need to do basically everything on my on but it provides the services to theoretically host a full HA cluster. +So the plan is to have for the beginning 3 Master Nodes and 2/3 Worker Nodes and an Managed Load Balancer in front of everything. +I thought about having 2 Cloud Networks one lb-network for the master nodes and the load balancer and a second one cluster network for the master and worker nodes. But with that approach every incoming traffic needs to get through the Masters to get terminated at the Ingress Controller which is running on the Worker, I like that approach because it allows me to use fewer targets on the Load Balancer to save some money also I could mostly isolate the workers from incoming traffic on a network level. Is that approach possible and even best practices or what do you recommend?</p> + + + https://stackoverflow.com/q/77635466 + 0 + Application (valhalla-server) not running (or accessible) )when exposed through clusterIP or NodePort service + + + + + kubexplore + https://stackoverflow.com/users/23076494 + + + 2023-12-10T16:07:16Z + 2023-12-10T16:33:29Z + <p>I have created a pod to deploy valhalla-server, yaml below:</p> +<pre><code>apiVersion: v1 +kind: Pod +metadata: + name: valahalla-pod + labels: + app: valahalla-app-pod # Updated label name +spec: + containers: + - name: docker-valahalla + image: ghcr.io/gis-ops/docker-valhalla/valhalla + env: + - name: tile_urls + value: &quot;https://download.geofabrik.de/europe/andorra-latest.osm.pbf ghcr.io/gis-ops/docker-valhalla/valhalla:latest&quot; + ports: + - containerPort: 8002 + volumeMounts: + - name: my-local-folder + mountPath: /custom_files + volumes: + - name: my-local-folder + hostPath: + path: /home/ubuntu/custom_files +</code></pre> +<p>When I exec into this pod, I am able to access the valhalla server on localhost on port 8002. But I have created a service for the pod, yaml below:</p> +<pre><code>apiVersion: v1 +kind: Service +metadata: + name: valahalla-service +spec: + selector: + app: valahalla-app-pod + ports: + - protocol: TCP + port: 80 # Port exposed by the service + targetPort: 8002 + type: NodePort +</code></pre> +<p>I am not able to access my application through this! It gives output when I curl by going into the pod but not when I try to access it from outisde.</p> +<p>I am using EKS on AWS for this.</p> +<p>I was using deployment before, I've tried using pod instead and changed service from clusterIP to NodePort.</p> + +`; + +Deno.test('getStackoverflowFeed', async () => { + const fetchWithTimeoutSpy = stub( + utils, + 'fetchWithTimeout', + returnsNext([ + new Promise((resolve) => { + resolve(new Response(responseTag, { status: 200 })); + }), + ]), + ); + + try { + const { source, items } = await getStackoverflowFeed( + supabaseClient, + undefined, + mockProfile, + { + ...mockSource, + options: { + stackoverflow: { type: 'tag', tag: 'kubernetes', sort: 'newest' }, + }, + }, + ); + feedutils.assertEqualsSource(source, { + 'id': 'stackoverflow-myuser-mycolumn-b33aefc859cbc9c75f22dc8de83b59e7', + 'columnId': 'mycolumn', + 'userId': 'myuser', + 'type': 'stackoverflow', + 'title': 'Newest questions tagged kubernetes - Stack Overflow', + 'options': { + 'stackoverflow': { + 'type': 'tag', + 'tag': 'kubernetes', + 'sort': 'newest', + 'url': + 'https://stackoverflow.com/feeds/tag?tagnames=kubernetes&sort=newest', + }, + }, + 'link': + 'https://stackoverflow.com/feeds/tag?tagnames=kubernetes&sort=newest', + }); + feedutils.assertEqualsItems(items, [{ + 'id': + 'stackoverflow-myuser-mycolumn-b33aefc859cbc9c75f22dc8de83b59e7-4ccc40394df08fa6092fa370ad44fa79', + 'userId': 'myuser', + 'columnId': 'mycolumn', + 'sourceId': + 'stackoverflow-myuser-mycolumn-b33aefc859cbc9c75f22dc8de83b59e7', + 'title': 'Kubernetes Ingress behind Cloud Load Balancer', + 'link': + 'https://stackoverflow.com/questions/77635559/kubernetes-ingress-behind-cloud-load-balancer', + 'description': + '

When using an Ingress Controller in Kubernetes the Ingress service is usually exposed via Load Balancer. Now I’m trining to understand on how this exactly works.\nAs I understand it the Ingress Controller is just running as an Pod like any other app and gets exposed via the Load Balancer.\nWhen configuring the external load balancer what target do I set, the Worker nodes or the master nodes, or does this even matter because I use a Service and then it’s automatically internally Load balanced?

\n

I try to get this Right so I can setup a Kubernetes Cluster in the Hetzner Cloud, because it has no managed service I need to do basically everything on my on but it provides the services to theoretically host a full HA cluster.\nSo the plan is to have for the beginning 3 Master Nodes and 2/3 Worker Nodes and an Managed Load Balancer in front of everything.\nI thought about having 2 Cloud Networks one lb-network for the master nodes and the load balancer and a second one cluster network for the master and worker nodes. But with that approach every incoming traffic needs to get through the Masters to get terminated at the Ingress Controller which is running on the Worker, I like that approach because it allows me to use fewer targets on the Load Balancer to save some money also I could mostly isolate the workers from incoming traffic on a network level. Is that approach possible and even best practices or what do you recommend?

', + 'publishedAt': 1702226384, + }, { + 'id': + 'stackoverflow-myuser-mycolumn-b33aefc859cbc9c75f22dc8de83b59e7-6f932d7a7105a5e38fa70de9bbad6fe5', + 'userId': 'myuser', + 'columnId': 'mycolumn', + 'sourceId': + 'stackoverflow-myuser-mycolumn-b33aefc859cbc9c75f22dc8de83b59e7', + 'title': + 'Application (valhalla-server) not running (or accessible) )when exposed through clusterIP or NodePort service', + 'link': + 'https://stackoverflow.com/questions/77635466/application-valhalla-server-not-running-or-accessible-when-exposed-through', + 'description': + '

I have created a pod to deploy valhalla-server, yaml below:

\n
apiVersion: v1\nkind: Pod\nmetadata:\n  name: valahalla-pod\n  labels:\n    app: valahalla-app-pod  # Updated label name\nspec:\n  containers:\n  - name: docker-valahalla\n    image: ghcr.io/gis-ops/docker-valhalla/valhalla\n    env:\n      - name: tile_urls\n        value: "https://download.geofabrik.de/europe/andorra-latest.osm.pbf ghcr.io/gis-ops/docker-valhalla/valhalla:latest"\n    ports:\n    - containerPort: 8002\n    volumeMounts:\n    - name: my-local-folder\n      mountPath: /custom_files\n  volumes:\n  - name: my-local-folder\n    hostPath:\n      path: /home/ubuntu/custom_files\n
\n

When I exec into this pod, I am able to access the valhalla server on localhost on port 8002. But I have created a service for the pod, yaml below:

\n
apiVersion: v1\nkind: Service\nmetadata:\n  name: valahalla-service\nspec:\n  selector:\n    app: valahalla-app-pod\n  ports:\n    - protocol: TCP\n      port: 80  # Port exposed by the service\n      targetPort: 8002\n  type: NodePort\n
\n

I am not able to access my application through this! It gives output when I curl by going into the pod but not when I try to access it from outisde.

\n

I am using EKS on AWS for this.

\n

I was using deployment before, I\'ve tried using pod instead and changed service from clusterIP to NodePort.

', + 'publishedAt': 1702224436, + }]); + } finally { + fetchWithTimeoutSpy.restore(); + } + + assertSpyCall(fetchWithTimeoutSpy, 0, { + args: [ + 'https://stackoverflow.com/feeds/tag?tagnames=kubernetes&sort=newest', + { method: 'get' }, + 5000, + ], + returned: new Promise((resolve) => { + resolve(new Response(responseTag, { status: 200 })); + }), + }); + assertSpyCalls(fetchWithTimeoutSpy, 1); +}); diff --git a/supabase/functions/_shared/feed/tumblr.ts b/supabase/functions/_shared/feed/tumblr.ts index 21edccb..4c2708f 100644 --- a/supabase/functions/_shared/feed/tumblr.ts +++ b/supabase/functions/_shared/feed/tumblr.ts @@ -8,8 +8,7 @@ import { Redis } from 'redis'; import { ISource } from '../models/source.ts'; import { IItem } from '../models/item.ts'; import { IProfile } from '../models/profile.ts'; -import { fetchWithTimeout } from '../utils/fetchWithTimeout.ts'; -import { log } from '../utils/log.ts'; +import { utils } from '../utils/index.ts'; /** * `isTumblrUrl` checks if the provided `url` is a valid Tumblr url. A url is @@ -50,11 +49,11 @@ export const getTumblrFeed = async ( * Get the RSS for the provided `tumblr` url and parse it. If a feed doesn't * contains an item we return an error. */ - const response = await fetchWithTimeout(source.options.tumblr, { + const response = await utils.fetchWithTimeout(source.options.tumblr, { method: 'get', }, 5000); const xml = await response.text(); - log('debug', 'Add source', { + utils.log('debug', 'Add source', { sourceType: 'tumblr', requestUrl: source.options.tumblr, responseStatus: response.status, diff --git a/supabase/functions/_shared/feed/tumblr_test.ts b/supabase/functions/_shared/feed/tumblr_test.ts new file mode 100644 index 0000000..f4406a5 --- /dev/null +++ b/supabase/functions/_shared/feed/tumblr_test.ts @@ -0,0 +1,178 @@ +import { assertEquals } from 'std/assert'; +import { createClient } from '@supabase/supabase-js'; +import { + assertSpyCall, + assertSpyCalls, + returnsNext, + stub, +} from 'std/testing/mock'; + +import { ISource } from '../models/source.ts'; +import { IProfile } from '../models/profile.ts'; +import { getTumblrFeed, isTumblrUrl } from './tumblr.ts'; +import { utils } from '../utils/index.ts'; +import { feedutils } from './utils/index.ts'; + +const supabaseClient = createClient('http://localhost:54321', 'test123'); +const mockProfile: IProfile = { + id: '', + tier: 'free', + createdAt: 0, + updatedAt: 0, +}; +const mockSource: ISource = { + id: '', + columnId: 'mycolumn', + userId: 'myuser', + type: 'medium', + title: '', +}; + +const response = ` + + + + Today on Tumblr + Tumblr (3.0; @todayontumblr) + https://todayontumblr.tumblr.com/ + + NEIL WON BEST ACTOR AT THE GAME AWARDS!!!!!! + <p ><a class="tumblr_blog" href="https://argetcross.tumblr.com/post/736098070263218176/neil-won-best-actor-at-the-game-awards">argetcross </a >:</p ><blockquote ><p >NEIL WON BEST ACTOR AT THE GAME AWARDS!!!!!!</p ><div class="npf_row"><figure class="tmblr-full" data-orig-height="945" data-orig-width="2048"><img src="https://64.media.tumblr.com/3721ad004e334d62e290e5062a296566/a5567169c27bf241-f9/s640x960/b8e88192aa512164126093c75681143fe851fead.jpg" data-orig-height="945" data-orig-width="2048" srcset="https://64.media.tumblr.com/3721ad004e334d62e290e5062a296566/a5567169c27bf241-f9/s75x75_c1/eff71b743942cc3094990943933886d7d289ae2f.jpg 75w, https://64.media.tumblr.com/3721ad004e334d62e290e5062a296566/a5567169c27bf241-f9/s100x200/f0d8dc8bb82087bfd457f9fcc92e8ec0a2d79250.jpg 100w, https://64.media.tumblr.com/3721ad004e334d62e290e5062a296566/a5567169c27bf241-f9/s250x400/fa2684d74d1a1321c7bece3f6bd0caa40392c36d.jpg 250w, https://64.media.tumblr.com/3721ad004e334d62e290e5062a296566/a5567169c27bf241-f9/s400x600/b578a131611c62237a1403a081ae8dbf8bb57b72.jpg 400w, https://64.media.tumblr.com/3721ad004e334d62e290e5062a296566/a5567169c27bf241-f9/s500x750/bed251d1aef5fffcc65ea69f8a113d0a99585081.jpg 500w, https://64.media.tumblr.com/3721ad004e334d62e290e5062a296566/a5567169c27bf241-f9/s540x810/3daeb676fcf06741f47af8f7b910b8e8368622e1.jpg 540w, https://64.media.tumblr.com/3721ad004e334d62e290e5062a296566/a5567169c27bf241-f9/s640x960/b8e88192aa512164126093c75681143fe851fead.jpg 640w, https://64.media.tumblr.com/3721ad004e334d62e290e5062a296566/a5567169c27bf241-f9/s1280x1920/2da89407179aa94cd58b581a58944fc2160f9d56.jpg 1280w, https://64.media.tumblr.com/3721ad004e334d62e290e5062a296566/a5567169c27bf241-f9/s2048x3072/49b9d11b154993cad04a1e2c61071573e184002e.jpg 2048w" sizes="(max-width: 1280px) 100vw, 1280px"/></figure ></div ></blockquote > + https://www.tumblr.com/todayontumblr/736339394445918208 + https://www.tumblr.com/todayontumblr/736339394445918208 + Sun, 10 Dec 2023 12:06:08 -0500 + today on tumblr + + + That &rsquo;s why they were rushing everyone &rsquo;s speeches cause Geoff wanted enough time to talk to his wife &hellip; + <p ><a class="tumblr_blog" href="https://orallech.tumblr.com/post/736101836478611456/thats-why-they-were-rushing-everyones-speeches">orallech </a >:</p ><blockquote ><p >That’s why they were rushing everyone’s speeches cause Geoff wanted enough time to talk to his wife kojima </p ></blockquote > + https://www.tumblr.com/todayontumblr/736335427403907072 + https://www.tumblr.com/todayontumblr/736335427403907072 + Sun, 10 Dec 2023 11:03:05 -0500 + today on tumblr + + + Muppet Fact #925 + <p ><a class="tumblr_blog" href="https://muppet-facts.tumblr.com/post/736100473317343232/muppet-fact-925">muppet-facts </a >:</p ><blockquote ><p >At the 2023 Game Awards, Gonzo presented the award for Best Debut Indie Game with Geoff Keighley. He said he &rsquo;s been playing <i >Tears of the Kingdom </i >. He lost days following a Cucco up the hill. He also has a conspiracy that a lot of games released this year follow a chicken theme. </p ><div class="npf_row"><figure class="tmblr-full" data-orig-height="2160" data-orig-width="3840"><img src="https://64.media.tumblr.com/7338226115b70192b7b37dae667dc0b3/217c86a94d5179ce-1e/s640x960/de1345504db6ac2e012d6aef542489fbfd8a522b.png" data-orig-height="2160" data-orig-width="3840" srcset="https://64.media.tumblr.com/7338226115b70192b7b37dae667dc0b3/217c86a94d5179ce-1e/s75x75_c1/24353b4aa077be11678ff83f73bcb2cdc02a0a82.png 75w, https://64.media.tumblr.com/7338226115b70192b7b37dae667dc0b3/217c86a94d5179ce-1e/s100x200/f8bae9a792c7644dee26379f44fb8d36f476213d.png 100w, https://64.media.tumblr.com/7338226115b70192b7b37dae667dc0b3/217c86a94d5179ce-1e/s250x400/f7d6d9833510950930a3e14e3bc345d067a30868.png 250w, https://64.media.tumblr.com/7338226115b70192b7b37dae667dc0b3/217c86a94d5179ce-1e/s400x600/0a16568c844f59cf493607a1cd6a32899ca5cf47.png 400w, https://64.media.tumblr.com/7338226115b70192b7b37dae667dc0b3/217c86a94d5179ce-1e/s500x750/c4635e448d70ab8200d5f29256a7153301a8e415.png 500w, https://64.media.tumblr.com/7338226115b70192b7b37dae667dc0b3/217c86a94d5179ce-1e/s540x810/d1f17ba75dd935012801d629591ee480ce8eca49.png 540w, https://64.media.tumblr.com/7338226115b70192b7b37dae667dc0b3/217c86a94d5179ce-1e/s640x960/de1345504db6ac2e012d6aef542489fbfd8a522b.png 640w, https://64.media.tumblr.com/7338226115b70192b7b37dae667dc0b3/217c86a94d5179ce-1e/s1280x1920/f7849216be34e6744bcf83e8d06e306ef486d439.png 1280w, https://64.media.tumblr.com/7338226115b70192b7b37dae667dc0b3/217c86a94d5179ce-1e/s2048x3072/673081b3df788fbe38d4237b236839f59ba339b8.png 2048w" sizes="(max-width: 1280px) 100vw, 1280px"/></figure ><figure class="tmblr-full" data-orig-height="1080" data-orig-width="1920"><img src="https://64.media.tumblr.com/4175b0d1d5a1b0fd5c0fe1bfe7b50c03/217c86a94d5179ce-c1/s640x960/f0dfd4aa4229944da9b0e525e6c9a0ec89143373.png" data-orig-height="1080" data-orig-width="1920" srcset="https://64.media.tumblr.com/4175b0d1d5a1b0fd5c0fe1bfe7b50c03/217c86a94d5179ce-c1/s75x75_c1/0268965777520275909841726829a2571b4dd4b1.png 75w, https://64.media.tumblr.com/4175b0d1d5a1b0fd5c0fe1bfe7b50c03/217c86a94d5179ce-c1/s100x200/e3b79f463b947c8857dddceda099e4cfe6e53e9c.png 100w, https://64.media.tumblr.com/4175b0d1d5a1b0fd5c0fe1bfe7b50c03/217c86a94d5179ce-c1/s250x400/4eeb69a20d2872719cf1e2e28b54367d28a33114.png 250w, https://64.media.tumblr.com/4175b0d1d5a1b0fd5c0fe1bfe7b50c03/217c86a94d5179ce-c1/s400x600/445c4b2e690ae3075b362a105af9b4aa45adee88.png 400w, https://64.media.tumblr.com/4175b0d1d5a1b0fd5c0fe1bfe7b50c03/217c86a94d5179ce-c1/s500x750/30a34daebebee02fc501e9ca7361d6f3221ed856.png 500w, https://64.media.tumblr.com/4175b0d1d5a1b0fd5c0fe1bfe7b50c03/217c86a94d5179ce-c1/s540x810/29f17a07b007f7476d4a321a56a29f333c32b93e.png 540w, https://64.media.tumblr.com/4175b0d1d5a1b0fd5c0fe1bfe7b50c03/217c86a94d5179ce-c1/s640x960/f0dfd4aa4229944da9b0e525e6c9a0ec89143373.png 640w, https://64.media.tumblr.com/4175b0d1d5a1b0fd5c0fe1bfe7b50c03/217c86a94d5179ce-c1/s1280x1920/6b40fc10a77481213526f9f4412aca16bc0ce173.png 1280w, https://64.media.tumblr.com/4175b0d1d5a1b0fd5c0fe1bfe7b50c03/217c86a94d5179ce-c1/s2048x3072/286ccace479893c0e6a7803080132f3b7869b634.png 1920w" sizes="(max-width: 1280px) 100vw, 1280px"/></figure ></div ><p ><b >Source: </b ></p ><p >The Game Awards 2023. December 7, 2023. </p ></blockquote > + https://www.tumblr.com/todayontumblr/736331601085120512 + https://www.tumblr.com/todayontumblr/736331601085120512 + Sun, 10 Dec 2023 10:02:16 -0500 + today on tumblr + + + Hardest image in gaming culture + <p ><a class="tumblr_blog" href="https://www.tumblr.com/temperedknight/736102767863742464/hardest-image-in-gaming-culture">temperedknight </a >:</p ><blockquote ><div class="npf_row"><figure class="tmblr-full" data-orig-height="521" data-orig-width="1063"><img src="https://64.media.tumblr.com/363392fc016d8c904fa3b86050b366a2/08a786e18a31fca4-84/s640x960/8ce0d3c4fa8efea0a196cb1921aa64cc43950f8d.png" data-orig-height="521" data-orig-width="1063" srcset="https://64.media.tumblr.com/363392fc016d8c904fa3b86050b366a2/08a786e18a31fca4-84/s75x75_c1/c86df7bd8018d29a3848bf9ebf68c645f3eb43ff.png 75w, https://64.media.tumblr.com/363392fc016d8c904fa3b86050b366a2/08a786e18a31fca4-84/s100x200/4d9c7c121ef92a8ba19983b2a5a56e5406a03e0c.png 100w, https://64.media.tumblr.com/363392fc016d8c904fa3b86050b366a2/08a786e18a31fca4-84/s250x400/2cf0925fa1dfd3fae5a991e88453f024be044f70.png 250w, https://64.media.tumblr.com/363392fc016d8c904fa3b86050b366a2/08a786e18a31fca4-84/s400x600/88f6685b3a8f0bafeb83ccb875d82d29269b268d.png 400w, https://64.media.tumblr.com/363392fc016d8c904fa3b86050b366a2/08a786e18a31fca4-84/s500x750/e084806a49fb3808cd10d8916bcaec53493fc181.png 500w, https://64.media.tumblr.com/363392fc016d8c904fa3b86050b366a2/08a786e18a31fca4-84/s540x810/4630feda50c2ec3c41a7d7413cae5f853cdb5034.png 540w, https://64.media.tumblr.com/363392fc016d8c904fa3b86050b366a2/08a786e18a31fca4-84/s640x960/8ce0d3c4fa8efea0a196cb1921aa64cc43950f8d.png 640w, https://64.media.tumblr.com/363392fc016d8c904fa3b86050b366a2/08a786e18a31fca4-84/s1280x1920/c48e8fb2dad3ff167d0a318025c6edfff19a49fc.png 1063w" sizes="(max-width: 1063px) 100vw, 1063px"/></figure ></div ><p >Hardest image in gaming culture </p ></blockquote > + https://www.tumblr.com/todayontumblr/736327882138386432 + https://www.tumblr.com/todayontumblr/736327882138386432 + Sun, 10 Dec 2023 09:03:09 -0500 + today on tumblr + + +`; + +Deno.test('isTumblrUrl', () => { + assertEquals( + isTumblrUrl('https://www.tumblr.com/todayontumblr'), + true, + ); + assertEquals(isTumblrUrl('https://www.google.de/'), false); +}); + +Deno.test('getTumblrFeed', async () => { + const fetchWithTimeoutSpy = stub( + utils, + 'fetchWithTimeout', + returnsNext([ + new Promise((resolve) => { + resolve(new Response(response, { status: 200 })); + }), + ]), + ); + + try { + const { source, items } = await getTumblrFeed( + supabaseClient, + undefined, + mockProfile, + { + ...mockSource, + options: { tumblr: 'https://www.tumblr.com/todayontumblr' }, + }, + ); + feedutils.assertEqualsSource(source, { + 'id': 'tumblr-myuser-mycolumn-8ce77e730a91955cd991349d0d6bfb6c', + 'columnId': 'mycolumn', + 'userId': 'myuser', + 'type': 'tumblr', + 'title': 'Today on Tumblr', + 'options': { 'tumblr': 'https://todayontumblr.tumblr.com/rss' }, + 'link': 'https://todayontumblr.tumblr.com/', + }); + feedutils.assertEqualsItems(items, [{ + 'id': + 'tumblr-myuser-mycolumn-8ce77e730a91955cd991349d0d6bfb6c-cadfdb150b18480df6d781f845d0fec5', + 'userId': 'myuser', + 'columnId': 'mycolumn', + 'sourceId': 'tumblr-myuser-mycolumn-8ce77e730a91955cd991349d0d6bfb6c', + 'title': 'NEIL WON BEST ACTOR AT THE GAME AWARDS!!!!!!', + 'link': 'https://www.tumblr.com/todayontumblr/736339394445918208', + 'media': + 'https://64.media.tumblr.com/3721ad004e334d62e290e5062a296566/a5567169c27bf241-f9/s640x960/b8e88192aa512164126093c75681143fe851fead.jpg', + 'description': + '

argetcross :

NEIL WON BEST ACTOR AT THE GAME AWARDS!!!!!!

', + 'publishedAt': 1702227968, + }, { + 'id': + 'tumblr-myuser-mycolumn-8ce77e730a91955cd991349d0d6bfb6c-9cab913d4df84b6a40d7f90de40e6b24', + 'userId': 'myuser', + 'columnId': 'mycolumn', + 'sourceId': 'tumblr-myuser-mycolumn-8ce77e730a91955cd991349d0d6bfb6c', + 'title': + 'That ’s why they were rushing everyone ’s speeches cause Geoff wanted enough time to talk to his wife …', + 'link': 'https://www.tumblr.com/todayontumblr/736335427403907072', + 'description': + '

orallech :

That’s why they were rushing everyone’s speeches cause Geoff wanted enough time to talk to his wife kojima

', + 'publishedAt': 1702224185, + }, { + 'id': + 'tumblr-myuser-mycolumn-8ce77e730a91955cd991349d0d6bfb6c-8c7e748f46996222cfbf724b7748be62', + 'userId': 'myuser', + 'columnId': 'mycolumn', + 'sourceId': 'tumblr-myuser-mycolumn-8ce77e730a91955cd991349d0d6bfb6c', + 'title': 'Muppet Fact #925', + 'link': 'https://www.tumblr.com/todayontumblr/736331601085120512', + 'media': + 'https://64.media.tumblr.com/7338226115b70192b7b37dae667dc0b3/217c86a94d5179ce-1e/s640x960/de1345504db6ac2e012d6aef542489fbfd8a522b.png', + 'description': + '

muppet-facts :

At the 2023 Game Awards, Gonzo presented the award for Best Debut Indie Game with Geoff Keighley. He said he ’s been playing Tears of the Kingdom . He lost days following a Cucco up the hill. He also has a conspiracy that a lot of games released this year follow a chicken theme.

Source:

The Game Awards 2023. December 7, 2023.

', + 'publishedAt': 1702220536, + }, { + 'id': + 'tumblr-myuser-mycolumn-8ce77e730a91955cd991349d0d6bfb6c-eb92be444fd1f7418a53a146c4b6366c', + 'userId': 'myuser', + 'columnId': 'mycolumn', + 'sourceId': 'tumblr-myuser-mycolumn-8ce77e730a91955cd991349d0d6bfb6c', + 'title': 'Hardest image in gaming culture', + 'link': 'https://www.tumblr.com/todayontumblr/736327882138386432', + 'media': + 'https://64.media.tumblr.com/363392fc016d8c904fa3b86050b366a2/08a786e18a31fca4-84/s640x960/8ce0d3c4fa8efea0a196cb1921aa64cc43950f8d.png', + 'description': + '

temperedknight :

Hardest image in gaming culture

', + 'publishedAt': 1702216989, + }]); + } finally { + fetchWithTimeoutSpy.restore(); + } + + assertSpyCall(fetchWithTimeoutSpy, 0, { + args: [ + 'https://todayontumblr.tumblr.com/rss', + { method: 'get' }, + 5000, + ], + returned: new Promise((resolve) => { + resolve(new Response(response, { status: 200 })); + }), + }); + assertSpyCalls(fetchWithTimeoutSpy, 1); +}); diff --git a/supabase/functions/_shared/feed/utils/index.ts b/supabase/functions/_shared/feed/utils/index.ts new file mode 100644 index 0000000..053f5a0 --- /dev/null +++ b/supabase/functions/_shared/feed/utils/index.ts @@ -0,0 +1,12 @@ +import { getFavicon } from './getFavicon.ts'; +import { uploadSourceIcon } from './uploadFile.ts'; +import { assertEqualsItems, assertEqualsSource } from './test.ts'; + +export type { Favicon } from './getFavicon.ts'; + +export const feedutils = { + getFavicon, + uploadSourceIcon, + assertEqualsItems, + assertEqualsSource, +}; diff --git a/supabase/functions/_shared/feed/utils/test.ts b/supabase/functions/_shared/feed/utils/test.ts new file mode 100644 index 0000000..6c6df49 --- /dev/null +++ b/supabase/functions/_shared/feed/utils/test.ts @@ -0,0 +1,32 @@ +import { assertEquals } from 'std/assert'; + +import { ISource } from '../../models/source.ts'; +import { IItem } from '../../models/item.ts'; + +export const assertEqualsSource = (actual: ISource, expected: ISource) => { + assertEquals(actual.id, expected.id); + assertEquals(actual.columnId, expected.columnId); + assertEquals(actual.userId, expected.userId); + assertEquals(actual.type, expected.type); + assertEquals(actual.title, expected.title); + assertEquals(actual.options, expected.options); + assertEquals(actual.link, expected.link); + assertEquals(actual.icon, expected.icon); +}; + +export const assertEqualsItems = (actual: IItem[], expected: IItem[]) => { + assertEquals(actual.length, expected.length); + + for (let i = 0; i < actual.length; i++) { + assertEquals(actual[i].id, expected[i].id); + assertEquals(actual[i].columnId, expected[i].columnId); + assertEquals(actual[i].userId, expected[i].userId); + assertEquals(actual[i].sourceId, expected[i].sourceId); + assertEquals(actual[i].title, expected[i].title); + assertEquals(actual[i].link, expected[i].link); + assertEquals(actual[i].media, expected[i].media); + assertEquals(actual[i].description, expected[i].description); + assertEquals(actual[i].author, expected[i].author); + assertEquals(actual[i].options, expected[i].options); + } +}; diff --git a/supabase/functions/_shared/feed/youtube.ts b/supabase/functions/_shared/feed/youtube.ts index 23329b6..0573f47 100644 --- a/supabase/functions/_shared/feed/youtube.ts +++ b/supabase/functions/_shared/feed/youtube.ts @@ -7,11 +7,10 @@ import { unescape } from 'lodash'; import { IItem } from '../models/item.ts'; import { ISource } from '../models/source.ts'; -import { uploadSourceIcon } from './utils/uploadFile.ts'; +import { feedutils } from './utils/index.ts'; import { IProfile } from '../models/profile.ts'; -import { fetchWithTimeout } from '../utils/fetchWithTimeout.ts'; import { FEEDDECK_SOURCE_YOUTUBE_API_KEY } from '../utils/constants.ts'; -import { log } from '../utils/log.ts'; +import { utils } from '../utils/index.ts'; /** * `isYoutubeUrl` checks if the provided `url` is a valid Youtube url. A url is @@ -76,11 +75,11 @@ export const getYoutubeFeed = async ( * Get the RSS for the provided `youtube` url and parse it. If a feed doesn't * contains an item we return an error. */ - const response = await fetchWithTimeout(source.options.youtube, { + const response = await utils.fetchWithTimeout(source.options.youtube, { method: 'get', }, 5000); const xml = await response.text(); - log('debug', 'Add source', { + utils.log('debug', 'Add source', { sourceType: 'youtube', requestUrl: source.options.youtube, responseStatus: response.status, @@ -103,7 +102,7 @@ export const getYoutubeFeed = async ( '', ), ); - source.icon = await uploadSourceIcon(supabaseClient, source); + source.icon = await feedutils.uploadSourceIcon(supabaseClient, source); } /** @@ -263,7 +262,7 @@ const getMedia = (entry: FeedEntry): string | undefined => { */ const getChannelId = async (url: string): Promise => { try { - const response = await fetchWithTimeout(url, { method: 'get' }, 5000); + const response = await utils.fetchWithTimeout(url, { method: 'get' }, 5000); const html = await response.text(); const match = html.match( /"https:\/\/www.youtube.com\/feeds\/videos.xml\?channel_id\=(.*?)"/, @@ -288,14 +287,9 @@ const getChannelId = async (url: string): Promise => { const getChannelIcon = async ( channelId: string, ): Promise => { - const youtubeApiKey = FEEDDECK_SOURCE_YOUTUBE_API_KEY; - if (!youtubeApiKey) { - return undefined; - } - try { - const response = await fetchWithTimeout( - `https://www.googleapis.com/youtube/v3/channels?id=${channelId}&part=id%2Csnippet&maxResults=1&key=${youtubeApiKey}`, + const response = await utils.fetchWithTimeout( + `https://www.googleapis.com/youtube/v3/channels?id=${channelId}&part=id%2Csnippet&maxResults=1&key=${FEEDDECK_SOURCE_YOUTUBE_API_KEY}`, { method: 'get' }, 5000, ); diff --git a/supabase/functions/_shared/feed/youtube_test.ts b/supabase/functions/_shared/feed/youtube_test.ts new file mode 100644 index 0000000..cb46dd3 --- /dev/null +++ b/supabase/functions/_shared/feed/youtube_test.ts @@ -0,0 +1,364 @@ +import { createClient } from '@supabase/supabase-js'; +import { + assertSpyCall, + assertSpyCalls, + returnsNext, + stub, +} from 'std/testing/mock'; + +import { ISource } from '../models/source.ts'; +import { IProfile } from '../models/profile.ts'; +import { getYoutubeFeed } from './youtube.ts'; +import { utils } from '../utils/index.ts'; +import { feedutils } from './utils/index.ts'; +import { assertEqualsItems, assertEqualsSource } from './utils/test.ts'; + +const supabaseClient = createClient('http://localhost:54321', 'test123'); +const mockProfile: IProfile = { + id: '', + tier: 'free', + createdAt: 0, + updatedAt: 0, +}; +const mockSource: ISource = { + id: '', + columnId: 'mycolumn', + userId: 'myuser', + type: 'medium', + title: '', +}; + +const responseYoutubeChannelId = + `
InfoPresseUrheberrechtKontaktCreatorWerbenEntwicklerImpressumVerträge hier kündigenNutzungsbedingungenDatenschutzRichtlinien & SicherheitWie funktioniert YouTube?Neue Funktionen testentagesschau - YouTube