From ffbed7366933e2bd0f06dac61b9e95cd090ca75b Mon Sep 17 00:00:00 2001 From: Rico Berger Date: Sun, 26 Nov 2023 17:03:54 +0100 Subject: [PATCH] [medium] Remove Spam (#79) When using Medium as a source it is possible that the a lot of items might be spam when following a tag. To reduce the spam, we filter these items based on a word list and an calculated score. The score is calculated by the number of words which are included in the title from our word list. --- supabase/functions/_shared/feed/medium.ts | 35 +++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/supabase/functions/_shared/feed/medium.ts b/supabase/functions/_shared/feed/medium.ts index 8d90aa9..22fc874 100644 --- a/supabase/functions/_shared/feed/medium.ts +++ b/supabase/functions/_shared/feed/medium.ts @@ -197,6 +197,41 @@ const skipEntry = ( return true; } + /** + * Skip entries which might be spam. To detect possible spam, we check the + * title of the entry against a list of words, when the title contains 3 or + * more of these words we consider the entry as spam. + */ + const filterWords = [ + 'cash', + 'loan', + 'customer', + 'care', + 'helpline', + 'number', + 'patti', + 'toll', + 'free', + 'paisa', + 'call', + 'kup', + 'niewykrywalnych', + 'fałszywych', + 'pieniędzy', + 'whatsapp', + ]; + const title = entry.title.value.toLowerCase(); + let score = 0; + + for (const word of filterWords) { + if (title.includes(word)) { + score += 1; + } + } + if (score >= 3) { + return true; + } + return false; };