github-starred/feeddeck
2
0
Fork 0
mirror of https://github.com/feeddeck/feeddeck.git synced 2026-03-11 17:47:47 -05:00
Code Issues 5 Packages Projects Releases 8 Wiki Activity

[x] Remove X Source (#162)

Browse Source 
X (Twitter) was never supported and the code was never used to add a
source. We kept it in the hope that it will be possible again to use X
within the app. Since this isn't the case it is time to remove the code.
This commit is contained in:
Rico Berger
2024-04-06 13:00:12 +02:00
committed by GitHub
parent eb9f84f650
commit 4c34d33ac0
12 changed files with 3 additions and 570 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
supabase/functions/_shared/feed/feed.ts
View File

@@ -19,7 +19,6 @@ import { IProfile } from '../models/profile.ts';
import { getNitterFeed } from './nitter.ts';
import { getMastodonFeed } from './mastodon.ts';
import { getFourChanFeed, isFourChanUrl } from './fourchan.ts';
// import { getXFeed } from './x.ts';
/**
* `getFeed` returns a feed which consist of a source and a list of items for
@@ -194,8 +193,6 @@ export const getFeed = async (
source,
feedData,
);
// case 'x':
// return await getXFeed(supabaseClient, redisClient, profile, source, data);
case 'youtube':
return await getYoutubeFeed(
supabaseClient,

295
supabase/functions/_shared/feed/x.ts
View File

@@ -1,295 +0,0 @@
import { SupabaseClient } from '@supabase/supabase-js';
import { Redis } from 'redis';
import { unescape } from 'lodash';
import { IItem } from '../models/item.ts';
import { ISource } from '../models/source.ts';
import { feedutils } from './utils/index.ts';
import { IProfile } from '../models/profile.ts';
import { utils } from '../utils/index.ts';
export const getXFeed = async (
supabaseClient: SupabaseClient,
_redisClient: Redis | undefined,
_profile: IProfile,
source: ISource,
_feedData: string | undefined,
): Promise<{ source: ISource; items: IItem[] }> => {
if (!source.options?.x || source.options.x.length === 0) {
throw new feedutils.FeedValidationError('Invalid source options');
}
if (source.options.x[0] !== '@') {
throw new feedutils.FeedValidationError('Invalid source options');
}
/**
* Get the feed for the provided X username, based on the content of the HTML
* returned for the syndication.twitter.com url.
*/
const response = await utils.fetchWithTimeout(
generateFeedUrl(source.options.x),
{ method: 'get' },
5000,
);
const html = await response.text();
const matches = html.match(
/script id="__NEXT_DATA__" type="application\/json">([^>]*)<\/script>/,
);
if (!matches || matches.length !== 2) {
throw new Error('Invalid feed');
}
const feed = JSON.parse(matches[1]) as Feed;
/**
* Generate a source id based on the user id, column id and the normalized
* `twitter` options. Besides that we also set the source type to `twitter`
* and the link for the source. In opposite to the other sources we do not use
* the title of the feed as the title for the source, instead we are using the
* user input as title.
*/
if (source.id === '') {
source.id = await generateSourceId(
source.userId,
source.columnId,
source.options.x,
);
}
source.type = 'x';
source.title = source.options.x;
source.link = `https://twitter.com/${source.options.x.slice(1)}`;
/**
* When the source doesn't has an icon yet and the user requested the feed of
* a user (string starts with `@`) we try to get an icon for the source from
* the first item in the returned feed.
*/
if (
!source.icon && source.options.x[0] === '@' &&
feed.props.pageProps.timeline.entries.length > 0 &&
feed.props.pageProps.timeline.entries[0].content.tweet.user
.profile_image_url_https
) {
source.icon = feed.props.pageProps.timeline.entries[0].content.tweet.user
.profile_image_url_https;
source.icon = await feedutils.uploadSourceIcon(supabaseClient, source);
}
/**
* Now that the source does contain all the required information we can start
* to generate the items for the source, by looping over all the feed entries.
* We only add the first 50 items from the feed, because we only keep the
* latest 50 items for each source in our deletion logic.
*/
const items: IItem[] = [];
for (
const [index, entry] of feed.props.pageProps.timeline.entries.entries()
) {
if (index === 50) {
break;
}
const media = getMedia(entry);
items.push({
id: await generateItemId(source.id, entry.content.tweet.id_str),
userId: source.userId,
columnId: source.columnId,
sourceId: source.id,
title: '',
link: `https://twitter.com${entry.content.tweet.permalink}`,
description: unescape(entry.content.tweet.full_text),
author: entry.content.tweet.user.screen_name,
options: media && media.length > 0 ? { media: media } : undefined,
publishedAt: Math.floor(
new Date(entry.content.tweet.created_at).getTime() / 1000,
),
});
}
return { source, items };
};
/**
* `generateFeedUrl` returns the url to get the Tweets of the provided username.
* Since we check before that the input must start with an `@` we do not need to
* check if the input is a valid username.
*/
const generateFeedUrl = (input: string): string => {
return `https://syndication.twitter.com/srv/timeline-profile/screen-name/${
input.slice(1)
}?showReplies=true`;
};
/**
* `generateSourceId` generates a unique source id based on the user id, column
* id and the link of the RSS feed. We use the MD5 algorithm for the link to
* generate the id.
*/
const generateSourceId = async (
userId: string,
columnId: string,
link: string,
): Promise<string> => {
return `x-${userId}-${columnId}-${await utils.md5(link)}`;
};
/**
* `generateItemId` generates a unique item id based on the source id and the
* identifier of the item. We use the MD5 algorithm for the identifier, which
* can be the link of the item or the id of the item.
*/
const generateItemId = async (
sourceId: string,
identifier: string,
): Promise<string> => {
return `${sourceId}-${await utils.md5(identifier)}`;
};
/**
* `getMedia` returns an image for the provided feed entry. To get the images we
* have to check the `entities.media`, `retweeted_status.media` and
* `extended_entities.media` properties of the feed entry.
*/
const getMedia = (entry: Entry): string[] | undefined => {
if (
entry.content.tweet.retweeted_status?.extended_entities?.media &&
entry.content.tweet.retweeted_status.extended_entities.media.length > 0
) {
const images = [];
for (
const media of entry.content.tweet.retweeted_status.extended_entities
.media
) {
if (media.type === 'photo') {
images.push(media.media_url_https);
}
}
return images;
}
if (
entry.content.tweet.retweeted_status?.entities?.media &&
entry.content.tweet.retweeted_status?.entities.media.length > 0
) {
const images = [];
for (const media of entry.content.tweet.retweeted_status.entities.media) {
if (media.type === 'photo') {
images.push(media.media_url_https);
}
}
return images;
}
if (
entry.content.tweet.retweeted_status?.media &&
entry.content.tweet.retweeted_status.media.length > 0
) {
const images = [];
for (const media of entry.content.tweet.retweeted_status.media) {
if (media.type === 'photo') {
images.push(media.media_url_https);
}
}
return images;
}
if (
entry.content.tweet.extended_entities?.media &&
entry.content.tweet.extended_entities.media.length > 0
) {
const images = [];
for (const media of entry.content.tweet.extended_entities.media) {
if (media.type === 'photo') {
images.push(media.media_url_https);
}
}
return images;
}
if (
entry.content.tweet.entities.media &&
entry.content.tweet.entities.media.length > 0
) {
const images = [];
for (const media of entry.content.tweet.entities.media) {
if (media.type === 'photo') {
images.push(media.media_url_https);
}
}
return images;
}
return undefined;
};
/**
* `Feed` is the interface for the returned data from Twitter for a users
* timeline.
*/
export interface Feed {
props: {
pageProps: {
timeline: {
entries: Entry[];
};
};
};
}
export interface Entry {
content: {
tweet: {
created_at: string;
entities: {
media?: {
media_url_https: string;
type: string;
}[];
};
full_text: string;
id_str: string;
permalink: string;
user: {
screen_name: string;
profile_image_url_https: string;
};
retweeted_status?: {
entities?: {
media?: {
media_url_https: string;
type: string;
}[];
};
extended_entities?: {
media?: {
media_url_https: string;
type: string;
}[];
};
media?: {
media_url_https: string;
type: string;
}[];
};
extended_entities?: {
media?: {
media_url_https: string;
type: string;
}[];
};
};
};
}

2
supabase/functions/_shared/models/source.ts
View File

@@ -16,7 +16,6 @@ export type TSourceType =
| 'rss'
| 'stackoverflow'
| 'tumblr'
| 'x'
| 'youtube'
| 'none';
@@ -46,6 +45,5 @@ export interface ISourceOptions {
rss?: string;
stackoverflow?: ISourceOptionsStackOverflow;
tumblr?: string;
x?: string;
youtube?: string;
}