From c0ca27b7ce6518758323e2922e66a3e3a008dbce Mon Sep 17 00:00:00 2001 From: Ollie H Date: Tue, 30 May 2023 18:08:49 -0700 Subject: [PATCH] Use proxy for fetching link meta (#716) * Use proxy for fetching link meta * Remove link meta test due to hitting proxy * setup different staging and prod proxy URLs --------- Co-authored-by: Ansh Nanda Co-authored-by: Paul Frazee --- __tests__/lib/link-meta.test.ts | 104 +------------------------------- package.json | 1 - src/lib/constants.ts | 15 +++++ src/lib/link-meta/link-meta.ts | 38 ++++++------ 4 files changed, 36 insertions(+), 122 deletions(-) diff --git a/__tests__/lib/link-meta.test.ts b/__tests__/lib/link-meta.test.ts index f0ca7a9d..504b11c2 100644 --- a/__tests__/lib/link-meta.test.ts +++ b/__tests__/lib/link-meta.test.ts @@ -1,106 +1,4 @@ -import { - LikelyType, - getLinkMeta, - getLikelyType, -} from '../../src/lib/link-meta/link-meta' -import {exampleComHtml} from './__mocks__/exampleComHtml' -import {BskyAgent} from '@atproto/api' -import {DEFAULT_SERVICE, RootStoreModel} from '../../src/state' - -describe('getLinkMeta', () => { - let rootStore: RootStoreModel - - beforeEach(() => { - rootStore = new RootStoreModel(new BskyAgent({service: DEFAULT_SERVICE})) - }) - - const inputs = [ - '', - 'httpbadurl', - 'https://example.com', - 'https://example.com/index.html', - 'https://example.com/image.png', - 'https://example.com/video.avi', - 'https://example.com/audio.ogg', - 'https://example.com/text.txt', - 'https://example.com/javascript.js', - 'https://bsky.app/', - 'https://bsky.app/index.html', - ] - const outputs = [ - { - error: 'Invalid URL', - likelyType: LikelyType.Other, - url: '', - }, - { - error: 'Invalid URL', - likelyType: LikelyType.Other, - url: 'httpbadurl', - }, - { - likelyType: LikelyType.HTML, - url: 'https://example.com', - title: 'Example Domain', - description: 'An example website', - }, - { - likelyType: LikelyType.HTML, - url: 'https://example.com/index.html', - title: 'Example Domain', - description: 'An example website', - }, - { - likelyType: LikelyType.Image, - url: 'https://example.com/image.png', - }, - { - likelyType: LikelyType.Video, - url: 'https://example.com/video.avi', - }, - { - likelyType: LikelyType.Audio, - url: 'https://example.com/audio.ogg', - }, - { - likelyType: LikelyType.Text, - url: 'https://example.com/text.txt', - }, - { - likelyType: LikelyType.Other, - url: 'https://example.com/javascript.js', - }, - { - likelyType: LikelyType.AtpData, - url: '/', - }, - { - likelyType: LikelyType.AtpData, - url: '/index.html', - }, - { - likelyType: LikelyType.Other, - url: '', - title: '', - }, - ] - it('correctly handles a set of text inputs', async () => { - for (let i = 0; i < inputs.length; i++) { - global.fetch = jest.fn().mockImplementationOnce(() => { - return new Promise((resolve, _reject) => { - resolve({ - ok: true, - status: 200, - text: () => exampleComHtml, - }) - }) - }) - const input = inputs[i] - const output = await getLinkMeta(rootStore, input) - expect(output).toEqual(outputs[i]) - } - }) -}) +import {LikelyType, getLikelyType} from '../../src/lib/link-meta/link-meta' describe('getLikelyType', () => { it('correctly handles non-parsed url', async () => { diff --git a/package.json b/package.json index e8e7a9e5..576360cd 100644 --- a/package.json +++ b/package.json @@ -85,7 +85,6 @@ "expo-updates": "~0.16.4", "fast-text-encoding": "^1.0.6", "graphemer": "^1.4.0", - "he": "^1.2.0", "history": "^5.3.0", "js-sha256": "^0.9.0", "lande": "^1.0.10", diff --git a/src/lib/constants.ts b/src/lib/constants.ts index ada7b205..170fe640 100644 --- a/src/lib/constants.ts +++ b/src/lib/constants.ts @@ -144,3 +144,18 @@ export const POST_IMG_MAX = { height: 2000, size: 1000000, } + +export const STAGING_LINK_META_PROXY = + 'https://cardyb.staging.bsky.dev/v1/extract?url=' + +export const PROD_LINK_META_PROXY = 'https://cardyb.bsky.app/v1/extract?url=' + +export function LINK_META_PROXY(serviceUrl: string) { + if (serviceUrl.includes('localhost')) { + return STAGING_LINK_META_PROXY + } else if (serviceUrl.includes('staging')) { + return STAGING_LINK_META_PROXY + } else { + return PROD_LINK_META_PROXY + } +} diff --git a/src/lib/link-meta/link-meta.ts b/src/lib/link-meta/link-meta.ts index 6c4ad538..6863798b 100644 --- a/src/lib/link-meta/link-meta.ts +++ b/src/lib/link-meta/link-meta.ts @@ -1,8 +1,7 @@ -import he from 'he' import {isBskyAppUrl} from '../strings/url-helpers' import {RootStoreModel} from 'state/index' import {extractBskyMeta} from './bsky' -import {extractHtmlMeta} from './html' +import {LINK_META_PROXY} from 'lib/constants' export enum LikelyType { HTML, @@ -54,26 +53,29 @@ export async function getLinkMeta( try { const controller = new AbortController() const to = setTimeout(() => controller.abort(), timeout || 5e3) - const httpRes = await fetch(url, { - headers: {accept: 'text/html'}, - signal: controller.signal, - }) - const httpResBody = await httpRes.text() + + const response = await fetch( + `${LINK_META_PROXY( + store.session.currentSession?.service || '', + )}${encodeURIComponent(url)}`, + ) + + const body = await response.json() clearTimeout(to) - const httpResMeta = extractHtmlMeta({ - html: httpResBody, - hostname: urlp?.hostname, - pathname: urlp?.pathname, - }) - meta.title = httpResMeta.title ? he.decode(httpResMeta.title) : undefined - meta.description = httpResMeta.description - ? he.decode(httpResMeta.description) - : undefined - meta.image = httpResMeta.image + + const {description, error, image, title} = body + + if (error !== '') { + throw new Error(error) + } + + meta.description = description + meta.image = image + meta.title = title } catch (e) { // failed console.error(e) - meta.error = 'Failed to fetch link' + meta.error = e instanceof Error ? e.toString() : 'Failed to fetch link' } return meta