Use proxy for fetching link meta (#716)

* Use proxy for fetching link meta

* Remove link meta test due to hitting proxy

* setup different staging and prod proxy URLs

---------

Co-authored-by: Ansh Nanda <anshnanda10@gmail.com>
Co-authored-by: Paul Frazee <pfrazee@gmail.com>

__tests__/lib/link-meta.test.ts

@@ -1,106 +1,4 @@
-import {
+import {LikelyType, getLikelyType} from '../../src/lib/link-meta/link-meta'
-  LikelyType,
-  getLinkMeta,
-  getLikelyType,
-} from '../../src/lib/link-meta/link-meta'
-import {exampleComHtml} from './__mocks__/exampleComHtml'
-import {BskyAgent} from '@atproto/api'
-import {DEFAULT_SERVICE, RootStoreModel} from '../../src/state'
-
-describe('getLinkMeta', () => {
-  let rootStore: RootStoreModel
-
-  beforeEach(() => {
-    rootStore = new RootStoreModel(new BskyAgent({service: DEFAULT_SERVICE}))
-  })
-
-  const inputs = [
-    '',
-    'httpbadurl',
-    'https://example.com',
-    'https://example.com/index.html',
-    'https://example.com/image.png',
-    'https://example.com/video.avi',
-    'https://example.com/audio.ogg',
-    'https://example.com/text.txt',
-    'https://example.com/javascript.js',
-    'https://bsky.app/',
-    'https://bsky.app/index.html',
-  ]
-  const outputs = [
-    {
-      error: 'Invalid URL',
-      likelyType: LikelyType.Other,
-      url: '',
-    },
-    {
-      error: 'Invalid URL',
-      likelyType: LikelyType.Other,
-      url: 'httpbadurl',
-    },
-    {
-      likelyType: LikelyType.HTML,
-      url: 'https://example.com',
-      title: 'Example Domain',
-      description: 'An example website',
-    },
-    {
-      likelyType: LikelyType.HTML,
-      url: 'https://example.com/index.html',
-      title: 'Example Domain',
-      description: 'An example website',
-    },
-    {
-      likelyType: LikelyType.Image,
-      url: 'https://example.com/image.png',
-    },
-    {
-      likelyType: LikelyType.Video,
-      url: 'https://example.com/video.avi',
-    },
-    {
-      likelyType: LikelyType.Audio,
-      url: 'https://example.com/audio.ogg',
-    },
-    {
-      likelyType: LikelyType.Text,
-      url: 'https://example.com/text.txt',
-    },
-    {
-      likelyType: LikelyType.Other,
-      url: 'https://example.com/javascript.js',
-    },
-    {
-      likelyType: LikelyType.AtpData,
-      url: '/',
-    },
-    {
-      likelyType: LikelyType.AtpData,
-      url: '/index.html',
-    },
-    {
-      likelyType: LikelyType.Other,
-      url: '',
-      title: '',
-    },
-  ]
-  it('correctly handles a set of text inputs', async () => {
-    for (let i = 0; i < inputs.length; i++) {
-      global.fetch = jest.fn().mockImplementationOnce(() => {
-        return new Promise((resolve, _reject) => {
-          resolve({
-            ok: true,
-            status: 200,
-            text: () => exampleComHtml,
-          })
-        })
-      })
-      const input = inputs[i]
-      const output = await getLinkMeta(rootStore, input)
-      expect(output).toEqual(outputs[i])
-    }
-  })
-})
 
 describe('getLikelyType', () => {
   it('correctly handles non-parsed url', async () => {

package.json

@@ -85,7 +85,6 @@
     "expo-updates": "~0.16.4",
     "fast-text-encoding": "^1.0.6",
     "graphemer": "^1.4.0",
-    "he": "^1.2.0",
     "history": "^5.3.0",
     "js-sha256": "^0.9.0",
     "lande": "^1.0.10",

src/lib/constants.ts

@@ -144,3 +144,18 @@ export const POST_IMG_MAX = {
   height: 2000,
   size: 1000000,
 }
+
+export const STAGING_LINK_META_PROXY =
+  'https://cardyb.staging.bsky.dev/v1/extract?url='
+
+export const PROD_LINK_META_PROXY = 'https://cardyb.bsky.app/v1/extract?url='
+
+export function LINK_META_PROXY(serviceUrl: string) {
+  if (serviceUrl.includes('localhost')) {
+    return STAGING_LINK_META_PROXY
+  } else if (serviceUrl.includes('staging')) {
+    return STAGING_LINK_META_PROXY
+  } else {
+    return PROD_LINK_META_PROXY
+  }
+}

src/lib/link-meta/link-meta.ts

@@ -1,8 +1,7 @@
-import he from 'he'
 import {isBskyAppUrl} from '../strings/url-helpers'
 import {RootStoreModel} from 'state/index'
 import {extractBskyMeta} from './bsky'
-import {extractHtmlMeta} from './html'
+import {LINK_META_PROXY} from 'lib/constants'
 
 export enum LikelyType {
   HTML,
@@ -54,26 +53,29 @@ export async function getLinkMeta(
   try {
     const controller = new AbortController()
     const to = setTimeout(() => controller.abort(), timeout || 5e3)
-    const httpRes = await fetch(url, {
+
-      headers: {accept: 'text/html'},
+    const response = await fetch(
-      signal: controller.signal,
+      `${LINK_META_PROXY(
-    })
+        store.session.currentSession?.service || '',
-    const httpResBody = await httpRes.text()
+      )}${encodeURIComponent(url)}`,
+    )
+
+    const body = await response.json()
     clearTimeout(to)
-    const httpResMeta = extractHtmlMeta({
+
-      html: httpResBody,
+    const {description, error, image, title} = body
-      hostname: urlp?.hostname,
+
-      pathname: urlp?.pathname,
+    if (error !== '') {
-    })
+      throw new Error(error)
-    meta.title = httpResMeta.title ? he.decode(httpResMeta.title) : undefined
+    }
-    meta.description = httpResMeta.description
+
-      ? he.decode(httpResMeta.description)
+    meta.description = description
-      : undefined
+    meta.image = image
-    meta.image = httpResMeta.image
+    meta.title = title
   } catch (e) {
     // failed
     console.error(e)
-    meta.error = 'Failed to fetch link'
+    meta.error = e instanceof Error ? e.toString() : 'Failed to fetch link'
   }
 
   return meta