Bsky short link service (#4542)

* bskylink: scaffold service w/ initial config and schema

* bskylink: implement link creation and redirects

* bskylink: tidy

* bskylink: tests

* bskylink: tidy, add error handler

* bskylink: add dockerfile

* bskylink: add build

* bskylink: fix some express plumbing

* bskyweb: proxy fallthrough routes to link service redirects

* bskyweb: build w/ link proxy

* Add AASA to bskylink (#4588)

---------

Co-authored-by: Hailey <me@haileyok.com>

bskylink/tests/index.ts

@@ -0,0 +1,84 @@
+import assert from 'node:assert'
+import {AddressInfo} from 'node:net'
+import {after, before, describe, it} from 'node:test'
+
+import {Database, envToCfg, LinkService, readEnv} from '../src/index.js'
+
+describe('link service', async () => {
+  let linkService: LinkService
+  let baseUrl: string
+  before(async () => {
+    const env = readEnv()
+    const cfg = envToCfg({
+      ...env,
+      hostnames: ['test.bsky.link'],
+      appHostname: 'test.bsky.app',
+      dbPostgresSchema: 'link_test',
+      dbPostgresUrl: process.env.DB_POSTGRES_URL,
+    })
+    const migrateDb = Database.postgres({
+      url: cfg.db.url,
+      schema: cfg.db.schema,
+    })
+    await migrateDb.migrateToLatestOrThrow()
+    await migrateDb.close()
+    linkService = await LinkService.create(cfg)
+    await linkService.start()
+    const {port} = linkService.server?.address() as AddressInfo
+    baseUrl = `http://localhost:${port}`
+  })
+
+  after(async () => {
+    await linkService?.destroy()
+  })
+
+  it('creates a starter pack link', async () => {
+    const link = await getLink('/start/did:example:alice/xxx')
+    const url = new URL(link)
+    assert.strictEqual(url.origin, 'https://test.bsky.link')
+    assert.match(url.pathname, /^\/[a-z0-9]+$/i)
+  })
+
+  it('normalizes input paths and provides same link each time.', async () => {
+    const link1 = await getLink('/start/did%3Aexample%3Abob/yyy')
+    const link2 = await getLink('/start/did:example:bob/yyy/')
+    assert.strictEqual(link1, link2)
+  })
+
+  it('serves permanent redirect, preserving query params.', async () => {
+    const link = await getLink('/start/did:example:carol/zzz/')
+    const [status, location] = await getRedirect(`${link}?a=b`)
+    assert.strictEqual(status, 301)
+    const locationUrl = new URL(location)
+    assert.strictEqual(
+      locationUrl.pathname + locationUrl.search,
+      '/start/did:example:carol/zzz?a=b',
+    )
+  })
+
+  async function getRedirect(link: string): Promise<[number, string]> {
+    const url = new URL(link)
+    const base = new URL(baseUrl)
+    url.protocol = base.protocol
+    url.host = base.host
+    const res = await fetch(url, {redirect: 'manual'})
+    await res.arrayBuffer() // drain
+    assert(
+      res.status === 301 || res.status === 303,
+      'response was not a redirect',
+    )
+    return [res.status, res.headers.get('location') ?? '']
+  }
+
+  async function getLink(path: string): Promise<string> {
+    const res = await fetch(new URL('/link', baseUrl), {
+      method: 'post',
+      headers: {'content-type': 'application/json'},
+      body: JSON.stringify({path}),
+    })
+    assert.strictEqual(res.status, 200)
+    const payload = await res.json()
+    assert(typeof payload.url === 'string')
+    return payload.url
+  }
+})

bskylink/tests/infra/_common.sh

@@ -0,0 +1,157 @@
+#!/usr/bin/env sh
+
+# Exit if any command fails
+set -e
+
+get_container_id() {
+  local compose_file=$1
+  local service=$2
+  if [ -z "${compose_file}" ] || [ -z "${service}" ]; then
+    echo "usage: get_container_id <compose_file> <service>"
+    exit 1
+  fi
+
+ # first line of jq normalizes for docker compose breaking change, see docker/compose#10958
+  docker compose --file $compose_file ps --format json --status running \
+    | jq -sc '.[] | if type=="array" then .[] else . end' | jq -s \
+    | jq -r '.[]? | select(.Service == "'${service}'") | .ID'
+}
+
+# Exports all environment variables
+export_env() {
+  export_pg_env
+}
+
+# Exports postgres environment variables
+export_pg_env() {
+  # Based on creds in compose.yaml
+  export PGPORT=5433
+  export PGHOST=localhost
+  export PGUSER=pg
+  export PGPASSWORD=password
+  export PGDATABASE=postgres
+  export DB_POSTGRES_URL="postgresql://pg:password@127.0.0.1:5433/postgres"
+}
+
+
+pg_clear() {
+  local pg_uri=$1
+
+  for schema_name in `psql "${pg_uri}" -c "SELECT schema_name FROM information_schema.schemata WHERE schema_name NOT LIKE 'pg_%' AND schema_name NOT LIKE 'information_schema';" -t`; do
+    psql "${pg_uri}" -c "DROP SCHEMA \"${schema_name}\" CASCADE;"
+  done
+}
+
+pg_init() {
+  local pg_uri=$1
+
+  psql "${pg_uri}" -c "CREATE SCHEMA IF NOT EXISTS \"public\";"
+}
+
+main_native() {
+  local services=${SERVICES}
+  local postgres_url_env_var=`[[ $services == *"db_test"* ]] && echo "DB_TEST_POSTGRES_URL" || echo "DB_POSTGRES_URL"`
+
+  postgres_url="${!postgres_url_env_var}"
+
+  if [ -n "${postgres_url}" ]; then
+    echo "Using ${postgres_url_env_var} (${postgres_url}) to connect to postgres."
+    pg_init "${postgres_url}"
+  else
+    echo "Postgres connection string missing did you set ${postgres_url_env_var}?"
+    exit 1
+  fi
+
+  cleanup() {
+    local services=$@
+
+    if [ -n "${postgres_url}" ] && [[ $services == *"db_test"* ]]; then
+      pg_clear "${postgres_url}" &> /dev/null
+    fi
+  }
+
+  # trap SIGINT and performs cleanup
+  trap "on_sigint ${services}" INT
+  on_sigint() {
+    cleanup $@
+    exit $?
+  }
+
+  # Run the arguments as a command
+  DB_POSTGRES_URL="${postgres_url}" \
+  "$@"
+  code=$?
+
+  cleanup ${services}
+
+  exit ${code}
+}
+
+main_docker() {
+  # Expect a SERVICES env var to be set with the docker service names
+  local services=${SERVICES}
+
+  dir=$(dirname $0)
+  compose_file="${dir}/docker-compose.yaml"
+
+  # whether this particular script started the container(s)
+  started_container=false
+
+  # performs cleanup as necessary, i.e. taking down containers
+  # if this script started them
+  cleanup() {
+    local services=$@
+    echo # newline
+    if $started_container; then
+      docker compose --file $compose_file rm --force --stop --volumes ${services}
+    fi
+  }
+
+  # trap SIGINT and performs cleanup
+  trap "on_sigint ${services}" INT
+  on_sigint() {
+    cleanup $@
+    exit $?
+  }
+
+  # check if all services are running already
+  not_running=false
+  for service in $services; do
+    container_id=$(get_container_id $compose_file $service)
+    if [ -z $container_id ]; then
+      not_running=true
+      break
+    fi
+  done
+
+  # if any are missing, recreate all services
+  if $not_running; then
+    started_container=true
+    docker compose --file $compose_file up --wait --force-recreate ${services}
+  else
+    echo "all services ${services} are already running"
+  fi
+
+  # do not exit when following commands fail, so we can intercept exit code & tear down docker
+  set +e
+
+  # setup environment variables and run args
+  export_env
+  "$@"
+  # save return code for later
+  code=$?
+
+  # performs cleanup as necessary
+  cleanup ${services}
+  exit ${code}
+}
+
+# Main entry point
+main() {
+  if ! docker ps >/dev/null 2>&1; then
+    echo "Docker unavailable. Running on host."
+    main_native $@
+  else
+    main_docker $@
+  fi
+}

bskylink/tests/infra/docker-compose.yaml

@@ -0,0 +1,27 @@
+version: '3.8'
+services:
+  # An ephermerally-stored postgres database for single-use test runs
+  db_test: &db_test
+    image: postgres:14.11-alpine
+    environment:
+      - POSTGRES_USER=pg
+      - POSTGRES_PASSWORD=password
+    ports:
+      - '5433:5432'
+    # Healthcheck ensures db is queryable when `docker-compose up --wait` completes
+    healthcheck:
+      test: 'pg_isready -U pg'
+      interval: 500ms
+      timeout: 10s
+      retries: 20
+  # A persistently-stored postgres database
+  db:
+    <<: *db_test
+    ports:
+      - '5432:5432'
+    healthcheck:
+      disable: true
+    volumes:
+      - link_db:/var/lib/postgresql/data
+volumes:
+  link_db:

bskylink/tests/infra/with-test-db.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env sh
+
+# Example usage:
+# ./with-test-db.sh psql postgresql://pg:password@localhost:5433/postgres -c 'select 1;'
+
+dir=$(dirname $0)
+. ${dir}/_common.sh
+
+SERVICES="db_test" main "$@"