From 58842d03a95af014cb44c3495d109e3bb6731fde Mon Sep 17 00:00:00 2001 From: bnewbold Date: Sat, 13 Apr 2024 12:20:06 -0700 Subject: [PATCH] rebased embedr (#3511) * skeleton of embedr service, based on bskyweb * embedr container setup * builds on this branch * actual routes * fix embedr go:embed * tweak embedr dockerfile * progress on embedr * fix path params * tweaks to build process * try to get embedr dockerfile to install embed deps * build this branch * updates to match sam's output HTML * try to unbreak embedr dockerfile * small embedr tweak * docker hack * get embed.js copied over to embedr * don't x-frame-options for embed.bsky.app * bskyembed: remove a console.log * use html/template for golang snippet generation * simplify embedr API fetches * missing file * Rm console.log fully --------- Co-authored-by: Dan Abramov --- .../workflows/build-and-push-embedr-aws.yaml | 57 +++++ Dockerfile.embedr | 78 ++++++ Makefile | 6 + bskyembed/src/screens/post.tsx | 3 - bskyweb/.gitignore | 6 + bskyweb/Makefile | 5 + bskyweb/README.embed.md | 52 ++++ bskyweb/cmd/embedr/.gitignore | 1 + bskyweb/cmd/embedr/handlers.go | 207 +++++++++++++++ bskyweb/cmd/embedr/main.go | 60 +++++ bskyweb/cmd/embedr/render.go | 16 ++ bskyweb/cmd/embedr/server.go | 236 ++++++++++++++++++ bskyweb/cmd/embedr/snippet.go | 71 ++++++ .../embedr-static/.well-known/security.txt | 4 + bskyweb/embedr-static/embed.js | 1 + bskyweb/embedr-static/favicon-16x16.png | Bin 0 -> 1731 bytes bskyweb/embedr-static/favicon-32x32.png | Bin 0 -> 2240 bytes bskyweb/embedr-static/favicon.png | Bin 0 -> 1412 bytes bskyweb/embedr-static/iframe-resize.js | 1 + bskyweb/embedr-static/ips-v4 | 30 +++ bskyweb/embedr-static/ips-v6 | 0 bskyweb/embedr-static/robots.txt | 9 + bskyweb/embedr-templates/error.html | 1 + bskyweb/embedr-templates/home.html | 8 + bskyweb/embedr-templates/oembed.html | 1 + bskyweb/embedr-templates/postEmbed.html | 1 + bskyweb/static.go | 3 + bskyweb/templates.go | 3 + package.json | 2 +- scripts/post-embed-build.js | 92 ++++--- 30 files changed, 912 insertions(+), 42 deletions(-) create mode 100644 .github/workflows/build-and-push-embedr-aws.yaml create mode 100644 Dockerfile.embedr create mode 100644 bskyweb/README.embed.md create mode 100644 bskyweb/cmd/embedr/.gitignore create mode 100644 bskyweb/cmd/embedr/handlers.go create mode 100644 bskyweb/cmd/embedr/main.go create mode 100644 bskyweb/cmd/embedr/render.go create mode 100644 bskyweb/cmd/embedr/server.go create mode 100644 bskyweb/cmd/embedr/snippet.go create mode 100644 bskyweb/embedr-static/.well-known/security.txt create mode 100644 bskyweb/embedr-static/embed.js create mode 100644 bskyweb/embedr-static/favicon-16x16.png create mode 100644 bskyweb/embedr-static/favicon-32x32.png create mode 100644 bskyweb/embedr-static/favicon.png create mode 100644 bskyweb/embedr-static/iframe-resize.js create mode 100644 bskyweb/embedr-static/ips-v4 create mode 100644 bskyweb/embedr-static/ips-v6 create mode 100644 bskyweb/embedr-static/robots.txt create mode 100644 bskyweb/embedr-templates/error.html create mode 100644 bskyweb/embedr-templates/home.html create mode 100644 bskyweb/embedr-templates/oembed.html create mode 100644 bskyweb/embedr-templates/postEmbed.html diff --git a/.github/workflows/build-and-push-embedr-aws.yaml b/.github/workflows/build-and-push-embedr-aws.yaml new file mode 100644 index 00000000..f7f24af9 --- /dev/null +++ b/.github/workflows/build-and-push-embedr-aws.yaml @@ -0,0 +1,57 @@ +name: build-and-push-embedr-aws +on: + push: + branches: + - main + - bnewbold/embedr + - bnewbold/embedr-rebase + +env: + REGISTRY: ${{ secrets.AWS_ECR_REGISTRY_USEAST2_PACKAGES_REGISTRY }} + USERNAME: ${{ secrets.AWS_ECR_REGISTRY_USEAST2_PACKAGES_USERNAME }} + PASSWORD: ${{ secrets.AWS_ECR_REGISTRY_USEAST2_PACKAGES_PASSWORD }} + IMAGE_NAME: embed + +jobs: + embedr-container-aws: + if: github.repository == 'bluesky-social/social-app' + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + id-token: write + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Setup Docker buildx + uses: docker/setup-buildx-action@v1 + + - name: Log into registry ${{ env.REGISTRY }} + uses: docker/login-action@v2 + with: + registry: ${{ env.REGISTRY }} + username: ${{ env.USERNAME}} + password: ${{ env.PASSWORD }} + + - name: Extract Docker metadata + id: meta + uses: docker/metadata-action@v4 + with: + images: | + ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + type=sha,enable=true,priority=100,prefix=,suffix=,format=long + + - name: Build and push Docker image + id: build-and-push + uses: docker/build-push-action@v4 + with: + context: . + push: ${{ github.event_name != 'pull_request' }} + file: ./Dockerfile.embedr + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/Dockerfile.embedr b/Dockerfile.embedr new file mode 100644 index 00000000..c7025165 --- /dev/null +++ b/Dockerfile.embedr @@ -0,0 +1,78 @@ +FROM golang:1.21-bullseye AS build-env + +WORKDIR /usr/src/social-app + +ENV DEBIAN_FRONTEND=noninteractive + +# Node +ENV NODE_VERSION=18 +ENV NVM_DIR=/usr/share/nvm + +# Go +ENV GODEBUG="netdns=go" +ENV GOOS="linux" +ENV GOARCH="amd64" +ENV CGO_ENABLED=1 +ENV GOEXPERIMENT="loopvar" + +COPY . . + +# +# Generate the JavaScript webpack. NOTE: this will change +# +RUN mkdir --parents $NVM_DIR && \ + wget \ + --output-document=/tmp/nvm-install.sh \ + https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.7/install.sh && \ + bash /tmp/nvm-install.sh + +RUN \. "$NVM_DIR/nvm.sh" && \ + nvm install $NODE_VERSION && \ + nvm use $NODE_VERSION && \ + npm install --global yarn && \ + yarn && \ + cd bskyembed && yarn install --frozen-lockfile && cd .. && \ + yarn intl:build && \ + yarn build-embed + +# DEBUG +RUN find ./bskyweb/embedr-static && find ./bskyweb/embedr-templates && find ./bskyembed/dist + +# hack around issue with empty directory and go:embed +RUN touch bskyweb/static/js/empty.txt + +# +# Generate the embedr Go binary. +# +RUN cd bskyweb/ && \ + go mod download && \ + go mod verify + +RUN cd bskyweb/ && \ + go build \ + -v \ + -trimpath \ + -tags timetzdata \ + -o /embedr \ + ./cmd/embedr + +FROM debian:bullseye-slim + +ENV GODEBUG=netdns=go +ENV TZ=Etc/UTC +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install --yes \ + dumb-init \ + ca-certificates + +ENTRYPOINT ["dumb-init", "--"] + +WORKDIR /embedr +COPY --from=build-env /embedr /usr/bin/embedr + +CMD ["/usr/bin/embedr"] + +LABEL org.opencontainers.image.source=https://github.com/bluesky-social/social-app +LABEL org.opencontainers.image.description="embed.bsky.app Web App" +LABEL org.opencontainers.image.licenses=MIT diff --git a/Makefile b/Makefile index c90abb78..a40d3761 100644 --- a/Makefile +++ b/Makefile @@ -13,6 +13,11 @@ build-web: ## Compile web bundle, copy to bskyweb directory yarn intl:build yarn build-web +.PHONY: build-web-embed +build-web-embed: ## Compile web embed bundle, copy to bskyweb/embedr* directories + yarn intl:build + yarn build-embed + .PHONY: test test: ## Run all tests NODE_ENV=test EXPO_PUBLIC_ENV=test yarn test @@ -28,6 +33,7 @@ lint: ## Run style checks and verify syntax .PHONY: deps deps: ## Installs dependent libs using 'yarn install' yarn install --frozen-lockfile + cd bskyembed && yarn install --frozen-lockfile .PHONY: nvm-setup nvm-setup: ## Use NVM to install and activate node+yarn diff --git a/bskyembed/src/screens/post.tsx b/bskyembed/src/screens/post.tsx index 76c92154..365227cd 100644 --- a/bskyembed/src/screens/post.tsx +++ b/bskyembed/src/screens/post.tsx @@ -17,9 +17,6 @@ const agent = new BskyAgent({ }) const uri = `at://${window.location.pathname.slice('/embed/'.length)}` - -console.log(uri) - if (!uri) { throw new Error('No uri in path') } diff --git a/bskyweb/.gitignore b/bskyweb/.gitignore index ace9fbf5..fad122a2 100644 --- a/bskyweb/.gitignore +++ b/bskyweb/.gitignore @@ -3,16 +3,22 @@ test-coverage.out # Don't check in the binary. /bskyweb +/embedr # Don't accidentally commit JS-generated code static/js/*.js static/js/*.map static/js/*.js.LICENSE.txt +static/js/empty.txt templates/scripts.html templates/*-embed.html static/embed/*.html static/embed/assets/*.js static/embed/assets/*.css +embedr-static/post-*.js +embedr-static/post-*.css +embedr-static/index-*.js +embedr-static/polyfills-*.js # Don't ignore this file !.gitignore diff --git a/bskyweb/Makefile b/bskyweb/Makefile index 6f979fa8..bb2da525 100644 --- a/bskyweb/Makefile +++ b/bskyweb/Makefile @@ -14,6 +14,7 @@ help: ## Print info about all commands .PHONY: build build: ## Build all executables go build ./cmd/bskyweb + go build ./cmd/embedr .PHONY: test test: ## Run all tests @@ -43,3 +44,7 @@ check: ## Compile everything, checking syntax (does not output binaries) .PHONY: run-dev-bskyweb run-dev-bskyweb: .env ## Runs 'bskyweb' for local dev GOLOG_LOG_LEVEL=info go run ./cmd/bskyweb serve + +.PHONY: run-dev-embedr +run-dev-embedr: .env ## Runs 'embedr' for local dev + GOLOG_LOG_LEVEL=info go run ./cmd/embedr serve diff --git a/bskyweb/README.embed.md b/bskyweb/README.embed.md new file mode 100644 index 00000000..8f19ef02 --- /dev/null +++ b/bskyweb/README.embed.md @@ -0,0 +1,52 @@ + +## oEmbed + + + +* URL scheme: `https://bsky.app/profile/*/post/*` +* API endpoint: `https://embed.bsky.app/oembed` + +Request params: + +- `url` (required): support both AT-URI and bsky.app URL +- `maxwidth` (optional): [220..550], 325 is default +- `maxheight` (not supported!) +- `format` (optional): only `json` supported + +Response format: + +- `type` (required): "rich" +- `version` (required): "1.0" +- `author_name` (optional): display name +- `author_url` (optional): profile URL +- `provider_name` (optional): "Bluesky Social" +- `provider_url` (optional): "https://bsky.app" +- `cache_age` (optional, integer seconds): 86400 (24 hours) (?) +- `width` (required): ? +- `height` (required): ? + +Not used: + +- title (optional): A text title, describing the resource. +- thumbnail_url (optional): A URL to a thumbnail image representing the resource. The thumbnail must respect any maxwidth and maxheight parameters. If this parameter is present, thumbnail_width and thumbnail_height must also be present. +- thumbnail_width (optional): The width of the optional thumbnail. If this parameter is present, thumbnail_url and thumbnail_height must also be present. +- thumbnail_height (optional): The height of the optional thumbnail. If this parameter is present, thumbnail_url and thumbnail_width must also be present. + +Only `json` is supported; `xml` is a 501. + +``` + +``` + + +## iframe URL + +`https://embed.bsky.app/embed//app.bsky.feed.post/` +`https://embed.bsky.app/static/embed.js` + +``` +
+

{{ post-text }}

+ — US Department of the Interior (@Interior) May 5, 2014 +
+``` diff --git a/bskyweb/cmd/embedr/.gitignore b/bskyweb/cmd/embedr/.gitignore new file mode 100644 index 00000000..c810652a --- /dev/null +++ b/bskyweb/cmd/embedr/.gitignore @@ -0,0 +1 @@ +/bskyweb diff --git a/bskyweb/cmd/embedr/handlers.go b/bskyweb/cmd/embedr/handlers.go new file mode 100644 index 00000000..2ab72be4 --- /dev/null +++ b/bskyweb/cmd/embedr/handlers.go @@ -0,0 +1,207 @@ +package main + +import ( + "context" + "errors" + "fmt" + "net/http" + "net/url" + "strconv" + "strings" + + appbsky "github.com/bluesky-social/indigo/api/bsky" + "github.com/bluesky-social/indigo/atproto/syntax" + + "github.com/labstack/echo/v4" +) + +var ErrPostNotFound = errors.New("post not found") +var ErrPostNotPublic = errors.New("post is not publicly accessible") + +func (srv *Server) getBlueskyPost(ctx context.Context, did syntax.DID, rkey syntax.RecordKey) (*appbsky.FeedDefs_PostView, error) { + + // fetch the post post (with extra context) + uri := fmt.Sprintf("at://%s/app.bsky.feed.post/%s", did, rkey) + tpv, err := appbsky.FeedGetPostThread(ctx, srv.xrpcc, 1, 0, uri) + if err != nil { + log.Warnf("failed to fetch post: %s\t%v", uri, err) + // TODO: detect 404, specifically? + return nil, ErrPostNotFound + } + + if tpv.Thread.FeedDefs_BlockedPost != nil { + return nil, ErrPostNotPublic + } else if tpv.Thread.FeedDefs_ThreadViewPost.Post == nil { + return nil, ErrPostNotFound + } + + postView := tpv.Thread.FeedDefs_ThreadViewPost.Post + for _, label := range postView.Author.Labels { + if label.Src == postView.Author.Did && label.Val == "!no-unauthenticated" { + return nil, ErrPostNotPublic + } + } + return postView, nil +} + +func (srv *Server) WebHome(c echo.Context) error { + return c.Render(http.StatusOK, "home.html", nil) +} + +type OEmbedResponse struct { + Type string `json:"type"` + Version string `json:"version"` + AuthorName string `json:"author_name,omitempty"` + AuthorURL string `json:"author_url,omitempty"` + ProviderName string `json:"provider_url,omitempty"` + CacheAge int `json:"cache_age,omitempty"` + Width int `json:"width,omitempty"` + Height *int `json:"height,omitempty"` + HTML string `json:"html,omitempty"` +} + +func (srv *Server) parseBlueskyURL(ctx context.Context, raw string) (*syntax.ATURI, error) { + + if raw == "" { + return nil, fmt.Errorf("empty url") + } + + // first try simple AT-URI + uri, err := syntax.ParseATURI(raw) + if nil == err { + return &uri, nil + } + + // then try bsky.app post URL + u, err := url.Parse(raw) + if err != nil { + return nil, err + } + if u.Hostname() != "bsky.app" { + return nil, fmt.Errorf("only bsky.app URLs currently supported") + } + pathParts := strings.Split(u.Path, "/") // NOTE: pathParts[0] will be empty string + if len(pathParts) != 5 || pathParts[1] != "profile" || pathParts[3] != "post" { + return nil, fmt.Errorf("only bsky.app post URLs currently supported") + } + atid, err := syntax.ParseAtIdentifier(pathParts[2]) + if err != nil { + return nil, err + } + rkey, err := syntax.ParseRecordKey(pathParts[4]) + if err != nil { + return nil, err + } + var did syntax.DID + if atid.IsHandle() { + ident, err := srv.dir.Lookup(ctx, *atid) + if err != nil { + return nil, err + } + did = ident.DID + } else { + did, err = atid.AsDID() + if err != nil { + return nil, err + } + } + + // TODO: don't really need to re-parse here, if we had test coverage + aturi, err := syntax.ParseATURI(fmt.Sprintf("at://%s/app.bsky.feed.post/%s", did, rkey)) + if err != nil { + return nil, err + } else { + return &aturi, nil + } +} + +func (srv *Server) WebOEmbed(c echo.Context) error { + formatParam := c.QueryParam("format") + if formatParam != "" && formatParam != "json" { + return c.String(http.StatusNotImplemented, "Unsupported oEmbed format: "+formatParam) + } + + // TODO: do we actually do something with width? + width := 550 + maxWidthParam := c.QueryParam("maxwidth") + if maxWidthParam != "" { + maxWidthInt, err := strconv.Atoi(maxWidthParam) + if err != nil || maxWidthInt < 220 || maxWidthInt > 550 { + return c.String(http.StatusBadRequest, "Invalid maxwidth (expected integer between 220 and 550)") + } + width = maxWidthInt + } + // NOTE: maxheight ignored + + aturi, err := srv.parseBlueskyURL(c.Request().Context(), c.QueryParam("url")) + if err != nil { + return c.String(http.StatusBadRequest, fmt.Sprintf("Expected 'url' to be bsky.app URL or AT-URI: %v", err)) + } + if aturi.Collection() != syntax.NSID("app.bsky.feed.post") { + return c.String(http.StatusNotImplemented, "Only posts (app.bsky.feed.post records) can be embedded currently") + } + did, err := aturi.Authority().AsDID() + if err != nil { + return err + } + + post, err := srv.getBlueskyPost(c.Request().Context(), did, aturi.RecordKey()) + if err == ErrPostNotFound { + return c.String(http.StatusNotFound, fmt.Sprintf("%v", err)) + } else if err == ErrPostNotPublic { + return c.String(http.StatusForbidden, fmt.Sprintf("%v", err)) + } else if err != nil { + return c.String(http.StatusInternalServerError, fmt.Sprintf("%v", err)) + } + + html, err := srv.postEmbedHTML(post) + if err != nil { + return c.String(http.StatusInternalServerError, fmt.Sprintf("%v", err)) + } + data := OEmbedResponse{ + Type: "rich", + Version: "1.0", + AuthorName: "@" + post.Author.Handle, + AuthorURL: fmt.Sprintf("https://bsky.app/profile/%s", post.Author.Handle), + ProviderName: "Bluesky Social", + CacheAge: 86400, + Width: width, + Height: nil, + HTML: html, + } + if post.Author.DisplayName != nil { + data.AuthorName = fmt.Sprintf("%s (@%s)", *post.Author.DisplayName, post.Author.Handle) + } + return c.JSON(http.StatusOK, data) +} + +func (srv *Server) WebPostEmbed(c echo.Context) error { + + // sanity check arguments. don't 4xx, just let app handle if not expected format + rkeyParam := c.Param("rkey") + rkey, err := syntax.ParseRecordKey(rkeyParam) + if err != nil { + return c.String(http.StatusBadRequest, fmt.Sprintf("Invalid RecordKey: %v", err)) + } + didParam := c.Param("did") + did, err := syntax.ParseDID(didParam) + if err != nil { + return c.String(http.StatusBadRequest, fmt.Sprintf("Invalid DID: %v", err)) + } + _ = rkey + _ = did + + // NOTE: this request was't really necessary; the JS will do the same fetch + /* + postView, err := srv.getBlueskyPost(ctx, did, rkey) + if err == ErrPostNotFound { + return c.String(http.StatusNotFound, fmt.Sprintf("%v", err)) + } else if err == ErrPostNotPublic { + return c.String(http.StatusForbidden, fmt.Sprintf("%v", err)) + } else if err != nil { + return c.String(http.StatusInternalServerError, fmt.Sprintf("%v", err)) + } + */ + + return c.Render(http.StatusOK, "postEmbed.html", nil) +} diff --git a/bskyweb/cmd/embedr/main.go b/bskyweb/cmd/embedr/main.go new file mode 100644 index 00000000..9f75ed69 --- /dev/null +++ b/bskyweb/cmd/embedr/main.go @@ -0,0 +1,60 @@ +package main + +import ( + "os" + + _ "github.com/joho/godotenv/autoload" + + logging "github.com/ipfs/go-log" + "github.com/urfave/cli/v2" +) + +var log = logging.Logger("embedr") + +func init() { + logging.SetAllLoggers(logging.LevelDebug) + //logging.SetAllLoggers(logging.LevelWarn) +} + +func main() { + run(os.Args) +} + +func run(args []string) { + + app := cli.App{ + Name: "embedr", + Usage: "web server for embed.bsky.app post embeds", + } + + app.Commands = []*cli.Command{ + &cli.Command{ + Name: "serve", + Usage: "run the server", + Action: serve, + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "appview-host", + Usage: "method, hostname, and port of PDS instance", + Value: "https://public.api.bsky.app", + EnvVars: []string{"ATP_APPVIEW_HOST"}, + }, + &cli.StringFlag{ + Name: "http-address", + Usage: "Specify the local IP/port to bind to", + Required: false, + Value: ":8100", + EnvVars: []string{"HTTP_ADDRESS"}, + }, + &cli.BoolFlag{ + Name: "debug", + Usage: "Enable debug mode", + Value: false, + Required: false, + EnvVars: []string{"DEBUG"}, + }, + }, + }, + } + app.RunAndExitOnError() +} diff --git a/bskyweb/cmd/embedr/render.go b/bskyweb/cmd/embedr/render.go new file mode 100644 index 00000000..cc8f0759 --- /dev/null +++ b/bskyweb/cmd/embedr/render.go @@ -0,0 +1,16 @@ +package main + +import ( + "html/template" + "io" + + "github.com/labstack/echo/v4" +) + +type Template struct { + templates *template.Template +} + +func (t *Template) Render(w io.Writer, name string, data interface{}, c echo.Context) error { + return t.templates.ExecuteTemplate(w, name, data) +} diff --git a/bskyweb/cmd/embedr/server.go b/bskyweb/cmd/embedr/server.go new file mode 100644 index 00000000..904b4df9 --- /dev/null +++ b/bskyweb/cmd/embedr/server.go @@ -0,0 +1,236 @@ +package main + +import ( + "context" + "errors" + "fmt" + "html/template" + "io/fs" + "net/http" + "os" + "os/signal" + "strings" + "syscall" + "time" + + "github.com/bluesky-social/indigo/atproto/identity" + "github.com/bluesky-social/indigo/util/cliutil" + "github.com/bluesky-social/indigo/xrpc" + "github.com/bluesky-social/social-app/bskyweb" + + "github.com/klauspost/compress/gzhttp" + "github.com/klauspost/compress/gzip" + "github.com/labstack/echo/v4" + "github.com/labstack/echo/v4/middleware" + "github.com/urfave/cli/v2" +) + +type Server struct { + echo *echo.Echo + httpd *http.Server + xrpcc *xrpc.Client + dir identity.Directory +} + +func serve(cctx *cli.Context) error { + debug := cctx.Bool("debug") + httpAddress := cctx.String("http-address") + appviewHost := cctx.String("appview-host") + + // Echo + e := echo.New() + + // create a new session (no auth) + xrpcc := &xrpc.Client{ + Client: cliutil.NewHttpClient(), + Host: appviewHost, + } + + // httpd + var ( + httpTimeout = 2 * time.Minute + httpMaxHeaderBytes = 2 * (1024 * 1024) + gzipMinSizeBytes = 1024 * 2 + gzipCompressionLevel = gzip.BestSpeed + gzipExceptMIMETypes = []string{"image/png"} + ) + + // Wrap the server handler in a gzip handler to compress larger responses. + gzipHandler, err := gzhttp.NewWrapper( + gzhttp.MinSize(gzipMinSizeBytes), + gzhttp.CompressionLevel(gzipCompressionLevel), + gzhttp.ExceptContentTypes(gzipExceptMIMETypes), + ) + if err != nil { + return err + } + + // + // server + // + server := &Server{ + echo: e, + xrpcc: xrpcc, + dir: identity.DefaultDirectory(), + } + + // Create the HTTP server. + server.httpd = &http.Server{ + Handler: gzipHandler(server), + Addr: httpAddress, + WriteTimeout: httpTimeout, + ReadTimeout: httpTimeout, + MaxHeaderBytes: httpMaxHeaderBytes, + } + + e.HideBanner = true + + tmpl := &Template{ + templates: template.Must(template.ParseFS(bskyweb.EmbedrTemplateFS, "embedr-templates/*.html")), + } + e.Renderer = tmpl + e.HTTPErrorHandler = server.errorHandler + + e.IPExtractor = echo.ExtractIPFromXFFHeader() + + // SECURITY: Do not modify without due consideration. + e.Use(middleware.SecureWithConfig(middleware.SecureConfig{ + ContentTypeNosniff: "nosniff", + // diable XFrameOptions; we're embedding here! + HSTSMaxAge: 31536000, // 365 days + // TODO: + // ContentSecurityPolicy + // XSSProtection + })) + e.Use(middleware.LoggerWithConfig(middleware.LoggerConfig{ + // Don't log requests for static content. + Skipper: func(c echo.Context) bool { + return strings.HasPrefix(c.Request().URL.Path, "/static") + }, + })) + e.Use(middleware.RateLimiterWithConfig(middleware.RateLimiterConfig{ + Skipper: middleware.DefaultSkipper, + Store: middleware.NewRateLimiterMemoryStoreWithConfig( + middleware.RateLimiterMemoryStoreConfig{ + Rate: 10, // requests per second + Burst: 30, // allow bursts + ExpiresIn: 3 * time.Minute, // garbage collect entries older than 3 minutes + }, + ), + IdentifierExtractor: func(ctx echo.Context) (string, error) { + id := ctx.RealIP() + return id, nil + }, + DenyHandler: func(c echo.Context, identifier string, err error) error { + return c.String(http.StatusTooManyRequests, "Your request has been rate limited. Please try again later. Contact support@bsky.app if you believe this was a mistake.\n") + }, + })) + + // redirect trailing slash to non-trailing slash. + // all of our current endpoints have no trailing slash. + e.Use(middleware.RemoveTrailingSlashWithConfig(middleware.TrailingSlashConfig{ + RedirectCode: http.StatusFound, + })) + + // + // configure routes + // + // static files + staticHandler := http.FileServer(func() http.FileSystem { + if debug { + log.Debugf("serving static file from the local file system") + return http.FS(os.DirFS("embedr-static")) + } + fsys, err := fs.Sub(bskyweb.EmbedrStaticFS, "embedr-static") + if err != nil { + log.Fatal(err) + } + return http.FS(fsys) + }()) + + e.GET("/robots.txt", echo.WrapHandler(staticHandler)) + e.GET("/ips-v4", echo.WrapHandler(staticHandler)) + e.GET("/ips-v6", echo.WrapHandler(staticHandler)) + e.GET("/.well-known/*", echo.WrapHandler(staticHandler)) + e.GET("/security.txt", func(c echo.Context) error { + return c.Redirect(http.StatusMovedPermanently, "/.well-known/security.txt") + }) + e.GET("/static/*", echo.WrapHandler(http.StripPrefix("/static/", staticHandler)), func(next echo.HandlerFunc) echo.HandlerFunc { + return func(c echo.Context) error { + path := c.Request().URL.Path + maxAge := 1 * (60 * 60) // default is 1 hour + + // Cache javascript and images files for 1 week, which works because + // they're always versioned (e.g. /static/js/main.64c14927.js) + if strings.HasPrefix(path, "/static/js/") || strings.HasPrefix(path, "/static/images/") { + maxAge = 7 * (60 * 60 * 24) // 1 week + } + + c.Response().Header().Set("Cache-Control", fmt.Sprintf("public, max-age=%d", maxAge)) + return next(c) + } + }) + + // actual routes + e.GET("/", server.WebHome) + e.GET("/iframe-resize.js", echo.WrapHandler(staticHandler)) + e.GET("/embed.js", echo.WrapHandler(staticHandler)) + e.GET("/oembed", server.WebOEmbed) + e.GET("/embed/:did/app.bsky.feed.post/:rkey", server.WebPostEmbed) + + // Start the server. + log.Infof("starting server address=%s", httpAddress) + go func() { + if err := server.httpd.ListenAndServe(); err != nil { + if !errors.Is(err, http.ErrServerClosed) { + log.Errorf("HTTP server shutting down unexpectedly: %s", err) + } + } + }() + + // Wait for a signal to exit. + log.Info("registering OS exit signal handler") + quit := make(chan struct{}) + exitSignals := make(chan os.Signal, 1) + signal.Notify(exitSignals, syscall.SIGINT, syscall.SIGTERM) + go func() { + sig := <-exitSignals + log.Infof("received OS exit signal: %s", sig) + + // Shut down the HTTP server. + if err := server.Shutdown(); err != nil { + log.Errorf("HTTP server shutdown error: %s", err) + } + + // Trigger the return that causes an exit. + close(quit) + }() + <-quit + log.Infof("graceful shutdown complete") + return nil +} + +func (srv *Server) ServeHTTP(rw http.ResponseWriter, req *http.Request) { + srv.echo.ServeHTTP(rw, req) +} + +func (srv *Server) Shutdown() error { + log.Info("shutting down") + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + return srv.httpd.Shutdown(ctx) +} + +func (srv *Server) errorHandler(err error, c echo.Context) { + code := http.StatusInternalServerError + if he, ok := err.(*echo.HTTPError); ok { + code = he.Code + } + c.Logger().Error(err) + data := map[string]interface{}{ + "statusCode": code, + } + c.Render(code, "error.html", data) +} diff --git a/bskyweb/cmd/embedr/snippet.go b/bskyweb/cmd/embedr/snippet.go new file mode 100644 index 00000000..e65f38a6 --- /dev/null +++ b/bskyweb/cmd/embedr/snippet.go @@ -0,0 +1,71 @@ +package main + +import ( + "bytes" + "fmt" + "html/template" + + appbsky "github.com/bluesky-social/indigo/api/bsky" + "github.com/bluesky-social/indigo/atproto/syntax" +) + +func (srv *Server) postEmbedHTML(postView *appbsky.FeedDefs_PostView) (string, error) { + // ensure that there isn't an injection from the URI + aturi, err := syntax.ParseATURI(postView.Uri) + if err != nil { + log.Error("bad AT-URI in reponse", "aturi", aturi, "err", err) + return "", err + } + + post, ok := postView.Record.Val.(*appbsky.FeedPost) + if !ok { + log.Error("bad post record value", "err", err) + return "", err + } + + const tpl = `
{{ .PostText }}

— {{ .PostAuthor }} {{ .PostIndexedAt }}
` + + t, err := template.New("snippet").Parse(tpl) + if err != nil { + log.Error("template parse error", "err", err) + return "", err + } + + var lang string + if len(post.Langs) > 0 { + lang = post.Langs[0] + } + var authorName string + if postView.Author.DisplayName != nil { + authorName = fmt.Sprintf("%s (@%s)", *postView.Author.DisplayName, postView.Author.Handle) + } else { + authorName = fmt.Sprintf("@%s", postView.Author.Handle) + } + fmt.Println(postView.Uri) + fmt.Println(fmt.Sprintf("%s", postView.Uri)) + data := struct { + PostURI template.URL + PostCID string + PostLang string + PostText string + PostAuthor string + PostIndexedAt string + WidgetURL template.URL + }{ + PostURI: template.URL(postView.Uri), + PostCID: postView.Cid, + PostLang: lang, + PostText: post.Text, + PostAuthor: authorName, + PostIndexedAt: postView.IndexedAt, // TODO: createdAt? + WidgetURL: template.URL("https://embed.bsky.app/static/embed.js"), + } + + var buf bytes.Buffer + err = t.Execute(&buf, data) + if err != nil { + log.Error("template parse error", "err", err) + return "", err + } + return buf.String(), nil +} diff --git a/bskyweb/embedr-static/.well-known/security.txt b/bskyweb/embedr-static/.well-known/security.txt new file mode 100644 index 00000000..8173cb72 --- /dev/null +++ b/bskyweb/embedr-static/.well-known/security.txt @@ -0,0 +1,4 @@ +Contact: mailto:security@bsky.app +Preferred-Languages: en +Canonical: https://bsky.app/.well-known/security.txt +Acknowledgements: https://github.com/bluesky-social/atproto/blob/main/CONTRIBUTORS.md diff --git a/bskyweb/embedr-static/embed.js b/bskyweb/embedr-static/embed.js new file mode 100644 index 00000000..15964a76 --- /dev/null +++ b/bskyweb/embedr-static/embed.js @@ -0,0 +1 @@ +/* embed javascript widget will go here */ diff --git a/bskyweb/embedr-static/favicon-16x16.png b/bskyweb/embedr-static/favicon-16x16.png new file mode 100644 index 0000000000000000000000000000000000000000..ea256e0569cee14f07f89970315bab56559207d3 GIT binary patch literal 1731 zcmeAS@N?(olHy`uVBq!ia0vp^0wB!61|;P_|4#%`jKx9jP7LeL$-D$|j-^I;ruq6Z zXaU(A3~Y=-49p-UK*+!-#lQ+?GcbfPO2gT4j2ciiOh7e;3_y}W6o}K>GZ|Q*>T7^B z2zUT7&?E>QkXezMlbcwQU!)LFl&@f{XR2oilw+B|0yaYg$lkPo5n=xVCb)S53z!jX zpgIO410!QALnA9ALj^+%D-&}oQ&Wb`59aL$N^ur=L>4nJa0`PlBg3pY5)2H?8!|&8 zN+NuHtdjF{^%7I^lT!66atlBvG1ydC0hzg}C5Z|ZxjA{oRu#5NU=>zCHb_`sNdc^+ zB->Ug!Z$#{Ilm}X!A#FU&p^qJOF==wrYI%ND#*nRsvXF)RmvzSDX`MlFE20GD>v55 zFG|-pw6wI;H!#vSGSUUA&@HaaD@m--%_~-h7y>iLCAB!YD6^m>Ge1uOWNuP#1_oB9X1WFzK!%MzhFUapoQqO{CSaHX z%|ytiAgRP=Mt)I9etwP}wp0W1QrG6AUiIgPFQZV<5D)u zIS4F2Zh5*mhFAzL4Yth|2^87;{OpdVU7d!~le~geN?lBrTs&u=p!$PZWvWsKo1pll z#K4Uky@Yd)xLx#8>{V%%??}*Dv&BFu%i!4AUyQqEmhbt#bMx<*D{AF-?<~LHD}H{i z+E{6}$(M5l`L$P-C`D$C! zK9PmOa!jOzDTOO{;VZG<~Y+oJ;ue zMA6mKuNM7qH!<84=Bn(oVD<**$#;`h*Ds1c8Kk1~;DDP_vzt@%`R7j>AGNz~d)xV% z<=vJ_?ibtc)-|;VxO_^C<_h~Dr9QJhe Kb6Mw<&;$S(wmP-| literal 0 HcmV?d00001 diff --git a/bskyweb/embedr-static/favicon-32x32.png b/bskyweb/embedr-static/favicon-32x32.png new file mode 100644 index 0000000000000000000000000000000000000000..a5ca7eed1e24b9554e417be0b81b5d0cf34bcdd5 GIT binary patch literal 2240 zcmaJ?3piA17(N*HW;MDfl4&Xlb2p7SjZutSx#iZ7Y(obVW|)~WDbY%)R4UX;#t^Bn zNEdA>o7*GFZq(C7Ek%SbWTLk0IYUvaJ@fqk|DFGP-}n3e|9sywd)-|f&|3Oh005w! z9N8Z5ibaNo8a#_)V&1`vlE}ls1}JWt*8{)QLL8@H7Z<=3wlx4XC0#%TfxtgNNgq&E z*Z|<8WboAvP%;_ED8m>s0MEcIcm(g^0jP2Oc6db|M|i+WW2-w)0)N3ID}6tgdz7Xk4_$oZ&noGBes5;;F2PI62l!gmk4 zz?Ha>E9SDfVu*#5Kb}m%Q^;O$`P1ki*#b|p1d*%#5B579TOkB1OrfG^qRA?gkW*<0 zgkAzcIBTk|9?L?(A4iP~Fo{S_Ppa{;22$+89mLXtnP0k*r3IZn;sf{YeJA!xFOFvC z+I68`A=9%)=msl~8rRWGw_I zSRB))-mBdB(76BNDx(#B8}hY-3`QEf%$@n17I(G-+8OFF_YJ#WbeA|xQ0kx01*VV{M z>=?BT4CbXJ$<;KyQkM$n+^B+LXyszFknZG`^{#hRIC*xJ36Iyk*Tm#*wa@Kp4;HAJ zV$7Y^GX^iVrJ=w}{rtAFMa;nH)msU318Zw?KVnF0X0@Z6Qnt@od_#`WjGKSKE$69c zW4E5X_h8cj9Y%Ah-IdtN0LUxlQ{|KT{VM->rp#D@$+bkcbBAb_VTUSq z-jwHtHD)<{V*1=6rmrZ*Uh~pg8MUK!m9d)ZlZ7A5B-iPG z3^CGs41d1INR6K}D4Onk)OT3j{Dq@FahJgdYQdf+z%aFtY8sai-*p6heD;>ekJ{o* z6D~|ELdn8RR=hUOQhFJr_X#pA=&`o(t&BVVqNCgRUd9e@sg&aX?z3A$RhrR5Wm>A* z2#y+;H?T;hwIzHg>dpHR;a}#Bu|Wyj8T;yV-EB&x(K)TNbDn1q0`S*g_14TYOvo%p zyG31R5rzgaQlrdXVnl literal 0 HcmV?d00001 diff --git a/bskyweb/embedr-static/favicon.png b/bskyweb/embedr-static/favicon.png new file mode 100644 index 0000000000000000000000000000000000000000..ddf55f4c815e6aa50cdb361c40dbf9e6ecfbe64a GIT binary patch literal 1412 zcmV-~1$+95P)~#+)E<0^kIYWrpF!JOY_r-~@3_fH^_P3Gkd?lzl#wZ9rYA zYRQfucI&HK*ulT|{4Cr0s#0}#)xX;S3l=Q6&){ey{Fio6(=HHnvaf~+G*n8x{rDU& zU=Dfmin^NMDG_z~V~1ZMzlI9*>8HJ=zU*ThuzXCfd8b{=xgtL3V06%|vvNdR8flXs zW_Q{?{BZl9_kwK<*-QN7-$6MdQn(gB#|UPS@Q79s>9x)9Jb}hFr8I0pc{=Q@|Ce9U z3z#uX69&30YrLS9-o(d8`jWR-p(+R$gt&dz(Gd11#D{cN#qb>5wCflz@CBvma1-jl z9U<3d<1cF(FLV?%qjOhIFzubV@A@%n)uO2b1aZQp+z_;cF&>&5FDO7KZ8(%0YQqU* z;Q_u9p3r`V?4PD`!Q%7fHAlwbpj7h;0QC+ zxi&sSA?UB*9vPnAfFsPgJQ&YTkVzWL`K77@u`-j~u3YRKUS zZ9;%M90e!v$m9t*_hXax;7~lhNR=kuXG##RbuwB<@APm#DjK|0NT?27(zF6?nM+J4EX79j6UAb1ZAQFJeJ2ZK*Tma%YSjh0W!-& zgd4x86DH1U;-o#>&dFO~$i{C>n#M!hQQ^yE7ZvWzu_IzurB37hpf_y?s{=-aB26JZ z%{oZ0B83BD8Rit>x#Ty~D$fz69FWJ%{&4aVs5B~gNa=D`x5{Vw4cc)|Y7i$fqUv%M z`(8dKn!jIyVb$5{$_@~6OHa1o?jwSoKo?r6gE~|7l^h_3Cu9njF3ch|)-05s8x)-` z(KOqMQo3fL(h8PR5ipk-qC~?t9{LJ8!NLg06xuG#8+^~KITVU4)(BWRyk6(djn`$I*-!=%SLVfE#2xAb@$%P*5YM@r>1ysGg+<+Z|TBvh2n)6#Ui z?C$zn$;@$;9E8PBT&rq)dTTWi>0BkcSM;R#shskS)mrmYCfX#S(pTzPQQbfN_=Xrq zXuZqA=wC3(7dSdtusqjV4dVS~f4R#!`j71VVrazu(K#%s{;d9iYr%pA^TBu8qIv!L S#kWBK0000 + + + +

embed.bsky.app homepage

+

could redirect to bsky.app? or show a "create embed" widget? + + diff --git a/bskyweb/embedr-templates/oembed.html b/bskyweb/embedr-templates/oembed.html new file mode 100644 index 00000000..646f0a48 --- /dev/null +++ b/bskyweb/embedr-templates/oembed.html @@ -0,0 +1 @@ +oembed JSON response will go here diff --git a/bskyweb/embedr-templates/postEmbed.html b/bskyweb/embedr-templates/postEmbed.html new file mode 100644 index 00000000..6329b3a1 --- /dev/null +++ b/bskyweb/embedr-templates/postEmbed.html @@ -0,0 +1 @@ +embed post HTML will go here diff --git a/bskyweb/static.go b/bskyweb/static.go index a67d189f..38adb833 100644 --- a/bskyweb/static.go +++ b/bskyweb/static.go @@ -4,3 +4,6 @@ import "embed" //go:embed static/* var StaticFS embed.FS + +//go:embed embedr-static/* +var EmbedrStaticFS embed.FS diff --git a/bskyweb/templates.go b/bskyweb/templates.go index ce3fa29a..a66965ab 100644 --- a/bskyweb/templates.go +++ b/bskyweb/templates.go @@ -4,3 +4,6 @@ import "embed" //go:embed templates/* var TemplateFS embed.FS + +//go:embed embedr-templates/* +var EmbedrTemplateFS embed.FS diff --git a/package.json b/package.json index e4a8de74..21e632aa 100644 --- a/package.json +++ b/package.json @@ -20,7 +20,7 @@ "build-ios": "yarn use-build-number-with-bump eas build -p ios", "build-android": "yarn use-build-number-with-bump eas build -p android", "build": "yarn use-build-number-with-bump eas build", - "build-embed": "cd bskyembed && yarn build && cd .. && node ./scripts/post-embed-build.js", + "build-embed": "cd bskyembed && yarn build && yarn build-snippet && cd .. && node ./scripts/post-embed-build.js", "start": "expo start --dev-client", "start:prod": "expo start --dev-client --no-dev --minify", "clean-cache": "rm -rf node_modules/.cache/babel-loader/*", diff --git a/scripts/post-embed-build.js b/scripts/post-embed-build.js index 5bece544..c0897e1b 100644 --- a/scripts/post-embed-build.js +++ b/scripts/post-embed-build.js @@ -1,49 +1,65 @@ -// const path = require('node:path') -// const fs = require('node:fs') +const path = require('node:path') +const fs = require('node:fs') -// const projectRoot = path.join(__dirname, '..') +const projectRoot = path.join(__dirname, '..') -// // copy embed assets to web-build +// copy embed assets to embedr -// const embedAssetSource = path.join( -// projectRoot, -// 'bskyembed', -// 'dist', -// 'static', -// 'embed', -// 'assets', -// ) +const embedAssetSource = path.join(projectRoot, 'bskyembed', 'dist', 'static') -// const embedAssetDest = path.join( -// projectRoot, -// 'web-build', -// 'static', -// 'embed', -// 'assets', -// ) +const embedAssetDest = path.join(projectRoot, 'bskyweb', 'embedr-static') -// fs.cpSync(embedAssetSource, embedAssetDest, {recursive: true}) +fs.cpSync(embedAssetSource, embedAssetDest, {recursive: true}) -// // copy entrypoint(s) to web-build +const embedEmbedJSSource = path.join( + projectRoot, + 'bskyembed', + 'dist', + 'embed.js', +) -// // additional entrypoints will need more work, but this'll do for now -// const embedHtmlSource = path.join( -// projectRoot, -// 'bskyembed', -// 'dist', -// 'index.html', -// ) +const embedEmbedJSDest = path.join( + projectRoot, + 'bskyweb', + 'embedr-static', + 'embed.js', +) -// const embedHtmlDest = path.join( -// projectRoot, -// 'web-build', -// 'static', -// 'embed', -// 'post.html', -// ) +fs.cpSync(embedEmbedJSSource, embedEmbedJSDest) -// fs.copyFileSync(embedHtmlSource, embedHtmlDest) +// copy entrypoint(s) to embedr -// console.log(`Copied embed assets to web-build`) +// additional entrypoints will need more work, but this'll do for now +const embedHomeHtmlSource = path.join( + projectRoot, + 'bskyembed', + 'dist', + 'index.html', +) -console.log('post-embed-build.js - waiting for embedr!') +const embedHomeHtmlDest = path.join( + projectRoot, + 'bskyweb', + 'embedr-templates', + 'home.html', +) + +fs.copyFileSync(embedHomeHtmlSource, embedHomeHtmlDest) + +const embedPostHtmlSource = path.join( + projectRoot, + 'bskyembed', + 'dist', + 'post.html', +) + +const embedPostHtmlDest = path.join( + projectRoot, + 'bskyweb', + 'embedr-templates', + 'postEmbed.html', +) + +fs.copyFileSync(embedPostHtmlSource, embedPostHtmlDest) + +console.log(`Copied embed assets to embedr`)