From a2f49bb08c4794bba36e903ec2402a83e936af79 Mon Sep 17 00:00:00 2001 From: bnewbold Date: Tue, 23 Jan 2024 13:16:32 -0800 Subject: [PATCH] more social card tweaks, and include in RSS as well (#2599) * move link expander to new file, add test, refactor a bit * text formatting: include indication if a quote post exists * rss: include expanded links --- bskyweb/cmd/bskyweb/formating.go | 57 ++++++++++++++++++ bskyweb/cmd/bskyweb/formatting_test.go | 39 ++++++++++++ bskyweb/cmd/bskyweb/rss.go | 7 ++- bskyweb/cmd/bskyweb/server.go | 50 ++-------------- .../bskyweb/testdata/atproto_embed_post.json | 60 +++++++++++++++++++ 5 files changed, 167 insertions(+), 46 deletions(-) create mode 100644 bskyweb/cmd/bskyweb/formating.go create mode 100644 bskyweb/cmd/bskyweb/formatting_test.go create mode 100644 bskyweb/cmd/bskyweb/testdata/atproto_embed_post.json diff --git a/bskyweb/cmd/bskyweb/formating.go b/bskyweb/cmd/bskyweb/formating.go new file mode 100644 index 00000000..023ba3f5 --- /dev/null +++ b/bskyweb/cmd/bskyweb/formating.go @@ -0,0 +1,57 @@ +package main + +import ( + "fmt" + "slices" + "strings" + + appbsky "github.com/bluesky-social/indigo/api/bsky" +) + +// Function to expand shortened links in rich text back to full urls, replacing shortened urls in social card meta tags and the noscript output. +// +// This essentially reverses the effect of the typescript function `shortenLinks()` in `src/lib/strings/rich-text-manip.ts` +func ExpandPostText(post *appbsky.FeedPost) string { + postText := post.Text + var charsAdded int = 0 + // iterate over facets, check if they're link facets, and if found, grab the uri + for _, facet := range post.Facets { + linkUri := "" + if slices.ContainsFunc(facet.Features, func(feat *appbsky.RichtextFacet_Features_Elem) bool { + if feat.RichtextFacet_Link == nil || feat.RichtextFacet_Link.LexiconTypeID != "app.bsky.richtext.facet#link" { + return false + } + + // bail out if bounds checks fail + if int(facet.Index.ByteStart)+charsAdded > len(postText) || int(facet.Index.ByteEnd)+charsAdded > len(postText) { + return false + } + linkText := postText[int(facet.Index.ByteStart)+charsAdded : int(facet.Index.ByteEnd)+charsAdded] + linkUri = feat.RichtextFacet_Link.Uri + + // only expand uris that have been shortened (as opposed to those with non-uri anchor text) + if strings.HasSuffix(linkText, "...") && strings.Contains(linkUri, linkText[0:len(linkText)-3]) { + return true + } + return false + }) { + // replace the shortened uri with the full length one from the facet using utf8 byte offsets + // NOTE: we already did bounds check above + postText = postText[0:int(facet.Index.ByteStart)+charsAdded] + linkUri + postText[int(facet.Index.ByteEnd)+charsAdded:] + charsAdded += len(linkUri) - int(facet.Index.ByteEnd-facet.Index.ByteStart) + } + } + // if the post has an embeded link and its url doesn't already appear in postText, append it to + // the end to avoid social cards with missing links + if post.Embed != nil && post.Embed.EmbedExternal != nil && post.Embed.EmbedExternal.External != nil { + externalURI := post.Embed.EmbedExternal.External.Uri + if !strings.Contains(postText, externalURI) { + postText = fmt.Sprintf("%s\n%s", postText, externalURI) + } + } + // TODO: could embed the actual post text? + if post.Embed != nil && (post.Embed.EmbedRecord != nil || post.Embed.EmbedRecordWithMedia != nil) { + postText = fmt.Sprintf("%s\n\n[contains quote post or other embeded content]", postText) + } + return postText +} diff --git a/bskyweb/cmd/bskyweb/formatting_test.go b/bskyweb/cmd/bskyweb/formatting_test.go new file mode 100644 index 00000000..1fbf8d5e --- /dev/null +++ b/bskyweb/cmd/bskyweb/formatting_test.go @@ -0,0 +1,39 @@ +package main + +import ( + "encoding/json" + "io" + "os" + "strings" + "testing" + + appbsky "github.com/bluesky-social/indigo/api/bsky" +) + +func loadPost(t *testing.T, p string) appbsky.FeedPost { + + f, err := os.Open(p) + if err != nil { + t.Fatal(err) + } + defer func() { _ = f.Close() }() + + postBytes, err := io.ReadAll(f) + if err != nil { + t.Fatal(err) + } + var post appbsky.FeedPost + if err := json.Unmarshal(postBytes, &post); err != nil { + t.Fatal(err) + } + return post +} + +func TestExpandPostText(t *testing.T) { + post := loadPost(t, "testdata/atproto_embed_post.json") + + text := ExpandPostText(&post) + if !strings.Contains(text, "https://github.com/snarfed/bridgy-fed") { + t.Fail() + } +} diff --git a/bskyweb/cmd/bskyweb/rss.go b/bskyweb/cmd/bskyweb/rss.go index b756b90e..76689abb 100644 --- a/bskyweb/cmd/bskyweb/rss.go +++ b/bskyweb/cmd/bskyweb/rss.go @@ -96,7 +96,10 @@ func (srv *Server) WebProfileRSS(c echo.Context) error { if err != nil { return err } - rec := p.Post.Record.Val.(*appbsky.FeedPost) + rec, ok := p.Post.Record.Val.(*appbsky.FeedPost) + if !ok { + continue + } // only top-level posts in RSS (no replies) if rec.Reply != nil { continue @@ -108,7 +111,7 @@ func (srv *Server) WebProfileRSS(c echo.Context) error { } posts = append(posts, Item{ Link: fmt.Sprintf("https://%s/profile/%s/post/%s", req.Host, pv.Handle, aturi.RecordKey().String()), - Description: rec.Text, + Description: ExpandPostText(rec), PubDate: pubDate, GUID: ItemGUID{ Value: aturi.String(), diff --git a/bskyweb/cmd/bskyweb/server.go b/bskyweb/cmd/bskyweb/server.go index 7571e536..8e7d618c 100644 --- a/bskyweb/cmd/bskyweb/server.go +++ b/bskyweb/cmd/bskyweb/server.go @@ -10,7 +10,6 @@ import ( "net/http" "os" "os/signal" - "slices" "strings" "syscall" "time" @@ -353,53 +352,16 @@ func (srv *Server) WebPost(c echo.Context) error { } } - data["postText"] = expandPostLinks(postView) + if postView.Record != nil { + postRecord, ok := postView.Record.Val.(*appbsky.FeedPost) + if ok { + data["postText"] = ExpandPostText(postRecord) + } + } return c.Render(http.StatusOK, "post.html", data) } -// function to expand shortened links in rich text back to full urls, replacing shortened urls in -// social card meta tags and the noscript output. this essentially reverses the effect -// of shortenLinks() in src/lib/strings/rich-text-manip.ts -func expandPostLinks(postView *appbsky.FeedDefs_PostView) string { - if postView.Record != nil { - rec := postView.Record.Val.(*appbsky.FeedPost) - postText := rec.Text - var charsAdded int64 = 0 - // iterate over facets, check if they're link facets, and if found, grab the uri - for _, facet := range rec.Facets { - linkUri := "" - if slices.ContainsFunc(facet.Features, func(feat *appbsky.RichtextFacet_Features_Elem) bool { - if feat.RichtextFacet_Link != nil && feat.RichtextFacet_Link.LexiconTypeID == "app.bsky.richtext.facet#link" { - linkUri = feat.RichtextFacet_Link.Uri - // only expand uris that have been shortened (as opposed to those with non-uri anchor text) - if int64(len(postText)) >= facet.Index.ByteEnd+charsAdded && - strings.HasSuffix(postText[facet.Index.ByteStart+charsAdded:facet.Index.ByteEnd+charsAdded], "...") && - strings.Contains(linkUri, postText[facet.Index.ByteStart+charsAdded:(facet.Index.ByteEnd+charsAdded)-3]) { - return true - } - } - return false - }) { - // replace the shortened uri with the full length one from the facet using utf8 byte offsets - if int64(len(postText)) >= facet.Index.ByteEnd+charsAdded { - postText = postText[0:facet.Index.ByteStart+charsAdded] + linkUri + postText[facet.Index.ByteEnd+charsAdded:] - charsAdded += int64(len(linkUri)) - (facet.Index.ByteEnd - facet.Index.ByteStart) - } - } - } - // if the post has an embeded link and its url doesn't already appear in postText, append it to - // the end to avoid social cards with missing links - if postView.Embed != nil && - postView.Embed.EmbedExternal_View != nil && - !strings.Contains(postText, postView.Embed.EmbedExternal_View.External.Uri) { - postText = fmt.Sprintf("%s\n%s", postText, postView.Embed.EmbedExternal_View.External.Uri) - } - return postText - } - return "" -} - func (srv *Server) WebProfile(c echo.Context) error { ctx := c.Request().Context() data := pongo2.Context{} diff --git a/bskyweb/cmd/bskyweb/testdata/atproto_embed_post.json b/bskyweb/cmd/bskyweb/testdata/atproto_embed_post.json new file mode 100644 index 00000000..2e54854e --- /dev/null +++ b/bskyweb/cmd/bskyweb/testdata/atproto_embed_post.json @@ -0,0 +1,60 @@ +{ + "$type": "app.bsky.feed.post", + "createdAt": "2023-12-04T19:30:03.024Z", + "embed": { + "$type": "app.bsky.embed.external", + "external": { + "description": "🕸 Bridges the IndieWeb to Mastodon and the fediverse via ActivityPub. - GitHub - snarfed/bridgy-fed: 🕸 Bridges the IndieWeb to Mastodon and the fediverse via ActivityPub.", + "thumb": { + "$type": "blob", + "ref": { + "$link": "bafkreidplhjcnrl2c74r3xs7nh7k7q3ny6ul7cgxr2fophblvdeky6t64e" + }, + "mimeType": "image/jpeg", + "size": 347998 + }, + "title": "GitHub - snarfed/bridgy-fed: 🕸 Bridges the IndieWeb to Mastodon and the fediverse via ActivityPub...", + "uri": "https://github.com/snarfed/bridgy-fed" + } + }, + "facets": [ + { + "features": [ + { + "$type": "app.bsky.richtext.facet#link", + "uri": "https://github.com/snarfed/bridgy-fed" + } + ], + "index": { + "byteEnd": 92, + "byteStart": 66 + } + }, + { + "features": [ + { + "$type": "app.bsky.richtext.facet#mention", + "did": "did:plc:fdme4gb7mu7zrie7peay7tst" + } + ], + "index": { + "byteEnd": 149, + "byteStart": 137 + } + } + ], + "langs": [ + "en" + ], + "reply": { + "parent": { + "cid": "bafyreifaidyl62p4snkdwsygviemsxyidi3cd7dxvjomh5644sovxhsppa", + "uri": "at://did:plc:ewvi7nxzyoun6zhxrhs64oiz/app.bsky.feed.post/3kfqklhpalh2c" + }, + "root": { + "cid": "bafyreibiimdwmsp5mqpm7utqcdmvo6fdqmofblp5obs3h7ub6652zyooci", + "uri": "at://did:plc:ewvi7nxzyoun6zhxrhs64oiz/app.bsky.feed.post/3kfqkkjdkic2e" + } + }, + "text": "Bridgy Fed is an open-source project — check out the code here: github.com/snarfed/brid...\n\nStay updated with the project by following @snarfed.org!" +}