more social card tweaks, and include in RSS as well (#2599)

* move link expander to new file, add test, refactor a bit

* text formatting: include indication if a quote post exists

* rss: include expanded links

bskyweb/cmd/bskyweb/formating.go

@@ -0,0 +1,57 @@
+package main
+
+import (
+	"fmt"
+	"slices"
+	"strings"
+
+	appbsky "github.com/bluesky-social/indigo/api/bsky"
+)
+
+// Function to expand shortened links in rich text back to full urls, replacing shortened urls in social card meta tags and the noscript output.
+//
+// This essentially reverses the effect of the typescript function `shortenLinks()` in `src/lib/strings/rich-text-manip.ts`
+func ExpandPostText(post *appbsky.FeedPost) string {
+	postText := post.Text
+	var charsAdded int = 0
+	// iterate over facets, check if they're link facets, and if found, grab the uri
+	for _, facet := range post.Facets {
+		linkUri := ""
+		if slices.ContainsFunc(facet.Features, func(feat *appbsky.RichtextFacet_Features_Elem) bool {
+			if feat.RichtextFacet_Link == nil || feat.RichtextFacet_Link.LexiconTypeID != "app.bsky.richtext.facet#link" {
+				return false
+			}
+
+			// bail out if bounds checks fail
+			if int(facet.Index.ByteStart)+charsAdded > len(postText) || int(facet.Index.ByteEnd)+charsAdded > len(postText) {
+				return false
+			}
+			linkText := postText[int(facet.Index.ByteStart)+charsAdded : int(facet.Index.ByteEnd)+charsAdded]
+			linkUri = feat.RichtextFacet_Link.Uri
+
+			// only expand uris that have been shortened (as opposed to those with non-uri anchor text)
+			if strings.HasSuffix(linkText, "...") && strings.Contains(linkUri, linkText[0:len(linkText)-3]) {
+				return true
+			}
+			return false
+		}) {
+			// replace the shortened uri with the full length one from the facet using utf8 byte offsets
+			// NOTE: we already did bounds check above
+			postText = postText[0:int(facet.Index.ByteStart)+charsAdded] + linkUri + postText[int(facet.Index.ByteEnd)+charsAdded:]
+			charsAdded += len(linkUri) - int(facet.Index.ByteEnd-facet.Index.ByteStart)
+		}
+	}
+	// if the post has an embeded link and its url doesn't already appear in postText, append it to
+	// the end to avoid social cards with missing links
+	if post.Embed != nil && post.Embed.EmbedExternal != nil && post.Embed.EmbedExternal.External != nil {
+		externalURI := post.Embed.EmbedExternal.External.Uri
+		if !strings.Contains(postText, externalURI) {
+			postText = fmt.Sprintf("%s\n%s", postText, externalURI)
+		}
+	}
+	// TODO: could embed the actual post text?
+	if post.Embed != nil && (post.Embed.EmbedRecord != nil || post.Embed.EmbedRecordWithMedia != nil) {
+		postText = fmt.Sprintf("%s\n\n[contains quote post or other embeded content]", postText)
+	}
+	return postText
+}

bskyweb/cmd/bskyweb/formatting_test.go

@@ -0,0 +1,39 @@
+package main
+
+import (
+	"encoding/json"
+	"io"
+	"os"
+	"strings"
+	"testing"
+
+	appbsky "github.com/bluesky-social/indigo/api/bsky"
+)
+
+func loadPost(t *testing.T, p string) appbsky.FeedPost {
+
+	f, err := os.Open(p)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer func() { _ = f.Close() }()
+
+	postBytes, err := io.ReadAll(f)
+	if err != nil {
+		t.Fatal(err)
+	}
+	var post appbsky.FeedPost
+	if err := json.Unmarshal(postBytes, &post); err != nil {
+		t.Fatal(err)
+	}
+	return post
+}
+
+func TestExpandPostText(t *testing.T) {
+	post := loadPost(t, "testdata/atproto_embed_post.json")
+
+	text := ExpandPostText(&post)
+	if !strings.Contains(text, "https://github.com/snarfed/bridgy-fed") {
+		t.Fail()
+	}
+}

bskyweb/cmd/bskyweb/rss.go

@@ -96,7 +96,10 @@ func (srv *Server) WebProfileRSS(c echo.Context) error {
 		if err != nil {
 			return err
 		}
-		rec := p.Post.Record.Val.(*appbsky.FeedPost)
+		rec, ok := p.Post.Record.Val.(*appbsky.FeedPost)
+		if !ok {
+			continue
+		}
 		// only top-level posts in RSS (no replies)
 		if rec.Reply != nil {
 			continue
@@ -108,7 +111,7 @@ func (srv *Server) WebProfileRSS(c echo.Context) error {
 		}
 		posts = append(posts, Item{
 			Link:        fmt.Sprintf("https://%s/profile/%s/post/%s", req.Host, pv.Handle, aturi.RecordKey().String()),
-			Description: rec.Text,
+			Description: ExpandPostText(rec),
 			PubDate:     pubDate,
 			GUID: ItemGUID{
 				Value:   aturi.String(),

bskyweb/cmd/bskyweb/server.go

@@ -10,7 +10,6 @@ import (
 	"net/http"
 	"os"
 	"os/signal"
-	"slices"
 	"strings"
 	"syscall"
 	"time"
@@ -353,53 +352,16 @@ func (srv *Server) WebPost(c echo.Context) error {
 		}
 	}
 
-	data["postText"] = expandPostLinks(postView)
+	if postView.Record != nil {
+		postRecord, ok := postView.Record.Val.(*appbsky.FeedPost)
+		if ok {
+			data["postText"] = ExpandPostText(postRecord)
+		}
+	}
 
 	return c.Render(http.StatusOK, "post.html", data)
 }
 
-// function to expand shortened links in rich text back to full urls, replacing shortened urls in
-// social card meta tags and the noscript output. this essentially reverses the effect
-// of shortenLinks() in src/lib/strings/rich-text-manip.ts
-func expandPostLinks(postView *appbsky.FeedDefs_PostView) string {
-	if postView.Record != nil {
-		rec := postView.Record.Val.(*appbsky.FeedPost)
-		postText := rec.Text
-		var charsAdded int64 = 0
-		// iterate over facets, check if they're link facets, and if found, grab the uri
-		for _, facet := range rec.Facets {
-			linkUri := ""
-			if slices.ContainsFunc(facet.Features, func(feat *appbsky.RichtextFacet_Features_Elem) bool {
-				if feat.RichtextFacet_Link != nil && feat.RichtextFacet_Link.LexiconTypeID == "app.bsky.richtext.facet#link" {
-					linkUri = feat.RichtextFacet_Link.Uri
-					// only expand uris that have been shortened (as opposed to those with non-uri anchor text)
-					if int64(len(postText)) >= facet.Index.ByteEnd+charsAdded &&
-						strings.HasSuffix(postText[facet.Index.ByteStart+charsAdded:facet.Index.ByteEnd+charsAdded], "...") &&
-						strings.Contains(linkUri, postText[facet.Index.ByteStart+charsAdded:(facet.Index.ByteEnd+charsAdded)-3]) {
-						return true
-					}
-				}
-				return false
-			}) {
-				// replace the shortened uri with the full length one from the facet using utf8 byte offsets
-				if int64(len(postText)) >= facet.Index.ByteEnd+charsAdded {
-					postText = postText[0:facet.Index.ByteStart+charsAdded] + linkUri + postText[facet.Index.ByteEnd+charsAdded:]
-					charsAdded += int64(len(linkUri)) - (facet.Index.ByteEnd - facet.Index.ByteStart)
-				}
-			}
-		}
-		// if the post has an embeded link and its url doesn't already appear in postText, append it to
-		// the end to avoid social cards with missing links
-		if postView.Embed != nil &&
-			postView.Embed.EmbedExternal_View != nil &&
-			!strings.Contains(postText, postView.Embed.EmbedExternal_View.External.Uri) {
-			postText = fmt.Sprintf("%s\n%s", postText, postView.Embed.EmbedExternal_View.External.Uri)
-		}
-		return postText
-	}
-	return ""
-}
-
 func (srv *Server) WebProfile(c echo.Context) error {
 	ctx := c.Request().Context()
 	data := pongo2.Context{}

bskyweb/cmd/bskyweb/testdata/atproto_embed_post.json

@@ -0,0 +1,60 @@
+{
+    "$type": "app.bsky.feed.post",
+    "createdAt": "2023-12-04T19:30:03.024Z",
+    "embed": {
+        "$type": "app.bsky.embed.external",
+        "external": {
+        "description": "🕸 Bridges the IndieWeb to Mastodon and the fediverse via ActivityPub. - GitHub - snarfed/bridgy-fed: 🕸 Bridges the IndieWeb to Mastodon and the fediverse via ActivityPub.",
+        "thumb": {
+            "$type": "blob",
+            "ref": {
+            "$link": "bafkreidplhjcnrl2c74r3xs7nh7k7q3ny6ul7cgxr2fophblvdeky6t64e"
+            },
+            "mimeType": "image/jpeg",
+            "size": 347998
+        },
+        "title": "GitHub - snarfed/bridgy-fed: 🕸 Bridges the IndieWeb to Mastodon and the fediverse via ActivityPub...",
+        "uri": "https://github.com/snarfed/bridgy-fed"
+        }
+    },
+    "facets": [
+        {
+        "features": [
+            {
+            "$type": "app.bsky.richtext.facet#link",
+            "uri": "https://github.com/snarfed/bridgy-fed"
+            }
+        ],
+        "index": {
+            "byteEnd": 92,
+            "byteStart": 66
+        }
+        },
+        {
+        "features": [
+            {
+            "$type": "app.bsky.richtext.facet#mention",
+            "did": "did:plc:fdme4gb7mu7zrie7peay7tst"
+            }
+        ],
+        "index": {
+            "byteEnd": 149,
+            "byteStart": 137
+        }
+        }
+    ],
+    "langs": [
+        "en"
+    ],
+    "reply": {
+        "parent": {
+        "cid": "bafyreifaidyl62p4snkdwsygviemsxyidi3cd7dxvjomh5644sovxhsppa",
+        "uri": "at://did:plc:ewvi7nxzyoun6zhxrhs64oiz/app.bsky.feed.post/3kfqklhpalh2c"
+        },
+        "root": {
+        "cid": "bafyreibiimdwmsp5mqpm7utqcdmvo6fdqmofblp5obs3h7ub6652zyooci",
+        "uri": "at://did:plc:ewvi7nxzyoun6zhxrhs64oiz/app.bsky.feed.post/3kfqkkjdkic2e"
+        }
+    },
+    "text": "Bridgy Fed is an open-source project — check out the code here: github.com/snarfed/brid...\n\nStay updated with the project by following @snarfed.org!"
+}