fix: sync latest Android and langpack updates

internal/app/bots/botfather.go

@@ -128,7 +128,7 @@ func (s *Service) sendServiceBotReply(ctx context.Context, botUserID, userID int
 		RecipientUserID:  userID,
 		RandomID:         s.botReplyRandomID(),
 		Message:          reply.Text,
-		Entities:         reply.Entities,
+		Entities:         serviceBotReplyEntities(reply.Text, reply.Entities),
 		Date:             int(s.now().Unix()),
 		RecipientBlocked: blocked,
 	}); err != nil {

internal/app/bots/service_bot_entities.go

@@ -0,0 +1,182 @@
+package bots
+
+import (
+	"sort"
+	"strings"
+	"unicode"
+	"unicode/utf8"
+
+	"telesrv/internal/domain"
+)
+
+type serviceBotEntitySpan struct {
+	start int
+	end   int
+}
+
+func serviceBotReplyEntities(text string, explicit []domain.MessageEntity) []domain.MessageEntity {
+	if text == "" && len(explicit) == 0 {
+		return nil
+	}
+	out := append([]domain.MessageEntity(nil), explicit...)
+	occupied := make([]serviceBotEntitySpan, 0, len(out)+8)
+	for _, entity := range out {
+		if entity.Length <= 0 {
+			continue
+		}
+		occupied = append(occupied, serviceBotEntitySpan{start: entity.Offset, end: entity.Offset + entity.Length})
+	}
+	appendEntity := func(entity domain.MessageEntity) {
+		if entity.Length <= 0 || len(out) >= domain.MaxMessageEntityCount {
+			return
+		}
+		span := serviceBotEntitySpan{start: entity.Offset, end: entity.Offset + entity.Length}
+		if serviceBotSpanOverlaps(span, occupied) {
+			return
+		}
+		out = append(out, entity)
+		occupied = append(occupied, span)
+	}
+	for _, span := range serviceBotURLByteSpans(text) {
+		offset, length := utf16Range(text, span.start, span.end)
+		appendEntity(domain.MessageEntity{Type: domain.MessageEntityURL, Offset: offset, Length: length})
+	}
+	for _, span := range serviceBotCommandByteSpans(text) {
+		offset, length := utf16Range(text, span.start, span.end)
+		appendEntity(domain.MessageEntity{Type: domain.MessageEntityBotCommand, Offset: offset, Length: length})
+	}
+	sort.SliceStable(out, func(i, j int) bool {
+		if out[i].Offset != out[j].Offset {
+			return out[i].Offset < out[j].Offset
+		}
+		return out[i].Length < out[j].Length
+	})
+	return out
+}
+
+func serviceBotURLByteSpans(text string) []serviceBotEntitySpan {
+	var spans []serviceBotEntitySpan
+	for i := 0; i < len(text); {
+		if !strings.HasPrefix(text[i:], "https://") && !strings.HasPrefix(text[i:], "http://") {
+			_, size := utf8.DecodeRuneInString(text[i:])
+			i += size
+			continue
+		}
+		start := i
+		i += len("http://")
+		if strings.HasPrefix(text[start:], "https://") {
+			i = start + len("https://")
+		}
+		for i < len(text) {
+			r, size := utf8.DecodeRuneInString(text[i:])
+			if unicode.IsSpace(r) || r == '<' || r == '>' {
+				break
+			}
+			i += size
+		}
+		end := trimServiceBotURLTrailingPunctuation(text, start, i)
+		if end > start {
+			spans = append(spans, serviceBotEntitySpan{start: start, end: end})
+		}
+		if i == start {
+			i++
+		}
+	}
+	return spans
+}
+
+func trimServiceBotURLTrailingPunctuation(text string, start, end int) int {
+	for end > start {
+		r, size := utf8.DecodeLastRuneInString(text[start:end])
+		if !strings.ContainsRune(".,;:!?)]}", r) {
+			break
+		}
+		end -= size
+	}
+	return end
+}
+
+func serviceBotCommandByteSpans(text string) []serviceBotEntitySpan {
+	var spans []serviceBotEntitySpan
+	for i := 0; i < len(text); {
+		r, size := utf8.DecodeRuneInString(text[i:])
+		if r != '/' || !serviceBotCommandStart(text, i) {
+			i += size
+			continue
+		}
+		start := i
+		i += size
+		commandStartEnd := i
+		for i < len(text) {
+			r, size = utf8.DecodeRuneInString(text[i:])
+			if !serviceBotCommandChar(r) {
+				break
+			}
+			i += size
+		}
+		if i == commandStartEnd {
+			continue
+		}
+		if i < len(text) {
+			r, size = utf8.DecodeRuneInString(text[i:])
+			if r == '@' {
+				mentionEnd := i + size
+				for mentionEnd < len(text) {
+					r, size = utf8.DecodeRuneInString(text[mentionEnd:])
+					if !serviceBotCommandChar(r) {
+						break
+					}
+					mentionEnd += size
+				}
+				if mentionEnd > i+size {
+					i = mentionEnd
+				}
+			}
+		}
+		spans = append(spans, serviceBotEntitySpan{start: start, end: i})
+	}
+	return spans
+}
+
+func serviceBotCommandStart(text string, byteIndex int) bool {
+	if byteIndex == 0 {
+		return true
+	}
+	prev, _ := utf8.DecodeLastRuneInString(text[:byteIndex])
+	if prev == ':' || prev == '/' || prev == '@' {
+		return false
+	}
+	return !serviceBotCommandChar(prev)
+}
+
+func serviceBotCommandChar(r rune) bool {
+	return r == '_' || ('0' <= r && r <= '9') || ('A' <= r && r <= 'Z') || ('a' <= r && r <= 'z')
+}
+
+func utf16Range(text string, startByte, endByte int) (int, int) {
+	offset := 0
+	for _, r := range text[:startByte] {
+		offset += utf16RuneLen(r)
+	}
+	length := 0
+	for _, r := range text[startByte:endByte] {
+		length += utf16RuneLen(r)
+	}
+	return offset, length
+}
+
+func utf16RuneLen(r rune) int {
+	if r > 0xFFFF {
+		return 2
+	}
+	return 1
+}
+
+func serviceBotSpanOverlaps(span serviceBotEntitySpan, occupied []serviceBotEntitySpan) bool {
+	for _, other := range occupied {
+		if span.start < other.end && other.start < span.end {
+			return true
+		}
+	}
+	return false
+}

internal/app/bots/service_bot_entities_test.go

@@ -0,0 +1,60 @@
+package bots
+
+import (
+	"testing"
+
+	"telesrv/internal/domain"
+)
+
+func TestServiceBotReplyEntitiesCommandsAndURLsUseUTF16Offsets(t *testing.T) {
+	text := "🙂 Send /cancel or https://telesrv.net/addstickers/fun_pack."
+	entities := serviceBotReplyEntities(text, nil)
+
+	assertEntity := func(typ domain.MessageEntityType, offset, length int) {
+		t.Helper()
+		for _, entity := range entities {
+			if entity.Type == typ && entity.Offset == offset && entity.Length == length {
+				return
+			}
+		}
+		t.Fatalf("entities %+v missing %s at offset=%d length=%d", entities, typ, offset, length)
+	}
+
+	assertEntity(domain.MessageEntityBotCommand, 8, len("/cancel"))
+	assertEntity(domain.MessageEntityURL, 19, len("https://telesrv.net/addstickers/fun_pack"))
+}
+
+func TestServiceBotReplyEntitiesSkipCommandsInsideURLsAndExplicitEntities(t *testing.T) {
+	text := "Token: abc/def\nhttps://telesrv.net/addstickers/fun_pack\nUse /help"
+	entities := serviceBotReplyEntities(text, []domain.MessageEntity{{
+		Type:   domain.MessageEntityCode,
+		Offset: len("Token: "),
+		Length: len("abc/def"),
+	}})
+
+	commandCount := 0
+	for _, entity := range entities {
+		if entity.Type == domain.MessageEntityBotCommand {
+			commandCount++
+		}
+		if entity.Type == domain.MessageEntityBotCommand && entity.Offset < len("Token: abc/def\nhttps://telesrv.net/") {
+			t.Fatalf("unexpected command entity inside code/url: %+v in %+v", entity, entities)
+		}
+	}
+	if commandCount != 1 {
+		t.Fatalf("bot command entities = %d in %+v, want only /help", commandCount, entities)
+	}
+}
+
+func TestServiceBotReplyEntitiesIgnoreBareSlashBeforeEmoji(t *testing.T) {
+	entities := serviceBotReplyEntities("not a command /🙂 but /help is", nil)
+	commandCount := 0
+	for _, entity := range entities {
+		if entity.Type == domain.MessageEntityBotCommand {
+			commandCount++
+		}
+	}
+	if commandCount != 1 {
+		t.Fatalf("bot command entities = %d in %+v, want only /help", commandCount, entities)
+	}
+}

internal/app/bots/stickersbot_test.go

@@ -29,7 +29,12 @@ func sendMessageToStickers(t *testing.T, svc *Service, messages *memory.MessageS
 	msg.From = domain.Peer{Type: domain.PeerTypeUser, ID: owner.ID}
 	msg.Peer = domain.Peer{Type: domain.PeerTypeUser, ID: domain.StickersBotUserID}
 	svc.respondAsStickers(owner.ID, msg)
-	list, err := messages.ListByUser(context.Background(), owner.ID, domain.MessageFilter{
+	return latestStickersReply(t, messages, owner.ID).Body
+}
+
+func latestStickersReply(t *testing.T, messages *memory.MessageStore, userID int64) domain.Message {
+	t.Helper()
+	list, err := messages.ListByUser(context.Background(), userID, domain.MessageFilter{
 		HasPeer: true,
 		Peer:    domain.Peer{Type: domain.PeerTypeUser, ID: domain.StickersBotUserID},
 		Limit:   100,
@@ -44,9 +49,9 @@ func sendMessageToStickers(t *testing.T, svc *Service, messages *memory.MessageS
 		}
 	}
 	if latest.ID == 0 {
-		t.Fatalf("no Stickers reply after message %+v", msg)
+		t.Fatal("no Stickers reply")
 	}
-	return latest.Body
+	return latest
 }
 
 func newStickersBotTestService(t *testing.T) (*Service, *memory.UserStore, *memory.BotStore, *memory.MessageStore, *stickersBotFakeCreator, *stickersBotFakeInstaller) {
@@ -138,6 +143,10 @@ func TestStickersBotSystemSeedStartAndCancel(t *testing.T) {
 	if reply := sendTextToStickers(t, svc, messages, owner, "/start"); !strings.Contains(reply, "/newpack") || !strings.Contains(reply, "/newemoji") || !strings.Contains(reply, "/addsticker") {
 		t.Fatalf("/start reply = %q, want help text", reply)
 	}
+	startReply := latestStickersReply(t, messages, owner.ID)
+	assertReplyEntityText(t, startReply, domain.MessageEntityBotCommand, "/newpack")
+	assertReplyEntityText(t, startReply, domain.MessageEntityBotCommand, "/newemoji")
+	assertReplyEntityText(t, startReply, domain.MessageEntityBotCommand, "/addsticker")
 	sendTextToStickers(t, svc, messages, owner, "/newpack")
 	if reply := sendTextToStickers(t, svc, messages, owner, "/cancel"); !strings.Contains(reply, "Cancelled") {
 		t.Fatalf("/cancel reply = %q, want cancelled", reply)
@@ -190,6 +199,8 @@ func TestStickersBotPublishStickerPack(t *testing.T) {
 	if !strings.Contains(reply, "https://telesrv.net/addstickers/fresh_pack") {
 		t.Fatalf("publish reply = %q, want addstickers link", reply)
 	}
+	publishReply := latestStickersReply(t, messages, owner.ID)
+	assertReplyEntityText(t, publishReply, domain.MessageEntityURL, "https://telesrv.net/addstickers/fresh_pack")
 	if len(creator.created) != 1 {
 		t.Fatalf("created requests = %d, want 1", len(creator.created))
 	}
@@ -454,6 +465,22 @@ func botCommandExists(commands []domain.BotCommand, want string) bool {
 	return false
 }
 
+func assertReplyEntityText(t *testing.T, msg domain.Message, typ domain.MessageEntityType, want string) {
+	t.Helper()
+	for _, entity := range msg.Entities {
+		if entity.Type != typ {
+			continue
+		}
+		if entity.Offset < 0 || entity.Length < 0 || entity.Offset+entity.Length > len(msg.Body) {
+			t.Fatalf("entity %+v out of ASCII bounds for %q", entity, msg.Body)
+		}
+		if got := msg.Body[entity.Offset : entity.Offset+entity.Length]; got == want {
+			return
+		}
+	}
+	t.Fatalf("message %q entities %+v missing %s entity for %q", msg.Body, msg.Entities, typ, want)
+}
+
 type stickersBotFakeCreator struct {
 	created []domain.CreateStickerSetRequest
 	sets    []domain.StickerSet

internal/app/langpack/parser_test.go

@@ -96,3 +96,27 @@ func TestSeedDirectoryWalksClientSubdirs(t *testing.T) {
 		t.Fatalf("weba pack = %+v", pack)
 	}
 }
+
+func TestBundledAndroidPersianLangPackParses(t *testing.T) {
+	path := filepath.Join("..", "..", "..", "data", "langpack", "android", "android_fa_v59634849.strings")
+	pack, err := ParseTDesktopFile(path)
+	if err != nil {
+		t.Fatalf("parse bundled android fa pack: %v", err)
+	}
+	if pack.LangPack != "android" || pack.LangCode != "fa" || pack.Version != 59634849 {
+		t.Fatalf("pack meta = %+v, want android/fa v59634849", pack)
+	}
+	if len(pack.Strings) < 10000 {
+		t.Fatalf("strings count = %d, want full android fa pack", len(pack.Strings))
+	}
+	wantPersian := "\u0641\u0627\u0631\u0633\u06cc"
+	for _, item := range pack.Strings {
+		if item.Key == "TranslateLanguageFA" {
+			if item.Value != wantPersian {
+				t.Fatalf("TranslateLanguageFA = %q, want %q", item.Value, wantPersian)
+			}
+			return
+		}
+	}
+	t.Fatalf("TranslateLanguageFA not found in bundled android fa pack")
+}