feat: sync langpack and auth updates

Sync telesrv commits 49f9bab and 04d9563 into the public mirror. Exclude private docs and runtime key material per sync rules.
This commit is contained in:
A 2026-07-18 00:07:41 +08:00
parent 6af61f26ba
commit 1292540350
244 changed files with 1552172 additions and 27668 deletions

View file

@ -34,7 +34,7 @@ func (s *testMailSender) Deliver(_ context.Context, req otpdelivery.Request) (ot
return otpdelivery.Result{}, nil
}
func TestConfiguredEmailLoginSendsAndLimitsAttempts(t *testing.T) {
func TestConfiguredEmailLoginSharesAttemptsAcrossOfficialCodeCarriers(t *testing.T) {
ctx := context.Background()
users := memory.NewUserStore()
authz := memory.NewAuthorizationStore()
@ -76,14 +76,14 @@ func TestConfiguredEmailLoginSendsAndLimitsAttempts(t *testing.T) {
if bad2 == bad1 {
bad2 = wrongCode(sender.code, '2')
}
if _, _, _, err := svc.SignInWithEmail(ctx, domain.Authorization{}, "+15550009101", hash, bad1); !errors.Is(err, ErrCodeInvalid) {
t.Fatalf("first bad SignInWithEmail err = %v, want ErrCodeInvalid", err)
if _, _, _, err := svc.SignIn(ctx, domain.Authorization{}, "+15550009101", hash, bad1); !errors.Is(err, ErrCodeInvalid) {
t.Fatalf("first bad WebK SignIn err = %v, want ErrCodeInvalid", err)
}
if _, _, _, err := svc.SignInWithEmail(ctx, domain.Authorization{}, "+15550009101", hash, bad2); !errors.Is(err, ErrCodeInvalid) {
t.Fatalf("second bad SignInWithEmail err = %v, want ErrCodeInvalid", err)
t.Fatalf("second bad native SignInWithEmail err = %v, want ErrCodeInvalid", err)
}
if _, _, _, err := svc.SignInWithEmail(ctx, domain.Authorization{}, "+15550009101", hash, sender.code); !errors.Is(err, ErrCodeExpired) {
t.Fatalf("SignInWithEmail after max attempts err = %v, want ErrCodeExpired", err)
if _, _, _, err := svc.SignIn(ctx, domain.Authorization{}, "+15550009101", hash, sender.code); !errors.Is(err, ErrCodeExpired) {
t.Fatalf("WebK SignIn after shared max attempts err = %v, want ErrCodeExpired", err)
}
}
@ -104,40 +104,160 @@ func wrongCode(code string, digit byte) string {
return string(out)
}
func TestConfiguredEmailLoginAcceptsCorrectCode(t *testing.T) {
func TestConfiguredEmailLoginAcceptsOfficialCodeCarriers(t *testing.T) {
tests := []struct {
name string
phone string
email string
webK bool
}{
{name: "webk_phone_code", phone: "15550009102", email: "webk@example.test", webK: true},
{name: "native_email_verification", phone: "15550009103", email: "native@example.test"},
}
for i, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
ctx := context.Background()
users := memory.NewUserStore()
authz := memory.NewAuthorizationStore()
u, err := users.Create(ctx, domain.User{Phone: tc.phone, FirstName: "Email"})
if err != nil {
t.Fatalf("create user: %v", err)
}
emails := &testLoginEmailStore{emails: map[string]string{tc.phone: tc.email}}
sender := &testMailSender{}
appDelivery := &captureLoginCodeDelivery{}
var key [8]byte
key[0] = byte(0x91 + i)
svc := NewService(users, authz, memory.NewCodeStore(), nil, nil, "12345",
WithLoginCodeDelivery(appDelivery),
WithLoginEmail(LoginEmailOptions{
Enabled: true,
CodeLength: 6,
Store: emails,
Sender: sender,
}))
hash, err := svc.SendCode(ctx, tc.phone)
if err != nil {
t.Fatalf("SendCode: %v", err)
}
if len(appDelivery.requests) != 1 || appDelivery.requests[0].Code != sender.code {
t.Fatalf("App-code delivery=%+v, want same email code", appDelivery.requests)
}
var got domain.User
var needSignUp bool
if tc.webK {
if _, _, _, err := svc.SignIn(ctx, domain.Authorization{AuthKeyID: key}, tc.phone, hash, "12345"); !errors.Is(err, ErrCodeInvalid) {
t.Fatalf("WebK development code err=%v, want ErrCodeInvalid for random email channel", err)
}
got, _, needSignUp, err = svc.SignIn(ctx, domain.Authorization{AuthKeyID: key}, tc.phone, hash, sender.code)
} else {
got, _, needSignUp, err = svc.SignInWithEmail(ctx, domain.Authorization{AuthKeyID: key}, tc.phone, hash, sender.code)
}
if err != nil {
t.Fatalf("sign in: %v", err)
}
if needSignUp || got.ID != u.ID {
t.Fatalf("sign in got user=%d needSignUp=%v, want %d/false", got.ID, needSignUp, u.ID)
}
})
}
}
func TestConfiguredEmailLoginViaWebKStillHonorsTwoFactor(t *testing.T) {
ctx := context.Background()
users := memory.NewUserStore()
authz := memory.NewAuthorizationStore()
u, err := users.Create(ctx, domain.User{Phone: "15550009102", FirstName: "Email"})
passwords := memory.NewPasswordStore()
u, err := users.Create(ctx, domain.User{Phone: "15550009104", FirstName: "Email"})
if err != nil {
t.Fatalf("create user: %v", err)
}
emails := &testLoginEmailStore{emails: map[string]string{"15550009102": "bob@example.test"}}
if err := passwords.Save(ctx, u.ID, domain.PasswordSettings{HasPassword: true}); err != nil {
t.Fatalf("save password settings: %v", err)
}
sender := &testMailSender{}
appDelivery := &captureLoginCodeDelivery{}
var key [8]byte
key[0] = 0x91
svc := NewService(users, authz, memory.NewCodeStore(), nil, nil, "12345",
WithLoginCodeDelivery(appDelivery),
WithPasswords(passwords),
WithLoginCodeDelivery(&captureLoginCodeDelivery{}),
WithLoginEmail(LoginEmailOptions{
Enabled: true,
CodeLength: 5,
Store: emails,
CodeLength: 6,
Store: &testLoginEmailStore{emails: map[string]string{u.Phone: "2fa@example.test"}},
Sender: sender,
}))
var key [8]byte
key[0] = 0x94
hash, err := svc.SendCode(ctx, "+15550009102")
hash, err := svc.SendCode(ctx, u.Phone)
if err != nil {
t.Fatalf("SendCode: %v", err)
}
if len(appDelivery.requests) != 1 || appDelivery.requests[0].Code != sender.code {
t.Fatalf("App-code delivery=%+v, want same email code", appDelivery.requests)
got, _, _, err := svc.SignIn(ctx, domain.Authorization{AuthKeyID: key}, u.Phone, hash, sender.code)
if !errors.Is(err, domain.ErrSessionPasswordNeeded) {
t.Fatalf("WebK email SignIn err=%v, want ErrSessionPasswordNeeded", err)
}
got, _, needSignUp, err := svc.SignInWithEmail(ctx, domain.Authorization{AuthKeyID: key}, "+15550009102", hash, sender.code)
if err != nil {
t.Fatalf("SignInWithEmail: %v", err)
if got.ID != u.ID {
t.Fatalf("WebK email SignIn user=%d, want pending 2FA user %d", got.ID, u.ID)
}
if needSignUp || got.ID != u.ID {
t.Fatalf("SignInWithEmail got user=%d needSignUp=%v, want %d/false", got.ID, needSignUp, u.ID)
if bound, found, err := svc.UserID(ctx, key); err != nil || found || bound != 0 {
t.Fatalf("UserID after WebK email SignIn with 2FA=%d found=%v err=%v, want not-found", bound, found, err)
}
}
func TestConfiguredEmailLoginHasSingleConsumerAcrossOfficialCodeCarriers(t *testing.T) {
ctx := context.Background()
users := memory.NewUserStore()
u, err := users.Create(ctx, domain.User{Phone: "15550009105", FirstName: "Email"})
if err != nil {
t.Fatalf("create user: %v", err)
}
sender := &testMailSender{}
svc := NewService(users, memory.NewAuthorizationStore(), memory.NewCodeStore(), nil, nil, "12345",
WithLoginCodeDelivery(&captureLoginCodeDelivery{}),
WithLoginEmail(LoginEmailOptions{
Enabled: true,
CodeLength: 6,
Store: &testLoginEmailStore{emails: map[string]string{u.Phone: "race@example.test"}},
Sender: sender,
}))
hash, err := svc.SendCode(ctx, u.Phone)
if err != nil {
t.Fatalf("SendCode: %v", err)
}
start := make(chan struct{})
results := make(chan error, 2)
var webKKey, nativeKey [8]byte
webKKey[0] = 0x95
nativeKey[0] = 0x96
go func() {
<-start
_, _, _, err := svc.SignIn(ctx, domain.Authorization{AuthKeyID: webKKey}, u.Phone, hash, sender.code)
results <- err
}()
go func() {
<-start
_, _, _, err := svc.SignInWithEmail(ctx, domain.Authorization{AuthKeyID: nativeKey}, u.Phone, hash, sender.code)
results <- err
}()
close(start)
accepted, expired := 0, 0
for range 2 {
err := <-results
switch {
case err == nil:
accepted++
case errors.Is(err, ErrCodeExpired):
expired++
default:
t.Fatalf("concurrent sign in err=%v, want nil or ErrCodeExpired", err)
}
}
if accepted != 1 || expired != 1 {
t.Fatalf("concurrent results accepted=%d expired=%d, want 1/1", accepted, expired)
}
}

View file

@ -895,7 +895,7 @@ func (s *Service) SignIn(ctx context.Context, auth domain.Authorization, phone,
if systemLoginPhoneForbidden(phone) {
return domain.User{}, domain.Message{}, false, ErrSystemUserLoginForbidden
}
_, existing, found, err := s.verifyLoginCode(ctx, phone, phoneCodeHash, code, false)
_, existing, found, err := s.verifyLoginCode(ctx, phone, phoneCodeHash, code)
if err != nil {
return domain.User{}, domain.Message{}, false, err
}
@ -905,17 +905,17 @@ func (s *Service) SignIn(ctx context.Context, auth domain.Authorization, phone,
return s.finishSignIn(ctx, auth, existing)
}
// SignInWithEmail 处理带 email_verification 的 auth.signIn:账号设置了登录邮箱后,新设备
// 的验证码改投递到邮箱,客户端凭邮箱码(而非短信码)登录。开启真实登录邮箱后必须匹配
// 随机邮箱码;未开启该特性时仍允许旧客户端把 phone channel 放进
// email_verification但必须精确匹配该 phone code不能再接受任意非空值。
// 两条路径共用 owner 绑定、原子尝试计数与 2FA 门控。
// SignInWithEmail 处理带 email_verification 的 auth.signIn。它与 SignIn
// 共享同一个登录凭证状态机TDesktop/Android 把邮箱码放在
// email_verificationWebK 把同一邮箱码放在 phone_codeTL 字段只是 proof
// carrier服务端签发记录的 channel 才表示实际投递渠道。所有渠道都必须精确
// 匹配签发码,并共用 owner 绑定、原子尝试计数、一次性消费与 2FA 门控。
func (s *Service) SignInWithEmail(ctx context.Context, auth domain.Authorization, phone, phoneCodeHash, code string) (domain.User, domain.Message, bool, error) {
phone = normalizePhone(phone)
if systemLoginPhoneForbidden(phone) {
return domain.User{}, domain.Message{}, false, ErrSystemUserLoginForbidden
}
_, existing, found, err := s.verifyLoginCode(ctx, phone, phoneCodeHash, strings.TrimSpace(code), true)
_, existing, found, err := s.verifyLoginCode(ctx, phone, phoneCodeHash, strings.TrimSpace(code))
if err != nil {
return domain.User{}, domain.Message{}, false, err
}
@ -929,7 +929,7 @@ func (s *Service) SignInWithEmail(ctx context.Context, auth domain.Authorization
// CodeStore verification. The phone owner is read both before and after that
// linearization point. A hash issued for an unregistered number therefore can
// never authorize whichever account happens to acquire that number later.
func (s *Service) verifyLoginCode(ctx context.Context, phone, phoneCodeHash, code string, emailPath bool) (store.PhoneCode, domain.User, bool, error) {
func (s *Service) verifyLoginCode(ctx context.Context, phone, phoneCodeHash, code string) (store.PhoneCode, domain.User, bool, error) {
rec, found, err := s.codes.Get(ctx, phoneCodeHash)
if err != nil {
return store.PhoneCode{}, domain.User{}, false, err
@ -944,11 +944,7 @@ func (s *Service) verifyLoginCode(ctx context.Context, phone, phoneCodeHash, cod
if rec.Phone != phone || rec.Purpose != "" {
return store.PhoneCode{}, domain.User{}, false, ErrCodeInvalid
}
channelAllowed := (rec.Channel == codeChannelPhone || rec.Channel == codeChannelSMS) && !emailPath
if emailPath {
channelAllowed = rec.Channel == codeChannelEmailLogin || (!s.loginEmailEnabled && rec.Channel == codeChannelPhone)
}
if !channelAllowed {
if !store.LoginCodeChannelVerifiable(rec.Channel) {
return store.PhoneCode{}, domain.User{}, false, ErrCodeInvalid
}
@ -1087,7 +1083,7 @@ func (s *Service) SignUp(ctx context.Context, auth domain.Authorization, phone,
s.invalidateLoginCodeDetached(ctx, phoneCodeHash, phone)
return domain.User{}, domain.Message{}, ErrCodeInvalid
}
if rec.Channel != codeChannelPhone && rec.Channel != codeChannelSMS && rec.Channel != codeChannelEmailLogin {
if !store.LoginCodeChannelVerifiable(rec.Channel) {
return domain.User{}, domain.Message{}, ErrCodeInvalid
}
if s.loginEmailRequireSetup && !rec.VerifiedEmail && strings.TrimSpace(rec.PendingEmail) == "" {
@ -1107,7 +1103,7 @@ func (s *Service) SignUp(ctx context.Context, auth domain.Authorization, phone,
return domain.User{}, domain.Message{}, ErrCodeExpired
}
rec = consumed
if rec.IssuedUserID != 0 || !rec.SignUpVerified || (rec.Channel != codeChannelPhone && rec.Channel != codeChannelSMS && rec.Channel != codeChannelEmailLogin) {
if rec.IssuedUserID != 0 || !rec.SignUpVerified || !store.LoginCodeChannelVerifiable(rec.Channel) {
return domain.User{}, domain.Message{}, ErrCodeInvalid
}
if current, currentFound, err := s.currentPhoneOwner(ctx, phone); err != nil {

View file

@ -341,6 +341,12 @@ func TestEmailSetupVerificationAuthorizesSignUpWithout777000Message(t *testing.T
if _, _, err := authSvc.SignUp(ctx, domain.Authorization{}, phone, hash, "Direct", "Email"); !errors.Is(err, ErrCodeInvalid) {
t.Fatalf("SignUp before email setup err=%v, want ErrCodeInvalid", err)
}
if _, _, _, err := authSvc.SignIn(ctx, domain.Authorization{}, phone, hash, "12345"); !errors.Is(err, ErrCodeInvalid) {
t.Fatalf("WebK SignIn with setup-required placeholder err=%v, want ErrCodeInvalid", err)
}
if _, _, _, err := authSvc.SignInWithEmail(ctx, domain.Authorization{}, phone, hash, "12345"); !errors.Is(err, ErrCodeInvalid) {
t.Fatalf("native SignInWithEmail with setup-required placeholder err=%v, want ErrCodeInvalid", err)
}
if _, _, err := accountSvc.SendLoginEmailCode(ctx, 0, phone, hash, "new@example.test", true); err != nil {
t.Fatalf("SendLoginEmailCode: %v", err)
}

View file

@ -0,0 +1,238 @@
package langpack
import (
"container/list"
"sync"
"telesrv/internal/domain"
)
const (
defaultLangPackCacheMaxBytes = int64(128 << 20)
defaultLangPackCacheMaxEntries = 256
defaultLanguageListCacheMaxEntries = 32
langPackStringValueHeaderBytes = int64(144)
langPackFixedHeaderBytes = int64(80)
)
type langPackCacheKind uint8
const (
langPackCacheRaw langPackCacheKind = iota
langPackCacheEffective
)
type langPackCacheKey struct {
pack string
code string
kind langPackCacheKind
}
func (k langPackCacheKey) singleflightKey() string {
return string(rune(k.kind)) + "\x00" + k.pack + "\x00" + k.code
}
type langPackCache struct {
mu sync.Mutex
maxBytes int64
maxEntries int
usedBytes int64
epoch uint64
ll *list.List
items map[langPackCacheKey]*list.Element
}
type langPackCacheEntry struct {
key langPackCacheKey
pack domain.LangPack
size int64
}
func newLangPackCache(maxBytes int64, maxEntries int) *langPackCache {
if maxBytes <= 0 || maxEntries <= 0 {
return nil
}
return &langPackCache{
maxBytes: maxBytes,
maxEntries: maxEntries,
ll: list.New(),
items: make(map[langPackCacheKey]*list.Element),
}
}
func (c *langPackCache) get(key langPackCacheKey) (domain.LangPack, bool) {
if c == nil {
return domain.LangPack{}, false
}
c.mu.Lock()
defer c.mu.Unlock()
element, ok := c.items[key]
if !ok {
return domain.LangPack{}, false
}
c.ll.MoveToFront(element)
return cloneLangPack(element.Value.(*langPackCacheEntry).pack), true
}
func (c *langPackCache) loadEpoch() uint64 {
if c == nil {
return 0
}
c.mu.Lock()
epoch := c.epoch
c.mu.Unlock()
return epoch
}
// putIfEpoch 返回 false 仅表示 load 期间发生过 flush调用方必须重载。
// 超大单项不进入缓存,但仍可安全返回给当前请求,因此返回 true。
func (c *langPackCache) putIfEpoch(key langPackCacheKey, pack domain.LangPack, loadEpoch uint64) bool {
if c == nil {
return true
}
size := estimateLangPackBytes(pack)
c.mu.Lock()
defer c.mu.Unlock()
if c.epoch != loadEpoch {
return false
}
if existing, ok := c.items[key]; ok {
c.remove(existing)
}
if size > c.maxBytes {
return true
}
entry := &langPackCacheEntry{key: key, pack: cloneLangPack(pack), size: size}
c.items[key] = c.ll.PushFront(entry)
c.usedBytes += size
for c.usedBytes > c.maxBytes || c.ll.Len() > c.maxEntries {
oldest := c.ll.Back()
if oldest == nil {
break
}
c.remove(oldest)
}
return true
}
func (c *langPackCache) flush() {
if c == nil {
return
}
c.mu.Lock()
c.epoch++
c.usedBytes = 0
c.ll.Init()
clear(c.items)
c.mu.Unlock()
}
func (c *langPackCache) remove(element *list.Element) {
entry := element.Value.(*langPackCacheEntry)
delete(c.items, entry.key)
c.ll.Remove(element)
c.usedBytes -= entry.size
}
func estimateLangPackBytes(pack domain.LangPack) int64 {
size := langPackFixedHeaderBytes + int64(len(pack.LangPack)+len(pack.LangCode))
size += int64(len(pack.Strings)) * langPackStringValueHeaderBytes
for _, item := range pack.Strings {
size += int64(len(item.Key) + len(item.Value) + len(item.ZeroValue) + len(item.OneValue) +
len(item.TwoValue) + len(item.FewValue) + len(item.ManyValue) + len(item.OtherValue))
}
return size
}
func cloneLangPack(pack domain.LangPack) domain.LangPack {
pack.Strings = append([]domain.LangPackString(nil), pack.Strings...)
return pack
}
type languageListCache struct {
mu sync.Mutex
maxEntries int
epoch uint64
ll *list.List
items map[string]*list.Element
}
type languageListCacheEntry struct {
pack string
languages []domain.LangPackLanguage
}
func newLanguageListCache(maxEntries int) *languageListCache {
if maxEntries <= 0 {
return nil
}
return &languageListCache{
maxEntries: maxEntries,
ll: list.New(),
items: make(map[string]*list.Element),
}
}
func (c *languageListCache) get(pack string) ([]domain.LangPackLanguage, bool) {
if c == nil {
return nil, false
}
c.mu.Lock()
defer c.mu.Unlock()
element, ok := c.items[pack]
if !ok {
return nil, false
}
c.ll.MoveToFront(element)
return cloneLanguages(element.Value.(*languageListCacheEntry).languages), true
}
func (c *languageListCache) loadEpoch() uint64 {
if c == nil {
return 0
}
c.mu.Lock()
epoch := c.epoch
c.mu.Unlock()
return epoch
}
func (c *languageListCache) putIfEpoch(pack string, languages []domain.LangPackLanguage, loadEpoch uint64) bool {
if c == nil {
return true
}
c.mu.Lock()
defer c.mu.Unlock()
if c.epoch != loadEpoch {
return false
}
if existing, ok := c.items[pack]; ok {
c.ll.Remove(existing)
delete(c.items, pack)
}
entry := &languageListCacheEntry{pack: pack, languages: cloneLanguages(languages)}
c.items[pack] = c.ll.PushFront(entry)
if c.ll.Len() > c.maxEntries {
oldest := c.ll.Back()
if oldest != nil {
delete(c.items, oldest.Value.(*languageListCacheEntry).pack)
c.ll.Remove(oldest)
}
}
return true
}
func (c *languageListCache) flush() {
if c == nil {
return
}
c.mu.Lock()
c.epoch++
c.ll.Init()
clear(c.items)
c.mu.Unlock()
}
func cloneLanguages(languages []domain.LangPackLanguage) []domain.LangPackLanguage {
return append([]domain.LangPackLanguage(nil), languages...)
}

View file

@ -0,0 +1,35 @@
package langpack
import (
"testing"
"telesrv/internal/domain"
)
func TestLangPackCachesRejectLoadsAcrossFlush(t *testing.T) {
packCache := newLangPackCache(1<<20, 8)
key := langPackCacheKey{pack: "tdesktop", code: "en", kind: langPackCacheRaw}
packEpoch := packCache.loadEpoch()
packCache.flush()
if packCache.putIfEpoch(key, domain.LangPack{
LangPack: "tdesktop",
LangCode: "en",
Version: 1,
Strings: []domain.LangPackString{{Key: "key", Value: "stale"}},
}, packEpoch) {
t.Fatal("pre-flush pack load was accepted")
}
if _, ok := packCache.get(key); ok {
t.Fatal("pre-flush pack load became visible")
}
languageCache := newLanguageListCache(8)
languageEpoch := languageCache.loadEpoch()
languageCache.flush()
if languageCache.putIfEpoch("tdesktop", []domain.LangPackLanguage{{LangCode: "en"}}, languageEpoch) {
t.Fatal("pre-flush language-list load was accepted")
}
if _, ok := languageCache.get("tdesktop"); ok {
t.Fatal("pre-flush language-list load became visible")
}
}

View file

@ -2,6 +2,7 @@ package langpack
import (
"fmt"
"math"
"os"
"path/filepath"
"regexp"
@ -12,6 +13,8 @@ import (
)
var tdesktopStringRE = regexp.MustCompile(`(?s)"((?:\\.|[^"\\])*)"\s*=\s*"((?:\\.|[^"\\])*)";`)
var langPackNameRE = regexp.MustCompile(`^[a-z0-9][a-z0-9_-]{0,31}$`)
var langCodeRE = regexp.MustCompile(`^[a-z0-9]{1,16}(?:-[a-z0-9]{1,16})*$`)
// ParseTDesktopFile 解析客户端 .strings 文件为 domain 语言包。
func ParseTDesktopFile(path string) (domain.LangPack, error) {
@ -82,6 +85,17 @@ func packFromFilename(path string) (domain.LangPack, error) {
if langPack == "" || langCode == "" {
return domain.LangPack{}, fmt.Errorf("invalid langpack filename %q", filepath.Base(path))
}
langPack = normalizePack(langPack)
langCode = normalizeCode(langCode)
if !langPackNameRE.MatchString(langPack) {
return domain.LangPack{}, fmt.Errorf("invalid langpack name %q in %q", langPack, filepath.Base(path))
}
if len(langCode) > 64 || !langCodeRE.MatchString(langCode) {
return domain.LangPack{}, fmt.Errorf("invalid language code %q in %q", langCode, filepath.Base(path))
}
if version <= 0 || version > math.MaxInt32 {
return domain.LangPack{}, fmt.Errorf("invalid langpack version %d in %q", version, filepath.Base(path))
}
return domain.LangPack{
LangPack: langPack,
LangCode: langCode,

View file

@ -4,6 +4,7 @@ import (
"context"
"os"
"path/filepath"
"strings"
"testing"
"telesrv/internal/store/memory"
@ -59,6 +60,23 @@ func TestParseClientLangPackFile(t *testing.T) {
}
}
func TestParseClientLangPackFileWithUnderscorePackName(t *testing.T) {
packDir := filepath.Join(t.TempDir(), "android_x")
if err := os.MkdirAll(packDir, 0o700); err != nil {
t.Fatalf("mkdir fixture: %v", err)
}
path := filepath.Join(packDir, "android_x_en_v42.strings")
writeLangPackFixture(t, path, `"TranslationMoreText" = "Translation Platform";`)
pack, err := ParseTDesktopFile(path)
if err != nil {
t.Fatalf("parse: %v", err)
}
if pack.LangPack != "android_x" || pack.LangCode != "en" || pack.Version != 42 {
t.Fatalf("pack meta = %+v", pack)
}
}
func TestSeedDirectoryWalksClientSubdirs(t *testing.T) {
root := t.TempDir()
for _, item := range []struct {
@ -98,13 +116,21 @@ func TestSeedDirectoryWalksClientSubdirs(t *testing.T) {
}
func TestBundledAndroidPersianLangPackParses(t *testing.T) {
path := filepath.Join("..", "..", "..", "data", "langpack", "android", "android_fa_v59634849.strings")
pack, err := ParseTDesktopFile(path)
root := filepath.Join("..", "..", "..", "data", "langpack", "android")
candidates, _, err := scanSeedCandidates(root)
if err != nil {
t.Fatalf("scan bundled android packs: %v", err)
}
candidate, ok := candidates["android\x00fa"]
if !ok {
t.Fatal("bundled android fa pack not found")
}
pack, err := ParseTDesktopFile(candidate.path)
if err != nil {
t.Fatalf("parse bundled android fa pack: %v", err)
}
if pack.LangPack != "android" || pack.LangCode != "fa" || pack.Version != 59634849 {
t.Fatalf("pack meta = %+v, want android/fa v59634849", pack)
if pack.LangPack != "android" || pack.LangCode != "fa" || pack.Version <= 0 {
t.Fatalf("pack meta = %+v, want versioned android/fa", pack)
}
if len(pack.Strings) < 10000 {
t.Fatalf("strings count = %d, want full android fa pack", len(pack.Strings))
@ -120,3 +146,120 @@ func TestBundledAndroidPersianLangPackParses(t *testing.T) {
}
t.Fatalf("TranslateLanguageFA not found in bundled android fa pack")
}
func TestSeedDirectoryReconcilesManifest(t *testing.T) {
ctx := context.Background()
root := t.TempDir()
packDir := filepath.Join(root, "tdesktop")
if err := os.MkdirAll(packDir, 0o700); err != nil {
t.Fatalf("mkdir pack dir: %v", err)
}
v1 := filepath.Join(packDir, "tdesktop_pt-BR_v1.strings")
writeLangPackFixture(t, v1, `
"lng_language_name" = "Português (Brasil)";
"lng_old" = "old";
`)
store := memory.NewLangPackStore()
service := NewService(store)
seeded, err := service.SeedDirectory(ctx, root)
if err != nil || seeded != 2 {
t.Fatalf("seed v1 = %d, %v", seeded, err)
}
pack, err := service.GetLangPack(ctx, "TDESKTOP", "pt_BR")
if err != nil || pack.LangCode != "pt-br" || pack.Version != 1 || len(pack.Strings) != 2 {
t.Fatalf("normalized pack = %+v, err %v", pack, err)
}
languages, err := service.ListLanguages(ctx, "tdesktop")
if err != nil || findLanguage(languages, "pt-br") == nil {
t.Fatalf("languages = %+v, err %v", languages, err)
}
if seeded, err := service.SeedDirectory(ctx, root); err != nil || seeded != 0 {
t.Fatalf("unchanged seed = %d, %v", seeded, err)
}
v2 := filepath.Join(packDir, "tdesktop_pt-br_v2.strings")
writeLangPackFixture(t, v2, `
"lng_language_name" = "Português do Brasil";
"lng_new" = "new";
`)
seeded, err = service.SeedDirectory(ctx, root)
if err != nil || seeded != 2 {
t.Fatalf("seed v2 = %d, %v", seeded, err)
}
pack, err = service.GetLangPack(ctx, "tdesktop", "pt-br")
if err != nil || pack.Version != 2 || len(pack.Strings) != 2 || stringValue(pack.Strings, "lng_old") != "" || stringValue(pack.Strings, "lng_new") != "new" {
t.Fatalf("replaced pack = %+v, err %v", pack, err)
}
if err := os.Remove(v1); err != nil {
t.Fatalf("remove v1: %v", err)
}
if err := os.Remove(v2); err != nil {
t.Fatalf("remove v2: %v", err)
}
if err := os.Remove(packDir); err != nil {
t.Fatalf("remove pack dir: %v", err)
}
if seeded, err := service.SeedDirectory(ctx, root); err != nil || seeded != 0 {
t.Fatalf("reconcile removed file = %d, %v", seeded, err)
}
languages, err = service.ListLanguages(ctx, "tdesktop")
if err != nil || len(languages) != 0 {
t.Fatalf("languages after removal = %+v, err %v", languages, err)
}
}
func TestSeedDirectoryRejectsVersionInvariantViolations(t *testing.T) {
ctx := context.Background()
root := t.TempDir()
packDir := filepath.Join(root, "tdesktop")
if err := os.MkdirAll(packDir, 0o700); err != nil {
t.Fatalf("mkdir pack dir: %v", err)
}
v2 := filepath.Join(packDir, "tdesktop_fr_v2.strings")
writeLangPackFixture(t, v2, `"lng_language_name" = "Français";`)
service := NewService(memory.NewLangPackStore())
if _, err := service.SeedDirectory(ctx, root); err != nil {
t.Fatalf("seed v2: %v", err)
}
writeLangPackFixture(t, v2, `"lng_language_name" = "Français modifié";`)
if _, err := service.SeedDirectory(ctx, root); err == nil || !strings.Contains(err.Error(), "without version bump") {
t.Fatalf("same-version mutation error = %v", err)
}
pack, err := service.GetLangPack(ctx, "tdesktop", "fr")
if err != nil || stringValue(pack.Strings, "lng_language_name") != "Français" {
t.Fatalf("pack changed after rejected mutation = %+v, err %v", pack, err)
}
if err := os.Remove(v2); err != nil {
t.Fatalf("remove v2: %v", err)
}
writeLangPackFixture(t, filepath.Join(packDir, "tdesktop_fr_v1.strings"), `"lng_language_name" = "Français";`)
if _, err := service.SeedDirectory(ctx, root); err == nil || !strings.Contains(err.Error(), "version rollback") {
t.Fatalf("version rollback error = %v", err)
}
}
func TestBundledLangPackDirectoryReconciles(t *testing.T) {
root := filepath.Join("..", "..", "..", "data", "langpack")
service := NewService(memory.NewLangPackStore())
seeded, err := service.SeedDirectory(context.Background(), root)
if err != nil {
t.Fatalf("seed bundled langpacks: %v", err)
}
if seeded < 50000 {
t.Fatalf("seeded bundled strings = %d, want full catalog", seeded)
}
if seeded, err := service.SeedDirectory(context.Background(), root); err != nil || seeded != 0 {
t.Fatalf("reconcile unchanged bundled langpacks = %d, %v", seeded, err)
}
}
func writeLangPackFixture(t *testing.T, path, content string) {
t.Helper()
if err := os.WriteFile(path, []byte(content), 0o600); err != nil {
t.Fatalf("write langpack fixture %q: %v", path, err)
}
}

View file

@ -2,14 +2,27 @@ package langpack
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"os"
"path/filepath"
"sort"
"strings"
"unicode/utf8"
"telesrv/internal/domain"
)
// SeedDirectory 将导出的 .strings 文件导入 LangPackStore。
type seedCandidate struct {
path string
meta domain.LangPack
}
// SeedDirectory 将导出的 .strings 文件清单原子对账到 LangPackStore。
// root 可直接指向 data/langpack也可指向包含 .strings 的具体平台目录。
func (s *Service) SeedDirectory(ctx context.Context, root string) (int, error) {
if s == nil || s.packs == nil || root == "" {
@ -23,33 +36,201 @@ func (s *Service) SeedDirectory(ctx context.Context, root string) (int, error) {
return 0, fmt.Errorf("stat langpack seed dir: %w", err)
}
seeded := 0
err := filepath.WalkDir(dir, func(path string, entry os.DirEntry, err error) error {
candidates, scopes, err := scanSeedCandidates(dir)
if err != nil {
return 0, fmt.Errorf("scan langpack seed dir: %w", err)
}
keys := make([]string, 0, len(candidates))
for key := range candidates {
keys = append(keys, key)
}
sort.Strings(keys)
seed := domain.LangPackSeed{
Catalog: seedCatalogID(dir),
Scopes: scopes,
Packs: make([]domain.LangPackSeedEntry, 0, len(keys)),
}
previous, err := s.packs.GetSeedCatalog(ctx, seed.Catalog)
if err != nil {
return 0, fmt.Errorf("get previous langpack seed catalog: %w", err)
}
previousByKey := make(map[string]domain.LangPackSeedCatalogEntry, len(previous.Packs))
for _, entry := range previous.Packs {
previousByKey[entry.LangPack+"\x00"+entry.LangCode] = entry
}
for _, key := range keys {
candidate := candidates[key]
sourceHash, err := fileSHA256(candidate.path)
if err != nil {
return 0, fmt.Errorf("hash langpack source %q: %w", candidate.path, err)
}
if old, ok := previousByKey[key]; ok &&
old.Version == candidate.meta.Version &&
old.SourceHash == sourceHash &&
old.ContentHash != "" && old.StringsCount > 0 {
seed.Packs = append(seed.Packs, domain.LangPackSeedEntry{
Pack: candidate.meta,
SourceHash: sourceHash,
ContentHash: old.ContentHash,
StringsCount: old.StringsCount,
ContentLoaded: false,
})
continue
}
pack, err := ParseTDesktopFile(candidate.path)
if err != nil {
return 0, err
}
pack, err = prepareSeedPack(pack)
if err != nil {
return 0, fmt.Errorf("prepare langpack %q: %w", candidate.path, err)
}
hash, err := langPackContentHash(pack)
if err != nil {
return 0, fmt.Errorf("hash langpack %q: %w", candidate.path, err)
}
seed.Packs = append(seed.Packs, domain.LangPackSeedEntry{
Pack: pack,
SourceHash: sourceHash,
ContentHash: hash,
StringsCount: len(pack.Strings),
ContentLoaded: true,
})
}
seeded, err := s.packs.ReconcileSeed(ctx, seed)
if err != nil {
return 0, fmt.Errorf("reconcile langpack seed: %w", err)
}
s.flushCaches()
return seeded, nil
}
func fileSHA256(path string) (string, error) {
file, err := os.Open(path)
if err != nil {
return "", err
}
defer file.Close()
hash := sha256.New()
if _, err := io.Copy(hash, file); err != nil {
return "", err
}
return hex.EncodeToString(hash.Sum(nil)), nil
}
func scanSeedCandidates(root string) (map[string]seedCandidate, []string, error) {
candidates := make(map[string]seedCandidate)
scopeSet := make(map[string]struct{})
hasChildDirs := false
hasFiles := false
err := filepath.WalkDir(root, func(path string, entry os.DirEntry, err error) error {
if err != nil {
return err
}
if entry.IsDir() || !strings.EqualFold(filepath.Ext(entry.Name()), ".strings") {
rel, err := filepath.Rel(root, path)
if err != nil {
return err
}
if entry.IsDir() {
if rel != "." && filepath.Dir(rel) == "." {
hasChildDirs = true
scope := normalizePack(entry.Name())
if langPackNameRE.MatchString(scope) {
scopeSet[scope] = struct{}{}
}
}
return nil
}
pack, err := ParseTDesktopFile(path)
if err != nil {
return err
}
existing, err := s.packs.GetPack(ctx, pack.LangPack, pack.LangCode, pack.Version)
if err != nil {
return err
}
if existing.Version >= pack.Version {
if !strings.EqualFold(filepath.Ext(entry.Name()), ".strings") {
return nil
}
if err := s.packs.UpsertPack(ctx, pack); err != nil {
hasFiles = true
meta, err := packFromFilename(path)
if err != nil {
return err
}
seeded += len(pack.Strings)
if relDir := filepath.Dir(rel); relDir != "." {
firstDir := strings.Split(relDir, string(filepath.Separator))[0]
scope := normalizePack(firstDir)
if !langPackNameRE.MatchString(scope) || scope != meta.LangPack {
return fmt.Errorf("langpack file %q is under pack directory %q", path, firstDir)
}
}
scopeSet[meta.LangPack] = struct{}{}
key := meta.LangPack + "\x00" + meta.LangCode
if previous, ok := candidates[key]; ok {
switch {
case meta.Version < previous.meta.Version:
return nil
case meta.Version == previous.meta.Version:
return fmt.Errorf("duplicate langpack version %s/%s v%d in %q and %q", meta.LangPack, meta.LangCode, meta.Version, previous.path, path)
}
}
candidates[key] = seedCandidate{path: path, meta: meta}
return nil
})
if err != nil {
return seeded, fmt.Errorf("walk langpack seed dir: %w", err)
return nil, nil, err
}
return seeded, nil
if !hasChildDirs && !hasFiles {
scope := normalizePack(filepath.Base(root))
if langPackNameRE.MatchString(scope) {
scopeSet[scope] = struct{}{}
}
}
scopes := make([]string, 0, len(scopeSet))
for scope := range scopeSet {
scopes = append(scopes, scope)
}
sort.Strings(scopes)
return candidates, scopes, nil
}
func prepareSeedPack(pack domain.LangPack) (domain.LangPack, error) {
pack.LangPack = normalizePack(pack.LangPack)
pack.LangCode = normalizeCode(pack.LangCode)
pack.FromVersion = 0
if len(pack.Strings) == 0 {
return domain.LangPack{}, errors.New("language file contains no strings")
}
deduplicated := make([]domain.LangPackString, 0, len(pack.Strings))
indexes := make(map[string]int, len(pack.Strings))
for _, item := range pack.Strings {
if item.Key == "" || utf8.RuneCountInString(item.Key) > 128 {
return domain.LangPack{}, fmt.Errorf("invalid string key %q", item.Key)
}
if index, exists := indexes[item.Key]; exists {
deduplicated[index] = item
continue
}
indexes[item.Key] = len(deduplicated)
deduplicated = append(deduplicated, item)
}
pack.Strings = deduplicated
sort.Slice(pack.Strings, func(i, j int) bool {
return pack.Strings[i].Key < pack.Strings[j].Key
})
return pack, nil
}
func langPackContentHash(pack domain.LangPack) (string, error) {
encoded, err := json.Marshal(pack)
if err != nil {
return "", err
}
sum := sha256.Sum256(encoded)
return hex.EncodeToString(sum[:]), nil
}
func seedCatalogID(root string) string {
base := normalizePack(filepath.Base(root))
if langPackNameRE.MatchString(base) {
return base
}
sum := sha256.Sum256([]byte(filepath.Clean(root)))
return "path-" + hex.EncodeToString(sum[:8])
}

View file

@ -4,18 +4,38 @@ import (
"context"
"strings"
"golang.org/x/sync/singleflight"
"golang.org/x/text/unicode/bidi"
"telesrv/internal/domain"
"telesrv/internal/store"
)
// Service 提供客户端语言包查询。
type Service struct {
packs store.LangPackStore
packs store.LangPackStore
packCache *langPackCache
languageCache *languageListCache
packLoads singleflight.Group
languageLoads singleflight.Group
}
// NewService 创建 langpack 服务。
func NewService(packs store.LangPackStore) *Service {
return &Service{packs: packs}
return newServiceWithCacheLimits(
packs,
defaultLangPackCacheMaxBytes,
defaultLangPackCacheMaxEntries,
defaultLanguageListCacheMaxEntries,
)
}
func newServiceWithCacheLimits(packs store.LangPackStore, maxBytes int64, maxEntries, languageEntries int) *Service {
return &Service{
packs: packs,
packCache: newLangPackCache(maxBytes, maxEntries),
languageCache: newLanguageListCache(languageEntries),
}
}
// GetLangPack 返回完整语言包。
@ -30,11 +50,23 @@ func (s *Service) GetDifference(ctx context.Context, langPack, langCode string,
if s == nil || s.packs == nil {
return domain.LangPack{LangPack: packName, LangCode: code, FromVersion: fromVersion}, nil
}
pack, err := s.packs.GetPack(ctx, packName, code, fromVersion)
var (
pack domain.LangPack
err error
)
if fromVersion == 0 {
pack, err = s.effectivePack(ctx, packName, code)
} else {
pack, err = s.rawPack(ctx, packName, code)
}
if err != nil {
return domain.LangPack{}, err
}
return s.overlayWebAStrings(ctx, pack, packName, code, fromVersion)
pack.FromVersion = fromVersion
if pack.Version <= fromVersion {
pack.Strings = nil
}
return pack, nil
}
// GetStrings 返回指定 key 的语言包字符串。
@ -44,29 +76,42 @@ func (s *Service) GetStrings(ctx context.Context, langPack, langCode string, key
if s == nil || s.packs == nil {
return domain.LangPack{LangPack: packName, LangCode: code}, nil
}
pack, err := s.packs.GetStrings(ctx, packName, code, keys)
pack, err := s.effectivePack(ctx, packName, code)
if err != nil {
return domain.LangPack{}, err
}
if len(keys) == 0 {
return s.overlayWebAStrings(ctx, pack, packName, code, 0)
}
missing := missingLangPackKeys(keys, pack.Strings)
if len(missing) == 0 || !shouldOverlayWebA(packName) {
return pack, nil
}
overlay, err := s.packs.GetStrings(ctx, "weba", code, missing)
if err != nil {
return domain.LangPack{}, err
wanted := make(map[string]struct{}, len(keys))
for _, key := range keys {
wanted[key] = struct{}{}
}
return mergeMissingLangPackStrings(pack, overlay), nil
selected := pack
selected.Strings = make([]domain.LangPackString, 0, len(keys))
for _, item := range pack.Strings {
if _, ok := wanted[item.Key]; ok {
selected.Strings = append(selected.Strings, item)
}
}
return selected, nil
}
// ListLanguages 返回已 seed 的语言包语言列表。
func (s *Service) ListLanguages(ctx context.Context, langPack string) ([]domain.LangPackLanguage, error) {
packName := normalizePack(langPack)
if s == nil || s.packs == nil {
return nil, nil
}
return s.cachedLanguages(ctx, packName)
}
func normalizePack(langPack string) string {
if langPack == "" {
pack := strings.ToLower(strings.TrimSpace(langPack))
if pack == "" {
return "tdesktop"
}
return langPack
return pack
}
func normalizeCode(langCode string) string {
@ -74,6 +119,7 @@ func normalizeCode(langCode string) string {
if code == "" {
return "en"
}
code = strings.ReplaceAll(code, "_", "-")
return strings.TrimSuffix(code, "-raw")
}
@ -86,15 +132,117 @@ func shouldOverlayWebA(langPack string) bool {
}
}
func (s *Service) overlayWebAStrings(ctx context.Context, pack domain.LangPack, langPack, langCode string, fromVersion int) (domain.LangPack, error) {
if fromVersion != 0 || !shouldOverlayWebA(langPack) {
return pack, nil
func (s *Service) rawPack(ctx context.Context, langPack, langCode string) (domain.LangPack, error) {
key := langPackCacheKey{pack: langPack, code: langCode, kind: langPackCacheRaw}
return s.cachedPack(ctx, key, func() (domain.LangPack, error) {
return s.packs.GetPack(ctx, langPack, langCode, 0)
})
}
func (s *Service) effectivePack(ctx context.Context, langPack, langCode string) (domain.LangPack, error) {
if !shouldOverlayWebA(langPack) {
return s.rawPack(ctx, langPack, langCode)
}
overlay, err := s.packs.GetPack(ctx, "weba", langCode, fromVersion)
if err != nil {
return domain.LangPack{}, err
key := langPackCacheKey{pack: langPack, code: langCode, kind: langPackCacheEffective}
return s.cachedPack(ctx, key, func() (domain.LangPack, error) {
pack, err := s.rawPack(ctx, langPack, langCode)
if err != nil {
return domain.LangPack{}, err
}
overlay, err := s.rawPack(ctx, "weba", langCode)
if err != nil {
return domain.LangPack{}, err
}
return mergeMissingLangPackStrings(pack, overlay), nil
})
}
type cachedPackLoadResult struct {
pack domain.LangPack
stable bool
}
func (s *Service) cachedPack(ctx context.Context, key langPackCacheKey, load func() (domain.LangPack, error)) (domain.LangPack, error) {
if s.packCache == nil {
return load()
}
return mergeMissingLangPackStrings(pack, overlay), nil
for {
if pack, ok := s.packCache.get(key); ok {
return pack, nil
}
value, err, _ := s.packLoads.Do(key.singleflightKey(), func() (any, error) {
if pack, ok := s.packCache.get(key); ok {
return cachedPackLoadResult{pack: pack, stable: true}, nil
}
loadEpoch := s.packCache.loadEpoch()
pack, err := load()
if err != nil {
return cachedPackLoadResult{}, err
}
return cachedPackLoadResult{
pack: pack,
stable: s.packCache.putIfEpoch(key, pack, loadEpoch),
}, nil
})
if err != nil {
return domain.LangPack{}, err
}
result := value.(cachedPackLoadResult)
if result.stable {
return cloneLangPack(result.pack), nil
}
if err := ctx.Err(); err != nil {
return domain.LangPack{}, err
}
}
}
type cachedLanguagesLoadResult struct {
languages []domain.LangPackLanguage
stable bool
}
func (s *Service) cachedLanguages(ctx context.Context, langPack string) ([]domain.LangPackLanguage, error) {
if languages, ok := s.languageCache.get(langPack); ok {
return languages, nil
}
for {
value, err, _ := s.languageLoads.Do(langPack, func() (any, error) {
if languages, ok := s.languageCache.get(langPack); ok {
return cachedLanguagesLoadResult{languages: languages, stable: true}, nil
}
loadEpoch := s.languageCache.loadEpoch()
languages, err := s.packs.ListLanguages(ctx, langPack)
if err != nil {
return cachedLanguagesLoadResult{}, err
}
for i := range languages {
languages[i] = completeLanguageMetadata(langPack, languages[i])
}
return cachedLanguagesLoadResult{
languages: languages,
stable: s.languageCache.putIfEpoch(langPack, languages, loadEpoch),
}, nil
})
if err != nil {
return nil, err
}
result := value.(cachedLanguagesLoadResult)
if result.stable {
return cloneLanguages(result.languages), nil
}
if err := ctx.Err(); err != nil {
return nil, err
}
}
}
func (s *Service) flushCaches() {
if s == nil {
return
}
s.packCache.flush()
s.languageCache.flush()
}
func mergeMissingLangPackStrings(pack, overlay domain.LangPack) domain.LangPack {
@ -121,6 +269,57 @@ func mergeMissingLangPackStrings(pack, overlay domain.LangPack) domain.LangPack
return pack
}
func completeLanguageMetadata(langPack string, lang domain.LangPackLanguage) domain.LangPackLanguage {
if lang.LangPack == "" {
lang.LangPack = langPack
}
lang.LangCode = normalizeCode(lang.LangCode)
if lang.PluralCode == "" {
lang.PluralCode = pluralCode(lang.LangCode)
}
if lang.NativeName == "" {
lang.NativeName = lang.Name
}
if lang.Name == "" {
lang.Name = lang.NativeName
}
if lang.Name == "" {
lang.Name = lang.LangCode
}
if lang.NativeName == "" {
lang.NativeName = lang.Name
}
if lang.StringsCount == 0 {
lang.StringsCount = lang.TranslatedCount
}
if lang.TranslatedCount == 0 {
lang.TranslatedCount = lang.StringsCount
}
lang.Official = true
lang.Rtl = lang.Rtl || isRTLText(lang.NativeName)
return lang
}
func pluralCode(langCode string) string {
if idx := strings.IndexAny(langCode, "-_"); idx > 0 {
return langCode[:idx]
}
return langCode
}
func isRTLText(value string) bool {
for _, r := range value {
properties, _ := bidi.LookupRune(r)
switch properties.Class() {
case bidi.R, bidi.AL:
return true
case bidi.L:
return false
}
}
return false
}
func missingLangPackKeys(keys []string, strings []domain.LangPackString) []string {
if len(keys) == 0 {
return nil

View file

@ -2,9 +2,12 @@ package langpack
import (
"context"
"sync"
"testing"
"time"
"telesrv/internal/domain"
"telesrv/internal/store"
"telesrv/internal/store/memory"
)
@ -71,6 +74,258 @@ func TestServiceNormalizesWebARawLangCode(t *testing.T) {
}
}
func TestListLanguagesUsesSeededPacks(t *testing.T) {
ctx := context.Background()
packs := memory.NewLangPackStore()
svc := NewService(packs)
if err := packs.UpsertPack(ctx, domain.LangPack{
LangPack: "tdesktop",
LangCode: "fr",
Version: 7,
Strings: []domain.LangPackString{
{Key: "lng_language_name", Value: "Français"},
{Key: "lng_test", Value: "Test"},
},
}); err != nil {
t.Fatalf("seed fr langpack: %v", err)
}
if err := packs.UpsertPack(ctx, domain.LangPack{
LangPack: "android",
LangCode: "fa",
Version: 8,
Strings: []domain.LangPackString{
{Key: "LanguageName", Value: "انگلیسی"},
{Key: "TranslateLanguageFA", Value: "فارسی"},
{Key: "lng_test", Value: "Test"},
},
}); err != nil {
t.Fatalf("seed fa langpack: %v", err)
}
if err := packs.UpsertPack(ctx, domain.LangPack{
LangPack: "tdesktop",
LangCode: "ckb",
Version: 9,
Strings: []domain.LangPackString{{Key: "lng_language_name", Value: "کوردی"}},
}); err != nil {
t.Fatalf("seed ckb langpack: %v", err)
}
tdesktop, err := svc.ListLanguages(ctx, "tdesktop")
if err != nil {
t.Fatalf("list tdesktop languages: %v", err)
}
fr := findLanguage(tdesktop, "fr")
if fr == nil || fr.Name != "Français" || fr.NativeName != "Français" || fr.PluralCode != "fr" || fr.StringsCount != 2 {
t.Fatalf("fr language = %+v", fr)
}
ckb := findLanguage(tdesktop, "ckb")
if ckb == nil || !ckb.Rtl {
t.Fatalf("ckb language = %+v, want file-derived rtl", ckb)
}
android, err := svc.ListLanguages(ctx, "android")
if err != nil {
t.Fatalf("list android languages: %v", err)
}
fa := findLanguage(android, "fa")
if fa == nil || fa.NativeName != "فارسی" || !fa.Rtl || fa.PluralCode != "fa" {
t.Fatalf("fa language = %+v", fa)
}
}
func TestServiceCachesLanguageResourcesAfterFirstRequest(t *testing.T) {
ctx := context.Background()
base := memory.NewLangPackStore()
for _, pack := range []domain.LangPack{
{
LangPack: "android",
LangCode: "en",
Version: 7,
Strings: []domain.LangPackString{
{Key: "LogOutTitle", Value: "Log Out"},
{Key: "NewMessageTitle", Value: "New Message"},
},
},
{
LangPack: "weba",
LangCode: "en",
Version: 12,
Strings: []domain.LangPackString{
{Key: "AccDescrPollVoteDown", Value: "Go to next unread poll vote"},
{Key: "NewMessageTitle", Value: "New Message from WebA"},
},
},
} {
if err := base.UpsertPack(ctx, pack); err != nil {
t.Fatalf("seed %s: %v", pack.LangPack, err)
}
}
counting := &countingLangPackStore{LangPackStore: base}
svc := NewService(counting)
first, err := svc.GetLangPack(ctx, "android", "en")
if err != nil {
t.Fatalf("first get langpack: %v", err)
}
first.Strings[0].Value = "caller mutation"
second, err := svc.GetLangPack(ctx, "android", "en")
if err != nil {
t.Fatalf("second get langpack: %v", err)
}
if got := stringValue(second.Strings, "LogOutTitle"); got != "Log Out" {
t.Fatalf("cached pack was mutated through caller alias: %q", got)
}
selected, err := svc.GetStrings(ctx, "android", "en", []string{"AccDescrPollVoteDown"})
if err != nil || stringValue(selected.Strings, "AccDescrPollVoteDown") == "" {
t.Fatalf("cached get strings = %+v, %v", selected, err)
}
if _, err := svc.GetDifference(ctx, "android", "en", 1); err != nil {
t.Fatalf("cached get difference: %v", err)
}
languages, err := svc.ListLanguages(ctx, "android")
if err != nil || len(languages) != 1 {
t.Fatalf("first list languages = %+v, %v", languages, err)
}
languages[0].Name = "caller mutation"
languages, err = svc.ListLanguages(ctx, "android")
if err != nil || len(languages) != 1 || languages[0].Name == "caller mutation" {
t.Fatalf("cached languages alias = %+v, %v", languages, err)
}
getPack, getStrings, listLanguages := counting.counts()
if getPack != 2 || getStrings != 0 || listLanguages != 1 {
t.Fatalf("store calls = getPack:%d getStrings:%d list:%d, want 2/0/1", getPack, getStrings, listLanguages)
}
}
func TestServiceCollapsesConcurrentLanguagePackLoads(t *testing.T) {
ctx := context.Background()
base := memory.NewLangPackStore()
if err := base.UpsertPack(ctx, domain.LangPack{
LangPack: "weba",
LangCode: "en",
Version: 3,
Strings: []domain.LangPackString{{Key: "NewMessageTitle", Value: "New Message"}},
}); err != nil {
t.Fatalf("seed weba: %v", err)
}
counting := &countingLangPackStore{LangPackStore: base, delay: 10 * time.Millisecond}
svc := NewService(counting)
start := make(chan struct{})
errs := make(chan error, 32)
var wg sync.WaitGroup
for range 32 {
wg.Add(1)
go func() {
defer wg.Done()
<-start
_, err := svc.GetLangPack(ctx, "weba", "en")
errs <- err
}()
}
close(start)
wg.Wait()
close(errs)
for err := range errs {
if err != nil {
t.Fatalf("concurrent get langpack: %v", err)
}
}
getPack, _, _ := counting.counts()
if getPack != 1 {
t.Fatalf("concurrent store getPack calls = %d, want 1", getPack)
}
}
func TestServiceLanguagePackCacheIsBounded(t *testing.T) {
ctx := context.Background()
base := memory.NewLangPackStore()
for _, code := range []string{"en", "fr"} {
if err := base.UpsertPack(ctx, domain.LangPack{
LangPack: "weba",
LangCode: code,
Version: 1,
Strings: []domain.LangPackString{{Key: "key", Value: code}},
}); err != nil {
t.Fatalf("seed %s: %v", code, err)
}
}
counting := &countingLangPackStore{LangPackStore: base}
svc := newServiceWithCacheLimits(counting, 1<<20, 1, 1)
for _, code := range []string{"en", "fr", "en"} {
if _, err := svc.GetLangPack(ctx, "weba", code); err != nil {
t.Fatalf("get %s: %v", code, err)
}
}
getPack, _, _ := counting.counts()
if getPack != 3 {
t.Fatalf("LRU store getPack calls = %d, want 3", getPack)
}
oversized := &countingLangPackStore{LangPackStore: base}
svc = newServiceWithCacheLimits(oversized, 1, 8, 1)
if _, err := svc.GetLangPack(ctx, "weba", "en"); err != nil {
t.Fatalf("first oversized get: %v", err)
}
if _, err := svc.GetLangPack(ctx, "weba", "en"); err != nil {
t.Fatalf("second oversized get: %v", err)
}
getPack, _, _ = oversized.counts()
if getPack != 2 {
t.Fatalf("oversized store getPack calls = %d, want 2", getPack)
}
}
type countingLangPackStore struct {
store.LangPackStore
mu sync.Mutex
delay time.Duration
getPack int
getStrings int
listLanguages int
}
func (s *countingLangPackStore) GetPack(ctx context.Context, langPack, langCode string, fromVersion int) (domain.LangPack, error) {
s.mu.Lock()
s.getPack++
delay := s.delay
s.mu.Unlock()
if delay > 0 {
time.Sleep(delay)
}
return s.LangPackStore.GetPack(ctx, langPack, langCode, fromVersion)
}
func (s *countingLangPackStore) GetStrings(ctx context.Context, langPack, langCode string, keys []string) (domain.LangPack, error) {
s.mu.Lock()
s.getStrings++
s.mu.Unlock()
return s.LangPackStore.GetStrings(ctx, langPack, langCode, keys)
}
func (s *countingLangPackStore) ListLanguages(ctx context.Context, langPack string) ([]domain.LangPackLanguage, error) {
s.mu.Lock()
s.listLanguages++
s.mu.Unlock()
return s.LangPackStore.ListLanguages(ctx, langPack)
}
func (s *countingLangPackStore) counts() (getPack, getStrings, listLanguages int) {
s.mu.Lock()
defer s.mu.Unlock()
return s.getPack, s.getStrings, s.listLanguages
}
func findLanguage(languages []domain.LangPackLanguage, code string) *domain.LangPackLanguage {
for i := range languages {
if languages[i].LangCode == code {
return &languages[i]
}
}
return nil
}
func stringValue(strings []domain.LangPackString, key string) string {
for _, item := range strings {
if item.Key == key {

View file

@ -9,6 +9,55 @@ type LangPack struct {
Strings []LangPackString
}
// LangPackSeed 是一次启动扫描得到的完整语言包文件清单。
type LangPackSeed struct {
Catalog string
Scopes []string
Packs []LangPackSeedEntry
}
// LangPackSeedEntry 记录一个语言包文件、源文件 hash 与规范化内容 hash。
type LangPackSeedEntry struct {
Pack LangPack
SourceHash string
ContentHash string
StringsCount int
ContentLoaded bool
}
// LangPackSeedCatalog 是上次成功对账后可用于跳过未变文件解析的清单快照。
type LangPackSeedCatalog struct {
Catalog string `json:"catalog"`
Scopes []string `json:"scopes"`
Packs []LangPackSeedCatalogEntry `json:"packs"`
}
// LangPackSeedCatalogEntry 只保存判断源文件是否变化所需的有界元数据。
type LangPackSeedCatalogEntry struct {
LangPack string `json:"lang_pack"`
LangCode string `json:"lang_code"`
Version int `json:"version"`
SourceHash string `json:"source_hash"`
ContentHash string `json:"content_hash"`
StringsCount int `json:"strings_count"`
}
// LangPackLanguage 是 langpack.getLanguages/getLanguage 返回的语言元数据。
type LangPackLanguage struct {
LangPack string
LangCode string
Name string
NativeName string
BaseLangCode string
PluralCode string
Official bool
Rtl bool
Beta bool
StringsCount int
TranslatedCount int
TranslationsURL string
}
// LangPackString 是语言包中的一个普通或复数形式字符串。
type LangPackString struct {
Key string

View file

@ -47,6 +47,57 @@ func TestEmailSentCodeUsesDeliveryLength(t *testing.T) {
}
}
func TestAuthSignInRoutesOfficialEmailCodeCarriers(t *testing.T) {
const (
phone = "+86 188 0000 0021"
hash = "hash-email-login"
code = "654321"
)
tests := []struct {
name string
request func() *tg.AuthSignInRequest
wantPhoneCodeCalls int
wantEmailCodeCalls int
}{
{
name: "webk_phone_code",
request: func() *tg.AuthSignInRequest {
return &tg.AuthSignInRequest{PhoneNumber: phone, PhoneCodeHash: hash, PhoneCode: code}
},
wantPhoneCodeCalls: 1,
},
{
name: "tdesktop_android_email_verification",
request: func() *tg.AuthSignInRequest {
req := &tg.AuthSignInRequest{PhoneNumber: phone, PhoneCodeHash: hash}
req.SetEmailVerification(&tg.EmailVerificationCode{Code: code})
return req
},
wantEmailCodeCalls: 1,
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
authSvc := &captureAuthService{signInUser: domain.User{ID: 100200301, Phone: "8618800000021", FirstName: "Alice"}}
r := New(Config{}, Deps{Auth: authSvc}, zaptest.NewLogger(t), fixedClock{now: time.Unix(1700000000, 0)})
if _, err := r.onAuthSignIn(context.Background(), tc.request()); err != nil {
t.Fatalf("onAuthSignIn: %v", err)
}
if authSvc.signInCount != tc.wantPhoneCodeCalls || authSvc.signInWithEmailCount != tc.wantEmailCodeCalls {
t.Fatalf("SignIn/SignInWithEmail calls=%d/%d, want %d/%d", authSvc.signInCount, authSvc.signInWithEmailCount, tc.wantPhoneCodeCalls, tc.wantEmailCodeCalls)
}
if tc.wantPhoneCodeCalls == 1 && (authSvc.signInPhone != phone || authSvc.signInHash != hash || authSvc.signInCode != code) {
t.Fatalf("SignIn proof=%q/%q/%q, want %q/%q/%q", authSvc.signInPhone, authSvc.signInHash, authSvc.signInCode, phone, hash, code)
}
if tc.wantEmailCodeCalls == 1 && (authSvc.signInWithEmailPhone != phone || authSvc.signInWithEmailHash != hash || authSvc.signInWithEmailCode != code) {
t.Fatalf("SignInWithEmail proof=%q/%q/%q, want %q/%q/%q", authSvc.signInWithEmailPhone, authSvc.signInWithEmailHash, authSvc.signInWithEmailCode, phone, hash, code)
}
})
}
}
func TestAccountVerifyEmailLoginSetupReturnsSentCodeSuccess(t *testing.T) {
user := domain.User{
ID: 100200300,

View file

@ -2,11 +2,32 @@ package rpc
import (
"encoding/json"
"telesrv/internal/domain"
"github.com/iamxvbaba/td/tg"
"sort"
"telesrv/internal/domain"
)
func tgLangPackLanguages(items []domain.LangPackLanguage) []tg.LangPackLanguage {
out := make([]tg.LangPackLanguage, 0, len(items))
for _, item := range items {
out = append(out, tg.LangPackLanguage{
Official: item.Official,
Rtl: item.Rtl,
Beta: item.Beta,
Name: item.Name,
NativeName: item.NativeName,
LangCode: item.LangCode,
BaseLangCode: item.BaseLangCode,
PluralCode: item.PluralCode,
StringsCount: item.StringsCount,
TranslatedCount: item.TranslatedCount,
TranslationsURL: item.TranslationsURL,
})
}
return out
}
func tgLangPackStrings(items []domain.LangPackString) []tg.LangPackStringClass {
out := make([]tg.LangPackStringClass, 0, len(items))
for _, item := range items {

View file

@ -775,6 +775,7 @@ type LangPackService interface {
GetLangPack(ctx context.Context, langPack, langCode string) (domain.LangPack, error)
GetDifference(ctx context.Context, langPack, langCode string, fromVersion int) (domain.LangPack, error)
GetStrings(ctx context.Context, langPack, langCode string, keys []string) (domain.LangPack, error)
ListLanguages(ctx context.Context, langPack string) ([]domain.LangPackLanguage, error)
}
// AIComposeService 抽象客户端输入框 AI 改写/润色与 aicompose tones 目录。

View file

@ -26,6 +26,9 @@ func wrapperTooDeepErr() error { return tgerr.New(400, "WRAPPER_TOO_DEEP") }
// inputConstructorInvalidErr 表示客户端传入的 TL 构造器不在当前 RPC 接受范围内。
func inputConstructorInvalidErr() error { return tgerr.New(400, "INPUT_CONSTRUCTOR_INVALID") }
// langCodeNotSupportedErr 表示请求的语言码没有已导入的语言包。
func langCodeNotSupportedErr() error { return tgerr.New(400, "LANG_CODE_NOT_SUPPORTED") }
// folderIDInvalidErr 表示客户端传入多个 folder peer 或非法 folder。
func folderIDInvalidErr() error { return tgerr.New(400, "FOLDER_ID_INVALID") }

View file

@ -17,15 +17,21 @@ func (r *Router) registerLangpack(d *tlprofile.Dispatcher) {
registerRPC[*tg.LangpackGetLanguagesRequest](d, tlprofile.SemanticMethodLangpackGetLanguages, func(ctx context.Context, layerRequest *tg.LangpackGetLanguagesRequest) (any, error) {
langPack := layerRequest.
LangPack
_ = langPack
return r.langpackLanguages(ctx, langPack), nil
languages, err := r.langpackLanguages(ctx, langPack)
if err != nil {
return nil, internalErr()
}
return languages, nil
})
registerRPC[*tg.LangpackGetLanguageRequest](d, tlprofile.SemanticMethodLangpackGetLanguage, func(ctx context.Context, req *tg.LangpackGetLanguageRequest) (any, error) {
if req == nil {
return nil, inputConstructorInvalidErr()
}
lang := r.langpackLanguage(ctx, req.LangPack, req.LangCode)
lang, err := r.langpackLanguage(ctx, req.LangPack, req.LangCode)
if err != nil {
return nil, err
}
return &lang, nil
})
registerRPC[*tg.LangpackGetLangPackRequest](d, tlprofile.SemanticMethodLangpackGetLangPack, func(ctx context.Context, req *tg.LangpackGetLangPackRequest) (any, error) {
@ -71,7 +77,7 @@ func langPackOrClient(ctx context.Context, langPack string) string {
return langPackFromClient(ctx)
}
func (r *Router) langpackLanguage(ctx context.Context, langPack, langCode string) tg.LangPackLanguage {
func (r *Router) langpackLanguage(ctx context.Context, langPack, langCode string) (tg.LangPackLanguage, error) {
if langCode == "" {
if info, ok := ClientInfoFrom(ctx); ok && info.LangCode != "" {
langCode = info.LangCode
@ -80,61 +86,31 @@ func (r *Router) langpackLanguage(ctx context.Context, langPack, langCode string
}
}
langCode = normalizeLangpackCode(langCode)
languages := r.langpackLanguages(ctx, langPack)
languages, err := r.langpackLanguages(ctx, langPack)
if err != nil {
return tg.LangPackLanguage{}, internalErr()
}
for _, lang := range languages {
if strings.ToLower(lang.LangCode) == langCode {
return lang
return lang, nil
}
}
for _, lang := range languages {
if strings.ToLower(lang.PluralCode) == langCode {
return lang
}
}
return languages[0]
return tg.LangPackLanguage{}, langCodeNotSupportedErr()
}
func (r *Router) langpackLanguages(ctx context.Context, langPack string) []tg.LangPackLanguage {
func (r *Router) langpackLanguages(ctx context.Context, langPack string) ([]tg.LangPackLanguage, error) {
if langPack == "" {
langPack = langPackFromClient(ctx)
}
langPack = strings.ToLower(langPack)
languages := []tg.LangPackLanguage{
{
Official: true,
Name: "English",
NativeName: "English",
LangCode: "en",
PluralCode: "en",
StringsCount: 0,
TranslatedCount: 0,
TranslationsURL: "",
},
{
Official: true,
Name: "Chinese (Simplified)",
NativeName: "Chinese (Simplified)",
LangCode: "zh-hans",
PluralCode: "zh",
StringsCount: 0,
TranslatedCount: 0,
TranslationsURL: "",
},
if r.deps.LangPack == nil {
return []tg.LangPackLanguage{}, nil
}
if langPack == "android" {
languages = append(languages, tg.LangPackLanguage{
Official: true,
Rtl: true,
Name: "Persian",
NativeName: "فارسی",
LangCode: "fa",
PluralCode: "fa",
StringsCount: 11002,
TranslatedCount: 11002,
TranslationsURL: "",
})
languages, err := r.deps.LangPack.ListLanguages(ctx, langPack)
if err != nil {
return nil, err
}
return languages
return tgLangPackLanguages(languages), nil
}
func langPackFromClient(ctx context.Context) string {
@ -171,5 +147,6 @@ func normalizeLangpackCode(langCode string) string {
if code == "" {
return "en"
}
code = strings.ReplaceAll(code, "_", "-")
return strings.TrimSuffix(code, "-raw")
}

View file

@ -2,6 +2,7 @@ package rpc
import (
"context"
"strings"
"testing"
"go.uber.org/zap/zaptest"
@ -9,10 +10,14 @@ import (
"github.com/iamxvbaba/td/bin"
"github.com/iamxvbaba/td/clock"
"github.com/iamxvbaba/td/tg"
applangpack "telesrv/internal/app/langpack"
"telesrv/internal/domain"
"telesrv/internal/store/memory"
)
func TestLangpackGetLanguagesCurrentAndLegacy(t *testing.T) {
r := New(Config{DC: 2, IP: "127.0.0.1", Port: 2398}, Deps{}, zaptest.NewLogger(t), clock.System)
r := newSeededLangpackRouter(t)
t.Run("current layer", func(t *testing.T) {
var in bin.Buffer
@ -47,7 +52,7 @@ func TestLangpackGetLanguagesCurrentAndLegacy(t *testing.T) {
}
func TestLangpackGetLanguage(t *testing.T) {
r := New(Config{DC: 2, IP: "127.0.0.1", Port: 2398}, Deps{}, zaptest.NewLogger(t), clock.System)
r := newSeededLangpackRouter(t)
var in bin.Buffer
if err := (&tg.LangpackGetLanguageRequest{LangPack: "tdesktop", LangCode: "zh-hans"}).Encode(&in); err != nil {
@ -69,21 +74,33 @@ func TestLangpackGetLanguage(t *testing.T) {
t.Fatalf("language = %+v, want zh-hans", lang)
}
raw := r.langpackLanguage(context.Background(), "tdesktop", "en-raw")
raw, err := r.langpackLanguage(context.Background(), "tdesktop", "en-raw")
if err != nil {
t.Fatalf("language(en-raw): %v", err)
}
if raw.LangCode != "en" {
t.Fatalf("language(en-raw) = %+v, want en", raw)
}
if _, err := r.langpackLanguage(context.Background(), "tdesktop", "zh"); err == nil || !strings.Contains(err.Error(), "LANG_CODE_NOT_SUPPORTED") {
t.Fatalf("language(zh) error = %v, want LANG_CODE_NOT_SUPPORTED", err)
}
}
func TestLangpackAndroidPersianLanguage(t *testing.T) {
r := New(Config{DC: 2, IP: "127.0.0.1", Port: 2398}, Deps{}, zaptest.NewLogger(t), clock.System)
r := newSeededLangpackRouter(t)
lang := r.langpackLanguage(context.Background(), "android", "fa")
if lang.LangCode != "fa" || lang.PluralCode != "fa" || lang.Name != "Persian" || lang.NativeName != "فارسی" || !lang.Rtl {
lang, err := r.langpackLanguage(context.Background(), "android", "fa")
if err != nil {
t.Fatalf("android fa language: %v", err)
}
if lang.LangCode != "fa" || lang.PluralCode != "fa" || lang.NativeName != "فارسی" || !lang.Rtl {
t.Fatalf("android fa language = %+v", lang)
}
languages := r.langpackLanguages(androidClientContext(), "")
languages, err := r.langpackLanguages(androidClientContext(), "")
if err != nil {
t.Fatalf("android languages: %v", err)
}
for _, item := range languages {
if item.LangCode == "fa" {
return
@ -191,6 +208,53 @@ func TestLegacyAccountRegisterDevice(t *testing.T) {
}
}
func newSeededLangpackRouter(t *testing.T) *Router {
t.Helper()
return New(Config{DC: 2, IP: "127.0.0.1", Port: 2398}, Deps{
LangPack: seededLangPackService(t),
}, zaptest.NewLogger(t), clock.System)
}
func seededLangPackService(t testing.TB) LangPackService {
t.Helper()
ctx := context.Background()
store := memory.NewLangPackStore()
for _, pack := range []domain.LangPack{
{
LangPack: "tdesktop",
LangCode: "en",
Version: 1,
Strings: []domain.LangPackString{{Key: "lng_language_name", Value: "English"}},
},
{
LangPack: "tdesktop",
LangCode: "zh-hans",
Version: 1,
Strings: []domain.LangPackString{{Key: "lng_language_name", Value: "简体中文"}},
},
{
LangPack: "android",
LangCode: "en",
Version: 1,
Strings: []domain.LangPackString{{Key: "LanguageName", Value: "English"}},
},
{
LangPack: "android",
LangCode: "fa",
Version: 1,
Strings: []domain.LangPackString{
{Key: "LanguageName", Value: "انگلیسی"},
{Key: "TranslateLanguageFA", Value: "فارسی"},
},
},
} {
if err := store.UpsertPack(ctx, pack); err != nil {
t.Fatalf("seed %s/%s: %v", pack.LangPack, pack.LangCode, err)
}
}
return applangpack.NewService(store)
}
func dispatchLangpackLanguages(t *testing.T, r *Router, ctx context.Context, in *bin.Buffer) []tg.LangPackLanguage {
t.Helper()
enc, err := r.Dispatch(ctx, [8]byte{}, 0, in)

View file

@ -1206,6 +1206,7 @@ func TestTDesktopStartupRPCsEncode(t *testing.T) {
LastName: "User",
Phone: "15550000000",
}},
LangPack: seededLangPackService(t),
}, zaptest.NewLogger(t), clock.System)
tests := []struct {

View file

@ -35,7 +35,14 @@ type captureAuthService struct {
completedPasswordKey [8]byte
completePasswordCount int
codeDelivery domain.AuthCodeDelivery
signInCount int
signInPhone string
signInHash string
signInCode string
signInWithEmailCount int
signInWithEmailPhone string
signInWithEmailHash string
signInWithEmailCode string
}
type blockingUserAuthService struct {
@ -188,15 +195,22 @@ func (s *captureAuthService) CancelCode(context.Context, string, string) error {
return nil
}
func (s *captureAuthService) SignIn(context.Context, domain.Authorization, string, string, string) (domain.User, domain.Message, bool, error) {
func (s *captureAuthService) SignIn(_ context.Context, _ domain.Authorization, phone, hash, code string) (domain.User, domain.Message, bool, error) {
s.signInCount++
s.signInPhone = phone
s.signInHash = hash
s.signInCode = code
if s.signInUser.ID != 0 {
return s.signInUser, domain.Message{}, false, nil
}
return domain.User{}, domain.Message{}, false, nil
}
func (s *captureAuthService) SignInWithEmail(context.Context, domain.Authorization, string, string, string) (domain.User, domain.Message, bool, error) {
func (s *captureAuthService) SignInWithEmail(_ context.Context, _ domain.Authorization, phone, hash, code string) (domain.User, domain.Message, bool, error) {
s.signInWithEmailCount++
s.signInWithEmailPhone = phone
s.signInWithEmailHash = hash
s.signInWithEmailCode = code
if s.signInUser.ID != 0 {
return s.signInUser, domain.Message{}, false, nil
}

View file

@ -16,6 +16,26 @@ const (
PhoneCodeChannelEmailSetupRequired = "email_setup_required"
)
// LoginCodeChannelVerifiable reports whether a code record belongs to the
// auth.signIn/auth.signUp state machine. The TL field carrying the proof
// (phone_code or email_verification) is deliberately not part of this
// decision: official clients use both fields for an email-delivered login
// code, while the record channel remains the server-side delivery fact.
func LoginCodeChannelVerifiable(channel string) bool {
switch channel {
case PhoneCodeChannelPhone, PhoneCodeChannelSMS, PhoneCodeChannelEmailLogin:
return true
default:
return false
}
}
// LoginCodeChannelTakeable includes the setup-required placeholder because
// cancel/resend may replace it, although it is never valid auth.signIn proof.
func LoginCodeChannelTakeable(channel string) bool {
return LoginCodeChannelVerifiable(channel) || channel == PhoneCodeChannelEmailSetupRequired
}
// PhoneCodeVersionCurrent is the only version accepted by the atomic login
// state machine. Version zero is the pre-state-machine shape and deliberately
// fails closed instead of being normalized on read.

View file

@ -10,5 +10,8 @@ import (
type LangPackStore interface {
GetPack(ctx context.Context, langPack, langCode string, fromVersion int) (domain.LangPack, error)
GetStrings(ctx context.Context, langPack, langCode string, keys []string) (domain.LangPack, error)
ListLanguages(ctx context.Context, langPack string) ([]domain.LangPackLanguage, error)
GetSeedCatalog(ctx context.Context, catalog string) (domain.LangPackSeedCatalog, error)
ReconcileSeed(ctx context.Context, seed domain.LangPackSeed) (int, error)
UpsertPack(ctx context.Context, pack domain.LangPack) error
}

View file

@ -2,19 +2,41 @@ package memory
import (
"context"
"fmt"
"sort"
"strings"
"sync"
"telesrv/internal/domain"
)
// LangPackStore 是 store.LangPackStore 的内存实现。
type LangPackStore struct {
mu sync.RWMutex
m map[string]domain.LangPack
mu sync.RWMutex
m map[string]domain.LangPack
seedHashes map[string]string
catalogs map[string]domain.LangPackSeedCatalog
}
// NewLangPackStore 创建内存 LangPackStore。
func NewLangPackStore() *LangPackStore {
return &LangPackStore{m: make(map[string]domain.LangPack)}
return &LangPackStore{
m: make(map[string]domain.LangPack),
seedHashes: make(map[string]string),
catalogs: make(map[string]domain.LangPackSeedCatalog),
}
}
func (s *LangPackStore) GetSeedCatalog(_ context.Context, catalog string) (domain.LangPackSeedCatalog, error) {
if catalog == "" {
catalog = "default"
}
s.mu.RLock()
state := s.catalogs[catalog]
s.mu.RUnlock()
state.Scopes = append([]string(nil), state.Scopes...)
state.Packs = append([]domain.LangPackSeedCatalogEntry(nil), state.Packs...)
return state, nil
}
func (s *LangPackStore) GetPack(_ context.Context, langPack, langCode string, fromVersion int) (domain.LangPack, error) {
@ -57,14 +79,171 @@ func (s *LangPackStore) GetStrings(_ context.Context, langPack, langCode string,
return out, nil
}
func (s *LangPackStore) ListLanguages(_ context.Context, langPack string) ([]domain.LangPackLanguage, error) {
s.mu.RLock()
packs := make([]domain.LangPack, 0)
for _, pack := range s.m {
if pack.LangPack == langPack {
packs = append(packs, pack)
}
}
s.mu.RUnlock()
sort.Slice(packs, func(i, j int) bool {
return packs[i].LangCode < packs[j].LangCode
})
out := make([]domain.LangPackLanguage, 0, len(packs))
for _, pack := range packs {
codeKey := langPackCodeKey(pack.LangCode)
lang := domain.LangPackLanguage{
LangPack: pack.LangPack,
LangCode: pack.LangCode,
StringsCount: len(pack.Strings),
TranslatedCount: len(pack.Strings),
}
for _, item := range pack.Strings {
switch item.Key {
case "LanguageNameInEnglish", "Localization.EnglishLanguageName":
if lang.Name == "" {
lang.Name = item.Value
}
case "lng_language_name", "Localization.LanguageName":
if lang.NativeName == "" {
lang.NativeName = item.Value
}
case "LanguageName":
if lang.NativeName == "" && pack.LangPack != "android" {
lang.NativeName = item.Value
}
case "TranslateLanguage" + codeKey, "PassportLanguage_" + codeKey:
if lang.NativeName == "" || pack.LangPack == "android" {
lang.NativeName = item.Value
}
}
}
out = append(out, lang)
}
return out, nil
}
func (s *LangPackStore) ReconcileSeed(_ context.Context, seed domain.LangPackSeed) (int, error) {
catalog := seed.Catalog
if catalog == "" {
catalog = "default"
}
scopes := make(map[string]struct{}, len(seed.Scopes))
for _, scope := range seed.Scopes {
scopes[scope] = struct{}{}
}
wanted := make(map[string]domain.LangPackSeedEntry, len(seed.Packs))
for _, entry := range seed.Packs {
if _, scoped := scopes[entry.Pack.LangPack]; !scoped {
return 0, fmt.Errorf("seeded langpack %s/%s is outside reconciliation scopes", entry.Pack.LangPack, entry.Pack.LangCode)
}
key := langPackKey(entry.Pack.LangPack, entry.Pack.LangCode)
if _, exists := wanted[key]; exists {
return 0, fmt.Errorf("duplicate seeded langpack %s/%s", entry.Pack.LangPack, entry.Pack.LangCode)
}
wanted[key] = entry
}
s.mu.Lock()
defer s.mu.Unlock()
for _, scope := range s.catalogs[catalog].Scopes {
scopes[scope] = struct{}{}
}
for key, entry := range wanted {
if err := validateSeedEntry(entry); err != nil {
return 0, err
}
current, exists := s.m[key]
if exists {
if entry.Pack.Version < current.Version {
return 0, fmt.Errorf("langpack version rollback %s/%s: %d < %d", entry.Pack.LangPack, entry.Pack.LangCode, entry.Pack.Version, current.Version)
}
if oldHash := s.seedHashes[key]; oldHash != "" && entry.Pack.Version == current.Version && oldHash != entry.ContentHash {
return 0, fmt.Errorf("langpack %s/%s v%d content changed without version bump", entry.Pack.LangPack, entry.Pack.LangCode, entry.Pack.Version)
}
}
unchanged := exists && current.Version == entry.Pack.Version && len(current.Strings) == entry.StringsCount && s.seedHashes[key] == entry.ContentHash
if !unchanged && !entry.ContentLoaded {
return 0, fmt.Errorf("langpack %s/%s content is required but was not loaded", entry.Pack.LangPack, entry.Pack.LangCode)
}
}
for key, pack := range s.m {
if _, scoped := scopes[pack.LangPack]; !scoped {
continue
}
if _, exists := wanted[key]; !exists {
delete(s.m, key)
delete(s.seedHashes, key)
}
}
written := 0
for key, entry := range wanted {
current, exists := s.m[key]
if exists && current.Version == entry.Pack.Version && len(current.Strings) == entry.StringsCount && s.seedHashes[key] == entry.ContentHash {
continue
}
pack := entry.Pack
pack.Strings = append([]domain.LangPackString(nil), pack.Strings...)
s.m[key] = pack
s.seedHashes[key] = entry.ContentHash
written += len(pack.Strings)
}
s.catalogs[catalog] = seedCatalogSnapshot(seed)
return written, nil
}
func (s *LangPackStore) UpsertPack(_ context.Context, pack domain.LangPack) error {
pack.Strings = append([]domain.LangPackString(nil), pack.Strings...)
s.mu.Lock()
s.m[langPackKey(pack.LangPack, pack.LangCode)] = pack
key := langPackKey(pack.LangPack, pack.LangCode)
s.m[key] = pack
delete(s.seedHashes, key)
clear(s.catalogs)
s.mu.Unlock()
return nil
}
func validateSeedEntry(entry domain.LangPackSeedEntry) error {
if entry.SourceHash == "" || entry.ContentHash == "" || entry.StringsCount <= 0 {
return fmt.Errorf("langpack %s/%s has incomplete seed metadata", entry.Pack.LangPack, entry.Pack.LangCode)
}
if entry.ContentLoaded && len(entry.Pack.Strings) != entry.StringsCount {
return fmt.Errorf("langpack %s/%s loaded %d strings, want %d", entry.Pack.LangPack, entry.Pack.LangCode, len(entry.Pack.Strings), entry.StringsCount)
}
return nil
}
func seedCatalogSnapshot(seed domain.LangPackSeed) domain.LangPackSeedCatalog {
state := domain.LangPackSeedCatalog{
Catalog: seed.Catalog,
Scopes: append([]string(nil), seed.Scopes...),
Packs: make([]domain.LangPackSeedCatalogEntry, 0, len(seed.Packs)),
}
for _, entry := range seed.Packs {
state.Packs = append(state.Packs, domain.LangPackSeedCatalogEntry{
LangPack: entry.Pack.LangPack,
LangCode: entry.Pack.LangCode,
Version: entry.Pack.Version,
SourceHash: entry.SourceHash,
ContentHash: entry.ContentHash,
StringsCount: entry.StringsCount,
})
}
return state
}
func langPackKey(langPack, langCode string) string {
return langPack + "\x00" + langCode
}
func langPackCodeKey(langCode string) string {
base := strings.ToUpper(strings.TrimSpace(langCode))
if idx := strings.IndexAny(base, "-_"); idx >= 0 {
base = base[:idx]
}
return base
}

View file

@ -191,11 +191,11 @@ func (s *CodeStore) InvalidateLoginCode(_ context.Context, hash, phone string) (
}
func loginCodeVerifiable(record store.PhoneCode) bool {
return record.Channel == store.PhoneCodeChannelPhone || record.Channel == store.PhoneCodeChannelSMS || record.Channel == store.PhoneCodeChannelEmailLogin
return store.LoginCodeChannelVerifiable(record.Channel)
}
func loginCodeTakeable(record store.PhoneCode) bool {
return loginCodeVerifiable(record) || record.Channel == store.PhoneCodeChannelEmailSetupRequired
return store.LoginCodeChannelTakeable(record.Channel)
}
func (s *CodeStore) liveCodeLocked(hash string) (codeEntry, bool) {

View file

@ -2,6 +2,7 @@ package postgres
import (
"context"
"encoding/json"
"errors"
"fmt"
@ -68,6 +69,68 @@ func (s *LangPackStore) GetStrings(ctx context.Context, langPack, langCode strin
return meta, nil
}
func (s *LangPackStore) ListLanguages(ctx context.Context, langPack string) ([]domain.LangPackLanguage, error) {
rows, err := s.q.ListLangPackLanguages(ctx, langPack)
if err != nil {
return nil, fmt.Errorf("list lang pack languages: %w", err)
}
out := make([]domain.LangPackLanguage, 0, len(rows))
for _, row := range rows {
out = append(out, domain.LangPackLanguage{
LangPack: row.LangPack,
LangCode: row.LangCode,
Name: row.Name,
NativeName: row.NativeName,
StringsCount: int(row.StringsCount),
TranslatedCount: int(row.StringsCount),
})
}
return out, nil
}
func (s *LangPackStore) GetSeedCatalog(ctx context.Context, catalog string) (domain.LangPackSeedCatalog, error) {
if catalog == "" {
catalog = "default"
}
encoded, err := s.q.GetLangPackSeedHash(ctx, langPackSeedManifestStateKey(catalog))
if errors.Is(err, pgx.ErrNoRows) {
return domain.LangPackSeedCatalog{Catalog: catalog}, nil
}
if err != nil {
return domain.LangPackSeedCatalog{}, fmt.Errorf("get langpack seed catalog %q: %w", catalog, err)
}
var state domain.LangPackSeedCatalog
if err := json.Unmarshal([]byte(encoded), &state); err != nil {
return domain.LangPackSeedCatalog{}, fmt.Errorf("decode langpack seed catalog %q: %w", catalog, err)
}
if state.Catalog != catalog {
return domain.LangPackSeedCatalog{}, fmt.Errorf("langpack seed catalog key %q contains catalog %q", catalog, state.Catalog)
}
return state, nil
}
func (s *LangPackStore) ReconcileSeed(ctx context.Context, seed domain.LangPackSeed) (int, error) {
txer, ok := s.db.(interface {
Begin(context.Context) (pgx.Tx, error)
})
if !ok {
return 0, errors.New("langpack seed reconciliation requires transaction support")
}
tx, err := txer.Begin(ctx)
if err != nil {
return 0, fmt.Errorf("begin langpack seed reconciliation: %w", err)
}
written, err := reconcileSeedWith(ctx, tx, s.q.WithTx(tx), seed)
if err != nil {
_ = tx.Rollback(ctx)
return 0, err
}
if err := tx.Commit(ctx); err != nil {
return 0, fmt.Errorf("commit langpack seed reconciliation: %w", err)
}
return written, nil
}
func (s *LangPackStore) UpsertPack(ctx context.Context, pack domain.LangPack) error {
if txer, ok := s.db.(interface {
Begin(context.Context) (pgx.Tx, error)
@ -77,7 +140,15 @@ func (s *LangPackStore) UpsertPack(ctx context.Context, pack domain.LangPack) er
return fmt.Errorf("begin lang pack upsert: %w", err)
}
q := s.q.WithTx(tx)
if err := upsertPackWith(ctx, q, pack); err != nil {
if err := replacePackWithCopy(ctx, tx, q, pack); err != nil {
_ = tx.Rollback(ctx)
return err
}
if err := q.DeleteLangPackSeedHash(ctx, langPackSeedStateKey(pack.LangPack, pack.LangCode)); err != nil {
_ = tx.Rollback(ctx)
return fmt.Errorf("delete lang pack seed hash: %w", err)
}
if err := invalidateLangPackSeedCatalogs(ctx, tx); err != nil {
_ = tx.Rollback(ctx)
return err
}
@ -86,10 +157,189 @@ func (s *LangPackStore) UpsertPack(ctx context.Context, pack domain.LangPack) er
}
return nil
}
return upsertPackWith(ctx, s.q, pack)
if err := replacePackWithQueries(ctx, s.q, pack); err != nil {
return err
}
if err := s.q.DeleteLangPackSeedHash(ctx, langPackSeedStateKey(pack.LangPack, pack.LangCode)); err != nil {
return fmt.Errorf("delete lang pack seed hash: %w", err)
}
if err := invalidateLangPackSeedCatalogs(ctx, s.db); err != nil {
return err
}
return nil
}
func upsertPackWith(ctx context.Context, q *sqlcgen.Queries, pack domain.LangPack) error {
func reconcileSeedWith(ctx context.Context, tx pgx.Tx, q *sqlcgen.Queries, seed domain.LangPackSeed) (int, error) {
catalog := seed.Catalog
if catalog == "" {
catalog = "default"
}
reconciledScopes := append([]string(nil), seed.Scopes...)
scopes := make(map[string]struct{}, len(seed.Scopes))
wanted := make(map[string]map[string]struct{}, len(seed.Scopes))
for _, scope := range seed.Scopes {
scopes[scope] = struct{}{}
wanted[scope] = make(map[string]struct{})
}
previousScopesJSON, err := q.GetLangPackSeedHash(ctx, langPackSeedCatalogStateKey(catalog))
if err == nil {
var previousScopes []string
if err := json.Unmarshal([]byte(previousScopesJSON), &previousScopes); err != nil {
return 0, fmt.Errorf("decode previous langpack seed scopes for %q: %w", catalog, err)
}
for _, scope := range previousScopes {
if _, exists := scopes[scope]; exists {
continue
}
scopes[scope] = struct{}{}
wanted[scope] = make(map[string]struct{})
reconciledScopes = append(reconciledScopes, scope)
}
} else if !errors.Is(err, pgx.ErrNoRows) {
return 0, fmt.Errorf("get previous langpack seed scopes for %q: %w", catalog, err)
}
for _, entry := range seed.Packs {
if err := validateSeedEntry(entry); err != nil {
return 0, err
}
if _, ok := scopes[entry.Pack.LangPack]; !ok {
return 0, fmt.Errorf("seeded langpack %s/%s is outside reconciliation scopes", entry.Pack.LangPack, entry.Pack.LangCode)
}
if _, exists := wanted[entry.Pack.LangPack][entry.Pack.LangCode]; exists {
return 0, fmt.Errorf("duplicate seeded langpack %s/%s", entry.Pack.LangPack, entry.Pack.LangCode)
}
wanted[entry.Pack.LangPack][entry.Pack.LangCode] = struct{}{}
}
for _, scope := range reconciledScopes {
codes, err := q.ListLangPackCodes(ctx, scope)
if err != nil {
return 0, fmt.Errorf("list existing langpack codes for %q: %w", scope, err)
}
for _, code := range codes {
if _, keep := wanted[scope][code]; keep {
continue
}
params := sqlcgen.DeleteLangPackStringsParams{LangPack: scope, LangCode: code}
if err := q.DeleteLangPackStrings(ctx, params); err != nil {
return 0, fmt.Errorf("delete removed langpack strings %s/%s: %w", scope, code, err)
}
if err := q.DeleteLangPackMeta(ctx, sqlcgen.DeleteLangPackMetaParams(params)); err != nil {
return 0, fmt.Errorf("delete removed langpack metadata %s/%s: %w", scope, code, err)
}
if err := q.DeleteLangPackSeedHash(ctx, langPackSeedStateKey(scope, code)); err != nil {
return 0, fmt.Errorf("delete removed langpack seed hash %s/%s: %w", scope, code, err)
}
}
}
written := 0
for _, entry := range seed.Packs {
pack := entry.Pack
meta, metaErr := q.GetLangPackMeta(ctx, sqlcgen.GetLangPackMetaParams{
LangPack: pack.LangPack,
LangCode: pack.LangCode,
})
metaFound := metaErr == nil
if metaErr != nil && !errors.Is(metaErr, pgx.ErrNoRows) {
return 0, fmt.Errorf("get existing langpack metadata %s/%s: %w", pack.LangPack, pack.LangCode, metaErr)
}
if metaFound && pack.Version < int(meta.Version) {
return 0, fmt.Errorf("langpack version rollback %s/%s: %d < %d", pack.LangPack, pack.LangCode, pack.Version, meta.Version)
}
stateKey := langPackSeedStateKey(pack.LangPack, pack.LangCode)
oldHash, hashErr := q.GetLangPackSeedHash(ctx, stateKey)
hashFound := hashErr == nil
if hashErr != nil && !errors.Is(hashErr, pgx.ErrNoRows) {
return 0, fmt.Errorf("get langpack seed hash %s/%s: %w", pack.LangPack, pack.LangCode, hashErr)
}
if metaFound && hashFound && pack.Version == int(meta.Version) && oldHash != entry.ContentHash {
return 0, fmt.Errorf("langpack %s/%s v%d content changed without version bump", pack.LangPack, pack.LangCode, pack.Version)
}
if metaFound && hashFound && pack.Version == int(meta.Version) && int(meta.StringsCount) == entry.StringsCount && oldHash == entry.ContentHash {
continue
}
if !entry.ContentLoaded {
return 0, fmt.Errorf("langpack %s/%s content is required but was not loaded", pack.LangPack, pack.LangCode)
}
if err := replacePackWithCopy(ctx, tx, q, pack); err != nil {
return 0, err
}
if err := q.PutLangPackSeedHash(ctx, sqlcgen.PutLangPackSeedHashParams{Key: stateKey, ContentHash: entry.ContentHash}); err != nil {
return 0, fmt.Errorf("put langpack seed hash %s/%s: %w", pack.LangPack, pack.LangCode, err)
}
written += len(pack.Strings)
}
encodedScopes, err := json.Marshal(seed.Scopes)
if err != nil {
return 0, fmt.Errorf("encode langpack seed scopes for %q: %w", catalog, err)
}
if err := q.PutLangPackSeedHash(ctx, sqlcgen.PutLangPackSeedHashParams{
Key: langPackSeedCatalogStateKey(catalog),
ContentHash: string(encodedScopes),
}); err != nil {
return 0, fmt.Errorf("put langpack seed scopes for %q: %w", catalog, err)
}
manifest := seedCatalogSnapshot(seed)
encodedManifest, err := json.Marshal(manifest)
if err != nil {
return 0, fmt.Errorf("encode langpack seed catalog %q: %w", catalog, err)
}
if err := q.PutLangPackSeedHash(ctx, sqlcgen.PutLangPackSeedHashParams{
Key: langPackSeedManifestStateKey(catalog),
ContentHash: string(encodedManifest),
}); err != nil {
return 0, fmt.Errorf("put langpack seed catalog %q: %w", catalog, err)
}
return written, nil
}
func replacePackWithCopy(ctx context.Context, tx pgx.Tx, q *sqlcgen.Queries, pack domain.LangPack) error {
params := sqlcgen.DeleteLangPackStringsParams{LangPack: pack.LangPack, LangCode: pack.LangCode}
if err := q.DeleteLangPackStrings(ctx, params); err != nil {
return fmt.Errorf("delete previous lang pack strings: %w", err)
}
if err := q.UpsertLangPackMeta(ctx, sqlcgen.UpsertLangPackMetaParams{
LangPack: pack.LangPack,
LangCode: pack.LangCode,
Version: int32(pack.Version),
StringsCount: int32(len(pack.Strings)),
}); err != nil {
return fmt.Errorf("upsert lang pack meta: %w", err)
}
if len(pack.Strings) == 0 {
return nil
}
count, err := tx.CopyFrom(
ctx,
pgx.Identifier{"lang_pack_strings"},
[]string{
"lang_pack", "lang_code", "key", "version", "pluralized", "value",
"zero_value", "one_value", "two_value", "few_value", "many_value", "other_value", "deleted",
},
pgx.CopyFromSlice(len(pack.Strings), func(i int) ([]any, error) {
item := pack.Strings[i]
return []any{
pack.LangPack, pack.LangCode, item.Key, int32(pack.Version), item.Pluralized, item.Value,
item.ZeroValue, item.OneValue, item.TwoValue, item.FewValue, item.ManyValue, item.OtherValue, item.Deleted,
}, nil
}),
)
if err != nil {
return fmt.Errorf("copy lang pack strings: %w", err)
}
if count != int64(len(pack.Strings)) {
return fmt.Errorf("copy lang pack strings wrote %d rows, want %d", count, len(pack.Strings))
}
return nil
}
func replacePackWithQueries(ctx context.Context, q *sqlcgen.Queries, pack domain.LangPack) error {
if err := q.DeleteLangPackStrings(ctx, sqlcgen.DeleteLangPackStringsParams{LangPack: pack.LangPack, LangCode: pack.LangCode}); err != nil {
return fmt.Errorf("delete previous lang pack strings: %w", err)
}
if err := q.UpsertLangPackMeta(ctx, sqlcgen.UpsertLangPackMetaParams{
LangPack: pack.LangPack,
LangCode: pack.LangCode,
@ -120,6 +370,54 @@ func upsertPackWith(ctx context.Context, q *sqlcgen.Queries, pack domain.LangPac
return nil
}
func langPackSeedStateKey(langPack, langCode string) string {
return "langpack:v1:entry:" + langPack + ":" + langCode
}
func langPackSeedCatalogStateKey(catalog string) string {
return "langpack:v1:catalog:" + catalog
}
func langPackSeedManifestStateKey(catalog string) string {
return "langpack:v2:catalog:" + catalog
}
func validateSeedEntry(entry domain.LangPackSeedEntry) error {
if entry.SourceHash == "" || entry.ContentHash == "" || entry.StringsCount <= 0 {
return fmt.Errorf("langpack %s/%s has incomplete seed metadata", entry.Pack.LangPack, entry.Pack.LangCode)
}
if entry.ContentLoaded && len(entry.Pack.Strings) != entry.StringsCount {
return fmt.Errorf("langpack %s/%s loaded %d strings, want %d", entry.Pack.LangPack, entry.Pack.LangCode, len(entry.Pack.Strings), entry.StringsCount)
}
return nil
}
func seedCatalogSnapshot(seed domain.LangPackSeed) domain.LangPackSeedCatalog {
state := domain.LangPackSeedCatalog{
Catalog: seed.Catalog,
Scopes: append([]string(nil), seed.Scopes...),
Packs: make([]domain.LangPackSeedCatalogEntry, 0, len(seed.Packs)),
}
for _, entry := range seed.Packs {
state.Packs = append(state.Packs, domain.LangPackSeedCatalogEntry{
LangPack: entry.Pack.LangPack,
LangCode: entry.Pack.LangCode,
Version: entry.Pack.Version,
SourceHash: entry.SourceHash,
ContentHash: entry.ContentHash,
StringsCount: entry.StringsCount,
})
}
return state
}
func invalidateLangPackSeedCatalogs(ctx context.Context, db sqlcgen.DBTX) error {
if _, err := db.Exec(ctx, `DELETE FROM seed_states WHERE key LIKE 'langpack:v2:catalog:%'`); err != nil {
return fmt.Errorf("invalidate langpack seed catalogs: %w", err)
}
return nil
}
func (s *LangPackStore) meta(ctx context.Context, langPack, langCode string) (domain.LangPack, bool, error) {
row, err := s.q.GetLangPackMeta(ctx, sqlcgen.GetLangPackMetaParams{
LangPack: langPack,

View file

@ -0,0 +1,164 @@
package postgres
import (
"context"
"path/filepath"
"strings"
"testing"
"time"
applangpack "telesrv/internal/app/langpack"
"telesrv/internal/domain"
)
func TestLangPackSeedReconciliationPostgres(t *testing.T) {
pool := testPool(t)
ctx := context.Background()
packName := "seedtest-" + randomSuffix(t)
store := NewLangPackStore(pool)
t.Cleanup(func() {
_, _ = pool.Exec(ctx, "DELETE FROM lang_pack_strings WHERE lang_pack = $1", packName)
_, _ = pool.Exec(ctx, "DELETE FROM lang_packs WHERE lang_pack = $1", packName)
_, _ = pool.Exec(ctx, "DELETE FROM seed_states WHERE key LIKE $1 OR key = $2 OR key = $3", "langpack:v1:entry:"+packName+":%", "langpack:v1:catalog:"+packName, "langpack:v2:catalog:"+packName)
})
seedV1 := domain.LangPackSeed{
Catalog: packName,
Scopes: []string{packName},
Packs: []domain.LangPackSeedEntry{{
SourceHash: "source-v1",
ContentHash: "hash-v1",
StringsCount: 2,
ContentLoaded: true,
Pack: domain.LangPack{
LangPack: packName,
LangCode: "fr",
Version: 1,
Strings: []domain.LangPackString{
{Key: "LanguageName", Value: "Français"},
{Key: "old", Value: "old"},
},
},
}},
}
written, err := store.ReconcileSeed(ctx, seedV1)
if err != nil || written != 2 {
t.Fatalf("reconcile v1 = %d, %v", written, err)
}
seedV1Unloaded := seedV1
seedV1Unloaded.Packs = append([]domain.LangPackSeedEntry(nil), seedV1.Packs...)
seedV1Unloaded.Packs[0].Pack.Strings = nil
seedV1Unloaded.Packs[0].ContentLoaded = false
if written, err := store.ReconcileSeed(ctx, seedV1Unloaded); err != nil || written != 0 {
t.Fatalf("reconcile unchanged v1 = %d, %v", written, err)
}
catalog, err := store.GetSeedCatalog(ctx, packName)
if err != nil || len(catalog.Packs) != 1 || catalog.Packs[0].SourceHash != "source-v1" {
t.Fatalf("seed catalog = %+v, %v", catalog, err)
}
seedV2 := domain.LangPackSeed{
Catalog: packName,
Scopes: []string{packName},
Packs: []domain.LangPackSeedEntry{{
SourceHash: "source-v2",
ContentHash: "hash-v2",
StringsCount: 2,
ContentLoaded: true,
Pack: domain.LangPack{
LangPack: packName,
LangCode: "fr",
Version: 2,
Strings: []domain.LangPackString{
{Key: "LanguageName", Value: "Français v2"},
{Key: "new", Value: "new"},
},
},
}},
}
written, err = store.ReconcileSeed(ctx, seedV2)
if err != nil || written != 2 {
t.Fatalf("reconcile v2 = %d, %v", written, err)
}
pack, err := store.GetPack(ctx, packName, "fr", 0)
if err != nil || pack.Version != 2 || len(pack.Strings) != 2 || postgresLangPackValue(pack.Strings, "old") != "" || postgresLangPackValue(pack.Strings, "new") != "new" {
t.Fatalf("replaced postgres pack = %+v, err %v", pack, err)
}
mutated := seedV2
mutated.Packs = append([]domain.LangPackSeedEntry(nil), seedV2.Packs...)
mutated.Packs[0].ContentHash = "changed-without-version"
if _, err := store.ReconcileSeed(ctx, mutated); err == nil || !strings.Contains(err.Error(), "without version bump") {
t.Fatalf("same-version mutation error = %v", err)
}
rollback := seedV2
rollback.Packs = append([]domain.LangPackSeedEntry(nil), seedV2.Packs...)
rollback.Packs[0].Pack.Version = 1
rollback.Packs[0].ContentHash = "rollback"
if _, err := store.ReconcileSeed(ctx, rollback); err == nil || !strings.Contains(err.Error(), "version rollback") {
t.Fatalf("rollback error = %v", err)
}
if written, err := store.ReconcileSeed(ctx, domain.LangPackSeed{Catalog: packName}); err != nil || written != 0 {
t.Fatalf("remove missing language = %d, %v", written, err)
}
languages, err := store.ListLanguages(ctx, packName)
if err != nil || len(languages) != 0 {
t.Fatalf("languages after manifest removal = %+v, err %v", languages, err)
}
}
func postgresLangPackValue(items []domain.LangPackString, key string) string {
for _, item := range items {
if item.Key == key {
return item.Value
}
}
return ""
}
func TestBundledLangPackSeedPostgres(t *testing.T) {
pool := testPool(t)
ctx := context.Background()
scopes := []string{"android", "android_x", "ios", "macos", "tdesktop", "weba"}
t.Cleanup(func() {
_, _ = pool.Exec(ctx, "DELETE FROM lang_pack_strings WHERE lang_pack = ANY($1::text[])", scopes)
_, _ = pool.Exec(ctx, "DELETE FROM lang_packs WHERE lang_pack = ANY($1::text[])", scopes)
for _, scope := range scopes {
_, _ = pool.Exec(ctx, "DELETE FROM seed_states WHERE key LIKE $1", "langpack:v1:entry:"+scope+":%")
}
_, _ = pool.Exec(ctx, "DELETE FROM seed_states WHERE key = 'langpack:v1:catalog:langpack'")
_, _ = pool.Exec(ctx, "DELETE FROM seed_states WHERE key = 'langpack:v2:catalog:langpack'")
})
service := applangpack.NewService(NewLangPackStore(pool))
root := filepath.Join("..", "..", "..", "data", "langpack")
started := time.Now()
seeded, err := service.SeedDirectory(ctx, root)
firstDuration := time.Since(started)
if err != nil {
t.Fatalf("seed bundled langpacks to postgres: %v", err)
}
if seeded < 50000 {
t.Fatalf("seeded bundled strings = %d, want full catalog", seeded)
}
started = time.Now()
seededAgain, err := service.SeedDirectory(ctx, root)
secondDuration := time.Since(started)
if err != nil || seededAgain != 0 {
t.Fatalf("reconcile unchanged bundled langpacks = %d, %v", seededAgain, err)
}
var metadataCount, activeCount int64
if err := pool.QueryRow(ctx, `
SELECT
(SELECT COALESCE(sum(strings_count), 0) FROM lang_packs WHERE lang_pack = ANY($1::text[])),
(SELECT count(*) FROM lang_pack_strings WHERE lang_pack = ANY($1::text[]) AND NOT deleted)
`, scopes).Scan(&metadataCount, &activeCount); err != nil {
t.Fatalf("count seeded langpack rows: %v", err)
}
if metadataCount != activeCount {
t.Fatalf("seeded strings_count = %d, active rows = %d", metadataCount, activeCount)
}
t.Logf("bundled langpack seed: first=%s unchanged=%s strings=%d", firstDuration, secondDuration, metadataCount)
}

View file

@ -30,6 +30,36 @@ ON CONFLICT (lang_pack, lang_code, key) DO UPDATE SET
deleted = EXCLUDED.deleted,
updated_at = now();
-- name: ListLangPackCodes :many
SELECT lang_code
FROM lang_packs
WHERE lang_pack = $1
ORDER BY lang_code;
-- name: DeleteLangPackStrings :exec
DELETE FROM lang_pack_strings
WHERE lang_pack = $1 AND lang_code = $2;
-- name: DeleteLangPackMeta :exec
DELETE FROM lang_packs
WHERE lang_pack = $1 AND lang_code = $2;
-- name: GetLangPackSeedHash :one
SELECT content_hash
FROM seed_states
WHERE key = $1;
-- name: PutLangPackSeedHash :exec
INSERT INTO seed_states (key, content_hash)
VALUES ($1, $2)
ON CONFLICT (key) DO UPDATE SET
content_hash = EXCLUDED.content_hash,
updated_at = now();
-- name: DeleteLangPackSeedHash :exec
DELETE FROM seed_states
WHERE key = $1;
-- name: ListLangPackStrings :many
SELECT
lang_pack, lang_code, key, version, pluralized, value,
@ -45,3 +75,48 @@ SELECT
FROM lang_pack_strings
WHERE lang_pack = $1 AND lang_code = $2 AND key = ANY(sqlc.arg(keys)::text[]) AND NOT deleted
ORDER BY key;
-- name: ListLangPackLanguages :many
SELECT
p.lang_pack,
p.lang_code,
p.version,
p.strings_count,
COALESCE(
MAX(s.value) FILTER (WHERE s.key = 'LanguageNameInEnglish'),
MAX(s.value) FILTER (WHERE s.key = 'Localization.EnglishLanguageName'),
''
)::text AS name,
CASE
WHEN p.lang_pack = 'android' THEN COALESCE(
MAX(s.value) FILTER (WHERE s.key = 'TranslateLanguage' || upper(split_part(replace(p.lang_code, '-', '_'), '_', 1))),
MAX(s.value) FILTER (WHERE s.key = 'PassportLanguage_' || upper(split_part(replace(p.lang_code, '-', '_'), '_', 1))),
MAX(s.value) FILTER (WHERE s.key = 'LanguageName'),
''
)
ELSE COALESCE(
MAX(s.value) FILTER (WHERE s.key = 'lng_language_name'),
MAX(s.value) FILTER (WHERE s.key = 'LanguageName'),
MAX(s.value) FILTER (WHERE s.key = 'Localization.LanguageName'),
MAX(s.value) FILTER (WHERE s.key = 'TranslateLanguage' || upper(split_part(replace(p.lang_code, '-', '_'), '_', 1))),
MAX(s.value) FILTER (WHERE s.key = 'PassportLanguage_' || upper(split_part(replace(p.lang_code, '-', '_'), '_', 1))),
''
)
END::text AS native_name
FROM lang_packs p
LEFT JOIN lang_pack_strings s
ON s.lang_pack = p.lang_pack
AND s.lang_code = p.lang_code
AND s.key = ANY(ARRAY[
'lng_language_name',
'LanguageName',
'LanguageNameInEnglish',
'Localization.LanguageName',
'Localization.EnglishLanguageName',
'TranslateLanguage' || upper(split_part(replace(p.lang_code, '-', '_'), '_', 1)),
'PassportLanguage_' || upper(split_part(replace(p.lang_code, '-', '_'), '_', 1))
]::text[])
AND NOT s.deleted
WHERE p.lang_pack = $1
GROUP BY p.lang_pack, p.lang_code, p.version, p.strings_count
ORDER BY p.lang_code;

View file

@ -9,6 +9,46 @@ import (
"context"
)
const deleteLangPackMeta = `-- name: DeleteLangPackMeta :exec
DELETE FROM lang_packs
WHERE lang_pack = $1 AND lang_code = $2
`
type DeleteLangPackMetaParams struct {
LangPack string
LangCode string
}
func (q *Queries) DeleteLangPackMeta(ctx context.Context, arg DeleteLangPackMetaParams) error {
_, err := q.db.Exec(ctx, deleteLangPackMeta, arg.LangPack, arg.LangCode)
return err
}
const deleteLangPackSeedHash = `-- name: DeleteLangPackSeedHash :exec
DELETE FROM seed_states
WHERE key = $1
`
func (q *Queries) DeleteLangPackSeedHash(ctx context.Context, key string) error {
_, err := q.db.Exec(ctx, deleteLangPackSeedHash, key)
return err
}
const deleteLangPackStrings = `-- name: DeleteLangPackStrings :exec
DELETE FROM lang_pack_strings
WHERE lang_pack = $1 AND lang_code = $2
`
type DeleteLangPackStringsParams struct {
LangPack string
LangCode string
}
func (q *Queries) DeleteLangPackStrings(ctx context.Context, arg DeleteLangPackStringsParams) error {
_, err := q.db.Exec(ctx, deleteLangPackStrings, arg.LangPack, arg.LangCode)
return err
}
const getLangPackMeta = `-- name: GetLangPackMeta :one
SELECT lang_pack, lang_code, version, strings_count
FROM lang_packs
@ -39,6 +79,19 @@ func (q *Queries) GetLangPackMeta(ctx context.Context, arg GetLangPackMetaParams
return i, err
}
const getLangPackSeedHash = `-- name: GetLangPackSeedHash :one
SELECT content_hash
FROM seed_states
WHERE key = $1
`
func (q *Queries) GetLangPackSeedHash(ctx context.Context, key string) (string, error) {
row := q.db.QueryRow(ctx, getLangPackSeedHash, key)
var content_hash string
err := row.Scan(&content_hash)
return content_hash, err
}
const getLangPackStringsByKeys = `-- name: GetLangPackStringsByKeys :many
SELECT
lang_pack, lang_code, key, version, pluralized, value,
@ -104,6 +157,115 @@ func (q *Queries) GetLangPackStringsByKeys(ctx context.Context, arg GetLangPackS
return items, nil
}
const listLangPackCodes = `-- name: ListLangPackCodes :many
SELECT lang_code
FROM lang_packs
WHERE lang_pack = $1
ORDER BY lang_code
`
func (q *Queries) ListLangPackCodes(ctx context.Context, langPack string) ([]string, error) {
rows, err := q.db.Query(ctx, listLangPackCodes, langPack)
if err != nil {
return nil, err
}
defer rows.Close()
var items []string
for rows.Next() {
var lang_code string
if err := rows.Scan(&lang_code); err != nil {
return nil, err
}
items = append(items, lang_code)
}
if err := rows.Err(); err != nil {
return nil, err
}
return items, nil
}
const listLangPackLanguages = `-- name: ListLangPackLanguages :many
SELECT
p.lang_pack,
p.lang_code,
p.version,
p.strings_count,
COALESCE(
MAX(s.value) FILTER (WHERE s.key = 'LanguageNameInEnglish'),
MAX(s.value) FILTER (WHERE s.key = 'Localization.EnglishLanguageName'),
''
)::text AS name,
CASE
WHEN p.lang_pack = 'android' THEN COALESCE(
MAX(s.value) FILTER (WHERE s.key = 'TranslateLanguage' || upper(split_part(replace(p.lang_code, '-', '_'), '_', 1))),
MAX(s.value) FILTER (WHERE s.key = 'PassportLanguage_' || upper(split_part(replace(p.lang_code, '-', '_'), '_', 1))),
MAX(s.value) FILTER (WHERE s.key = 'LanguageName'),
''
)
ELSE COALESCE(
MAX(s.value) FILTER (WHERE s.key = 'lng_language_name'),
MAX(s.value) FILTER (WHERE s.key = 'LanguageName'),
MAX(s.value) FILTER (WHERE s.key = 'Localization.LanguageName'),
MAX(s.value) FILTER (WHERE s.key = 'TranslateLanguage' || upper(split_part(replace(p.lang_code, '-', '_'), '_', 1))),
MAX(s.value) FILTER (WHERE s.key = 'PassportLanguage_' || upper(split_part(replace(p.lang_code, '-', '_'), '_', 1))),
''
)
END::text AS native_name
FROM lang_packs p
LEFT JOIN lang_pack_strings s
ON s.lang_pack = p.lang_pack
AND s.lang_code = p.lang_code
AND s.key = ANY(ARRAY[
'lng_language_name',
'LanguageName',
'LanguageNameInEnglish',
'Localization.LanguageName',
'Localization.EnglishLanguageName',
'TranslateLanguage' || upper(split_part(replace(p.lang_code, '-', '_'), '_', 1)),
'PassportLanguage_' || upper(split_part(replace(p.lang_code, '-', '_'), '_', 1))
]::text[])
AND NOT s.deleted
WHERE p.lang_pack = $1
GROUP BY p.lang_pack, p.lang_code, p.version, p.strings_count
ORDER BY p.lang_code
`
type ListLangPackLanguagesRow struct {
LangPack string
LangCode string
Version int32
StringsCount int32
Name string
NativeName string
}
func (q *Queries) ListLangPackLanguages(ctx context.Context, langPack string) ([]ListLangPackLanguagesRow, error) {
rows, err := q.db.Query(ctx, listLangPackLanguages, langPack)
if err != nil {
return nil, err
}
defer rows.Close()
var items []ListLangPackLanguagesRow
for rows.Next() {
var i ListLangPackLanguagesRow
if err := rows.Scan(
&i.LangPack,
&i.LangCode,
&i.Version,
&i.StringsCount,
&i.Name,
&i.NativeName,
); err != nil {
return nil, err
}
items = append(items, i)
}
if err := rows.Err(); err != nil {
return nil, err
}
return items, nil
}
const listLangPackStrings = `-- name: ListLangPackStrings :many
SELECT
lang_pack, lang_code, key, version, pluralized, value,
@ -168,6 +330,24 @@ func (q *Queries) ListLangPackStrings(ctx context.Context, arg ListLangPackStrin
return items, nil
}
const putLangPackSeedHash = `-- name: PutLangPackSeedHash :exec
INSERT INTO seed_states (key, content_hash)
VALUES ($1, $2)
ON CONFLICT (key) DO UPDATE SET
content_hash = EXCLUDED.content_hash,
updated_at = now()
`
type PutLangPackSeedHashParams struct {
Key string
ContentHash string
}
func (q *Queries) PutLangPackSeedHash(ctx context.Context, arg PutLangPackSeedHashParams) error {
_, err := q.db.Exec(ctx, putLangPackSeedHash, arg.Key, arg.ContentHash)
return err
}
const upsertLangPackMeta = `-- name: UpsertLangPackMeta :exec
INSERT INTO lang_packs (lang_pack, lang_code, version, strings_count)
VALUES ($1, $2, $3, $4)