feat: sync langpack and auth updates
Sync telesrv commits 49f9bab and 04d9563 into the public mirror. Exclude private docs and runtime key material per sync rules.
This commit is contained in:
parent
6af61f26ba
commit
1292540350
244 changed files with 1552172 additions and 27668 deletions
238
internal/app/langpack/cache.go
Normal file
238
internal/app/langpack/cache.go
Normal file
|
|
@ -0,0 +1,238 @@
|
|||
package langpack
|
||||
|
||||
import (
|
||||
"container/list"
|
||||
"sync"
|
||||
|
||||
"telesrv/internal/domain"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultLangPackCacheMaxBytes = int64(128 << 20)
|
||||
defaultLangPackCacheMaxEntries = 256
|
||||
defaultLanguageListCacheMaxEntries = 32
|
||||
langPackStringValueHeaderBytes = int64(144)
|
||||
langPackFixedHeaderBytes = int64(80)
|
||||
)
|
||||
|
||||
type langPackCacheKind uint8
|
||||
|
||||
const (
|
||||
langPackCacheRaw langPackCacheKind = iota
|
||||
langPackCacheEffective
|
||||
)
|
||||
|
||||
type langPackCacheKey struct {
|
||||
pack string
|
||||
code string
|
||||
kind langPackCacheKind
|
||||
}
|
||||
|
||||
func (k langPackCacheKey) singleflightKey() string {
|
||||
return string(rune(k.kind)) + "\x00" + k.pack + "\x00" + k.code
|
||||
}
|
||||
|
||||
type langPackCache struct {
|
||||
mu sync.Mutex
|
||||
maxBytes int64
|
||||
maxEntries int
|
||||
usedBytes int64
|
||||
epoch uint64
|
||||
ll *list.List
|
||||
items map[langPackCacheKey]*list.Element
|
||||
}
|
||||
|
||||
type langPackCacheEntry struct {
|
||||
key langPackCacheKey
|
||||
pack domain.LangPack
|
||||
size int64
|
||||
}
|
||||
|
||||
func newLangPackCache(maxBytes int64, maxEntries int) *langPackCache {
|
||||
if maxBytes <= 0 || maxEntries <= 0 {
|
||||
return nil
|
||||
}
|
||||
return &langPackCache{
|
||||
maxBytes: maxBytes,
|
||||
maxEntries: maxEntries,
|
||||
ll: list.New(),
|
||||
items: make(map[langPackCacheKey]*list.Element),
|
||||
}
|
||||
}
|
||||
|
||||
func (c *langPackCache) get(key langPackCacheKey) (domain.LangPack, bool) {
|
||||
if c == nil {
|
||||
return domain.LangPack{}, false
|
||||
}
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
element, ok := c.items[key]
|
||||
if !ok {
|
||||
return domain.LangPack{}, false
|
||||
}
|
||||
c.ll.MoveToFront(element)
|
||||
return cloneLangPack(element.Value.(*langPackCacheEntry).pack), true
|
||||
}
|
||||
|
||||
func (c *langPackCache) loadEpoch() uint64 {
|
||||
if c == nil {
|
||||
return 0
|
||||
}
|
||||
c.mu.Lock()
|
||||
epoch := c.epoch
|
||||
c.mu.Unlock()
|
||||
return epoch
|
||||
}
|
||||
|
||||
// putIfEpoch 返回 false 仅表示 load 期间发生过 flush,调用方必须重载。
|
||||
// 超大单项不进入缓存,但仍可安全返回给当前请求,因此返回 true。
|
||||
func (c *langPackCache) putIfEpoch(key langPackCacheKey, pack domain.LangPack, loadEpoch uint64) bool {
|
||||
if c == nil {
|
||||
return true
|
||||
}
|
||||
size := estimateLangPackBytes(pack)
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
if c.epoch != loadEpoch {
|
||||
return false
|
||||
}
|
||||
if existing, ok := c.items[key]; ok {
|
||||
c.remove(existing)
|
||||
}
|
||||
if size > c.maxBytes {
|
||||
return true
|
||||
}
|
||||
entry := &langPackCacheEntry{key: key, pack: cloneLangPack(pack), size: size}
|
||||
c.items[key] = c.ll.PushFront(entry)
|
||||
c.usedBytes += size
|
||||
for c.usedBytes > c.maxBytes || c.ll.Len() > c.maxEntries {
|
||||
oldest := c.ll.Back()
|
||||
if oldest == nil {
|
||||
break
|
||||
}
|
||||
c.remove(oldest)
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (c *langPackCache) flush() {
|
||||
if c == nil {
|
||||
return
|
||||
}
|
||||
c.mu.Lock()
|
||||
c.epoch++
|
||||
c.usedBytes = 0
|
||||
c.ll.Init()
|
||||
clear(c.items)
|
||||
c.mu.Unlock()
|
||||
}
|
||||
|
||||
func (c *langPackCache) remove(element *list.Element) {
|
||||
entry := element.Value.(*langPackCacheEntry)
|
||||
delete(c.items, entry.key)
|
||||
c.ll.Remove(element)
|
||||
c.usedBytes -= entry.size
|
||||
}
|
||||
|
||||
func estimateLangPackBytes(pack domain.LangPack) int64 {
|
||||
size := langPackFixedHeaderBytes + int64(len(pack.LangPack)+len(pack.LangCode))
|
||||
size += int64(len(pack.Strings)) * langPackStringValueHeaderBytes
|
||||
for _, item := range pack.Strings {
|
||||
size += int64(len(item.Key) + len(item.Value) + len(item.ZeroValue) + len(item.OneValue) +
|
||||
len(item.TwoValue) + len(item.FewValue) + len(item.ManyValue) + len(item.OtherValue))
|
||||
}
|
||||
return size
|
||||
}
|
||||
|
||||
func cloneLangPack(pack domain.LangPack) domain.LangPack {
|
||||
pack.Strings = append([]domain.LangPackString(nil), pack.Strings...)
|
||||
return pack
|
||||
}
|
||||
|
||||
type languageListCache struct {
|
||||
mu sync.Mutex
|
||||
maxEntries int
|
||||
epoch uint64
|
||||
ll *list.List
|
||||
items map[string]*list.Element
|
||||
}
|
||||
|
||||
type languageListCacheEntry struct {
|
||||
pack string
|
||||
languages []domain.LangPackLanguage
|
||||
}
|
||||
|
||||
func newLanguageListCache(maxEntries int) *languageListCache {
|
||||
if maxEntries <= 0 {
|
||||
return nil
|
||||
}
|
||||
return &languageListCache{
|
||||
maxEntries: maxEntries,
|
||||
ll: list.New(),
|
||||
items: make(map[string]*list.Element),
|
||||
}
|
||||
}
|
||||
|
||||
func (c *languageListCache) get(pack string) ([]domain.LangPackLanguage, bool) {
|
||||
if c == nil {
|
||||
return nil, false
|
||||
}
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
element, ok := c.items[pack]
|
||||
if !ok {
|
||||
return nil, false
|
||||
}
|
||||
c.ll.MoveToFront(element)
|
||||
return cloneLanguages(element.Value.(*languageListCacheEntry).languages), true
|
||||
}
|
||||
|
||||
func (c *languageListCache) loadEpoch() uint64 {
|
||||
if c == nil {
|
||||
return 0
|
||||
}
|
||||
c.mu.Lock()
|
||||
epoch := c.epoch
|
||||
c.mu.Unlock()
|
||||
return epoch
|
||||
}
|
||||
|
||||
func (c *languageListCache) putIfEpoch(pack string, languages []domain.LangPackLanguage, loadEpoch uint64) bool {
|
||||
if c == nil {
|
||||
return true
|
||||
}
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
if c.epoch != loadEpoch {
|
||||
return false
|
||||
}
|
||||
if existing, ok := c.items[pack]; ok {
|
||||
c.ll.Remove(existing)
|
||||
delete(c.items, pack)
|
||||
}
|
||||
entry := &languageListCacheEntry{pack: pack, languages: cloneLanguages(languages)}
|
||||
c.items[pack] = c.ll.PushFront(entry)
|
||||
if c.ll.Len() > c.maxEntries {
|
||||
oldest := c.ll.Back()
|
||||
if oldest != nil {
|
||||
delete(c.items, oldest.Value.(*languageListCacheEntry).pack)
|
||||
c.ll.Remove(oldest)
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (c *languageListCache) flush() {
|
||||
if c == nil {
|
||||
return
|
||||
}
|
||||
c.mu.Lock()
|
||||
c.epoch++
|
||||
c.ll.Init()
|
||||
clear(c.items)
|
||||
c.mu.Unlock()
|
||||
}
|
||||
|
||||
func cloneLanguages(languages []domain.LangPackLanguage) []domain.LangPackLanguage {
|
||||
return append([]domain.LangPackLanguage(nil), languages...)
|
||||
}
|
||||
35
internal/app/langpack/cache_test.go
Normal file
35
internal/app/langpack/cache_test.go
Normal file
|
|
@ -0,0 +1,35 @@
|
|||
package langpack
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"telesrv/internal/domain"
|
||||
)
|
||||
|
||||
func TestLangPackCachesRejectLoadsAcrossFlush(t *testing.T) {
|
||||
packCache := newLangPackCache(1<<20, 8)
|
||||
key := langPackCacheKey{pack: "tdesktop", code: "en", kind: langPackCacheRaw}
|
||||
packEpoch := packCache.loadEpoch()
|
||||
packCache.flush()
|
||||
if packCache.putIfEpoch(key, domain.LangPack{
|
||||
LangPack: "tdesktop",
|
||||
LangCode: "en",
|
||||
Version: 1,
|
||||
Strings: []domain.LangPackString{{Key: "key", Value: "stale"}},
|
||||
}, packEpoch) {
|
||||
t.Fatal("pre-flush pack load was accepted")
|
||||
}
|
||||
if _, ok := packCache.get(key); ok {
|
||||
t.Fatal("pre-flush pack load became visible")
|
||||
}
|
||||
|
||||
languageCache := newLanguageListCache(8)
|
||||
languageEpoch := languageCache.loadEpoch()
|
||||
languageCache.flush()
|
||||
if languageCache.putIfEpoch("tdesktop", []domain.LangPackLanguage{{LangCode: "en"}}, languageEpoch) {
|
||||
t.Fatal("pre-flush language-list load was accepted")
|
||||
}
|
||||
if _, ok := languageCache.get("tdesktop"); ok {
|
||||
t.Fatal("pre-flush language-list load became visible")
|
||||
}
|
||||
}
|
||||
|
|
@ -2,6 +2,7 @@ package langpack
|
|||
|
||||
import (
|
||||
"fmt"
|
||||
"math"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
|
|
@ -12,6 +13,8 @@ import (
|
|||
)
|
||||
|
||||
var tdesktopStringRE = regexp.MustCompile(`(?s)"((?:\\.|[^"\\])*)"\s*=\s*"((?:\\.|[^"\\])*)";`)
|
||||
var langPackNameRE = regexp.MustCompile(`^[a-z0-9][a-z0-9_-]{0,31}$`)
|
||||
var langCodeRE = regexp.MustCompile(`^[a-z0-9]{1,16}(?:-[a-z0-9]{1,16})*$`)
|
||||
|
||||
// ParseTDesktopFile 解析客户端 .strings 文件为 domain 语言包。
|
||||
func ParseTDesktopFile(path string) (domain.LangPack, error) {
|
||||
|
|
@ -82,6 +85,17 @@ func packFromFilename(path string) (domain.LangPack, error) {
|
|||
if langPack == "" || langCode == "" {
|
||||
return domain.LangPack{}, fmt.Errorf("invalid langpack filename %q", filepath.Base(path))
|
||||
}
|
||||
langPack = normalizePack(langPack)
|
||||
langCode = normalizeCode(langCode)
|
||||
if !langPackNameRE.MatchString(langPack) {
|
||||
return domain.LangPack{}, fmt.Errorf("invalid langpack name %q in %q", langPack, filepath.Base(path))
|
||||
}
|
||||
if len(langCode) > 64 || !langCodeRE.MatchString(langCode) {
|
||||
return domain.LangPack{}, fmt.Errorf("invalid language code %q in %q", langCode, filepath.Base(path))
|
||||
}
|
||||
if version <= 0 || version > math.MaxInt32 {
|
||||
return domain.LangPack{}, fmt.Errorf("invalid langpack version %d in %q", version, filepath.Base(path))
|
||||
}
|
||||
return domain.LangPack{
|
||||
LangPack: langPack,
|
||||
LangCode: langCode,
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ import (
|
|||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"telesrv/internal/store/memory"
|
||||
|
|
@ -59,6 +60,23 @@ func TestParseClientLangPackFile(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func TestParseClientLangPackFileWithUnderscorePackName(t *testing.T) {
|
||||
packDir := filepath.Join(t.TempDir(), "android_x")
|
||||
if err := os.MkdirAll(packDir, 0o700); err != nil {
|
||||
t.Fatalf("mkdir fixture: %v", err)
|
||||
}
|
||||
path := filepath.Join(packDir, "android_x_en_v42.strings")
|
||||
writeLangPackFixture(t, path, `"TranslationMoreText" = "Translation Platform";`)
|
||||
|
||||
pack, err := ParseTDesktopFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("parse: %v", err)
|
||||
}
|
||||
if pack.LangPack != "android_x" || pack.LangCode != "en" || pack.Version != 42 {
|
||||
t.Fatalf("pack meta = %+v", pack)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSeedDirectoryWalksClientSubdirs(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
for _, item := range []struct {
|
||||
|
|
@ -98,13 +116,21 @@ func TestSeedDirectoryWalksClientSubdirs(t *testing.T) {
|
|||
}
|
||||
|
||||
func TestBundledAndroidPersianLangPackParses(t *testing.T) {
|
||||
path := filepath.Join("..", "..", "..", "data", "langpack", "android", "android_fa_v59634849.strings")
|
||||
pack, err := ParseTDesktopFile(path)
|
||||
root := filepath.Join("..", "..", "..", "data", "langpack", "android")
|
||||
candidates, _, err := scanSeedCandidates(root)
|
||||
if err != nil {
|
||||
t.Fatalf("scan bundled android packs: %v", err)
|
||||
}
|
||||
candidate, ok := candidates["android\x00fa"]
|
||||
if !ok {
|
||||
t.Fatal("bundled android fa pack not found")
|
||||
}
|
||||
pack, err := ParseTDesktopFile(candidate.path)
|
||||
if err != nil {
|
||||
t.Fatalf("parse bundled android fa pack: %v", err)
|
||||
}
|
||||
if pack.LangPack != "android" || pack.LangCode != "fa" || pack.Version != 59634849 {
|
||||
t.Fatalf("pack meta = %+v, want android/fa v59634849", pack)
|
||||
if pack.LangPack != "android" || pack.LangCode != "fa" || pack.Version <= 0 {
|
||||
t.Fatalf("pack meta = %+v, want versioned android/fa", pack)
|
||||
}
|
||||
if len(pack.Strings) < 10000 {
|
||||
t.Fatalf("strings count = %d, want full android fa pack", len(pack.Strings))
|
||||
|
|
@ -120,3 +146,120 @@ func TestBundledAndroidPersianLangPackParses(t *testing.T) {
|
|||
}
|
||||
t.Fatalf("TranslateLanguageFA not found in bundled android fa pack")
|
||||
}
|
||||
|
||||
func TestSeedDirectoryReconcilesManifest(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
root := t.TempDir()
|
||||
packDir := filepath.Join(root, "tdesktop")
|
||||
if err := os.MkdirAll(packDir, 0o700); err != nil {
|
||||
t.Fatalf("mkdir pack dir: %v", err)
|
||||
}
|
||||
v1 := filepath.Join(packDir, "tdesktop_pt-BR_v1.strings")
|
||||
writeLangPackFixture(t, v1, `
|
||||
"lng_language_name" = "Português (Brasil)";
|
||||
"lng_old" = "old";
|
||||
`)
|
||||
|
||||
store := memory.NewLangPackStore()
|
||||
service := NewService(store)
|
||||
seeded, err := service.SeedDirectory(ctx, root)
|
||||
if err != nil || seeded != 2 {
|
||||
t.Fatalf("seed v1 = %d, %v", seeded, err)
|
||||
}
|
||||
pack, err := service.GetLangPack(ctx, "TDESKTOP", "pt_BR")
|
||||
if err != nil || pack.LangCode != "pt-br" || pack.Version != 1 || len(pack.Strings) != 2 {
|
||||
t.Fatalf("normalized pack = %+v, err %v", pack, err)
|
||||
}
|
||||
languages, err := service.ListLanguages(ctx, "tdesktop")
|
||||
if err != nil || findLanguage(languages, "pt-br") == nil {
|
||||
t.Fatalf("languages = %+v, err %v", languages, err)
|
||||
}
|
||||
if seeded, err := service.SeedDirectory(ctx, root); err != nil || seeded != 0 {
|
||||
t.Fatalf("unchanged seed = %d, %v", seeded, err)
|
||||
}
|
||||
|
||||
v2 := filepath.Join(packDir, "tdesktop_pt-br_v2.strings")
|
||||
writeLangPackFixture(t, v2, `
|
||||
"lng_language_name" = "Português do Brasil";
|
||||
"lng_new" = "new";
|
||||
`)
|
||||
seeded, err = service.SeedDirectory(ctx, root)
|
||||
if err != nil || seeded != 2 {
|
||||
t.Fatalf("seed v2 = %d, %v", seeded, err)
|
||||
}
|
||||
pack, err = service.GetLangPack(ctx, "tdesktop", "pt-br")
|
||||
if err != nil || pack.Version != 2 || len(pack.Strings) != 2 || stringValue(pack.Strings, "lng_old") != "" || stringValue(pack.Strings, "lng_new") != "new" {
|
||||
t.Fatalf("replaced pack = %+v, err %v", pack, err)
|
||||
}
|
||||
|
||||
if err := os.Remove(v1); err != nil {
|
||||
t.Fatalf("remove v1: %v", err)
|
||||
}
|
||||
if err := os.Remove(v2); err != nil {
|
||||
t.Fatalf("remove v2: %v", err)
|
||||
}
|
||||
if err := os.Remove(packDir); err != nil {
|
||||
t.Fatalf("remove pack dir: %v", err)
|
||||
}
|
||||
if seeded, err := service.SeedDirectory(ctx, root); err != nil || seeded != 0 {
|
||||
t.Fatalf("reconcile removed file = %d, %v", seeded, err)
|
||||
}
|
||||
languages, err = service.ListLanguages(ctx, "tdesktop")
|
||||
if err != nil || len(languages) != 0 {
|
||||
t.Fatalf("languages after removal = %+v, err %v", languages, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSeedDirectoryRejectsVersionInvariantViolations(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
root := t.TempDir()
|
||||
packDir := filepath.Join(root, "tdesktop")
|
||||
if err := os.MkdirAll(packDir, 0o700); err != nil {
|
||||
t.Fatalf("mkdir pack dir: %v", err)
|
||||
}
|
||||
v2 := filepath.Join(packDir, "tdesktop_fr_v2.strings")
|
||||
writeLangPackFixture(t, v2, `"lng_language_name" = "Français";`)
|
||||
service := NewService(memory.NewLangPackStore())
|
||||
if _, err := service.SeedDirectory(ctx, root); err != nil {
|
||||
t.Fatalf("seed v2: %v", err)
|
||||
}
|
||||
|
||||
writeLangPackFixture(t, v2, `"lng_language_name" = "Français modifié";`)
|
||||
if _, err := service.SeedDirectory(ctx, root); err == nil || !strings.Contains(err.Error(), "without version bump") {
|
||||
t.Fatalf("same-version mutation error = %v", err)
|
||||
}
|
||||
pack, err := service.GetLangPack(ctx, "tdesktop", "fr")
|
||||
if err != nil || stringValue(pack.Strings, "lng_language_name") != "Français" {
|
||||
t.Fatalf("pack changed after rejected mutation = %+v, err %v", pack, err)
|
||||
}
|
||||
|
||||
if err := os.Remove(v2); err != nil {
|
||||
t.Fatalf("remove v2: %v", err)
|
||||
}
|
||||
writeLangPackFixture(t, filepath.Join(packDir, "tdesktop_fr_v1.strings"), `"lng_language_name" = "Français";`)
|
||||
if _, err := service.SeedDirectory(ctx, root); err == nil || !strings.Contains(err.Error(), "version rollback") {
|
||||
t.Fatalf("version rollback error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBundledLangPackDirectoryReconciles(t *testing.T) {
|
||||
root := filepath.Join("..", "..", "..", "data", "langpack")
|
||||
service := NewService(memory.NewLangPackStore())
|
||||
seeded, err := service.SeedDirectory(context.Background(), root)
|
||||
if err != nil {
|
||||
t.Fatalf("seed bundled langpacks: %v", err)
|
||||
}
|
||||
if seeded < 50000 {
|
||||
t.Fatalf("seeded bundled strings = %d, want full catalog", seeded)
|
||||
}
|
||||
if seeded, err := service.SeedDirectory(context.Background(), root); err != nil || seeded != 0 {
|
||||
t.Fatalf("reconcile unchanged bundled langpacks = %d, %v", seeded, err)
|
||||
}
|
||||
}
|
||||
|
||||
func writeLangPackFixture(t *testing.T, path, content string) {
|
||||
t.Helper()
|
||||
if err := os.WriteFile(path, []byte(content), 0o600); err != nil {
|
||||
t.Fatalf("write langpack fixture %q: %v", path, err)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,14 +2,27 @@ package langpack
|
|||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"unicode/utf8"
|
||||
|
||||
"telesrv/internal/domain"
|
||||
)
|
||||
|
||||
// SeedDirectory 将导出的 .strings 文件导入 LangPackStore。
|
||||
type seedCandidate struct {
|
||||
path string
|
||||
meta domain.LangPack
|
||||
}
|
||||
|
||||
// SeedDirectory 将导出的 .strings 文件清单原子对账到 LangPackStore。
|
||||
// root 可直接指向 data/langpack,也可指向包含 .strings 的具体平台目录。
|
||||
func (s *Service) SeedDirectory(ctx context.Context, root string) (int, error) {
|
||||
if s == nil || s.packs == nil || root == "" {
|
||||
|
|
@ -23,33 +36,201 @@ func (s *Service) SeedDirectory(ctx context.Context, root string) (int, error) {
|
|||
return 0, fmt.Errorf("stat langpack seed dir: %w", err)
|
||||
}
|
||||
|
||||
seeded := 0
|
||||
err := filepath.WalkDir(dir, func(path string, entry os.DirEntry, err error) error {
|
||||
candidates, scopes, err := scanSeedCandidates(dir)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("scan langpack seed dir: %w", err)
|
||||
}
|
||||
keys := make([]string, 0, len(candidates))
|
||||
for key := range candidates {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
|
||||
seed := domain.LangPackSeed{
|
||||
Catalog: seedCatalogID(dir),
|
||||
Scopes: scopes,
|
||||
Packs: make([]domain.LangPackSeedEntry, 0, len(keys)),
|
||||
}
|
||||
previous, err := s.packs.GetSeedCatalog(ctx, seed.Catalog)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("get previous langpack seed catalog: %w", err)
|
||||
}
|
||||
previousByKey := make(map[string]domain.LangPackSeedCatalogEntry, len(previous.Packs))
|
||||
for _, entry := range previous.Packs {
|
||||
previousByKey[entry.LangPack+"\x00"+entry.LangCode] = entry
|
||||
}
|
||||
for _, key := range keys {
|
||||
candidate := candidates[key]
|
||||
sourceHash, err := fileSHA256(candidate.path)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("hash langpack source %q: %w", candidate.path, err)
|
||||
}
|
||||
if old, ok := previousByKey[key]; ok &&
|
||||
old.Version == candidate.meta.Version &&
|
||||
old.SourceHash == sourceHash &&
|
||||
old.ContentHash != "" && old.StringsCount > 0 {
|
||||
seed.Packs = append(seed.Packs, domain.LangPackSeedEntry{
|
||||
Pack: candidate.meta,
|
||||
SourceHash: sourceHash,
|
||||
ContentHash: old.ContentHash,
|
||||
StringsCount: old.StringsCount,
|
||||
ContentLoaded: false,
|
||||
})
|
||||
continue
|
||||
}
|
||||
pack, err := ParseTDesktopFile(candidate.path)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
pack, err = prepareSeedPack(pack)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("prepare langpack %q: %w", candidate.path, err)
|
||||
}
|
||||
hash, err := langPackContentHash(pack)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("hash langpack %q: %w", candidate.path, err)
|
||||
}
|
||||
seed.Packs = append(seed.Packs, domain.LangPackSeedEntry{
|
||||
Pack: pack,
|
||||
SourceHash: sourceHash,
|
||||
ContentHash: hash,
|
||||
StringsCount: len(pack.Strings),
|
||||
ContentLoaded: true,
|
||||
})
|
||||
}
|
||||
|
||||
seeded, err := s.packs.ReconcileSeed(ctx, seed)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("reconcile langpack seed: %w", err)
|
||||
}
|
||||
s.flushCaches()
|
||||
return seeded, nil
|
||||
}
|
||||
|
||||
func fileSHA256(path string) (string, error) {
|
||||
file, err := os.Open(path)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer file.Close()
|
||||
hash := sha256.New()
|
||||
if _, err := io.Copy(hash, file); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return hex.EncodeToString(hash.Sum(nil)), nil
|
||||
}
|
||||
|
||||
func scanSeedCandidates(root string) (map[string]seedCandidate, []string, error) {
|
||||
candidates := make(map[string]seedCandidate)
|
||||
scopeSet := make(map[string]struct{})
|
||||
hasChildDirs := false
|
||||
hasFiles := false
|
||||
|
||||
err := filepath.WalkDir(root, func(path string, entry os.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if entry.IsDir() || !strings.EqualFold(filepath.Ext(entry.Name()), ".strings") {
|
||||
rel, err := filepath.Rel(root, path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if entry.IsDir() {
|
||||
if rel != "." && filepath.Dir(rel) == "." {
|
||||
hasChildDirs = true
|
||||
scope := normalizePack(entry.Name())
|
||||
if langPackNameRE.MatchString(scope) {
|
||||
scopeSet[scope] = struct{}{}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
pack, err := ParseTDesktopFile(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
existing, err := s.packs.GetPack(ctx, pack.LangPack, pack.LangCode, pack.Version)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if existing.Version >= pack.Version {
|
||||
if !strings.EqualFold(filepath.Ext(entry.Name()), ".strings") {
|
||||
return nil
|
||||
}
|
||||
if err := s.packs.UpsertPack(ctx, pack); err != nil {
|
||||
hasFiles = true
|
||||
meta, err := packFromFilename(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
seeded += len(pack.Strings)
|
||||
if relDir := filepath.Dir(rel); relDir != "." {
|
||||
firstDir := strings.Split(relDir, string(filepath.Separator))[0]
|
||||
scope := normalizePack(firstDir)
|
||||
if !langPackNameRE.MatchString(scope) || scope != meta.LangPack {
|
||||
return fmt.Errorf("langpack file %q is under pack directory %q", path, firstDir)
|
||||
}
|
||||
}
|
||||
scopeSet[meta.LangPack] = struct{}{}
|
||||
key := meta.LangPack + "\x00" + meta.LangCode
|
||||
if previous, ok := candidates[key]; ok {
|
||||
switch {
|
||||
case meta.Version < previous.meta.Version:
|
||||
return nil
|
||||
case meta.Version == previous.meta.Version:
|
||||
return fmt.Errorf("duplicate langpack version %s/%s v%d in %q and %q", meta.LangPack, meta.LangCode, meta.Version, previous.path, path)
|
||||
}
|
||||
}
|
||||
candidates[key] = seedCandidate{path: path, meta: meta}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return seeded, fmt.Errorf("walk langpack seed dir: %w", err)
|
||||
return nil, nil, err
|
||||
}
|
||||
return seeded, nil
|
||||
if !hasChildDirs && !hasFiles {
|
||||
scope := normalizePack(filepath.Base(root))
|
||||
if langPackNameRE.MatchString(scope) {
|
||||
scopeSet[scope] = struct{}{}
|
||||
}
|
||||
}
|
||||
|
||||
scopes := make([]string, 0, len(scopeSet))
|
||||
for scope := range scopeSet {
|
||||
scopes = append(scopes, scope)
|
||||
}
|
||||
sort.Strings(scopes)
|
||||
return candidates, scopes, nil
|
||||
}
|
||||
|
||||
func prepareSeedPack(pack domain.LangPack) (domain.LangPack, error) {
|
||||
pack.LangPack = normalizePack(pack.LangPack)
|
||||
pack.LangCode = normalizeCode(pack.LangCode)
|
||||
pack.FromVersion = 0
|
||||
if len(pack.Strings) == 0 {
|
||||
return domain.LangPack{}, errors.New("language file contains no strings")
|
||||
}
|
||||
deduplicated := make([]domain.LangPackString, 0, len(pack.Strings))
|
||||
indexes := make(map[string]int, len(pack.Strings))
|
||||
for _, item := range pack.Strings {
|
||||
if item.Key == "" || utf8.RuneCountInString(item.Key) > 128 {
|
||||
return domain.LangPack{}, fmt.Errorf("invalid string key %q", item.Key)
|
||||
}
|
||||
if index, exists := indexes[item.Key]; exists {
|
||||
deduplicated[index] = item
|
||||
continue
|
||||
}
|
||||
indexes[item.Key] = len(deduplicated)
|
||||
deduplicated = append(deduplicated, item)
|
||||
}
|
||||
pack.Strings = deduplicated
|
||||
sort.Slice(pack.Strings, func(i, j int) bool {
|
||||
return pack.Strings[i].Key < pack.Strings[j].Key
|
||||
})
|
||||
return pack, nil
|
||||
}
|
||||
|
||||
func langPackContentHash(pack domain.LangPack) (string, error) {
|
||||
encoded, err := json.Marshal(pack)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
sum := sha256.Sum256(encoded)
|
||||
return hex.EncodeToString(sum[:]), nil
|
||||
}
|
||||
|
||||
func seedCatalogID(root string) string {
|
||||
base := normalizePack(filepath.Base(root))
|
||||
if langPackNameRE.MatchString(base) {
|
||||
return base
|
||||
}
|
||||
sum := sha256.Sum256([]byte(filepath.Clean(root)))
|
||||
return "path-" + hex.EncodeToString(sum[:8])
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4,18 +4,38 @@ import (
|
|||
"context"
|
||||
"strings"
|
||||
|
||||
"golang.org/x/sync/singleflight"
|
||||
"golang.org/x/text/unicode/bidi"
|
||||
|
||||
"telesrv/internal/domain"
|
||||
"telesrv/internal/store"
|
||||
)
|
||||
|
||||
// Service 提供客户端语言包查询。
|
||||
type Service struct {
|
||||
packs store.LangPackStore
|
||||
packs store.LangPackStore
|
||||
packCache *langPackCache
|
||||
languageCache *languageListCache
|
||||
packLoads singleflight.Group
|
||||
languageLoads singleflight.Group
|
||||
}
|
||||
|
||||
// NewService 创建 langpack 服务。
|
||||
func NewService(packs store.LangPackStore) *Service {
|
||||
return &Service{packs: packs}
|
||||
return newServiceWithCacheLimits(
|
||||
packs,
|
||||
defaultLangPackCacheMaxBytes,
|
||||
defaultLangPackCacheMaxEntries,
|
||||
defaultLanguageListCacheMaxEntries,
|
||||
)
|
||||
}
|
||||
|
||||
func newServiceWithCacheLimits(packs store.LangPackStore, maxBytes int64, maxEntries, languageEntries int) *Service {
|
||||
return &Service{
|
||||
packs: packs,
|
||||
packCache: newLangPackCache(maxBytes, maxEntries),
|
||||
languageCache: newLanguageListCache(languageEntries),
|
||||
}
|
||||
}
|
||||
|
||||
// GetLangPack 返回完整语言包。
|
||||
|
|
@ -30,11 +50,23 @@ func (s *Service) GetDifference(ctx context.Context, langPack, langCode string,
|
|||
if s == nil || s.packs == nil {
|
||||
return domain.LangPack{LangPack: packName, LangCode: code, FromVersion: fromVersion}, nil
|
||||
}
|
||||
pack, err := s.packs.GetPack(ctx, packName, code, fromVersion)
|
||||
var (
|
||||
pack domain.LangPack
|
||||
err error
|
||||
)
|
||||
if fromVersion == 0 {
|
||||
pack, err = s.effectivePack(ctx, packName, code)
|
||||
} else {
|
||||
pack, err = s.rawPack(ctx, packName, code)
|
||||
}
|
||||
if err != nil {
|
||||
return domain.LangPack{}, err
|
||||
}
|
||||
return s.overlayWebAStrings(ctx, pack, packName, code, fromVersion)
|
||||
pack.FromVersion = fromVersion
|
||||
if pack.Version <= fromVersion {
|
||||
pack.Strings = nil
|
||||
}
|
||||
return pack, nil
|
||||
}
|
||||
|
||||
// GetStrings 返回指定 key 的语言包字符串。
|
||||
|
|
@ -44,29 +76,42 @@ func (s *Service) GetStrings(ctx context.Context, langPack, langCode string, key
|
|||
if s == nil || s.packs == nil {
|
||||
return domain.LangPack{LangPack: packName, LangCode: code}, nil
|
||||
}
|
||||
pack, err := s.packs.GetStrings(ctx, packName, code, keys)
|
||||
pack, err := s.effectivePack(ctx, packName, code)
|
||||
if err != nil {
|
||||
return domain.LangPack{}, err
|
||||
}
|
||||
if len(keys) == 0 {
|
||||
return s.overlayWebAStrings(ctx, pack, packName, code, 0)
|
||||
}
|
||||
missing := missingLangPackKeys(keys, pack.Strings)
|
||||
if len(missing) == 0 || !shouldOverlayWebA(packName) {
|
||||
return pack, nil
|
||||
}
|
||||
overlay, err := s.packs.GetStrings(ctx, "weba", code, missing)
|
||||
if err != nil {
|
||||
return domain.LangPack{}, err
|
||||
wanted := make(map[string]struct{}, len(keys))
|
||||
for _, key := range keys {
|
||||
wanted[key] = struct{}{}
|
||||
}
|
||||
return mergeMissingLangPackStrings(pack, overlay), nil
|
||||
selected := pack
|
||||
selected.Strings = make([]domain.LangPackString, 0, len(keys))
|
||||
for _, item := range pack.Strings {
|
||||
if _, ok := wanted[item.Key]; ok {
|
||||
selected.Strings = append(selected.Strings, item)
|
||||
}
|
||||
}
|
||||
return selected, nil
|
||||
}
|
||||
|
||||
// ListLanguages 返回已 seed 的语言包语言列表。
|
||||
func (s *Service) ListLanguages(ctx context.Context, langPack string) ([]domain.LangPackLanguage, error) {
|
||||
packName := normalizePack(langPack)
|
||||
if s == nil || s.packs == nil {
|
||||
return nil, nil
|
||||
}
|
||||
return s.cachedLanguages(ctx, packName)
|
||||
}
|
||||
|
||||
func normalizePack(langPack string) string {
|
||||
if langPack == "" {
|
||||
pack := strings.ToLower(strings.TrimSpace(langPack))
|
||||
if pack == "" {
|
||||
return "tdesktop"
|
||||
}
|
||||
return langPack
|
||||
return pack
|
||||
}
|
||||
|
||||
func normalizeCode(langCode string) string {
|
||||
|
|
@ -74,6 +119,7 @@ func normalizeCode(langCode string) string {
|
|||
if code == "" {
|
||||
return "en"
|
||||
}
|
||||
code = strings.ReplaceAll(code, "_", "-")
|
||||
return strings.TrimSuffix(code, "-raw")
|
||||
}
|
||||
|
||||
|
|
@ -86,15 +132,117 @@ func shouldOverlayWebA(langPack string) bool {
|
|||
}
|
||||
}
|
||||
|
||||
func (s *Service) overlayWebAStrings(ctx context.Context, pack domain.LangPack, langPack, langCode string, fromVersion int) (domain.LangPack, error) {
|
||||
if fromVersion != 0 || !shouldOverlayWebA(langPack) {
|
||||
return pack, nil
|
||||
func (s *Service) rawPack(ctx context.Context, langPack, langCode string) (domain.LangPack, error) {
|
||||
key := langPackCacheKey{pack: langPack, code: langCode, kind: langPackCacheRaw}
|
||||
return s.cachedPack(ctx, key, func() (domain.LangPack, error) {
|
||||
return s.packs.GetPack(ctx, langPack, langCode, 0)
|
||||
})
|
||||
}
|
||||
|
||||
func (s *Service) effectivePack(ctx context.Context, langPack, langCode string) (domain.LangPack, error) {
|
||||
if !shouldOverlayWebA(langPack) {
|
||||
return s.rawPack(ctx, langPack, langCode)
|
||||
}
|
||||
overlay, err := s.packs.GetPack(ctx, "weba", langCode, fromVersion)
|
||||
if err != nil {
|
||||
return domain.LangPack{}, err
|
||||
key := langPackCacheKey{pack: langPack, code: langCode, kind: langPackCacheEffective}
|
||||
return s.cachedPack(ctx, key, func() (domain.LangPack, error) {
|
||||
pack, err := s.rawPack(ctx, langPack, langCode)
|
||||
if err != nil {
|
||||
return domain.LangPack{}, err
|
||||
}
|
||||
overlay, err := s.rawPack(ctx, "weba", langCode)
|
||||
if err != nil {
|
||||
return domain.LangPack{}, err
|
||||
}
|
||||
return mergeMissingLangPackStrings(pack, overlay), nil
|
||||
})
|
||||
}
|
||||
|
||||
type cachedPackLoadResult struct {
|
||||
pack domain.LangPack
|
||||
stable bool
|
||||
}
|
||||
|
||||
func (s *Service) cachedPack(ctx context.Context, key langPackCacheKey, load func() (domain.LangPack, error)) (domain.LangPack, error) {
|
||||
if s.packCache == nil {
|
||||
return load()
|
||||
}
|
||||
return mergeMissingLangPackStrings(pack, overlay), nil
|
||||
for {
|
||||
if pack, ok := s.packCache.get(key); ok {
|
||||
return pack, nil
|
||||
}
|
||||
value, err, _ := s.packLoads.Do(key.singleflightKey(), func() (any, error) {
|
||||
if pack, ok := s.packCache.get(key); ok {
|
||||
return cachedPackLoadResult{pack: pack, stable: true}, nil
|
||||
}
|
||||
loadEpoch := s.packCache.loadEpoch()
|
||||
pack, err := load()
|
||||
if err != nil {
|
||||
return cachedPackLoadResult{}, err
|
||||
}
|
||||
return cachedPackLoadResult{
|
||||
pack: pack,
|
||||
stable: s.packCache.putIfEpoch(key, pack, loadEpoch),
|
||||
}, nil
|
||||
})
|
||||
if err != nil {
|
||||
return domain.LangPack{}, err
|
||||
}
|
||||
result := value.(cachedPackLoadResult)
|
||||
if result.stable {
|
||||
return cloneLangPack(result.pack), nil
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return domain.LangPack{}, err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type cachedLanguagesLoadResult struct {
|
||||
languages []domain.LangPackLanguage
|
||||
stable bool
|
||||
}
|
||||
|
||||
func (s *Service) cachedLanguages(ctx context.Context, langPack string) ([]domain.LangPackLanguage, error) {
|
||||
if languages, ok := s.languageCache.get(langPack); ok {
|
||||
return languages, nil
|
||||
}
|
||||
for {
|
||||
value, err, _ := s.languageLoads.Do(langPack, func() (any, error) {
|
||||
if languages, ok := s.languageCache.get(langPack); ok {
|
||||
return cachedLanguagesLoadResult{languages: languages, stable: true}, nil
|
||||
}
|
||||
loadEpoch := s.languageCache.loadEpoch()
|
||||
languages, err := s.packs.ListLanguages(ctx, langPack)
|
||||
if err != nil {
|
||||
return cachedLanguagesLoadResult{}, err
|
||||
}
|
||||
for i := range languages {
|
||||
languages[i] = completeLanguageMetadata(langPack, languages[i])
|
||||
}
|
||||
return cachedLanguagesLoadResult{
|
||||
languages: languages,
|
||||
stable: s.languageCache.putIfEpoch(langPack, languages, loadEpoch),
|
||||
}, nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result := value.(cachedLanguagesLoadResult)
|
||||
if result.stable {
|
||||
return cloneLanguages(result.languages), nil
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Service) flushCaches() {
|
||||
if s == nil {
|
||||
return
|
||||
}
|
||||
s.packCache.flush()
|
||||
s.languageCache.flush()
|
||||
}
|
||||
|
||||
func mergeMissingLangPackStrings(pack, overlay domain.LangPack) domain.LangPack {
|
||||
|
|
@ -121,6 +269,57 @@ func mergeMissingLangPackStrings(pack, overlay domain.LangPack) domain.LangPack
|
|||
return pack
|
||||
}
|
||||
|
||||
func completeLanguageMetadata(langPack string, lang domain.LangPackLanguage) domain.LangPackLanguage {
|
||||
if lang.LangPack == "" {
|
||||
lang.LangPack = langPack
|
||||
}
|
||||
lang.LangCode = normalizeCode(lang.LangCode)
|
||||
if lang.PluralCode == "" {
|
||||
lang.PluralCode = pluralCode(lang.LangCode)
|
||||
}
|
||||
if lang.NativeName == "" {
|
||||
lang.NativeName = lang.Name
|
||||
}
|
||||
if lang.Name == "" {
|
||||
lang.Name = lang.NativeName
|
||||
}
|
||||
if lang.Name == "" {
|
||||
lang.Name = lang.LangCode
|
||||
}
|
||||
if lang.NativeName == "" {
|
||||
lang.NativeName = lang.Name
|
||||
}
|
||||
if lang.StringsCount == 0 {
|
||||
lang.StringsCount = lang.TranslatedCount
|
||||
}
|
||||
if lang.TranslatedCount == 0 {
|
||||
lang.TranslatedCount = lang.StringsCount
|
||||
}
|
||||
lang.Official = true
|
||||
lang.Rtl = lang.Rtl || isRTLText(lang.NativeName)
|
||||
return lang
|
||||
}
|
||||
|
||||
func pluralCode(langCode string) string {
|
||||
if idx := strings.IndexAny(langCode, "-_"); idx > 0 {
|
||||
return langCode[:idx]
|
||||
}
|
||||
return langCode
|
||||
}
|
||||
|
||||
func isRTLText(value string) bool {
|
||||
for _, r := range value {
|
||||
properties, _ := bidi.LookupRune(r)
|
||||
switch properties.Class() {
|
||||
case bidi.R, bidi.AL:
|
||||
return true
|
||||
case bidi.L:
|
||||
return false
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func missingLangPackKeys(keys []string, strings []domain.LangPackString) []string {
|
||||
if len(keys) == 0 {
|
||||
return nil
|
||||
|
|
|
|||
|
|
@ -2,9 +2,12 @@ package langpack
|
|||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"telesrv/internal/domain"
|
||||
"telesrv/internal/store"
|
||||
"telesrv/internal/store/memory"
|
||||
)
|
||||
|
||||
|
|
@ -71,6 +74,258 @@ func TestServiceNormalizesWebARawLangCode(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func TestListLanguagesUsesSeededPacks(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
packs := memory.NewLangPackStore()
|
||||
svc := NewService(packs)
|
||||
if err := packs.UpsertPack(ctx, domain.LangPack{
|
||||
LangPack: "tdesktop",
|
||||
LangCode: "fr",
|
||||
Version: 7,
|
||||
Strings: []domain.LangPackString{
|
||||
{Key: "lng_language_name", Value: "Français"},
|
||||
{Key: "lng_test", Value: "Test"},
|
||||
},
|
||||
}); err != nil {
|
||||
t.Fatalf("seed fr langpack: %v", err)
|
||||
}
|
||||
if err := packs.UpsertPack(ctx, domain.LangPack{
|
||||
LangPack: "android",
|
||||
LangCode: "fa",
|
||||
Version: 8,
|
||||
Strings: []domain.LangPackString{
|
||||
{Key: "LanguageName", Value: "انگلیسی"},
|
||||
{Key: "TranslateLanguageFA", Value: "فارسی"},
|
||||
{Key: "lng_test", Value: "Test"},
|
||||
},
|
||||
}); err != nil {
|
||||
t.Fatalf("seed fa langpack: %v", err)
|
||||
}
|
||||
if err := packs.UpsertPack(ctx, domain.LangPack{
|
||||
LangPack: "tdesktop",
|
||||
LangCode: "ckb",
|
||||
Version: 9,
|
||||
Strings: []domain.LangPackString{{Key: "lng_language_name", Value: "کوردی"}},
|
||||
}); err != nil {
|
||||
t.Fatalf("seed ckb langpack: %v", err)
|
||||
}
|
||||
|
||||
tdesktop, err := svc.ListLanguages(ctx, "tdesktop")
|
||||
if err != nil {
|
||||
t.Fatalf("list tdesktop languages: %v", err)
|
||||
}
|
||||
fr := findLanguage(tdesktop, "fr")
|
||||
if fr == nil || fr.Name != "Français" || fr.NativeName != "Français" || fr.PluralCode != "fr" || fr.StringsCount != 2 {
|
||||
t.Fatalf("fr language = %+v", fr)
|
||||
}
|
||||
ckb := findLanguage(tdesktop, "ckb")
|
||||
if ckb == nil || !ckb.Rtl {
|
||||
t.Fatalf("ckb language = %+v, want file-derived rtl", ckb)
|
||||
}
|
||||
|
||||
android, err := svc.ListLanguages(ctx, "android")
|
||||
if err != nil {
|
||||
t.Fatalf("list android languages: %v", err)
|
||||
}
|
||||
fa := findLanguage(android, "fa")
|
||||
if fa == nil || fa.NativeName != "فارسی" || !fa.Rtl || fa.PluralCode != "fa" {
|
||||
t.Fatalf("fa language = %+v", fa)
|
||||
}
|
||||
}
|
||||
|
||||
func TestServiceCachesLanguageResourcesAfterFirstRequest(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
base := memory.NewLangPackStore()
|
||||
for _, pack := range []domain.LangPack{
|
||||
{
|
||||
LangPack: "android",
|
||||
LangCode: "en",
|
||||
Version: 7,
|
||||
Strings: []domain.LangPackString{
|
||||
{Key: "LogOutTitle", Value: "Log Out"},
|
||||
{Key: "NewMessageTitle", Value: "New Message"},
|
||||
},
|
||||
},
|
||||
{
|
||||
LangPack: "weba",
|
||||
LangCode: "en",
|
||||
Version: 12,
|
||||
Strings: []domain.LangPackString{
|
||||
{Key: "AccDescrPollVoteDown", Value: "Go to next unread poll vote"},
|
||||
{Key: "NewMessageTitle", Value: "New Message from WebA"},
|
||||
},
|
||||
},
|
||||
} {
|
||||
if err := base.UpsertPack(ctx, pack); err != nil {
|
||||
t.Fatalf("seed %s: %v", pack.LangPack, err)
|
||||
}
|
||||
}
|
||||
counting := &countingLangPackStore{LangPackStore: base}
|
||||
svc := NewService(counting)
|
||||
|
||||
first, err := svc.GetLangPack(ctx, "android", "en")
|
||||
if err != nil {
|
||||
t.Fatalf("first get langpack: %v", err)
|
||||
}
|
||||
first.Strings[0].Value = "caller mutation"
|
||||
second, err := svc.GetLangPack(ctx, "android", "en")
|
||||
if err != nil {
|
||||
t.Fatalf("second get langpack: %v", err)
|
||||
}
|
||||
if got := stringValue(second.Strings, "LogOutTitle"); got != "Log Out" {
|
||||
t.Fatalf("cached pack was mutated through caller alias: %q", got)
|
||||
}
|
||||
selected, err := svc.GetStrings(ctx, "android", "en", []string{"AccDescrPollVoteDown"})
|
||||
if err != nil || stringValue(selected.Strings, "AccDescrPollVoteDown") == "" {
|
||||
t.Fatalf("cached get strings = %+v, %v", selected, err)
|
||||
}
|
||||
if _, err := svc.GetDifference(ctx, "android", "en", 1); err != nil {
|
||||
t.Fatalf("cached get difference: %v", err)
|
||||
}
|
||||
|
||||
languages, err := svc.ListLanguages(ctx, "android")
|
||||
if err != nil || len(languages) != 1 {
|
||||
t.Fatalf("first list languages = %+v, %v", languages, err)
|
||||
}
|
||||
languages[0].Name = "caller mutation"
|
||||
languages, err = svc.ListLanguages(ctx, "android")
|
||||
if err != nil || len(languages) != 1 || languages[0].Name == "caller mutation" {
|
||||
t.Fatalf("cached languages alias = %+v, %v", languages, err)
|
||||
}
|
||||
|
||||
getPack, getStrings, listLanguages := counting.counts()
|
||||
if getPack != 2 || getStrings != 0 || listLanguages != 1 {
|
||||
t.Fatalf("store calls = getPack:%d getStrings:%d list:%d, want 2/0/1", getPack, getStrings, listLanguages)
|
||||
}
|
||||
}
|
||||
|
||||
func TestServiceCollapsesConcurrentLanguagePackLoads(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
base := memory.NewLangPackStore()
|
||||
if err := base.UpsertPack(ctx, domain.LangPack{
|
||||
LangPack: "weba",
|
||||
LangCode: "en",
|
||||
Version: 3,
|
||||
Strings: []domain.LangPackString{{Key: "NewMessageTitle", Value: "New Message"}},
|
||||
}); err != nil {
|
||||
t.Fatalf("seed weba: %v", err)
|
||||
}
|
||||
counting := &countingLangPackStore{LangPackStore: base, delay: 10 * time.Millisecond}
|
||||
svc := NewService(counting)
|
||||
start := make(chan struct{})
|
||||
errs := make(chan error, 32)
|
||||
var wg sync.WaitGroup
|
||||
for range 32 {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
<-start
|
||||
_, err := svc.GetLangPack(ctx, "weba", "en")
|
||||
errs <- err
|
||||
}()
|
||||
}
|
||||
close(start)
|
||||
wg.Wait()
|
||||
close(errs)
|
||||
for err := range errs {
|
||||
if err != nil {
|
||||
t.Fatalf("concurrent get langpack: %v", err)
|
||||
}
|
||||
}
|
||||
getPack, _, _ := counting.counts()
|
||||
if getPack != 1 {
|
||||
t.Fatalf("concurrent store getPack calls = %d, want 1", getPack)
|
||||
}
|
||||
}
|
||||
|
||||
func TestServiceLanguagePackCacheIsBounded(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
base := memory.NewLangPackStore()
|
||||
for _, code := range []string{"en", "fr"} {
|
||||
if err := base.UpsertPack(ctx, domain.LangPack{
|
||||
LangPack: "weba",
|
||||
LangCode: code,
|
||||
Version: 1,
|
||||
Strings: []domain.LangPackString{{Key: "key", Value: code}},
|
||||
}); err != nil {
|
||||
t.Fatalf("seed %s: %v", code, err)
|
||||
}
|
||||
}
|
||||
counting := &countingLangPackStore{LangPackStore: base}
|
||||
svc := newServiceWithCacheLimits(counting, 1<<20, 1, 1)
|
||||
for _, code := range []string{"en", "fr", "en"} {
|
||||
if _, err := svc.GetLangPack(ctx, "weba", code); err != nil {
|
||||
t.Fatalf("get %s: %v", code, err)
|
||||
}
|
||||
}
|
||||
getPack, _, _ := counting.counts()
|
||||
if getPack != 3 {
|
||||
t.Fatalf("LRU store getPack calls = %d, want 3", getPack)
|
||||
}
|
||||
|
||||
oversized := &countingLangPackStore{LangPackStore: base}
|
||||
svc = newServiceWithCacheLimits(oversized, 1, 8, 1)
|
||||
if _, err := svc.GetLangPack(ctx, "weba", "en"); err != nil {
|
||||
t.Fatalf("first oversized get: %v", err)
|
||||
}
|
||||
if _, err := svc.GetLangPack(ctx, "weba", "en"); err != nil {
|
||||
t.Fatalf("second oversized get: %v", err)
|
||||
}
|
||||
getPack, _, _ = oversized.counts()
|
||||
if getPack != 2 {
|
||||
t.Fatalf("oversized store getPack calls = %d, want 2", getPack)
|
||||
}
|
||||
}
|
||||
|
||||
type countingLangPackStore struct {
|
||||
store.LangPackStore
|
||||
mu sync.Mutex
|
||||
delay time.Duration
|
||||
getPack int
|
||||
getStrings int
|
||||
listLanguages int
|
||||
}
|
||||
|
||||
func (s *countingLangPackStore) GetPack(ctx context.Context, langPack, langCode string, fromVersion int) (domain.LangPack, error) {
|
||||
s.mu.Lock()
|
||||
s.getPack++
|
||||
delay := s.delay
|
||||
s.mu.Unlock()
|
||||
if delay > 0 {
|
||||
time.Sleep(delay)
|
||||
}
|
||||
return s.LangPackStore.GetPack(ctx, langPack, langCode, fromVersion)
|
||||
}
|
||||
|
||||
func (s *countingLangPackStore) GetStrings(ctx context.Context, langPack, langCode string, keys []string) (domain.LangPack, error) {
|
||||
s.mu.Lock()
|
||||
s.getStrings++
|
||||
s.mu.Unlock()
|
||||
return s.LangPackStore.GetStrings(ctx, langPack, langCode, keys)
|
||||
}
|
||||
|
||||
func (s *countingLangPackStore) ListLanguages(ctx context.Context, langPack string) ([]domain.LangPackLanguage, error) {
|
||||
s.mu.Lock()
|
||||
s.listLanguages++
|
||||
s.mu.Unlock()
|
||||
return s.LangPackStore.ListLanguages(ctx, langPack)
|
||||
}
|
||||
|
||||
func (s *countingLangPackStore) counts() (getPack, getStrings, listLanguages int) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return s.getPack, s.getStrings, s.listLanguages
|
||||
}
|
||||
|
||||
func findLanguage(languages []domain.LangPackLanguage, code string) *domain.LangPackLanguage {
|
||||
for i := range languages {
|
||||
if languages[i].LangCode == code {
|
||||
return &languages[i]
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func stringValue(strings []domain.LangPackString, key string) string {
|
||||
for _, item := range strings {
|
||||
if item.Key == key {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue