feat: sync langpack and auth updates
Sync telesrv commits 49f9bab and 04d9563 into the public mirror. Exclude private docs and runtime key material per sync rules.
This commit is contained in:
A
parent
6af61f26ba
commit
1292540350
244 changed files with 1552172 additions and 27668 deletions
|
|
@ -5,14 +5,20 @@
|
|||
// 用法:
|
||||
//
|
||||
// langpackfetch languages [pack] 列出某 pack(默认 android)的可用语言
|
||||
// langpackfetch all <out_dir> [pack...] 拉取所有官方语言及 manifest
|
||||
// langpackfetch <out_dir> <langCode> [pack...] 拉取语言包(默认 packs = android ios macos)
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
|
@ -26,13 +32,46 @@ const (
|
|||
tdesktopAPIHash = "344583e45741c457fe1862106095a5eb"
|
||||
)
|
||||
|
||||
var (
|
||||
officialPacks = []string{"android", "android_x", "ios", "macos", "tdesktop", "weba", "webk"}
|
||||
packNameRE = regexp.MustCompile(`^[a-z0-9][a-z0-9_-]{0,31}$`)
|
||||
langCodeRE = regexp.MustCompile(`^[a-z0-9]{1,16}(?:[-_][a-z0-9]{1,16})*$`)
|
||||
)
|
||||
|
||||
type manifest struct {
|
||||
Schema int `json:"schema"`
|
||||
Packs []packManifest `json:"packs"`
|
||||
}
|
||||
|
||||
type packManifest struct {
|
||||
Name string `json:"name"`
|
||||
Languages []languageManifest `json:"languages"`
|
||||
}
|
||||
|
||||
type languageManifest struct {
|
||||
LangCode string `json:"lang_code"`
|
||||
Name string `json:"name"`
|
||||
NativeName string `json:"native_name"`
|
||||
BaseLangCode string `json:"base_lang_code,omitempty"`
|
||||
PluralCode string `json:"plural_code"`
|
||||
Official bool `json:"official"`
|
||||
RTL bool `json:"rtl,omitempty"`
|
||||
Beta bool `json:"beta,omitempty"`
|
||||
StringsCount int `json:"strings_count"`
|
||||
TranslatedCount int `json:"translated_count"`
|
||||
TranslationsURL string `json:"translations_url,omitempty"`
|
||||
Version int `json:"version"`
|
||||
File string `json:"file"`
|
||||
SHA256 string `json:"sha256"`
|
||||
}
|
||||
|
||||
func main() {
|
||||
if len(os.Args) < 2 {
|
||||
fmt.Fprintln(os.Stderr, "usage:\n langpackfetch languages [pack]\n langpackfetch <out_dir> <langCode> [pack...]")
|
||||
fmt.Fprintln(os.Stderr, "usage:\n langpackfetch languages [pack]\n langpackfetch all <out_dir> [pack...]\n langpackfetch <out_dir> <langCode> [pack...]")
|
||||
os.Exit(2)
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 6*time.Minute)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
|
||||
defer cancel()
|
||||
|
||||
client := telegram.NewClient(tdesktopAPIID, tdesktopAPIHash, telegram.Options{})
|
||||
|
|
@ -55,6 +94,16 @@ func main() {
|
|||
}
|
||||
return nil
|
||||
}
|
||||
if os.Args[1] == "all" {
|
||||
if len(os.Args) < 3 {
|
||||
return fmt.Errorf("all needs <out_dir>")
|
||||
}
|
||||
packs := os.Args[3:]
|
||||
if len(packs) == 0 {
|
||||
packs = officialPacks
|
||||
}
|
||||
return fetchAll(ctx, api, os.Args[2], packs)
|
||||
}
|
||||
|
||||
outRoot := os.Args[1]
|
||||
langCode := "en"
|
||||
|
|
@ -74,7 +123,7 @@ func main() {
|
|||
fmt.Fprintf(os.Stderr, "skip %s/%s: %v\n", pack, langCode, err)
|
||||
continue
|
||||
}
|
||||
if err := writePack(outRoot, pack, diff); err != nil {
|
||||
if _, err := writePack(outRoot, pack, diff); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
|
@ -85,10 +134,88 @@ func main() {
|
|||
}
|
||||
}
|
||||
|
||||
func writePack(root, pack string, diff *tg.LangPackDifference) error {
|
||||
func fetchAll(ctx context.Context, api *tg.Client, root string, packs []string) error {
|
||||
seen := make(map[string]struct{}, len(packs))
|
||||
result := manifest{Schema: 1}
|
||||
for _, pack := range packs {
|
||||
pack = strings.ToLower(strings.TrimSpace(pack))
|
||||
if !packNameRE.MatchString(pack) {
|
||||
return fmt.Errorf("invalid lang pack %q", pack)
|
||||
}
|
||||
if _, ok := seen[pack]; ok {
|
||||
continue
|
||||
}
|
||||
seen[pack] = struct{}{}
|
||||
|
||||
languages, err := api.LangpackGetLanguages(ctx, pack)
|
||||
if err != nil {
|
||||
return fmt.Errorf("getLanguages %q: %w", pack, err)
|
||||
}
|
||||
sort.Slice(languages, func(i, j int) bool { return languages[i].LangCode < languages[j].LangCode })
|
||||
pm := packManifest{Name: pack}
|
||||
for _, language := range languages {
|
||||
if !language.Official {
|
||||
continue
|
||||
}
|
||||
code := strings.ToLower(strings.TrimSpace(language.LangCode))
|
||||
if !langCodeRE.MatchString(code) {
|
||||
return fmt.Errorf("invalid language code %q returned for %q", language.LangCode, pack)
|
||||
}
|
||||
diff, err := api.LangpackGetLangPack(ctx, &tg.LangpackGetLangPackRequest{
|
||||
LangPack: pack,
|
||||
LangCode: code,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("getLangPack %s/%s: %w", pack, code, err)
|
||||
}
|
||||
written, err := writePack(root, pack, diff)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
rel, err := filepath.Rel(root, written.Path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
pm.Languages = append(pm.Languages, languageManifest{
|
||||
LangCode: code,
|
||||
Name: language.Name,
|
||||
NativeName: language.NativeName,
|
||||
BaseLangCode: language.BaseLangCode,
|
||||
PluralCode: language.PluralCode,
|
||||
Official: language.Official,
|
||||
RTL: language.Rtl,
|
||||
Beta: language.Beta,
|
||||
StringsCount: language.StringsCount,
|
||||
TranslatedCount: language.TranslatedCount,
|
||||
TranslationsURL: language.TranslationsURL,
|
||||
Version: diff.Version,
|
||||
File: filepath.ToSlash(rel),
|
||||
SHA256: written.SHA256,
|
||||
})
|
||||
}
|
||||
fmt.Printf("pack %s complete: %d official languages\n", pack, len(pm.Languages))
|
||||
result.Packs = append(result.Packs, pm)
|
||||
}
|
||||
encoded, err := json.MarshalIndent(result, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return writeFileAtomic(filepath.Join(root, "official-language-packs.json"), append(encoded, '\n'))
|
||||
}
|
||||
|
||||
type writtenPack struct {
|
||||
Path string
|
||||
SHA256 string
|
||||
}
|
||||
|
||||
func writePack(root, pack string, diff *tg.LangPackDifference) (writtenPack, error) {
|
||||
pack = strings.ToLower(strings.TrimSpace(pack))
|
||||
if !packNameRE.MatchString(pack) {
|
||||
return writtenPack{}, fmt.Errorf("invalid lang pack %q", pack)
|
||||
}
|
||||
dir := filepath.Join(root, pack)
|
||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||
return err
|
||||
return writtenPack{}, err
|
||||
}
|
||||
var b strings.Builder
|
||||
count := 0
|
||||
|
|
@ -116,13 +243,51 @@ func writePack(root, pack string, diff *tg.LangPackDifference) error {
|
|||
}
|
||||
langCode := diff.LangCode
|
||||
if langCode == "" {
|
||||
langCode = "unknown"
|
||||
return writtenPack{}, fmt.Errorf("empty language code returned for %q", pack)
|
||||
}
|
||||
langCode = strings.ToLower(langCode)
|
||||
if !langCodeRE.MatchString(langCode) {
|
||||
return writtenPack{}, fmt.Errorf("invalid language code %q returned for %q", diff.LangCode, pack)
|
||||
}
|
||||
name := fmt.Sprintf("%s_%s_v%d.strings", pack, langCode, diff.Version)
|
||||
path := filepath.Join(dir, name)
|
||||
if err := os.WriteFile(path, []byte(b.String()), 0o644); err != nil {
|
||||
data := []byte(b.String())
|
||||
if err := writeFileAtomic(path, data); err != nil {
|
||||
return writtenPack{}, err
|
||||
}
|
||||
sum := sha256.Sum256(data)
|
||||
fmt.Printf("wrote %s (%d strings, version %d)\n", path, count, diff.Version)
|
||||
return writtenPack{Path: path, SHA256: hex.EncodeToString(sum[:])}, nil
|
||||
}
|
||||
|
||||
func writeFileAtomic(path string, data []byte) error {
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("wrote %s (%d strings, version %d)\n", path, count, diff.Version)
|
||||
tmp, err := os.CreateTemp(filepath.Dir(path), ".langpack-*.tmp")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
tmpName := tmp.Name()
|
||||
defer os.Remove(tmpName)
|
||||
if err := tmp.Chmod(0o644); err != nil {
|
||||
_ = tmp.Close()
|
||||
return err
|
||||
}
|
||||
if _, err := tmp.Write(data); err != nil {
|
||||
_ = tmp.Close()
|
||||
return err
|
||||
}
|
||||
if err := tmp.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.Rename(tmpName, path); err != nil {
|
||||
if removeErr := os.Remove(path); removeErr != nil && !os.IsNotExist(removeErr) {
|
||||
return fmt.Errorf("replace %q: %w (remove existing: %v)", path, err, removeErr)
|
||||
}
|
||||
if err := os.Rename(tmpName, path); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
|
|
|||
78
cmd/langpackfetch/main_test.go
Normal file
78
cmd/langpackfetch/main_test.go
Normal file
|
|
@ -0,0 +1,78 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/iamxvbaba/td/tg"
|
||||
)
|
||||
|
||||
func TestWritePack(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
diff := &tg.LangPackDifference{
|
||||
LangCode: "pt-br",
|
||||
Version: 42,
|
||||
Strings: []tg.LangPackStringClass{
|
||||
&tg.LangPackString{Key: "plain", Value: "value"},
|
||||
&tg.LangPackStringPluralized{Key: "items", OneValue: "one", OtherValue: "many"},
|
||||
&tg.LangPackStringDeleted{Key: "removed"},
|
||||
},
|
||||
}
|
||||
written, err := writePack(root, "tdesktop", diff)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
wantPath := filepath.Join(root, "tdesktop", "tdesktop_pt-br_v42.strings")
|
||||
if written.Path != wantPath {
|
||||
t.Fatalf("path = %q, want %q", written.Path, wantPath)
|
||||
}
|
||||
data, err := os.ReadFile(wantPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
text := string(data)
|
||||
for _, want := range []string{`"plain" = "value";`, `"items#one" = "one";`, `"items#other" = "many";`} {
|
||||
if !strings.Contains(text, want) {
|
||||
t.Errorf("output does not contain %q: %s", want, text)
|
||||
}
|
||||
}
|
||||
if strings.Contains(text, "removed") {
|
||||
t.Errorf("deleted key was emitted: %s", text)
|
||||
}
|
||||
sum := sha256.Sum256(data)
|
||||
if written.SHA256 != hex.EncodeToString(sum[:]) {
|
||||
t.Fatalf("sha256 = %q, want %x", written.SHA256, sum)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWritePackRejectsPathComponents(t *testing.T) {
|
||||
_, err := writePack(t.TempDir(), "../android", &tg.LangPackDifference{LangCode: "en", Version: 1})
|
||||
if err == nil {
|
||||
t.Fatal("expected invalid pack error")
|
||||
}
|
||||
_, err = writePack(t.TempDir(), "android", &tg.LangPackDifference{LangCode: "../en", Version: 1})
|
||||
if err == nil {
|
||||
t.Fatal("expected invalid language code error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestWriteFileAtomicReplacesExisting(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "manifest.json")
|
||||
if err := writeFileAtomic(path, []byte("first")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := writeFileAtomic(path, []byte("second")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if string(data) != "second" {
|
||||
t.Fatalf("content = %q, want second", data)
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue