updates for server files size limits

cmd/telesrv-admin/readstore.go

@@ -2573,36 +2573,73 @@ FROM photos p
 // than physical when the same blob is attributed to more than one
 // document/photo, but a purged file with no file_blobs rows left correctly
 // contributes 0 to both, never a stale non-zero "ghost" size).
+//
+// Every field here deliberately EXCLUDES system/bundled content (owner_user_id
+// = 0 on the documents/photos row -- the built-in sticker packs, emoji sets,
+// default wallpapers, GIF catalog, and system-bot avatars this server seeds
+// at every boot; see internal/app/files's Seed* functions). None of that is
+// something an operator manages through storage retention/purge -- it isn't
+// one of the Photo/Video/GIF/Music/Voice/File/Avatar categories those
+// controls target, it's permanent server furniture -- so counting it here
+// alongside real user uploads made every number on this page mean "user
+// storage plus an unpredictable pile of bundled assets" instead of just
+// answering "how much space are my users actually using". SystemBytes below
+// is the one exception: it reports that excluded total separately, purely
+// for an operator's own curiosity/disk-accounting, never folded into the
+// other totals.
 type StorageStatsRow struct {
 	PhysicalBytes int64 `json:"PhysicalBytes,string"`
 	LogicalBytes  int64 `json:"LogicalBytes,string"`
-	// UnattributedBytes, DocumentCount, PhotoCount and AccountCount all count
-	// only items that still own real file_blobs bytes -- documents/photos
-	// rows themselves are kept forever after a hard-retention purge (so a
-	// message can still render "here was a file"), so counting rows instead
-	// of live bytes would keep growing even as the actual content becomes
-	// physically empty, diverging further and further from PhysicalBytes
-	// above.
-	UnattributedBytes int64 `json:"UnattributedBytes,string"`
-	DocumentCount     int64 `json:"DocumentCount,string"`
-	PhotoCount        int64 `json:"PhotoCount,string"`
-	AccountCount      int64 `json:"AccountCount,string"`
-	BackendKind       string
+	// SystemBytes is the physical size of excluded system/bundled content
+	// (owner_user_id = 0) -- shown separately so the gap between this and
+	// what `docker exec ... mc du` or the MinIO console reports isn't a
+	// mystery, but never added into PhysicalBytes/LogicalBytes/DocumentCount/
+	// PhotoCount/AccountCount above.
+	SystemBytes int64 `json:"SystemBytes,string"`
+	// DocumentCount, PhotoCount and AccountCount all count only items that
+	// still own real file_blobs bytes -- documents/photos rows are
+	// deliberately kept forever after a hard-retention purge (so a message
+	// can still render "here was a file"), so counting rows instead of live
+	// bytes would keep growing even as the actual content becomes physically
+	// empty, diverging further and further from PhysicalBytes above.
+	DocumentCount int64 `json:"DocumentCount,string"`
+	PhotoCount    int64 `json:"PhotoCount,string"`
+	AccountCount  int64 `json:"AccountCount,string"`
+	BackendKind   string
 }
 
 // StorageStats returns the admin panel's storage overview.
 func (s *readStore) StorageStats(ctx context.Context) (StorageStatsRow, error) {
 	var stats StorageStatsRow
-	if err := s.pool.QueryRow(ctx, `SELECT COALESCE(SUM(size), 0)::bigint FROM file_blobs`).Scan(&stats.PhysicalBytes); err != nil {
+	// Physical usage dedups by (backend, object_key) like the unfiltered
+	// version used to, but only counts an object if at least one real user's
+	// (owner_user_id <> 0) document/photo still references it -- content
+	// shared between a system asset and a real upload (content-addressed
+	// storage, so only possible via a byte-for-byte coincidental duplicate)
+	// still counts, since a real user genuinely has that data stored.
+	if err := s.pool.QueryRow(ctx, `
+SELECT COALESCE(SUM(size), 0)::bigint FROM (
+    SELECT DISTINCT ON (fb.backend, fb.object_key) fb.backend, fb.object_key, fb.size
+    FROM file_blobs fb
+    WHERE EXISTS (
+        SELECT 1 FROM documents d
+        WHERE d.owner_user_id <> 0
+          AND (fb.location_key = 'doc:' || d.id::text OR fb.location_key LIKE 'doc:' || d.id::text || ':%')
+    ) OR EXISTS (
+        SELECT 1 FROM photos p
+        WHERE p.owner_user_id <> 0
+          AND (fb.location_key = 'photo:' || p.id::text OR fb.location_key LIKE 'photo:' || p.id::text || ':%')
+    )
+) x`).Scan(&stats.PhysicalBytes); err != nil {
 		return StorageStatsRow{}, fmt.Errorf("sum physical blob bytes: %w", err)
 	}
 	if err := s.pool.QueryRow(ctx, `
-SELECT COALESCE(SUM(size), 0)::bigint FROM (`+perOwnerMediaSizeSQL+`) x`).Scan(&stats.LogicalBytes); err != nil {
+SELECT COALESCE(SUM(size), 0)::bigint FROM (`+perOwnerMediaSizeSQL+`) x WHERE owner_user_id <> 0`).Scan(&stats.LogicalBytes); err != nil {
 		return StorageStatsRow{}, fmt.Errorf("sum logical media bytes: %w", err)
 	}
 	if err := s.pool.QueryRow(ctx, `
-SELECT COALESCE(SUM(size), 0)::bigint FROM (`+perOwnerMediaSizeSQL+`) x WHERE owner_user_id = 0`).Scan(&stats.UnattributedBytes); err != nil {
-		return StorageStatsRow{}, fmt.Errorf("sum unattributed media bytes: %w", err)
+SELECT COALESCE(SUM(size), 0)::bigint FROM (`+perOwnerMediaSizeSQL+`) x WHERE owner_user_id = 0`).Scan(&stats.SystemBytes); err != nil {
+		return StorageStatsRow{}, fmt.Errorf("sum system media bytes: %w", err)
 	}
 	// Documents/Photos/AccountCount all count only items that still own real
 	// file_blobs bytes -- documents/photos rows are deliberately kept forever
@@ -2610,9 +2647,10 @@ SELECT COALESCE(SUM(size), 0)::bigint FROM (`+perOwnerMediaSizeSQL+`) x WHERE ow
 	// a file"), so a plain count(*) would keep growing even as everything it
 	// counts becomes physically empty, wildly diverging from PhysicalBytes
 	// above and making the overview page look broken/confusing rather than
-	// informative.
+	// informative. owner_user_id <> 0 excludes system/bundled content -- see
+	// StorageStatsRow's doc comment.
 	if err := s.pool.QueryRow(ctx, `
-SELECT count(*)::bigint FROM documents d WHERE EXISTS (
+SELECT count(*)::bigint FROM documents d WHERE d.owner_user_id <> 0 AND EXISTS (
     SELECT 1 FROM file_blobs fb
     WHERE fb.location_key = 'doc:' || d.id::text
        OR fb.location_key LIKE 'doc:' || d.id::text || ':%'
@@ -2620,7 +2658,7 @@ SELECT count(*)::bigint FROM documents d WHERE EXISTS (
 		return StorageStatsRow{}, fmt.Errorf("count documents: %w", err)
 	}
 	if err := s.pool.QueryRow(ctx, `
-SELECT count(*)::bigint FROM photos p WHERE EXISTS (
+SELECT count(*)::bigint FROM photos p WHERE p.owner_user_id <> 0 AND EXISTS (
     SELECT 1 FROM file_blobs fb
     WHERE fb.location_key = 'photo:' || p.id::text
        OR fb.location_key LIKE 'photo:' || p.id::text || ':%'

cmd/telesrv-admin/web/dist/index.html

@@ -23,7 +23,7 @@
       })();
     </script>
 
-    <script type="module" crossorigin src="/assets/index-DCHv80n3.js"></script>
+    <script type="module" crossorigin src="/assets/index-DwjnA8dD.js"></script>
     <link rel="stylesheet" crossorigin href="/assets/index-B7j2PW0q.css">
   </head>
   <body>

cmd/telesrv-admin/web/src/pages/StoragePage.tsx

@@ -119,9 +119,8 @@ function StorageOverviewTab({ navigate }: { navigate: Navigate }) {
         <Metric label={"Photos"} value={stats ? formatQuantity(stats.PhotoCount) : "-"} />
         <Metric label={"Accounts with media"} value={stats ? formatQuantity(stats.AccountCount) : "-"} />
         <Metric
-          label={"Unattributed"}
-          value={stats ? formatBytes(stats.UnattributedBytes) : "-"}
-          tone={stats && Number(stats.UnattributedBytes) > 0 ? "warn" : "neutral"}
+          label={"System/bundled content"}
+          value={stats ? formatBytes(stats.SystemBytes) : "-"}
         />
       </div>
 

cmd/telesrv-admin/web/src/types.ts

@@ -727,7 +727,7 @@ export type SharedDeviceGroupListResponse = {
 export type StorageStatsResponse = {
   PhysicalBytes: string;
   LogicalBytes: string;
-  UnattributedBytes: string;
+  SystemBytes: string;
   DocumentCount: string;
   PhotoCount: string;
   AccountCount: string;

cmd/telesrv/main.go

@@ -1611,6 +1611,7 @@ func run(logger *zap.Logger) error {
 			help.WithMapboxToken(cfg.MapboxToken),
 			help.WithEmailSignupEnable(cfg.EmailSignupEnable),
 			help.WithEmailSignupPhonePrefixes(cfg.EmailSignupPhonePrefixes),
+			help.WithMaxUploadFileBytes(cfg.StorageMaxUploadFileBytes),
 			help.WithAccountFreezeProvider(adminService),
 		),
 		AppUpdates:                 appUpdateResolver,

internal/app/help/service.go

@@ -82,6 +82,7 @@ type Service struct {
 	mapboxToken              string
 	emailSignupEnable        bool
 	emailSignupPhonePrefixes []string
+	maxUploadFileBytes       int64
 
 	appConfigOnce  sync.Once
 	appConfigCache domain.AppConfig
@@ -129,6 +130,42 @@ func WithEmailSignupPhonePrefixes(prefixes []string) Option {
 	}
 }
 
+// maxUploadFilePartBytes mirrors internal/app/files.MaxUploadPartBytes (not
+// imported to avoid pulling the whole files package into help just for one
+// constant): the wire size of one upload chunk, used to convert
+// TELESRV_STORAGE_MAX_UPLOAD_FILE_BYTES into the part-count unit
+// upload_max_fileparts_default/_premium are declared in.
+const maxUploadFilePartBytes = 524288
+
+// WithMaxUploadFileBytes overrides the stock upload_max_fileparts_default/
+// _premium app config keys (already in tdesktopDefaultAppConfigBase, values
+// 4000/8000 -- 2000/4000 MB at 512KB/part) with TELESRV_STORAGE_MAX_UPLOAD_FILE_BYTES's
+// part-count equivalent, when configured smaller than the protocol default.
+// tdesktop's Data::PremiumLimits::uploadMaxDefault/Premium already read
+// these two keys (data_premium_limits.cpp) and feed both the pre-upload size
+// check and its warning dialog (localimageloader.cpp's FileSizeLimit/
+// FileSizePremiumLimit, storage_media_prepare.cpp) -- so overriding the
+// number here is enough to make an adapted client warn about (and reject)
+// files above this self-hosted server's real ceiling, instead of only its
+// own hardcoded ~2/4GB assumption. 0 (unlimited, the default) leaves the
+// stock 4000/8000 values untouched.
+func WithMaxUploadFileBytes(bytes int64) Option {
+	return func(s *Service) {
+		s.maxUploadFileBytes = bytes
+	}
+}
+
+// maxUploadFileParts converts a configured byte ceiling to the number of
+// 512KB parts it takes to hold it, rounding up (a partial final part still
+// needs a whole slot) and never below 1.
+func maxUploadFileParts(bytes int64) int64 {
+	parts := (bytes + maxUploadFilePartBytes - 1) / maxUploadFilePartBytes
+	if parts < 1 {
+		return 1
+	}
+	return parts
+}
+
 // NewService 创建 help 服务。
 func NewService(appConfigs store.AppConfigStore, countries store.CountryStore, opts ...Option) *Service {
 	s := &Service{
@@ -143,12 +180,12 @@ func NewService(appConfigs store.AppConfigStore, countries store.CountryStore, o
 	return s
 }
 
-func defaultAppConfig(mapboxToken string, emailSignupEnable bool, emailSignupPhonePrefixes []string) domain.AppConfig {
-	jsonBytes := defaultAppConfigJSON(mapboxToken, emailSignupEnable, emailSignupPhonePrefixes)
-	return domain.AppConfig{Client: tdesktopClient, Hash: defaultAppConfigHashFor(mapboxToken, emailSignupEnable, emailSignupPhonePrefixes), JSON: jsonBytes}
+func defaultAppConfig(mapboxToken string, emailSignupEnable bool, emailSignupPhonePrefixes []string, maxUploadFileBytes int64) domain.AppConfig {
+	jsonBytes := defaultAppConfigJSON(mapboxToken, emailSignupEnable, emailSignupPhonePrefixes, maxUploadFileBytes)
+	return domain.AppConfig{Client: tdesktopClient, Hash: defaultAppConfigHashFor(mapboxToken, emailSignupEnable, emailSignupPhonePrefixes, maxUploadFileBytes), JSON: jsonBytes}
 }
 
-func defaultAppConfigJSON(mapboxToken string, emailSignupEnable bool, emailSignupPhonePrefixes []string) []byte {
+func defaultAppConfigJSON(mapboxToken string, emailSignupEnable bool, emailSignupPhonePrefixes []string, maxUploadFileBytes int64) []byte {
 	androidInvoiceBilling := `,"premium_playmarket_direct_currency_list":` + compatandroid.DirectInvoiceCurrenciesJSON()
 	base := tdesktopDefaultAppConfigBase + tdesktopNoForwardsAppConfig + androidInvoiceBilling
 	if emailSignupEnable {
@@ -159,6 +196,11 @@ func defaultAppConfigJSON(mapboxToken string, emailSignupEnable bool, emailSignu
 			}
 		}
 	}
+	if maxUploadFileBytes > 0 {
+		parts := strconv.FormatInt(maxUploadFileParts(maxUploadFileBytes), 10)
+		base = strings.Replace(base, `"upload_max_fileparts_default":4000`, `"upload_max_fileparts_default":`+parts, 1)
+		base = strings.Replace(base, `"upload_max_fileparts_premium":8000`, `"upload_max_fileparts_premium":`+parts, 1)
+	}
 	if mapboxToken == "" {
 		return []byte(base + `}`)
 	}
@@ -170,7 +212,7 @@ func defaultAppConfigJSON(mapboxToken string, emailSignupEnable bool, emailSignu
 	return []byte(base + `,"tdesktop_config_map":{"maps":` + tokenJSON + `,"geo":` + tokenJSON + `,"bmaps":` + tokenJSON + `,"bgeo":` + tokenJSON + `}}`)
 }
 
-func defaultAppConfigHashFor(mapboxToken string, emailSignupEnable bool, emailSignupPhonePrefixes []string) int {
+func defaultAppConfigHashFor(mapboxToken string, emailSignupEnable bool, emailSignupPhonePrefixes []string, maxUploadFileBytes int64) int {
 	h := defaultAppConfigHash
 	if emailSignupEnable {
 		h += 1000003 // large odd offset so toggling the flag always changes the hash
@@ -178,6 +220,9 @@ func defaultAppConfigHashFor(mapboxToken string, emailSignupEnable bool, emailSi
 			h += 1 + int(crc32.ChecksumIEEE([]byte(strings.Join(emailSignupPhonePrefixes, ",")))&0x3fffffff)
 		}
 	}
+	if maxUploadFileBytes > 0 {
+		h += 1 + int(crc32.ChecksumIEEE([]byte(strconv.FormatInt(maxUploadFileBytes, 10)))&0x3fffffff)
+	}
 	if mapboxToken == "" {
 		return h
 	}
@@ -243,9 +288,9 @@ func (s *Service) accountAppConfig(ctx context.Context, userID int64, base domai
 
 func (s *Service) loadAppConfig(ctx context.Context) domain.AppConfig {
 	if s == nil {
-		return defaultAppConfig("", false, nil)
+		return defaultAppConfig("", false, nil, 0)
 	}
-	defaultCfg := defaultAppConfig(s.mapboxToken, s.emailSignupEnable, s.emailSignupPhonePrefixes)
+	defaultCfg := defaultAppConfig(s.mapboxToken, s.emailSignupEnable, s.emailSignupPhonePrefixes, s.maxUploadFileBytes)
 	s.appConfigOnce.Do(func() {
 		if s.appConfigs == nil {
 			s.appConfigCache = defaultCfg

internal/app/help/service_premium_test.go

@@ -119,6 +119,66 @@ func containsJSONCurrency(values []any, want string) bool {
 	return false
 }
 
+func TestAppConfigKeepsStockUploadPartsLimitsByDefault(t *testing.T) {
+	cfg, notModified, err := (*Service)(nil).GetAppConfig(context.Background(), 0, 0)
+	if err != nil || notModified {
+		t.Fatalf("GetAppConfig = notModified %v err %v", notModified, err)
+	}
+	var decoded map[string]any
+	if err := json.Unmarshal(cfg.JSON, &decoded); err != nil {
+		t.Fatalf("app config json invalid: %v", err)
+	}
+	if got := decoded["upload_max_fileparts_default"]; got != float64(4000) {
+		t.Fatalf("upload_max_fileparts_default = %v, want 4000 (unconfigured limit)", got)
+	}
+	if got := decoded["upload_max_fileparts_premium"]; got != float64(8000) {
+		t.Fatalf("upload_max_fileparts_premium = %v, want 8000 (unconfigured limit)", got)
+	}
+}
+
+func TestAppConfigUsesConfiguredMaxUploadFileBytesAndHash(t *testing.T) {
+	// 500MiB / 512KiB-per-part = 1000 parts exactly.
+	svc := NewService(nil, nil, WithMaxUploadFileBytes(500*1024*1024))
+	cfg, notModified, err := svc.GetAppConfig(context.Background(), 0, 0)
+	if err != nil || notModified {
+		t.Fatalf("GetAppConfig = notModified %v err %v", notModified, err)
+	}
+	if cfg.Hash == defaultAppConfigHash {
+		t.Fatalf("hash = %d, want limit-specific hash", cfg.Hash)
+	}
+	if _, notModified, err := svc.GetAppConfig(context.Background(), 0, cfg.Hash); err != nil || !notModified {
+		t.Fatalf("GetAppConfig(hash) = notModified %v err %v, want notModified", notModified, err)
+	}
+	var decoded map[string]any
+	if err := json.Unmarshal(cfg.JSON, &decoded); err != nil {
+		t.Fatalf("app config json invalid: %v", err)
+	}
+	if got := decoded["upload_max_fileparts_default"]; got != float64(1000) {
+		t.Fatalf("upload_max_fileparts_default = %v, want 1000", got)
+	}
+	if got := decoded["upload_max_fileparts_premium"]; got != float64(1000) {
+		t.Fatalf("upload_max_fileparts_premium = %v, want 1000", got)
+	}
+}
+
+func TestMaxUploadFilePartsRoundsUpAndFloorsAtOne(t *testing.T) {
+	cases := []struct {
+		bytes int64
+		want  int64
+	}{
+		{0, 1},
+		{1, 1},
+		{maxUploadFilePartBytes, 1},
+		{maxUploadFilePartBytes + 1, 2},
+		{500 * 1024 * 1024, 1000},
+	}
+	for _, c := range cases {
+		if got := maxUploadFileParts(c.bytes); got != c.want {
+			t.Errorf("maxUploadFileParts(%d) = %d, want %d", c.bytes, got, c.want)
+		}
+	}
+}
+
 func TestAppConfigOmitsMapboxTokenByDefault(t *testing.T) {
 	cfg, notModified, err := (*Service)(nil).GetAppConfig(context.Background(), 0, 0)
 	if err != nil || notModified {