fixes and ui improvements

cmd/telesrv-admin/readstore.go

@@ -2574,8 +2574,15 @@ FROM photos p
 // document/photo, but a purged file with no file_blobs rows left correctly
 // contributes 0 to both, never a stale non-zero "ghost" size).
 type StorageStatsRow struct {
-	PhysicalBytes     int64 `json:"PhysicalBytes,string"`
+	PhysicalBytes int64 `json:"PhysicalBytes,string"`
-	LogicalBytes      int64 `json:"LogicalBytes,string"`
+	LogicalBytes  int64 `json:"LogicalBytes,string"`
+	// UnattributedBytes, DocumentCount, PhotoCount and AccountCount all count
+	// only items that still own real file_blobs bytes -- documents/photos
+	// rows themselves are kept forever after a hard-retention purge (so a
+	// message can still render "here was a file"), so counting rows instead
+	// of live bytes would keep growing even as the actual content becomes
+	// physically empty, diverging further and further from PhysicalBytes
+	// above.
 	UnattributedBytes int64 `json:"UnattributedBytes,string"`
 	DocumentCount     int64 `json:"DocumentCount,string"`
 	PhotoCount        int64 `json:"PhotoCount,string"`
@@ -2597,14 +2604,31 @@ SELECT COALESCE(SUM(size), 0)::bigint FROM (`+perOwnerMediaSizeSQL+`) x`).Scan(&
 SELECT COALESCE(SUM(size), 0)::bigint FROM (`+perOwnerMediaSizeSQL+`) x WHERE owner_user_id = 0`).Scan(&stats.UnattributedBytes); err != nil {
 		return StorageStatsRow{}, fmt.Errorf("sum unattributed media bytes: %w", err)
 	}
-	if err := s.pool.QueryRow(ctx, `SELECT count(*)::bigint FROM documents`).Scan(&stats.DocumentCount); err != nil {
+	// Documents/Photos/AccountCount all count only items that still own real
+	// file_blobs bytes -- documents/photos rows are deliberately kept forever
+	// after a hard-retention purge (so a message can still render "here was
+	// a file"), so a plain count(*) would keep growing even as everything it
+	// counts becomes physically empty, wildly diverging from PhysicalBytes
+	// above and making the overview page look broken/confusing rather than
+	// informative.
+	if err := s.pool.QueryRow(ctx, `
+SELECT count(*)::bigint FROM documents d WHERE EXISTS (
+    SELECT 1 FROM file_blobs fb
+    WHERE fb.location_key = 'doc:' || d.id::text
+       OR fb.location_key LIKE 'doc:' || d.id::text || ':%'
+)`).Scan(&stats.DocumentCount); err != nil {
 		return StorageStatsRow{}, fmt.Errorf("count documents: %w", err)
 	}
-	if err := s.pool.QueryRow(ctx, `SELECT count(*)::bigint FROM photos`).Scan(&stats.PhotoCount); err != nil {
+	if err := s.pool.QueryRow(ctx, `
+SELECT count(*)::bigint FROM photos p WHERE EXISTS (
+    SELECT 1 FROM file_blobs fb
+    WHERE fb.location_key = 'photo:' || p.id::text
+       OR fb.location_key LIKE 'photo:' || p.id::text || ':%'
+)`).Scan(&stats.PhotoCount); err != nil {
 		return StorageStatsRow{}, fmt.Errorf("count photos: %w", err)
 	}
 	if err := s.pool.QueryRow(ctx, `
-SELECT count(DISTINCT owner_user_id)::bigint FROM (`+perOwnerMediaSizeSQL+`) x WHERE owner_user_id <> 0`).Scan(&stats.AccountCount); err != nil {
+SELECT count(DISTINCT owner_user_id)::bigint FROM (`+perOwnerMediaSizeSQL+`) x WHERE owner_user_id <> 0 AND size > 0`).Scan(&stats.AccountCount); err != nil {
 		return StorageStatsRow{}, fmt.Errorf("count storage accounts: %w", err)
 	}
 	stats.BackendKind = strings.ToLower(strings.TrimSpace(os.Getenv("TELESRV_BLOB_BACKEND")))
@@ -2680,9 +2704,14 @@ func (s *readStore) ListAccountStorageUsage(ctx context.Context, q string, sortB
 	args = append(args, offset, limit+1)
 	rows, err := s.pool.Query(ctx, `
 WITH totals AS (
+    -- size > 0 excludes documents/photos whose file_blobs bytes have already
+    -- been purged -- their row is kept forever (see perOwnerMediaSizeSQL's
+    -- doc comment) so counting every row here would keep FileCount growing
+    -- long after Bytes has settled at (or near) 0, same mismatch this query
+    -- used to have before it was joined through file_blobs at all.
     SELECT owner_user_id, SUM(size)::bigint AS bytes, COUNT(*)::bigint AS file_count
     FROM (`+perOwnerMediaSizeSQL+`) x
-    WHERE owner_user_id <> 0
+    WHERE owner_user_id <> 0 AND size > 0
     GROUP BY owner_user_id
 )
 SELECT t.owner_user_id, COALESCE(u.username, ''), COALESCE(u.first_name, ''), t.bytes, t.file_count

cmd/telesrv-admin/web/dist/index.html

@@ -23,8 +23,8 @@
       })();
     </script>
 
-    <script type="module" crossorigin src="/assets/index-CMLbsGwc.js"></script>
+    <script type="module" crossorigin src="/assets/index-NtDKRYU1.js"></script>
-    <link rel="stylesheet" crossorigin href="/assets/index-B7hI8ol7.css">
+    <link rel="stylesheet" crossorigin href="/assets/index-CU8XZPsy.css">
   </head>
   <body>
     <div id="root"></div>

cmd/telesrv-admin/web/src/pages/StoragePage.tsx

@@ -420,6 +420,9 @@ function CategoryRetentionModal({
           <p className="env-field-desc">
             {"Leave a category at 0 to inherit the shared Retention age. The mode switch still applies to all of them -- these only change how old that one category's media must be."}
           </p>
+          <p className="modal-warn-note">
+            {"Danger zone: Avatar and GIF are shared buckets, not per-user ones. Avatar also covers the built-in system bots' own profile photos (BotFather, Stickers, ChatBot, VerifyBot, the official system account), and GIF also covers the server's bundled GIF catalog (@gif) -- a short age here purges those right along with ordinary user media."}
+          </p>
           {CATEGORY_AGE_FIELDS.map((field) => (
             <DurationField
               key={field.key}
@@ -488,6 +491,9 @@ function ManualPurgeStorageModal({ onClose }: { onClose: () => void }) {
           <p className="env-field-desc">
             {"Deletes the file bytes of every document/photo matching the categories below, right now -- independent of the retention mode/age configured above. The message/profile-photo itself is never deleted, only its file; a purged item starts showing as unavailable. Leave \"Created before\" empty to purge everything in the selected categories, regardless of age."}
           </p>
+          <p className="modal-warn-note">
+            {"Danger zone: Avatar and GIF are shared buckets, not per-user ones. Selecting Avatar also purges the built-in system bots' own profile photos (BotFather, Stickers, ChatBot, VerifyBot, the official system account) -- these are bundled with the server and restore themselves automatically on the next restart. Selecting GIF also purges the server's bundled GIF catalog (@gif) entries, and those do NOT restore themselves: the catalog only re-imports a GIF whose source file is still sitting in TELESRV_GIF_SEED_DIR, so a purge is permanent for any catalog entry whose original file was since moved/deleted."}
+          </p>
           <div className="attr-block">
             <label className="checkline">
               <input type="checkbox" checked={allSelected} onChange={toggleAll} />

cmd/telesrv-admin/web/src/styles/04-modal-and-login.css

@@ -461,6 +461,22 @@
   border-radius: var(--radius);
 }
 
+/* Warns that an Avatar/GIF retention control also reaches built-in server
+   resources (system bot avatars, the bundled GIF catalog), not just
+   ordinary user media -- see StoragePage.tsx's CategoryRetentionModal and
+   ManualPurgeStorageModal. */
+.modal-warn-note {
+  padding: 9px 12px;
+  background: var(--warn-tint);
+  border: 1px solid var(--warn-border);
+  border-radius: var(--radius);
+  color: var(--warn);
+  font-size: 12px;
+  font-weight: 700;
+  line-height: 1.4;
+  margin: 0 0 10px;
+}
+
 .secret-reveal-label {
   display: flex;
   align-items: center;

internal/app/files/botfather_avatar_seed.go

@@ -12,36 +12,36 @@ import (
 //go:embed seedassets/botfather_avatar.jpg
 var botFatherAvatarJPG []byte
 
-// SeedBotFatherAvatar idempotently seeds the built-in BotFather account's
+// SeedBotFatherAvatar seeds the built-in BotFather account's profile photo
-// profile photo from the bundled avatar, mirroring SeedOfficialSystemAvatar:
+// from the bundled avatar, mirroring SeedOfficialSystemAvatar: writes it
-// writes it under the fixed domain.BotFatherUserPhotoID so the photo/blob
+// under the fixed domain.BotFatherUserPhotoID so the photo/blob layer and
-// layer and the pure domain.BotFatherUser() struct literal stay in sync
+// the pure domain.BotFatherUser() struct literal stay in sync across
-// across restarts, and registers it as the account's *current* profile photo
+// restarts, and registers it as the account's *current* profile photo so
-// so users.getFullUser resolves it too. Returns true if it actually wrote a
+// users.getFullUser resolves it too. Deliberately re-upserts the photo row
-// new photo.
+// AND its file_blobs bytes on every boot (not just when the photos row is
+// missing) -- storage retention/manual-purge only ever deletes file_blobs
+// bytes, never the photos row itself, so a "skip if the row exists" check
+// used to leave this bot's avatar permanently blank after any purge that
+// swept the Avatar category, even though the row it checked for was still
+// right there. Returns true if it actually (re)wrote the photo.
 func (s *Service) SeedBotFatherAvatar(ctx context.Context) (bool, error) {
 	photoID := domain.BotFatherUserPhotoID
-	wrote := false
+	sizes, err := s.putPhotoStaticSizes(ctx, photoID, botFatherAvatarJPG, photoSizeSpecsForAvatar(botFatherAvatarJPG))
-	if _, found, err := s.media.GetPhoto(ctx, photoID); err != nil {
+	if err != nil {
 		return false, err
-	} else if !found {
-		sizes, err := s.putPhotoStaticSizes(ctx, photoID, botFatherAvatarJPG, photoSizeSpecsForAvatar(botFatherAvatarJPG))
-		if err != nil {
-			return false, err
-		}
-		photo := domain.Photo{
-			ID:            photoID,
-			AccessHash:    domain.BotFatherUserPhotoAccessHash,
-			FileReference: randomFileReference(),
-			Date:          int(time.Now().Unix()),
-			DCID:          s.dc,
-			Sizes:         sizes,
-		}
-		if err := s.media.PutPhoto(ctx, photo); err != nil {
-			return false, err
-		}
-		wrote = true
 	}
+	newPhoto := domain.Photo{
+		ID:            photoID,
+		AccessHash:    domain.BotFatherUserPhotoAccessHash,
+		FileReference: randomFileReference(),
+		Date:          int(time.Now().Unix()),
+		DCID:          s.dc,
+		Sizes:         sizes,
+	}
+	if err := s.media.PutPhoto(ctx, newPhoto); err != nil {
+		return false, err
+	}
+	wrote := true
 	photo, ok, err := s.SetCurrentProfilePhoto(ctx, domain.PeerTypeUser, domain.BotFatherUserID, photoID, int(time.Now().Unix()))
 	if err != nil {
 		return false, err

internal/app/files/chatbot_avatar_seed.go

@@ -12,36 +12,35 @@ import (
 //go:embed seedassets/chatbot_avatar.png
 var chatBotAvatarPNG []byte
 
-// SeedChatBotAvatar idempotently seeds the built-in @ChatBot account's
+// SeedChatBotAvatar seeds the built-in @ChatBot account's profile photo from
-// profile photo from the bundled avatar, mirroring SeedBotFatherAvatar:
+// the bundled avatar, mirroring SeedOfficialSystemAvatar: writes it under the
-// writes it under the fixed domain.ChatBotUserPhotoID so the photo/blob
+// fixed domain.ChatBotUserPhotoID so the photo/blob layer and the pure
-// layer and the pure domain.ChatBotUser() struct literal stay in sync
+// domain.ChatBotUser() struct literal stay in sync across restarts, and
-// across restarts, and registers it as the account's *current* profile photo
+// registers it as the account's *current* profile photo so
-// so users.getFullUser resolves it too. Returns true if it actually wrote a
+// users.getFullUser resolves it too. Deliberately re-upserts the photo row
-// new photo.
+// AND its file_blobs bytes on every boot (not just when the photos row is
+// missing) -- see SeedBotFatherAvatar's doc comment for why a "skip if the
+// row exists" check is wrong here (storage retention/manual-purge only ever
+// deletes file_blobs bytes, never the photos row). Returns true if it
+// actually (re)wrote the photo.
 func (s *Service) SeedChatBotAvatar(ctx context.Context) (bool, error) {
 	photoID := domain.ChatBotUserPhotoID
-	wrote := false
+	sizes, err := s.putPhotoStaticSizes(ctx, photoID, chatBotAvatarPNG, photoSizeSpecsForAvatar(chatBotAvatarPNG))
-	if _, found, err := s.media.GetPhoto(ctx, photoID); err != nil {
+	if err != nil {
 		return false, err
-	} else if !found {
-		sizes, err := s.putPhotoStaticSizes(ctx, photoID, chatBotAvatarPNG, photoSizeSpecsForAvatar(chatBotAvatarPNG))
-		if err != nil {
-			return false, err
-		}
-		photo := domain.Photo{
-			ID:            photoID,
-			AccessHash:    domain.ChatBotUserPhotoAccessHash,
-			FileReference: randomFileReference(),
-			Date:          int(time.Now().Unix()),
-			DCID:          s.dc,
-			Sizes:         sizes,
-		}
-		if err := s.media.PutPhoto(ctx, photo); err != nil {
-			return false, err
-		}
-		wrote = true
 	}
+	newPhoto := domain.Photo{
+		ID:            photoID,
+		AccessHash:    domain.ChatBotUserPhotoAccessHash,
+		FileReference: randomFileReference(),
+		Date:          int(time.Now().Unix()),
+		DCID:          s.dc,
+		Sizes:         sizes,
+	}
+	if err := s.media.PutPhoto(ctx, newPhoto); err != nil {
+		return false, err
+	}
+	wrote := true
 	photo, ok, err := s.SetCurrentProfilePhoto(ctx, domain.PeerTypeUser, domain.ChatBotUserID, photoID, int(time.Now().Unix()))
 	if err != nil {
 		return false, err

internal/app/files/stickers_bot_avatar_seed.go

@@ -12,36 +12,35 @@ import (
 //go:embed seedassets/stickers_avatar.png
 var stickersBotAvatarPNG []byte
 
-// SeedStickersBotAvatar idempotently seeds the built-in @Stickers account's
+// SeedStickersBotAvatar seeds the built-in @Stickers account's profile photo
-// profile photo from the bundled avatar, mirroring SeedBotFatherAvatar:
+// from the bundled avatar, mirroring SeedOfficialSystemAvatar: writes it
-// writes it under the fixed domain.StickersBotUserPhotoID so the photo/blob
+// under the fixed domain.StickersBotUserPhotoID so the photo/blob layer and
-// layer and the pure domain.StickersBotUser() struct literal stay in sync
+// the pure domain.StickersBotUser() struct literal stay in sync across
-// across restarts, and registers it as the account's *current* profile photo
+// restarts, and registers it as the account's *current* profile photo so
-// so users.getFullUser resolves it too. Returns true if it actually wrote a
+// users.getFullUser resolves it too. Deliberately re-upserts the photo row
-// new photo.
+// AND its file_blobs bytes on every boot (not just when the photos row is
+// missing) -- see SeedBotFatherAvatar's doc comment for why a "skip if the
+// row exists" check is wrong here (storage retention/manual-purge only ever
+// deletes file_blobs bytes, never the photos row). Returns true if it
+// actually (re)wrote the photo.
 func (s *Service) SeedStickersBotAvatar(ctx context.Context) (bool, error) {
 	photoID := domain.StickersBotUserPhotoID
-	wrote := false
+	sizes, err := s.putPhotoStaticSizes(ctx, photoID, stickersBotAvatarPNG, photoSizeSpecsForAvatar(stickersBotAvatarPNG))
-	if _, found, err := s.media.GetPhoto(ctx, photoID); err != nil {
+	if err != nil {
 		return false, err
-	} else if !found {
-		sizes, err := s.putPhotoStaticSizes(ctx, photoID, stickersBotAvatarPNG, photoSizeSpecsForAvatar(stickersBotAvatarPNG))
-		if err != nil {
-			return false, err
-		}
-		photo := domain.Photo{
-			ID:            photoID,
-			AccessHash:    domain.StickersBotUserPhotoAccessHash,
-			FileReference: randomFileReference(),
-			Date:          int(time.Now().Unix()),
-			DCID:          s.dc,
-			Sizes:         sizes,
-		}
-		if err := s.media.PutPhoto(ctx, photo); err != nil {
-			return false, err
-		}
-		wrote = true
 	}
+	newPhoto := domain.Photo{
+		ID:            photoID,
+		AccessHash:    domain.StickersBotUserPhotoAccessHash,
+		FileReference: randomFileReference(),
+		Date:          int(time.Now().Unix()),
+		DCID:          s.dc,
+		Sizes:         sizes,
+	}
+	if err := s.media.PutPhoto(ctx, newPhoto); err != nil {
+		return false, err
+	}
+	wrote := true
 	photo, ok, err := s.SetCurrentProfilePhoto(ctx, domain.PeerTypeUser, domain.StickersBotUserID, photoID, int(time.Now().Unix()))
 	if err != nil {
 		return false, err

internal/app/files/verifybot_avatar_seed.go

@@ -12,36 +12,35 @@ import (
 //go:embed seedassets/verifybot_avatar.png
 var verifyBotAvatarPNG []byte
 
-// SeedVerifyBotAvatar idempotently seeds the built-in @verifybot account's
+// SeedVerifyBotAvatar seeds the built-in @verifybot account's profile photo
-// profile photo from the bundled avatar, mirroring SeedChatBotAvatar: writes
+// from the bundled avatar, mirroring SeedOfficialSystemAvatar: writes it
-// it under the fixed domain.VerifyBotUserPhotoID so the photo/blob layer and
+// under the fixed domain.VerifyBotUserPhotoID so the photo/blob layer and
 // the pure domain.VerifyBotUser() struct literal stay in sync across
 // restarts, and registers it as the account's *current* profile photo so
-// users.getFullUser resolves it too. Returns true if it actually wrote a new
+// users.getFullUser resolves it too. Deliberately re-upserts the photo row
-// photo.
+// AND its file_blobs bytes on every boot (not just when the photos row is
+// missing) -- see SeedBotFatherAvatar's doc comment for why a "skip if the
+// row exists" check is wrong here (storage retention/manual-purge only ever
+// deletes file_blobs bytes, never the photos row). Returns true if it
+// actually (re)wrote the photo.
 func (s *Service) SeedVerifyBotAvatar(ctx context.Context) (bool, error) {
 	photoID := domain.VerifyBotUserPhotoID
-	wrote := false
+	sizes, err := s.putPhotoStaticSizes(ctx, photoID, verifyBotAvatarPNG, photoSizeSpecsForAvatar(verifyBotAvatarPNG))
-	if _, found, err := s.media.GetPhoto(ctx, photoID); err != nil {
+	if err != nil {
 		return false, err
-	} else if !found {
-		sizes, err := s.putPhotoStaticSizes(ctx, photoID, verifyBotAvatarPNG, photoSizeSpecsForAvatar(verifyBotAvatarPNG))
-		if err != nil {
-			return false, err
-		}
-		photo := domain.Photo{
-			ID:            photoID,
-			AccessHash:    domain.VerifyBotUserPhotoAccessHash,
-			FileReference: randomFileReference(),
-			Date:          int(time.Now().Unix()),
-			DCID:          s.dc,
-			Sizes:         sizes,
-		}
-		if err := s.media.PutPhoto(ctx, photo); err != nil {
-			return false, err
-		}
-		wrote = true
 	}
+	newPhoto := domain.Photo{
+		ID:            photoID,
+		AccessHash:    domain.VerifyBotUserPhotoAccessHash,
+		FileReference: randomFileReference(),
+		Date:          int(time.Now().Unix()),
+		DCID:          s.dc,
+		Sizes:         sizes,
+	}
+	if err := s.media.PutPhoto(ctx, newPhoto); err != nil {
+		return false, err
+	}
+	wrote := true
 	photo, ok, err := s.SetCurrentProfilePhoto(ctx, domain.PeerTypeUser, domain.VerifyBotUserID, photoID, int(time.Now().Unix()))
 	if err != nil {
 		return false, err

internal/store/postgres/media_refs.go

@@ -349,6 +349,21 @@ func (s *MediaStore) DeleteFileBlobsForDocument(ctx context.Context, id int64) (
 				return fmt.Errorf("delete file blob row: %w", err)
 			}
 		}
+		// gif_catalog and user_sticker_collections entries are pure picker
+		// metadata -- unlike a chat message (which keeps rendering "media
+		// unavailable" off the documents row after its bytes are gone), a
+		// picker entry has no placeholder concept: once the underlying gif's
+		// bytes are purged it's just a dead thumbnail nobody can open, so it
+		// gets removed here too, in the same transaction as the actual blob
+		// purge, instead of drifting out of sync until some separate sweep
+		// happens to notice. No-op (0 rows) for the overwhelming majority of
+		// documents, which are never in either table.
+		if _, err := tx.Exec(ctx, `DELETE FROM gif_catalog WHERE document_id = $1`, id); err != nil {
+			return fmt.Errorf("delete gif catalog entry for purged document: %w", err)
+		}
+		if _, err := tx.Exec(ctx, `DELETE FROM user_sticker_collections WHERE document_id = $1`, id); err != nil {
+			return fmt.Errorf("delete sticker collection entries for purged document: %w", err)
+		}
 		return nil
 	})
 	if err != nil {