fixes for storage managament

cmd/telesrv-admin/readstore.go

@@ -2539,28 +2539,40 @@ const (
 )
 
 // perOwnerMediaSizeSQL is shared between StorageStats and
-// ListAccountStorageUsage: each document row's size is its stored column;
-// each photo row has no single size column (JSONB sizes holds one entry per
-// rendition), so its "attributed size" is the largest rendition -- the
-// dominant cost, thumbnails are comparatively tiny. This is an
-// approximation (not the exact sum of every rendition's blob bytes, which
-// would require joining file_blobs by location_key prefix) chosen for admin
-// visibility, not billing precision.
+// ListAccountStorageUsage: attributes each document/photo's REAL remaining
+// bytes (the sum of every file_blobs row it still owns -- main body plus
+// thumbnail/rendition variants) to its owner, one row per document/photo so
+// an outer COUNT(*)/SUM(size) aggregate still gets an accurate file count
+// alongside the byte total. This used to read documents.size / the largest
+// photo rendition size directly -- a static value on the metadata row that
+// survives a hard-retention purge unchanged, so it kept counting bytes for
+// files whose blobs were long gone. Joining through file_blobs instead means
+// a purged item correctly contributes 0: there is nothing left to attribute.
 const perOwnerMediaSizeSQL = `
-SELECT owner_user_id, size FROM documents
+SELECT d.owner_user_id, COALESCE((
+    SELECT SUM(fb.size) FROM file_blobs fb
+    WHERE fb.location_key = 'doc:' || d.id::text
+       OR fb.location_key LIKE 'doc:' || d.id::text || ':%'
+), 0) AS size
+FROM documents d
 UNION ALL
 SELECT p.owner_user_id, COALESCE((
-    SELECT MAX((elem->>'size')::bigint) FROM jsonb_array_elements(p.sizes) elem
+    SELECT SUM(fb.size) FROM file_blobs fb
+    WHERE fb.location_key = 'photo:' || p.id::text
+       OR fb.location_key LIKE 'photo:' || p.id::text || ':%'
 ), 0) AS size
 FROM photos p
 `
 
 // StorageStatsRow is the admin panel's storage overview: physical bytes
 // (from file_blobs, backend-dedup-aware -- what's actually consuming disk
-// or S3) versus logical bytes (sum of the same approximate per-row
-// attribution the per-account breakdown uses, which can legitimately be
-// higher than physical when identical content is shared by more than one
-// document/photo).
+// or S3, deduplicated exactly once across the whole system) versus logical
+// bytes (the SAME real, still-existing file_blobs bytes as physical, just
+// summed per-owner via perOwnerMediaSizeSQL without deduplicating content
+// shared across accounts/documents -- so logical can legitimately be higher
+// than physical when the same blob is attributed to more than one
+// document/photo, but a purged file with no file_blobs rows left correctly
+// contributes 0 to both, never a stale non-zero "ghost" size).
 type StorageStatsRow struct {
 	PhysicalBytes     int64 `json:"PhysicalBytes,string"`
 	LogicalBytes      int64 `json:"LogicalBytes,string"`

cmd/telesrv-admin/server.go

@@ -136,6 +136,7 @@ func (s *server) routes() http.Handler {
 	mux.Handle("POST /api/actions/set-gif-catalog-category", s.requireAuthAPI(http.HandlerFunc(s.handleSetGifCatalogCategoryAPI)))
 	mux.Handle("POST /api/actions/auto-categorize-gif-catalog", s.requireAuthAPI(http.HandlerFunc(s.handleAutoCategorizeGifCatalogAPI)))
 	mux.Handle("POST /api/actions/delete-uncategorized-gifs", s.requireAuthAPI(http.HandlerFunc(s.handleDeleteUncategorizedGifsAPI)))
+	mux.Handle("POST /api/actions/storage-manual-purge", s.requireAuthAPI(http.HandlerFunc(s.handleStorageManualPurgeAPI)))
 	mux.Handle("POST /api/actions/delete-gif-catalog-entry", s.requireAuthAPI(http.HandlerFunc(s.handleDeleteGifCatalogEntryAPI)))
 	mux.Handle("POST /api/actions/mint-collectible-username", s.requireAuthAPI(http.HandlerFunc(s.handleMintCollectibleUsernameAPI)))
 	mux.Handle("POST /api/actions/transfer-collectible-username", s.requireAuthAPI(http.HandlerFunc(s.handleTransferCollectibleUsernameAPI)))
@@ -2040,6 +2041,37 @@ func (s *server) handleDeleteUncategorizedGifsAPI(w http.ResponseWriter, r *http
 	writeCommandResultAPI(w, result, err)
 }
 
+// storageManualPurgeAPIRequest mirrors ManualPurgeStorageRequest's frontend
+// payload (see StoragePage.tsx's manual purge modal): categories/include_avatars
+// go through as native JSON, created_before is an ISO/RFC3339 date string
+// (produced by `new Date(x).toISOString()` on the frontend, same convention
+// as freeze_until -- see setAccountFrozenAPIRequest.Until) that encoding/json
+// parses straight into *time.Time; the field left undefined by the frontend
+// (no date entered) decodes to nil, meaning no age filter at all.
+type storageManualPurgeAPIRequest struct {
+	CommandID      string     `json:"command_id"`
+	Reason         string     `json:"reason"`
+	Confirm        bool       `json:"confirm"`
+	Categories     []string   `json:"categories"`
+	IncludeAvatars bool       `json:"include_avatars"`
+	CreatedBefore  *time.Time `json:"created_before"`
+}
+
+func (s *server) handleStorageManualPurgeAPI(w http.ResponseWriter, r *http.Request) {
+	var body storageManualPurgeAPIRequest
+	if !decodeAction(w, r, &body) {
+		return
+	}
+	req := admin.ManualPurgeStorageRequest{
+		CommandMeta:    s.commandMetaFromAPI(r, body.CommandID, body.Reason, body.Confirm, "storage-manual-purge"),
+		Categories:     body.Categories,
+		IncludeAvatars: body.IncludeAvatars,
+		CreatedBefore:  body.CreatedBefore,
+	}
+	result, err := s.callAdminAPI(r.Context(), "/v1/storage/manual-purge", req)
+	writeCommandResultAPI(w, result, err)
+}
+
 type deleteGifCatalogEntryAPIRequest struct {
 	CommandID string `json:"command_id"`
 	Reason    string `json:"reason"`

cmd/telesrv-admin/web/dist/index.html

@@ -23,7 +23,7 @@
       })();
     </script>
 
-    <script type="module" crossorigin src="/assets/index-By5gK7SA.js"></script>
+    <script type="module" crossorigin src="/assets/index-CMLbsGwc.js"></script>
     <link rel="stylesheet" crossorigin href="/assets/index-B7hI8ol7.css">
   </head>
   <body>

cmd/telesrv-admin/web/src/pages/Dashboard.tsx

@@ -151,12 +151,12 @@ export function Dashboard({ navigate }: { navigate: Navigate }) {
           icon={<HardDrive />}
           label="Disk free"
           percent={
-            host?.Ready && host.DiskTotalBytes > 0
+            host?.Ready && host.DiskReady && host.DiskTotalBytes > 0
               ? ((host.DiskTotalBytes - host.DiskFreeBytes) / host.DiskTotalBytes) * 100
               : undefined
           }
-          valueText={host?.Ready ? formatBytes(String(host.DiskFreeBytes)) : "…"}
-          sub={host?.Ready ? `of ${formatBytes(String(host.DiskTotalBytes))}` : undefined}
+          valueText={host?.Ready && host.DiskReady ? formatBytes(String(host.DiskFreeBytes)) : "…"}
+          sub={host?.Ready && host.DiskReady ? `of ${formatBytes(String(host.DiskTotalBytes))}` : "no reading yet"}
           warnAbove={85}
         />
       </Section>

cmd/telesrv-admin/web/src/pages/StoragePage.tsx

@@ -1,4 +1,4 @@
-import { ArrowDown, ArrowUp, ArrowUpDown, ChevronDown, Loader2, RefreshCw, Search, Settings2, X } from "lucide-react";
+import { ArrowDown, ArrowUp, ArrowUpDown, ChevronDown, ChevronRight, Loader2, RefreshCw, Search, Settings2, X } from "lucide-react";
 import { useEffect, useMemo, useState } from "react";
 import { createPortal } from "react-dom";
 import { api, errorMessage } from "../api";
@@ -38,7 +38,7 @@ function SortableHeader({
   );
 }
 
-function StorageOverviewTab() {
+function StorageOverviewTab({ navigate }: { navigate: Navigate }) {
   const [stats, setStats] = useState<StorageStatsResponse | null>(null);
   const [rows, setRows] = useState<AccountStorageRow[]>([]);
   const [hasMore, setHasMore] = useState(false);
@@ -148,6 +148,7 @@ function StorageOverviewTab() {
               <SortableHeader label={"Account"} sortKey="username" activeKey={sortKey} desc={sortDesc} onSort={toggleSort} />
               <SortableHeader label={"Storage used"} sortKey="bytes" activeKey={sortKey} desc={sortDesc} onSort={toggleSort} />
               <SortableHeader label={"Files"} sortKey="files" activeKey={sortKey} desc={sortDesc} onSort={toggleSort} />
+              <th></th>
             </tr>
           </thead>
           <tbody>
@@ -157,9 +158,10 @@ function StorageOverviewTab() {
                 <td>{displayUsername(row.Username) || row.FirstName || "-"}</td>
                 <td className="mono">{formatBytes(row.Bytes)}</td>
                 <td className="mono">{formatQuantity(row.FileCount)}</td>
+                <td><button className="row-link" type="button" onClick={() => navigate(`/accounts/${row.UserID}`)}>{"Details"} <ChevronRight size={14} /></button></td>
               </tr>
             ))}
-            {rows.length === 0 && <EmptyRow colSpan={4} />}
+            {rows.length === 0 && <EmptyRow colSpan={5} />}
           </tbody>
         </table>
       </div>
@@ -174,7 +176,7 @@ function StorageOverviewTab() {
   );
 }
 
-export function StoragePage({ navigate: _navigate }: { navigate: Navigate }) {
+export function StoragePage({ navigate }: { navigate: Navigate }) {
   const [tab, setTab] = useState<"overview" | "limits">("overview");
   return (
     <PageFrame title={"Storage"} eyebrow={"Media / Storage usage"}>
@@ -186,7 +188,7 @@ export function StoragePage({ navigate: _navigate }: { navigate: Navigate }) {
           {"Limits & Retention"}
         </button>
       </div>
-      {tab === "overview" ? <StorageOverviewTab /> : <LimitsRetentionSection />}
+      {tab === "overview" ? <StorageOverviewTab navigate={navigate} /> : <LimitsRetentionSection />}
     </PageFrame>
   );
 }
@@ -438,6 +440,92 @@ function CategoryRetentionModal({
   );
 }
 
+// ManualPurgeStorageModal lets an operator delete media blob bytes right now
+// by hand-picked category (and, optionally, avatars) with an optional
+// "created before" age cutoff -- the manual counterpart of the automatic
+// hard-retention sweep above. Leaving the date empty purges everything
+// matching the selected categories regardless of age. Deletion semantics are
+// identical to "hard" retention mode: only the file bytes are removed, never
+// the document/photo row, so an affected message still renders its
+// placeholder (see internal/app/files.Service.ManualPurge's doc comment).
+// Follows the same modal-backdrop/command-modal structure as
+// CategoryRetentionModal, but hands the actual mutation off to ActionButton
+// (reason + dry-run + confirm) since -- unlike the settings above -- this is
+// an immediate, irreversible delete rather than a saved .env value.
+function ManualPurgeStorageModal({ onClose }: { onClose: () => void }) {
+  const [selected, setSelected] = useState<Record<string, boolean>>({});
+  const [dateValue, setDateValue] = useState("");
+
+  const documentFields = CATEGORY_AGE_FIELDS.filter((f) => f.key !== "avatar");
+  const allSelected = CATEGORY_AGE_FIELDS.every((f) => selected[f.key]);
+
+  function toggle(key: string) {
+    setSelected((prev) => ({ ...prev, [key]: !prev[key] }));
+  }
+
+  function toggleAll() {
+    const next = !allSelected;
+    const nextSelected: Record<string, boolean> = {};
+    for (const field of CATEGORY_AGE_FIELDS) nextSelected[field.key] = next;
+    setSelected(nextSelected);
+  }
+
+  const chosenCategories = documentFields.filter((f) => selected[f.key]).map((f) => f.key);
+  const includeAvatars = Boolean(selected.avatar);
+  const canSubmit = chosenCategories.length > 0 || includeAvatars;
+
+  return createPortal(
+    <div className="modal-backdrop" role="presentation">
+      <section className="modal command-modal" role="dialog" aria-modal="true" aria-label={"Manually purge storage"}>
+        <div className="modal-head">
+          <div>
+            <div className="eyebrow">{"Limits & Retention"}</div>
+            <h2>{"Manually purge storage"}</h2>
+          </div>
+          <button className="icon-btn" type="button" onClick={onClose} aria-label={"Close"}><X size={15} /></button>
+        </div>
+        <div className="command-body">
+          <p className="env-field-desc">
+            {"Deletes the file bytes of every document/photo matching the categories below, right now -- independent of the retention mode/age configured above. The message/profile-photo itself is never deleted, only its file; a purged item starts showing as unavailable. Leave \"Created before\" empty to purge everything in the selected categories, regardless of age."}
+          </p>
+          <div className="attr-block">
+            <label className="checkline">
+              <input type="checkbox" checked={allSelected} onChange={toggleAll} />
+              {" "}{"Select all"}
+            </label>
+            {CATEGORY_AGE_FIELDS.map((field) => (
+              <label key={field.key} className="checkline">
+                <input type="checkbox" checked={Boolean(selected[field.key])} onChange={() => toggle(field.key)} />
+                {" "}{field.label}
+              </label>
+            ))}
+          </div>
+          <label className="duration-field">
+            <span>{"Created before (optional)"}</span>
+            <input type="date" value={dateValue} onChange={(event) => setDateValue(event.target.value)} />
+            <span className="env-field-desc">{"Empty = no age limit, purge everything matching the selected categories."}</span>
+          </label>
+        </div>
+        <div className="modal-actions">
+          <button className="btn" type="button" onClick={onClose}>{"Close"}</button>
+          <ActionButton
+            disabled={!canSubmit}
+            label={"Purge selected storage"}
+            tone="danger"
+            path="/api/actions/storage-manual-purge"
+            payload={() => ({
+              categories: chosenCategories,
+              include_avatars: includeAvatars,
+              created_before: dateValue ? new Date(dateValue).toISOString() : undefined
+            })}
+          />
+        </div>
+      </section>
+    </div>,
+    document.body
+  );
+}
+
 function LimitsRetentionSection() {
   const [loaded, setLoaded] = useState(false);
   const [error, setError] = useState("");
@@ -451,6 +539,7 @@ function LimitsRetentionSection() {
   const [categoryAgeMinutes, setCategoryAgeMinutes] = useState<Record<string, string>>({});
   const [categoryModalOpen, setCategoryModalOpen] = useState(false);
   const [evictionEnable, setEvictionEnable] = useState(false);
+  const [manualPurgeOpen, setManualPurgeOpen] = useState(false);
 
   async function load() {
     setError("");
@@ -621,6 +710,16 @@ function LimitsRetentionSection() {
             </p>
           </div>
 
+          <div className="attr-block" style={{ marginTop: "1.5em" }}>
+            <button className="btn danger icon-text" type="button" onClick={() => setManualPurgeOpen(true)}>
+              <Settings2 size={15} /> {"Manually purge storage..."}
+            </button>
+            <span className="env-field-desc">
+              {"Delete media file bytes right now by hand-picked category and an optional age cutoff, independent of the retention settings above."}
+            </span>
+          </div>
+          {manualPurgeOpen && <ManualPurgeStorageModal onClose={() => setManualPurgeOpen(false)} />}
+
           <div className="gift-table-actions env-save-row">
             <ActionButton
               tone="warn"

cmd/telesrv-admin/web/src/types.ts

@@ -753,6 +753,11 @@ export type HostStatsSnapshot = {
   MemTotalBytes: number;
   DiskFreeBytes: number;
   DiskTotalBytes: number;
+  // False when the disk-space sample itself failed (wrong path, not
+  // created yet, etc) -- DiskFreeBytes/DiskTotalBytes are stale/zero in
+  // that case, not "the disk is actually full". Independent of Ready,
+  // which only covers CPU/memory.
+  DiskReady: boolean;
   Ready: boolean;
 };