fixes for storage managament
This commit is contained in:
onysd
parent
e6bfe2d444
commit
8ef2b58bf9
29 changed files with 1768 additions and 63 deletions
|
|
@ -64,7 +64,7 @@ func TestHardRetentionPurgesBlobBytesButKeepsMetadataRow(t *testing.T) {
|
|||
})
|
||||
|
||||
cutoff := time.Now().Add(-24 * time.Hour)
|
||||
ids, err := media.ListDocumentIDsForHardRetentionOlderThan(ctx, domain.MediaCategoryNone, cutoff, 1000)
|
||||
ids, err := media.ListDocumentIDsForHardRetentionOlderThan(ctx, domain.MediaCategoryNone, &cutoff, 1000)
|
||||
if err != nil {
|
||||
t.Fatalf("ListDocumentIDsForHardRetentionOlderThan: %v", err)
|
||||
}
|
||||
|
|
@ -95,7 +95,7 @@ func TestHardRetentionPurgesBlobBytesButKeepsMetadataRow(t *testing.T) {
|
|||
|
||||
// Idempotent / self-terminating: a second sweep pass no longer selects
|
||||
// this document, since it no longer owns any file_blobs row.
|
||||
ids, err = media.ListDocumentIDsForHardRetentionOlderThan(ctx, domain.MediaCategoryNone, cutoff, 1000)
|
||||
ids, err = media.ListDocumentIDsForHardRetentionOlderThan(ctx, domain.MediaCategoryNone, &cutoff, 1000)
|
||||
if err != nil {
|
||||
t.Fatalf("ListDocumentIDsForHardRetentionOlderThan (2nd pass): %v", err)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -236,48 +236,90 @@ func (s *MediaStore) DeleteDocumentAndBlobs(ctx context.Context, id int64) ([]do
|
|||
return blobs, nil
|
||||
}
|
||||
|
||||
// nullableTimestamptz converts an optional cutoff into the nullable
|
||||
// pgtype.Timestamptz the hard-retention List*/Count* queries expect: nil
|
||||
// means "no age filter at all" (sqlc.narg(cutoff) IS NULL in the query),
|
||||
// matching the manual-purge admin action's "no date = everything" contract.
|
||||
// The automatic sweep (files.Service.DeleteBlobBytesForMediaOlderThan)
|
||||
// always passes a non-nil cutoff.
|
||||
func nullableTimestamptz(cutoff *time.Time) pgtype.Timestamptz {
|
||||
if cutoff == nil {
|
||||
return pgtype.Timestamptz{}
|
||||
}
|
||||
return pgtype.Timestamptz{Time: *cutoff, Valid: true}
|
||||
}
|
||||
|
||||
// ListDocumentIDsForHardRetentionOlderThan returns document ids in the given
|
||||
// category older than cutoff (by upload/created_at) that still own at least
|
||||
// one file_blobs row, oldest first, up to limit -- the "hard" retention
|
||||
// sweep's candidate list. Unlike ListOrphanedDocumentIDsOlderThan, this
|
||||
// ignores media_references entirely: a document still referenced by a live
|
||||
// message is exactly as eligible as an orphaned one.
|
||||
func (s *MediaStore) ListDocumentIDsForHardRetentionOlderThan(ctx context.Context, category domain.MediaCategory, cutoff time.Time, limit int) ([]int64, error) {
|
||||
// message is exactly as eligible as an orphaned one. cutoff may be nil,
|
||||
// meaning no age filter at all (every document in the category is a
|
||||
// candidate) -- used by the manual purge admin action.
|
||||
func (s *MediaStore) ListDocumentIDsForHardRetentionOlderThan(ctx context.Context, category domain.MediaCategory, cutoff *time.Time, limit int) ([]int64, error) {
|
||||
if limit <= 0 {
|
||||
return nil, nil
|
||||
}
|
||||
return s.q.ListDocumentIDsForHardRetentionOlderThan(ctx, sqlcgen.ListDocumentIDsForHardRetentionOlderThanParams{
|
||||
Cutoff: pgtype.Timestamptz{Time: cutoff, Valid: true},
|
||||
Cutoff: nullableTimestamptz(cutoff),
|
||||
Category: int16(category),
|
||||
BatchLimit: int32(limit),
|
||||
})
|
||||
}
|
||||
|
||||
// CountDocumentsForHardRetention is the exact-count counterpart of
|
||||
// ListDocumentIDsForHardRetentionOlderThan, for the manual purge admin
|
||||
// action's dry-run preview.
|
||||
func (s *MediaStore) CountDocumentsForHardRetention(ctx context.Context, category domain.MediaCategory, cutoff *time.Time) (int, error) {
|
||||
n, err := s.q.CountDocumentsForHardRetention(ctx, sqlcgen.CountDocumentsForHardRetentionParams{
|
||||
Cutoff: nullableTimestamptz(cutoff),
|
||||
Category: int16(category),
|
||||
})
|
||||
return int(n), err
|
||||
}
|
||||
|
||||
// ListPhotoIDsForHardRetentionOlderThan is the photo counterpart of
|
||||
// ListDocumentIDsForHardRetentionOlderThan (excluding a live avatar -- see
|
||||
// ListAvatarPhotoIDsForHardRetentionOlderThan).
|
||||
func (s *MediaStore) ListPhotoIDsForHardRetentionOlderThan(ctx context.Context, cutoff time.Time, limit int) ([]int64, error) {
|
||||
// ListAvatarPhotoIDsForHardRetentionOlderThan). cutoff may be nil -- see
|
||||
// ListDocumentIDsForHardRetentionOlderThan.
|
||||
func (s *MediaStore) ListPhotoIDsForHardRetentionOlderThan(ctx context.Context, cutoff *time.Time, limit int) ([]int64, error) {
|
||||
if limit <= 0 {
|
||||
return nil, nil
|
||||
}
|
||||
return s.q.ListPhotoIDsForHardRetentionOlderThan(ctx, sqlcgen.ListPhotoIDsForHardRetentionOlderThanParams{
|
||||
Cutoff: pgtype.Timestamptz{Time: cutoff, Valid: true},
|
||||
Cutoff: nullableTimestamptz(cutoff),
|
||||
BatchLimit: int32(limit),
|
||||
})
|
||||
}
|
||||
|
||||
// CountPhotosForHardRetention is the exact-count counterpart of
|
||||
// ListPhotoIDsForHardRetentionOlderThan.
|
||||
func (s *MediaStore) CountPhotosForHardRetention(ctx context.Context, cutoff *time.Time) (int, error) {
|
||||
n, err := s.q.CountPhotosForHardRetention(ctx, nullableTimestamptz(cutoff))
|
||||
return int(n), err
|
||||
}
|
||||
|
||||
// ListAvatarPhotoIDsForHardRetentionOlderThan is the "Avatar" category
|
||||
// counterpart of ListPhotoIDsForHardRetentionOlderThan.
|
||||
func (s *MediaStore) ListAvatarPhotoIDsForHardRetentionOlderThan(ctx context.Context, cutoff time.Time, limit int) ([]int64, error) {
|
||||
// counterpart of ListPhotoIDsForHardRetentionOlderThan. cutoff may be nil --
|
||||
// see ListDocumentIDsForHardRetentionOlderThan.
|
||||
func (s *MediaStore) ListAvatarPhotoIDsForHardRetentionOlderThan(ctx context.Context, cutoff *time.Time, limit int) ([]int64, error) {
|
||||
if limit <= 0 {
|
||||
return nil, nil
|
||||
}
|
||||
return s.q.ListAvatarPhotoIDsForHardRetentionOlderThan(ctx, sqlcgen.ListAvatarPhotoIDsForHardRetentionOlderThanParams{
|
||||
Cutoff: pgtype.Timestamptz{Time: cutoff, Valid: true},
|
||||
Cutoff: nullableTimestamptz(cutoff),
|
||||
BatchLimit: int32(limit),
|
||||
})
|
||||
}
|
||||
|
||||
// CountAvatarPhotosForHardRetention is the exact-count counterpart of
|
||||
// ListAvatarPhotoIDsForHardRetentionOlderThan.
|
||||
func (s *MediaStore) CountAvatarPhotosForHardRetention(ctx context.Context, cutoff *time.Time) (int, error) {
|
||||
n, err := s.q.CountAvatarPhotosForHardRetention(ctx, nullableTimestamptz(cutoff))
|
||||
return int(n), err
|
||||
}
|
||||
|
||||
// DeleteFileBlobsForDocument deletes every file_blobs row a document owns
|
||||
// (main body + thumbnail variants), returning what was deleted so the caller
|
||||
// can physically remove each object from its backend once confirming (via
|
||||
|
|
|
|||
|
|
@ -277,9 +277,14 @@ LIMIT sqlc.arg(batch_limit)::int;
|
|||
-- rows but deliberately leaves this documents row in place), it naturally
|
||||
-- drops out of this query on the next pass. category scopes to one
|
||||
-- documents.category bucket per sweep tick -- see
|
||||
-- ListOrphanedDocumentIDsOlderThan for the 0/None fallback note.
|
||||
-- ListOrphanedDocumentIDsOlderThan for the 0/None fallback note. cutoff is
|
||||
-- nullable: NULL means no age filter at all (every document in the category
|
||||
-- is a candidate, regardless of created_at) -- used by the manual purge admin
|
||||
-- action, which unlike the automatic sweep can target "everything" with no
|
||||
-- age cutoff. The automatic sweep (files.Service.DeleteBlobBytesForMediaOlderThan)
|
||||
-- always passes a real, non-null cutoff.
|
||||
SELECT d.id FROM documents d
|
||||
WHERE d.created_at < sqlc.arg(cutoff)::timestamptz
|
||||
WHERE (sqlc.narg(cutoff)::timestamptz IS NULL OR d.created_at < sqlc.narg(cutoff)::timestamptz)
|
||||
AND d.category = sqlc.arg(category)::smallint
|
||||
AND EXISTS (
|
||||
SELECT 1 FROM file_blobs fb
|
||||
|
|
@ -289,12 +294,26 @@ WHERE d.created_at < sqlc.arg(cutoff)::timestamptz
|
|||
ORDER BY d.created_at ASC
|
||||
LIMIT sqlc.arg(batch_limit)::int;
|
||||
|
||||
-- name: CountDocumentsForHardRetention :one
|
||||
-- Exact-count counterpart of ListDocumentIDsForHardRetentionOlderThan, used
|
||||
-- by the manual purge admin action's dry-run preview: a LIMIT-capped list
|
||||
-- length is not good enough there, the operator needs the real total.
|
||||
SELECT COUNT(*)::int FROM documents d
|
||||
WHERE (sqlc.narg(cutoff)::timestamptz IS NULL OR d.created_at < sqlc.narg(cutoff)::timestamptz)
|
||||
AND d.category = sqlc.arg(category)::smallint
|
||||
AND EXISTS (
|
||||
SELECT 1 FROM file_blobs fb
|
||||
WHERE fb.location_key = 'doc:' || d.id::text
|
||||
OR fb.location_key LIKE 'doc:' || d.id::text || ':%'
|
||||
);
|
||||
|
||||
-- name: ListPhotoIDsForHardRetentionOlderThan :many
|
||||
-- See ListDocumentIDsForHardRetentionOlderThan -- same "hard" retention
|
||||
-- candidate selection, for photos. Excludes photos currently active as
|
||||
-- someone's avatar -- see ListAvatarPhotoIDsForHardRetentionOlderThan.
|
||||
-- candidate selection (including the nullable cutoff), for photos. Excludes
|
||||
-- photos currently active as someone's avatar -- see
|
||||
-- ListAvatarPhotoIDsForHardRetentionOlderThan.
|
||||
SELECT p.id FROM photos p
|
||||
WHERE p.created_at < sqlc.arg(cutoff)::timestamptz
|
||||
WHERE (sqlc.narg(cutoff)::timestamptz IS NULL OR p.created_at < sqlc.narg(cutoff)::timestamptz)
|
||||
AND NOT EXISTS (SELECT 1 FROM profile_photos pp WHERE pp.photo_id = p.id AND pp.active)
|
||||
AND EXISTS (
|
||||
SELECT 1 FROM file_blobs fb
|
||||
|
|
@ -304,13 +323,25 @@ WHERE p.created_at < sqlc.arg(cutoff)::timestamptz
|
|||
ORDER BY p.created_at ASC
|
||||
LIMIT sqlc.arg(batch_limit)::int;
|
||||
|
||||
-- name: CountPhotosForHardRetention :one
|
||||
-- Exact-count counterpart of ListPhotoIDsForHardRetentionOlderThan -- see
|
||||
-- CountDocumentsForHardRetention.
|
||||
SELECT COUNT(*)::int FROM photos p
|
||||
WHERE (sqlc.narg(cutoff)::timestamptz IS NULL OR p.created_at < sqlc.narg(cutoff)::timestamptz)
|
||||
AND NOT EXISTS (SELECT 1 FROM profile_photos pp WHERE pp.photo_id = p.id AND pp.active)
|
||||
AND EXISTS (
|
||||
SELECT 1 FROM file_blobs fb
|
||||
WHERE fb.location_key = 'photo:' || p.id::text
|
||||
OR fb.location_key LIKE 'photo:' || p.id::text || ':%'
|
||||
);
|
||||
|
||||
-- name: ListAvatarPhotoIDsForHardRetentionOlderThan :many
|
||||
-- Same as ListPhotoIDsForHardRetentionOlderThan but only photos currently
|
||||
-- active as someone's avatar (profile_photos.active) -- lets the Avatar
|
||||
-- category carry its own retention age, independent of ordinary shared-media
|
||||
-- photos.
|
||||
SELECT p.id FROM photos p
|
||||
WHERE p.created_at < sqlc.arg(cutoff)::timestamptz
|
||||
WHERE (sqlc.narg(cutoff)::timestamptz IS NULL OR p.created_at < sqlc.narg(cutoff)::timestamptz)
|
||||
AND EXISTS (SELECT 1 FROM profile_photos pp WHERE pp.photo_id = p.id AND pp.active)
|
||||
AND EXISTS (
|
||||
SELECT 1 FROM file_blobs fb
|
||||
|
|
@ -320,6 +351,17 @@ WHERE p.created_at < sqlc.arg(cutoff)::timestamptz
|
|||
ORDER BY p.created_at ASC
|
||||
LIMIT sqlc.arg(batch_limit)::int;
|
||||
|
||||
-- name: CountAvatarPhotosForHardRetention :one
|
||||
-- Exact-count counterpart of ListAvatarPhotoIDsForHardRetentionOlderThan.
|
||||
SELECT COUNT(*)::int FROM photos p
|
||||
WHERE (sqlc.narg(cutoff)::timestamptz IS NULL OR p.created_at < sqlc.narg(cutoff)::timestamptz)
|
||||
AND EXISTS (SELECT 1 FROM profile_photos pp WHERE pp.photo_id = p.id AND pp.active)
|
||||
AND EXISTS (
|
||||
SELECT 1 FROM file_blobs fb
|
||||
WHERE fb.location_key = 'photo:' || p.id::text
|
||||
OR fb.location_key LIKE 'photo:' || p.id::text || ':%'
|
||||
);
|
||||
|
||||
-- name: ListOldestDocumentsForEviction :many
|
||||
-- Active eviction (TELESRV_STORAGE_EVICTION_ENABLE): candidates are every
|
||||
-- document that still owns at least one file_blobs row, oldest-uploaded
|
||||
|
|
|
|||
|
|
@ -279,7 +279,7 @@ func TestHardRetentionCategoryFilterOnlyReturnsMatchingCategory(t *testing.T) {
|
|||
}
|
||||
|
||||
cutoff := time.Now().Add(-24 * time.Hour)
|
||||
videoIDs, err := media.ListDocumentIDsForHardRetentionOlderThan(ctx, domain.MediaCategoryVideo, cutoff, 1000)
|
||||
videoIDs, err := media.ListDocumentIDsForHardRetentionOlderThan(ctx, domain.MediaCategoryVideo, &cutoff, 1000)
|
||||
if err != nil {
|
||||
t.Fatalf("ListDocumentIDsForHardRetentionOlderThan(Video): %v", err)
|
||||
}
|
||||
|
|
@ -287,7 +287,7 @@ func TestHardRetentionCategoryFilterOnlyReturnsMatchingCategory(t *testing.T) {
|
|||
t.Fatalf("video-category candidates = %v, want to include %d and exclude %d", videoIDs, videoID, musicID)
|
||||
}
|
||||
|
||||
musicIDs, err := media.ListDocumentIDsForHardRetentionOlderThan(ctx, domain.MediaCategoryMusic, cutoff, 1000)
|
||||
musicIDs, err := media.ListDocumentIDsForHardRetentionOlderThan(ctx, domain.MediaCategoryMusic, &cutoff, 1000)
|
||||
if err != nil {
|
||||
t.Fatalf("ListDocumentIDsForHardRetentionOlderThan(Music): %v", err)
|
||||
}
|
||||
|
|
@ -296,6 +296,123 @@ func TestHardRetentionCategoryFilterOnlyReturnsMatchingCategory(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
// TestManualPurgeNilCutoffIgnoresAge guards the manual purge admin action's
|
||||
// query-layer contract: a nil cutoff on the hard-retention List/Count
|
||||
// queries must mean "no age filter at all" -- a document created just now is
|
||||
// still a candidate -- while a real, non-nil cutoff still filters normally.
|
||||
// This is the query-layer half of files.Service.ManualPurge/CountManualPurgeCandidates's
|
||||
// "no date = everything matching the categories, regardless of age" contract
|
||||
// (the automatic sweep, by contrast, never passes a nil cutoff).
|
||||
func TestManualPurgeNilCutoffIgnoresAge(t *testing.T) {
|
||||
pool := testPool(t)
|
||||
ctx := context.Background()
|
||||
media := NewMediaStore(pool)
|
||||
|
||||
freshID := time.Now().UnixNano()
|
||||
locationKey := "doc:" + strconv.FormatInt(freshID, 10)
|
||||
t.Cleanup(func() {
|
||||
_, _ = pool.Exec(context.Background(), "DELETE FROM documents WHERE id = $1", freshID)
|
||||
_, _ = pool.Exec(context.Background(), "DELETE FROM file_blobs WHERE location_key = $1", locationKey)
|
||||
})
|
||||
|
||||
if err := media.PutDocument(ctx, domain.Document{ID: freshID, MimeType: "application/octet-stream", Size: 128}); err != nil {
|
||||
t.Fatalf("PutDocument: %v", err)
|
||||
}
|
||||
// Deliberately NOT backdated -- created_at stays "just now", unlike every
|
||||
// other hard-retention test in this file, since the whole point here is
|
||||
// that a nil cutoff must not exempt a fresh document.
|
||||
if _, err := pool.Exec(ctx, "UPDATE documents SET category = $2 WHERE id = $1", freshID, int16(domain.MediaCategoryFile)); err != nil {
|
||||
t.Fatalf("categorize document: %v", err)
|
||||
}
|
||||
blob := postgresTestBlob(locationKey, "manual-purge-nil-cutoff", 128, "application/octet-stream")
|
||||
if err := media.PutFileBlob(ctx, blob); err != nil {
|
||||
t.Fatalf("PutFileBlob: %v", err)
|
||||
}
|
||||
|
||||
// nil cutoff: the fresh document IS a candidate.
|
||||
ids, err := media.ListDocumentIDsForHardRetentionOlderThan(ctx, domain.MediaCategoryFile, nil, 1000)
|
||||
if err != nil {
|
||||
t.Fatalf("ListDocumentIDsForHardRetentionOlderThan(nil cutoff): %v", err)
|
||||
}
|
||||
if !containsInt64(ids, freshID) {
|
||||
t.Fatalf("nil-cutoff candidates = %v, want to include fresh document %d", ids, freshID)
|
||||
}
|
||||
count, err := media.CountDocumentsForHardRetention(ctx, domain.MediaCategoryFile, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("CountDocumentsForHardRetention(nil cutoff): %v", err)
|
||||
}
|
||||
if count < 1 {
|
||||
t.Fatalf("nil-cutoff count = %d, want >= 1 (must include fresh document)", count)
|
||||
}
|
||||
|
||||
// A real cutoff in the past still excludes the fresh document.
|
||||
pastCutoff := time.Now().Add(-24 * time.Hour)
|
||||
ids, err = media.ListDocumentIDsForHardRetentionOlderThan(ctx, domain.MediaCategoryFile, &pastCutoff, 1000)
|
||||
if err != nil {
|
||||
t.Fatalf("ListDocumentIDsForHardRetentionOlderThan(past cutoff): %v", err)
|
||||
}
|
||||
if containsInt64(ids, freshID) {
|
||||
t.Fatalf("past-cutoff candidates = %v, want to exclude fresh document %d", ids, freshID)
|
||||
}
|
||||
count, err = media.CountDocumentsForHardRetention(ctx, domain.MediaCategoryFile, &pastCutoff)
|
||||
if err != nil {
|
||||
t.Fatalf("CountDocumentsForHardRetention(past cutoff): %v", err)
|
||||
}
|
||||
// Some other, unrelated old File-category document could in principle
|
||||
// exist in this shared test database, so this only asserts the fresh
|
||||
// document itself dropped out, via the id-level checks above; the count
|
||||
// assertion below just guards against the query going the wrong
|
||||
// direction entirely (e.g. cutoff being ignored and still counting
|
||||
// everything).
|
||||
allCount, err := media.CountDocumentsForHardRetention(ctx, domain.MediaCategoryFile, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("CountDocumentsForHardRetention(nil, recheck): %v", err)
|
||||
}
|
||||
if count > allCount {
|
||||
t.Fatalf("past-cutoff count %d > nil-cutoff count %d, cutoff filter is backwards", count, allCount)
|
||||
}
|
||||
}
|
||||
|
||||
// TestManualPurgePhotoAndAvatarNilCutoffIgnoresAge is the photo/avatar
|
||||
// counterpart of TestManualPurgeNilCutoffIgnoresAge.
|
||||
func TestManualPurgePhotoAndAvatarNilCutoffIgnoresAge(t *testing.T) {
|
||||
pool := testPool(t)
|
||||
ctx := context.Background()
|
||||
media := NewMediaStore(pool)
|
||||
|
||||
photoID := time.Now().UnixNano()
|
||||
locationKey := "photo:" + strconv.FormatInt(photoID, 10)
|
||||
t.Cleanup(func() {
|
||||
_, _ = pool.Exec(context.Background(), "DELETE FROM photos WHERE id = $1", photoID)
|
||||
_, _ = pool.Exec(context.Background(), "DELETE FROM file_blobs WHERE location_key = $1", locationKey)
|
||||
})
|
||||
|
||||
if err := media.PutPhoto(ctx, domain.Photo{ID: photoID, AccessHash: 1}); err != nil {
|
||||
t.Fatalf("PutPhoto: %v", err)
|
||||
}
|
||||
blob := postgresTestBlob(locationKey, "manual-purge-photo-nil-cutoff", 64, "image/jpeg")
|
||||
if err := media.PutFileBlob(ctx, blob); err != nil {
|
||||
t.Fatalf("PutFileBlob: %v", err)
|
||||
}
|
||||
|
||||
ids, err := media.ListPhotoIDsForHardRetentionOlderThan(ctx, nil, 1000)
|
||||
if err != nil {
|
||||
t.Fatalf("ListPhotoIDsForHardRetentionOlderThan(nil cutoff): %v", err)
|
||||
}
|
||||
if !containsInt64(ids, photoID) {
|
||||
t.Fatalf("nil-cutoff photo candidates = %v, want to include fresh photo %d", ids, photoID)
|
||||
}
|
||||
|
||||
pastCutoff := time.Now().Add(-24 * time.Hour)
|
||||
ids, err = media.ListPhotoIDsForHardRetentionOlderThan(ctx, &pastCutoff, 1000)
|
||||
if err != nil {
|
||||
t.Fatalf("ListPhotoIDsForHardRetentionOlderThan(past cutoff): %v", err)
|
||||
}
|
||||
if containsInt64(ids, photoID) {
|
||||
t.Fatalf("past-cutoff photo candidates = %v, want to exclude fresh photo %d", ids, photoID)
|
||||
}
|
||||
}
|
||||
|
||||
// TestEvictionListsOldestAcrossDocumentsAndPhotos guards Part 3's eviction
|
||||
// query layer: ListOldestMediaForEviction must return candidates from both
|
||||
// tables (interleaving by created_at is done by the caller in Go, see
|
||||
|
|
|
|||
|
|
@ -79,6 +79,51 @@ func (q *Queries) CountAvailableReactions(ctx context.Context) (int32, error) {
|
|||
return total, err
|
||||
}
|
||||
|
||||
const countAvatarPhotosForHardRetention = `-- name: CountAvatarPhotosForHardRetention :one
|
||||
SELECT COUNT(*)::int FROM photos p
|
||||
WHERE ($1::timestamptz IS NULL OR p.created_at < $1::timestamptz)
|
||||
AND EXISTS (SELECT 1 FROM profile_photos pp WHERE pp.photo_id = p.id AND pp.active)
|
||||
AND EXISTS (
|
||||
SELECT 1 FROM file_blobs fb
|
||||
WHERE fb.location_key = 'photo:' || p.id::text
|
||||
OR fb.location_key LIKE 'photo:' || p.id::text || ':%'
|
||||
)
|
||||
`
|
||||
|
||||
// Exact-count counterpart of ListAvatarPhotoIDsForHardRetentionOlderThan.
|
||||
func (q *Queries) CountAvatarPhotosForHardRetention(ctx context.Context, cutoff pgtype.Timestamptz) (int32, error) {
|
||||
row := q.db.QueryRow(ctx, countAvatarPhotosForHardRetention, cutoff)
|
||||
var column_1 int32
|
||||
err := row.Scan(&column_1)
|
||||
return column_1, err
|
||||
}
|
||||
|
||||
const countDocumentsForHardRetention = `-- name: CountDocumentsForHardRetention :one
|
||||
SELECT COUNT(*)::int FROM documents d
|
||||
WHERE ($1::timestamptz IS NULL OR d.created_at < $1::timestamptz)
|
||||
AND d.category = $2::smallint
|
||||
AND EXISTS (
|
||||
SELECT 1 FROM file_blobs fb
|
||||
WHERE fb.location_key = 'doc:' || d.id::text
|
||||
OR fb.location_key LIKE 'doc:' || d.id::text || ':%'
|
||||
)
|
||||
`
|
||||
|
||||
type CountDocumentsForHardRetentionParams struct {
|
||||
Cutoff pgtype.Timestamptz
|
||||
Category int16
|
||||
}
|
||||
|
||||
// Exact-count counterpart of ListDocumentIDsForHardRetentionOlderThan, used
|
||||
// by the manual purge admin action's dry-run preview: a LIMIT-capped list
|
||||
// length is not good enough there, the operator needs the real total.
|
||||
func (q *Queries) CountDocumentsForHardRetention(ctx context.Context, arg CountDocumentsForHardRetentionParams) (int32, error) {
|
||||
row := q.db.QueryRow(ctx, countDocumentsForHardRetention, arg.Cutoff, arg.Category)
|
||||
var column_1 int32
|
||||
err := row.Scan(&column_1)
|
||||
return column_1, err
|
||||
}
|
||||
|
||||
const countFileBlobRefs = `-- name: CountFileBlobRefs :one
|
||||
SELECT COUNT(*)::int FROM file_blobs WHERE backend = $1::text AND object_key = $2::text
|
||||
`
|
||||
|
|
@ -95,6 +140,26 @@ func (q *Queries) CountFileBlobRefs(ctx context.Context, arg CountFileBlobRefsPa
|
|||
return column_1, err
|
||||
}
|
||||
|
||||
const countPhotosForHardRetention = `-- name: CountPhotosForHardRetention :one
|
||||
SELECT COUNT(*)::int FROM photos p
|
||||
WHERE ($1::timestamptz IS NULL OR p.created_at < $1::timestamptz)
|
||||
AND NOT EXISTS (SELECT 1 FROM profile_photos pp WHERE pp.photo_id = p.id AND pp.active)
|
||||
AND EXISTS (
|
||||
SELECT 1 FROM file_blobs fb
|
||||
WHERE fb.location_key = 'photo:' || p.id::text
|
||||
OR fb.location_key LIKE 'photo:' || p.id::text || ':%'
|
||||
)
|
||||
`
|
||||
|
||||
// Exact-count counterpart of ListPhotoIDsForHardRetentionOlderThan -- see
|
||||
// CountDocumentsForHardRetention.
|
||||
func (q *Queries) CountPhotosForHardRetention(ctx context.Context, cutoff pgtype.Timestamptz) (int32, error) {
|
||||
row := q.db.QueryRow(ctx, countPhotosForHardRetention, cutoff)
|
||||
var column_1 int32
|
||||
err := row.Scan(&column_1)
|
||||
return column_1, err
|
||||
}
|
||||
|
||||
const countProfilePhotos = `-- name: CountProfilePhotos :one
|
||||
SELECT count(*)::int AS total
|
||||
FROM profile_photos
|
||||
|
|
@ -866,7 +931,7 @@ func (q *Queries) ListAvatarOrphanedPhotoIDsOlderThan(ctx context.Context, arg L
|
|||
|
||||
const listAvatarPhotoIDsForHardRetentionOlderThan = `-- name: ListAvatarPhotoIDsForHardRetentionOlderThan :many
|
||||
SELECT p.id FROM photos p
|
||||
WHERE p.created_at < $1::timestamptz
|
||||
WHERE ($1::timestamptz IS NULL OR p.created_at < $1::timestamptz)
|
||||
AND EXISTS (SELECT 1 FROM profile_photos pp WHERE pp.photo_id = p.id AND pp.active)
|
||||
AND EXISTS (
|
||||
SELECT 1 FROM file_blobs fb
|
||||
|
|
@ -908,7 +973,7 @@ func (q *Queries) ListAvatarPhotoIDsForHardRetentionOlderThan(ctx context.Contex
|
|||
|
||||
const listDocumentIDsForHardRetentionOlderThan = `-- name: ListDocumentIDsForHardRetentionOlderThan :many
|
||||
SELECT d.id FROM documents d
|
||||
WHERE d.created_at < $1::timestamptz
|
||||
WHERE ($1::timestamptz IS NULL OR d.created_at < $1::timestamptz)
|
||||
AND d.category = $2::smallint
|
||||
AND EXISTS (
|
||||
SELECT 1 FROM file_blobs fb
|
||||
|
|
@ -933,7 +998,12 @@ type ListDocumentIDsForHardRetentionOlderThanParams struct {
|
|||
// rows but deliberately leaves this documents row in place), it naturally
|
||||
// drops out of this query on the next pass. category scopes to one
|
||||
// documents.category bucket per sweep tick -- see
|
||||
// ListOrphanedDocumentIDsOlderThan for the 0/None fallback note.
|
||||
// ListOrphanedDocumentIDsOlderThan for the 0/None fallback note. cutoff is
|
||||
// nullable: NULL means no age filter at all (every document in the category
|
||||
// is a candidate, regardless of created_at) -- used by the manual purge admin
|
||||
// action, which unlike the automatic sweep can target "everything" with no
|
||||
// age cutoff. The automatic sweep (files.Service.DeleteBlobBytesForMediaOlderThan)
|
||||
// always passes a real, non-null cutoff.
|
||||
func (q *Queries) ListDocumentIDsForHardRetentionOlderThan(ctx context.Context, arg ListDocumentIDsForHardRetentionOlderThanParams) ([]int64, error) {
|
||||
rows, err := q.db.Query(ctx, listDocumentIDsForHardRetentionOlderThan, arg.Cutoff, arg.Category, arg.BatchLimit)
|
||||
if err != nil {
|
||||
|
|
@ -1207,7 +1277,7 @@ func (q *Queries) ListOrphanedPhotoIDsOlderThan(ctx context.Context, arg ListOrp
|
|||
|
||||
const listPhotoIDsForHardRetentionOlderThan = `-- name: ListPhotoIDsForHardRetentionOlderThan :many
|
||||
SELECT p.id FROM photos p
|
||||
WHERE p.created_at < $1::timestamptz
|
||||
WHERE ($1::timestamptz IS NULL OR p.created_at < $1::timestamptz)
|
||||
AND NOT EXISTS (SELECT 1 FROM profile_photos pp WHERE pp.photo_id = p.id AND pp.active)
|
||||
AND EXISTS (
|
||||
SELECT 1 FROM file_blobs fb
|
||||
|
|
@ -1224,8 +1294,9 @@ type ListPhotoIDsForHardRetentionOlderThanParams struct {
|
|||
}
|
||||
|
||||
// See ListDocumentIDsForHardRetentionOlderThan -- same "hard" retention
|
||||
// candidate selection, for photos. Excludes photos currently active as
|
||||
// someone's avatar -- see ListAvatarPhotoIDsForHardRetentionOlderThan.
|
||||
// candidate selection (including the nullable cutoff), for photos. Excludes
|
||||
// photos currently active as someone's avatar -- see
|
||||
// ListAvatarPhotoIDsForHardRetentionOlderThan.
|
||||
func (q *Queries) ListPhotoIDsForHardRetentionOlderThan(ctx context.Context, arg ListPhotoIDsForHardRetentionOlderThanParams) ([]int64, error) {
|
||||
rows, err := q.db.Query(ctx, listPhotoIDsForHardRetentionOlderThan, arg.Cutoff, arg.BatchLimit)
|
||||
if err != nil {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue