fix for retention logic
This commit is contained in:
parent
70c0ba44f0
commit
e6bfe2d444
35 changed files with 2108 additions and 132 deletions
|
|
@ -105,6 +105,30 @@ RETURNING media_kind, media_id`, string(refKind), refKey)
|
|||
return nil
|
||||
}
|
||||
|
||||
// ListMediaReferences returns every live reference row for (kind, mediaID) --
|
||||
// used by the storage retention sweep to turn a hard-retention/eviction blob
|
||||
// purge into a visible notice on every message that still embeds the purged
|
||||
// media. See internal/app/files.notifyRetentionPurge.
|
||||
func (s *MediaStore) ListMediaReferences(ctx context.Context, kind domain.MediaKind, mediaID int64) ([]domain.MediaReference, error) {
|
||||
rows, err := s.q.ListMediaReferences(ctx, sqlcgen.ListMediaReferencesParams{
|
||||
MediaKind: string(kind),
|
||||
MediaID: mediaID,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list media references: %w", err)
|
||||
}
|
||||
out := make([]domain.MediaReference, 0, len(rows))
|
||||
for _, r := range rows {
|
||||
out = append(out, domain.MediaReference{
|
||||
Kind: domain.MediaKind(r.MediaKind),
|
||||
MediaID: r.MediaID,
|
||||
RefKind: domain.MediaRefKind(r.RefKind),
|
||||
RefKey: r.RefKey,
|
||||
})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// ---- storage retention sweep ----
|
||||
|
||||
// OrphanDocumentIfUnreferenced marks a document orphaned right now if
|
||||
|
|
@ -128,20 +152,22 @@ WHERE id = $1
|
|||
return tag.RowsAffected() > 0, nil
|
||||
}
|
||||
|
||||
// ListOrphanedDocumentIDsOlderThan returns document ids whose orphaned_at is
|
||||
// set and older than cutoff, oldest first, up to limit.
|
||||
func (s *MediaStore) ListOrphanedDocumentIDsOlderThan(ctx context.Context, cutoff time.Time, limit int) ([]int64, error) {
|
||||
// ListOrphanedDocumentIDsOlderThan returns document ids in the given category
|
||||
// whose orphaned_at is set and older than cutoff, oldest first, up to limit.
|
||||
func (s *MediaStore) ListOrphanedDocumentIDsOlderThan(ctx context.Context, category domain.MediaCategory, cutoff time.Time, limit int) ([]int64, error) {
|
||||
if limit <= 0 {
|
||||
return nil, nil
|
||||
}
|
||||
return s.q.ListOrphanedDocumentIDsOlderThan(ctx, sqlcgen.ListOrphanedDocumentIDsOlderThanParams{
|
||||
Cutoff: pgtype.Timestamptz{Time: cutoff, Valid: true},
|
||||
Category: int16(category),
|
||||
BatchLimit: int32(limit),
|
||||
})
|
||||
}
|
||||
|
||||
// ListOrphanedPhotoIDsOlderThan returns photo ids whose orphaned_at is set
|
||||
// and older than cutoff, oldest first, up to limit.
|
||||
// ListOrphanedPhotoIDsOlderThan returns photo ids (excluding a live avatar --
|
||||
// see ListAvatarOrphanedPhotoIDsOlderThan) whose orphaned_at is set and older
|
||||
// than cutoff, oldest first, up to limit.
|
||||
func (s *MediaStore) ListOrphanedPhotoIDsOlderThan(ctx context.Context, cutoff time.Time, limit int) ([]int64, error) {
|
||||
if limit <= 0 {
|
||||
return nil, nil
|
||||
|
|
@ -152,6 +178,19 @@ func (s *MediaStore) ListOrphanedPhotoIDsOlderThan(ctx context.Context, cutoff t
|
|||
})
|
||||
}
|
||||
|
||||
// ListAvatarOrphanedPhotoIDsOlderThan is the "Avatar" category counterpart of
|
||||
// ListOrphanedPhotoIDsOlderThan -- see the query's doc comment for why this
|
||||
// is expected to stay empty in practice under "orphan" mode.
|
||||
func (s *MediaStore) ListAvatarOrphanedPhotoIDsOlderThan(ctx context.Context, cutoff time.Time, limit int) ([]int64, error) {
|
||||
if limit <= 0 {
|
||||
return nil, nil
|
||||
}
|
||||
return s.q.ListAvatarOrphanedPhotoIDsOlderThan(ctx, sqlcgen.ListAvatarOrphanedPhotoIDsOlderThanParams{
|
||||
Cutoff: pgtype.Timestamptz{Time: cutoff, Valid: true},
|
||||
BatchLimit: int32(limit),
|
||||
})
|
||||
}
|
||||
|
||||
// CountFileBlobRefs reports how many file_blobs rows still point at
|
||||
// (backend, objectKey) -- the caller must not physically delete the object
|
||||
// from that backend while this is > 0 (content-addressed storage: the same
|
||||
|
|
@ -197,24 +236,26 @@ func (s *MediaStore) DeleteDocumentAndBlobs(ctx context.Context, id int64) ([]do
|
|||
return blobs, nil
|
||||
}
|
||||
|
||||
// ListDocumentIDsForHardRetentionOlderThan returns document ids older than
|
||||
// cutoff (by upload/created_at) that still own at least one file_blobs row,
|
||||
// oldest first, up to limit -- the "hard" retention sweep's candidate list.
|
||||
// Unlike ListOrphanedDocumentIDsOlderThan, this ignores media_references
|
||||
// entirely: a document still referenced by a live message is exactly as
|
||||
// eligible as an orphaned one.
|
||||
func (s *MediaStore) ListDocumentIDsForHardRetentionOlderThan(ctx context.Context, cutoff time.Time, limit int) ([]int64, error) {
|
||||
// ListDocumentIDsForHardRetentionOlderThan returns document ids in the given
|
||||
// category older than cutoff (by upload/created_at) that still own at least
|
||||
// one file_blobs row, oldest first, up to limit -- the "hard" retention
|
||||
// sweep's candidate list. Unlike ListOrphanedDocumentIDsOlderThan, this
|
||||
// ignores media_references entirely: a document still referenced by a live
|
||||
// message is exactly as eligible as an orphaned one.
|
||||
func (s *MediaStore) ListDocumentIDsForHardRetentionOlderThan(ctx context.Context, category domain.MediaCategory, cutoff time.Time, limit int) ([]int64, error) {
|
||||
if limit <= 0 {
|
||||
return nil, nil
|
||||
}
|
||||
return s.q.ListDocumentIDsForHardRetentionOlderThan(ctx, sqlcgen.ListDocumentIDsForHardRetentionOlderThanParams{
|
||||
Cutoff: pgtype.Timestamptz{Time: cutoff, Valid: true},
|
||||
Category: int16(category),
|
||||
BatchLimit: int32(limit),
|
||||
})
|
||||
}
|
||||
|
||||
// ListPhotoIDsForHardRetentionOlderThan is the photo counterpart of
|
||||
// ListDocumentIDsForHardRetentionOlderThan.
|
||||
// ListDocumentIDsForHardRetentionOlderThan (excluding a live avatar -- see
|
||||
// ListAvatarPhotoIDsForHardRetentionOlderThan).
|
||||
func (s *MediaStore) ListPhotoIDsForHardRetentionOlderThan(ctx context.Context, cutoff time.Time, limit int) ([]int64, error) {
|
||||
if limit <= 0 {
|
||||
return nil, nil
|
||||
|
|
@ -225,6 +266,18 @@ func (s *MediaStore) ListPhotoIDsForHardRetentionOlderThan(ctx context.Context,
|
|||
})
|
||||
}
|
||||
|
||||
// ListAvatarPhotoIDsForHardRetentionOlderThan is the "Avatar" category
|
||||
// counterpart of ListPhotoIDsForHardRetentionOlderThan.
|
||||
func (s *MediaStore) ListAvatarPhotoIDsForHardRetentionOlderThan(ctx context.Context, cutoff time.Time, limit int) ([]int64, error) {
|
||||
if limit <= 0 {
|
||||
return nil, nil
|
||||
}
|
||||
return s.q.ListAvatarPhotoIDsForHardRetentionOlderThan(ctx, sqlcgen.ListAvatarPhotoIDsForHardRetentionOlderThanParams{
|
||||
Cutoff: pgtype.Timestamptz{Time: cutoff, Valid: true},
|
||||
BatchLimit: int32(limit),
|
||||
})
|
||||
}
|
||||
|
||||
// DeleteFileBlobsForDocument deletes every file_blobs row a document owns
|
||||
// (main body + thumbnail variants), returning what was deleted so the caller
|
||||
// can physically remove each object from its backend once confirming (via
|
||||
|
|
@ -292,6 +345,32 @@ func (s *MediaStore) DeleteFileBlobsForPhoto(ctx context.Context, id int64) ([]d
|
|||
return blobs, nil
|
||||
}
|
||||
|
||||
// ListOldestMediaForEviction returns up to limit documents and limit photos
|
||||
// still owning file_blobs bytes, oldest-uploaded first, for the active
|
||||
// eviction sweep to interleave by actual created_at (not drain one table
|
||||
// before the other) -- see domain.EvictionCandidate.
|
||||
func (s *MediaStore) ListOldestMediaForEviction(ctx context.Context, limit int) ([]domain.EvictionCandidate, error) {
|
||||
if limit <= 0 {
|
||||
return nil, nil
|
||||
}
|
||||
docs, err := s.q.ListOldestDocumentsForEviction(ctx, int32(limit))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list oldest documents for eviction: %w", err)
|
||||
}
|
||||
photos, err := s.q.ListOldestPhotosForEviction(ctx, int32(limit))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list oldest photos for eviction: %w", err)
|
||||
}
|
||||
out := make([]domain.EvictionCandidate, 0, len(docs)+len(photos))
|
||||
for _, d := range docs {
|
||||
out = append(out, domain.EvictionCandidate{Kind: domain.MediaKindDocument, MediaID: d.ID, CreatedAt: d.CreatedAt.Time})
|
||||
}
|
||||
for _, p := range photos {
|
||||
out = append(out, domain.EvictionCandidate{Kind: domain.MediaKindPhoto, MediaID: p.ID, CreatedAt: p.CreatedAt.Time})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// DeletePhotoAndBlobs deletes a photo row and every file_blobs row it owns
|
||||
// (one per rendition size), returning what was deleted so the caller can
|
||||
// physically remove each object from its backend once confirming (via
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue