fix for retention logic
This commit is contained in:
onysd
parent
70c0ba44f0
commit
e6bfe2d444
35 changed files with 2108 additions and 132 deletions
|
|
@ -12,7 +12,12 @@ import (
|
|||
)
|
||||
|
||||
func (s *ChannelStore) EditChannelMessage(ctx context.Context, req domain.EditChannelMessageRequest) (domain.EditChannelMessageResult, error) {
|
||||
if req.UserID == 0 || req.ChannelID == 0 || req.ID <= 0 {
|
||||
retentionPurge := req.RetentionPurge && req.RetentionPurgeAction != nil
|
||||
// RetentionPurge is a server-internal edit with no acting user (the
|
||||
// storage retention sweep, not an RPC caller) -- UserID==0 is normally
|
||||
// invalid, and req.UserID would otherwise also need to name an existing
|
||||
// channel member for getChannelForMember below.
|
||||
if (req.UserID == 0 && !retentionPurge) || req.ChannelID == 0 || req.ID <= 0 {
|
||||
return domain.EditChannelMessageResult{}, domain.ErrChannelInvalid
|
||||
}
|
||||
beginner, ok := s.db.(txBeginner)
|
||||
|
|
@ -44,7 +49,17 @@ func (s *ChannelStore) EditChannelMessage(ctx context.Context, req domain.EditCh
|
|||
_ = tx.Rollback(ctx)
|
||||
}
|
||||
}()
|
||||
channel, member, err := s.getChannelForMember(ctx, tx, req.UserID, req.ChannelID)
|
||||
var (
|
||||
channel domain.Channel
|
||||
member domain.ChannelMember
|
||||
)
|
||||
if retentionPurge {
|
||||
// No membership requirement: this is a server-internal edit, not an
|
||||
// action taken by any particular channel member.
|
||||
channel, err = s.channelByID(ctx, tx, req.ChannelID)
|
||||
} else {
|
||||
channel, member, err = s.getChannelForMember(ctx, tx, req.UserID, req.ChannelID)
|
||||
}
|
||||
if err != nil {
|
||||
return domain.EditChannelMessageResult{}, err
|
||||
}
|
||||
|
|
@ -52,6 +67,17 @@ func (s *ChannelStore) EditChannelMessage(ctx context.Context, req domain.EditCh
|
|||
if err != nil {
|
||||
return domain.EditChannelMessageResult{}, err
|
||||
}
|
||||
if retentionPurge {
|
||||
if msg.Deleted {
|
||||
return domain.EditChannelMessageResult{}, domain.ErrMessageIDInvalid
|
||||
}
|
||||
if msg.Action != nil {
|
||||
// 幂等守卫：已经是服务消息(含已被本路径转换过的 retention 通知)
|
||||
// 不再重复编辑 -- 见 EditMessageRequest.RetentionPurge 同款守卫。
|
||||
return domain.EditChannelMessageResult{}, domain.ErrMessageNotModified
|
||||
}
|
||||
return s.applyRetentionPurgeChannelMessage(ctx, tx, channel, msg, req)
|
||||
}
|
||||
if msg.Deleted || msg.Action != nil {
|
||||
return domain.EditChannelMessageResult{}, domain.ErrMessageIDInvalid
|
||||
}
|
||||
|
|
@ -283,6 +309,74 @@ WHERE channel_id = $1 AND id = $2`, req.ChannelID, req.ID, mediaJSON); err != ni
|
|||
return domain.EditChannelMessageResult{Channel: channel, Message: msg, Event: event, ServiceMessage: serviceMsg, ServiceEvent: serviceEvent, Recipients: recipients}, nil
|
||||
}
|
||||
|
||||
// applyRetentionPurgeChannelMessage turns msg into a real service message
|
||||
// (messageActionCustomAction via req.RetentionPurgeAction) in place. Unlike a
|
||||
// normal channel edit -- which only ever replaces body/entities/media --
|
||||
// channel service actions live in a structurally separate Action column
|
||||
// (see tgChannelMessage: m.Action != nil renders tg.MessageService instead of
|
||||
// tg.Message), so this clears body/media and sets Action instead of touching
|
||||
// them. Reuses the same pts/durable-event/admin-log machinery as a normal
|
||||
// edit so getChannelDifference and the fanout dispatcher see it identically.
|
||||
func (s *ChannelStore) applyRetentionPurgeChannelMessage(ctx context.Context, tx pgx.Tx, channel domain.Channel, msg domain.ChannelMessage, req domain.EditChannelMessageRequest) (domain.EditChannelMessageResult, error) {
|
||||
actionJSON, err := marshalJSON(req.RetentionPurgeAction, "{}")
|
||||
if err != nil {
|
||||
return domain.EditChannelMessageResult{}, fmt.Errorf("encode retention purge channel action: %w", err)
|
||||
}
|
||||
pts, err := s.reserveChannelPts(ctx, tx, req.ChannelID)
|
||||
if err != nil {
|
||||
return domain.EditChannelMessageResult{}, fmt.Errorf("allocate retention purge channel pts: %w", err)
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `
|
||||
UPDATE channel_messages
|
||||
SET body = '', entities = '[]'::jsonb, media = '{}'::jsonb, action = $3, edit_date = $4, pts = $5, updated_at = now()
|
||||
WHERE channel_id = $1 AND id = $2`, req.ChannelID, req.ID, actionJSON, req.EditDate, pts); err != nil {
|
||||
return domain.EditChannelMessageResult{}, fmt.Errorf("update retention purge channel message: %w", err)
|
||||
}
|
||||
prevMsg := msg
|
||||
msg.Body = ""
|
||||
msg.Entities = nil
|
||||
msg.Media = nil
|
||||
msg.Action = req.RetentionPurgeAction
|
||||
msg.EditDate = req.EditDate
|
||||
msg.Pts = pts
|
||||
// 媒体索引/media_references 靠这次替换后的（空）媒体重建：原文档/照片
|
||||
// 不再被这条消息引用，storage retention 的孤儿判定据此推进。
|
||||
if err := replaceChannelMediaIndexTx(ctx, tx, req.ChannelID, req.ID, msg.Date, msg.Media, nil); err != nil {
|
||||
return domain.EditChannelMessageResult{}, err
|
||||
}
|
||||
event := domain.ChannelUpdateEvent{
|
||||
ChannelID: req.ChannelID,
|
||||
Type: domain.ChannelUpdateEditMessage,
|
||||
Pts: pts,
|
||||
PtsCount: 1,
|
||||
Date: req.EditDate,
|
||||
Message: msg,
|
||||
SenderUserID: req.UserID,
|
||||
}
|
||||
if err := insertChannelEventTx(ctx, tx, event); err != nil {
|
||||
return domain.EditChannelMessageResult{}, err
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `UPDATE channels SET pts = $2, updated_at = now() WHERE id = $1`, req.ChannelID, pts); err != nil {
|
||||
return domain.EditChannelMessageResult{}, fmt.Errorf("update retention purge channel pts: %w", err)
|
||||
}
|
||||
channel.Pts = pts
|
||||
if err := s.insertChannelAdminLogTx(ctx, tx, domain.ChannelAdminLogEvent{
|
||||
ChannelID: req.ChannelID,
|
||||
UserID: req.UserID,
|
||||
Date: req.EditDate,
|
||||
Type: domain.ChannelAdminLogEditMessage,
|
||||
PrevMessage: &prevMsg,
|
||||
NewMessage: &msg,
|
||||
}); err != nil {
|
||||
return domain.EditChannelMessageResult{}, err
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return domain.EditChannelMessageResult{}, fmt.Errorf("commit retention purge channel message: %w", err)
|
||||
}
|
||||
recipients, _ := s.ListActiveChannelMemberIDs(ctx, req.UserID, req.ChannelID, 0)
|
||||
return domain.EditChannelMessageResult{Channel: channel, Message: msg, Event: event, Recipients: recipients}, nil
|
||||
}
|
||||
|
||||
func isChannelTodoParticipantEdit(req domain.EditChannelMessageRequest, msg domain.ChannelMessage) bool {
|
||||
if !req.AllowTodoParticipantMutation || req.SetReplyMarkup || req.Media == nil || req.Media.Kind != domain.MessageMediaKindTodo || req.Media.Todo == nil {
|
||||
return false
|
||||
|
|
|
|||
|
|
@ -64,7 +64,7 @@ func TestHardRetentionPurgesBlobBytesButKeepsMetadataRow(t *testing.T) {
|
|||
})
|
||||
|
||||
cutoff := time.Now().Add(-24 * time.Hour)
|
||||
ids, err := media.ListDocumentIDsForHardRetentionOlderThan(ctx, cutoff, 1000)
|
||||
ids, err := media.ListDocumentIDsForHardRetentionOlderThan(ctx, domain.MediaCategoryNone, cutoff, 1000)
|
||||
if err != nil {
|
||||
t.Fatalf("ListDocumentIDsForHardRetentionOlderThan: %v", err)
|
||||
}
|
||||
|
|
@ -95,7 +95,7 @@ func TestHardRetentionPurgesBlobBytesButKeepsMetadataRow(t *testing.T) {
|
|||
|
||||
// Idempotent / self-terminating: a second sweep pass no longer selects
|
||||
// this document, since it no longer owns any file_blobs row.
|
||||
ids, err = media.ListDocumentIDsForHardRetentionOlderThan(ctx, cutoff, 1000)
|
||||
ids, err = media.ListDocumentIDsForHardRetentionOlderThan(ctx, domain.MediaCategoryNone, cutoff, 1000)
|
||||
if err != nil {
|
||||
t.Fatalf("ListDocumentIDsForHardRetentionOlderThan (2nd pass): %v", err)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -24,9 +24,30 @@ ON CONFLICT (channel_id, id, category) DO NOTHING`, channelID, id, int16(c), dat
|
|||
return fmt.Errorf("insert channel media index: %w", err)
|
||||
}
|
||||
}
|
||||
if err := setDocumentCategoryTx(ctx, tx, media); err != nil {
|
||||
return err
|
||||
}
|
||||
return addMediaReferencesTx(ctx, tx, media, domain.MediaRefKindChannelMessage, channelMessageRefKey(channelID, id))
|
||||
}
|
||||
|
||||
// setDocumentCategoryTx persists media.Document's retention-sweep category
|
||||
// (see domain.DocumentMediaCategory) onto its documents row, alongside the
|
||||
// message_box_media/channel_message_media index write these callers already
|
||||
// do -- zero new classification logic, just also stamping the already
|
||||
// computed value directly onto documents.category so the per-category
|
||||
// storage retention sweep (internal/app/files/retention.go) can filter on it
|
||||
// without a join. No-op for non-document media.
|
||||
func setDocumentCategoryTx(ctx context.Context, tx pgx.Tx, media *domain.MessageMedia) error {
|
||||
if media == nil || media.Kind != domain.MessageMediaKindDocument || media.Document == nil || media.Document.ID == 0 {
|
||||
return nil
|
||||
}
|
||||
category := domain.DocumentMediaCategory(media)
|
||||
if _, err := tx.Exec(ctx, `UPDATE documents SET category = $2 WHERE id = $1`, media.Document.ID, int16(category)); err != nil {
|
||||
return fmt.Errorf("set document category: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// deleteChannelMediaIndexTx 清掉一条频道消息的全部索引行(编辑改媒体前先清后插)。
|
||||
// 只在 replaceChannelMediaIndexTx 内被调用;真正的消息删除不清 *_media 分类索引
|
||||
// (读时靠 JOIN deleted 过滤,见文件头注释),但仍需在此清掉 media_references,
|
||||
|
|
@ -57,6 +78,9 @@ ON CONFLICT (owner_user_id, box_id, category) DO NOTHING`, ownerUserID, boxID, p
|
|||
return fmt.Errorf("insert message box media index: %w", err)
|
||||
}
|
||||
}
|
||||
if err := setDocumentCategoryTx(ctx, tx, media); err != nil {
|
||||
return err
|
||||
}
|
||||
return addMediaReferencesTx(ctx, tx, media, domain.MediaRefKindMessageBox, messageBoxRefKey(ownerUserID, boxID))
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -105,6 +105,30 @@ RETURNING media_kind, media_id`, string(refKind), refKey)
|
|||
return nil
|
||||
}
|
||||
|
||||
// ListMediaReferences returns every live reference row for (kind, mediaID) --
|
||||
// used by the storage retention sweep to turn a hard-retention/eviction blob
|
||||
// purge into a visible notice on every message that still embeds the purged
|
||||
// media. See internal/app/files.notifyRetentionPurge.
|
||||
func (s *MediaStore) ListMediaReferences(ctx context.Context, kind domain.MediaKind, mediaID int64) ([]domain.MediaReference, error) {
|
||||
rows, err := s.q.ListMediaReferences(ctx, sqlcgen.ListMediaReferencesParams{
|
||||
MediaKind: string(kind),
|
||||
MediaID: mediaID,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list media references: %w", err)
|
||||
}
|
||||
out := make([]domain.MediaReference, 0, len(rows))
|
||||
for _, r := range rows {
|
||||
out = append(out, domain.MediaReference{
|
||||
Kind: domain.MediaKind(r.MediaKind),
|
||||
MediaID: r.MediaID,
|
||||
RefKind: domain.MediaRefKind(r.RefKind),
|
||||
RefKey: r.RefKey,
|
||||
})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// ---- storage retention sweep ----
|
||||
|
||||
// OrphanDocumentIfUnreferenced marks a document orphaned right now if
|
||||
|
|
@ -128,20 +152,22 @@ WHERE id = $1
|
|||
return tag.RowsAffected() > 0, nil
|
||||
}
|
||||
|
||||
// ListOrphanedDocumentIDsOlderThan returns document ids whose orphaned_at is
|
||||
// set and older than cutoff, oldest first, up to limit.
|
||||
func (s *MediaStore) ListOrphanedDocumentIDsOlderThan(ctx context.Context, cutoff time.Time, limit int) ([]int64, error) {
|
||||
// ListOrphanedDocumentIDsOlderThan returns document ids in the given category
|
||||
// whose orphaned_at is set and older than cutoff, oldest first, up to limit.
|
||||
func (s *MediaStore) ListOrphanedDocumentIDsOlderThan(ctx context.Context, category domain.MediaCategory, cutoff time.Time, limit int) ([]int64, error) {
|
||||
if limit <= 0 {
|
||||
return nil, nil
|
||||
}
|
||||
return s.q.ListOrphanedDocumentIDsOlderThan(ctx, sqlcgen.ListOrphanedDocumentIDsOlderThanParams{
|
||||
Cutoff: pgtype.Timestamptz{Time: cutoff, Valid: true},
|
||||
Category: int16(category),
|
||||
BatchLimit: int32(limit),
|
||||
})
|
||||
}
|
||||
|
||||
// ListOrphanedPhotoIDsOlderThan returns photo ids whose orphaned_at is set
|
||||
// and older than cutoff, oldest first, up to limit.
|
||||
// ListOrphanedPhotoIDsOlderThan returns photo ids (excluding a live avatar --
|
||||
// see ListAvatarOrphanedPhotoIDsOlderThan) whose orphaned_at is set and older
|
||||
// than cutoff, oldest first, up to limit.
|
||||
func (s *MediaStore) ListOrphanedPhotoIDsOlderThan(ctx context.Context, cutoff time.Time, limit int) ([]int64, error) {
|
||||
if limit <= 0 {
|
||||
return nil, nil
|
||||
|
|
@ -152,6 +178,19 @@ func (s *MediaStore) ListOrphanedPhotoIDsOlderThan(ctx context.Context, cutoff t
|
|||
})
|
||||
}
|
||||
|
||||
// ListAvatarOrphanedPhotoIDsOlderThan is the "Avatar" category counterpart of
|
||||
// ListOrphanedPhotoIDsOlderThan -- see the query's doc comment for why this
|
||||
// is expected to stay empty in practice under "orphan" mode.
|
||||
func (s *MediaStore) ListAvatarOrphanedPhotoIDsOlderThan(ctx context.Context, cutoff time.Time, limit int) ([]int64, error) {
|
||||
if limit <= 0 {
|
||||
return nil, nil
|
||||
}
|
||||
return s.q.ListAvatarOrphanedPhotoIDsOlderThan(ctx, sqlcgen.ListAvatarOrphanedPhotoIDsOlderThanParams{
|
||||
Cutoff: pgtype.Timestamptz{Time: cutoff, Valid: true},
|
||||
BatchLimit: int32(limit),
|
||||
})
|
||||
}
|
||||
|
||||
// CountFileBlobRefs reports how many file_blobs rows still point at
|
||||
// (backend, objectKey) -- the caller must not physically delete the object
|
||||
// from that backend while this is > 0 (content-addressed storage: the same
|
||||
|
|
@ -197,24 +236,26 @@ func (s *MediaStore) DeleteDocumentAndBlobs(ctx context.Context, id int64) ([]do
|
|||
return blobs, nil
|
||||
}
|
||||
|
||||
// ListDocumentIDsForHardRetentionOlderThan returns document ids older than
|
||||
// cutoff (by upload/created_at) that still own at least one file_blobs row,
|
||||
// oldest first, up to limit -- the "hard" retention sweep's candidate list.
|
||||
// Unlike ListOrphanedDocumentIDsOlderThan, this ignores media_references
|
||||
// entirely: a document still referenced by a live message is exactly as
|
||||
// eligible as an orphaned one.
|
||||
func (s *MediaStore) ListDocumentIDsForHardRetentionOlderThan(ctx context.Context, cutoff time.Time, limit int) ([]int64, error) {
|
||||
// ListDocumentIDsForHardRetentionOlderThan returns document ids in the given
|
||||
// category older than cutoff (by upload/created_at) that still own at least
|
||||
// one file_blobs row, oldest first, up to limit -- the "hard" retention
|
||||
// sweep's candidate list. Unlike ListOrphanedDocumentIDsOlderThan, this
|
||||
// ignores media_references entirely: a document still referenced by a live
|
||||
// message is exactly as eligible as an orphaned one.
|
||||
func (s *MediaStore) ListDocumentIDsForHardRetentionOlderThan(ctx context.Context, category domain.MediaCategory, cutoff time.Time, limit int) ([]int64, error) {
|
||||
if limit <= 0 {
|
||||
return nil, nil
|
||||
}
|
||||
return s.q.ListDocumentIDsForHardRetentionOlderThan(ctx, sqlcgen.ListDocumentIDsForHardRetentionOlderThanParams{
|
||||
Cutoff: pgtype.Timestamptz{Time: cutoff, Valid: true},
|
||||
Category: int16(category),
|
||||
BatchLimit: int32(limit),
|
||||
})
|
||||
}
|
||||
|
||||
// ListPhotoIDsForHardRetentionOlderThan is the photo counterpart of
|
||||
// ListDocumentIDsForHardRetentionOlderThan.
|
||||
// ListDocumentIDsForHardRetentionOlderThan (excluding a live avatar -- see
|
||||
// ListAvatarPhotoIDsForHardRetentionOlderThan).
|
||||
func (s *MediaStore) ListPhotoIDsForHardRetentionOlderThan(ctx context.Context, cutoff time.Time, limit int) ([]int64, error) {
|
||||
if limit <= 0 {
|
||||
return nil, nil
|
||||
|
|
@ -225,6 +266,18 @@ func (s *MediaStore) ListPhotoIDsForHardRetentionOlderThan(ctx context.Context,
|
|||
})
|
||||
}
|
||||
|
||||
// ListAvatarPhotoIDsForHardRetentionOlderThan is the "Avatar" category
|
||||
// counterpart of ListPhotoIDsForHardRetentionOlderThan.
|
||||
func (s *MediaStore) ListAvatarPhotoIDsForHardRetentionOlderThan(ctx context.Context, cutoff time.Time, limit int) ([]int64, error) {
|
||||
if limit <= 0 {
|
||||
return nil, nil
|
||||
}
|
||||
return s.q.ListAvatarPhotoIDsForHardRetentionOlderThan(ctx, sqlcgen.ListAvatarPhotoIDsForHardRetentionOlderThanParams{
|
||||
Cutoff: pgtype.Timestamptz{Time: cutoff, Valid: true},
|
||||
BatchLimit: int32(limit),
|
||||
})
|
||||
}
|
||||
|
||||
// DeleteFileBlobsForDocument deletes every file_blobs row a document owns
|
||||
// (main body + thumbnail variants), returning what was deleted so the caller
|
||||
// can physically remove each object from its backend once confirming (via
|
||||
|
|
@ -292,6 +345,32 @@ func (s *MediaStore) DeleteFileBlobsForPhoto(ctx context.Context, id int64) ([]d
|
|||
return blobs, nil
|
||||
}
|
||||
|
||||
// ListOldestMediaForEviction returns up to limit documents and limit photos
|
||||
// still owning file_blobs bytes, oldest-uploaded first, for the active
|
||||
// eviction sweep to interleave by actual created_at (not drain one table
|
||||
// before the other) -- see domain.EvictionCandidate.
|
||||
func (s *MediaStore) ListOldestMediaForEviction(ctx context.Context, limit int) ([]domain.EvictionCandidate, error) {
|
||||
if limit <= 0 {
|
||||
return nil, nil
|
||||
}
|
||||
docs, err := s.q.ListOldestDocumentsForEviction(ctx, int32(limit))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list oldest documents for eviction: %w", err)
|
||||
}
|
||||
photos, err := s.q.ListOldestPhotosForEviction(ctx, int32(limit))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list oldest photos for eviction: %w", err)
|
||||
}
|
||||
out := make([]domain.EvictionCandidate, 0, len(docs)+len(photos))
|
||||
for _, d := range docs {
|
||||
out = append(out, domain.EvictionCandidate{Kind: domain.MediaKindDocument, MediaID: d.ID, CreatedAt: d.CreatedAt.Time})
|
||||
}
|
||||
for _, p := range photos {
|
||||
out = append(out, domain.EvictionCandidate{Kind: domain.MediaKindPhoto, MediaID: p.ID, CreatedAt: p.CreatedAt.Time})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// DeletePhotoAndBlobs deletes a photo row and every file_blobs row it owns
|
||||
// (one per rendition size), returning what was deleted so the caller can
|
||||
// physically remove each object from its backend once confirming (via
|
||||
|
|
|
|||
|
|
@ -83,9 +83,18 @@ func (s *MessageStore) EditMessage(ctx context.Context, req domain.EditMessageRe
|
|||
}
|
||||
authorEdit := target.Outgoing && target.MessageSenderID == req.OwnerUserID && target.FromUserID == req.OwnerUserID
|
||||
viaBotEdit := req.ViaBotEditBotID != 0 && target.ViaBotID == req.ViaBotEditBotID
|
||||
if !authorEdit && !viaBotEdit && !req.WebPageResolve && !validTodoParticipantEdit(req, target, oldEntities) {
|
||||
if !authorEdit && !viaBotEdit && !req.WebPageResolve && !req.RetentionPurge && !validTodoParticipantEdit(req, target, oldEntities) {
|
||||
return res, domain.ErrMessageAuthorRequired
|
||||
}
|
||||
if req.RetentionPurge {
|
||||
// 幂等守卫：已经是这条 retention 通知的消息不再重复编辑（同一被回收
|
||||
// 媒体可能被多个 box 的 media_references 行各引用一次，比如自己与对端
|
||||
// 各自的 box——同一条共享 private_message 只需真正编辑一次）。
|
||||
if targetMedia, err := decodeMessageMedia(target.MediaJson); err == nil &&
|
||||
targetMedia != nil && targetMedia.Kind == domain.MessageMediaKindService {
|
||||
return res, domain.ErrMessageNotModified
|
||||
}
|
||||
}
|
||||
richChanged := req.SetRichMessage && !richMessagesEqual(targetRich, req.RichMessage)
|
||||
if req.Media == nil && !req.SetReplyMarkup && !richChanged && target.Body == req.Message && target.HideEdited == req.HideEdited && sameMessageEntities(oldEntities, req.Entities) {
|
||||
return res, domain.ErrMessageNotModified
|
||||
|
|
|
|||
|
|
@ -202,6 +202,16 @@ WHERE id = sqlc.arg(media_id)::bigint AND orphaned_at IS NOT NULL;
|
|||
UPDATE photos SET orphaned_at = NULL
|
||||
WHERE id = sqlc.arg(media_id)::bigint AND orphaned_at IS NOT NULL;
|
||||
|
||||
-- name: ListMediaReferences :many
|
||||
-- Every live reference to a document/photo -- used by the storage retention
|
||||
-- sweep to turn a hard-retention/eviction blob purge into a visible notice on
|
||||
-- every message that still embeds the purged media (see
|
||||
-- files.notifyRetentionPurge). profile_photo/sticker_set/gift refs have no
|
||||
-- message to edit and are filtered by the caller, not this query.
|
||||
SELECT media_kind, media_id, ref_kind, ref_key
|
||||
FROM media_references
|
||||
WHERE media_kind = sqlc.arg(media_kind)::text AND media_id = sqlc.arg(media_id)::bigint;
|
||||
|
||||
-- name: RemoveMediaReference :exec
|
||||
DELETE FROM media_references
|
||||
WHERE media_kind = sqlc.arg(media_kind)::text
|
||||
|
|
@ -226,15 +236,36 @@ WHERE id = sqlc.arg(media_id)::bigint
|
|||
);
|
||||
|
||||
-- name: ListOrphanedDocumentIDsOlderThan :many
|
||||
-- category selects one documents.category bucket per sweep tick (per-category
|
||||
-- retention age, see internal/app/files/retention.go) -- 0 (MediaCategoryNone)
|
||||
-- covers unclassified documents (e.g. stickers), which always use the shared
|
||||
-- global age since there is no per-category override for that bucket.
|
||||
SELECT id FROM documents
|
||||
WHERE orphaned_at IS NOT NULL AND orphaned_at < sqlc.arg(cutoff)::timestamptz
|
||||
AND category = sqlc.arg(category)::smallint
|
||||
ORDER BY orphaned_at ASC
|
||||
LIMIT sqlc.arg(batch_limit)::int;
|
||||
|
||||
-- name: ListOrphanedPhotoIDsOlderThan :many
|
||||
SELECT id FROM photos
|
||||
WHERE orphaned_at IS NOT NULL AND orphaned_at < sqlc.arg(cutoff)::timestamptz
|
||||
ORDER BY orphaned_at ASC
|
||||
-- Excludes photos currently active as someone's avatar -- see
|
||||
-- ListAvatarOrphanedPhotoIDsOlderThan for that split-off bucket.
|
||||
SELECT p.id FROM photos p
|
||||
WHERE p.orphaned_at IS NOT NULL AND p.orphaned_at < sqlc.arg(cutoff)::timestamptz
|
||||
AND NOT EXISTS (SELECT 1 FROM profile_photos pp WHERE pp.photo_id = p.id AND pp.active)
|
||||
ORDER BY p.orphaned_at ASC
|
||||
LIMIT sqlc.arg(batch_limit)::int;
|
||||
|
||||
-- name: ListAvatarOrphanedPhotoIDsOlderThan :many
|
||||
-- Same as ListOrphanedPhotoIDsOlderThan but only photos currently active as
|
||||
-- someone's avatar (profile_photos.active) -- lets the Avatar category carry
|
||||
-- its own retention age. In practice a live avatar is never orphaned (an
|
||||
-- active profile_photos row is itself a media_references entry), so this
|
||||
-- bucket is expected to stay empty under "orphan" mode; kept for symmetry
|
||||
-- with the "hard" mode avatar split, which is the one that actually matters.
|
||||
SELECT p.id FROM photos p
|
||||
WHERE p.orphaned_at IS NOT NULL AND p.orphaned_at < sqlc.arg(cutoff)::timestamptz
|
||||
AND EXISTS (SELECT 1 FROM profile_photos pp WHERE pp.photo_id = p.id AND pp.active)
|
||||
ORDER BY p.orphaned_at ASC
|
||||
LIMIT sqlc.arg(batch_limit)::int;
|
||||
|
||||
-- name: ListDocumentIDsForHardRetentionOlderThan :many
|
||||
|
|
@ -244,9 +275,12 @@ LIMIT sqlc.arg(batch_limit)::int;
|
|||
-- keeps this sweep from re-selecting the same document forever: once its
|
||||
-- blob bytes are purged (DeleteFileBlobsForDocument removes the file_blobs
|
||||
-- rows but deliberately leaves this documents row in place), it naturally
|
||||
-- drops out of this query on the next pass.
|
||||
-- drops out of this query on the next pass. category scopes to one
|
||||
-- documents.category bucket per sweep tick -- see
|
||||
-- ListOrphanedDocumentIDsOlderThan for the 0/None fallback note.
|
||||
SELECT d.id FROM documents d
|
||||
WHERE d.created_at < sqlc.arg(cutoff)::timestamptz
|
||||
AND d.category = sqlc.arg(category)::smallint
|
||||
AND EXISTS (
|
||||
SELECT 1 FROM file_blobs fb
|
||||
WHERE fb.location_key = 'doc:' || d.id::text
|
||||
|
|
@ -257,9 +291,11 @@ LIMIT sqlc.arg(batch_limit)::int;
|
|||
|
||||
-- name: ListPhotoIDsForHardRetentionOlderThan :many
|
||||
-- See ListDocumentIDsForHardRetentionOlderThan -- same "hard" retention
|
||||
-- candidate selection, for photos.
|
||||
-- candidate selection, for photos. Excludes photos currently active as
|
||||
-- someone's avatar -- see ListAvatarPhotoIDsForHardRetentionOlderThan.
|
||||
SELECT p.id FROM photos p
|
||||
WHERE p.created_at < sqlc.arg(cutoff)::timestamptz
|
||||
AND NOT EXISTS (SELECT 1 FROM profile_photos pp WHERE pp.photo_id = p.id AND pp.active)
|
||||
AND EXISTS (
|
||||
SELECT 1 FROM file_blobs fb
|
||||
WHERE fb.location_key = 'photo:' || p.id::text
|
||||
|
|
@ -268,6 +304,52 @@ WHERE p.created_at < sqlc.arg(cutoff)::timestamptz
|
|||
ORDER BY p.created_at ASC
|
||||
LIMIT sqlc.arg(batch_limit)::int;
|
||||
|
||||
-- name: ListAvatarPhotoIDsForHardRetentionOlderThan :many
|
||||
-- Same as ListPhotoIDsForHardRetentionOlderThan but only photos currently
|
||||
-- active as someone's avatar (profile_photos.active) -- lets the Avatar
|
||||
-- category carry its own retention age, independent of ordinary shared-media
|
||||
-- photos.
|
||||
SELECT p.id FROM photos p
|
||||
WHERE p.created_at < sqlc.arg(cutoff)::timestamptz
|
||||
AND EXISTS (SELECT 1 FROM profile_photos pp WHERE pp.photo_id = p.id AND pp.active)
|
||||
AND EXISTS (
|
||||
SELECT 1 FROM file_blobs fb
|
||||
WHERE fb.location_key = 'photo:' || p.id::text
|
||||
OR fb.location_key LIKE 'photo:' || p.id::text || ':%'
|
||||
)
|
||||
ORDER BY p.created_at ASC
|
||||
LIMIT sqlc.arg(batch_limit)::int;
|
||||
|
||||
-- name: ListOldestDocumentsForEviction :many
|
||||
-- Active eviction (TELESRV_STORAGE_EVICTION_ENABLE): candidates are every
|
||||
-- document that still owns at least one file_blobs row, oldest-uploaded
|
||||
-- first, REGARDLESS of category or age -- unlike the retention sweeps above,
|
||||
-- eviction only cares about reclaiming bytes once the total physical budget
|
||||
-- (TELESRV_STORAGE_MAX_TOTAL_BYTES) is exceeded. created_at is returned so
|
||||
-- the caller can interleave these with ListOldestPhotosForEviction by actual
|
||||
-- age instead of draining one table before touching the other.
|
||||
SELECT d.id, d.created_at FROM documents d
|
||||
WHERE EXISTS (
|
||||
SELECT 1 FROM file_blobs fb
|
||||
WHERE fb.location_key = 'doc:' || d.id::text
|
||||
OR fb.location_key LIKE 'doc:' || d.id::text || ':%'
|
||||
)
|
||||
ORDER BY d.created_at ASC
|
||||
LIMIT sqlc.arg(batch_limit)::int;
|
||||
|
||||
-- name: ListOldestPhotosForEviction :many
|
||||
-- See ListOldestDocumentsForEviction -- same oldest-first eviction candidate
|
||||
-- selection, for photos (avatars included: eviction is bytes-only and does
|
||||
-- not honor the Avatar category's separate retention age).
|
||||
SELECT p.id, p.created_at FROM photos p
|
||||
WHERE EXISTS (
|
||||
SELECT 1 FROM file_blobs fb
|
||||
WHERE fb.location_key = 'photo:' || p.id::text
|
||||
OR fb.location_key LIKE 'photo:' || p.id::text || ':%'
|
||||
)
|
||||
ORDER BY p.created_at ASC
|
||||
LIMIT sqlc.arg(batch_limit)::int;
|
||||
|
||||
-- name: CountFileBlobRefs :one
|
||||
SELECT COUNT(*)::int FROM file_blobs WHERE backend = sqlc.arg(backend)::text AND object_key = sqlc.arg(object_key)::text;
|
||||
|
||||
|
|
|
|||
360
internal/store/postgres/retention_v2_integration_test.go
Normal file
360
internal/store/postgres/retention_v2_integration_test.go
Normal file
|
|
@ -0,0 +1,360 @@
|
|||
package postgres
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strconv"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"telesrv/internal/domain"
|
||||
)
|
||||
|
||||
// TestRetentionPurgeBypassProducesServiceMessage_PrivateMessage exercises
|
||||
// Part 1 of the storage retention v2 plan end-to-end against a real
|
||||
// Postgres: EditMessage with RetentionPurge=true bypasses the ordinary
|
||||
// author check (the storage retention sweep is not the message's author) and
|
||||
// replaces the message media with a messageActionCustomAction service
|
||||
// payload, and a second retention-purge edit on the now-already-service
|
||||
// message is a no-op (ErrMessageNotModified) -- guarding the idempotency
|
||||
// guard that keeps a shared media_references duplicate (owner+peer both
|
||||
// referencing the same purged document) from double-editing the same box.
|
||||
func TestRetentionPurgeBypassProducesServiceMessage_PrivateMessage(t *testing.T) {
|
||||
pool := testPool(t)
|
||||
ctx := context.Background()
|
||||
suffix := randomSuffix(t)
|
||||
|
||||
users := NewUserStore(pool)
|
||||
sender, err := users.Create(ctx, domain.User{AccessHash: 81, Phone: "+1997" + suffix + "01", FirstName: "RetentionSender"})
|
||||
if err != nil {
|
||||
t.Fatalf("create sender: %v", err)
|
||||
}
|
||||
recipient, err := users.Create(ctx, domain.User{AccessHash: 82, Phone: "+1997" + suffix + "02", FirstName: "RetentionRecipient"})
|
||||
if err != nil {
|
||||
t.Fatalf("create recipient: %v", err)
|
||||
}
|
||||
ids := []int64{sender.ID, recipient.ID}
|
||||
t.Cleanup(func() {
|
||||
_, _ = pool.Exec(ctx, "DELETE FROM message_boxes WHERE owner_user_id = ANY($1::bigint[])", ids)
|
||||
_, _ = pool.Exec(ctx, "DELETE FROM private_messages WHERE sender_user_id = ANY($1::bigint[])", ids)
|
||||
_, _ = pool.Exec(ctx, "DELETE FROM dialogs WHERE user_id = ANY($1::bigint[])", ids)
|
||||
_, _ = pool.Exec(ctx, "DELETE FROM users WHERE id = ANY($1::bigint[])", ids)
|
||||
})
|
||||
|
||||
messages := NewMessageStore(pool, WithMessageAllocators(&perUserCounterAllocator{}))
|
||||
sent, err := messages.SendPrivateText(ctx, domain.SendPrivateTextRequest{
|
||||
SenderUserID: sender.ID,
|
||||
RecipientUserID: recipient.ID,
|
||||
RandomID: time.Now().UnixNano(),
|
||||
Message: "a file worth keeping",
|
||||
Media: &domain.MessageMedia{
|
||||
Kind: domain.MessageMediaKindDocument,
|
||||
Document: &domain.Document{ID: time.Now().UnixNano(), AccessHash: 1, MimeType: "application/octet-stream"},
|
||||
},
|
||||
Date: int(time.Now().Unix()),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("send private media: %v", err)
|
||||
}
|
||||
|
||||
noticeMedia := &domain.MessageMedia{
|
||||
Kind: domain.MessageMediaKindService,
|
||||
ServiceAction: &domain.MessageServiceAction{
|
||||
Kind: domain.MessageServiceActionCustomText,
|
||||
Text: domain.RetentionPurgeNoticeText,
|
||||
},
|
||||
}
|
||||
req := domain.EditMessageRequest{
|
||||
OwnerUserID: sender.ID,
|
||||
Peer: domain.Peer{Type: domain.PeerTypeUser, ID: recipient.ID},
|
||||
ID: sent.SenderMessage.ID,
|
||||
SetRichMessage: true,
|
||||
Media: noticeMedia,
|
||||
RetentionPurge: true,
|
||||
}
|
||||
res, err := messages.EditMessage(ctx, req)
|
||||
if err != nil {
|
||||
t.Fatalf("EditMessage(RetentionPurge): %v", err)
|
||||
}
|
||||
self := res.Self()
|
||||
if self.Message.Media == nil || self.Message.Media.Kind != domain.MessageMediaKindService {
|
||||
t.Fatalf("edited message media = %+v, want service kind", self.Message.Media)
|
||||
}
|
||||
if self.Message.Media.ServiceAction == nil || self.Message.Media.ServiceAction.Kind != domain.MessageServiceActionCustomText {
|
||||
t.Fatalf("edited message service action = %+v, want custom_text", self.Message.Media.ServiceAction)
|
||||
}
|
||||
if self.Message.Media.ServiceAction.Text != domain.RetentionPurgeNoticeText {
|
||||
t.Fatalf("edited message notice text = %q, want %q", self.Message.Media.ServiceAction.Text, domain.RetentionPurgeNoticeText)
|
||||
}
|
||||
|
||||
// A second retention-purge edit is a no-op: the message is already the
|
||||
// notice, so this must not keep bumping pts / re-writing it forever.
|
||||
if _, err := messages.EditMessage(ctx, req); err != domain.ErrMessageNotModified {
|
||||
t.Fatalf("second RetentionPurge edit err = %v, want ErrMessageNotModified", err)
|
||||
}
|
||||
|
||||
// A non-author, non-bypass edit on someone else's message must still be
|
||||
// rejected -- RetentionPurge only bypasses the author check when it is
|
||||
// actually set, not for ordinary edits in general.
|
||||
ordinary := req
|
||||
ordinary.RetentionPurge = false
|
||||
ordinary.OwnerUserID = recipient.ID
|
||||
ordinary.Peer = domain.Peer{Type: domain.PeerTypeUser, ID: sender.ID}
|
||||
if _, err := messages.EditMessage(ctx, ordinary); err != domain.ErrMessageAuthorRequired {
|
||||
t.Fatalf("non-bypass edit by non-author err = %v, want ErrMessageAuthorRequired", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRetentionPurgeBypassProducesServiceMessage_ChannelMessage is the
|
||||
// channel counterpart of the private-message test above: channel messages
|
||||
// store their service action in a structurally separate Action column (see
|
||||
// tgChannelMessage), so applyRetentionPurgeChannelMessage must clear body/
|
||||
// media and set Action instead of replacing Media like the private path.
|
||||
func TestRetentionPurgeBypassProducesServiceMessage_ChannelMessage(t *testing.T) {
|
||||
pool := testPool(t)
|
||||
ctx := context.Background()
|
||||
suffix := randomSuffix(t)
|
||||
|
||||
users := NewUserStore(pool)
|
||||
owner, err := users.Create(ctx, domain.User{AccessHash: 83, Phone: "+1997" + suffix + "03", FirstName: "RetentionChOwner"})
|
||||
if err != nil {
|
||||
t.Fatalf("create owner: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _, _ = pool.Exec(ctx, "DELETE FROM users WHERE id = $1", owner.ID) })
|
||||
|
||||
channels := NewChannelStore(pool)
|
||||
created, err := channels.CreateChannel(ctx, domain.CreateChannelRequest{CreatorUserID: owner.ID, Title: "Retention " + suffix, Megagroup: true, Date: 1700003000})
|
||||
if err != nil {
|
||||
t.Fatalf("create channel: %v", err)
|
||||
}
|
||||
channelID := created.Channel.ID
|
||||
t.Cleanup(func() { _, _ = pool.Exec(ctx, "DELETE FROM channels WHERE id = $1", channelID) })
|
||||
|
||||
sent, err := channels.SendChannelMessage(ctx, domain.SendChannelMessageRequest{
|
||||
UserID: owner.ID, ChannelID: channelID, RandomID: 771122, Message: "old file",
|
||||
Media: &domain.MessageMedia{Kind: domain.MessageMediaKindDocument, Document: &domain.Document{ID: time.Now().UnixNano(), AccessHash: 1}},
|
||||
Date: 1700003001,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("send channel message: %v", err)
|
||||
}
|
||||
|
||||
req := domain.EditChannelMessageRequest{
|
||||
ChannelID: channelID,
|
||||
ID: sent.Message.ID,
|
||||
RetentionPurge: true,
|
||||
RetentionPurgeAction: &domain.ChannelMessageAction{Type: domain.ChannelActionCustomText, Text: domain.RetentionPurgeNoticeText},
|
||||
}
|
||||
res, err := channels.EditChannelMessage(ctx, req)
|
||||
if err != nil {
|
||||
t.Fatalf("EditChannelMessage(RetentionPurge): %v", err)
|
||||
}
|
||||
if res.Message.Action == nil || res.Message.Action.Type != domain.ChannelActionCustomText {
|
||||
t.Fatalf("edited channel message action = %+v, want custom_text", res.Message.Action)
|
||||
}
|
||||
if res.Message.Action.Text != domain.RetentionPurgeNoticeText {
|
||||
t.Fatalf("edited channel message notice text = %q, want %q", res.Message.Action.Text, domain.RetentionPurgeNoticeText)
|
||||
}
|
||||
if res.Message.Body != "" || !res.Message.Media.IsZero() {
|
||||
t.Fatalf("edited channel message body/media not cleared: body=%q media=%+v", res.Message.Body, res.Message.Media)
|
||||
}
|
||||
|
||||
// Idempotency guard: a second retention-purge edit on an already-service
|
||||
// message must not succeed again.
|
||||
if _, err := channels.EditChannelMessage(ctx, req); err != domain.ErrMessageNotModified {
|
||||
t.Fatalf("second RetentionPurge channel edit err = %v, want ErrMessageNotModified", err)
|
||||
}
|
||||
|
||||
// Read-back via ListChannelHistory renders as a service message too.
|
||||
hist, err := channels.ListChannelHistory(ctx, owner.ID, domain.ChannelHistoryFilter{ChannelID: channelID, Limit: 10})
|
||||
if err != nil {
|
||||
t.Fatalf("list channel history: %v", err)
|
||||
}
|
||||
var found bool
|
||||
for _, m := range hist.Messages {
|
||||
if m.ID == sent.Message.ID {
|
||||
found = true
|
||||
if m.Action == nil || m.Action.Type != domain.ChannelActionCustomText {
|
||||
t.Fatalf("history action = %+v, want custom_text", m.Action)
|
||||
}
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Fatalf("purged message %d not found in channel history", sent.Message.ID)
|
||||
}
|
||||
}
|
||||
|
||||
// TestDocumentCategoryPopulatedOnSend guards Part 2's schema/write-path
|
||||
// change: sending a document message stamps the already-computed
|
||||
// domain.DocumentMediaCategory value directly onto the documents row, at the
|
||||
// same point the shared message_box_media index is written -- zero new
|
||||
// classification logic.
|
||||
func TestDocumentCategoryPopulatedOnSend(t *testing.T) {
|
||||
pool := testPool(t)
|
||||
ctx := context.Background()
|
||||
suffix := randomSuffix(t)
|
||||
|
||||
users := NewUserStore(pool)
|
||||
sender, err := users.Create(ctx, domain.User{AccessHash: 84, Phone: "+1997" + suffix + "04", FirstName: "CatSender"})
|
||||
if err != nil {
|
||||
t.Fatalf("create sender: %v", err)
|
||||
}
|
||||
recipient, err := users.Create(ctx, domain.User{AccessHash: 85, Phone: "+1997" + suffix + "05", FirstName: "CatRecipient"})
|
||||
if err != nil {
|
||||
t.Fatalf("create recipient: %v", err)
|
||||
}
|
||||
ids := []int64{sender.ID, recipient.ID}
|
||||
docID := time.Now().UnixNano()
|
||||
t.Cleanup(func() {
|
||||
_, _ = pool.Exec(ctx, "DELETE FROM message_boxes WHERE owner_user_id = ANY($1::bigint[])", ids)
|
||||
_, _ = pool.Exec(ctx, "DELETE FROM private_messages WHERE sender_user_id = ANY($1::bigint[])", ids)
|
||||
_, _ = pool.Exec(ctx, "DELETE FROM dialogs WHERE user_id = ANY($1::bigint[])", ids)
|
||||
_, _ = pool.Exec(ctx, "DELETE FROM users WHERE id = ANY($1::bigint[])", ids)
|
||||
_, _ = pool.Exec(ctx, "DELETE FROM documents WHERE id = $1", docID)
|
||||
_, _ = pool.Exec(ctx, "DELETE FROM media_references WHERE media_kind = 'document' AND media_id = $1", docID)
|
||||
})
|
||||
|
||||
media := NewMediaStore(pool)
|
||||
if err := media.PutDocument(ctx, domain.Document{ID: docID, MimeType: "video/mp4", Size: 2048}); err != nil {
|
||||
t.Fatalf("PutDocument: %v", err)
|
||||
}
|
||||
|
||||
messages := NewMessageStore(pool, WithMessageAllocators(&perUserCounterAllocator{}))
|
||||
_, err = messages.SendPrivateText(ctx, domain.SendPrivateTextRequest{
|
||||
SenderUserID: sender.ID,
|
||||
RecipientUserID: recipient.ID,
|
||||
RandomID: time.Now().UnixNano(),
|
||||
Media: &domain.MessageMedia{
|
||||
Kind: domain.MessageMediaKindDocument,
|
||||
Document: &domain.Document{
|
||||
ID: docID, AccessHash: 1, MimeType: "video/mp4",
|
||||
Attributes: []domain.DocumentAttribute{{Kind: domain.DocAttrVideo, W: 640, H: 480, Duration: 5}},
|
||||
},
|
||||
},
|
||||
Date: int(time.Now().Unix()),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("send video document: %v", err)
|
||||
}
|
||||
|
||||
var category int16
|
||||
if err := pool.QueryRow(ctx, "SELECT category FROM documents WHERE id = $1", docID).Scan(&category); err != nil {
|
||||
t.Fatalf("read document category: %v", err)
|
||||
}
|
||||
if domain.MediaCategory(category) != domain.MediaCategoryVideo {
|
||||
t.Fatalf("document category = %d, want %d (Video)", category, domain.MediaCategoryVideo)
|
||||
}
|
||||
}
|
||||
|
||||
// TestHardRetentionCategoryFilterOnlyReturnsMatchingCategory guards Part 2's
|
||||
// query-layer change: ListDocumentIDsForHardRetentionOlderThan now takes a
|
||||
// category argument and must only return documents in that exact bucket,
|
||||
// even when another old, blob-owning document sits in a different category.
|
||||
func TestHardRetentionCategoryFilterOnlyReturnsMatchingCategory(t *testing.T) {
|
||||
pool := testPool(t)
|
||||
ctx := context.Background()
|
||||
media := NewMediaStore(pool)
|
||||
|
||||
videoID := time.Now().UnixNano()
|
||||
musicID := videoID + 1
|
||||
t.Cleanup(func() {
|
||||
_, _ = pool.Exec(context.Background(), "DELETE FROM documents WHERE id = ANY($1::bigint[])", []int64{videoID, musicID})
|
||||
_, _ = pool.Exec(context.Background(), "DELETE FROM file_blobs WHERE location_key = ANY($1::text[])",
|
||||
[]string{"doc:" + strconv.FormatInt(videoID, 10), "doc:" + strconv.FormatInt(musicID, 10)})
|
||||
})
|
||||
|
||||
for _, d := range []struct {
|
||||
id int64
|
||||
category domain.MediaCategory
|
||||
}{{videoID, domain.MediaCategoryVideo}, {musicID, domain.MediaCategoryMusic}} {
|
||||
if err := media.PutDocument(ctx, domain.Document{ID: d.id, MimeType: "application/octet-stream", Size: 512}); err != nil {
|
||||
t.Fatalf("PutDocument %d: %v", d.id, err)
|
||||
}
|
||||
if _, err := pool.Exec(ctx, "UPDATE documents SET created_at = now() - interval '100 days', category = $2 WHERE id = $1", d.id, int16(d.category)); err != nil {
|
||||
t.Fatalf("backdate/categorize document %d: %v", d.id, err)
|
||||
}
|
||||
blob := postgresTestBlob("doc:"+strconv.FormatInt(d.id, 10), "cat-filter", 512, "application/octet-stream")
|
||||
if err := media.PutFileBlob(ctx, blob); err != nil {
|
||||
t.Fatalf("PutFileBlob %d: %v", d.id, err)
|
||||
}
|
||||
}
|
||||
|
||||
cutoff := time.Now().Add(-24 * time.Hour)
|
||||
videoIDs, err := media.ListDocumentIDsForHardRetentionOlderThan(ctx, domain.MediaCategoryVideo, cutoff, 1000)
|
||||
if err != nil {
|
||||
t.Fatalf("ListDocumentIDsForHardRetentionOlderThan(Video): %v", err)
|
||||
}
|
||||
if !containsInt64(videoIDs, videoID) || containsInt64(videoIDs, musicID) {
|
||||
t.Fatalf("video-category candidates = %v, want to include %d and exclude %d", videoIDs, videoID, musicID)
|
||||
}
|
||||
|
||||
musicIDs, err := media.ListDocumentIDsForHardRetentionOlderThan(ctx, domain.MediaCategoryMusic, cutoff, 1000)
|
||||
if err != nil {
|
||||
t.Fatalf("ListDocumentIDsForHardRetentionOlderThan(Music): %v", err)
|
||||
}
|
||||
if !containsInt64(musicIDs, musicID) || containsInt64(musicIDs, videoID) {
|
||||
t.Fatalf("music-category candidates = %v, want to include %d and exclude %d", musicIDs, musicID, videoID)
|
||||
}
|
||||
}
|
||||
|
||||
// TestEvictionListsOldestAcrossDocumentsAndPhotos guards Part 3's eviction
|
||||
// query layer: ListOldestMediaForEviction must return candidates from both
|
||||
// tables (interleaving by created_at is done by the caller in Go, see
|
||||
// files.Service.EvictOldestMediaOverBudget) so the oldest-overall item can be
|
||||
// picked regardless of which table it lives in.
|
||||
func TestEvictionListsOldestAcrossDocumentsAndPhotos(t *testing.T) {
|
||||
pool := testPool(t)
|
||||
ctx := context.Background()
|
||||
media := NewMediaStore(pool)
|
||||
|
||||
docID := time.Now().UnixNano()
|
||||
photoID := docID + 1
|
||||
t.Cleanup(func() {
|
||||
_, _ = pool.Exec(context.Background(), "DELETE FROM documents WHERE id = $1", docID)
|
||||
_, _ = pool.Exec(context.Background(), "DELETE FROM photos WHERE id = $1", photoID)
|
||||
_, _ = pool.Exec(context.Background(), "DELETE FROM file_blobs WHERE location_key = ANY($1::text[])",
|
||||
[]string{"doc:" + strconv.FormatInt(docID, 10), "photo:" + strconv.FormatInt(photoID, 10)})
|
||||
})
|
||||
|
||||
if err := media.PutDocument(ctx, domain.Document{ID: docID, MimeType: "application/octet-stream", Size: 256}); err != nil {
|
||||
t.Fatalf("PutDocument: %v", err)
|
||||
}
|
||||
// The document is the older of the two (200 days vs 100 days) -- it must
|
||||
// sort before the photo in the merged eviction candidate list.
|
||||
if _, err := pool.Exec(ctx, "UPDATE documents SET created_at = now() - interval '200 days' WHERE id = $1", docID); err != nil {
|
||||
t.Fatalf("backdate document: %v", err)
|
||||
}
|
||||
if err := media.PutFileBlob(ctx, postgresTestBlob("doc:"+strconv.FormatInt(docID, 10), "evict-doc", 256, "application/octet-stream")); err != nil {
|
||||
t.Fatalf("PutFileBlob doc: %v", err)
|
||||
}
|
||||
|
||||
if err := media.PutPhoto(ctx, domain.Photo{ID: photoID, AccessHash: 1}); err != nil {
|
||||
t.Fatalf("PutPhoto: %v", err)
|
||||
}
|
||||
if _, err := pool.Exec(ctx, "UPDATE photos SET created_at = now() - interval '100 days' WHERE id = $1", photoID); err != nil {
|
||||
t.Fatalf("backdate photo: %v", err)
|
||||
}
|
||||
if err := media.PutFileBlob(ctx, postgresTestBlob("photo:"+strconv.FormatInt(photoID, 10), "evict-photo", 256, "image/jpeg")); err != nil {
|
||||
t.Fatalf("PutFileBlob photo: %v", err)
|
||||
}
|
||||
|
||||
candidates, err := media.ListOldestMediaForEviction(ctx, 1000)
|
||||
if err != nil {
|
||||
t.Fatalf("ListOldestMediaForEviction: %v", err)
|
||||
}
|
||||
var doc, photo *domain.EvictionCandidate
|
||||
for i := range candidates {
|
||||
c := candidates[i]
|
||||
switch {
|
||||
case c.Kind == domain.MediaKindDocument && c.MediaID == docID:
|
||||
doc = &candidates[i]
|
||||
case c.Kind == domain.MediaKindPhoto && c.MediaID == photoID:
|
||||
photo = &candidates[i]
|
||||
}
|
||||
}
|
||||
if doc == nil || photo == nil {
|
||||
t.Fatalf("eviction candidates missing doc/photo: %+v", candidates)
|
||||
}
|
||||
if !doc.CreatedAt.Before(photo.CreatedAt) {
|
||||
t.Fatalf("document created_at %v not before photo created_at %v (document should be older)", doc.CreatedAt, photo.CreatedAt)
|
||||
}
|
||||
}
|
||||
|
|
@ -825,20 +825,103 @@ func (q *Queries) ListAvailableReactions(ctx context.Context) ([]AvailableReacti
|
|||
return items, nil
|
||||
}
|
||||
|
||||
const listAvatarOrphanedPhotoIDsOlderThan = `-- name: ListAvatarOrphanedPhotoIDsOlderThan :many
|
||||
SELECT p.id FROM photos p
|
||||
WHERE p.orphaned_at IS NOT NULL AND p.orphaned_at < $1::timestamptz
|
||||
AND EXISTS (SELECT 1 FROM profile_photos pp WHERE pp.photo_id = p.id AND pp.active)
|
||||
ORDER BY p.orphaned_at ASC
|
||||
LIMIT $2::int
|
||||
`
|
||||
|
||||
type ListAvatarOrphanedPhotoIDsOlderThanParams struct {
|
||||
Cutoff pgtype.Timestamptz
|
||||
BatchLimit int32
|
||||
}
|
||||
|
||||
// Same as ListOrphanedPhotoIDsOlderThan but only photos currently active as
|
||||
// someone's avatar (profile_photos.active) -- lets the Avatar category carry
|
||||
// its own retention age. In practice a live avatar is never orphaned (an
|
||||
// active profile_photos row is itself a media_references entry), so this
|
||||
// bucket is expected to stay empty under "orphan" mode; kept for symmetry
|
||||
// with the "hard" mode avatar split, which is the one that actually matters.
|
||||
func (q *Queries) ListAvatarOrphanedPhotoIDsOlderThan(ctx context.Context, arg ListAvatarOrphanedPhotoIDsOlderThanParams) ([]int64, error) {
|
||||
rows, err := q.db.Query(ctx, listAvatarOrphanedPhotoIDsOlderThan, arg.Cutoff, arg.BatchLimit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var items []int64
|
||||
for rows.Next() {
|
||||
var id int64
|
||||
if err := rows.Scan(&id); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items = append(items, id)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
const listAvatarPhotoIDsForHardRetentionOlderThan = `-- name: ListAvatarPhotoIDsForHardRetentionOlderThan :many
|
||||
SELECT p.id FROM photos p
|
||||
WHERE p.created_at < $1::timestamptz
|
||||
AND EXISTS (SELECT 1 FROM profile_photos pp WHERE pp.photo_id = p.id AND pp.active)
|
||||
AND EXISTS (
|
||||
SELECT 1 FROM file_blobs fb
|
||||
WHERE fb.location_key = 'photo:' || p.id::text
|
||||
OR fb.location_key LIKE 'photo:' || p.id::text || ':%'
|
||||
)
|
||||
ORDER BY p.created_at ASC
|
||||
LIMIT $2::int
|
||||
`
|
||||
|
||||
type ListAvatarPhotoIDsForHardRetentionOlderThanParams struct {
|
||||
Cutoff pgtype.Timestamptz
|
||||
BatchLimit int32
|
||||
}
|
||||
|
||||
// Same as ListPhotoIDsForHardRetentionOlderThan but only photos currently
|
||||
// active as someone's avatar (profile_photos.active) -- lets the Avatar
|
||||
// category carry its own retention age, independent of ordinary shared-media
|
||||
// photos.
|
||||
func (q *Queries) ListAvatarPhotoIDsForHardRetentionOlderThan(ctx context.Context, arg ListAvatarPhotoIDsForHardRetentionOlderThanParams) ([]int64, error) {
|
||||
rows, err := q.db.Query(ctx, listAvatarPhotoIDsForHardRetentionOlderThan, arg.Cutoff, arg.BatchLimit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var items []int64
|
||||
for rows.Next() {
|
||||
var id int64
|
||||
if err := rows.Scan(&id); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items = append(items, id)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
const listDocumentIDsForHardRetentionOlderThan = `-- name: ListDocumentIDsForHardRetentionOlderThan :many
|
||||
SELECT d.id FROM documents d
|
||||
WHERE d.created_at < $1::timestamptz
|
||||
AND d.category = $2::smallint
|
||||
AND EXISTS (
|
||||
SELECT 1 FROM file_blobs fb
|
||||
WHERE fb.location_key = 'doc:' || d.id::text
|
||||
OR fb.location_key LIKE 'doc:' || d.id::text || ':%'
|
||||
)
|
||||
ORDER BY d.created_at ASC
|
||||
LIMIT $2::int
|
||||
LIMIT $3::int
|
||||
`
|
||||
|
||||
type ListDocumentIDsForHardRetentionOlderThanParams struct {
|
||||
Cutoff pgtype.Timestamptz
|
||||
Category int16
|
||||
BatchLimit int32
|
||||
}
|
||||
|
||||
|
|
@ -848,9 +931,11 @@ type ListDocumentIDsForHardRetentionOlderThanParams struct {
|
|||
// keeps this sweep from re-selecting the same document forever: once its
|
||||
// blob bytes are purged (DeleteFileBlobsForDocument removes the file_blobs
|
||||
// rows but deliberately leaves this documents row in place), it naturally
|
||||
// drops out of this query on the next pass.
|
||||
// drops out of this query on the next pass. category scopes to one
|
||||
// documents.category bucket per sweep tick -- see
|
||||
// ListOrphanedDocumentIDsOlderThan for the 0/None fallback note.
|
||||
func (q *Queries) ListDocumentIDsForHardRetentionOlderThan(ctx context.Context, arg ListDocumentIDsForHardRetentionOlderThanParams) ([]int64, error) {
|
||||
rows, err := q.db.Query(ctx, listDocumentIDsForHardRetentionOlderThan, arg.Cutoff, arg.BatchLimit)
|
||||
rows, err := q.db.Query(ctx, listDocumentIDsForHardRetentionOlderThan, arg.Cutoff, arg.Category, arg.BatchLimit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
@ -917,20 +1002,156 @@ func (q *Queries) ListFileBlobsByLocationPrefix(ctx context.Context, arg ListFil
|
|||
return items, nil
|
||||
}
|
||||
|
||||
const listMediaReferences = `-- name: ListMediaReferences :many
|
||||
SELECT media_kind, media_id, ref_kind, ref_key
|
||||
FROM media_references
|
||||
WHERE media_kind = $1::text AND media_id = $2::bigint
|
||||
`
|
||||
|
||||
type ListMediaReferencesParams struct {
|
||||
MediaKind string
|
||||
MediaID int64
|
||||
}
|
||||
|
||||
type ListMediaReferencesRow struct {
|
||||
MediaKind string
|
||||
MediaID int64
|
||||
RefKind string
|
||||
RefKey string
|
||||
}
|
||||
|
||||
// Every live reference to a document/photo -- used by the storage retention
|
||||
// sweep to turn a hard-retention/eviction blob purge into a visible notice on
|
||||
// every message that still embeds the purged media (see
|
||||
// files.notifyRetentionPurge). profile_photo/sticker_set/gift refs have no
|
||||
// message to edit and are filtered by the caller, not this query.
|
||||
func (q *Queries) ListMediaReferences(ctx context.Context, arg ListMediaReferencesParams) ([]ListMediaReferencesRow, error) {
|
||||
rows, err := q.db.Query(ctx, listMediaReferences, arg.MediaKind, arg.MediaID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var items []ListMediaReferencesRow
|
||||
for rows.Next() {
|
||||
var i ListMediaReferencesRow
|
||||
if err := rows.Scan(
|
||||
&i.MediaKind,
|
||||
&i.MediaID,
|
||||
&i.RefKind,
|
||||
&i.RefKey,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items = append(items, i)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
const listOldestDocumentsForEviction = `-- name: ListOldestDocumentsForEviction :many
|
||||
SELECT d.id, d.created_at FROM documents d
|
||||
WHERE EXISTS (
|
||||
SELECT 1 FROM file_blobs fb
|
||||
WHERE fb.location_key = 'doc:' || d.id::text
|
||||
OR fb.location_key LIKE 'doc:' || d.id::text || ':%'
|
||||
)
|
||||
ORDER BY d.created_at ASC
|
||||
LIMIT $1::int
|
||||
`
|
||||
|
||||
type ListOldestDocumentsForEvictionRow struct {
|
||||
ID int64
|
||||
CreatedAt pgtype.Timestamptz
|
||||
}
|
||||
|
||||
// Active eviction (TELESRV_STORAGE_EVICTION_ENABLE): candidates are every
|
||||
// document that still owns at least one file_blobs row, oldest-uploaded
|
||||
// first, REGARDLESS of category or age -- unlike the retention sweeps above,
|
||||
// eviction only cares about reclaiming bytes once the total physical budget
|
||||
// (TELESRV_STORAGE_MAX_TOTAL_BYTES) is exceeded. created_at is returned so
|
||||
// the caller can interleave these with ListOldestPhotosForEviction by actual
|
||||
// age instead of draining one table before touching the other.
|
||||
func (q *Queries) ListOldestDocumentsForEviction(ctx context.Context, batchLimit int32) ([]ListOldestDocumentsForEvictionRow, error) {
|
||||
rows, err := q.db.Query(ctx, listOldestDocumentsForEviction, batchLimit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var items []ListOldestDocumentsForEvictionRow
|
||||
for rows.Next() {
|
||||
var i ListOldestDocumentsForEvictionRow
|
||||
if err := rows.Scan(&i.ID, &i.CreatedAt); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items = append(items, i)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
const listOldestPhotosForEviction = `-- name: ListOldestPhotosForEviction :many
|
||||
SELECT p.id, p.created_at FROM photos p
|
||||
WHERE EXISTS (
|
||||
SELECT 1 FROM file_blobs fb
|
||||
WHERE fb.location_key = 'photo:' || p.id::text
|
||||
OR fb.location_key LIKE 'photo:' || p.id::text || ':%'
|
||||
)
|
||||
ORDER BY p.created_at ASC
|
||||
LIMIT $1::int
|
||||
`
|
||||
|
||||
type ListOldestPhotosForEvictionRow struct {
|
||||
ID int64
|
||||
CreatedAt pgtype.Timestamptz
|
||||
}
|
||||
|
||||
// See ListOldestDocumentsForEviction -- same oldest-first eviction candidate
|
||||
// selection, for photos (avatars included: eviction is bytes-only and does
|
||||
// not honor the Avatar category's separate retention age).
|
||||
func (q *Queries) ListOldestPhotosForEviction(ctx context.Context, batchLimit int32) ([]ListOldestPhotosForEvictionRow, error) {
|
||||
rows, err := q.db.Query(ctx, listOldestPhotosForEviction, batchLimit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var items []ListOldestPhotosForEvictionRow
|
||||
for rows.Next() {
|
||||
var i ListOldestPhotosForEvictionRow
|
||||
if err := rows.Scan(&i.ID, &i.CreatedAt); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items = append(items, i)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
const listOrphanedDocumentIDsOlderThan = `-- name: ListOrphanedDocumentIDsOlderThan :many
|
||||
SELECT id FROM documents
|
||||
WHERE orphaned_at IS NOT NULL AND orphaned_at < $1::timestamptz
|
||||
AND category = $2::smallint
|
||||
ORDER BY orphaned_at ASC
|
||||
LIMIT $2::int
|
||||
LIMIT $3::int
|
||||
`
|
||||
|
||||
type ListOrphanedDocumentIDsOlderThanParams struct {
|
||||
Cutoff pgtype.Timestamptz
|
||||
Category int16
|
||||
BatchLimit int32
|
||||
}
|
||||
|
||||
// category selects one documents.category bucket per sweep tick (per-category
|
||||
// retention age, see internal/app/files/retention.go) -- 0 (MediaCategoryNone)
|
||||
// covers unclassified documents (e.g. stickers), which always use the shared
|
||||
// global age since there is no per-category override for that bucket.
|
||||
func (q *Queries) ListOrphanedDocumentIDsOlderThan(ctx context.Context, arg ListOrphanedDocumentIDsOlderThanParams) ([]int64, error) {
|
||||
rows, err := q.db.Query(ctx, listOrphanedDocumentIDsOlderThan, arg.Cutoff, arg.BatchLimit)
|
||||
rows, err := q.db.Query(ctx, listOrphanedDocumentIDsOlderThan, arg.Cutoff, arg.Category, arg.BatchLimit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
@ -950,9 +1171,10 @@ func (q *Queries) ListOrphanedDocumentIDsOlderThan(ctx context.Context, arg List
|
|||
}
|
||||
|
||||
const listOrphanedPhotoIDsOlderThan = `-- name: ListOrphanedPhotoIDsOlderThan :many
|
||||
SELECT id FROM photos
|
||||
WHERE orphaned_at IS NOT NULL AND orphaned_at < $1::timestamptz
|
||||
ORDER BY orphaned_at ASC
|
||||
SELECT p.id FROM photos p
|
||||
WHERE p.orphaned_at IS NOT NULL AND p.orphaned_at < $1::timestamptz
|
||||
AND NOT EXISTS (SELECT 1 FROM profile_photos pp WHERE pp.photo_id = p.id AND pp.active)
|
||||
ORDER BY p.orphaned_at ASC
|
||||
LIMIT $2::int
|
||||
`
|
||||
|
||||
|
|
@ -961,6 +1183,8 @@ type ListOrphanedPhotoIDsOlderThanParams struct {
|
|||
BatchLimit int32
|
||||
}
|
||||
|
||||
// Excludes photos currently active as someone's avatar -- see
|
||||
// ListAvatarOrphanedPhotoIDsOlderThan for that split-off bucket.
|
||||
func (q *Queries) ListOrphanedPhotoIDsOlderThan(ctx context.Context, arg ListOrphanedPhotoIDsOlderThanParams) ([]int64, error) {
|
||||
rows, err := q.db.Query(ctx, listOrphanedPhotoIDsOlderThan, arg.Cutoff, arg.BatchLimit)
|
||||
if err != nil {
|
||||
|
|
@ -984,6 +1208,7 @@ func (q *Queries) ListOrphanedPhotoIDsOlderThan(ctx context.Context, arg ListOrp
|
|||
const listPhotoIDsForHardRetentionOlderThan = `-- name: ListPhotoIDsForHardRetentionOlderThan :many
|
||||
SELECT p.id FROM photos p
|
||||
WHERE p.created_at < $1::timestamptz
|
||||
AND NOT EXISTS (SELECT 1 FROM profile_photos pp WHERE pp.photo_id = p.id AND pp.active)
|
||||
AND EXISTS (
|
||||
SELECT 1 FROM file_blobs fb
|
||||
WHERE fb.location_key = 'photo:' || p.id::text
|
||||
|
|
@ -999,7 +1224,8 @@ type ListPhotoIDsForHardRetentionOlderThanParams struct {
|
|||
}
|
||||
|
||||
// See ListDocumentIDsForHardRetentionOlderThan -- same "hard" retention
|
||||
// candidate selection, for photos.
|
||||
// candidate selection, for photos. Excludes photos currently active as
|
||||
// someone's avatar -- see ListAvatarPhotoIDsForHardRetentionOlderThan.
|
||||
func (q *Queries) ListPhotoIDsForHardRetentionOlderThan(ctx context.Context, arg ListPhotoIDsForHardRetentionOlderThanParams) ([]int64, error) {
|
||||
rows, err := q.db.Query(ctx, listPhotoIDsForHardRetentionOlderThan, arg.Cutoff, arg.BatchLimit)
|
||||
if err != nil {
|
||||
|
|
|
|||
|
|
@ -1252,6 +1252,7 @@ type Document struct {
|
|||
CreatedAt pgtype.Timestamptz
|
||||
OwnerUserID int64
|
||||
OrphanedAt pgtype.Timestamptz
|
||||
Category int16
|
||||
}
|
||||
|
||||
type EncryptedFile struct {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue