This commit is contained in:
onysd 2026-09-01 12:50:18 +03:00
parent 21a0856587
commit e8dc967e6a
26 changed files with 1373 additions and 481 deletions

View file

@ -954,27 +954,31 @@ ORDER BY g.last_active_at DESC, g.device_model, g.system_version, g.platform, g.
return groups, hasMore, nil return groups, hasMore, nil
} }
// BroadcastRow is one system-broadcast campaign, with sent/failed counts // BroadcastRow is one system-broadcast campaign. SentCount/FailedCount/
// derived live from broadcast_recipients (never stored, so they can't drift). // MaterializedCount are maintained incrementally by the delivery worker as
// it closes out each recipient row (see internal/app/broadcast); for an
// "all"-mode campaign still enumerating, TargetCount grows until
// EnumerationDone.
type BroadcastRow struct { type BroadcastRow struct {
ID int64 ID int64
Message string Message string
TargetMode string TargetMode string
TotalCount int TargetCount int64
SentCount int MaterializedCount int64
FailedCount int SentCount int64
FailedCount int64
EnumerationDone bool
CreatedBy string CreatedBy string
CreatedAt time.Time CreatedAt time.Time
} }
const broadcastRowColumns = ` const broadcastRowColumns = `
b.id, b.message, b.target_mode, b.total_count, b.created_by, b.created_at, b.id, b.message, b.target_mode, b.target_count, b.materialized_count,
count(*) FILTER (WHERE r.status = 'sent')::int AS sent_count, b.sent_count, b.failed_count, b.enumeration_done, b.created_by, b.created_at`
count(*) FILTER (WHERE r.status = 'failed')::int AS failed_count`
func scanBroadcastRow(row interface{ Scan(...any) error }, item *BroadcastRow) error { func scanBroadcastRow(row interface{ Scan(...any) error }, item *BroadcastRow) error {
return row.Scan(&item.ID, &item.Message, &item.TargetMode, &item.TotalCount, &item.CreatedBy, &item.CreatedAt, return row.Scan(&item.ID, &item.Message, &item.TargetMode, &item.TargetCount, &item.MaterializedCount,
&item.SentCount, &item.FailedCount) &item.SentCount, &item.FailedCount, &item.EnumerationDone, &item.CreatedBy, &item.CreatedAt)
} }
// ListBroadcasts pages campaigns newest-first. // ListBroadcasts pages campaigns newest-first.
@ -985,9 +989,7 @@ func (s *readStore) ListBroadcasts(ctx context.Context, beforeID int64, limit in
rows, err := s.pool.Query(ctx, ` rows, err := s.pool.Query(ctx, `
SELECT `+broadcastRowColumns+` SELECT `+broadcastRowColumns+`
FROM broadcasts b FROM broadcasts b
LEFT JOIN broadcast_recipients r ON r.broadcast_id = b.id
WHERE $1::bigint = 0 OR b.id < $1 WHERE $1::bigint = 0 OR b.id < $1
GROUP BY b.id
ORDER BY b.id DESC ORDER BY b.id DESC
LIMIT $2`, beforeID, limit+1) LIMIT $2`, beforeID, limit+1)
if err != nil { if err != nil {

View file

@ -17,6 +17,8 @@ import (
"strings" "strings"
"time" "time"
"github.com/iamxvbaba/td/tg"
"telesrv/internal/admin" "telesrv/internal/admin"
"telesrv/internal/domain" "telesrv/internal/domain"
"telesrv/internal/hoststats" "telesrv/internal/hoststats"
@ -303,6 +305,10 @@ func (s *server) handleSession(w http.ResponseWriter, r *http.Request) {
// tells "the old admin process died and a new one answered" apart // tells "the old admin process died and a new one answered" apart
// from "the old one is just slow to respond". // from "the old one is just slow to respond".
"boot_id": bootID, "boot_id": bootID,
// api_layer is the MTProto TL schema layer this server binary speaks
// (tg.Layer), shown in the sidebar footer above the build/commit line
// so an operator can tell at a glance which protocol layer is live.
"api_layer": tg.Layer,
// build is this admin binary's own commit -- shown under "Version" // build is this admin binary's own commit -- shown under "Version"
// in the sidebar footer so an operator can tell at a glance which // in the sidebar footer so an operator can tell at a glance which
// build is actually running, independent of the app version string. // build is actually running, independent of the app version string.
@ -936,10 +942,13 @@ type createBroadcastAPIRequest struct {
UserIDs []int64 `json:"user_ids,omitempty"` UserIDs []int64 `json:"user_ids,omitempty"`
} }
// handleCreateBroadcastAPI resolves "all users" into an explicit id list // handleCreateBroadcastAPI forwards a broadcast create straight to the admin
// before forwarding to the admin API: the admin service always receives an // API. "all" mode is no longer pre-resolved into an explicit id list here:
// already-resolved recipient list, never "every user" as a live concept it // the admin service snapshots the current eligible user set itself and the
// would have to know how to enumerate itself. // broadcast worker enumerates it incrementally, so "every user" never has
// to cross this boundary (or the one after it) as a potentially huge id
// slice. Only "selected" mode carries UserIDs, already an operator-picked
// list bounded by domain.MaxBroadcastSelectedRecipients.
func (s *server) handleCreateBroadcastAPI(w http.ResponseWriter, r *http.Request) { func (s *server) handleCreateBroadcastAPI(w http.ResponseWriter, r *http.Request) {
var body createBroadcastAPIRequest var body createBroadcastAPIRequest
if !decodeAction(w, r, &body) { if !decodeAction(w, r, &body) {
@ -947,16 +956,7 @@ func (s *server) handleCreateBroadcastAPI(w http.ResponseWriter, r *http.Request
} }
userIDs := body.UserIDs userIDs := body.UserIDs
if body.TargetMode == "all" { if body.TargetMode == "all" {
if s.read == nil { userIDs = nil
writeAPIError(w, http.StatusServiceUnavailable, "read store is not configured")
return
}
all, err := s.read.ListAllAccountIDs(r.Context())
if err != nil {
writeAPIError(w, http.StatusInternalServerError, err.Error())
return
}
userIDs = all
} }
req := admin.CreateBroadcastRequest{ req := admin.CreateBroadcastRequest{
CommandMeta: s.commandMetaFromAPI(r, body.CommandID, body.Reason, body.Confirm, "create-broadcast"), CommandMeta: s.commandMetaFromAPI(r, body.CommandID, body.Reason, body.Confirm, "create-broadcast"),

View file

@ -23,7 +23,7 @@
})(); })();
</script> </script>
<script type="module" crossorigin src="/assets/index-CwTwvGWj.js"></script> <script type="module" crossorigin src="/assets/index-TfcI68oK.js"></script>
<link rel="stylesheet" crossorigin href="/assets/index-0MvM-hpw.css"> <link rel="stylesheet" crossorigin href="/assets/index-0MvM-hpw.css">
</head> </head>
<body> <body>

View file

@ -42,7 +42,7 @@ export function App() {
return ( return (
<PermissionsProvider permissions={session.permissions ?? []} hideThirdPartyVerification={session.hide_third_party_verification ?? true}> <PermissionsProvider permissions={session.permissions ?? []} hideThirdPartyVerification={session.hide_third_party_verification ?? true}>
<Shell actor={session.actor} build={session.build} route={route} navigate={navigate} onLogout={() => setSession(null)}> <Shell actor={session.actor} apiLayer={session.api_layer} build={session.build} route={route} navigate={navigate} onLogout={() => setSession(null)}>
<Routes route={route} navigate={navigate} /> <Routes route={route} navigate={navigate} />
</Shell> </Shell>
</PermissionsProvider> </PermissionsProvider>

View file

@ -8,9 +8,11 @@ import { MultiUserPicker } from "./EntityPicker";
type TargetMode = "all" | "selected"; type TargetMode = "all" | "selected";
// CreateBroadcastModal composes the message and target list, then hands off to // CreateBroadcastModal composes the message and target list, then hands off to
// ActionButton for the usual dry-run/confirm flow. "All users" is resolved to an // ActionButton for the usual dry-run/confirm flow. "All users" is never
// explicit id list server-side (cmd/telesrv-admin/server.go), not here -- the // resolved into an id list at all -- the admin service snapshots the
// picker only ever deals with an actual, visible list of accounts. // current eligible user set itself and a background worker enumerates it
// incrementally, so this only ever sends user_ids for "selected" mode,
// where the picker deals with an actual, visible list of accounts.
export function CreateBroadcastModal({ onClose, onCreated }: { onClose: () => void; onCreated: () => void }) { export function CreateBroadcastModal({ onClose, onCreated }: { onClose: () => void; onCreated: () => void }) {
const [message, setMessage] = useState(""); const [message, setMessage] = useState("");
const [targetMode, setTargetMode] = useState<TargetMode>("all"); const [targetMode, setTargetMode] = useState<TargetMode>("all");

View file

@ -41,6 +41,7 @@ export function BootScreen() {
export function Shell({ export function Shell({
actor, actor,
apiLayer,
build, build,
route, route,
navigate, navigate,
@ -48,6 +49,7 @@ export function Shell({
children children
}: { }: {
actor: string; actor: string;
apiLayer?: number;
build?: { commit: string; short_commit: string; dirty: boolean; build_time: string }; build?: { commit: string; short_commit: string; dirty: boolean; build_time: string };
route: RouteState; route: RouteState;
navigate: Navigate; navigate: Navigate;
@ -178,6 +180,9 @@ export function Shell({
</nav> </nav>
<div className="sidebar-status"> <div className="sidebar-status">
<span className="sidebar-label">{"Version: O7"}</span> <span className="sidebar-label">{"Version: O7"}</span>
{typeof apiLayer === "number" && (
<span className="sidebar-label sidebar-api-layer">{`API layer: ${apiLayer}`}</span>
)}
{build?.short_commit && ( {build?.short_commit && (
<span className="sidebar-label sidebar-build" title={build.commit + (build.dirty ? " (uncommitted changes)" : "")}> <span className="sidebar-label sidebar-build" title={build.commit + (build.dirty ? " (uncommitted changes)" : "")}>
{`Build: ${build.short_commit}${build.dirty ? "+" : ""}`} {`Build: ${build.short_commit}${build.dirty ? "+" : ""}`}

View file

@ -68,7 +68,7 @@ export function BroadcastsPage() {
}, []); }, []);
const rows = data?.rows ?? []; const rows = data?.rows ?? [];
const inFlight = rows.filter((row) => row.SentCount + row.FailedCount < row.TotalCount).length; const inFlight = rows.filter((row) => !row.EnumerationDone || row.SentCount + row.FailedCount < row.TargetCount).length;
const canGoPrev = history.length > 0 && !busy; const canGoPrev = history.length > 0 && !busy;
const canGoNext = Boolean(data?.has_more) && !busy; const canGoNext = Boolean(data?.has_more) && !busy;
@ -110,7 +110,7 @@ export function BroadcastsPage() {
<tbody> <tbody>
{rows.map((row) => { {rows.map((row) => {
const delivered = row.SentCount + row.FailedCount; const delivered = row.SentCount + row.FailedCount;
const done = row.TotalCount > 0 && delivered >= row.TotalCount; const done = row.EnumerationDone && row.TargetCount > 0 && delivered >= row.TargetCount;
return ( return (
<tr key={row.ID}> <tr key={row.ID}>
<td className="mono">{row.ID}</td> <td className="mono">{row.ID}</td>
@ -118,7 +118,7 @@ export function BroadcastsPage() {
<td>{row.TargetMode === "all" ? <Badge tone="warn">{"All users"}</Badge> : <Badge>{"Selected"}</Badge>}</td> <td>{row.TargetMode === "all" ? <Badge tone="warn">{"All users"}</Badge> : <Badge>{"Selected"}</Badge>}</td>
<td>{row.SentCount}</td> <td>{row.SentCount}</td>
<td>{row.FailedCount > 0 ? <Badge tone="danger">{row.FailedCount}</Badge> : row.FailedCount}</td> <td>{row.FailedCount > 0 ? <Badge tone="danger">{row.FailedCount}</Badge> : row.FailedCount}</td>
<td>{row.TotalCount}</td> <td>{row.TargetCount}</td>
<td>{row.CreatedBy || "-"}</td> <td>{row.CreatedBy || "-"}</td>
<td> <td>
{formatDate(row.CreatedAt)} {formatDate(row.CreatedAt)}

View file

@ -584,6 +584,9 @@ export type AdminSession = {
// comment. Used by Server Settings' Restart/Update flow to detect a // comment. Used by Server Settings' Restart/Update flow to detect a
// genuinely new admin process after asking it to bounce. // genuinely new admin process after asking it to bounce.
boot_id?: string; boot_id?: string;
// The MTProto TL schema layer this server binary speaks -- shown in the
// sidebar footer above the build/commit line.
api_layer?: number;
// This admin binary's own build -- shown under "Version" in the sidebar // This admin binary's own build -- shown under "Version" in the sidebar
// footer so an operator can tell which build is actually running. // footer so an operator can tell which build is actually running.
build?: { build?: {
@ -794,9 +797,11 @@ export type BroadcastRow = {
ID: number; ID: number;
Message: string; Message: string;
TargetMode: string; TargetMode: string;
TotalCount: number; TargetCount: number;
MaterializedCount: number;
SentCount: number; SentCount: number;
FailedCount: number; FailedCount: number;
EnumerationDone: boolean;
CreatedBy: string; CreatedBy: string;
CreatedAt: string; CreatedAt: string;
}; };

View file

@ -1705,8 +1705,12 @@ func run(logger *zap.Logger) error {
// all/selected users) are delivered from the same kind of durable outbox as // all/selected users) are delivered from the same kind of durable outbox as
// applicant notifications above: an admin creating one for every user must // applicant notifications above: an admin creating one for every user must
// not wait on however long sending to all of them takes. // not wait on however long sending to all of them takes.
go broadcastapp.NewWorker(broadcastService, logger.Named("broadcast").Named("delivery"), go broadcastapp.NewWorker(broadcastService, broadcastapp.WorkerConfig{
cfg.BroadcastWorkerInterval, cfg.BroadcastWorkerBatch).Run(ctx) Interval: cfg.BroadcastWorkerInterval,
Lease: cfg.BroadcastWorkerLease,
MaterializeBatch: cfg.BroadcastWorkerMaterializeBatch,
DeliveryBatch: cfg.BroadcastWorkerBatch,
}, logger.Named("broadcast").Named("delivery")).Run(ctx)
moderationActionOptions := []moderationapp.ActionExecutorOption{ moderationActionOptions := []moderationapp.ActionExecutorOption{
moderationapp.WithAccountDeletionNotifier(router), moderationapp.WithAccountDeletionNotifier(router),
} }

View file

@ -1,2 +0,0 @@
DROP TABLE IF EXISTS broadcast_recipients;
DROP TABLE IF EXISTS broadcasts;

View file

@ -1,54 +0,0 @@
CREATE TABLE broadcasts (
id bigserial PRIMARY KEY,
message text NOT NULL CHECK (message <> '' AND octet_length(message) <= 4096),
target_mode varchar(16) NOT NULL CHECK (target_mode IN ('all', 'selected')),
snapshot_max_user_id bigint NOT NULL DEFAULT 0,
enumeration_cursor_user_id bigint NOT NULL DEFAULT 0,
enumeration_done boolean NOT NULL DEFAULT false,
target_count bigint NOT NULL DEFAULT 0 CHECK (target_count >= 0),
materialized_count bigint NOT NULL DEFAULT 0 CHECK (materialized_count >= 0),
sent_count bigint NOT NULL DEFAULT 0 CHECK (sent_count >= 0),
failed_count bigint NOT NULL DEFAULT 0 CHECK (failed_count >= 0),
created_by varchar(128) NOT NULL DEFAULT '',
created_at timestamptz NOT NULL DEFAULT now(),
CHECK (enumeration_cursor_user_id >= 0 AND enumeration_cursor_user_id <= snapshot_max_user_id),
CHECK (sent_count + failed_count <= materialized_count)
);
CREATE TABLE broadcast_recipients (
id bigserial PRIMARY KEY,
broadcast_id bigint NOT NULL REFERENCES broadcasts(id) ON DELETE CASCADE,
user_id bigint NOT NULL,
status varchar(16) NOT NULL DEFAULT 'pending'
CHECK (status IN ('pending', 'processing', 'sent', 'failed')),
attempts integer NOT NULL DEFAULT 0 CHECK (attempts >= 0),
next_attempt_at timestamptz NOT NULL DEFAULT now(),
lease_token varchar(64) NOT NULL DEFAULT '',
lease_until timestamptz,
last_error varchar(500) NOT NULL DEFAULT '',
private_message_id bigint NOT NULL DEFAULT 0,
message_box_id integer NOT NULL DEFAULT 0,
pts integer NOT NULL DEFAULT 0,
sent_at timestamptz,
created_at timestamptz NOT NULL DEFAULT now(),
updated_at timestamptz NOT NULL DEFAULT now(),
UNIQUE (broadcast_id, user_id),
CHECK (
(status = 'sent' AND private_message_id > 0 AND message_box_id > 0 AND pts > 0 AND sent_at IS NOT NULL)
OR
(status <> 'sent' AND private_message_id = 0 AND message_box_id = 0 AND pts = 0 AND sent_at IS NULL)
),
CHECK (
(status = 'processing' AND lease_token <> '' AND lease_until IS NOT NULL)
OR
(status <> 'processing' AND lease_token = '' AND lease_until IS NULL)
)
);
CREATE INDEX broadcasts_enumeration_idx ON broadcasts (id)
WHERE target_mode = 'all' AND NOT enumeration_done;
CREATE INDEX broadcast_recipients_pending_idx ON broadcast_recipients (next_attempt_at, id)
WHERE status = 'pending';
CREATE INDEX broadcast_recipients_processing_idx ON broadcast_recipients (lease_until, id)
WHERE status = 'processing';
CREATE INDEX broadcast_recipients_broadcast_idx ON broadcast_recipients (broadcast_id, id);

View file

@ -1,3 +0,0 @@
ALTER TABLE broadcasts
DROP CONSTRAINT IF EXISTS broadcasts_entities_array_check,
DROP COLUMN IF EXISTS entities;

View file

@ -1,6 +0,0 @@
ALTER TABLE broadcasts
ADD COLUMN entities jsonb NOT NULL DEFAULT '[]'::jsonb;
ALTER TABLE broadcasts
ADD CONSTRAINT broadcasts_entities_array_check
CHECK (jsonb_typeof(entities) = 'array');

View file

@ -0,0 +1,41 @@
DROP INDEX IF EXISTS public.broadcast_recipients_processing_idx;
DROP INDEX IF EXISTS public.broadcast_recipients_pending_next_attempt_idx;
ALTER TABLE public.broadcast_recipients
DROP CONSTRAINT IF EXISTS broadcast_recipients_lease_check,
DROP CONSTRAINT IF EXISTS broadcast_recipients_sent_tracking_check,
DROP CONSTRAINT IF EXISTS broadcast_recipients_attempts_check,
DROP CONSTRAINT IF EXISTS broadcast_recipients_status_check;
ALTER TABLE public.broadcast_recipients
DROP COLUMN IF EXISTS updated_at,
DROP COLUMN IF EXISTS pts,
DROP COLUMN IF EXISTS message_box_id,
DROP COLUMN IF EXISTS private_message_id,
DROP COLUMN IF EXISTS lease_until,
DROP COLUMN IF EXISTS lease_token,
DROP COLUMN IF EXISTS next_attempt_at;
DROP INDEX IF EXISTS public.broadcasts_enumeration_idx;
ALTER TABLE public.broadcasts
DROP CONSTRAINT IF EXISTS broadcasts_sent_failed_within_materialized_check,
DROP CONSTRAINT IF EXISTS broadcasts_enumeration_cursor_check,
DROP CONSTRAINT IF EXISTS broadcasts_entities_array_check;
ALTER TABLE public.broadcasts
DROP COLUMN IF EXISTS failed_count,
DROP COLUMN IF EXISTS sent_count,
DROP COLUMN IF EXISTS materialized_count,
DROP COLUMN IF EXISTS enumeration_done,
DROP COLUMN IF EXISTS enumeration_cursor_user_id,
DROP COLUMN IF EXISTS snapshot_max_user_id,
DROP COLUMN IF EXISTS entities;
ALTER TABLE public.broadcasts
DROP CONSTRAINT IF EXISTS broadcasts_target_count_check;
ALTER TABLE public.broadcasts
ALTER COLUMN target_count TYPE integer,
ALTER COLUMN target_count SET DEFAULT 0;
ALTER TABLE public.broadcasts
RENAME COLUMN target_count TO total_count;

View file

@ -0,0 +1,131 @@
-- Upgrades our existing broadcasts/broadcast_recipients tables (created by
-- 20260714003131_system_broadcasts.up.sql, already applied in production) to
-- upstream gramsrv's richer design: formatted entities, incremental
-- materialization of "all"-target campaigns instead of one giant upfront
-- INSERT, and a lease-based delivery worker safe against duplicate sends if
-- the worker restarts mid-cycle.
--
-- This is an ALTER-based migration on purpose: it must not DROP/CREATE these
-- tables, because production already has rows in them (including 'sent'
-- rows from the old code path that never recorded a private_message_id/
-- message_box_id/pts, since that tracking didn't exist yet). The CHECK
-- constraints below explicitly carve out that legacy shape as valid
-- alongside the new fully-tracked shape, so this migration applies cleanly
-- against live data without any hand-editing.
-- broadcasts: rename total_count -> target_count (upstream's name for the
-- same "how many recipients this campaign targets" figure) and widen it to
-- bigint to match. A rename is metadata-only, so existing values survive
-- untouched.
ALTER TABLE public.broadcasts
RENAME COLUMN total_count TO target_count;
ALTER TABLE public.broadcasts
ALTER COLUMN target_count TYPE bigint,
ALTER COLUMN target_count SET DEFAULT 0;
ALTER TABLE public.broadcasts
ADD CONSTRAINT broadcasts_target_count_check CHECK (target_count >= 0);
ALTER TABLE public.broadcasts
ADD COLUMN entities jsonb NOT NULL DEFAULT '[]'::jsonb,
ADD COLUMN snapshot_max_user_id bigint NOT NULL DEFAULT 0,
ADD COLUMN enumeration_cursor_user_id bigint NOT NULL DEFAULT 0,
ADD COLUMN enumeration_done boolean NOT NULL DEFAULT false,
ADD COLUMN materialized_count bigint NOT NULL DEFAULT 0 CHECK (materialized_count >= 0),
ADD COLUMN sent_count bigint NOT NULL DEFAULT 0 CHECK (sent_count >= 0),
ADD COLUMN failed_count bigint NOT NULL DEFAULT 0 CHECK (failed_count >= 0);
ALTER TABLE public.broadcasts
ADD CONSTRAINT broadcasts_entities_array_check CHECK (jsonb_typeof(entities) = 'array');
-- Backfill: every pre-existing broadcast was created by the old code path,
-- which inserted every recipient row upfront in one transaction -- so from
-- the new model's point of view enumeration is already complete and fully
-- materialized for every one of them.
UPDATE public.broadcasts
SET enumeration_done = true,
enumeration_cursor_user_id = snapshot_max_user_id,
materialized_count = target_count;
-- Backfill sent_count/failed_count from the actual recipient rows rather
-- than trusting target_count, so old broadcasts read correctly under the
-- new derived-elsewhere-no-more columns instead of showing zeros.
UPDATE public.broadcasts b
SET sent_count = counts.sent_count,
failed_count = counts.failed_count
FROM (
SELECT broadcast_id,
count(*) FILTER (WHERE status = 'sent')::bigint AS sent_count,
count(*) FILTER (WHERE status = 'failed')::bigint AS failed_count
FROM public.broadcast_recipients
GROUP BY broadcast_id
) AS counts
WHERE b.id = counts.broadcast_id;
ALTER TABLE public.broadcasts
ADD CONSTRAINT broadcasts_enumeration_cursor_check
CHECK (enumeration_cursor_user_id >= 0 AND enumeration_cursor_user_id <= snapshot_max_user_id),
ADD CONSTRAINT broadcasts_sent_failed_within_materialized_check
CHECK (sent_count + failed_count <= materialized_count);
CREATE INDEX broadcasts_enumeration_idx ON public.broadcasts (id)
WHERE target_mode = 'all' AND NOT enumeration_done;
-- broadcast_recipients: add the lease-based delivery columns. Defaults give
-- every existing row sane values (next_attempt_at = now(), no active lease,
-- no tracked delivery identifiers) with no data loss.
ALTER TABLE public.broadcast_recipients
ADD COLUMN next_attempt_at timestamptz NOT NULL DEFAULT now(),
ADD COLUMN lease_token varchar(64) NOT NULL DEFAULT '',
ADD COLUMN lease_until timestamptz,
ADD COLUMN private_message_id bigint NOT NULL DEFAULT 0,
ADD COLUMN message_box_id integer NOT NULL DEFAULT 0,
ADD COLUMN pts integer NOT NULL DEFAULT 0,
ADD COLUMN updated_at timestamptz NOT NULL DEFAULT now();
-- The original table had no CHECK on status at all, so widening the allowed
-- set to include 'processing' needs no data fixup -- every existing row is
-- already 'pending', 'sent', or 'failed'.
ALTER TABLE public.broadcast_recipients
ADD CONSTRAINT broadcast_recipients_status_check
CHECK (status IN ('pending', 'processing', 'sent', 'failed')),
ADD CONSTRAINT broadcast_recipients_attempts_check CHECK (attempts >= 0);
-- The landmine: upstream's CHECK requires every 'sent' row to carry a
-- positive private_message_id/message_box_id/pts. Production already has
-- 'sent' rows from before that tracking existed, all with those columns at
-- their just-added default of 0. Rather than reject that data (or worse,
-- silently corrupt it with fabricated ids), this CHECK treats
-- "sent with all three still 0" as a legitimate legacy/untracked case,
-- alongside the real "sent with all three populated" case. New code always
-- populates them on a genuine send, so only pre-migration rows will ever
-- take the legacy branch.
ALTER TABLE public.broadcast_recipients
ADD CONSTRAINT broadcast_recipients_sent_tracking_check
CHECK (
(status = 'sent' AND sent_at IS NOT NULL AND (
(private_message_id > 0 AND message_box_id > 0 AND pts > 0)
OR
(private_message_id = 0 AND message_box_id = 0 AND pts = 0)
))
OR
(status <> 'sent' AND private_message_id = 0 AND message_box_id = 0 AND pts = 0 AND sent_at IS NULL)
);
-- No legacy carve-out needed here: lease_token/lease_until default to
-- ''/NULL, and no pre-existing row is 'processing' (that status didn't
-- exist before this migration), so every existing row already satisfies the
-- "not processing => no lease" branch.
ALTER TABLE public.broadcast_recipients
ADD CONSTRAINT broadcast_recipients_lease_check
CHECK (
(status = 'processing' AND lease_token <> '' AND lease_until IS NOT NULL)
OR
(status <> 'processing' AND lease_token = '' AND lease_until IS NULL)
);
CREATE INDEX broadcast_recipients_pending_next_attempt_idx
ON public.broadcast_recipients (next_attempt_at, id)
WHERE status = 'pending';
CREATE INDEX broadcast_recipients_processing_idx
ON public.broadcast_recipients (lease_until, id)
WHERE status = 'processing';

View file

@ -222,13 +222,18 @@ type AccountService interface {
// BroadcastService creates and lists system broadcast campaigns (a message // BroadcastService creates and lists system broadcast campaigns (a message
// from domain.OfficialSystemUserID to all or a hand-picked list of users). // from domain.OfficialSystemUserID to all or a hand-picked list of users).
// Delivery itself happens out-of-band via a worker draining the durable // Both recipient enumeration and delivery happen out-of-band via a worker
// recipient outbox created here -- this interface only enqueues and reads // draining the durable outbox created here -- this interface only enqueues
// back, so CreateBroadcast never blocks on however many recipients there // and reads back, so CreateBroadcast never blocks on however many
// are. Resolving "all users" into an explicit id list is the caller's job // recipients there are.
// (cmd/telesrv-admin's readstore, the same place every other account list //
// query already lives), not this service's -- it always receives an // For domain.BroadcastTargetAll, recipientUserIDs must be empty: the
// already-resolved id list. // service snapshots the current max eligible user id itself and the worker
// enumerates it incrementally, so a huge user base never has to cross the
// admin HTTP boundary as an explicit id list. For
// domain.BroadcastTargetSelected, recipientUserIDs is the operator-picked
// list, already resolved by the caller (cmd/telesrv-admin's readstore
// proxy, the same place every other account list query already lives).
type BroadcastService interface { type BroadcastService interface {
Create(ctx context.Context, message string, targetMode domain.BroadcastTargetMode, recipientUserIDs []int64, createdBy string) (domain.Broadcast, error) Create(ctx context.Context, message string, targetMode domain.BroadcastTargetMode, recipientUserIDs []int64, createdBy string) (domain.Broadcast, error)
List(ctx context.Context, beforeID int64, limit int) ([]domain.Broadcast, bool, error) List(ctx context.Context, beforeID int64, limit int) ([]domain.Broadcast, bool, error)
@ -900,11 +905,11 @@ type DeleteBotRequest struct {
BotUserID int64 `json:"bot_user_id"` BotUserID int64 `json:"bot_user_id"`
} }
// CreateBroadcastRequest's UserIDs is always an already-resolved recipient // CreateBroadcastRequest's UserIDs carries the operator-picked recipient
// list -- for TargetMode "all" the caller (cmd/telesrv-admin's readstore // list for TargetMode "selected" only. For TargetMode "all", UserIDs must be
// proxy) has already turned "every user" into an explicit id list before // empty: the admin service snapshots the current eligible user set itself
// this reaches the admin service, so CreateBroadcast never has to know how // and the broadcast worker enumerates it incrementally, so "every user"
// to enumerate accounts itself. // never has to cross the admin HTTP boundary as an explicit id list.
type CreateBroadcastRequest struct { type CreateBroadcastRequest struct {
CommandMeta CommandMeta
Message string `json:"message"` Message string `json:"message"`
@ -1770,11 +1775,11 @@ func (s *Service) DeleteBot(ctx context.Context, req DeleteBotRequest) (CommandR
} }
// CreateBroadcast enqueues a system-broadcast (a message from // CreateBroadcast enqueues a system-broadcast (a message from
// domain.OfficialSystemUserID) to an already-resolved recipient list. // domain.OfficialSystemUserID) to all users or an already-resolved
// Delivery happens out-of-band via the broadcast worker draining the durable // "selected" recipient list. Both recipient enumeration (for "all") and
// recipient rows this creates -- the command completes as soon as the // delivery happen out-of-band via the broadcast worker -- the command
// recipient snapshot is written, never waiting on however many sends that // completes as soon as the campaign is snapshotted, never waiting on however
// implies. // many sends that implies.
func (s *Service) CreateBroadcast(ctx context.Context, req CreateBroadcastRequest) (CommandResult, error) { func (s *Service) CreateBroadcast(ctx context.Context, req CreateBroadcastRequest) (CommandResult, error) {
if s == nil || s.broadcast == nil { if s == nil || s.broadcast == nil {
return CommandResult{}, fmt.Errorf("admin broadcast dependency is not configured") return CommandResult{}, fmt.Errorf("admin broadcast dependency is not configured")
@ -1783,13 +1788,25 @@ func (s *Service) CreateBroadcast(ctx context.Context, req CreateBroadcastReques
if message == "" { if message == "" {
return CommandResult{}, domain.ErrBroadcastMessageEmpty return CommandResult{}, domain.ErrBroadcastMessageEmpty
} }
if len(message) > domain.MaxBroadcastMessageBytes {
return CommandResult{}, domain.ErrBroadcastMessageTooLong
}
targetMode := domain.BroadcastTargetMode(req.TargetMode) targetMode := domain.BroadcastTargetMode(req.TargetMode)
if targetMode != domain.BroadcastTargetAll && targetMode != domain.BroadcastTargetSelected { switch targetMode {
case domain.BroadcastTargetAll:
if len(req.UserIDs) != 0 {
return CommandResult{}, domain.ErrBroadcastInvalid return CommandResult{}, domain.ErrBroadcastInvalid
} }
case domain.BroadcastTargetSelected:
if len(req.UserIDs) == 0 { if len(req.UserIDs) == 0 {
return CommandResult{}, domain.ErrBroadcastNoRecipients return CommandResult{}, domain.ErrBroadcastNoRecipients
} }
if len(req.UserIDs) > domain.MaxBroadcastSelectedRecipients {
return CommandResult{}, domain.ErrBroadcastInvalid
}
default:
return CommandResult{}, domain.ErrBroadcastInvalid
}
return s.runCommand(ctx, req.CommandMeta, ActionCreateBroadcast, 0, domain.Peer{}, req, func() (CommandResult, error) { return s.runCommand(ctx, req.CommandMeta, ActionCreateBroadcast, 0, domain.Peer{}, req, func() (CommandResult, error) {
details := map[string]any{ details := map[string]any{
"target_mode": string(targetMode), "target_mode": string(targetMode),
@ -1804,7 +1821,7 @@ func (s *Service) CreateBroadcast(ctx context.Context, req CreateBroadcastReques
return CommandResult{Details: details}, err return CommandResult{Details: details}, err
} }
details["broadcast_id"] = created.ID details["broadcast_id"] = created.ID
details["total_count"] = created.TotalCount details["target_count"] = created.TargetCount
return CommandResult{Message: "broadcast created", Details: details}, nil return CommandResult{Message: "broadcast created", Details: details}, nil
}) })
} }

View file

@ -1,10 +1,35 @@
// Package broadcast implements admin-triggered system message campaigns: // Package broadcast implements admin-triggered system message campaigns:
// sending a message from the official system account (domain.OfficialSystemUserID, // sending a message from the official system account (domain.OfficialSystemUserID,
// 777000) to every user or a hand-picked list. Delivery is a durable outbox // 777000) to every user or a hand-picked list.
// (store.BroadcastStore's recipient rows) drained by a periodic Worker, //
// mirroring internal/app/verification's notification outbox -- the admin // A "selected" campaign's recipient rows are all inserted at creation, since
// action only snapshots the recipient list and returns, never sending // that list is bounded by domain.MaxBroadcastSelectedRecipients. An "all"
// potentially thousands of messages inline within one HTTP request. // campaign instead only snapshots the current max eligible user id at
// creation, and store.BroadcastStore.MaterializeBroadcastRecipients walks
// that range incrementally, a bounded batch per worker cycle -- so creating
// a campaign for a large user base is a single cheap insert, not one giant
// blocking transaction.
//
// Delivery is a lease-based claim cycle (store.BroadcastStore.
// ClaimBroadcastRecipients/CompleteBroadcastRecipient/ReleaseBroadcastRecipient):
// a worker leases a bounded batch of eligible rows for a fixed duration,
// delivers each one, and closes it out. A lease that is never renewed simply
// expires, so a worker crash mid-cycle cannot strand rows in 'processing'
// forever, and a future multi-instance worker can run the same cycle
// concurrently without two instances ever believing they hold the same
// row's lease at once.
//
// Delivery itself goes through messageSender.SendPrivateText -- the same
// store-layer send path internal/app/bots's sendServiceBotReplyResult calls
// directly (bypassing the auth-checked app.messages.Service wrapper, which
// requires SenderUserID == the authenticated caller) -- rather than
// duplicating message/pts/dispatch-outbox creation here. SendPrivateText's
// random_id dedup is what actually closes the small race a lease alone
// leaves open: if a lease expires and gets reclaimed while the original
// holder's send is still in flight, both attempts use the same
// (broadcastID, userID)-derived random id (see stableBroadcastRandomID), so
// the store resolves them to the very same message instead of sending
// twice, no matter which claim ends up recording it.
package broadcast package broadcast
import ( import (
@ -13,6 +38,8 @@ import (
"hash/fnv" "hash/fnv"
"strconv" "strconv"
"strings" "strings"
"time"
"unicode/utf8"
"go.uber.org/zap" "go.uber.org/zap"
@ -21,10 +48,7 @@ import (
) )
// messageSender is the narrow port this package needs from // messageSender is the narrow port this package needs from
// store.MessageStore: sending a message with an arbitrary SenderUserID, the // store.MessageStore: sending a message with an arbitrary SenderUserID.
// way internal/app/bots's sendServiceBotReplyResult calls it directly at the
// store layer rather than through the auth-checked app.messages.Service
// wrapper (which requires SenderUserID == the authenticated caller).
type messageSender interface { type messageSender interface {
SendPrivateText(ctx context.Context, req domain.SendPrivateTextRequest) (domain.SendPrivateTextResult, error) SendPrivateText(ctx context.Context, req domain.SendPrivateTextRequest) (domain.SendPrivateTextResult, error)
} }
@ -69,22 +93,80 @@ func WithLogger(log *zap.Logger) Option {
// Ready reports whether both the store and the sender are wired. // Ready reports whether both the store and the sender are wired.
func (s *Service) Ready() bool { return s != nil && s.store != nil && s.messages != nil } func (s *Service) Ready() bool { return s != nil && s.store != nil && s.messages != nil }
// Create validates and snapshots a new broadcast's recipient set, then func normalizeRequest(message string, mode domain.BroadcastTargetMode, selectedUserIDs []int64) (string, []int64, error) {
// returns immediately: delivery happens asynchronously via RunSendCycle, so message = strings.TrimSpace(message)
// this never blocks an admin HTTP request on however many recipients there if message == "" {
// are. return "", nil, domain.ErrBroadcastMessageEmpty
func (s *Service) Create(ctx context.Context, message string, targetMode domain.BroadcastTargetMode, recipientUserIDs []int64, createdBy string) (domain.Broadcast, error) { }
if !utf8.ValidString(message) || len(message) > domain.MaxBroadcastMessageBytes {
return "", nil, domain.ErrBroadcastMessageTooLong
}
switch mode {
case domain.BroadcastTargetAll:
if len(selectedUserIDs) != 0 {
return "", nil, domain.ErrBroadcastInvalid
}
return message, nil, nil
case domain.BroadcastTargetSelected:
if len(selectedUserIDs) == 0 {
return "", nil, domain.ErrBroadcastNoRecipients
}
if len(selectedUserIDs) > domain.MaxBroadcastSelectedRecipients {
return "", nil, domain.ErrBroadcastInvalid
}
seen := make(map[int64]struct{}, len(selectedUserIDs))
ids := make([]int64, 0, len(selectedUserIDs))
for _, userID := range selectedUserIDs {
if userID <= 0 || domain.IsSystemUserID(userID) {
return "", nil, domain.ErrBroadcastRecipientInvalid
}
if _, ok := seen[userID]; ok {
return "", nil, domain.ErrBroadcastRecipientInvalid
}
seen[userID] = struct{}{}
ids = append(ids, userID)
}
return message, ids, nil
default:
return "", nil, domain.ErrBroadcastInvalid
}
}
// Preview validates and counts a campaign's intended recipient set without
// creating anything, so an admin UI can show "this will reach N users"
// before committing.
func (s *Service) Preview(ctx context.Context, message string, mode domain.BroadcastTargetMode, selectedUserIDs []int64) (int64, error) {
if s == nil || s.store == nil {
return 0, fmt.Errorf("broadcast store is not configured")
}
_, ids, err := normalizeRequest(message, mode, selectedUserIDs)
if err != nil {
return 0, err
}
return s.store.PreviewBroadcastRecipients(ctx, mode, ids)
}
// Create validates and snapshots a new broadcast, then returns immediately:
// delivery (and, for "all" mode, recipient enumeration itself) happens
// asynchronously via the Worker's RunCycle, so this never blocks an admin
// HTTP request on however many recipients there are.
//
// Entities are derived automatically from the plain-text message (mentions,
// hashtags, cashtags, bot commands -- see domain.DetectAutomaticMessageEntities),
// not operator-composed: there is currently no admin UI for hand-authoring
// bold/italic/link spans on a broadcast, so this only gets a broadcast the
// same clickable-entity rendering any other plain-text message with an
// @mention or #hashtag already gets.
func (s *Service) Create(ctx context.Context, message string, mode domain.BroadcastTargetMode, selectedUserIDs []int64, createdBy string) (domain.Broadcast, error) {
if s == nil || s.store == nil { if s == nil || s.store == nil {
return domain.Broadcast{}, fmt.Errorf("broadcast store is not configured") return domain.Broadcast{}, fmt.Errorf("broadcast store is not configured")
} }
message = strings.TrimSpace(message) message, ids, err := normalizeRequest(message, mode, selectedUserIDs)
if message == "" { if err != nil {
return domain.Broadcast{}, domain.ErrBroadcastMessageEmpty return domain.Broadcast{}, err
} }
if targetMode != domain.BroadcastTargetAll && targetMode != domain.BroadcastTargetSelected { entities := domain.DetectAutomaticMessageEntities(message, nil)
return domain.Broadcast{}, domain.ErrBroadcastInvalid return s.store.CreateBroadcast(ctx, message, entities, mode, ids, strings.TrimSpace(createdBy))
}
return s.store.CreateBroadcast(ctx, message, targetMode, recipientUserIDs, createdBy)
} }
// List pages broadcasts newest-first. // List pages broadcasts newest-first.
@ -103,51 +185,80 @@ func (s *Service) Get(ctx context.Context, id int64) (domain.Broadcast, bool, er
return s.store.BroadcastByID(ctx, id) return s.store.BroadcastByID(ctx, id)
} }
// RunSendCycle drains up to limit pending recipient rows, sending each from // CycleResult reports one worker cycle's outcome.
// domain.OfficialSystemUserID. One recipient's failure (blocked account, type CycleResult struct {
// deleted account, transient error) never blocks the rest of the batch. Materialized int
func (s *Service) RunSendCycle(ctx context.Context, limit int) (sent int, err error) { Claimed int
if s == nil || !s.Ready() { Sent int
return 0, nil Failed int
} }
pending, err := s.store.PendingBroadcastRecipients(ctx, limit)
// RunCycle advances "all"-mode enumeration by up to materializeBatch rows,
// then claims up to deliveryBatch eligible recipient rows under leaseToken
// for lease, delivering each one via SendPrivateText. One recipient's
// failure (blocked account, deleted account, transient error) never blocks
// the rest of the batch.
func (s *Service) RunCycle(ctx context.Context, leaseToken string, materializeBatch, deliveryBatch int, lease time.Duration) (CycleResult, error) {
var result CycleResult
if !s.Ready() {
return result, nil
}
materialized, err := s.store.MaterializeBroadcastRecipients(ctx, materializeBatch)
if err != nil { if err != nil {
return 0, err return result, err
} }
for _, recipient := range pending { result.Materialized = materialized
claims, err := s.store.ClaimBroadcastRecipients(ctx, leaseToken, deliveryBatch, lease)
if err != nil {
return result, err
}
result.Claimed = len(claims)
for _, claim := range claims {
if err := ctx.Err(); err != nil { if err := ctx.Err(); err != nil {
return sent, err return result, err
} }
_, sendErr := s.messages.SendPrivateText(ctx, domain.SendPrivateTextRequest{ if err := s.deliverClaim(ctx, claim); err != nil {
result.Failed++
if releaseErr := s.store.ReleaseBroadcastRecipient(ctx, claim, err.Error()); releaseErr != nil {
s.log.Warn("release broadcast recipient failed",
zap.Int64("recipient_id", claim.RecipientID),
zap.Int64("broadcast_id", claim.BroadcastID),
zap.Error(releaseErr))
}
continue
}
result.Sent++
}
return result, nil
}
func (s *Service) deliverClaim(ctx context.Context, claim store.BroadcastRecipientClaim) error {
send, err := s.messages.SendPrivateText(ctx, domain.SendPrivateTextRequest{
SenderUserID: domain.OfficialSystemUserID, SenderUserID: domain.OfficialSystemUserID,
RecipientUserID: recipient.UserID, RecipientUserID: claim.UserID,
// A stable id derived from (broadcast, recipient) makes reprocessing // A stable id derived from (broadcast, recipient) makes redelivering
// this exact row idempotent at the store layer's random_id dedup, // this exact row idempotent at the store layer's random_id dedup: if
// instead of risking a duplicate message if this worker crashes // this claim's lease expires and gets reclaimed while a prior send is
// between sending and marking the row delivered. // still in flight, both resolve to the same message instead of
RandomID: stableBroadcastRandomID(recipient.BroadcastID, recipient.UserID), // sending twice.
Message: recipient.Message, RandomID: stableBroadcastRandomID(claim.BroadcastID, claim.UserID),
Message: claim.Message,
Entities: claim.Entities,
}) })
if sendErr != nil { if err != nil {
if markErr := s.store.MarkBroadcastRecipientFailed(ctx, recipient.RecipientID, sendErr.Error()); markErr != nil { return err
s.log.Warn("mark broadcast recipient failed",
zap.Int64("recipient_id", recipient.RecipientID), zap.Error(markErr))
} }
continue msg := send.RecipientMessage
if err := s.store.CompleteBroadcastRecipient(ctx, claim, msg.UID, msg.ID, msg.Pts); err != nil {
return err
} }
if markErr := s.store.MarkBroadcastRecipientSent(ctx, recipient.RecipientID); markErr != nil { return nil
s.log.Warn("mark broadcast recipient sent",
zap.Int64("recipient_id", recipient.RecipientID), zap.Error(markErr))
continue
}
sent++
}
return sent, nil
} }
// stableBroadcastRandomID derives a random_id from (broadcastID, userID) so // stableBroadcastRandomID derives a random_id from (broadcastID, userID) so
// re-processing the same recipient row (after a crash, before it was marked // re-processing the same recipient row (after a lease is reclaimed, before
// delivered) resolves to the same send instead of a duplicate message. // it was recorded delivered) resolves to the same send instead of a
// duplicate message.
func stableBroadcastRandomID(broadcastID, userID int64) int64 { func stableBroadcastRandomID(broadcastID, userID int64) int64 {
h := fnv.New64a() h := fnv.New64a()
_, _ = h.Write([]byte(strconv.FormatInt(broadcastID, 10) + ":" + strconv.FormatInt(userID, 10))) _, _ = h.Write([]byte(strconv.FormatInt(broadcastID, 10) + ":" + strconv.FormatInt(userID, 10)))

View file

@ -4,6 +4,7 @@ import (
"context" "context"
"errors" "errors"
"testing" "testing"
"time"
"telesrv/internal/domain" "telesrv/internal/domain"
"telesrv/internal/store/memory" "telesrv/internal/store/memory"
@ -12,6 +13,7 @@ import (
type fakeSender struct { type fakeSender struct {
sent []domain.SendPrivateTextRequest sent []domain.SendPrivateTextRequest
failFor map[int64]bool // fail every send to this recipient user id failFor map[int64]bool // fail every send to this recipient user id
nextID int
} }
func (f *fakeSender) SendPrivateText(_ context.Context, req domain.SendPrivateTextRequest) (domain.SendPrivateTextResult, error) { func (f *fakeSender) SendPrivateText(_ context.Context, req domain.SendPrivateTextRequest) (domain.SendPrivateTextResult, error) {
@ -19,14 +21,21 @@ func (f *fakeSender) SendPrivateText(_ context.Context, req domain.SendPrivateTe
return domain.SendPrivateTextResult{}, errors.New("simulated send failure") return domain.SendPrivateTextResult{}, errors.New("simulated send failure")
} }
f.sent = append(f.sent, req) f.sent = append(f.sent, req)
return domain.SendPrivateTextResult{}, nil f.nextID++
return domain.SendPrivateTextResult{
RecipientMessage: domain.Message{
ID: f.nextID,
UID: int64(f.nextID),
Pts: f.nextID,
},
}, nil
} }
func TestCreateValidatesInput(t *testing.T) { func TestCreateValidatesInput(t *testing.T) {
svc := NewService(memory.NewBroadcastStore(), WithMessageSender(&fakeSender{})) svc := NewService(memory.NewBroadcastStore(), WithMessageSender(&fakeSender{}))
ctx := context.Background() ctx := context.Background()
if _, err := svc.Create(ctx, " ", domain.BroadcastTargetAll, []int64{1}, "admin"); !errors.Is(err, domain.ErrBroadcastMessageEmpty) { if _, err := svc.Create(ctx, " ", domain.BroadcastTargetAll, nil, "admin"); !errors.Is(err, domain.ErrBroadcastMessageEmpty) {
t.Fatalf("empty message: err = %v, want ErrBroadcastMessageEmpty", err) t.Fatalf("empty message: err = %v, want ErrBroadcastMessageEmpty", err)
} }
if _, err := svc.Create(ctx, "hi", domain.BroadcastTargetMode("bogus"), []int64{1}, "admin"); !errors.Is(err, domain.ErrBroadcastInvalid) { if _, err := svc.Create(ctx, "hi", domain.BroadcastTargetMode("bogus"), []int64{1}, "admin"); !errors.Is(err, domain.ErrBroadcastInvalid) {
@ -35,21 +44,33 @@ func TestCreateValidatesInput(t *testing.T) {
if _, err := svc.Create(ctx, "hi", domain.BroadcastTargetSelected, nil, "admin"); !errors.Is(err, domain.ErrBroadcastNoRecipients) { if _, err := svc.Create(ctx, "hi", domain.BroadcastTargetSelected, nil, "admin"); !errors.Is(err, domain.ErrBroadcastNoRecipients) {
t.Fatalf("no recipients: err = %v, want ErrBroadcastNoRecipients", err) t.Fatalf("no recipients: err = %v, want ErrBroadcastNoRecipients", err)
} }
tooMany := make([]int64, domain.MaxBroadcastSelectedRecipients+1)
for i := range tooMany {
tooMany[i] = int64(1000 + i)
}
if _, err := svc.Create(ctx, "hi", domain.BroadcastTargetSelected, tooMany, "admin"); !errors.Is(err, domain.ErrBroadcastInvalid) {
t.Fatalf("too many recipients: err = %v, want ErrBroadcastInvalid", err)
}
created, err := svc.Create(ctx, " News! ", domain.BroadcastTargetSelected, []int64{10, 20, 20}, "admin") // A duplicate id in the selected list is rejected outright, not silently
// collapsed: the caller's list should already be a set.
if _, err := svc.Create(ctx, "hi", domain.BroadcastTargetSelected, []int64{10, 20, 20}, "admin"); !errors.Is(err, domain.ErrBroadcastRecipientInvalid) {
t.Fatalf("duplicate recipient: err = %v, want ErrBroadcastRecipientInvalid", err)
}
created, err := svc.Create(ctx, " News! ", domain.BroadcastTargetSelected, []int64{10, 20}, "admin")
if err != nil { if err != nil {
t.Fatalf("Create: %v", err) t.Fatalf("Create: %v", err)
} }
if created.Message != "News!" { if created.Message != "News!" {
t.Fatalf("Message = %q, want trimmed %q", created.Message, "News!") t.Fatalf("Message = %q, want trimmed %q", created.Message, "News!")
} }
// The duplicate recipient (20 twice) collapses to one row. if created.TargetCount != 2 {
if created.TotalCount != 2 { t.Fatalf("TargetCount = %d, want 2", created.TargetCount)
t.Fatalf("TotalCount = %d, want 2 (duplicate recipient collapsed)", created.TotalCount)
} }
} }
func TestRunSendCycleDeliversAndCounts(t *testing.T) { func TestRunCycleDeliversAndCounts(t *testing.T) {
store := memory.NewBroadcastStore() store := memory.NewBroadcastStore()
sender := &fakeSender{} sender := &fakeSender{}
svc := NewService(store, WithMessageSender(sender)) svc := NewService(store, WithMessageSender(sender))
@ -60,12 +81,12 @@ func TestRunSendCycleDeliversAndCounts(t *testing.T) {
t.Fatalf("Create: %v", err) t.Fatalf("Create: %v", err)
} }
sent, err := svc.RunSendCycle(ctx, 10) result, err := svc.RunCycle(ctx, "lease-1", 100, 10, 30*time.Second)
if err != nil { if err != nil {
t.Fatalf("RunSendCycle: %v", err) t.Fatalf("RunCycle: %v", err)
} }
if sent != 3 { if result.Sent != 3 {
t.Fatalf("sent = %d, want 3", sent) t.Fatalf("Sent = %d, want 3", result.Sent)
} }
if len(sender.sent) != 3 { if len(sender.sent) != 3 {
t.Fatalf("sender received %d sends, want 3", len(sender.sent)) t.Fatalf("sender received %d sends, want 3", len(sender.sent))
@ -88,16 +109,16 @@ func TestRunSendCycleDeliversAndCounts(t *testing.T) {
} }
// A second cycle finds nothing left pending. // A second cycle finds nothing left pending.
sent, err = svc.RunSendCycle(ctx, 10) result, err = svc.RunCycle(ctx, "lease-2", 100, 10, 30*time.Second)
if err != nil { if err != nil {
t.Fatalf("RunSendCycle (second): %v", err) t.Fatalf("RunCycle (second): %v", err)
} }
if sent != 0 { if result.Claimed != 0 {
t.Fatalf("second cycle sent = %d, want 0 (nothing pending)", sent) t.Fatalf("second cycle claimed = %d, want 0 (nothing pending)", result.Claimed)
} }
} }
func TestRunSendCycleRetriesThenTerminatesFailures(t *testing.T) { func TestRunCycleRetriesThenTerminatesFailures(t *testing.T) {
store := memory.NewBroadcastStore() store := memory.NewBroadcastStore()
sender := &fakeSender{failFor: map[int64]bool{999: true}} sender := &fakeSender{failFor: map[int64]bool{999: true}}
svc := NewService(store, WithMessageSender(sender)) svc := NewService(store, WithMessageSender(sender))
@ -110,22 +131,49 @@ func TestRunSendCycleRetriesThenTerminatesFailures(t *testing.T) {
// Run one cycle per attempt, up to the cap; the row must stay pending // Run one cycle per attempt, up to the cap; the row must stay pending
// (retried) below the cap and become terminal at it. // (retried) below the cap and become terminal at it.
for i := 0; i < domain.MaxBroadcastRecipientAttempts; i++ { for i := 0; i < domain.MaxBroadcastRecipientAttempts; i++ {
sent, err := svc.RunSendCycle(ctx, 10) result, err := svc.RunCycle(ctx, "lease", 100, 10, 30*time.Second)
if err != nil { if err != nil {
t.Fatalf("RunSendCycle attempt %d: %v", i+1, err) t.Fatalf("RunCycle attempt %d: %v", i+1, err)
} }
if sent != 0 { if result.Sent != 0 || result.Failed != 1 {
t.Fatalf("attempt %d: sent = %d, want 0 (always fails)", i+1, sent) t.Fatalf("attempt %d: sent=%d failed=%d, want sent=0 failed=1 (always fails)", i+1, result.Sent, result.Failed)
} }
} }
// One more cycle: the row is now terminal ('failed'), so PendingBroadcastRecipients // One more cycle: the row is now terminal ('failed'), so nothing is left
// must not return it, and RunSendCycle finds nothing left to attempt. // to claim.
pending, err := store.PendingBroadcastRecipients(ctx, 10) result, err := svc.RunCycle(ctx, "lease-final", 100, 10, 30*time.Second)
if err != nil { if err != nil {
t.Fatalf("PendingBroadcastRecipients: %v", err) t.Fatalf("RunCycle (final): %v", err)
} }
if len(pending) != 0 { if result.Claimed != 0 {
t.Fatalf("pending = %+v, want empty (recipient should be terminally failed)", pending) t.Fatalf("final cycle claimed = %d, want 0 (recipient should be terminally failed)", result.Claimed)
}
}
func TestCreateAllModeSnapshotsWithoutExplicitIDs(t *testing.T) {
store := memory.NewBroadcastStore()
store.SeedEligibleUsers([]int64{1, 2, 3})
sender := &fakeSender{}
svc := NewService(store, WithMessageSender(sender))
ctx := context.Background()
created, err := svc.Create(ctx, "hello all", domain.BroadcastTargetAll, nil, "admin")
if err != nil {
t.Fatalf("Create: %v", err)
}
if created.TargetMode != domain.BroadcastTargetAll {
t.Fatalf("TargetMode = %q, want all", created.TargetMode)
}
result, err := svc.RunCycle(ctx, "lease", 100, 100, 30*time.Second)
if err != nil {
t.Fatalf("RunCycle: %v", err)
}
if result.Materialized != 3 {
t.Fatalf("Materialized = %d, want 3", result.Materialized)
}
if result.Sent != 3 {
t.Fatalf("Sent = %d, want 3", result.Sent)
} }
} }

View file

@ -2,59 +2,69 @@ package broadcast
import ( import (
"context" "context"
"crypto/rand"
"encoding/hex"
"time" "time"
"go.uber.org/zap" "go.uber.org/zap"
) )
// defaultInterval/defaultBatch match the shipped // WorkerConfig tunes the periodic materialize+delivery cycle. Non-positive
// TELESRV_BROADCAST_WORKER_INTERVAL/_BATCH defaults. // or out-of-range fields fall back to the defaults below (matching the
const ( // shipped TELESRV_BROADCAST_WORKER_* defaults).
defaultInterval = 3 * time.Second type WorkerConfig struct {
defaultBatch = 50 Interval time.Duration
) Lease time.Duration
MaterializeBatch int
DeliveryBatch int
}
// Worker drains the broadcast delivery outbox. // Worker drains the broadcast delivery outbox and, for "all"-mode
// campaigns, the recipient-enumeration backlog.
// //
// A broadcast is created together with its recipient snapshot, never with the // A broadcast is created together with only its target snapshot, never with
// sends themselves: an admin creating a broadcast for every user must not // the enumeration or the sends themselves: an admin creating a broadcast for
// wait on however long that takes. Delivery is therefore a separate, // every user must not wait on however long that would take. Both
// retrying cycle over durable rows, and this worker is only its cadence. // materialization and delivery are therefore a separate, retrying cycle over
// durable rows, and this worker is only its cadence.
type Worker struct { type Worker struct {
service *Service service *Service
logger *zap.Logger config WorkerConfig
interval time.Duration log *zap.Logger
batch int
} }
// NewWorker creates the periodic delivery worker. Non-positive // NewWorker creates the periodic worker.
// interval/batch fall back to the shipped defaults. func NewWorker(service *Service, config WorkerConfig, log *zap.Logger) *Worker {
func NewWorker(service *Service, logger *zap.Logger, interval time.Duration, batch int) *Worker { if config.Interval <= 0 {
if logger == nil { config.Interval = 3 * time.Second
logger = zap.NewNop()
} }
if interval <= 0 { if config.Lease <= 0 {
interval = defaultInterval config.Lease = 30 * time.Second
} }
if batch <= 0 { if config.MaterializeBatch <= 0 || config.MaterializeBatch > 1000 {
batch = defaultBatch config.MaterializeBatch = 200
} }
return &Worker{service: service, logger: logger, interval: interval, batch: batch} if config.DeliveryBatch <= 0 || config.DeliveryBatch > 500 {
config.DeliveryBatch = 50
}
if log == nil {
log = zap.NewNop()
}
return &Worker{service: service, config: config, log: log}
} }
// Run delivers one batch immediately and then on every tick until ctx is // Run advances one cycle immediately and then on every tick until ctx is
// done. A not-ready service (missing store/sender) exits immediately with // done. A not-ready service (missing store/sender) exits immediately with
// one explicit log line instead of ticking forever over a no-op. // one explicit log line instead of ticking forever over a no-op.
func (w *Worker) Run(ctx context.Context) { func (w *Worker) Run(ctx context.Context) {
if w == nil { if w == nil || w.service == nil || !w.service.Ready() {
return if w != nil && w.log != nil {
w.log.Info("broadcast delivery worker disabled: not configured")
} }
if !w.service.Ready() {
w.logger.Info("broadcast delivery worker disabled: not configured")
return return
} }
w.runOnce(ctx) w.runOnce(ctx)
ticker := time.NewTicker(w.interval) ticker := time.NewTicker(w.config.Interval)
defer ticker.Stop() defer ticker.Stop()
for { for {
select { select {
@ -67,18 +77,23 @@ func (w *Worker) Run(ctx context.Context) {
} }
func (w *Worker) runOnce(ctx context.Context) { func (w *Worker) runOnce(ctx context.Context) {
if w == nil || w.service == nil { var tokenBytes [16]byte
if _, err := rand.Read(tokenBytes[:]); err != nil {
w.log.Error("generate broadcast lease token", zap.Error(err))
return return
} }
sent, err := w.service.RunSendCycle(ctx, w.batch) result, err := w.service.RunCycle(ctx, hex.EncodeToString(tokenBytes[:]), w.config.MaterializeBatch, w.config.DeliveryBatch, w.config.Lease)
if err != nil { if err != nil {
if ctx.Err() != nil { if ctx.Err() == nil {
w.log.Warn("broadcast delivery cycle failed", zap.Error(err))
}
return return
} }
w.logger.Warn("broadcast delivery cycle failed", zap.Int("sent", sent), zap.Int("batch", w.batch), zap.Error(err)) if result.Materialized > 0 || result.Claimed > 0 {
return w.log.Info("broadcast delivery cycle completed",
} zap.Int("materialized", result.Materialized),
if sent > 0 { zap.Int("claimed", result.Claimed),
w.logger.Info("broadcast delivery cycle completed", zap.Int("sent", sent), zap.Int("batch", w.batch)) zap.Int("sent", result.Sent),
zap.Int("failed", result.Failed))
} }
} }

View file

@ -623,11 +623,19 @@ type Config struct {
BotVerificationRequestRateWindow time.Duration BotVerificationRequestRateWindow time.Duration
// BroadcastWorkerInterval / BroadcastWorkerBatch drive the system-broadcast // BroadcastWorkerInterval / BroadcastWorkerBatch drive the system-broadcast
// delivery worker (internal/app/broadcast): an admin-created broadcast is // delivery worker (internal/app/broadcast): an admin-created broadcast is
// snapshotted into a durable per-recipient outbox immediately, and this // snapshotted immediately, and this worker both enumerates "all"-mode
// worker drains it in batches, so sending to thousands of users never blocks // recipients incrementally and drains delivery in batches, so sending to
// the admin action itself. // thousands of users never blocks the admin action itself.
BroadcastWorkerInterval time.Duration BroadcastWorkerInterval time.Duration
// BroadcastWorkerBatch bounds one cycle's delivery claims.
BroadcastWorkerBatch int BroadcastWorkerBatch int
// BroadcastWorkerMaterializeBatch bounds one cycle's "all"-mode recipient
// enumeration inserts.
BroadcastWorkerMaterializeBatch int
// BroadcastWorkerLease bounds how long a delivery worker holds a claimed
// recipient row before another cycle is allowed to reclaim it (e.g. after
// a crash mid-delivery).
BroadcastWorkerLease time.Duration
// HideThirdPartyVerification hides third-party bot verification instead of // HideThirdPartyVerification hides third-party bot verification instead of
// removing it: the admin panel drops its "Third-party marks" nav entry and // removing it: the admin panel drops its "Third-party marks" nav entry and
@ -1073,6 +1081,8 @@ func Load() (Config, error) {
BotVerificationRequestRateWindow: envDurationOr("TELESRV_BOT_VERIFICATION_REQUEST_RATE_WINDOW", 24*time.Hour), BotVerificationRequestRateWindow: envDurationOr("TELESRV_BOT_VERIFICATION_REQUEST_RATE_WINDOW", 24*time.Hour),
BroadcastWorkerInterval: envDurationOr("TELESRV_BROADCAST_WORKER_INTERVAL", 3*time.Second), BroadcastWorkerInterval: envDurationOr("TELESRV_BROADCAST_WORKER_INTERVAL", 3*time.Second),
BroadcastWorkerBatch: envIntOr("TELESRV_BROADCAST_WORKER_BATCH", 50), BroadcastWorkerBatch: envIntOr("TELESRV_BROADCAST_WORKER_BATCH", 50),
BroadcastWorkerMaterializeBatch: envIntOr("TELESRV_BROADCAST_WORKER_MATERIALIZE_BATCH", 200),
BroadcastWorkerLease: envDurationOr("TELESRV_BROADCAST_WORKER_LEASE", 30*time.Second),
HideThirdPartyVerification: envBoolOr("TELESRV_HIDE_THIRD_PARTY_VERIFICATION", true), HideThirdPartyVerification: envBoolOr("TELESRV_HIDE_THIRD_PARTY_VERIFICATION", true),
GroupCallCheckTTL: envDurationOr("TELESRV_GROUPCALL_CHECK_TTL", 45*time.Second), GroupCallCheckTTL: envDurationOr("TELESRV_GROUPCALL_CHECK_TTL", 45*time.Second),
@ -1387,6 +1397,15 @@ func validateVerificationConfig(cfg Config) error {
if cfg.BroadcastWorkerInterval <= 0 { if cfg.BroadcastWorkerInterval <= 0 {
return fmt.Errorf("TELESRV_BROADCAST_WORKER_INTERVAL must be positive") return fmt.Errorf("TELESRV_BROADCAST_WORKER_INTERVAL must be positive")
} }
if cfg.BroadcastWorkerLease <= 0 {
return fmt.Errorf("TELESRV_BROADCAST_WORKER_LEASE must be positive")
}
if cfg.BroadcastWorkerMaterializeBatch <= 0 || cfg.BroadcastWorkerMaterializeBatch > 1000 {
return fmt.Errorf("TELESRV_BROADCAST_WORKER_MATERIALIZE_BATCH must be 1..1000")
}
if cfg.BroadcastWorkerBatch <= 0 || cfg.BroadcastWorkerBatch > 500 {
return fmt.Errorf("TELESRV_BROADCAST_WORKER_BATCH must be 1..500")
}
if cfg.VerificationMaxActivePerUser < 0 || cfg.VerificationMaxActivePerUser > 50 { if cfg.VerificationMaxActivePerUser < 0 || cfg.VerificationMaxActivePerUser > 50 {
return fmt.Errorf("TELESRV_VERIFICATION_MAX_ACTIVE_PER_USER must be 0..50") return fmt.Errorf("TELESRV_VERIFICATION_MAX_ACTIVE_PER_USER must be 0..50")
} }

View file

@ -9,10 +9,11 @@ import (
type BroadcastTargetMode string type BroadcastTargetMode string
const ( const (
// BroadcastTargetAll snapshots every non-bot, non-system account at // BroadcastTargetAll snapshots every non-bot, non-system account as of
// creation time (mirrors the exclusion cmd/telesrv-admin's CountAccounts // creation time (mirrors the exclusion cmd/telesrv-admin's CountAccounts
// already applies: real users only, not @BotFather/@Stickers/@ChatBot/777000 // already applies: real users only, not @BotFather/@Stickers/@ChatBot/777000
// itself). // itself) by recording the current max user id and enumerating up to it
// incrementally, rather than resolving the whole list inline.
BroadcastTargetAll BroadcastTargetMode = "all" BroadcastTargetAll BroadcastTargetMode = "all"
// BroadcastTargetSelected sends only to the operator-picked user list // BroadcastTargetSelected sends only to the operator-picked user list
// carried on the create request. // carried on the create request.
@ -24,6 +25,11 @@ type BroadcastRecipientStatus string
const ( const (
BroadcastRecipientPending BroadcastRecipientStatus = "pending" BroadcastRecipientPending BroadcastRecipientStatus = "pending"
// BroadcastRecipientProcessing means a delivery worker currently holds a
// time-bounded lease on this row (see LeaseToken/LeaseUntil). If the
// worker dies before finishing, the lease simply expires and another
// worker cycle reclaims the row -- no separate crash-recovery pass needed.
BroadcastRecipientProcessing BroadcastRecipientStatus = "processing"
BroadcastRecipientSent BroadcastRecipientStatus = "sent" BroadcastRecipientSent BroadcastRecipientStatus = "sent"
// BroadcastRecipientFailed is terminal: MaxBroadcastRecipientAttempts was // BroadcastRecipientFailed is terminal: MaxBroadcastRecipientAttempts was
// reached, so the worker stops retrying this row. A blocked or deleted // reached, so the worker stops retrying this row. A blocked or deleted
@ -35,36 +41,90 @@ const (
// worker gives up and marks the row permanently failed. // worker gives up and marks the row permanently failed.
const MaxBroadcastRecipientAttempts = 5 const MaxBroadcastRecipientAttempts = 5
// MaxBroadcastMessageBytes caps a broadcast's message body, matching the
// broadcasts.message CHECK added in
// deploy/migrations/20260901000024_broadcast_lease_delivery_and_entities.up.sql.
const MaxBroadcastMessageBytes = 4096
// MaxBroadcastSelectedRecipients caps how many user ids one "selected"-mode
// broadcast may carry in its create request, so a hand-built recipient list
// can't smuggle in an "all users" sized payload through the wrong target mode.
const MaxBroadcastSelectedRecipients = 200
// Broadcast is one admin-triggered system message campaign, sent from // Broadcast is one admin-triggered system message campaign, sent from
// OfficialSystemUserID (777000) to every recipient snapshotted into // OfficialSystemUserID (777000) to every recipient targeted by TargetMode.
// broadcast_recipients at creation time. SentCount/FailedCount are derived //
// from the recipient rows at read time, not stored, so they can never drift. // For BroadcastTargetAll, recipient rows are not all inserted at creation:
// SnapshotMaxUserID/EnumerationCursorUserID/EnumerationDone track an
// incremental keyset walk over the users table (see
// store.BroadcastStore.MaterializeBroadcastRecipients), so creating a
// campaign for a large user base is a single cheap insert, not one giant
// blocking transaction. MaterializedCount is how many recipient rows exist
// so far; TargetCount is the (possibly still-growing, for "all") total this
// campaign is aimed at. SentCount/FailedCount are maintained incrementally
// by the delivery worker as it closes out each recipient row.
type Broadcast struct { type Broadcast struct {
ID int64 ID int64
Message string Message string
Entities []MessageEntity
TargetMode BroadcastTargetMode TargetMode BroadcastTargetMode
TotalCount int TargetCount int64
SentCount int MaterializedCount int64
FailedCount int SentCount int64
FailedCount int64
EnumerationDone bool
CreatedBy string CreatedBy string
CreatedAt time.Time CreatedAt time.Time
} }
// BroadcastRecipient is one durable outbox row: one user's delivery state // BroadcastRecipient is one durable outbox row: one user's delivery state
// for one broadcast. // for one broadcast.
//
// A worker claims a batch of eligible rows by writing LeaseToken/LeaseUntil
// (see store.BroadcastStore.ClaimBroadcastRecipients), delivers the message,
// then either closes the row as 'sent' (recording PrivateMessageID/
// MessageBoxID/Pts, the same identifiers domain.Message carries, so a
// campaign's delivery history is independently auditable without joining
// back through the shared message store) or releases it back to 'pending'
// (or terminally 'failed', once MaxBroadcastRecipientAttempts is reached) on
// error. A lease that is never renewed simply expires, so a worker that
// crashes mid-delivery cannot leave a row stuck in 'processing' forever.
type BroadcastRecipient struct { type BroadcastRecipient struct {
ID int64 ID int64
BroadcastID int64 BroadcastID int64
UserID int64 UserID int64
Status BroadcastRecipientStatus Status BroadcastRecipientStatus
Attempts int Attempts int
// NextAttemptAt gates retries with exponential backoff after a failed
// delivery; a 'pending' row isn't eligible for claiming again until then.
NextAttemptAt time.Time
LeaseToken string
LeaseUntil *time.Time
LastError string LastError string
// PrivateMessageID/MessageBoxID/Pts identify the delivered message once
// Status is 'sent'. A pre-migration row that was marked 'sent' before
// this tracking existed carries all three as zero -- see the CHECK
// constraint added in
// deploy/migrations/20260901000024_broadcast_lease_delivery_and_entities.up.sql,
// which treats that as a legitimate legacy/untracked case.
PrivateMessageID int64
MessageBoxID int
Pts int
SentAt *time.Time SentAt *time.Time
CreatedAt time.Time
UpdatedAt time.Time
} }
var ( var (
ErrBroadcastInvalid = errors.New("broadcast invalid") ErrBroadcastInvalid = errors.New("broadcast invalid")
ErrBroadcastMessageEmpty = errors.New("broadcast message is empty") ErrBroadcastMessageEmpty = errors.New("broadcast message is empty")
ErrBroadcastMessageTooLong = errors.New("broadcast message exceeds the maximum length")
ErrBroadcastNoRecipients = errors.New("broadcast has no recipients") ErrBroadcastNoRecipients = errors.New("broadcast has no recipients")
ErrBroadcastRecipientInvalid = errors.New("broadcast recipient invalid")
ErrBroadcastNotFound = errors.New("broadcast not found") ErrBroadcastNotFound = errors.New("broadcast not found")
// ErrBroadcastLeaseLost means the delivery worker's lease on a recipient
// row was reclaimed (expired and re-claimed by another cycle, or the row
// otherwise changed underneath it) before delivery finished. The caller
// should simply drop the result: the row is someone else's to finish now.
ErrBroadcastLeaseLost = errors.New("broadcast recipient lease lost")
) )

View file

@ -2,43 +2,76 @@ package store
import ( import (
"context" "context"
"time"
"telesrv/internal/domain" "telesrv/internal/domain"
) )
// BroadcastStore persists system broadcast campaigns and their durable // BroadcastStore persists system broadcast campaigns and their durable
// per-recipient delivery outbox. // per-recipient delivery outbox.
//
// A BroadcastTargetAll campaign is not fully enumerated at creation:
// CreateBroadcast only snapshots the target user-id range (the current max
// user id) and returns. MaterializeBroadcastRecipients then advances that
// campaign's enumeration a bounded batch at a time, so a huge user base
// never blocks the admin's create call, or any one worker cycle, on a
// single giant INSERT. ClaimBroadcastRecipients/ReleaseBroadcastRecipient/
// CompleteBroadcastRecipient implement a lease-based handoff for the
// delivery half of the cycle: a worker claims a bounded batch of eligible
// rows under a time-limited lease, and either completes or releases each
// one it processes. A lease that is never renewed simply expires, so a
// worker crash mid-cycle cannot strand a row in 'processing' forever, and
// two workers can never believe they both hold the same row's lease at once.
type BroadcastStore interface { type BroadcastStore interface {
// CreateBroadcast inserts the broadcast row and one pending recipient row // PreviewBroadcastRecipients validates and counts the intended recipient
// per id in recipientUserIDs, in a single transaction: a broadcast with // set without creating anything -- for "selected" mode this also
// zero recipients (an empty "selected" list, or an "all" snapshot taken // validates every id names a real, non-bot, non-system account.
// when there happen to be no eligible users) is rejected with PreviewBroadcastRecipients(ctx context.Context, mode domain.BroadcastTargetMode, selectedUserIDs []int64) (int64, error)
// domain.ErrBroadcastNoRecipients rather than created empty. // CreateBroadcast inserts the broadcast row. For BroadcastTargetAll this
CreateBroadcast(ctx context.Context, message string, targetMode domain.BroadcastTargetMode, recipientUserIDs []int64, createdBy string) (domain.Broadcast, error) // only snapshots the current max user id and target count; no recipient
// PendingBroadcastRecipients returns undelivered outbox rows across every // rows are inserted here (see MaterializeBroadcastRecipients). For
// broadcast, oldest first, each carrying its broadcast's message text so // BroadcastTargetSelected, the given ids are validated and their
// the worker can send without a second round trip per row. // recipient rows are inserted immediately, since that list is already
PendingBroadcastRecipients(ctx context.Context, limit int) ([]PendingBroadcastRecipient, error) // bounded by domain.MaxBroadcastSelectedRecipients.
// MarkBroadcastRecipientSent closes a recipient row as delivered. CreateBroadcast(ctx context.Context, message string, entities []domain.MessageEntity, mode domain.BroadcastTargetMode, selectedUserIDs []int64, createdBy string) (domain.Broadcast, error)
MarkBroadcastRecipientSent(ctx context.Context, recipientID int64) error // MaterializeBroadcastRecipients advances one "all"-mode campaign's
// MarkBroadcastRecipientFailed records a failed attempt. The row stays // enumeration by up to limit newly-inserted recipient rows, and reports
// 'pending' (retried on the next cycle) until attempts reaches // how many were inserted. A campaign with nothing left to enumerate (or
// domain.MaxBroadcastRecipientAttempts, at which point it becomes the // no "all"-mode campaign still enumerating at all) returns 0, nil.
// terminal 'failed' status. MaterializeBroadcastRecipients(ctx context.Context, limit int) (int, error)
MarkBroadcastRecipientFailed(ctx context.Context, recipientID int64, reason string) error // ClaimBroadcastRecipients atomically leases up to limit eligible rows
// ListBroadcasts pages broadcasts newest-first, each with sent/failed // (pending, or processing under an expired lease) to leaseToken for
// counts derived live from its recipient rows. // lease, returning each claim together with its broadcast's message and
// entities so the caller can deliver without a second round trip.
ClaimBroadcastRecipients(ctx context.Context, leaseToken string, limit int, lease time.Duration) ([]BroadcastRecipientClaim, error)
// CompleteBroadcastRecipient closes a claimed row as delivered, recording
// the message identifiers the send produced, and advances its
// broadcast's sent_count. It is a no-op returning
// domain.ErrBroadcastLeaseLost if the claim's lease was lost (expired
// and reclaimed, or otherwise no longer matches) in the meantime --
// safe to call even after a duplicate/idempotent resend, since the
// caller is expected to tolerate that error.
CompleteBroadcastRecipient(ctx context.Context, claim BroadcastRecipientClaim, privateMessageID int64, messageBoxID int, pts int) error
// ReleaseBroadcastRecipient returns a claimed row to 'pending' (with
// backoff) after a failed delivery attempt, or to the terminal 'failed'
// once domain.MaxBroadcastRecipientAttempts is reached, and advances its
// broadcast's failed_count in that terminal case.
ReleaseBroadcastRecipient(ctx context.Context, claim BroadcastRecipientClaim, cause string) error
// ListBroadcasts pages broadcasts newest-first.
ListBroadcasts(ctx context.Context, beforeID int64, limit int) ([]domain.Broadcast, bool, error) ListBroadcasts(ctx context.Context, beforeID int64, limit int) ([]domain.Broadcast, bool, error)
// BroadcastByID returns one broadcast with derived counts. // BroadcastByID returns one broadcast.
BroadcastByID(ctx context.Context, id int64) (domain.Broadcast, bool, error) BroadcastByID(ctx context.Context, id int64) (domain.Broadcast, bool, error)
} }
// PendingBroadcastRecipient is one undelivered outbox row, joined with its // BroadcastRecipientClaim is one recipient row leased for delivery, carrying
// broadcast's message text. // its broadcast's message text and entities so the worker doesn't need a
type PendingBroadcastRecipient struct { // second lookup before sending.
type BroadcastRecipientClaim struct {
RecipientID int64 RecipientID int64
BroadcastID int64 BroadcastID int64
UserID int64 UserID int64
Attempts int Attempts int
LeaseToken string
Message string Message string
Entities []domain.MessageEntity
} }

View file

@ -11,137 +11,257 @@ import (
) )
// BroadcastStore is the in-memory implementation of store.BroadcastStore, // BroadcastStore is the in-memory implementation of store.BroadcastStore,
// used by admin/app unit tests. // used by admin/app unit tests. It has no concept of a "users table" to
// snapshot against for "all" mode, so callers seed eligible user ids via
// SeedEligibleUsers; MaterializeBroadcastRecipients walks that fixed set the
// same way the postgres backend walks a keyset range.
type BroadcastStore struct { type BroadcastStore struct {
mu sync.Mutex mu sync.Mutex
broadcasts map[int64]domain.Broadcast broadcasts map[int64]domain.Broadcast
recipients map[int64]*memBroadcastRecipient recipients map[int64]*domain.BroadcastRecipient
eligibleUsers []int64 // sorted ascending, mirrors "all non-bot, non-system users"
nextBID int64 nextBID int64
nextRID int64 nextRID int64
} }
type memBroadcastRecipient struct {
domain.BroadcastRecipient
message string
}
func NewBroadcastStore() *BroadcastStore { func NewBroadcastStore() *BroadcastStore {
return &BroadcastStore{ return &BroadcastStore{
broadcasts: make(map[int64]domain.Broadcast), broadcasts: make(map[int64]domain.Broadcast),
recipients: make(map[int64]*memBroadcastRecipient), recipients: make(map[int64]*domain.BroadcastRecipient),
} }
} }
var _ store.BroadcastStore = (*BroadcastStore)(nil) var _ store.BroadcastStore = (*BroadcastStore)(nil)
func (s *BroadcastStore) CreateBroadcast(_ context.Context, message string, targetMode domain.BroadcastTargetMode, recipientUserIDs []int64, createdBy string) (domain.Broadcast, error) { // SeedEligibleUsers sets the fixed set of user ids "all"-mode targets and
if len(recipientUserIDs) == 0 { // PreviewBroadcastRecipients/CreateBroadcast/MaterializeBroadcastRecipients
return domain.Broadcast{}, domain.ErrBroadcastNoRecipients // enumerate over, mirroring the postgres store's live users-table query.
} func (s *BroadcastStore) SeedEligibleUsers(userIDs []int64) {
s.mu.Lock() s.mu.Lock()
defer s.mu.Unlock() defer s.mu.Unlock()
s.eligibleUsers = append([]int64(nil), userIDs...)
sort.Slice(s.eligibleUsers, func(i, j int) bool { return s.eligibleUsers[i] < s.eligibleUsers[j] })
}
func isEligibleSelected(userID int64) bool {
return userID > 0 && !domain.IsSystemUserID(userID)
}
func (s *BroadcastStore) PreviewBroadcastRecipients(_ context.Context, mode domain.BroadcastTargetMode, selectedUserIDs []int64) (int64, error) {
s.mu.Lock()
defer s.mu.Unlock()
switch mode {
case domain.BroadcastTargetAll:
if len(s.eligibleUsers) == 0 {
return 0, domain.ErrBroadcastNoRecipients
}
return int64(len(s.eligibleUsers)), nil
case domain.BroadcastTargetSelected:
if len(selectedUserIDs) == 0 {
return 0, domain.ErrBroadcastNoRecipients
}
for _, id := range selectedUserIDs {
if !isEligibleSelected(id) {
return 0, domain.ErrBroadcastRecipientInvalid
}
}
return int64(len(selectedUserIDs)), nil
default:
return 0, domain.ErrBroadcastInvalid
}
}
func (s *BroadcastStore) CreateBroadcast(_ context.Context, message string, entities []domain.MessageEntity, mode domain.BroadcastTargetMode, selectedUserIDs []int64, createdBy string) (domain.Broadcast, error) {
s.mu.Lock()
defer s.mu.Unlock()
switch mode {
case domain.BroadcastTargetAll:
if len(s.eligibleUsers) == 0 {
return domain.Broadcast{}, domain.ErrBroadcastNoRecipients
}
s.nextBID++ s.nextBID++
b := domain.Broadcast{ b := domain.Broadcast{
ID: s.nextBID, ID: s.nextBID, Message: message, Entities: entities, TargetMode: mode,
Message: message, TargetCount: int64(len(s.eligibleUsers)), CreatedBy: createdBy, CreatedAt: time.Now().UTC(),
TargetMode: targetMode,
CreatedBy: createdBy,
CreatedAt: time.Now().UTC(),
} }
seen := make(map[int64]bool, len(recipientUserIDs)) s.broadcasts[b.ID] = b
for _, userID := range recipientUserIDs { return b, nil
case domain.BroadcastTargetSelected:
if len(selectedUserIDs) == 0 {
return domain.Broadcast{}, domain.ErrBroadcastNoRecipients
}
for _, id := range selectedUserIDs {
if !isEligibleSelected(id) {
return domain.Broadcast{}, domain.ErrBroadcastRecipientInvalid
}
}
s.nextBID++
b := domain.Broadcast{
ID: s.nextBID, Message: message, Entities: entities, TargetMode: mode,
EnumerationDone: true, CreatedBy: createdBy, CreatedAt: time.Now().UTC(),
}
seen := make(map[int64]bool, len(selectedUserIDs))
for _, userID := range selectedUserIDs {
if seen[userID] { if seen[userID] {
continue continue
} }
seen[userID] = true seen[userID] = true
s.nextRID++ s.nextRID++
s.recipients[s.nextRID] = &memBroadcastRecipient{ s.recipients[s.nextRID] = &domain.BroadcastRecipient{
BroadcastRecipient: domain.BroadcastRecipient{ ID: s.nextRID, BroadcastID: b.ID, UserID: userID,
ID: s.nextRID, Status: domain.BroadcastRecipientPending, NextAttemptAt: time.Now().UTC(),
BroadcastID: b.ID,
UserID: userID,
Status: domain.BroadcastRecipientPending,
},
message: message,
} }
b.TotalCount++ b.TargetCount++
b.MaterializedCount++
} }
s.broadcasts[b.ID] = b s.broadcasts[b.ID] = b
return b, nil return b, nil
default:
return domain.Broadcast{}, domain.ErrBroadcastInvalid
}
} }
func (s *BroadcastStore) PendingBroadcastRecipients(_ context.Context, limit int) ([]store.PendingBroadcastRecipient, error) { func (s *BroadcastStore) MaterializeBroadcastRecipients(_ context.Context, limit int) (int, error) {
if limit <= 0 || limit > 200 { if limit <= 0 || limit > 1000 {
limit = 50 limit = 100
} }
s.mu.Lock() s.mu.Lock()
defer s.mu.Unlock() defer s.mu.Unlock()
// Iteration order over a map is unspecified; sort by recipient id (assigned var ids []int64
// in creation order) so this matches the postgres backend's "oldest first". for id, b := range s.broadcasts {
ids := make([]int64, 0, len(s.recipients)) if b.TargetMode == domain.BroadcastTargetAll && !b.EnumerationDone {
ids = append(ids, id)
}
}
if len(ids) == 0 {
return 0, nil
}
sortInt64s(ids)
bid := ids[0]
b := s.broadcasts[bid]
inserted := 0
for _, userID := range s.eligibleUsers {
if int64(inserted) >= int64(limit) {
break
}
if s.hasRecipient(bid, userID) {
continue
}
s.nextRID++
s.recipients[s.nextRID] = &domain.BroadcastRecipient{
ID: s.nextRID, BroadcastID: bid, UserID: userID,
Status: domain.BroadcastRecipientPending, NextAttemptAt: time.Now().UTC(),
}
b.MaterializedCount++
inserted++
}
if inserted < limit {
b.EnumerationDone = true
b.TargetCount = b.MaterializedCount
}
s.broadcasts[bid] = b
return inserted, nil
}
func (s *BroadcastStore) hasRecipient(broadcastID, userID int64) bool {
for _, r := range s.recipients {
if r.BroadcastID == broadcastID && r.UserID == userID {
return true
}
}
return false
}
func (s *BroadcastStore) ClaimBroadcastRecipients(_ context.Context, leaseToken string, limit int, lease time.Duration) ([]store.BroadcastRecipientClaim, error) {
if leaseToken == "" {
return nil, domain.ErrBroadcastInvalid
}
if limit <= 0 || limit > 500 {
limit = 50
}
if lease <= 0 {
lease = 30 * time.Second
}
s.mu.Lock()
defer s.mu.Unlock()
var ids []int64
now := time.Now().UTC()
for id, r := range s.recipients { for id, r := range s.recipients {
if r.Status == domain.BroadcastRecipientPending { eligible := (r.Status == domain.BroadcastRecipientPending && !r.NextAttemptAt.After(now)) ||
(r.Status == domain.BroadcastRecipientProcessing && r.LeaseUntil != nil && !r.LeaseUntil.After(now))
if eligible {
ids = append(ids, id) ids = append(ids, id)
} }
} }
sortInt64s(ids) sortInt64s(ids)
out := make([]store.PendingBroadcastRecipient, 0, limit) if len(ids) > limit {
for _, id := range ids { ids = ids[:limit]
if len(out) >= limit {
break
} }
out := make([]store.BroadcastRecipientClaim, 0, len(ids))
until := now.Add(lease)
for _, id := range ids {
r := s.recipients[id] r := s.recipients[id]
out = append(out, store.PendingBroadcastRecipient{ r.Status = domain.BroadcastRecipientProcessing
RecipientID: r.ID, BroadcastID: r.BroadcastID, UserID: r.UserID, Attempts: r.Attempts, Message: r.message, r.Attempts++
r.LeaseToken = leaseToken
r.LeaseUntil = &until
r.UpdatedAt = now
b := s.broadcasts[r.BroadcastID]
out = append(out, store.BroadcastRecipientClaim{
RecipientID: r.ID, BroadcastID: r.BroadcastID, UserID: r.UserID,
Attempts: r.Attempts, LeaseToken: leaseToken, Message: b.Message, Entities: b.Entities,
}) })
} }
return out, nil return out, nil
} }
func (s *BroadcastStore) MarkBroadcastRecipientSent(_ context.Context, recipientID int64) error { func (s *BroadcastStore) CompleteBroadcastRecipient(_ context.Context, claim store.BroadcastRecipientClaim, privateMessageID int64, messageBoxID int, pts int) error {
s.mu.Lock() s.mu.Lock()
defer s.mu.Unlock() defer s.mu.Unlock()
r, ok := s.recipients[recipientID] r, ok := s.recipients[claim.RecipientID]
if !ok || r.Status != domain.BroadcastRecipientPending { if !ok || r.Status != domain.BroadcastRecipientProcessing || r.LeaseToken != claim.LeaseToken {
return nil return domain.ErrBroadcastLeaseLost
} }
r.Status = domain.BroadcastRecipientSent
now := time.Now().UTC() now := time.Now().UTC()
r.SentAt = &now r.Status = domain.BroadcastRecipientSent
r.LeaseToken = ""
r.LeaseUntil = nil
r.LastError = "" r.LastError = ""
r.PrivateMessageID = privateMessageID
r.MessageBoxID = messageBoxID
r.Pts = pts
r.SentAt = &now
r.UpdatedAt = now
b := s.broadcasts[claim.BroadcastID]
b.SentCount++
s.broadcasts[claim.BroadcastID] = b
return nil return nil
} }
func (s *BroadcastStore) MarkBroadcastRecipientFailed(_ context.Context, recipientID int64, reason string) error { func (s *BroadcastStore) ReleaseBroadcastRecipient(_ context.Context, claim store.BroadcastRecipientClaim, cause string) error {
s.mu.Lock() s.mu.Lock()
defer s.mu.Unlock() defer s.mu.Unlock()
r, ok := s.recipients[recipientID] r, ok := s.recipients[claim.RecipientID]
if !ok || r.Status != domain.BroadcastRecipientPending { if !ok || r.Status != domain.BroadcastRecipientProcessing || r.LeaseToken != claim.LeaseToken {
return nil return nil
} }
r.Attempts++ now := time.Now().UTC()
r.LastError = reason r.LeaseToken = ""
r.LeaseUntil = nil
r.LastError = cause
r.UpdatedAt = now
if r.Attempts >= domain.MaxBroadcastRecipientAttempts { if r.Attempts >= domain.MaxBroadcastRecipientAttempts {
r.Status = domain.BroadcastRecipientFailed r.Status = domain.BroadcastRecipientFailed
b := s.broadcasts[claim.BroadcastID]
b.FailedCount++
s.broadcasts[claim.BroadcastID] = b
} else {
r.Status = domain.BroadcastRecipientPending
r.NextAttemptAt = now
} }
return nil return nil
} }
func (s *BroadcastStore) countsFor(broadcastID int64) (sent, failed int) {
for _, r := range s.recipients {
if r.BroadcastID != broadcastID {
continue
}
switch r.Status {
case domain.BroadcastRecipientSent:
sent++
case domain.BroadcastRecipientFailed:
failed++
}
}
return sent, failed
}
func (s *BroadcastStore) ListBroadcasts(_ context.Context, beforeID int64, limit int) ([]domain.Broadcast, bool, error) { func (s *BroadcastStore) ListBroadcasts(_ context.Context, beforeID int64, limit int) ([]domain.Broadcast, bool, error) {
if limit <= 0 || limit > 200 { if limit <= 0 || limit > 200 {
limit = 50 limit = 50
@ -161,9 +281,7 @@ func (s *BroadcastStore) ListBroadcasts(_ context.Context, beforeID int64, limit
} }
out := make([]domain.Broadcast, 0, len(ids)) out := make([]domain.Broadcast, 0, len(ids))
for _, id := range ids { for _, id := range ids {
b := s.broadcasts[id] out = append(out, s.broadcasts[id])
b.SentCount, b.FailedCount = s.countsFor(id)
out = append(out, b)
} }
return out, hasMore, nil return out, hasMore, nil
} }
@ -175,7 +293,6 @@ func (s *BroadcastStore) BroadcastByID(_ context.Context, id int64) (domain.Broa
if !ok { if !ok {
return domain.Broadcast{}, false, nil return domain.Broadcast{}, false, nil
} }
b.SentCount, b.FailedCount = s.countsFor(id)
return b, true, nil return b, true, nil
} }

View file

@ -2,7 +2,10 @@ package postgres
import ( import (
"context" "context"
"errors"
"fmt" "fmt"
"strings"
"time"
"github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5"
@ -12,7 +15,10 @@ import (
) )
// BroadcastStore persists system broadcast campaigns (see // BroadcastStore persists system broadcast campaigns (see
// deploy/migrations/20260714003131_system_broadcasts.up.sql). // deploy/migrations/20260714003131_system_broadcasts.up.sql, extended by
// deploy/migrations/20260901000024_broadcast_lease_delivery_and_entities.up.sql
// with entities, incremental "all"-mode materialization and lease-based
// delivery claims).
type BroadcastStore struct { type BroadcastStore struct {
db sqlcgen.DBTX db sqlcgen.DBTX
} }
@ -24,37 +30,113 @@ func NewBroadcastStore(db sqlcgen.DBTX) *BroadcastStore {
var _ store.BroadcastStore = (*BroadcastStore)(nil) var _ store.BroadcastStore = (*BroadcastStore)(nil)
// CreateBroadcast inserts the broadcast row and one pending recipient row per const eligibleBroadcastUsersSQL = `
// id, deduplicating recipientUserIDs (a "selected" list built by hand in the FROM users
// panel could otherwise carry a repeat) via ON CONFLICT DO NOTHING against WHERE NOT is_bot
// the (broadcast_id, user_id) unique constraint. AND deleted_at IS NULL
func (s *BroadcastStore) CreateBroadcast(ctx context.Context, message string, targetMode domain.BroadcastTargetMode, recipientUserIDs []int64, createdBy string) (domain.Broadcast, error) { AND id <> ALL($1::bigint[])`
if len(recipientUserIDs) == 0 {
return domain.Broadcast{}, domain.ErrBroadcastNoRecipients // PreviewBroadcastRecipients counts (and, for "selected", validates) the
// intended recipient set without creating anything.
func (s *BroadcastStore) PreviewBroadcastRecipients(ctx context.Context, mode domain.BroadcastTargetMode, selectedUserIDs []int64) (int64, error) {
switch mode {
case domain.BroadcastTargetAll:
var count int64
if err := s.db.QueryRow(ctx, `SELECT count(*) `+eligibleBroadcastUsersSQL, domain.SystemUserIDs()).Scan(&count); err != nil {
return 0, fmt.Errorf("count broadcast recipients: %w", err)
}
if count == 0 {
return 0, domain.ErrBroadcastNoRecipients
}
return count, nil
case domain.BroadcastTargetSelected:
return validateSelectedBroadcastUsers(ctx, s.db, selectedUserIDs)
default:
return 0, domain.ErrBroadcastInvalid
}
}
func validateSelectedBroadcastUsers(ctx context.Context, db sqlcgen.DBTX, selectedUserIDs []int64) (int64, error) {
if len(selectedUserIDs) == 0 {
return 0, domain.ErrBroadcastNoRecipients
}
var count int64
if err := db.QueryRow(ctx, `
SELECT count(*)
FROM users
WHERE id = ANY($1::bigint[])
AND NOT is_bot
AND deleted_at IS NULL
AND id <> ALL($2::bigint[])`, selectedUserIDs, domain.SystemUserIDs()).Scan(&count); err != nil {
return 0, fmt.Errorf("validate broadcast recipients: %w", err)
}
if count != int64(len(selectedUserIDs)) {
return 0, domain.ErrBroadcastRecipientInvalid
}
return count, nil
}
// CreateBroadcast inserts the broadcast row. For "all" mode it only
// snapshots the current max eligible user id and target count -- recipient
// rows are inserted incrementally by MaterializeBroadcastRecipients, not
// here. For "selected" mode, whose recipient list is already bounded by
// domain.MaxBroadcastSelectedRecipients, every recipient row is inserted in
// the same transaction as the broadcast itself, deduplicating via
// ON CONFLICT DO NOTHING against the (broadcast_id, user_id) unique
// constraint (a hand-built selected list could otherwise carry a repeat).
func (s *BroadcastStore) CreateBroadcast(ctx context.Context, message string, entities []domain.MessageEntity, mode domain.BroadcastTargetMode, selectedUserIDs []int64, createdBy string) (domain.Broadcast, error) {
entitiesJSON, err := encodeMessageEntities(entities)
if err != nil {
return domain.Broadcast{}, fmt.Errorf("encode broadcast entities: %w", err)
} }
var out domain.Broadcast var out domain.Broadcast
err := withTx(ctx, s.db, "create broadcast", func(tx pgx.Tx) error { err = withTx(ctx, s.db, "create broadcast", func(tx pgx.Tx) error {
switch mode {
case domain.BroadcastTargetAll:
var maxUserID, count int64
if err := tx.QueryRow(ctx, ` if err := tx.QueryRow(ctx, `
INSERT INTO broadcasts (message, target_mode, total_count, created_by) SELECT COALESCE(max(id), 0), count(*) `+eligibleBroadcastUsersSQL, domain.SystemUserIDs()).Scan(&maxUserID, &count); err != nil {
VALUES ($1, $2, $3, $4) return fmt.Errorf("snapshot broadcast recipients: %w", err)
RETURNING id, message, target_mode, total_count, created_by, created_at`,
message, string(targetMode), len(recipientUserIDs), createdBy,
).Scan(&out.ID, &out.Message, &out.TargetMode, &out.TotalCount, &out.CreatedBy, &out.CreatedAt); err != nil {
return fmt.Errorf("insert broadcast: %w", err)
} }
batch := &pgx.Batch{} if count == 0 {
for _, userID := range recipientUserIDs { return domain.ErrBroadcastNoRecipients
batch.Queue(` }
row := tx.QueryRow(ctx, `
INSERT INTO broadcasts (
message, entities, target_mode, snapshot_max_user_id, enumeration_done,
target_count, created_by
) VALUES ($1, $2::jsonb, 'all', $3, false, $4, $5)
RETURNING `+broadcastColumns,
message, string(entitiesJSON), maxUserID, count, createdBy,
)
if err := scanBroadcastRow(row, &out); err != nil {
return fmt.Errorf("insert all-user broadcast: %w", err)
}
case domain.BroadcastTargetSelected:
count, err := validateSelectedBroadcastUsers(ctx, tx, selectedUserIDs)
if err != nil {
return err
}
row := tx.QueryRow(ctx, `
INSERT INTO broadcasts (
message, entities, target_mode, enumeration_done, target_count,
materialized_count, created_by
) VALUES ($1, $2::jsonb, 'selected', true, $3, $3, $4)
RETURNING `+broadcastColumns,
message, string(entitiesJSON), count, createdBy,
)
if err := scanBroadcastRow(row, &out); err != nil {
return fmt.Errorf("insert selected broadcast: %w", err)
}
if _, err := tx.Exec(ctx, `
INSERT INTO broadcast_recipients (broadcast_id, user_id) INSERT INTO broadcast_recipients (broadcast_id, user_id)
VALUES ($1, $2) SELECT $1, user_id
ON CONFLICT (broadcast_id, user_id) DO NOTHING`, out.ID, userID) FROM unnest($2::bigint[]) AS selected(user_id)
} ON CONFLICT (broadcast_id, user_id) DO NOTHING`, out.ID, selectedUserIDs); err != nil {
results := tx.SendBatch(ctx, batch) return fmt.Errorf("insert selected broadcast recipients: %w", err)
defer results.Close()
for range recipientUserIDs {
if _, err := results.Exec(); err != nil {
return fmt.Errorf("insert broadcast recipient: %w", err)
} }
default:
return domain.ErrBroadcastInvalid
} }
return nil return nil
}) })
@ -64,93 +146,217 @@ ON CONFLICT (broadcast_id, user_id) DO NOTHING`, out.ID, userID)
return out, nil return out, nil
} }
// PendingBroadcastRecipients returns undelivered outbox rows, oldest first, // MaterializeBroadcastRecipients advances one all-user campaign with a
// each carrying its broadcast's message text. // single bounded keyset INSERT, picking whichever "all"-mode campaign still
func (s *BroadcastStore) PendingBroadcastRecipients(ctx context.Context, limit int) ([]store.PendingBroadcastRecipient, error) { // has enumeration left (oldest first) under FOR UPDATE SKIP LOCKED, so
if limit <= 0 || limit > 200 { // concurrent worker cycles never step on each other's progress.
func (s *BroadcastStore) MaterializeBroadcastRecipients(ctx context.Context, limit int) (int, error) {
if limit <= 0 || limit > 1000 {
limit = 100
}
var inserted int
err := s.db.QueryRow(ctx, `
WITH campaign AS (
SELECT id, snapshot_max_user_id, enumeration_cursor_user_id
FROM broadcasts
WHERE target_mode = 'all' AND NOT enumeration_done
ORDER BY id
FOR UPDATE SKIP LOCKED
LIMIT 1
), candidates AS (
SELECT u.id
FROM campaign c
JOIN LATERAL (
SELECT id
FROM users
WHERE id > c.enumeration_cursor_user_id
AND id <= c.snapshot_max_user_id
AND NOT is_bot
AND deleted_at IS NULL
AND id <> ALL($1::bigint[])
ORDER BY id
LIMIT $2
) u ON true
), materialized AS (
INSERT INTO broadcast_recipients (broadcast_id, user_id)
SELECT c.id, candidate.id
FROM campaign c
CROSS JOIN candidates candidate
ON CONFLICT (broadcast_id, user_id) DO NOTHING
RETURNING user_id
), progress AS (
UPDATE broadcasts b
SET enumeration_cursor_user_id = COALESCE((SELECT max(id) FROM candidates), b.snapshot_max_user_id),
enumeration_done = (SELECT count(*) FROM candidates) < $2,
materialized_count = b.materialized_count + (SELECT count(*) FROM materialized),
target_count = CASE
WHEN (SELECT count(*) FROM candidates) < $2
THEN b.materialized_count + (SELECT count(*) FROM materialized)
ELSE b.target_count
END
FROM campaign c
WHERE b.id = c.id
RETURNING b.id
)
SELECT count(*)::int FROM materialized`, domain.SystemUserIDs(), limit).Scan(&inserted)
if errors.Is(err, pgx.ErrNoRows) {
return 0, nil
}
if err != nil {
return 0, fmt.Errorf("materialize broadcast recipients: %w", err)
}
return inserted, nil
}
// ClaimBroadcastRecipients atomically leases up to limit eligible rows
// (pending, or processing under an expired lease) to leaseToken, joining
// each claim with its broadcast's message and entities.
func (s *BroadcastStore) ClaimBroadcastRecipients(ctx context.Context, leaseToken string, limit int, lease time.Duration) ([]store.BroadcastRecipientClaim, error) {
if strings.TrimSpace(leaseToken) == "" || len(leaseToken) > 64 {
return nil, domain.ErrBroadcastInvalid
}
if limit <= 0 || limit > 500 {
limit = 50 limit = 50
} }
leaseSeconds := int(lease / time.Second)
if leaseSeconds <= 0 || leaseSeconds > 3600 {
leaseSeconds = 30
}
rows, err := s.db.Query(ctx, ` rows, err := s.db.Query(ctx, `
SELECT r.id, r.broadcast_id, r.user_id, r.attempts, b.message WITH candidates AS (
FROM broadcast_recipients r SELECT id
JOIN broadcasts b ON b.id = r.broadcast_id FROM broadcast_recipients
WHERE r.status = 'pending' WHERE (status = 'pending' AND next_attempt_at <= now())
ORDER BY r.id OR (status = 'processing' AND lease_until <= now())
LIMIT $1`, limit) ORDER BY id
FOR UPDATE SKIP LOCKED
LIMIT $1
), claimed AS (
UPDATE broadcast_recipients r
SET status = 'processing',
attempts = attempts + 1,
lease_token = $2,
lease_until = now() + make_interval(secs => $3),
updated_at = now()
FROM candidates c
WHERE r.id = c.id
RETURNING r.id, r.broadcast_id, r.user_id, r.attempts
)
SELECT c.id, c.broadcast_id, c.user_id, c.attempts, b.message, b.entities::text
FROM claimed c
JOIN broadcasts b ON b.id = c.broadcast_id
ORDER BY c.id`, limit, leaseToken, leaseSeconds)
if err != nil { if err != nil {
return nil, fmt.Errorf("list pending broadcast recipients: %w", err) return nil, fmt.Errorf("claim broadcast recipients: %w", err)
} }
defer rows.Close() defer rows.Close()
out := make([]store.PendingBroadcastRecipient, 0, limit) out := make([]store.BroadcastRecipientClaim, 0, limit)
for rows.Next() { for rows.Next() {
var item store.PendingBroadcastRecipient var item store.BroadcastRecipientClaim
if err := rows.Scan(&item.RecipientID, &item.BroadcastID, &item.UserID, &item.Attempts, &item.Message); err != nil { var entitiesJSON string
return nil, fmt.Errorf("scan pending broadcast recipient: %w", err) item.LeaseToken = leaseToken
if err := rows.Scan(&item.RecipientID, &item.BroadcastID, &item.UserID, &item.Attempts, &item.Message, &entitiesJSON); err != nil {
return nil, fmt.Errorf("scan broadcast recipient claim: %w", err)
} }
entities, err := decodeMessageEntities(entitiesJSON)
if err != nil {
return nil, fmt.Errorf("decode broadcast recipient claim entities: %w", err)
}
item.Entities = entities
out = append(out, item) out = append(out, item)
} }
if err := rows.Err(); err != nil { if err := rows.Err(); err != nil {
return nil, fmt.Errorf("iterate pending broadcast recipients: %w", err) return nil, fmt.Errorf("iterate broadcast recipient claims: %w", err)
} }
return out, nil return out, nil
} }
// MarkBroadcastRecipientSent closes a recipient row as delivered. Closing an // CompleteBroadcastRecipient closes a claimed row as delivered and advances
// already-closed row is a no-op: the outbox is exactly-once, not // its broadcast's sent_count in the same transaction.
// at-least-once. func (s *BroadcastStore) CompleteBroadcastRecipient(ctx context.Context, claim store.BroadcastRecipientClaim, privateMessageID int64, messageBoxID int, pts int) error {
func (s *BroadcastStore) MarkBroadcastRecipientSent(ctx context.Context, recipientID int64) error { return withTx(ctx, s.db, "complete broadcast recipient", func(tx pgx.Tx) error {
if _, err := s.db.Exec(ctx, ` tag, err := tx.Exec(ctx, `
UPDATE broadcast_recipients UPDATE broadcast_recipients
SET status = 'sent', sent_at = now(), last_error = '' SET status = 'sent', lease_token = '', lease_until = NULL,
WHERE id = $1 AND status = 'pending'`, recipientID); err != nil { last_error = '', private_message_id = $3, message_box_id = $4,
return fmt.Errorf("mark broadcast recipient sent: %w", err) pts = $5, sent_at = now(), updated_at = now()
WHERE id = $1 AND status = 'processing' AND lease_token = $2`,
claim.RecipientID, claim.LeaseToken, privateMessageID, messageBoxID, pts)
if err != nil {
return fmt.Errorf("complete broadcast recipient: %w", err)
}
if tag.RowsAffected() != 1 {
return domain.ErrBroadcastLeaseLost
}
if _, err := tx.Exec(ctx, `
UPDATE broadcasts SET sent_count = sent_count + 1 WHERE id = $1`, claim.BroadcastID); err != nil {
return fmt.Errorf("advance broadcast sent count: %w", err)
}
return nil
})
}
// ReleaseBroadcastRecipient returns a claimed row to 'pending' with backoff,
// or to the terminal 'failed' once domain.MaxBroadcastRecipientAttempts is
// reached, advancing failed_count in that terminal case.
func (s *BroadcastStore) ReleaseBroadcastRecipient(ctx context.Context, claim store.BroadcastRecipientClaim, cause string) error {
if len(cause) > 500 {
cause = cause[:500]
}
_, err := s.db.Exec(ctx, `
WITH changed AS (
UPDATE broadcast_recipients
SET status = CASE WHEN attempts >= $3 THEN 'failed' ELSE 'pending' END,
next_attempt_at = CASE
WHEN attempts >= $3 THEN next_attempt_at
ELSE now() + make_interval(secs => LEAST(300, (1 << LEAST(attempts, 8))))
END,
lease_token = '',
lease_until = NULL,
last_error = $4,
updated_at = now()
WHERE id = $1
AND status = 'processing'
AND lease_token = $2
RETURNING broadcast_id, status
)
UPDATE broadcasts b
SET failed_count = failed_count + 1
FROM changed c
WHERE b.id = c.broadcast_id AND c.status = 'failed'`, claim.RecipientID, claim.LeaseToken, domain.MaxBroadcastRecipientAttempts, cause)
if err != nil {
return fmt.Errorf("release broadcast recipient: %w", err)
} }
return nil return nil
} }
// MarkBroadcastRecipientFailed records a failed delivery attempt. The row const broadcastColumns = `
// stays 'pending' (retried on the next cycle) until attempts reaches id, message, entities::text, target_mode, target_count, materialized_count,
// domain.MaxBroadcastRecipientAttempts, at which point it becomes the sent_count, failed_count, enumeration_done, created_by, created_at`
// terminal 'failed' status so a permanently blocked/deleted recipient
// doesn't spin forever alongside real deliveries.
func (s *BroadcastStore) MarkBroadcastRecipientFailed(ctx context.Context, recipientID int64, reason string) error {
if len(reason) > 500 {
reason = reason[:500]
}
if _, err := s.db.Exec(ctx, `
UPDATE broadcast_recipients
SET attempts = attempts + 1,
last_error = $2,
status = CASE WHEN attempts + 1 >= $3 THEN 'failed' ELSE 'pending' END
WHERE id = $1 AND status = 'pending'`, recipientID, reason, domain.MaxBroadcastRecipientAttempts); err != nil {
return fmt.Errorf("mark broadcast recipient failed: %w", err)
}
return nil
}
const broadcastSelectColumns = `
b.id, b.message, b.target_mode, b.total_count, b.created_by, b.created_at,
count(*) FILTER (WHERE r.status = 'sent')::int AS sent_count,
count(*) FILTER (WHERE r.status = 'failed')::int AS failed_count`
func scanBroadcastRow(row interface{ Scan(...any) error }, item *domain.Broadcast) error { func scanBroadcastRow(row interface{ Scan(...any) error }, item *domain.Broadcast) error {
return row.Scan(&item.ID, &item.Message, &item.TargetMode, &item.TotalCount, &item.CreatedBy, &item.CreatedAt, var entitiesJSON string
&item.SentCount, &item.FailedCount) if err := row.Scan(&item.ID, &item.Message, &entitiesJSON, &item.TargetMode, &item.TargetCount, &item.MaterializedCount,
&item.SentCount, &item.FailedCount, &item.EnumerationDone, &item.CreatedBy, &item.CreatedAt); err != nil {
return err
}
entities, err := decodeMessageEntities(entitiesJSON)
if err != nil {
return fmt.Errorf("decode broadcast entities: %w", err)
}
item.Entities = entities
return nil
} }
// ListBroadcasts pages broadcasts newest-first, each with sent/failed counts // ListBroadcasts pages broadcasts newest-first.
// derived live from its recipient rows (never stored, so they can't drift).
func (s *BroadcastStore) ListBroadcasts(ctx context.Context, beforeID int64, limit int) ([]domain.Broadcast, bool, error) { func (s *BroadcastStore) ListBroadcasts(ctx context.Context, beforeID int64, limit int) ([]domain.Broadcast, bool, error) {
if limit <= 0 || limit > 200 { if limit <= 0 || limit > 200 {
limit = 50 limit = 50
} }
rows, err := s.db.Query(ctx, ` rows, err := s.db.Query(ctx, `SELECT `+broadcastColumns+`
SELECT `+broadcastSelectColumns+` FROM broadcasts
FROM broadcasts b WHERE $1::bigint = 0 OR id < $1
LEFT JOIN broadcast_recipients r ON r.broadcast_id = b.id ORDER BY id DESC
WHERE $1::bigint = 0 OR b.id < $1
GROUP BY b.id
ORDER BY b.id DESC
LIMIT $2`, beforeID, limit+1) LIMIT $2`, beforeID, limit+1)
if err != nil { if err != nil {
return nil, false, fmt.Errorf("list broadcasts: %w", err) return nil, false, fmt.Errorf("list broadcasts: %w", err)
@ -174,17 +380,14 @@ LIMIT $2`, beforeID, limit+1)
return out, hasMore, nil return out, hasMore, nil
} }
// BroadcastByID returns one broadcast with derived counts. // BroadcastByID returns one broadcast.
func (s *BroadcastStore) BroadcastByID(ctx context.Context, id int64) (domain.Broadcast, bool, error) { func (s *BroadcastStore) BroadcastByID(ctx context.Context, id int64) (domain.Broadcast, bool, error) {
var item domain.Broadcast var item domain.Broadcast
err := scanBroadcastRow(s.db.QueryRow(ctx, ` err := scanBroadcastRow(s.db.QueryRow(ctx, `SELECT `+broadcastColumns+`
SELECT `+broadcastSelectColumns+` FROM broadcasts
FROM broadcasts b WHERE id = $1`, id), &item)
LEFT JOIN broadcast_recipients r ON r.broadcast_id = b.id
WHERE b.id = $1
GROUP BY b.id`, id), &item)
if err != nil { if err != nil {
if err == pgx.ErrNoRows { if errors.Is(err, pgx.ErrNoRows) {
return domain.Broadcast{}, false, nil return domain.Broadcast{}, false, nil
} }
return domain.Broadcast{}, false, fmt.Errorf("get broadcast: %w", err) return domain.Broadcast{}, false, fmt.Errorf("get broadcast: %w", err)

View file

@ -0,0 +1,144 @@
package postgres
import (
"context"
"testing"
"telesrv/deploy"
)
const (
broadcastLeaseDeliveryMigrationUp = "migrations/20260901000024_broadcast_lease_delivery_and_entities.up.sql"
broadcastLeaseDeliveryMigrationDown = "migrations/20260901000024_broadcast_lease_delivery_and_entities.down.sql"
)
// TestBroadcastLeaseDeliveryMigrationBackfillsLegacyDataPostgres proves the
// ALTER-based migration in
// deploy/migrations/20260901000024_broadcast_lease_delivery_and_entities.up.sql
// applies cleanly against pre-existing broadcasts/broadcast_recipients rows
// shaped by the original 20260714003131_system_broadcasts.up.sql schema --
// specifically 'sent' recipient rows that predate private_message_id/
// message_box_id/pts tracking, which the new sent-tracking CHECK constraint
// must accept as a legitimate legacy case rather than reject.
func TestBroadcastLeaseDeliveryMigrationBackfillsLegacyDataPostgres(t *testing.T) {
pool := testPool(t)
ctx := context.Background()
upSQL, err := deploy.Migrations.ReadFile(broadcastLeaseDeliveryMigrationUp)
if err != nil {
t.Fatalf("read up migration: %v", err)
}
downSQL, err := deploy.Migrations.ReadFile(broadcastLeaseDeliveryMigrationDown)
if err != nil {
t.Fatalf("read down migration: %v", err)
}
tx, err := pool.Begin(ctx)
if err != nil {
t.Fatalf("begin broadcast lease delivery migration test: %v", err)
}
defer func() { _ = tx.Rollback(context.Background()) }()
// Return the schema to its pre-migration (20260714003131) shape.
if _, err := tx.Exec(ctx, string(downSQL)); err != nil {
t.Fatalf("revert broadcast lease delivery migration: %v", err)
}
// Seed fixtures shaped exactly like production rows created before this
// migration existed: a broadcast with only total_count, and recipient
// rows in every legacy status -- including a 'sent' row that carries no
// delivery-identifier tracking at all, since that tracking didn't exist
// yet when it was created.
var broadcastID int64
if err := tx.QueryRow(ctx, `
INSERT INTO public.broadcasts (message, target_mode, total_count, created_by)
VALUES ('legacy campaign', 'selected', 3, 'legacy-admin')
RETURNING id`).Scan(&broadcastID); err != nil {
t.Fatalf("insert legacy broadcast fixture: %v", err)
}
rows := []struct {
userID int64
status string
}{
{userID: 9_100_000_000_030_001, status: "sent"},
{userID: 9_100_000_000_030_002, status: "pending"},
{userID: 9_100_000_000_030_003, status: "failed"},
}
for _, r := range rows {
var sentAtClause string
if r.status == "sent" {
sentAtClause = ", sent_at = now()"
}
if _, err := tx.Exec(ctx, `
INSERT INTO public.broadcast_recipients (broadcast_id, user_id, status)
VALUES ($1, $2, $3)`, broadcastID, r.userID, r.status); err != nil {
t.Fatalf("insert legacy recipient fixture (status=%s): %v", r.status, err)
}
if sentAtClause != "" {
if _, err := tx.Exec(ctx, `
UPDATE public.broadcast_recipients SET sent_at = now() WHERE broadcast_id = $1 AND user_id = $2`, broadcastID, r.userID); err != nil {
t.Fatalf("stamp legacy sent_at fixture: %v", err)
}
}
}
// Re-apply the migration under test. This must not fail against the
// legacy 'sent' row above (private_message_id/message_box_id/pts all
// still at their just-added zero default).
if _, err := tx.Exec(ctx, string(upSQL)); err != nil {
t.Fatalf("apply broadcast lease delivery migration over legacy data: %v", err)
}
var targetCount, materializedCount, sentCount, failedCount int64
var enumerationDone bool
if err := tx.QueryRow(ctx, `
SELECT target_count, materialized_count, sent_count, failed_count, enumeration_done
FROM public.broadcasts WHERE id = $1`, broadcastID).Scan(&targetCount, &materializedCount, &sentCount, &failedCount, &enumerationDone); err != nil {
t.Fatalf("read migrated broadcast: %v", err)
}
if targetCount != 3 {
t.Fatalf("target_count = %d, want 3 (renamed from total_count)", targetCount)
}
if materializedCount != 3 {
t.Fatalf("materialized_count = %d, want 3 (backfilled from target_count)", materializedCount)
}
if sentCount != 1 {
t.Fatalf("sent_count = %d, want 1 (backfilled from recipient rows)", sentCount)
}
if failedCount != 1 {
t.Fatalf("failed_count = %d, want 1 (backfilled from recipient rows)", failedCount)
}
if !enumerationDone {
t.Fatalf("enumeration_done = false, want true (pre-existing campaigns were fully enumerated at creation)")
}
var status string
var privateMessageID int64
var messageBoxID, pts int
if err := tx.QueryRow(ctx, `
SELECT status, private_message_id, message_box_id, pts
FROM public.broadcast_recipients
WHERE broadcast_id = $1 AND user_id = $2`, broadcastID, rows[0].userID).Scan(&status, &privateMessageID, &messageBoxID, &pts); err != nil {
t.Fatalf("read migrated legacy sent recipient: %v", err)
}
if status != "sent" || privateMessageID != 0 || messageBoxID != 0 || pts != 0 {
t.Fatalf("legacy sent recipient = status=%q private_message_id=%d message_box_id=%d pts=%d, want sent/0/0/0 (untracked legacy case accepted)",
status, privateMessageID, messageBoxID, pts)
}
// A properly-tracked 'sent' row (what new code always writes, via
// CompleteBroadcastRecipient) must also satisfy the CHECK.
if _, err := tx.Exec(ctx, `
INSERT INTO public.broadcast_recipients (broadcast_id, user_id, status, sent_at, private_message_id, message_box_id, pts)
VALUES ($1, $2, 'sent', now(), 1, 1, 1)`, broadcastID, int64(9_100_000_000_030_099)); err != nil {
t.Fatalf("insert of a properly-tracked 'sent' row failed: %v", err)
}
// But a 'sent' row with only some tracking columns populated -- neither
// the legacy all-zero case nor the fully-tracked case -- must still be
// rejected.
if _, err := tx.Exec(ctx, `
INSERT INTO public.broadcast_recipients (broadcast_id, user_id, status, sent_at, private_message_id)
VALUES ($1, $2, 'sent', now(), 1)`, broadcastID, int64(9_100_000_000_030_098)); err == nil {
t.Fatalf("insert of a partially-tracked 'sent' row unexpectedly succeeded")
}
}