Fase 1b: cableado del matcher (ingesta polling, runner shadow/full, comparador)
- config: bloque matcher (MATCHER_MODE off/shadow/full, IRON_PASSWORD, audiencia, poll, batch) + validacion - db: colecciones subscriptions/activefires/notifications_shadow/matcher_state + ensureMatcherIndexes (2dsphere en notifications_shadow para el dedupe 500m) - matcher/context: MatchContext sobre Mongo real (candidatas geo $near 1000km + audiencia; dedupe geo $near 500m; lang del owner) - matcher/ingest: polling de activefires por createdAt>checkpoint (Mongo 3.2 sin change streams), checkpoint persistido en matcher_state - matcher/runner: loop shadow(->notifications_shadow)/full(->notifications), idempotente por dedupe+checkpoint - matcher/compare: comparador de shadow (shadowOnly=spam, realOnly=faltantes, contentMismatches) — el gate antes del cutover - index: arranca el matcher; fake-repo ampliado (upsert, sort, $gt/$lt) - docs/env/README actualizados 73 tests en verde, typecheck y build OK.
This commit is contained in:
parent
b3aeb6108f
commit
a1ed206985
13 changed files with 531 additions and 19 deletions
52
test/compare.test.ts
Normal file
52
test/compare.test.ts
Normal file
|
|
@ -0,0 +1,52 @@
|
|||
import { describe, expect, it } from 'vitest';
|
||||
import { ObjectId } from 'mongodb';
|
||||
import { compareShadow, notifKey } from '../src/matcher/compare.js';
|
||||
import type { NotificationDoc } from '../src/types.js';
|
||||
|
||||
function n(userId: string, lon: number, lat: number, content = 'c'): NotificationDoc {
|
||||
return {
|
||||
_id: new ObjectId(),
|
||||
userId,
|
||||
content,
|
||||
geo: { type: 'Point', coordinates: [lon, lat] },
|
||||
type: 'mobile',
|
||||
when: new Date('2026-07-13T12:00:00Z'),
|
||||
sealed: 's',
|
||||
};
|
||||
}
|
||||
|
||||
describe('compareShadow', () => {
|
||||
it('matches identical sets', () => {
|
||||
const a = [n('u1', -3.7, 40.4), n('u2', -4, 41)];
|
||||
const b = [n('u1', -3.7, 40.4), n('u2', -4, 41)];
|
||||
const diff = compareShadow(a, b);
|
||||
expect(diff.matched).toBe(2);
|
||||
expect(diff.shadowOnly).toHaveLength(0);
|
||||
expect(diff.realOnly).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('flags shadowOnly (spam risk) when we generate an extra notification', () => {
|
||||
const shadow = [n('u1', -3.7, 40.4), n('u3', -5, 42)];
|
||||
const real = [n('u1', -3.7, 40.4)];
|
||||
const diff = compareShadow(shadow, real);
|
||||
expect(diff.shadowOnly).toEqual([notifKey(n('u3', -5, 42))]);
|
||||
expect(diff.realOnly).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('flags realOnly (missing) when node-red had one we do not', () => {
|
||||
const shadow = [n('u1', -3.7, 40.4)];
|
||||
const real = [n('u1', -3.7, 40.4), n('u2', -4, 41)];
|
||||
const diff = compareShadow(shadow, real);
|
||||
expect(diff.realOnly).toEqual([notifKey(n('u2', -4, 41))]);
|
||||
expect(diff.shadowOnly).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('reports content mismatches on matched keys', () => {
|
||||
const shadow = [n('u1', -3.7, 40.4, 'nuevo texto')];
|
||||
const real = [n('u1', -3.7, 40.4, 'viejo texto')];
|
||||
const diff = compareShadow(shadow, real);
|
||||
expect(diff.matched).toBe(1);
|
||||
expect(diff.contentMismatches).toHaveLength(1);
|
||||
expect(diff.contentMismatches[0]).toMatchObject({ shadow: 'nuevo texto', real: 'viejo texto' });
|
||||
});
|
||||
});
|
||||
|
|
@ -47,8 +47,12 @@ function matchField(docVal: any, cond: any): boolean {
|
|||
return Array.isArray(v) && v.some((x) => valEq(docVal, x));
|
||||
case '$gte':
|
||||
return docVal != null && docVal >= (v as any);
|
||||
case '$gt':
|
||||
return docVal != null && docVal > (v as any);
|
||||
case '$lte':
|
||||
return docVal != null && docVal <= (v as any);
|
||||
case '$lt':
|
||||
return docVal != null && docVal < (v as any);
|
||||
case '$exists':
|
||||
return (docVal !== undefined) === v;
|
||||
default:
|
||||
|
|
@ -70,6 +74,7 @@ function applyUpdate(doc: Doc, update: Doc): void {
|
|||
|
||||
class FakeCollection {
|
||||
docs: Doc[] = [];
|
||||
collectionName = 'fake';
|
||||
private uniqueFields: string[] | null = null;
|
||||
|
||||
async createIndex(spec: Doc, opts?: { unique?: boolean }): Promise<string> {
|
||||
|
|
@ -102,11 +107,19 @@ class FakeCollection {
|
|||
return this.docs.find((d) => matches(d, filter)) ?? null;
|
||||
}
|
||||
|
||||
async updateOne(filter: Doc, update: Doc): Promise<{ modifiedCount: number }> {
|
||||
async updateOne(filter: Doc, update: Doc, opts?: { upsert?: boolean }): Promise<{ modifiedCount: number; upsertedCount: number }> {
|
||||
const d = this.docs.find((x) => matches(x, filter));
|
||||
if (!d) return { modifiedCount: 0 };
|
||||
if (!d) {
|
||||
if (opts?.upsert) {
|
||||
const created: Doc = { ...filter };
|
||||
applyUpdate(created, update);
|
||||
this.docs.push(created);
|
||||
return { modifiedCount: 0, upsertedCount: 1 };
|
||||
}
|
||||
return { modifiedCount: 0, upsertedCount: 0 };
|
||||
}
|
||||
applyUpdate(d, update);
|
||||
return { modifiedCount: 1 };
|
||||
return { modifiedCount: 1, upsertedCount: 0 };
|
||||
}
|
||||
|
||||
async updateMany(filter: Doc, update: Doc): Promise<{ modifiedCount: number }> {
|
||||
|
|
@ -138,6 +151,16 @@ class FakeCollection {
|
|||
project() {
|
||||
return cursor;
|
||||
},
|
||||
sort(spec: Doc) {
|
||||
const [field, dir] = Object.entries(spec)[0] as [string, number];
|
||||
result = [...result].sort((a, b) => {
|
||||
const av = a[field];
|
||||
const bv = b[field];
|
||||
if (av === bv) return 0;
|
||||
return (av < bv ? -1 : 1) * (dir < 0 ? -1 : 1);
|
||||
});
|
||||
return cursor;
|
||||
},
|
||||
limit(n: number) {
|
||||
result = result.slice(0, n);
|
||||
return cursor;
|
||||
|
|
@ -151,15 +174,16 @@ class FakeCollection {
|
|||
}
|
||||
|
||||
export function makeRepos(): Repos {
|
||||
const notifications = new FakeCollection();
|
||||
const users = new FakeCollection();
|
||||
const sends = new FakeCollection();
|
||||
return {
|
||||
client: {} as any,
|
||||
db: {} as any,
|
||||
notifications: notifications as any,
|
||||
users: users as any,
|
||||
sends: sends as any,
|
||||
notifications: new FakeCollection() as any,
|
||||
users: new FakeCollection() as any,
|
||||
sends: new FakeCollection() as any,
|
||||
subscriptions: new FakeCollection() as any,
|
||||
activefires: new FakeCollection() as any,
|
||||
notificationsShadow: new FakeCollection() as any,
|
||||
matcherState: new FakeCollection() as any,
|
||||
close: async () => {},
|
||||
};
|
||||
}
|
||||
|
|
|
|||
111
test/matcher-runner.test.ts
Normal file
111
test/matcher-runner.test.ts
Normal file
|
|
@ -0,0 +1,111 @@
|
|||
import { beforeEach, describe, expect, it } from 'vitest';
|
||||
import { ObjectId } from 'mongodb';
|
||||
import type { Repos } from '../src/db.js';
|
||||
import type { Config } from '../src/config.js';
|
||||
import { createMatcherRunner } from '../src/matcher/runner.js';
|
||||
import { pollNewFires, advanceCheckpoint, newestCreatedAt } from '../src/matcher/ingest.js';
|
||||
import type { MatchContext } from '../src/matcher/matcher.js';
|
||||
import type { FireDoc } from '../src/matcher/types.js';
|
||||
import { makeRepos } from './fake-repo.js';
|
||||
import { silentLog, testCfg } from './helpers.js';
|
||||
|
||||
let repos: Repos;
|
||||
beforeEach(() => {
|
||||
repos = makeRepos();
|
||||
});
|
||||
|
||||
function matcherCfg(mode: 'off' | 'shadow' | 'full', over: Partial<Config['matcher']> = {}): Config {
|
||||
return testCfg({ matcher: { mode, audienceTypes: ['web', 'mobile'], pollIntervalMs: 1000, batchSize: 2000, ironPassword: 'x'.repeat(32), ...over } });
|
||||
}
|
||||
|
||||
function fireDoc(over: Partial<FireDoc> = {}): FireDoc {
|
||||
return { _id: new ObjectId(), lat: 40.4, lon: -3.7, when: new Date('2026-07-13T12:00:00Z'), type: 'viirs', createdAt: new Date('2026-07-13T12:05:00Z'), ...over };
|
||||
}
|
||||
|
||||
describe('ingest', () => {
|
||||
it('polls fires created after the checkpoint, ascending', async () => {
|
||||
await repos.activefires.insertMany([
|
||||
fireDoc({ createdAt: new Date('2026-07-13T10:00:00Z') }),
|
||||
fireDoc({ createdAt: new Date('2026-07-13T12:00:00Z') }),
|
||||
fireDoc({ createdAt: new Date('2026-07-13T13:00:00Z') }),
|
||||
] as never);
|
||||
await advanceCheckpoint(repos, new Date('2026-07-13T11:00:00Z'));
|
||||
const cfg = matcherCfg('shadow');
|
||||
|
||||
const fires = await pollNewFires(cfg, repos);
|
||||
expect(fires).toHaveLength(2); // only the 12:00 and 13:00 ones
|
||||
expect(fires[0]!.createdAt! < fires[1]!.createdAt!).toBe(true);
|
||||
});
|
||||
|
||||
it('newestCreatedAt returns the max', () => {
|
||||
const max = newestCreatedAt([
|
||||
fireDoc({ createdAt: new Date('2026-07-13T12:00:00Z') }),
|
||||
fireDoc({ createdAt: new Date('2026-07-13T14:00:00Z') }),
|
||||
]);
|
||||
expect(max).toEqual(new Date('2026-07-13T14:00:00Z'));
|
||||
});
|
||||
});
|
||||
|
||||
describe('matcher runner', () => {
|
||||
// Deterministic context so we don't need geo indexes in the fake.
|
||||
function fakeCtx(over: Partial<MatchContext> = {}): MatchContext {
|
||||
return {
|
||||
candidateSubs: async () => [
|
||||
{ _id: new ObjectId(), location: { lat: 40.4, lon: -3.7 }, geo: { type: 'Point', coordinates: [-3.7, 40.4] }, distance: 50, owner: 'u1', type: 'mobile' },
|
||||
],
|
||||
existsNotifNear: async () => false,
|
||||
userLang: async () => 'es',
|
||||
ironPassword: 'x'.repeat(32),
|
||||
audienceTypes: new Set(['web', 'mobile']),
|
||||
now: () => new Date('2026-07-13T12:06:00Z'),
|
||||
...over,
|
||||
};
|
||||
}
|
||||
|
||||
it('does nothing when mode is off', async () => {
|
||||
const cfg = matcherCfg('off');
|
||||
const r = createMatcherRunner(cfg, repos, silentLog, fakeCtx());
|
||||
const res = await r.runOnce();
|
||||
expect(res).toEqual({ fires: 0, generated: 0 });
|
||||
});
|
||||
|
||||
it('shadow mode writes to notifications_shadow, not notifications', async () => {
|
||||
await repos.activefires.insertOne(fireDoc() as never);
|
||||
await advanceCheckpoint(repos, new Date('2026-07-13T00:00:00Z'));
|
||||
const cfg = matcherCfg('shadow');
|
||||
const r = createMatcherRunner(cfg, repos, silentLog, fakeCtx());
|
||||
|
||||
const res = await r.runOnce();
|
||||
|
||||
expect(res.generated).toBe(1);
|
||||
expect(await repos.notificationsShadow.countDocuments({})).toBe(1);
|
||||
expect(await repos.notifications.countDocuments({})).toBe(0);
|
||||
// checkpoint advanced
|
||||
const state = await repos.matcherState.findOne({ _id: 'fires' });
|
||||
expect(state?.checkpoint).toEqual(fireDoc().createdAt);
|
||||
});
|
||||
|
||||
it('full mode writes to notifications', async () => {
|
||||
await repos.activefires.insertOne(fireDoc() as never);
|
||||
await advanceCheckpoint(repos, new Date('2026-07-13T00:00:00Z'));
|
||||
const cfg = matcherCfg('full');
|
||||
const r = createMatcherRunner(cfg, repos, silentLog, fakeCtx());
|
||||
|
||||
const res = await r.runOnce();
|
||||
|
||||
expect(res.generated).toBe(1);
|
||||
expect(await repos.notifications.countDocuments({})).toBe(1);
|
||||
const n = await repos.notifications.findOne({});
|
||||
expect(n?.type).toBe('mobile');
|
||||
expect(n?.sealed).toContain('Fe26.2');
|
||||
});
|
||||
|
||||
it('generates nothing for fires with no matching subs', async () => {
|
||||
await repos.activefires.insertOne(fireDoc() as never);
|
||||
await advanceCheckpoint(repos, new Date('2026-07-13T00:00:00Z'));
|
||||
const cfg = matcherCfg('shadow');
|
||||
const r = createMatcherRunner(cfg, repos, silentLog, fakeCtx({ candidateSubs: async () => [] }));
|
||||
const res = await r.runOnce();
|
||||
expect(res.generated).toBe(0);
|
||||
});
|
||||
});
|
||||
Loading…
Add table
Add a link
Reference in a new issue