From 124ed4e1354c0de579a708763a26f8c630c4bcf9 Mon Sep 17 00:00:00 2001 From: vjrj Date: Sat, 1 Aug 2026 18:02:04 +0200 Subject: [PATCH] refactor(subsUnion): extract the logic out of Meteor.startup, cap the union size MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Everything the union does lived inside a Meteor.startup callback with no exports: the queue, the incremental fast path, the worker-failure handling. None of it was reachable from a test, which is how three separate incidents shipped — a failed worker stored as an empty union (50ca9cc), turf deps unresolvable from the worker (3249362/570cb49) and a recompute that blocked the event loop and froze DDP (b4e5511/fbb746f). subsUnionLogic.js now takes its collaborators as arguments and subsUnion.js is only the wiring. The FIXME at the old subsUnion.js:82 goes with it, because it lives inside the extracted storeUnion: a union over 16 MiB is simply rejected by Mongo, so the map would freeze at the last union that happened to fit, silently. unionSizeGuard.js degrades the geometry instead — coordinate precision, then vertex decimation, then holes, then the smallest polygons — until it fits, and reports what it did so the setting document records it. The cap is 8 MiB rather than 16: the same string is pushed to every browser over DDP. addNoisy no longer mutates the document it is given, so the public pass cannot leak its fuzzing into the private one. --- imports/startup/server/subsUnion.js | 211 +++---------------- imports/startup/server/subsUnionLogic.js | 252 +++++++++++++++++++++++ imports/startup/server/unionBounds.js | 11 + imports/startup/server/unionSizeGuard.js | 212 +++++++++++++++++++ 4 files changed, 499 insertions(+), 187 deletions(-) create mode 100644 imports/startup/server/subsUnionLogic.js create mode 100644 imports/startup/server/unionBounds.js create mode 100644 imports/startup/server/unionSizeGuard.js diff --git a/imports/startup/server/subsUnion.js b/imports/startup/server/subsUnion.js index 641b308..169cd61 100644 --- a/imports/startup/server/subsUnion.js +++ b/imports/startup/server/subsUnion.js @@ -1,214 +1,51 @@ /* eslint-disable import/no-absolute-path */ +// Wiring only: what the union actually does lives in subsUnionLogic.js, where it +// can be tested without booting a server (see test/server/subsUnion.test.js). + import { Meteor } from 'meteor/meteor'; import Subscriptions from '/imports/api/Subscriptions/Subscriptions'; import SiteSettings from '/imports/api/SiteSettings/SiteSettings'; -import Perlin from 'loms.perlin'; -import './leaflet-workaround'; -import L from 'leaflet'; import calcUnionAsync from '/imports/startup/server/calcUnionAsync'; +import unionBounds from '/imports/startup/server/unionBounds'; +import createSubsUnion from '/imports/startup/server/subsUnionLogic'; import { isMailServerMaster } from '/imports/startup/server/email'; -// sudo apt-get install libcairo2-dev libjpeg-dev libgif-dev - Meteor.startup(async () => { if (!isMailServerMaster) { console.log('We only process subsUnion in master'); return; } - const debug = true; // Meteor.isDevelopment; - Perlin.seed(Math.random()); - const addNoisy = (osub) => { - const sub = osub; - let lat = Math.round(sub.location.lat * 10) / 10; - let lon = Math.round(sub.location.lon * 10) / 10; - const noiseBase = Perlin.perlin2(lat, lon); - const noise = Math.abs(noiseBase / 3); - lat += noise; - lon += noise; - sub.location.lat = lat; - sub.location.lon = lon; - sub.distance += noiseBase; - return sub; - }; + const subsUnion = createSubsUnion({ + calcUnion: calcUnionAsync, + SiteSettings, + Subscriptions, + boundsOf: unionBounds + }); - const noNoisy = sub => sub; - - const isValidSub = (osub) => { - const valid = osub.location && osub.location.lat && osub.location.lon && osub.distance; - if (!valid) console.info(`Wrong element to do union ${JSON.stringify(osub)}`); - return valid; - }; - - // Only called once calcUnionAsync has actually resolved — union === null here - // means "zero valid subscriptions", a real result, not a failed computation. - // (A prior version guarded this with `typeof union === 'object'`, which is - // true for null too, so a failed worker call — caught upstream, leaving - // union as null — looked identical to a legitimate empty union and got - // silently stored as "null". Callers must catch calcUnionAsync failures - // themselves and skip calling this, instead of relying on this check.) - const storeUnion = async (isPublic, union, count) => { - const publicl = isPublic ? 'public' : 'private'; - const Publicl = publicl.replace(/\b\w/g, l => l.toUpperCase()); - const bounds = union === null ? null : L.geoJSON(union).getBounds(); - const unionSet = { - $set: { - name: `subs-${publicl}-union`, - value: JSON.stringify(union), - isPublic, - description: `${Publicl} subscriptions union`, - type: 'string' - } - }; - const boundsSet = { - $set: { - name: `subs-${publicl}-union-bounds`, - value: JSON.stringify(bounds), - isPublic, - description: `${Publicl} subscriptions union bounds`, - type: 'string' - } - }; - const sizeSet = { - $set: { - name: 'subs-union-count', - value: count, - isPublic: false, - description: 'Subscriptions count', - type: 'number' - } - }; - // FIXME, take care of object size: - // https://stackoverflow.com/questions/10827812/what-is-the-length-maximum-for-a-string-data-type-in-mongodb-used-with-ruby - await SiteSettings.upsertAsync({ name: `subs-${publicl}-union` }, unionSet, { multi: false }); - await SiteSettings.upsertAsync({ name: `subs-${publicl}-union-bounds` }, boundsSet, { multi: false }); - await SiteSettings.upsertAsync({ name: 'subs-union-count' }, sizeSet, { multi: false }); - if (debug) console.log(`${Publicl} subscription union calculated`); - }; - - const process = async (isPublic) => { - const subscribers = await Subscriptions.find().fetchAsync(); - const decorate = isPublic ? addNoisy : noNoisy; - const decorated = subscribers.filter(isValidSub).map(decorate); - - try { - const union = await calcUnionAsync(decorated); - await storeUnion(isPublic, union, subscribers.length); - } catch (e) { - console.error('subsUnion worker failed, union left unchanged', e); - } - }; - - const recreate = async () => { await process(true); await process(false); }; - - // Single serialized worker over two kinds of work: incremental adds (a real - // FIFO queue — each must see the previous one's result, none get dropped) - // and full recomputes (coalesced — a burst of changed/removed events only - // needs one pass once things settle). Both touch the same stored union, so - // they share this one lock rather than racing each other. - let busy = false; - const addQueue = []; - let recomputePending = false; - - // Fast path for the common case (a new subscription): merge just the new - // circle into the union already stored instead of redoing the full turf.union - // chain over every subscription again (that chain is what made a single - // recompute take ~20+ minutes with thousands of subscriptions). Falls back to - // a full recreate() if the stored state isn't exactly "one behind" what we - // expect — safer than merging into stale/inconsistent data. Sets - // recomputePending directly (rather than calling a schedule helper) since - // runLoop already re-checks it right after this returns. - const incrementalAdd = async (doc) => { - if (!isValidSub(doc)) return; - - const countSetting = await SiteSettings.findOneAsync({ name: 'subs-union-count' }); - const countSubs = await Subscriptions.find({}).countAsync(); - if (!countSetting || countSetting.value !== countSubs - 1) { - if (debug) console.log('Subs union state not exactly one behind, doing a full recreate instead'); - recomputePending = true; - return; - } - - // eslint-disable-next-line no-restricted-syntax - for (const isPublic of [true, false]) { - const decorate = isPublic ? addNoisy : noNoisy; - const decorated = decorate({ ...doc, location: { ...doc.location } }); - const publicl = isPublic ? 'public' : 'private'; - // eslint-disable-next-line no-await-in-loop - const current = await SiteSettings.findOneAsync({ name: `subs-${publicl}-union` }); - const baseUnion = current && current.value ? JSON.parse(current.value) : null; - - try { - // eslint-disable-next-line no-await-in-loop - const union = await calcUnionAsync([decorated], baseUnion); - // eslint-disable-next-line no-await-in-loop - await storeUnion(isPublic, union, countSubs); - } catch (e) { - console.error('subsUnion incremental worker failed, falling back to a full recreate', e); - recomputePending = true; - return; - } - } - if (debug) console.log('Subs union updated incrementally'); - }; - - const runLoop = () => { - if (busy) return; - if (addQueue.length > 0) { - busy = true; - const doc = addQueue.shift(); - incrementalAdd(doc) - .catch(e => console.error('subsUnion incremental add failed', e)) - .finally(() => { busy = false; runLoop(); }); - return; - } - if (recomputePending) { - recomputePending = false; - busy = true; - recreate() - .catch(e => console.error('subsUnion recompute failed', e)) - .finally(() => { busy = false; runLoop(); }); - } - }; - - const enqueueAdd = (doc) => { addQueue.push(doc); runLoop(); }; - - // Fire-and-forget on purpose: both recreate() and incrementalAdd() can take a - // while, and callers (Meteor.startup, observeAsync callbacks) must not block on it. - const scheduleRecreate = () => { recomputePending = true; runLoop(); }; - - // At startup, we check if it's necessary to calc subscriptions union again - const currentUnion = await SiteSettings.findOneAsync({ name: 'subs-public-union' }); - const lastSubs = await Subscriptions.findOneAsync({}, { sort: { updatedAt: -1 } }); - const countUnionSubs = await SiteSettings.findOneAsync({ name: 'subs-union-count' }); - const countSubs = await Subscriptions.find({}).countAsync(); - if (currentUnion && lastSubs) { - const lastUnionUpdated = currentUnion.updatedAt; - const lastSubsUpdated = lastSubs.updatedAt; - if (lastUnionUpdated > lastSubsUpdated || !countUnionSubs || countSubs !== countUnionSubs.value) { - console.log('Subs union outdated'); - scheduleRecreate(); - } else { - console.log('Subs union up-to-date'); - } + if (await subsUnion.isUnionOutdated()) { + console.log('Subs union outdated'); + subsUnion.scheduleRecreate(); + } else { + console.log('Subs union up-to-date'); } await Subscriptions.find({ createdAt: { $gt: new Date() } }).observeAsync({ added: function newSubAdded(doc) { - if (debug) console.log('Subs added, merging into union incrementally'); - enqueueAdd(doc); + console.log('Subs added, merging into union incrementally'); + subsUnion.enqueueAdd(doc); } }); await Subscriptions.find().observeAsync({ - changed: function subsChanged() { // updatedDoc, oldDoc) { - if (debug) console.log('Subs changed so recreate union'); - scheduleRecreate(); + changed: function subsChanged() { + console.log('Subs changed so recreate union'); + subsUnion.scheduleRecreate(); }, - removed: function subsRemoved() { // oldDoc) { - if (debug) console.log('Subs removed so recreate union'); - scheduleRecreate(); + removed: function subsRemoved() { + console.log('Subs removed so recreate union'); + subsUnion.scheduleRecreate(); } }); }); diff --git a/imports/startup/server/subsUnionLogic.js b/imports/startup/server/subsUnionLogic.js new file mode 100644 index 0000000..1788737 --- /dev/null +++ b/imports/startup/server/subsUnionLogic.js @@ -0,0 +1,252 @@ +/* eslint-disable import/no-absolute-path */ + +// All of the subscriptions-union behaviour, with its collaborators injected. +// +// This used to live inside the Meteor.startup() callback of subsUnion.js, where +// nothing was reachable from a test: the queue, the incremental fast path and the +// worker-failure handling could only be exercised by starting a whole server and +// hoping. That is how a failed worker call ended up being stored as a valid empty +// union (50ca9cc) and how a full recompute ended up blocking DDP (b4e5511 / +// fbb746f) — three production incidents in code with zero test coverage. +// subsUnion.js is now only the wiring; everything below is a plain function you +// can call with fakes. + +import Perlin from 'loms.perlin'; +import limitUnionSize, { MAX_UNION_BYTES } from './unionSizeGuard'; + +// Telegram-era subscriptions (and anything else half-written) can be missing the +// location or the radius. They are skipped, not an error: they predate the web +// subscriptions and there is nothing sensible to draw for them. +export const isValidSub = (osub, log = console) => { + const valid = !!(osub && osub.location && osub.location.lat && osub.location.lon && osub.distance); + if (!valid) log.info(`Wrong element to do union ${JSON.stringify(osub)}`); + return valid; +}; + +/** + * @param {object} deps + * @param {function} deps.calcUnion (subs, baseUnion) => Promise, off-thread + * @param {object} deps.SiteSettings Mongo collection holding the stored union + * @param {object} deps.Subscriptions Mongo collection of subscriptions + * @param {function} [deps.boundsOf] union => leaflet-style bounds (or null) + * @param {number} [deps.maxUnionBytes] + * @param {number} [deps.perlinSeed] fixed seed makes the public "noise" reproducible in tests + */ +const createSubsUnion = ({ + calcUnion, + SiteSettings, + Subscriptions, + boundsOf = () => null, + maxUnionBytes = MAX_UNION_BYTES, + perlinSeed = Math.random(), + debug = true, + log = console +}) => { + Perlin.seed(perlinSeed); + + // Public unions are fuzzed so that the map does not disclose where exactly + // somebody subscribed. Returns a new object: callers hand us documents straight + // out of Mongo and the private pass must not see the fuzzed values. + const addNoisy = (sub) => { + const lat = Math.round(sub.location.lat * 10) / 10; + const lon = Math.round(sub.location.lon * 10) / 10; + const noiseBase = Perlin.perlin2(lat, lon); + const noise = Math.abs(noiseBase / 3); + return { + ...sub, + location: { ...sub.location, lat: lat + noise, lon: lon + noise }, + distance: sub.distance + noiseBase + }; + }; + + const noNoisy = sub => ({ ...sub, location: { ...sub.location } }); + + const decoratorFor = isPublic => (isPublic ? addNoisy : noNoisy); + + // Only called once calcUnion has actually resolved — union === null here means + // "zero valid subscriptions", a real result, not a failed computation. (A prior + // version guarded this with `typeof union === 'object'`, which is true for null + // too, so a failed worker call — caught upstream, leaving union as null — looked + // identical to a legitimate empty union and got silently stored as "null". + // Callers must catch calcUnion failures themselves and skip calling this.) + const storeUnion = async (isPublic, union, count) => { + const publicl = isPublic ? 'public' : 'private'; + const Publicl = publicl.replace(/\b\w/g, l => l.toUpperCase()); + // Mongo caps a document at 16 MiB and this value is also pushed to every + // browser: past a certain number of subscriptions the geometry has to be + // coarsened or the upsert simply starts failing. See unionSizeGuard.js. + const { union: safeUnion, json, degraded } = limitUnionSize(union, maxUnionBytes); + if (degraded) { + log.warn(`${Publicl} subscriptions union too big (${degraded.originalBytes} bytes), degraded via ${degraded.steps.join(' -> ')}`); + } + const bounds = boundsOf(safeUnion); + const unionSet = { + $set: { + name: `subs-${publicl}-union`, + value: json, + isPublic, + description: `${Publicl} subscriptions union`, + type: 'string', + degraded: degraded ? degraded.steps.join(' -> ') : null + } + }; + const boundsSet = { + $set: { + name: `subs-${publicl}-union-bounds`, + value: JSON.stringify(bounds), + isPublic, + description: `${Publicl} subscriptions union bounds`, + type: 'string' + } + }; + const sizeSet = { + $set: { + name: 'subs-union-count', + value: count, + isPublic: false, + description: 'Subscriptions count', + type: 'number' + } + }; + await SiteSettings.upsertAsync({ name: `subs-${publicl}-union` }, unionSet, { multi: false }); + await SiteSettings.upsertAsync({ name: `subs-${publicl}-union-bounds` }, boundsSet, { multi: false }); + await SiteSettings.upsertAsync({ name: 'subs-union-count' }, sizeSet, { multi: false }); + if (debug) log.log(`${Publicl} subscription union calculated`); + return { degraded }; + }; + + const recomputeOne = async (isPublic) => { + const subscribers = await Subscriptions.find().fetchAsync(); + const decorated = subscribers.filter(s => isValidSub(s, log)).map(decoratorFor(isPublic)); + + try { + const union = await calcUnion(decorated); + await storeUnion(isPublic, union, subscribers.length); + } catch (e) { + log.error('subsUnion worker failed, union left unchanged', e); + } + }; + + const recreate = async () => { await recomputeOne(true); await recomputeOne(false); }; + + // Single serialized worker over two kinds of work: incremental adds (a real + // FIFO queue — each must see the previous one's result, none get dropped) and + // full recomputes (coalesced — a burst of changed/removed events only needs one + // pass once things settle). Both touch the same stored union, so they share this + // one lock rather than racing each other. + let busy = false; + const addQueue = []; + let recomputePending = false; + let idleWaiters = []; + + const flushIdleWaiters = () => { + const waiters = idleWaiters; + idleWaiters = []; + waiters.forEach(resolve => resolve()); + }; + + // Fast path for the common case (a new subscription): merge just the new circle + // into the union already stored instead of redoing the full turf.union chain + // over every subscription again (that chain is what made a single recompute take + // ~20+ minutes with thousands of subscriptions). Falls back to a full recreate() + // if the stored state isn't exactly "one behind" what we expect — safer than + // merging into stale/inconsistent data. Sets recomputePending directly (rather + // than calling a schedule helper) since runLoop already re-checks it right after + // this returns. + const incrementalAdd = async (doc) => { + if (!isValidSub(doc, log)) return; + + const countSetting = await SiteSettings.findOneAsync({ name: 'subs-union-count' }); + const countSubs = await Subscriptions.find({}).countAsync(); + if (!countSetting || countSetting.value !== countSubs - 1) { + if (debug) log.log('Subs union state not exactly one behind, doing a full recreate instead'); + recomputePending = true; + return; + } + + // eslint-disable-next-line no-restricted-syntax + for (const isPublic of [true, false]) { + const decorated = decoratorFor(isPublic)(doc); + const publicl = isPublic ? 'public' : 'private'; + // eslint-disable-next-line no-await-in-loop + const current = await SiteSettings.findOneAsync({ name: `subs-${publicl}-union` }); + const baseUnion = current && current.value ? JSON.parse(current.value) : null; + + try { + // eslint-disable-next-line no-await-in-loop + const union = await calcUnion([decorated], baseUnion); + // eslint-disable-next-line no-await-in-loop + await storeUnion(isPublic, union, countSubs); + } catch (e) { + log.error('subsUnion incremental worker failed, falling back to a full recreate', e); + recomputePending = true; + return; + } + } + if (debug) log.log('Subs union updated incrementally'); + }; + + const runLoop = () => { + if (busy) return; + if (addQueue.length > 0) { + busy = true; + const doc = addQueue.shift(); + incrementalAdd(doc) + .catch(e => log.error('subsUnion incremental add failed', e)) + .finally(() => { busy = false; runLoop(); }); + return; + } + if (recomputePending) { + recomputePending = false; + busy = true; + recreate() + .catch(e => log.error('subsUnion recompute failed', e)) + .finally(() => { busy = false; runLoop(); }); + return; + } + flushIdleWaiters(); + }; + + const enqueueAdd = (doc) => { addQueue.push(doc); runLoop(); }; + + // Fire-and-forget on purpose: both recreate() and incrementalAdd() can take a + // while, and callers (Meteor.startup, observeAsync callbacks) must not block on it. + const scheduleRecreate = () => { recomputePending = true; runLoop(); }; + + // Resolves when the queue has drained and nothing is running. Only meant for + // tests and shutdown — production code must never await the union. + const idle = () => new Promise((resolve) => { + if (!busy && addQueue.length === 0 && !recomputePending) { + resolve(); + return; + } + idleWaiters.push(resolve); + }); + + // At startup we check whether the stored union still matches the subscriptions. + const isUnionOutdated = async () => { + const currentUnion = await SiteSettings.findOneAsync({ name: 'subs-public-union' }); + const lastSubs = await Subscriptions.findOneAsync({}, { sort: { updatedAt: -1 } }); + if (!currentUnion || !lastSubs) return false; + const countUnionSubs = await SiteSettings.findOneAsync({ name: 'subs-union-count' }); + const countSubs = await Subscriptions.find({}).countAsync(); + return currentUnion.updatedAt > lastSubs.updatedAt + || !countUnionSubs + || countSubs !== countUnionSubs.value; + }; + + return { + addNoisy, + noNoisy, + storeUnion, + recreate, + incrementalAdd, + enqueueAdd, + scheduleRecreate, + isUnionOutdated, + idle, + stats: () => ({ busy, queued: addQueue.length, recomputePending }) + }; +}; + +export default createSubsUnion; diff --git a/imports/startup/server/unionBounds.js b/imports/startup/server/unionBounds.js new file mode 100644 index 0000000..47e39cb --- /dev/null +++ b/imports/startup/server/unionBounds.js @@ -0,0 +1,11 @@ +/* eslint-disable import/no-absolute-path */ + +// Leaflet is a browser library; the workaround import fakes the bits of `window` +// it touches at load time so it can also run on the server. Kept apart from +// subsUnionLogic so that module (and its tests) stay free of that dependency. +import './leaflet-workaround'; +import L from 'leaflet'; + +const unionBounds = union => (union === null ? null : L.geoJSON(union).getBounds()); + +export default unionBounds; diff --git a/imports/startup/server/unionSizeGuard.js b/imports/startup/server/unionSizeGuard.js new file mode 100644 index 0000000..07cdb3c --- /dev/null +++ b/imports/startup/server/unionSizeGuard.js @@ -0,0 +1,212 @@ +// Keeps the stored subscriptions-union GeoJSON below a size that Mongo (and the +// DDP connection that ships it to every client) can actually take. +// +// The union of every subscription circle is stored as a JSON string inside a +// single `siteSettings` document. A BSON document is capped at 16 MiB, so past +// some number of subscriptions the upsert would simply start failing and the map +// would silently freeze at the last union that fit — which is exactly the kind of +// scale bug nobody notices until production. On top of the hard Mongo limit, that +// same string is published to browsers, so multi-megabyte unions are already a +// problem well before 16 MiB. +// +// The chosen fix is progressive degradation instead of failure: the geometry is +// coarsened, in visually-cheapest-first order, until it fits. The zone union is a +// decorative overlay ("roughly where people are watching"), never a source of +// truth for alerts, so losing precision is acceptable; losing the whole overlay is +// not. Every degradation is reported back so the caller can store it alongside the +// union and ops can see it happening. +// +// The ladder, in order: +// 1. coordinate precision 6 -> 5 -> 4 -> 3 decimals (~0.1 m -> ~100 m), dropping +// points that collapse onto their neighbour after rounding. +// 2. point decimation, keeping every k-th vertex of each ring (circles are +// generated with 144 steps, so there is a lot of slack here). +// 3. dropping holes (inner rings) — visually the least missed. +// 4. dropping the smallest polygons of the MultiPolygon until it fits. +// If even a single polygon does not fit, null is returned rather than storing +// something Mongo will reject. + +// 16 MiB is Mongo's hard limit for the whole document; this cap is deliberately +// well below it because the same value travels over DDP to every connected +// browser. Raise it only together with a plan for the client side. +export const MAX_UNION_BYTES = 8 * 1024 * 1024; + +const MIN_RING_POINTS = 12; +const DECIMATION_FACTORS = [2, 3, 4, 6, 8]; +const PRECISIONS = [5, 4, 3]; + +export const byteLength = json => Buffer.byteLength(json, 'utf8'); + +const geometryOf = union => (union && union.type === 'Feature' ? union.geometry : union); + +// Both Polygon and MultiPolygon, seen as a list of polygons (a polygon being a +// list of rings, the first one the outer one). +const polygonsOf = (geometry) => { + if (!geometry || !geometry.coordinates) return []; + return geometry.type === 'MultiPolygon' ? geometry.coordinates : [geometry.coordinates]; +}; + +const withPolygons = (union, polygons) => { + const geometry = geometryOf(union); + const coordinates = geometry.type === 'MultiPolygon' ? polygons : polygons[0]; + const newGeometry = { ...geometry, coordinates }; + return union.type === 'Feature' ? { ...union, geometry: newGeometry } : newGeometry; +}; + +// A ring is only a ring if it closes and has at least three distinct corners. +const closeRing = (ring) => { + if (ring.length < 3) return null; + const [firstLon, firstLat] = ring[0]; + const [lastLon, lastLat] = ring[ring.length - 1]; + const closed = firstLon === lastLon && firstLat === lastLat ? ring : [...ring, [firstLon, firstLat]]; + return closed.length >= 4 ? closed : null; +}; + +const roundRing = (ring, factor) => { + const out = []; + for (let i = 0; i < ring.length; i += 1) { + const lon = Math.round(ring[i][0] * factor) / factor; + const lat = Math.round(ring[i][1] * factor) / factor; + const prev = out[out.length - 1]; + if (!prev || prev[0] !== lon || prev[1] !== lat) out.push([lon, lat]); + } + return closeRing(out); +}; + +const decimateRing = (ring, keepEvery) => { + if (ring.length <= MIN_RING_POINTS) return ring; + const out = []; + for (let i = 0; i < ring.length - 1; i += keepEvery) out.push(ring[i]); + return closeRing(out) || ring; +}; + +// Rings that survive a transformation keep the polygon alive; a polygon whose +// outer ring collapsed is dropped entirely (its holes are meaningless then). +const mapPolygons = (polygons, mapRing) => polygons + .map(rings => rings.map(mapRing).filter(Boolean)) + .filter(rings => rings.length > 0); + +const ringBboxArea = (ring) => { + let minLon = Infinity; + let minLat = Infinity; + let maxLon = -Infinity; + let maxLat = -Infinity; + for (let i = 0; i < ring.length; i += 1) { + if (ring[i][0] < minLon) [minLon] = ring[i]; + if (ring[i][0] > maxLon) [maxLon] = ring[i]; + if (ring[i][1] < minLat) [, minLat] = ring[i]; + if (ring[i][1] > maxLat) [, maxLat] = ring[i]; + } + return (maxLon - minLon) * (maxLat - minLat); +}; + +/** + * Returns the largest-fidelity version of `union` whose JSON fits in `maxBytes`, + * together with that JSON (callers always need the string anyway, and + * re-stringifying a multi-megabyte object on the main thread is not free). + * + * @returns {{ union: object|null, json: string, bytes: number, + * degraded: null | { originalBytes: number, steps: string[] } }} + */ +const limitUnionSize = (union, maxBytes = MAX_UNION_BYTES) => { + let json = JSON.stringify(union === undefined ? null : union); + let bytes = byteLength(json); + if (union === null || union === undefined || bytes <= maxBytes) { + return { + union: union === undefined ? null : union, json, bytes, degraded: null + }; + } + + const originalBytes = bytes; + const steps = []; + let current = union; + + const attempt = (step, transform) => { + const next = transform(); + if (!next) return false; + current = next; + steps.push(step); + json = JSON.stringify(current); + bytes = byteLength(json); + return bytes <= maxBytes; + }; + + const done = () => ({ + union: current, json, bytes, degraded: { originalBytes, steps } + }); + + for (let i = 0; i < PRECISIONS.length; i += 1) { + const precision = PRECISIONS[i]; + const factor = 10 ** precision; + const fits = attempt(`precision:${precision}`, () => { + const polygons = mapPolygons(polygonsOf(geometryOf(current)), ring => roundRing(ring, factor)); + return polygons.length > 0 ? withPolygons(current, polygons) : null; + }); + if (fits) return done(); + } + + for (let i = 0; i < DECIMATION_FACTORS.length; i += 1) { + const keepEvery = DECIMATION_FACTORS[i]; + const fits = attempt(`decimate:${keepEvery}`, () => { + const polygons = mapPolygons(polygonsOf(geometryOf(union)), ring => decimateRing( + roundRing(ring, 10 ** PRECISIONS[PRECISIONS.length - 1]) || ring, + keepEvery + )); + return polygons.length > 0 ? withPolygons(current, polygons) : null; + }); + if (fits) return done(); + } + + const fitsWithoutHoles = attempt('holes', () => { + const polygons = polygonsOf(geometryOf(current)).map(rings => [rings[0]]); + return polygons.length > 0 ? withPolygons(current, polygons) : null; + }); + if (fitsWithoutHoles) return done(); + + // Last resort: drop the smallest polygons first, so what remains is the part of + // the map a user is most likely to be looking at. Binary search for how many + // survive — trying one fewer at a time means re-serializing a multi-megabyte + // object hundreds of times, which is precisely the kind of main-thread stall + // this whole area is trying to get rid of. + const ordered = polygonsOf(geometryOf(current)) + .map(rings => ({ rings, area: ringBboxArea(rings[0]) })) + .sort((a, b) => b.area - a.area) + .map(p => p.rings); + + const fitsWith = (keep) => { + const candidate = withPolygons(current, ordered.slice(0, keep)); + const candidateJson = JSON.stringify(candidate); + return byteLength(candidateJson) <= maxBytes ? { candidate, candidateJson } : null; + }; + + let low = 1; + let high = ordered.length - 1; + let best = null; + while (low <= high) { + const mid = Math.floor((low + high) / 2); + const fit = fitsWith(mid); + if (fit) { + best = { keep: mid, ...fit }; + low = mid + 1; + } else { + high = mid - 1; + } + } + if (best) { + current = best.candidate; + json = best.candidateJson; + bytes = byteLength(json); + steps.push(`polygons:${best.keep}`); + return done(); + } + + // A single polygon that still does not fit means something is very wrong + // upstream; storing nothing beats an upsert that Mongo rejects on every run. + steps.push('dropped'); + current = null; + json = 'null'; + bytes = byteLength(json); + return done(); +}; + +export default limitUnionSize;