diff --git a/imports/startup/server/subsUnion.js b/imports/startup/server/subsUnion.js index 641b308..169cd61 100644 --- a/imports/startup/server/subsUnion.js +++ b/imports/startup/server/subsUnion.js @@ -1,214 +1,51 @@ /* eslint-disable import/no-absolute-path */ +// Wiring only: what the union actually does lives in subsUnionLogic.js, where it +// can be tested without booting a server (see test/server/subsUnion.test.js). + import { Meteor } from 'meteor/meteor'; import Subscriptions from '/imports/api/Subscriptions/Subscriptions'; import SiteSettings from '/imports/api/SiteSettings/SiteSettings'; -import Perlin from 'loms.perlin'; -import './leaflet-workaround'; -import L from 'leaflet'; import calcUnionAsync from '/imports/startup/server/calcUnionAsync'; +import unionBounds from '/imports/startup/server/unionBounds'; +import createSubsUnion from '/imports/startup/server/subsUnionLogic'; import { isMailServerMaster } from '/imports/startup/server/email'; -// sudo apt-get install libcairo2-dev libjpeg-dev libgif-dev - Meteor.startup(async () => { if (!isMailServerMaster) { console.log('We only process subsUnion in master'); return; } - const debug = true; // Meteor.isDevelopment; - Perlin.seed(Math.random()); - const addNoisy = (osub) => { - const sub = osub; - let lat = Math.round(sub.location.lat * 10) / 10; - let lon = Math.round(sub.location.lon * 10) / 10; - const noiseBase = Perlin.perlin2(lat, lon); - const noise = Math.abs(noiseBase / 3); - lat += noise; - lon += noise; - sub.location.lat = lat; - sub.location.lon = lon; - sub.distance += noiseBase; - return sub; - }; + const subsUnion = createSubsUnion({ + calcUnion: calcUnionAsync, + SiteSettings, + Subscriptions, + boundsOf: unionBounds + }); - const noNoisy = sub => sub; - - const isValidSub = (osub) => { - const valid = osub.location && osub.location.lat && osub.location.lon && osub.distance; - if (!valid) console.info(`Wrong element to do union ${JSON.stringify(osub)}`); - return valid; - }; - - // Only called once calcUnionAsync has actually resolved — union === null here - // means "zero valid subscriptions", a real result, not a failed computation. - // (A prior version guarded this with `typeof union === 'object'`, which is - // true for null too, so a failed worker call — caught upstream, leaving - // union as null — looked identical to a legitimate empty union and got - // silently stored as "null". Callers must catch calcUnionAsync failures - // themselves and skip calling this, instead of relying on this check.) - const storeUnion = async (isPublic, union, count) => { - const publicl = isPublic ? 'public' : 'private'; - const Publicl = publicl.replace(/\b\w/g, l => l.toUpperCase()); - const bounds = union === null ? null : L.geoJSON(union).getBounds(); - const unionSet = { - $set: { - name: `subs-${publicl}-union`, - value: JSON.stringify(union), - isPublic, - description: `${Publicl} subscriptions union`, - type: 'string' - } - }; - const boundsSet = { - $set: { - name: `subs-${publicl}-union-bounds`, - value: JSON.stringify(bounds), - isPublic, - description: `${Publicl} subscriptions union bounds`, - type: 'string' - } - }; - const sizeSet = { - $set: { - name: 'subs-union-count', - value: count, - isPublic: false, - description: 'Subscriptions count', - type: 'number' - } - }; - // FIXME, take care of object size: - // https://stackoverflow.com/questions/10827812/what-is-the-length-maximum-for-a-string-data-type-in-mongodb-used-with-ruby - await SiteSettings.upsertAsync({ name: `subs-${publicl}-union` }, unionSet, { multi: false }); - await SiteSettings.upsertAsync({ name: `subs-${publicl}-union-bounds` }, boundsSet, { multi: false }); - await SiteSettings.upsertAsync({ name: 'subs-union-count' }, sizeSet, { multi: false }); - if (debug) console.log(`${Publicl} subscription union calculated`); - }; - - const process = async (isPublic) => { - const subscribers = await Subscriptions.find().fetchAsync(); - const decorate = isPublic ? addNoisy : noNoisy; - const decorated = subscribers.filter(isValidSub).map(decorate); - - try { - const union = await calcUnionAsync(decorated); - await storeUnion(isPublic, union, subscribers.length); - } catch (e) { - console.error('subsUnion worker failed, union left unchanged', e); - } - }; - - const recreate = async () => { await process(true); await process(false); }; - - // Single serialized worker over two kinds of work: incremental adds (a real - // FIFO queue — each must see the previous one's result, none get dropped) - // and full recomputes (coalesced — a burst of changed/removed events only - // needs one pass once things settle). Both touch the same stored union, so - // they share this one lock rather than racing each other. - let busy = false; - const addQueue = []; - let recomputePending = false; - - // Fast path for the common case (a new subscription): merge just the new - // circle into the union already stored instead of redoing the full turf.union - // chain over every subscription again (that chain is what made a single - // recompute take ~20+ minutes with thousands of subscriptions). Falls back to - // a full recreate() if the stored state isn't exactly "one behind" what we - // expect — safer than merging into stale/inconsistent data. Sets - // recomputePending directly (rather than calling a schedule helper) since - // runLoop already re-checks it right after this returns. - const incrementalAdd = async (doc) => { - if (!isValidSub(doc)) return; - - const countSetting = await SiteSettings.findOneAsync({ name: 'subs-union-count' }); - const countSubs = await Subscriptions.find({}).countAsync(); - if (!countSetting || countSetting.value !== countSubs - 1) { - if (debug) console.log('Subs union state not exactly one behind, doing a full recreate instead'); - recomputePending = true; - return; - } - - // eslint-disable-next-line no-restricted-syntax - for (const isPublic of [true, false]) { - const decorate = isPublic ? addNoisy : noNoisy; - const decorated = decorate({ ...doc, location: { ...doc.location } }); - const publicl = isPublic ? 'public' : 'private'; - // eslint-disable-next-line no-await-in-loop - const current = await SiteSettings.findOneAsync({ name: `subs-${publicl}-union` }); - const baseUnion = current && current.value ? JSON.parse(current.value) : null; - - try { - // eslint-disable-next-line no-await-in-loop - const union = await calcUnionAsync([decorated], baseUnion); - // eslint-disable-next-line no-await-in-loop - await storeUnion(isPublic, union, countSubs); - } catch (e) { - console.error('subsUnion incremental worker failed, falling back to a full recreate', e); - recomputePending = true; - return; - } - } - if (debug) console.log('Subs union updated incrementally'); - }; - - const runLoop = () => { - if (busy) return; - if (addQueue.length > 0) { - busy = true; - const doc = addQueue.shift(); - incrementalAdd(doc) - .catch(e => console.error('subsUnion incremental add failed', e)) - .finally(() => { busy = false; runLoop(); }); - return; - } - if (recomputePending) { - recomputePending = false; - busy = true; - recreate() - .catch(e => console.error('subsUnion recompute failed', e)) - .finally(() => { busy = false; runLoop(); }); - } - }; - - const enqueueAdd = (doc) => { addQueue.push(doc); runLoop(); }; - - // Fire-and-forget on purpose: both recreate() and incrementalAdd() can take a - // while, and callers (Meteor.startup, observeAsync callbacks) must not block on it. - const scheduleRecreate = () => { recomputePending = true; runLoop(); }; - - // At startup, we check if it's necessary to calc subscriptions union again - const currentUnion = await SiteSettings.findOneAsync({ name: 'subs-public-union' }); - const lastSubs = await Subscriptions.findOneAsync({}, { sort: { updatedAt: -1 } }); - const countUnionSubs = await SiteSettings.findOneAsync({ name: 'subs-union-count' }); - const countSubs = await Subscriptions.find({}).countAsync(); - if (currentUnion && lastSubs) { - const lastUnionUpdated = currentUnion.updatedAt; - const lastSubsUpdated = lastSubs.updatedAt; - if (lastUnionUpdated > lastSubsUpdated || !countUnionSubs || countSubs !== countUnionSubs.value) { - console.log('Subs union outdated'); - scheduleRecreate(); - } else { - console.log('Subs union up-to-date'); - } + if (await subsUnion.isUnionOutdated()) { + console.log('Subs union outdated'); + subsUnion.scheduleRecreate(); + } else { + console.log('Subs union up-to-date'); } await Subscriptions.find({ createdAt: { $gt: new Date() } }).observeAsync({ added: function newSubAdded(doc) { - if (debug) console.log('Subs added, merging into union incrementally'); - enqueueAdd(doc); + console.log('Subs added, merging into union incrementally'); + subsUnion.enqueueAdd(doc); } }); await Subscriptions.find().observeAsync({ - changed: function subsChanged() { // updatedDoc, oldDoc) { - if (debug) console.log('Subs changed so recreate union'); - scheduleRecreate(); + changed: function subsChanged() { + console.log('Subs changed so recreate union'); + subsUnion.scheduleRecreate(); }, - removed: function subsRemoved() { // oldDoc) { - if (debug) console.log('Subs removed so recreate union'); - scheduleRecreate(); + removed: function subsRemoved() { + console.log('Subs removed so recreate union'); + subsUnion.scheduleRecreate(); } }); }); diff --git a/imports/startup/server/subsUnionLogic.js b/imports/startup/server/subsUnionLogic.js new file mode 100644 index 0000000..1788737 --- /dev/null +++ b/imports/startup/server/subsUnionLogic.js @@ -0,0 +1,252 @@ +/* eslint-disable import/no-absolute-path */ + +// All of the subscriptions-union behaviour, with its collaborators injected. +// +// This used to live inside the Meteor.startup() callback of subsUnion.js, where +// nothing was reachable from a test: the queue, the incremental fast path and the +// worker-failure handling could only be exercised by starting a whole server and +// hoping. That is how a failed worker call ended up being stored as a valid empty +// union (50ca9cc) and how a full recompute ended up blocking DDP (b4e5511 / +// fbb746f) — three production incidents in code with zero test coverage. +// subsUnion.js is now only the wiring; everything below is a plain function you +// can call with fakes. + +import Perlin from 'loms.perlin'; +import limitUnionSize, { MAX_UNION_BYTES } from './unionSizeGuard'; + +// Telegram-era subscriptions (and anything else half-written) can be missing the +// location or the radius. They are skipped, not an error: they predate the web +// subscriptions and there is nothing sensible to draw for them. +export const isValidSub = (osub, log = console) => { + const valid = !!(osub && osub.location && osub.location.lat && osub.location.lon && osub.distance); + if (!valid) log.info(`Wrong element to do union ${JSON.stringify(osub)}`); + return valid; +}; + +/** + * @param {object} deps + * @param {function} deps.calcUnion (subs, baseUnion) => Promise, off-thread + * @param {object} deps.SiteSettings Mongo collection holding the stored union + * @param {object} deps.Subscriptions Mongo collection of subscriptions + * @param {function} [deps.boundsOf] union => leaflet-style bounds (or null) + * @param {number} [deps.maxUnionBytes] + * @param {number} [deps.perlinSeed] fixed seed makes the public "noise" reproducible in tests + */ +const createSubsUnion = ({ + calcUnion, + SiteSettings, + Subscriptions, + boundsOf = () => null, + maxUnionBytes = MAX_UNION_BYTES, + perlinSeed = Math.random(), + debug = true, + log = console +}) => { + Perlin.seed(perlinSeed); + + // Public unions are fuzzed so that the map does not disclose where exactly + // somebody subscribed. Returns a new object: callers hand us documents straight + // out of Mongo and the private pass must not see the fuzzed values. + const addNoisy = (sub) => { + const lat = Math.round(sub.location.lat * 10) / 10; + const lon = Math.round(sub.location.lon * 10) / 10; + const noiseBase = Perlin.perlin2(lat, lon); + const noise = Math.abs(noiseBase / 3); + return { + ...sub, + location: { ...sub.location, lat: lat + noise, lon: lon + noise }, + distance: sub.distance + noiseBase + }; + }; + + const noNoisy = sub => ({ ...sub, location: { ...sub.location } }); + + const decoratorFor = isPublic => (isPublic ? addNoisy : noNoisy); + + // Only called once calcUnion has actually resolved — union === null here means + // "zero valid subscriptions", a real result, not a failed computation. (A prior + // version guarded this with `typeof union === 'object'`, which is true for null + // too, so a failed worker call — caught upstream, leaving union as null — looked + // identical to a legitimate empty union and got silently stored as "null". + // Callers must catch calcUnion failures themselves and skip calling this.) + const storeUnion = async (isPublic, union, count) => { + const publicl = isPublic ? 'public' : 'private'; + const Publicl = publicl.replace(/\b\w/g, l => l.toUpperCase()); + // Mongo caps a document at 16 MiB and this value is also pushed to every + // browser: past a certain number of subscriptions the geometry has to be + // coarsened or the upsert simply starts failing. See unionSizeGuard.js. + const { union: safeUnion, json, degraded } = limitUnionSize(union, maxUnionBytes); + if (degraded) { + log.warn(`${Publicl} subscriptions union too big (${degraded.originalBytes} bytes), degraded via ${degraded.steps.join(' -> ')}`); + } + const bounds = boundsOf(safeUnion); + const unionSet = { + $set: { + name: `subs-${publicl}-union`, + value: json, + isPublic, + description: `${Publicl} subscriptions union`, + type: 'string', + degraded: degraded ? degraded.steps.join(' -> ') : null + } + }; + const boundsSet = { + $set: { + name: `subs-${publicl}-union-bounds`, + value: JSON.stringify(bounds), + isPublic, + description: `${Publicl} subscriptions union bounds`, + type: 'string' + } + }; + const sizeSet = { + $set: { + name: 'subs-union-count', + value: count, + isPublic: false, + description: 'Subscriptions count', + type: 'number' + } + }; + await SiteSettings.upsertAsync({ name: `subs-${publicl}-union` }, unionSet, { multi: false }); + await SiteSettings.upsertAsync({ name: `subs-${publicl}-union-bounds` }, boundsSet, { multi: false }); + await SiteSettings.upsertAsync({ name: 'subs-union-count' }, sizeSet, { multi: false }); + if (debug) log.log(`${Publicl} subscription union calculated`); + return { degraded }; + }; + + const recomputeOne = async (isPublic) => { + const subscribers = await Subscriptions.find().fetchAsync(); + const decorated = subscribers.filter(s => isValidSub(s, log)).map(decoratorFor(isPublic)); + + try { + const union = await calcUnion(decorated); + await storeUnion(isPublic, union, subscribers.length); + } catch (e) { + log.error('subsUnion worker failed, union left unchanged', e); + } + }; + + const recreate = async () => { await recomputeOne(true); await recomputeOne(false); }; + + // Single serialized worker over two kinds of work: incremental adds (a real + // FIFO queue — each must see the previous one's result, none get dropped) and + // full recomputes (coalesced — a burst of changed/removed events only needs one + // pass once things settle). Both touch the same stored union, so they share this + // one lock rather than racing each other. + let busy = false; + const addQueue = []; + let recomputePending = false; + let idleWaiters = []; + + const flushIdleWaiters = () => { + const waiters = idleWaiters; + idleWaiters = []; + waiters.forEach(resolve => resolve()); + }; + + // Fast path for the common case (a new subscription): merge just the new circle + // into the union already stored instead of redoing the full turf.union chain + // over every subscription again (that chain is what made a single recompute take + // ~20+ minutes with thousands of subscriptions). Falls back to a full recreate() + // if the stored state isn't exactly "one behind" what we expect — safer than + // merging into stale/inconsistent data. Sets recomputePending directly (rather + // than calling a schedule helper) since runLoop already re-checks it right after + // this returns. + const incrementalAdd = async (doc) => { + if (!isValidSub(doc, log)) return; + + const countSetting = await SiteSettings.findOneAsync({ name: 'subs-union-count' }); + const countSubs = await Subscriptions.find({}).countAsync(); + if (!countSetting || countSetting.value !== countSubs - 1) { + if (debug) log.log('Subs union state not exactly one behind, doing a full recreate instead'); + recomputePending = true; + return; + } + + // eslint-disable-next-line no-restricted-syntax + for (const isPublic of [true, false]) { + const decorated = decoratorFor(isPublic)(doc); + const publicl = isPublic ? 'public' : 'private'; + // eslint-disable-next-line no-await-in-loop + const current = await SiteSettings.findOneAsync({ name: `subs-${publicl}-union` }); + const baseUnion = current && current.value ? JSON.parse(current.value) : null; + + try { + // eslint-disable-next-line no-await-in-loop + const union = await calcUnion([decorated], baseUnion); + // eslint-disable-next-line no-await-in-loop + await storeUnion(isPublic, union, countSubs); + } catch (e) { + log.error('subsUnion incremental worker failed, falling back to a full recreate', e); + recomputePending = true; + return; + } + } + if (debug) log.log('Subs union updated incrementally'); + }; + + const runLoop = () => { + if (busy) return; + if (addQueue.length > 0) { + busy = true; + const doc = addQueue.shift(); + incrementalAdd(doc) + .catch(e => log.error('subsUnion incremental add failed', e)) + .finally(() => { busy = false; runLoop(); }); + return; + } + if (recomputePending) { + recomputePending = false; + busy = true; + recreate() + .catch(e => log.error('subsUnion recompute failed', e)) + .finally(() => { busy = false; runLoop(); }); + return; + } + flushIdleWaiters(); + }; + + const enqueueAdd = (doc) => { addQueue.push(doc); runLoop(); }; + + // Fire-and-forget on purpose: both recreate() and incrementalAdd() can take a + // while, and callers (Meteor.startup, observeAsync callbacks) must not block on it. + const scheduleRecreate = () => { recomputePending = true; runLoop(); }; + + // Resolves when the queue has drained and nothing is running. Only meant for + // tests and shutdown — production code must never await the union. + const idle = () => new Promise((resolve) => { + if (!busy && addQueue.length === 0 && !recomputePending) { + resolve(); + return; + } + idleWaiters.push(resolve); + }); + + // At startup we check whether the stored union still matches the subscriptions. + const isUnionOutdated = async () => { + const currentUnion = await SiteSettings.findOneAsync({ name: 'subs-public-union' }); + const lastSubs = await Subscriptions.findOneAsync({}, { sort: { updatedAt: -1 } }); + if (!currentUnion || !lastSubs) return false; + const countUnionSubs = await SiteSettings.findOneAsync({ name: 'subs-union-count' }); + const countSubs = await Subscriptions.find({}).countAsync(); + return currentUnion.updatedAt > lastSubs.updatedAt + || !countUnionSubs + || countSubs !== countUnionSubs.value; + }; + + return { + addNoisy, + noNoisy, + storeUnion, + recreate, + incrementalAdd, + enqueueAdd, + scheduleRecreate, + isUnionOutdated, + idle, + stats: () => ({ busy, queued: addQueue.length, recomputePending }) + }; +}; + +export default createSubsUnion; diff --git a/imports/startup/server/unionBounds.js b/imports/startup/server/unionBounds.js new file mode 100644 index 0000000..47e39cb --- /dev/null +++ b/imports/startup/server/unionBounds.js @@ -0,0 +1,11 @@ +/* eslint-disable import/no-absolute-path */ + +// Leaflet is a browser library; the workaround import fakes the bits of `window` +// it touches at load time so it can also run on the server. Kept apart from +// subsUnionLogic so that module (and its tests) stay free of that dependency. +import './leaflet-workaround'; +import L from 'leaflet'; + +const unionBounds = union => (union === null ? null : L.geoJSON(union).getBounds()); + +export default unionBounds; diff --git a/imports/startup/server/unionSizeGuard.js b/imports/startup/server/unionSizeGuard.js new file mode 100644 index 0000000..07cdb3c --- /dev/null +++ b/imports/startup/server/unionSizeGuard.js @@ -0,0 +1,212 @@ +// Keeps the stored subscriptions-union GeoJSON below a size that Mongo (and the +// DDP connection that ships it to every client) can actually take. +// +// The union of every subscription circle is stored as a JSON string inside a +// single `siteSettings` document. A BSON document is capped at 16 MiB, so past +// some number of subscriptions the upsert would simply start failing and the map +// would silently freeze at the last union that fit — which is exactly the kind of +// scale bug nobody notices until production. On top of the hard Mongo limit, that +// same string is published to browsers, so multi-megabyte unions are already a +// problem well before 16 MiB. +// +// The chosen fix is progressive degradation instead of failure: the geometry is +// coarsened, in visually-cheapest-first order, until it fits. The zone union is a +// decorative overlay ("roughly where people are watching"), never a source of +// truth for alerts, so losing precision is acceptable; losing the whole overlay is +// not. Every degradation is reported back so the caller can store it alongside the +// union and ops can see it happening. +// +// The ladder, in order: +// 1. coordinate precision 6 -> 5 -> 4 -> 3 decimals (~0.1 m -> ~100 m), dropping +// points that collapse onto their neighbour after rounding. +// 2. point decimation, keeping every k-th vertex of each ring (circles are +// generated with 144 steps, so there is a lot of slack here). +// 3. dropping holes (inner rings) — visually the least missed. +// 4. dropping the smallest polygons of the MultiPolygon until it fits. +// If even a single polygon does not fit, null is returned rather than storing +// something Mongo will reject. + +// 16 MiB is Mongo's hard limit for the whole document; this cap is deliberately +// well below it because the same value travels over DDP to every connected +// browser. Raise it only together with a plan for the client side. +export const MAX_UNION_BYTES = 8 * 1024 * 1024; + +const MIN_RING_POINTS = 12; +const DECIMATION_FACTORS = [2, 3, 4, 6, 8]; +const PRECISIONS = [5, 4, 3]; + +export const byteLength = json => Buffer.byteLength(json, 'utf8'); + +const geometryOf = union => (union && union.type === 'Feature' ? union.geometry : union); + +// Both Polygon and MultiPolygon, seen as a list of polygons (a polygon being a +// list of rings, the first one the outer one). +const polygonsOf = (geometry) => { + if (!geometry || !geometry.coordinates) return []; + return geometry.type === 'MultiPolygon' ? geometry.coordinates : [geometry.coordinates]; +}; + +const withPolygons = (union, polygons) => { + const geometry = geometryOf(union); + const coordinates = geometry.type === 'MultiPolygon' ? polygons : polygons[0]; + const newGeometry = { ...geometry, coordinates }; + return union.type === 'Feature' ? { ...union, geometry: newGeometry } : newGeometry; +}; + +// A ring is only a ring if it closes and has at least three distinct corners. +const closeRing = (ring) => { + if (ring.length < 3) return null; + const [firstLon, firstLat] = ring[0]; + const [lastLon, lastLat] = ring[ring.length - 1]; + const closed = firstLon === lastLon && firstLat === lastLat ? ring : [...ring, [firstLon, firstLat]]; + return closed.length >= 4 ? closed : null; +}; + +const roundRing = (ring, factor) => { + const out = []; + for (let i = 0; i < ring.length; i += 1) { + const lon = Math.round(ring[i][0] * factor) / factor; + const lat = Math.round(ring[i][1] * factor) / factor; + const prev = out[out.length - 1]; + if (!prev || prev[0] !== lon || prev[1] !== lat) out.push([lon, lat]); + } + return closeRing(out); +}; + +const decimateRing = (ring, keepEvery) => { + if (ring.length <= MIN_RING_POINTS) return ring; + const out = []; + for (let i = 0; i < ring.length - 1; i += keepEvery) out.push(ring[i]); + return closeRing(out) || ring; +}; + +// Rings that survive a transformation keep the polygon alive; a polygon whose +// outer ring collapsed is dropped entirely (its holes are meaningless then). +const mapPolygons = (polygons, mapRing) => polygons + .map(rings => rings.map(mapRing).filter(Boolean)) + .filter(rings => rings.length > 0); + +const ringBboxArea = (ring) => { + let minLon = Infinity; + let minLat = Infinity; + let maxLon = -Infinity; + let maxLat = -Infinity; + for (let i = 0; i < ring.length; i += 1) { + if (ring[i][0] < minLon) [minLon] = ring[i]; + if (ring[i][0] > maxLon) [maxLon] = ring[i]; + if (ring[i][1] < minLat) [, minLat] = ring[i]; + if (ring[i][1] > maxLat) [, maxLat] = ring[i]; + } + return (maxLon - minLon) * (maxLat - minLat); +}; + +/** + * Returns the largest-fidelity version of `union` whose JSON fits in `maxBytes`, + * together with that JSON (callers always need the string anyway, and + * re-stringifying a multi-megabyte object on the main thread is not free). + * + * @returns {{ union: object|null, json: string, bytes: number, + * degraded: null | { originalBytes: number, steps: string[] } }} + */ +const limitUnionSize = (union, maxBytes = MAX_UNION_BYTES) => { + let json = JSON.stringify(union === undefined ? null : union); + let bytes = byteLength(json); + if (union === null || union === undefined || bytes <= maxBytes) { + return { + union: union === undefined ? null : union, json, bytes, degraded: null + }; + } + + const originalBytes = bytes; + const steps = []; + let current = union; + + const attempt = (step, transform) => { + const next = transform(); + if (!next) return false; + current = next; + steps.push(step); + json = JSON.stringify(current); + bytes = byteLength(json); + return bytes <= maxBytes; + }; + + const done = () => ({ + union: current, json, bytes, degraded: { originalBytes, steps } + }); + + for (let i = 0; i < PRECISIONS.length; i += 1) { + const precision = PRECISIONS[i]; + const factor = 10 ** precision; + const fits = attempt(`precision:${precision}`, () => { + const polygons = mapPolygons(polygonsOf(geometryOf(current)), ring => roundRing(ring, factor)); + return polygons.length > 0 ? withPolygons(current, polygons) : null; + }); + if (fits) return done(); + } + + for (let i = 0; i < DECIMATION_FACTORS.length; i += 1) { + const keepEvery = DECIMATION_FACTORS[i]; + const fits = attempt(`decimate:${keepEvery}`, () => { + const polygons = mapPolygons(polygonsOf(geometryOf(union)), ring => decimateRing( + roundRing(ring, 10 ** PRECISIONS[PRECISIONS.length - 1]) || ring, + keepEvery + )); + return polygons.length > 0 ? withPolygons(current, polygons) : null; + }); + if (fits) return done(); + } + + const fitsWithoutHoles = attempt('holes', () => { + const polygons = polygonsOf(geometryOf(current)).map(rings => [rings[0]]); + return polygons.length > 0 ? withPolygons(current, polygons) : null; + }); + if (fitsWithoutHoles) return done(); + + // Last resort: drop the smallest polygons first, so what remains is the part of + // the map a user is most likely to be looking at. Binary search for how many + // survive — trying one fewer at a time means re-serializing a multi-megabyte + // object hundreds of times, which is precisely the kind of main-thread stall + // this whole area is trying to get rid of. + const ordered = polygonsOf(geometryOf(current)) + .map(rings => ({ rings, area: ringBboxArea(rings[0]) })) + .sort((a, b) => b.area - a.area) + .map(p => p.rings); + + const fitsWith = (keep) => { + const candidate = withPolygons(current, ordered.slice(0, keep)); + const candidateJson = JSON.stringify(candidate); + return byteLength(candidateJson) <= maxBytes ? { candidate, candidateJson } : null; + }; + + let low = 1; + let high = ordered.length - 1; + let best = null; + while (low <= high) { + const mid = Math.floor((low + high) / 2); + const fit = fitsWith(mid); + if (fit) { + best = { keep: mid, ...fit }; + low = mid + 1; + } else { + high = mid - 1; + } + } + if (best) { + current = best.candidate; + json = best.candidateJson; + bytes = byteLength(json); + steps.push(`polygons:${best.keep}`); + return done(); + } + + // A single polygon that still does not fit means something is very wrong + // upstream; storing nothing beats an upsert that Mongo rejects on every run. + steps.push('dropped'); + current = null; + json = 'null'; + bytes = byteLength(json); + return done(); +}; + +export default limitUnionSize;