perf(subsUnion): unión en árbol, menos vértices, timeout y cota de memoria
Cuatro endurecimientos salidos del banco de carga (bench/union-bench.js), cada uno con su número: 1. Unión en ÁRBOL en vez de en cadena. La cadena unía siempre contra un polígono acumulado que no paraba de crecer. Con 1.000 suscripciones repartidas por el mundo —el peor caso, círculos que no se solapan— eran 238 s; en árbol son 3,6 s, con un documento idéntico. Con datos realistas (España) 10.000 suscripciones bajan de 39,3 s a 23,5 s. test/server/calcUnionAsync.test.js comprueba la equivalencia contra una referencia calculada en cadena. 2. 64 vértices por círculo en vez de 144, configurable con `private.unionCircleSteps`. Con 10.000 suscripciones: 23,5 s → 10,0 s y el pico de RSS de 777 MB → 421 MB. En pantalla un círculo de 64 lados sigue siendo un círculo, y la capa de zonas es un adorno difuso; el host de despliegue tiene 5,9 GB para todo el stack. 3. Timeout del worker (10 min por defecto, `private.unionWorkerTimeoutMs`). Un worker que no contestaba dejaba la promesa colgada y con ella la cola entera: `busy` se quedaba en true y no se volvía a calcular ninguna unión hasta reiniciar, sin que nada fallase. Se reintenta una vez ante un error real, nunca ante un timeout. 4. Cota de heap del worker (1 GB, `private.unionWorkerMaxHeapMb`), para que un caso patológico muera con ERR_WORKER_OUT_OF_MEMORY —que esta capa convierte en un rechazo, dejando la unión anterior intacta— en vez de que el kernel se lleve por delante el proceso de Meteor. Además unionTelemetry.js manda a GlitchTip las uniones que pasan de un minuto y las que hay que degradar por tamaño, y registra duración/tamaño de cada una. El único aviso de que la unión iba mal era, hasta ahora, que el mapa se quedaba viejo. Smoke REST byte-idéntico. 90 tests en verde.
This commit is contained in:
parent
072e76aaed
commit
56f53e2c23
7 changed files with 479 additions and 39 deletions
|
|
@ -6,7 +6,11 @@ import fs from 'fs';
|
|||
import path from 'path';
|
||||
import { Worker } from 'worker_threads';
|
||||
import { expect } from 'chai';
|
||||
import calcUnionAsync from '/imports/startup/server/calcUnionAsync';
|
||||
import { Meteor } from 'meteor/meteor';
|
||||
import tcircle from '@turf/circle';
|
||||
import tunion from '@turf/union';
|
||||
import ttruncate from '@turf/truncate';
|
||||
import calcUnionAsync, { timeoutMs } from '/imports/startup/server/calcUnionAsync';
|
||||
|
||||
// These run the real worker thread against the real @turf modules. The point is
|
||||
// not the geometry (turf is not ours to test) but the plumbing that broke twice
|
||||
|
|
@ -99,6 +103,125 @@ describe('calcUnionAsync worker plumbing', function () {
|
|||
expect(rejected).to.equal(true);
|
||||
});
|
||||
|
||||
// The worker unites in a tree instead of in a chain (fase 12: 238 s -> 3,6 s on
|
||||
// the pathological case). The union is associative and commutative, so the
|
||||
// result must match what the old chain produced — computed here with the same
|
||||
// turf the worker uses, in the original order.
|
||||
describe('tree merge equivalence', function () {
|
||||
// La referencia se calcula aquí con 144 vértices, así que el worker tiene que
|
||||
// usar los mismos: con distinto número de vértices los polígonos son
|
||||
// distintos por definición, y no habría nada que comparar.
|
||||
let originalSteps;
|
||||
beforeEach(function () {
|
||||
originalSteps = Meteor.settings.private.unionCircleSteps;
|
||||
Meteor.settings.private.unionCircleSteps = 144;
|
||||
});
|
||||
afterEach(function () {
|
||||
Meteor.settings.private.unionCircleSteps = originalSteps;
|
||||
});
|
||||
|
||||
const bboxOf = (union) => {
|
||||
const g = union.geometry || union;
|
||||
const polys = g.type === 'MultiPolygon' ? g.coordinates : [g.coordinates];
|
||||
let minLon = Infinity; let minLat = Infinity; let maxLon = -Infinity; let maxLat = -Infinity;
|
||||
polys.forEach(poly => poly.forEach(ring => ring.forEach(([lon, lat]) => {
|
||||
if (lon < minLon) minLon = lon;
|
||||
if (lon > maxLon) maxLon = lon;
|
||||
if (lat < minLat) minLat = lat;
|
||||
if (lat > maxLat) maxLat = lat;
|
||||
})));
|
||||
return [minLon, minLat, maxLon, maxLat];
|
||||
};
|
||||
|
||||
const sequentialReference = (subs) => {
|
||||
const circles = subs.map(s => tcircle(
|
||||
[s.location.lon, s.location.lat], s.distance, { units: 'kilometers', steps: 144 }
|
||||
));
|
||||
let acc = ttruncate(circles[0], { precision: 6, coordinates: 2 });
|
||||
for (let i = 1; i < circles.length; i += 1) {
|
||||
acc = ttruncate(tunion(acc, circles[i]), { precision: 6, coordinates: 2 });
|
||||
}
|
||||
return acc;
|
||||
};
|
||||
|
||||
it('gives the same geometry as the old chain for overlapping circles', async function () {
|
||||
const subs = [];
|
||||
for (let i = 0; i < 12; i += 1) subs.push(subAt(40 + (i * 0.15), -4 + (i * 0.1), 25));
|
||||
|
||||
const fromWorker = await calcUnionAsync(subs);
|
||||
const reference = sequentialReference(subs);
|
||||
|
||||
expect(fromWorker.geometry.type).to.equal(reference.geometry.type);
|
||||
bboxOf(fromWorker).forEach((v, i) => {
|
||||
expect(v).to.be.closeTo(bboxOf(reference)[i], 1e-6);
|
||||
});
|
||||
});
|
||||
|
||||
it('gives the same geometry for circles that do not touch each other', async function () {
|
||||
const subs = [subAt(43.5, -5.9, 10), subAt(40.4, -3.7, 10), subAt(28.1, -15.4, 10), subAt(39.6, 2.9, 10)];
|
||||
|
||||
const fromWorker = await calcUnionAsync(subs);
|
||||
const reference = sequentialReference(subs);
|
||||
|
||||
expect(fromWorker.geometry.type).to.equal('MultiPolygon');
|
||||
expect(fromWorker.geometry.coordinates).to.have.lengthOf(reference.geometry.coordinates.length);
|
||||
bboxOf(fromWorker).forEach((v, i) => {
|
||||
expect(v).to.be.closeTo(bboxOf(reference)[i], 1e-6);
|
||||
});
|
||||
});
|
||||
|
||||
it('merges into a base union with the same result either way', async function () {
|
||||
const base = await calcUnionAsync([subAt(43.5, -5.9, 20)]);
|
||||
const merged = await calcUnionAsync([subAt(43.6, -5.9, 20), subAt(43.7, -5.9, 20)], base);
|
||||
const reference = sequentialReference([subAt(43.5, -5.9, 20), subAt(43.6, -5.9, 20), subAt(43.7, -5.9, 20)]);
|
||||
|
||||
bboxOf(merged).forEach((v, i) => {
|
||||
expect(v).to.be.closeTo(bboxOf(reference)[i], 1e-4);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
// A worker that never answers used to hang the promise forever, taking the
|
||||
// whole queue with it (busy stays true and no union is ever computed again).
|
||||
describe('timeout', function () {
|
||||
let original;
|
||||
beforeEach(function () {
|
||||
original = Meteor.settings.private.unionWorkerTimeoutMs;
|
||||
});
|
||||
afterEach(function () {
|
||||
Meteor.settings.private.unionWorkerTimeoutMs = original;
|
||||
});
|
||||
|
||||
it('rejects instead of hanging when the worker takes too long', async function () {
|
||||
// 1 ms: not even spawning the thread fits in that.
|
||||
Meteor.settings.private.unionWorkerTimeoutMs = 1;
|
||||
|
||||
let error = null;
|
||||
try {
|
||||
await calcUnionAsync([subAt(43.5, -5.9)]);
|
||||
} catch (e) {
|
||||
error = e;
|
||||
}
|
||||
expect(error, 'calcUnionAsync resolved instead of timing out').to.not.equal(null);
|
||||
expect(error.isTimeout).to.equal(true);
|
||||
expect(error.message).to.match(/did not answer/);
|
||||
});
|
||||
|
||||
it('uses the configured bound, and a generous default when there is none', async function () {
|
||||
delete Meteor.settings.private.unionWorkerTimeoutMs;
|
||||
expect(timeoutMs()).to.equal(10 * 60 * 1000);
|
||||
|
||||
Meteor.settings.private.unionWorkerTimeoutMs = 1234;
|
||||
expect(timeoutMs()).to.equal(1234);
|
||||
});
|
||||
|
||||
it('still resolves normally when the bound is generous', async function () {
|
||||
Meteor.settings.private.unionWorkerTimeoutMs = 60000;
|
||||
const union = await calcUnionAsync([subAt(43.5, -5.9)]);
|
||||
expect(union.geometry.type).to.equal('Polygon');
|
||||
});
|
||||
});
|
||||
|
||||
// The whole reason the worker exists (b4e5511 / fbb746f): a union big enough to
|
||||
// take real CPU time must not stop the event loop from running timers, because
|
||||
// that is what froze DDP and disconnected every browser.
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue