feat(species): expand catalog to ~1200 edible species from Wikidata

Grow the bundled species catalog from 14 hand-curated entries to ~1200
edible/cultivated species, internationalized in 13 languages (es, en, fr,
de, it, pt, ca, gl, eu, ar, zh, ja, ru — Latin + Arabic RTL + CJK + Cyrillic).

- Add a reproducible generator (tool/gen_species_catalog.dart) that queries
  Wikidata (CC0, no attribution burden) in two phases, filters out
  non-vernacular noise (author citations, ranks, initials) and applies a
  relevance floor, then merges hand-curated, authoritative core-crop data
  (tool/curated_overrides.json: names, family, viability_years). GBIF is used
  only as an identifier. The generated species.json (v3) is committed.
- Carry wikidata_qid and gbif_key through the parse/seed pipeline; the columns
  already existed, so no DB migration.
- Rewrite seedBundled to one read + one batch (was a SELECT per species on
  every startup — a real cost at ~1200 rows) and keep it idempotent with
  backfill of the new reference fields.
- Move species search filtering to SQL (LIKE) so a large catalog is not pulled
  into memory on every keystroke.
- Cover the generator transform, the generated asset, and the new fields with
  tests.
This commit is contained in:
vjrj 2026-07-10 02:00:47 +02:00
parent ba87bf2719
commit 959fa2551c
9 changed files with 68483 additions and 120 deletions

View file

@ -0,0 +1,159 @@
import 'package:flutter_test/flutter_test.dart';
import '../../tool/gen_species_catalog.dart';
void main() {
group('isVernacularName', () {
test('accepts genuine common names', () {
expect(isVernacularName('Tomate', 'solanum lycopersicum'), isTrue);
expect(isVernacularName("Queen Anne's lace", 'daucus carota'), isTrue);
expect(isVernacularName('番茄', 'solanum lycopersicum'), isTrue);
});
test('rejects the scientific name itself', () {
expect(
isVernacularName('Solanum lycopersicum', 'solanum lycopersicum'),
isFalse,
);
});
test('rejects scientific noise: commas, parentheses, ranks, initials', () {
expect(isVernacularName('Remusatia formosana , Hayata', 'x'), isFalse);
expect(isVernacularName('滇常山(原变种)', 'x'), isFalse);
expect(isVernacularName('Brassica oleracea var. acephala', 'x'), isFalse);
expect(isVernacularName('Z. mays', 'x'), isFalse);
expect(isVernacularName('Allium cepa L.', 'x'), isFalse);
expect(isVernacularName('Prunus × domestica', 'x'), isFalse);
});
});
group('buildCatalog', () {
test('emits version 3 and sorts by scientific name', () {
final catalog = buildCatalog([
SpeciesRecord(
qid: 'Q2',
scientificName: 'Zea mays',
common: {
'en': {'Maize'},
},
),
SpeciesRecord(
qid: 'Q1',
scientificName: 'Allium cepa',
common: {
'en': {'Onion'},
},
),
], const {});
expect(catalog['version'], 3);
expect((catalog['species'] as List).map((e) => e['scientific_name']), [
'Allium cepa',
'Zea mays',
]);
});
test('dedups by scientific name, merging names and backfilling ids', () {
final catalog = buildCatalog([
SpeciesRecord(
qid: 'Q13223',
scientificName: 'Solanum lycopersicum',
family: 'Solanaceae',
common: {
'es': {'tomate'},
},
),
SpeciesRecord(
qid: 'Q-dupe',
scientificName: 'Solanum lycopersicum',
gbifKey: 2930137,
common: {
'en': {'tomato'},
'es': {'tomatera'},
},
),
], const {});
final entry = (catalog['species'] as List).single as Map;
expect(entry['wikidata_qid'], 'Q13223'); // first entity kept
expect(entry['family'], 'Solanaceae');
expect(entry['gbif_key'], 2930137); // backfilled from the duplicate
expect(entry['common']['es'], containsAll(['tomate', 'tomatera']));
expect(entry['common']['en'], ['tomato']);
});
test('drops names that are not real vernaculars', () {
final catalog = buildCatalog([
SpeciesRecord(
qid: 'Q1',
scientificName: 'Solanum lycopersicum',
common: {
'es': {'Solanum lycopersicum', 'Tomate'},
},
),
], const {});
expect((catalog['species'] as List).single['common']['es'], ['Tomate']);
});
test('minLanguages is a relevance floor for Wikidata-only species', () {
final records = [
SpeciesRecord(
qid: 'Q1',
scientificName: 'Obscura ignota',
common: {
'zh': {'仅中文'},
},
),
SpeciesRecord(
qid: 'Q2',
scientificName: 'Nota bene',
common: {
'en': {'well known'},
'es': {'bien conocida'},
},
),
];
final kept =
(buildCatalog(records, const {}, minLanguages: 2)['species'] as List)
.map((e) => e['scientific_name'])
.toList();
expect(kept, ['Nota bene']); // the 1-language species is filtered out
});
test('curated overrides win, come first, and bypass the floor', () {
final catalog = buildCatalog(
[
SpeciesRecord(
qid: 'Q081',
scientificName: 'Daucus carota',
family: 'Apiaceae',
common: {
// Wikidata's messy taxon labels for the wild carrot.
'en': {'Devils Plague', 'wild carrot'},
},
),
],
const {
'Daucus carota': CuratedOverride(
family: 'Apiaceae',
viabilityYears: 3,
common: {
'es': ['Zanahoria'],
'en': ['Carrot'],
},
),
},
// Would be filtered as it has only one Wikidata language, but curated
// species are always kept.
minLanguages: 4,
);
final entry = (catalog['species'] as List).single as Map;
expect(entry['viability_years'], 3);
expect(entry['common']['es'], ['Zanahoria']);
// Curated name first, then the Wikidata vernaculars.
expect(entry['common']['en'].first, 'Carrot');
expect(entry['common']['en'], contains('wild carrot'));
});
});
}