feat(species): expand catalog to ~1200 edible species from Wikidata
Grow the bundled species catalog from 14 hand-curated entries to ~1200 edible/cultivated species, internationalized in 13 languages (es, en, fr, de, it, pt, ca, gl, eu, ar, zh, ja, ru — Latin + Arabic RTL + CJK + Cyrillic). - Add a reproducible generator (tool/gen_species_catalog.dart) that queries Wikidata (CC0, no attribution burden) in two phases, filters out non-vernacular noise (author citations, ranks, initials) and applies a relevance floor, then merges hand-curated, authoritative core-crop data (tool/curated_overrides.json: names, family, viability_years). GBIF is used only as an identifier. The generated species.json (v3) is committed. - Carry wikidata_qid and gbif_key through the parse/seed pipeline; the columns already existed, so no DB migration. - Rewrite seedBundled to one read + one batch (was a SELECT per species on every startup — a real cost at ~1200 rows) and keep it idempotent with backfill of the new reference fields. - Move species search filtering to SQL (LIKE) so a large catalog is not pulled into memory on every keystroke. - Cover the generator transform, the generated asset, and the new fields with tests.
This commit is contained in:
parent
ba87bf2719
commit
959fa2551c
9 changed files with 68483 additions and 120 deletions
159
apps/app_seeds/test/tool/gen_species_catalog_test.dart
Normal file
159
apps/app_seeds/test/tool/gen_species_catalog_test.dart
Normal file
|
|
@ -0,0 +1,159 @@
|
|||
import 'package:flutter_test/flutter_test.dart';
|
||||
|
||||
import '../../tool/gen_species_catalog.dart';
|
||||
|
||||
void main() {
|
||||
group('isVernacularName', () {
|
||||
test('accepts genuine common names', () {
|
||||
expect(isVernacularName('Tomate', 'solanum lycopersicum'), isTrue);
|
||||
expect(isVernacularName("Queen Anne's lace", 'daucus carota'), isTrue);
|
||||
expect(isVernacularName('番茄', 'solanum lycopersicum'), isTrue);
|
||||
});
|
||||
|
||||
test('rejects the scientific name itself', () {
|
||||
expect(
|
||||
isVernacularName('Solanum lycopersicum', 'solanum lycopersicum'),
|
||||
isFalse,
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects scientific noise: commas, parentheses, ranks, initials', () {
|
||||
expect(isVernacularName('Remusatia formosana , Hayata', 'x'), isFalse);
|
||||
expect(isVernacularName('滇常山(原变种)', 'x'), isFalse);
|
||||
expect(isVernacularName('Brassica oleracea var. acephala', 'x'), isFalse);
|
||||
expect(isVernacularName('Z. mays', 'x'), isFalse);
|
||||
expect(isVernacularName('Allium cepa L.', 'x'), isFalse);
|
||||
expect(isVernacularName('Prunus × domestica', 'x'), isFalse);
|
||||
});
|
||||
});
|
||||
|
||||
group('buildCatalog', () {
|
||||
test('emits version 3 and sorts by scientific name', () {
|
||||
final catalog = buildCatalog([
|
||||
SpeciesRecord(
|
||||
qid: 'Q2',
|
||||
scientificName: 'Zea mays',
|
||||
common: {
|
||||
'en': {'Maize'},
|
||||
},
|
||||
),
|
||||
SpeciesRecord(
|
||||
qid: 'Q1',
|
||||
scientificName: 'Allium cepa',
|
||||
common: {
|
||||
'en': {'Onion'},
|
||||
},
|
||||
),
|
||||
], const {});
|
||||
|
||||
expect(catalog['version'], 3);
|
||||
expect((catalog['species'] as List).map((e) => e['scientific_name']), [
|
||||
'Allium cepa',
|
||||
'Zea mays',
|
||||
]);
|
||||
});
|
||||
|
||||
test('dedups by scientific name, merging names and backfilling ids', () {
|
||||
final catalog = buildCatalog([
|
||||
SpeciesRecord(
|
||||
qid: 'Q13223',
|
||||
scientificName: 'Solanum lycopersicum',
|
||||
family: 'Solanaceae',
|
||||
common: {
|
||||
'es': {'tomate'},
|
||||
},
|
||||
),
|
||||
SpeciesRecord(
|
||||
qid: 'Q-dupe',
|
||||
scientificName: 'Solanum lycopersicum',
|
||||
gbifKey: 2930137,
|
||||
common: {
|
||||
'en': {'tomato'},
|
||||
'es': {'tomatera'},
|
||||
},
|
||||
),
|
||||
], const {});
|
||||
|
||||
final entry = (catalog['species'] as List).single as Map;
|
||||
expect(entry['wikidata_qid'], 'Q13223'); // first entity kept
|
||||
expect(entry['family'], 'Solanaceae');
|
||||
expect(entry['gbif_key'], 2930137); // backfilled from the duplicate
|
||||
expect(entry['common']['es'], containsAll(['tomate', 'tomatera']));
|
||||
expect(entry['common']['en'], ['tomato']);
|
||||
});
|
||||
|
||||
test('drops names that are not real vernaculars', () {
|
||||
final catalog = buildCatalog([
|
||||
SpeciesRecord(
|
||||
qid: 'Q1',
|
||||
scientificName: 'Solanum lycopersicum',
|
||||
common: {
|
||||
'es': {'Solanum lycopersicum', 'Tomate'},
|
||||
},
|
||||
),
|
||||
], const {});
|
||||
expect((catalog['species'] as List).single['common']['es'], ['Tomate']);
|
||||
});
|
||||
|
||||
test('minLanguages is a relevance floor for Wikidata-only species', () {
|
||||
final records = [
|
||||
SpeciesRecord(
|
||||
qid: 'Q1',
|
||||
scientificName: 'Obscura ignota',
|
||||
common: {
|
||||
'zh': {'仅中文'},
|
||||
},
|
||||
),
|
||||
SpeciesRecord(
|
||||
qid: 'Q2',
|
||||
scientificName: 'Nota bene',
|
||||
common: {
|
||||
'en': {'well known'},
|
||||
'es': {'bien conocida'},
|
||||
},
|
||||
),
|
||||
];
|
||||
final kept =
|
||||
(buildCatalog(records, const {}, minLanguages: 2)['species'] as List)
|
||||
.map((e) => e['scientific_name'])
|
||||
.toList();
|
||||
expect(kept, ['Nota bene']); // the 1-language species is filtered out
|
||||
});
|
||||
|
||||
test('curated overrides win, come first, and bypass the floor', () {
|
||||
final catalog = buildCatalog(
|
||||
[
|
||||
SpeciesRecord(
|
||||
qid: 'Q081',
|
||||
scientificName: 'Daucus carota',
|
||||
family: 'Apiaceae',
|
||||
common: {
|
||||
// Wikidata's messy taxon labels for the wild carrot.
|
||||
'en': {'Devil’s Plague', 'wild carrot'},
|
||||
},
|
||||
),
|
||||
],
|
||||
const {
|
||||
'Daucus carota': CuratedOverride(
|
||||
family: 'Apiaceae',
|
||||
viabilityYears: 3,
|
||||
common: {
|
||||
'es': ['Zanahoria'],
|
||||
'en': ['Carrot'],
|
||||
},
|
||||
),
|
||||
},
|
||||
// Would be filtered as it has only one Wikidata language, but curated
|
||||
// species are always kept.
|
||||
minLanguages: 4,
|
||||
);
|
||||
|
||||
final entry = (catalog['species'] as List).single as Map;
|
||||
expect(entry['viability_years'], 3);
|
||||
expect(entry['common']['es'], ['Zanahoria']);
|
||||
// Curated name first, then the Wikidata vernaculars.
|
||||
expect(entry['common']['en'].first, 'Carrot');
|
||||
expect(entry['common']['en'], contains('wild carrot'));
|
||||
});
|
||||
});
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue