feat(species): expand catalog to ~1200 edible species from Wikidata
Grow the bundled species catalog from 14 hand-curated entries to ~1200 edible/cultivated species, internationalized in 13 languages (es, en, fr, de, it, pt, ca, gl, eu, ar, zh, ja, ru — Latin + Arabic RTL + CJK + Cyrillic). - Add a reproducible generator (tool/gen_species_catalog.dart) that queries Wikidata (CC0, no attribution burden) in two phases, filters out non-vernacular noise (author citations, ranks, initials) and applies a relevance floor, then merges hand-curated, authoritative core-crop data (tool/curated_overrides.json: names, family, viability_years). GBIF is used only as an identifier. The generated species.json (v3) is committed. - Carry wikidata_qid and gbif_key through the parse/seed pipeline; the columns already existed, so no DB migration. - Rewrite seedBundled to one read + one batch (was a SELECT per species on every startup — a real cost at ~1200 rows) and keep it idempotent with backfill of the new reference fields. - Move species search filtering to SQL (LIKE) so a large catalog is not pulled into memory on every keystroke. - Cover the generator transform, the generated asset, and the new fields with tests.
This commit is contained in:
parent
ba87bf2719
commit
959fa2551c
9 changed files with 68483 additions and 120 deletions
84
apps/app_seeds/test/data/species_catalog_asset_test.dart
Normal file
84
apps/app_seeds/test/data/species_catalog_asset_test.dart
Normal file
|
|
@ -0,0 +1,84 @@
|
|||
import 'dart:convert';
|
||||
import 'dart:io';
|
||||
|
||||
import 'package:flutter_test/flutter_test.dart';
|
||||
import 'package:tane/data/species_catalog.dart';
|
||||
|
||||
/// Guards the committed, generated catalog asset (produced by
|
||||
/// `tool/gen_species_catalog.dart`). Reads the file directly — no Flutter asset
|
||||
/// bundle needed — so it runs as a plain VM test.
|
||||
void main() {
|
||||
// `flutter test` runs with the package root as the working directory.
|
||||
final file = File('assets/catalog/species.json');
|
||||
final json = jsonDecode(file.readAsStringSync()) as Map<String, dynamic>;
|
||||
final species = (json['species'] as List).cast<Map<String, dynamic>>();
|
||||
|
||||
test('is the current (version 3) generated catalog', () {
|
||||
expect(json['version'], 3);
|
||||
});
|
||||
|
||||
test('is a broad catalog, not the old starter set', () {
|
||||
// The expanded catalog holds ~1200 species (was 14 by hand). The floor
|
||||
// leaves headroom for Wikidata drift across regenerations.
|
||||
expect(species.length, greaterThan(900));
|
||||
});
|
||||
|
||||
test('every entry has a scientific name and at least one localized name', () {
|
||||
for (final e in species) {
|
||||
final sci = e['scientific_name'];
|
||||
expect(sci, isA<String>());
|
||||
expect((sci as String).trim(), isNotEmpty);
|
||||
|
||||
final common = e['common'] as Map<String, dynamic>;
|
||||
expect(common, isNotEmpty, reason: '$sci has no common names');
|
||||
for (final entry in common.entries) {
|
||||
final names = (entry.value as List).cast<String>();
|
||||
expect(names, isNotEmpty);
|
||||
expect(names.every((n) => n.trim().isNotEmpty), isTrue);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
test('identifiers and reference fields are well typed', () {
|
||||
for (final e in species) {
|
||||
expect(e['wikidata_qid'], matches(r'^Q\d+$'));
|
||||
if (e.containsKey('gbif_key')) expect(e['gbif_key'], isA<int>());
|
||||
if (e.containsKey('viability_years')) {
|
||||
expect(e['viability_years'], isA<int>());
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
test('scientific names are unique', () {
|
||||
final names = species.map((e) => e['scientific_name'] as String).toList();
|
||||
expect(names.length, names.toSet().length);
|
||||
});
|
||||
|
||||
test('is genuinely multilingual (non-Latin scripts present)', () {
|
||||
int withLang(String code) =>
|
||||
species.where((e) => (e['common'] as Map).containsKey(code)).length;
|
||||
// International by design: Arabic (RTL) and CJK must be represented, not
|
||||
// an es/en-only catalog.
|
||||
expect(withLang('ar'), greaterThan(0), reason: 'no Arabic names');
|
||||
expect(withLang('zh'), greaterThan(0), reason: 'no Chinese names');
|
||||
expect(withLang('ja'), greaterThan(0), reason: 'no Japanese names');
|
||||
});
|
||||
|
||||
test('keeps the curated core crops with their authoritative names', () {
|
||||
final byName = {for (final e in species) e['scientific_name'] as String: e};
|
||||
final tomato = byName['Solanum lycopersicum'];
|
||||
final carrot = byName['Daucus carota'];
|
||||
expect(tomato, isNotNull);
|
||||
expect(carrot, isNotNull);
|
||||
// Curated names lead (not Wikidata's messier taxon labels) and viability
|
||||
// reference data survives the regeneration.
|
||||
expect((tomato!['common'] as Map)['es'], contains('Tomate'));
|
||||
expect((carrot!['common'] as Map)['es'], contains('Zanahoria'));
|
||||
expect(carrot['viability_years'], 3);
|
||||
});
|
||||
|
||||
test('parses through the app loader without error', () {
|
||||
final seeds = parseSpeciesCatalog(file.readAsStringSync());
|
||||
expect(seeds.length, species.length);
|
||||
});
|
||||
}
|
||||
|
|
@ -39,12 +39,14 @@ void main() {
|
|||
'parseSpeciesCatalog reads scientific name, family and common names',
|
||||
() {
|
||||
const json = '''
|
||||
{"version":1,"species":[
|
||||
{"scientific_name":"Zea mays","family":"Poaceae","common":{"en":["Maize"],"es":["Maíz"]}}
|
||||
{"version":3,"species":[
|
||||
{"scientific_name":"Zea mays","family":"Poaceae","wikidata_qid":"Q11575","gbif_key":2703077,"common":{"en":["Maize"],"es":["Maíz"]}}
|
||||
]}''';
|
||||
final seeds = parseSpeciesCatalog(json);
|
||||
expect(seeds.single.scientificName, 'Zea mays');
|
||||
expect(seeds.single.family, 'Poaceae');
|
||||
expect(seeds.single.wikidataQid, 'Q11575');
|
||||
expect(seeds.single.gbifKey, 2703077);
|
||||
expect(seeds.single.commonNames['es'], ['Maíz']);
|
||||
},
|
||||
);
|
||||
|
|
@ -80,4 +82,86 @@ void main() {
|
|||
await repo.seedBundled(_seeds);
|
||||
expect(await repo.search(' '), isEmpty);
|
||||
});
|
||||
|
||||
test('seedBundled stores wikidata id and gbif key', () async {
|
||||
await repo.seedBundled(const [
|
||||
SpeciesSeed(
|
||||
scientificName: 'Solanum lycopersicum',
|
||||
family: 'Solanaceae',
|
||||
wikidataQid: 'Q13223',
|
||||
gbifKey: 2930137,
|
||||
commonNames: {
|
||||
'en': ['Tomato'],
|
||||
},
|
||||
),
|
||||
]);
|
||||
|
||||
final row = await db.select(db.species).getSingle();
|
||||
expect(row.wikidataQid, 'Q13223');
|
||||
expect(row.gbifKey, 2930137);
|
||||
});
|
||||
|
||||
test('seedBundled backfills reference fields added later', () async {
|
||||
// First seed lacks the enrichment fields (an older catalog).
|
||||
await repo.seedBundled(const [
|
||||
SpeciesSeed(
|
||||
scientificName: 'Allium cepa',
|
||||
commonNames: {
|
||||
'en': ['Onion'],
|
||||
},
|
||||
),
|
||||
]);
|
||||
// A later catalog carries them; identity is preserved, fields are filled.
|
||||
await repo.seedBundled(const [
|
||||
SpeciesSeed(
|
||||
scientificName: 'Allium cepa',
|
||||
wikidataQid: 'Q23485',
|
||||
gbifKey: 2857697,
|
||||
viabilityYears: 1,
|
||||
commonNames: {
|
||||
'en': ['Onion'],
|
||||
},
|
||||
),
|
||||
]);
|
||||
|
||||
final rows = await db.select(db.species).get();
|
||||
expect(rows.length, 1);
|
||||
expect(rows.single.wikidataQid, 'Q23485');
|
||||
expect(rows.single.gbifKey, 2857697);
|
||||
expect(rows.single.viabilityYears, 1);
|
||||
});
|
||||
|
||||
test('search matches a non-Latin common name', () async {
|
||||
await repo.seedBundled(const [
|
||||
SpeciesSeed(
|
||||
scientificName: 'Solanum lycopersicum',
|
||||
commonNames: {
|
||||
'ar': ['طماطم'],
|
||||
'zh': ['番茄'],
|
||||
},
|
||||
),
|
||||
]);
|
||||
|
||||
final ar = await repo.search('طماطم', languageCode: 'ar');
|
||||
expect(ar.single.scientificName, 'Solanum lycopersicum');
|
||||
expect(ar.single.commonName, 'طماطم');
|
||||
|
||||
final zh = await repo.search('番茄', languageCode: 'zh');
|
||||
expect(zh.single.commonName, '番茄');
|
||||
});
|
||||
|
||||
test('search respects the limit', () async {
|
||||
final many = [
|
||||
for (var i = 0; i < 20; i++)
|
||||
SpeciesSeed(
|
||||
scientificName: 'Genus species$i',
|
||||
commonNames: const {
|
||||
'en': ['Plant'],
|
||||
},
|
||||
),
|
||||
];
|
||||
await repo.seedBundled(many);
|
||||
final results = await repo.search('Genus', limit: 5);
|
||||
expect(results.length, 5);
|
||||
});
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue