feat(species): expand catalog to ~1200 edible species from Wikidata
Grow the bundled species catalog from 14 hand-curated entries to ~1200 edible/cultivated species, internationalized in 13 languages (es, en, fr, de, it, pt, ca, gl, eu, ar, zh, ja, ru — Latin + Arabic RTL + CJK + Cyrillic). - Add a reproducible generator (tool/gen_species_catalog.dart) that queries Wikidata (CC0, no attribution burden) in two phases, filters out non-vernacular noise (author citations, ranks, initials) and applies a relevance floor, then merges hand-curated, authoritative core-crop data (tool/curated_overrides.json: names, family, viability_years). GBIF is used only as an identifier. The generated species.json (v3) is committed. - Carry wikidata_qid and gbif_key through the parse/seed pipeline; the columns already existed, so no DB migration. - Rewrite seedBundled to one read + one batch (was a SELECT per species on every startup — a real cost at ~1200 rows) and keep it idempotent with backfill of the new reference fields. - Move species search filtering to SQL (LIKE) so a large catalog is not pulled into memory on every keystroke. - Cover the generator transform, the generated asset, and the new fields with tests.
This commit is contained in:
parent
ba87bf2719
commit
959fa2551c
9 changed files with 68483 additions and 120 deletions
368
apps/app_seeds/tool/gen_species_catalog.dart
Normal file
368
apps/app_seeds/tool/gen_species_catalog.dart
Normal file
|
|
@ -0,0 +1,368 @@
|
|||
// Regenerates the bundled species catalog (`assets/catalog/species.json`) from
|
||||
// Wikidata (CC0). Dev-only tool — it is NOT shipped in the app; the app only
|
||||
// reads the committed JSON it produces (local-first, offline).
|
||||
//
|
||||
// dart run tool/gen_species_catalog.dart # writes species.json (v3)
|
||||
// dart run tool/gen_species_catalog.dart --dry # prints stats, writes nothing
|
||||
//
|
||||
// Names come only from Wikidata labels/aliases (CC0 → no attribution burden).
|
||||
// GBIF is used solely as an identifier (`gbif_key`, a fact, not copyrightable).
|
||||
// Seed-longevity (`viability_years`) and authoritative core-crop names are
|
||||
// merged from the hand-maintained `tool/curated_overrides.json`. See
|
||||
// `tool/README.md`.
|
||||
//
|
||||
// The pure transform (`buildCatalog`) is separated from the network fetch so it
|
||||
// is unit-testable without hitting the endpoint.
|
||||
import 'dart:convert';
|
||||
import 'dart:io';
|
||||
|
||||
/// Locale set bundled at launch — a multilingual seed, not a ceiling. Covers
|
||||
/// Latin, Arabic (RTL), CJK (Chinese + Japanese) and Cyrillic scripts. Order
|
||||
/// here is the order common names are written in the JSON.
|
||||
const List<String> kCatalogLanguages = [
|
||||
'es',
|
||||
'en',
|
||||
'fr',
|
||||
'de',
|
||||
'it',
|
||||
'pt',
|
||||
'ca',
|
||||
'gl',
|
||||
'eu',
|
||||
'ar',
|
||||
'zh',
|
||||
'ja',
|
||||
'ru',
|
||||
];
|
||||
|
||||
const String _endpoint = 'https://query.wikidata.org/sparql';
|
||||
|
||||
/// Wikimedia asks bots to send a descriptive, contactable User-Agent.
|
||||
const String _userAgent =
|
||||
'TanemakiSpeciesCatalogBot/1.0 (https://tanemaki.app; vjrj@comunes.org)';
|
||||
|
||||
const String _catalogNote =
|
||||
'Edible / cultivated plant species, generated from Wikidata (CC0) by '
|
||||
'tool/gen_species_catalog.dart — see tool/README.md to regenerate. `common` '
|
||||
'is a locale-keyed map (Wikidata labels + aliases) the app resolves by the '
|
||||
"user's locale with graceful fallback, never assuming es/en only. "
|
||||
'`gbif_key` links the GBIF taxonomic backbone (identifier only). '
|
||||
'`viability_years` is public-domain agricultural-extension seed longevity '
|
||||
'merged from tool/curated_overrides.json (not in Wikidata); a conservative '
|
||||
'single value used to warn about aging lots.';
|
||||
|
||||
/// Hand-curated, authoritative data for a core crop (from
|
||||
/// `curated_overrides.json`). Its names win over Wikidata's; its family and
|
||||
/// viability override; a curated species is always kept.
|
||||
class CuratedOverride {
|
||||
const CuratedOverride({
|
||||
this.family,
|
||||
this.viabilityYears,
|
||||
this.common = const {},
|
||||
});
|
||||
|
||||
final String? family;
|
||||
final int? viabilityYears;
|
||||
final Map<String, List<String>> common;
|
||||
}
|
||||
|
||||
/// One aggregated species before it is serialized. Mutable during aggregation.
|
||||
class SpeciesRecord {
|
||||
SpeciesRecord({
|
||||
required this.qid,
|
||||
required this.scientificName,
|
||||
this.family,
|
||||
this.gbifKey,
|
||||
Map<String, Set<String>>? common,
|
||||
}) : common = common ?? {};
|
||||
|
||||
final String qid;
|
||||
final String scientificName;
|
||||
String? family;
|
||||
int? gbifKey;
|
||||
|
||||
/// language code → set of names (dedup, insertion-ordered).
|
||||
final Map<String, Set<String>> common;
|
||||
}
|
||||
|
||||
/// Whether [name] reads as a genuine vernacular rather than a scientific name,
|
||||
/// synonym or author citation. Wikidata aliases are full of the latter (commas,
|
||||
/// parentheses, hybrid `×`, `var.`/`subsp.` ranks, or a straight repeat of the
|
||||
/// binomial); those are noise in a common-name list.
|
||||
bool isVernacularName(String name, String scientificNameLower) {
|
||||
final n = name.trim();
|
||||
if (n.isEmpty) return false;
|
||||
if (n.toLowerCase() == scientificNameLower) return false;
|
||||
// Commas/parentheses (ASCII and CJK full-width) and the hybrid sign mark
|
||||
// author citations, synonym lists and hybrid formulas — never a vernacular.
|
||||
if (n.contains(RegExp(r'[,()（），×]'))) return false;
|
||||
if (n.contains(RegExp(r'\b(var|subsp|ssp|f|cv|nothosubsp)\.'))) return false;
|
||||
// A lone initial + period is an abbreviated genus or author citation
|
||||
// ("Z. mays", "Allium cepa L."), not a vernacular.
|
||||
if (n.contains(RegExp(r'\b[A-Z]\.'))) return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
/// Pure transform: dedup by scientific name, keep only [languages], drop
|
||||
/// non-vernacular noise, require a name in at least [minLanguages] languages
|
||||
/// (a relevance floor — obscure taxa carry a vernacular in only one), merge the
|
||||
/// hand-curated [curated] overrides (authoritative names/family/viability that
|
||||
/// win over Wikidata and are always kept), and emit an ordered, JSON-ready
|
||||
/// catalog map (`{version, note, species: [...]}`). No network, no I/O.
|
||||
Map<String, dynamic> buildCatalog(
|
||||
Iterable<SpeciesRecord> records,
|
||||
Map<String, CuratedOverride> curated, {
|
||||
List<String> languages = kCatalogLanguages,
|
||||
int minLanguages = 1,
|
||||
}) {
|
||||
// Dedup by scientific name; merge names and backfill family/gbif across the
|
||||
// duplicate entities Wikidata sometimes has for one accepted binomial.
|
||||
final byName = <String, SpeciesRecord>{};
|
||||
for (final r in records) {
|
||||
final name = r.scientificName.trim();
|
||||
if (name.isEmpty) continue;
|
||||
final existing = byName[name];
|
||||
if (existing == null) {
|
||||
byName[name] = r;
|
||||
continue;
|
||||
}
|
||||
existing.family ??= r.family;
|
||||
existing.gbifKey ??= r.gbifKey;
|
||||
for (final entry in r.common.entries) {
|
||||
existing.common.putIfAbsent(entry.key, () => {}).addAll(entry.value);
|
||||
}
|
||||
}
|
||||
|
||||
final species = <Map<String, dynamic>>[];
|
||||
for (final r in byName.values) {
|
||||
final sciLower = r.scientificName.trim().toLowerCase();
|
||||
final override = curated[r.scientificName];
|
||||
final common = <String, List<String>>{};
|
||||
for (final lang in languages) {
|
||||
final ordered = <String>[];
|
||||
final seen = <String>{};
|
||||
// Curated names first (authoritative, kept as-is), then Wikidata's
|
||||
// vernaculars, de-duplicated case-insensitively.
|
||||
for (final n in override?.common[lang] ?? const <String>[]) {
|
||||
if (n.trim().isNotEmpty && seen.add(n.toLowerCase())) ordered.add(n);
|
||||
}
|
||||
for (final n in r.common[lang] ?? const <String>{}) {
|
||||
final v = n.trim();
|
||||
if (!isVernacularName(v, sciLower)) continue;
|
||||
if (seen.add(v.toLowerCase())) ordered.add(v);
|
||||
}
|
||||
if (ordered.isNotEmpty) common[lang] = ordered;
|
||||
}
|
||||
// A relevance floor for Wikidata-only species: obscure taxa carry a real
|
||||
// vernacular in only one language (a lone CJK/Cyrillic label), well-known
|
||||
// crops in many. Curated species bypass the floor.
|
||||
if (override == null && common.length < minLanguages) continue;
|
||||
if (common.isEmpty) continue;
|
||||
|
||||
final family = override?.family ?? r.family;
|
||||
final viability = override?.viabilityYears;
|
||||
species.add({
|
||||
'scientific_name': r.scientificName,
|
||||
'family': ?family,
|
||||
'wikidata_qid': r.qid,
|
||||
if (r.gbifKey != null) 'gbif_key': r.gbifKey,
|
||||
'common': common,
|
||||
'viability_years': ?viability,
|
||||
});
|
||||
}
|
||||
|
||||
species.sort(
|
||||
(a, b) => (a['scientific_name'] as String).compareTo(
|
||||
b['scientific_name'] as String,
|
||||
),
|
||||
);
|
||||
|
||||
return {'version': 3, 'note': _catalogNote, 'species': species};
|
||||
}
|
||||
|
||||
Future<void> main(List<String> args) async {
|
||||
final dryRun = args.contains('--dry');
|
||||
final toolDir = File(Platform.script.toFilePath()).parent;
|
||||
final overridesFile = File('${toolDir.path}/curated_overrides.json');
|
||||
final outFile = File('${toolDir.path}/../assets/catalog/species.json');
|
||||
|
||||
final curated = _readCuratedOverrides(overridesFile);
|
||||
stdout.writeln('Loaded ${curated.length} curated overrides.');
|
||||
|
||||
final client = HttpClient()..userAgent = _userAgent;
|
||||
try {
|
||||
stdout.writeln('Querying Wikidata for edible/cultivated species…');
|
||||
final skeleton = await _fetchSpeciesSkeleton(client);
|
||||
stdout.writeln(
|
||||
'Found ${skeleton.length} candidate species. Fetching '
|
||||
'names in ${kCatalogLanguages.length} languages…',
|
||||
);
|
||||
await _fetchLabels(client, skeleton);
|
||||
|
||||
final catalog = buildCatalog(skeleton.values, curated, minLanguages: 4);
|
||||
final count = (catalog['species'] as List).length;
|
||||
stdout.writeln('Built catalog: $count species with bundled-locale names.');
|
||||
|
||||
if (dryRun) {
|
||||
stdout.writeln('--dry: not writing ${outFile.path}');
|
||||
return;
|
||||
}
|
||||
const encoder = JsonEncoder.withIndent(' ');
|
||||
await outFile.writeAsString('${encoder.convert(catalog)}\n');
|
||||
stdout.writeln('Wrote ${outFile.path}');
|
||||
} finally {
|
||||
client.close();
|
||||
}
|
||||
}
|
||||
|
||||
Map<String, CuratedOverride> _readCuratedOverrides(File file) {
|
||||
if (!file.existsSync()) return {};
|
||||
final data = jsonDecode(file.readAsStringSync()) as Map<String, dynamic>;
|
||||
final species = (data['species'] as Map<String, dynamic>);
|
||||
return species.map((name, raw) {
|
||||
final o = raw as Map<String, dynamic>;
|
||||
final common = (o['common'] as Map<String, dynamic>? ?? const {}).map(
|
||||
(lang, names) => MapEntry(lang, (names as List).cast<String>()),
|
||||
);
|
||||
return MapEntry(
|
||||
name,
|
||||
CuratedOverride(
|
||||
family: o['family'] as String?,
|
||||
viabilityYears: (o['viability_years'] as num?)?.toInt(),
|
||||
common: common,
|
||||
),
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
/// Phase 1: the list of species QIDs with scientific name, family and GBIF key.
|
||||
Future<Map<String, SpeciesRecord>> _fetchSpeciesSkeleton(
|
||||
HttpClient client,
|
||||
) async {
|
||||
// Taxa with a binomial (or lower) scientific name — CONTAINS(" ") keeps
|
||||
// species and botanical varieties (relevant for seed savers: e.g. the many
|
||||
// Brassica oleracea var. cultivars) while dropping genus-level rows — that are
|
||||
// edible/cultivated, reached three ways: a crop, the source of a food, or the
|
||||
// source of a food ingredient. Taxon rank is intentionally not required: many
|
||||
// cultivated taxa in Wikidata leave P105 unset.
|
||||
const query = r'''
|
||||
SELECT ?item ?sci ?gbif WHERE {
|
||||
?item wdt:P225 ?sci .
|
||||
FILTER(CONTAINS(?sci, " "))
|
||||
{
|
||||
?item wdt:P279* wd:Q235352 . # a crop
|
||||
} UNION {
|
||||
?item wdt:P1672 ?use . # this taxon is source of …
|
||||
?use wdt:P279* wd:Q2095 . # … a food
|
||||
} UNION {
|
||||
?item wdt:P1672 ?use .
|
||||
?use wdt:P279* wd:Q25403900 . # … a food ingredient
|
||||
} UNION {
|
||||
?food wdt:P1582 ?item . # … the natural source of a food
|
||||
?food wdt:P279* wd:Q2095 .
|
||||
} UNION {
|
||||
?item wdt:P366 ?use . # this taxon has use …
|
||||
?use wdt:P279* wd:Q2095 . # … food
|
||||
}
|
||||
OPTIONAL { ?item wdt:P846 ?gbif . }
|
||||
}''';
|
||||
final rows = await _runSparql(client, query);
|
||||
final out = <String, SpeciesRecord>{};
|
||||
for (final row in rows) {
|
||||
final qid = _qid(row['item']?['value'] as String?);
|
||||
final sci = row['sci']?['value'] as String?;
|
||||
if (qid == null || sci == null) continue;
|
||||
final rec = out.putIfAbsent(
|
||||
qid,
|
||||
() => SpeciesRecord(qid: qid, scientificName: sci),
|
||||
);
|
||||
final gbif = row['gbif']?['value'] as String?;
|
||||
if (gbif != null) rec.gbifKey ??= int.tryParse(gbif);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/// Phase 2: labels + aliases (and the family, walked up `P171*`) in the bundled
|
||||
/// languages, batched over QIDs to keep each query under the endpoint timeout.
|
||||
Future<void> _fetchLabels(
|
||||
HttpClient client,
|
||||
Map<String, SpeciesRecord> skeleton,
|
||||
) async {
|
||||
final langFilter = kCatalogLanguages.map((l) => '"$l"').join(', ');
|
||||
final qids = skeleton.keys.toList();
|
||||
const batchSize = 100;
|
||||
for (var i = 0; i < qids.length; i += batchSize) {
|
||||
final batch = qids.skip(i).take(batchSize);
|
||||
final values = batch.map((q) => 'wd:$q').join(' ');
|
||||
// `kind` orders the primary label ("a", rdfs:label — usually the real
|
||||
// vernacular) ahead of aliases ("b", skos:altLabel — often synonyms).
|
||||
final query =
|
||||
'''
|
||||
SELECT ?item ?label ?kind ?family WHERE {
|
||||
VALUES ?item { $values }
|
||||
OPTIONAL {
|
||||
?item wdt:P171* ?fam . # walk up to the family-rank ancestor
|
||||
?fam wdt:P105 wd:Q35409 ;
|
||||
wdt:P225 ?family .
|
||||
}
|
||||
{ ?item rdfs:label ?label . BIND("a" AS ?kind) }
|
||||
UNION
|
||||
{ ?item skos:altLabel ?label . BIND("b" AS ?kind) }
|
||||
FILTER(LANG(?label) IN ($langFilter))
|
||||
}''';
|
||||
final rows = await _runSparql(client, query);
|
||||
// Primary labels first so they land first in each language's ordered set.
|
||||
rows.sort(
|
||||
(a, b) => (a['kind']?['value'] as String? ?? 'b').compareTo(
|
||||
b['kind']?['value'] as String? ?? 'b',
|
||||
),
|
||||
);
|
||||
for (final row in rows) {
|
||||
final qid = _qid(row['item']?['value'] as String?);
|
||||
final label = row['label'];
|
||||
if (qid == null || label == null) continue;
|
||||
final rec = skeleton[qid];
|
||||
if (rec == null) continue;
|
||||
rec.family ??= row['family']?['value'] as String?;
|
||||
final lang = label['xml:lang'] as String?;
|
||||
final value = label['value'] as String?;
|
||||
if (lang == null || value == null) continue;
|
||||
rec.common.putIfAbsent(lang, () => {}).add(value);
|
||||
}
|
||||
stdout.writeln(
|
||||
' labels: ${(i + batchSize).clamp(0, qids.length)}'
|
||||
'/${qids.length}',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Runs a SPARQL SELECT and returns the `results.bindings` list.
|
||||
Future<List<Map<String, dynamic>>> _runSparql(
|
||||
HttpClient client,
|
||||
String query,
|
||||
) async {
|
||||
final uri = Uri.parse(
|
||||
_endpoint,
|
||||
).replace(queryParameters: {'query': query, 'format': 'json'});
|
||||
final request = await client.getUrl(uri);
|
||||
request.headers.set(
|
||||
HttpHeaders.acceptHeader,
|
||||
'application/sparql-results+json',
|
||||
);
|
||||
final response = await request.close();
|
||||
if (response.statusCode != 200) {
|
||||
final body = await response.transform(utf8.decoder).join();
|
||||
throw HttpException('SPARQL ${response.statusCode}: $body');
|
||||
}
|
||||
final body = await response.transform(utf8.decoder).join();
|
||||
final decoded = jsonDecode(body) as Map<String, dynamic>;
|
||||
final bindings = (decoded['results']?['bindings'] as List?) ?? const [];
|
||||
return bindings.cast<Map<String, dynamic>>();
|
||||
}
|
||||
|
||||
/// Extracts the `Qxxxx` id from a Wikidata entity URI.
|
||||
String? _qid(String? uri) {
|
||||
if (uri == null) return null;
|
||||
final i = uri.lastIndexOf('/');
|
||||
return i == -1 ? uri : uri.substring(i + 1);
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue