feat(species): expand catalog to ~1200 edible species from Wikidata
Grow the bundled species catalog from 14 hand-curated entries to ~1200 edible/cultivated species, internationalized in 13 languages (es, en, fr, de, it, pt, ca, gl, eu, ar, zh, ja, ru — Latin + Arabic RTL + CJK + Cyrillic). - Add a reproducible generator (tool/gen_species_catalog.dart) that queries Wikidata (CC0, no attribution burden) in two phases, filters out non-vernacular noise (author citations, ranks, initials) and applies a relevance floor, then merges hand-curated, authoritative core-crop data (tool/curated_overrides.json: names, family, viability_years). GBIF is used only as an identifier. The generated species.json (v3) is committed. - Carry wikidata_qid and gbif_key through the parse/seed pipeline; the columns already existed, so no DB migration. - Rewrite seedBundled to one read + one batch (was a SELECT per species on every startup — a real cost at ~1200 rows) and keep it idempotent with backfill of the new reference fields. - Move species search filtering to SQL (LIKE) so a large catalog is not pulled into memory on every keystroke. - Cover the generator transform, the generated asset, and the new fields with tests.
This commit is contained in:
parent
ba87bf2719
commit
959fa2551c
9 changed files with 68483 additions and 120 deletions
|
|
@ -18,6 +18,8 @@ List<SpeciesSeed> parseSpeciesCatalog(String jsonString) {
|
|||
return SpeciesSeed(
|
||||
scientificName: e['scientific_name'] as String,
|
||||
family: e['family'] as String?,
|
||||
wikidataQid: e['wikidata_qid'] as String?,
|
||||
gbifKey: (e['gbif_key'] as num?)?.toInt(),
|
||||
commonNames: common,
|
||||
viabilityYears: (e['viability_years'] as num?)?.toInt(),
|
||||
);
|
||||
|
|
|
|||
|
|
@ -8,6 +8,8 @@ class SpeciesSeed {
|
|||
const SpeciesSeed({
|
||||
required this.scientificName,
|
||||
this.family,
|
||||
this.wikidataQid,
|
||||
this.gbifKey,
|
||||
this.commonNames = const {},
|
||||
this.viabilityYears,
|
||||
});
|
||||
|
|
@ -15,6 +17,14 @@ class SpeciesSeed {
|
|||
final String scientificName;
|
||||
final String? family;
|
||||
|
||||
/// Wikidata entity id (e.g. `Q13223`), the CC0 source of this row; null when
|
||||
/// the catalog was authored by hand. Kept for de-duplication and later
|
||||
/// online enrichment.
|
||||
final String? wikidataQid;
|
||||
|
||||
/// GBIF taxon key linking to the taxonomic backbone; null if unknown.
|
||||
final int? gbifKey;
|
||||
|
||||
/// language code → list of common names.
|
||||
final Map<String, List<String>> commonNames;
|
||||
|
||||
|
|
@ -51,112 +61,194 @@ class SpeciesRepository {
|
|||
final String nodeId;
|
||||
|
||||
/// Idempotently inserts bundled species (keyed by scientific name). Safe to
|
||||
/// call on every startup. Existing rows keep their identity, but a bundled
|
||||
/// row still missing [SpeciesSeed.viabilityYears] is backfilled — so the
|
||||
/// reference data reaches installs seeded before the field was bundled.
|
||||
/// call on every startup. Existing rows keep their identity, but bundled
|
||||
/// reference fields still missing on a row ([SpeciesSeed.viabilityYears],
|
||||
/// [SpeciesSeed.wikidataQid], [SpeciesSeed.gbifKey]) are backfilled — so data
|
||||
/// added to the catalog reaches installs seeded before the field was bundled.
|
||||
///
|
||||
/// Reads all existing species once and writes in a single batch: with a
|
||||
/// catalog of ~1000 species a per-row SELECT on every startup would be a real
|
||||
/// cost on the encrypted database.
|
||||
Future<void> seedBundled(List<SpeciesSeed> seeds) async {
|
||||
final stamp = Hlc.zero(nodeId).pack();
|
||||
await _db.transaction(() async {
|
||||
final existing = {
|
||||
for (final s in await _db.select(_db.species).get())
|
||||
s.scientificName: s,
|
||||
};
|
||||
|
||||
final newSpecies = <SpeciesCompanion>[];
|
||||
final newNames = <SpeciesCommonNamesCompanion>[];
|
||||
final backfills = <({String id, SpeciesCompanion patch})>[];
|
||||
|
||||
for (final seed in seeds) {
|
||||
final existing =
|
||||
await (_db.select(_db.species)
|
||||
..where((s) => s.scientificName.equals(seed.scientificName)))
|
||||
.getSingleOrNull();
|
||||
if (existing != null) {
|
||||
// Backfill bundled reference data added after the row was seeded.
|
||||
if (existing.viabilityYears == null && seed.viabilityYears != null) {
|
||||
await (_db.update(
|
||||
_db.species,
|
||||
)..where((s) => s.id.equals(existing.id))).write(
|
||||
SpeciesCompanion(viabilityYears: Value(seed.viabilityYears)),
|
||||
);
|
||||
}
|
||||
final row = existing[seed.scientificName];
|
||||
if (row != null) {
|
||||
final patch = _backfillPatch(row, seed);
|
||||
if (patch != null) backfills.add((id: row.id, patch: patch));
|
||||
continue;
|
||||
}
|
||||
|
||||
final speciesId = idGen.newId();
|
||||
await _db
|
||||
.into(_db.species)
|
||||
.insert(
|
||||
SpeciesCompanion.insert(
|
||||
id: speciesId,
|
||||
createdAt: 0,
|
||||
updatedAt: stamp,
|
||||
lastAuthor: nodeId,
|
||||
scientificName: seed.scientificName,
|
||||
family: Value(seed.family),
|
||||
isBundled: const Value(true),
|
||||
viabilityYears: Value(seed.viabilityYears),
|
||||
),
|
||||
);
|
||||
newSpecies.add(
|
||||
SpeciesCompanion.insert(
|
||||
id: speciesId,
|
||||
createdAt: 0,
|
||||
updatedAt: stamp,
|
||||
lastAuthor: nodeId,
|
||||
scientificName: seed.scientificName,
|
||||
family: Value(seed.family),
|
||||
wikidataQid: Value(seed.wikidataQid),
|
||||
gbifKey: Value(seed.gbifKey),
|
||||
isBundled: const Value(true),
|
||||
viabilityYears: Value(seed.viabilityYears),
|
||||
),
|
||||
);
|
||||
|
||||
for (final entry in seed.commonNames.entries) {
|
||||
for (final name in entry.value) {
|
||||
await _db
|
||||
.into(_db.speciesCommonNames)
|
||||
.insert(
|
||||
SpeciesCommonNamesCompanion.insert(
|
||||
id: idGen.newId(),
|
||||
createdAt: 0,
|
||||
updatedAt: stamp,
|
||||
lastAuthor: nodeId,
|
||||
speciesId: speciesId,
|
||||
name: name,
|
||||
language: Value(entry.key),
|
||||
),
|
||||
);
|
||||
newNames.add(
|
||||
SpeciesCommonNamesCompanion.insert(
|
||||
id: idGen.newId(),
|
||||
createdAt: 0,
|
||||
updatedAt: stamp,
|
||||
lastAuthor: nodeId,
|
||||
speciesId: speciesId,
|
||||
name: name,
|
||||
language: Value(entry.key),
|
||||
),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (newSpecies.isEmpty && newNames.isEmpty && backfills.isEmpty) return;
|
||||
|
||||
await _db.batch((b) {
|
||||
b.insertAll(_db.species, newSpecies);
|
||||
b.insertAll(_db.speciesCommonNames, newNames);
|
||||
for (final entry in backfills) {
|
||||
b.update(
|
||||
_db.species,
|
||||
entry.patch,
|
||||
where: (s) => s.id.equals(entry.id),
|
||||
);
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
/// A partial update carrying only the bundled reference fields a stored row is
|
||||
/// still missing, or null when the row is already complete.
|
||||
SpeciesCompanion? _backfillPatch(Specy row, SpeciesSeed seed) {
|
||||
var patch = const SpeciesCompanion();
|
||||
var changed = false;
|
||||
if (row.viabilityYears == null && seed.viabilityYears != null) {
|
||||
patch = patch.copyWith(viabilityYears: Value(seed.viabilityYears));
|
||||
changed = true;
|
||||
}
|
||||
if (row.wikidataQid == null && seed.wikidataQid != null) {
|
||||
patch = patch.copyWith(wikidataQid: Value(seed.wikidataQid));
|
||||
changed = true;
|
||||
}
|
||||
if (row.gbifKey == null && seed.gbifKey != null) {
|
||||
patch = patch.copyWith(gbifKey: Value(seed.gbifKey));
|
||||
changed = true;
|
||||
}
|
||||
return changed ? patch : null;
|
||||
}
|
||||
|
||||
/// Searches species by scientific name or a common name (case-insensitive
|
||||
/// substring). The catalog is small, so it is filtered in Dart. Returns up to
|
||||
/// [limit] matches, each with a best common name for [languageCode].
|
||||
/// substring). Filtering runs in SQL (`LIKE`) so a large catalog is not pulled
|
||||
/// into memory on every keystroke. Returns up to [limit] matches, each with a
|
||||
/// best common name for [languageCode].
|
||||
Future<List<SpeciesMatch>> search(
|
||||
String query, {
|
||||
String languageCode = 'en',
|
||||
int limit = 8,
|
||||
}) async {
|
||||
final q = query.trim().toLowerCase();
|
||||
final q = query.trim();
|
||||
if (q.isEmpty) return const [];
|
||||
|
||||
final species = await (_db.select(
|
||||
_db.species,
|
||||
)..where((s) => s.isDeleted.equals(false))).get();
|
||||
final commons = await (_db.select(
|
||||
_db.speciesCommonNames,
|
||||
)..where((n) => n.isDeleted.equals(false))).get();
|
||||
// SQLite `LIKE` is case-insensitive for ASCII (matching the previous
|
||||
// `toLowerCase().contains` behaviour); `%`/`_`/`\` in the query are escaped
|
||||
// so they match literally rather than acting as wildcards.
|
||||
final pattern = '%${_escapeLike(q)}%';
|
||||
// A few extra candidates so the final scientific-name sort/limit is stable.
|
||||
final scan = limit * 4;
|
||||
|
||||
final byScientific =
|
||||
await (_db.select(_db.species)
|
||||
..where(
|
||||
(s) =>
|
||||
s.isDeleted.equals(false) &
|
||||
s.scientificName.like(pattern, escapeChar: r'\'),
|
||||
)
|
||||
..limit(scan))
|
||||
.get();
|
||||
|
||||
final byCommon =
|
||||
await (_db.select(_db.speciesCommonNames)
|
||||
..where(
|
||||
(n) =>
|
||||
n.isDeleted.equals(false) &
|
||||
n.name.like(pattern, escapeChar: r'\'),
|
||||
)
|
||||
..limit(scan))
|
||||
.get();
|
||||
|
||||
final speciesById = {for (final s in byScientific) s.id: s};
|
||||
final missing = byCommon
|
||||
.map((n) => n.speciesId)
|
||||
.where((id) => !speciesById.containsKey(id))
|
||||
.toSet();
|
||||
if (missing.isNotEmpty) {
|
||||
final more =
|
||||
await (_db.select(_db.species)..where(
|
||||
(s) => s.isDeleted.equals(false) & s.id.isIn(missing.toList()),
|
||||
))
|
||||
.get();
|
||||
for (final s in more) {
|
||||
speciesById[s.id] = s;
|
||||
}
|
||||
}
|
||||
if (speciesById.isEmpty) return const [];
|
||||
|
||||
// Resolve a localized label only for the matched species.
|
||||
final names =
|
||||
await (_db.select(_db.speciesCommonNames)..where(
|
||||
(n) =>
|
||||
n.isDeleted.equals(false) &
|
||||
n.speciesId.isIn(speciesById.keys.toList()),
|
||||
))
|
||||
.get();
|
||||
final namesBySpecies = <String, List<({String name, String? language})>>{};
|
||||
for (final c in commons) {
|
||||
namesBySpecies.putIfAbsent(c.speciesId, () => []).add((
|
||||
name: c.name,
|
||||
language: c.language,
|
||||
for (final n in names) {
|
||||
namesBySpecies.putIfAbsent(n.speciesId, () => []).add((
|
||||
name: n.name,
|
||||
language: n.language,
|
||||
));
|
||||
}
|
||||
|
||||
final matches = <SpeciesMatch>[];
|
||||
for (final s in species) {
|
||||
final names = namesBySpecies[s.id] ?? const [];
|
||||
final matchesScientific = s.scientificName.toLowerCase().contains(q);
|
||||
final matchesCommon = names.any((n) => n.name.toLowerCase().contains(q));
|
||||
if (matchesScientific || matchesCommon) {
|
||||
matches.add(
|
||||
SpeciesMatch(
|
||||
id: s.id,
|
||||
scientificName: s.scientificName,
|
||||
family: s.family,
|
||||
commonName: _bestName(names, languageCode),
|
||||
),
|
||||
);
|
||||
}
|
||||
}
|
||||
final matches = [
|
||||
for (final s in speciesById.values)
|
||||
SpeciesMatch(
|
||||
id: s.id,
|
||||
scientificName: s.scientificName,
|
||||
family: s.family,
|
||||
commonName: _bestName(namesBySpecies[s.id] ?? const [], languageCode),
|
||||
),
|
||||
];
|
||||
matches.sort((a, b) => a.scientificName.compareTo(b.scientificName));
|
||||
return matches.take(limit).toList();
|
||||
}
|
||||
|
||||
/// Escapes SQL `LIKE` metacharacters so a user query matches literally.
|
||||
String _escapeLike(String input) => input
|
||||
.replaceAll(r'\', r'\\')
|
||||
.replaceAll('%', r'\%')
|
||||
.replaceAll('_', r'\_');
|
||||
|
||||
String? _bestName(
|
||||
List<({String name, String? language})> names,
|
||||
String languageCode,
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue