comunes/tane
1
0
Fork 0
Code Issues Pull requests Projects Releases 7 Packages Wiki Activity Actions

Merge branch 'claude/cool-swanson-81a747': expand species catalog to ~1200 edible species

Browse source 
# Conflicts:
#	apps/app_seeds/assets/catalog/species.json
#	apps/app_seeds/lib/data/species_repository.dart
This commit is contained in:
vjrj 2026-07-10 02:04:51 +02:00
commit 46ba85dad0
9 changed files with 68482 additions and 175 deletions
Download patch file Download diff file Expand all files Collapse all files

67617
apps/app_seeds/assets/catalog/species.json
View file

File diff suppressed because it is too large Load diff

4
apps/app_seeds/lib/data/species_catalog.dart
View file

@ -18,10 +18,10 @@ List<SpeciesSeed> parseSpeciesCatalog(String jsonString) {
return SpeciesSeed(
scientificName: e['scientific_name'] as String,
family: e['family'] as String?,
wikidataQid: e['wikidata_qid'] as String?,
gbifKey: (e['gbif_key'] as num?)?.toInt(),
commonNames: common,
viabilityYears: (e['viability_years'] as num?)?.toInt(),
gbifKey: (e['gbif_key'] as num?)?.toInt(),
wikidataQid: e['wikidata_qid'] as String?,
);
}).toList();
}

262
apps/app_seeds/lib/data/species_repository.dart
View file

@ -8,27 +8,28 @@ class SpeciesSeed {
const SpeciesSeed({
required this.scientificName,
this.family,
this.wikidataQid,
this.gbifKey,
this.commonNames = const {},
this.viabilityYears,
this.gbifKey,
this.wikidataQid,
});
final String scientificName;
final String? family;
/// Wikidata entity id (e.g. `Q13223`), the CC0 source of this row; null when
/// the catalog was authored by hand. Kept for de-duplication and to feed the
/// derived Wikipedia (locale-aware) reference link.
final String? wikidataQid;
/// GBIF backbone taxon key; null if unknown. Feeds the derived reference link.
final int? gbifKey;
/// language code → list of common names.
final Map<String, List<String>> commonNames;
/// Typical seed longevity in years (bundled reference data); null if unknown.
final int? viabilityYears;
/// GBIF backbone taxon key; null if unknown. Feeds the derived reference link.
final int? gbifKey;
/// Wikidata item id (e.g. `Q23501`); null if unknown. Feeds the derived
/// Wikipedia (locale-aware) reference link.
final String? wikidataQid;
}
/// A catalog match surfaced to the UI (with a best common name for the locale).
@ -60,129 +61,194 @@ class SpeciesRepository {
final String nodeId;
/// Idempotently inserts bundled species (keyed by scientific name). Safe to
/// call on every startup. Existing rows keep their identity, but a bundled
/// row still missing [SpeciesSeed.viabilityYears] is backfilled — so the
/// reference data reaches installs seeded before the field was bundled.
/// call on every startup. Existing rows keep their identity, but bundled
/// reference fields still missing on a row ([SpeciesSeed.viabilityYears],
/// [SpeciesSeed.wikidataQid], [SpeciesSeed.gbifKey]) are backfilled — so data
/// added to the catalog reaches installs seeded before the field was bundled.
///
/// Reads all existing species once and writes in a single batch: with a
/// catalog of ~1000 species a per-row SELECT on every startup would be a real
/// cost on the encrypted database.
Future<void> seedBundled(List<SpeciesSeed> seeds) async {
final stamp = Hlc.zero(nodeId).pack();
await _db.transaction(() async {
final existing = {
for (final s in await _db.select(_db.species).get())
s.scientificName: s,
};
final newSpecies = <SpeciesCompanion>[];
final newNames = <SpeciesCommonNamesCompanion>[];
final backfills = <({String id, SpeciesCompanion patch})>[];
for (final seed in seeds) {
final existing =
await (_db.select(_db.species)
..where((s) => s.scientificName.equals(seed.scientificName)))
.getSingleOrNull();
if (existing != null) {
// Backfill bundled reference data added after the row was seeded, so
// it reaches installs seeded before these fields were bundled. Each
// field is only written when the row still lacks it.
final backfill = SpeciesCompanion(
viabilityYears:
existing.viabilityYears == null && seed.viabilityYears != null
? Value(seed.viabilityYears)
: const Value.absent(),
gbifKey: existing.gbifKey == null && seed.gbifKey != null
? Value(seed.gbifKey)
: const Value.absent(),
wikidataQid:
existing.wikidataQid == null && seed.wikidataQid != null
? Value(seed.wikidataQid)
: const Value.absent(),
);
if (backfill.viabilityYears.present ||
backfill.gbifKey.present ||
backfill.wikidataQid.present) {
await (_db.update(
_db.species,
)..where((s) => s.id.equals(existing.id))).write(backfill);
}
final row = existing[seed.scientificName];
if (row != null) {
final patch = _backfillPatch(row, seed);
if (patch != null) backfills.add((id: row.id, patch: patch));
continue;
}
final speciesId = idGen.newId();
await _db
.into(_db.species)
.insert(
SpeciesCompanion.insert(
id: speciesId,
createdAt: 0,
updatedAt: stamp,
lastAuthor: nodeId,
scientificName: seed.scientificName,
family: Value(seed.family),
isBundled: const Value(true),
viabilityYears: Value(seed.viabilityYears),
gbifKey: Value(seed.gbifKey),
wikidataQid: Value(seed.wikidataQid),
),
);
newSpecies.add(
SpeciesCompanion.insert(
id: speciesId,
createdAt: 0,
updatedAt: stamp,
lastAuthor: nodeId,
scientificName: seed.scientificName,
family: Value(seed.family),
wikidataQid: Value(seed.wikidataQid),
gbifKey: Value(seed.gbifKey),
isBundled: const Value(true),
viabilityYears: Value(seed.viabilityYears),
),
);
for (final entry in seed.commonNames.entries) {
for (final name in entry.value) {
await _db
.into(_db.speciesCommonNames)
.insert(
SpeciesCommonNamesCompanion.insert(
id: idGen.newId(),
createdAt: 0,
updatedAt: stamp,
lastAuthor: nodeId,
speciesId: speciesId,
name: name,
language: Value(entry.key),
),
);
newNames.add(
SpeciesCommonNamesCompanion.insert(
id: idGen.newId(),
createdAt: 0,
updatedAt: stamp,
lastAuthor: nodeId,
speciesId: speciesId,
name: name,
language: Value(entry.key),
),
);
}
}
}
if (newSpecies.isEmpty && newNames.isEmpty && backfills.isEmpty) return;
await _db.batch((b) {
b.insertAll(_db.species, newSpecies);
b.insertAll(_db.speciesCommonNames, newNames);
for (final entry in backfills) {
b.update(
_db.species,
entry.patch,
where: (s) => s.id.equals(entry.id),
);
}
});
});
}
/// A partial update carrying only the bundled reference fields a stored row is
/// still missing, or null when the row is already complete.
SpeciesCompanion? _backfillPatch(Specy row, SpeciesSeed seed) {
var patch = const SpeciesCompanion();
var changed = false;
if (row.viabilityYears == null && seed.viabilityYears != null) {
patch = patch.copyWith(viabilityYears: Value(seed.viabilityYears));
changed = true;
}
if (row.wikidataQid == null && seed.wikidataQid != null) {
patch = patch.copyWith(wikidataQid: Value(seed.wikidataQid));
changed = true;
}
if (row.gbifKey == null && seed.gbifKey != null) {
patch = patch.copyWith(gbifKey: Value(seed.gbifKey));
changed = true;
}
return changed ? patch : null;
}
/// Searches species by scientific name or a common name (case-insensitive
/// substring). The catalog is small, so it is filtered in Dart. Returns up to
/// [limit] matches, each with a best common name for [languageCode].
/// substring). Filtering runs in SQL (`LIKE`) so a large catalog is not pulled
/// into memory on every keystroke. Returns up to [limit] matches, each with a
/// best common name for [languageCode].
Future<List<SpeciesMatch>> search(
String query, {
String languageCode = 'en',
int limit = 8,
}) async {
final q = query.trim().toLowerCase();
final q = query.trim();
if (q.isEmpty) return const [];
final species = await (_db.select(
_db.species,
)..where((s) => s.isDeleted.equals(false))).get();
final commons = await (_db.select(
_db.speciesCommonNames,
)..where((n) => n.isDeleted.equals(false))).get();
// SQLite `LIKE` is case-insensitive for ASCII (matching the previous
// `toLowerCase().contains` behaviour); `%`/`_`/`\` in the query are escaped
// so they match literally rather than acting as wildcards.
final pattern = '%${_escapeLike(q)}%';
// A few extra candidates so the final scientific-name sort/limit is stable.
final scan = limit * 4;
final byScientific =
await (_db.select(_db.species)
..where(
(s) =>
s.isDeleted.equals(false) &
s.scientificName.like(pattern, escapeChar: r'\'),
)
..limit(scan))
.get();
final byCommon =
await (_db.select(_db.speciesCommonNames)
..where(
(n) =>
n.isDeleted.equals(false) &
n.name.like(pattern, escapeChar: r'\'),
)
..limit(scan))
.get();
final speciesById = {for (final s in byScientific) s.id: s};
final missing = byCommon
.map((n) => n.speciesId)
.where((id) => !speciesById.containsKey(id))
.toSet();
if (missing.isNotEmpty) {
final more =
await (_db.select(_db.species)..where(
(s) => s.isDeleted.equals(false) & s.id.isIn(missing.toList()),
))
.get();
for (final s in more) {
speciesById[s.id] = s;
}
}
if (speciesById.isEmpty) return const [];
// Resolve a localized label only for the matched species.
final names =
await (_db.select(_db.speciesCommonNames)..where(
(n) =>
n.isDeleted.equals(false) &
n.speciesId.isIn(speciesById.keys.toList()),
))
.get();
final namesBySpecies = <String, List<({String name, String? language})>>{};
for (final c in commons) {
namesBySpecies.putIfAbsent(c.speciesId, () => []).add((
name: c.name,
language: c.language,
for (final n in names) {
namesBySpecies.putIfAbsent(n.speciesId, () => []).add((
name: n.name,
language: n.language,
));
}
final matches = <SpeciesMatch>[];
for (final s in species) {
final names = namesBySpecies[s.id] ?? const [];
final matchesScientific = s.scientificName.toLowerCase().contains(q);
final matchesCommon = names.any((n) => n.name.toLowerCase().contains(q));
if (matchesScientific || matchesCommon) {
matches.add(
SpeciesMatch(
id: s.id,
scientificName: s.scientificName,
family: s.family,
commonName: _bestName(names, languageCode),
),
);
}
}
final matches = [
for (final s in speciesById.values)
SpeciesMatch(
id: s.id,
scientificName: s.scientificName,
family: s.family,
commonName: _bestName(namesBySpecies[s.id] ?? const [], languageCode),
),
];
matches.sort((a, b) => a.scientificName.compareTo(b.scientificName));
return matches.take(limit).toList();
}
/// Escapes SQL `LIKE` metacharacters so a user query matches literally.
String _escapeLike(String input) => input
.replaceAll(r'\', r'\\')
.replaceAll('%', r'\%')
.replaceAll('_', r'\_');
String? _bestName(
List<({String name, String? language})> names,
String languageCode,

84
apps/app_seeds/test/data/species_catalog_asset_test.dart Normal file
View file

@ -0,0 +1,84 @@
import 'dart:convert';
import 'dart:io';
import 'package:flutter_test/flutter_test.dart';
import 'package:tane/data/species_catalog.dart';
/// Guards the committed, generated catalog asset (produced by
/// `tool/gen_species_catalog.dart`). Reads the file directly — no Flutter asset
/// bundle needed — so it runs as a plain VM test.
void main() {
// `flutter test` runs with the package root as the working directory.
final file = File('assets/catalog/species.json');
final json = jsonDecode(file.readAsStringSync()) as Map<String, dynamic>;
final species = (json['species'] as List).cast<Map<String, dynamic>>();
test('is the current (version 3) generated catalog', () {
expect(json['version'], 3);
});
test('is a broad catalog, not the old starter set', () {
// The expanded catalog holds ~1200 species (was 14 by hand). The floor
// leaves headroom for Wikidata drift across regenerations.
expect(species.length, greaterThan(900));
});
test('every entry has a scientific name and at least one localized name', () {
for (final e in species) {
final sci = e['scientific_name'];
expect(sci, isA<String>());
expect((sci as String).trim(), isNotEmpty);
final common = e['common'] as Map<String, dynamic>;
expect(common, isNotEmpty, reason: '$sci has no common names');
for (final entry in common.entries) {
final names = (entry.value as List).cast<String>();
expect(names, isNotEmpty);
expect(names.every((n) => n.trim().isNotEmpty), isTrue);
}
}
});
test('identifiers and reference fields are well typed', () {
for (final e in species) {
expect(e['wikidata_qid'], matches(r'^Q\d+$'));
if (e.containsKey('gbif_key')) expect(e['gbif_key'], isA<int>());
if (e.containsKey('viability_years')) {
expect(e['viability_years'], isA<int>());
}
}
});
test('scientific names are unique', () {
final names = species.map((e) => e['scientific_name'] as String).toList();
expect(names.length, names.toSet().length);
});
test('is genuinely multilingual (non-Latin scripts present)', () {
int withLang(String code) =>
species.where((e) => (e['common'] as Map).containsKey(code)).length;
// International by design: Arabic (RTL) and CJK must be represented, not
// an es/en-only catalog.
expect(withLang('ar'), greaterThan(0), reason: 'no Arabic names');
expect(withLang('zh'), greaterThan(0), reason: 'no Chinese names');
expect(withLang('ja'), greaterThan(0), reason: 'no Japanese names');
});
test('keeps the curated core crops with their authoritative names', () {
final byName = {for (final e in species) e['scientific_name'] as String: e};
final tomato = byName['Solanum lycopersicum'];
final carrot = byName['Daucus carota'];
expect(tomato, isNotNull);
expect(carrot, isNotNull);
// Curated names lead (not Wikidata's messier taxon labels) and viability
// reference data survives the regeneration.
expect((tomato!['common'] as Map)['es'], contains('Tomate'));
expect((carrot!['common'] as Map)['es'], contains('Zanahoria'));
expect(carrot['viability_years'], 3);
});
test('parses through the app loader without error', () {
final seeds = parseSpeciesCatalog(file.readAsStringSync());
expect(seeds.length, species.length);
});
}

88
apps/app_seeds/test/data/species_repository_test.dart
View file

@ -40,12 +40,14 @@ void main() {
'parseSpeciesCatalog reads scientific name, family and common names',
() {
const json = '''
{"version":1,"species":[
{"scientific_name":"Zea mays","family":"Poaceae","common":{"en":["Maize"],"es":["Maíz"]}}
{"version":3,"species":[
{"scientific_name":"Zea mays","family":"Poaceae","wikidata_qid":"Q11575","gbif_key":2703077,"common":{"en":["Maize"],"es":["Maíz"]}}
]}''';
final seeds = parseSpeciesCatalog(json);
expect(seeds.single.scientificName, 'Zea mays');
expect(seeds.single.family, 'Poaceae');
expect(seeds.single.wikidataQid, 'Q11575');
expect(seeds.single.gbifKey, 2703077);
expect(seeds.single.commonNames['es'], ['Maíz']);
},
);
@ -140,4 +142,86 @@ void main() {
await repo.seedBundled(_seeds);
expect(await repo.search(' '), isEmpty);
});
test('seedBundled stores wikidata id and gbif key', () async {
await repo.seedBundled(const [
SpeciesSeed(
scientificName: 'Solanum lycopersicum',
family: 'Solanaceae',
wikidataQid: 'Q13223',
gbifKey: 2930137,
commonNames: {
'en': ['Tomato'],
},
),
]);
final row = await db.select(db.species).getSingle();
expect(row.wikidataQid, 'Q13223');
expect(row.gbifKey, 2930137);
});
test('seedBundled backfills reference fields added later', () async {
// First seed lacks the enrichment fields (an older catalog).
await repo.seedBundled(const [
SpeciesSeed(
scientificName: 'Allium cepa',
commonNames: {
'en': ['Onion'],
},
),
]);
// A later catalog carries them; identity is preserved, fields are filled.
await repo.seedBundled(const [
SpeciesSeed(
scientificName: 'Allium cepa',
wikidataQid: 'Q23485',
gbifKey: 2857697,
viabilityYears: 1,
commonNames: {
'en': ['Onion'],
},
),
]);
final rows = await db.select(db.species).get();
expect(rows.length, 1);
expect(rows.single.wikidataQid, 'Q23485');
expect(rows.single.gbifKey, 2857697);
expect(rows.single.viabilityYears, 1);
});
test('search matches a non-Latin common name', () async {
await repo.seedBundled(const [
SpeciesSeed(
scientificName: 'Solanum lycopersicum',
commonNames: {
'ar': ['طماطم'],
'zh': ['番茄'],
},
),
]);
final ar = await repo.search('طماطم', languageCode: 'ar');
expect(ar.single.scientificName, 'Solanum lycopersicum');
expect(ar.single.commonName, 'طماطم');
final zh = await repo.search('番茄', languageCode: 'zh');
expect(zh.single.commonName, '番茄');
});
test('search respects the limit', () async {
final many = [
for (var i = 0; i < 20; i++)
SpeciesSeed(
scientificName: 'Genus species$i',
commonNames: const {
'en': ['Plant'],
},
),
];
await repo.seedBundled(many);
final results = await repo.search('Genus', limit: 5);
expect(results.length, 5);
});
}

159
apps/app_seeds/test/tool/gen_species_catalog_test.dart Normal file
View file

@ -0,0 +1,159 @@
import 'package:flutter_test/flutter_test.dart';
import '../../tool/gen_species_catalog.dart';
void main() {
group('isVernacularName', () {
test('accepts genuine common names', () {
expect(isVernacularName('Tomate', 'solanum lycopersicum'), isTrue);
expect(isVernacularName("Queen Anne's lace", 'daucus carota'), isTrue);
expect(isVernacularName('番茄', 'solanum lycopersicum'), isTrue);
});
test('rejects the scientific name itself', () {
expect(
isVernacularName('Solanum lycopersicum', 'solanum lycopersicum'),
isFalse,
);
});
test('rejects scientific noise: commas, parentheses, ranks, initials', () {
expect(isVernacularName('Remusatia formosana , Hayata', 'x'), isFalse);
expect(isVernacularName('滇常山（原变种）', 'x'), isFalse);
expect(isVernacularName('Brassica oleracea var. acephala', 'x'), isFalse);
expect(isVernacularName('Z. mays', 'x'), isFalse);
expect(isVernacularName('Allium cepa L.', 'x'), isFalse);
expect(isVernacularName('Prunus × domestica', 'x'), isFalse);
});
});
group('buildCatalog', () {
test('emits version 3 and sorts by scientific name', () {
final catalog = buildCatalog([
SpeciesRecord(
qid: 'Q2',
scientificName: 'Zea mays',
common: {
'en': {'Maize'},
},
),
SpeciesRecord(
qid: 'Q1',
scientificName: 'Allium cepa',
common: {
'en': {'Onion'},
},
),
], const {});
expect(catalog['version'], 3);
expect((catalog['species'] as List).map((e) => e['scientific_name']), [
'Allium cepa',
'Zea mays',
]);
});
test('dedups by scientific name, merging names and backfilling ids', () {
final catalog = buildCatalog([
SpeciesRecord(
qid: 'Q13223',
scientificName: 'Solanum lycopersicum',
family: 'Solanaceae',
common: {
'es': {'tomate'},
},
),
SpeciesRecord(
qid: 'Q-dupe',
scientificName: 'Solanum lycopersicum',
gbifKey: 2930137,
common: {
'en': {'tomato'},
'es': {'tomatera'},
},
),
], const {});
final entry = (catalog['species'] as List).single as Map;
expect(entry['wikidata_qid'], 'Q13223'); // first entity kept
expect(entry['family'], 'Solanaceae');
expect(entry['gbif_key'], 2930137); // backfilled from the duplicate
expect(entry['common']['es'], containsAll(['tomate', 'tomatera']));
expect(entry['common']['en'], ['tomato']);
});
test('drops names that are not real vernaculars', () {
final catalog = buildCatalog([
SpeciesRecord(
qid: 'Q1',
scientificName: 'Solanum lycopersicum',
common: {
'es': {'Solanum lycopersicum', 'Tomate'},
},
),
], const {});
expect((catalog['species'] as List).single['common']['es'], ['Tomate']);
});
test('minLanguages is a relevance floor for Wikidata-only species', () {
final records = [
SpeciesRecord(
qid: 'Q1',
scientificName: 'Obscura ignota',
common: {
'zh': {'仅中文'},
},
),
SpeciesRecord(
qid: 'Q2',
scientificName: 'Nota bene',
common: {
'en': {'well known'},
'es': {'bien conocida'},
},
),
];
final kept =
(buildCatalog(records, const {}, minLanguages: 2)['species'] as List)
.map((e) => e['scientific_name'])
.toList();
expect(kept, ['Nota bene']); // the 1-language species is filtered out
});
test('curated overrides win, come first, and bypass the floor', () {
final catalog = buildCatalog(
[
SpeciesRecord(
qid: 'Q081',
scientificName: 'Daucus carota',
family: 'Apiaceae',
common: {
// Wikidata's messy taxon labels for the wild carrot.
'en': {'Devil’s Plague', 'wild carrot'},
},
),
],
const {
'Daucus carota': CuratedOverride(
family: 'Apiaceae',
viabilityYears: 3,
common: {
'es': ['Zanahoria'],
'en': ['Carrot'],
},
),
},
// Would be filtered as it has only one Wikidata language, but curated
// species are always kept.
minLanguages: 4,
);
final entry = (catalog['species'] as List).single as Map;
expect(entry['viability_years'], 3);
expect(entry['common']['es'], ['Zanahoria']);
// Curated name first, then the Wikidata vernaculars.
expect(entry['common']['en'].first, 'Carrot');
expect(entry['common']['en'], contains('wild carrot'));
});
});
}

56
apps/app_seeds/tool/README.md Normal file
View file

@ -0,0 +1,56 @@
# Species catalog generator
Regenerates [`assets/catalog/species.json`](../assets/catalog/species.json) — the
bundled, offline, multilingual catalog of edible / cultivated plant species the
app seeds into the encrypted database at first run.
```sh
cd apps/app_seeds
dart run tool/gen_species_catalog.dart # writes species.json (version 3)
dart run tool/gen_species_catalog.dart --dry # prints stats, writes nothing
```
The generated JSON is **committed** to the repo — the app never queries the
network for it (local-first, offline).
## Sources & licensing
- **Names** come only from **Wikidata** labels and aliases. Wikidata is **CC0**
(public domain), so the bundled data carries **no attribution burden** and is
clean for redistribution under the app's AGPL-3.0 license.
- **`gbif_key`** links the [GBIF](https://www.gbif.org/) taxonomic backbone. It
is used **only as an identifier** (a fact, not copyrightable text) — no GBIF
vernacular text is bundled, so no CC-BY attribution is triggered.
- **Curated core crops** (names, family and `viability_years`) live in
[`curated_overrides.json`](curated_overrides.json), hand-maintained. For a
listed species the curated common names **win** (placed first, never dropped)
and its family/viability override Wikidata; Wikidata still adds more languages.
A curated species is always kept even if it would miss the relevance floor.
- **`viability_years`** (seed longevity) is **not** in Wikidata — it comes only
from the curated overrides (public-domain agricultural-extension figures).
Species without an entry simply get no age-based expiry warning.
## What it does
1. **Phase 1 — species list (SPARQL):** taxa with a binomial name that are a
*crop* (`P279* Q235352`), the source/ingredient of a *food*
(`P1672`/`P366`/reverse `P1582` → `P279* Q2095`), with scientific name
(`P225`) and GBIF key (`P846`).
2. **Phase 2 — names + family:** `rdfs:label` (primary) + `skos:altLabel`
(aliases), batched over the species and filtered to the bundled languages;
the family is walked up `P171*` to the family-rank ancestor.
3. **Transform (`buildCatalog`, pure & unit-tested):** dedup by scientific name;
drop non-vernacular noise (the binomial itself, author citations, `var.`/`×`
ranks, abbreviated initials); merge the curated overrides (authoritative,
first); require a real vernacular in **≥ 4 languages** as a relevance floor so
the long tail of obscure single-label taxa is dropped; sort; serialize.
The current run yields **~1200 species** (`assets/catalog/species.json` ≈ 1.5 MB).
## Bundled languages
`es, en, fr, de, it, pt, ca, gl, eu, ar, zh, ja, ru` — a multilingual **seed**,
not a ceiling (Latin + Arabic RTL + CJK + Cyrillic). Edit `kCatalogLanguages` in
[`gen_species_catalog.dart`](gen_species_catalog.dart) to widen coverage, then
regenerate. A species with no name in any bundled language is dropped (add its
language rather than assuming es/en).

19
apps/app_seeds/tool/curated_overrides.json Normal file
View file

@ -0,0 +1,19 @@
{
"note": "Hand-curated, authoritative data for core crops, merged into the generated catalog by gen_species_catalog.dart. For a listed species the curated common names win (placed first, never dropped by the vernacular filter), and its family/viability_years override Wikidata; Wikidata still enriches with additional languages. A curated species is always kept even if it would miss the relevance floor. `viability_years` is public-domain agricultural-extension seed longevity (Wikidata has none); a conservative single value used to warn about aging lots. Extend this list to pin the names of species that matter to your community.",
"species": {
"Solanum lycopersicum": { "family": "Solanaceae", "viability_years": 5, "common": { "es": ["Tomate"], "en": ["Tomato"] } },
"Solanum melongena": { "family": "Solanaceae", "viability_years": 5, "common": { "es": ["Berenjena"], "en": ["Aubergine", "Eggplant"] } },
"Capsicum annuum": { "family": "Solanaceae", "viability_years": 3, "common": { "es": ["Pimiento", "Guindilla"], "en": ["Pepper", "Chilli"] } },
"Phaseolus vulgaris": { "family": "Fabaceae", "viability_years": 3, "common": { "es": ["Judía", "Alubia", "Habichuela"], "en": ["Common bean"] } },
"Vicia faba": { "family": "Fabaceae", "viability_years": 4, "common": { "es": ["Haba"], "en": ["Broad bean", "Fava bean"] } },
"Pisum sativum": { "family": "Fabaceae", "viability_years": 3, "common": { "es": ["Guisante"], "en": ["Pea"] } },
"Zea mays": { "family": "Poaceae", "viability_years": 2, "common": { "es": ["Maíz"], "en": ["Maize", "Corn"] } },
"Cucurbita pepo": { "family": "Cucurbitaceae", "viability_years": 5, "common": { "es": ["Calabacín", "Calabaza"], "en": ["Courgette", "Zucchini", "Squash"] } },
"Cucumis sativus": { "family": "Cucurbitaceae", "viability_years": 5, "common": { "es": ["Pepino"], "en": ["Cucumber"] } },
"Lactuca sativa": { "family": "Asteraceae", "viability_years": 3, "common": { "es": ["Lechuga"], "en": ["Lettuce"] } },
"Beta vulgaris": { "family": "Amaranthaceae", "viability_years": 4, "common": { "es": ["Acelga", "Remolacha"], "en": ["Chard", "Beetroot"] } },
"Allium cepa": { "family": "Amaryllidaceae", "viability_years": 1, "common": { "es": ["Cebolla"], "en": ["Onion"] } },
"Allium sativum": { "family": "Amaryllidaceae", "viability_years": 1, "common": { "es": ["Ajo"], "en": ["Garlic"] } },
"Daucus carota": { "family": "Apiaceae", "viability_years": 3, "common": { "es": ["Zanahoria"], "en": ["Carrot"] } }
}
}

368
apps/app_seeds/tool/gen_species_catalog.dart Normal file
View file

@ -0,0 +1,368 @@
// Regenerates the bundled species catalog (`assets/catalog/species.json`) from
// Wikidata (CC0). Dev-only tool — it is NOT shipped in the app; the app only
// reads the committed JSON it produces (local-first, offline).
//
// dart run tool/gen_species_catalog.dart # writes species.json (v3)
// dart run tool/gen_species_catalog.dart --dry # prints stats, writes nothing
//
// Names come only from Wikidata labels/aliases (CC0 → no attribution burden).
// GBIF is used solely as an identifier (`gbif_key`, a fact, not copyrightable).
// Seed-longevity (`viability_years`) and authoritative core-crop names are
// merged from the hand-maintained `tool/curated_overrides.json`. See
// `tool/README.md`.
//
// The pure transform (`buildCatalog`) is separated from the network fetch so it
// is unit-testable without hitting the endpoint.
import 'dart:convert';
import 'dart:io';
/// Locale set bundled at launch — a multilingual seed, not a ceiling. Covers
/// Latin, Arabic (RTL), CJK (Chinese + Japanese) and Cyrillic scripts. Order
/// here is the order common names are written in the JSON.
const List<String> kCatalogLanguages = [
'es',
'en',
'fr',
'de',
'it',
'pt',
'ca',
'gl',
'eu',
'ar',
'zh',
'ja',
'ru',
];
const String _endpoint = 'https://query.wikidata.org/sparql';
/// Wikimedia asks bots to send a descriptive, contactable User-Agent.
const String _userAgent =
'TanemakiSpeciesCatalogBot/1.0 (https://tanemaki.app; vjrj@comunes.org)';
const String _catalogNote =
'Edible / cultivated plant species, generated from Wikidata (CC0) by '
'tool/gen_species_catalog.dart — see tool/README.md to regenerate. `common` '
'is a locale-keyed map (Wikidata labels + aliases) the app resolves by the '
"user's locale with graceful fallback, never assuming es/en only. "
'`gbif_key` links the GBIF taxonomic backbone (identifier only). '
'`viability_years` is public-domain agricultural-extension seed longevity '
'merged from tool/curated_overrides.json (not in Wikidata); a conservative '
'single value used to warn about aging lots.';
/// Hand-curated, authoritative data for a core crop (from
/// `curated_overrides.json`). Its names win over Wikidata's; its family and
/// viability override; a curated species is always kept.
class CuratedOverride {
const CuratedOverride({
this.family,
this.viabilityYears,
this.common = const {},
});
final String? family;
final int? viabilityYears;
final Map<String, List<String>> common;
}
/// One aggregated species before it is serialized. Mutable during aggregation.
class SpeciesRecord {
SpeciesRecord({
required this.qid,
required this.scientificName,
this.family,
this.gbifKey,
Map<String, Set<String>>? common,
}) : common = common ?? {};
final String qid;
final String scientificName;
String? family;
int? gbifKey;
/// language code → set of names (dedup, insertion-ordered).
final Map<String, Set<String>> common;
}
/// Whether [name] reads as a genuine vernacular rather than a scientific name,
/// synonym or author citation. Wikidata aliases are full of the latter (commas,
/// parentheses, hybrid `×`, `var.`/`subsp.` ranks, or a straight repeat of the
/// binomial); those are noise in a common-name list.
bool isVernacularName(String name, String scientificNameLower) {
final n = name.trim();
if (n.isEmpty) return false;
if (n.toLowerCase() == scientificNameLower) return false;
// Commas/parentheses (ASCII and CJK full-width) and the hybrid sign mark
// author citations, synonym lists and hybrid formulas — never a vernacular.
if (n.contains(RegExp(r'[,()（），×]'))) return false;
if (n.contains(RegExp(r'\b(var|subsp|ssp|f|cv|nothosubsp)\.'))) return false;
// A lone initial + period is an abbreviated genus or author citation
// ("Z. mays", "Allium cepa L."), not a vernacular.
if (n.contains(RegExp(r'\b[A-Z]\.'))) return false;
return true;
}
/// Pure transform: dedup by scientific name, keep only [languages], drop
/// non-vernacular noise, require a name in at least [minLanguages] languages
/// (a relevance floor — obscure taxa carry a vernacular in only one), merge the
/// hand-curated [curated] overrides (authoritative names/family/viability that
/// win over Wikidata and are always kept), and emit an ordered, JSON-ready
/// catalog map (`{version, note, species: [...]}`). No network, no I/O.
Map<String, dynamic> buildCatalog(
Iterable<SpeciesRecord> records,
Map<String, CuratedOverride> curated, {
List<String> languages = kCatalogLanguages,
int minLanguages = 1,
}) {
// Dedup by scientific name; merge names and backfill family/gbif across the
// duplicate entities Wikidata sometimes has for one accepted binomial.
final byName = <String, SpeciesRecord>{};
for (final r in records) {
final name = r.scientificName.trim();
if (name.isEmpty) continue;
final existing = byName[name];
if (existing == null) {
byName[name] = r;
continue;
}
existing.family ??= r.family;
existing.gbifKey ??= r.gbifKey;
for (final entry in r.common.entries) {
existing.common.putIfAbsent(entry.key, () => {}).addAll(entry.value);
}
}
final species = <Map<String, dynamic>>[];
for (final r in byName.values) {
final sciLower = r.scientificName.trim().toLowerCase();
final override = curated[r.scientificName];
final common = <String, List<String>>{};
for (final lang in languages) {
final ordered = <String>[];
final seen = <String>{};
// Curated names first (authoritative, kept as-is), then Wikidata's
// vernaculars, de-duplicated case-insensitively.
for (final n in override?.common[lang] ?? const <String>[]) {
if (n.trim().isNotEmpty && seen.add(n.toLowerCase())) ordered.add(n);
}
for (final n in r.common[lang] ?? const <String>{}) {
final v = n.trim();
if (!isVernacularName(v, sciLower)) continue;
if (seen.add(v.toLowerCase())) ordered.add(v);
}
if (ordered.isNotEmpty) common[lang] = ordered;
}
// A relevance floor for Wikidata-only species: obscure taxa carry a real
// vernacular in only one language (a lone CJK/Cyrillic label), well-known
// crops in many. Curated species bypass the floor.
if (override == null && common.length < minLanguages) continue;
if (common.isEmpty) continue;
final family = override?.family ?? r.family;
final viability = override?.viabilityYears;
species.add({
'scientific_name': r.scientificName,
'family': ?family,
'wikidata_qid': r.qid,
if (r.gbifKey != null) 'gbif_key': r.gbifKey,
'common': common,
'viability_years': ?viability,
});
}
species.sort(
(a, b) => (a['scientific_name'] as String).compareTo(
b['scientific_name'] as String,
),
);
return {'version': 3, 'note': _catalogNote, 'species': species};
}
Future<void> main(List<String> args) async {
final dryRun = args.contains('--dry');
final toolDir = File(Platform.script.toFilePath()).parent;
final overridesFile = File('${toolDir.path}/curated_overrides.json');
final outFile = File('${toolDir.path}/../assets/catalog/species.json');
final curated = _readCuratedOverrides(overridesFile);
stdout.writeln('Loaded ${curated.length} curated overrides.');
final client = HttpClient()..userAgent = _userAgent;
try {
stdout.writeln('Querying Wikidata for edible/cultivated species…');
final skeleton = await _fetchSpeciesSkeleton(client);
stdout.writeln(
'Found ${skeleton.length} candidate species. Fetching '
'names in ${kCatalogLanguages.length} languages…',
);
await _fetchLabels(client, skeleton);
final catalog = buildCatalog(skeleton.values, curated, minLanguages: 4);
final count = (catalog['species'] as List).length;
stdout.writeln('Built catalog: $count species with bundled-locale names.');
if (dryRun) {
stdout.writeln('--dry: not writing ${outFile.path}');
return;
}
const encoder = JsonEncoder.withIndent(' ');
await outFile.writeAsString('${encoder.convert(catalog)}\n');
stdout.writeln('Wrote ${outFile.path}');
} finally {
client.close();
}
}
Map<String, CuratedOverride> _readCuratedOverrides(File file) {
if (!file.existsSync()) return {};
final data = jsonDecode(file.readAsStringSync()) as Map<String, dynamic>;
final species = (data['species'] as Map<String, dynamic>);
return species.map((name, raw) {
final o = raw as Map<String, dynamic>;
final common = (o['common'] as Map<String, dynamic>? ?? const {}).map(
(lang, names) => MapEntry(lang, (names as List).cast<String>()),
);
return MapEntry(
name,
CuratedOverride(
family: o['family'] as String?,
viabilityYears: (o['viability_years'] as num?)?.toInt(),
common: common,
),
);
});
}
/// Phase 1: the list of species QIDs with scientific name, family and GBIF key.
Future<Map<String, SpeciesRecord>> _fetchSpeciesSkeleton(
HttpClient client,
) async {
// Taxa with a binomial (or lower) scientific name — CONTAINS(" ") keeps
// species and botanical varieties (relevant for seed savers: e.g. the many
// Brassica oleracea var. cultivars) while dropping genus-level rows — that are
// edible/cultivated, reached three ways: a crop, the source of a food, or the
// source of a food ingredient. Taxon rank is intentionally not required: many
// cultivated taxa in Wikidata leave P105 unset.
const query = r'''
SELECT ?item ?sci ?gbif WHERE {
?item wdt:P225 ?sci .
FILTER(CONTAINS(?sci, " "))
{
?item wdt:P279* wd:Q235352 . # a crop
} UNION {
?item wdt:P1672 ?use . # this taxon is source of …
?use wdt:P279* wd:Q2095 . # … a food
} UNION {
?item wdt:P1672 ?use .
?use wdt:P279* wd:Q25403900 . # … a food ingredient
} UNION {
?food wdt:P1582 ?item . # … the natural source of a food
?food wdt:P279* wd:Q2095 .
} UNION {
?item wdt:P366 ?use . # this taxon has use …
?use wdt:P279* wd:Q2095 . # … food
}
OPTIONAL { ?item wdt:P846 ?gbif . }
}''';
final rows = await _runSparql(client, query);
final out = <String, SpeciesRecord>{};
for (final row in rows) {
final qid = _qid(row['item']?['value'] as String?);
final sci = row['sci']?['value'] as String?;
if (qid == null || sci == null) continue;
final rec = out.putIfAbsent(
qid,
() => SpeciesRecord(qid: qid, scientificName: sci),
);
final gbif = row['gbif']?['value'] as String?;
if (gbif != null) rec.gbifKey ??= int.tryParse(gbif);
}
return out;
}
/// Phase 2: labels + aliases (and the family, walked up `P171*`) in the bundled
/// languages, batched over QIDs to keep each query under the endpoint timeout.
Future<void> _fetchLabels(
HttpClient client,
Map<String, SpeciesRecord> skeleton,
) async {
final langFilter = kCatalogLanguages.map((l) => '"$l"').join(', ');
final qids = skeleton.keys.toList();
const batchSize = 100;
for (var i = 0; i < qids.length; i += batchSize) {
final batch = qids.skip(i).take(batchSize);
final values = batch.map((q) => 'wd:$q').join(' ');
// `kind` orders the primary label ("a", rdfs:label — usually the real
// vernacular) ahead of aliases ("b", skos:altLabel — often synonyms).
final query =
'''
SELECT ?item ?label ?kind ?family WHERE {
VALUES ?item { $values }
OPTIONAL {
?item wdt:P171* ?fam . # walk up to the family-rank ancestor
?fam wdt:P105 wd:Q35409 ;
wdt:P225 ?family .
}
{ ?item rdfs:label ?label . BIND("a" AS ?kind) }
UNION
{ ?item skos:altLabel ?label . BIND("b" AS ?kind) }
FILTER(LANG(?label) IN ($langFilter))
}''';
final rows = await _runSparql(client, query);
// Primary labels first so they land first in each language's ordered set.
rows.sort(
(a, b) => (a['kind']?['value'] as String? ?? 'b').compareTo(
b['kind']?['value'] as String? ?? 'b',
),
);
for (final row in rows) {
final qid = _qid(row['item']?['value'] as String?);
final label = row['label'];
if (qid == null || label == null) continue;
final rec = skeleton[qid];
if (rec == null) continue;
rec.family ??= row['family']?['value'] as String?;
final lang = label['xml:lang'] as String?;
final value = label['value'] as String?;
if (lang == null || value == null) continue;
rec.common.putIfAbsent(lang, () => {}).add(value);
}
stdout.writeln(
' labels: ${(i + batchSize).clamp(0, qids.length)}'
'/${qids.length}',
);
}
}
/// Runs a SPARQL SELECT and returns the `results.bindings` list.
Future<List<Map<String, dynamic>>> _runSparql(
HttpClient client,
String query,
) async {
final uri = Uri.parse(
_endpoint,
).replace(queryParameters: {'query': query, 'format': 'json'});
final request = await client.getUrl(uri);
request.headers.set(
HttpHeaders.acceptHeader,
'application/sparql-results+json',
);
final response = await request.close();
if (response.statusCode != 200) {
final body = await response.transform(utf8.decoder).join();
throw HttpException('SPARQL ${response.statusCode}: $body');
}
final body = await response.transform(utf8.decoder).join();
final decoded = jsonDecode(body) as Map<String, dynamic>;
final bindings = (decoded['results']?['bindings'] as List?) ?? const [];
return bindings.cast<Map<String, dynamic>>();
}
/// Extracts the `Qxxxx` id from a Wikidata entity URI.
String? _qid(String? uri) {
if (uri == null) return null;
final i = uri.lastIndexOf('/');
return i == -1 ? uri : uri.substring(i + 1);
}