tane/apps/app_seeds/tool/gen_species_catalog.dart

// Regenerates the bundled species catalog (`assets/catalog/species.json`) from
// Wikidata (CC0). Dev-only tool — it is NOT shipped in the app; the app only
// reads the committed JSON it produces (local-first, offline).
//
//   dart run tool/gen_species_catalog.dart          # writes species.json (v3)
//   dart run tool/gen_species_catalog.dart --dry     # prints stats, writes nothing
//
// Names come only from Wikidata labels/aliases (CC0 → no attribution burden).
// GBIF is used solely as an identifier (`gbif_key`, a fact, not copyrightable).
// Seed-longevity (`viability_years`) and authoritative core-crop names are
// merged from the hand-maintained `tool/curated_overrides.json`. See
// `tool/README.md`.
//
// The pure transform (`buildCatalog`) is separated from the network fetch so it
// is unit-testable without hitting the endpoint.
import 'dart:convert';
import 'dart:io';

/// Locale set bundled at launch — a multilingual seed, not a ceiling. Covers
/// Latin, Arabic (RTL), CJK (Chinese + Japanese) and Cyrillic scripts. Order
/// here is the order common names are written in the JSON.
const List<String> kCatalogLanguages = [
  'es',
  'en',
  'fr',
  'de',
  'it',
  'pt',
  'ca',
  'gl',
  'eu',
  'ar',
  'zh',
  'ja',
  'ru',
];

const String _endpoint = 'https://query.wikidata.org/sparql';

/// Wikimedia asks bots to send a descriptive, contactable User-Agent.
const String _userAgent =
    'TaneSpeciesCatalogBot/1.0 (https://tane.comunes.org; vjrj@comunes.org)';

const String _catalogNote =
    'Edible / cultivated plant species, generated from Wikidata (CC0) by '
    'tool/gen_species_catalog.dart — see tool/README.md to regenerate. `common` '
    'is a locale-keyed map (Wikidata labels + aliases) the app resolves by the '
    "user's locale with graceful fallback, never assuming es/en only. "
    '`gbif_key` links the GBIF taxonomic backbone (identifier only). '
    '`viability_years` is public-domain agricultural-extension seed longevity '
    'merged from tool/curated_overrides.json (not in Wikidata); a conservative '
    'single value used to warn about aging lots.';

/// Hand-curated, authoritative data for a core crop (from
/// `curated_overrides.json`). Its names win over Wikidata's; its family and
/// viability override; a curated species is always kept.
class CuratedOverride {
  const CuratedOverride({
    this.family,
    this.viabilityYears,
    this.common = const {},
  });

  final String? family;
  final int? viabilityYears;
  final Map<String, List<String>> common;
}

/// One aggregated species before it is serialized. Mutable during aggregation.
class SpeciesRecord {
  SpeciesRecord({
    required this.qid,
    required this.scientificName,
    this.family,
    this.gbifKey,
    Map<String, Set<String>>? common,
  }) : common = common ?? {};

  final String qid;
  final String scientificName;
  String? family;
  int? gbifKey;

  /// language code → set of names (dedup, insertion-ordered).
  final Map<String, Set<String>> common;
}

/// Whether [name] reads as a genuine vernacular rather than a scientific name,
/// synonym or author citation. Wikidata aliases are full of the latter (commas,
/// parentheses, hybrid `×`, `var.`/`subsp.` ranks, or a straight repeat of the
/// binomial); those are noise in a common-name list.
bool isVernacularName(String name, String scientificNameLower) {
  final n = name.trim();
  if (n.isEmpty) return false;
  if (n.toLowerCase() == scientificNameLower) return false;
  // Commas/parentheses (ASCII and CJK full-width) and the hybrid sign mark
  // author citations, synonym lists and hybrid formulas — never a vernacular.
  if (n.contains(RegExp(r'[,()（），×]'))) return false;
  if (n.contains(RegExp(r'\b(var|subsp|ssp|f|cv|nothosubsp)\.'))) return false;
  // A lone initial + period is an abbreviated genus or author citation
  // ("Z. mays", "Allium cepa L."), not a vernacular.
  if (n.contains(RegExp(r'\b[A-Z]\.'))) return false;
  return true;
}

/// Pure transform: dedup by scientific name, keep only [languages], drop
/// non-vernacular noise, require a name in at least [minLanguages] languages
/// (a relevance floor — obscure taxa carry a vernacular in only one), merge the
/// hand-curated [curated] overrides (authoritative names/family/viability that
/// win over Wikidata and are always kept), and emit an ordered, JSON-ready
/// catalog map (`{version, note, species: [...]}`). No network, no I/O.
Map<String, dynamic> buildCatalog(
  Iterable<SpeciesRecord> records,
  Map<String, CuratedOverride> curated, {
  List<String> languages = kCatalogLanguages,
  int minLanguages = 1,
}) {
  // Dedup by scientific name; merge names and backfill family/gbif across the
  // duplicate entities Wikidata sometimes has for one accepted binomial.
  final byName = <String, SpeciesRecord>{};
  for (final r in records) {
    final name = r.scientificName.trim();
    if (name.isEmpty) continue;
    final existing = byName[name];
    if (existing == null) {
      byName[name] = r;
      continue;
    }
    existing.family ??= r.family;
    existing.gbifKey ??= r.gbifKey;
    for (final entry in r.common.entries) {
      existing.common.putIfAbsent(entry.key, () => {}).addAll(entry.value);
    }
  }

  final species = <Map<String, dynamic>>[];
  for (final r in byName.values) {
    final sciLower = r.scientificName.trim().toLowerCase();
    final override = curated[r.scientificName];
    final common = <String, List<String>>{};
    for (final lang in languages) {
      final ordered = <String>[];
      final seen = <String>{};
      // Curated names first (authoritative, kept as-is), then Wikidata's
      // vernaculars, de-duplicated case-insensitively.
      for (final n in override?.common[lang] ?? const <String>[]) {
        if (n.trim().isNotEmpty && seen.add(n.toLowerCase())) ordered.add(n);
      }
      for (final n in r.common[lang] ?? const <String>{}) {
        final v = n.trim();
        if (!isVernacularName(v, sciLower)) continue;
        if (seen.add(v.toLowerCase())) ordered.add(v);
      }
      if (ordered.isNotEmpty) common[lang] = ordered;
    }
    // A relevance floor for Wikidata-only species: obscure taxa carry a real
    // vernacular in only one language (a lone CJK/Cyrillic label), well-known
    // crops in many. Curated species bypass the floor.
    if (override == null && common.length < minLanguages) continue;
    if (common.isEmpty) continue;

    final family = override?.family ?? r.family;
    final viability = override?.viabilityYears;
    species.add({
      'scientific_name': r.scientificName,
      'family': ?family,
      'wikidata_qid': r.qid,
      if (r.gbifKey != null) 'gbif_key': r.gbifKey,
      'common': common,
      'viability_years': ?viability,
    });
  }

  species.sort(
    (a, b) => (a['scientific_name'] as String).compareTo(
      b['scientific_name'] as String,
    ),
  );

  return {'version': 3, 'note': _catalogNote, 'species': species};
}

Future<void> main(List<String> args) async {
  final dryRun = args.contains('--dry');
  final toolDir = File(Platform.script.toFilePath()).parent;
  final overridesFile = File('${toolDir.path}/curated_overrides.json');
  final outFile = File('${toolDir.path}/../assets/catalog/species.json');

  final curated = _readCuratedOverrides(overridesFile);
  stdout.writeln('Loaded ${curated.length} curated overrides.');

  final client = HttpClient()..userAgent = _userAgent;
  try {
    stdout.writeln('Querying Wikidata for edible/cultivated species…');
    final skeleton = await _fetchSpeciesSkeleton(client);
    stdout.writeln(
      'Found ${skeleton.length} candidate species. Fetching '
      'names in ${kCatalogLanguages.length} languages…',
    );
    await _fetchLabels(client, skeleton);

    final catalog = buildCatalog(skeleton.values, curated, minLanguages: 4);
    final count = (catalog['species'] as List).length;
    stdout.writeln('Built catalog: $count species with bundled-locale names.');

    if (dryRun) {
      stdout.writeln('--dry: not writing ${outFile.path}');
      return;
    }
    const encoder = JsonEncoder.withIndent('  ');
    await outFile.writeAsString('${encoder.convert(catalog)}\n');
    stdout.writeln('Wrote ${outFile.path}');
  } finally {
    client.close();
  }
}

Map<String, CuratedOverride> _readCuratedOverrides(File file) {
  if (!file.existsSync()) return {};
  final data = jsonDecode(file.readAsStringSync()) as Map<String, dynamic>;
  final species = (data['species'] as Map<String, dynamic>);
  return species.map((name, raw) {
    final o = raw as Map<String, dynamic>;
    final common = (o['common'] as Map<String, dynamic>? ?? const {}).map(
      (lang, names) => MapEntry(lang, (names as List).cast<String>()),
    );
    return MapEntry(
      name,
      CuratedOverride(
        family: o['family'] as String?,
        viabilityYears: (o['viability_years'] as num?)?.toInt(),
        common: common,
      ),
    );
  });
}

/// Phase 1: the list of species QIDs with scientific name, family and GBIF key.
Future<Map<String, SpeciesRecord>> _fetchSpeciesSkeleton(
  HttpClient client,
) async {
  // Taxa with a binomial (or lower) scientific name — CONTAINS(" ") keeps
  // species and botanical varieties (relevant for seed savers: e.g. the many
  // Brassica oleracea var. cultivars) while dropping genus-level rows — that are
  // edible/cultivated, reached three ways: a crop, the source of a food, or the
  // source of a food ingredient. Taxon rank is intentionally not required: many
  // cultivated taxa in Wikidata leave P105 unset.
  const query = r'''
SELECT ?item ?sci ?gbif WHERE {
  ?item wdt:P225 ?sci .
  FILTER(CONTAINS(?sci, " "))
  {
    ?item wdt:P279* wd:Q235352 .       # a crop
  } UNION {
    ?item wdt:P1672 ?use .             # this taxon is source of …
    ?use wdt:P279* wd:Q2095 .          # … a food
  } UNION {
    ?item wdt:P1672 ?use .
    ?use wdt:P279* wd:Q25403900 .      # … a food ingredient
  } UNION {
    ?food wdt:P1582 ?item .           # … the natural source of a food
    ?food wdt:P279* wd:Q2095 .
  } UNION {
    ?item wdt:P366 ?use .             # this taxon has use …
    ?use wdt:P279* wd:Q2095 .         # … food
  }
  OPTIONAL { ?item wdt:P846 ?gbif . }
}''';
  final rows = await _runSparql(client, query);
  final out = <String, SpeciesRecord>{};
  for (final row in rows) {
    final qid = _qid(row['item']?['value'] as String?);
    final sci = row['sci']?['value'] as String?;
    if (qid == null || sci == null) continue;
    final rec = out.putIfAbsent(
      qid,
      () => SpeciesRecord(qid: qid, scientificName: sci),
    );
    final gbif = row['gbif']?['value'] as String?;
    if (gbif != null) rec.gbifKey ??= int.tryParse(gbif);
  }
  return out;
}

/// Phase 2: labels + aliases (and the family, walked up `P171*`) in the bundled
/// languages, batched over QIDs to keep each query under the endpoint timeout.
Future<void> _fetchLabels(
  HttpClient client,
  Map<String, SpeciesRecord> skeleton,
) async {
  final langFilter = kCatalogLanguages.map((l) => '"$l"').join(', ');
  final qids = skeleton.keys.toList();
  const batchSize = 100;
  for (var i = 0; i < qids.length; i += batchSize) {
    final batch = qids.skip(i).take(batchSize);
    final values = batch.map((q) => 'wd:$q').join(' ');
    // `kind` orders the primary label ("a", rdfs:label — usually the real
    // vernacular) ahead of aliases ("b", skos:altLabel — often synonyms).
    final query =
        '''
SELECT ?item ?label ?kind ?family WHERE {
  VALUES ?item { $values }
  OPTIONAL {
    ?item wdt:P171* ?fam .              # walk up to the family-rank ancestor
    ?fam wdt:P105 wd:Q35409 ;
         wdt:P225 ?family .
  }
  { ?item rdfs:label ?label . BIND("a" AS ?kind) }
  UNION
  { ?item skos:altLabel ?label . BIND("b" AS ?kind) }
  FILTER(LANG(?label) IN ($langFilter))
}''';
    final rows = await _runSparql(client, query);
    // Primary labels first so they land first in each language's ordered set.
    rows.sort(
      (a, b) => (a['kind']?['value'] as String? ?? 'b').compareTo(
        b['kind']?['value'] as String? ?? 'b',
      ),
    );
    for (final row in rows) {
      final qid = _qid(row['item']?['value'] as String?);
      final label = row['label'];
      if (qid == null || label == null) continue;
      final rec = skeleton[qid];
      if (rec == null) continue;
      rec.family ??= row['family']?['value'] as String?;
      final lang = label['xml:lang'] as String?;
      final value = label['value'] as String?;
      if (lang == null || value == null) continue;
      rec.common.putIfAbsent(lang, () => {}).add(value);
    }
    stdout.writeln(
      '  labels: ${(i + batchSize).clamp(0, qids.length)}'
      '/${qids.length}',
    );
  }
}

/// Runs a SPARQL SELECT and returns the `results.bindings` list.
Future<List<Map<String, dynamic>>> _runSparql(
  HttpClient client,
  String query,
) async {
  final uri = Uri.parse(
    _endpoint,
  ).replace(queryParameters: {'query': query, 'format': 'json'});
  final request = await client.getUrl(uri);
  request.headers.set(
    HttpHeaders.acceptHeader,
    'application/sparql-results+json',
  );
  final response = await request.close();
  if (response.statusCode != 200) {
    final body = await response.transform(utf8.decoder).join();
    throw HttpException('SPARQL ${response.statusCode}: $body');
  }
  final body = await response.transform(utf8.decoder).join();
  final decoded = jsonDecode(body) as Map<String, dynamic>;
  final bindings = (decoded['results']?['bindings'] as List?) ?? const [];
  return bindings.cast<Map<String, dynamic>>();
}

/// Extracts the `Qxxxx` id from a Wikidata entity URI.
String? _qid(String? uri) {
  if (uri == null) return null;
  final i = uri.lastIndexOf('/');
  return i == -1 ? uri : uri.substring(i + 1);
}