feat(inventory): additive CSV import for bulk digitization
Import a spreadsheet list (the same 21-column schema the CSV export already produces) as new varieties/lots. Tolerant reader: columns matched by header name in any order, only variety_label required, unknown enums fall back to a safe default, rows sharing variety_id collapse into one variety with several lots. Species re-linked by scientific name. Additive by design (fresh ids + HLC), so it never overwrites existing rows; JSON stays the canonical id-reconciling import. Adds a hand-rolled RFC 4180 parser (no new dependency), an importCsv path in VarietyRepository/ExportImportService, an 'Import from CSV' tile in the backup section, and en/es strings. Note: i18n .g.dart also carries pre-existing About/intro strings from the working tree (regenerated by slang).
This commit is contained in:
parent
21072633b5
commit
89b61bc9e4
11 changed files with 1070 additions and 2 deletions
|
|
@ -1,3 +1,5 @@
|
|||
import 'package:commons_core/commons_core.dart' show QuantityKind;
|
||||
|
||||
import '../../db/database.dart';
|
||||
import '../../db/enums.dart';
|
||||
import 'inventory_snapshot.dart';
|
||||
|
|
@ -141,4 +143,338 @@ class InventoryCsvCodec {
|
|||
}
|
||||
return '"${cell.replaceAll('"', '""')}"';
|
||||
}
|
||||
|
||||
/// Parses a CSV back into an **additive** import (the canonical import format
|
||||
/// is JSON; this is a convenience for lists a collective already keeps in a
|
||||
/// spreadsheet). Reader tolerance per data-model §5.2:
|
||||
///
|
||||
/// - columns are matched **by header name** (any order); unknown columns are
|
||||
/// ignored, missing columns are treated as empty;
|
||||
/// - only `variety_label` is required — a row without it is skipped;
|
||||
/// - unknown enum values fall back to a safe default (`lot_type` → seed,
|
||||
/// `offer_status` → private) or are dropped (`quantity_kind`, `presentation`);
|
||||
/// - rows sharing a non-empty `variety_id` collapse into one variety holding
|
||||
/// several lots; rows without a `variety_id` each become their own variety.
|
||||
///
|
||||
/// Sync metadata (`variety_id`/`lot_id`/timestamps) and the derived
|
||||
/// `latest_germination_rate` are **not** carried over: the repository stamps
|
||||
/// fresh ids and HLCs so an import never overwrites existing rows.
|
||||
CsvImport decode(String source) {
|
||||
final table = _parse(source);
|
||||
if (table.length < 2) return const CsvImport(varieties: [], skippedRows: 0);
|
||||
|
||||
final header = table.first;
|
||||
final index = <String, int>{};
|
||||
for (var i = 0; i < header.length; i++) {
|
||||
index[header[i].trim()] = i;
|
||||
}
|
||||
String? cell(List<String> row, String name) {
|
||||
final i = index[name];
|
||||
if (i == null || i >= row.length) return null;
|
||||
final value = row[i].trim();
|
||||
return value.isEmpty ? null : value;
|
||||
}
|
||||
|
||||
final byGroup = <String, _CsvVarietyBuilder>{};
|
||||
final builders = <_CsvVarietyBuilder>[];
|
||||
var skipped = 0;
|
||||
|
||||
for (final row in table.skip(1)) {
|
||||
if (row.every((c) => c.trim().isEmpty)) continue; // blank line
|
||||
final label = cell(row, 'variety_label');
|
||||
if (label == null) {
|
||||
skipped++;
|
||||
continue;
|
||||
}
|
||||
|
||||
final groupId = cell(row, 'variety_id');
|
||||
_CsvVarietyBuilder builder;
|
||||
if (groupId != null && byGroup.containsKey(groupId)) {
|
||||
builder = byGroup[groupId]!;
|
||||
} else {
|
||||
builder = _CsvVarietyBuilder(
|
||||
label: label,
|
||||
category: cell(row, 'category'),
|
||||
scientificName: cell(row, 'species_scientific_name'),
|
||||
cultivarName: cell(row, 'cultivar_name'),
|
||||
notes: cell(row, 'variety_notes'),
|
||||
vernacularNames: _parseVernacular(cell(row, 'vernacular_names')),
|
||||
links: _parseLinks(cell(row, 'external_links')),
|
||||
);
|
||||
builders.add(builder);
|
||||
if (groupId != null) byGroup[groupId] = builder;
|
||||
}
|
||||
|
||||
final lot = _parseLot(row, cell);
|
||||
if (lot != null) builder.lots.add(lot);
|
||||
}
|
||||
|
||||
return CsvImport(
|
||||
varieties: [for (final b in builders) b.build()],
|
||||
skippedRows: skipped,
|
||||
);
|
||||
}
|
||||
|
||||
CsvImportLot? _parseLot(List<String> row, String? Function(List<String>, String) cell) {
|
||||
final typeCell = cell(row, 'lot_type');
|
||||
final year = cell(row, 'harvest_year');
|
||||
final month = cell(row, 'harvest_month');
|
||||
final kind = cell(row, 'quantity_kind');
|
||||
final precise = cell(row, 'quantity_precise');
|
||||
final label = cell(row, 'quantity_label');
|
||||
final presentation = cell(row, 'presentation');
|
||||
final storage = cell(row, 'storage_location');
|
||||
final offer = cell(row, 'offer_status');
|
||||
|
||||
final hasLot = [
|
||||
typeCell,
|
||||
year,
|
||||
month,
|
||||
kind,
|
||||
precise,
|
||||
label,
|
||||
presentation,
|
||||
storage,
|
||||
offer,
|
||||
].any((c) => c != null);
|
||||
if (!hasLot) return null;
|
||||
|
||||
return CsvImportLot(
|
||||
type: _enumOr(LotType.values, typeCell, LotType.seed),
|
||||
harvestYear: year == null ? null : int.tryParse(year),
|
||||
harvestMonth: month == null ? null : int.tryParse(month),
|
||||
quantityKind: _enumName(QuantityKind.values, kind),
|
||||
quantityPrecise: precise == null ? null : double.tryParse(precise),
|
||||
quantityLabel: label,
|
||||
presentation: _enumOrNull(Presentation.values, presentation),
|
||||
storageLocation: storage,
|
||||
offerStatus: _enumOr(OfferStatus.values, offer, OfferStatus.private),
|
||||
);
|
||||
}
|
||||
|
||||
/// `name (lang); name2` → vernacular names; the trailing `(xx)` is the ISO
|
||||
/// language, mirroring the export join in [encode].
|
||||
List<CsvImportName> _parseVernacular(String? cell) {
|
||||
if (cell == null) return const [];
|
||||
final out = <CsvImportName>[];
|
||||
for (final part in cell.split(';')) {
|
||||
final text = part.trim();
|
||||
if (text.isEmpty) continue;
|
||||
final match = RegExp(r'^(.*)\(([^()]+)\)$').firstMatch(text);
|
||||
if (match != null) {
|
||||
out.add(
|
||||
CsvImportName(
|
||||
name: match.group(1)!.trim(),
|
||||
language: match.group(2)!.trim(),
|
||||
),
|
||||
);
|
||||
} else {
|
||||
out.add(CsvImportName(name: text));
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/// `title|url; url2` → external links, mirroring the export join in [encode].
|
||||
List<CsvImportLink> _parseLinks(String? cell) {
|
||||
if (cell == null) return const [];
|
||||
final out = <CsvImportLink>[];
|
||||
for (final part in cell.split(';')) {
|
||||
final text = part.trim();
|
||||
if (text.isEmpty) continue;
|
||||
final pipe = text.indexOf('|');
|
||||
if (pipe >= 0) {
|
||||
out.add(
|
||||
CsvImportLink(
|
||||
url: text.substring(pipe + 1).trim(),
|
||||
title: text.substring(0, pipe).trim(),
|
||||
),
|
||||
);
|
||||
} else {
|
||||
out.add(CsvImportLink(url: text));
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/// Returns [name] only if it is a valid enum value name, else null — used to
|
||||
/// keep `quantity_kind` as the stored enum key while dropping unknown ones.
|
||||
String? _enumName<T extends Enum>(List<T> values, String? name) {
|
||||
if (name == null) return null;
|
||||
return values.asNameMap().containsKey(name) ? name : null;
|
||||
}
|
||||
|
||||
T _enumOr<T extends Enum>(List<T> values, String? name, T fallback) =>
|
||||
_enumOrNull(values, name) ?? fallback;
|
||||
|
||||
T? _enumOrNull<T extends Enum>(List<T> values, String? name) {
|
||||
if (name == null) return null;
|
||||
return values.asNameMap()[name];
|
||||
}
|
||||
|
||||
/// A minimal RFC 4180 parser: fields may be quoted, quotes inside a quoted
|
||||
/// field are doubled, and quoted fields may contain commas and newlines.
|
||||
/// Handles both `\r\n` and `\n` line endings; a trailing newline does not
|
||||
/// produce a spurious empty row.
|
||||
List<List<String>> _parse(String input) {
|
||||
final rows = <List<String>>[];
|
||||
var row = <String>[];
|
||||
final field = StringBuffer();
|
||||
var inQuotes = false;
|
||||
var i = 0;
|
||||
while (i < input.length) {
|
||||
final c = input[i];
|
||||
if (inQuotes) {
|
||||
if (c == '"') {
|
||||
if (i + 1 < input.length && input[i + 1] == '"') {
|
||||
field.write('"');
|
||||
i += 2;
|
||||
continue;
|
||||
}
|
||||
inQuotes = false;
|
||||
i++;
|
||||
continue;
|
||||
}
|
||||
field.write(c);
|
||||
i++;
|
||||
continue;
|
||||
}
|
||||
if (c == '"') {
|
||||
inQuotes = true;
|
||||
i++;
|
||||
continue;
|
||||
}
|
||||
if (c == ',') {
|
||||
row.add(field.toString());
|
||||
field.clear();
|
||||
i++;
|
||||
continue;
|
||||
}
|
||||
if (c == '\r' || c == '\n') {
|
||||
row.add(field.toString());
|
||||
field.clear();
|
||||
rows.add(row);
|
||||
row = <String>[];
|
||||
if (c == '\r' && i + 1 < input.length && input[i + 1] == '\n') {
|
||||
i += 2;
|
||||
} else {
|
||||
i++;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
field.write(c);
|
||||
i++;
|
||||
}
|
||||
row.add(field.toString());
|
||||
final trailingEmpty = rows.isNotEmpty && row.length == 1 && row.first.isEmpty;
|
||||
if (!trailingEmpty) rows.add(row);
|
||||
return rows;
|
||||
}
|
||||
}
|
||||
|
||||
/// The result of decoding a CSV for an additive import: the varieties to add
|
||||
/// (each with its lots, names and links) plus how many rows were skipped for
|
||||
/// lacking a `variety_label`.
|
||||
class CsvImport {
|
||||
const CsvImport({required this.varieties, required this.skippedRows});
|
||||
|
||||
final List<CsvImportVariety> varieties;
|
||||
final int skippedRows;
|
||||
}
|
||||
|
||||
/// One variety parsed from CSV, to be inserted fresh (new id + HLC).
|
||||
class CsvImportVariety {
|
||||
const CsvImportVariety({
|
||||
required this.label,
|
||||
this.category,
|
||||
this.scientificName,
|
||||
this.cultivarName,
|
||||
this.notes,
|
||||
this.vernacularNames = const [],
|
||||
this.links = const [],
|
||||
this.lots = const [],
|
||||
});
|
||||
|
||||
final String label;
|
||||
final String? category;
|
||||
final String? scientificName;
|
||||
final String? cultivarName;
|
||||
final String? notes;
|
||||
final List<CsvImportName> vernacularNames;
|
||||
final List<CsvImportLink> links;
|
||||
final List<CsvImportLot> lots;
|
||||
}
|
||||
|
||||
/// One lot parsed from a CSV row.
|
||||
class CsvImportLot {
|
||||
const CsvImportLot({
|
||||
this.type = LotType.seed,
|
||||
this.harvestYear,
|
||||
this.harvestMonth,
|
||||
this.quantityKind,
|
||||
this.quantityPrecise,
|
||||
this.quantityLabel,
|
||||
this.presentation,
|
||||
this.storageLocation,
|
||||
this.offerStatus = OfferStatus.private,
|
||||
});
|
||||
|
||||
final LotType type;
|
||||
final int? harvestYear;
|
||||
final int? harvestMonth;
|
||||
final String? quantityKind;
|
||||
final double? quantityPrecise;
|
||||
final String? quantityLabel;
|
||||
final Presentation? presentation;
|
||||
final String? storageLocation;
|
||||
final OfferStatus offerStatus;
|
||||
}
|
||||
|
||||
/// One vernacular name parsed from CSV.
|
||||
class CsvImportName {
|
||||
const CsvImportName({required this.name, this.language});
|
||||
|
||||
final String name;
|
||||
final String? language;
|
||||
}
|
||||
|
||||
/// One external link parsed from CSV.
|
||||
class CsvImportLink {
|
||||
const CsvImportLink({required this.url, this.title});
|
||||
|
||||
final String url;
|
||||
final String? title;
|
||||
}
|
||||
|
||||
/// Mutable accumulator: several CSV rows can contribute lots to one variety.
|
||||
class _CsvVarietyBuilder {
|
||||
_CsvVarietyBuilder({
|
||||
required this.label,
|
||||
this.category,
|
||||
this.scientificName,
|
||||
this.cultivarName,
|
||||
this.notes,
|
||||
this.vernacularNames = const [],
|
||||
this.links = const [],
|
||||
});
|
||||
|
||||
final String label;
|
||||
final String? category;
|
||||
final String? scientificName;
|
||||
final String? cultivarName;
|
||||
final String? notes;
|
||||
final List<CsvImportName> vernacularNames;
|
||||
final List<CsvImportLink> links;
|
||||
final List<CsvImportLot> lots = [];
|
||||
|
||||
CsvImportVariety build() => CsvImportVariety(
|
||||
label: label,
|
||||
category: category,
|
||||
scientificName: scientificName,
|
||||
cultivarName: cultivarName,
|
||||
notes: notes,
|
||||
vernacularNames: vernacularNames,
|
||||
links: links,
|
||||
lots: List.unmodifiable(lots),
|
||||
);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ import 'package:equatable/equatable.dart';
|
|||
import '../db/database.dart';
|
||||
import '../db/enums.dart';
|
||||
import 'export_import/import_reconciler.dart';
|
||||
import 'export_import/inventory_csv_codec.dart';
|
||||
import 'export_import/inventory_snapshot.dart';
|
||||
|
||||
/// A lightweight row for the inventory list (only what the list renders).
|
||||
|
|
@ -961,6 +962,113 @@ class VarietyRepository {
|
|||
return summary;
|
||||
}
|
||||
|
||||
/// Additive import of a spreadsheet [csv]: every parsed variety is inserted
|
||||
/// fresh (new UUIDv7 + local HLC), so it never overwrites existing rows and
|
||||
/// re-importing the same file adds it again. The canonical, id-reconciling
|
||||
/// import is JSON ([importInventory]); this is the low-friction path for a
|
||||
/// list a collective already keeps in a spreadsheet.
|
||||
///
|
||||
/// Species are re-linked by scientific name against the local catalog
|
||||
/// (unmatched → null). Returns the number of varieties added.
|
||||
Future<ImportSummary> importCsv(CsvImport csv) async {
|
||||
var inserted = 0;
|
||||
await _db.transaction(() async {
|
||||
for (final v in csv.varieties) {
|
||||
final speciesId = await _resolveSpeciesByName(v.scientificName);
|
||||
final varietyId = idGen.newId();
|
||||
final (created, updated) = _stamp();
|
||||
await _db
|
||||
.into(_db.varieties)
|
||||
.insert(
|
||||
VarietiesCompanion.insert(
|
||||
id: varietyId,
|
||||
label: v.label,
|
||||
createdAt: created,
|
||||
updatedAt: updated,
|
||||
lastAuthor: nodeId,
|
||||
category: Value(v.category),
|
||||
cultivarName: Value(v.cultivarName),
|
||||
notes: Value(v.notes),
|
||||
speciesId: Value(speciesId),
|
||||
),
|
||||
);
|
||||
inserted++;
|
||||
|
||||
for (final lot in v.lots) {
|
||||
final (created, updated) = _stamp();
|
||||
await _db
|
||||
.into(_db.lots)
|
||||
.insert(
|
||||
LotsCompanion.insert(
|
||||
id: idGen.newId(),
|
||||
varietyId: varietyId,
|
||||
createdAt: created,
|
||||
updatedAt: updated,
|
||||
lastAuthor: nodeId,
|
||||
type: Value(lot.type),
|
||||
harvestYear: Value(lot.harvestYear),
|
||||
harvestMonth: Value(lot.harvestMonth),
|
||||
quantityKind: Value(lot.quantityKind),
|
||||
quantityPrecise: Value(lot.quantityPrecise),
|
||||
quantityLabel: Value(lot.quantityLabel),
|
||||
presentation: Value(lot.presentation),
|
||||
storageLocation: Value(lot.storageLocation),
|
||||
offerStatus: Value(lot.offerStatus),
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
for (final name in v.vernacularNames) {
|
||||
final (created, updated) = _stamp();
|
||||
await _db
|
||||
.into(_db.varietyVernacularNames)
|
||||
.insert(
|
||||
VarietyVernacularNamesCompanion.insert(
|
||||
id: idGen.newId(),
|
||||
varietyId: varietyId,
|
||||
name: name.name,
|
||||
createdAt: created,
|
||||
updatedAt: updated,
|
||||
lastAuthor: nodeId,
|
||||
language: Value(name.language),
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
for (final link in v.links) {
|
||||
final (created, updated) = _stamp();
|
||||
await _db
|
||||
.into(_db.externalLinks)
|
||||
.insert(
|
||||
ExternalLinksCompanion.insert(
|
||||
id: idGen.newId(),
|
||||
createdAt: created,
|
||||
updatedAt: updated,
|
||||
lastAuthor: nodeId,
|
||||
parentType: ParentType.variety,
|
||||
parentId: varietyId,
|
||||
url: link.url,
|
||||
title: Value(link.title),
|
||||
),
|
||||
);
|
||||
}
|
||||
}
|
||||
});
|
||||
return ImportSummary(inserted: inserted);
|
||||
}
|
||||
|
||||
/// Resolves a single scientific name to a local catalog species id, or null
|
||||
/// when blank or unmatched.
|
||||
Future<String?> _resolveSpeciesByName(String? scientificName) async {
|
||||
final name = scientificName?.trim();
|
||||
if (name == null || name.isEmpty) return null;
|
||||
final local = await (_db.select(_db.species)..where(
|
||||
(s) => s.scientificName.equals(name) & s.isDeleted.equals(false),
|
||||
))
|
||||
.getSingleOrNull();
|
||||
return local?.id;
|
||||
}
|
||||
|
||||
/// Maps incoming species ids to local catalog ids by scientific name.
|
||||
Future<Map<String, String?>> _resolveSpecies(
|
||||
Map<String, String> speciesNamesById,
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue