Importer update add support for meta data and multiple inserts for some entities like addresses and phones, updated other things

This commit is contained in:
Simon Pocrnjič
2025-10-09 22:28:48 +02:00
parent c8029c9eb0
commit 0598261cdc
27 changed files with 2517 additions and 375 deletions
+427 -41
View File
@@ -32,8 +32,9 @@ public function simulate(Import $import, int $limit = 100, bool $verbose = false
$delimiter = $meta['forced_delimiter'] ?? $meta['detected_delimiter'] ?? ',';
$columns = is_array($meta['columns'] ?? null) ? $meta['columns'] : [];
$targetToSource = $this->buildTargetLookup($import);
if (! $targetToSource) {
// Build both flat and grouped lookups
[$targetToSource, $groupedLookup] = $this->buildTargetLookup($import);
if (! $targetToSource && ! $groupedLookup) {
return $this->errorPayload('Ni shranjenih mapiranj za ta uvoz.');
}
@@ -47,6 +48,8 @@ public function simulate(Import $import, int $limit = 100, bool $verbose = false
// Discover mapped entity roots and then filter by supported list with safe fallbacks
$detectedRoots = $this->detectEntityRoots($targetToSource);
// Roots that support multiple grouped entries
$multiRoots = $this->loadSupportsMultipleRoots();
$supported = $this->loadSupportedEntityRoots();
$entityRoots = $this->filterEntityRoots($detectedRoots, $supported, $targetToSource);
$summaries = $this->initSummaries($entityRoots);
@@ -104,6 +107,26 @@ public function simulate(Import $import, int $limit = 100, bool $verbose = false
return null;
};
// Grouped values accessor: returns [group => value] for given root.field
$groupVals = function (string $root, string $field) use ($assoc, $groupedLookup, $targetToSource) {
$out = [];
$key = $root.'.'.$field;
if (isset($groupedLookup[$root])) {
foreach ($groupedLookup[$root] as $g => $fields) {
if (isset($fields[$field])) {
$col = $fields[$field];
$out[$g] = $assoc[$col] ?? null;
}
}
}
// Also include ungrouped flat mapping as default group when no explicit group exists
if (isset($targetToSource[$key]) && ! isset($out[''])) {
$out[''] = $assoc[$targetToSource[$key]] ?? null;
}
return $out;
};
// Contract
if (isset($entityRoots['contract'])) {
[$contractEntity, $summaries, $contractCache] = $this->simulateContract($val, $summaries, $contractCache, $val('contract.reference'));
@@ -116,6 +139,49 @@ public function simulate(Import $import, int $limit = 100, bool $verbose = false
$contractEntity['action'] = 'reactivate';
$contractEntity['reactivation'] = true;
}
// Attach contract meta preview from mappings (group-aware)
$metaGroups = [];
// Grouped contract.meta.* via groupedLookup
if (isset($groupedLookup['contract'])) {
foreach ($groupedLookup['contract'] as $g => $fields) {
foreach ($fields as $f => $srcCol) {
if (str_starts_with($f, 'meta.')) {
$key = substr($f, strlen('meta.'));
if ($key !== '') {
if (! isset($metaGroups[$g])) {
$metaGroups[$g] = [];
}
$metaGroups[$g][$key] = [
'title' => $srcCol,
'value' => $assoc[$srcCol] ?? null,
];
}
}
}
}
}
// Flat contract.meta.* (no group): assign to group '1'
foreach ($targetToSource as $tf => $srcCol) {
if (str_starts_with($tf, 'contract.meta.')) {
$key = substr($tf, strlen('contract.meta.'));
if ($key !== '') {
$g = '1';
if (! isset($metaGroups[$g])) {
$metaGroups[$g] = [];
}
// Do not override grouped if already present
if (! isset($metaGroups[$g][$key])) {
$metaGroups[$g][$key] = [
'title' => $srcCol,
'value' => $assoc[$srcCol] ?? null,
];
}
}
}
}
if (! empty($metaGroups)) {
$contractEntity['meta'] = $metaGroups;
}
$rowEntities['contract'] = $contractEntity + [
'action_label' => $translatedActions[$contractEntity['action']] ?? $contractEntity['action'],
];
@@ -156,22 +222,45 @@ public function simulate(Import $import, int $limit = 100, bool $verbose = false
}
$reference = $val($rootKey.'.reference');
$identityCandidates = $this->genericIdentityCandidates($rootKey, $val);
[$genericEntity, $summaries, $genericCaches, $genericExistingIdentities, $genericSeenIdentities]
= $this->simulateGenericRoot(
$rootKey,
$val,
$summaries,
$genericCaches,
$reference,
$identityCandidates,
$genericExistingIdentities,
$genericSeenIdentities,
$verbose,
$targetToSource,
);
$rowEntities[$rootKey] = $genericEntity + [
'action_label' => $translatedActions[$genericEntity['action']] ?? $genericEntity['action'],
];
if (isset($multiRoots[$rootKey]) && $multiRoots[$rootKey] === true) {
// Multi-item simulation per group
[$items, $summaries, $genericCaches, $genericExistingIdentities, $genericSeenIdentities]
= $this->simulateGenericRootMulti(
$rootKey,
$val,
$groupVals,
$summaries,
$genericCaches,
$identityCandidates,
$genericExistingIdentities,
$genericSeenIdentities,
$verbose,
$targetToSource
);
// Add action labels and attach
$rowEntities[$rootKey] = array_map(function ($ent) use ($translatedActions) {
$ent['action_label'] = $translatedActions[$ent['action']] ?? $ent['action'];
return $ent;
}, $items);
} else {
[$genericEntity, $summaries, $genericCaches, $genericExistingIdentities, $genericSeenIdentities]
= $this->simulateGenericRoot(
$rootKey,
$val,
$summaries,
$genericCaches,
$reference,
$identityCandidates,
$genericExistingIdentities,
$genericSeenIdentities,
$verbose,
$targetToSource,
);
$rowEntities[$rootKey] = $genericEntity + [
'action_label' => $translatedActions[$genericEntity['action']] ?? $genericEntity['action'],
];
}
}
// Attach chain entities (client_case, person) if contract already existed
@@ -317,8 +406,22 @@ public function simulate(Import $import, int $limit = 100, bool $verbose = false
// If existing contract: upgrade generic email/phone/address entities (already simulated) to mark as chain if corresponding person attached
if ($existingContract && isset($rowEntities['person']['id'])) {
foreach (['email', 'phone', 'address'] as $gRoot) {
if (isset($rowEntities[$gRoot]) && ! ($rowEntities[$gRoot]['existing_chain'] ?? false)) {
$rowEntities[$gRoot]['existing_chain'] = true; // mark for UI toggle
if (! isset($rowEntities[$gRoot])) {
continue;
}
$valRef = $rowEntities[$gRoot];
// If multi items array, set flag on each item
if (is_array($valRef) && isset($valRef[0]) && is_array($valRef[0])) {
foreach ($rowEntities[$gRoot] as &$it) {
if (! ($it['existing_chain'] ?? false)) {
$it['existing_chain'] = true;
}
}
unset($it);
} else {
if (! ($rowEntities[$gRoot]['existing_chain'] ?? false)) {
$rowEntities[$gRoot]['existing_chain'] = true; // mark for UI toggle
}
}
}
}
@@ -352,7 +455,21 @@ public function simulate(Import $import, int $limit = 100, bool $verbose = false
if ($verbose) {
foreach ($rowEntities as $eroot => &$ent) {
$tf = $eroot.'.reference';
if (isset($targetToSource[$tf])) {
if (! isset($targetToSource[$tf])) {
continue;
}
if (is_array($ent) && isset($ent[0]) && is_array($ent[0])) {
foreach ($ent as &$item) {
$item['sources'] = $item['sources'] ?? [];
if (! isset($item['sources'][$tf])) {
$item['sources'][$tf] = [
'source_column' => $targetToSource[$tf],
'value' => $val($tf),
];
}
}
unset($item);
} else {
$ent['sources'] = $ent['sources'] ?? [];
if (! isset($ent['sources'][$tf])) {
$ent['sources'][$tf] = [
@@ -404,7 +521,6 @@ public function simulate(Import $import, int $limit = 100, bool $verbose = false
foreach (['reference', 'identity_used', 'identity_candidates', 'full_name', 'first_name', 'last_name', 'address', 'country', 'nu', 'value'] as $k) {
if (isset($ent[$k]) && $ent[$k]) {
$hasData = true;
break;
}
}
// Some entities (e.g. payment) do not have 'action'; treat them as non-empty if they have data or status
@@ -455,47 +571,88 @@ private function buildTargetLookup(Import $import): array
$mappings = \DB::table('import_mappings')
->where('import_id', $import->id)
->orderBy('position')
->get(['source_column', 'target_field']);
->get(['source_column', 'target_field', 'options']);
$lookup = [];
$grouped = [];
foreach ($mappings as $m) {
$target = trim((string) $m->target_field);
$source = trim((string) $m->source_column);
if ($target === '' || $source === '') {
continue;
}
// Parse grouping and field
$group = null;
$root = null;
$rest = null;
$restEffective = null;
$opts = [];
if (preg_match('/^([a-zA-Z0-9_]+)\\[([^\]]+)\\]\\.([a-zA-Z0-9_]+)$/', $target, $mm)) {
$root = $mm[1];
$group = (string) $mm[2];
$rest = $mm[3];
$restEffective = $rest;
} else {
if (str_contains($target, '.')) {
[$root, $rest] = explode('.', $target, 2);
}
try {
$opts = is_array($m->options) ? $m->options : (json_decode((string) $m->options, true) ?: []);
} catch (\Throwable) {
$opts = [];
}
if (is_array($opts) && array_key_exists('group', $opts) && $opts['group'] !== '' && $opts['group'] !== null) {
$group = (string) $opts['group'];
}
$restEffective = $rest;
// Alias meta with options.key => meta.{key}
if ($rest === 'meta' && is_array($opts) && ! empty($opts['key'])) {
$restEffective = 'meta'.'.'.(string) $opts['key'];
}
}
// Register flat lookups
if (! isset($lookup[$target])) {
$lookup[$target] = $source;
}
// If mapping uses *.client_ref, also register *.reference alias for simulation reference purposes
if ($rest !== null) {
$normRoot = $this->normalizeRoot((string) $root);
$tfNorm = $normRoot.'.'.$restEffective;
if (! isset($lookup[$tfNorm])) {
$lookup[$tfNorm] = $source;
}
if (str_ends_with((string) $root, 's')) {
$sing = substr((string) $root, 0, -1);
if ($sing) {
$tfSing = $sing.'.'.$restEffective;
if (! isset($lookup[$tfSing])) {
$lookup[$tfSing] = $source;
}
}
}
}
if (str_ends_with($target, '.client_ref')) {
$alias = substr($target, 0, -strlen('.client_ref')).'.reference';
if (! isset($lookup[$alias])) {
$lookup[$alias] = $source;
}
}
if (str_contains($target, '.')) {
[$root, $rest] = explode('.', $target, 2);
$norm = $this->normalizeRoot($root);
if ($norm !== $root) {
$alt = $norm.'.'.$rest;
if (! isset($lookup[$alt])) {
$lookup[$alt] = $source;
}
// Register grouped lookup per normalized root
if ($group !== null && $rest !== null) {
$normRoot = $this->normalizeRoot((string) $root);
if (! isset($grouped[$normRoot])) {
$grouped[$normRoot] = [];
}
if (str_ends_with($root, 's')) {
$sing = substr($root, 0, -1);
if ($sing && $sing !== $root) {
$alt2 = $sing.'.'.$rest;
if (! isset($lookup[$alt2])) {
$lookup[$alt2] = $source;
}
}
if (! isset($grouped[$normRoot][$group])) {
$grouped[$normRoot][$group] = [];
}
$grouped[$normRoot][$group][$restEffective] = $source;
}
}
return $lookup;
return [$lookup, $grouped];
}
private function readFileRows(Import $import, bool $hasHeader, string $delimiter, array $columns, int $limit): array
@@ -1173,6 +1330,235 @@ private function loadSupportedEntityRoots(): array
}
}
/**
* Load which canonical roots support multiple items (grouped) from import_entities.supports_multiple.
* Falls back to known defaults if table/column is unavailable.
*/
private function loadSupportsMultipleRoots(): array
{
$map = [];
try {
if (\Schema::hasTable('import_entities') && \Schema::hasColumn('import_entities', 'supports_multiple')) {
$rows = \App\Models\ImportEntity::query()->get(['key', 'canonical_root', 'supports_multiple']);
foreach ($rows as $r) {
$root = $r->canonical_root ?: $r->key;
if (! $root) {
continue;
}
$norm = $this->normalizeRoot($root);
$map[$norm] = (bool) $r->supports_multiple;
}
}
} catch (\Throwable) {
// ignore and fallback
}
if (empty($map)) {
// Conservative defaults: only known contact types are multi
$map = [
'email' => true,
'phone' => true,
'address' => true,
];
}
return $map;
}
/**
* Simulate generic root that supports multiple items (grouped). Returns array of entities per group.
* Mirrors simulateGenericRoot per item while grouping by provided group values.
*
* @return array{0: array<int, array>, 1: array, 2: array, 3: array, 4: array}
*/
private function simulateGenericRootMulti(
string $root,
callable $val,
callable $groupVals,
array $summaries,
array $genericCaches,
array $identityCandidatesBase,
array $genericExistingIdentities,
array $genericSeenIdentities,
bool $verbose = false,
array $targetToSource = [],
): array {
// Build per-group entities
$items = [];
// Determine fields per root to preview
$previewFields = [];
if ($root === 'email') {
$previewFields = ['value'];
} elseif ($root === 'phone') {
$previewFields = ['nu'];
} elseif ($root === 'address') {
$previewFields = ['address', 'country']; // postal_code not present
}
// Collect groups seen for this root
$groups = [];
foreach ($previewFields as $pf) {
$map = $groupVals($root, $pf);
foreach ($map as $g => $v) {
$groups[$g] = true;
}
}
// If no groups but flat mapping exists, still simulate single default item
if (empty($groups)) {
$groups[''] = true;
}
// Ensure summary bucket exists and count total_rows once per row (not per item)
if (! isset($summaries[$root])) {
$summaries[$root] = [
'root' => $root,
'total_rows' => 0,
'create' => 0,
'update' => 0,
'missing_ref' => 0,
'invalid' => 0,
'duplicate' => 0,
'duplicate_db' => 0,
];
}
$summaries[$root]['total_rows']++;
// For each group, produce an item entity similar to simulateGenericRoot
foreach (array_keys($groups) as $g) {
// Build reference and identity candidates per group
$reference = null;
if ($root === 'email') {
$reference = $groupVals('email', 'value')[$g] ?? null;
} elseif ($root === 'phone') {
$reference = $groupVals('phone', 'nu')[$g] ?? null;
} elseif ($root === 'address') {
$reference = $groupVals('address', 'address')[$g] ?? null;
}
$identityCandidates = $identityCandidatesBase;
// Override with group-specific identity when present
if ($root === 'email') {
$v = $groupVals('email', 'value')[$g] ?? null;
if ($v) {
$identityCandidates = ['value:'.mb_strtolower(trim((string) $v))];
}
} elseif ($root === 'phone') {
$nu = $groupVals('phone', 'nu')[$g] ?? null;
if ($nu) {
$norm = preg_replace('/\D+/', '', (string) $nu) ?? '';
if ($norm) {
$identityCandidates = ['nu:'.$norm];
}
}
} elseif ($root === 'address') {
$addr = $groupVals('address', 'address')[$g] ?? null;
$country = $groupVals('address', 'country')[$g] ?? null;
if ($addr || $country) {
$key = mb_strtolower(trim((string) ($addr ?? ''))).'|'.mb_strtolower(trim((string) ($country ?? '')));
$identityCandidates = ['addr:'.$key];
}
}
// Query existing by reference when available
$modelClass = $this->modelClassForGeneric($root);
$record = null;
if ($reference) {
if (! isset($genericCaches[$root])) {
$genericCaches[$root] = [];
}
if (array_key_exists($reference, $genericCaches[$root])) {
$record = $genericCaches[$root][$reference];
} elseif ($modelClass && class_exists($modelClass)) {
try {
if (\Schema::hasColumn((new $modelClass)->getTable(), 'reference')) {
$record = $modelClass::query()->where('reference', $reference)->first(['id', 'reference']);
}
} catch (\Throwable) {
$record = null;
}
$genericCaches[$root][$reference] = $record; // may be null
}
}
$entity = [
'reference' => $reference,
'id' => $record?->id,
'exists' => (bool) $record,
'action' => $reference ? ($record ? 'update' : 'create') : 'skip',
'group' => $g,
'identity_candidates' => $identityCandidates,
];
// Previews
if ($root === 'email') {
$entity['value'] = $groupVals('email', 'value')[$g] ?? null;
} elseif ($root === 'phone') {
$entity['nu'] = $groupVals('phone', 'nu')[$g] ?? null;
} elseif ($root === 'address') {
$entity['address'] = $groupVals('address', 'address')[$g] ?? null;
$entity['country'] = $groupVals('address', 'country')[$g] ?? null;
}
if ($verbose) {
$srcs = [];
foreach ($targetToSource as $tf => $col) {
if (str_starts_with($tf, $root.'.')) {
// only include fields that match this group if grouped mapping exists
// We can't easily map back to group here without reverse index; include anyway for now
$srcs[$tf] = [
'source_column' => $col,
'value' => $val($tf),
];
}
}
if ($srcs) {
$entity['sources'] = $entity['sources'] ?? [];
$entity['sources'] += $srcs;
}
}
if (! $reference) {
$summaries[$root]['missing_ref']++;
} elseif ($record) {
$summaries[$root]['update']++;
} else {
$summaries[$root]['create']++;
}
// Duplicate detection per root across items in stream
foreach ($identityCandidates as $identity) {
if ($identity === null || $identity === '') {
continue;
}
if (! isset($genericExistingIdentities[$root])) {
$genericExistingIdentities[$root] = $this->loadExistingGenericIdentities($root);
}
if (isset($genericExistingIdentities[$root][$identity])) {
$entity['duplicate_db'] = true;
$entity['identity_used'] = $identity;
$summaries[$root]['duplicate_db']++;
break;
}
if (! isset($genericSeenIdentities[$root])) {
$genericSeenIdentities[$root] = [];
}
if (isset($genericSeenIdentities[$root][$identity])) {
$entity['duplicate'] = true;
$entity['identity_used'] = $identity;
$summaries[$root]['duplicate']++;
break;
}
$genericSeenIdentities[$root][$identity] = true;
$entity['identity_used'] = $identity;
break;
}
$items[] = $entity;
}
return [$items, $summaries, $genericCaches, $genericExistingIdentities, $genericSeenIdentities];
}
/* ------------------------------- Localization ------------------------------- */
private function actionTranslations(): array