sipo/Teren-app
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Importer update add support for meta data and multiple inserts for some entities like addresses and phones, updated other things

Browse Source
This commit is contained in:
Simon Pocrnjič
2025-10-09 22:28:48 +02:00
parent c8029c9eb0
commit 0598261cdc
27 changed files with 2517 additions and 375 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/Services/ImportProcessor.php
+382 -48
View File
@@ -62,7 +62,7 @@ public function process(Import $import, ?Authenticatable $user = null): array
->get(['source_column', 'target_field', 'transform', 'apply_mode', 'options']);
// Load dynamic entity config
[$rootAliasMap, $fieldAliasMap, $validRoots] = $this->loadImportEntityConfig();
[$rootAliasMap, $fieldAliasMap, $validRoots, $supportsMultiple] = $this->loadImportEntityConfig();
// Normalize aliases (plural/legacy roots, field names) before validation
$mappings = $this->normalizeMappings($mappings, $rootAliasMap, $fieldAliasMap);
// Validate mapping roots early to avoid silent failures due to typos
@@ -219,9 +219,10 @@ public function process(Import $import, ?Authenticatable $user = null): array
if ($isPg) {
// No DB changes were made for this row; nothing to roll back explicitly.
}
continue; // proceed to next CSV row
}
[$recordType, $mapped] = $this->applyMappings($rawAssoc, $mappings);
[$recordType, $mapped] = $this->applyMappings($rawAssoc, $mappings, $supportsMultiple);
// Determine row-level reactivation intent: precedence row > import > template
$rowReactivate = false;
@@ -751,9 +752,10 @@ public function process(Import $import, ?Authenticatable $user = null): array
// Resolve by contact values next
if (! $personIdForRow) {
$emailVal = trim((string) ($mapped['email']['value'] ?? ''));
$phoneNu = trim((string) ($mapped['phone']['nu'] ?? ''));
$addrLine = trim((string) ($mapped['address']['address'] ?? ''));
// consider first values from multi groups if present
$emailVal = trim((string) ($this->firstFromMulti($mapped, 'email', 'value') ?? ''));
$phoneNu = trim((string) ($this->firstFromMulti($mapped, 'phone', 'nu') ?? ''));
$addrLine = trim((string) ($this->firstFromMulti($mapped, 'address', 'address') ?? ''));
// Try to resolve by existing contacts first
if ($emailVal !== '') {
@@ -814,22 +816,57 @@ public function process(Import $import, ?Authenticatable $user = null): array
$contactChanged = false;
if ($personIdForRow) {
if (! empty($mapped['email'] ?? [])) {
$r = $this->upsertEmail($personIdForRow, $mapped['email'], $mappings);
if (in_array($r['action'] ?? 'skipped', ['inserted', 'updated'])) {
$contactChanged = true;
}
}
if (! empty($mapped['address'] ?? [])) {
$r = $this->upsertAddress($personIdForRow, $mapped['address'], $mappings);
if (in_array($r['action'] ?? 'skipped', ['inserted', 'updated'])) {
$contactChanged = true;
}
}
if (! empty($mapped['phone'] ?? [])) {
$r = $this->upsertPhone($personIdForRow, $mapped['phone'], $mappings);
if (in_array($r['action'] ?? 'skipped', ['inserted', 'updated'])) {
$contactChanged = true;
// Fan-out for multi-supported roots; backward compatible for single hashes
foreach (['email' => 'upsertEmail', 'address' => 'upsertAddress', 'phone' => 'upsertPhone'] as $root => $method) {
if (isset($mapped[$root]) && is_array($mapped[$root])) {
// If it's a grouped map (supports multiple), iterate groups; else treat as single data hash
$data = $mapped[$root];
$isGrouped = $this->isGroupedMulti($data);
if ($isGrouped) {
// De-duplicate grouped items within the same row by their unique key per root
$keyField = $root === 'email' ? 'value' : ($root === 'phone' ? 'nu' : 'address');
$normalizer = function ($v) use ($root) {
if ($v === null) {
return null;
}
$s = trim((string) $v);
if ($s === '') {
return '';
}
if ($root === 'email') {
return mb_strtolower($s);
}
if ($root === 'phone') {
// Keep leading + and digits only for comparison
$s = preg_replace('/[^0-9+]/', '', $s) ?? $s;
// Collapse multiple + to single leading
$s = ltrim($s, '+');
return '+'.$s;
}
// address: normalize whitespace and lowercase for comparison
$s = preg_replace('/\s+/', ' ', $s) ?? $s;
return mb_strtolower(trim($s));
};
$data = $this->dedupeGroupedItems($data, $keyField, $normalizer);
foreach ($data as $grp => $payload) {
if (empty(array_filter($payload, fn ($v) => ! is_null($v) && trim((string) $v) !== ''))) {
continue; // skip empty group
}
$r = $this->{$method}($personIdForRow, $payload, $mappings);
if (in_array($r['action'] ?? 'skipped', ['inserted', 'updated'])) {
$contactChanged = true;
}
}
} else {
if (! empty($data)) {
$r = $this->{$method}($personIdForRow, $data, $mappings);
if (in_array($r['action'] ?? 'skipped', ['inserted', 'updated'])) {
$contactChanged = true;
}
}
}
}
}
}
@@ -986,7 +1023,7 @@ private function buildRowAssoc(array $row, ?array $header): array
return $assoc;
}
private function applyMappings(array $raw, $mappings): array
protected function applyMappings(array $raw, $mappings, array $supportsMultiple): array
{
$recordType = null;
$mapped = [];
@@ -1025,11 +1062,135 @@ private function applyMappings(array $raw, $mappings): array
// detect record type from first segment, e.g., "account.balance_amount"
$parts = explode('.', $target);
if (! $recordType && isset($parts[0])) {
$recordType = $parts[0];
$rootWithBracket = $parts[0] ?? '';
// Support bracket grouping like address[1].city
$group = null;
$root = $rootWithBracket;
if (preg_match('/^(?P<base>[a-zA-Z_][a-zA-Z0-9_]*)(\[(?P<grp>[^\]]+)\])$/', $rootWithBracket, $m)) {
$root = $m['base'];
$group = $m['grp'];
}
if (! $recordType && isset($root)) {
$recordType = $root;
}
// If this root supports multiple, determine group id: prefer mapping options.group, else bracket, else '1'
$supportsMulti = $supportsMultiple[$root] ?? false;
// Special handling for meta mappings: contract.meta.key (supports options.key and options.type)
if ($root === 'contract' && isset($parts[1]) && str_starts_with($parts[1], 'meta')) {
// Path could be meta.key or meta[key]
$metaKey = null;
$metaType = null;
// support dot path contract.meta.someKey
if (isset($parts[2])) {
$metaKey = $parts[2];
} else {
// support contract.meta[someKey]
if (preg_match('/^meta\[(?P<k>[^\]]+)\]$/', $parts[1], $mm)) {
$metaKey = $mm['k'];
}
}
if ($metaKey === null || $metaKey === '') {
// fallback: read key from mapping options.key if present
$opts = $map->options ?? null;
if (is_string($opts)) {
$opts = json_decode($opts, true) ?: [];
}
if (is_array($opts) && ! empty($opts['key'])) {
$metaKey = (string) $opts['key'];
}
if (is_array($opts) && ! empty($opts['type'])) {
$metaType = is_string($opts['type']) ? strtolower($opts['type']) : null;
}
} else {
// we still may have options.type
$opts = $map->options ?? null;
if (is_string($opts)) {
$opts = json_decode($opts, true) ?: [];
}
if (is_array($opts) && ! empty($opts['type'])) {
$metaType = is_string($opts['type']) ? strtolower($opts['type']) : null;
}
}
if ($metaKey !== null && $metaKey !== '') {
// group-aware bucket for meta entries
$groupOpt = $this->mappingOptionGroup($map);
$grp = ($groupOpt !== null && $groupOpt !== '') ? (string) $groupOpt : ($group ?? '1');
if (! isset($mapped['contract'])) {
$mapped['contract'] = [];
}
if (! isset($mapped['contract']['meta']) || ! is_array($mapped['contract']['meta'])) {
$mapped['contract']['meta'] = [];
}
if (! isset($mapped['contract']['meta'][$grp])) {
$mapped['contract']['meta'][$grp] = [];
}
// Optionally coerce the value based on provided type
$coerced = $value;
$metaType = in_array($metaType, ['string', 'number', 'date', 'boolean'], true) ? $metaType : null;
if ($metaType === 'number') {
if (is_string($coerced)) {
$norm = $this->normalizeDecimal($coerced);
$coerced = is_numeric($norm) ? (float) $norm : $coerced;
}
} elseif ($metaType === 'boolean') {
if (is_string($coerced)) {
$lc = strtolower(trim($coerced));
if (in_array($lc, ['1', 'true', 'yes', 'y'], true)) {
$coerced = true;
} elseif (in_array($lc, ['0', 'false', 'no', 'n'], true)) {
$coerced = false;
}
} else {
$coerced = (bool) $coerced;
}
} elseif ($metaType === 'date') {
$coerced = is_scalar($coerced) ? $this->normalizeDate((string) $coerced) : null;
} else {
// string or unspecified: cast scalars to string for consistency
if (is_scalar($coerced)) {
$coerced = (string) $coerced;
}
}
// Store as structure with title, value and optional type
$entry = [
'title' => is_string($src) ? $src : (string) $src,
'value' => $coerced,
];
if ($metaType !== null) {
$entry['type'] = $metaType;
}
$mapped['contract']['meta'][$grp][$metaKey] = $entry;
continue;
}
}
if ($supportsMulti) {
$groupOpt = $this->mappingOptionGroup($map);
$grp = ($groupOpt !== null && $groupOpt !== '') ? (string) $groupOpt : ($group ?? '1');
// rebuild target path to exclude bracket part
$field = $parts[1] ?? null;
if ($field !== null) {
if (! isset($mapped[$root]) || ! is_array($mapped[$root])) {
$mapped[$root] = [];
}
if (! isset($mapped[$root][$grp]) || ! is_array($mapped[$root][$grp])) {
$mapped[$root][$grp] = [];
}
$mapped[$root][$grp][$field] = $value;
}
} else {
// single item root: assign field or root as appropriate
$field = $parts[1] ?? null;
if ($field !== null) {
if (! isset($mapped[$root]) || ! is_array($mapped[$root])) {
$mapped[$root] = [];
}
$mapped[$root][$field] = $value;
} else {
$mapped[$root] = $value;
}
}
// build nested array by dot notation
$this->arraySetDot($mapped, $target, $value);
}
return [$recordType, $mapped];
@@ -1190,18 +1351,15 @@ private function upsertAccount(Import $import, array $mapped, $mappings): array
}
if ($existing) {
if (empty($applyUpdate)) {
return ['action' => 'skipped', 'message' => 'No fields marked for update'];
}
// Only update fields that are set; skip nulls to avoid wiping unintentionally
// Build non-null changes for account fields
$changes = array_filter($applyUpdate, fn ($v) => ! is_null($v));
if (empty($changes)) {
return ['action' => 'skipped', 'message' => 'No non-null changes'];
}
// Track balance change
$oldBalance = (float) ($existing->balance_amount ?? 0);
$existing->fill($changes);
$existing->save();
// Note: meta merging for contracts is handled in upsertContractChain, not here
if (! empty($changes)) {
$existing->fill($changes);
$existing->save();
}
// If balance_amount changed and this wasn't caused by a payment (we are in account upsert), log an activity with before/after
if (array_key_exists('balance_amount', $changes)) {
@@ -1421,18 +1579,71 @@ private function upsertContractChain(Import $import, array $mapped, $mappings):
}
if ($existing) {
if (empty($applyUpdate)) {
// Return existing contract reference even when skipped so callers can treat as resolved
return ['action' => 'skipped', 'message' => 'No contract fields marked for update', 'contract' => $existing];
}
// 1) Prepare contract field changes (non-null)
$changes = array_filter($applyUpdate, fn ($v) => ! is_null($v));
if (empty($changes)) {
return ['action' => 'skipped', 'message' => 'No non-null contract changes', 'contract' => $existing];
// 2) Prepare meta changes if provided via mapping
$metaUpdated = false;
$metaAppliedKeys = [];
if (! empty($contractData['meta'] ?? null) && is_array($contractData['meta'])) {
// Flatten incoming grouped meta to key => {title, value}
$incomingMeta = [];
foreach ($contractData['meta'] as $grp => $entries) {
if (! is_array($entries)) {
continue;
}
foreach ($entries as $k => $v) {
// v is expected as [title, value]
$incomingMeta[$k] = $v;
}
}
if (! empty($incomingMeta)) {
$currentMeta = is_array($existing->meta ?? null) ? $existing->meta : (json_decode((string) $existing->meta, true) ?: []);
foreach ($incomingMeta as $k => $entry) {
$newVal = is_array($entry) && array_key_exists('value', $entry) ? $entry['value'] : $entry;
$newTitle = is_array($entry) && array_key_exists('title', $entry) ? $entry['title'] : null;
$newType = is_array($entry) && array_key_exists('type', $entry) ? $entry['type'] : null;
$curEntry = $currentMeta[$k] ?? null;
$curVal = is_array($curEntry) && array_key_exists('value', $curEntry) ? $curEntry['value'] : $curEntry;
$curTitle = is_array($curEntry) && array_key_exists('title', $curEntry) ? $curEntry['title'] : null;
$curType = is_array($curEntry) && array_key_exists('type', $curEntry) ? $curEntry['type'] : null;
// Update when value differs, or title differs, or type differs
$shouldUpdate = ($newVal !== $curVal) || ($newTitle !== null && $newTitle !== $curTitle) || ($newType !== null && $newType !== $curType);
if ($shouldUpdate) {
if (is_array($entry)) {
$currentMeta[$k] = $entry;
} else {
$currentMeta[$k] = ['title' => (string) $k, 'value' => $newVal];
}
$metaUpdated = true;
$metaAppliedKeys[] = $k;
}
}
if ($metaUpdated) {
$existing->meta = $currentMeta;
}
}
}
if (empty($changes) && ! $metaUpdated) {
// Nothing to change
return ['action' => 'skipped', 'message' => 'No contract fields or meta changes', 'contract' => $existing];
}
if (! empty($changes)) {
$existing->fill($changes);
}
$existing->fill($changes);
$existing->save();
return ['action' => 'updated', 'contract' => $existing, 'applied_fields' => $changes];
// Build applied fields info, include meta keys if any
$applied = $changes;
if ($metaUpdated && ! empty($metaAppliedKeys)) {
foreach ($metaAppliedKeys as $k) {
$applied['meta:'.$k] = 'updated';
}
}
return ['action' => 'updated', 'contract' => $existing, 'applied_fields' => $applied];
} else {
if (empty($applyInsert)) {
return ['action' => 'skipped', 'message' => 'No contract fields marked for insert'];
@@ -1459,6 +1670,21 @@ private function upsertContractChain(Import $import, array $mapped, $mappings):
// ensure required defaults
$data['start_date'] = $data['start_date'] ?? now()->toDateString();
$data['type_id'] = $data['type_id'] ?? $this->getDefaultContractTypeId();
// Merge meta for create if provided
if (! empty($contractData['meta'] ?? null) && is_array($contractData['meta'])) {
$incomingMeta = [];
foreach ($contractData['meta'] as $grp => $entries) {
if (! is_array($entries)) {
continue;
}
foreach ($entries as $k => $v) {
$incomingMeta[$k] = $v;
}
}
if (! empty($incomingMeta)) {
$data['meta'] = $incomingMeta;
}
}
$created = Contract::create($data);
return ['action' => 'inserted', 'contract' => $created, 'applied_fields' => $data];
@@ -1503,6 +1729,7 @@ private function normalizeDate(?string $raw): ?string
if ($ts === false) {
return null;
}
return date('Y-m-d', $ts);
}
@@ -1712,6 +1939,7 @@ private function rowIsEffectivelyEmpty(array $rawAssoc): bool
}
}
}
return true;
}
@@ -1775,7 +2003,15 @@ private function normalizeTargetField(string $target, array $rootAliasMap, array
return $target;
}
$parts = explode('.', $target);
$root = $parts[0] ?? '';
$rootWithBracket = $parts[0] ?? '';
// Extract optional bracket group from root (e.g., address[1]) but preserve it after aliasing
$bracket = null;
if (preg_match('/^(?P<base>[a-zA-Z_][a-zA-Z0-9_]*)(\[(?P<grp>[^\]]+)\])$/', $rootWithBracket, $m)) {
$root = $m['base'];
$bracket = $m['grp'];
} else {
$root = $rootWithBracket;
}
$field = $parts[1] ?? null;
// Root aliases (plural to canonical) from DB
@@ -1791,18 +2027,21 @@ private function normalizeTargetField(string $target, array $rootAliasMap, array
// Rebuild
if ($field !== null) {
return $root.'.'.$field;
$rootOut = $bracket !== null ? ($root.'['.$bracket.']') : $root;
return $rootOut.'.'.$field;
}
return $root;
return $bracket !== null ? ($root.'['.$bracket.']') : $root;
}
private function loadImportEntityConfig(): array
protected function loadImportEntityConfig(): array
{
$entities = ImportEntity::all();
$rootAliasMap = [];
$fieldAliasMap = [];
$validRoots = [];
$supportsMultiple = [];
foreach ($entities as $ent) {
$canonical = $ent->canonical_root;
$validRoots[] = $canonical;
@@ -1817,13 +2056,108 @@ private function loadImportEntityConfig(): array
$aliases['__default'] = $aliases['__default'] ?? null;
}
$fieldAliasMap[$canonical] = $aliases;
$supportsMultiple[$canonical] = (bool) ($ent->supports_multiple ?? false);
}
// sensible defaults when DB empty
if (empty($validRoots)) {
$validRoots = ['person', 'contract', 'account', 'address', 'phone', 'email', 'client_case'];
$supportsMultiple = [
'address' => true,
'phone' => true,
'email' => true,
];
}
return [$rootAliasMap, $fieldAliasMap, $validRoots];
return [$rootAliasMap, $fieldAliasMap, $validRoots, $supportsMultiple];
}
/**
* Get mapping options.group if provided.
*/
private function mappingOptionGroup(object $map): ?string
{
$raw = $map->options ?? null;
if (is_string($raw)) {
try {
$json = json_decode($raw, true, 512, JSON_THROW_ON_ERROR);
return isset($json['group']) ? (string) $json['group'] : null;
} catch (\Throwable $e) {
return null;
}
}
if (is_array($raw)) {
return isset($raw['group']) ? (string) $raw['group'] : null;
}
return null;
}
/**
* Determine if a mapped root is a grouped multi structure.
*/
private function isGroupedMulti(mixed $data): bool
{
if (! is_array($data)) {
return false;
}
// Consider grouped if first element is itself an array
foreach ($data as $k => $v) {
return is_array($v);
}
return false;
}
/**
* Read first value from a multi-group or single map for a given root/field.
*/
private function firstFromMulti(array $mapped, string $root, string $field): mixed
{
if (! isset($mapped[$root])) {
return null;
}
$data = $mapped[$root];
if ($this->isGroupedMulti($data)) {
foreach ($data as $grp => $payload) {
if (isset($payload[$field]) && $payload[$field] !== null && trim((string) $payload[$field]) !== '') {
return $payload[$field];
}
}
return null;
}
return $data[$field] ?? null;
}
/**
* Remove duplicates from grouped items by comparing a key field across groups after normalization.
* Keeps the first occurrence and drops later duplicates. Empty/blank keys are kept only once.
*
* @param array<string,array> $grouped e.g. ['1' => ['value' => 'a'], '2' => ['value' => 'a']]
* @param string $keyField e.g. 'value' for email, 'nu' for phone, 'address' for address
* @param callable|null $normalizer function(string|null): string|null normalizes comparison key
* @return array<string,array>
*/
protected function dedupeGroupedItems(array $grouped, string $keyField, ?callable $normalizer = null): array
{
$seen = [];
$out = [];
foreach ($grouped as $grp => $payload) {
$raw = $payload[$keyField] ?? null;
$key = $normalizer ? $normalizer($raw) : (is_null($raw) ? null : trim((string) $raw));
$key = $key === '' ? '' : $key; // ensure empty string stays empty
$finger = is_null($key) ? '__NULL__' : (string) $key;
if (array_key_exists($finger, $seen)) {
// duplicate => skip
continue;
}
$seen[$finger] = true;
$out[$grp] = $payload;
}
return $out;
}
private function findOrCreatePersonId(array $p): ?int