changes 0230092025

This commit is contained in:
Simon Pocrnjič
2025-09-30 00:06:47 +02:00
parent 1fddf959f0
commit a2bb75fdcc
31 changed files with 2729 additions and 628 deletions
+98 -9
View File
@@ -5,15 +5,79 @@
class CsvImportService
{
/**
* Read the first line of a file; returns null on failure.
* Normalize a line to UTF-8 and strip BOM / control characters for robust splitting.
*/
public function readFirstLine(string $path): ?string
private function normalizeLine(string $line): string
{
// Strip UTF-8 BOM
if (str_starts_with($line, "\xEF\xBB\xBF")) {
$line = substr($line, 3);
}
// Detect UTF-16 BOMs
$hasNulls = strpos($line, "\x00") !== false;
if (str_starts_with($line, "\xFF\xFE")) {
// UTF-16LE BOM
$line = substr($line, 2);
$line = function_exists('mb_convert_encoding') ? @mb_convert_encoding($line, 'UTF-8', 'UTF-16LE') : preg_replace('/\x00/', '', $line);
} elseif (str_starts_with($line, "\xFE\xFF")) {
// UTF-16BE BOM
$line = substr($line, 2);
$line = function_exists('mb_convert_encoding') ? @mb_convert_encoding($line, 'UTF-8', 'UTF-16BE') : preg_replace('/\x00/', '', $line);
} elseif ($hasNulls) {
// Likely UTF-16 without BOM, try LE then BE
if (function_exists('mb_convert_encoding')) {
$try = @mb_convert_encoding($line, 'UTF-8', 'UTF-16LE');
if ($try !== false) {
$line = $try;
} else {
$try = @mb_convert_encoding($line, 'UTF-8', 'UTF-16BE');
if ($try !== false) {
$line = $try;
} else {
$line = preg_replace('/\x00/', '', $line);
}
}
} else {
$line = preg_replace('/\x00/', '', $line);
}
} else {
// Non UTF-16: try detect common encodings and convert to UTF-8 if needed
if (function_exists('mb_detect_encoding') && function_exists('mb_convert_encoding')) {
// Use default detection order for portability across environments
$enc = @mb_detect_encoding($line, null, true);
if ($enc && strtoupper($enc) !== 'UTF-8') {
$line = @mb_convert_encoding($line, 'UTF-8', $enc) ?: $line;
}
}
}
// Replace non-breaking space with regular space
$line = str_replace("\xC2\xA0", ' ', $line);
return $line;
}
/**
* Read the first meaningful (non-empty after normalization) line of a file; returns null on failure.
* Skips BOM-only lines and leading blank lines. Limits scanning to first 50 lines to be safe.
*/
public function readFirstMeaningfulLine(string $path): ?string
{
$fh = @fopen($path, 'r');
if (!$fh) return null;
$line = fgets($fh);
if (! $fh) {
return null;
}
$line = null;
$limit = 50;
while ($limit-- > 0 && ($raw = fgets($fh)) !== false) {
$normalized = $this->normalizeLine($raw);
if (trim($normalized) !== '') {
$line = $normalized;
break;
}
}
fclose($fh);
return $line === false ? null : $line;
return $line;
}
/**
@@ -24,14 +88,15 @@ public function readFirstLine(string $path): ?string
public function detectColumnsFromCsv(string $path, bool $hasHeader): array
{
// Use actual tab character for TSV; keep other common delimiters
$delims = [',',';','|',"\t"];
$delims = [',', ';', '|', "\t"];
$bestDelim = ',';
$bestCols = [];
$firstLine = $this->readFirstLine($path);
$firstLine = $this->readFirstMeaningfulLine($path);
if ($firstLine === null) {
return [$bestDelim, []];
}
// Already normalized by readFirstMeaningfulLine
$maxCount = 0;
foreach ($delims as $d) {
@@ -44,12 +109,27 @@ public function detectColumnsFromCsv(string $path, bool $hasHeader): array
}
}
if (!$hasHeader) {
// Fallback: if str_getcsv failed to split but we clearly see delimiters, do a simple explode
if ($maxCount <= 1) {
foreach (["\t", ';', ',', '|'] as $d) {
if (substr_count($firstLine, $d) >= 1) {
$parts = explode($d, $firstLine);
if (count($parts) > $maxCount) {
$bestDelim = $d;
$bestCols = $parts;
$maxCount = count($parts);
}
}
}
}
if (! $hasHeader) {
// return positional indices 0..N-1
$cols = [];
for ($i = 0; $i < $maxCount; $i++) {
$cols[] = (string) $i;
}
return [$bestDelim, $cols];
}
@@ -57,6 +137,7 @@ public function detectColumnsFromCsv(string $path, bool $hasHeader): array
$clean = array_map(function ($v) {
$v = trim((string) $v);
$v = preg_replace('/\s+/', ' ', $v);
return $v;
}, $bestCols);
@@ -69,16 +150,23 @@ public function detectColumnsFromCsv(string $path, bool $hasHeader): array
*/
public function parseColumnsFromCsv(string $path, string $delimiter, bool $hasHeader): array
{
$firstLine = $this->readFirstLine($path);
$firstLine = $this->readFirstMeaningfulLine($path);
if ($firstLine === null) {
return [];
}
// Already normalized by readFirstMeaningfulLine
$row = str_getcsv($firstLine, $delimiter);
$count = is_array($row) ? count($row) : 0;
// Fallback explode if str_getcsv failed to split
if ($count <= 1 && substr_count($firstLine, $delimiter) >= 1) {
$row = explode($delimiter, $firstLine);
$count = count($row);
}
if ($hasHeader) {
return array_map(function ($v) {
$v = trim((string) $v);
$v = preg_replace('/\s+/', ' ', $v);
return $v;
}, $row ?: []);
}
@@ -86,6 +174,7 @@ public function parseColumnsFromCsv(string $path, string $delimiter, bool $hasHe
for ($i = 0; $i < $count; $i++) {
$cols[] = (string) $i;
}
return $cols;
}
}
+588 -50
View File
@@ -4,12 +4,15 @@
use App\Models\Account;
use App\Models\AccountType;
use App\Models\Activity;
use App\Models\Client;
use App\Models\ClientCase;
use App\Models\Contract;
use App\Models\ContractType;
use App\Models\Decision;
use App\Models\Email;
use App\Models\Import;
use App\Models\ImportEntity;
use App\Models\ImportEvent;
use App\Models\ImportRow;
use App\Models\Person\AddressType;
@@ -57,6 +60,13 @@ public function process(Import $import, ?Authenticatable $user = null): array
->where('import_id', $import->id)
->get(['source_column', 'target_field', 'transform', 'apply_mode', 'options']);
// Load dynamic entity config
[$rootAliasMap, $fieldAliasMap, $validRoots] = $this->loadImportEntityConfig();
// Normalize aliases (plural/legacy roots, field names) before validation
$mappings = $this->normalizeMappings($mappings, $rootAliasMap, $fieldAliasMap);
// Validate mapping roots early to avoid silent failures due to typos
$this->validateMappingRoots($mappings, $validRoots);
$header = $import->meta['columns'] ?? null;
// Prefer explicitly chosen delimiter, then template meta, else detected
$delimiter = $import->meta['forced_delimiter']
@@ -66,6 +76,8 @@ public function process(Import $import, ?Authenticatable $user = null): array
$hasHeader = (bool) ($import->meta['has_header'] ?? true);
$path = Storage::disk($import->disk)->path($import->path);
// Note: Do not auto-detect or infer mappings/fields beyond what the template mapping provides
// Parse file and create import_rows with mapped_data
$fh = @fopen($path, 'r');
if (! $fh) {
@@ -95,12 +107,55 @@ public function process(Import $import, ?Authenticatable $user = null): array
if ($hasHeader) {
$first = fgetcsv($fh, 0, $delimiter);
$rowNum++;
// use actual detected header if not already stored
if (! $header) {
$header = array_map(fn ($v) => trim((string) $v), $first ?: []);
// Always use the actual header from the file for parsing
$header = array_map(fn ($v) => $this->sanitizeHeaderName((string) $v), $first ?: []);
// Heuristic: if header parsed as a single column but contains common delimiters, warn about mismatch
if (count($header) === 1) {
$rawHeader = $first[0] ?? '';
if (is_string($rawHeader) && (str_contains($rawHeader, ';') || str_contains($rawHeader, "\t"))) {
ImportEvent::create([
'import_id' => $import->id,
'user_id' => $user?->getAuthIdentifier(),
'event' => 'delimiter_mismatch_suspected',
'level' => 'warning',
'message' => 'Header parsed as a single column. Suspected delimiter mismatch. Set a forced delimiter in the template or import settings.',
'context' => [
'current_delimiter' => $delimiter,
'raw_header' => $rawHeader,
],
]);
}
}
// Preflight: warn if any mapped source columns are not present in the header (exact match)
$headerSet = [];
foreach ($header as $h) {
$headerSet[$h] = true;
}
$missingSources = [];
foreach ($mappings as $map) {
$src = (string) ($map->source_column ?? '');
if ($src !== '' && ! array_key_exists($src, $headerSet)) {
$missingSources[] = $src;
}
}
if (! empty($missingSources)) {
ImportEvent::create([
'import_id' => $import->id,
'user_id' => $user?->getAuthIdentifier(),
'event' => 'source_columns_missing_in_header',
'level' => 'warning',
'message' => 'Some mapped source columns are not present in the file header (exact match required).',
'context' => [
'missing' => $missingSources,
'header' => $header,
],
]);
}
}
// If mapping contains contract.reference, we require each row to successfully resolve/create a contract
$requireContract = $this->mappingIncludes($mappings, 'contract.reference');
while (($row = fgetcsv($fh, 0, $delimiter)) !== false) {
$rowNum++;
$total++;
@@ -108,6 +163,8 @@ public function process(Import $import, ?Authenticatable $user = null): array
$rawAssoc = $this->buildRowAssoc($row, $header);
[$recordType, $mapped] = $this->applyMappings($rawAssoc, $mappings);
// Do not auto-derive or fallback values; only use explicitly mapped fields
$importRow = ImportRow::create([
'import_id' => $import->id,
'row_number' => $rowNum,
@@ -122,6 +179,31 @@ public function process(Import $import, ?Authenticatable $user = null): array
if (isset($mapped['contract'])) {
$contractResult = $this->upsertContractChain($import, $mapped, $mappings);
if ($contractResult['action'] === 'skipped') {
// Even if no contract fields were updated, we may still need to apply template meta
// like attaching a segment or creating an activity. Do that if we have the contract.
if (isset($contractResult['contract']) && $contractResult['contract'] instanceof Contract) {
try {
$this->postContractActions($import, $contractResult['contract']);
ImportEvent::create([
'import_id' => $import->id,
'user_id' => $user?->getAuthIdentifier(),
'import_row_id' => $importRow->id,
'event' => 'post_contract_actions_applied',
'level' => 'info',
'message' => 'Applied template post-actions on existing contract.',
'context' => ['contract_id' => $contractResult['contract']->id],
]);
} catch (\Throwable $e) {
ImportEvent::create([
'import_id' => $import->id,
'user_id' => $user?->getAuthIdentifier(),
'import_row_id' => $importRow->id,
'event' => 'post_contract_action_failed',
'level' => 'warning',
'message' => $e->getMessage(),
]);
}
}
$skipped++;
$importRow->update(['status' => 'skipped']);
ImportEvent::create([
@@ -148,15 +230,64 @@ public function process(Import $import, ?Authenticatable $user = null): array
'message' => ucfirst($contractResult['action']).' contract',
'context' => ['id' => $contractResult['contract']->id],
]);
// Post-contract actions from template/import meta
try {
$this->postContractActions($import, $contractResult['contract']);
} catch (\Throwable $e) {
ImportEvent::create([
'import_id' => $import->id,
'user_id' => $user?->getAuthIdentifier(),
'import_row_id' => $importRow->id,
'event' => 'post_contract_action_failed',
'level' => 'warning',
'message' => $e->getMessage(),
]);
}
} else {
$invalid++;
$importRow->update(['status' => 'invalid', 'errors' => [$contractResult['message'] ?? 'Contract processing failed']]);
}
}
// Enforce hard requirement: if template mapped contract.reference but we didn't resolve/create a contract, mark row invalid and continue
if ($requireContract) {
$contractEnsured = false;
if ($contractResult && isset($contractResult['contract']) && $contractResult['contract'] instanceof Contract) {
$contractEnsured = true;
}
if (! $contractEnsured) {
$srcCol = $this->findSourceColumnFor($mappings, 'contract.reference');
$rawVal = $srcCol !== null ? ($rawAssoc[$srcCol] ?? null) : null;
$extra = $srcCol !== null ? ' Source column: "'.$srcCol.'" value: '.(is_null($rawVal) || $rawVal === '' ? '(empty)' : (is_scalar($rawVal) ? (string) $rawVal : json_encode($rawVal))) : '';
$msg = 'Row '.$rowNum.': Contract was required (contract.reference mapped) but not created/resolved. '.($contractResult['message'] ?? '').$extra;
// Avoid double-counting invalid if already set by contract processing
if ($importRow->status !== 'invalid') {
$invalid++;
$importRow->update(['status' => 'invalid', 'errors' => [$msg]]);
}
ImportEvent::create([
'import_id' => $import->id,
'user_id' => $user?->getAuthIdentifier(),
'import_row_id' => $importRow->id,
'event' => 'row_invalid',
'level' => 'error',
'message' => $msg,
]);
// Skip further processing for this row
continue;
}
}
// Accounts
$accountResult = null;
if (isset($mapped['account'])) {
// If a contract was just created or resolved above, pass its id to account mapping for this row
if ($contractResult && isset($contractResult['contract']) && $contractResult['contract'] instanceof Contract) {
$mapped['account']['contract_id'] = $contractResult['contract']->id;
}
$accountResult = $this->upsertAccount($import, $mapped, $mappings);
if ($accountResult['action'] === 'skipped') {
$skipped++;
@@ -168,6 +299,7 @@ public function process(Import $import, ?Authenticatable $user = null): array
'event' => 'row_skipped',
'level' => 'info',
'message' => $accountResult['message'] ?? 'Skipped (no changes).',
'context' => $accountResult['context'] ?? null,
]);
} elseif ($accountResult['action'] === 'inserted' || $accountResult['action'] === 'updated') {
$imported++;
@@ -191,7 +323,7 @@ public function process(Import $import, ?Authenticatable $user = null): array
}
}
// Contacts: resolve person strictly via Contract -> ClientCase -> Person, contacts, or identifiers
// Contacts: resolve person via Contract/Account chain, client_case.client_ref, contacts, or identifiers
$personIdForRow = null;
// Prefer person from contract created/updated above
if ($contractResult && isset($contractResult['contract']) && $contractResult['contract'] instanceof Contract) {
@@ -222,6 +354,16 @@ public function process(Import $import, ?Authenticatable $user = null): array
}
}
}
// Resolve by client_case.client_ref for this client (prefer reusing existing person)
if (! $personIdForRow && $import->client_id && ! empty($mapped['client_case']['client_ref'] ?? null)) {
$cc = ClientCase::where('client_id', $import->client_id)
->where('client_ref', $mapped['client_case']['client_ref'])
->first();
if ($cc) {
$personIdForRow = $cc->person_id ?: null;
}
}
// Resolve by contact values next
if (! $personIdForRow) {
$emailVal = trim((string) ($mapped['email']['value'] ?? ''));
@@ -240,6 +382,22 @@ public function process(Import $import, ?Authenticatable $user = null): array
}
// If still no person but we have any contact value, auto-create a minimal person
// BUT if we can map to an existing client_case by client_ref, reuse that case and set person there (avoid separate person rows)
if (! $personIdForRow && ($emailVal !== '' || $phoneNu !== '' || $addrLine !== '')) {
if ($import->client_id && ! empty($mapped['client_case']['client_ref'] ?? null)) {
$cc = ClientCase::where('client_id', $import->client_id)
->where('client_ref', $mapped['client_case']['client_ref'])
->first();
if ($cc) {
$pid = $cc->person_id ?: $this->createMinimalPersonId();
if (! $cc->person_id) {
$cc->person_id = $pid;
$cc->save();
}
$personIdForRow = $pid;
}
}
}
if (! $personIdForRow && ($emailVal !== '' || $phoneNu !== '' || $addrLine !== '')) {
$personIdForRow = $this->createMinimalPersonId();
ImportEvent::create([
@@ -384,12 +542,29 @@ private function applyMappings(array $raw, $mappings): array
}
$value = $raw[$src] ?? null;
// very basic transforms
if ($map->transform === 'trim') {
$value = is_string($value) ? trim($value) : $value;
}
if ($map->transform === 'upper') {
$value = is_string($value) ? strtoupper($value) : $value;
// Transform chain support: e.g. "trim|decimal" or "upper|alnum"
$transform = (string) ($map->transform ?? '');
if ($transform !== '') {
$parts = explode('|', $transform);
foreach ($parts as $t) {
$t = trim($t);
if ($t === 'trim') {
$value = is_string($value) ? trim($value) : $value;
} elseif ($t === 'upper') {
$value = is_string($value) ? strtoupper($value) : $value;
} elseif ($t === 'lower') {
$value = is_string($value) ? strtolower($value) : $value;
} elseif ($t === 'digits' || $t === 'numeric') {
$value = is_string($value) ? preg_replace('/[^0-9]/', '', $value) : $value;
} elseif ($t === 'decimal') {
$value = is_string($value) ? $this->normalizeDecimal($value) : $value;
} elseif ($t === 'alnum') {
$value = is_string($value) ? preg_replace('/[^A-Za-z0-9]/', '', $value) : $value;
} elseif ($t === 'ref') {
// Reference safe: keep letters+digits only, uppercase
$value = is_string($value) ? strtoupper(preg_replace('/[^A-Za-z0-9]/', '', $value)) : $value;
}
}
}
// detect record type from first segment, e.g., "account.balance_amount"
@@ -423,6 +598,16 @@ private function upsertAccount(Import $import, array $mapped, $mappings): array
$acc = $mapped['account'] ?? [];
$contractId = $acc['contract_id'] ?? null;
$reference = $acc['reference'] ?? null;
// Determine if the template includes any contract mappings; if not, do not create contracts here
$hasContractRoot = $this->mappingsContainRoot($mappings, 'contract');
// Normalize references (remove spaces) for consistent matching
if (! is_null($reference)) {
$reference = preg_replace('/\s+/', '', trim((string) $reference));
$acc['reference'] = $reference;
}
if (! empty($acc['contract_reference'] ?? null)) {
$acc['contract_reference'] = preg_replace('/\s+/', '', trim((string) $acc['contract_reference']));
}
// If contract_id not provided, attempt to resolve by contract reference for the selected client
if (! $contractId) {
$contractRef = $acc['contract_reference'] ?? ($mapped['contract']['reference'] ?? null);
@@ -436,25 +621,17 @@ private function upsertAccount(Import $import, array $mapped, $mappings): array
->first();
if ($existingContract) {
$contractId = $existingContract->id;
} else {
// 2) Not found: attempt to resolve debtor via identifiers or provided person, then create case+contract
// Try strong identifiers first
} elseif ($hasContractRoot) {
// Only create a new contract if the template explicitly includes contract mappings
// Resolve debtor via identifiers or provided person
$personId = $this->findPersonIdByIdentifiers($mapped['person'] ?? []);
// Create from provided person data if unresolved
if (! $personId) {
$personId = $this->findOrCreatePersonId($mapped['person'] ?? []);
}
// Last resort, create minimal
if (! $personId) {
$personId = $this->createMinimalPersonId();
}
// Use the selected client for this import to tie the case/contract
if (! $clientId) {
return ['action' => 'skipped', 'message' => 'Client required to create contract'];
}
$resolvedClientId = $clientId;
$clientCaseId = $this->findOrCreateClientCaseId($resolvedClientId, $personId);
// Build minimal/new contract
$clientCaseId = $this->findOrCreateClientCaseId($clientId, $personId, $mapped['client_case']['client_ref'] ?? null);
$contractFields = $mapped['contract'] ?? [];
$newContractData = [
'client_case_id' => $clientCaseId,
@@ -465,11 +642,13 @@ private function upsertAccount(Import $import, array $mapped, $mappings): array
$newContractData[$k] = $contractFields[$k];
}
}
// ensure required fields on contracts
$newContractData['start_date'] = $newContractData['start_date'] ?? now()->toDateString();
$newContractData['type_id'] = $newContractData['type_id'] ?? $this->getDefaultContractTypeId();
$createdContract = Contract::create($newContractData);
$contractId = $createdContract->id;
} else {
// Do not create contracts implicitly when not mapped in the template
$contractId = null;
}
if ($contractId) {
$acc['contract_id'] = $contractId;
@@ -477,17 +656,35 @@ private function upsertAccount(Import $import, array $mapped, $mappings): array
}
}
}
// Default account.reference to contract reference if missing
if (! $reference) {
$contractRef = $acc['contract_reference'] ?? ($mapped['contract']['reference'] ?? null);
if ($contractRef) {
$reference = $contractRef;
// Fallback: if account.reference is empty but contract.reference is present, use it
if ((is_null($reference) || $reference === '') && ! empty($mapped['contract']['reference'] ?? null)) {
$reference = preg_replace('/\s+/', '', trim((string) $mapped['contract']['reference']));
if ($reference !== '') {
$acc['reference'] = $reference;
$mapped['account'] = $acc;
}
}
// Do not default or infer account.reference from other fields; rely solely on mapped values
if (! $contractId || ! $reference) {
return ['action' => 'skipped', 'message' => 'Missing contract_id/reference'];
$issues = [];
if (! $contractId) {
$issues[] = 'contract_id unresolved';
}
if (! $reference) {
$issues[] = 'account.reference empty';
}
$candidateContractRef = $acc['contract_reference'] ?? ($mapped['contract']['reference'] ?? null);
return [
'action' => 'skipped',
'message' => 'Prerequisite missing: '.implode(' & ', $issues),
'context' => [
'has_contract_root_mapped' => $hasContractRoot,
'candidate_contract_reference' => $candidateContractRef,
'account_reference_provided' => $reference,
'account_fields_present' => array_keys(array_filter($acc, fn ($v) => ! is_null($v) && $v !== '')),
],
];
}
$existing = Account::query()
@@ -498,6 +695,7 @@ private function upsertAccount(Import $import, array $mapped, $mappings): array
// Build applyable data based on apply_mode
$applyInsert = [];
$applyUpdate = [];
$applyModeByField = [];
foreach ($mappings as $map) {
if (! $map->target_field) {
continue;
@@ -511,7 +709,11 @@ private function upsertAccount(Import $import, array $mapped, $mappings): array
continue;
}
$value = $acc[$field] ?? null;
if (in_array($field, ['balance_amount','initial_amount'], true) && is_string($value)) {
$value = $this->normalizeDecimal($value);
}
$mode = $map->apply_mode ?? 'both';
$applyModeByField[$field] = $mode;
if (in_array($mode, ['insert', 'both'])) {
$applyInsert[$field] = $value;
}
@@ -535,6 +737,15 @@ private function upsertAccount(Import $import, array $mapped, $mappings): array
// also include contract hints for downstream contact resolution
return ['action' => 'updated', 'account' => $existing, 'contract_id' => $contractId];
} else {
// On insert: if initial_amount is not provided but balance_amount is, allow defaulting
// Only when the mapping for initial_amount is 'insert' or 'both', or unmapped (null).
$initMode = $applyModeByField['initial_amount'] ?? null;
if ((! array_key_exists('initial_amount', $applyInsert) || is_null($applyInsert['initial_amount'] ?? null))
&& array_key_exists('balance_amount', $applyInsert)
&& ($applyInsert['balance_amount'] !== null && $applyInsert['balance_amount'] !== '')
&& ($initMode === null || in_array($initMode, ['insert','both'], true))) {
$applyInsert['initial_amount'] = $applyInsert['balance_amount'];
}
if (empty($applyInsert)) {
return ['action' => 'skipped', 'message' => 'No fields marked for insert'];
}
@@ -552,6 +763,18 @@ private function upsertAccount(Import $import, array $mapped, $mappings): array
}
}
private function mappingsContainRoot($mappings, string $root): bool
{
foreach ($mappings as $map) {
$target = (string) ($map->target_field ?? '');
if ($target !== '' && str_starts_with($target, $root.'.')) {
return true;
}
}
return false;
}
private function findPersonIdByIdentifiers(array $p): ?int
{
$tax = $p['tax_number'] ?? null;
@@ -576,6 +799,10 @@ private function upsertContractChain(Import $import, array $mapped, $mappings):
{
$contractData = $mapped['contract'] ?? [];
$reference = $contractData['reference'] ?? null;
if (! is_null($reference)) {
$reference = preg_replace('/\s+/', '', trim((string) $reference));
$contractData['reference'] = $reference;
}
if (! $reference) {
return ['action' => 'invalid', 'message' => 'Missing contract.reference'];
}
@@ -605,26 +832,53 @@ private function upsertContractChain(Import $import, array $mapped, $mappings):
// If we still need to insert, we must resolve clientCaseId, but avoid creating new person/case unless necessary
if (! $existing && ! $clientCaseId) {
// Resolve by identifiers or provided person; do not use Client->person
$personId = null;
if (! empty($mapped['person'] ?? [])) {
$personId = $this->findPersonIdByIdentifiers($mapped['person']);
if (! $personId) {
$personId = $this->findOrCreatePersonId($mapped['person']);
$clientRef = $mapped['client_case']['client_ref'] ?? null;
// First, if we have a client and client_ref, try to reuse existing case to avoid creating extra persons
if ($clientId && $clientRef) {
$cc = ClientCase::where('client_id', $clientId)->where('client_ref', $clientRef)->first();
if ($cc) {
// Reuse this case
$clientCaseId = $cc->id;
// If case has no person yet and we have mapped person identifiers/data, set it once
if (! $cc->person_id) {
$pid = null;
if (! empty($mapped['person'] ?? [])) {
$pid = $this->findPersonIdByIdentifiers($mapped['person']);
if (! $pid) {
$pid = $this->findOrCreatePersonId($mapped['person']);
}
}
if (! $pid) {
$pid = $this->createMinimalPersonId();
}
$cc->person_id = $pid;
$cc->save();
}
}
}
// As a last resort, create a minimal person for this client
if ($clientId && ! $personId) {
$personId = $this->createMinimalPersonId();
}
if ($clientId && $personId) {
$clientCaseId = $this->findOrCreateClientCaseId($clientId, $personId);
} elseif ($personId) {
// require an import client to attach case/contract
return ['action' => 'invalid', 'message' => 'Import must be linked to a client to create a case'];
} else {
return ['action' => 'invalid', 'message' => 'Unable to resolve client_case (need import client)'];
if (! $clientCaseId) {
// Resolve by identifiers or provided person; do not use Client->person
$personId = null;
if (! empty($mapped['person'] ?? [])) {
$personId = $this->findPersonIdByIdentifiers($mapped['person']);
if (! $personId) {
$personId = $this->findOrCreatePersonId($mapped['person']);
}
}
// As a last resort, create a minimal person for this client
if ($clientId && ! $personId) {
$personId = $this->createMinimalPersonId();
}
if ($clientId && $personId) {
$clientCaseId = $this->findOrCreateClientCaseId($clientId, $personId, $clientRef);
} elseif ($personId) {
// require an import client to attach case/contract
return ['action' => 'invalid', 'message' => 'Import must be linked to a client to create a case'];
} else {
return ['action' => 'invalid', 'message' => 'Unable to resolve client_case (need import client)'];
}
}
}
@@ -644,6 +898,9 @@ private function upsertContractChain(Import $import, array $mapped, $mappings):
continue;
}
$value = $contractData[$field] ?? null;
if ($field === 'reference' && ! is_null($value)) {
$value = preg_replace('/\s+/', '', trim((string) $value));
}
$mode = $map->apply_mode ?? 'both';
if (in_array($mode, ['insert', 'both'])) {
$applyInsert[$field] = $value;
@@ -655,11 +912,12 @@ private function upsertContractChain(Import $import, array $mapped, $mappings):
if ($existing) {
if (empty($applyUpdate)) {
return ['action' => 'skipped', 'message' => 'No contract fields marked for update'];
// Return existing contract reference even when skipped so callers can treat as resolved
return ['action' => 'skipped', 'message' => 'No contract fields marked for update', 'contract' => $existing];
}
$changes = array_filter($applyUpdate, fn ($v) => ! is_null($v));
if (empty($changes)) {
return ['action' => 'skipped', 'message' => 'No non-null contract changes'];
return ['action' => 'skipped', 'message' => 'No non-null contract changes', 'contract' => $existing];
}
$existing->fill($changes);
$existing->save();
@@ -681,6 +939,197 @@ private function upsertContractChain(Import $import, array $mapped, $mappings):
}
}
private function sanitizeHeaderName(string $v): string
{
// Strip UTF-8 BOM and trim whitespace/control characters
$v = preg_replace('/^\xEF\xBB\xBF/', '', $v) ?? $v;
return trim($v);
}
private function findSourceColumnFor($mappings, string $targetField): ?string
{
foreach ($mappings as $map) {
if ((string) ($map->target_field ?? '') === $targetField) {
$src = (string) ($map->source_column ?? '');
return $src !== '' ? $src : null;
}
}
return null;
}
// Removed auto-detection helpers by request: no pattern scanning or fallback derivation
private function normalizeDecimal(string $raw): string
{
// Keep digits, comma, dot, and minus to detect separators
$s = preg_replace('/[^0-9,\.-]/', '', $raw) ?? '';
$s = trim($s);
if ($s === '') {
return $s;
}
$lastComma = strrpos($s, ',');
$lastDot = strrpos($s, '.');
// Determine decimal separator by last occurrence
$decimalSep = null;
if ($lastComma !== false || $lastDot !== false) {
if ($lastComma === false) {
$decimalSep = '.';
} elseif ($lastDot === false) {
$decimalSep = ',';
} else {
$decimalSep = $lastComma > $lastDot ? ',' : '.';
}
}
// Remove all thousand separators (the other one) and unify decimal to '.'
if ($decimalSep === ',') {
// remove all dots
$s = str_replace('.', '', $s);
// replace last comma with dot
$pos = strrpos($s, ',');
if ($pos !== false) {
$s[$pos] = '.';
}
// remove any remaining commas (unlikely)
$s = str_replace(',', '', $s);
} elseif ($decimalSep === '.') {
// remove all commas
$s = str_replace(',', '', $s);
// dot already decimal
} else {
// no decimal separator: remove commas/dots entirely
$s = str_replace([',', '.'], '', $s);
}
// Collapse multiple minus signs, keep leading only
$s = ltrim($s, '+');
$neg = false;
if (str_starts_with($s, '-')) {
$neg = true;
$s = ltrim($s, '-');
}
// Remove any stray minus signs
$s = str_replace('-', '', $s);
if ($neg) {
$s = '-'.$s;
}
return $s;
}
/**
* Ensure mapping roots are recognized; fail fast if unknown roots found.
*/
private function validateMappingRoots($mappings, array $validRoots): void
{
foreach ($mappings as $map) {
$target = (string) ($map->target_field ?? '');
if ($target === '') {
continue;
}
$root = explode('.', $target)[0];
if (! in_array($root, $validRoots, true)) {
// Common typos guidance
$hint = '';
if (str_starts_with($root, 'contract')) {
$hint = ' Did you mean "contract"?';
}
throw new \InvalidArgumentException('Unknown mapping root "'.$root.'" in target_field "'.$target.'".'.$hint);
}
}
}
private function mappingIncludes($mappings, string $targetField): bool
{
foreach ($mappings as $map) {
if ((string) ($map->target_field ?? '') === $targetField) {
return true;
}
}
return false;
}
/**
* Normalize mapping target_field to canonical forms.
* Examples:
* - contracts.reference => contract.reference
* - accounts.balance_amount => account.balance_amount
* - person_phones.nu => phone.nu
* - person_addresses.address => address.address
* - emails.email|emails.value => email.value
*/
private function normalizeMappings($mappings, array $rootAliasMap, array $fieldAliasMap)
{
$normalized = [];
foreach ($mappings as $map) {
$clone = clone $map;
$clone->target_field = $this->normalizeTargetField((string) ($map->target_field ?? ''), $rootAliasMap, $fieldAliasMap);
$normalized[] = $clone;
}
return collect($normalized);
}
private function normalizeTargetField(string $target, array $rootAliasMap, array $fieldAliasMap): string
{
if ($target === '') {
return $target;
}
$parts = explode('.', $target);
$root = $parts[0] ?? '';
$field = $parts[1] ?? null;
// Root aliases (plural to canonical) from DB
$root = $rootAliasMap[$root] ?? $root;
// Field aliases per root from DB
$aliases = $fieldAliasMap[$root] ?? [];
if ($field === null && isset($aliases['__default'])) {
$field = $aliases['__default'];
} elseif (isset($aliases[$field])) {
$field = $aliases[$field];
}
// Rebuild
if ($field !== null) {
return $root.'.'.$field;
}
return $root;
}
private function loadImportEntityConfig(): array
{
$entities = ImportEntity::all();
$rootAliasMap = [];
$fieldAliasMap = [];
$validRoots = [];
foreach ($entities as $ent) {
$canonical = $ent->canonical_root;
$validRoots[] = $canonical;
foreach ((array) ($ent->aliases ?? []) as $alias) {
$rootAliasMap[$alias] = $canonical;
}
// Also ensure canonical maps to itself
$rootAliasMap[$canonical] = $canonical;
$aliases = (array) ($ent->field_aliases ?? []);
// Allow default field per entity via '__default'
if (is_array($ent->fields) && count($ent->fields)) {
$aliases['__default'] = $aliases['__default'] ?? null;
}
$fieldAliasMap[$canonical] = $aliases;
}
// sensible defaults when DB empty
if (empty($validRoots)) {
$validRoots = ['person', 'contract', 'account', 'address', 'phone', 'email', 'client_case'];
}
return [$rootAliasMap, $fieldAliasMap, $validRoots];
}
private function findOrCreatePersonId(array $p): ?int
{
// Basic dedup: by tax_number, ssn, else full_name
@@ -776,14 +1225,30 @@ private function findOrCreateClientId(int $personId): int
return Client::create(['person_id' => $personId])->id;
}
private function findOrCreateClientCaseId(int $clientId, int $personId): int
private function findOrCreateClientCaseId(int $clientId, int $personId, ?string $clientRef = null): int
{
// Prefer existing by client_ref if provided
if ($clientRef) {
$cc = ClientCase::where('client_id', $clientId)
->where('client_ref', $clientRef)
->first();
if ($cc) {
// Ensure person_id is set (if missing) when matching by client_ref
if (! $cc->person_id) {
$cc->person_id = $personId;
$cc->save();
}
return $cc->id;
}
}
// Fallback: by (client_id, person_id)
$cc = ClientCase::where('client_id', $clientId)->where('person_id', $personId)->first();
if ($cc) {
return $cc->id;
}
return ClientCase::create(['client_id' => $clientId, 'person_id' => $personId])->id;
return ClientCase::create(['client_id' => $clientId, 'person_id' => $personId, 'client_ref' => $clientRef])->id;
}
private function upsertEmail(int $personId, array $emailData, $mappings): array
@@ -948,4 +1413,77 @@ private function upsertPhone(int $personId, array $phoneData, $mappings): array
return ['action' => 'inserted', 'phone' => $created];
}
}
/**
* After a contract is inserted/updated, attach default segment and create an activity
* using decision_id from import/template meta. Activity note includes template name.
*/
private function postContractActions(Import $import, Contract $contract): void
{
$meta = $import->meta ?? [];
$segmentId = (int) ($meta['segment_id'] ?? 0);
$decisionId = (int) ($meta['decision_id'] ?? 0);
$templateName = (string) ($meta['template_name'] ?? optional($import->template)->name ?? '');
$actionId = (int) ($meta['action_id'] ?? 0);
// Attach segment to contract as the main (active) segment if provided
if ($segmentId > 0) {
// Ensure the segment exists on the client case and is active
$ccSeg = \DB::table('client_case_segment')
->where('client_case_id', $contract->client_case_id)
->where('segment_id', $segmentId)
->first();
if (! $ccSeg) {
\DB::table('client_case_segment')->insert([
'client_case_id' => $contract->client_case_id,
'segment_id' => $segmentId,
'active' => true,
'created_at' => now(),
'updated_at' => now(),
]);
} elseif (! $ccSeg->active) {
\DB::table('client_case_segment')
->where('id', $ccSeg->id)
->update(['active' => true, 'updated_at' => now()]);
}
// Deactivate all other segments for this contract to make this the main one
\DB::table('contract_segment')
->where('contract_id', $contract->id)
->where('segment_id', '!=', $segmentId)
->update(['active' => false, 'updated_at' => now()]);
// Upsert the selected segment as active for this contract
$pivot = \DB::table('contract_segment')
->where('contract_id', $contract->id)
->where('segment_id', $segmentId)
->first();
if ($pivot) {
if (! $pivot->active) {
\DB::table('contract_segment')
->where('id', $pivot->id)
->update(['active' => true, 'updated_at' => now()]);
}
} else {
\DB::table('contract_segment')->insert([
'contract_id' => $contract->id,
'segment_id' => $segmentId,
'active' => true,
'created_at' => now(),
'updated_at' => now(),
]);
}
}
// Create activity if decision provided
if ($decisionId > 0) {
Activity::create([
'decision_id' => $decisionId,
'action_id' => $actionId > 0 ? $actionId : null,
'contract_id' => $contract->id,
'client_case_id' => $contract->client_case_id,
'note' => trim('Imported via template'.($templateName ? ': '.$templateName : '')),
]);
}
}
}