sipo/Teren-app
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

changes 0230092025

Browse Source
This commit is contained in:
Simon Pocrnjič
2025-09-30 00:06:47 +02:00
parent 1fddf959f0
commit a2bb75fdcc
31 changed files with 2729 additions and 628 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/Http/Controllers/ImportController.php
+55 -34
View File
@@ -4,16 +4,15 @@
use App\Models\Client;
use App\Models\Import;
use App\Models\ImportTemplate;
use App\Models\ImportRow;
use App\Models\ImportEvent;
use App\Models\ImportTemplate;
use App\Services\CsvImportService;
use App\Services\ImportProcessor;
use Illuminate\Http\Request;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Storage;
use Illuminate\Support\Str;
use Inertia\Inertia;
use App\Services\CsvImportService;
class ImportController extends Controller
{
@@ -64,7 +63,7 @@ public function index(Request $request)
'full_name' => $imp->client->person->full_name,
] : null,
] : null,
'template' => $imp->template ? [ 'id' => $imp->template->id, 'name' => $imp->template->name ] : null,
'template' => $imp->template ? ['id' => $imp->template->id, 'name' => $imp->template->name] : null,
];
}, $imports['data']);
@@ -99,7 +98,6 @@ public function create(Request $request)
DB::raw('person.full_name as name'),
]);
return Inertia::render('Imports/Create', [
'templates' => $templates,
'clients' => $clients,
@@ -129,7 +127,7 @@ public function store(Request $request)
// Resolve client_uuid to client_id if provided
$clientId = null;
if (!empty($validated['client_uuid'] ?? null)) {
if (! empty($validated['client_uuid'] ?? null)) {
$clientId = Client::where('uuid', $validated['client_uuid'])->value('id');
}
@@ -163,6 +161,7 @@ public function process(Import $import, Request $request, ImportProcessor $proce
{
$import->update(['status' => 'validating', 'started_at' => now()]);
$result = $processor->process($import, user: $request->user());
return response()->json($result);
}
@@ -188,25 +187,44 @@ public function columns(Request $request, Import $import, CsvImportService $csv)
if ($tplDelimiter) {
$explicitDelimiter = (string) $tplDelimiter;
}
} elseif (!empty($import->meta['forced_delimiter'] ?? null)) {
} elseif (! empty($import->meta['forced_delimiter'] ?? null)) {
$explicitDelimiter = (string) $import->meta['forced_delimiter'];
}
// Only implement CSV/TSV detection for now; others can be added later
if (!in_array($import->source_type, ['csv','txt'])) {
return response()->json([
'columns' => [],
'note' => 'Column preview supported for CSV/TXT at this step.',
]);
// Prefer CSV/TXT; if source_type is unknown, attempt best-effort based on file extension
$treatAsText = in_array($import->source_type, ['csv', 'txt']);
if (! $treatAsText) {
$orig = strtolower(pathinfo($import->original_name ?? '', PATHINFO_EXTENSION));
if (in_array($orig, ['csv', 'txt'])) {
$treatAsText = true;
}
}
$fullPath = Storage::disk($import->disk)->path($import->path);
if ($explicitDelimiter !== null && $explicitDelimiter !== '') {
$columns = $csv->parseColumnsFromCsv($fullPath, $explicitDelimiter, $hasHeader);
$delimiter = $explicitDelimiter;
} else {
[$delimiter, $columns] = $csv->detectColumnsFromCsv($fullPath, $hasHeader);
}
$fullPath = Storage::disk($import->disk)->path($import->path);
$note = '';
if ($treatAsText) {
if ($explicitDelimiter !== null && $explicitDelimiter !== '') {
$columns = $csv->parseColumnsFromCsv($fullPath, $explicitDelimiter, $hasHeader);
$delimiter = $explicitDelimiter;
} else {
[$delimiter, $columns] = $csv->detectColumnsFromCsv($fullPath, $hasHeader);
// Backstop: if single column but file clearly has separators, try common ones
if (is_array($columns) && count($columns) <= 1) {
foreach ([';', "\t", '|', ' ', ','] as $try) {
$alt = $csv->parseColumnsFromCsv($fullPath, $try, $hasHeader);
if (is_array($alt) && count($alt) > 1) {
$delimiter = $try;
$columns = $alt;
$note = 'Delimiter auto-detection backstopped to '.json_encode($try);
break;
}
}
}
}
} else {
// Best-effort: try detect anyway
[$delimiter, $columns] = $csv->detectColumnsFromCsv($fullPath, $hasHeader);
}
// Save meta
$meta = $import->meta ?? [];
@@ -225,6 +243,7 @@ public function columns(Request $request, Import $import, CsvImportService $csv)
'columns' => $columns,
'has_header' => $hasHeader,
'detected_delimiter' => $delimiter,
'note' => $note,
]);
}
@@ -236,9 +255,9 @@ public function saveMappings(Request $request, Import $import)
$data = $request->validate([
'mappings' => 'required|array',
'mappings.*.source_column' => 'required|string',
'mappings.*.entity' => 'nullable|string|in:person,person_addresses,person_phones,emails,accounts,contracts',
'mappings.*.entity' => 'nullable|string|in:person,person_addresses,person_phones,emails,accounts,contracts,client_cases',
'mappings.*.target_field' => 'required|string',
'mappings.*.transform' => 'nullable|string|in:trim,upper,lower',
'mappings.*.transform' => 'nullable|string|in:trim,upper,lower,decimal,ref',
'mappings.*.apply_mode' => 'nullable|string|in:insert,update,both',
'mappings.*.options' => 'nullable|array',
]);
@@ -247,14 +266,14 @@ public function saveMappings(Request $request, Import $import)
$now = now();
$existing = \DB::table('import_mappings')
->where('import_id', $import->id)
->get(['id','source_column','position']);
->get(['id', 'source_column', 'position']);
$bySource = [];
$dupes = [];
foreach ($existing as $row) {
$src = (string) $row->source_column;
if (!array_key_exists($src, $bySource)) {
$bySource[$src] = [ 'id' => $row->id, 'position' => $row->position ];
if (! array_key_exists($src, $bySource)) {
$bySource[$src] = ['id' => $row->id, 'position' => $row->position];
} else {
$dupes[$src] = ($dupes[$src] ?? []);
$dupes[$src][] = $row->id;
@@ -262,7 +281,9 @@ public function saveMappings(Request $request, Import $import)
}
$basePosition = (int) (\DB::table('import_mappings')->where('import_id', $import->id)->max('position') ?? -1);
$inserted = 0; $updated = 0; $deduped = 0;
$inserted = 0;
$updated = 0;
$deduped = 0;
foreach ($data['mappings'] as $pos => $m) {
$src = (string) $m['source_column'];
@@ -281,7 +302,7 @@ public function saveMappings(Request $request, Import $import)
\DB::table('import_mappings')->where('id', $bySource[$src]['id'])->update($payload);
$updated++;
// Remove duplicates if any
if (!empty($dupes[$src])) {
if (! empty($dupes[$src])) {
$deleted = \DB::table('import_mappings')->whereIn('id', $dupes[$src])->delete();
$deduped += (int) $deleted;
unset($dupes[$src]);
@@ -325,8 +346,9 @@ public function getMappings(Import $import)
'transform',
'apply_mode',
'options',
'position'
'position',
]);
return response()->json(['mappings' => $rows]);
}
@@ -339,7 +361,8 @@ public function getEvents(Import $import)
->where('import_id', $import->id)
->orderByDesc('id')
->limit($limit)
->get(['id','created_at','level','event','message','import_row_id','context']);
->get(['id', 'created_at', 'level', 'event', 'message', 'import_row_id', 'context']);
return response()->json(['events' => $events]);
}
@@ -363,7 +386,6 @@ public function show(Import $import)
'clients.uuid as client_uuid',
]);
$clients = Client::query()
->join('person', 'person.id', '=', 'clients.person_id')
->orderBy('person.full_name')
@@ -371,7 +393,7 @@ public function show(Import $import)
->get([
'clients.id',
'clients.uuid',
'person.full_name as name'
'person.full_name as name',
]);
// Import client
@@ -383,7 +405,6 @@ public function show(Import $import)
'person.full_name as name',
]);
// Render a dedicated page to continue the import
return Inertia::render('Imports/Import', [
'import' => [
@@ -398,11 +419,11 @@ public function show(Import $import)
'imported_rows' => $import->imported_rows,
'invalid_rows' => $import->invalid_rows,
'valid_rows' => $import->valid_rows,
'finished_at' => $import->finished_at
'finished_at' => $import->finished_at,
],
'templates' => $templates,
'clients' => $clients,
'client' => $client
'client' => $client,
]);
}
}
app/Http/Controllers/ImportEntityController.php
+115
View File
@@ -0,0 +1,115 @@
<?php
namespace App\Http\Controllers;
use App\Models\ImportEntity;
use Illuminate\Http\Request;
class ImportEntityController extends Controller
{
public function index()
{
$entities = ImportEntity::query()
->orderByRaw("(ui->>'order')::int nulls last")
->get();
// Fallback: if no entities are seeded yet, return a sensible default set
if ($entities->isEmpty()) {
$entities = collect([
[
'key' => 'person',
'canonical_root' => 'person',
'label' => 'Person',
'fields' => ['first_name', 'last_name', 'full_name', 'gender', 'birthday', 'tax_number', 'social_security_number', 'description'],
'ui' => ['order' => 1],
],
[
'key' => 'person_addresses',
'canonical_root' => 'address',
'label' => 'Person Addresses',
'fields' => ['address', 'country', 'type_id', 'description'],
'ui' => ['order' => 2],
],
[
'key' => 'person_phones',
'canonical_root' => 'phone',
'label' => 'Person Phones',
'fields' => ['nu', 'country_code', 'type_id', 'description'],
'ui' => ['order' => 3],
],
[
'key' => 'emails',
'canonical_root' => 'email',
'label' => 'Emails',
'fields' => ['value', 'is_primary', 'label'],
'ui' => ['order' => 4],
],
[
'key' => 'contracts',
'canonical_root' => 'contract',
'label' => 'Contracts',
'fields' => ['reference', 'start_date', 'end_date', 'description', 'type_id', 'client_case_id'],
'ui' => ['order' => 5],
],
[
'key' => 'accounts',
'canonical_root' => 'account',
'label' => 'Accounts',
'fields' => ['reference', 'balance_amount', 'contract_id', 'contract_reference', 'type_id', 'active', 'description'],
'ui' => ['order' => 6],
],
]);
} else {
// Ensure fields are arrays for frontend consumption
$entities = $entities->map(function ($ent) {
$ent->fields = is_array($ent->fields) ? $ent->fields : [];
return $ent;
});
}
return response()->json(['entities' => $entities]);
}
public function suggest(Request $request)
{
$cols = $request->input('columns', []);
if (! is_array($cols)) {
$cols = [];
}
$entities = ImportEntity::all();
$suggestions = [];
foreach ($cols as $col) {
$s = $this->suggestFor($col, $entities);
if ($s) {
$suggestions[$col] = $s;
}
}
return response()->json(['suggestions' => $suggestions]);
}
private function suggestFor(string $source, $entities): ?array
{
$s = trim(mb_strtolower($source));
foreach ($entities as $ent) {
$rules = (array) ($ent->rules ?? []);
foreach ($rules as $rule) {
$pattern = $rule['pattern'] ?? null;
$field = $rule['field'] ?? null;
if (! $pattern || ! $field) {
continue;
}
if (@preg_match($pattern, $s)) {
return [
'entity' => $ent->key, // UI key (plural except person)
'field' => $field,
'canonical_root' => $ent->canonical_root,
];
}
}
}
return null;
}
}
app/Http/Controllers/ImportTemplateController.php
+136 -24
View File
@@ -2,10 +2,13 @@
namespace App\Http\Controllers;
use App\Models\Action;
use App\Models\Client;
use App\Models\Decision;
use App\Models\Import;
use App\Models\ImportTemplate;
use App\Models\ImportTemplateMapping;
use App\Models\Client;
use App\Models\Segment;
use Illuminate\Http\Request;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Str;
@@ -21,7 +24,7 @@ public function index()
->get();
return Inertia::render('Imports/Templates/Index', [
'templates' => $templates->map(fn($t) => [
'templates' => $templates->map(fn ($t) => [
'uuid' => $t->uuid,
'name' => $t->name,
'description' => $t->description,
@@ -48,8 +51,22 @@ public function create()
DB::raw('person.full_name as name'),
]);
$segments = Segment::query()->orderBy('name')->get(['id', 'name']);
$decisions = Decision::query()->orderBy('name')->get(['id', 'name']);
$actions = Action::with(['decisions:id,name'])
->orderBy('name')
->get(['id', 'name'])
->map(fn ($a) => [
'id' => $a->id,
'name' => $a->name,
'decisions' => $a->decisions->map(fn ($d) => ['id' => $d->id, 'name' => $d->name])->values(),
]);
return Inertia::render('Imports/Templates/Create', [
'clients' => $clients,
'segments' => $segments,
'decisions' => $decisions,
'actions' => $actions,
]);
}
@@ -57,8 +74,15 @@ public function store(Request $request)
{
// Normalize payload to be resilient to UI variations
$raw = $request->all();
// Allow passing default segment/decision either inside meta or as top-level
if (isset($raw['segment_id']) && ! isset($raw['meta']['segment_id'])) {
$raw['meta']['segment_id'] = $raw['segment_id'];
}
if (isset($raw['decision_id']) && ! isset($raw['meta']['decision_id'])) {
$raw['meta']['decision_id'] = $raw['decision_id'];
}
// Resolve client by uuid if provided, or cast string numeric to int
if (!empty($raw['client_uuid'] ?? null)) {
if (! empty($raw['client_uuid'] ?? null)) {
$raw['client_id'] = Client::where('uuid', $raw['client_uuid'])->value('id');
} elseif (isset($raw['client_id']) && is_string($raw['client_id']) && ctype_digit($raw['client_id'])) {
$raw['client_id'] = (int) $raw['client_id'];
@@ -66,8 +90,13 @@ public function store(Request $request)
// Normalize entities to array of strings
if (isset($raw['entities']) && is_array($raw['entities'])) {
$raw['entities'] = array_values(array_filter(array_map(function ($e) {
if (is_string($e)) return $e;
if (is_array($e) && array_key_exists('value', $e)) return (string) $e['value'];
if (is_string($e)) {
return $e;
}
if (is_array($e) && array_key_exists('value', $e)) {
return (string) $e['value'];
}
return null;
}, $raw['entities'])));
}
@@ -84,14 +113,29 @@ public function store(Request $request)
'entities.*' => 'string|in:person,person_addresses,person_phones,emails,accounts,contracts',
'mappings' => 'array',
'mappings.*.source_column' => 'required|string',
'mappings.*.entity' => 'nullable|string|in:person,person_addresses,person_phones,emails,accounts,contracts',
'mappings.*.entity' => 'nullable|string|in:person,person_addresses,person_phones,emails,accounts,contracts,client_cases',
'mappings.*.target_field' => 'nullable|string',
'mappings.*.transform' => 'nullable|string|max:50',
'mappings.*.apply_mode' => 'nullable|string|in:insert,update,both',
'mappings.*.options' => 'nullable|array',
'mappings.*.position' => 'nullable|integer',
'meta' => 'nullable|array',
'meta.segment_id' => 'nullable|integer|exists:segments,id',
'meta.decision_id' => 'nullable|integer|exists:decisions,id',
'meta.action_id' => 'nullable|integer|exists:actions,id',
])->validate();
// Ensure decision belongs to action if both provided in meta
$meta = $data['meta'] ?? [];
if (! empty($meta['action_id']) && ! empty($meta['decision_id'])) {
$belongs = \DB::table('action_decision')
->where('action_id', $meta['action_id'])
->where('decision_id', $meta['decision_id'])
->exists();
if (! $belongs) {
return back()->withErrors(['meta.decision_id' => 'Selected decision is not associated with the chosen action.'])->withInput();
}
}
$template = null;
DB::transaction(function () use (&$template, $request, $data) {
$template = ImportTemplate::create([
@@ -104,9 +148,12 @@ public function store(Request $request)
'user_id' => $request->user()?->id,
'client_id' => $data['client_id'] ?? null,
'is_active' => $data['is_active'] ?? true,
'meta' => [
'meta' => array_filter([
'entities' => $data['entities'] ?? [],
],
'segment_id' => data_get($data, 'meta.segment_id'),
'decision_id' => data_get($data, 'meta.decision_id'),
'action_id' => data_get($data, 'meta.action_id'),
], fn ($v) => ! is_null($v) && $v !== ''),
]);
foreach (($data['mappings'] ?? []) as $m) {
@@ -144,6 +191,17 @@ public function edit(ImportTemplate $template)
DB::raw('person.full_name as name'),
]);
$segments = Segment::query()->orderBy('name')->get(['id', 'name']);
$decisions = Decision::query()->orderBy('name')->get(['id', 'name']);
$actions = Action::with(['decisions:id,name'])
->orderBy('name')
->get(['id', 'name'])
->map(fn ($a) => [
'id' => $a->id,
'name' => $a->name,
'decisions' => $a->decisions->map(fn ($d) => ['id' => $d->id, 'name' => $d->name])->values(),
]);
return Inertia::render('Imports/Templates/Edit', [
'template' => [
'uuid' => $template->uuid,
@@ -155,9 +213,12 @@ public function edit(ImportTemplate $template)
'client_uuid' => $template->client?->uuid,
'sample_headers' => $template->sample_headers,
'meta' => $template->meta,
'mappings' => $template->mappings()->orderBy('position')->get(['id','entity','source_column','target_field','transform','apply_mode','options','position']),
'mappings' => $template->mappings()->orderBy('position')->get(['id', 'entity', 'source_column', 'target_field', 'transform', 'apply_mode', 'options', 'position']),
],
'clients' => $clients,
'segments' => $segments,
'decisions' => $decisions,
'actions' => $actions,
]);
}
@@ -171,7 +232,7 @@ public function addMapping(Request $request, ImportTemplate $template)
}
$data = validator($raw, [
'source_column' => 'required|string',
'entity' => 'nullable|string|in:person,person_addresses,person_phones,emails,accounts,contracts',
'entity' => 'nullable|string|in:person,person_addresses,person_phones,emails,accounts,contracts,client_cases',
'target_field' => 'nullable|string',
'transform' => 'nullable|string|in:trim,upper,lower',
'apply_mode' => 'nullable|string|in:insert,update,both',
@@ -193,6 +254,7 @@ public function addMapping(Request $request, ImportTemplate $template)
'options' => $data['options'] ?? $existing->options,
'position' => $data['position'] ?? $existing->position,
]);
return redirect()->route('importTemplates.edit', ['template' => $template->uuid])
->with('info', 'Mapping already existed. Updated existing mapping.');
} else {
@@ -207,6 +269,7 @@ public function addMapping(Request $request, ImportTemplate $template)
'options' => $data['options'] ?? null,
'position' => $position,
]);
return redirect()->route('importTemplates.edit', ['template' => $template->uuid])
->with('success', 'Mapping added');
}
@@ -216,7 +279,7 @@ public function addMapping(Request $request, ImportTemplate $template)
public function update(Request $request, ImportTemplate $template)
{
$raw = $request->all();
if (!empty($raw['client_uuid'] ?? null)) {
if (! empty($raw['client_uuid'] ?? null)) {
$raw['client_id'] = Client::where('uuid', $raw['client_uuid'])->value('id');
}
$data = validator($raw, [
@@ -229,16 +292,35 @@ public function update(Request $request, ImportTemplate $template)
'sample_headers' => 'nullable|array',
'meta' => 'nullable|array',
'meta.delimiter' => 'nullable|string|max:4',
'meta.segment_id' => 'nullable|integer|exists:segments,id',
'meta.decision_id' => 'nullable|integer|exists:decisions,id',
'meta.action_id' => 'nullable|integer|exists:actions,id',
])->validate();
// Validate decision/action consistency on update as well
$meta = $data['meta'] ?? [];
if (! empty($meta['action_id']) && ! empty($meta['decision_id'])) {
$belongs = \DB::table('action_decision')
->where('action_id', $meta['action_id'])
->where('decision_id', $meta['decision_id'])
->exists();
if (! $belongs) {
return back()->withErrors(['meta.decision_id' => 'Selected decision is not associated with the chosen action.'])->withInput();
}
}
// Merge meta safely, preserving existing keys when not provided
$newMeta = $template->meta ?? [];
if (array_key_exists('meta', $data) && is_array($data['meta'])) {
$newMeta = array_merge($newMeta, $data['meta']);
// Drop empty delimiter to allow auto-detect
if (array_key_exists('delimiter', $newMeta) && (!is_string($newMeta['delimiter']) || trim((string) $newMeta['delimiter']) === '')) {
if (array_key_exists('delimiter', $newMeta) && (! is_string($newMeta['delimiter']) || trim((string) $newMeta['delimiter']) === '')) {
unset($newMeta['delimiter']);
}
foreach (['segment_id', 'decision_id', 'action_id'] as $k) {
if (array_key_exists($k, $newMeta) && ($newMeta[$k] === '' || is_null($newMeta[$k]))) {
unset($newMeta[$k]);
}
}
}
$template->update([
@@ -266,14 +348,14 @@ public function bulkAddMappings(Request $request, ImportTemplate $template)
}
$data = validator($raw, [
'sources' => 'required|string', // comma and/or newline separated
'entity' => 'nullable|string|in:person,person_addresses,person_phones,emails,accounts,contracts',
'entity' => 'nullable|string|in:person,person_addresses,person_phones,emails,accounts,contracts,client_cases',
'default_field' => 'nullable|string', // if provided, used as the field name for all entries
'apply_mode' => 'nullable|string|in:insert,update,both',
'transform' => 'nullable|string|in:trim,upper,lower',
])->validate();
$list = preg_split('/\r?\n|,/', $data['sources']);
$list = array_values(array_filter(array_map(fn($s) => trim($s), $list), fn($s) => $s !== ''));
$list = array_values(array_filter(array_map(fn ($s) => trim($s), $list), fn ($s) => $s !== ''));
if (empty($list)) {
return redirect()->route('importTemplates.edit', ['template' => $template->uuid])
@@ -286,14 +368,15 @@ public function bulkAddMappings(Request $request, ImportTemplate $template)
$entity = $data['entity'] ?? null;
$defaultField = $data['default_field'] ?? null; // allows forcing a specific field for all
$created = 0; $updated = 0;
$created = 0;
$updated = 0;
DB::transaction(function () use ($template, $list, $apply, $transform, $entity, $defaultField, $basePosition, &$created, &$updated) {
foreach ($list as $idx => $source) {
$targetField = null;
if ($defaultField) {
$targetField = $entity ? ($entity . '.' . $defaultField) : $defaultField;
$targetField = $entity ? ($entity.'.'.$defaultField) : $defaultField;
} elseif ($entity) {
$targetField = $entity . '.' . $source;
$targetField = $entity.'.'.$source;
}
$existing = ImportTemplateMapping::where('import_template_id', $template->id)
@@ -327,10 +410,17 @@ public function bulkAddMappings(Request $request, ImportTemplate $template)
});
$msg = [];
if ($created) $msg[] = "$created created";
if ($updated) $msg[] = "$updated updated";
if ($created) {
$msg[] = "$created created";
}
if ($updated) {
$msg[] = "$updated updated";
}
$text = 'Mappings processed';
if (!empty($msg)) $text .= ': ' . implode(', ', $msg);
if (! empty($msg)) {
$text .= ': '.implode(', ', $msg);
}
return redirect()->route('importTemplates.edit', ['template' => $template->uuid])
->with('success', $text);
}
@@ -338,7 +428,9 @@ public function bulkAddMappings(Request $request, ImportTemplate $template)
// Update an existing mapping
public function updateMapping(Request $request, ImportTemplate $template, ImportTemplateMapping $mapping)
{
if ($mapping->import_template_id !== $template->id) abort(404);
if ($mapping->import_template_id !== $template->id) {
abort(404);
}
$raw = $request->all();
if (array_key_exists('transform', $raw) && $raw['transform'] === '') {
$raw['transform'] = null;
@@ -361,6 +453,7 @@ public function updateMapping(Request $request, ImportTemplate $template, Import
'options' => $data['options'] ?? null,
'position' => $data['position'] ?? $mapping->position,
]);
return redirect()->route('importTemplates.edit', ['template' => $template->uuid])
->with('success', 'Mapping updated');
}
@@ -368,8 +461,11 @@ public function updateMapping(Request $request, ImportTemplate $template, Import
// Delete a mapping
public function deleteMapping(ImportTemplate $template, ImportTemplateMapping $mapping)
{
if ($mapping->import_template_id !== $template->id) abort(404);
if ($mapping->import_template_id !== $template->id) {
abort(404);
}
$mapping->delete();
return redirect()->route('importTemplates.edit', ['template' => $template->uuid])
->with('success', 'Mapping deleted');
}
@@ -385,11 +481,14 @@ public function reorderMappings(Request $request, ImportTemplate $template)
// Ensure all ids belong to template
$validIds = ImportTemplateMapping::where('import_template_id', $template->id)
->whereIn('id', $ids)->pluck('id')->all();
if (count($validIds) !== count($ids)) abort(422, 'Invalid mapping ids');
if (count($validIds) !== count($ids)) {
abort(422, 'Invalid mapping ids');
}
// Apply new positions
foreach ($ids as $idx => $id) {
ImportTemplateMapping::where('id', $id)->update(['position' => $idx]);
}
return redirect()->route('importTemplates.edit', ['template' => $template->uuid])
->with('success', 'Mappings reordered');
}
@@ -422,7 +521,20 @@ public function applyToImport(Request $request, ImportTemplate $template, Import
$copied++;
}
$import->update(['import_template_id' => $template->id]);
// Merge default actions (segment/decision) into import meta for processing
$importMeta = $import->meta ?? [];
$tplMeta = $template->meta ?? [];
$merged = array_merge($importMeta, array_filter([
'segment_id' => $tplMeta['segment_id'] ?? null,
'decision_id' => $tplMeta['decision_id'] ?? null,
'action_id' => $tplMeta['action_id'] ?? null,
'template_name' => $template->name,
], fn ($v) => ! is_null($v) && $v !== ''));
$import->update([
'import_template_id' => $template->id,
'meta' => $merged,
]);
});
return response()->json(['ok' => true, 'copied' => $copied, 'cleared' => $clear]);
app/Models/ClientCase.php
+1
View File
@@ -23,6 +23,7 @@ class ClientCase extends Model
protected $fillable = [
'client_id',
'person_id',
'client_ref',
];
protected $hidden = [
app/Models/ImportEntity.php
+27
View File
@@ -0,0 +1,27 @@
<?php
namespace App\Models;
use Illuminate\Database\Eloquent\Model;
class ImportEntity extends Model
{
protected $fillable = [
'key',
'canonical_root',
'label',
'fields',
'field_aliases',
'aliases',
'rules',
'ui',
];
protected $casts = [
'fields' => 'array',
'field_aliases' => 'array',
'aliases' => 'array',
'rules' => 'array',
'ui' => 'array',
];
}
app/Services/CsvImportService.php
+98 -9
View File
@@ -5,15 +5,79 @@
class CsvImportService
{
/**
* Read the first line of a file; returns null on failure.
* Normalize a line to UTF-8 and strip BOM / control characters for robust splitting.
*/
public function readFirstLine(string $path): ?string
private function normalizeLine(string $line): string
{
// Strip UTF-8 BOM
if (str_starts_with($line, "\xEF\xBB\xBF")) {
$line = substr($line, 3);
}
// Detect UTF-16 BOMs
$hasNulls = strpos($line, "\x00") !== false;
if (str_starts_with($line, "\xFF\xFE")) {
// UTF-16LE BOM
$line = substr($line, 2);
$line = function_exists('mb_convert_encoding') ? @mb_convert_encoding($line, 'UTF-8', 'UTF-16LE') : preg_replace('/\x00/', '', $line);
} elseif (str_starts_with($line, "\xFE\xFF")) {
// UTF-16BE BOM
$line = substr($line, 2);
$line = function_exists('mb_convert_encoding') ? @mb_convert_encoding($line, 'UTF-8', 'UTF-16BE') : preg_replace('/\x00/', '', $line);
} elseif ($hasNulls) {
// Likely UTF-16 without BOM, try LE then BE
if (function_exists('mb_convert_encoding')) {
$try = @mb_convert_encoding($line, 'UTF-8', 'UTF-16LE');
if ($try !== false) {
$line = $try;
} else {
$try = @mb_convert_encoding($line, 'UTF-8', 'UTF-16BE');
if ($try !== false) {
$line = $try;
} else {
$line = preg_replace('/\x00/', '', $line);
}
}
} else {
$line = preg_replace('/\x00/', '', $line);
}
} else {
// Non UTF-16: try detect common encodings and convert to UTF-8 if needed
if (function_exists('mb_detect_encoding') && function_exists('mb_convert_encoding')) {
// Use default detection order for portability across environments
$enc = @mb_detect_encoding($line, null, true);
if ($enc && strtoupper($enc) !== 'UTF-8') {
$line = @mb_convert_encoding($line, 'UTF-8', $enc) ?: $line;
}
}
}
// Replace non-breaking space with regular space
$line = str_replace("\xC2\xA0", ' ', $line);
return $line;
}
/**
* Read the first meaningful (non-empty after normalization) line of a file; returns null on failure.
* Skips BOM-only lines and leading blank lines. Limits scanning to first 50 lines to be safe.
*/
public function readFirstMeaningfulLine(string $path): ?string
{
$fh = @fopen($path, 'r');
if (!$fh) return null;
$line = fgets($fh);
if (! $fh) {
return null;
}
$line = null;
$limit = 50;
while ($limit-- > 0 && ($raw = fgets($fh)) !== false) {
$normalized = $this->normalizeLine($raw);
if (trim($normalized) !== '') {
$line = $normalized;
break;
}
}
fclose($fh);
return $line === false ? null : $line;
return $line;
}
/**
@@ -24,14 +88,15 @@ public function readFirstLine(string $path): ?string
public function detectColumnsFromCsv(string $path, bool $hasHeader): array
{
// Use actual tab character for TSV; keep other common delimiters
$delims = [',',';','|',"\t"];
$delims = [',', ';', '|', "\t"];
$bestDelim = ',';
$bestCols = [];
$firstLine = $this->readFirstLine($path);
$firstLine = $this->readFirstMeaningfulLine($path);
if ($firstLine === null) {
return [$bestDelim, []];
}
// Already normalized by readFirstMeaningfulLine
$maxCount = 0;
foreach ($delims as $d) {
@@ -44,12 +109,27 @@ public function detectColumnsFromCsv(string $path, bool $hasHeader): array
}
}
if (!$hasHeader) {
// Fallback: if str_getcsv failed to split but we clearly see delimiters, do a simple explode
if ($maxCount <= 1) {
foreach (["\t", ';', ',', '|'] as $d) {
if (substr_count($firstLine, $d) >= 1) {
$parts = explode($d, $firstLine);
if (count($parts) > $maxCount) {
$bestDelim = $d;
$bestCols = $parts;
$maxCount = count($parts);
}
}
}
}
if (! $hasHeader) {
// return positional indices 0..N-1
$cols = [];
for ($i = 0; $i < $maxCount; $i++) {
$cols[] = (string) $i;
}
return [$bestDelim, $cols];
}
@@ -57,6 +137,7 @@ public function detectColumnsFromCsv(string $path, bool $hasHeader): array
$clean = array_map(function ($v) {
$v = trim((string) $v);
$v = preg_replace('/\s+/', ' ', $v);
return $v;
}, $bestCols);
@@ -69,16 +150,23 @@ public function detectColumnsFromCsv(string $path, bool $hasHeader): array
*/
public function parseColumnsFromCsv(string $path, string $delimiter, bool $hasHeader): array
{
$firstLine = $this->readFirstLine($path);
$firstLine = $this->readFirstMeaningfulLine($path);
if ($firstLine === null) {
return [];
}
// Already normalized by readFirstMeaningfulLine
$row = str_getcsv($firstLine, $delimiter);
$count = is_array($row) ? count($row) : 0;
// Fallback explode if str_getcsv failed to split
if ($count <= 1 && substr_count($firstLine, $delimiter) >= 1) {
$row = explode($delimiter, $firstLine);
$count = count($row);
}
if ($hasHeader) {
return array_map(function ($v) {
$v = trim((string) $v);
$v = preg_replace('/\s+/', ' ', $v);
return $v;
}, $row ?: []);
}
@@ -86,6 +174,7 @@ public function parseColumnsFromCsv(string $path, string $delimiter, bool $hasHe
for ($i = 0; $i < $count; $i++) {
$cols[] = (string) $i;
}
return $cols;
}
}
app/Services/ImportProcessor.php
+588 -50
View File
@@ -4,12 +4,15 @@
use App\Models\Account;
use App\Models\AccountType;
use App\Models\Activity;
use App\Models\Client;
use App\Models\ClientCase;
use App\Models\Contract;
use App\Models\ContractType;
use App\Models\Decision;
use App\Models\Email;
use App\Models\Import;
use App\Models\ImportEntity;
use App\Models\ImportEvent;
use App\Models\ImportRow;
use App\Models\Person\AddressType;
@@ -57,6 +60,13 @@ public function process(Import $import, ?Authenticatable $user = null): array
->where('import_id', $import->id)
->get(['source_column', 'target_field', 'transform', 'apply_mode', 'options']);
// Load dynamic entity config
[$rootAliasMap, $fieldAliasMap, $validRoots] = $this->loadImportEntityConfig();
// Normalize aliases (plural/legacy roots, field names) before validation
$mappings = $this->normalizeMappings($mappings, $rootAliasMap, $fieldAliasMap);
// Validate mapping roots early to avoid silent failures due to typos
$this->validateMappingRoots($mappings, $validRoots);
$header = $import->meta['columns'] ?? null;
// Prefer explicitly chosen delimiter, then template meta, else detected
$delimiter = $import->meta['forced_delimiter']
@@ -66,6 +76,8 @@ public function process(Import $import, ?Authenticatable $user = null): array
$hasHeader = (bool) ($import->meta['has_header'] ?? true);
$path = Storage::disk($import->disk)->path($import->path);
// Note: Do not auto-detect or infer mappings/fields beyond what the template mapping provides
// Parse file and create import_rows with mapped_data
$fh = @fopen($path, 'r');
if (! $fh) {
@@ -95,12 +107,55 @@ public function process(Import $import, ?Authenticatable $user = null): array
if ($hasHeader) {
$first = fgetcsv($fh, 0, $delimiter);
$rowNum++;
// use actual detected header if not already stored
if (! $header) {
$header = array_map(fn ($v) => trim((string) $v), $first ?: []);
// Always use the actual header from the file for parsing
$header = array_map(fn ($v) => $this->sanitizeHeaderName((string) $v), $first ?: []);
// Heuristic: if header parsed as a single column but contains common delimiters, warn about mismatch
if (count($header) === 1) {
$rawHeader = $first[0] ?? '';
if (is_string($rawHeader) && (str_contains($rawHeader, ';') || str_contains($rawHeader, "\t"))) {
ImportEvent::create([
'import_id' => $import->id,
'user_id' => $user?->getAuthIdentifier(),
'event' => 'delimiter_mismatch_suspected',
'level' => 'warning',
'message' => 'Header parsed as a single column. Suspected delimiter mismatch. Set a forced delimiter in the template or import settings.',
'context' => [
'current_delimiter' => $delimiter,
'raw_header' => $rawHeader,
],
]);
}
}
// Preflight: warn if any mapped source columns are not present in the header (exact match)
$headerSet = [];
foreach ($header as $h) {
$headerSet[$h] = true;
}
$missingSources = [];
foreach ($mappings as $map) {
$src = (string) ($map->source_column ?? '');
if ($src !== '' && ! array_key_exists($src, $headerSet)) {
$missingSources[] = $src;
}
}
if (! empty($missingSources)) {
ImportEvent::create([
'import_id' => $import->id,
'user_id' => $user?->getAuthIdentifier(),
'event' => 'source_columns_missing_in_header',
'level' => 'warning',
'message' => 'Some mapped source columns are not present in the file header (exact match required).',
'context' => [
'missing' => $missingSources,
'header' => $header,
],
]);
}
}
// If mapping contains contract.reference, we require each row to successfully resolve/create a contract
$requireContract = $this->mappingIncludes($mappings, 'contract.reference');
while (($row = fgetcsv($fh, 0, $delimiter)) !== false) {
$rowNum++;
$total++;
@@ -108,6 +163,8 @@ public function process(Import $import, ?Authenticatable $user = null): array
$rawAssoc = $this->buildRowAssoc($row, $header);
[$recordType, $mapped] = $this->applyMappings($rawAssoc, $mappings);
// Do not auto-derive or fallback values; only use explicitly mapped fields
$importRow = ImportRow::create([
'import_id' => $import->id,
'row_number' => $rowNum,
@@ -122,6 +179,31 @@ public function process(Import $import, ?Authenticatable $user = null): array
if (isset($mapped['contract'])) {
$contractResult = $this->upsertContractChain($import, $mapped, $mappings);
if ($contractResult['action'] === 'skipped') {
// Even if no contract fields were updated, we may still need to apply template meta
// like attaching a segment or creating an activity. Do that if we have the contract.
if (isset($contractResult['contract']) && $contractResult['contract'] instanceof Contract) {
try {
$this->postContractActions($import, $contractResult['contract']);
ImportEvent::create([
'import_id' => $import->id,
'user_id' => $user?->getAuthIdentifier(),
'import_row_id' => $importRow->id,
'event' => 'post_contract_actions_applied',
'level' => 'info',
'message' => 'Applied template post-actions on existing contract.',
'context' => ['contract_id' => $contractResult['contract']->id],
]);
} catch (\Throwable $e) {
ImportEvent::create([
'import_id' => $import->id,
'user_id' => $user?->getAuthIdentifier(),
'import_row_id' => $importRow->id,
'event' => 'post_contract_action_failed',
'level' => 'warning',
'message' => $e->getMessage(),
]);
}
}
$skipped++;
$importRow->update(['status' => 'skipped']);
ImportEvent::create([
@@ -148,15 +230,64 @@ public function process(Import $import, ?Authenticatable $user = null): array
'message' => ucfirst($contractResult['action']).' contract',
'context' => ['id' => $contractResult['contract']->id],
]);
// Post-contract actions from template/import meta
try {
$this->postContractActions($import, $contractResult['contract']);
} catch (\Throwable $e) {
ImportEvent::create([
'import_id' => $import->id,
'user_id' => $user?->getAuthIdentifier(),
'import_row_id' => $importRow->id,
'event' => 'post_contract_action_failed',
'level' => 'warning',
'message' => $e->getMessage(),
]);
}
} else {
$invalid++;
$importRow->update(['status' => 'invalid', 'errors' => [$contractResult['message'] ?? 'Contract processing failed']]);
}
}
// Enforce hard requirement: if template mapped contract.reference but we didn't resolve/create a contract, mark row invalid and continue
if ($requireContract) {
$contractEnsured = false;
if ($contractResult && isset($contractResult['contract']) && $contractResult['contract'] instanceof Contract) {
$contractEnsured = true;
}
if (! $contractEnsured) {
$srcCol = $this->findSourceColumnFor($mappings, 'contract.reference');
$rawVal = $srcCol !== null ? ($rawAssoc[$srcCol] ?? null) : null;
$extra = $srcCol !== null ? ' Source column: "'.$srcCol.'" value: '.(is_null($rawVal) || $rawVal === '' ? '(empty)' : (is_scalar($rawVal) ? (string) $rawVal : json_encode($rawVal))) : '';
$msg = 'Row '.$rowNum.': Contract was required (contract.reference mapped) but not created/resolved. '.($contractResult['message'] ?? '').$extra;
// Avoid double-counting invalid if already set by contract processing
if ($importRow->status !== 'invalid') {
$invalid++;
$importRow->update(['status' => 'invalid', 'errors' => [$msg]]);
}
ImportEvent::create([
'import_id' => $import->id,
'user_id' => $user?->getAuthIdentifier(),
'import_row_id' => $importRow->id,
'event' => 'row_invalid',
'level' => 'error',
'message' => $msg,
]);
// Skip further processing for this row
continue;
}
}
// Accounts
$accountResult = null;
if (isset($mapped['account'])) {
// If a contract was just created or resolved above, pass its id to account mapping for this row
if ($contractResult && isset($contractResult['contract']) && $contractResult['contract'] instanceof Contract) {
$mapped['account']['contract_id'] = $contractResult['contract']->id;
}
$accountResult = $this->upsertAccount($import, $mapped, $mappings);
if ($accountResult['action'] === 'skipped') {
$skipped++;
@@ -168,6 +299,7 @@ public function process(Import $import, ?Authenticatable $user = null): array
'event' => 'row_skipped',
'level' => 'info',
'message' => $accountResult['message'] ?? 'Skipped (no changes).',
'context' => $accountResult['context'] ?? null,
]);
} elseif ($accountResult['action'] === 'inserted' || $accountResult['action'] === 'updated') {
$imported++;
@@ -191,7 +323,7 @@ public function process(Import $import, ?Authenticatable $user = null): array
}
}
// Contacts: resolve person strictly via Contract -> ClientCase -> Person, contacts, or identifiers
// Contacts: resolve person via Contract/Account chain, client_case.client_ref, contacts, or identifiers
$personIdForRow = null;
// Prefer person from contract created/updated above
if ($contractResult && isset($contractResult['contract']) && $contractResult['contract'] instanceof Contract) {
@@ -222,6 +354,16 @@ public function process(Import $import, ?Authenticatable $user = null): array
}
}
}
// Resolve by client_case.client_ref for this client (prefer reusing existing person)
if (! $personIdForRow && $import->client_id && ! empty($mapped['client_case']['client_ref'] ?? null)) {
$cc = ClientCase::where('client_id', $import->client_id)
->where('client_ref', $mapped['client_case']['client_ref'])
->first();
if ($cc) {
$personIdForRow = $cc->person_id ?: null;
}
}
// Resolve by contact values next
if (! $personIdForRow) {
$emailVal = trim((string) ($mapped['email']['value'] ?? ''));
@@ -240,6 +382,22 @@ public function process(Import $import, ?Authenticatable $user = null): array
}
// If still no person but we have any contact value, auto-create a minimal person
// BUT if we can map to an existing client_case by client_ref, reuse that case and set person there (avoid separate person rows)
if (! $personIdForRow && ($emailVal !== '' || $phoneNu !== '' || $addrLine !== '')) {
if ($import->client_id && ! empty($mapped['client_case']['client_ref'] ?? null)) {
$cc = ClientCase::where('client_id', $import->client_id)
->where('client_ref', $mapped['client_case']['client_ref'])
->first();
if ($cc) {
$pid = $cc->person_id ?: $this->createMinimalPersonId();
if (! $cc->person_id) {
$cc->person_id = $pid;
$cc->save();
}
$personIdForRow = $pid;
}
}
}
if (! $personIdForRow && ($emailVal !== '' || $phoneNu !== '' || $addrLine !== '')) {
$personIdForRow = $this->createMinimalPersonId();
ImportEvent::create([
@@ -384,12 +542,29 @@ private function applyMappings(array $raw, $mappings): array
}
$value = $raw[$src] ?? null;
// very basic transforms
if ($map->transform === 'trim') {
$value = is_string($value) ? trim($value) : $value;
}
if ($map->transform === 'upper') {
$value = is_string($value) ? strtoupper($value) : $value;
// Transform chain support: e.g. "trim|decimal" or "upper|alnum"
$transform = (string) ($map->transform ?? '');
if ($transform !== '') {
$parts = explode('|', $transform);
foreach ($parts as $t) {
$t = trim($t);
if ($t === 'trim') {
$value = is_string($value) ? trim($value) : $value;
} elseif ($t === 'upper') {
$value = is_string($value) ? strtoupper($value) : $value;
} elseif ($t === 'lower') {
$value = is_string($value) ? strtolower($value) : $value;
} elseif ($t === 'digits' || $t === 'numeric') {
$value = is_string($value) ? preg_replace('/[^0-9]/', '', $value) : $value;
} elseif ($t === 'decimal') {
$value = is_string($value) ? $this->normalizeDecimal($value) : $value;
} elseif ($t === 'alnum') {
$value = is_string($value) ? preg_replace('/[^A-Za-z0-9]/', '', $value) : $value;
} elseif ($t === 'ref') {
// Reference safe: keep letters+digits only, uppercase
$value = is_string($value) ? strtoupper(preg_replace('/[^A-Za-z0-9]/', '', $value)) : $value;
}
}
}
// detect record type from first segment, e.g., "account.balance_amount"
@@ -423,6 +598,16 @@ private function upsertAccount(Import $import, array $mapped, $mappings): array
$acc = $mapped['account'] ?? [];
$contractId = $acc['contract_id'] ?? null;
$reference = $acc['reference'] ?? null;
// Determine if the template includes any contract mappings; if not, do not create contracts here
$hasContractRoot = $this->mappingsContainRoot($mappings, 'contract');
// Normalize references (remove spaces) for consistent matching
if (! is_null($reference)) {
$reference = preg_replace('/\s+/', '', trim((string) $reference));
$acc['reference'] = $reference;
}
if (! empty($acc['contract_reference'] ?? null)) {
$acc['contract_reference'] = preg_replace('/\s+/', '', trim((string) $acc['contract_reference']));
}
// If contract_id not provided, attempt to resolve by contract reference for the selected client
if (! $contractId) {
$contractRef = $acc['contract_reference'] ?? ($mapped['contract']['reference'] ?? null);
@@ -436,25 +621,17 @@ private function upsertAccount(Import $import, array $mapped, $mappings): array
->first();
if ($existingContract) {
$contractId = $existingContract->id;
} else {
// 2) Not found: attempt to resolve debtor via identifiers or provided person, then create case+contract
// Try strong identifiers first
} elseif ($hasContractRoot) {
// Only create a new contract if the template explicitly includes contract mappings
// Resolve debtor via identifiers or provided person
$personId = $this->findPersonIdByIdentifiers($mapped['person'] ?? []);
// Create from provided person data if unresolved
if (! $personId) {
$personId = $this->findOrCreatePersonId($mapped['person'] ?? []);
}
// Last resort, create minimal
if (! $personId) {
$personId = $this->createMinimalPersonId();
}
// Use the selected client for this import to tie the case/contract
if (! $clientId) {
return ['action' => 'skipped', 'message' => 'Client required to create contract'];
}
$resolvedClientId = $clientId;
$clientCaseId = $this->findOrCreateClientCaseId($resolvedClientId, $personId);
// Build minimal/new contract
$clientCaseId = $this->findOrCreateClientCaseId($clientId, $personId, $mapped['client_case']['client_ref'] ?? null);
$contractFields = $mapped['contract'] ?? [];
$newContractData = [
'client_case_id' => $clientCaseId,
@@ -465,11 +642,13 @@ private function upsertAccount(Import $import, array $mapped, $mappings): array
$newContractData[$k] = $contractFields[$k];
}
}
// ensure required fields on contracts
$newContractData['start_date'] = $newContractData['start_date'] ?? now()->toDateString();
$newContractData['type_id'] = $newContractData['type_id'] ?? $this->getDefaultContractTypeId();
$createdContract = Contract::create($newContractData);
$contractId = $createdContract->id;
} else {
// Do not create contracts implicitly when not mapped in the template
$contractId = null;
}
if ($contractId) {
$acc['contract_id'] = $contractId;
@@ -477,17 +656,35 @@ private function upsertAccount(Import $import, array $mapped, $mappings): array
}
}
}
// Default account.reference to contract reference if missing
if (! $reference) {
$contractRef = $acc['contract_reference'] ?? ($mapped['contract']['reference'] ?? null);
if ($contractRef) {
$reference = $contractRef;
// Fallback: if account.reference is empty but contract.reference is present, use it
if ((is_null($reference) || $reference === '') && ! empty($mapped['contract']['reference'] ?? null)) {
$reference = preg_replace('/\s+/', '', trim((string) $mapped['contract']['reference']));
if ($reference !== '') {
$acc['reference'] = $reference;
$mapped['account'] = $acc;
}
}
// Do not default or infer account.reference from other fields; rely solely on mapped values
if (! $contractId || ! $reference) {
return ['action' => 'skipped', 'message' => 'Missing contract_id/reference'];
$issues = [];
if (! $contractId) {
$issues[] = 'contract_id unresolved';
}
if (! $reference) {
$issues[] = 'account.reference empty';
}
$candidateContractRef = $acc['contract_reference'] ?? ($mapped['contract']['reference'] ?? null);
return [
'action' => 'skipped',
'message' => 'Prerequisite missing: '.implode(' & ', $issues),
'context' => [
'has_contract_root_mapped' => $hasContractRoot,
'candidate_contract_reference' => $candidateContractRef,
'account_reference_provided' => $reference,
'account_fields_present' => array_keys(array_filter($acc, fn ($v) => ! is_null($v) && $v !== '')),
],
];
}
$existing = Account::query()
@@ -498,6 +695,7 @@ private function upsertAccount(Import $import, array $mapped, $mappings): array
// Build applyable data based on apply_mode
$applyInsert = [];
$applyUpdate = [];
$applyModeByField = [];
foreach ($mappings as $map) {
if (! $map->target_field) {
continue;
@@ -511,7 +709,11 @@ private function upsertAccount(Import $import, array $mapped, $mappings): array
continue;
}
$value = $acc[$field] ?? null;
if (in_array($field, ['balance_amount','initial_amount'], true) && is_string($value)) {
$value = $this->normalizeDecimal($value);
}
$mode = $map->apply_mode ?? 'both';
$applyModeByField[$field] = $mode;
if (in_array($mode, ['insert', 'both'])) {
$applyInsert[$field] = $value;
}
@@ -535,6 +737,15 @@ private function upsertAccount(Import $import, array $mapped, $mappings): array
// also include contract hints for downstream contact resolution
return ['action' => 'updated', 'account' => $existing, 'contract_id' => $contractId];
} else {
// On insert: if initial_amount is not provided but balance_amount is, allow defaulting
// Only when the mapping for initial_amount is 'insert' or 'both', or unmapped (null).
$initMode = $applyModeByField['initial_amount'] ?? null;
if ((! array_key_exists('initial_amount', $applyInsert) || is_null($applyInsert['initial_amount'] ?? null))
&& array_key_exists('balance_amount', $applyInsert)
&& ($applyInsert['balance_amount'] !== null && $applyInsert['balance_amount'] !== '')
&& ($initMode === null || in_array($initMode, ['insert','both'], true))) {
$applyInsert['initial_amount'] = $applyInsert['balance_amount'];
}
if (empty($applyInsert)) {
return ['action' => 'skipped', 'message' => 'No fields marked for insert'];
}
@@ -552,6 +763,18 @@ private function upsertAccount(Import $import, array $mapped, $mappings): array
}
}
private function mappingsContainRoot($mappings, string $root): bool
{
foreach ($mappings as $map) {
$target = (string) ($map->target_field ?? '');
if ($target !== '' && str_starts_with($target, $root.'.')) {
return true;
}
}
return false;
}
private function findPersonIdByIdentifiers(array $p): ?int
{
$tax = $p['tax_number'] ?? null;
@@ -576,6 +799,10 @@ private function upsertContractChain(Import $import, array $mapped, $mappings):
{
$contractData = $mapped['contract'] ?? [];
$reference = $contractData['reference'] ?? null;
if (! is_null($reference)) {
$reference = preg_replace('/\s+/', '', trim((string) $reference));
$contractData['reference'] = $reference;
}
if (! $reference) {
return ['action' => 'invalid', 'message' => 'Missing contract.reference'];
}
@@ -605,26 +832,53 @@ private function upsertContractChain(Import $import, array $mapped, $mappings):
// If we still need to insert, we must resolve clientCaseId, but avoid creating new person/case unless necessary
if (! $existing && ! $clientCaseId) {
// Resolve by identifiers or provided person; do not use Client->person
$personId = null;
if (! empty($mapped['person'] ?? [])) {
$personId = $this->findPersonIdByIdentifiers($mapped['person']);
if (! $personId) {
$personId = $this->findOrCreatePersonId($mapped['person']);
$clientRef = $mapped['client_case']['client_ref'] ?? null;
// First, if we have a client and client_ref, try to reuse existing case to avoid creating extra persons
if ($clientId && $clientRef) {
$cc = ClientCase::where('client_id', $clientId)->where('client_ref', $clientRef)->first();
if ($cc) {
// Reuse this case
$clientCaseId = $cc->id;
// If case has no person yet and we have mapped person identifiers/data, set it once
if (! $cc->person_id) {
$pid = null;
if (! empty($mapped['person'] ?? [])) {
$pid = $this->findPersonIdByIdentifiers($mapped['person']);
if (! $pid) {
$pid = $this->findOrCreatePersonId($mapped['person']);
}
}
if (! $pid) {
$pid = $this->createMinimalPersonId();
}
$cc->person_id = $pid;
$cc->save();
}
}
}
// As a last resort, create a minimal person for this client
if ($clientId && ! $personId) {
$personId = $this->createMinimalPersonId();
}
if ($clientId && $personId) {
$clientCaseId = $this->findOrCreateClientCaseId($clientId, $personId);
} elseif ($personId) {
// require an import client to attach case/contract
return ['action' => 'invalid', 'message' => 'Import must be linked to a client to create a case'];
} else {
return ['action' => 'invalid', 'message' => 'Unable to resolve client_case (need import client)'];
if (! $clientCaseId) {
// Resolve by identifiers or provided person; do not use Client->person
$personId = null;
if (! empty($mapped['person'] ?? [])) {
$personId = $this->findPersonIdByIdentifiers($mapped['person']);
if (! $personId) {
$personId = $this->findOrCreatePersonId($mapped['person']);
}
}
// As a last resort, create a minimal person for this client
if ($clientId && ! $personId) {
$personId = $this->createMinimalPersonId();
}
if ($clientId && $personId) {
$clientCaseId = $this->findOrCreateClientCaseId($clientId, $personId, $clientRef);
} elseif ($personId) {
// require an import client to attach case/contract
return ['action' => 'invalid', 'message' => 'Import must be linked to a client to create a case'];
} else {
return ['action' => 'invalid', 'message' => 'Unable to resolve client_case (need import client)'];
}
}
}
@@ -644,6 +898,9 @@ private function upsertContractChain(Import $import, array $mapped, $mappings):
continue;
}
$value = $contractData[$field] ?? null;
if ($field === 'reference' && ! is_null($value)) {
$value = preg_replace('/\s+/', '', trim((string) $value));
}
$mode = $map->apply_mode ?? 'both';
if (in_array($mode, ['insert', 'both'])) {
$applyInsert[$field] = $value;
@@ -655,11 +912,12 @@ private function upsertContractChain(Import $import, array $mapped, $mappings):
if ($existing) {
if (empty($applyUpdate)) {
return ['action' => 'skipped', 'message' => 'No contract fields marked for update'];
// Return existing contract reference even when skipped so callers can treat as resolved
return ['action' => 'skipped', 'message' => 'No contract fields marked for update', 'contract' => $existing];
}
$changes = array_filter($applyUpdate, fn ($v) => ! is_null($v));
if (empty($changes)) {
return ['action' => 'skipped', 'message' => 'No non-null contract changes'];
return ['action' => 'skipped', 'message' => 'No non-null contract changes', 'contract' => $existing];
}
$existing->fill($changes);
$existing->save();
@@ -681,6 +939,197 @@ private function upsertContractChain(Import $import, array $mapped, $mappings):
}
}
private function sanitizeHeaderName(string $v): string
{
// Strip UTF-8 BOM and trim whitespace/control characters
$v = preg_replace('/^\xEF\xBB\xBF/', '', $v) ?? $v;
return trim($v);
}
private function findSourceColumnFor($mappings, string $targetField): ?string
{
foreach ($mappings as $map) {
if ((string) ($map->target_field ?? '') === $targetField) {
$src = (string) ($map->source_column ?? '');
return $src !== '' ? $src : null;
}
}
return null;
}
// Removed auto-detection helpers by request: no pattern scanning or fallback derivation
private function normalizeDecimal(string $raw): string
{
// Keep digits, comma, dot, and minus to detect separators
$s = preg_replace('/[^0-9,\.-]/', '', $raw) ?? '';
$s = trim($s);
if ($s === '') {
return $s;
}
$lastComma = strrpos($s, ',');
$lastDot = strrpos($s, '.');
// Determine decimal separator by last occurrence
$decimalSep = null;
if ($lastComma !== false || $lastDot !== false) {
if ($lastComma === false) {
$decimalSep = '.';
} elseif ($lastDot === false) {
$decimalSep = ',';
} else {
$decimalSep = $lastComma > $lastDot ? ',' : '.';
}
}
// Remove all thousand separators (the other one) and unify decimal to '.'
if ($decimalSep === ',') {
// remove all dots
$s = str_replace('.', '', $s);
// replace last comma with dot
$pos = strrpos($s, ',');
if ($pos !== false) {
$s[$pos] = '.';
}
// remove any remaining commas (unlikely)
$s = str_replace(',', '', $s);
} elseif ($decimalSep === '.') {
// remove all commas
$s = str_replace(',', '', $s);
// dot already decimal
} else {
// no decimal separator: remove commas/dots entirely
$s = str_replace([',', '.'], '', $s);
}
// Collapse multiple minus signs, keep leading only
$s = ltrim($s, '+');
$neg = false;
if (str_starts_with($s, '-')) {
$neg = true;
$s = ltrim($s, '-');
}
// Remove any stray minus signs
$s = str_replace('-', '', $s);
if ($neg) {
$s = '-'.$s;
}
return $s;
}
/**
* Ensure mapping roots are recognized; fail fast if unknown roots found.
*/
private function validateMappingRoots($mappings, array $validRoots): void
{
foreach ($mappings as $map) {
$target = (string) ($map->target_field ?? '');
if ($target === '') {
continue;
}
$root = explode('.', $target)[0];
if (! in_array($root, $validRoots, true)) {
// Common typos guidance
$hint = '';
if (str_starts_with($root, 'contract')) {
$hint = ' Did you mean "contract"?';
}
throw new \InvalidArgumentException('Unknown mapping root "'.$root.'" in target_field "'.$target.'".'.$hint);
}
}
}
private function mappingIncludes($mappings, string $targetField): bool
{
foreach ($mappings as $map) {
if ((string) ($map->target_field ?? '') === $targetField) {
return true;
}
}
return false;
}
/**
* Normalize mapping target_field to canonical forms.
* Examples:
* - contracts.reference => contract.reference
* - accounts.balance_amount => account.balance_amount
* - person_phones.nu => phone.nu
* - person_addresses.address => address.address
* - emails.email|emails.value => email.value
*/
private function normalizeMappings($mappings, array $rootAliasMap, array $fieldAliasMap)
{
$normalized = [];
foreach ($mappings as $map) {
$clone = clone $map;
$clone->target_field = $this->normalizeTargetField((string) ($map->target_field ?? ''), $rootAliasMap, $fieldAliasMap);
$normalized[] = $clone;
}
return collect($normalized);
}
private function normalizeTargetField(string $target, array $rootAliasMap, array $fieldAliasMap): string
{
if ($target === '') {
return $target;
}
$parts = explode('.', $target);
$root = $parts[0] ?? '';
$field = $parts[1] ?? null;
// Root aliases (plural to canonical) from DB
$root = $rootAliasMap[$root] ?? $root;
// Field aliases per root from DB
$aliases = $fieldAliasMap[$root] ?? [];
if ($field === null && isset($aliases['__default'])) {
$field = $aliases['__default'];
} elseif (isset($aliases[$field])) {
$field = $aliases[$field];
}
// Rebuild
if ($field !== null) {
return $root.'.'.$field;
}
return $root;
}
private function loadImportEntityConfig(): array
{
$entities = ImportEntity::all();
$rootAliasMap = [];
$fieldAliasMap = [];
$validRoots = [];
foreach ($entities as $ent) {
$canonical = $ent->canonical_root;
$validRoots[] = $canonical;
foreach ((array) ($ent->aliases ?? []) as $alias) {
$rootAliasMap[$alias] = $canonical;
}
// Also ensure canonical maps to itself
$rootAliasMap[$canonical] = $canonical;
$aliases = (array) ($ent->field_aliases ?? []);
// Allow default field per entity via '__default'
if (is_array($ent->fields) && count($ent->fields)) {
$aliases['__default'] = $aliases['__default'] ?? null;
}
$fieldAliasMap[$canonical] = $aliases;
}
// sensible defaults when DB empty
if (empty($validRoots)) {
$validRoots = ['person', 'contract', 'account', 'address', 'phone', 'email', 'client_case'];
}
return [$rootAliasMap, $fieldAliasMap, $validRoots];
}
private function findOrCreatePersonId(array $p): ?int
{
// Basic dedup: by tax_number, ssn, else full_name
@@ -776,14 +1225,30 @@ private function findOrCreateClientId(int $personId): int
return Client::create(['person_id' => $personId])->id;
}
private function findOrCreateClientCaseId(int $clientId, int $personId): int
private function findOrCreateClientCaseId(int $clientId, int $personId, ?string $clientRef = null): int
{
// Prefer existing by client_ref if provided
if ($clientRef) {
$cc = ClientCase::where('client_id', $clientId)
->where('client_ref', $clientRef)
->first();
if ($cc) {
// Ensure person_id is set (if missing) when matching by client_ref
if (! $cc->person_id) {
$cc->person_id = $personId;
$cc->save();
}
return $cc->id;
}
}
// Fallback: by (client_id, person_id)
$cc = ClientCase::where('client_id', $clientId)->where('person_id', $personId)->first();
if ($cc) {
return $cc->id;
}
return ClientCase::create(['client_id' => $clientId, 'person_id' => $personId])->id;
return ClientCase::create(['client_id' => $clientId, 'person_id' => $personId, 'client_ref' => $clientRef])->id;
}
private function upsertEmail(int $personId, array $emailData, $mappings): array
@@ -948,4 +1413,77 @@ private function upsertPhone(int $personId, array $phoneData, $mappings): array
return ['action' => 'inserted', 'phone' => $created];
}
}
/**
* After a contract is inserted/updated, attach default segment and create an activity
* using decision_id from import/template meta. Activity note includes template name.
*/
private function postContractActions(Import $import, Contract $contract): void
{
$meta = $import->meta ?? [];
$segmentId = (int) ($meta['segment_id'] ?? 0);
$decisionId = (int) ($meta['decision_id'] ?? 0);
$templateName = (string) ($meta['template_name'] ?? optional($import->template)->name ?? '');
$actionId = (int) ($meta['action_id'] ?? 0);
// Attach segment to contract as the main (active) segment if provided
if ($segmentId > 0) {
// Ensure the segment exists on the client case and is active
$ccSeg = \DB::table('client_case_segment')
->where('client_case_id', $contract->client_case_id)
->where('segment_id', $segmentId)
->first();
if (! $ccSeg) {
\DB::table('client_case_segment')->insert([
'client_case_id' => $contract->client_case_id,
'segment_id' => $segmentId,
'active' => true,
'created_at' => now(),
'updated_at' => now(),
]);
} elseif (! $ccSeg->active) {
\DB::table('client_case_segment')
->where('id', $ccSeg->id)
->update(['active' => true, 'updated_at' => now()]);
}
// Deactivate all other segments for this contract to make this the main one
\DB::table('contract_segment')
->where('contract_id', $contract->id)
->where('segment_id', '!=', $segmentId)
->update(['active' => false, 'updated_at' => now()]);
// Upsert the selected segment as active for this contract
$pivot = \DB::table('contract_segment')
->where('contract_id', $contract->id)
->where('segment_id', $segmentId)
->first();
if ($pivot) {
if (! $pivot->active) {
\DB::table('contract_segment')
->where('id', $pivot->id)
->update(['active' => true, 'updated_at' => now()]);
}
} else {
\DB::table('contract_segment')->insert([
'contract_id' => $contract->id,
'segment_id' => $segmentId,
'active' => true,
'created_at' => now(),
'updated_at' => now(),
]);
}
}
// Create activity if decision provided
if ($decisionId > 0) {
Activity::create([
'decision_id' => $decisionId,
'action_id' => $actionId > 0 ? $actionId : null,
'contract_id' => $contract->id,
'client_case_id' => $contract->client_case_id,
'note' => trim('Imported via template'.($templateName ? ': '.$templateName : '')),
]);
}
}
}