with([ 'client:id,uuid,person_id', 'client.person:id,uuid,full_name', 'template:id,name', ]) ->orderByDesc('created_at') ->paginate(15); $imports = [ 'data' => $paginator->items(), 'links' => [ 'first' => $paginator->url(1), 'last' => $paginator->url($paginator->lastPage()), 'prev' => $paginator->previousPageUrl(), 'next' => $paginator->nextPageUrl(), ], 'meta' => [ 'current_page' => $paginator->currentPage(), 'from' => $paginator->firstItem(), 'last_page' => $paginator->lastPage(), 'path' => $paginator->path(), 'per_page' => $paginator->perPage(), 'to' => $paginator->lastItem(), 'total' => $paginator->total(), ], ]; // Map items into a simpler shape $imports['data'] = array_map(function ($imp) { return [ 'id' => $imp->id, 'uuid' => $imp->uuid, 'created_at' => $imp->created_at, 'original_name' => $imp->original_name, 'size' => $imp->size, 'status' => $imp->status, 'client' => $imp->client ? [ 'id' => $imp->client_id, 'uuid' => $imp->client->uuid, 'person' => $imp->client->person ? [ 'uuid' => $imp->client->person->uuid, 'full_name' => $imp->client->person->full_name, ] : null, ] : null, 'template' => $imp->template ? ['id' => $imp->template->id, 'name' => $imp->template->name] : null, ]; }, $imports['data']); return Inertia::render('Imports/Index', [ 'imports' => $imports, ]); } // Show the import creation page public function create(Request $request) { $templates = ImportTemplate::query() ->leftJoin('clients', 'clients.id', '=', 'import_templates.client_id') ->where('import_templates.is_active', true) ->orderBy('import_templates.name') ->get([ 'import_templates.id', 'import_templates.uuid', 'import_templates.name', 'import_templates.source_type', 'import_templates.default_record_type', 'import_templates.client_id', DB::raw('clients.uuid as client_uuid'), ]); $clients = Client::query() ->join('person', 'person.id', '=', 'clients.person_id') ->orderBy('person.full_name') ->get([ 'clients.id', 'clients.uuid', DB::raw('person.full_name as name'), ]); return Inertia::render('Imports/Create', [ 'templates' => $templates, 'clients' => $clients, // no existing import on create ]); } // Create a new import job, store file, and return basic info public function store(Request $request) { $validated = $request->validate([ 'file' => 'required|file|max:20480', // 20MB; adjust as needed 'source_type' => 'nullable|string|in:csv,xml,xls,xlsx,json,txt', 'sheet_name' => 'nullable|string|max:64', 'has_header' => 'nullable|boolean', 'import_template_id' => 'nullable|integer|exists:import_templates,id', 'client_uuid' => 'nullable|string|exists:clients,uuid', ]); $file = $validated['file']; $ext = strtolower($file->getClientOriginalExtension()); $sourceType = $validated['source_type'] ?? ($ext === 'txt' ? 'csv' : $ext); $uuid = (string) Str::uuid(); $disk = 'local'; $path = $file->storeAs('imports', $uuid.'.'.$ext, $disk); // Resolve client_uuid to client_id if provided $clientId = null; if (! empty($validated['client_uuid'] ?? null)) { $clientId = Client::where('uuid', $validated['client_uuid'])->value('id'); } $import = Import::create([ 'uuid' => $uuid, 'user_id' => $request->user()?->id, 'import_template_id' => $validated['import_template_id'] ?? null, 'client_id' => $clientId, 'source_type' => $sourceType, 'file_name' => basename($path), 'original_name' => $file->getClientOriginalName(), 'disk' => $disk, 'path' => $path, 'size' => $file->getSize(), 'sheet_name' => $validated['sheet_name'] ?? null, 'status' => 'uploaded', 'show_missing' => false, 'meta' => [ 'has_header' => $validated['has_header'] ?? true, ], ]); return response()->json([ 'id' => $import->id, 'uuid' => $import->uuid, 'status' => $import->status, 'show_missing' => (bool) ($import->show_missing ?? false), ]); } // Kick off processing of an import - simple synchronous step for now public function process(Import $import, Request $request, ImportProcessor $processor) { $import->update(['status' => 'validating', 'started_at' => now()]); $result = $processor->process($import, user: $request->user()); return response()->json($result); } // Analyze the uploaded file and return column headers or positional indices public function columns(Request $request, Import $import, CsvImportService $csv) { $validated = $request->validate([ 'has_header' => 'nullable|boolean', 'delimiter' => 'nullable|string|max:4', ]); $hasHeader = array_key_exists('has_header', $validated) ? (bool) $validated['has_header'] : (bool) ($import->meta['has_header'] ?? true); // Resolve delimiter preference: explicit param > template meta > existing meta > auto-detect $explicitDelimiter = null; if (array_key_exists('delimiter', $validated) && $validated['delimiter'] !== null && $validated['delimiter'] !== '') { $explicitDelimiter = (string) $validated['delimiter']; } elseif ($import->import_template_id) { // Try reading template meta for a default delimiter $tplDelimiter = optional(ImportTemplate::find($import->import_template_id))->meta['delimiter'] ?? null; if ($tplDelimiter) { $explicitDelimiter = (string) $tplDelimiter; } } elseif (! empty($import->meta['forced_delimiter'] ?? null)) { $explicitDelimiter = (string) $import->meta['forced_delimiter']; } // Prefer CSV/TXT; if source_type is unknown, attempt best-effort based on file extension $treatAsText = in_array($import->source_type, ['csv', 'txt']); if (! $treatAsText) { $orig = strtolower(pathinfo($import->original_name ?? '', PATHINFO_EXTENSION)); if (in_array($orig, ['csv', 'txt'])) { $treatAsText = true; } } $fullPath = Storage::disk($import->disk)->path($import->path); $note = ''; if ($treatAsText) { if ($explicitDelimiter !== null && $explicitDelimiter !== '') { $columns = $csv->parseColumnsFromCsv($fullPath, $explicitDelimiter, $hasHeader); $delimiter = $explicitDelimiter; } else { [$delimiter, $columns] = $csv->detectColumnsFromCsv($fullPath, $hasHeader); // Backstop: if single column but file clearly has separators, try common ones if (is_array($columns) && count($columns) <= 1) { foreach ([';', "\t", '|', ' ', ','] as $try) { $alt = $csv->parseColumnsFromCsv($fullPath, $try, $hasHeader); if (is_array($alt) && count($alt) > 1) { $delimiter = $try; $columns = $alt; $note = 'Delimiter auto-detection backstopped to '.json_encode($try); break; } } } } } else { // Best-effort: try detect anyway [$delimiter, $columns] = $csv->detectColumnsFromCsv($fullPath, $hasHeader); } // Save meta $meta = $import->meta ?? []; $meta['has_header'] = $hasHeader; $meta['detected_delimiter'] = $delimiter; $meta['columns'] = $columns; if ($explicitDelimiter) { $meta['forced_delimiter'] = $explicitDelimiter; } $import->update([ 'meta' => $meta, 'status' => $import->status === 'uploaded' ? 'parsed' : $import->status, ]); return response()->json([ 'columns' => $columns, 'has_header' => $hasHeader, 'detected_delimiter' => $delimiter, 'note' => $note, ]); } // CSV helpers moved to App\Services\CsvImportService // Save ad-hoc mappings for a specific import (when no template is selected) public function saveMappings(Request $request, Import $import) { $data = $request->validate([ 'mappings' => 'required|array', 'mappings.*.source_column' => 'required|string', 'mappings.*.entity' => 'nullable|string|in:person,person_addresses,person_phones,emails,accounts,contracts,client_cases,payments', 'mappings.*.target_field' => 'required|string', 'mappings.*.transform' => 'nullable|string|in:trim,upper,lower,decimal,ref', 'mappings.*.apply_mode' => 'nullable|string|in:insert,update,both,keyref', 'mappings.*.options' => 'nullable|array', ]); // Upsert by (import_id, source_column): update existing rows; insert new ones; avoid duplicates $now = now(); $existing = \DB::table('import_mappings') ->where('import_id', $import->id) ->get(['id', 'source_column', 'position']); $bySource = []; $dupes = []; foreach ($existing as $row) { $src = (string) $row->source_column; if (! array_key_exists($src, $bySource)) { $bySource[$src] = ['id' => $row->id, 'position' => $row->position]; } else { $dupes[$src] = ($dupes[$src] ?? []); $dupes[$src][] = $row->id; } } $basePosition = (int) (\DB::table('import_mappings')->where('import_id', $import->id)->max('position') ?? -1); $inserted = 0; $updated = 0; $deduped = 0; foreach ($data['mappings'] as $pos => $m) { $src = (string) $m['source_column']; $payload = [ 'entity' => $m['entity'] ?? null, 'target_field' => $m['target_field'], 'transform' => $m['transform'] ?? null, 'apply_mode' => $m['apply_mode'] ?? 'both', 'options' => $m['options'] ?? null, 'position' => $pos, // keep UI order 'updated_at' => $now, ]; if (array_key_exists($src, $bySource)) { // Update first occurrence \DB::table('import_mappings')->where('id', $bySource[$src]['id'])->update($payload); $updated++; // Remove duplicates if any if (! empty($dupes[$src])) { $deleted = \DB::table('import_mappings')->whereIn('id', $dupes[$src])->delete(); $deduped += (int) $deleted; unset($dupes[$src]); } } else { // Insert new \DB::table('import_mappings')->insert([ 'import_id' => $import->id, 'entity' => $payload['entity'], 'source_column' => $src, 'target_field' => $payload['target_field'], 'transform' => $payload['transform'], 'apply_mode' => $payload['apply_mode'], 'options' => $payload['options'], 'position' => ++$basePosition, 'created_at' => $now, 'updated_at' => $now, ]); $inserted++; } } // Mark this as ad-hoc mapping usage $import->update(['import_template_id' => null]); return response()->json(['ok' => true, 'saved' => ($inserted + $updated), 'inserted' => $inserted, 'updated' => $updated, 'deduped' => $deduped]); } // Fetch current mappings for an import (after applying a template or saving ad-hoc mappings) public function getMappings(Import $import) { $rows = \DB::table('import_mappings') ->where('import_id', $import->id) ->orderBy('position') ->orderBy('id') ->get([ 'id', 'entity', 'source_column', 'target_field', 'transform', 'apply_mode', 'options', 'position', ]); return response()->json(['mappings' => $rows]); } /** * List active, non-archived contracts for the import's client that are NOT present * in the processed import file (based on mapped contract.reference values). * Only available when contract.reference mapping apply_mode is 'keyref'. */ public function missingContracts(Import $import) { // Ensure client context is available if (empty($import->client_id)) { return response()->json(['error' => 'Import has no client bound.'], 422); } // Respect optional feature flag on import if (! (bool) ($import->show_missing ?? false)) { return response()->json(['error' => 'Missing contracts listing is disabled for this import.'], 422); } // Check that this import's mappings set contract.reference to keyref mode $mappings = \DB::table('import_mappings') ->where('import_id', $import->id) ->get(['target_field', 'apply_mode']); $isKeyref = false; foreach ($mappings as $map) { $tf = strtolower((string) ($map->target_field ?? '')); $am = strtolower((string) ($map->apply_mode ?? '')); if (in_array($tf, ['contract.reference', 'contracts.reference'], true) && $am === 'keyref') { $isKeyref = true; break; } } if (! $isKeyref) { return response()->json(['error' => 'Missing contracts are only available for keyref mapping on contract.reference.'], 422); } // Collect referenced contract references from processed rows $present = []; foreach (\App\Models\ImportRow::query()->where('import_id', $import->id)->get(['mapped_data']) as $row) { $md = $row->mapped_data ?? []; if (is_array($md) && isset($md['contract']['reference'])) { $ref = (string) $md['contract']['reference']; if ($ref !== '') { $present[] = preg_replace('/\s+/', '', trim($ref)); } } } $present = array_values(array_unique(array_filter($present))); // Query active, non-archived contracts for this client that were not in import // Include person full_name (owner of the client case) and aggregate active accounts' balance_amount $contractsQ = \App\Models\Contract::query() ->join('client_cases', 'contracts.client_case_id', '=', 'client_cases.id') ->join('person', 'person.id', '=', 'client_cases.person_id') ->leftJoin('accounts', function ($join) { $join->on('accounts.contract_id', '=', 'contracts.id') ->where('accounts.active', 1); }) ->where('client_cases.client_id', $import->client_id) ->where('contracts.active', 1) ->whereNull('contracts.deleted_at') ->when(count($present) > 0, function ($q) use ($present) { $q->whereNotIn('contracts.reference', $present); }) ->groupBy('contracts.uuid', 'contracts.reference', 'client_cases.uuid', 'person.full_name') ->orderBy('contracts.reference') ->get([ 'contracts.uuid as uuid', 'contracts.reference as reference', 'client_cases.uuid as case_uuid', 'person.full_name as full_name', \DB::raw('COALESCE(SUM(accounts.balance_amount), 0) as balance_amount'), ]); return response()->json([ 'missing' => $contractsQ, 'count' => $contractsQ->count(), ]); } /** * Update import options (e.g., booleans like show_missing, reactivate) from the UI. */ public function updateOptions(Request $request, Import $import) { $data = $request->validate([ 'show_missing' => 'nullable|boolean', 'reactivate' => 'nullable|boolean', ]); $payload = []; if (array_key_exists('show_missing', $data)) { $payload['show_missing'] = (bool) $data['show_missing']; } if (array_key_exists('reactivate', $data)) { $payload['reactivate'] = (bool) $data['reactivate']; } if (! empty($payload)) { $import->update($payload); } return response()->json([ 'ok' => true, 'import' => [ 'id' => $import->id, 'uuid' => $import->uuid, 'show_missing' => (bool) ($import->show_missing ?? false), 'reactivate' => (bool) ($import->reactivate ?? false), ], ]); } // Fetch recent import events (logs) for an import public function getEvents(Import $import) { $limit = (int) request()->query('limit', 200); $limit = max(1, min($limit, 1000)); $events = ImportEvent::query() ->where('import_id', $import->id) ->orderByDesc('id') ->limit($limit) ->get(['id', 'created_at', 'level', 'event', 'message', 'import_row_id', 'context']); return response()->json(['events' => $events]); } // Preview (up to N) raw CSV rows for an import for mapping review public function preview(Import $import, Request $request) { $validated = $request->validate([ 'limit' => 'nullable|integer|min:1|max:500', ]); $limit = (int) ($validated['limit'] ?? 200); // Determine header/delimiter the same way as columns() stored them $meta = $import->meta ?? []; $hasHeader = (bool) ($meta['has_header'] ?? true); // Forced delimiter overrides everything; else detected; fallback comma $delimiter = $meta['forced_delimiter'] ?? $meta['detected_delimiter'] ?? ','; $rows = []; $columns = []; $truncated = false; $path = Storage::disk($import->disk)->path($import->path); if (! is_readable($path)) { return response()->json([ 'error' => 'File not readable', ], 422); } $fh = @fopen($path, 'r'); if (! $fh) { return response()->json([ 'error' => 'Unable to open file', ], 422); } try { if ($hasHeader) { $header = fgetcsv($fh, 0, $delimiter) ?: []; $columns = array_map(function ($h) { return is_string($h) ? trim($h) : (string) $h; }, $header); } else { // Use meta stored columns when available, else infer later from widest row $columns = is_array($meta['columns'] ?? null) ? $meta['columns'] : []; } $count = 0; $widest = count($columns); while (($data = fgetcsv($fh, 0, $delimiter)) !== false) { if ($count >= $limit) { $truncated = true; break; } // Track widest for non-header scenario if (! $hasHeader) { $widest = max($widest, count($data)); } $rows[] = $data; $count++; } if (! $hasHeader && $widest > count($columns)) { // Generate positional column labels if missing $columns = []; for ($i = 0; $i < $widest; $i++) { $columns[] = 'col_'.($i + 1); } } } finally { fclose($fh); } // Normalize each row into assoc keyed by columns (pad/truncate as needed) $assocRows = []; foreach ($rows as $r) { $assoc = []; foreach ($columns as $i => $colName) { $assoc[$colName] = array_key_exists($i, $r) ? $r[$i] : null; } $assocRows[] = $assoc; } return response()->json([ 'columns' => $columns, 'rows' => $assocRows, 'limit' => $limit, 'truncated' => $truncated, 'has_header' => $hasHeader, ]); } /** * Simulate application of payment rows for a payments import without persisting changes. * Returns per-row projected balance changes and resolution of contract / account references. */ public function simulatePayments(Import $import, Request $request) { // Delegate to the generic simulate method for backward compatibility. return $this->simulate($import, $request); } /** * Generic simulation endpoint: projects what would happen if the import were processed * using the first N rows and current saved mappings. Works for both payments and non-payments * templates. For payments templates, payment-specific summaries/entities will be included * automatically by the simulation service when mappings contain the payment root. */ public function simulate(Import $import, Request $request) { $validated = $request->validate([ 'limit' => 'nullable|integer|min:1|max:500', 'verbose' => 'nullable|boolean', ]); $limit = (int) ($validated['limit'] ?? 100); $verbose = (bool) ($validated['verbose'] ?? false); $service = app(\App\Services\ImportSimulationService::class); $result = $service->simulate($import, $limit, $verbose); return response()->json($result); } // Show an existing import by UUID to continue where left off public function show(Import $import) { // Load templates (global + client specific) and clients for selection on continue page $templates = ImportTemplate::query() ->leftJoin('clients', 'clients.id', '=', 'import_templates.client_id') ->where('import_templates.is_active', true) ->where('import_templates.id', $import->import_template_id) ->orderBy('import_templates.name') ->get([ 'import_templates.id', 'import_templates.uuid', 'import_templates.name', 'import_templates.source_type', 'import_templates.default_record_type', 'import_templates.client_id', 'import_templates.meta', 'clients.uuid as client_uuid', ]); $clients = Client::query() ->join('person', 'person.id', '=', 'clients.person_id') ->orderBy('person.full_name') ->where('clients.id', $import->client_id) ->get([ 'clients.id', 'clients.uuid', 'person.full_name as name', ]); // Import client $client = Client::query() ->join('person', 'person.id', '=', 'clients.person_id') ->where('clients.id', $import->client_id) ->firstOrFail([ 'clients.uuid as uuid', 'person.full_name as name', ]); // Render a dedicated page to continue the import return Inertia::render('Imports/Import', [ 'import' => [ 'id' => $import->id, 'uuid' => $import->uuid, 'status' => $import->status, 'meta' => $import->meta, 'client_id' => $import->client_id, 'client_uuid' => optional($client)->uuid, 'import_template_id' => $import->import_template_id, 'show_missing' => (bool) ($import->show_missing ?? false), 'reactivate' => (bool) ($import->reactivate ?? false), 'total_rows' => $import->total_rows, 'imported_rows' => $import->imported_rows, 'invalid_rows' => $import->invalid_rows, 'valid_rows' => $import->valid_rows, 'finished_at' => $import->finished_at, ], 'templates' => $templates, 'clients' => $clients, 'client' => $client, ]); } // Delete an import if not finished (statuses allowed: uploaded, mapping, processing_failed etc.) public function destroy(Request $request, Import $import) { // Only allow deletion if not completed or processing if (in_array($import->status, ['completed', 'processing'])) { return back()->with([ 'ok' => false, 'message' => 'Import can not be deleted in its current status.', ], 422); } // Attempt to delete stored file try { if ($import->disk && $import->path && Storage::disk($import->disk)->exists($import->path)) { Storage::disk($import->disk)->delete($import->path); } } catch (\Throwable $e) { // Log event but proceed with deletion ImportEvent::create([ 'import_id' => $import->id, 'user_id' => $request->user()?->getAuthIdentifier(), 'event' => 'file_delete_failed', 'level' => 'warning', 'message' => 'Failed to delete import file: '.$e->getMessage(), ]); } // Clean up related events/rows optionally (soft approach: rely on FKs if cascade configured) // If not cascaded, we could manually delete; check quickly // Assuming foreign key ON DELETE CASCADE for import_rows & import_events $import->delete(); return back()->with(['ok' => true]); } }