Teren-app/app/Http/Controllers/ImportController.php
2025-10-13 21:14:10 +02:00

698 lines
27 KiB
PHP

<?php
namespace App\Http\Controllers;
use App\Models\Account;
use App\Models\Client;
use App\Models\Contract;
use App\Models\Import;
use App\Models\ImportEvent;
use App\Models\ImportTemplate;
use App\Services\CsvImportService;
use App\Services\ImportProcessor;
use Illuminate\Http\Request;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Storage;
use Illuminate\Support\Str;
use Inertia\Inertia;
class ImportController extends Controller
{
// List imports (paginated)
public function index(Request $request)
{
$paginator = Import::query()
->with([
'client:id,uuid,person_id',
'client.person:id,uuid,full_name',
'template:id,name',
])
->orderByDesc('created_at')
->paginate(15);
$imports = [
'data' => $paginator->items(),
'links' => [
'first' => $paginator->url(1),
'last' => $paginator->url($paginator->lastPage()),
'prev' => $paginator->previousPageUrl(),
'next' => $paginator->nextPageUrl(),
],
'meta' => [
'current_page' => $paginator->currentPage(),
'from' => $paginator->firstItem(),
'last_page' => $paginator->lastPage(),
'path' => $paginator->path(),
'per_page' => $paginator->perPage(),
'to' => $paginator->lastItem(),
'total' => $paginator->total(),
],
];
// Map items into a simpler shape
$imports['data'] = array_map(function ($imp) {
return [
'id' => $imp->id,
'uuid' => $imp->uuid,
'created_at' => $imp->created_at,
'original_name' => $imp->original_name,
'size' => $imp->size,
'status' => $imp->status,
'client' => $imp->client ? [
'id' => $imp->client_id,
'uuid' => $imp->client->uuid,
'person' => $imp->client->person ? [
'uuid' => $imp->client->person->uuid,
'full_name' => $imp->client->person->full_name,
] : null,
] : null,
'template' => $imp->template ? ['id' => $imp->template->id, 'name' => $imp->template->name] : null,
];
}, $imports['data']);
return Inertia::render('Imports/Index', [
'imports' => $imports,
]);
}
// Show the import creation page
public function create(Request $request)
{
$templates = ImportTemplate::query()
->leftJoin('clients', 'clients.id', '=', 'import_templates.client_id')
->where('import_templates.is_active', true)
->orderBy('import_templates.name')
->get([
'import_templates.id',
'import_templates.uuid',
'import_templates.name',
'import_templates.source_type',
'import_templates.default_record_type',
'import_templates.client_id',
DB::raw('clients.uuid as client_uuid'),
]);
$clients = Client::query()
->join('person', 'person.id', '=', 'clients.person_id')
->orderBy('person.full_name')
->get([
'clients.id',
'clients.uuid',
DB::raw('person.full_name as name'),
]);
return Inertia::render('Imports/Create', [
'templates' => $templates,
'clients' => $clients,
// no existing import on create
]);
}
// Create a new import job, store file, and return basic info
public function store(Request $request)
{
$validated = $request->validate([
'file' => 'required|file|max:20480', // 20MB; adjust as needed
'source_type' => 'nullable|string|in:csv,xml,xls,xlsx,json,txt',
'sheet_name' => 'nullable|string|max:64',
'has_header' => 'nullable|boolean',
'import_template_id' => 'nullable|integer|exists:import_templates,id',
'client_uuid' => 'nullable|string|exists:clients,uuid',
]);
$file = $validated['file'];
$ext = strtolower($file->getClientOriginalExtension());
$sourceType = $validated['source_type'] ?? ($ext === 'txt' ? 'csv' : $ext);
$uuid = (string) Str::uuid();
$disk = 'local';
$path = $file->storeAs('imports', $uuid.'.'.$ext, $disk);
// Resolve client_uuid to client_id if provided
$clientId = null;
if (! empty($validated['client_uuid'] ?? null)) {
$clientId = Client::where('uuid', $validated['client_uuid'])->value('id');
}
$import = Import::create([
'uuid' => $uuid,
'user_id' => $request->user()?->id,
'import_template_id' => $validated['import_template_id'] ?? null,
'client_id' => $clientId,
'source_type' => $sourceType,
'file_name' => basename($path),
'original_name' => $file->getClientOriginalName(),
'disk' => $disk,
'path' => $path,
'size' => $file->getSize(),
'sheet_name' => $validated['sheet_name'] ?? null,
'status' => 'uploaded',
'show_missing' => false,
'meta' => [
'has_header' => $validated['has_header'] ?? true,
],
]);
return response()->json([
'id' => $import->id,
'uuid' => $import->uuid,
'status' => $import->status,
'show_missing' => (bool) ($import->show_missing ?? false),
]);
}
// Kick off processing of an import - simple synchronous step for now
public function process(Import $import, Request $request, ImportProcessor $processor)
{
$import->update(['status' => 'validating', 'started_at' => now()]);
$result = $processor->process($import, user: $request->user());
return response()->json($result);
}
// Analyze the uploaded file and return column headers or positional indices
public function columns(Request $request, Import $import, CsvImportService $csv)
{
$validated = $request->validate([
'has_header' => 'nullable|boolean',
'delimiter' => 'nullable|string|max:4',
]);
$hasHeader = array_key_exists('has_header', $validated)
? (bool) $validated['has_header']
: (bool) ($import->meta['has_header'] ?? true);
// Resolve delimiter preference: explicit param > template meta > existing meta > auto-detect
$explicitDelimiter = null;
if (array_key_exists('delimiter', $validated) && $validated['delimiter'] !== null && $validated['delimiter'] !== '') {
$explicitDelimiter = (string) $validated['delimiter'];
} elseif ($import->import_template_id) {
// Try reading template meta for a default delimiter
$tplDelimiter = optional(ImportTemplate::find($import->import_template_id))->meta['delimiter'] ?? null;
if ($tplDelimiter) {
$explicitDelimiter = (string) $tplDelimiter;
}
} elseif (! empty($import->meta['forced_delimiter'] ?? null)) {
$explicitDelimiter = (string) $import->meta['forced_delimiter'];
}
// Prefer CSV/TXT; if source_type is unknown, attempt best-effort based on file extension
$treatAsText = in_array($import->source_type, ['csv', 'txt']);
if (! $treatAsText) {
$orig = strtolower(pathinfo($import->original_name ?? '', PATHINFO_EXTENSION));
if (in_array($orig, ['csv', 'txt'])) {
$treatAsText = true;
}
}
$fullPath = Storage::disk($import->disk)->path($import->path);
$note = '';
if ($treatAsText) {
if ($explicitDelimiter !== null && $explicitDelimiter !== '') {
$columns = $csv->parseColumnsFromCsv($fullPath, $explicitDelimiter, $hasHeader);
$delimiter = $explicitDelimiter;
} else {
[$delimiter, $columns] = $csv->detectColumnsFromCsv($fullPath, $hasHeader);
// Backstop: if single column but file clearly has separators, try common ones
if (is_array($columns) && count($columns) <= 1) {
foreach ([';', "\t", '|', ' ', ','] as $try) {
$alt = $csv->parseColumnsFromCsv($fullPath, $try, $hasHeader);
if (is_array($alt) && count($alt) > 1) {
$delimiter = $try;
$columns = $alt;
$note = 'Delimiter auto-detection backstopped to '.json_encode($try);
break;
}
}
}
}
} else {
// Best-effort: try detect anyway
[$delimiter, $columns] = $csv->detectColumnsFromCsv($fullPath, $hasHeader);
}
// Save meta
$meta = $import->meta ?? [];
$meta['has_header'] = $hasHeader;
$meta['detected_delimiter'] = $delimiter;
$meta['columns'] = $columns;
if ($explicitDelimiter) {
$meta['forced_delimiter'] = $explicitDelimiter;
}
$import->update([
'meta' => $meta,
'status' => $import->status === 'uploaded' ? 'parsed' : $import->status,
]);
return response()->json([
'columns' => $columns,
'has_header' => $hasHeader,
'detected_delimiter' => $delimiter,
'note' => $note,
]);
}
// CSV helpers moved to App\Services\CsvImportService
// Save ad-hoc mappings for a specific import (when no template is selected)
public function saveMappings(Request $request, Import $import)
{
$data = $request->validate([
'mappings' => 'required|array',
'mappings.*.source_column' => 'required|string',
'mappings.*.entity' => 'nullable|string|in:person,person_addresses,person_phones,emails,accounts,contracts,client_cases,payments',
'mappings.*.target_field' => 'required|string',
'mappings.*.transform' => 'nullable|string|in:trim,upper,lower,decimal,ref',
'mappings.*.apply_mode' => 'nullable|string|in:insert,update,both,keyref',
'mappings.*.options' => 'nullable|array',
]);
// Upsert by (import_id, source_column): update existing rows; insert new ones; avoid duplicates
$now = now();
$existing = \DB::table('import_mappings')
->where('import_id', $import->id)
->get(['id', 'source_column', 'position']);
$bySource = [];
$dupes = [];
foreach ($existing as $row) {
$src = (string) $row->source_column;
if (! array_key_exists($src, $bySource)) {
$bySource[$src] = ['id' => $row->id, 'position' => $row->position];
} else {
$dupes[$src] = ($dupes[$src] ?? []);
$dupes[$src][] = $row->id;
}
}
$basePosition = (int) (\DB::table('import_mappings')->where('import_id', $import->id)->max('position') ?? -1);
$inserted = 0;
$updated = 0;
$deduped = 0;
foreach ($data['mappings'] as $pos => $m) {
$src = (string) $m['source_column'];
$payload = [
'entity' => $m['entity'] ?? null,
'target_field' => $m['target_field'],
'transform' => $m['transform'] ?? null,
'apply_mode' => $m['apply_mode'] ?? 'both',
'options' => $m['options'] ?? null,
'position' => $pos, // keep UI order
'updated_at' => $now,
];
if (array_key_exists($src, $bySource)) {
// Update first occurrence
\DB::table('import_mappings')->where('id', $bySource[$src]['id'])->update($payload);
$updated++;
// Remove duplicates if any
if (! empty($dupes[$src])) {
$deleted = \DB::table('import_mappings')->whereIn('id', $dupes[$src])->delete();
$deduped += (int) $deleted;
unset($dupes[$src]);
}
} else {
// Insert new
\DB::table('import_mappings')->insert([
'import_id' => $import->id,
'entity' => $payload['entity'],
'source_column' => $src,
'target_field' => $payload['target_field'],
'transform' => $payload['transform'],
'apply_mode' => $payload['apply_mode'],
'options' => $payload['options'],
'position' => ++$basePosition,
'created_at' => $now,
'updated_at' => $now,
]);
$inserted++;
}
}
// Mark this as ad-hoc mapping usage
$import->update(['import_template_id' => null]);
return response()->json(['ok' => true, 'saved' => ($inserted + $updated), 'inserted' => $inserted, 'updated' => $updated, 'deduped' => $deduped]);
}
// Fetch current mappings for an import (after applying a template or saving ad-hoc mappings)
public function getMappings(Import $import)
{
$rows = \DB::table('import_mappings')
->where('import_id', $import->id)
->orderBy('position')
->orderBy('id')
->get([
'id',
'entity',
'source_column',
'target_field',
'transform',
'apply_mode',
'options',
'position',
]);
return response()->json(['mappings' => $rows]);
}
/**
* List active, non-archived contracts for the import's client that are NOT present
* in the processed import file (based on mapped contract.reference values).
* Only available when contract.reference mapping apply_mode is 'keyref'.
*/
public function missingContracts(Import $import)
{
// Ensure client context is available
if (empty($import->client_id)) {
return response()->json(['error' => 'Import has no client bound.'], 422);
}
// Respect optional feature flag on import
if (! (bool) ($import->show_missing ?? false)) {
return response()->json(['error' => 'Missing contracts listing is disabled for this import.'], 422);
}
// Check that this import's mappings set contract.reference to keyref mode
$mappings = \DB::table('import_mappings')
->where('import_id', $import->id)
->get(['target_field', 'apply_mode']);
$isKeyref = false;
foreach ($mappings as $map) {
$tf = strtolower((string) ($map->target_field ?? ''));
$am = strtolower((string) ($map->apply_mode ?? ''));
if (in_array($tf, ['contract.reference', 'contracts.reference'], true) && $am === 'keyref') {
$isKeyref = true;
break;
}
}
if (! $isKeyref) {
return response()->json(['error' => 'Missing contracts are only available for keyref mapping on contract.reference.'], 422);
}
// Collect referenced contract references from processed rows
$present = [];
foreach (\App\Models\ImportRow::query()->where('import_id', $import->id)->get(['mapped_data']) as $row) {
$md = $row->mapped_data ?? [];
if (is_array($md) && isset($md['contract']['reference'])) {
$ref = (string) $md['contract']['reference'];
if ($ref !== '') {
$present[] = preg_replace('/\s+/', '', trim($ref));
}
}
}
$present = array_values(array_unique(array_filter($present)));
// Query active, non-archived contracts for this client that were not in import
// Include person full_name (owner of the client case) and aggregate active accounts' balance_amount
$contractsQ = \App\Models\Contract::query()
->join('client_cases', 'contracts.client_case_id', '=', 'client_cases.id')
->join('person', 'person.id', '=', 'client_cases.person_id')
->leftJoin('accounts', function ($join) {
$join->on('accounts.contract_id', '=', 'contracts.id')
->where('accounts.active', 1);
})
->where('client_cases.client_id', $import->client_id)
->where('contracts.active', 1)
->whereNull('contracts.deleted_at')
->when(count($present) > 0, function ($q) use ($present) {
$q->whereNotIn('contracts.reference', $present);
})
->groupBy('contracts.uuid', 'contracts.reference', 'client_cases.uuid', 'person.full_name')
->orderBy('contracts.reference')
->get([
'contracts.uuid as uuid',
'contracts.reference as reference',
'client_cases.uuid as case_uuid',
'person.full_name as full_name',
\DB::raw('COALESCE(SUM(accounts.balance_amount), 0) as balance_amount'),
]);
return response()->json([
'missing' => $contractsQ,
'count' => $contractsQ->count(),
]);
}
/**
* Update import options (e.g., booleans like show_missing, reactivate) from the UI.
*/
public function updateOptions(Request $request, Import $import)
{
$data = $request->validate([
'show_missing' => 'nullable|boolean',
'reactivate' => 'nullable|boolean',
]);
$payload = [];
if (array_key_exists('show_missing', $data)) {
$payload['show_missing'] = (bool) $data['show_missing'];
}
if (array_key_exists('reactivate', $data)) {
$payload['reactivate'] = (bool) $data['reactivate'];
}
if (! empty($payload)) {
$import->update($payload);
}
return response()->json([
'ok' => true,
'import' => [
'id' => $import->id,
'uuid' => $import->uuid,
'show_missing' => (bool) ($import->show_missing ?? false),
'reactivate' => (bool) ($import->reactivate ?? false),
],
]);
}
// Fetch recent import events (logs) for an import
public function getEvents(Import $import)
{
$limit = (int) request()->query('limit', 200);
$limit = max(1, min($limit, 1000));
$events = ImportEvent::query()
->where('import_id', $import->id)
->orderByDesc('id')
->limit($limit)
->get(['id', 'created_at', 'level', 'event', 'message', 'import_row_id', 'context']);
return response()->json(['events' => $events]);
}
// Preview (up to N) raw CSV rows for an import for mapping review
public function preview(Import $import, Request $request)
{
$validated = $request->validate([
'limit' => 'nullable|integer|min:1|max:500',
]);
$limit = (int) ($validated['limit'] ?? 200);
// Determine header/delimiter the same way as columns() stored them
$meta = $import->meta ?? [];
$hasHeader = (bool) ($meta['has_header'] ?? true);
// Forced delimiter overrides everything; else detected; fallback comma
$delimiter = $meta['forced_delimiter']
?? $meta['detected_delimiter']
?? ',';
$rows = [];
$columns = [];
$truncated = false;
$path = Storage::disk($import->disk)->path($import->path);
if (! is_readable($path)) {
return response()->json([
'error' => 'File not readable',
], 422);
}
$fh = @fopen($path, 'r');
if (! $fh) {
return response()->json([
'error' => 'Unable to open file',
], 422);
}
try {
if ($hasHeader) {
$header = fgetcsv($fh, 0, $delimiter) ?: [];
$columns = array_map(function ($h) {
return is_string($h) ? trim($h) : (string) $h;
}, $header);
} else {
// Use meta stored columns when available, else infer later from widest row
$columns = is_array($meta['columns'] ?? null) ? $meta['columns'] : [];
}
$count = 0;
$widest = count($columns);
while (($data = fgetcsv($fh, 0, $delimiter)) !== false) {
if ($count >= $limit) {
$truncated = true;
break;
}
// Track widest for non-header scenario
if (! $hasHeader) {
$widest = max($widest, count($data));
}
$rows[] = $data;
$count++;
}
if (! $hasHeader && $widest > count($columns)) {
// Generate positional column labels if missing
$columns = [];
for ($i = 0; $i < $widest; $i++) {
$columns[] = 'col_'.($i + 1);
}
}
} finally {
fclose($fh);
}
// Normalize each row into assoc keyed by columns (pad/truncate as needed)
$assocRows = [];
foreach ($rows as $r) {
$assoc = [];
foreach ($columns as $i => $colName) {
$assoc[$colName] = array_key_exists($i, $r) ? $r[$i] : null;
}
$assocRows[] = $assoc;
}
return response()->json([
'columns' => $columns,
'rows' => $assocRows,
'limit' => $limit,
'truncated' => $truncated,
'has_header' => $hasHeader,
]);
}
/**
* Simulate application of payment rows for a payments import without persisting changes.
* Returns per-row projected balance changes and resolution of contract / account references.
*/
public function simulatePayments(Import $import, Request $request)
{
// Delegate to the generic simulate method for backward compatibility.
return $this->simulate($import, $request);
}
/**
* Generic simulation endpoint: projects what would happen if the import were processed
* using the first N rows and current saved mappings. Works for both payments and non-payments
* templates. For payments templates, payment-specific summaries/entities will be included
* automatically by the simulation service when mappings contain the payment root.
*/
public function simulate(Import $import, Request $request)
{
$validated = $request->validate([
'limit' => 'nullable|integer|min:1|max:500',
'verbose' => 'nullable|boolean',
]);
$limit = (int) ($validated['limit'] ?? 100);
$verbose = (bool) ($validated['verbose'] ?? false);
$service = app(\App\Services\ImportSimulationService::class);
$result = $service->simulate($import, $limit, $verbose);
return response()->json($result);
}
// Show an existing import by UUID to continue where left off
public function show(Import $import)
{
// Load templates (global + client specific) and clients for selection on continue page
$templates = ImportTemplate::query()
->leftJoin('clients', 'clients.id', '=', 'import_templates.client_id')
->where('import_templates.is_active', true)
->where('import_templates.id', $import->import_template_id)
->orderBy('import_templates.name')
->get([
'import_templates.id',
'import_templates.uuid',
'import_templates.name',
'import_templates.source_type',
'import_templates.default_record_type',
'import_templates.client_id',
'import_templates.meta',
'clients.uuid as client_uuid',
]);
$clients = Client::query()
->join('person', 'person.id', '=', 'clients.person_id')
->orderBy('person.full_name')
->where('clients.id', $import->client_id)
->get([
'clients.id',
'clients.uuid',
'person.full_name as name',
]);
// Import client
$client = Client::query()
->join('person', 'person.id', '=', 'clients.person_id')
->where('clients.id', $import->client_id)
->firstOrFail([
'clients.uuid as uuid',
'person.full_name as name',
]);
// Render a dedicated page to continue the import
return Inertia::render('Imports/Import', [
'import' => [
'id' => $import->id,
'uuid' => $import->uuid,
'status' => $import->status,
'meta' => $import->meta,
'client_id' => $import->client_id,
'client_uuid' => optional($client)->uuid,
'import_template_id' => $import->import_template_id,
'show_missing' => (bool) ($import->show_missing ?? false),
'reactivate' => (bool) ($import->reactivate ?? false),
'total_rows' => $import->total_rows,
'imported_rows' => $import->imported_rows,
'invalid_rows' => $import->invalid_rows,
'valid_rows' => $import->valid_rows,
'finished_at' => $import->finished_at,
],
'templates' => $templates,
'clients' => $clients,
'client' => $client,
]);
}
// Delete an import if not finished (statuses allowed: uploaded, mapping, processing_failed etc.)
public function destroy(Request $request, Import $import)
{
// Only allow deletion if not completed or processing
if (in_array($import->status, ['completed', 'processing'])) {
return back()->with([
'ok' => false,
'message' => 'Import can not be deleted in its current status.',
], 422);
}
// Attempt to delete stored file
try {
if ($import->disk && $import->path && Storage::disk($import->disk)->exists($import->path)) {
Storage::disk($import->disk)->delete($import->path);
}
} catch (\Throwable $e) {
// Log event but proceed with deletion
ImportEvent::create([
'import_id' => $import->id,
'user_id' => $request->user()?->getAuthIdentifier(),
'event' => 'file_delete_failed',
'level' => 'warning',
'message' => 'Failed to delete import file: '.$e->getMessage(),
]);
}
// Clean up related events/rows optionally (soft approach: rely on FKs if cascade configured)
// If not cascaded, we could manually delete; check quickly
// Assuming foreign key ON DELETE CASCADE for import_rows & import_events
$import->delete();
return back()->with(['ok' => true]);
}
}