430 lines
17 KiB
PHP
430 lines
17 KiB
PHP
<?php
|
|
|
|
namespace App\Http\Controllers;
|
|
|
|
use App\Models\Client;
|
|
use App\Models\Import;
|
|
use App\Models\ImportEvent;
|
|
use App\Models\ImportTemplate;
|
|
use App\Services\CsvImportService;
|
|
use App\Services\ImportProcessor;
|
|
use Illuminate\Http\Request;
|
|
use Illuminate\Support\Facades\DB;
|
|
use Illuminate\Support\Facades\Storage;
|
|
use Illuminate\Support\Str;
|
|
use Inertia\Inertia;
|
|
|
|
class ImportController extends Controller
|
|
{
|
|
// List imports (paginated)
|
|
public function index(Request $request)
|
|
{
|
|
$paginator = Import::query()
|
|
->with([
|
|
'client:id,uuid,person_id',
|
|
'client.person:id,uuid,full_name',
|
|
'template:id,name',
|
|
])
|
|
->orderByDesc('created_at')
|
|
->paginate(15);
|
|
|
|
$imports = [
|
|
'data' => $paginator->items(),
|
|
'links' => [
|
|
'first' => $paginator->url(1),
|
|
'last' => $paginator->url($paginator->lastPage()),
|
|
'prev' => $paginator->previousPageUrl(),
|
|
'next' => $paginator->nextPageUrl(),
|
|
],
|
|
'meta' => [
|
|
'current_page' => $paginator->currentPage(),
|
|
'from' => $paginator->firstItem(),
|
|
'last_page' => $paginator->lastPage(),
|
|
'path' => $paginator->path(),
|
|
'per_page' => $paginator->perPage(),
|
|
'to' => $paginator->lastItem(),
|
|
'total' => $paginator->total(),
|
|
],
|
|
];
|
|
// Map items into a simpler shape
|
|
$imports['data'] = array_map(function ($imp) {
|
|
return [
|
|
'id' => $imp->id,
|
|
'uuid' => $imp->uuid,
|
|
'created_at' => $imp->created_at,
|
|
'original_name' => $imp->original_name,
|
|
'size' => $imp->size,
|
|
'status' => $imp->status,
|
|
'client' => $imp->client ? [
|
|
'id' => $imp->client_id,
|
|
'uuid' => $imp->client->uuid,
|
|
'person' => $imp->client->person ? [
|
|
'uuid' => $imp->client->person->uuid,
|
|
'full_name' => $imp->client->person->full_name,
|
|
] : null,
|
|
] : null,
|
|
'template' => $imp->template ? ['id' => $imp->template->id, 'name' => $imp->template->name] : null,
|
|
];
|
|
}, $imports['data']);
|
|
|
|
return Inertia::render('Imports/Index', [
|
|
'imports' => $imports,
|
|
]);
|
|
}
|
|
|
|
// Show the import creation page
|
|
public function create(Request $request)
|
|
{
|
|
$templates = ImportTemplate::query()
|
|
->leftJoin('clients', 'clients.id', '=', 'import_templates.client_id')
|
|
->where('import_templates.is_active', true)
|
|
->orderBy('import_templates.name')
|
|
->get([
|
|
'import_templates.id',
|
|
'import_templates.uuid',
|
|
'import_templates.name',
|
|
'import_templates.source_type',
|
|
'import_templates.default_record_type',
|
|
'import_templates.client_id',
|
|
DB::raw('clients.uuid as client_uuid'),
|
|
]);
|
|
|
|
$clients = Client::query()
|
|
->join('person', 'person.id', '=', 'clients.person_id')
|
|
->orderBy('person.full_name')
|
|
->get([
|
|
'clients.id',
|
|
'clients.uuid',
|
|
DB::raw('person.full_name as name'),
|
|
]);
|
|
|
|
return Inertia::render('Imports/Create', [
|
|
'templates' => $templates,
|
|
'clients' => $clients,
|
|
// no existing import on create
|
|
]);
|
|
}
|
|
|
|
// Create a new import job, store file, and return basic info
|
|
public function store(Request $request)
|
|
{
|
|
$validated = $request->validate([
|
|
'file' => 'required|file|max:20480', // 20MB; adjust as needed
|
|
'source_type' => 'nullable|string|in:csv,xml,xls,xlsx,json,txt',
|
|
'sheet_name' => 'nullable|string|max:64',
|
|
'has_header' => 'nullable|boolean',
|
|
'import_template_id' => 'nullable|integer|exists:import_templates,id',
|
|
'client_uuid' => 'nullable|string|exists:clients,uuid',
|
|
]);
|
|
|
|
$file = $validated['file'];
|
|
$ext = strtolower($file->getClientOriginalExtension());
|
|
$sourceType = $validated['source_type'] ?? ($ext === 'txt' ? 'csv' : $ext);
|
|
|
|
$uuid = (string) Str::uuid();
|
|
$disk = 'local';
|
|
$path = $file->storeAs('imports', $uuid.'.'.$ext, $disk);
|
|
|
|
// Resolve client_uuid to client_id if provided
|
|
$clientId = null;
|
|
if (! empty($validated['client_uuid'] ?? null)) {
|
|
$clientId = Client::where('uuid', $validated['client_uuid'])->value('id');
|
|
}
|
|
|
|
$import = Import::create([
|
|
'uuid' => $uuid,
|
|
'user_id' => $request->user()?->id,
|
|
'import_template_id' => $validated['import_template_id'] ?? null,
|
|
'client_id' => $clientId,
|
|
'source_type' => $sourceType,
|
|
'file_name' => basename($path),
|
|
'original_name' => $file->getClientOriginalName(),
|
|
'disk' => $disk,
|
|
'path' => $path,
|
|
'size' => $file->getSize(),
|
|
'sheet_name' => $validated['sheet_name'] ?? null,
|
|
'status' => 'uploaded',
|
|
'meta' => [
|
|
'has_header' => $validated['has_header'] ?? true,
|
|
],
|
|
]);
|
|
|
|
return response()->json([
|
|
'id' => $import->id,
|
|
'uuid' => $import->uuid,
|
|
'status' => $import->status,
|
|
]);
|
|
}
|
|
|
|
// Kick off processing of an import - simple synchronous step for now
|
|
public function process(Import $import, Request $request, ImportProcessor $processor)
|
|
{
|
|
$import->update(['status' => 'validating', 'started_at' => now()]);
|
|
$result = $processor->process($import, user: $request->user());
|
|
|
|
return response()->json($result);
|
|
}
|
|
|
|
// Analyze the uploaded file and return column headers or positional indices
|
|
public function columns(Request $request, Import $import, CsvImportService $csv)
|
|
{
|
|
$validated = $request->validate([
|
|
'has_header' => 'nullable|boolean',
|
|
'delimiter' => 'nullable|string|max:4',
|
|
]);
|
|
|
|
$hasHeader = array_key_exists('has_header', $validated)
|
|
? (bool) $validated['has_header']
|
|
: (bool) ($import->meta['has_header'] ?? true);
|
|
|
|
// Resolve delimiter preference: explicit param > template meta > existing meta > auto-detect
|
|
$explicitDelimiter = null;
|
|
if (array_key_exists('delimiter', $validated) && $validated['delimiter'] !== null && $validated['delimiter'] !== '') {
|
|
$explicitDelimiter = (string) $validated['delimiter'];
|
|
} elseif ($import->import_template_id) {
|
|
// Try reading template meta for a default delimiter
|
|
$tplDelimiter = optional(ImportTemplate::find($import->import_template_id))->meta['delimiter'] ?? null;
|
|
if ($tplDelimiter) {
|
|
$explicitDelimiter = (string) $tplDelimiter;
|
|
}
|
|
} elseif (! empty($import->meta['forced_delimiter'] ?? null)) {
|
|
$explicitDelimiter = (string) $import->meta['forced_delimiter'];
|
|
}
|
|
|
|
// Prefer CSV/TXT; if source_type is unknown, attempt best-effort based on file extension
|
|
$treatAsText = in_array($import->source_type, ['csv', 'txt']);
|
|
if (! $treatAsText) {
|
|
$orig = strtolower(pathinfo($import->original_name ?? '', PATHINFO_EXTENSION));
|
|
if (in_array($orig, ['csv', 'txt'])) {
|
|
$treatAsText = true;
|
|
}
|
|
}
|
|
|
|
$fullPath = Storage::disk($import->disk)->path($import->path);
|
|
$note = '';
|
|
if ($treatAsText) {
|
|
if ($explicitDelimiter !== null && $explicitDelimiter !== '') {
|
|
$columns = $csv->parseColumnsFromCsv($fullPath, $explicitDelimiter, $hasHeader);
|
|
$delimiter = $explicitDelimiter;
|
|
} else {
|
|
[$delimiter, $columns] = $csv->detectColumnsFromCsv($fullPath, $hasHeader);
|
|
// Backstop: if single column but file clearly has separators, try common ones
|
|
if (is_array($columns) && count($columns) <= 1) {
|
|
foreach ([';', "\t", '|', ' ', ','] as $try) {
|
|
$alt = $csv->parseColumnsFromCsv($fullPath, $try, $hasHeader);
|
|
if (is_array($alt) && count($alt) > 1) {
|
|
$delimiter = $try;
|
|
$columns = $alt;
|
|
$note = 'Delimiter auto-detection backstopped to '.json_encode($try);
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
} else {
|
|
// Best-effort: try detect anyway
|
|
[$delimiter, $columns] = $csv->detectColumnsFromCsv($fullPath, $hasHeader);
|
|
}
|
|
|
|
// Save meta
|
|
$meta = $import->meta ?? [];
|
|
$meta['has_header'] = $hasHeader;
|
|
$meta['detected_delimiter'] = $delimiter;
|
|
$meta['columns'] = $columns;
|
|
if ($explicitDelimiter) {
|
|
$meta['forced_delimiter'] = $explicitDelimiter;
|
|
}
|
|
$import->update([
|
|
'meta' => $meta,
|
|
'status' => $import->status === 'uploaded' ? 'parsed' : $import->status,
|
|
]);
|
|
|
|
return response()->json([
|
|
'columns' => $columns,
|
|
'has_header' => $hasHeader,
|
|
'detected_delimiter' => $delimiter,
|
|
'note' => $note,
|
|
]);
|
|
}
|
|
|
|
// CSV helpers moved to App\Services\CsvImportService
|
|
|
|
// Save ad-hoc mappings for a specific import (when no template is selected)
|
|
public function saveMappings(Request $request, Import $import)
|
|
{
|
|
$data = $request->validate([
|
|
'mappings' => 'required|array',
|
|
'mappings.*.source_column' => 'required|string',
|
|
'mappings.*.entity' => 'nullable|string|in:person,person_addresses,person_phones,emails,accounts,contracts,client_cases',
|
|
'mappings.*.target_field' => 'required|string',
|
|
'mappings.*.transform' => 'nullable|string|in:trim,upper,lower,decimal,ref',
|
|
'mappings.*.apply_mode' => 'nullable|string|in:insert,update,both',
|
|
'mappings.*.options' => 'nullable|array',
|
|
]);
|
|
|
|
// Upsert by (import_id, source_column): update existing rows; insert new ones; avoid duplicates
|
|
$now = now();
|
|
$existing = \DB::table('import_mappings')
|
|
->where('import_id', $import->id)
|
|
->get(['id', 'source_column', 'position']);
|
|
|
|
$bySource = [];
|
|
$dupes = [];
|
|
foreach ($existing as $row) {
|
|
$src = (string) $row->source_column;
|
|
if (! array_key_exists($src, $bySource)) {
|
|
$bySource[$src] = ['id' => $row->id, 'position' => $row->position];
|
|
} else {
|
|
$dupes[$src] = ($dupes[$src] ?? []);
|
|
$dupes[$src][] = $row->id;
|
|
}
|
|
}
|
|
|
|
$basePosition = (int) (\DB::table('import_mappings')->where('import_id', $import->id)->max('position') ?? -1);
|
|
$inserted = 0;
|
|
$updated = 0;
|
|
$deduped = 0;
|
|
|
|
foreach ($data['mappings'] as $pos => $m) {
|
|
$src = (string) $m['source_column'];
|
|
$payload = [
|
|
'entity' => $m['entity'] ?? null,
|
|
'target_field' => $m['target_field'],
|
|
'transform' => $m['transform'] ?? null,
|
|
'apply_mode' => $m['apply_mode'] ?? 'both',
|
|
'options' => $m['options'] ?? null,
|
|
'position' => $pos, // keep UI order
|
|
'updated_at' => $now,
|
|
];
|
|
|
|
if (array_key_exists($src, $bySource)) {
|
|
// Update first occurrence
|
|
\DB::table('import_mappings')->where('id', $bySource[$src]['id'])->update($payload);
|
|
$updated++;
|
|
// Remove duplicates if any
|
|
if (! empty($dupes[$src])) {
|
|
$deleted = \DB::table('import_mappings')->whereIn('id', $dupes[$src])->delete();
|
|
$deduped += (int) $deleted;
|
|
unset($dupes[$src]);
|
|
}
|
|
} else {
|
|
// Insert new
|
|
\DB::table('import_mappings')->insert([
|
|
'import_id' => $import->id,
|
|
'entity' => $payload['entity'],
|
|
'source_column' => $src,
|
|
'target_field' => $payload['target_field'],
|
|
'transform' => $payload['transform'],
|
|
'apply_mode' => $payload['apply_mode'],
|
|
'options' => $payload['options'],
|
|
'position' => ++$basePosition,
|
|
'created_at' => $now,
|
|
'updated_at' => $now,
|
|
]);
|
|
$inserted++;
|
|
}
|
|
}
|
|
|
|
// Mark this as ad-hoc mapping usage
|
|
$import->update(['import_template_id' => null]);
|
|
|
|
return response()->json(['ok' => true, 'saved' => ($inserted + $updated), 'inserted' => $inserted, 'updated' => $updated, 'deduped' => $deduped]);
|
|
}
|
|
|
|
// Fetch current mappings for an import (after applying a template or saving ad-hoc mappings)
|
|
public function getMappings(Import $import)
|
|
{
|
|
$rows = \DB::table('import_mappings')
|
|
->where('import_id', $import->id)
|
|
->orderBy('position')
|
|
->orderBy('id')
|
|
->get([
|
|
'id',
|
|
'entity',
|
|
'source_column',
|
|
'target_field',
|
|
'transform',
|
|
'apply_mode',
|
|
'options',
|
|
'position',
|
|
]);
|
|
|
|
return response()->json(['mappings' => $rows]);
|
|
}
|
|
|
|
// Fetch recent import events (logs) for an import
|
|
public function getEvents(Import $import)
|
|
{
|
|
$limit = (int) request()->query('limit', 200);
|
|
$limit = max(1, min($limit, 1000));
|
|
$events = ImportEvent::query()
|
|
->where('import_id', $import->id)
|
|
->orderByDesc('id')
|
|
->limit($limit)
|
|
->get(['id', 'created_at', 'level', 'event', 'message', 'import_row_id', 'context']);
|
|
|
|
return response()->json(['events' => $events]);
|
|
}
|
|
|
|
// Show an existing import by UUID to continue where left off
|
|
public function show(Import $import)
|
|
{
|
|
// Load templates (global + client specific) and clients for selection on continue page
|
|
$templates = ImportTemplate::query()
|
|
->leftJoin('clients', 'clients.id', '=', 'import_templates.client_id')
|
|
->where('import_templates.is_active', true)
|
|
->where('import_templates.id', $import->import_template_id)
|
|
->orderBy('import_templates.name')
|
|
->get([
|
|
'import_templates.id',
|
|
'import_templates.uuid',
|
|
'import_templates.name',
|
|
'import_templates.source_type',
|
|
'import_templates.default_record_type',
|
|
'import_templates.client_id',
|
|
'import_templates.meta',
|
|
'clients.uuid as client_uuid',
|
|
]);
|
|
|
|
$clients = Client::query()
|
|
->join('person', 'person.id', '=', 'clients.person_id')
|
|
->orderBy('person.full_name')
|
|
->where('clients.id', $import->client_id)
|
|
->get([
|
|
'clients.id',
|
|
'clients.uuid',
|
|
'person.full_name as name',
|
|
]);
|
|
|
|
// Import client
|
|
$client = Client::query()
|
|
->join('person', 'person.id', '=', 'clients.person_id')
|
|
->where('clients.id', $import->client_id)
|
|
->firstOrFail([
|
|
'clients.uuid as uuid',
|
|
'person.full_name as name',
|
|
]);
|
|
|
|
// Render a dedicated page to continue the import
|
|
return Inertia::render('Imports/Import', [
|
|
'import' => [
|
|
'id' => $import->id,
|
|
'uuid' => $import->uuid,
|
|
'status' => $import->status,
|
|
'meta' => $import->meta,
|
|
'client_id' => $import->client_id,
|
|
'client_uuid' => optional($client)->uuid,
|
|
'import_template_id' => $import->import_template_id,
|
|
'total_rows' => $import->total_rows,
|
|
'imported_rows' => $import->imported_rows,
|
|
'invalid_rows' => $import->invalid_rows,
|
|
'valid_rows' => $import->valid_rows,
|
|
'finished_at' => $import->finished_at,
|
|
],
|
|
'templates' => $templates,
|
|
'clients' => $clients,
|
|
'client' => $client,
|
|
]);
|
|
}
|
|
}
|