sipo/Teren-app
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/master' into Development

Browse Source
This commit is contained in:
Simon Pocrnjič
2025-12-21 21:22:36 +01:00
parent 85922bdac0 adc2a64687
commit ee641586c3
10 changed files with 481 additions and 338 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/Services/Archiving/ArchiveExecutor.php
+3 -1
View File
@@ -69,6 +69,8 @@ public function executeSetting(ArchiveSetting $setting, ?array $context = null,
$entities = $flat;
}
// dd($entities);
foreach ($entities as $entityDef) {
$rawTable = $entityDef['table'] ?? null;
if (! $rawTable) {
@@ -97,7 +99,7 @@ public function executeSetting(ArchiveSetting $setting, ?array $context = null,
// Process in batches to avoid locking large tables
while (true) {
$query = DB::table($table)->whereNull('deleted_at');
if (Schema::hasColumn($table, 'active')) {
if (Schema::hasColumn($table, 'active') && ! $reactivate) {
$query->where('active', 1);
}
// Apply context filters or chain derived filters
app/Services/ImportProcessor.php
+153 -21
View File
@@ -25,14 +25,17 @@
use App\Models\Person\PersonType;
use App\Models\Person\PhoneType;
use Illuminate\Contracts\Auth\Authenticatable;
use Illuminate\Database\QueryException;
use Illuminate\Support\Carbon;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Log;
use Illuminate\Support\Facades\Storage;
class ImportProcessor
{
/**
* Track contracts that already existed and were matched during history imports.
*
* @var array<int,bool>
*/
private array $historyFoundContractIds = [];
@@ -180,10 +183,7 @@ public function process(Import $import, ?Authenticatable $user = null): array
}
// Preflight: warn if any mapped source columns are not present in the header (exact match)
$headerSet = [];
foreach ($header as $h) {
$headerSet[$h] = true;
}
$missingSources = [];
// Regex validation removed per request; rely on basic length/placeholder checks only
foreach ($mappings as $map) {
$src = (string) ($map->source_column ?? '');
if ($src !== '' && ! array_key_exists($src, $headerSet)) {
@@ -216,7 +216,28 @@ public function process(Import $import, ?Authenticatable $user = null): array
if ($isPg) {
// Establish a savepoint so a failing row does not poison the whole transaction
DB::statement('SAVEPOINT import_row_'.$rowNum);
try {
DB::statement('SAVEPOINT import_row_'.$rowNum);
} catch (\Throwable $se) {
Log::error('Import savepoint_failed', [
'import_id' => $import->id,
'row_number' => $rowNum,
'exception' => $this->exceptionContext($se),
]);
ImportEvent::create([
'import_id' => $import->id,
'user_id' => $user?->getAuthIdentifier(),
'event' => 'savepoint_failed',
'level' => 'error',
'message' => 'Failed to create savepoint; transaction already aborted.',
'context' => [
'row_number' => $rowNum,
'exception' => $this->exceptionContext($se),
],
]);
throw $se; // abort import so root cause surfaces
}
}
// Scope variables per row so they aren't reused after exception
@@ -1067,13 +1088,38 @@ public function process(Import $import, ?Authenticatable $user = null): array
}
}
} catch (\Throwable $e) {
$rollbackFailed = false;
$rollbackError = null;
if ($isPg) {
// Roll back only this row's work
try {
DB::statement('ROLLBACK TO SAVEPOINT import_row_'.$rowNum);
} catch (\Throwable $ignored) { /* noop */
} catch (\Throwable $ignored) {
$rollbackFailed = true;
$rollbackError = $ignored;
}
}
if ($rollbackFailed) {
Log::error('Import row_rollback_failed', [
'import_id' => $import->id,
'row_number' => $rowNum,
'exception' => $this->exceptionContext($rollbackError ?? $e),
]);
// Abort the whole import if we cannot rollback to the row savepoint (transaction is poisoned)
ImportEvent::create([
'import_id' => $import->id,
'user_id' => $user?->getAuthIdentifier(),
'event' => 'row_rollback_failed',
'level' => 'error',
'message' => 'Rollback to savepoint failed; aborting import.',
'context' => [
'row_number' => $rowNum,
'exception' => $this->exceptionContext($rollbackError ?? $e),
],
]);
throw $rollbackError ?? $e;
}
// Ensure importRow exists for logging if failure happened before its creation
if (! $importRow) {
try {
@@ -1100,6 +1146,12 @@ public function process(Import $import, ?Authenticatable $user = null): array
}
$failedRows[] = $rowNum;
$invalid++;
Log::error('Import row_exception', [
'import_id' => $import->id,
'row_number' => $rowNum,
'exception' => $this->exceptionContext($e),
'raw_preview' => isset($rawAssoc) ? $this->buildRawDataPreview($rawAssoc) : [],
]);
try {
ImportEvent::create([
'import_id' => $import->id,
@@ -1117,6 +1169,12 @@ public function process(Import $import, ?Authenticatable $user = null): array
],
]);
} catch (\Throwable $evtErr) {
Log::error('Import row_exception_event_failed', [
'import_id' => $import->id,
'row_number' => $rowNum,
'exception' => $this->exceptionContext($evtErr),
'original_exception' => $this->exceptionContext($e),
]);
// Swallow secondary failure to ensure loop continues
}
@@ -1189,12 +1247,17 @@ public function process(Import $import, ?Authenticatable $user = null): array
// Mark failed and log after rollback (so no partial writes persist)
$import->refresh();
$import->update(['status' => 'failed', 'failed_at' => now()]);
Log::error('Import processing_failed', [
'import_id' => $import->id,
'exception' => $this->exceptionContext($e),
]);
ImportEvent::create([
'import_id' => $import->id,
'user_id' => $user?->getAuthIdentifier(),
'event' => 'processing_failed',
'level' => 'error',
'message' => $e->getMessage(),
'message' => $this->safeErrorMessage($e->getMessage()),
'context' => $this->exceptionContext($e),
]);
return ['ok' => false, 'status' => 'failed', 'error' => $e->getMessage()];
@@ -1920,6 +1983,8 @@ private function upsertActivity(Import $import, array $mapped, $mappings, ?array
} elseif (in_array($field, ['action_id', 'decision_id', 'user_id'], true)) {
$normalized = is_null($value) ? null : (int) $value;
} elseif (is_string($normalized)) {
// Clean invalid UTF-8 sequences from string fields
$normalized = mb_convert_encoding($normalized, 'UTF-8', 'UTF-8');
$normalized = trim($normalized);
}
if (in_array($applyMode, ['both', 'insert'], true)) {
@@ -2009,7 +2074,7 @@ private function upsertActivity(Import $import, array $mapped, $mappings, ?array
}
$data = array_filter($applyInsert, fn ($v) => ! is_null($v));
$activityModel = new Activity();
$activityModel = new Activity;
$activityModel->forceFill($data);
if (array_key_exists('created_at', $data)) {
// Preserve provided timestamps by disabling automatic timestamps for this save
@@ -2233,6 +2298,7 @@ private function upsertContractChain(Import $import, array $mapped, $mappings, b
if ($existing) {
if ($historyImport) {
$this->historyFoundContractIds[$existing->id] = true;
return ['action' => 'skipped_history', 'contract' => $existing, 'message' => 'Existing contract left unchanged (history import)'];
}
// 1) Prepare contract field changes (non-null)
@@ -2475,13 +2541,57 @@ private function safeErrorMessage(string $msg): string
}
// Fallback strip invalid bytes
$msg = @iconv('UTF-8', 'UTF-8//IGNORE', $msg) ?: $msg;
if (strlen($msg) > 500) {
$msg = substr($msg, 0, 497).'...';
// Use mb_strlen and mb_substr for UTF-8 safety
if (mb_strlen($msg) > 500) {
$msg = mb_substr($msg, 0, 497).'...';
}
return $msg;
}
/**
* Extract structured exception details for logging.
*/
private function exceptionContext(\Throwable $e): array
{
$ctx = [
'exception' => get_class($e),
'message' => $this->safeErrorMessage($e->getMessage()),
'code' => $e->getCode(),
'file' => $e->getFile().':'.$e->getLine(),
];
if (method_exists($e, 'getPrevious') && $e->getPrevious()) {
$prev = $e->getPrevious();
$ctx['previous'] = [
'exception' => get_class($prev),
'message' => $this->safeErrorMessage($prev->getMessage()),
'code' => $prev->getCode(),
'file' => $prev->getFile().':'.$prev->getLine(),
];
}
if ($e instanceof QueryException) {
$ctx['sql'] = $e->getSql();
$ctx['bindings'] = $e->getBindings();
$info = $e->errorInfo ?? null;
if (is_array($info)) {
$ctx['sqlstate'] = $info[0] ?? null;
$ctx['driver_error_code'] = $info[1] ?? null;
$ctx['driver_error_message'] = $info[2] ?? null;
}
} elseif (property_exists($e, 'errorInfo')) {
$info = $e->errorInfo;
if (is_array($info)) {
$ctx['sqlstate'] = $info[0] ?? null;
$ctx['driver_error_code'] = $info[1] ?? null;
$ctx['driver_error_message'] = $info[2] ?? null;
}
}
return $ctx;
}
/**
* Build a trimmed raw data preview (first 8 columns, truncated values) for logging.
*/
@@ -2522,9 +2632,9 @@ private function formatAppliedFieldMessage(string $root, array $fields): string
} else {
$disp = method_exists($v, '__toString') ? (string) $v : gettype($v);
}
// Truncate very long values for log safety
if (strlen($disp) > 60) {
$disp = substr($disp, 0, 57).'...';
// Truncate very long values for log safety (use mb_substr for UTF-8 safety)
if (mb_strlen($disp) > 60) {
$disp = mb_substr($disp, 0, 57).'...';
}
$parts[] = $k.'='.$disp;
}
@@ -3000,8 +3110,10 @@ private function upsertEmail(int $personId, array $emailData, $mappings): array
private function upsertAddress(int $personId, array $addrData, $mappings): array
{
$addressLine = trim((string) ($addrData['address'] ?? ''));
// Normalize whitespace
// Normalize whitespace: collapse multiples and tighten around separators
$addressLine = preg_replace('/\s+/', ' ', $addressLine);
$addressLine = preg_replace('/\s*([,;\/])\s*/', '$1 ', $addressLine);
$addressLine = trim($addressLine);
// Skip common placeholders or missing values
if ($addressLine === '' || $addressLine === '0' || strcasecmp($addressLine, '#N/A') === 0 || preg_match('/^(#?n\/?a|na|null|none)$/i', $addressLine)) {
return ['action' => 'skipped', 'message' => 'No address value'];
@@ -3009,15 +3121,21 @@ private function upsertAddress(int $personId, array $addrData, $mappings): array
if (mb_strlen($addressLine) < 3) {
return ['action' => 'skipped', 'message' => 'Invalid address value'];
}
// Allow only basic address characters to avoid noisy special chars
if (! preg_match('/^[A-Za-z0-9\\s\\.,\\-\\/\\#\\\'"\\(\\)&]+$/', $addressLine)) {
return ['action' => 'skipped', 'message' => 'Invalid address value'];
}
// If identical address already exists anywhere, skip to avoid constraint violation
/*$existingAny = PersonAddress::where('address', $addressLine)->first();
if ($existingAny) {
return ['action' => 'skipped', 'message' => 'Address already exists in database'];
}*/
// Default country SLO if not provided
if (! isset($addrData['country']) || $addrData['country'] === null || $addrData['country'] === '') {
$addrData['country'] = 'SLO';
}
$existing = PersonAddress::where('person_id', $personId)->where('address', $addressLine)->first();
// Compare addresses with all spaces removed to handle whitespace variations
$addressLineNoSpaces = preg_replace('/\s+/', '', $addressLine);
$existing = PersonAddress::where('person_id', $personId)
->whereRaw("REPLACE(address, ' ', '') = ?", [$addressLineNoSpaces])
->first();
$applyInsert = [];
$applyUpdate = [];
foreach ($mappings as $map) {
@@ -3060,9 +3178,23 @@ private function upsertAddress(int $personId, array $addrData, $mappings): array
$data['person_id'] = $personId;
$data['country'] = $data['country'] ?? 'SLO';
$data['type_id'] = $data['type_id'] ?? $this->getDefaultAddressTypeId();
$created = PersonAddress::create($data);
try {
$created = PersonAddress::create($data);
return ['action' => 'inserted', 'address' => $created];
return ['action' => 'inserted', 'address' => $created];
} catch (QueryException $e) {
// If unique constraint violation, skip instead of aborting
Log::warning('Address constraint violation during import', [
'person_id' => $personId,
'address' => $addressLine,
'error' => $e->getMessage(),
]);
if ($e->getCode() === '23505' || str_contains($e->getMessage(), 'unique') || str_contains($e->getMessage(), 'duplicate')) {
return ['action' => 'skipped', 'message' => 'Address already exists (constraint violation)'];
}
throw $e;
}
}
}