442 lines
14 KiB
PHP
442 lines
14 KiB
PHP
<?php
|
|
|
|
namespace App\Jobs;
|
|
|
|
use App\Enums\LogType;
|
|
use App\Exceptions\AbortXmlImportException;
|
|
use App\Exceptions\InvalidXmlException;
|
|
use ErrorException;
|
|
use Illuminate\Contracts\Filesystem\FileNotFoundException;
|
|
use Illuminate\Database\QueryException;
|
|
use Illuminate\Support\Facades\Log;
|
|
use Illuminate\Support\Facades\Mail;
|
|
use Illuminate\Support\Str;
|
|
use RecursiveArrayIterator;
|
|
use RecursiveIteratorIterator;
|
|
use SimpleXMLElement;
|
|
use Throwable;
|
|
use TypeError;
|
|
|
|
abstract class XmlJobs extends XArcadiaFlux
|
|
{
|
|
|
|
/**
|
|
* invalid xml folder
|
|
*
|
|
* @var string
|
|
*/
|
|
protected $invalidFolderName = 'invalid';
|
|
|
|
/**
|
|
* The columns to take from the XML
|
|
* subcolumns can be added with a dot
|
|
*
|
|
* @var array
|
|
*/
|
|
protected $takeColumns = [];
|
|
|
|
/**
|
|
* The columns allowed to be null
|
|
*
|
|
* @var array
|
|
*/
|
|
protected $nullableFields = [];
|
|
|
|
/**
|
|
* Need to encapsulate the freeking XML
|
|
* Because provided one is dirty
|
|
*
|
|
* @var boolean
|
|
*/
|
|
protected $encapsulate = false;
|
|
|
|
/**
|
|
* Parse Data from the XML and gives it to the callback
|
|
*
|
|
* @param string $entityName
|
|
* @param string $modelClass
|
|
* @return void
|
|
*/
|
|
// phpcs:ignore Generic.Metrics.CyclomaticComplexity.TooHigh
|
|
protected function parseData(string $entityName, string $modelClass, callable $callback = null): void
|
|
{
|
|
/** @var Filesystem */
|
|
$disk = $this->getDisk();
|
|
$this->log('Démarrage du traitement');
|
|
$files = $disk->files();
|
|
|
|
$progress = 1;
|
|
$total = count(array_filter($files, function ($filename) {
|
|
if ($this->fileHasToBeIgnored($filename)) {
|
|
return false;
|
|
}
|
|
return true;
|
|
}));
|
|
|
|
foreach ($files as $filename) {
|
|
if ($this->fileHasToBeIgnored($filename)) {
|
|
continue;
|
|
}
|
|
|
|
$this->log(sprintf(' traitement %d sur %s ', $progress++, $total));
|
|
|
|
try {
|
|
/** @var string $data */
|
|
$data = $disk->get($filename);
|
|
try {
|
|
if ($this->encapsulate) {
|
|
$data = sprintf('<Root>%s</Root>', $data);
|
|
/** @var SimpleXMLElement|false */
|
|
$xmlContent = simplexml_load_string($data);
|
|
} else {
|
|
/** @var SimpleXMLElement|false */
|
|
$xmlContent = simplexml_load_string($data, null, LIBXML_NOCDATA | LIBXML_PARSEHUGE);
|
|
}
|
|
if ($xmlContent === false) {
|
|
throw new InvalidXmlException('XML is Invalid (simplexml_load_string returned false)');
|
|
}
|
|
} catch (ErrorException $e) {
|
|
throw new InvalidXmlException('XML is Invalid', 0, $e);
|
|
}
|
|
|
|
$data = $this->extract($xmlContent);
|
|
|
|
$lines = [];
|
|
foreach ($data[$entityName] as $traca) {
|
|
$line = $this->processFields($traca);
|
|
if ($callback) {
|
|
$callback($xmlContent, $line);
|
|
}
|
|
$lines[] = $line;
|
|
}
|
|
|
|
// Mise en base des données
|
|
foreach ($lines as $line) {
|
|
/** @var array Unique Key columns */
|
|
$uniqueColumnsKey = [];
|
|
foreach ($this->uniqueKeyField as $uField) {
|
|
$uniqueColumnsKey[$uField] = $line[$uField];
|
|
}
|
|
forward_static_call_array(
|
|
[$modelClass, 'updateOrCreate'],
|
|
[$uniqueColumnsKey, $line]
|
|
);
|
|
}
|
|
} catch (InvalidXmlException $e) {
|
|
$message = $e->getMessage();
|
|
if (get_class($e) === QueryException::class) {
|
|
$message = implode(' ', $e->errorInfo);
|
|
}
|
|
$this->log(
|
|
sprintf(
|
|
'File %s could not be parsed => %s on line %d',
|
|
$filename,
|
|
$message,
|
|
$e->getLine()
|
|
),
|
|
LogType::critical()
|
|
);
|
|
$this->sendInvalidXmlEmail($filename, $e);
|
|
$this->moveToInvalid($filename);
|
|
continue;
|
|
} catch (ErrorException | TypeError $e) {
|
|
$this->log(
|
|
sprintf(
|
|
'CRITICAL: File %s could not be parsed => %s on line %d',
|
|
$filename,
|
|
$e->getMessage(),
|
|
$e->getLine()
|
|
),
|
|
LogType::critical()
|
|
);
|
|
$this->sendErrorParseEmailToServiceWeb($e);
|
|
continue; // ignore the file
|
|
} catch (AbortXmlImportException $e) {
|
|
$this->log(
|
|
sprintf(
|
|
'ERROR: File %s could not be parsed => %s on line %d',
|
|
$filename,
|
|
$e->getMessage(),
|
|
$e->getLine()
|
|
),
|
|
LogType::error()
|
|
);
|
|
$this->sendErrorParseEmailToServiceWeb($e);
|
|
continue; // ignore the file
|
|
} catch (FileNotFoundException | QueryException $e) {
|
|
// This happens because of concurent xml jobs read
|
|
$message = $e->getMessage();
|
|
if (get_class($e) === QueryException::class) {
|
|
$message = implode(' ', $e->errorInfo);
|
|
}
|
|
$this->log(
|
|
sprintf(
|
|
'ERROR: File %s could not be read (not found) => %s on line %d',
|
|
$filename,
|
|
$message,
|
|
$e->getLine()
|
|
),
|
|
LogType::debug()
|
|
);
|
|
continue; // ignore the file
|
|
}
|
|
$this->moveToArchives($filename);
|
|
}
|
|
|
|
// Clear old archives
|
|
$this->clearOldArchives();
|
|
}
|
|
|
|
/**
|
|
* Recursive extract to array
|
|
*
|
|
* @param \SimpleXMLElement|false $sxe
|
|
* @return array
|
|
*/
|
|
protected function extract($sxe = null): array
|
|
{
|
|
if (!$sxe instanceof SimpleXMLElement) {
|
|
return array();
|
|
}
|
|
$extract = array();
|
|
foreach ($sxe->children() as $key => $value) {
|
|
if ($value->count()) {
|
|
$extract[$key][] = $this->extract($value);
|
|
continue;
|
|
}
|
|
$value = strval($value);
|
|
if (array_key_exists($key, $extract) and is_array($extract[$key])) {
|
|
$extract[$key][] = strlen($value) ? $value : null;
|
|
} else {
|
|
$extract[$key] = strlen($value) ? $value : null;
|
|
}
|
|
}
|
|
return $extract;
|
|
}
|
|
|
|
/**
|
|
* Process fields to gather data
|
|
*
|
|
* @param array $fields
|
|
* @return array
|
|
* @throws AbortXmlImportException
|
|
*/
|
|
protected function processFields(array $fields): array
|
|
{
|
|
$result = [];
|
|
foreach ($fields as $key => $field) {
|
|
if (!($key = $this->shouldFieldBeConsidered($key))) {
|
|
continue;
|
|
}
|
|
if (!$this->isFieldNullable($key) && is_null($field)) {
|
|
throw new AbortXmlImportException(sprintf('The field %s is not allowed to be null', $key));
|
|
}
|
|
if (is_array($field)) {
|
|
// Flatten data
|
|
$it = new RecursiveIteratorIterator(new RecursiveArrayIterator($field));
|
|
foreach ($it as $k => $f) {
|
|
$ko = strtolower(sprintf('%s.%s', $key, $k));
|
|
if (!$this->isFieldNullable($ko) && is_null($f)) {
|
|
throw new AbortXmlImportException(sprintf(
|
|
'%s : The field %s is not allowed to be null',
|
|
get_class($this),
|
|
$ko
|
|
));
|
|
}
|
|
if (!($k = $this->shouldFieldBeConsidered($ko))) {
|
|
continue;
|
|
}
|
|
$result[$k] = $f;
|
|
}
|
|
} else {
|
|
$result[$key] = $field;
|
|
}
|
|
}
|
|
return $result;
|
|
}
|
|
|
|
/**
|
|
* Is the filed nullable
|
|
*
|
|
* @param string $key
|
|
* @return boolean
|
|
*/
|
|
private function isFieldNullable(string $key): bool
|
|
{
|
|
if (in_array($key, $this->nullableFields)) {
|
|
return true;
|
|
}
|
|
return false;
|
|
}
|
|
|
|
/**
|
|
* - replace key name with the corresponding alias
|
|
* - strtolower the key
|
|
* - snake_case the key
|
|
*
|
|
* @param string $key
|
|
* @return string|false
|
|
*/
|
|
private function shouldFieldBeConsidered(string $key)
|
|
{
|
|
$key = strtolower($key);
|
|
foreach ($this->takeColumns as $k => $alias) {
|
|
if (strtolower($k) === $key) {
|
|
return Str::snake(strtolower($alias));
|
|
}
|
|
}
|
|
return false;
|
|
}
|
|
|
|
/**
|
|
* Is the file to be ignored
|
|
*
|
|
* @param string $filename
|
|
* @return boolean
|
|
*/
|
|
protected function fileHasToBeIgnored(string $filename): bool
|
|
{
|
|
return in_array($filename, $this->ignoreFiles) or !$this->checkExtension($filename, 'xml');
|
|
}
|
|
|
|
/**
|
|
* Check if the file is an xml file
|
|
*
|
|
* @param string $filename
|
|
* @return boolean
|
|
*/
|
|
protected function isFileXml(string $filename): bool
|
|
{
|
|
$isXml = strtolower(pathinfo($filename, \PATHINFO_EXTENSION)) === 'xml';
|
|
if (!$isXml) {
|
|
Log::warning(sprintf('%s : unwanted file type in %s', get_class($this), $filename));
|
|
}
|
|
return $isXml;
|
|
}
|
|
|
|
protected function sendErrorParseEmailToServiceWeb(Throwable $e)
|
|
{
|
|
Mail::send([
|
|
'html' => 'exception-handler::emails.exception',
|
|
'text' => 'exception-handler::emails.exception-plain'
|
|
], [
|
|
'appName' => config('app.name'),
|
|
'code' => $e->getCode(),
|
|
'line' => $e->getLine(),
|
|
'emessage' => $this->getAnonymizedMessage($e->getMessage()),
|
|
'file' => str_replace(\realpath(\sprintf('%s/../', \app_path())), '', $e->getFile()),
|
|
'stackTrace' => $this->getAnonymizedStackTrace($e),
|
|
'uri' => app()->runningInConsole() ?
|
|
'CONSOLE' : (
|
|
(request() and request()->fullUrl()) ?
|
|
request()->fullUrl() : 'none')
|
|
], function ($m) {
|
|
/** @var \Illuminate\Mail\Message $m */
|
|
$m->to(
|
|
config('mail.contactsList.serviceweb'),
|
|
sprintf('%s Exception Handler', config('app.name'))
|
|
)->subject('Need help !, an error occured in your application, please check Logs ASAP');
|
|
});
|
|
}
|
|
|
|
/**
|
|
* Anonymize at least Sql queries
|
|
*
|
|
* @param string $message
|
|
* @return string
|
|
*/
|
|
private function getAnonymizedMessage(string $message): string
|
|
{
|
|
$message = \preg_replace('/^(.*)( \(SQL:.*\))$/', '$1', $message) ?? $message;
|
|
return $message;
|
|
}
|
|
|
|
/**
|
|
* Anonymize stack trace
|
|
*
|
|
* @param \Exception $e
|
|
* @return string
|
|
*/
|
|
private function getAnonymizedStackTrace(Throwable $e): string
|
|
{
|
|
$projectPath = \realpath(\sprintf('%s/../', \app_path()));
|
|
$traceStack = $e->getTrace();
|
|
$traceStr = '';
|
|
$maxStackStage = 0;
|
|
foreach ($traceStack as $stackStage => $stack) {
|
|
$maxStackStage = $stackStage;
|
|
$traceStr .= \sprintf(
|
|
'#%s %s(%d): ',
|
|
$stackStage,
|
|
str_replace($projectPath, '', $stack['file']),
|
|
$stack['line']
|
|
);
|
|
$traceStr .= "{$stack['class']}->{$stack['function']}(";
|
|
$traceArgs = '';
|
|
if (isset($stack['args'])) {
|
|
$argsCount = count($stack['args']);
|
|
for ($i = 0; $i < $argsCount; $i++) {
|
|
$type = \gettype($stack['args'][$i]);
|
|
switch ($type) {
|
|
case 'object':
|
|
$traceArgs .= sprintf('Object(%s)', \get_class($stack['args'][$i]));
|
|
break;
|
|
case 'array':
|
|
$traceArgs .= sprintf('Array(%d)', \count($stack['args'][$i]));
|
|
break;
|
|
case 'string':
|
|
$traceArgs .= sprintf('String(%d)', \strlen($stack['args'][$i]));
|
|
break;
|
|
default:
|
|
$traceArgs .= \gettype($stack['args'][$i]);
|
|
}
|
|
$traceArgs .= ($i !== ($argsCount - 1) ? ',' : '');
|
|
}
|
|
$traceStr .= $traceArgs;
|
|
}
|
|
$traceStr .= ")\n";
|
|
}
|
|
$maxStackStage++;
|
|
$traceStr .= "#$maxStackStage {main}";
|
|
return $traceStr;
|
|
}
|
|
|
|
protected function sendInvalidXmlEmail(string $filename, Throwable $e)
|
|
{
|
|
$message = $e->getPrevious()->getMessage();
|
|
Mail::send('emails.invalidxml', [
|
|
'strError' => sprintf(
|
|
'File %s could not be parsed => %s on line %d',
|
|
$filename,
|
|
$message,
|
|
$e->getLine()
|
|
),
|
|
'filename' => $filename,
|
|
'exception' => $e->getPrevious()
|
|
], function ($m) {
|
|
$m->to(
|
|
config('mail.contactsList.invalidInco'),
|
|
'Le Gaulois Exception Handler'
|
|
)->subject('You sent me an invalid XML file, please check its validity before resend ASAP.');
|
|
});
|
|
}
|
|
|
|
/**
|
|
* Move the file to invalid folder
|
|
*
|
|
* @param string $filename
|
|
* @return void
|
|
*/
|
|
protected function moveToInvalid(string $filename): void
|
|
{
|
|
if (!$this->getDisk()->move($filename, sprintf('%s/%s', $this->invalidFolderName, $filename))) {
|
|
$this->log(
|
|
sprintf(
|
|
'File %s could not be moved to invalid folder name',
|
|
$filename
|
|
),
|
|
LogType::warning()
|
|
);
|
|
}
|
|
}
|
|
}
|