365 lines
11 KiB
PHP
365 lines
11 KiB
PHP
<?php
|
|
|
|
namespace App\Jobs;
|
|
|
|
use App\Enums\LogType;
|
|
use App\Exceptions\AbortXmlImportException;
|
|
use App\Exceptions\InvalidXmlException;
|
|
use ErrorException;
|
|
use Illuminate\Contracts\Filesystem\FileNotFoundException;
|
|
use Illuminate\Database\QueryException;
|
|
use Illuminate\Support\Facades\Log;
|
|
use Illuminate\Support\Facades\Mail;
|
|
use Illuminate\Support\Str;
|
|
use RecursiveArrayIterator;
|
|
use RecursiveIteratorIterator;
|
|
use SimpleXMLElement;
|
|
use Throwable;
|
|
use TypeError;
|
|
|
|
abstract class XmlJobs extends XArcadiaFlux
|
|
{
|
|
|
|
/**
|
|
* invalid xml folder
|
|
*
|
|
* @var string
|
|
*/
|
|
protected $invalidFolderName = 'invalid';
|
|
|
|
/**
|
|
* The columns to take from the XML
|
|
* subcolumns can be added with a dot
|
|
*
|
|
* @var array
|
|
*/
|
|
protected $takeColumns = [];
|
|
|
|
/**
|
|
* The columns allowed to be null
|
|
*
|
|
* @var array
|
|
*/
|
|
protected $nullableFields = [];
|
|
|
|
/**
|
|
* Need to encapsulate the freeking XML
|
|
* Because provided one is dirty
|
|
*
|
|
* @var boolean
|
|
*/
|
|
protected $encapsulate = false;
|
|
|
|
/**
|
|
* Parse Data from the XML and gives it to the callback
|
|
*
|
|
* @param string $entityName
|
|
* @param string $modelClass
|
|
* @return void
|
|
*/
|
|
// phpcs:ignore Generic.Metrics.CyclomaticComplexity.TooHigh
|
|
protected function parseData(string $entityName, string $modelClass, callable $callback = null): void
|
|
{
|
|
/** @var Filesystem */
|
|
$disk = $this->getDisk();
|
|
$this->log('Démarrage du traitement');
|
|
$files = $disk->files();
|
|
|
|
$progress = 1;
|
|
$total = count(array_filter($files, function ($filename) {
|
|
if ($this->fileHasToBeIgnored($filename)) {
|
|
return false;
|
|
}
|
|
return true;
|
|
}));
|
|
|
|
foreach ($files as $filename) {
|
|
if ($this->fileHasToBeIgnored($filename)) {
|
|
continue;
|
|
}
|
|
|
|
$this->log(sprintf(' traitement %d sur %s ', $progress++, $total));
|
|
|
|
try {
|
|
/** @var string $data */
|
|
$data = $disk->get($filename);
|
|
try {
|
|
if ($this->encapsulate) {
|
|
$data = sprintf('<Root>%s</Root>', $data);
|
|
/** @var SimpleXMLElement|false */
|
|
$xmlContent = simplexml_load_string($data);
|
|
} else {
|
|
/** @var SimpleXMLElement|false */
|
|
$xmlContent = simplexml_load_string($data, null, LIBXML_NOCDATA | LIBXML_PARSEHUGE);
|
|
}
|
|
if ($xmlContent === false) {
|
|
throw new InvalidXmlException('XML is Invalid (simplexml_load_string returned false)');
|
|
}
|
|
} catch (ErrorException $e) {
|
|
throw new InvalidXmlException('XML is Invalid', 0, $e);
|
|
}
|
|
|
|
$data = $this->extract($xmlContent);
|
|
|
|
$lines = [];
|
|
foreach ($data[$entityName] as $traca) {
|
|
$line = $this->processFields($traca);
|
|
if ($callback) {
|
|
$callback($xmlContent, $line);
|
|
}
|
|
$lines[] = $line;
|
|
}
|
|
|
|
// Mise en base des données
|
|
foreach ($lines as $line) {
|
|
/** @var array Unique Key columns */
|
|
$uniqueColumnsKey = [];
|
|
foreach ($this->uniqueKeyField as $uField) {
|
|
$uniqueColumnsKey[$uField] = $line[$uField];
|
|
}
|
|
forward_static_call_array(
|
|
[$modelClass, 'updateOrCreate'],
|
|
[$uniqueColumnsKey, $line]
|
|
);
|
|
}
|
|
} catch (InvalidXmlException $e) {
|
|
$message = $e->getMessage();
|
|
if (get_class($e) === QueryException::class) {
|
|
$message = implode(' ', $e->errorInfo);
|
|
}
|
|
$this->log(
|
|
sprintf(
|
|
'File %s could not be parsed => %s on line %d',
|
|
$filename,
|
|
$message,
|
|
$e->getLine()
|
|
),
|
|
LogType::critical()
|
|
);
|
|
$this->sendInvalidXmlEmail($filename, $e);
|
|
$this->moveToInvalid($filename);
|
|
continue;
|
|
} catch (ErrorException | TypeError $e) {
|
|
$this->log(
|
|
sprintf(
|
|
'CRITICAL: File %s could not be parsed => %s on line %d',
|
|
$filename,
|
|
$e->getMessage(),
|
|
$e->getLine()
|
|
),
|
|
LogType::critical()
|
|
);
|
|
$this->sendErrorParseEmailToServiceWeb($e);
|
|
continue; // ignore the file
|
|
} catch (AbortXmlImportException $e) {
|
|
$this->log(
|
|
sprintf(
|
|
'ERROR: File %s could not be parsed => %s on line %d',
|
|
$filename,
|
|
$e->getMessage(),
|
|
$e->getLine()
|
|
),
|
|
LogType::error()
|
|
);
|
|
$this->sendErrorParseEmailToServiceWeb($e);
|
|
continue; // ignore the file
|
|
} catch (FileNotFoundException | QueryException $e) {
|
|
// This happens because of concurent xml jobs read
|
|
$message = $e->getMessage();
|
|
if (get_class($e) === QueryException::class) {
|
|
$message = implode(' ', $e->errorInfo);
|
|
}
|
|
$this->log(
|
|
sprintf(
|
|
'ERROR: File %s could not be read (not found) => %s on line %d',
|
|
$filename,
|
|
$message,
|
|
$e->getLine()
|
|
),
|
|
LogType::debug()
|
|
);
|
|
continue; // ignore the file
|
|
}
|
|
$this->moveToArchives($filename);
|
|
}
|
|
|
|
// Clear old archives
|
|
$this->clearOldArchives();
|
|
}
|
|
|
|
/**
|
|
* Recursive extract to array
|
|
*
|
|
* @param \SimpleXMLElement|false $sxe
|
|
* @return array
|
|
*/
|
|
protected function extract($sxe = null): array
|
|
{
|
|
if (!$sxe instanceof SimpleXMLElement) {
|
|
return array();
|
|
}
|
|
$extract = array();
|
|
foreach ($sxe->children() as $key => $value) {
|
|
if ($value->count()) {
|
|
$extract[$key][] = $this->extract($value);
|
|
continue;
|
|
}
|
|
$value = strval($value);
|
|
if (array_key_exists($key, $extract) and is_array($extract[$key])) {
|
|
$extract[$key][] = strlen($value) ? $value : null;
|
|
} else {
|
|
$extract[$key] = strlen($value) ? $value : null;
|
|
}
|
|
}
|
|
return $extract;
|
|
}
|
|
|
|
/**
|
|
* Process fields to gather data
|
|
*
|
|
* @param array $fields
|
|
* @return array
|
|
* @throws AbortXmlImportException
|
|
*/
|
|
protected function processFields(array $fields): array
|
|
{
|
|
$result = [];
|
|
foreach ($fields as $key => $field) {
|
|
if (!($key = $this->shouldFieldBeConsidered($key))) {
|
|
continue;
|
|
}
|
|
if (!$this->isFieldNullable($key) && is_null($field)) {
|
|
throw new AbortXmlImportException(sprintf('The field %s is not allowed to be null', $key));
|
|
}
|
|
if (is_array($field)) {
|
|
// Flatten data
|
|
$it = new RecursiveIteratorIterator(new RecursiveArrayIterator($field));
|
|
foreach ($it as $k => $f) {
|
|
$ko = strtolower(sprintf('%s.%s', $key, $k));
|
|
if (!$this->isFieldNullable($ko) && is_null($f)) {
|
|
throw new AbortXmlImportException(sprintf(
|
|
'%s : The field %s is not allowed to be null',
|
|
get_class($this),
|
|
$ko
|
|
));
|
|
}
|
|
if (!($k = $this->shouldFieldBeConsidered($ko))) {
|
|
continue;
|
|
}
|
|
$result[$k] = $f;
|
|
}
|
|
} else {
|
|
$result[$key] = $field;
|
|
}
|
|
}
|
|
return $result;
|
|
}
|
|
|
|
/**
|
|
* Is the filed nullable
|
|
*
|
|
* @param string $key
|
|
* @return boolean
|
|
*/
|
|
private function isFieldNullable(string $key): bool
|
|
{
|
|
if (in_array($key, $this->nullableFields)) {
|
|
return true;
|
|
}
|
|
return false;
|
|
}
|
|
|
|
/**
|
|
* - replace key name with the corresponding alias
|
|
* - strtolower the key
|
|
* - snake_case the key
|
|
*
|
|
* @param string $key
|
|
* @return string|false
|
|
*/
|
|
private function shouldFieldBeConsidered(string $key)
|
|
{
|
|
$key = strtolower($key);
|
|
foreach ($this->takeColumns as $k => $alias) {
|
|
if (strtolower($k) === $key) {
|
|
return Str::snake(strtolower($alias));
|
|
}
|
|
}
|
|
return false;
|
|
}
|
|
|
|
/**
|
|
* Is the file to be ignored
|
|
*
|
|
* @param string $filename
|
|
* @return boolean
|
|
*/
|
|
protected function fileHasToBeIgnored(string $filename): bool
|
|
{
|
|
return in_array($filename, $this->ignoreFiles) or !$this->checkExtension($filename, 'xml');
|
|
}
|
|
|
|
/**
|
|
* Check if the file is an xml file
|
|
*
|
|
* @param string $filename
|
|
* @return boolean
|
|
*/
|
|
protected function isFileXml(string $filename): bool
|
|
{
|
|
$isXml = strtolower(pathinfo($filename, \PATHINFO_EXTENSION)) === 'xml';
|
|
if (!$isXml) {
|
|
Log::warning(sprintf('%s : unwanted file type in %s', get_class($this), $filename));
|
|
}
|
|
return $isXml;
|
|
}
|
|
|
|
protected function sendErrorParseEmailToServiceWeb(Throwable $e)
|
|
{
|
|
Mail::send('emails.exception', ['exception' => $e], function ($m) {
|
|
$m->to(
|
|
config('mail.contactsList.serviceweb'),
|
|
'Le Gaulois Exception Handler'
|
|
)->subject('Could not eat INCO, because of its toxicity !');
|
|
});
|
|
}
|
|
|
|
protected function sendInvalidXmlEmail(string $filename, Throwable $e)
|
|
{
|
|
$message = $e->getPrevious()->getMessage();
|
|
Mail::send('emails.invalidxml', [
|
|
'strError' => sprintf(
|
|
'File %s could not be parsed => %s on line %d',
|
|
$filename,
|
|
$message,
|
|
$e->getLine()
|
|
),
|
|
'filename' => $filename,
|
|
'exception' => $e->getPrevious()
|
|
], function ($m) {
|
|
$m->to(
|
|
config('mail.contactsList.invalidInco'),
|
|
'Le Gaulois Exception Handler'
|
|
)->subject('You sent me an invalid XML file, please check its validity before resend ASAP.');
|
|
});
|
|
}
|
|
|
|
/**
|
|
* Move the file to invalid folder
|
|
*
|
|
* @param string $filename
|
|
* @return void
|
|
*/
|
|
protected function moveToInvalid(string $filename): void
|
|
{
|
|
if (!$this->getDisk()->move($filename, sprintf('%s/%s', $this->invalidFolderName, $filename))) {
|
|
$this->log(
|
|
sprintf(
|
|
'File %s could not be moved to invalid folder name',
|
|
$filename
|
|
),
|
|
LogType::warning()
|
|
);
|
|
}
|
|
}
|
|
}
|