434 lines
12 KiB
PHP
434 lines
12 KiB
PHP
<?php
|
|
|
|
namespace App\Jobs;
|
|
|
|
use App\Enums\LogType;
|
|
use App\Exceptions\AbortXmlImportException;
|
|
use ErrorException;
|
|
use Exception;
|
|
use Illuminate\Bus\Queueable;
|
|
use Illuminate\Contracts\Filesystem\Filesystem;
|
|
use Illuminate\Contracts\Queue\ShouldQueue;
|
|
use Illuminate\Foundation\Bus\Dispatchable;
|
|
use Illuminate\Queue\InteractsWithQueue;
|
|
use Illuminate\Queue\SerializesModels;
|
|
use Illuminate\Support\Carbon;
|
|
use Illuminate\Support\Facades\Log;
|
|
use Illuminate\Support\Facades\Storage;
|
|
use Illuminate\Support\Str;
|
|
use RecursiveArrayIterator;
|
|
use RecursiveIteratorIterator;
|
|
use SimpleXMLElement;
|
|
use TypeError;
|
|
|
|
abstract class XmlJobs implements ShouldQueue
|
|
{
|
|
use Dispatchable;
|
|
use InteractsWithQueue;
|
|
use Queueable;
|
|
use SerializesModels;
|
|
|
|
/**
|
|
* The Laravel disk name
|
|
*
|
|
* @var string
|
|
*/
|
|
protected $disk = null;
|
|
|
|
/**
|
|
* The columns to take from the XML
|
|
* subcolumns can be added with a dot
|
|
*
|
|
* @var array
|
|
*/
|
|
protected $takeColumns = [];
|
|
|
|
/**
|
|
* The columns allowed to be null
|
|
*
|
|
* @var array
|
|
*/
|
|
protected $nullableFields = [];
|
|
|
|
/**
|
|
* This is for Laravel Console usage
|
|
*/
|
|
public $console = null;
|
|
|
|
/**
|
|
* The relative folder where to archive data
|
|
*
|
|
* @var string
|
|
*/
|
|
private $archiveFolderName = 'archives';
|
|
|
|
/**
|
|
* The files extension to consider
|
|
*
|
|
* @var string
|
|
*/
|
|
private $wantedFileExt = 'xml';
|
|
|
|
/**
|
|
* Need to encapsulate the freeking XML
|
|
* Because provided one is dirty
|
|
*
|
|
* @var boolean
|
|
*/
|
|
protected $encapsulate = false;
|
|
|
|
/**
|
|
* The files to ignore
|
|
*
|
|
* @var array
|
|
*/
|
|
private $ignoreFiles = [
|
|
'.gitignore', '.gitkeep'
|
|
];
|
|
|
|
/**
|
|
* Parse Data from the XML and gives it to the callback
|
|
*
|
|
* @param string $entityName
|
|
* @param string $modelClass
|
|
* @return void
|
|
*/
|
|
protected function parseData(string $entityName, string $modelClass, callable $callback = null): void
|
|
{
|
|
/** @var Filesystem */
|
|
$disk = $this->getDisk();
|
|
$this->log('Démarrage du traitement');
|
|
$files = $disk->files();
|
|
|
|
$progress = 1;
|
|
$total = count(array_filter($files, function ($filename) {
|
|
if ($this->fileHasToBeIgnored($filename)) {
|
|
return false;
|
|
}
|
|
return true;
|
|
}));
|
|
|
|
foreach ($files as $filename) {
|
|
if ($this->fileHasToBeIgnored($filename)) {
|
|
continue;
|
|
}
|
|
|
|
$this->log(sprintf(' traitement %d sur %s ', $progress++, $total));
|
|
|
|
/** @var string $data */
|
|
$data = $disk->get($filename);
|
|
|
|
if ($this->encapsulate) {
|
|
$data = sprintf('<Root>%s</Root>', $data);
|
|
/** @var SimpleXMLElement|false */
|
|
$xmlContent = simplexml_load_string($data);
|
|
} else {
|
|
/** @var SimpleXMLElement|false */
|
|
$xmlContent = simplexml_load_string($data, null, LIBXML_NOCDATA | LIBXML_PARSEHUGE);
|
|
}
|
|
|
|
if ($xmlContent === false) {
|
|
$this->log(
|
|
sprintf('File %s could not be parsed (simplexml_load_string)', $filename),
|
|
LogType::critical()
|
|
);
|
|
continue; // ignore the file
|
|
}
|
|
|
|
$data = $this->extract($xmlContent);
|
|
|
|
try {
|
|
$lines = [];
|
|
|
|
foreach ($data[$entityName] as $traca) {
|
|
$line = $this->processFields($traca);
|
|
if ($callback) {
|
|
$callback($xmlContent, $line);
|
|
}
|
|
$lines[] = $line;
|
|
}
|
|
|
|
// Mise en base des données
|
|
foreach ($lines as $line) {
|
|
/** @var array Unique Key columns */
|
|
$uniqueColumnsKey = [];
|
|
foreach ($this->uniqueKeyField as $uField) {
|
|
$uniqueColumnsKey[$uField] = $line[$uField];
|
|
}
|
|
forward_static_call_array(
|
|
[$modelClass, 'updateOrCreate'],
|
|
[$uniqueColumnsKey, $line]
|
|
);
|
|
}
|
|
} catch (ErrorException | TypeError $e) {
|
|
$this->log(
|
|
sprintf(
|
|
'File %s could not be parsed => %s on line %d',
|
|
$filename,
|
|
$e->getMessage(),
|
|
$e->getLine()
|
|
),
|
|
LogType::critical()
|
|
);
|
|
continue; // ignore the file
|
|
} catch (AbortXmlImportException $e) {
|
|
$this->log(
|
|
sprintf(
|
|
'File %s could not be parsed => %s on line %d',
|
|
$filename,
|
|
$e->getMessage(),
|
|
$e->getLine()
|
|
),
|
|
LogType::error()
|
|
);
|
|
continue; // ignore the file
|
|
}
|
|
|
|
$this->moveToArchives($filename);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Recursive extract to array
|
|
*
|
|
* @param \SimpleXMLElement|false $sxe
|
|
* @return array
|
|
*/
|
|
protected function extract($sxe = null): array
|
|
{
|
|
if (!$sxe instanceof SimpleXMLElement) {
|
|
return array();
|
|
}
|
|
$extract = array();
|
|
foreach ($sxe->children() as $key => $value) {
|
|
if ($value->count()) {
|
|
$extract[$key][] = $this->extract($value);
|
|
continue;
|
|
}
|
|
$value = strval($value);
|
|
if (array_key_exists($key, $extract) and is_array($extract[$key])) {
|
|
$extract[$key][] = strlen($value) ? $value : null;
|
|
} else {
|
|
$extract[$key] = strlen($value) ? $value : null;
|
|
}
|
|
}
|
|
return $extract;
|
|
}
|
|
|
|
/**
|
|
* Process fields to gather data
|
|
*
|
|
* @param array $fields
|
|
* @return array
|
|
* @throws AbortXmlImportException
|
|
*/
|
|
protected function processFields(array $fields): array
|
|
{
|
|
$result = [];
|
|
foreach ($fields as $key => $field) {
|
|
if (!($key = $this->shouldFieldBeConsidered($key))) {
|
|
continue;
|
|
}
|
|
if (!$this->isFieldNullable($key) && is_null($field)) {
|
|
throw new AbortXmlImportException(sprintf('The field %s is not allowed to be null', $key));
|
|
}
|
|
if (is_array($field)) {
|
|
// Flatten data
|
|
$it = new RecursiveIteratorIterator(new RecursiveArrayIterator($field));
|
|
foreach ($it as $k => $f) {
|
|
$ko = strtolower(sprintf('%s.%s', $key, $k));
|
|
if (!$this->isFieldNullable($ko) && is_null($f)) {
|
|
throw new AbortXmlImportException(sprintf('%s : The field %s is not allowed to be null', get_class($this), $ko));
|
|
}
|
|
if (!($k = $this->shouldFieldBeConsidered($ko))) {
|
|
continue;
|
|
}
|
|
$result[$k] = $f;
|
|
}
|
|
} else {
|
|
$result[$key] = $field;
|
|
}
|
|
}
|
|
return $result;
|
|
}
|
|
|
|
/**
|
|
* Is the filed nullable
|
|
*
|
|
* @param string $key
|
|
* @return boolean
|
|
*/
|
|
private function isFieldNullable(string $key): bool
|
|
{
|
|
if (in_array($key, $this->nullableFields)) {
|
|
return true;
|
|
}
|
|
return false;
|
|
}
|
|
|
|
/**
|
|
* - replace key name with the corresponding alias
|
|
* - strtolower the key
|
|
* - snake_case the key
|
|
*
|
|
* @param string $key
|
|
* @return string|false
|
|
*/
|
|
private function shouldFieldBeConsidered(string $key)
|
|
{
|
|
$key = strtolower($key);
|
|
foreach ($this->takeColumns as $k => $alias) {
|
|
if (strtolower($k) === $key) {
|
|
return Str::snake(strtolower($alias));
|
|
}
|
|
}
|
|
return false;
|
|
}
|
|
|
|
/**
|
|
* Is the file to be ignored
|
|
*
|
|
* @param string $filename
|
|
* @return boolean
|
|
*/
|
|
protected function fileHasToBeIgnored(string $filename): bool
|
|
{
|
|
return in_array($filename, $this->ignoreFiles) or !$this->isFileXml($filename);
|
|
}
|
|
|
|
/**
|
|
* Check if the file is an xml file
|
|
*
|
|
* @param string $filename
|
|
* @return boolean
|
|
*/
|
|
protected function isFileXml(string $filename): bool
|
|
{
|
|
$isXml = strtolower(pathinfo($filename, \PATHINFO_EXTENSION)) === 'xml';
|
|
if (!$isXml) {
|
|
Log::warning(sprintf('%s : unwanted file type in %s', get_class($this), $filename));
|
|
}
|
|
return $isXml;
|
|
}
|
|
|
|
/**
|
|
* Move the file to archives
|
|
*
|
|
* @param string $filename
|
|
* @return void
|
|
*/
|
|
protected function moveToArchives(string $filename): void
|
|
{
|
|
if (!$this->getDisk()->move($filename, sprintf('%s/%s', $this->archiveFolderName, $filename))) {
|
|
$this->log(
|
|
sprintf(
|
|
'File %s could not be moved to archives',
|
|
$filename
|
|
),
|
|
LogType::warning()
|
|
);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Suppression des fichiers dans
|
|
* archives qui ont plus de 2j
|
|
*
|
|
* @return void
|
|
*/
|
|
private function clearOldArchives(): void
|
|
{
|
|
$disk = $this->getDisk();
|
|
$archives = $disk->files($this->archiveFolderName);
|
|
|
|
// /** @var File */
|
|
foreach ($archives as $archive) {
|
|
if (in_array($archive, $this->ignoreFiles)) {
|
|
continue;
|
|
}
|
|
|
|
if ($disk->lastModified($archive) < strtotime(Carbon::now()->subDays(2))) {
|
|
if (!$disk->delete($archive)) {
|
|
Log::critical(sprintf('%s : Could not delete file %s', get_class($this), $archive));
|
|
};
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Get the concerned disk
|
|
*
|
|
* @return Filesystem
|
|
*/
|
|
protected function getDisk(): Filesystem
|
|
{
|
|
return Storage::disk($this->disk);
|
|
}
|
|
|
|
/**
|
|
* Logs data
|
|
*
|
|
* @param string $message
|
|
* @param LogType $logtype
|
|
* @return void
|
|
*/
|
|
protected function log(string $message, LogType $logtype = null): void
|
|
{
|
|
if (!$logtype) {
|
|
$logtype = LogType::debug();
|
|
}
|
|
$message = sprintf('%s : %s', get_class($this), $message);
|
|
switch ($logtype->value) {
|
|
case LogType::debug()->value:
|
|
Log::debug($message);
|
|
break;
|
|
case LogType::info()->value:
|
|
Log::info($message);
|
|
break;
|
|
case LogType::log()->value:
|
|
Log::log($message);
|
|
break;
|
|
case LogType::warning()->value:
|
|
Log::warning($message);
|
|
break;
|
|
case LogType::error()->value:
|
|
Log::error($message);
|
|
break;
|
|
case LogType::critical()->value:
|
|
Log::critical($message);
|
|
break;
|
|
default:
|
|
throw new Exception(sprintf('%s : unhandled log type %s', get_class($this), $logtype->label));
|
|
}
|
|
$this->consolePrint($message, $logtype);
|
|
}
|
|
|
|
/**
|
|
* Prints to the console
|
|
*
|
|
* @param string $message
|
|
* @param LogType $logtype
|
|
* @return void
|
|
*/
|
|
protected function consolePrint(string $message, LogType $logtype): void
|
|
{
|
|
if ($this->console) {
|
|
switch ($logtype->value) {
|
|
case LogType::debug()->value:
|
|
case LogType::info()->value:
|
|
case LogType::log()->value:
|
|
$this->console->info($message);
|
|
break;
|
|
case LogType::warning()->value:
|
|
$this->console->comment($message);
|
|
break;
|
|
case LogType::error()->value:
|
|
case LogType::critical()->value:
|
|
$this->console->error($message);
|
|
break;
|
|
default:
|
|
throw new Exception(sprintf('%s : unhandled log type %s', get_class($this), $logtype->label));
|
|
}
|
|
}
|
|
}
|
|
}
|