ldc-legaulois/app/Jobs/XmlJobs.php
2021-04-08 12:49:58 +02:00

434 lines
12 KiB
PHP

<?php
namespace App\Jobs;
use App\Enums\LogType;
use App\Exceptions\AbortXmlImportException;
use ErrorException;
use Exception;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Filesystem\Filesystem;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use Illuminate\Support\Carbon;
use Illuminate\Support\Facades\Log;
use Illuminate\Support\Facades\Storage;
use Illuminate\Support\Str;
use RecursiveArrayIterator;
use RecursiveIteratorIterator;
use SimpleXMLElement;
use TypeError;
abstract class XmlJobs implements ShouldQueue
{
use Dispatchable;
use InteractsWithQueue;
use Queueable;
use SerializesModels;
/**
* The Laravel disk name
*
* @var string
*/
protected $disk = null;
/**
* The columns to take from the XML
* subcolumns can be added with a dot
*
* @var array
*/
protected $takeColumns = [];
/**
* The columns allowed to be null
*
* @var array
*/
protected $nullableFields = [];
/**
* This is for Laravel Console usage
*/
public $console = null;
/**
* The relative folder where to archive data
*
* @var string
*/
private $archiveFolderName = 'archives';
/**
* The files extension to consider
*
* @var string
*/
private $wantedFileExt = 'xml';
/**
* Need to encapsulate the freeking XML
* Because provided one is dirty
*
* @var boolean
*/
protected $encapsulate = false;
/**
* The files to ignore
*
* @var array
*/
private $ignoreFiles = [
'.gitignore', '.gitkeep'
];
/**
* Parse Data from the XML and gives it to the callback
*
* @param string $entityName
* @param string $modelClass
* @return void
*/
protected function parseData(string $entityName, string $modelClass, callable $callback = null): void
{
/** @var Filesystem */
$disk = $this->getDisk();
$this->log('Démarrage du traitement');
$files = $disk->files();
$progress = 1;
$total = count(array_filter($files, function ($filename) {
if ($this->fileHasToBeIgnored($filename)) {
return false;
}
return true;
}));
foreach ($files as $filename) {
if ($this->fileHasToBeIgnored($filename)) {
continue;
}
$this->log(sprintf(' traitement %d sur %s ', $progress++, $total));
/** @var string $data */
$data = $disk->get($filename);
if ($this->encapsulate) {
$data = sprintf('<Root>%s</Root>', $data);
/** @var SimpleXMLElement|false */
$xmlContent = simplexml_load_string($data);
} else {
/** @var SimpleXMLElement|false */
$xmlContent = simplexml_load_string($data, null, LIBXML_NOCDATA | LIBXML_PARSEHUGE);
}
if ($xmlContent === false) {
$this->log(
sprintf('File %s could not be parsed (simplexml_load_string)', $filename),
LogType::critical()
);
continue; // ignore the file
}
$data = $this->extract($xmlContent);
try {
$lines = [];
foreach ($data[$entityName] as $traca) {
$line = $this->processFields($traca);
if ($callback) {
$callback($xmlContent, $line);
}
$lines[] = $line;
}
// Mise en base des données
foreach ($lines as $line) {
/** @var array Unique Key columns */
$uniqueColumnsKey = [];
foreach ($this->uniqueKeyField as $uField) {
$uniqueColumnsKey[$uField] = $line[$uField];
}
forward_static_call_array(
[$modelClass, 'updateOrCreate'],
[$uniqueColumnsKey, $line]
);
}
} catch (ErrorException | TypeError $e) {
$this->log(
sprintf(
'File %s could not be parsed => %s on line %d',
$filename,
$e->getMessage(),
$e->getLine()
),
LogType::critical()
);
continue; // ignore the file
} catch (AbortXmlImportException $e) {
$this->log(
sprintf(
'File %s could not be parsed => %s on line %d',
$filename,
$e->getMessage(),
$e->getLine()
),
LogType::error()
);
continue; // ignore the file
}
$this->moveToArchives($filename);
}
}
/**
* Recursive extract to array
*
* @param \SimpleXMLElement|false $sxe
* @return array
*/
protected function extract($sxe = null): array
{
if (!$sxe instanceof SimpleXMLElement) {
return array();
}
$extract = array();
foreach ($sxe->children() as $key => $value) {
if ($value->count()) {
$extract[$key][] = $this->extract($value);
continue;
}
$value = strval($value);
if (array_key_exists($key, $extract) and is_array($extract[$key])) {
$extract[$key][] = strlen($value) ? $value : null;
} else {
$extract[$key] = strlen($value) ? $value : null;
}
}
return $extract;
}
/**
* Process fields to gather data
*
* @param array $fields
* @return array
* @throws AbortXmlImportException
*/
protected function processFields(array $fields): array
{
$result = [];
foreach ($fields as $key => $field) {
if (!($key = $this->shouldFieldBeConsidered($key))) {
continue;
}
if (!$this->isFieldNullable($key) && is_null($field)) {
throw new AbortXmlImportException(sprintf('The field %s is not allowed to be null', $key));
}
if (is_array($field)) {
// Flatten data
$it = new RecursiveIteratorIterator(new RecursiveArrayIterator($field));
foreach ($it as $k => $f) {
$ko = strtolower(sprintf('%s.%s', $key, $k));
if (!$this->isFieldNullable($ko) && is_null($f)) {
throw new AbortXmlImportException(sprintf('%s : The field %s is not allowed to be null', get_class($this), $ko));
}
if (!($k = $this->shouldFieldBeConsidered($ko))) {
continue;
}
$result[$k] = $f;
}
} else {
$result[$key] = $field;
}
}
return $result;
}
/**
* Is the filed nullable
*
* @param string $key
* @return boolean
*/
private function isFieldNullable(string $key): bool
{
if (in_array($key, $this->nullableFields)) {
return true;
}
return false;
}
/**
* - replace key name with the corresponding alias
* - strtolower the key
* - snake_case the key
*
* @param string $key
* @return string|false
*/
private function shouldFieldBeConsidered(string $key)
{
$key = strtolower($key);
foreach ($this->takeColumns as $k => $alias) {
if (strtolower($k) === $key) {
return Str::snake(strtolower($alias));
}
}
return false;
}
/**
* Is the file to be ignored
*
* @param string $filename
* @return boolean
*/
protected function fileHasToBeIgnored(string $filename): bool
{
return in_array($filename, $this->ignoreFiles) or !$this->isFileXml($filename);
}
/**
* Check if the file is an xml file
*
* @param string $filename
* @return boolean
*/
protected function isFileXml(string $filename): bool
{
$isXml = strtolower(pathinfo($filename, \PATHINFO_EXTENSION)) === 'xml';
if (!$isXml) {
Log::warning(sprintf('%s : unwanted file type in %s', get_class($this), $filename));
}
return $isXml;
}
/**
* Move the file to archives
*
* @param string $filename
* @return void
*/
protected function moveToArchives(string $filename): void
{
if (!$this->getDisk()->move($filename, sprintf('%s/%s', $this->archiveFolderName, $filename))) {
$this->log(
sprintf(
'File %s could not be moved to archives',
$filename
),
LogType::warning()
);
}
}
/**
* Suppression des fichiers dans
* archives qui ont plus de 2j
*
* @return void
*/
private function clearOldArchives(): void
{
$disk = $this->getDisk();
$archives = $disk->files($this->archiveFolderName);
// /** @var File */
foreach ($archives as $archive) {
if (in_array($archive, $this->ignoreFiles)) {
continue;
}
if ($disk->lastModified($archive) < strtotime(Carbon::now()->subDays(2))) {
if (!$disk->delete($archive)) {
Log::critical(sprintf('%s : Could not delete file %s', get_class($this), $archive));
};
}
}
}
/**
* Get the concerned disk
*
* @return Filesystem
*/
protected function getDisk(): Filesystem
{
return Storage::disk($this->disk);
}
/**
* Logs data
*
* @param string $message
* @param LogType $logtype
* @return void
*/
protected function log(string $message, LogType $logtype = null): void
{
if (!$logtype) {
$logtype = LogType::debug();
}
$message = sprintf('%s : %s', get_class($this), $message);
switch ($logtype->value) {
case LogType::debug()->value:
Log::debug($message);
break;
case LogType::info()->value:
Log::info($message);
break;
case LogType::log()->value:
Log::log($message);
break;
case LogType::warning()->value:
Log::warning($message);
break;
case LogType::error()->value:
Log::error($message);
break;
case LogType::critical()->value:
Log::critical($message);
break;
default:
throw new Exception(sprintf('%s : unhandled log type %s', get_class($this), $logtype->label));
}
$this->consolePrint($message, $logtype);
}
/**
* Prints to the console
*
* @param string $message
* @param LogType $logtype
* @return void
*/
protected function consolePrint(string $message, LogType $logtype): void
{
if ($this->console) {
switch ($logtype->value) {
case LogType::debug()->value:
case LogType::info()->value:
case LogType::log()->value:
$this->console->info($message);
break;
case LogType::warning()->value:
$this->console->comment($message);
break;
case LogType::error()->value:
case LogType::critical()->value:
$this->console->error($message);
break;
default:
throw new Exception(sprintf('%s : unhandled log type %s', get_class($this), $logtype->label));
}
}
}
}