Ticket 754: Rewrite Arcadia Xml in job:

- composer require kwaadpepper/enum
- add MAIL_LIST_SERVICEWEB var to .env
- add XmlJobs, XArcadiaFlux
- ArcadiaLink is now a Job instead of a simple comand
- Add real progressbar to XArcadiaFlux
- Add XmlArcadiaMediasIn to Kernel to execute twice daily on one server
- Add Lib\Utils class with sendEmailErrorManually until laravel bump is done
- Changed XmlJobs can now handle xml files which have only one entity within
- Add guessed XSD
- Changed XmlJobs can now validate xml with xsd before parse
- Changed XmlJobs Report Query Exceptions on save
- Added XmlJobs Add save callback to handle save
- Extensive tests
This commit is contained in:
Jérémy Munsch 2022-03-25 12:41:00 +01:00
parent 4b397a1dd4
commit 3adefc6fdc
No known key found for this signature in database
GPG key ID: 55196BEE445F6049
23 changed files with 1415 additions and 164 deletions

View file

@ -40,3 +40,5 @@ LDAP_PASSWORD=adminpassword
LDAP_USE_SSL=false
ELASTICSEARCH_HOSTS=
MAIL_LIST_SERVICEWEB=

View file

@ -1,80 +0,0 @@
<?php
namespace App\Console\Commands;
use App\Models\Media;
use Illuminate\Console\Command;
use Illuminate\Support\Facades\Storage;
class ArcadiaLink extends Command
{
/**
* The name and signature of the console command.
*
* @var string
*/
protected $signature = 'arcadialink:process';
/**
* The console command description.
*
* @var string
*/
protected $description = 'Command description';
/**
* Create a new command instance.
*
* @return void
*/
public function __construct()
{
parent::__construct();
}
/**
* Execute the console command.
*
* @return mixed
*/
public function handle()
{
$this->info('Démarrage du traitement');
$files = Storage::disk('arcadia_link_r')->files();
$bar = $this->output->createProgressBar(count($files)-1);
$bar->start();
foreach ($files as $filename) {
if($filename == ".gitignore" || $filename == ".gitkeep") continue;
// ---- Récupération du contenu
$files = Storage::disk('arcadia_link_r')->get($filename);
$fileContent = simplexml_load_string($files, null, LIBXML_NOCDATA);
$media = Media::withTrashed()->find($fileContent->code_DAM);
$media->arcadia_id = $fileContent->codeArcadia;
$media->version = $fileContent->version;
$media->isLocked = 0;
$media->save();
// Suppression dans la supervision Arcadia Link
$a = \App\Models\ArcadiaLink::where('media_id', $fileContent->code_DAM)->first();
if ($a)
$a->delete();
// ---- Index dans Elasticsearch
$media = Media::withTrashed()->find($media->id);
$esExist = Media::searchByQuery(array('match' => array('arcadia_id' => "$media->arcadia_id")));
if ($esExist->totalHits() > 0)
$media->removeFromIndex();
$media->addToIndex();
Storage::disk('arcadia_link_r')->delete($filename);
$bar->advance();
}
$bar->finish();
$this->line("");
$this->info('Fin du traitement');
}
}

View file

@ -0,0 +1,40 @@
<?php
namespace App\Console\Commands;
use App\Jobs\Arcadia\XmlArcadiaMediasIn;
use Illuminate\Console\Command;
class ArcadiaLinkCommand extends Command
{
/**
* The name and signature of the console command.
*
* @var string
*/
protected $signature = 'arcadialink:process';
/**
* The console command description.
*
* @var string
*/
protected $description = 'Lecture du flux Arcadia entrant (medias). Run the XmlArcadiaMediasIn manually';
/**
* Execute the console command.
*
* @return mixed
*/
public function handle()
{
$this->info('Running the Job, this could take some time.');
$job = new XmlArcadiaMediasIn();
$job->console = $this;
$job->handle();
$this->line("");
$this->info('Fin du traitement');
}
}

View file

@ -2,6 +2,8 @@
namespace App\Console;
use App\Console\Commands\ArcadiaLinkCommand;
use App\Jobs\Arcadia\XmlArcadiaMediasIn;
use Illuminate\Console\Scheduling\Schedule;
use Illuminate\Foundation\Console\Kernel as ConsoleKernel;
@ -13,19 +15,19 @@ class Kernel extends ConsoleKernel
* @var array
*/
protected $commands = [
//
ArcadiaLinkCommand::class
];
/**
* Define the application's command schedule.
*
* @param \Illuminate\Console\Scheduling\Schedule $schedule
* @param \Illuminate\Console\Scheduling\Schedule $schedule
* @return void
*/
protected function schedule(Schedule $schedule)
{
// $schedule->command('inspire')
// ->hourly();
$schedule->job(XmlArcadiaMediasIn::class)->name('arcadia_medias_in')
->onOneServer()->cron('*/2 * * * *');
}
/**
@ -35,7 +37,7 @@ class Kernel extends ConsoleKernel
*/
protected function commands()
{
$this->load(__DIR__.'/Commands');
$this->load(__DIR__ . '/Commands');
require base_path('routes/console.php');
}

17
app/Enums/LogType.php Normal file
View file

@ -0,0 +1,17 @@
<?php
namespace App\Enums;
use Kwaadpepper\Enum\BaseEnumRoutable;
/**
* @method static self debug()
* @method static self log()
* @method static self warning()
* @method static self info()
* @method static self error()
* @method static self critical()
*/
final class LogType extends BaseEnumRoutable
{
}

View file

@ -0,0 +1,9 @@
<?php
namespace App\Exceptions;
use ErrorException;
class AbortXmlImportException extends ErrorException
{
}

View file

@ -0,0 +1,9 @@
<?php
namespace App\Exceptions;
use ErrorException;
class ElasticSearchException extends ErrorException
{
}

View file

@ -0,0 +1,9 @@
<?php
namespace App\Exceptions;
use ErrorException;
class InvalidXmlException extends ErrorException
{
}

View file

@ -0,0 +1,284 @@
<?php
namespace App\Jobs\Arcadia;
use App\Enums\LogType;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Filesystem\Filesystem;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use Illuminate\Support\Carbon;
use Illuminate\Support\Facades\Log;
use Illuminate\Support\Facades\Storage;
abstract class XArcadiaFlux implements ShouldQueue
{
use Dispatchable;
use InteractsWithQueue;
use Queueable;
use SerializesModels;
/**
* The Laravel disk name
*
* @var string
*/
protected $disk = null;
/**
* This is for Laravel Console usage
* @var \Illuminate\Console\Command|null
*/
public $console = null;
/**
* This is for Laravel Console usage
* @var \Symfony\Component\Console\Helper\ProgressBar|null
*/
public $progressBar = null;
/**
* The relative folder where to archive data
*
* @var string
*/
private $archiveFolderName = 'archives';
/**
* The files to ignore
*
* @var array
*/
protected $ignoreFiles = [
'.gitignore', '.gitkeep'
];
/**
* Check if the file extension
*
* @param string $filename
* @param string $extension
* @return boolean
*/
protected function checkExtension(string $filename, string $extension): bool
{
$isExtension = strtolower(pathinfo($filename, \PATHINFO_EXTENSION)) === $extension;
if (!$isExtension) {
Log::warning(sprintf(
'%s : unwanted file type in %s wanted %s',
get_class($this),
$filename,
$extension
));
}
return $isExtension;
}
/**
* Move the file to archives
*
* @param string $filename
* @return void
*/
protected function moveToArchives(string $filename): void
{
$newFileName = $filename;
if (!$this->getDisk()->exists($filename)) {
$this->log(
sprintf(
'File %s could not be moved to archives (not found)',
$filename
),
LogType::warning()
);
return;
}
if ($this->getDisk()->exists(sprintf('%s/%s', $this->archiveFolderName, $filename))) {
$ext = \pathinfo($filename, \PATHINFO_EXTENSION);
$newFileName = \sprintf(
'%s_%d.%s',
\substr($filename, 0, 0 - (strlen($ext) + 1)),
\time(),
$ext
);
}
try {
if (!$this->getDisk()->move($filename, sprintf('%s/%s', $this->archiveFolderName, $newFileName))) {
$this->log(
sprintf(
'File %s could not be moved to archives',
$filename
),
LogType::warning()
);
}
} catch (\Exception $e) {
$this->log(
sprintf(
'File %s could not be moved to archives',
$filename
),
LogType::warning()
);
}
}
/**
* Suppression des fichiers dans
* archives qui ont plus de 2j
*
* @return void
*/
protected function clearOldArchives(): void
{
$disk = $this->getDisk();
$archives = $disk->files($this->archiveFolderName);
// /** @var File */
foreach ($archives as $archive) {
$pi = \pathinfo($archive, \PATHINFO_BASENAME);
if (in_array($pi, $this->ignoreFiles)) {
continue;
}
if ($disk->lastModified($archive) < strtotime(Carbon::now()->subDays(2))) {
if (!$disk->delete($archive)) {
Log::critical(sprintf('%s : Could not delete file %s', get_class($this), $archive));
};
}
}
}
/**
* Get the concerned disk
*
* @return Filesystem
*/
protected function getDisk(): Filesystem
{
return Storage::disk($this->disk);
}
/**
* Logs data
*
* @param string $message
* @param LogType $logtype
* @return void
* @throws \ErrorException If logtype is not handled.
*/
protected function log(string $message, LogType $logtype = null): void
{
if (!$logtype) {
$logtype = LogType::debug();
}
$message = sprintf('%s : %s', get_class($this), $message);
switch ($logtype->value) {
case LogType::debug()->value:
Log::debug($message);
break;
case LogType::info()->value:
Log::info($message);
break;
case LogType::log()->value:
Log::log($message);
break;
case LogType::warning()->value:
Log::warning($message);
break;
case LogType::error()->value:
Log::error($message);
break;
case LogType::critical()->value:
Log::critical($message);
break;
default:
throw new \ErrorException(sprintf('%s : unhandled log type %s', get_class($this), $logtype->label));
} //end switch
$this->consolePrint($message, $logtype);
}
/**
* Prints to the console
*
* @param string $message
* @param LogType $logtype
* @return void
* @throws \ErrorException If logtype is not handled.
*/
protected function consolePrint(string $message, LogType $logtype): void
{
$this->clearProgressBarFromLine();
if ($this->console) {
switch ($logtype->value) {
case LogType::debug()->value:
case LogType::info()->value:
case LogType::log()->value:
$this->console->info($message);
break;
case LogType::warning()->value:
$this->console->comment($message);
break;
case LogType::error()->value:
case LogType::critical()->value:
$this->console->error($message);
break;
default:
throw new \ErrorException(sprintf('%s : unhandled log type %s', get_class($this), $logtype->label));
}
}
}
/**
* Start ProgressBar
*
* @param integer $total
* @return void
*/
protected function startProgressBar(int $total)
{
if ($this->console) {
$this->progressBar = $this->console->getOutput()
->createProgressBar($total - 1);
$this->progressBar->start();
}
}
/**
* Clear the line to write someting
*
* @return void
*/
protected function clearProgressBarFromLine()
{
if ($this->progressBar) {
$this->progressBar->clear();
}
}
/**
* Increment 1 on progress bar
*
* @return void
*/
protected function incrementProgressBar()
{
if ($this->progressBar) {
$this->progressBar->advance();
}
}
/**
* End the progress bar
*
* @return void
*/
protected function endProgressBar()
{
if ($this->progressBar) {
$this->progressBar->finish();
}
}
}

View file

@ -0,0 +1,145 @@
<?php
namespace App\Jobs\Arcadia;
use App\Exceptions\AbortXmlImportException;
use App\Exceptions\ElasticSearchException;
use App\Lib\Utils;
use App\Models\ArcadiaLink;
use App\Models\Media;
use Illuminate\Database\Eloquent\ModelNotFoundException;
class XmlArcadiaMediasIn extends XmlJobs
{
/**
* The xsd file path used to validate files
*
* @var string|null
*/
protected $xsdSchemaPath = 'resources/documents/arcadia/incoming_medias.xsd';
/**
* Disk name from config/filesystems.php
*
* @var string
*/
protected $disk = 'arcadia_link_r';
/**
* The columns to take from the XML subcolumns can be added with a dot
*
* @var array
*/
protected $takeColumns = [
'code_DAM' => 'id',
'codeArcadia' => 'arcadia_id',
'version' => 'version'
];
/**
* The columns allowed to be null
*
* @var array
*/
protected $nullableFields = [];
/**
* Fields that are unique
*
* @var array
*/
protected $uniqueKeyField = [
'id'
];
/**
* Execute the job.
*
* @return void
*/
public function handle()
{
//phpcs:ignore Generic.CodeAnalysis.UnusedFunctionParameter.FoundInExtendedClassBeforeLastUsed
$this->parseData('MEDIA_VISUEL', Media::class, function (\SimpleXMLElement $xmlData, array &$line) {
// * After XML parse Callback.
// * Unlock media
$line['isLocked'] = 0;
}, function (array $uniqueColumnsKey, $line) {
// * DO the DB save yourself callback
try {
/** @var \App\Models\Media $media */
$media = Media::withTrashed()->where($uniqueColumnsKey)->firstOrFail();
$media->arcadia_id = $line['arcadia_id'];
$media->version = $line['version'];
$media->isLocked = 0;
$media->saveOrFail();
} catch (ModelNotFoundException $e) {
$e = new AbortXmlImportException(
sprintf('La fiche media \'%s\' ne peut être trouvé', \json_encode($uniqueColumnsKey)),
0,
1,
__FILE__,
__LINE__,
$e
);
throw $e;
}//end try
//phpcs:ignore Generic.CodeAnalysis.UnusedFunctionParameter.FoundInExtendedClassBeforeLastUsed
}, function (\SimpleXMLElement $xmlData, array &$line) {
// * After DB save Callback.
$media = null;
try {
/** @var \App\Models\Media $media */
$media = Media::withTrashed()->findOrFail($line['id']);
} catch (ModelNotFoundException $e) {
$e = new AbortXmlImportException(
"La fiche media '%s' ne peut être trouvé",
0,
1,
__FILE__,
__LINE__,
$e
);
throw $e;
}
try {
// * Suppression dans la supervision Arcadia Link
$arcadiaLink = ArcadiaLink::whereHas('media', function ($q) use ($media) {
$q->whereId($media->id);
})->first();
if ($arcadiaLink) {
$arcadiaLink->delete();
}
// * Index ElasticSearch
$esExist = Media::searchByQuery([
'match' => ['arcadia_id' => $media->arcadia_id ?? '']
]);
// * Remove all from elastic if any duplicates
if ($esExist->totalHits() > 0) {
$media->removeFromIndex();
}
// * Save into elastic
$media->addToIndex();
} catch (\Exception $e) {
$report = new ElasticSearchException(
'Something went wrong when updating media into elastic search',
0,
__FILE__,
__LINE__,
$e
);
// ! Someting went wrong
report($report);
// ! TODO: Remove this after adding kwaadpepper/exception-handler.
Utils::sendEmailErrorManually($report);
throw new AbortXmlImportException($report->getMessage());
} //end try
});
}
}

View file

@ -0,0 +1,467 @@
<?php
namespace App\Jobs\Arcadia;
use App\Enums\LogType;
use App\Exceptions\AbortXmlImportException;
use App\Exceptions\InvalidXmlException;
use App\Lib\Utils;
use Illuminate\Contracts\Filesystem\FileNotFoundException;
use Illuminate\Database\QueryException;
use Illuminate\Support\Facades\File;
use Illuminate\Support\Facades\Log;
use Illuminate\Support\Facades\Mail;
use Illuminate\Support\Str;
abstract class XmlJobs extends XArcadiaFlux
{
/**
* The xsd file path used to validate files
*
* @var string|null
*/
protected $xsdSchemaPath = null;
/**
* The xsd file content used to validate files
*
* @var string|null
*/
protected $xsdSchema = null;
/**
* invalid xml folder
*
* @var string
*/
protected $invalidFolderName = 'invalid';
/**
* The columns to take from the XML
* subcolumns can be added with a dot
*
* @var array
*/
protected $takeColumns = [];
/**
* The columns allowed to be null
*
* @var array
*/
protected $nullableFields = [];
/**
* Need to encapsulate the freeking XML
* Because provided one is dirty
*
* @var boolean
*/
protected $encapsulate = false;
/**
* Parse Data from the XML and gives it to the callback
*
* @param string $entityName
* @param string $modelClass
* @param callable $afterParseCallback After parse is done.
* @param callable $saveCallback Instead of updateOrCreate, execute $saveCallback.
* @param callable $afterSaveCallback After save is done.
* @return void
* @throws InvalidXmlException If xml file is not valid.
* @phpcs:disable Generic.Metrics.CyclomaticComplexity.MaxExceeded
*/
protected function parseData(
string $entityName,
string $modelClass,
callable $afterParseCallback = null,
callable $saveCallback = null,
callable $afterSaveCallback = null
): void {
/** @var Filesystem */
$disk = $this->getDisk();
$this->log('Démarrage du traitement');
$this->assertXsdIsHereIfAny();
$files = $disk->files();
$total = count(array_filter($files, function ($filename) {
if ($this->fileHasToBeIgnored($filename)) {
return false;
}
return true;
}));
$this->startProgressBar($total);
foreach ($files as $filename) {
if ($this->fileHasToBeIgnored($filename)) {
continue;
}
$this->incrementProgressBar();
try {
/** @var string $data */
$data = $disk->get($filename);
$this->validateXmlFile($filename, $data);
try {
if ($this->encapsulate) {
$data = sprintf('<Root>%s</Root>', $data);
/** @var \SimpleXMLElement|false */
$xmlContent = $data;
} else {
/** @var \SimpleXMLElement|false */
$xmlContent = simplexml_load_string($data, null, LIBXML_NOCDATA | LIBXML_PARSEHUGE);
}
if ($xmlContent === false) {
throw new InvalidXmlException('XML is Invalid (simplexml_load_string returned false)');
}
} catch (\ErrorException $e) {
throw new InvalidXmlException('XML is Invalid', 0, $e);
}
$data = $this->extract($xmlContent);
$lines = [];
// * Multiple Entities within file
if (isset($data[$entityName])) {
foreach ($data[$entityName] as $traca) {
$line = $this->processFields($traca);
if ($afterParseCallback) {
$afterParseCallback($xmlContent, $line);
}
$lines[] = $line;
}
} else {
// * Only one entity within file
$line = $this->processFields($data);
if ($afterParseCallback) {
$afterParseCallback($xmlContent, $line);
}
$lines[] = $line;
}
// * Mise en base des données
foreach ($lines as $line) {
/** @var array Unique Key columns */
$uniqueColumnsKey = [];
foreach ($this->uniqueKeyField as $uField) {
$uniqueColumnsKey[$uField] = $line[$uField];
}
if (count($uniqueColumnsKey)) {
if ($saveCallback) {
$saveCallback($uniqueColumnsKey, $line);
} else {
forward_static_call_array(
[$modelClass, 'updateOrCreate'],
[$uniqueColumnsKey, $line]
);
}
if ($afterSaveCallback) {
$afterSaveCallback($xmlContent, $line);
}
}
} //end foreach
} catch (InvalidXmlException $e) {
$message = $e->getMessage();
$this->log(
sprintf(
'File %s could not be parsed => %s on line %d',
$filename,
$message,
$e->getLine()
),
LogType::critical()
);
$this->sendInvalidXmlEmail($filename, $e);
$this->moveToInvalid($filename);
continue;
} catch (AbortXmlImportException $e) {
$this->log(
sprintf(
'ERROR: File %s could not be parsed => %s on line %d',
$filename,
$e->getMessage(),
$e->getLine()
),
LogType::error()
);
report($e);
// ! TODO: Remove this after adding kwaadpepper/exception-handler.
Utils::sendEmailErrorManually($e);
continue;
// * ignore the file
} catch (QueryException $e) {
$this->log(implode(' ', $e->errorInfo), LogType::critical());
report($e);
// ! TODO: Remove this after adding kwaadpepper/exception-handler.
Utils::sendEmailErrorManually($e);
continue;
} catch (FileNotFoundException $e) {
// This happens because of concurent xml jobs read.
$message = $e->getMessage();
$this->log(
sprintf(
'ERROR: File %s could not be read (not found) => %s on line %d',
$filename,
$message,
$e->getLine()
),
LogType::debug()
);
continue;
// * ignore the file
} catch (\Exception | \ErrorException | \TypeError $e) {
$this->log(
sprintf(
'File %s could not be parsed => %s on line %d',
$filename,
$e->getMessage(),
$e->getLine()
),
LogType::critical()
);
report($e);
// ! TODO: Remove this after adding kwaadpepper/exception-handler.
Utils::sendEmailErrorManually($e);
continue;
// * ignore the file
} //end try
$this->moveToArchives($filename);
} //end foreach
// Clear old archives.
$this->clearOldArchives();
$this->endProgressBar();
}
/**
* Recursive extract to array
*
* @param \SimpleXMLElement|false $sxe
* @return array
*/
protected function extract($sxe = null): array
{
if (!$sxe instanceof \SimpleXMLElement) {
return array();
}
$extract = array();
foreach ($sxe->children() as $key => $value) {
if ($value->count()) {
$extract[$key][] = $this->extract($value);
continue;
}
$value = strval($value);
if (array_key_exists($key, $extract) and is_array($extract[$key])) {
$extract[$key][] = strlen($value) ? $value : null;
} else {
$extract[$key] = strlen($value) ? $value : null;
}
}
return $extract;
}
/**
* Process fields to gather data
*
* @param array $fields
* @return array
* @throws AbortXmlImportException If a field could not be validated.
*/
protected function processFields(array $fields): array
{
$result = [];
foreach ($fields as $key => $field) {
if (!($key = $this->shouldFieldBeConsidered($key))) {
continue;
}
if (!$this->isFieldNullable($key) && is_null($field)) {
throw new AbortXmlImportException(sprintf('The field %s is not allowed to be null', $key));
}
if (is_array($field)) {
// * Flatten data
$it = new \RecursiveIteratorIterator(new \RecursiveArrayIterator($field));
foreach ($it as $k => $f) {
$ko = strtolower(sprintf('%s.%s', $key, $k));
if (!$this->isFieldNullable($ko) && is_null($f)) {
throw new AbortXmlImportException(sprintf(
'%s : The field %s is not allowed to be null',
get_class($this),
$ko
));
}
if (!($k = $this->shouldFieldBeConsidered($ko))) {
continue;
}
$result[$k] = $f;
}
} else {
$result[$key] = $field;
}
} //end foreach
return $result;
}
/**
* Is the filed nullable
*
* @param string $key
* @return boolean
*/
private function isFieldNullable(string $key): bool
{
if (in_array($key, $this->nullableFields)) {
return true;
}
return false;
}
/**
* - replace key name with the corresponding alias
* - strtolower the key
* - snake_case the key
*
* @param string $key
* @return string|false
*/
private function shouldFieldBeConsidered(string $key)
{
$key = strtolower($key);
foreach ($this->takeColumns as $k => $alias) {
if (strtolower($k) === $key) {
return Str::snake(strtolower($alias));
}
}
return false;
}
/**
* Is the file to be ignored
*
* @param string $filename
* @return boolean
*/
protected function fileHasToBeIgnored(string $filename): bool
{
return in_array($filename, $this->ignoreFiles) or !$this->checkExtension($filename, 'xml');
}
/**
* Check if the file is an xml file
*
* @param string $filename
* @return boolean
*/
protected function isFileXml(string $filename): bool
{
$isXml = strtolower(pathinfo($filename, \PATHINFO_EXTENSION)) === 'xml';
if (!$isXml) {
Log::warning(sprintf('%s : unwanted file type in %s', get_class($this), $filename));
}
return $isXml;
}
/**
* Send invalid XML email
*
* @param string $filename
* @param \Throwable $e
* @return void
*/
protected function sendInvalidXmlEmail(string $filename, \Throwable $e)
{
$message = $e->getPrevious()->getMessage();
Mail::send('mails.invalidxml', [
'strError' => sprintf(
'File %s could not be parsed => %s on line %d',
$filename,
$message,
$e->getLine()
),
'filename' => $filename,
'exception' => $e->getPrevious()
], function ($m) {
$m->to(
config('mail.contactsList.invalidXml'),
'XML Exception Handler'
)->subject('You sent me an invalid XML file, please check its validity before resend ASAP.');
});
}
/**
* Move the file to invalid folder
*
* @param string $filename
* @return void
*/
protected function moveToInvalid(string $filename): void
{
if (!$this->getDisk()->move($filename, sprintf('%s/%s', $this->invalidFolderName, $filename))) {
$this->log(
sprintf(
'File %s could not be moved to invalid folder name',
$filename
),
LogType::warning()
);
}
}
/**
* Assert XSD is readable if configured
*
* @return void
* @throws \Illuminate\Contracts\Filesystem\FileNotFoundException If xsd file cant be read.
*/
private function assertXsdIsHereIfAny()
{
if ($this->xsdSchemaPath) {
$this->xsdSchema = File::get($this->xsdSchemaPath);
}
}
/**
* Validate Xml file against XSD if loaded
*
* @param string $fileName
* @param string $fileContent
* @return void
* @throws InvalidXmlException If XML could not be parsed or validated against XSD.
* @phpcs:disable Squiz.Commenting.FunctionCommentThrowTag.WrongNumber
*/
private function validateXmlFile(string $fileName, string $fileContent)
{
if ($this->xsdSchema) {
$domDocument = new \DOMDocument('1.0', 'UTF-8');
try {
if (!$domDocument->loadXML($fileContent)) {
throw new InvalidXmlException("XML Validation $fileName content could not be loaded to validate.");
}
if (!$domDocument->schemaValidateSource($this->xsdSchema, LIBXML_SCHEMA_CREATE)) {
throw new InvalidXmlException(
"XML Validation $fileName content could not be validated against xsd."
);
}
} catch (\Exception $e) {
if (\get_class($e) === InvalidXmlException::class) {
throw $e;
}
throw new InvalidXmlException(
"XML Validation $fileName content could not be validated against xsd.",
0,
1,
__FILE__,
__LINE__,
$e
);
} //end try
}//end if
}
}

129
app/Lib/Utils.php Normal file
View file

@ -0,0 +1,129 @@
<?php
namespace App\Lib;
use Illuminate\Support\Facades\Log;
use Illuminate\Support\Facades\Mail;
/**
* Some utilities methods
*
* @author Jérémy Munsch <jeremy.munsch@ldc.fr>
*/
class Utils
{
/**
* Manually send an exception to team instead of going trough the error handler
* (requires package kwaadpepper/exception-handler)
*
* @param \Throwable $e
* @return void
*/
public static function sendEmailErrorManually(\Throwable $e)
{
Mail::send([
'html' => 'exception-handler.exception',
'text' => 'exception-handler.exception-plain'
], [
'appName' => config('app.name'),
'code' => $e->getCode(),
'line' => $e->getLine(),
'ename' => \get_class($e),
'emessage' => self::getAnonymizedMessage($e->getMessage()),
'file' => str_replace(\realpath(\sprintf('%s/../', \app_path())), '', $e->getFile()),
'stackTrace' => self::getAnonymizedStackTrace($e),
'uri' => app()->runningInConsole() ?
'CONSOLE' : (
(request() and request()->fullUrl()) ?
request()->fullUrl() : 'none')
], function ($m) {
/** @var \Illuminate\Mail\Message $m */
$m->to(
config('mail.contactsList.serviceweb'),
sprintf('%s Exception Handler', config('app.name'))
)->subject('Need help !, an error occured in your application, please check Logs ASAP');
});
Log::alert(sprintf('Envoie d\'une erreur par mail : %s', json_encode($e)));
}
/**
* Anonymize at least Sql queries
*
* @param string $message
* @return string
*/
private static function getAnonymizedMessage(string $message): string
{
$message = \preg_replace('/^(.*)( \(SQL:.*\))$/', '$1', $message) ?? $message;
return $message;
}
/**
* Anonymize stack trace
*
* @param \Exception $e
* @return string
*/
private static function getAnonymizedStackTrace(\Throwable $e): string
{
$projectPath = \realpath(\sprintf('%s/../', \app_path()));
$traceStack = $e->getTrace();
$traceStr = '';
$maxStackStage = 0;
foreach ($traceStack as $stackStage => $stack) {
$maxStackStage = $stackStage;
$traceStr .= \sprintf(
'#%s %s(%d): ',
$stackStage,
str_replace($projectPath, '', $stack['file'] ?? ''),
$stack['line'] ?? ''
);
if (isset($stack['class'])) {
$traceStr .= "{$stack['class']}->{$stack['function']}(";
if (isset($stack['args'])) {
$traceStr .= self::handleArgs($stack);
}
$traceStr .= ")\n";
} elseif (isset($stack['function'])) {
$traceStr .= "{$stack['function']}(";
if (isset($stack['args'])) {
$traceStr .= self::handleArgs($stack);
}
$traceStr .= ")\n";
}
}
$maxStackStage++;
$traceStr .= "#$maxStackStage {main}";
return $traceStr;
}
/**
* Handle Args
*
* @param array $stack
* @return string
*/
private static function handleArgs(array $stack): string
{
$traceArgs = '';
$argsCount = count($stack['args']);
for ($i = 0; $i < $argsCount; $i++) {
$type = \gettype($stack['args'][$i]);
switch ($type) {
case 'object':
$traceArgs .= sprintf('Object(%s)', \get_class($stack['args'][$i]));
break;
case 'array':
$traceArgs .= sprintf('Array(%d)', \count($stack['args'][$i]));
break;
case 'string':
$traceArgs .= sprintf('String(%d)', \strlen($stack['args'][$i]));
break;
default:
$traceArgs .= \gettype($stack['args'][$i]);
}
$traceArgs .= ($i !== ($argsCount - 1) ? ',' : '');
}
return $traceArgs;
}
}

View file

@ -7,15 +7,46 @@ use Carbon\Carbon;
class ArcadiaLink extends Model
{
/**
* *The table associated with the model.
*
* @var string
*/
protected $table = 'arcadia_link';
public function getDateFAttribute() {
// * GETTERS
/**
* Get dateF attribute
*
* @return string
* @phpcs:disable Generic.NamingConventions.CamelCapsFunctionName.ScopeNotCamelCaps
*/
public function getDateFAttribute()
{
return Carbon::createFromFormat('Y-m-d H:i:s', $this->created_at)->format('d/m/Y H:i:s');
}
public function getMediaUuidAttribute() {
/**
* Get mediaUuid attribute
*
* @return string|null
*/
public function getMediaUuidAttribute()
{
$m = Media::find($this->media_id);
return $m->uid;
}
// * RELATIONSHIPS
/**
* BelongsTo media relation
*
* @return \Illuminate\Database\Eloquent\Relations\BelongsTo
*/
public function media()
{
return $this->belongsTo(Media::class);
}
}

View file

@ -31,6 +31,16 @@ class Media extends Model
'typemedia', 'societes', 'marques'
];
/**
* The attributes that are mass assignable.
*
* @var array
*/
protected $fillable = [
// ! id est obligatoire ici pour Jobs/XmlArcadiaMediasIn
'id'
];
/**
* ES The elasticsearch settings.
*

View file

@ -14,6 +14,7 @@
"elasticquent/elasticquent": "dev-master",
"fideloper/proxy": "^4.0",
"james-heinrich/getid3": "^1.9",
"kwaadpepper/enum": "^2.1",
"laravel/framework": "5.8.*",
"laravel/tinker": "^1.0",
"laravelcollective/html": "^5.8",

62
composer.lock generated
View file

@ -4,7 +4,7 @@
"Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
"This file is @generated automatically"
],
"content-hash": "906079f4dfcfc034b2c3617a0fc16ee9",
"content-hash": "68a7a8d32fb9300acb3a4735c3bb1580",
"packages": [
{
"name": "adldap2/adldap2",
@ -934,6 +934,66 @@
],
"time": "2019-02-07T17:30:03+00:00"
},
{
"name": "kwaadpepper/enum",
"version": "2.1.2",
"source": {
"type": "git",
"url": "https://github.com/Kwaadpepper/enum.git",
"reference": "bbd08723895cad5a41bc0d57e565fe64fa13ca41"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/Kwaadpepper/enum/zipball/bbd08723895cad5a41bc0d57e565fe64fa13ca41",
"reference": "bbd08723895cad5a41bc0d57e565fe64fa13ca41",
"shasum": ""
},
"require": {
"php": "^7.3"
},
"require-dev": {
"orchestra/testbench": "^5",
"phpunit/php-code-coverage": "^9.2",
"phpunit/phpunit": "~9.0",
"slevomat/coding-standard": "^7.0",
"squizlabs/php_codesniffer": "^3.5"
},
"type": "library",
"extra": {
"laravel": {
"providers": [
"Kwaadpepper\\Enum\\EnumServiceProvider"
]
}
},
"autoload": {
"psr-4": {
"Kwaadpepper\\Enum\\": "src/"
}
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
"authors": [
{
"name": "Jérémy Munsch",
"email": "github@jeremydev.ovh",
"homepage": "https://jeremydev.ovh"
}
],
"description": "Enums to use for any project",
"homepage": "https://github.com/kwaadpepper/enum",
"keywords": [
"enum",
"laravel"
],
"support": {
"issues": "https://github.com/Kwaadpepper/enum/issues",
"source": "https://github.com/Kwaadpepper/enum/tree/2.1.2"
},
"time": "2021-11-18T16:52:08+00:00"
},
{
"name": "laravel/framework",
"version": "v5.8.30",

View file

@ -3,56 +3,56 @@
return [
/*
|--------------------------------------------------------------------------
| Mail Driver
|--------------------------------------------------------------------------
|
| Laravel supports both SMTP and PHP's "mail" function as drivers for the
| sending of e-mail. You may specify which one you're using throughout
| your application here. By default, Laravel is setup for SMTP mail.
|
| Supported: "smtp", "sendmail", "mailgun", "mandrill", "ses",
| "sparkpost", "postmark", "log", "array"
|
|--------------------------------------------------------------------------
| Mail Driver
|--------------------------------------------------------------------------
|
| Laravel supports both SMTP and PHP's "mail" function as drivers for the
| sending of e-mail. You may specify which one you're using throughout
| your application here. By default, Laravel is setup for SMTP mail.
|
| Supported: "smtp", "sendmail", "mailgun", "ses",
| "postmark", "log", "array"
|
*/
'driver' => env('MAIL_DRIVER', 'smtp'),
'driver' => env('MAIL_MAILER', 'smtp'),
/*
|--------------------------------------------------------------------------
| SMTP Host Address
|--------------------------------------------------------------------------
|
| Here you may provide the host address of the SMTP server used by your
| applications. A default option is provided that is compatible with
| the Mailgun mail service which will provide reliable deliveries.
|
|--------------------------------------------------------------------------
| SMTP Host Address
|--------------------------------------------------------------------------
|
| Here you may provide the host address of the SMTP server used by your
| applications. A default option is provided that is compatible with
| the Mailgun mail service which will provide reliable deliveries.
|
*/
'host' => env('MAIL_HOST', 'smtp.mailgun.org'),
/*
|--------------------------------------------------------------------------
| SMTP Host Port
|--------------------------------------------------------------------------
|
| This is the SMTP port used by your application to deliver e-mails to
| users of the application. Like the host we have set this value to
| stay compatible with the Mailgun e-mail application by default.
|
|--------------------------------------------------------------------------
| SMTP Host Port
|--------------------------------------------------------------------------
|
| This is the SMTP port used by your application to deliver e-mails to
| users of the application. Like the host we have set this value to
| stay compatible with the Mailgun e-mail application by default.
|
*/
'port' => env('MAIL_PORT', 587),
/*
|--------------------------------------------------------------------------
| Global "From" Address
|--------------------------------------------------------------------------
|
| You may wish for all e-mails sent by your application to be sent from
| the same address. Here, you may specify a name and address that is
| used globally for all e-mails that are sent by your application.
|
|--------------------------------------------------------------------------
| Global "From" Address
|--------------------------------------------------------------------------
|
| You may wish for all e-mails sent by your application to be sent from
| the same address. Here, you may specify a name and address that is
| used globally for all e-mails that are sent by your application.
|
*/
'from' => [
@ -61,27 +61,27 @@ return [
],
/*
|--------------------------------------------------------------------------
| E-Mail Encryption Protocol
|--------------------------------------------------------------------------
|
| Here you may specify the encryption protocol that should be used when
| the application send e-mail messages. A sensible default using the
| transport layer security protocol should provide great security.
|
|--------------------------------------------------------------------------
| E-Mail Encryption Protocol
|--------------------------------------------------------------------------
|
| Here you may specify the encryption protocol that should be used when
| the application send e-mail messages. A sensible default using the
| transport layer security protocol should provide great security.
|
*/
'encryption' => env('MAIL_ENCRYPTION', 'tls'),
/*
|--------------------------------------------------------------------------
| SMTP Server Username
|--------------------------------------------------------------------------
|
| If your SMTP server requires a username for authentication, you should
| set it here. This will get used to authenticate with your server on
| connection. You may also set the "password" value below this one.
|
|--------------------------------------------------------------------------
| SMTP Server Username
|--------------------------------------------------------------------------
|
| If your SMTP server requires a username for authentication, you should
| set it here. This will get used to authenticate with your server on
| connection. You may also set the "password" value below this one.
|
*/
'username' => env('MAIL_USERNAME'),
@ -89,27 +89,27 @@ return [
'password' => env('MAIL_PASSWORD'),
/*
|--------------------------------------------------------------------------
| Sendmail System Path
|--------------------------------------------------------------------------
|
| When using the "sendmail" driver to send e-mails, we will need to know
| the path to where Sendmail lives on this server. A default path has
| been provided here, which will work well on most of your systems.
|
|--------------------------------------------------------------------------
| Sendmail System Path
|--------------------------------------------------------------------------
|
| When using the "sendmail" driver to send e-mails, we will need to know
| the path to where Sendmail lives on this server. A default path has
| been provided here, which will work well on most of your systems.
|
*/
'sendmail' => '/usr/sbin/sendmail -bs',
/*
|--------------------------------------------------------------------------
| Markdown Mail Settings
|--------------------------------------------------------------------------
|
| If you are using Markdown based email rendering, you may configure your
| theme and component paths here, allowing you to customize the design
| of the emails. Or, you may simply stick with the Laravel defaults!
|
|--------------------------------------------------------------------------
| Markdown Mail Settings
|--------------------------------------------------------------------------
|
| If you are using Markdown based email rendering, you may configure your
| theme and component paths here, allowing you to customize the design
| of the emails. Or, you may simply stick with the Laravel defaults!
|
*/
'markdown' => [
@ -121,16 +121,22 @@ return [
],
/*
|--------------------------------------------------------------------------
| Log Channel
|--------------------------------------------------------------------------
|
| If you are using the "log" driver, you may specify the logging channel
| if you prefer to keep mail messages separate from other log entries
| for simpler reading. Otherwise, the default channel will be used.
|
|--------------------------------------------------------------------------
| Log Channel
|--------------------------------------------------------------------------
|
| If you are using the "log" driver, you may specify the logging channel
| if you prefer to keep mail messages separate from other log entries
| for simpler reading. Otherwise, the default channel will be used.
|
*/
'log_channel' => env('MAIL_LOG_CHANNEL'),
// * Contacts List
'contactsList' => [
'invalidXml' => env('MAIL_LIST_SERVICEWEB', 'equipe.web@ldc.fr'),
'serviceweb' => env('MAIL_LIST_SERVICEWEB', 'equipe.web@ldc.fr')
],
];

View file

@ -0,0 +1,20 @@
<?xml version="1.0" encoding="UTF-8"?>
<!-- < XSD deviné, basé sur la BDD et cette seule affirmation "les 3 champs sont obligatoires" @author Jérémy Munsch jeremy.munsch@ldc.fr > -->
<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema" elementFormDefault="qualified" attributeFormDefault="unqualified">
<xs:element name="MEDIA_VISUEL">
<xs:complexType>
<xs:sequence>
<xs:element name="codeArcadia" type="_positive_unsigned_integer_without_leading_zeros" />
<xs:element name="code_DAM" type="_positive_unsigned_integer_without_leading_zeros"/>
<xs:element name="version" type="_positive_unsigned_integer_without_leading_zeros"/>
</xs:sequence>
</xs:complexType>
</xs:element>
<xs:simpleType name="_positive_unsigned_integer_without_leading_zeros">
<xs:restriction base="xs:positiveInteger">
<xs:pattern value="[123456789]\d*"/>
</xs:restriction>
</xs:simpleType>
</xs:schema>

10
resources/lang/en.json Normal file
View file

@ -0,0 +1,10 @@
{
"error_occured_in_app_check_trace": "An error occured on :appName",
"check_trace": "Please check logs in order to fix this.",
"uri": "uri",
"date": "date",
"code": "code",
"file": "file",
"line": "line",
"anonymized_stack_trace": "anonymized stack trace"
}

10
resources/lang/fr.json Normal file
View file

@ -0,0 +1,10 @@
{
"error_occured_in_app": "Une erreur est survenue sur :appName",
"check_trace": "Veuillez vérifier les logs pour réparer ceci.",
"uri": "uri",
"date": "date",
"code": "code",
"file": "fichier",
"line": "ligne",
"anonymized_stack_trace": "pile d'appels anonymisée"
}

View file

@ -0,0 +1,17 @@
{{ $appName }} Exception Handler
{{ __('error_occured_in_app_check_trace', [
'appName' => $appName,
]) }}
{{ __('check_trace') }}
{{ $ename }}
{{ $emessage }}
{{ __('uri') }} : {{ $uri }}
{{ __('date') }} : {{ date('Y-m-d H:i:s') }}
{{ __('code') }} : {{ $code }}
{{ __('file') }} : {{ $file }}
{{ __('line') }} : {{ $line }}
{{ __('anonymized_stack_trace') }} :
{!! nl2br($stackTrace) !!}

View file

@ -0,0 +1,25 @@
<!DOCTYPE html
PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<title>{{ $appName }} Exception Handler</title>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
</head>
<body>
<h1>{{ __('error_occured_in_app', ['appName' => $appName]) }}</h1>
<p>{{ __('check_trace') }}</p>
<h2>{{ $ename }}</h2>
<h3>{{ $emessage }}</h3>
<p>{{ __('uri') }} : {{ $uri }}</p>
<p>{{ __('date') }} : {{ date('Y-m-d H:i:s') }}</p>
<p>{{ __('code') }} : {{ $code }}</p>
<p>{{ __('file') }} : {{ $file }}</p>
<p>{{ __('line') }} : {{ $line }}</p><br>
<p>{{ __('anonymized_stack_trace') }} :</p>
<p>{!! nl2br($stackTrace) !!}</p>
</body>
</html>

View file

@ -0,0 +1,28 @@
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<title>XML Exception Handler: Unvalid XML</title>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
</head>
<body>
<h1>{{ config('app.name') }} à reçu un fichier xml non valide !!</h1>
<h3>{{ $strError }}</h3>
<h4>FICHIER: `{{ $filename }}`</h4>
<p>{{ $exception->getMessage() }}</p>
<p>
<b>date</b> : {{ date('Y-m-d H:i:s') }}<br>
<b>code</b> : {{$exception->getCode() }}<br>
<b>file</b> : {{$exception->getFile() }}<br>
<b>line</b> : {{$exception->getLine() }}<br>
<p>
<b>Merci de corriger le flux, puis de renvoyer le fichier en provoquant un mouvement EAI.</b><br><br>
<b>Pour rappel</b>&nbsp;: Il est recommandé de valider un fichier XML à l'aide d'un <b>DTD</b><br>
<a href="https://www.w3schools.com/xml/xml_dtd.asp">https://www.w3schools.com/xml/xml_dtd.asp</a><br>
ou bien <b>XSD</b> <a href="https://www.w3schools.com/xml/schema_intro.asp">https://www.w3schools.com/xml/schema_intro.asp</a><br><br>
<b>Pourquoi&nbsp;?</b><br>
Pour que les concepteurs puissent identifier les types de données qui transitent (les champs et leurs caratéristiques)<br>
Mais aussi valider les fichiers reçus.
</p>
<p>En l'absence de données valides, cette fiche va être mise de coté.</p>
</body>
</html>