From dcfb3f14bbdc964d2e154c079cb27af5bf556c3e Mon Sep 17 00:00:00 2001
From: Olivier Legat <olivier.legat@ldc.fr>
Date: Sat, 2 Apr 2022 10:27:41 +0000
Subject: [PATCH] Update xml jobs helpers added Schema validation step adapt
 code to new substructure code/libelle => keep libelle only added nom field to
 DB and data extraction added changelog.md

---
 app/Jobs/XArcadiaFlux.php                     |  48 ++--
 app/Jobs/XmlIncoReader.php                    | 106 +++++++--
 app/Jobs/XmlJobs.php                          | 101 ++++++---
 app/Lib/Utils.php                             | 210 ++++++++++++++++++
 app/Models/XmlInco.php                        | 135 -----------
 changelog.md                                  |  12 +
 ...21_11_19_102629_create_xml_incos_table.php |   2 +-
 7 files changed, 408 insertions(+), 206 deletions(-)
 create mode 100644 app/Lib/Utils.php
 create mode 100644 changelog.md

diff --git a/app/Jobs/XArcadiaFlux.php b/app/Jobs/XArcadiaFlux.php
index 89dabef..56818a0 100644
--- a/app/Jobs/XArcadiaFlux.php
+++ b/app/Jobs/XArcadiaFlux.php
@@ -77,21 +77,37 @@ abstract class XArcadiaFlux implements ShouldQueue
      */
     protected function moveToArchives(string $filename): void
     {
-        $archiveFile = sprintf('%s/%s', $this->archiveFolderName, $filename);
-        if ($this->getDisk()->exists($archiveFile)) {
+        $newFileName = $filename;
+        if (!$this->getDisk()->exists($filename)) {
             $this->log(
                 sprintf(
-                    'File %s exists, and therefore must be delete before archive',
-                    $archiveFile
+                    'File %s could not be moved to archives (not found)',
+                    $filename
                 ),
-                LogType::info()
+                LogType::warning()
             );
-            if (!$this->getDisk()->delete($archiveFile)) {
-                Log::critical(sprintf('%s : Could not delete file %s', get_class($this), $archiveFile));
-            };
+            return;
         }
-
-        if (!$this->getDisk()->move($filename, sprintf('%s/%s', $this->archiveFolderName, $filename))) {
+        if ($this->getDisk()->exists(sprintf('%s/%s', $this->archiveFolderName, $filename))) {
+            $ext         = \pathinfo($filename, \PATHINFO_EXTENSION);
+            $newFileName = \sprintf(
+                '%s_%d.%s',
+                \substr($filename, 0, 0 - (strlen($ext) + 1)),
+                \time(),
+                $ext
+            );
+        }
+        try {
+            if (!$this->getDisk()->move($filename, sprintf('%s/%s', $this->archiveFolderName, $newFileName))) {
+                $this->log(
+                    sprintf(
+                        'File %s could not be moved to archives',
+                        $filename
+                    ),
+                    LogType::warning()
+                );
+            }
+        } catch (Exception $e) {
             $this->log(
                 sprintf(
                     'File %s could not be moved to archives',
@@ -104,13 +120,13 @@ abstract class XArcadiaFlux implements ShouldQueue
 
     /**
      * Suppression des fichiers dans
-     * archives qui ont plus de 30j
+     * archives qui ont plus de 2j
      *
      * @return void
      */
     protected function clearOldArchives(): void
     {
-        $disk = $this->getDisk();
+        $disk     = $this->getDisk();
         $archives = $disk->files($this->archiveFolderName);
 
         // /** @var File */
@@ -120,7 +136,7 @@ abstract class XArcadiaFlux implements ShouldQueue
                 continue;
             }
 
-            if ($disk->lastModified($archive) < strtotime(Carbon::now()->subDays(30))) {
+            if ($disk->lastModified($archive) < strtotime(Carbon::now()->subDays(2))) {
                 if (!$disk->delete($archive)) {
                     Log::critical(sprintf('%s : Could not delete file %s', get_class($this), $archive));
                 };
@@ -141,7 +157,7 @@ abstract class XArcadiaFlux implements ShouldQueue
     /**
      * Logs data
      *
-     * @param string $message
+     * @param string  $message
      * @param LogType $logtype
      * @return void
      */
@@ -172,14 +188,14 @@ abstract class XArcadiaFlux implements ShouldQueue
                 break;
             default:
                 throw new Exception(sprintf('%s : unhandled log type %s', get_class($this), $logtype->label));
-        }
+        }//end switch
         $this->consolePrint($message, $logtype);
     }
 
     /**
      * Prints to the console
      *
-     * @param string $message
+     * @param string  $message
      * @param LogType $logtype
      * @return void
      */
diff --git a/app/Jobs/XmlIncoReader.php b/app/Jobs/XmlIncoReader.php
index 3a2998e..b85d311 100644
--- a/app/Jobs/XmlIncoReader.php
+++ b/app/Jobs/XmlIncoReader.php
@@ -13,6 +13,8 @@ class XmlIncoReader extends XmlJobs
 
     protected $imagesPath = 'images';
 
+    protected $xsdFile = 'xsd/incollector.xsd';
+
     /**
      * éléments à la racine : nom du noeud
      * sous éléments d'un noeud PARENT :
@@ -61,6 +63,7 @@ class XmlIncoReader extends XmlJobs
 
 
         'INCO_COMPO' => 'inco_compo', // Consider this sub array
+        'INCO_COMPO.NOM' => 'nom',
         'INCO_COMPO.TAILLE_REF' => 'taille_ref',
         'INCO_COMPO.VAL_ENERG_KJ_NUT' => 'val_energ_kj_nut',
         'INCO_COMPO.VAL_ENERG_KJ_R' => 'val_energ_kj_r',  //_R apports de la taille_ref
@@ -168,7 +171,7 @@ class XmlIncoReader extends XmlJobs
         'allegations',
         'astuces', 'arg_vente',
         'pds_moy_theo',
-        'lib_conso', 'lst_ing',
+        'lib_conso', 'lst_ing', 'lst_allergene',
         'allegat_sante', 'allegat_nutrit', 'allegat_autre',
         'inco_compo',
         'inco_compo.nom',
@@ -432,32 +435,89 @@ class XmlIncoReader extends XmlJobs
                 $disk = $this->getDisk();
 
                 $info = getimagesizefromstring($imageData);
-                $mime = $info['mime'];
-                switch ($mime) {
-                    case 'image/jpeg':
-                        $imgExtension = 'jpg';
-                        break;
-                    case 'image/png':
-                        $imgExtension = 'png';
-                        break;
-                    case 'image/gif':
-                        $imgExtension = 'gif';
-                        break;
-                    default:
-                        throw new AbortXmlImportException(sprintf('Unknown image type %s', $imgName));
+                if($info !== false) {
+                    $mime = $info['mime'];
+                    switch ($mime) {
+                        case 'image/jpeg':
+                            $imgExtension = 'jpg';
+                            break;
+                        case 'image/png':
+                            $imgExtension = 'png';
+                            break;
+                        case 'image/gif':
+                            $imgExtension = 'gif';
+                            break;
+                        default:
+                            throw new AbortXmlImportException(sprintf('Unknown image type %s', $imgName));
+                    }
+
+                    $imgNamePath = sprintf('%s.%s', $imgNamePath, $imgExtension);
+
+                    if (!$disk->put($imgNamePath, $imageData)) {
+                        throw new AbortXmlImportException(sprintf('The image %s could not be saved', $imgNamePath));
+                    }
+
+                    $arImgPaths[$index] = sprintf('%s/%s', $this->disk, $imgNamePath);
+                    $index++;
+                } else {
+                    //probably not an image could be a PDF, PPT, AI anything is possible
                 }
-
-                $imgNamePath = sprintf('%s.%s', $imgNamePath, $imgExtension);
-
-                if (!$disk->put($imgNamePath, $imageData)) {
-                    throw new AbortXmlImportException(sprintf('The image %s could not be saved', $imgNamePath));
-                }
-
-                $arImgPaths[$index] = sprintf('%s/%s', $this->disk, $imgNamePath);
-                $index++;
             }
         }
 
         return $arImgPaths;
     }
+
+    /**
+     * Recursive extract to array
+     * introduce sort of Hook to transform
+     *           <TYP_PORTION>
+     *           <code><![CDATA[2]]></code>
+     *           <libelle><![CDATA[Approximatif]]></libelle>
+     *           </TYP_PORTION>
+     *           extracted as "TYP_PORTION" => array:1 [
+     *           0 => array:2 [
+     *               "code" => "2"
+     *               "libelle" => "Approximatif"
+     *             ]
+     *           ]
+     *           into this
+     *           <TYP_PORTION><![CDATA[Approximatif]]></TYP_PORTION>
+     *           extracted as "TYP_PORTION" => "Approximatif"
+     *
+     *
+     * @param \SimpleXMLElement|false $sxe
+     * @return array
+     */
+    protected function extract($sxe = null): array
+    {
+
+        if (!$sxe instanceof SimpleXMLElement) {
+            return array();
+        }
+        $extract = array();
+        foreach ($sxe->children() as $key => $value) {
+            if ($value->count()) {
+                $extraction = $this->extract($value);
+
+                if(count($extraction)
+                && array_key_exists("code",$extraction)
+                && array_key_exists("libelle",$extraction)) {
+                    $val = strval($extraction['libelle']);
+                    $extract[$key] = strlen($val) ? $val : null;
+                    continue;
+                }
+
+                $extract[$key][] = $extraction; //$this->extract($value);
+                continue;
+            }
+            $value = strval($value);
+            if (array_key_exists($key, $extract) and is_array($extract[$key])) {
+                $extract[$key][] = strlen($value) ? $value : null;
+            } else {
+                $extract[$key] = strlen($value) ? $value : null;
+            }
+        }
+        return $extract;
+    }
 }
diff --git a/app/Jobs/XmlJobs.php b/app/Jobs/XmlJobs.php
index d87ef6d..f54ef0e 100644
--- a/app/Jobs/XmlJobs.php
+++ b/app/Jobs/XmlJobs.php
@@ -5,9 +5,9 @@ namespace App\Jobs;
 use App\Enums\LogType;
 use App\Exceptions\AbortXmlImportException;
 use App\Exceptions\InvalidXmlException;
+use App\Lib\Utils;
 use ErrorException;
 use Illuminate\Contracts\Filesystem\FileNotFoundException;
-use Illuminate\Contracts\Filesystem\Filesystem;
 use Illuminate\Database\QueryException;
 use Illuminate\Support\Facades\Log;
 use Illuminate\Support\Facades\Mail;
@@ -20,7 +20,6 @@ use TypeError;
 
 abstract class XmlJobs extends XArcadiaFlux
 {
-
     /**
      * invalid xml folder
      *
@@ -28,6 +27,13 @@ abstract class XmlJobs extends XArcadiaFlux
      */
     protected $invalidFolderName = 'invalid';
 
+    /**
+     * xsd file location
+     *
+     * @var string
+     */
+    protected $xsdFile = '';
+
     /**
      * The columns to take from the XML
      * subcolumns can be added with a dot
@@ -67,7 +73,7 @@ abstract class XmlJobs extends XArcadiaFlux
         $files = $disk->files();
 
         $progress = 1;
-        $total = count(array_filter($files, function ($filename) {
+        $total    = count(array_filter($files, function ($filename) {
             if ($this->fileHasToBeIgnored($filename)) {
                 return false;
             }
@@ -80,11 +86,11 @@ abstract class XmlJobs extends XArcadiaFlux
             }
 
             $this->log(sprintf(' traitement %d sur %s ', $progress++, $total));
-
             try {
                 /** @var string $data */
                 $data = $disk->get($filename);
                 try {
+                    $this->validateSchema($data);
                     if ($this->encapsulate) {
                         $data = sprintf('<Root>%s</Root>', $data);
                         /** @var SimpleXMLElement|false */
@@ -93,14 +99,8 @@ abstract class XmlJobs extends XArcadiaFlux
                         /** @var SimpleXMLElement|false */
                         $xmlContent = simplexml_load_string($data, null, LIBXML_NOCDATA | LIBXML_PARSEHUGE);
                     }
-
                     if ($xmlContent === false) {
                         throw new InvalidXmlException('XML is Invalid (simplexml_load_string returned false)');
-                        // $this->log(
-                        //     sprintf('File %s could not be parsed (simplexml_load_string)', $filename),
-                        //     LogType::critical()
-                        // );
-                        // continue; // ignore the file
                     }
                 } catch (ErrorException $e) {
                     throw new InvalidXmlException('XML is Invalid', 0, $e);
@@ -124,12 +124,10 @@ abstract class XmlJobs extends XArcadiaFlux
                     foreach ($this->uniqueKeyField as $uField) {
                         $uniqueColumnsKey[$uField] = $line[$uField];
                     }
-                    $savedData = forward_static_call_array(
+                    forward_static_call_array(
                         [$modelClass, 'updateOrCreate'],
                         [$uniqueColumnsKey, $line]
                     );
-                    // @todo : should we put that method in a interface ?
-                    $savedData->updateLocalProductInfo();
                 }
             } catch (InvalidXmlException $e) {
                 $message = $e->getMessage();
@@ -159,7 +157,8 @@ abstract class XmlJobs extends XArcadiaFlux
                     LogType::critical()
                 );
                 $this->sendErrorParseEmailToServiceWeb($e);
-                continue; // ignore the file
+                continue;
+// ignore the file
             } catch (AbortXmlImportException $e) {
                 $this->log(
                     sprintf(
@@ -171,8 +170,8 @@ abstract class XmlJobs extends XArcadiaFlux
                     LogType::error()
                 );
                 $this->sendErrorParseEmailToServiceWeb($e);
-                $this->moveToInvalid($filename);
-                continue; // ignore the file
+                continue;
+// ignore the file
             } catch (FileNotFoundException | QueryException $e) {
                 // This happens because of concurent xml jobs read
                 $message = $e->getMessage();
@@ -188,10 +187,11 @@ abstract class XmlJobs extends XArcadiaFlux
                     ),
                     LogType::debug()
                 );
-                continue; // ignore the file
-            }
+                continue;
+// ignore the file
+            }//end try
             $this->moveToArchives($filename);
-        }
+        }//end foreach
 
         // Clear old archives
         $this->clearOldArchives();
@@ -261,7 +261,7 @@ abstract class XmlJobs extends XArcadiaFlux
             } else {
                 $result[$key] = $field;
             }
-        }
+        }//end foreach
         return $result;
     }
 
@@ -309,6 +309,23 @@ abstract class XmlJobs extends XArcadiaFlux
         return in_array($filename, $this->ignoreFiles) or !$this->checkExtension($filename, 'xml');
     }
 
+    /**
+     * Validate xml data against xsd file if provided
+     *
+     * @return void
+     */
+    protected function validateSchema($data) {
+        if($this->xsdFile) {
+            //check against XSD
+            $xsd = storage_path($this->disk."/".$this->xsdFile);
+            $domDocument= new \DOMDocument();
+            $domDocument->loadXML($data);
+            //free some memroy to avoid overload
+            unset($data);
+            $domDocument->schemaValidate($xsd);
+        }
+    }
+
     /**
      * Check if the file is an xml file
      *
@@ -326,15 +343,7 @@ abstract class XmlJobs extends XArcadiaFlux
 
     protected function sendErrorParseEmailToServiceWeb(Throwable $e)
     {
-        Mail::send('emails.exception', [
-            'exception' => $e,
-            'appName' => config('app.name'),
-        ], function ($m) {
-            $m->to(
-                config('mail.contactsList.serviceweb'),
-                sprintf('%s Exception Handler', config('app.name'))
-            )->subject('Could not eat INCO, because of its toxicity !');
-        });
+        Utils::sendEmailErrorManually($e);
     }
 
     protected function sendInvalidXmlEmail(string $filename, Throwable $e)
@@ -366,10 +375,40 @@ abstract class XmlJobs extends XArcadiaFlux
      */
     protected function moveToInvalid(string $filename): void
     {
-        if (!$this->getDisk()->move($filename, sprintf('%s/%s', $this->invalidFolderName, $filename))) {
+        $newFileName = $filename;
+        if (!$this->getDisk()->exists($filename)) {
             $this->log(
                 sprintf(
-                    'File %s could not be moved to invalid folder name',
+                    'File %s could not be moved to invalid (not found)',
+                    $filename
+                ),
+                LogType::warning()
+            );
+            return;
+        }
+        if ($this->getDisk()->exists(sprintf('%s/%s', $this->invalidFolderName, $filename))) {
+            $ext         = \pathinfo($filename, \PATHINFO_EXTENSION);
+            $newFileName = \sprintf(
+                '%s_%d.%s',
+                \substr($filename, 0, 0 - (strlen($ext) + 1)),
+                \time(),
+                $ext
+            );
+        }
+        try {
+            if (!$this->getDisk()->move($filename, sprintf('%s/%s', $this->invalidFolderName, $newFileName))) {
+                $this->log(
+                    sprintf(
+                        'File %s could not be moved to invalid',
+                        $filename
+                    ),
+                    LogType::warning()
+                );
+            }
+        } catch (Exception $e) {
+            $this->log(
+                sprintf(
+                    'File %s could not be moved to invalid',
                     $filename
                 ),
                 LogType::warning()
diff --git a/app/Lib/Utils.php b/app/Lib/Utils.php
new file mode 100644
index 0000000..041e8fa
--- /dev/null
+++ b/app/Lib/Utils.php
@@ -0,0 +1,210 @@
+<?php
+
+namespace App\Lib;
+
+use Exception;
+use Illuminate\Support\Facades\Log;
+use Illuminate\Support\Facades\Mail;
+use Throwable;
+
+/**
+ * Some utilities methods
+ *
+ * @author Jérémy Munsch <jeremy.munsch@ldc.fr>
+ */
+// phpcs:disable Generic.Files.LineLength.TooLong
+class Utils
+{
+    /**
+     * Replace stange accents
+     *
+     * https://stackoverflow.com/questions/3371697/replacing-accented-characters-php
+     * @author Jérémy Munsch <jeremy.munsch@jeremydev.ovh>
+     * @param string $subject
+     * @return string
+     */
+    public static function replaceSpecChar(string $subject): string
+    {
+        $char_map = array(
+            "ъ" => "-", "ь" => "-", "Ъ" => "-", "Ь" => "-",
+            "А" => "A", "Ă" => "A", "Ǎ" => "A", "Ą" => "A", "À" => "A", "Ã" => "A", "Á" => "A", "Æ" => "A", "Â" => "A", "Å" => "A", "Ǻ" => "A", "Ā" => "A", "א" => "A",
+            "Б" => "B", "ב" => "B", "Þ" => "B",
+            "Ĉ" => "C", "Ć" => "C", "Ç" => "C", "Ц" => "C", "צ" => "C", "Ċ" => "C", "Č" => "C", "©" => "C", "ץ" => "C",
+            "Д" => "D", "Ď" => "D", "Đ" => "D", "ד" => "D", "Ð" => "D",
+            "È" => "E", "Ę" => "E", "É" => "E", "Ë" => "E", "Ê" => "E", "Е" => "E", "Ē" => "E", "Ė" => "E", "Ě" => "E", "Ĕ" => "E", "Є" => "E", "Ə" => "E", "ע" => "E",
+            "Ф" => "F", "Ƒ" => "F",
+            "Ğ" => "G", "Ġ" => "G", "Ģ" => "G", "Ĝ" => "G", "Г" => "G", "ג" => "G", "Ґ" => "G",
+            "ח" => "H", "Ħ" => "H", "Х" => "H", "Ĥ" => "H", "ה" => "H",
+            "I" => "I", "Ï" => "I", "Î" => "I", "Í" => "I", "Ì" => "I", "Į" => "I", "Ĭ" => "I", "I" => "I", "И" => "I", "Ĩ" => "I", "Ǐ" => "I", "י" => "I", "Ї" => "I", "Ī" => "I", "І" => "I",
+            "Й" => "J", "Ĵ" => "J",
+            "ĸ" => "K", "כ" => "K", "Ķ" => "K", "К" => "K", "ך" => "K",
+            "Ł" => "L", "Ŀ" => "L", "Л" => "L", "Ļ" => "L", "Ĺ" => "L", "Ľ" => "L", "ל" => "L",
+            "מ" => "M", "М" => "M", "ם" => "M",
+            "Ñ" => "N", "Ń" => "N", "Н" => "N", "Ņ" => "N", "ן" => "N", "Ŋ" => "N", "נ" => "N", "ʼn" => "N", "Ň" => "N",
+            "Ø" => "O", "Ó" => "O", "Ò" => "O", "Ô" => "O", "Õ" => "O", "О" => "O", "Ő" => "O", "Ŏ" => "O", "Ō" => "O", "Ǿ" => "O", "Ǒ" => "O", "Ơ" => "O",
+            "פ" => "P", "ף" => "P", "П" => "P",
+            "ק" => "Q",
+            "Ŕ" => "R", "Ř" => "R", "Ŗ" => "R", "ר" => "R", "Р" => "R", "®" => "R",
+            "Ş" => "S", "Ś" => "S", "Ș" => "S", "Š" => "S", "С" => "S", "Ŝ" => "S", "ס" => "S",
+            "Т" => "T", "Ț" => "T", "ט" => "T", "Ŧ" => "T", "ת" => "T", "Ť" => "T", "Ţ" => "T",
+            "Ù" => "U", "Û" => "U", "Ú" => "U", "Ū" => "U", "У" => "U", "Ũ" => "U", "Ư" => "U", "Ǔ" => "U", "Ų" => "U", "Ŭ" => "U", "Ů" => "U", "Ű" => "U", "Ǖ" => "U", "Ǜ" => "U", "Ǚ" => "U", "Ǘ" => "U",
+            "В" => "V", "ו" => "V",
+            "Ý" => "Y", "Ы" => "Y", "Ŷ" => "Y", "Ÿ" => "Y",
+            "Ź" => "Z", "Ž" => "Z", "Ż" => "Z", "З" => "Z", "ז" => "Z",
+            "а" => "a", "ă" => "a", "ǎ" => "a", "ą" => "a", "à" => "a", "ã" => "a", "á" => "a", "æ" => "a", "â" => "a", "å" => "a", "ǻ" => "a", "ā" => "a", "א" => "a",
+            "б" => "b", "ב" => "b", "þ" => "b",
+            "ĉ" => "c", "ć" => "c", "ç" => "c", "ц" => "c", "צ" => "c", "ċ" => "c", "č" => "c", "©" => "c", "ץ" => "c",
+            "Ч" => "ch", "ч" => "ch",
+            "д" => "d", "ď" => "d", "đ" => "d", "ד" => "d", "ð" => "d",
+            "è" => "e", "ę" => "e", "é" => "e", "ë" => "e", "ê" => "e", "е" => "e", "ē" => "e", "ė" => "e", "ě" => "e", "ĕ" => "e", "є" => "e", "ə" => "e", "ע" => "e",
+            "ф" => "f", "ƒ" => "f",
+            "ğ" => "g", "ġ" => "g", "ģ" => "g", "ĝ" => "g", "г" => "g", "ג" => "g", "ґ" => "g",
+            "ח" => "h", "ħ" => "h", "х" => "h", "ĥ" => "h", "ה" => "h",
+            "i" => "i", "ï" => "i", "î" => "i", "í" => "i", "ì" => "i", "į" => "i", "ĭ" => "i", "ı" => "i", "и" => "i", "ĩ" => "i", "ǐ" => "i", "י" => "i", "ї" => "i", "ī" => "i", "і" => "i",
+            "й" => "j", "Й" => "j", "Ĵ" => "j", "ĵ" => "j",
+            "ĸ" => "k", "כ" => "k", "ķ" => "k", "к" => "k", "ך" => "k",
+            "ł" => "l", "ŀ" => "l", "л" => "l", "ļ" => "l", "ĺ" => "l", "ľ" => "l", "ל" => "l",
+            "מ" => "m", "м" => "m", "ם" => "m",
+            "ñ" => "n", "ń" => "n", "н" => "n", "ņ" => "n", "ן" => "n", "ŋ" => "n", "נ" => "n", "ʼn" => "n", "ň" => "n",
+            "ø" => "o", "ó" => "o", "ò" => "o", "ô" => "o", "õ" => "o", "о" => "o", "ő" => "o", "ŏ" => "o", "ō" => "o", "ǿ" => "o", "ǒ" => "o", "ơ" => "o",
+            "פ" => "p", "ף" => "p", "п" => "p",
+            "ק" => "q",
+            "ŕ" => "r", "ř" => "r", "ŗ" => "r", "ר" => "r", "р" => "r", "®" => "r",
+            "ş" => "s", "ś" => "s", "ș" => "s", "š" => "s", "с" => "s", "ŝ" => "s", "ס" => "s",
+            "т" => "t", "ț" => "t", "ט" => "t", "ŧ" => "t", "ת" => "t", "ť" => "t", "ţ" => "t",
+            "ù" => "u", "û" => "u", "ú" => "u", "ū" => "u", "у" => "u", "ũ" => "u", "ư" => "u", "ǔ" => "u", "ų" => "u", "ŭ" => "u", "ů" => "u", "ű" => "u", "ǖ" => "u", "ǜ" => "u", "ǚ" => "u", "ǘ" => "u",
+            "в" => "v", "ו" => "v",
+            "ý" => "y", "ы" => "y", "ŷ" => "y", "ÿ" => "y",
+            "ź" => "z", "ž" => "z", "ż" => "z", "з" => "z", "ז" => "z", "ſ" => "z",
+            "™" => "tm",
+            "@" => "at",
+            "Ä" => "ae", "Ǽ" => "ae", "ä" => "ae", "æ" => "ae", "ǽ" => "ae",
+            "ij" => "ij", "IJ" => "ij",
+            "я" => "ja", "Я" => "ja",
+            "Э" => "je", "э" => "je",
+            "ё" => "jo", "Ё" => "jo",
+            "ю" => "ju", "Ю" => "ju",
+            "œ" => "oe", "Œ" => "oe", "ö" => "oe", "Ö" => "oe",
+            "щ" => "sch", "Щ" => "sch",
+            "ш" => "sh", "Ш" => "sh",
+            "ß" => "ss",
+            "Ü" => "ue",
+            "Ж" => "zh", "ж" => "zh",
+        );
+        return self::mbStrtr($subject, $char_map);
+    }
+
+    /**
+     * Multibyte strtr
+     *
+     * @param string       $str
+     * @param string|array $from
+     * @param string|null  $to
+     * @return string
+     */
+    public static function mbStrtr($str, $from, $to = null): string
+    {
+        if (is_array($from)) {
+            $from = array_map('utf8_decode', $from);
+            $from = array_map('utf8_decode', array_flip($from));
+            return utf8_encode(strtr(utf8_decode($str), array_flip($from)));
+        }
+        return utf8_encode(strtr(utf8_decode($str), utf8_decode($from), utf8_decode($to)));
+    }
+
+    /**
+     * Manually send an exception to team instead of going trough the error handler
+     * (requires package kwaadpepper/exception-handler)
+     *
+     * @param Throwable $e
+     * @return void
+     */
+    public static function sendEmailErrorManually(Throwable $e)
+    {
+        Mail::send([
+            'html' => 'exception-handler::emails.exception',
+            'text' => 'exception-handler::emails.exception-plain'
+        ], [
+            'appName' => config('app.name'),
+            'code' => $e->getCode(),
+            'line' => $e->getLine(),
+            'ename' => \get_class($e),
+            'emessage' => self::getAnonymizedMessage($e->getMessage()),
+            'file' => str_replace(\realpath(\sprintf('%s/../', \app_path())), '', $e->getFile()),
+            'stackTrace' => self::getAnonymizedStackTrace($e),
+            'uri' => app()->runningInConsole() ?
+                'CONSOLE' : (
+                (request() and request()->fullUrl()) ?
+                    request()->fullUrl() : 'none')
+        ], function ($m) {
+            /** @var \Illuminate\Mail\Message $m */
+            $m->to(
+                config('mail.contactsList.serviceweb'),
+                sprintf('%s Exception Handler', config('app.name'))
+            )->subject('Need help !, an error occured in your application, please check Logs ASAP');
+        });
+        Log::alert(sprintf('Envoie d\'une erreur par mail : %s', json_encode($e)));
+    }
+
+    /**
+     * Anonymize at least Sql queries
+     *
+     * @param string $message
+     * @return string
+     */
+    private static function getAnonymizedMessage(string $message): string
+    {
+        $message = \preg_replace('/^(.*)( \(SQL:.*\))$/', '$1', $message) ?? $message;
+        return $message;
+    }
+
+    /**
+     * Anonymize stack trace
+     *
+     * @param \Exception $e
+     * @return string
+     */
+    private static function getAnonymizedStackTrace(Throwable $e): string
+    {
+        $projectPath   = \realpath(\sprintf('%s/../', \app_path()));
+        $traceStack    = $e->getTrace();
+        $traceStr      = '';
+        $maxStackStage = 0;
+        foreach ($traceStack as $stackStage => $stack) {
+            $maxStackStage = $stackStage;
+            $traceStr     .= \sprintf(
+                '#%s %s(%d): ',
+                $stackStage,
+                str_replace($projectPath, '', $stack['file']),
+                $stack['line']
+            );
+            $traceStr     .= "{$stack['class']}->{$stack['function']}(";
+            $traceArgs     = '';
+            if (isset($stack['args'])) {
+                $argsCount = count($stack['args']);
+                for ($i = 0; $i < $argsCount; $i++) {
+                    $type = \gettype($stack['args'][$i]);
+                    switch ($type) {
+                        case 'object':
+                            $traceArgs .= sprintf('Object(%s)', \get_class($stack['args'][$i]));
+                            break;
+                        case 'array':
+                            $traceArgs .= sprintf('Array(%d)', \count($stack['args'][$i]));
+                            break;
+                        case 'string':
+                            $traceArgs .= sprintf('String(%d)', \strlen($stack['args'][$i]));
+                            break;
+                        default:
+                            $traceArgs .= \gettype($stack['args'][$i]);
+                    }
+                    $traceArgs .= ($i !== ($argsCount - 1) ? ',' : '');
+                }
+                $traceStr .= $traceArgs;
+            }//end if
+            $traceStr .= ")\n";
+        }//end foreach
+        $maxStackStage++;
+        $traceStr .= "#$maxStackStage {main}";
+        return $traceStr;
+    }
+}
diff --git a/app/Models/XmlInco.php b/app/Models/XmlInco.php
index f33346b..e35f2b0 100644
--- a/app/Models/XmlInco.php
+++ b/app/Models/XmlInco.php
@@ -97,139 +97,4 @@ class XmlInco extends Model
     {
         return sprintf('%s - %s', $this->id, $this->descriptif);
     }
-
-    public function updateLocalProductInfo()
-    {
-        $foodProduct = FoodProduct::where('no_art', $this->id)->first();
-
-        if (!empty($foodProduct)) {
-            //info produits à suggerer lors de l'édition
-            // this->descriptif (doit aller dans la table products)
-            // $table->text('preparation')->nullable(); => $this->conseil_prepa
-            // $table->text('selling_argument')->nullable(); => $this->arg_vente
-
-            //infos produit maj automatique
-            $foodProduct->ingredients = $this->lst_ing;
-            $foodProduct->allergens   = $this->lib_allergene;
-            //$foodProduct->servings    = $this->nb_portion; // problem les types de correspondent pas numeric = string (exemple this->nb_portion = '1/2')
-
-            $nutri_values = $this->nutritional_values;
-            if (!empty($nutri_values)) {
-                //maj des info nutri locales
-                //pour la ref
-                $foodProduct->nv_ref_weight_g = $nutri_values["taille_ref"];
-                $foodProduct->nv_kj           = $nutri_values["val_energ_kj_r"]; //Energie Kj
-                $foodProduct->nv_kcal         = $nutri_values["val_energ_kcal_r"]; //Energie Kcal
-                $foodProduct->nv_fat_g        = $nutri_values["mat_grass_r"]; //graisses
-                $foodProduct->nv_sat_fat_g    = $nutri_values["ags_r"]; //dont acides gras saturés
-                $foodProduct->nv_carb_g       = $nutri_values["glucides_r"]; //Glucides
-                $foodProduct->nv_sugar_g      = $nutri_values["sucre_r"]; //dont sucres
-                $foodProduct->nv_fiber_g      = $nutri_values["fib_alim_r"]; //fibres alimentaires
-                $foodProduct->nv_protein_g    = $nutri_values["proteine_r"]; //protéines
-                $foodProduct->nv_salt_g       = $nutri_values["sel_r"]; //sel
-
-                // $foodProduct-> = $nutri_values["sodium_r"];
-                // $foodProduct-> = $nutri_values["ener_e14_r"];
-                // $foodProduct-> = $nutri_values["fruits_r"];
-                // $foodProduct-> = $nutri_values["fruits_coque_sech_r"];
-                // $foodProduct-> = $nutri_values["ener_kjo_r"];
-                // $foodProduct-> = $nutri_values["fatsat_r"];
-                // $foodProduct-> = $nutri_values["choval_r"];
-                // $foodProduct-> = $nutri_values["sugar_r"];
-                // $foodProduct-> = $nutri_values["fibtg_r"];
-                // $foodProduct-> = $nutri_values["pro_r"];
-                // $foodProduct-> = $nutri_values["salteq_r"];
-
-
-                //valeurs nutritionnelles par portion
-                $foodProduct->nv_serving_size_g    = $nutri_values["taille_port"]; //taille portion en grammes
-                $foodProduct->nv_serving_kj        = $nutri_values["val_energ_kj_p"]; //Energie Kj
-                $foodProduct->nv_serving_kcal      = $nutri_values["val_energ_kcal_p"]; //Energie Kcal
-                $foodProduct->nv_serving_fat_g     = $nutri_values["mat_grass_p"]; //graisses
-                $foodProduct->nv_serving_sat_fat_g = $nutri_values["ags_p"]; //dont acides gras saturés
-                $foodProduct->nv_serving_carb_g    = $nutri_values["glucides_p"]; //Glucides
-                $foodProduct->nv_serving_sugar_g   = $nutri_values["sucre_p"]; //dont sucres
-                $foodProduct->nv_serving_fiber_g   = $nutri_values["fib_alim_p"]; //fibres alimentaires
-                $foodProduct->nv_serving_protein_g = $nutri_values["proteine_p"]; //protéines
-                $foodProduct->nv_serving_salt_g    = $nutri_values["sel_p"]; //sel
-
-                // $foodProduct-> = $nutri_values["sodium_p"];
-                // $foodProduct-> = $nutri_values["fruits_p"];
-
-
-                //apports journaliers
-                $foodProduct->nv_daily_energy  = $nutri_values["pourcent_apport_ref_kjo_r"]; //Energie % recommandé
-                $foodProduct->nv_daily_fat     = $nutri_values["pourcent_apport_ref_fat_r"]; //graisses
-                $foodProduct->nv_daily_sat_fat = $nutri_values["pourcent_apport_ref_fatsat_r"]; //dont acides gras saturés
-                $foodProduct->nv_daily_carb    = $nutri_values["pourcent_apport_ref_choval_r"]; //Glucides
-                $foodProduct->nv_daily_sugar   = $nutri_values["pourcent_apport_ref_sugar_r"]; //dont sucres
-                $foodProduct->nv_daily_fiber   = $nutri_values["pourcent_apport_ref_fibtg_r"]; //fibres alimentaires
-                $foodProduct->nv_daily_protein = $nutri_values["pourcent_apport_ref_pro_r"]; //protéines
-                $foodProduct->nv_daily_salt    = $nutri_values["pourcent_apport_ref_salteq_r"]; //sel
-
-                //$foodProduct-> = $nutri_values["pourcent_apport_ref_e14_r"];
-
-                //nutriscore
-                $foodProduct->nutriscore = $nutri_values["classement"];
-
-
-                //Infos n'ayant pas de mapping
-                // $foodProduct-> = $nutri_values["val_energ_kj_nut"];
-                // $foodProduct-> = $nutri_values["val_energ_kcal_nut"];
-                // $foodProduct-> = $nutri_values["mat_grass_nut"];
-                // $foodProduct-> = $nutri_values["ags_nut"];
-                // $foodProduct-> = $nutri_values["glucides_nut"];
-                // $foodProduct-> = $nutri_values["sucre_nut"];
-                // $foodProduct-> = $nutri_values["sucre_r"];
-                // $foodProduct-> = $nutri_values["fib_alim_nut"];
-                // $foodProduct-> = $nutri_values["proteine_nut"];
-                // $foodProduct-> = $nutri_values["sel_nut"];
-                // $foodProduct-> = $nutri_values["apport_ref"];
-                // $foodProduct-> = $nutri_values["fruits_coque_sech_p"];
-                // $foodProduct-> = $nutri_values["ener_e14_p"];
-                // $foodProduct-> = $nutri_values["pourcent_apport_ref_e14_p"];
-                // $foodProduct-> = $nutri_values["ener_kjo_p"];
-                // $foodProduct-> = $nutri_values["pourcent_apport_ref_kjo_p"];
-                // $foodProduct-> = $nutri_values["fat_p"];
-                // $foodProduct-> = $nutri_values["pourcent_apport_ref_fat_p"];
-                // $foodProduct-> = $nutri_values["fatsat_p"];
-                // $foodProduct-> = $nutri_values["pourcent_apport_ref_fatsat_p"];
-                // $foodProduct-> = $nutri_values["choval_p"];
-                // $foodProduct-> = $nutri_values["pourcent_apport_ref_choval_p"];
-                // $foodProduct-> = $nutri_values["sugar_p"];
-                // $foodProduct-> = $nutri_values["pourcent_apport_ref_sugar_p"];
-                // $foodProduct-> = $nutri_values["fibtg_p"];
-                // $foodProduct-> = $nutri_values["pourcent_apport_ref_fibtg_p"];
-                // $foodProduct-> = $nutri_values["pro_p"];
-                // $foodProduct-> = $nutri_values["pourcent_apport_ref_pro_p"];
-                // $foodProduct-> = $nutri_values["salteq_p"];
-                // $foodProduct-> = $nutri_values["pourcent_apport_ref_salteq_p"];
-                // $foodProduct-> = $nutri_values["apport_port"];
-                // $foodProduct-> = $nutri_values["part_p"];
-                // $foodProduct-> = $nutri_values["part_g"];
-                // $foodProduct-> = $nutri_values["part_l"];
-                // $foodProduct-> = $nutri_values["p_sur_l"];
-                // $foodProduct-> = $nutri_values["pourcent_ags"];
-                // $foodProduct-> = $nutri_values["pourcent_agi"];
-                // $foodProduct-> = $nutri_values["pourcent_ss"];
-                // $foodProduct-> = $nutri_values["pourcent_amidon"];
-                // $foodProduct-> = $nutri_values["composante_n"];
-                // $foodProduct-> = $nutri_values["composante_p"];
-                // $foodProduct-> = $nutri_values["pourcent_fruits"];
-                // $foodProduct-> = $nutri_values["mat_grass_r_prec"];
-                // $foodProduct-> = $nutri_values["ags_r_prec"];
-                // $foodProduct-> = $nutri_values["glucides_r_prec"];
-                // $foodProduct-> = $nutri_values["sucre_r_prec"];
-                // $foodProduct-> = $nutri_values["fib_alim_r_prec"];
-                // $foodProduct-> = $nutri_values["proteine_r_prec"];
-                // $foodProduct-> = $nutri_values["sel_r_prec"];
-                // $foodProduct-> = $nutri_values["sodium_r_prec"];
-                // $foodProduct-> = $nutri_values["fruits_r_prec"];
-                // $foodProduct-> = $nutri_values["fruits_coque_sech_r_prec"];
-                // $foodProduct-> = $nutri_values["score_nutri"];
-            }
-
-            $foodProduct->save();
-        }
-    }
 }
diff --git a/changelog.md b/changelog.md
new file mode 100644
index 0000000..0979324
--- /dev/null
+++ b/changelog.md
@@ -0,0 +1,12 @@
+# Changelog
+
+All notable changes to `Incollector` will be documented in this file.
+
+# version 0.0.2 - 2022-04-01
+- update Xml job helpers
+- add new field 'NOM' to data collection
+- adapt to code/libelle nodes
+- schema validation if xsd fil provided
+
+# Version 0.0.1 - 2021-11-19
+First release version
diff --git a/database/migrations/2021_11_19_102629_create_xml_incos_table.php b/database/migrations/2021_11_19_102629_create_xml_incos_table.php
index 092ced7..fbd3c42 100644
--- a/database/migrations/2021_11_19_102629_create_xml_incos_table.php
+++ b/database/migrations/2021_11_19_102629_create_xml_incos_table.php
@@ -16,7 +16,7 @@ class CreateXmlIncosTable extends Migration
     {
         Schema::create('xml_incos', function (Blueprint $table) {
             $table->id();
-
+            $table->string('nom')->nullable();
             $table->boolean('prod_saison');
 
             $table->string('allegations')->nullable();