From 71cf7d5c9abca2bc61e91ee8308c547755efec6c Mon Sep 17 00:00:00 2001 From: Olivier Legat Date: Wed, 21 Apr 2021 11:06:40 +0200 Subject: [PATCH] inco mise en place --- app/Console/Commands/XmlIncoReader.php | 38 ++ app/Enums/LogType.php | 83 ++++ app/Exceptions/AbortXmlImportException.php | 9 + app/Jobs/XArcadiaFlux.php | 193 ++++++++ app/Jobs/XmlIncoReader.php | 439 ++++++++++++++++++ app/Jobs/XmlJobs.php | 280 +++++++++++ app/Models/XmlInco.php | 32 ++ config/filesystems.php | 4 + .../2021_04_21_092030_create_jobs_table.php | 36 ++ ...21_04_21_101455_create_xml_incos_table.php | 50 ++ storage/.gitignore | 1 + storage/inco/.gitkeep | 0 storage/inco/archives/.gitkeep | 0 storage/inco/images/.gitkeep | 0 14 files changed, 1165 insertions(+) create mode 100644 app/Console/Commands/XmlIncoReader.php create mode 100644 app/Enums/LogType.php create mode 100644 app/Exceptions/AbortXmlImportException.php create mode 100644 app/Jobs/XArcadiaFlux.php create mode 100644 app/Jobs/XmlIncoReader.php create mode 100644 app/Jobs/XmlJobs.php create mode 100644 app/Models/XmlInco.php create mode 100644 database/migrations/2021_04_21_092030_create_jobs_table.php create mode 100644 database/migrations/2021_04_21_101455_create_xml_incos_table.php create mode 100644 storage/.gitignore create mode 100644 storage/inco/.gitkeep create mode 100644 storage/inco/archives/.gitkeep create mode 100644 storage/inco/images/.gitkeep diff --git a/app/Console/Commands/XmlIncoReader.php b/app/Console/Commands/XmlIncoReader.php new file mode 100644 index 00000000..6e7c53bc --- /dev/null +++ b/app/Console/Commands/XmlIncoReader.php @@ -0,0 +1,38 @@ +info('Running the Job, this could take some time.'); + $job = new JobsXmlIncoReader(); + $job->console = $this; + $job->handle(); + $this->info('Job completed'); + } +} diff --git a/app/Enums/LogType.php b/app/Enums/LogType.php new file mode 100644 index 00000000..2c6fbc0b --- /dev/null +++ b/app/Enums/LogType.php @@ -0,0 +1,83 @@ +value = $value; + $obj->label = $label; + + return $obj; + } + + /** + * @param string $name + * + * @return int|string + * + */ + public function __get(string $name) + { + if ($name === 'label') { + return $this->label; + } + + if ($name === 'value') { + return $this->value; + } + } + + public static function debug() { + return self::makeElement("debug"); + } + + public static function log() { + return self::makeElement("log"); + } + + public static function warning() { + return self::makeElement("warning"); + } + + public static function info() { + return self::makeElement("info"); + } + + public static function error() { + return self::makeElement("error"); + } + + public static function critical() { + return self::makeElement("critical"); + } +} diff --git a/app/Exceptions/AbortXmlImportException.php b/app/Exceptions/AbortXmlImportException.php new file mode 100644 index 00000000..60e11533 --- /dev/null +++ b/app/Exceptions/AbortXmlImportException.php @@ -0,0 +1,9 @@ +getDisk()->move($filename, sprintf('%s/%s', $this->archiveFolderName, $filename))) { + $this->log( + sprintf( + 'File %s could not be moved to archives', + $filename + ), + LogType::warning() + ); + } + } + + /** + * Suppression des fichiers dans + * archives qui ont plus de 30j + * + * @return void + */ + protected function clearOldArchives(): void + { + $disk = $this->getDisk(); + $archives = $disk->files($this->archiveFolderName); + + // /** @var File */ + foreach ($archives as $archive) { + $pi = \pathinfo($archive, \PATHINFO_BASENAME); + if (in_array($pi, $this->ignoreFiles)) { + continue; + } + + if ($disk->lastModified($archive) < strtotime(Carbon::now()->subDays(30))) { + if (!$disk->delete($archive)) { + Log::critical(sprintf('%s : Could not delete file %s', get_class($this), $archive)); + }; + } + } + } + + /** + * Get the concerned disk + * + * @return Filesystem + */ + protected function getDisk(): Filesystem + { + return Storage::disk($this->disk); + } + + /** + * Logs data + * + * @param string $message + * @param LogType $logtype + * @return void + */ + protected function log(string $message, LogType $logtype = null): void + { + if (!$logtype) { + $logtype = LogType::debug(); + } + $message = sprintf('%s : %s', get_class($this), $message); + switch ($logtype->value) { + case LogType::debug()->value: + Log::debug($message); + break; + case LogType::info()->value: + Log::info($message); + break; + case LogType::log()->value: + Log::log($message); + break; + case LogType::warning()->value: + Log::warning($message); + break; + case LogType::error()->value: + Log::error($message); + break; + case LogType::critical()->value: + Log::critical($message); + break; + default: + throw new Exception(sprintf('%s : unhandled log type %s', get_class($this), $logtype->label)); + } + $this->consolePrint($message, $logtype); + } + + /** + * Prints to the console + * + * @param string $message + * @param LogType $logtype + * @return void + */ + protected function consolePrint(string $message, LogType $logtype): void + { + if ($this->console) { + switch ($logtype->value) { + case LogType::debug()->value: + case LogType::info()->value: + case LogType::log()->value: + $this->console->info($message); + break; + case LogType::warning()->value: + $this->console->comment($message); + break; + case LogType::error()->value: + case LogType::critical()->value: + $this->console->error($message); + break; + default: + throw new Exception(sprintf('%s : unhandled log type %s', get_class($this), $logtype->label)); + } + } + } +} diff --git a/app/Jobs/XmlIncoReader.php b/app/Jobs/XmlIncoReader.php new file mode 100644 index 00000000..b33dbda3 --- /dev/null +++ b/app/Jobs/XmlIncoReader.php @@ -0,0 +1,439 @@ + 'id', // primary key on model + + 'ID_SOCIETE' => 'id_societe', + 'NOM_SOCIETE' => 'nom_societe', + + 'PROD_SAISON' => 'prod_saison', + 'NOM' => 'nom', + 'NUMERO' => 'numero', + 'TAILLE_REF' => 'taille_ref', + 'ALLEGATIONS' => 'allegations', + 'CONSEIL_PREPA' => 'conseil_prepa', + 'ASTUCES' => 'astuces', + 'DESCRIPTIF' => 'descriptif', + 'ARG_VENTE' => 'arg_vente', + 'NB_PORTION' => 'nb_portion', + 'TYP_PORTION' => 'typ_portion', + 'PDS_MOY_THEO' => 'pds_moy_theo', + 'LIB_CONSO' => 'lib_conso', + 'LST_ING' => 'lst_ing', + + // list alergene + 'LIB_ALLERGENE' => 'lib_allergene', + + // Allegations + 'ALLEGAT_SANTE' => 'allegat_sante', + 'ALLEGAT_NUTRIT' => 'allegat_nutrit', + 'ALLEGAT_AUTRE' => 'allegat_autre', + + + 'INCO_COMPO' => 'inco_compo', // Consider this sub array + 'INCO_COMPO.VAL_ENERG_KJ_NUT' => 'val_energ_kj_nut', + 'INCO_COMPO.VAL_ENERG_KJ_R' => 'val_energ_kj_r', + 'INCO_COMPO.VAL_ENERG_KCAL_R' => 'val_energ_kcal_r', + 'INCO_COMPO.VAL_ENERG_KCAL_NUT' => 'val_energ_kcal_nut', + 'INCO_COMPO.MAT_GRASS_R' => 'mat_grass_r', + 'INCO_COMPO.MAT_GRASS_NUT' => 'mat_grass_nut', + 'INCO_COMPO.AGS_R' => 'ags_r', + 'INCO_COMPO.AGS_NUT' => 'ags_nut', + 'INCO_COMPO.GLUCIDES_R' => 'glucides_r', + 'INCO_COMPO.GLUCIDES_NUT' => 'glucides_nut', + 'INCO_COMPO.SUCRE_NUT' => 'sucre_nut', + 'INCO_COMPO.SUCRE_R' => 'sucre_r', + 'INCO_COMPO.FIB_ALIM_NUT' => 'fib_alim_nut', + 'INCO_COMPO.FIB_ALIM_R' => 'fib_alim_r', + 'INCO_COMPO.PROTEINE_R' => 'proteine_r', + 'INCO_COMPO.PROTEINE_NUT' => 'proteine_nut', + 'INCO_COMPO.SEL_R' => 'sel_r', + 'INCO_COMPO.SEL_NUT' => 'sel_nut', + 'INCO_COMPO.SODIUM_R' => 'sodium_r', + 'INCO_COMPO.ENER_E14_R' => 'ener_e14_r', + 'INCO_COMPO.FRUITS_R' => 'fruits_r', + 'INCO_COMPO.FRUITS_COQUE_SECH_R' => 'fruits_coque_sech_r', + 'INCO_COMPO.POURCENT_APPORT_REF_E14_R' => 'pourcent_apport_ref_e14_r', + 'INCO_COMPO.ENER_KJO_R' => 'ener_kjo_r', + 'INCO_COMPO.POURCENT_APPORT_REF_KJO_R' => 'pourcent_apport_ref_kjo_r', + 'INCO_COMPO.POURCENT_APPORT_REF_FAT_R' => 'pourcent_apport_ref_fat_r', + 'INCO_COMPO.FATSAT_R' => 'fatsat_r', + 'INCO_COMPO.POURCENT_APPORT_REF_FATSAT_R' => 'pourcent_apport_ref_fatsat_r', + 'INCO_COMPO.CHOVAL_R' => 'choval_r', + 'INCO_COMPO.POURCENT_APPORT_REF_CHOVAL_R' => 'pourcent_apport_ref_choval_r', + 'INCO_COMPO.SUGAR_R' => 'sugar_r', + 'INCO_COMPO.POURCENT_APPORT_REF_SUGAR_R' => 'pourcent_apport_ref_sugar_r', + 'INCO_COMPO.FIBTG_R' => 'fibtg_r', + 'INCO_COMPO.POURCENT_APPORT_REF_FIBTG_R' => 'pourcent_apport_ref_fibtg_r', + 'INCO_COMPO.PRO_R' => 'pro_r', + 'INCO_COMPO.POURCENT_APPORT_REF_PRO_R' => 'pourcent_apport_ref_pro_r', + 'INCO_COMPO.SALTEQ_R' => 'salteq_r', + 'INCO_COMPO.POURCENT_APPORT_REF_SALTEQ_R' => 'pourcent_apport_ref_salteq_r', + 'INCO_COMPO.APPORT_REF' => 'apport_ref', + 'INCO_COMPO.TAILLE_PORT' => 'taille_port', + 'INCO_COMPO.VAL_ENERG_KJ_P' => 'val_energ_kj_p', + 'INCO_COMPO.VAL_ENERG_KCAL_P' => 'val_energ_kcal_p', + 'INCO_COMPO.MAT_GRASS_P' => 'mat_grass_p', + 'INCO_COMPO.AGS_P' => 'ags_p', + 'INCO_COMPO.GLUCIDES_P' => 'glucides_p', + 'INCO_COMPO.SUCRE_P' => 'sucre_p', + 'INCO_COMPO.FIB_ALIM_P' => 'fib_alim_p', + 'INCO_COMPO.PROTEINE_P' => 'proteine_p', + 'INCO_COMPO.SODIUM_P' => 'sodium_p', + 'INCO_COMPO.FRUITS_P' => 'fruits_p', + 'INCO_COMPO.SEL_P' => 'sel_p', + 'INCO_COMPO.FRUITS_COQUE_SECH_P' => 'fruits_coque_sech_p', + 'INCO_COMPO.ENER_E14_P' => 'ener_e14_p', + 'INCO_COMPO.POURCENT_APPORT_REF_E14_P' => 'pourcent_apport_ref_e14_p', + 'INCO_COMPO.ENER_KJO_P' => 'ener_kjo_p', + 'INCO_COMPO.POURCENT_APPORT_REF_KJO_P' => 'pourcent_apport_ref_kjo_p', + 'INCO_COMPO.FAT_P' => 'fat_p', + 'INCO_COMPO.POURCENT_APPORT_REF_FAT_P' => 'pourcent_apport_ref_fat_p', + 'INCO_COMPO.FATSAT_P' => 'fatsat_p', + 'INCO_COMPO.POURCENT_APPORT_REF_FATSAT_P' => 'pourcent_apport_ref_fatsat_p', + 'INCO_COMPO.CHOVAL_P' => 'CHOVAL_P', + 'INCO_COMPO.POURCENT_APPORT_REF_CHOVAL_P' => 'pourcent_apport_ref_choval_p', + 'INCO_COMPO.SUGAR_P' => 'SUGAR_P', + 'INCO_COMPO.POURCENT_APPORT_REF_SUGAR_P' => 'pourcent_apport_ref_sugar_p', + 'INCO_COMPO.FIBTG_P' => 'fibtg_p', + 'INCO_COMPO.POURCENT_APPORT_REF_FIBTG_P' => 'pourcent_apport_ref_fibtg_p', + 'INCO_COMPO.PRO_P' => 'pro_p', + 'INCO_COMPO.POURCENT_APPORT_REF_PRO_P' => 'pourcent_apport_ref_pro_p', + 'INCO_COMPO.SALTEQ_P' => 'salteq_p', + 'INCO_COMPO.POURCENT_APPORT_REF_SALTEQ_P' => 'pourcent_apport_ref_salteq_p', + 'INCO_COMPO.APPORT_PORT' => 'apport_port', + 'INCO_COMPO.PART_P' => 'part_p', + 'INCO_COMPO.PART_G' => 'part_g', + 'INCO_COMPO.PART_L' => 'part_l', + 'INCO_COMPO.P_SUR_L' => 'p_sur_l', + 'INCO_COMPO.POURCENT_AGS' => 'pourcent_ags', + 'INCO_COMPO.POURCENT_AGI' => 'pourcent_agi', + 'INCO_COMPO.POURCENT_SS' => 'pourcent_ss', + 'INCO_COMPO.POURCENT_AMIDON' => 'pourcent_amidon', + 'INCO_COMPO.COMPOSANTE_N' => 'composante_n', + 'INCO_COMPO.COMPOSANTE_P' => 'composante_p', + 'INCO_COMPO.POURCENT_FRUITS' => 'pourcent_fruits', + 'INCO_COMPO.SCORE_NUTRI' => 'score_nutri', + 'INCO_COMPO.CLASSEMENT' => 'classement', + 'INCO_COMPO.MAT_GRASS_R_PREC' => 'mat_grass_r_prec', + 'INCO_COMPO.AGS_R_PREC' => 'ags_r_prec', + 'INCO_COMPO.GLUCIDES_R_PREC' => 'glucides_r_prec', + 'INCO_COMPO.SUCRE_R_PREC' => 'sucre_r_prec', + 'INCO_COMPO.FIB_ALIM_R_PREC' => 'fib_alim_r_prec', + 'INCO_COMPO.PROTEINE_R_PREC' => 'proteine_r_prec', + 'INCO_COMPO.SEL_R_PREC' => 'sel_r_prec', + 'INCO_COMPO.SODIUM_R_PREC' => 'sodium_r_prec', + 'INCO_COMPO.FRUITS_R_PREC' => 'fruits_r_prec', + 'INCO_COMPO.FRUITS_COQUE_SECH_R_PREC' => 'fruits_coque_sech_r_prec', + ]; + + protected $nullableFields = [ + 'taille_ref', + 'conseil_prepa', + 'typ_portion', + 'nb_portion', + 'descriptif', + 'prod_saison', + 'allegations', + 'astuces', 'arg_vente', + 'pds_moy_theo', + 'lib_conso', 'lst_ing', + 'allegat_sante', 'allegat_nutrit', 'allegat_autre', + 'inco_compo', + 'inco_compo.nom', + 'inco_compo.numero', + 'inco_compo.taille', + 'inco_compo.taille_ref', + 'inco_compo.val_energ_kj_nut', + 'inco_compo.val_energ_kj_r', + 'inco_compo.val_energ_kcal_r', + 'inco_compo.val_energ_kcal_nut', + 'inco_compo.mat_grass_r', + 'inco_compo.mat_grass_nut', + 'inco_compo.ags_r', + 'inco_compo.ags_nut', + 'inco_compo.glucides_r', + 'inco_compo.glucides_nut', + 'inco_compo.sucre_nut', + 'inco_compo.sucre_r', + 'inco_compo.fib_alim_nut', + 'inco_compo.fib_alim_r', + 'inco_compo.proteine_r', + 'inco_compo.proteine_nut', + 'inco_compo.sel_r', + 'inco_compo.sel_nut', + 'inco_compo.sodium_r', + 'inco_compo.ener_e14_r', + 'inco_compo.fruits_r', + 'inco_compo.fruits_coque_sech_r', + 'inco_compo.pourcent_apport_ref_e14_r', + 'inco_compo.ener_kjo_r', + 'inco_compo.pourcent_apport_ref_kjo_r', + 'inco_compo.pourcent_apport_ref_fat_r', + 'inco_compo.fatsat_r', + 'inco_compo.pourcent_apport_ref_fatsat_r', + 'inco_compo.choval_r', + 'inco_compo.pourcent_apport_ref_choval_r', + 'inco_compo.sugar_r', + 'inco_compo.pourcent_apport_ref_sugar_r', + 'inco_compo.fibtg_r', + 'inco_compo.pourcent_apport_ref_fibtg_r', + 'inco_compo.pro_r', + 'inco_compo.pourcent_apport_ref_pro_r', + 'inco_compo.salteq_r', + 'inco_compo.pourcent_apport_ref_salteq_r', + 'inco_compo.apport_ref', + 'inco_compo.taille_port', + 'inco_compo.val_energ_kj_p', + 'inco_compo.val_energ_kcal_p', + 'inco_compo.mat_grass_p', + 'inco_compo.ags_p', + 'inco_compo.glucides_p', + 'inco_compo.sucre_p', + 'inco_compo.fib_alim_p', + 'inco_compo.proteine_p', + 'inco_compo.sodium_p', + 'inco_compo.fruits_p', + 'inco_compo.sel_p', + 'inco_compo.fruits_coque_sech_p', + 'inco_compo.ener_e14_p', + 'inco_compo.pourcent_apport_ref_e14_p', + 'inco_compo.ener_kjo_p', + 'inco_compo.pourcent_apport_ref_kjo_p', + 'inco_compo.fat_p', + 'inco_compo.pourcent_apport_ref_fat_p', + 'inco_compo.fatsat_p', + 'inco_compo.pourcent_apport_ref_fatsat_p', + 'inco_compo.choval_p', + 'inco_compo.pourcent_apport_ref_choval_p', + 'inco_compo.sugar_p', + 'inco_compo.pourcent_apport_ref_sugar_p', + 'inco_compo.fibtg_p', + 'inco_compo.pourcent_apport_ref_fibtg_p', + 'inco_compo.pro_p', + 'inco_compo.pourcent_apport_ref_pro_p', + 'inco_compo.salteq_p', + 'inco_compo.pourcent_apport_ref_salteq_p', + 'inco_compo.apport_port', + 'inco_compo.part_p', + 'inco_compo.part_g', + 'inco_compo.part_l', + 'inco_compo.p_sur_l', + 'inco_compo.pourcent_ags', + 'inco_compo.pourcent_agi', + 'inco_compo.pourcent_ss', + 'inco_compo.pourcent_amidon', + 'inco_compo.composante_n', + 'inco_compo.composante_p', + 'inco_compo.pourcent_fruits', + 'inco_compo.score_nutri', + 'inco_compo.classement', + 'inco_compo.mat_grass_r_prec', + 'inco_compo.ags_r_prec', + 'inco_compo.glucides_r_prec', + 'inco_compo.sucre_r_prec', + 'inco_compo.fib_alim_r_prec', + 'inco_compo.proteine_r_prec', + 'inco_compo.sel_r_prec', + 'inco_compo.sodium_r_prec', + 'inco_compo.fruits_r_prec', + 'inco_compo.fruits_coque_sech_r_prec' + ]; + + protected $uniqueKeyField = ['id']; + + protected $mergeColumns = [ + 'allegations' => [ + 'allegat_sante', + 'allegat_nutrit', + 'allegat_autre' + ], + 'nutritional_values' => [ + 'val_energ_kj_nut', + 'val_energ_kj_r', + 'val_energ_kcal_r', + 'val_energ_kcal_nut', + 'mat_grass_r', + 'mat_grass_nut', + 'ags_r', + 'ags_nut', + 'glucides_r', + 'glucides_nut', + 'sucre_nut', + 'sucre_r', + 'fib_alim_nut', + 'fib_alim_r', + 'proteine_r', + 'proteine_nut', + 'sel_r', + 'sel_nut', + 'sodium_r', + 'ener_e14_r', + 'fruits_r', + 'fruits_coque_sech_r', + 'pourcent_apport_ref_e14_r', + 'ener_kjo_r', + 'pourcent_apport_ref_kjo_r', + 'pourcent_apport_ref_fat_r', + 'fatsat_r', + 'pourcent_apport_ref_fatsat_r', + 'choval_r', + 'pourcent_apport_ref_choval_r', + 'sugar_r', + 'pourcent_apport_ref_sugar_r', + 'fibtg_r', + 'pourcent_apport_ref_fibtg_r', + 'pro_r', + 'pourcent_apport_ref_pro_r', + 'salteq_r', + 'pourcent_apport_ref_salteq_r', + 'apport_ref', + 'taille_port', + 'val_energ_kj_p', + 'val_energ_kcal_p', + 'mat_grass_p', + 'ags_p', + 'glucides_p', + 'sucre_p', + 'fib_alim_p', + 'proteine_p', + 'sodium_p', + 'fruits_p', + 'sel_p', + 'fruits_coque_sech_p', + 'ener_e14_p', + 'pourcent_apport_ref_e14_p', + 'ener_kjo_p', + 'pourcent_apport_ref_kjo_p', + 'fat_p', + 'pourcent_apport_ref_fat_p', + 'fatsat_p', + 'pourcent_apport_ref_fatsat_p', + 'choval_p', + 'pourcent_apport_ref_choval_p', + 'sugar_p', + 'pourcent_apport_ref_sugar_p', + 'fibtg_p', + 'pourcent_apport_ref_fibtg_p', + 'pro_p', + 'pourcent_apport_ref_pro_p', + 'salteq_p', + 'pourcent_apport_ref_salteq_p', + 'apport_port', + 'part_p', + 'part_g', + 'part_l', + 'p_sur_l', + 'pourcent_ags', + 'pourcent_agi', + 'pourcent_ss', + 'pourcent_amidon', + 'composante_n', + 'composante_p', + 'pourcent_fruits', + 'score_nutri', + 'classement', + 'mat_grass_r_prec', + 'ags_r_prec', + 'glucides_r_prec', + 'sucre_r_prec', + 'fib_alim_r_prec', + 'proteine_r_prec', + 'sel_r_prec', + 'sodium_r_prec', + 'fruits_r_prec', + 'fruits_coque_sech_r_prec', + ] + ]; + + public function handle() + { + + $this->parseData('INCO', XmlInco::class, function (SimpleXMLElement $xmlData, array &$line) { + + // merge columns + foreach ($this->mergeColumns as $key => $values) { + $line[$key] = []; + // merge columns and remove merged columns from line + array_map(function ($kValue) use (&$line, $key) { + if (array_key_exists($kValue, $line)) { + $line[$key][$kValue] = $line[$kValue]; + unset($line[$kValue]); + } + }, $values); + if ($key == 'allegations') { + $line[$key] = implode(',', array_filter($line[$key])); + $line[$key] = strlen($line[$key]) ? $line[$key] : null; + } + if ($key == 'nutritional_values') { + $line[$key] = json_encode($line[$key], \JSON_FORCE_OBJECT); + } + } + + // Boolean cast prod_saison + $line['prod_saison'] = isset($line['prod_saison']) ? + (strtolower($line['prod_saison']) == 'non' ? false : true) : false; + + $imageFileName = $this->parseImage($xmlData); + + $line['picture'] = '/storage/images/inco_default.png'; + + // Ajout du chemin de l'image dans les champs a envoyé en base + if ($imageFileName !== null) { + $line['picture'] = sprintf('%s/%s', $this->disk, $imageFileName); + } + }); + } + + private function parseImage(SimpleXMLElement $xmlData) + { + $imageData = trim((string)$xmlData->INCO->MEDIA_VISUEL_EAI->MEDIA_VISUEL_EAI); + if (strlen($imageData)) { + $imageData = base64_decode($imageData); + $imgName = strval($xmlData->INCO->INCO_COMPO->TBL_INCO_COMPO->ID_INCO); + if(empty($imgName)) { + $imgName = strval($xmlData->INCO->NO_ART); + } + $imgNamePath = sprintf('%s/%s', rtrim($this->imagesPath, '/'), $imgName); + + $disk = $this->getDisk(); + + $info = getimagesizefromstring($imageData); + $mime = $info['mime']; + switch ($mime) { + case 'image/jpeg': + $imgExtension = 'jpg'; + break; + case 'image/png': + $imgExtension = 'png'; + break; + case 'image/gif': + $imgExtension = 'gif'; + break; + default: + throw new AbortXmlImportException(sprintf('Unknown image type %s', $imgName)); + } + + $imgNamePath = sprintf('%s.%s', $imgNamePath, $imgExtension); + + if (!$disk->put($imgNamePath, $imageData)) { + throw new AbortXmlImportException(sprintf('The image %s could not be saved', $imgNamePath)); + } + + return $imgNamePath; + } + } +} diff --git a/app/Jobs/XmlJobs.php b/app/Jobs/XmlJobs.php new file mode 100644 index 00000000..6acc66de --- /dev/null +++ b/app/Jobs/XmlJobs.php @@ -0,0 +1,280 @@ +getDisk(); + $this->log('Démarrage du traitement'); + $files = $disk->files(); + + $progress = 1; + $total = count(array_filter($files, function ($filename) { + if ($this->fileHasToBeIgnored($filename)) { + return false; + } + return true; + })); + + foreach ($files as $filename) { + if ($this->fileHasToBeIgnored($filename)) { + continue; + } + + $this->log(sprintf(' traitement %d sur %s ', $progress++, $total)); + + /** @var string $data */ + $data = $disk->get($filename); + + try { + if ($this->encapsulate) { + $data = sprintf('%s', $data); + /** @var SimpleXMLElement|false */ + $xmlContent = simplexml_load_string($data); + } else { + /** @var SimpleXMLElement|false */ + $xmlContent = simplexml_load_string($data, null, LIBXML_NOCDATA | LIBXML_PARSEHUGE); + } + + if ($xmlContent === false) { + $this->log( + sprintf('File %s could not be parsed (simplexml_load_string)', $filename), + LogType::critical() + ); + continue; // ignore the file + } + + $data = $this->extract($xmlContent); + + + $lines = []; + + foreach ($data[$entityName] as $traca) { + $line = $this->processFields($traca); + if ($callback) { + $callback($xmlContent, $line); + } + $lines[] = $line; + } + + // Mise en base des données + foreach ($lines as $line) { + /** @var array Unique Key columns */ + $uniqueColumnsKey = []; + foreach ($this->uniqueKeyField as $uField) { + $uniqueColumnsKey[$uField] = $line[$uField]; + } + forward_static_call_array( + [$modelClass, 'updateOrCreate'], + [$uniqueColumnsKey, $line] + ); + } + } catch (ErrorException | TypeError | QueryException $e) { + $message = $e->getMessage(); + if (get_class($e) === QueryException::class) { + $message = implode(' ', $e->errorInfo); + } + $this->log( + sprintf( + 'File %s could not be parsed => %s on line %d', + $filename, + $message, + $e->getLine() + ), + LogType::critical() + ); + continue; // ignore the file + } catch (AbortXmlImportException $e) { + $this->log( + sprintf( + 'File %s could not be parsed => %s on line %d', + $filename, + $e->getMessage(), + $e->getLine() + ), + LogType::error() + ); + continue; // ignore the file + } + + $this->moveToArchives($filename); + } + + // Clear old archives + $this->clearOldArchives(); + } + + /** + * Recursive extract to array + * + * @param \SimpleXMLElement|false $sxe + * @return array + */ + protected function extract($sxe = null): array + { + if (!$sxe instanceof SimpleXMLElement) { + return array(); + } + $extract = array(); + foreach ($sxe->children() as $key => $value) { + if ($value->count()) { + $extract[$key][] = $this->extract($value); + continue; + } + $value = strval($value); + if (array_key_exists($key, $extract) and is_array($extract[$key])) { + $extract[$key][] = strlen($value) ? $value : null; + } else { + $extract[$key] = strlen($value) ? $value : null; + } + } + return $extract; + } + + /** + * Process fields to gather data + * + * @param array $fields + * @return array + * @throws AbortXmlImportException + */ + protected function processFields(array $fields): array + { + $result = []; + foreach ($fields as $key => $field) { + if (!($key = $this->shouldFieldBeConsidered($key))) { + continue; + } + if (!$this->isFieldNullable($key) && is_null($field)) { + throw new AbortXmlImportException(sprintf('The field %s is not allowed to be null', $key)); + } + if (is_array($field)) { + // Flatten data + $it = new RecursiveIteratorIterator(new RecursiveArrayIterator($field)); + foreach ($it as $k => $f) { + $ko = strtolower(sprintf('%s.%s', $key, $k)); + if (!$this->isFieldNullable($ko) && is_null($f)) { + throw new AbortXmlImportException(sprintf( + '%s : The field %s is not allowed to be null', + get_class($this), + $ko + )); + } + if (!($k = $this->shouldFieldBeConsidered($ko))) { + continue; + } + $result[$k] = $f; + } + } else { + $result[$key] = $field; + } + } + return $result; + } + + /** + * Is the filed nullable + * + * @param string $key + * @return boolean + */ + private function isFieldNullable(string $key): bool + { + if (in_array($key, $this->nullableFields)) { + return true; + } + return false; + } + + /** + * - replace key name with the corresponding alias + * - strtolower the key + * - snake_case the key + * + * @param string $key + * @return string|false + */ + private function shouldFieldBeConsidered(string $key) + { + $key = strtolower($key); + foreach ($this->takeColumns as $k => $alias) { + if (strtolower($k) === $key) { + return Str::snake(strtolower($alias)); + } + } + return false; + } + + /** + * Is the file to be ignored + * + * @param string $filename + * @return boolean + */ + protected function fileHasToBeIgnored(string $filename): bool + { + return in_array($filename, $this->ignoreFiles) or !$this->checkExtension($filename, 'xml'); + } + + /** + * Check if the file is an xml file + * + * @param string $filename + * @return boolean + */ + protected function isFileXml(string $filename): bool + { + $isXml = strtolower(pathinfo($filename, \PATHINFO_EXTENSION)) === 'xml'; + if (!$isXml) { + Log::warning(sprintf('%s : unwanted file type in %s', get_class($this), $filename)); + } + return $isXml; + } +} diff --git a/app/Models/XmlInco.php b/app/Models/XmlInco.php new file mode 100644 index 00000000..5472272d --- /dev/null +++ b/app/Models/XmlInco.php @@ -0,0 +1,32 @@ +id, $this->descriptif); + } +} diff --git a/config/filesystems.php b/config/filesystems.php index 94c81126..3cb8811a 100644 --- a/config/filesystems.php +++ b/config/filesystems.php @@ -65,6 +65,10 @@ return [ 'endpoint' => env('AWS_ENDPOINT'), ], + 'inco' => [ + 'driver' => 'local', + 'root' => storage_path('inco'), + ], ], /* diff --git a/database/migrations/2021_04_21_092030_create_jobs_table.php b/database/migrations/2021_04_21_092030_create_jobs_table.php new file mode 100644 index 00000000..1be9e8a8 --- /dev/null +++ b/database/migrations/2021_04_21_092030_create_jobs_table.php @@ -0,0 +1,36 @@ +bigIncrements('id'); + $table->string('queue')->index(); + $table->longText('payload'); + $table->unsignedTinyInteger('attempts'); + $table->unsignedInteger('reserved_at')->nullable(); + $table->unsignedInteger('available_at'); + $table->unsignedInteger('created_at'); + }); + } + + /** + * Reverse the migrations. + * + * @return void + */ + public function down() + { + Schema::dropIfExists('jobs'); + } +} diff --git a/database/migrations/2021_04_21_101455_create_xml_incos_table.php b/database/migrations/2021_04_21_101455_create_xml_incos_table.php new file mode 100644 index 00000000..8684526a --- /dev/null +++ b/database/migrations/2021_04_21_101455_create_xml_incos_table.php @@ -0,0 +1,50 @@ +id(); + + $table->boolean('prod_saison'); + + $table->string('allegations')->nullable(); + $table->text('descriptif')->nullable(); + $table->string('picture'); + $table->text('astuces')->nullable(); + $table->string('lib_conso')->nullable(); + $table->string('nb_portion', 20)->nullable(); + $table->string('typ_portion', 200)->nullable(); + $table->string('pds_moy_theo', 20)->nullable(); + + $table->text('conseil_prepa')->nullable(); + $table->text('arg_vente')->nullable(); + $table->text('lst_ing')->nullable(); + + $table->text('nutritional_values'); + + $table->timestamps(); + }); + } + + /** + * Reverse the migrations. + * + * @return void + */ + public function down() + { + Schema::dropIfExists('inco_datas'); + } +} diff --git a/storage/.gitignore b/storage/.gitignore new file mode 100644 index 00000000..e3a41975 --- /dev/null +++ b/storage/.gitignore @@ -0,0 +1 @@ +inco/* diff --git a/storage/inco/.gitkeep b/storage/inco/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/storage/inco/archives/.gitkeep b/storage/inco/archives/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/storage/inco/images/.gitkeep b/storage/inco/images/.gitkeep new file mode 100644 index 00000000..e69de29b