getDisk(); $this->log('Démarrage du traitement'); $this->assertXsdIsHereIfAny(); $files = $disk->files(); $total = count(array_filter($files, function (string $filename) { if ($this->fileHasToBeIgnored($filename)) { return false; } return true; })); $this->startProgressBar($total); /** @var string $filename */ foreach ($files as $filename) { if ($this->fileHasToBeIgnored($filename)) { continue; } $this->incrementProgressBar(); try { /** @var string $data */ $data = $disk->get($filename); $this->validateXmlFile($filename, $data); try { if ($this->encapsulate) { $data = sprintf('%s', $data); /** @var \SimpleXMLElement|false */ $xmlContent = $data; } else { /** @var \SimpleXMLElement|false */ $xmlContent = simplexml_load_string($data, null, LIBXML_NOCDATA | LIBXML_PARSEHUGE); } if ($xmlContent === false) { throw new InvalidXmlException('XML is Invalid (simplexml_load_string returned false)'); } } catch (\ErrorException $e) { throw new InvalidXmlException('XML is Invalid', 0, 1, __FILE__, __LINE__, $e); } $data = $this->extract($xmlContent); $lines = []; // * Multiple Entities within file if (isset($data[$entityName])) { foreach ($data[$entityName] as $traca) { $line = $this->processFields($traca); if ($afterParseCallback) { $afterParseCallback($xmlContent, $line); } $lines[] = $line; } } else { // * Only one entity within file $line = $this->processFields($data); if ($afterParseCallback) { $afterParseCallback($xmlContent, $line); } $lines[] = $line; } // * Mise en base des données foreach ($lines as $line) { /** @var array Unique Key columns */ $uniqueColumnsKey = []; foreach ($this->uniqueKeyField as $uField) { $uniqueColumnsKey[$uField] = $line[$uField]; } if (count($uniqueColumnsKey)) { if ($saveCallback) { $saveCallback($uniqueColumnsKey, $line); } else { forward_static_call_array( [$modelClass, 'updateOrCreate'], [$uniqueColumnsKey, $line] ); } if ($afterSaveCallback) { $afterSaveCallback($xmlContent, $line); } } } //end foreach } catch (InvalidXmlException $e) { $message = $e->getMessage(); $this->log( sprintf( 'File %s could not be parsed => %s on line %d', $filename, $message, $e->getLine() ), LogType::critical ); $this->sendInvalidXmlEmail($filename, $e); $this->moveToInvalid($filename); continue; } catch (AbortXmlImportException $e) { $this->log( sprintf( 'ERROR: File %s could not be parsed => %s on line %d', $filename, $e->getMessage(), $e->getLine() ), LogType::error ); report($e); continue; // * ignore the file } catch (QueryException $e) { $this->log(implode(' ', $e->errorInfo), LogType::critical); report($e); continue; } catch (FileNotFoundException $e) { // This happens because of concurent xml jobs read. $message = $e->getMessage(); $this->log( sprintf( 'ERROR: File %s could not be read (not found) => %s on line %d', $filename, $message, $e->getLine() ), LogType::debug ); continue; // * ignore the file } catch (\Exception | \ErrorException | \TypeError $e) { $this->log( sprintf( 'File %s could not be parsed => %s on line %d', $filename, $e->getMessage(), $e->getLine() ), LogType::critical ); report($e); continue; // * ignore the file } //end try $this->moveToArchives($filename); } //end foreach // Clear old archives. $this->clearOldArchives(); $this->endProgressBar(); } /** * Recursive extract to array * * @param \SimpleXMLElement|false $sxe * @return array */ protected function extract($sxe = null): array { if (!$sxe instanceof \SimpleXMLElement) { return array(); } $extract = array(); foreach ($sxe->children() as $key => $value) { if ($value->count()) { $extract[$key][] = $this->extract($value); continue; } $value = strval($value); if (array_key_exists($key, $extract) and is_array($extract[$key])) { $extract[$key][] = strlen($value) ? $value : null; } else { $extract[$key] = strlen($value) ? $value : null; } } return $extract; } /** * Process fields to gather data * * @param array $fields * @return array * @throws AbortXmlImportException If a field could not be validated. */ protected function processFields(array $fields): array { $result = []; foreach ($fields as $key => $field) { if (!($key = $this->shouldFieldBeConsidered($key))) { continue; } if (!$this->isFieldNullable($key) && is_null($field)) { throw new AbortXmlImportException(sprintf('The field %s is not allowed to be null', $key)); } if (is_array($field)) { // * Flatten data $it = new \RecursiveIteratorIterator(new \RecursiveArrayIterator($field)); foreach ($it as $k => $f) { $ko = strtolower(sprintf('%s.%s', $key, $k)); if (!$this->isFieldNullable($ko) && is_null($f)) { throw new AbortXmlImportException(sprintf( '%s : The field %s is not allowed to be null', get_class($this), $ko )); } if (!($k = $this->shouldFieldBeConsidered($ko))) { continue; } $result[$k] = $f; } } else { $result[$key] = $field; } } //end foreach return $result; } /** * Is the filed nullable * * @param string $key * @return boolean */ private function isFieldNullable(string $key): bool { if (in_array($key, $this->nullableFields)) { return true; } return false; } /** * - replace key name with the corresponding alias * - strtolower the key * - snake_case the key * * @param string $key * @return string|false */ private function shouldFieldBeConsidered(string $key) { $key = strtolower($key); foreach ($this->takeColumns as $k => $alias) { if (strtolower($k) === $key) { return Str::snake(strtolower($alias)); } } return false; } /** * Is the file to be ignored * * @param string $filename * @return boolean */ protected function fileHasToBeIgnored(string $filename): bool { return in_array($filename, $this->ignoreFiles) or !$this->checkExtension($filename, 'xml'); } /** * Check if the file is an xml file * * @param string $filename * @return boolean */ protected function isFileXml(string $filename): bool { $isXml = strtolower(pathinfo($filename, \PATHINFO_EXTENSION)) === 'xml'; if (!$isXml) { Log::warning(sprintf('%s : unwanted file type in %s', get_class($this), $filename)); } return $isXml; } /** * Send invalid XML email * * @param string $filename * @param \Throwable $e * @return void */ protected function sendInvalidXmlEmail(string $filename, \Throwable $e) { $message = $e->getPrevious()->getMessage(); Mail::send('mails.invalidxml', [ 'strError' => sprintf( 'File %s could not be parsed => %s on line %d', $filename, $message, $e->getLine() ), 'filename' => $filename, 'exception' => $e->getPrevious() ], function (Message $m) { $m->to( config('mail.contactsList.invalidXml'), 'XML Exception Handler' )->subject('You sent me an invalid XML file, please check its validity before resend ASAP.'); }); } /** * Move the file to invalid folder * * @param string $filename * @return void */ protected function moveToInvalid(string $filename): void { if (!$this->getDisk()->move($filename, sprintf('%s/%s', $this->invalidFolderName, $filename))) { $this->log( sprintf( 'File %s could not be moved to invalid folder name', $filename ), LogType::warning ); } } /** * Assert XSD is readable if configured * * @return void * @throws \Illuminate\Contracts\Filesystem\FileNotFoundException If xsd file cant be read. */ private function assertXsdIsHereIfAny() { if ($this->xsdSchemaPath) { $this->xsdSchema = File::get($this->xsdSchemaPath); } } /** * Validate Xml file against XSD if loaded * * @param string $fileName * @param string $fileContent * @return void * @throws InvalidXmlException If XML could not be parsed or validated against XSD. * @phpcs:disable Squiz.Commenting.FunctionCommentThrowTag.WrongNumber */ private function validateXmlFile(string $fileName, string $fileContent) { if ($this->xsdSchema) { $domDocument = new \DOMDocument('1.0', 'UTF-8'); try { if (!$domDocument->loadXML($fileContent)) { throw new InvalidXmlException("XML Validation $fileName content could not be loaded to validate."); } if (!$domDocument->schemaValidateSource($this->xsdSchema, LIBXML_SCHEMA_CREATE)) { throw new InvalidXmlException( "XML Validation $fileName content could not be validated against xsd." ); } } catch (\Exception $e) { if (\get_class($e) === InvalidXmlException::class) { throw $e; } throw new InvalidXmlException( "XML Validation $fileName content could not be validated against xsd.", 0, 1, __FILE__, __LINE__, $e ); } //end try }//end if } }