getDisk(); $this->log('Démarrage du traitement'); $files = $disk->files(); $progress = 1; $total = count(array_filter($files, function ($filename) { if ($this->fileHasToBeIgnored($filename)) { return false; } return true; })); foreach ($files as $filename) { if ($this->fileHasToBeIgnored($filename)) { continue; } $this->log(sprintf(' traitement %d sur %s ', $progress++, $total)); try { /** @var string $data */ $data = $disk->get($filename); try { $this->validateSchema($data); if ($this->encapsulate) { $data = sprintf('%s', $data); /** @var SimpleXMLElement|false */ $xmlContent = simplexml_load_string($data); } else { /** @var SimpleXMLElement|false */ $xmlContent = simplexml_load_string($data, null, LIBXML_NOCDATA | LIBXML_PARSEHUGE); } if ($xmlContent === false) { throw new InvalidXmlException('XML is Invalid (simplexml_load_string returned false)'); } } catch (ErrorException $e) { throw new InvalidXmlException('XML is Invalid', 0, $e); } $data = $this->extract($xmlContent); $lines = []; foreach ($data[$entityName] as $traca) { $line = $this->processFields($traca); if ($callback) { $callback($xmlContent, $line); } $lines[] = $line; } // Mise en base des données foreach ($lines as $line) { /** @var array Unique Key columns */ $uniqueColumnsKey = []; foreach ($this->uniqueKeyField as $uField) { $uniqueColumnsKey[$uField] = $line[$uField]; } forward_static_call_array( [$modelClass, 'updateOrCreate'], [$uniqueColumnsKey, $line] ); } } catch (InvalidXmlException $e) { $message = $e->getMessage(); if (get_class($e) === QueryException::class) { $message = implode(' ', $e->errorInfo); } $this->log( sprintf( 'File %s could not be parsed => %s on line %d', $filename, $message, $e->getLine() ), LogType::critical() ); $this->sendInvalidXmlEmail($filename, $e); $this->moveToInvalid($filename); continue; } catch (ErrorException | TypeError $e) { $this->log( sprintf( 'CRITICAL: File %s could not be parsed => %s on line %d', $filename, $e->getMessage(), $e->getLine() ), LogType::critical() ); $this->sendErrorParseEmailToServiceWeb($e); continue; // ignore the file } catch (AbortXmlImportException $e) { $this->log( sprintf( 'ERROR: File %s could not be parsed => %s on line %d', $filename, $e->getMessage(), $e->getLine() ), LogType::error() ); $this->sendErrorParseEmailToServiceWeb($e); continue; // ignore the file } catch (FileNotFoundException | QueryException $e) { // This happens because of concurent xml jobs read $message = $e->getMessage(); if (get_class($e) === QueryException::class) { $message = implode(' ', $e->errorInfo); } $this->log( sprintf( 'ERROR: File %s could not be read (not found) => %s on line %d', $filename, $message, $e->getLine() ), LogType::debug() ); continue; // ignore the file }//end try $this->moveToArchives($filename); }//end foreach // Clear old archives $this->clearOldArchives(); } /** * Recursive extract to array * * @param \SimpleXMLElement|false $sxe * @return array */ protected function extract($sxe = null): array { if (!$sxe instanceof SimpleXMLElement) { return array(); } $extract = array(); foreach ($sxe->children() as $key => $value) { if ($value->count()) { $extract[$key][] = $this->extract($value); continue; } $value = strval($value); if (array_key_exists($key, $extract) and is_array($extract[$key])) { $extract[$key][] = strlen($value) ? $value : null; } else { $extract[$key] = strlen($value) ? $value : null; } } return $extract; } /** * Process fields to gather data * * @param array $fields * @return array * @throws AbortXmlImportException */ protected function processFields(array $fields): array { $result = []; foreach ($fields as $key => $field) { if (!($key = $this->shouldFieldBeConsidered($key))) { continue; } if (!$this->isFieldNullable($key) && is_null($field)) { throw new AbortXmlImportException(sprintf('The field %s is not allowed to be null', $key)); } if (is_array($field)) { // Flatten data $it = new RecursiveIteratorIterator(new RecursiveArrayIterator($field)); foreach ($it as $k => $f) { $ko = strtolower(sprintf('%s.%s', $key, $k)); if (!$this->isFieldNullable($ko) && is_null($f)) { throw new AbortXmlImportException(sprintf( '%s : The field %s is not allowed to be null', get_class($this), $ko )); } if (!($k = $this->shouldFieldBeConsidered($ko))) { continue; } $result[$k] = $f; } } else { $result[$key] = $field; } }//end foreach return $result; } /** * Is the filed nullable * * @param string $key * @return boolean */ private function isFieldNullable(string $key): bool { if (in_array($key, $this->nullableFields)) { return true; } return false; } /** * - replace key name with the corresponding alias * - strtolower the key * - snake_case the key * * @param string $key * @return string|false */ private function shouldFieldBeConsidered(string $key) { $key = strtolower($key); foreach ($this->takeColumns as $k => $alias) { if (strtolower($k) === $key) { return Str::snake(strtolower($alias)); } } return false; } /** * Is the file to be ignored * * @param string $filename * @return boolean */ protected function fileHasToBeIgnored(string $filename): bool { return in_array($filename, $this->ignoreFiles) or !$this->checkExtension($filename, 'xml'); } /** * Validate xml data against xsd file if provided * * @return void */ protected function validateSchema($data) { if($this->xsdFile) { //check against XSD $xsd = storage_path($this->disk."/".$this->xsdFile); $domDocument= new \DOMDocument(); $domDocument->loadXML($data); //free some memroy to avoid overload unset($data); $domDocument->schemaValidate($xsd); } } /** * Check if the file is an xml file * * @param string $filename * @return boolean */ protected function isFileXml(string $filename): bool { $isXml = strtolower(pathinfo($filename, \PATHINFO_EXTENSION)) === 'xml'; if (!$isXml) { Log::warning(sprintf('%s : unwanted file type in %s', get_class($this), $filename)); } return $isXml; } protected function sendErrorParseEmailToServiceWeb(Throwable $e) { Utils::sendEmailErrorManually($e); } protected function sendInvalidXmlEmail(string $filename, Throwable $e) { $message = $e->getPrevious()->getMessage(); Mail::send('emails.invalidxml', [ 'strError' => sprintf( 'File %s could not be parsed => %s on line %d', $filename, $message, $e->getLine() ), 'appName' => config('app.name'), 'filename' => $filename, 'exception' => $e->getPrevious() ], function ($m) { $m->to( config('mail.contactsList.invalidInco'), sprintf('%s Exception Handler', config('app.name')) )->subject('You sent me an invalid XML file, please check its validity before resend ASAP.'); }); } /** * Move the file to invalid folder * * @param string $filename * @return void */ protected function moveToInvalid(string $filename): void { $newFileName = $filename; if (!$this->getDisk()->exists($filename)) { $this->log( sprintf( 'File %s could not be moved to invalid (not found)', $filename ), LogType::warning() ); return; } if ($this->getDisk()->exists(sprintf('%s/%s', $this->invalidFolderName, $filename))) { $ext = \pathinfo($filename, \PATHINFO_EXTENSION); $newFileName = \sprintf( '%s_%d.%s', \substr($filename, 0, 0 - (strlen($ext) + 1)), \time(), $ext ); } try { if (!$this->getDisk()->move($filename, sprintf('%s/%s', $this->invalidFolderName, $newFileName))) { $this->log( sprintf( 'File %s could not be moved to invalid', $filename ), LogType::warning() ); } } catch (Exception $e) { $this->log( sprintf( 'File %s could not be moved to invalid', $filename ), LogType::warning() ); } } }