getDisk(); $this->log('Démarrage du traitement'); $files = $disk->files(); $progress = 1; $total = count(array_filter($files, function ($filename) { if ($this->fileHasToBeIgnored($filename)) { return false; } return true; })); foreach ($files as $filename) { if ($this->fileHasToBeIgnored($filename)) { continue; } $this->log(sprintf(' traitement %d sur %s ', $progress++, $total)); try { /** @var string $data */ $data = $disk->get($filename); try { if ($this->encapsulate) { $data = sprintf('%s', $data); /** @var SimpleXMLElement|false */ $xmlContent = simplexml_load_string($data); } else { /** @var SimpleXMLElement|false */ $xmlContent = simplexml_load_string($data, null, LIBXML_NOCDATA | LIBXML_PARSEHUGE); } if ($xmlContent === false) { throw new InvalidXmlException('XML is Invalid (simplexml_load_string returned false)'); } } catch (ErrorException $e) { throw new InvalidXmlException('XML is Invalid', 0, $e); }//end try $data = $this->extract($xmlContent); $lines = []; foreach ($data[$entityName] as $traca) { $line = $this->processFields($traca); if ($callback) { $callback($xmlContent, $line); } $lines[] = $line; } // Mise en base des données. foreach ($lines as $line) { /** @var array Unique Key columns */ $uniqueColumnsKey = []; foreach ($this->uniqueKeyField as $uField) { $uniqueColumnsKey[$uField] = $line[$uField]; } $savedData = forward_static_call_array( [$modelClass, 'updateOrCreate'], [$uniqueColumnsKey, $line] ); $savedData->updateLocalProductInfo(); } } catch (InvalidXmlException $e) { $message = $e->getMessage(); if (get_class($e) === QueryException::class) { $message = implode(' ', $e->errorInfo); } $this->log( sprintf( 'File %s could not be parsed => %s on line %d', $filename, $message, $e->getLine() ), LogType::critical() ); $this->sendInvalidXmlEmail($filename, $e); $this->moveToInvalid($filename); continue; } catch (ErrorException | TypeError $e) { $this->log( sprintf( 'CRITICAL: File %s could not be parsed => %s on line %d', $filename, $e->getMessage(), $e->getLine() ), LogType::critical() ); $this->sendErrorParseEmailToServiceWeb($e); continue; // Ignore the file. } catch (AbortXmlImportException $e) { $this->log( sprintf( 'ERROR: File %s could not be parsed => %s on line %d', $filename, $e->getMessage(), $e->getLine() ), LogType::error() ); $this->sendErrorParseEmailToServiceWeb($e); $this->moveToInvalid($filename); continue; // Ignore the file. } catch (FileNotFoundException | QueryException $e) { // This happens because of concurent xml jobs read. $message = $e->getMessage(); if (get_class($e) === QueryException::class) { $message = implode(' ', $e->errorInfo); } $this->log( sprintf( 'ERROR: File %s could not be read (not found) => %s on line %d', $filename, $message, $e->getLine() ), LogType::debug() ); continue; // Ignore the file. }//end try $this->moveToArchives($filename); }//end foreach // Clear old archives. $this->clearOldArchives(); } /** * Recursive extract to array * * @param \SimpleXMLElement|false $sxe * @return array */ protected function extract($sxe = null): array { if (!$sxe instanceof SimpleXMLElement) { return array(); } $extract = array(); foreach ($sxe->children() as $key => $value) { if ($value->count()) { $extract[$key][] = $this->extract($value); continue; } $value = strval($value); if (array_key_exists($key, $extract) and is_array($extract[$key])) { $extract[$key][] = strlen($value) ? $value : null; } else { $extract[$key] = strlen($value) ? $value : null; } } return $extract; } /** * Process fields to gather data * * @param array $fields * @return array * @throws AbortXmlImportException When field should not be null. */ protected function processFields(array $fields): array { $result = []; foreach ($fields as $key => $field) { if (!($key = $this->shouldFieldBeConsidered($key))) { continue; } if (!$this->isFieldNullable($key) && is_null($field)) { throw new AbortXmlImportException(sprintf('The field %s is not allowed to be null', $key)); } if (is_array($field)) { // Flatten data. $it = new RecursiveIteratorIterator(new RecursiveArrayIterator($field)); foreach ($it as $k => $f) { $ko = strtolower(sprintf('%s.%s', $key, $k)); if (!$this->isFieldNullable($ko) && is_null($f)) { throw new AbortXmlImportException(sprintf( '%s : The field %s is not allowed to be null', get_class($this), $ko )); } if (!($k = $this->shouldFieldBeConsidered($ko))) { continue; } $result[$k] = $f; } } else { $result[$key] = $field; } }//end foreach return $result; } /** * Is the filed nullable * * @param string $key * @return boolean */ private function isFieldNullable(string $key): bool { if (in_array($key, $this->nullableFields)) { return true; } return false; } /** * - replace key name with the corresponding alias * - strtolower the key * - snake_case the key * * @param string $key * @return string|false */ private function shouldFieldBeConsidered(string $key) { $key = strtolower($key); foreach ($this->takeColumns as $k => $alias) { if (strtolower($k) === $key) { return Str::snake(strtolower($alias)); } } return false; } /** * Is the file to be ignored * * @param string $filename * @return boolean */ protected function fileHasToBeIgnored(string $filename): bool { return in_array($filename, $this->ignoreFiles) or !$this->checkExtension($filename, 'xml'); } /** * Check if the file is an xml file * * @param string $filename * @return boolean */ protected function isFileXml(string $filename): bool { $isXml = strtolower(pathinfo($filename, \PATHINFO_EXTENSION)) === 'xml'; if (!$isXml) { Log::warning(sprintf('%s : unwanted file type in %s', get_class($this), $filename)); } return $isXml; } /** * Send Error Mail * * @param Throwable $e * @return void */ protected function sendErrorParseEmailToServiceWeb(Throwable $e) { Mail::send('emails.exception', [ 'exception' => $e, 'appName' => config('app.name'), ], function ($m) { $m->to( config('mail.contactsList.serviceweb'), sprintf('%s Exception Handler', config('app.name')) )->subject('Could not eat INCO, because of its toxicity !'); }); } /** * Send Mail Invalid XML * * @param string $filename * @param Throwable $e * @return void */ protected function sendInvalidXmlEmail(string $filename, Throwable $e) { $message = $e->getPrevious()->getMessage(); Mail::send('emails.invalidxml', [ 'strError' => sprintf( 'File %s could not be parsed => %s on line %d', $filename, $message, $e->getLine() ), 'appName' => config('app.name'), 'filename' => $filename, 'exception' => $e->getPrevious() ], function ($m) { $m->to( config('mail.contactsList.invalidInco'), sprintf('%s Exception Handler', config('app.name')) )->subject('You sent me an invalid XML file, please check its validity before resend ASAP.'); }); } /** * Move the file to invalid folder * * @param string $filename * @return void */ protected function moveToInvalid(string $filename): void { if (!$this->getDisk()->move($filename, sprintf('%s/%s', $this->invalidFolderName, $filename))) { $this->log( sprintf( 'File %s could not be moved to invalid folder name', $filename ), LogType::warning() ); } } }