ldc-dam/app/Jobs/Arcadia/XmlJobs.php

<?php

namespace App\Jobs\Arcadia;

use App\Enums\LogType;
use App\Exceptions\AbortXmlImportException;
use App\Exceptions\InvalidXmlException;
use App\Lib\Utils;
use Illuminate\Contracts\Filesystem\FileNotFoundException;
use Illuminate\Database\QueryException;
use Illuminate\Support\Facades\File;
use Illuminate\Support\Facades\Log;
use Illuminate\Support\Facades\Mail;
use Illuminate\Support\Str;

abstract class XmlJobs extends XArcadiaFlux
{
    /**
     * The xsd file path used to validate files
     *
     * @var string|null
     */
    protected $xsdSchemaPath = null;

    /**
     * The xsd file content used to validate files
     *
     * @var string|null
     */
    protected $xsdSchema = null;

    /**
     * invalid xml folder
     *
     * @var string
     */
    protected $invalidFolderName = 'invalid';

    /**
     * The columns to take from the XML
     * subcolumns can be added with a dot
     *
     * @var array
     */
    protected $takeColumns = [];

    /**
     * The columns allowed to be null
     *
     * @var array
     */
    protected $nullableFields = [];

    /**
     * Need to encapsulate the freeking XML
     * Because provided one is dirty
     *
     * @var boolean
     */
    protected $encapsulate = false;

    /**
     * Parse Data from the XML and gives it to the callback
     *
     * @param string   $entityName
     * @param string   $modelClass
     * @param callable $afterParseCallback After parse is done.
     * @param callable $saveCallback       Instead of updateOrCreate, execute $saveCallback.
     * @param callable $afterSaveCallback  After save is done.
     * @return void
     * @throws InvalidXmlException If xml file is not valid.
     * @phpcs:disable Generic.Metrics.CyclomaticComplexity.MaxExceeded
     */
    protected function parseData(
        string $entityName,
        string $modelClass,
        callable $afterParseCallback = null,
        callable $saveCallback = null,
        callable $afterSaveCallback = null
    ): void {
        /** @var Filesystem */
        $disk = $this->getDisk();
        $this->log('Démarrage du traitement');

        $this->assertXsdIsHereIfAny();

        $files = $disk->files();

        $total = count(array_filter($files, function ($filename) {
            if ($this->fileHasToBeIgnored($filename)) {
                return false;
            }
            return true;
        }));

        $this->startProgressBar($total);

        foreach ($files as $filename) {
            if ($this->fileHasToBeIgnored($filename)) {
                continue;
            }

            $this->incrementProgressBar();

            try {
                /** @var string $data */
                $data = $disk->get($filename);

                $this->validateXmlFile($filename, $data);

                try {
                    if ($this->encapsulate) {
                        $data = sprintf('<Root>%s</Root>', $data);
                        /** @var \SimpleXMLElement|false */
                        $xmlContent = $data;
                    } else {
                        /** @var \SimpleXMLElement|false */
                        $xmlContent = simplexml_load_string($data, null, LIBXML_NOCDATA | LIBXML_PARSEHUGE);
                    }
                    if ($xmlContent === false) {
                        throw new InvalidXmlException('XML is Invalid (simplexml_load_string returned false)');
                    }
                } catch (\ErrorException $e) {
                    throw new InvalidXmlException('XML is Invalid', 0, $e);
                }

                $data  = $this->extract($xmlContent);
                $lines = [];

                // * Multiple Entities within file
                if (isset($data[$entityName])) {
                    foreach ($data[$entityName] as $traca) {
                        $line = $this->processFields($traca);
                        if ($afterParseCallback) {
                            $afterParseCallback($xmlContent, $line);
                        }
                        $lines[] = $line;
                    }
                } else {
                    // * Only one entity within file
                    $line = $this->processFields($data);
                    if ($afterParseCallback) {
                        $afterParseCallback($xmlContent, $line);
                    }
                    $lines[] = $line;
                }


                // * Mise en base des données
                foreach ($lines as $line) {
                    /** @var array Unique Key columns */
                    $uniqueColumnsKey = [];
                    foreach ($this->uniqueKeyField as $uField) {
                        $uniqueColumnsKey[$uField] = $line[$uField];
                    }
                    if (count($uniqueColumnsKey)) {
                        if ($saveCallback) {
                            $saveCallback($uniqueColumnsKey, $line);
                        } else {
                            forward_static_call_array(
                                [$modelClass, 'updateOrCreate'],
                                [$uniqueColumnsKey, $line]
                            );
                        }
                        if ($afterSaveCallback) {
                            $afterSaveCallback($xmlContent, $line);
                        }
                    }
                } //end foreach
            } catch (InvalidXmlException $e) {
                $message = $e->getMessage();
                $this->log(
                    sprintf(
                        'File %s could not be parsed => %s on line %d',
                        $filename,
                        $message,
                        $e->getLine()
                    ),
                    LogType::critical()
                );
                $this->sendInvalidXmlEmail($filename, $e);
                $this->moveToInvalid($filename);
                continue;
            } catch (AbortXmlImportException $e) {
                $this->log(
                    sprintf(
                        'ERROR: File %s could not be parsed => %s on line %d',
                        $filename,
                        $e->getMessage(),
                        $e->getLine()
                    ),
                    LogType::error()
                );
                report($e);
                // ! TODO: Remove this after adding kwaadpepper/exception-handler.
                Utils::sendEmailErrorManually($e);
                continue;
                // * ignore the file
            } catch (QueryException $e) {
                $this->log(implode(' ', $e->errorInfo), LogType::critical());
                report($e);
                // ! TODO: Remove this after adding kwaadpepper/exception-handler.
                Utils::sendEmailErrorManually($e);
                continue;
            } catch (FileNotFoundException $e) {
                // This happens because of concurent xml jobs read.
                $message = $e->getMessage();
                $this->log(
                    sprintf(
                        'ERROR: File %s could not be read (not found) => %s on line %d',
                        $filename,
                        $message,
                        $e->getLine()
                    ),
                    LogType::debug()
                );
                continue;
                // * ignore the file
            } catch (\Exception | \ErrorException | \TypeError $e) {
                $this->log(
                    sprintf(
                        'File %s could not be parsed => %s on line %d',
                        $filename,
                        $e->getMessage(),
                        $e->getLine()
                    ),
                    LogType::critical()
                );
                report($e);
                // ! TODO: Remove this after adding kwaadpepper/exception-handler.
                Utils::sendEmailErrorManually($e);
                continue;
                // * ignore the file
            } //end try
            $this->moveToArchives($filename);
        } //end foreach

        // Clear old archives.
        $this->clearOldArchives();

        $this->endProgressBar();
    }

    /**
     * Recursive extract to array
     *
     * @param \SimpleXMLElement|false $sxe
     * @return array
     */
    protected function extract($sxe = null): array
    {
        if (!$sxe instanceof \SimpleXMLElement) {
            return array();
        }
        $extract = array();
        foreach ($sxe->children() as $key => $value) {
            if ($value->count()) {
                $extract[$key][] = $this->extract($value);
                continue;
            }
            $value = strval($value);
            if (array_key_exists($key, $extract) and is_array($extract[$key])) {
                $extract[$key][] = strlen($value) ? $value : null;
            } else {
                $extract[$key] = strlen($value) ? $value : null;
            }
        }
        return $extract;
    }

    /**
     * Process fields to gather data
     *
     * @param array $fields
     * @return array
     * @throws AbortXmlImportException If a field could not be validated.
     */
    protected function processFields(array $fields): array
    {
        $result = [];
        foreach ($fields as $key => $field) {
            if (!($key = $this->shouldFieldBeConsidered($key))) {
                continue;
            }
            if (!$this->isFieldNullable($key) && is_null($field)) {
                throw new AbortXmlImportException(sprintf('The field %s is not allowed to be null', $key));
            }
            if (is_array($field)) {
                // * Flatten data
                $it = new \RecursiveIteratorIterator(new \RecursiveArrayIterator($field));
                foreach ($it as $k => $f) {
                    $ko = strtolower(sprintf('%s.%s', $key, $k));
                    if (!$this->isFieldNullable($ko) && is_null($f)) {
                        throw new AbortXmlImportException(sprintf(
                            '%s : The field %s is not allowed to be null',
                            get_class($this),
                            $ko
                        ));
                    }
                    if (!($k = $this->shouldFieldBeConsidered($ko))) {
                        continue;
                    }
                    $result[$k] = $f;
                }
            } else {
                $result[$key] = $field;
            }
        } //end foreach
        return $result;
    }

    /**
     * Is the filed nullable
     *
     * @param string $key
     * @return boolean
     */
    private function isFieldNullable(string $key): bool
    {
        if (in_array($key, $this->nullableFields)) {
            return true;
        }
        return false;
    }

    /**
     * - replace key name with the corresponding alias
     * - strtolower the key
     * - snake_case the key
     *
     * @param string $key
     * @return string|false
     */
    private function shouldFieldBeConsidered(string $key)
    {
        $key = strtolower($key);
        foreach ($this->takeColumns as $k => $alias) {
            if (strtolower($k) === $key) {
                return Str::snake(strtolower($alias));
            }
        }
        return false;
    }

    /**
     * Is the file to be ignored
     *
     * @param string $filename
     * @return boolean
     */
    protected function fileHasToBeIgnored(string $filename): bool
    {
        return in_array($filename, $this->ignoreFiles) or !$this->checkExtension($filename, 'xml');
    }

    /**
     * Check if the file is an xml file
     *
     * @param string $filename
     * @return boolean
     */
    protected function isFileXml(string $filename): bool
    {
        $isXml = strtolower(pathinfo($filename, \PATHINFO_EXTENSION)) === 'xml';
        if (!$isXml) {
            Log::warning(sprintf('%s : unwanted file type in %s', get_class($this), $filename));
        }
        return $isXml;
    }

    /**
     * Send invalid XML email
     *
     * @param string     $filename
     * @param \Throwable $e
     * @return void
     */
    protected function sendInvalidXmlEmail(string $filename, \Throwable $e)
    {
        $message = $e->getPrevious()->getMessage();
        Mail::send('mails.invalidxml', [
            'strError' => sprintf(
                'File %s could not be parsed => %s on line %d',
                $filename,
                $message,
                $e->getLine()
            ),
            'filename' => $filename,
            'exception' => $e->getPrevious()
        ], function ($m) {
            $m->to(
                config('mail.contactsList.invalidXml'),
                'XML Exception Handler'
            )->subject('You sent me an invalid XML file, please check its validity before resend ASAP.');
        });
    }

    /**
     * Move the file to invalid folder
     *
     * @param string $filename
     * @return void
     */
    protected function moveToInvalid(string $filename): void
    {
        if (!$this->getDisk()->move($filename, sprintf('%s/%s', $this->invalidFolderName, $filename))) {
            $this->log(
                sprintf(
                    'File %s could not be moved to invalid folder name',
                    $filename
                ),
                LogType::warning()
            );
        }
    }

    /**
     * Assert XSD is readable if configured
     *
     * @return void
     * @throws \Illuminate\Contracts\Filesystem\FileNotFoundException If xsd file cant be read.
     */
    private function assertXsdIsHereIfAny()
    {
        if ($this->xsdSchemaPath) {
            $this->xsdSchema = File::get($this->xsdSchemaPath);
        }
    }

    /**
     * Validate Xml file against XSD if loaded
     *
     * @param string $fileName
     * @param string $fileContent
     * @return void
     * @throws InvalidXmlException If XML could not be parsed or validated against XSD.
     * @phpcs:disable Squiz.Commenting.FunctionCommentThrowTag.WrongNumber
     */
    private function validateXmlFile(string $fileName, string $fileContent)
    {
        if ($this->xsdSchema) {
            $domDocument = new \DOMDocument('1.0', 'UTF-8');
            try {
                if (!$domDocument->loadXML($fileContent)) {
                    throw new InvalidXmlException("XML Validation $fileName content could not be loaded to validate.");
                }
                if (!$domDocument->schemaValidateSource($this->xsdSchema, LIBXML_SCHEMA_CREATE)) {
                    throw new InvalidXmlException(
                        "XML Validation $fileName content could not be validated against xsd."
                    );
                }
            } catch (\Exception $e) {
                if (\get_class($e) === InvalidXmlException::class) {
                    throw $e;
                }
                throw new InvalidXmlException(
                    "XML Validation $fileName content could not be validated against xsd.",
                    0,
                    1,
                    __FILE__,
                    __LINE__,
                    $e
                );
            } //end try
        }//end if
    }
}