cii-text-analysis/Lib/RawDataLoader.php

<?php

declare(strict_types=1);

namespace Classifier\Lib;

class RawDataLoader
{
    private const RAW_DATA_PATH = __DIR__ . '/../data/raw';

    private const SMS_SEPARATOR = "\"\n\"";
    private const SMS_FILE = self::RAW_DATA_PATH . '/sms';

    private const SMS_SEPARATOR_DEV = "|\n|";
    private const SMS_FILE_DEV = self::RAW_DATA_PATH . '/sms_dev_maria_suff';

    private const MAIL_SEPARATOR = "___";
    private const MAIL_FILE = self::RAW_DATA_PATH . '/mails';

    private const MIXED_TA_LOG_SEPARATOR = "###";
    private const MIXED_TA_LOG_FILE = self::RAW_DATA_PATH . '/t.log';

    public function __construct()
    {
        if (!file_exists(self::RAW_DATA_PATH)) {
            mkdir(self::RAW_DATA_PATH);
        }

        if (!file_exists(self::SMS_FILE)) {
            throw new \Exception('SMS file not found');
        }
        if (!file_exists(self::MAIL_FILE)) {
            throw new \Exception('Mail file not found');
        }
        if (!file_exists(self::MIXED_TA_LOG_FILE)) {
            throw new \Exception('Mixed TA log file not found');
        }
    }

    /** @return string[] */
    public function getSanitizedStrings(): array
    {
        return array_merge(
            // $this->loadMailData(),
            // $this->loadSmsData2(),
            // $this->loadSmsDataDev(),
            $this->loadMixedTaLogData()
        );
    }

    private function sanitize(string $value): string
    {
        $output = $this->fromUt8ToPlainText($value);
        $output = $this->safelyRemoveEmptyLines($output);

        return $output;
    }


    private function fromUt8ToPlainText(string $value): string
    {
        // * Supprimer les balises
        $output = strip_tags($value);

        // * Convertir les entités html (ex: &amp; devient &)
        $output = html_entity_decode($output, ENT_QUOTES | ENT_HTML5, 'UTF-8');

        return $output;
    }

    private function safelyRemoveEmptyLines(string $value): string
    {
        // * Remplace les lignes ne contenant que des espaces/tabs par une chaîne vide
        $result = preg_replace('/^[\s\t]*[\r\n]+/m', '', $value);

        // * Remplace les sauts de lignes avec un `\n` unique
        $result = preg_replace('/\R/', "\n", $result);

        return $result ?? $value;
    }

    private function loadMailData(): array
    {
        /** @var string[] */
        $output = [];
        $data = $this->loadData(self::MAIL_FILE);

        $token = strtok($data, self::MAIL_SEPARATOR);
        while ($token !== false) {
            $sanitizedToken = $this->sanitize($token);

            $sanitizedToken = mb_trim(str_replace('\\\'', '\'', $sanitizedToken));

            if (!empty($sanitizedToken)) {
                $output[] = $sanitizedToken;
            }
            $token = strtok(self::MAIL_SEPARATOR);
        }

        return $output;
    }

    private function loadSmsData(): array
    {
        /** @var string[] */
        $output = [];
        $data = $this->loadData(self::SMS_FILE);

        $token = strtok($data, self::SMS_SEPARATOR);
        while ($token !== false) {
            $sanitizedToken = $this->sanitize($token);

            $sanitizedToken = mb_trim(str_replace('\\\'', '\'', $sanitizedToken));

            if (!empty($sanitizedToken)) {
                $output[] = $sanitizedToken;
            }
            $token = strtok(self::SMS_SEPARATOR);
        }

        return $output;
    }

    private function loadSmsData2(): array
    {
        /** @var string[] */
        $output = [];
        $data = $this->loadData(self::SMS_FILE);

        $splitted = preg_split('/' . preg_quote(self::SMS_SEPARATOR, '/') . '/', $data);

        foreach ($splitted as $token) {
            $sanitizedToken = $this->sanitize($token);

            $sanitizedToken = mb_trim(str_replace('\\\'', '\'', $sanitizedToken));

            if (!empty($sanitizedToken)) {
                $output[] = $sanitizedToken;
            }
        }

        return $output;
    }

    private function loadSmsDataDev(): array
    {
        /** @var string[] */
        $output = [];
        $data = $this->loadData(self::SMS_FILE_DEV);

        $splitted = preg_split('/' . preg_quote(self::SMS_SEPARATOR_DEV, '/') . '/', $data);

        foreach ($splitted as $token) {
            $sanitizedToken = $this->sanitize($token);

            $sanitizedToken = mb_trim(str_replace('\\\'', '\'', $sanitizedToken));
            $sanitizedToken = preg_replace('/¶/', '', $sanitizedToken);
            $sanitizedToken = preg_replace('/^\|/', '', $sanitizedToken);
            $sanitizedToken = preg_replace('/\|$/', '', $sanitizedToken);
            $sanitizedToken = trim($sanitizedToken);

            if (!empty($sanitizedToken)) {
                $output[] = $sanitizedToken;
            }
        }

        return $output;
    }

    private function loadMixedTaLogData(): array
    {
        /** @var string[] */
        $output = [];
        $data = $this->loadData(self::MIXED_TA_LOG_FILE);

        $splitted = preg_split('/' . preg_quote(self::MIXED_TA_LOG_SEPARATOR, '/') . '/', $data);

        shuffle($splitted);

        foreach ($splitted as $token) {
            $sanitizedToken = $this->sanitize($token);

            $sanitizedToken = mb_trim(str_replace('\\\'', '\'', $sanitizedToken));
            $sanitizedToken = preg_replace('/¶/', '', $sanitizedToken);
            $sanitizedToken = preg_replace('/^\|/', '', $sanitizedToken);
            $sanitizedToken = preg_replace('/\|$/', '', $sanitizedToken);
            $sanitizedToken = preg_replace(['/\\n/', '/\\\n/', '/\\\\n/'], "\n", $sanitizedToken);
            $sanitizedToken = preg_replace('/\\t/', "\t", $sanitizedToken);
            $sanitizedToken = trim($sanitizedToken);

            if (!empty($sanitizedToken)) {
                $output[] = $sanitizedToken;
            }
        }

        return $output;
    }

    private function loadData(string $file): string
    {
        return file_get_contents($file);
    }
}