From e7e1a986953130e7f74296421d33ebf910be9522 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9my=20Munsch?= Date: Mon, 19 Apr 2021 10:30:11 +0200 Subject: [PATCH] CRM Migrate from old CRM (Le Gaulois) --- .env.example | 9 + .gitignore | 2 + MIG/CPlatosChiffrement.class.php | 69 +++++ MIG/Db.php | 96 ++++++ MIG/export.php | 90 ++++++ MIG/make_tmp_table.sql | 9 + MIG/migrate.php | 80 +++++ MIG/pwd.php | 89 ++++++ app/Console/Commands/migrateCrmUsers.php | 77 +++++ app/Exceptions/AbortRowImportException.php | 14 + app/Lib/MigrateOldData.php | 211 +++++++++++++ composer.json | 1 + config/database.php | 23 ++ .../migrations/LeGaulois2016Migration.php | 293 ++++++++++++++++++ 14 files changed, 1063 insertions(+) create mode 100644 MIG/CPlatosChiffrement.class.php create mode 100644 MIG/Db.php create mode 100644 MIG/export.php create mode 100644 MIG/make_tmp_table.sql create mode 100644 MIG/migrate.php create mode 100644 MIG/pwd.php create mode 100644 app/Console/Commands/migrateCrmUsers.php create mode 100644 app/Exceptions/AbortRowImportException.php create mode 100644 app/Lib/MigrateOldData.php create mode 100644 database/migrations/LeGaulois2016Migration.php diff --git a/.env.example b/.env.example index 7dc51e1..57565b6 100644 --- a/.env.example +++ b/.env.example @@ -14,6 +14,15 @@ DB_DATABASE=laravel DB_USERNAME=root DB_PASSWORD= +DB_CONNECTION_OLD=mysql +DB_HOST_OLD=127.0.0.1 +DB_PORT_OLD=3306 +DB_DATABASE_OLD=ldccrm_12_2020 +DB_USERNAME_OLD=root +DB_PASSWORD_OLD= + +MIGRATIONPAGINATION=20000 + BROADCAST_DRIVER=log CACHE_DRIVER=file QUEUE_CONNECTION=sync diff --git a/.gitignore b/.gitignore index 6dc44ac..8ec1046 100644 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,5 @@ Homestead.json Homestead.yaml npm-debug.log yarn-error.log +storage/crm_migration.csv +MIG/errors_sql.txt diff --git a/MIG/CPlatosChiffrement.class.php b/MIG/CPlatosChiffrement.class.php new file mode 100644 index 0000000..01de94e --- /dev/null +++ b/MIG/CPlatosChiffrement.class.php @@ -0,0 +1,69 @@ +setBlockLength(128); + // keys are null-padded to the closest valid size + // longer than the longest key and it's truncated + $cipher->setKeyLength(128); + $cipher->setKey($key); + // the IV defaults to all-NULLs if not explicitly defined + $cipher->setIV($iv); + $cipher->disablePadding(); + $length = strlen($text); + $pad = 32 - ($length % 32); + $text = str_pad($text, $length + $pad, chr(0)); + return base64_encode($cipher->encrypt($text)); + } + private function decryptRJ256($key, $iv, $text) + { + $cipher = new Rijndael('cbc'); // could use CRYPT_RIJNDAEL_MODE_CBC + $cipher->setBlockLength(128); + // keys are null-padded to the closest valid size + // longer than the longest key and it's truncated + $cipher->setKeyLength(128); + $cipher->setKey($key); + // the IV defaults to all-NULLs if not explicitly defined + $cipher->setIV($iv); + $cipher->disablePadding(); + return $cipher->decrypt(base64_decode($text)); + } +} diff --git a/MIG/Db.php b/MIG/Db.php new file mode 100644 index 0000000..0f47bbd --- /dev/null +++ b/MIG/Db.php @@ -0,0 +1,96 @@ + \PDO::ERRMODE_EXCEPTION)); + } catch (\Exception $e) { + die($e->getMessage()); + } + } + return self::$instance; + } + + public function request($statement, $params = false) + { + try { + $db = self::Instance(); + $query = $db->prepare($statement); + if ($params) { + $query->execute($params); + } else { + $query->execute(); + } + } catch (\PDOException $e) { + die($e->getMessage()); + } + $query->setFetchMode(\PDO::FETCH_ASSOC); + $result = $query->fetch(); + $query->closeCursor(); + return ($result); + } + + public function requestAll($statement, $params = false) + { + try { + $db = self::Instance(); + $query = $db->prepare($statement); + if ($params) { + $query->execute($params); + } else { + $query->execute(); + } + } catch (\PDOException $e) { + die($e->getMessage()); + } + $query->setFetchMode(\PDO::FETCH_ASSOC); + $result = $query->fetchAll(); + $query->closeCursor(); + return ($result); + } + + public function requestPush($statement, $params = false) + { + try { + $db = self::Instance(); + $query = $db->prepare($statement); + if ($params) + $query->execute($params); + else + $query->execute(); + return true; + } catch (\PDOException $e) { + die($e->getMessage()); + } + } + + public function requestPushInsert($statement, $params) + { + try { + $db = self::Instance(); + $query = $db->prepare($statement); + $query->execute($params); + return $db->lastInsertId(); + } catch (\PDOException $e) { + file_put_contents("errors_sql.txt", implode(',', $params) . PHP_EOL, FILE_APPEND); + } + } +} diff --git a/MIG/export.php b/MIG/export.php new file mode 100644 index 0000000..d3777b6 --- /dev/null +++ b/MIG/export.php @@ -0,0 +1,90 @@ +requestAll("SELECT * +FROM useraccount +INNER JOIN siteuser su ON useraccount.user_id = su.siteuser_useraccountId +WHERE su.siteuser_siteId = :siteid +AND user_status = 1 +AND user_lastUpdateDate >= '2018-03-01' +AND (user_emailCheckStatus <> 0 OR user_emailCheckStatus IS NULL) +AND (user_lastCnxDateHeure IS NOT NULL AND user_lastCnxDateHeure <> '0000-00-00 00:00:00')", ['siteid' => $siteId]); + +foreach ($accounts as $account) { + $isSingleOptin = false; + + $oldCRMid = $account['user_id']; + + $civility = $account['user_civilite']; + $firstname = $account['user_prenom']; + $lastname = $account['user_nom']; + + $birthdate = $account['user_dateNaissance']; + + $address1 = !empty($account['user_adresse1']) ? $account['user_adresse1'] : null; + $address2 = !empty($account['user_adresse2']) ? $account['user_adresse2'] : null; + $postcode = (!empty($account['user_codePostal']) && $account['user_codePostal'] !== 0 ? $account['user_codePostal'] : null); + $city = !empty($account['user_ville']) ? $account['user_ville'] : null; + + $optin = $account['user_inscritNewsletter']; + $haschcode = $account['user_hashcode']; + $status = $account['user_status']; + + $created_at = $account['user_creationDate']; + $updated_at = $account['user_lastUpdateDate']; + + $email = $account['user_email']; + $password = CPlatosChiffrement::decrypt($account['user_pwd']); + + $last_cnx = $account['user_lastCnxDateHeure']; + + if (empty($firstname) && empty($lastname)) + $isSingleOptin = true; + + // Récupération des champs complémentaires + $fields = [['fieldkey' => 'created_at', 'fieldvalue' => $created_at], ['fieldkey' => 'updated_at', 'fieldvalue' => $updated_at], ['fieldkey' => 'last_cnx', 'fieldvalue' => $last_cnx]]; + + if (!$isSingleOptin) { + $crmUser = $apiCrm->createCustomer($email, $password, $firstname, $lastname, $optin, $birthdate, null, $address1, $address2, $city, $postcode, "France"); + if ($crmUser->status === 'OK') { + $newCRMId = $crmUser->user->crm_id; + $activate = $apiCrm->activateCustomer($newCRMId); + if ($activate->status === 'NOK') { + file_put_contents("Export_errors.txt", "OLD CRM ID : $oldCRMid - NEW CRM ID : $newCRMId - ACTIVATE API ERROR : " . $activate->code . " - " . $activate->message . PHP_EOL, FILE_APPEND); + } + } else { + file_put_contents("Export_errors.txt", "OLD CRM ID : $oldCRMid - API ERROR : " . $crmUser->code . " - " . $crmUser->message . PHP_EOL, FILE_APPEND); + continue; + } + } else { + $result = $apiCrm->createSingleOptin($email); + + if ($result->status === 'OK') { + $newCRMId = "SINGLEOPTIN"; + } else { + file_put_contents("Export_errors.txt", "OLD CRM ID : $oldCRMid - API ERROR : " . $crmUser->code . " - " . $crmUser->message . PHP_EOL, FILE_APPEND); + continue; + } + } + write("$oldCRMid;$newCRMId;$email;" . json_encode($fields) . PHP_EOL, $siteName); +} + +function write($text, $siteName) +{ + if (!file_exists("Export_$siteName.csv")) + file_put_contents("Export_$siteName.csv", "\xEF\xBB\xBF", FILE_APPEND); // BOM UTF-8 + file_put_contents("Export_$siteName.csv", $text, FILE_APPEND); +} diff --git a/MIG/make_tmp_table.sql b/MIG/make_tmp_table.sql new file mode 100644 index 0000000..54f2600 --- /dev/null +++ b/MIG/make_tmp_table.sql @@ -0,0 +1,9 @@ +CREATE TABLE `legaulois_migration` ( + `id` BIGINT(20) NOT NULL AUTO_INCREMENT, + `user_id` BIGINT(20) NOT NULL DEFAULT '0', + `pwd` VARCHAR(255) NOT NULL DEFAULT '0', + PRIMARY KEY (`id`) +) +COLLATE='utf8_unicode_ci' +ENGINE=InnoDB +AUTO_INCREMENT=0; diff --git a/MIG/migrate.php b/MIG/migrate.php new file mode 100644 index 0000000..588a177 --- /dev/null +++ b/MIG/migrate.php @@ -0,0 +1,80 @@ +fieldkey){ + case 'created_at': + if (!$isSingleOptin) { + $query = "UPDATE customers SET created_at = :nvalue WHERE id = :newid"; + $db->requestPush($query, ['nvalue' => $otherField->fieldvalue, 'newid' => $newCrmId]); + $query = "UPDATE customer_infos SET created_at = :nvalue WHERE customer_id = :newid AND brand_id = :brandid"; + $db->requestPush($query, ['nvalue' => $otherField->fieldvalue, 'newid' => $newCrmId, 'brandid' => $newBrandId]); + $query = "UPDATE customer_login SET created_at = :nvalue WHERE customer_id = :newid AND brand_id = :brandid"; + $db->requestPush($query, ['nvalue' => $otherField->fieldvalue, 'newid' => $newCrmId, 'brandid' => $newBrandId]); + $query = "UPDATE customer_opt_in_history SET created_at = :nvalue WHERE customer_id = :newid AND brand_id = :brandid"; + $db->requestPush($query, ['nvalue' => $otherField->fieldvalue, 'newid' => $newCrmId, 'brandid' => $newBrandId]); + }else{ + $query = "UPDATE single_optin SET created_at = :nvalue, updated_at = :nvalue WHERE email = :email AND brand_id = :brandid"; + $db->requestPush($query, ['nvalue' => $otherField->fieldvalue, 'email' => $email, 'brandid' => $newBrandId]); + } + break; + case 'updated_at': + if (!$isSingleOptin) { + $query = "UPDATE customers SET updated_at = :nvalue WHERE id = :newid"; + $db->requestPush($query, ['nvalue' => $otherField->fieldvalue, 'newid' => $newCrmId]); + $query = "UPDATE customer_infos SET updated_at = :nvalue WHERE customer_id = :newid AND brand_id = :brandid"; + $db->requestPush($query, ['nvalue' => $otherField->fieldvalue, 'newid' => $newCrmId, 'brandid' => $newBrandId]); + $query = "UPDATE customer_login SET updated_at = :nvalue WHERE customer_id = :newid AND brand_id = :brandid"; + $db->requestPush($query, ['nvalue' => $otherField->fieldvalue, 'newid' => $newCrmId, 'brandid' => $newBrandId]); + $query = "UPDATE customer_opt_in_history SET updated_at = :nvalue WHERE customer_id = :newid AND brand_id = :brandid"; + $db->requestPush($query, ['nvalue' => $otherField->fieldvalue, 'newid' => $newCrmId, 'brandid' => $newBrandId]); + } + break; + case 'last_cnx': + if (!$isSingleOptin) { + $query = "UPDATE customer_login SET last_connection = :nvalue WHERE customer_id = :newid AND brand_id = :brandid"; + $db->requestPush($query, ['nvalue' => $otherField->fieldvalue, 'newid' => $newCrmId, 'brandid' => $newBrandId]); + } + break; + + } + //var_dump($otherField); + } +} diff --git a/MIG/pwd.php b/MIG/pwd.php new file mode 100644 index 0000000..f1b43f2 --- /dev/null +++ b/MIG/pwd.php @@ -0,0 +1,89 @@ +requestPush("CREATE TABLE IF NOT EXISTS $table ( + `id` BIGINT(20) NOT NULL AUTO_INCREMENT, + `user_id` BIGINT(20) NOT NULL DEFAULT '0', + `firstname` VARCHAR(255) NOT NULL DEFAULT '0', + `lastname` VARCHAR(255) NOT NULL DEFAULT '0', + `birthdate` VARCHAR(255) NULL DEFAULT '0', + `address` VARCHAR(255) NULL DEFAULT '0', + `address2` VARCHAR(255) NULL DEFAULT '0', + `city` VARCHAR(255) NULL DEFAULT '0', + `postcode` VARCHAR(255) NULL DEFAULT '0', + `optin` INT(11) NOT NULL DEFAULT '0', + `email` VARCHAR(255) NOT NULL DEFAULT '0', + `pwd` VARCHAR(255) NOT NULL DEFAULT '0', + `created_at` TIMESTAMP NOT NULL, + `updated_at` TIMESTAMP NOT NULL, + `last_cnx` TIMESTAMP NOT NULL, + PRIMARY KEY (`id`) +) COLLATE='utf8_unicode_ci' ENGINE=InnoDB AUTO_INCREMENT=0;"); + +$getAccounts = function ($skip) use ($db, $siteId, $maxdate){ + return $db->requestAll("SELECT * +FROM useraccount +INNER JOIN siteuser su ON useraccount.user_id = su.siteuser_useraccountId +WHERE su.siteuser_siteId = :siteid +AND user_id >= 701697 +AND user_status = 1 +AND user_lastUpdateDate >= :maxdate +AND (user_emailCheckStatus <> 0 OR user_emailCheckStatus IS NULL) +AND (user_lastCnxDateHeure IS NOT NULL AND user_lastCnxDateHeure <> '0000-00-00 00:00:00') +AND user_lastCnxDateHeure >= :maxdate +AND user_email IS NOT NULL LIMIT 4600 OFFSET ". ($skip * 4600), ['siteid' => $siteId, 'maxdate' => $maxdate]); +}; +$k = 0; +while (count($accounts = $getAccounts($k++))) { + $sql = []; + $toto = ""; + foreach ($accounts as $account){ + $crm_id = $account['user_id']; + $civility = $account['user_civilite']; + $firstname = ucfirst($account['user_prenom']); + $lastname = ucfirst($account['user_nom']); + $birthdate = $account['user_dateNaissance']; + $address1 = !empty($account['user_adresse1']) ? $account['user_adresse1'] : null; + $address2 = !empty($account['user_adresse2']) ? $account['user_adresse2'] : null; + $postcode = (!empty($account['user_codePostal']) && $account['user_codePostal'] !== 0 ? $account['user_codePostal'] : null); + $city = !empty($account['user_ville']) ? $account['user_ville'] : null; + $optin = $account['user_inscritNewsletter']; + $haschcode = $account['user_hashcode']; + $status = $account['user_status']; + $created_at = $account['user_creationDate']; + $updated_at = $account['user_lastUpdateDate']; + $email = strtolower($account['user_email']); + //$password = CPlatosChiffrement::decrypt($account['user_pwd']); + $password = "toto"; + $last_cnx = $account['user_lastCnxDateHeure']; + $datas = [ + $crm_id, + $firstname, + $lastname, + $birthdate, + $address1, + $address2, + $city, + $postcode, + $optin, + $email, + $password, + $created_at, + $updated_at, + $last_cnx + ]; + $toto = "(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"; + $db->requestPushInsert("INSERT INTO $table (user_id, firstname, lastname, birthdate, address, address2, city, postcode, optin, email, pwd, created_at, updated_at, last_cnx) VALUES $toto", $datas); + } +} diff --git a/app/Console/Commands/migrateCrmUsers.php b/app/Console/Commands/migrateCrmUsers.php new file mode 100644 index 0000000..da55ab4 --- /dev/null +++ b/app/Console/Commands/migrateCrmUsers.php @@ -0,0 +1,77 @@ +ask('What is the tmp table name with users data ?'); + $table = 'legaulois_migration'; + + try { + DB::connection(config('database.oldDB'))->table($table)->exists(); + } catch (Exception $e) { + $this->error(sprintf('%s table not found on old crm database', $table)); + return -1; + } + + $brandname = $this->ask('What is the brand name ?'); + $brand = Brand::where('name', 'LIKE', "%$brandname%")->first(); + if (!$brand) { + if (!$this->confirm(sprintf('Would you like to use %s ? ', $brandname))) { + $this->info('Command cancelled'); + return 0; + } + if ($this->confirm('Would you like to create a new one ?')) { + $brand = new Brand(); + $brand->name = $brandname; + $brand->save(); + } else { + $this->info('Command cancelled'); + return 0; + } + } + + $mgOldData = new MigrateOldData(); + $mgOldData->runMigration($brand, $table); + } catch (Exception $e) { + $this->error(sprintf('Failed %s', $e)); + return -1; + } + return 0; + } +} diff --git a/app/Exceptions/AbortRowImportException.php b/app/Exceptions/AbortRowImportException.php new file mode 100644 index 0000000..bda67ee --- /dev/null +++ b/app/Exceptions/AbortRowImportException.php @@ -0,0 +1,14 @@ + 2020) + * @author Jérémy Munsch + */ +class AbortRowImportException extends Exception +{ +} diff --git a/app/Lib/MigrateOldData.php b/app/Lib/MigrateOldData.php new file mode 100644 index 0000000..38eb886 --- /dev/null +++ b/app/Lib/MigrateOldData.php @@ -0,0 +1,211 @@ +table = $table; + $this->brand = $brand; + $this->instruct(); + $this->up(); + } + + public function instruct(): array + { + $this->initFile(); + $cleanDegeu = function (string $str) { + return html_entity_decode( + trim(str_replace('\\', '', ucwords(stripcslashes(strtolower($str))))), + ENT_QUOTES | ENT_HTML5, + 'UTF-8' + ); + }; + return [ + 'Users' => [ + 'table' => $this->table, + 'model' => Customer::class, + 'comment' => sprintf('Migrating %s customers', $this->brand->name), + 'bind' => [ + 'email' => [ + 'apply' => function ($v, $model) use ($cleanDegeu) { + $v = \mb_convert_encoding($v, "UTF-8"); + $v = explode('#|#', $v); + $oldId = $v[0]; + $first_name = $cleanDegeu($v[1]); + $last_name = $cleanDegeu($v[2]); + $birth_date = $v[3]; + $address = $cleanDegeu($v[4]); + $address2 = $cleanDegeu($v[5]); + $city = $cleanDegeu($v[6]); + $postcode = $v[7]; + $user_inscritNewsletter = $v[8]; + $email = strtolower($v[9]); + $pwd = $v[10]; + $created_at = $v[11]; + $updated_at = $v[12]; + $last_cnx = $v[13]; + + // Just save Single optin + if (empty($first_name) && empty($last_name)) { + $sglOptin = SingleOptin::where('email', strtolower($email)) + ->where('brand_id', $this->brand->id)->firstOrNew(); + $sglOptin->email = strtolower($email); + $sglOptin->brand_id = $this->brand->id; + $sglOptin->save(); + } else { + // Validate Data + $d = [ + 'first_name' => $first_name, + 'last_name' => $last_name, + 'address' => $address, + 'address2' => $address2, + 'city' => $city, + 'postcode' => $postcode, + ]; + + $validator = Validator::make($d, [ + 'first_name' => 'min:2', + 'last_name' => 'min:2' + ]); + + if ($validator->fails()) { + throw new AbortRowImportException(); + } + + $rules = [ + 'address' => 'string|min:2', + 'address2' => 'string|min:2', + 'city' => 'string|min:2', + 'postcode' => 'digits_between:4,5' + ]; + + $validate = function (array $data, array $rule) { + return Validator::make($data, $rule)->fails(); + }; + + foreach ($rules as $key => $rule) { + if ($validate([$key => $d[$key]], [$key => $rule])) { + ${$key} = null; + } + } + + $model = Customer::where('email', $email)->firstOrNew(); + $model->email = $email; + $model->created_at = $created_at; + $model->updated_at = $updated_at; + $model->save(); + + // filtre birthdate puisque c'est du sale dans la vielle crm + try { + $expl = explode('-', $birth_date); + $d = Carbon::parse($birth_date); + if ($d->toDateString() === sprintf('%s-%s-%s', $expl[2] ?? '', $expl[1] ?? '', $expl[0] ?? '')) { + $birth_date = $d->toDateString(); + } + } catch (InvalidFormatException $e) { + $birth_date = null; + } + + + if ($birth_date === '0000-00-00') { + $birth_date = null; + } + + $city = !empty($city) ? $city : null; + $postcode = !empty($postcode) ? $postcode : null; + $address = !empty($address) ? $address : null; + $address2 = !empty($address2) ? $address2 : null; + + $cInfo = new CustomerInfos(); + $cInfo->hashcode = Str::uuid(); + $cInfo->firstname = ucfirst(strtolower($first_name)); + $cInfo->lastname = ucfirst(strtolower($last_name)); + $cInfo->birth_date = strlen($birth_date) ? $birth_date : null; + $cInfo->address = $address; + $cInfo->address2 = $address2; + $cInfo->city = $city; + $cInfo->postcode = $postcode; + $cInfo->opt_in = $user_inscritNewsletter ? true : false; + $cInfo->brand_id = $this->brand->id; + $cInfo->customer_id = $model->id; + $cInfo->created_at = $created_at; + $cInfo->updated_at = $updated_at; + $cInfo->save(); + + $cLogin = new CustomerLogin(); + $cLogin->password = Hash::make(trim($pwd)); + $cLogin->brand_id = $this->brand->id; + $cLogin->customer_id = $model->id; + $cLogin->status = true; + $cLogin->created_at = $created_at; + $cLogin->updated_at = $updated_at; + $cLogin->last_connection = $last_cnx; + $cLogin->save(); + + $this->write( + [ + 'new' => $model->id, + 'old' => $oldId, + 'hashcode' => $cInfo->hashcode, + 'created_at' => $created_at, + 'updated_at' => $updated_at + ] + ); + } + throw new AbortRowImportException(); + }, + 'name' => [ + 'user_id', + 'firstname', + 'lastname', + 'birthdate', + 'address', + 'address2', + 'city', + 'postcode', + 'optin', + 'email', + 'pwd', + 'created_at', + 'updated_at', + 'last_cnx' + ] + ], + ] + ], + ]; + } + + public function initFile() + { + File::put(storage_path('crm_migration.csv'), sprintf('new;old;hashcode;created_at;updated_at') . PHP_EOL); + } + + public function write(array $data) + { + foreach ($data as &$d) { + $d = sprintf('"%s"', $d); + } + File::append(storage_path('crm_migration.csv'), implode(';', $data) . PHP_EOL); + } +} diff --git a/composer.json b/composer.json index f298c95..d20e776 100644 --- a/composer.json +++ b/composer.json @@ -39,6 +39,7 @@ "psr-4": { "App\\": "app/", "Database\\Factories\\": "database/factories/", + "Database\\migrations\\": "database/migrations/", "Database\\Seeders\\": "database/seeders/" } }, diff --git a/config/database.php b/config/database.php index b42d9b3..eae8298 100644 --- a/config/database.php +++ b/config/database.php @@ -33,6 +33,9 @@ return [ | */ + 'oldDB' => 'oldcrm', + 'newDB' => 'mysql', + 'connections' => [ 'sqlite' => [ @@ -63,6 +66,26 @@ return [ ]) : [], ], + 'oldcrm' => [ + 'driver' => 'mysql', + 'url' => env('DATABASE_URL_OLD'), + 'host' => env('DB_HOST_OLD', '127.0.0.1'), + 'port' => env('DB_PORT_OLD', '3306'), + 'database' => env('DB_DATABASE_OLD', 'forge'), + 'username' => env('DB_USERNAME_OLD', 'forge'), + 'password' => env('DB_PASSWORD_OLD', ''), + 'unix_socket' => env('DB_SOCKET_OLD', ''), + 'charset' => 'utf8mb4', + 'collation' => 'utf8mb4_unicode_ci', + 'prefix' => '', + 'prefix_indexes' => true, + 'strict' => true, + 'engine' => null, + 'options' => extension_loaded('pdo_mysql') ? array_filter([ + PDO::MYSQL_ATTR_SSL_CA => env('MYSQL_ATTR_SSL_CA'), + ]) : [], + ], + 'pgsql' => [ 'driver' => 'pgsql', 'url' => env('DATABASE_URL'), diff --git a/database/migrations/LeGaulois2016Migration.php b/database/migrations/LeGaulois2016Migration.php new file mode 100644 index 0000000..7de6c6d --- /dev/null +++ b/database/migrations/LeGaulois2016Migration.php @@ -0,0 +1,293 @@ + + * Examples + *[ + * 'content_ocb' => [ + * 'model' => OCBContent::class, + * 'bind' => [ + * 'id' => 'ocb_id', + * 'picture' => 'ocb_img', + * 'content' => 'ocb_content', + * 'status' => 'ocb_is_published', + * 'type' => [ + * 'apply' => function ($value) { + * switch ($value) { + * case 'VOLAILLE': + * return OCBType::poultry(); + * case 'LAPIN': + * return OCBType::rabbit(); + * case 'OEUF': + * return OCBType::egg(); + * } + * }, + * 'name' => 'ocb_type', + * 'default' => '' + * ], + * 'order' => 'ocb_position', + * 'created_at' => [ + * 'name' => '', + * 'default' => date('Y-m-d H:i:s') + * ], + * 'updated_at' => [ + * 'name' => '', + * 'default' => date('Y-m-d H:i:s') + * ] + * ] + * ], + * 'Recipe ratings' => [ + * 'table' => 'recipe', + * 'comment' => 'Add ratings to recipes', + * 'model' => Rating::class, + * 'bind' => [ + * 'anyprop' => [ + * 'apply' => function ($recipe_id) { + * $recipe_rating = $this->oldDB->table('recipe_rating') + * ->select('recipe_rating_value') + * ->where('recipe_rating_recipe_id', '=', $recipe_id) + * ->get(); + * $count = $recipe_rating->count(); + * $sum = $recipe_rating->sum(function ($obj) { + * return $obj->recipe_rating_value; + * }); + * $this->newDB->table('ratings') + * ->insert([ + * 'total' => $sum, + * 'count' => $count, + * 'rating' => $count ? floatval($sum / $count) : 0, + * 'recipes_id' => $recipe_id, + * 'created_at' => date('Y-m-d H:i:s'), + * 'updated_at' => date('Y-m-d H:i:s'), + * ]); + * throw new AbortRowImportException(); + * }, + * 'name' => 'recipe_id' + * ] + * ] + * ], + *] + * + * @param array $instructions + * @return void + */ + + +/** + * Helps to migration from legaulois2016 to legaulois2020 + * @author Jérémy Munsch + */ +abstract class LeGaulois2016Migration extends Migration +{ + + /** @var ConsoleOutput */ + private $output = null; + + /** @var ConsoleSectionOutput */ + private $outputSection = null; + + /** @var \Illuminate\Database\ConnectionInterface */ + private $oldDB = null; + + /** @var \Illuminate\Database\ConnectionInterface */ + private $newDB = null; + + /** @var array */ + private $instructions = null; + + public function __construct() + { + $oldDB = config(sprintf('database.connections.%s', config('database.oldDB'))); + $newDB = config(sprintf('database.connections.%s', config('database.newDB'))); + if (!$oldDB) { + throw new Exception(sprintf('Unknown database connection %s for newDB', config('database.newDB'))); + } + if (!$newDB) { + throw new Exception(sprintf('Unknown database connection %s for oldDB', config('database.oldDB'))); + } + $this->oldDB = DB::connection(config('database.oldDB')); + $this->newDB = DB::connection(config('database.newDB')); + } + + public function up() + { + $this->instructions = $this->instruct(); + $this->processData(); + } + + public function down() + { + // Do nothing + } + + public function __get(string $name) + { + if ($name === 'oldDB') { + return $this->oldDB; + } + if ($name === 'newDB') { + return $this->newDB; + } + } + + /** + * Set migration instructions to execute on "up" only + * @param array $instructions + * see Instructions in \App\Database\Migrations\LeGaulois2016Migration + */ + abstract public function instruct(): array; + + /** + * Core execution on instruction + * + * @return void + */ + private function processData() + { + $getOldData = function ($keyOrKeys, $array) { + if (is_array($keyOrKeys)) { + $o = []; + foreach ($keyOrKeys as $key) { + $o[] = strval($array[$key]); + } + return implode('#|#', $o); + } + return $array[$keyOrKeys] ?? null; + }; + + foreach ($this->instructions as $oldTableName => $modelData) { + $migrationName = $oldTableName; + $oldTableName = $modelData['table'] ?? $oldTableName; + $rowNbrs = $this->oldDB->table($oldTableName)->count(); + $rowInserted = 0; + + $this->consoleLog($this->renderProgress($migrationName, $rowInserted, $rowNbrs)); + + $i = 0; + $page = 1; + $perPage = env('MIGRATIONPAGINATION', 300); + do { + $results = $this->oldDB->table($oldTableName)->paginate($perPage, ['*'], 'page', $page); + $i += $perPage; + $page++; + foreach ($results as $oldData) { + $model = null; + try { // Try to get exisiting model ID (to avoid SQL Integrity violations doing delete and create) + $model = $modelData['model']::findOrFail( + $getOldData($modelData['bind']['id'] ?? 0, (array)$oldData) + ); + } catch (ModelNotFoundException $e) { + $model = new $modelData['model'](); + } + try { // bind each property to it's data and save + foreach ($modelData['bind'] as $modelAttr => $oldModelAttr) { + if (is_array($oldModelAttr)) { + if (array_key_exists('apply', $oldModelAttr)) { + $argC = (new ReflectionFunction($oldModelAttr['apply']))->getNumberOfParameters(); + $argV = [ + $getOldData($oldModelAttr['name'], (array)$oldData) + ]; + if ($argC > 1) { + $argV[] = $model; + } + $model[$modelAttr] = + call_user_func_array($oldModelAttr['apply'], $argV) ?? + $oldModelAttr['default'] ?? + null; + } else { + $model[$modelAttr] = + $getOldData($oldModelAttr['name'], (array)$oldData) ?? + $oldModelAttr['default'] ?? + null; + } + } else { + $model[$modelAttr] = ((array)$oldData)[$oldModelAttr]; + } + } + $model->save(); + } catch (QueryException $e) { + File::append(\storage_path('error_migration.txt'), implode(',', $e->getBindings()) . PHP_EOL); + } catch (AbortRowImportException $e) { + // DO nothing + } finally { + ++$rowInserted; + if (rand(0, 15) == 10) { // prevent screen ficklering + $this->consoleLog($this->renderProgress($migrationName, $rowInserted, $rowNbrs)); + } + } + } + } while ($i <= $rowNbrs); + if (array_key_exists('comment', $modelData)) { + $migrationName .= sprintf(' <%s> ', $modelData['comment']); + } + $terminal = new Terminal(); + $consoleLength = $terminal->getWidth(); + + $r = round(80 * 100 / $consoleLength) - strlen($migrationName); + $r = $r >= 0 ? $r : 0; + $this->consoleLog(sprintf('%s%smigrated', $migrationName, str_repeat(' ', $r))); + $this->consoleLog("", true); // Cariage return + } + } + + /** + * Prints a message to the console + * + * @param string $message + * @param boolean $newLine + * @return void + */ + private function consoleLog(string $message, bool $newLine = false) + { + if (!$this->output) { + $this->output = new ConsoleOutput(); + } + if ($newLine) { + $this->outputSection = null; + $this->output->write(sprintf("%s", $message)); + } else { + if (!$this->outputSection) { + $this->outputSection = $this->output->section(); + } + $this->outputSection->overwrite(sprintf('%s', $message)); + } + } + + /** + * Render the progressbar to a string + * + * @param string $oldTableName + * @param integer $count + * @param integer $total + * @return string + */ + private function renderProgress(string $oldTableName, int $count, int $total) + { + $total = !$total ? 1 : $total; + $oldTableName = sprintf('%s ', $oldTableName); + $percent = $count * 100 / $total; + $percentText = sprintf(' [%d of %d]', $count, $total); + $terminal = new Terminal(); + $consoleLength = $terminal->getWidth(); + $bar = $consoleLength - strlen($oldTableName) - strlen($percentText); + $bar = $bar < 0 ? 0 : $bar; + $prog = round($percent / 100 * $bar); + $rep = $bar - $prog; + return $oldTableName . str_repeat('#', $prog) . str_repeat('_', $rep >= 0 ? $rep : 0) . $percentText; + } +}