From 612962a37230053b9c4b659488b4890e157dd784 Mon Sep 17 00:00:00 2001 From: Nekrasov Ilya Date: Wed, 28 Mar 2018 15:39:11 +0300 Subject: [PATCH] Initial commit --- .gitignore | 5 + .scrutinizer.yml | 13 + LICENSE | 22 ++ README.md | 2 + composer.json | 20 ++ phpunit.xml | 18 ++ src/Step.php | 296 ++++++++++++++++++++++ src/StopStepException.php | 10 + src/StopSyncException.php | 10 + src/Sync.php | 499 ++++++++++++++++++++++++++++++++++++++ 10 files changed, 895 insertions(+) create mode 100644 .gitignore create mode 100644 .scrutinizer.yml create mode 100644 LICENSE create mode 100644 README.md create mode 100644 composer.json create mode 100644 phpunit.xml create mode 100644 src/Step.php create mode 100644 src/StopStepException.php create mode 100644 src/StopSyncException.php create mode 100644 src/Sync.php diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..0b27101 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +/vendor +composer.phar +composer.lock +.DS_Store +/.idea diff --git a/.scrutinizer.yml b/.scrutinizer.yml new file mode 100644 index 0000000..983ce4d --- /dev/null +++ b/.scrutinizer.yml @@ -0,0 +1,13 @@ +filter: + paths: + - 'src/*' + excluded_paths: + - 'vendor/*' + - 'tests/*' +tools: + php_cs_fixer: + config: { level: psr2 } +checks: + php: + code_rating: true + duplication: true \ No newline at end of file diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..c357b8a --- /dev/null +++ b/LICENSE @@ -0,0 +1,22 @@ +The MIT License (MIT) + +Copyright (c) 2015 Nekrasov Ilya + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + diff --git a/README.md b/README.md new file mode 100644 index 0000000..b5ae937 --- /dev/null +++ b/README.md @@ -0,0 +1,2 @@ +## Not for public usage + diff --git a/composer.json b/composer.json new file mode 100644 index 0000000..621deae --- /dev/null +++ b/composer.json @@ -0,0 +1,20 @@ +{ + "name": "arrilot/bitrix-sync", + "license": "MIT", + "keywords": [], + "authors": [ + { + "name": "Nekrasov Ilya", + "email": "nekrasov.ilya90@gmail.com" + } + ], + "homepage": "https://github.com/arrilot/bitrix-sync", + "require": { + "php": ">=5.6.9" + }, + "autoload": { + "psr-4": { + "Arrilot\\BitrixSync\\": "src/" + } + } +} diff --git a/phpunit.xml b/phpunit.xml new file mode 100644 index 0000000..c1d8bce --- /dev/null +++ b/phpunit.xml @@ -0,0 +1,18 @@ + + + + + tests + + + diff --git a/src/Step.php b/src/Step.php new file mode 100644 index 0000000..17217df --- /dev/null +++ b/src/Step.php @@ -0,0 +1,296 @@ +name ? $this->name : get_called_class(); + } + + /** + * @return string + */ + public function getStatus() + { + return $this->status; + } + + /** + * @return $this + */ + protected function logCurrentMemoryUsage() + { + $usage = memory_get_usage(true); + if ($usage < 1024) { + $this->logger->info('Current memory usage: ' . $usage . ' B'); + } elseif ($usage < 1024 * 1024) { + $this->logger->info('Current memory usage: ' . $usage / 1024 . ' KB'); + } else { + $this->logger->info('Current memory usage: ' . $usage / 1024 / 1024 . ' MB'); + } + + return $this; + } + + /** + * @return $this + */ + protected function logPeakMemoryUsage() + { + $usage = memory_get_peak_usage(true); + if ($usage < 1024) { + $this->logger->info('Peak memory usage: ' . $usage . ' B'); + } elseif ($usage < 1024 * 1024) { + $this->logger->info('Peak memory usage: ' . $usage / 1024 . ' KB'); + } else { + $this->logger->info('Peak memory usage: ' . $usage / 1024 / 1024 . ' MB'); + } + + return $this; + } + + /** + * @return $this + */ + protected function logSqlQueriesSinceStepStart() + { + if ($this->sqlLogsBitrix) { + $log = \Bitrix\Main\Application::getConnection()->getTracker()->getQueries(); + $count = count($log); + $totalTime = 0; + $details = []; + foreach ($log as $entry) { + $type = strtok(ltrim($entry->getSql()), ' '); + $type = strtolower(strtok($type, "\r\n" )); + if (!isset($details[$type])) { + $details[$type] = ['count' => 0, 'total_time' => 0]; + } + $details[$type]['count']++; + $details[$type]['total_time'] += $entry->getTime(); + $totalTime += $entry->getTime(); + } + $this->logger->info( + sprintf('SQL запросов bitrix с начала шага: %s, время выполнения %s сек.', $count, $totalTime), + $details + ); + } + + if ($this->sqlLogsIlluminate) { + $log = \Illuminate\Database\Capsule\Manager::getQueryLog(); + $count = count($log); + $totalTime = 0; + $details = []; + foreach ($log as $entry) { + $type = strtok($entry['query'], ' '); + if (!isset($details[$type])) { + $details[$type] = ['count' => 0, 'total_time' => 0]; + } + $details[$type]['count']++; + $details[$type]['total_time'] += $entry['time'] / 1000; + $totalTime += $entry['time'] / 1000; + } + $this->logger->info( + sprintf('SQL запросов illuminate/database с начала шага: %s, время выполнения %s сек.', $count, $totalTime), + $details + ); + } + + return $this; + } + + /** + * Эти методы позволяют вклиниться в этапы жизненного цикла шага. + * Выполняются эти методы именно в таком порядке. + * Сам метод peform выполняетс между onAfterLogStart и onBeforeLogFinish + */ + public function onBeforeLogStart() { } + public function onAfterLogStart() { } + public function onBeforeLogFinish() { } + public function onAfterLogFinish() { } + + /** + * Завершает шаг как пропущенный. + * @param string $message + * @throws StopStepException + */ + public function stopAsSkipped($message = '') + { + $this->status = 'skipped'; + throw new StopStepException($message); + } + + /** + * Завершает шаг как успешно завершенный. + * @param string $message + * @throws StopStepException + */ + public function stopAsFinished($message = '') + { + $this->status = 'finished'; + throw new StopStepException($message); + } + + /** + * Завершает шаг как проваленный. + * @param string $message + * @throws StopStepException + */ + public function stopAsFailed($message = '') + { + $this->status = 'failed'; + throw new StopStepException($message); + } + + /** + * Завершает всю синхронизацию. + * @param string $message + * @throws StopSyncException + */ + public function stopEverything($message = '') + { + $this->status = 'failed'; + throw new StopSyncException($message); + } + + /** + * @param Logger $logger + * @return $this + */ + public function setLogger(Logger $logger) + { + $this->logger = $logger; + + return $this; + } + + /** + * @param $data + * @return $this + */ + public function setSharedData(&$data) + { + $this->shared = &$data; + + return $this; + } + + /** + * @return $this + */ + public function logStart() + { + $this->startTime = microtime(true); + $this->logger->info('=============================================='); + $this->logger->info(sprintf('Шаг "%s" начат', $this->getName())); + + return $this; + } + + /** + * @return $this + */ + public function logFinish() + { + $this->logger->info(sprintf('Шаг "%s" завершён', $this->getName())); + $time = microtime(true) - $this->startTime; + if ($time > 60) { + $this->logger->info("Затраченное время: " . $time / 60 ." минут"); + } else { + $this->logger->info("Затраченное время: " . $time ." секунд"); + } + + $this->logSqlQueriesSinceStepStart(); + $this->flushSqlLogs(); + $this->logCurrentMemoryUsage(); + $this->logPeakMemoryUsage(); + + return $this; + } + + /** + * Установка параметров для логирования SQL запросов. + * + * @param $sqlLogsBitrix + * @param $sqlLogsIlluminate + * @return $this + */ + public function setSqlLoggingParams($sqlLogsBitrix, $sqlLogsIlluminate) + { + $this->sqlLogsBitrix = $sqlLogsBitrix; + $this->sqlLogsIlluminate = $sqlLogsIlluminate; + + return $this; + } + + /** + * Обнуление sql-трэкеров после завершения шага + */ + public function flushSqlLogs() + { + if ($this->sqlLogsBitrix) { + \Bitrix\Main\Application::getConnection()->getTracker()->reset(); + } + + if ($this->sqlLogsIlluminate) { + \Illuminate\Database\Capsule\Manager::flushQueryLog(); + } + } +} diff --git a/src/StopStepException.php b/src/StopStepException.php new file mode 100644 index 0000000..d48f6dd --- /dev/null +++ b/src/StopStepException.php @@ -0,0 +1,10 @@ +name = $name; + $this->env = $this->env(); + + $this->logDir = $this->logDir($name); + $this->logger = new Logger($this->name); + $this->formatterForLogger = new LineFormatter("[%datetime%] %level_name%: %message% %context%\n"); + $this->logFile = $this->logDir . '/' . date('Y_m_d_H_i_s') . '.log'; + $handler = (new StreamHandler($this->logFile))->setFormatter($this->formatterForLogger); + $this->logger->pushHandler($handler); + + $this->lockFile = $this->logDir . '/' . $this->name . '_is_in_process.lock'; + } + + /** + * Запуск синхронизации + */ + public function perform() + { + $this->logger->info('Синхронизация начата'); + $this->adjustPhpSettings(); + $this->preventOverlapping(); + $this->normalizeSteps(); + $this->validateSteps(); + + if ($this->sqlLogsBitrix) { + $this->doProfileSql(); + } + + foreach ($this->steps as $step) { + $step + ->setLogger($this->logger) + ->setSqlLoggingParams($this->sqlLogsBitrix, $this->sqlLogsIlluminate) + ->setSharedData($this->sharedData); + + $step->onBeforeLogStart(); + $step->logStart(); + $step->onBeforeLogStart(); + + try { + $step->perform(); + } catch (StopStepException $e) { + if ($step->getStatus() === 'failed') { + $status = 'проваленный'; + } elseif($step->getStatus() === 'skipped') { + $status = 'пропущенный'; + } else { + $status = 'успешный'; + } + $trace = $e->getTrace(); + $this->logger->info('Шаг завершён как '. $status . '.', [ + 'message' => $e->getMessage(), + 'class' => $trace[0]['class'], + 'line' => $trace[0]['line'], + ]); + + } catch (StopSyncException $e) { + $trace = $e->getTrace(); + $this->logger->info('Получена команда на завершение синхронизации.', [ + 'message' => $e->getMessage(), + 'class' => $trace[0]['class'], + 'line' => $trace[0]['line'], + ]); + $step->onBeforeLogFinish(); + $step->logFinish(); + $step->onAfterLogFinish(); + break; + } + + $step->onBeforeLogFinish(); + $step->logFinish(); + $step->onAfterLogFinish(); + + unset($step); + } + + $this->logger->info('=============================================='); + $this->logger->info('Синхронизация завершена'); + + $this->doEmailFinalLog(); + } + + /** + * Установка шагов синхронизации. + * + * @param array $steps + * @return $this + */ + public function setSteps(array $steps) + { + $this->steps = $steps; + + return $this; + } + + /** + * Перезапись названия окружения. + * + * @param $env + * @return $this + */ + public function setEnv($env) + { + $this->env = $env; + + return $this; + } + + /** + * Установка значения по-умолчанию для массива передаваемого из шага в шаг. + * + * @param array $data + * @return $this + */ + public function setSharedData(array $data) + { + $this->sharedData = $data; + + return $this; + } + + /** + * Включает дублирование логов в вывод symfony/console. + * Требуется дополнительный пакет `symfony/monolog-bridge` + * + * @param OutputInterface $output + * @return $this + */ + public function sendOutputToSymfonyConsoleOutput(OutputInterface $output) + { + if (!class_exists('\Symfony\Bridge\Monolog\Handler\ConsoleHandler')) { + throw new LogicException('Необходимо выполнить `composer require symfony/monolog-bridge` для использования данного метода'); + } + + $verbosityLevelMap = array( + OutputInterface::VERBOSITY_QUIET => Logger::ERROR, + OutputInterface::VERBOSITY_NORMAL => Logger::INFO, + OutputInterface::VERBOSITY_VERBOSE => Logger::INFO, + OutputInterface::VERBOSITY_VERY_VERBOSE => Logger::INFO, + OutputInterface::VERBOSITY_DEBUG => Logger::DEBUG, + ); + + $formatter = new ConsoleFormatter(['format' => "%datetime% %start_tag%%level_name%%end_tag% %message%%context%\n"]); + $handler = (new ConsoleHandler($output, true, $verbosityLevelMap))->setFormatter($formatter); + $this->logger->pushHandler($handler); + + return $this; + } + + /** + * Включает дублирование логов в echo + * + * @return $this + */ + public function sendOutputToEcho() + { + $handler = (new StreamHandler('php://stdout'))->setFormatter($this->formatterForLogger); + $this->logger->pushHandler($handler); + + return $this; + } + + /** + * Getter for logger + */ + public function getLogger() + { + return $this->logger; + } + + /** + * @param HandlerInterface $handler + * @return Sync + */ + public function pushLogHandler(HandlerInterface $handler) + { + $this->logger->pushHandler($handler); + + return $this; + } + + /** + * Директория куда попадают логи. + * Важно чтобы в неё не попадало ничего другого кроме файлов создаваемых данным классом + * иначе при очистке логов они могут быть удалены. + * + * @param $name + * @return string + */ + protected function logDir($name) + { + return logs_path('syncs/'.$name); + } + + /** + * Название окружения (dev/production). + * Используется в письмах для понимания откуда они пришли. + * + * @return string + */ + protected function env() + { + return class_exists('\Arrilot\DotEnv\DotEnv') ? \Arrilot\DotEnv\DotEnv::get('APP_ENV', 'production') : ''; + } + + /** + * Email с которого посылаются письма. + * + * @return string + */ + protected function emailFrom() + { + $emailFrom = Option::get('main', 'email_from'); + return $emailFrom ? $emailFrom : 'mail@greensight.ru'; + } + + /** + * Имя сайта которое будет отображено в письмах + * + * @return string + */ + protected function siteName() + { + return Option::get('main', 'site_name'); + } + + /** + * Метод для донастройки php/приложения перед началом синхронизации. + */ + protected function adjustPhpSettings() + { + $connection = \Bitrix\Main\Application::getConnection(); + + // выставим максимальный таймаут (8 часов) чтобы mysql не отваливался по нему в случае, + // если есть какой-то долгий шаг без запросов в базу. + $connection->query('SET wait_timeout=28800'); + if ($this->checkIlluminate()) { + \Illuminate\Database\Capsule\Manager::select('SET wait_timeout=28800'); + } + } + + /** + * Непосредственно включает профилирование SQL-запросов. + * @return $this + */ + protected function doProfileSql() + { + global $DB; + $connection = \Bitrix\Main\Application::getConnection(); + $connection->setTracker(null); + $connection->startTracker(true); + $DB->ShowSqlStat = true; + $DB->sqlTracker = $connection->getTracker(); + + if ($this->checkIlluminate()) { + \Illuminate\Database\Capsule\Manager::enableQueryLog(); + $this->sqlLogsIlluminate = true; + } + + return $this; + } + + /** + * @return bool + */ + protected function checkIlluminate() + { + return class_exists('Illuminate\Database\Capsule\Manager'); + } + + /** + * Выключает защиту от наложения синхронизаций друг на друга. + * + * @return $this + */ + public function allowOverlapping() + { + $this->allowOverlapping = true; + + return $this; + } + + /** + * @return $this + */ + public function profileSql() + { + $this->sqlLogsBitrix = true; + + return $this; + } + + /** + * Посылать ошибки на email. По-умолчанию посылает только критические + * + * @param array|string $emails + * @param int $level + * @return $this + */ + public function emailErrorsTo($emails, $level = Logger::ALERT) + { + $title = sprintf('%s, %s: ошибка синхронизации "%s"', $this->siteName(), $this->env, $this->name); + $handler = (new NativeMailerHandler($emails, $title, $this->emailFrom(), $level))->setFormatter($this->formatterForLogger); + $this->logger->pushHandler($handler); + + return $this; + } + + /** + * Послать окончательный лог синхронизации после её окончания на указанный email/email-ы. + * + * @param array|string $emails + * @return $this + */ + public function emailFinalLogTo($emails) + { + $this->emailsForFinalLog = is_array($emails) ? $emails : (array) $emails; + + return $this; + } + + /** + * Удаляет все логи старше чем $days дней. + * + * @param $days + */ + public function cleanOldLogs($days = 30) + { + $fileSystemIterator = new FilesystemIterator($this->logDir); + $now = time(); + foreach ($fileSystemIterator as $file) { + if ($now - $file->getCTime() >= 60 * 60 * 24 * $days) { + unlink($this->logDir . '/' . $file->getFilename()); + } + } + } + + /** + * Защита от наложения синхронизаций друг на друга. + */ + protected function preventOverlapping() + { + if ($this->allowOverlapping) { + return; + } + + // проверяем существование lock-файла + if (file_exists($this->lockFile)) { + $error = $this->env . ': файл '.$this->lockFile. ' уже существует. Импорт остановлен.'; + $this->logger->alert($error); + throw new RuntimeException($error); + } + + // создаем lock-файл + $fp = fopen($this->lockFile, "w"); + fclose($fp); + + // удаляем lock-файл при завершении скрипта + register_shutdown_function(function ($lockFile) { + unlink($lockFile); + }, $this->lockFile); + } + + /** + * Нормализация формата шагов. + */ + protected function normalizeSteps() + { + foreach ($this->steps as $i => $step) { + if (is_string($step)) { + $step = new $step; + $this->steps[$i] = $step; + } + } + } + + /** + * Валидация шагов синхронизации. + */ + protected function validateSteps() + { + $names = []; + foreach ($this->steps as $step) { + if (!$step instanceof Step) { + throw new LogicException(get_class($step) . ' is not an instance of ' . Step::class); + } + $names[] = get_class($step); + } + + foreach ($this->steps as $i => $step) { + foreach ($step->dependsOn as $depends) { + if (!in_array($depends, $names)) { + throw new LogicException(sprintf('There is no "%s" step in sync but "%s" depends on it', $depends, get_class($step))); + } + + if (array_search($depends, $names) > $i) { + throw new LogicException(sprintf('Step "%s" depends on step "%s" and must be run after it', get_class($step), $depends)); + } + } + } + } + + /** + * Непосредственно отправка лога с результатами синхронизации. + */ + protected function doEmailFinalLog() + { + if (!$this->emailsForFinalLog) { + return; + } + + $headers = + "From: ".$this->emailFrom().PHP_EOL. + "Content-Type: text/plain; charset=utf-8".PHP_EOL. + "Content-Transfer-Encoding: 8bit"; + + $subject = sprintf('%s, %s: синхронизация "%s" завершёна', $this->siteName, $this->env, $this->name); + $message = file_get_contents($this->logFile); + if (!$message) { + $message = 'Не удалось получить файл-лог ' . $this->logFile; + } + + mail(implode(',', $this->emailsForFinalLog), $subject, $message, $headers); + } +}