diff --git a/api/.env b/api/.env index 8caf27abc..06c02c64d 100644 --- a/api/.env +++ b/api/.env @@ -30,3 +30,8 @@ DATABASE_URL="postgresql://db_user:db_password@127.0.0.1:5432/db_name?serverVers ###> nelmio/cors-bundle ### CORS_ALLOW_ORIGIN='^https?://(localhost|127\.0\.0\.1)(:[0-9]+)?$' ###< nelmio/cors-bundle ### + +###> symfony/sendinblue-mailer ### +# MAILER_DSN=sendinblue+api://KEY@default +# MAILER_DSN=sendinblue+smtp://USERNAME:PASSWORD@default +###< symfony/sendinblue-mailer ### diff --git a/api/migrations/Version20230228084820.php b/api/migrations/Version20230228084820.php index 7255ac41a..51a9a8365 100644 --- a/api/migrations/Version20230228084820.php +++ b/api/migrations/Version20230228084820.php @@ -31,7 +31,6 @@ public function up(Schema $schema): void public function down(Schema $schema): void { // this down() migration is auto-generated, please modify it to your needs - $this->addSql('CREATE SCHEMA public'); $this->addSql('ALTER TABLE cronjob DROP reference'); $this->addSql('ALTER TABLE cronjob DROP version'); $this->addSql('ALTER TABLE collection_entity DROP reference'); diff --git a/api/migrations/Version20230228095524.php b/api/migrations/Version20230228095524.php index 31c68443d..b81793b5c 100644 --- a/api/migrations/Version20230228095524.php +++ b/api/migrations/Version20230228095524.php @@ -27,7 +27,6 @@ public function up(Schema $schema): void public function down(Schema $schema): void { // this down() migration is auto-generated, please modify it to your needs - $this->addSql('CREATE SCHEMA public'); $this->addSql('ALTER TABLE endpoint DROP reference'); $this->addSql('ALTER TABLE endpoint DROP version'); } diff --git a/api/migrations/Version20230303141510.php b/api/migrations/Version20230303141510.php index 822b540f6..2c2afd818 100644 --- a/api/migrations/Version20230303141510.php +++ b/api/migrations/Version20230303141510.php @@ -27,7 +27,6 @@ public function up(Schema $schema): void public function down(Schema $schema): void { // this down() migration is auto-generated, please modify it to your needs - $this->addSql('CREATE SCHEMA public'); $this->addSql('ALTER TABLE gateway DROP endpoints_config'); } } diff --git a/api/migrations/Version20230309160743.php b/api/migrations/Version20230309160743.php index d3ee7bebe..3da4c568c 100644 --- a/api/migrations/Version20230309160743.php +++ b/api/migrations/Version20230309160743.php @@ -27,7 +27,6 @@ public function up(Schema $schema): void public function down(Schema $schema): void { // this down() migration is auto-generated, please modify it to your needs - $this->addSql('CREATE SCHEMA public'); $this->addSql('ALTER TABLE application DROP configuration'); } } diff --git a/api/migrations/Version20230328151236.php b/api/migrations/Version20230328151236.php index 3595cc2e6..aecf9f48b 100644 --- a/api/migrations/Version20230328151236.php +++ b/api/migrations/Version20230328151236.php @@ -52,7 +52,6 @@ public function up(Schema $schema): void public function down(Schema $schema): void { // This down() migration is auto-generated, please modify it to your needs. - $this->addSql('CREATE SCHEMA public'); $this->addSql('ALTER TABLE security_group DROP reference'); $this->addSql('ALTER TABLE security_group DROP version'); $this->addSql('ALTER TABLE "user" DROP reference'); diff --git a/api/migrations/Version20230504111926.php b/api/migrations/Version20230504111926.php index 45bcf9466..9cb7cad35 100644 --- a/api/migrations/Version20230504111926.php +++ b/api/migrations/Version20230504111926.php @@ -29,7 +29,6 @@ public function up(Schema $schema): void public function down(Schema $schema): void { // this down() migration is auto-generated, please modify it to your needs - $this->addSql('CREATE SCHEMA public'); $this->addSql('DROP TABLE "gateway_audit_trail"'); $this->addSql('ALTER TABLE audit_trail ALTER id TYPE UUID'); $this->addSql('ALTER TABLE audit_trail ALTER id DROP DEFAULT'); diff --git a/api/migrations/Version20230602151620.php b/api/migrations/Version20230602151620.php index e8b03f5d0..a6a6a49cd 100644 --- a/api/migrations/Version20230602151620.php +++ b/api/migrations/Version20230602151620.php @@ -26,7 +26,6 @@ public function up(Schema $schema): void public function down(Schema $schema): void { // this down() migration is auto-generated, please modify it to your needs - $this->addSql('CREATE SCHEMA public'); $this->addSql('DROP INDEX entity_attribute_unique'); } } diff --git a/api/migrations/Version20230626125323.php b/api/migrations/Version20230626125323.php index 2f4f71ec3..dfb936b4b 100644 --- a/api/migrations/Version20230626125323.php +++ b/api/migrations/Version20230626125323.php @@ -26,7 +26,6 @@ public function up(Schema $schema): void public function down(Schema $schema): void { // this down() migration is auto-generated, please modify it to your needs - $this->addSql('CREATE SCHEMA public'); $this->addSql('ALTER TABLE attribute ALTER allow_cascade DROP DEFAULT'); } } diff --git a/api/migrations/Version20230922133622.php b/api/migrations/Version20230922133622.php index 03fe88ae2..12ef7ca0e 100644 --- a/api/migrations/Version20230922133622.php +++ b/api/migrations/Version20230922133622.php @@ -27,7 +27,6 @@ public function up(Schema $schema): void public function down(Schema $schema): void { // this down() migration is auto-generated, please modify it to your needs - $this->addSql('CREATE SCHEMA public'); $this->addSql('ALTER TABLE action DROP user_id'); $this->addSql('ALTER TABLE cronjob DROP user_id'); } diff --git a/api/migrations/Version20230926112500.php b/api/migrations/Version20230926112500.php index cbbde0146..8456f62ce 100644 --- a/api/migrations/Version20230926112500.php +++ b/api/migrations/Version20230926112500.php @@ -26,7 +26,6 @@ public function up(Schema $schema): void public function down(Schema $schema): void { // this down() migration is auto-generated, please modify it to your needs - $this->addSql('CREATE SCHEMA public'); - $this->addSql('ALTER TABLE "user" DROP organisation_id'); + $this->addSql('ALTER TABLE "user" RENAME COLUMN organization_id TO organisation_id'); } } diff --git a/api/migrations/Version20231123112218.php b/api/migrations/Version20231123112218.php new file mode 100644 index 000000000..d9f5e1d82 --- /dev/null +++ b/api/migrations/Version20231123112218.php @@ -0,0 +1,60 @@ +logging boolean to Gateway->configLogging array'; + + }//end getDescription() + + + /** + * Migrate up. + * + * @param Schema $schema Schema. + * @return void + */ + public function up(Schema $schema): void + { + $this->addSql('ALTER TABLE gateway ALTER logging TYPE TEXT'); + $this->addSql('ALTER TABLE gateway ALTER logging SET DEFAULT \'a:10:{s:10:"callMethod";b:1;s:7:"callUrl";b:1;s:9:"callQuery";b:1;s:15:"callContentType";b:1;s:8:"callBody";b:1;s:18:"responseStatusCode";b:1;s:19:"responseContentType";b:1;s:12:"responseBody";b:1;s:16:"maxCharCountBody";i:500;s:21:"maxCharCountErrorBody";i:2000;}\''); + $this->addSql('UPDATE gateway SET logging = \'a:10:{s:10:"callMethod";b:1;s:7:"callUrl";b:1;s:9:"callQuery";b:1;s:15:"callContentType";b:1;s:8:"callBody";b:1;s:18:"responseStatusCode";b:1;s:19:"responseContentType";b:1;s:12:"responseBody";b:1;s:16:"maxCharCountBody";i:500;s:21:"maxCharCountErrorBody";i:2000;}\''); + $this->addSql('ALTER TABLE gateway ALTER logging SET NOT NULL'); + $this->addSql('COMMENT ON COLUMN gateway.logging IS \'(DC2Type:array)\''); + $this->addSql('ALTER TABLE gateway RENAME COLUMN logging TO logging_config'); + }//end up() + + /** + * Migrate down. + * + * @param Schema $schema Schema. + * @return void + */ + public function down(Schema $schema): void + { + $this->addSql('ALTER TABLE gateway RENAME COLUMN logging_config TO logging'); + $this->addSql('ALTER TABLE gateway ALTER logging DROP NOT NULL'); + $this->addSql('ALTER TABLE gateway ALTER logging DROP DEFAULT'); + $this->addSql('UPDATE gateway SET logging = NULL'); + $this->addSql('ALTER TABLE gateway ALTER logging TYPE BOOLEAN USING logging::boolean'); + $this->addSql('COMMENT ON COLUMN gateway.logging IS NULL'); + }//end down() +}//end class diff --git a/api/src/Entity/Gateway.php b/api/src/Entity/Gateway.php index fc0e8c491..64888e31e 100644 --- a/api/src/Entity/Gateway.php +++ b/api/src/Entity/Gateway.php @@ -559,11 +559,25 @@ class Gateway private ?string $documentation = null; /** - * Setting logging to true will couse ALL responses to be logged (normaly we only log errors). Doing so wil dramaticly slow down the gateway and couse an increase in database size. This is not recomended outside of development purposes. + * Configuration for logging, when an api call is made on the source we can log some information for this call. + * With this array you can enable/disable what will be logged. * - * @ORM\Column(type="boolean", nullable=true) + * @Groups({"read","read_secure","write"}) + * + * @ORM\Column(type="array") */ - private $logging; + private array $loggingConfig = [ + 'callMethod' => true, + 'callUrl' => true, + 'callQuery' => true, + 'callContentType' => true, + 'callBody' => true, + 'responseStatusCode' => true, + 'responseContentType' => true, + 'responseBody' => true, + 'maxCharCountBody' => 500, + 'maxCharCountErrorBody' => 2000, + ]; /** * @var array ... @@ -792,6 +806,7 @@ public function fromSchema(array $schema): self array_key_exists('jwtId', $schema) ? $this->setJwtId($schema['jwtId']) : ''; array_key_exists('username', $schema) ? $this->setUsername($schema['username']) : ''; array_key_exists('documentation', $schema) ? $this->setDocumentation($schema['documentation']) : ''; + array_key_exists('loggingConfig', $schema) ? $this->setLoggingConfig($schema['loggingConfig']) : ''; array_key_exists('headers', $schema) ? $this->setHeaders($schema['headers']) : ''; array_key_exists('translationConfig', $schema) ? $this->setTranslationConfig($schema['translationConfig']) : ''; array_key_exists('type', $schema) ? $this->setType($schema['type']) : ''; @@ -826,6 +841,7 @@ public function toSchema(): array 'jwtId' => $this->getJwtId(), 'username' => $this->getUsername(), 'documentation' => $this->getDocumentation(), + 'loggingConfig' => $this->getLoggingConfig(), 'headers' => $this->getHeaders(), 'translationConfig' => $this->getTranslationConfig(), 'type' => $this->getType(), @@ -857,6 +873,7 @@ public function export(): ?array 'password' => $this->getPassword(), 'apikey' => $this->getApikey(), 'documentation' => $this->getDocumentation(), + 'loggingConfig' => $this->getLoggingConfig(), 'headers' => $this->getHeaders(), 'translationConfig' => $this->getTranslationConfig(), 'type' => $this->getType(), @@ -1129,14 +1146,14 @@ public function setDocumentation(?string $documentation): self return $this; } - public function getLogging(): ?bool + public function getLoggingConfig(): ?array { - return $this->logging; + return $this->loggingConfig; } - public function setLogging(?bool $logging): self + public function setLoggingConfig(array $loggingConfig): self { - $this->logging = $logging; + $this->loggingConfig = array_merge($loggingConfig); return $this; } diff --git a/api/src/Logger/SessionDataProcessor.php b/api/src/Logger/SessionDataProcessor.php index 1e0152568..e16385ee2 100644 --- a/api/src/Logger/SessionDataProcessor.php +++ b/api/src/Logger/SessionDataProcessor.php @@ -4,6 +4,7 @@ use App\Event\ActionEvent; use Doctrine\ORM\EntityManagerInterface; +use Exception; use Psr\EventDispatcher\EventDispatcherInterface; use Symfony\Component\HttpFoundation\RequestStack; use Symfony\Component\HttpFoundation\Session\SessionInterface; @@ -45,27 +46,105 @@ public function __construct( /** * Update the context with data from the session and the request stack. * - * @param array $context The context to update. + * @param array $record The log record to update the context for. * * @return array The updated context. */ - public function updateContext(array $context): array + public function updateContext(array $record): array { + $context = $record['context']; + $context['session'] = $this->session->getId(); - $context['process'] = $this->session->has('process') ? $this->session->get('process') : ''; - $context['endpoint'] = $this->session->has('endpoint') ? $this->session->get('endpoint') : ''; - $context['schema'] = $this->session->has('schema') ? $this->session->get('schema') : ''; + $context['process'] = $this->session->has('process') === true ? $this->session->get('process') : ''; + $context['endpoint'] = $this->session->has('endpoint') === true ? $this->session->get('endpoint') : ''; + $context['schema'] = $this->session->has('schema') === true ? $this->session->get('schema') : ''; $context['object'] = $this->session->has('object') === true ? $this->session->get('object') : ''; - $context['cronjob'] = $this->session->has('cronjob') ? $this->session->get('cronjob') : ''; - $context['action'] = $this->session->has('action') ? $this->session->get('action') : ''; - $context['mapping'] = $this->session->has('mapping') ? $this->session->get('mapping') : ''; - $context['source'] = $this->session->has('source') ? $this->session->get('source') : ''; - $context['plugin'] = isset($record['data']['plugin']) === true ? $record['data']['plugin'] : ''; - $context['user'] = $this->session->has('user') ? $this->session->get('user') : ''; - $context['organization'] = $this->session->has('organization') ? $this->session->get('organization') : ''; - $context['application'] = $this->session->has('application') ? $this->session->get('application') : ''; - $context['host'] = $this->requestStack->getMainRequest() ? $this->requestStack->getMainRequest()->getHost() : ''; - $context['ip'] = $this->requestStack->getMainRequest() ? $this->requestStack->getMainRequest()->getClientIp() : ''; + $context['cronjob'] = $this->session->has('cronjob') === true ? $this->session->get('cronjob') : ''; + $context['action'] = $this->session->has('action') === true ? $this->session->get('action') : ''; + $context['mapping'] = $this->session->has('mapping') === true ? $this->session->get('mapping') : ''; + $context['source'] = $this->session->has('source') === true ? $this->session->get('source') : ''; + + // Add more to context if we are dealing with a log containing sourceCall data. + if (isset($context['sourceCall']) === true) { + $context = $this->updateSourceCallContext($context, $record['level_name']); + } + + $context['user'] = $this->session->has('user') === true ? $this->session->get('user') : ''; + $context['organization'] = $this->session->has('organization') === true ? $this->session->get('organization') : ''; + $context['application'] = $this->session->has('application') === true ? $this->session->get('application') : ''; + $context['host'] = $this->requestStack->getMainRequest() !== null ? $this->requestStack->getMainRequest()->getHost() : ''; + $context['ip'] = $this->requestStack->getMainRequest() !== null ? $this->requestStack->getMainRequest()->getClientIp() : ''; + $context['method'] = $this->requestStack->getMainRequest() !== null ? $this->requestStack->getMainRequest()->getMethod() : ''; + + // Add more to context for higher level logs. + if (in_array($record['level_name'], ['ERROR', 'CRITICAL', 'ALERT', 'EMERGENCY']) === true) { + $context = $this->addErrorContext($context, $record['level_name']); + } + + return $context; + } + + /** + * Update the context for Source call logs. + * + * @param array $context The log context we are updating. + * @param string $levelName The level name of the log record we are updating the context for. + * + * @return array The updated context. + */ + private function updateSourceCallContext(array $context, string $levelName): array + { + if (in_array($levelName, ['ERROR', 'CRITICAL', 'ALERT', 'EMERGENCY']) === true) { + $maxStrLength = ($context['sourceCall']['maxCharCountErrorBody'] ?? 2000); + unset($context['sourceCall']['maxCharCountBody']); + } else { + $maxStrLength = ($context['sourceCall']['maxCharCountBody'] ?? 500); + unset($context['sourceCall']['maxCharCountErrorBody']); + } + + if (isset($context['sourceCall']['callQuery']) === true) { + $context['sourceCall']['callQuery'] = json_encode($context['sourceCall']['callQuery']); + if ($context['sourceCall']['callQuery'] === "[]") { + $context['sourceCall']['callQuery'] = ''; + } + } + + if (isset($context['sourceCall']['callBody']) === true && strlen($context['sourceCall']['callBody']) > $maxStrLength) { + $context['sourceCall']['callBody'] = substr($context['sourceCall']['callBody'], 0, $maxStrLength).'...'; + } + + if (isset($context['sourceCall']['responseBody']) === true && strlen($context['sourceCall']['responseBody']) > $maxStrLength) { + $context['sourceCall']['responseBody'] = substr($context['sourceCall']['responseBody'], 0, $maxStrLength).'...'; + } + + return $context; + } + + /** + * Update the context with data from the session and the request stack. For log records with level ERROR or higher. + * See: https://github.com/Seldaek/monolog/blob/main/doc/01-usage.md for all possible log levels. + * + * @param array $context The log context we are updating. + * @param string $levelName The level name of the log record we are updating the context for. + * + * @return array The updated context. + */ + private function addErrorContext(array $context, string $levelName): array + { + $context['pathRaw'] = $this->requestStack->getMainRequest() !== null ? $this->requestStack->getMainRequest()->getPathInfo() : ''; + $context['querystring'] = $this->requestStack->getMainRequest() !== null ? $this->requestStack->getMainRequest()->getQueryString() : ''; + $context['mongoDBFilter'] = $this->session->has('mongoDBFilter') === true ? json_encode($this->session->get('mongoDBFilter')) : ''; + $context['contentType'] = $this->requestStack->getMainRequest() !== null ? $this->requestStack->getMainRequest()->getContentType() : ''; + + // Do not log entire body for normal errors, only critical and higher. + if ($this->requestStack->getMainRequest() !== null && $levelName !== 'ERROR') { + try { + $context['body'] = $this->requestStack->getMainRequest()->toArray(); + } catch (Exception $exception) { + $context['body'] = ''; + } + $context['crude_body'] = $this->requestStack->getMainRequest() ? $this->requestStack->getMainRequest()->getContent() : ''; + } return $context; } @@ -85,7 +164,7 @@ public function dispatchLogCreateAction(array $record): array $this->entityManager->getConnection()->getSchemaManager()->listDatabases() ) === true && $this->entityManager->getConnection()->getSchemaManager()->tablesExist('action') === true - && in_array($record['level_name'], ['DEBUG', 'INFO', 'NOTICE', 'WARNING']) === false + && in_array($record['level_name'], ['ERROR', 'CRITICAL', 'ALERT', 'EMERGENCY']) === true ){ $event = new ActionEvent('commongateway.action.event', $record, 'commongateway.log.create'); @@ -106,7 +185,7 @@ public function dispatchLogCreateAction(array $record): array */ public function __invoke(array $record): array { - $record['context'] = $this->updateContext($record['context']); + $record['context'] = $this->updateContext($record); return $this->dispatchLogCreateAction($record); } diff --git a/api/symfony.lock b/api/symfony.lock index 0664d2039..d20a5abee 100644 --- a/api/symfony.lock +++ b/api/symfony.lock @@ -785,6 +785,15 @@ "symfony/security-http": { "version": "v5.3.6" }, + "symfony/sendinblue-mailer": { + "version": "5.3", + "recipe": { + "repo": "github.com/symfony/recipes", + "branch": "main", + "version": "5.2", + "ref": "ae1cf494ce06b9a4578a8445c610402a1676ee8d" + } + }, "symfony/serializer": { "version": "v5.3.4" },