From 178161aaac315e06e460ee465c0cdfd4201f2d71 Mon Sep 17 00:00:00 2001 From: Jim <58939809+jamesrwarren@users.noreply.github.com> Date: Wed, 15 Nov 2023 15:24:46 +0000 Subject: [PATCH] DDPB-3703 move to graviton based single cache resource per account (#1442) --- api/app/api.env | 1 + api/app/config/parameters.yml.dist | 2 + api/app/config/services.yml | 1 + api/app/config/services_test.yml | 1 + api/app/src/Controller/AuthController.php | 5 +- api/app/src/Security/RedisUserProvider.php | 7 +- .../BruteForce/AttemptsInTimeChecker.php | 10 ++- .../AttemptsIncrementalWaitingChecker.php | 10 ++- .../Unit/Service/Auth/UserProviderTest.php | 4 +- .../app/confd/conf.d/parameters.hml.toml | 1 + .../app/confd/templates/parameters.yml.tmpl | 3 +- client/app/admin.env | 2 +- client/app/config/packages/snc_redis.yml | 2 +- client/app/config/parameters.yml.dist | 1 + client/app/config/services.yml | 1 + client/app/frontend.env | 4 +- .../app/src/Command/ProcessLayCSVCommand.php | 31 ++++---- .../app/src/Command/ProcessOrgCSVCommand.php | 7 +- .../src/Controller/Admin/IndexController.php | 76 ++++++++++-------- .../Availability/RedisAvailability.php | 10 ++- .../Client/TokenStorage/RedisStorage.php | 11 +-- .../Client/TokenStorage/RedisStorageTest.php | 7 +- .../app/confd/conf.d/parameters.hml.toml | 1 + .../app/confd/templates/parameters.yml.tmpl | 1 + docker-compose.yml | 1 + terraform/account/elasticache.tf | 77 +++++++++++++++++++ terraform/account/elasticache_parameters.tf | 10 --- terraform/environment/admin_service.tf | 5 +- terraform/environment/admin_sg.tf | 7 ++ terraform/environment/api_service.tf | 4 + terraform/environment/api_sg.tf | 7 ++ terraform/environment/check_csv_uploaded.tf | 9 ++- terraform/environment/dns_private.tf | 4 +- terraform/environment/elasticache.tf | 49 ++++++++++++ terraform/environment/front_service.tf | 3 +- terraform/environment/front_sg.tf | 7 ++ terraform/environment/integration_test_v2.tf | 4 + terraform/environment/smoke_test.tf | 4 + .../environment/sync_to_sirius_checklists.tf | 6 +- .../environment/sync_to_sirius_documents.tf | 6 +- 40 files changed, 309 insertions(+), 93 deletions(-) create mode 100644 terraform/account/elasticache.tf delete mode 100644 terraform/account/elasticache_parameters.tf create mode 100644 terraform/environment/elasticache.tf diff --git a/api/app/api.env b/api/app/api.env index 36e69c5c6c..9ad3785cbd 100644 --- a/api/app/api.env +++ b/api/app/api.env @@ -47,3 +47,4 @@ SSM_LOCALSTACK=true SECRETS_MANAGER_LOCALSTACK=true JWT_HOST=http://frontend-webserver +WORKSPACE=local diff --git a/api/app/config/parameters.yml.dist b/api/app/config/parameters.yml.dist index 4291ccd57a..6cc777812b 100755 --- a/api/app/config/parameters.yml.dist +++ b/api/app/config/parameters.yml.dist @@ -39,3 +39,5 @@ parameters: version: 'latest' region: 'eu-west-1' validate: true + + workspace: local diff --git a/api/app/config/services.yml b/api/app/config/services.yml index d5dac6f318..a94c9002b1 100755 --- a/api/app/config/services.yml +++ b/api/app/config/services.yml @@ -53,6 +53,7 @@ services: $projectDir: "%kernel.project_dir%" $redis: "@snc_redis.default" $frontendHost: "%env(JWT_HOST)%" + $workspace: "%workspace%" # makes classes in src/ available to be used as services # this creates a service per class whose id is the fully-qualified class name diff --git a/api/app/config/services_test.yml b/api/app/config/services_test.yml index 8ef423e3db..7b144b54c7 100644 --- a/api/app/config/services_test.yml +++ b/api/app/config/services_test.yml @@ -6,6 +6,7 @@ services: bind: $fixtureParams: "%fixtures%" $symfonyEnvironment: "%kernel.environment%" + $workspace: "%workspace%" App\Service\BruteForce\AttemptsInTimeChecker: public: true diff --git a/api/app/src/Controller/AuthController.php b/api/app/src/Controller/AuthController.php index 23e128f9fa..651bd978bd 100644 --- a/api/app/src/Controller/AuthController.php +++ b/api/app/src/Controller/AuthController.php @@ -24,7 +24,8 @@ public function __construct( private RestFormatter $restFormatter, private JWTService $JWTService, private LoggerInterface $logger, - private TokenStorageInterface $tokenStorage + private TokenStorageInterface $tokenStorage, + private string $workspace ) { } @@ -53,7 +54,7 @@ public function login( $em->flush(); // Now doing this inline rather than injecting RedisUserProvider - $authToken = $user->getId().'_'.sha1(microtime().spl_object_hash($user).rand(1, 999)); + $authToken = $this->workspace.'_'.$user->getId().'_'.sha1(microtime().spl_object_hash($user).rand(1, 999)); $redis->set($authToken, serialize($this->tokenStorage->getToken())); // add token into response diff --git a/api/app/src/Security/RedisUserProvider.php b/api/app/src/Security/RedisUserProvider.php index 1b22232353..7cf1d964b0 100644 --- a/api/app/src/Security/RedisUserProvider.php +++ b/api/app/src/Security/RedisUserProvider.php @@ -24,8 +24,9 @@ public function __construct( private Client $redis, private LoggerInterface $logger, private array $options, - private UserRepository $userRepository) - { + private UserRepository $userRepository, + private string $workspace + ) { $this->timeoutSeconds = $options['timeout_seconds']; } @@ -34,7 +35,7 @@ public function __construct( */ public function generateRandomTokenAndStore(User $user) { - $token = $user->getId().'_'.sha1(microtime().spl_object_hash($user).rand(1, 999)); + $token = $this->workspace.'_'.$user->getId().'_'.sha1(microtime().spl_object_hash($user).rand(1, 999)); $this->redis->set($token, $user->getId()); $this->redis->expire($token, $this->timeoutSeconds); diff --git a/api/app/src/Service/BruteForce/AttemptsInTimeChecker.php b/api/app/src/Service/BruteForce/AttemptsInTimeChecker.php index 6e96b18ec2..9c48478c49 100644 --- a/api/app/src/Service/BruteForce/AttemptsInTimeChecker.php +++ b/api/app/src/Service/BruteForce/AttemptsInTimeChecker.php @@ -21,16 +21,22 @@ class AttemptsInTimeChecker */ private $redisPrefix; - public function __construct(PredisClient $redis, $prefix = null) + /** + * @var string + */ + private $workspace; + + public function __construct(PredisClient $redis, $workspace, $prefix = null) { $this->redis = $redis; $this->redisPrefix = $prefix; $this->triggers = []; + $this->workspace = $workspace; } public function setRedisPrefix($redisPrefix) { - $this->redisPrefix = $redisPrefix; + $this->redisPrefix = $this->workspace.'_'.$redisPrefix; return $this; } diff --git a/api/app/src/Service/BruteForce/AttemptsIncrementalWaitingChecker.php b/api/app/src/Service/BruteForce/AttemptsIncrementalWaitingChecker.php index 119e304abd..5e26bd85b4 100644 --- a/api/app/src/Service/BruteForce/AttemptsIncrementalWaitingChecker.php +++ b/api/app/src/Service/BruteForce/AttemptsIncrementalWaitingChecker.php @@ -25,17 +25,23 @@ class AttemptsIncrementalWaitingChecker private array $secondsBeforeNextAttempt; - public function __construct(PredisClient $redis, $prefix = null) + /** + * @var string + */ + private $workspace; + + public function __construct(PredisClient $redis, $workspace, $prefix = null) { $this->redis = $redis; $this->redisPrefix = $prefix; $this->freezeRules = []; $this->secondsBeforeNextAttempt = []; + $this->workspace = $workspace; } public function setRedisPrefix($redisPrefix) { - $this->redisPrefix = $redisPrefix; + $this->redisPrefix = $this->workspace.'_'.$redisPrefix; return $this; } diff --git a/api/app/tests/Unit/Service/Auth/UserProviderTest.php b/api/app/tests/Unit/Service/Auth/UserProviderTest.php index dbe9e3d68e..b2d3d548e2 100644 --- a/api/app/tests/Unit/Service/Auth/UserProviderTest.php +++ b/api/app/tests/Unit/Service/Auth/UserProviderTest.php @@ -24,7 +24,7 @@ public function setUp(): void $this->logger = m::stub('Symfony\Bridge\Monolog\Logger'); $options = ['timeout_seconds' => 7]; - $this->userProvider = new RedisUserProvider($this->em, $this->redis, $this->logger, $options, $this->repo); + $this->userProvider = new RedisUserProvider($this->em, $this->redis, $this->logger, $options, $this->repo, 'testing'); } public function testloadUserByUsernameRedisNotFound() @@ -75,7 +75,7 @@ public function testgenerateRandomTokenAndStore() 'getId' => 123, ]); - $tokenMatchPattern = '/^123_[0-9a-f]{5,40}[\d]{1,}/'; + $tokenMatchPattern = '/^testing_123_[0-9a-f]{5,40}[\d]{1,}/'; $this->redis->shouldReceive('set')->with(\Mockery::pattern($tokenMatchPattern), 123)->atLeast(1); $this->redis->shouldReceive('expire')->with(\Mockery::pattern($tokenMatchPattern), 7)->atLeast(1); diff --git a/api/docker/app/confd/conf.d/parameters.hml.toml b/api/docker/app/confd/conf.d/parameters.hml.toml index 747a219ee7..e8fb27ad5a 100644 --- a/api/docker/app/confd/conf.d/parameters.hml.toml +++ b/api/docker/app/confd/conf.d/parameters.hml.toml @@ -15,4 +15,5 @@ keys = [ "/smtp", "/ssm/localstack", "/url", + "/workspace" ] diff --git a/api/docker/app/confd/templates/parameters.yml.tmpl b/api/docker/app/confd/templates/parameters.yml.tmpl index ffe5850010..7945613093 100644 --- a/api/docker/app/confd/templates/parameters.yml.tmpl +++ b/api/docker/app/confd/templates/parameters.yml.tmpl @@ -49,7 +49,8 @@ parameters: database_ssl_mode: {{ getv "/database/ssl" }} locale: en secret: {{ getv "/secret" }} - redis_dsn: '{{getv "/redis/dsn" }}' + redis_dsn: '{{ getv "/redis/dsn" }}' log_level: warning verbose_log_level: notice log_path: /var/log/app/application.log + workspace: {{ getv "/workspace" }} diff --git a/client/app/admin.env b/client/app/admin.env index 2fe903a455..db2e611397 100644 --- a/client/app/admin.env +++ b/client/app/admin.env @@ -3,7 +3,7 @@ API_CLIENT_SECRET=api-admin-key ROLE=admin SESSION_REDIS_DSN=redis://redis-frontend -SESSION_PREFIX=dd_session_admin +SESSION_PREFIX=dd_admin NGINX_APP_NAME=admin diff --git a/client/app/config/packages/snc_redis.yml b/client/app/config/packages/snc_redis.yml index 600bfeeef0..900b0507d7 100644 --- a/client/app/config/packages/snc_redis.yml +++ b/client/app/config/packages/snc_redis.yml @@ -6,4 +6,4 @@ snc_redis: dsn: "%redis_dsn%" session: client: default - prefix: "%session_prefix%" + prefix: "%workspace%_%session_prefix%" diff --git a/client/app/config/parameters.yml.dist b/client/app/config/parameters.yml.dist index 1664dec7f4..1a4dcff008 100644 --- a/client/app/config/parameters.yml.dist +++ b/client/app/config/parameters.yml.dist @@ -77,3 +77,4 @@ parameters: htmltopdf_address: "http://htmltopdf:80" pa_pro_report_csv_filename: pa_pro_report.csv lay_report_csv_filename: layDeputyReport.csv + workspace: local diff --git a/client/app/config/services.yml b/client/app/config/services.yml index b578949fbd..c191fef47c 100644 --- a/client/app/config/services.yml +++ b/client/app/config/services.yml @@ -43,6 +43,7 @@ services: $hostedEnv: "%env(ENVIRONMENT)%" $projectDir: "%kernel.project_dir%" $sessionPrefix: "%session_prefix%" + $workspace: "%workspace%" App\: resource: "../src/" diff --git a/client/app/frontend.env b/client/app/frontend.env index 1991d11803..fe85242d41 100644 --- a/client/app/frontend.env +++ b/client/app/frontend.env @@ -7,7 +7,7 @@ SESSION_COOKIE_SECURE=false ROLE=front SESSION_REDIS_DSN=redis://redis-frontend -SESSION_PREFIX=dd_session_front +SESSION_PREFIX=dd_front GA_DEFAULT=UA-57312200-1 GA_GDS= @@ -63,3 +63,5 @@ ENVIRONMENT=local SSM_LOCALSTACK=true SECRETS_MANAGER_LOCALSTACK=true + +WORKSPACE=local diff --git a/client/app/src/Command/ProcessLayCSVCommand.php b/client/app/src/Command/ProcessLayCSVCommand.php index bb8431d146..4b9ac4d05d 100644 --- a/client/app/src/Command/ProcessLayCSVCommand.php +++ b/client/app/src/Command/ProcessLayCSVCommand.php @@ -18,9 +18,9 @@ use Symfony\Component\Console\Input\InputInterface; use Symfony\Component\Console\Output\OutputInterface; use Symfony\Component\DependencyInjection\ParameterBag\ParameterBagInterface; -use Throwable; -class ProcessLayCSVCommand extends Command { +class ProcessLayCSVCommand extends Command +{ protected static $defaultName = 'digideps:process-lay-csv'; private const CHUNK_SIZE = 50; @@ -38,11 +38,13 @@ public function __construct( private Mailer $mailer, private LoggerInterface $logger, private ClientInterface $redis, + private string $workspace ) { parent::__construct(); } - protected function configure(): void { + protected function configure(): void + { $this ->setDescription('Process the Lay Deputies CSV from the S3 bucket') ->addArgument('email', InputArgument::REQUIRED, 'Email address to send results to'); @@ -57,7 +59,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int $this->s3->getObject([ 'Bucket' => $bucket, 'Key' => $layReportFile, - 'SaveAs' => "/tmp/layReport.csv" + 'SaveAs' => '/tmp/layReport.csv', ]); } catch (S3Exception $e) { if (in_array($e->getAwsErrorCode(), S3Storage::MISSING_FILE_AWS_ERROR_CODES)) { @@ -67,17 +69,18 @@ protected function execute(InputInterface $input, OutputInterface $output): int } } - $data = $this->csvToArray("/tmp/layReport.csv"); + $data = $this->csvToArray('/tmp/layReport.csv'); $this->process($data, $input->getArgument('email')); - if (!unlink("/tmp/layReport.csv")) { + if (!unlink('/tmp/layReport.csv')) { $this->logger->log('error', 'Unable to delete file /tmp/layReport.csv.'); } return 0; } - private function csvToArray(string $fileName) { + private function csvToArray(string $fileName) + { try { return (new CsvToArray($fileName, false, false)) ->setOptionalColumns([ @@ -100,17 +103,18 @@ private function csvToArray(string $fileName) { ]) ->setUnexpectedColumns(['LastReportDay', 'DeputyOrganisation']) ->getData(); - } catch (Throwable $e) { + } catch (\Throwable $e) { $this->logger->log('error', sprintf('Error processing CSV file: %s', $e->getMessage())); } } - private function process(mixed $data, string $email) { + private function process(mixed $data, string $email) + { $this->restClient->delete('/pre-registration/delete'); $chunks = array_chunk($data, self::CHUNK_SIZE); - $this->redis->set('lay-csv-processing', 'processing'); + $this->redis->set($this->workspace.'-lay-csv-processing', 'processing'); foreach ($chunks as $index => $chunk) { $compressedChunk = CsvUploader::compressData($chunk); @@ -121,13 +125,14 @@ private function process(mixed $data, string $email) { $this->storeOutput($upload); } - $this->redis->set('lay-csv-processing', 'completed'); - $this->redis->set('lay-csv-completed-date', date('Y-m-d H:i:s')); + $this->redis->set($this->workspace.'-lay-csv-processing', 'completed'); + $this->redis->set($this->workspace.'-lay-csv-completed-date', date('Y-m-d H:i:s')); $this->mailer->sendProcessLayCSVEmail($email, $this->output); } - private function storeOutput(array $output) { + private function storeOutput(array $output) + { if (!empty($output['errors'])) { $this->output['errors'] = array_merge($this->output['errors'], $output['errors']); } diff --git a/client/app/src/Command/ProcessOrgCSVCommand.php b/client/app/src/Command/ProcessOrgCSVCommand.php index b85be174e0..9513fe1ebe 100644 --- a/client/app/src/Command/ProcessOrgCSVCommand.php +++ b/client/app/src/Command/ProcessOrgCSVCommand.php @@ -49,6 +49,7 @@ public function __construct( private Mailer $mailer, private LoggerInterface $logger, private ClientInterface $redis, + private string $workspace ) { parent::__construct(); } @@ -137,7 +138,7 @@ private function process(mixed $data, string $email) { $chunks = array_chunk($data, self::CHUNK_SIZE); - $this->redis->set('org-csv-processing', 'processing'); + $this->redis->set($this->workspace.'-org-csv-processing', 'processing'); foreach ($chunks as $index => $chunk) { try { @@ -156,8 +157,8 @@ private function process(mixed $data, string $email) $this->logger->log('notice', 'Successfully processed all chunks'); - $this->redis->set('org-csv-processing', 'completed'); - $this->redis->set('org-csv-completed-date', date('Y-m-d H:i:s')); + $this->redis->set($this->workspace.'-org-csv-processing', 'completed'); + $this->redis->set($this->workspace.'-org-csv-completed-date', date('Y-m-d H:i:s')); $this->mailer->sendProcessOrgCSVEmail($email, $this->output); } diff --git a/client/app/src/Controller/Admin/IndexController.php b/client/app/src/Controller/Admin/IndexController.php index 63982d9416..cb5f38bc5a 100644 --- a/client/app/src/Controller/Admin/IndexController.php +++ b/client/app/src/Controller/Admin/IndexController.php @@ -21,11 +21,8 @@ use App\Service\OrgService; use Aws\S3\Exception\S3Exception; use Aws\S3\S3Client; -use DateTime; -use Exception; use Predis\ClientInterface; use Psr\Log\LoggerInterface; -use RuntimeException; use Sensio\Bundle\FrameworkExtraBundle\Configuration\Security; use Sensio\Bundle\FrameworkExtraBundle\Configuration\Template; use Symfony\Bundle\FrameworkBundle\Console\Application; @@ -35,7 +32,6 @@ use Symfony\Component\Form\Extension\Core\Type\ChoiceType; use Symfony\Component\Form\Extension\Core\Type\SubmitType; use Symfony\Component\Form\FormError; -use Symfony\Component\Form\FormInterface; use Symfony\Component\HttpFoundation\RedirectResponse; use Symfony\Component\HttpFoundation\Request; use Symfony\Component\HttpFoundation\Response; @@ -46,7 +42,6 @@ use Symfony\Component\Security\Core\Authentication\Token\Storage\TokenStorageInterface; use Symfony\Contracts\EventDispatcher\EventDispatcherInterface; use Symfony\Contracts\Translation\TranslatorInterface; -use Throwable; /** * @Route("/admin") @@ -65,13 +60,16 @@ public function __construct( private ParameterBagInterface $params, private KernelInterface $kernel, private EventDispatcherInterface $dispatcher, - private S3Client $s3 + private S3Client $s3, + private string $workspace ) { } /** * @Route("/", name="admin_homepage") + * * @Security("is_granted('ROLE_ADMIN') or is_granted('ROLE_AD')") + * * @Template("@App/Admin/Index/index.html.twig") */ public function indexAction(Request $request) @@ -104,7 +102,9 @@ public function indexAction(Request $request) /** * @Route("/user-add", name="admin_add_user") + * * @Security("is_granted('ROLE_ADMIN') or is_granted('ROLE_AD')") + * * @Template("@App/Admin/Index/addUser.html.twig") * * @return array|RedirectResponse @@ -118,7 +118,7 @@ public function addUserAction(Request $request) // add user try { if (!$this->isGranted(EntityDir\User::ROLE_SUPER_ADMIN) && EntityDir\User::ROLE_SUPER_ADMIN == $form->getData()->getRoleName()) { - throw new RuntimeException('Cannot add admin from non-admin user'); + throw new \RuntimeException('Cannot add admin from non-admin user'); } $this->userApi->createUser($form->getData()); @@ -141,10 +141,10 @@ public function addUserAction(Request $request) /** * @Route("/user/{id}", name="admin_user_view", requirements={"id":"\d+"}) + * * @Security("is_granted('ROLE_ADMIN')") - * @Template("@App/Admin/Index/viewUser.html.twig") * - * @param $id + * @Template("@App/Admin/Index/viewUser.html.twig") * * @return User[]|Response */ @@ -152,19 +152,21 @@ public function viewAction($id) { try { return ['user' => $this->getPopulatedUser($id)]; - } catch (Throwable $e) { + } catch (\Throwable $e) { return $this->renderNotFound(); } } /** * @Route("/edit-user", name="admin_editUser", methods={"GET", "POST"}) + * * @Security("is_granted('ROLE_ADMIN') or is_granted('ROLE_AD')") + * * @Template("@App/Admin/Index/editUser.html.twig") * * @return array|Response * - * @throws Throwable + * @throws \Throwable */ public function editUserAction(Request $request, TranslatorInterface $translator) { @@ -173,13 +175,13 @@ public function editUserAction(Request $request, TranslatorInterface $translator try { /* @var User $user */ $user = $this->getPopulatedUser($filter); - } catch (Throwable $e) { + } catch (\Throwable $e) { return $this->renderNotFound(); } try { $this->denyAccessUnlessGranted('edit-user', $user); - } catch (Throwable $e) { + } catch (\Throwable $e) { $accessErrorMessage = 'You do not have permission to edit this user'; return $this->render('@App/Admin/Index/error.html.twig', [ @@ -198,7 +200,7 @@ public function editUserAction(Request $request, TranslatorInterface $translator $this->restClient->put('user/'.$user->getId(), $updateUser, ['admin_edit_user']); $this->addFlash('notice', 'Your changes were saved'); $this->redirectToRoute('admin_editUser', ['filter' => $user->getId()]); - } catch (Throwable $e) { + } catch (\Throwable $e) { switch ((int) $e->getCode()) { case 422: $form->get('email')->addError(new FormError($translator->trans('editUserForm.email.existingError', [], 'admin'))); @@ -229,9 +231,6 @@ public function editUserAction(Request $request, TranslatorInterface $translator return $view; } - /** - * @param $id - */ private function getPopulatedUser($id): User { /* @var User $user */ @@ -252,6 +251,7 @@ private function renderNotFound(): Response /** * @Route("/edit-ndr/{id}", name="admin_editNdr", methods={"POST"}) + * * @Security("is_granted('ROLE_ADMIN') or is_granted('ROLE_AD')") * * @param string $id @@ -280,7 +280,9 @@ public function editNdrAction(Request $request, $id) /** * @Route("/delete-confirm/{id}", name="admin_delete_confirm", methods={"GET"}) + * * @Security("is_granted('ROLE_ADMIN') or is_granted('ROLE_AD')") + * * @Template("@App/Admin/Index/deleteConfirm.html.twig") * * @param int $id @@ -299,6 +301,7 @@ public function deleteConfirmAction($id) /** * @Route("/delete/{id}", name="admin_delete", methods={"GET"}) + * * @Security("is_granted('ROLE_ADMIN') or is_granted('ROLE_AD')") * * @param int $id @@ -317,7 +320,7 @@ public function deleteAction($id) } return $this->redirect($this->generateUrl('admin_homepage')); - } catch (Throwable $e) { + } catch (\Throwable $e) { $this->logger->warning( sprintf('Error while deleting deputy: %s', $e->getMessage()), ['deputy_email' => $user->getEmail()] @@ -331,7 +334,9 @@ public function deleteAction($id) /** * @Route("/upload", name="admin_upload") + * * @Security("is_granted('ROLE_ADMIN') or is_granted('ROLE_AD')") + * * @Template("@App/Admin/Index/upload.html.twig") */ public function uploadAction(Request $request, RouterInterface $router) @@ -365,9 +370,12 @@ public function uploadAction(Request $request, RouterInterface $router) /** * @Route("/pre-registration-upload", name="pre_registration_upload") + * * @Security("is_granted('ROLE_SUPER_ADMIN')") + * * @Template("@App/Admin/Index/uploadUsers.html.twig") - * @throws Exception + * + * @throws \Exception */ public function uploadUsersAction(Request $request, ClientInterface $redisClient) { @@ -423,11 +431,11 @@ public function uploadUsersAction(Request $request, ClientInterface $redisClient foreach ($chunks as $k => $chunk) { $compressedData = CsvUploader::compressData($chunk); - $redisClient->set('chunk'.$k, $compressedData); + $redisClient->set($this->workspace.'_chunk'.$k, $compressedData); } return $this->redirect($this->generateUrl('pre_registration_upload', ['nOfChunks' => count($chunks)])); - } catch (Throwable $e) { + } catch (\Throwable $e) { $message = $e->getMessage(); if ($e instanceof RestClientException && isset($e->getData()['message'])) { @@ -452,7 +460,7 @@ public function uploadUsersAction(Request $request, ClientInterface $redisClient $application->run($input, $output); }); - $this->addFlash("notice", "CSV import process started. Keep an eye on your emails for completion."); + $this->addFlash('notice', 'CSV import process started. Keep an eye on your emails for completion.'); return $this->redirect($this->generateUrl('pre_registration_upload')); } @@ -491,16 +499,19 @@ public function uploadUsersAction(Request $request, ClientInterface $redisClient 'processStatusDate' => $processCompletedDate, 'fileUploadedInfo' => [ 'fileName' => $layReportFile, - 'date' => $bucketFileInfo['LastModified'] + 'date' => $bucketFileInfo['LastModified'], ], ]; } /** * @Route("/org-csv-upload", name="admin_org_upload") + * * @Security("is_granted('ROLE_SUPER_ADMIN')") + * * @Template("@App/Admin/Index/uploadOrgUsers.html.twig") - * @throws Exception + * + * @throws \Exception */ public function uploadOrgUsersAction(Request $request, ClientInterface $redisClient) { @@ -524,8 +535,9 @@ public function uploadOrgUsersAction(Request $request, ClientInterface $redisCli $this->orgService->setLogging($outputStreamResponse); $redirectUrl = $this->generateUrl('admin_org_upload'); + return $this->orgService->process($data, $redirectUrl); - } catch (Throwable $e) { + } catch (\Throwable $e) { $message = $e->getMessage(); if ($e instanceof RestClientException && isset($e->getData()['message'])) { @@ -534,7 +546,7 @@ public function uploadOrgUsersAction(Request $request, ClientInterface $redisCli if ($outputStreamResponse) { $this->addFlash('error', $message); - exit(); + exit; } else { $form->get('file')->addError(new FormError($message)); } @@ -555,8 +567,7 @@ public function uploadOrgUsersAction(Request $request, ClientInterface $redisCli $application->run($input, $output); }); - - $this->addFlash("notice", "CSV import process started. Keep an eye on your emails for completion."); + $this->addFlash('notice', 'CSV import process started. Keep an eye on your emails for completion.'); return $this->redirect($this->generateUrl('admin_org_upload')); } @@ -592,27 +603,28 @@ public function uploadOrgUsersAction(Request $request, ClientInterface $redisCli 'processStatusDate' => $processCompletedDate, 'fileUploadedInfo' => [ 'fileName' => $paProReportFile, - 'date' => $bucketFileInfo['LastModified'] + 'date' => $bucketFileInfo['LastModified'], ], ]; } /** * @Route("/send-activation-link/{email}", name="admin_send_activation_link") + * * @Security("is_granted('ROLE_ADMIN') or is_granted('ROLE_AD')") **/ public function sendUserActivationLinkAction(string $email, LoggerInterface $logger) { try { $this->userApi->activate($email, 'pass-reset'); - } catch (Throwable $e) { + } catch (\Throwable $e) { $logger->debug($e->getMessage()); } return new Response('[Link sent]'); } - private function handleOrgUploadForm($fileName): Throwable|Exception|array + private function handleOrgUploadForm($fileName): \Throwable|\Exception|array { return (new CsvToArray($fileName, false)) ->setExpectedColumns([ @@ -652,7 +664,7 @@ private function handleOrgUploadForm($fileName): Throwable|Exception|array ->getData(); } - private function handleLayUploadForm($fileName): Throwable|Exception|array + private function handleLayUploadForm($fileName): \Throwable|\Exception|array { return (new CsvToArray($fileName, false, true)) ->setOptionalColumns([ diff --git a/client/app/src/Service/Availability/RedisAvailability.php b/client/app/src/Service/Availability/RedisAvailability.php index 9036699f99..3cec0173e5 100644 --- a/client/app/src/Service/Availability/RedisAvailability.php +++ b/client/app/src/Service/Availability/RedisAvailability.php @@ -7,24 +7,26 @@ class RedisAvailability extends ServiceAvailabilityAbstract { - const TEST_KEY = 'RedisAvailabilityTestKey'; + public const TEST_KEY = 'RedisAvailabilityTestKey'; private ContainerInterface $container; private ClientInterface $redis; + private string $workspace; - public function __construct(ContainerInterface $container, ClientInterface $redis) + public function __construct(ContainerInterface $container, ClientInterface $redis, $workspace) { $this->isHealthy = false; $this->container = $container; $this->redis = $redis; + $this->workspace = $workspace; } public function ping() { try { - $this->redis->set(self::TEST_KEY, 'valueSaved'); + $this->redis->set($this->workspace.'_'.self::TEST_KEY, 'valueSaved'); - if ('valueSaved' == $this->redis->get(self::TEST_KEY)) { + if ('valueSaved' == $this->redis->get($this->workspace.'_'.self::TEST_KEY)) { $this->isHealthy = true; } } catch (\Throwable $e) { diff --git a/client/app/src/Service/Client/TokenStorage/RedisStorage.php b/client/app/src/Service/Client/TokenStorage/RedisStorage.php index 165e8ba38d..2c9f04fd30 100644 --- a/client/app/src/Service/Client/TokenStorage/RedisStorage.php +++ b/client/app/src/Service/Client/TokenStorage/RedisStorage.php @@ -9,23 +9,24 @@ class RedisStorage extends TokenStorage { public function __construct( private PredisClientInterface $redis, - private string $sessionPrefix + private string $sessionPrefix, + private string $workspace ) { } public function get($id) { - return $this->redis->get($this->sessionPrefix.$id); + return $this->redis->get($this->workspace.'_'.$this->sessionPrefix.$id); } public function set($id, $value) { - return $this->redis->set($this->sessionPrefix.$id, $value); + return $this->redis->set($this->workspace.'_'.$this->sessionPrefix.$id, $value); } public function remove($id) { - $this->redis->set($this->sessionPrefix.$id, null); - $this->redis->expire($this->sessionPrefix.$id, 0); + $this->redis->set($this->workspace.'_'.$this->sessionPrefix.$id, null); + $this->redis->expire($this->workspace.'_'.$this->sessionPrefix.$id, 0); } } diff --git a/client/app/tests/phpunit/Service/Client/TokenStorage/RedisStorageTest.php b/client/app/tests/phpunit/Service/Client/TokenStorage/RedisStorageTest.php index 6fa803e157..4690b1cdcc 100644 --- a/client/app/tests/phpunit/Service/Client/TokenStorage/RedisStorageTest.php +++ b/client/app/tests/phpunit/Service/Client/TokenStorage/RedisStorageTest.php @@ -22,8 +22,9 @@ public function setUp(): void { $this->redis = m::mock(Client::class); $this->prefix = 'prefix'; + $this->workspace = 'testing'; - $this->object = new RedisStorage($this->redis, $this->prefix); + $this->object = new RedisStorage($this->redis, $this->prefix, $this->workspace); } public function testGet() @@ -31,7 +32,7 @@ public function testGet() $value = 'v'; $id = 1; - $this->redis->shouldReceive('get')->with($this->prefix . $id)->andReturn($value); + $this->redis->shouldReceive('get')->with($this->workspace.'_'.$this->prefix.$id)->andReturn($value); $this->assertEquals($value, $this->object->get($id)); } @@ -42,7 +43,7 @@ public function testSet() $returnValue = 'rv'; $id = 1; - $this->redis->shouldReceive('set')->with($this->prefix . $id, $value)->andReturn($returnValue); + $this->redis->shouldReceive('set')->with($this->workspace.'_'.$this->prefix.$id, $value)->andReturn($returnValue); $this->assertEquals($returnValue, $this->object->set($id, $value)); } diff --git a/client/docker/app/confd/conf.d/parameters.hml.toml b/client/docker/app/confd/conf.d/parameters.hml.toml index bf1bf64957..9de0a2626f 100644 --- a/client/docker/app/confd/conf.d/parameters.hml.toml +++ b/client/docker/app/confd/conf.d/parameters.hml.toml @@ -2,6 +2,7 @@ src = "parameters.yml.tmpl" dest = "/var/www/config/parameters.yml" keys = [ + "/workspace", "/opg/docker/tag", "/session", "/ga", diff --git a/client/docker/app/confd/templates/parameters.yml.tmpl b/client/docker/app/confd/templates/parameters.yml.tmpl index 4ab1d476cf..7951c9a2c7 100644 --- a/client/docker/app/confd/templates/parameters.yml.tmpl +++ b/client/docker/app/confd/templates/parameters.yml.tmpl @@ -1,5 +1,6 @@ parameters: locale: en + workspace: {{ getv "/workspace" }} secret: {{ getv "/secret" }} api_base_url: {{ getv "/api/url" }} api_client_secret: {{ getv "/api/client/secret" }} diff --git a/docker-compose.yml b/docker-compose.yml index f4d83b3bf1..4ecca08576 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -126,6 +126,7 @@ services: environment: APP_ENV: ${APP_ENV:-dev} APP_DEBUG: ${APP_DEBUG:-0} + WORKSPACE: local env_file: - ./api/app/api.env - ./api/app/tests/Behat/test.env diff --git a/terraform/account/elasticache.tf b/terraform/account/elasticache.tf new file mode 100644 index 0000000000..9a7890dabb --- /dev/null +++ b/terraform/account/elasticache.tf @@ -0,0 +1,77 @@ +# see comments for ticket ddpb-3661 for extra details on in transit encryption decisions +resource "aws_elasticache_replication_group" "cache_api" { + automatic_failover_enabled = true + engine = "redis" + engine_version = "6.x" + parameter_group_name = "api-cache-params" + replication_group_id = "api-redis-${local.account.name}" + description = "Replication Group for API" + node_type = "cache.m7g.large" + num_cache_clusters = 2 + port = 6379 + subnet_group_name = local.account.ec_subnet_group + security_group_ids = [aws_security_group.cache_api_sg.id] + snapshot_retention_limit = 1 + apply_immediately = true + at_rest_encryption_enabled = true + #tfsec:ignore:aws-elasticache-enable-in-transit-encryption - too much of a performance hit. To be re-evaluated. + transit_encryption_enabled = false + tags = merge({ + InstanceName = "api-${local.account.name}" + Stack = local.account.name + }, local.default_tags) +} + +resource "aws_security_group" "cache_api_sg" { + name = "${local.account.name}-account-cache-api" + vpc_id = aws_vpc.main.id + tags = merge(local.default_tags, { Name = "${local.account.name}-account-cache--api" }) + + lifecycle { + create_before_destroy = true + } +} + +# see comments for ticket ddpb-3661 for extra details on in transit encryption decisions +resource "aws_elasticache_replication_group" "front_api" { + automatic_failover_enabled = true + engine = "redis" + engine_version = "6.x" + parameter_group_name = "default.redis6.x" + replication_group_id = "frontend-redis-${local.account.name}" + description = "Replication Group for Front and Admin" + node_type = "cache.m7g.large" + num_cache_clusters = 2 + port = 6379 + subnet_group_name = local.account.ec_subnet_group + security_group_ids = [aws_security_group.cache_front_sg.id] + snapshot_retention_limit = 1 + apply_immediately = true + at_rest_encryption_enabled = true + #tfsec:ignore:aws-elasticache-enable-in-transit-encryption - too much of a performance hit. To be re-evaluated. + transit_encryption_enabled = false + tags = merge({ + InstanceName = "front-${local.account.name}" + Stack = local.account.name + }, local.default_tags) +} + +resource "aws_security_group" "cache_front_sg" { + name = "${local.account.name}-account-cache-frontend" + vpc_id = aws_vpc.main.id + tags = merge(local.default_tags, { Name = "${local.account.name}-account-cache-frontend" }) + + lifecycle { + create_before_destroy = true + } +} + +resource "aws_elasticache_parameter_group" "digideps" { + name = "api-cache-params" + family = "redis6.x" + + parameter { + name = "maxmemory-policy" + value = "allkeys-lru" + } +} diff --git a/terraform/account/elasticache_parameters.tf b/terraform/account/elasticache_parameters.tf deleted file mode 100644 index 909b64b987..0000000000 --- a/terraform/account/elasticache_parameters.tf +++ /dev/null @@ -1,10 +0,0 @@ -resource "aws_elasticache_parameter_group" "digideps" { - name = "api-cache-params" - family = "redis6.x" - - parameter { - name = "maxmemory-policy" - value = "allkeys-lru" - } - -} diff --git a/terraform/environment/admin_service.tf b/terraform/environment/admin_service.tf index 114d55319d..76c6e93e81 100644 --- a/terraform/environment/admin_service.tf +++ b/terraform/environment/admin_service.tf @@ -160,14 +160,15 @@ locals { { name = "S3_BUCKETNAME", value = "pa-uploads-${local.environment}" }, { name = "S3_SIRIUS_BUCKET", value = "digideps.${local.account.sirius_environment}.eu-west-1.sirius.opg.justice.gov.uk" }, { name = "SESSION_REDIS_DSN", value = "redis://${aws_route53_record.frontend_redis.fqdn}" }, - { name = "SESSION_PREFIX", value = "dd_session_admin" }, + { name = "SESSION_PREFIX", value = "dd_admin" }, { name = "APP_ENV", value = local.account.app_env }, { name = "OPG_DOCKER_TAG", value = var.OPG_DOCKER_TAG }, { name = "HTMLTOPDF_ADDRESS", value = "http://${local.htmltopdf_service_fqdn}" }, { name = "ENVIRONMENT", value = local.environment }, { name = "NGINX_APP_NAME", value = "admin" }, { name = "PA_PRO_REPORT_CSV_FILENAME", value = "paProDeputyReport.csv" }, - { name = "LAY_REPORT_CSV_FILENAME", value = "layDeputyReport.csv" } + { name = "LAY_REPORT_CSV_FILENAME", value = "layDeputyReport.csv" }, + { name = "WORKSPACE", value = local.environment } ] } ) diff --git a/terraform/environment/admin_sg.tf b/terraform/environment/admin_sg.tf index bce24f5cf1..edcb5db42c 100644 --- a/terraform/environment/admin_sg.tf +++ b/terraform/environment/admin_sg.tf @@ -30,6 +30,13 @@ locals { target_type = "security_group_id" target = module.frontend_cache_security_group.id } + # cache_front = { + # port = 6379 + # type = "egress" + # protocol = "tcp" + # target_type = "security_group_id" + # target = data.aws_security_group.front_cache_sg.id + # } api = { port = 80 type = "egress" diff --git a/terraform/environment/api_service.tf b/terraform/environment/api_service.tf index d21fc71550..f187a970a5 100644 --- a/terraform/environment/api_service.tf +++ b/terraform/environment/api_service.tf @@ -215,6 +215,10 @@ locals { name = "SECRETS_PREFIX", value = join("", [local.secrets_prefix, "/"]) }, + { + name = "WORKSPACE", + value = local.environment + }, ] } ) diff --git a/terraform/environment/api_sg.tf b/terraform/environment/api_sg.tf index 656b1e2497..61d959406d 100644 --- a/terraform/environment/api_sg.tf +++ b/terraform/environment/api_sg.tf @@ -13,6 +13,13 @@ locals { target_type = "security_group_id" target = module.api_cache_security_group.id } + # cache_api = { + # port = 6379 + # type = "egress" + # protocol = "tcp" + # target_type = "security_group_id" + # target = data.aws_security_group.api_cache_sg.id + # } rds = { port = 5432 type = "egress" diff --git a/terraform/environment/check_csv_uploaded.tf b/terraform/environment/check_csv_uploaded.tf index 403678a88c..3bf25eba65 100644 --- a/terraform/environment/check_csv_uploaded.tf +++ b/terraform/environment/check_csv_uploaded.tf @@ -184,8 +184,13 @@ locals { value = "redis://${aws_route53_record.frontend_redis.fqdn}" }, { - name = "SESSION_PREFIX", - value = "dd_session_front" } + name = "SESSION_PREFIX", + value = "dd_front" + }, + { + name = "WORKSPACE", + value = local.environment + } ] } diff --git a/terraform/environment/dns_private.tf b/terraform/environment/dns_private.tf index 3ec3098ada..45c7349323 100644 --- a/terraform/environment/dns_private.tf +++ b/terraform/environment/dns_private.tf @@ -14,13 +14,15 @@ resource "aws_route53_record" "frontend_redis" { type = "CNAME" zone_id = aws_route53_zone.internal.id records = [aws_elasticache_replication_group.frontend.primary_endpoint_address] - ttl = 300 + # records = [data.aws_elasticache_replication_group.front_cache_cluster.primary_endpoint_address] + ttl = 300 } resource "aws_route53_record" "api_redis" { name = "api-redis" type = "CNAME" zone_id = aws_route53_zone.internal.id + # records = [data.aws_elasticache_replication_group.api_cache_cluster.primary_endpoint_address] records = [aws_elasticache_replication_group.api.primary_endpoint_address] ttl = 300 } diff --git a/terraform/environment/elasticache.tf b/terraform/environment/elasticache.tf new file mode 100644 index 0000000000..1e5f3a5355 --- /dev/null +++ b/terraform/environment/elasticache.tf @@ -0,0 +1,49 @@ +## Frontend Elasticache +# +#data "aws_elasticache_replication_group" "front_cache_cluster" { +# replication_group_id = "frontend-redis-${local.account.name}" +#} +# +#data "aws_security_group" "front_cache_sg" { +# name = "${local.account.name}-account-cache-frontend" +#} +# +#resource "aws_security_group_rule" "admin_to_redis" { +# description = "Admin to to front cache cluster" +# from_port = 6379 +# to_port = 6379 +# type = "ingress" +# protocol = "tcp" +# security_group_id = data.aws_security_group.front_cache_sg.id +# source_security_group_id = module.admin_service_security_group.id +#} +# +#resource "aws_security_group_rule" "front_to_redis" { +# description = "Frontend to front cache cluster" +# from_port = 6379 +# to_port = 6379 +# type = "ingress" +# protocol = "tcp" +# security_group_id = data.aws_security_group.front_cache_sg.id +# source_security_group_id = module.front_service_security_group.id +#} +# +## API Elasticache +# +#data "aws_elasticache_replication_group" "api_cache_cluster" { +# replication_group_id = "api-redis-${local.account.name}" +#} +# +#data "aws_security_group" "api_cache_sg" { +# name = "${local.account.name}-account-cache-api" +#} +# +#resource "aws_security_group_rule" "api_to_redis" { +# description = "Api to Api cache cluster" +# from_port = 6379 +# to_port = 6379 +# type = "ingress" +# protocol = "tcp" +# security_group_id = data.aws_security_group.api_cache_sg.id +# source_security_group_id = module.api_service_security_group.id +#} diff --git a/terraform/environment/front_service.tf b/terraform/environment/front_service.tf index c48dd9aec9..7d0e01e164 100644 --- a/terraform/environment/front_service.tf +++ b/terraform/environment/front_service.tf @@ -165,7 +165,8 @@ locals { { name = "S3_BUCKETNAME", value = "pa-uploads-${local.environment}" }, { name = "SECRETS_PREFIX", value = join("", [local.secrets_prefix, "/"]) }, { name = "SESSION_REDIS_DSN", value = "redis://${aws_route53_record.frontend_redis.fqdn}" }, - { name = "SESSION_PREFIX", value = "dd_session_front" } + { name = "SESSION_PREFIX", value = "dd_front" }, + { name = "WORKSPACE", value = local.environment } ] } ) diff --git a/terraform/environment/front_sg.tf b/terraform/environment/front_sg.tf index 08ecf5e2fb..e32e2dcd1f 100644 --- a/terraform/environment/front_sg.tf +++ b/terraform/environment/front_sg.tf @@ -30,6 +30,13 @@ locals { target_type = "security_group_id" target = module.frontend_cache_security_group.id } + # cache_front = { + # port = 6379 + # type = "egress" + # protocol = "tcp" + # target_type = "security_group_id" + # target = data.aws_security_group.front_cache_sg.id + # } api = { port = 80 type = "egress" diff --git a/terraform/environment/integration_test_v2.tf b/terraform/environment/integration_test_v2.tf index 841c49ee0d..b5de60ad5a 100644 --- a/terraform/environment/integration_test_v2.tf +++ b/terraform/environment/integration_test_v2.tf @@ -162,6 +162,10 @@ locals { { name = "REDIS_DSN", value = "redis://${aws_route53_record.api_redis.fqdn}" + }, + { + name = "WORKSPACE", + value = local.environment } ] } diff --git a/terraform/environment/smoke_test.tf b/terraform/environment/smoke_test.tf index 9ffcc82c88..7d700e8c90 100644 --- a/terraform/environment/smoke_test.tf +++ b/terraform/environment/smoke_test.tf @@ -92,6 +92,10 @@ locals { { name = "NONADMIN_HOST", value = "https://${aws_route53_record.front.fqdn}" + }, + { + name = "WORKSPACE", + value = local.environment } ] } diff --git a/terraform/environment/sync_to_sirius_checklists.tf b/terraform/environment/sync_to_sirius_checklists.tf index 072dca569f..7dba1ce59c 100644 --- a/terraform/environment/sync_to_sirius_checklists.tf +++ b/terraform/environment/sync_to_sirius_checklists.tf @@ -154,7 +154,7 @@ locals { }, { name = "SESSION_PREFIX", - value = "dd_session_check" + value = "dd_check" }, { name = "EMAIL_SEND_INTERNAL", @@ -179,6 +179,10 @@ locals { { name = "HTMLTOPDF_ADDRESS", value = "http://${local.htmltopdf_service_fqdn}" + }, + { + name = "WORKSPACE", + value = local.environment } ] } diff --git a/terraform/environment/sync_to_sirius_documents.tf b/terraform/environment/sync_to_sirius_documents.tf index 99082ec235..32aa7a39c1 100644 --- a/terraform/environment/sync_to_sirius_documents.tf +++ b/terraform/environment/sync_to_sirius_documents.tf @@ -146,7 +146,7 @@ locals { }, { name = "SESSION_PREFIX", - value = "dd_session_check" + value = "dd_check" }, { name = "EMAIL_SEND_INTERNAL", @@ -167,6 +167,10 @@ locals { { name = "PARAMETER_PREFIX", value = local.parameter_prefix + }, + { + name = "WORKSPACE", + value = local.environment } ] }