From 0f192f83b94c578b57176d415ae9551a7a8f7bef Mon Sep 17 00:00:00 2001 From: Akshay Mankar Date: Thu, 17 Oct 2024 17:19:56 +0200 Subject: [PATCH] Enable manual usage of locally running wire-server (#4176) * dockerephemeral: Use inbucket for SMTP * nginz/local-conf: Update list of endpoints * run-services: Add config tuned for manual usage and an option to turn it on * Makefile: Add target to run services tuned for manual usage * Docs * changelog Co-authored-by: Sven Tennie --- Makefile | 5 +- changelog.d/5-internal/fix-nginx-paths | 1 + changelog.d/5-internal/inbucket | 1 + changelog.d/5-internal/make-crm | 1 + deploy/dockerephemeral/docker-compose.yaml | 6 +- docs/src/developer/developer/building.md | 23 +++++ hack/bin/cabal-run-integration.sh | 6 +- integration/test/Testlib/JSON.hs | 17 ++++ integration/test/Testlib/RunServices.hs | 69 ++++++++++++-- .../integration-test/conf/nginz/nginx.conf | 93 ++++++++++--------- 10 files changed, 164 insertions(+), 58 deletions(-) create mode 100644 changelog.d/5-internal/fix-nginx-paths create mode 100644 changelog.d/5-internal/inbucket create mode 100644 changelog.d/5-internal/make-crm diff --git a/Makefile b/Makefile index 1413ff10e82..faee19b3dff 100644 --- a/Makefile +++ b/Makefile @@ -126,6 +126,9 @@ ci: cr: c db-migrate ./dist/run-services +crm: c db-migrate + ./dist/run-services -m + # Run integration from new test suite # Usage: make devtest # Usage: TEST_INCLUDE=test1,test2 make devtest @@ -134,7 +137,7 @@ devtest: ghcid --command 'cabal repl integration' --test='Testlib.Run.mainI []' .PHONY: sanitize-pr -sanitize-pr: +sanitize-pr: make lint-all-shallow make git-add-cassandra-schema @git diff-files --quiet -- || ( echo "There are unstaged changes, please take a look, consider committing them, and try again."; exit 1 ) diff --git a/changelog.d/5-internal/fix-nginx-paths b/changelog.d/5-internal/fix-nginx-paths new file mode 100644 index 00000000000..0d7bd115c65 --- /dev/null +++ b/changelog.d/5-internal/fix-nginx-paths @@ -0,0 +1 @@ +nginz/local-conf: Update list of endpoints \ No newline at end of file diff --git a/changelog.d/5-internal/inbucket b/changelog.d/5-internal/inbucket new file mode 100644 index 00000000000..12334d3b1d6 --- /dev/null +++ b/changelog.d/5-internal/inbucket @@ -0,0 +1 @@ +dockerephemeral: Use inbucket for SMTP \ No newline at end of file diff --git a/changelog.d/5-internal/make-crm b/changelog.d/5-internal/make-crm new file mode 100644 index 00000000000..eb4df600ece --- /dev/null +++ b/changelog.d/5-internal/make-crm @@ -0,0 +1 @@ +Makefile: Add target `crm` to run services tuned for manual usage \ No newline at end of file diff --git a/deploy/dockerephemeral/docker-compose.yaml b/deploy/dockerephemeral/docker-compose.yaml index debbdb32fa4..13061660d8c 100644 --- a/deploy/dockerephemeral/docker-compose.yaml +++ b/deploy/dockerephemeral/docker-compose.yaml @@ -58,9 +58,11 @@ services: basic_smtp: # needed for demo setup container_name: demo_wire_smtp - image: ixdotai/smtp:v0.5.2 + image: inbucket/inbucket:latest ports: - - 127.0.0.1:2500:25 + - 127.0.0.1:2500:2500 + - 127.0.0.1:1100:1100 + - 127.0.0.1:9000:9000 networks: - demo_wire diff --git a/docs/src/developer/developer/building.md b/docs/src/developer/developer/building.md index 2096fe9d93f..223026e8ca8 100644 --- a/docs/src/developer/developer/building.md +++ b/docs/src/developer/developer/building.md @@ -218,3 +218,26 @@ After all containers are up you can use these Makefile targets to run the tests ``` `TASTY_NUM_THREADS` can also be set to other values, it defaults to number of cores available. + +## How to run the webapp locally against locally running backend + +1. Clone the webapp from: https://github.com/wireapp/wire-webapp +2. Install these depedencies needed for the webapp: + 1. nodejs + 2. yarn + 3. mkcert +3. Copy `.env.localhost` to `.env` and uncomment the local section +4. Run the webapp using: + ```bash + yarn + yarn start + ``` +4. From wire-server repo start the dependencies using: + ```bash + ./deploy/dockerephemeral/run.sh + ``` +5. From wire-server repo start the backend using: + ```bash + make crm + ``` +6. Go to http://localhost:8081 in the browser. diff --git a/hack/bin/cabal-run-integration.sh b/hack/bin/cabal-run-integration.sh index 66daccfb538..582b2de874e 100755 --- a/hack/bin/cabal-run-integration.sh +++ b/hack/bin/cabal-run-integration.sh @@ -49,13 +49,15 @@ run_integration_tests() { then cd "$TOP_LEVEL" "$TOP_LEVEL/dist/run-services" \ - "$TOP_LEVEL/dist/integration" \ - "${@:2}" + -- \ + "$TOP_LEVEL/dist/integration" \ + "${@:2}" else service_dir="$TOP_LEVEL/services/$package" cd "$service_dir" "$TOP_LEVEL/dist/run-services" \ + -- \ "$TOP_LEVEL/dist/$package-integration" \ -s "$service_dir/$package.integration.yaml" \ -i "$TOP_LEVEL/services/integration.yaml" \ diff --git a/integration/test/Testlib/JSON.hs b/integration/test/Testlib/JSON.hs index 31eaf24ce14..96ee6da2492 100644 --- a/integration/test/Testlib/JSON.hs +++ b/integration/test/Testlib/JSON.hs @@ -262,6 +262,23 @@ setField :: setField selector v x = do modifyField @a @Value selector (\_ -> pure (toJSON v)) x +-- | Merges fields if the old and new are both Objects or Arrays. Otherwise new +-- field overwrites the old completely +mergeField :: forall a b. (HasCallStack, MakesValue a, ToJSON b) => String -> b -> a -> App Value +mergeField selector v x = do + modifyField @a @Value + selector + ( \case + Just (Object old) -> case toJSON v of + (Object new) -> pure $ Object (new <> old) + nonObjectNew -> pure nonObjectNew + Just (Array old) -> case toJSON v of + (Array new) -> pure $ Array (old <> new) + nonArrayNew -> pure nonArrayNew + _ -> pure (toJSON v) + ) + x + member :: (HasCallStack, MakesValue a) => String -> a -> App Bool member k x = KM.member (KM.fromString k) <$> (make x >>= asObject) diff --git a/integration/test/Testlib/RunServices.hs b/integration/test/Testlib/RunServices.hs index c2ee022185a..a88686b2979 100644 --- a/integration/test/Testlib/RunServices.hs +++ b/integration/test/Testlib/RunServices.hs @@ -1,10 +1,10 @@ -module Testlib.RunServices where +module Testlib.RunServices (main) where import Control.Concurrent import Control.Monad.Codensity +import Options.Applicative import System.Directory -import System.Environment (getArgs) -import System.Exit (exitWith) +import System.Exit import System.FilePath import System.Posix (getWorkingDirectory) import System.Process @@ -31,23 +31,43 @@ findProjectRoot path = do Nothing -> pure Nothing Just p -> findProjectRoot p +data Opts = Opts + { withManualTestingOverrides :: Bool, + runSubprocess :: [String] + } + deriving (Show) + +optsParser :: Parser Opts +optsParser = + Opts + <$> switch + ( long "with-manual-testing-overrides" + <> short 'm' + <> help "Run services with settings tuned for manual app usage (not recommended for running integration tests)" + ) + <*> many + ( strArgument + ( metavar "COMMAND_WITH_ARGS" + <> help "When specified, the command will be run after services have started and service will be killed after the command exits" + ) + ) + main :: IO () main = do cwd <- getWorkingDirectory mbProjectRoot <- findProjectRoot cwd + opts <- execParser (info (optsParser <**> helper) fullDesc) cfg <- case mbProjectRoot of Nothing -> error "Could not find project root. Please make sure you call run-services from somewhere in wire-server." Just projectRoot -> pure $ joinPath [projectRoot, "services/integration.yaml"] - args <- getArgs - - let run = case args of + let run = case opts.runSubprocess of [] -> do putStrLn "services started" - forever (threadDelay 1000000000) + forever (threadDelay maxBound) _ -> do - let cp = proc "sh" (["-c", "exec \"$@\"", "--"] <> args) + let cp = proc "sh" (["-c", "exec \"$@\"", "--"] <> opts.runSubprocess) (_, _, _, ph) <- createProcess cp exitWith =<< waitForProcess ph @@ -57,6 +77,37 @@ main = do $ do _modifyEnv <- traverseConcurrentlyCodensity - (\r -> void $ startDynamicBackend r mempty) + ( \r -> + void + $ if opts.withManualTestingOverrides + then startDynamicBackend r manualTestingOverrides + else startDynamicBackend r mempty + ) [backendA, backendB] liftIO run + +manualTestingOverrides :: ServiceOverrides +manualTestingOverrides = + let smtpEndpoint = object ["host" .= "localhost", "port" .= (2500 :: Int)] + authSettings = + object + [ "userTokenTimeout" .= (4838400 :: Int), + "sessionTokenTimeout" .= (86400 :: Int), + "accessTokenTimeout" .= (900 :: Int), + "providerTokenTimeout" .= (900 :: Int), + "legalHoldUserTokenTimeout" .= (4838400 :: Int), + "legalHoldAccessTokenTimeout" .= (900 :: Int) + ] + in def + { brigCfg = + mergeField "emailSMS.email.smtpEndpoint" smtpEndpoint + >=> setField "emailSMS.email.smtpConnType" "plain" + >=> removeField "emailSMS.email.sesQueue" + >=> removeField "emailSMS.email.sesEndpoint" + >=> mergeField "zauth.authSettings" authSettings + >=> setField @_ @Int "optSettings.setActivationTimeout" 3600 + >=> setField @_ @Int "optSettings.setVerificationTimeout" 3600 + >=> setField @_ @Int "optSettings.setTeamInvitationTimeout" 3600 + >=> setField @_ @Int "optSettings.setUserCookieRenewAge" 1209600 + >=> removeField "optSettings.setSuspendInactiveUsers" + } diff --git a/services/nginz/integration-test/conf/nginz/nginx.conf b/services/nginz/integration-test/conf/nginz/nginx.conf index 95b560f7b1d..c887411e18f 100644 --- a/services/nginz/integration-test/conf/nginz/nginx.conf +++ b/services/nginz/integration-test/conf/nginz/nginx.conf @@ -155,6 +155,11 @@ http { # FUTUREWORK(federation): are any other settings # (e.g. timeouts, body size, buffers, headers,...) # useful/recommended/important-for-security?) + } + + location /api-version { + include common_response_no_zauth.conf; + proxy_pass http://brig; } # Brig Endpoints @@ -201,12 +206,12 @@ http { proxy_pass http://brig; } - location /activate { + location ~* ^(/v[0-9]+)?/activate { include common_response_no_zauth.conf; proxy_pass http://brig; } - location /login { + location ~* ^(/v[0-9]+)?/login { include common_response_no_zauth.conf; proxy_pass http://brig; } @@ -221,50 +226,50 @@ http { proxy_pass http://brig; } - location /verification-code/send { + location ~* ^(/v[0-9]+)?/verification-code/send { include common_response_no_zauth.conf; proxy_pass http://brig; } ## brig authenticated endpoints - location ~* ^(/v[0-9]+)?/self$ { + location ~* ^(/v[0-9]+)?/self { include common_response_with_zauth.conf; oauth_scope self; proxy_pass http://brig; } - location /users { + location ~* ^(/v[0-9]+)?/users { include common_response_with_zauth.conf; proxy_pass http://brig; } - location /list-users { + location ~* ^(/v[0-9]+)?/list-users { include common_response_with_zauth.conf; proxy_pass http://brig; } - location /search { + location ~* ^(/v[0-9]+)?/search { include common_response_with_zauth.conf; proxy_pass http://brig; } - location /list-connections { + location ~* ^(/v[0-9]+)?/list-connections { include common_response_with_zauth.conf; proxy_pass http://brig; } - location ~* ^/teams/([^/]+)/search$ { + location ~* ^(/v[0-9]+)?/teams/([^/]+)/search$ { include common_response_with_zauth.conf; proxy_pass http://brig; } - location ~* /teams/([^/]+)/services { + location ~* ^(/v[0-9]+)?/teams/([^/]+)/services { include common_response_with_zauth.conf; proxy_pass http://brig; } - location /connections { + location ~* ^(/v[0-9]+)?/connections { include common_response_with_zauth.conf; proxy_pass http://brig; } @@ -279,17 +284,17 @@ http { proxy_pass http://brig; } - location /properties { + location ~* ^(/v[0-9]+)?/properties { include common_response_with_zauth.conf; proxy_pass http://brig; } - location /calls/config { + location ~* ^(/v[0-9]+)?/calls/config { include common_response_with_zauth.conf; proxy_pass http://brig; } - location ~* ^/teams/([^/]*)/size$ { + location ~* ^(/v[0-9]+)?/teams/([^/]*)/size$ { include common_response_with_zauth.conf; proxy_pass http://brig; } @@ -304,39 +309,39 @@ http { proxy_pass http://brig; } - location ~* ^/oauth/clients/([^/]*)$ { + location ~* ^(/v[0-9]+)?/oauth/clients/([^/]*)$ { include common_response_with_zauth.conf; proxy_pass http://brig; } - location ~* ^/oauth/authorization/codes$ { + location ~* ^(/v[0-9]+)?/oauth/authorization/codes$ { include common_response_with_zauth.conf; proxy_pass http://brig; } - location /oauth/token { + location ~* ^(/v[0-9]+)?/oauth/token { include common_response_no_zauth.conf; proxy_pass http://brig; } - location /oauth/revoke { + location ~* ^(/v[0-9]+)?/oauth/revoke { include common_response_no_zauth.conf; proxy_pass http://brig; } - location /oauth/applications { + location ~* ^(/v[0-9]+)?/oauth/applications { include common_response_with_zauth.conf; proxy_pass http://brig; } # Cargohold Endpoints - location /assets { + location ~* ^(/v[0-9]+)?/assets { include common_response_with_zauth.conf; proxy_pass http://cargohold; } - location /bot/assets { + location ~* ^(/v[0-9]+)?/bot/assets { include common_response_with_zauth.conf; proxy_pass http://cargohold; } @@ -370,62 +375,62 @@ http { proxy_pass http://galley; } - location ~* ^/conversations/([^/]*)/otr/messages { + location ~* ^(/v[0-9]+)?/conversations/([^/]*)/otr/messages { include common_response_with_zauth.conf; proxy_pass http://galley; } - location ~* ^/conversations/([^/]*)/([^/]*)/proteus/messages { + location ~* ^(/v[0-9]+)?/conversations/([^/]*)/([^/]*)/proteus/messages { include common_response_with_zauth.conf; proxy_pass http://galley; } - location ~* ^/conversations/([^/]*)/([^/]*)/protocol { + location ~* ^(/v[0-9]+)?/conversations/([^/]*)/([^/]*)/protocol { include common_response_with_zauth.conf; proxy_pass http://galley; } - location /broadcast { + location ~* ^(/v[0-9]+)?/broadcast { include common_response_with_zauth.conf; proxy_pass http://galley; } - location /bot/conversation { + location ~* ^(/v[0-9]+)?/bot/conversation { include common_response_with_zauth.conf; proxy_pass http://galley; } - location /bot/messages { + location ~* ^(/v[0-9]+)?/bot/messages { include common_response_with_zauth.conf; proxy_pass http://galley; } - location ~* ^/teams$ { + location ~* ^(/v[0-9]+)?/teams$ { include common_response_with_zauth.conf; proxy_pass http://galley; } - location ~* ^/teams/([^/]*)$ { + location ~* ^(/v[0-9]+)?/teams/([^/]*)$ { include common_response_with_zauth.conf; proxy_pass http://galley; } - location ~* ^/teams/([^/]*)/members(.*) { + location ~* ^(/v[0-9]+)?/teams/([^/]*)/members(.*) { include common_response_with_zauth.conf; proxy_pass http://galley; } - location ~* ^/teams/([^/]*)/conversations(.*) { + location ~* ^(/v[0-9]+)?/teams/([^/]*)/conversations(.*) { include common_response_with_zauth.conf; proxy_pass http://galley; } - location ~* ^/teams/([^/]*)/features { + location ~* ^(/v[0-9]+)?/teams/([^/]*)/features { include common_response_with_zauth.conf; proxy_pass http://galley; } - location ~* ^/teams/([^/]*)/features/([^/]*) { + location ~* ^(/v[0-9]+)?/teams/([^/]*)/features/([^/]*) { include common_response_with_zauth.conf; proxy_pass http://galley; } @@ -441,22 +446,22 @@ http { proxy_pass http://galley; } - location ~* ^/teams/([^/]*)/legalhold(.*) { + location ~* ^(/v[0-9]+)?/teams/([^/]*)/legalhold(.*) { include common_response_with_zauth.conf; proxy_pass http://galley; } - location ~* ^/teams/([^/]*)/members/csv$ { + location ~* ^(/v[0-9]+)?/teams/([^/]*)/members/csv$ { include common_response_with_zauth.conf; proxy_pass http://galley; } - location /mls/welcome { + location ~* ^(/v[0-9]+)?/mls/welcome { include common_response_with_zauth.conf; proxy_pass http://galley; } - location /mls/messages { + location ~* ^(/v[0-9]+)?/mls/messages { include common_response_with_zauth.conf; proxy_pass http://galley; } @@ -473,31 +478,31 @@ http { # Gundeck Endpoints - location /push { + location ~* ^(/v[0-9]+)?/push { include common_response_with_zauth.conf; proxy_pass http://gundeck; } - location /presences { + location ~* ^(/v[0-9]+)?/presences { include common_response_with_zauth.conf; proxy_pass http://gundeck; } - location ~* ^(/v[0-9]+)?/notifications$ { + location ~* ^(/v[0-9]+)?/notifications { include common_response_with_zauth.conf; proxy_pass http://gundeck; } # Proxy Endpoints - location /proxy { + location ~* ^(/v[0-9]+)?/proxy { include common_response_with_zauth.conf; proxy_pass http://proxy; } # Cannon Endpoints - location /await { + location ~* ^(/v[0-9]+)?/await { include common_response_with_zauth.conf; proxy_pass http://cannon; @@ -508,12 +513,12 @@ http { # Spar Endpoints - location /sso { + location ~* ^(/v[0-9]+)?/sso { include common_response_no_zauth.conf; proxy_pass http://spar; } - location /identity-providers { + location ~* ^(/v[0-9]+)?/identity-providers { include common_response_with_zauth.conf; proxy_pass http://spar; }