From 0a75340b8d7a94db4cdb26563325d06bcd6a05c8 Mon Sep 17 00:00:00 2001 From: Eric Le Goff Date: Tue, 23 May 2017 16:04:10 +0200 Subject: [PATCH 1/4] Connected to #114 --- docs/developer_guide/running.rst | 149 ++++++++++++++++++++++++++++++- 1 file changed, 146 insertions(+), 3 deletions(-) diff --git a/docs/developer_guide/running.rst b/docs/developer_guide/running.rst index 2054a55..85f0382 100644 --- a/docs/developer_guide/running.rst +++ b/docs/developer_guide/running.rst @@ -87,9 +87,9 @@ REPL with boot and in the REPL run the commands listed below:: $ export ES_HOST=localhost $ export ES_PORT=9300 - $ export CONFIG_PATH=ssclj-conf.edn + $ export CONFIG_NAME=ssclj-conf.edn - $ boot repl + $ boot dev-env repl boot.user=> (require '[com.sixsq.slipstream.ssclj.app.server :as server :reload true]) nil boot.user=> (def stop-fn (server/start 8201)) @@ -112,7 +112,8 @@ Typical content looks like:: :make-pool? true}} The ``ssclj-conf.edn`` is part of the source code and located under -``resources/`` subdirectory, which gets appended to the classpath. +``test-resources/`` subdirectory, which gets appended to the classpath thanks to the ``dev-env`` +option for the ``boot`` command above. The service's log file can be found under ``logs/ssclj-.log`` @@ -223,6 +224,148 @@ that both ``jar`` and ``conf`` artifacts should be added. ${project.version} +Starting the HTTP Server +------------------------ + +`Nginx `__ is required to serve SlipsStream pages. + +* Nginx installation + +Linux users should install it from the official `documentation `__ page. + +Mac OS X users can simply run :: + + brew install nginx + +* Nginx configuration + +By default, the main Nginx configuration file is named ``nginx.conf`` and placed in the directory ``/usr/local/nginx/conf``, ``/etc/nginx``, or ``/usr/local/etc/nginx``. + +It should contain the following :: + + worker_processes 1; + + events { + worker_connections 1024; + } + + + http { + include mime.types; + default_type application/octet-stream; + sendfile on; + keepalive_timeout 65; + include servers/*.conf; + } + + + +Create a ``servers`` directory realative to your ``nginx.conf`` location and add the following two files into it + +- ``slipstream.conf``:: + + upstream slipstream_servers { + server 127.0.0.1:8080; + + keepalive 50; + } + + upstream ssclj_servers { + server 127.0.0.1:8201; + + keepalive 50; + } + + ssl_session_cache shared:SSL:1m; + ssl_session_timeout 30m; + ssl_session_tickets on; + #ssl_dhparam /etc/nginx/ssl/dhparam.pem; + ssl_prefer_server_ciphers on; + ssl_protocols TLSv1 TLSv1.1 TLSv1.2; + ssl_ecdh_curve prime256v1; + ssl_ciphers "ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:DHE-RSA-AES256-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES256-SHA:ECDHE-RSA-AES256-SHA:DHE-RSA-AES256-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES128-SHA:DHE-RSA-AES128-SHA:!aNULL:!eNULL:!EXPORT:!DES:!MD5:!PSK:!RC4:!3DES:AES128-GCM-SHA256"; + resolver 8.8.8.8 8.8.4.4; + #ssl_stapling on; + #ssl_stapling_verify on; + + # Tells browsers to ONLY connect via HTTPS to SlipStream. + # The timeout is set to 1 year, which is reset with each visit. + #add_header Strict-Transport-Security "max-age=31536000; includeSubdomains" always; + + server { + listen 443 ssl http2; # deferred reuseport; + + ssl_certificate /usr/local/etc/nginx/ssl/nginx.crt; + ssl_certificate_key /usr/local/etc/nginx/ssl/nginx.key; + + # Include SlipStream common configuration parameters + location / { + proxy_pass http://slipstream_servers; + include servers/slipstream-proxy.params; + } + + location /auth { + proxy_pass http://ssclj_servers; + include servers/slipstream-proxy.params; + } + + location /api { + proxy_pass http://ssclj_servers; + include servers/slipstream-proxy.params; + } + + } + + - and ``slipstream-proxy.params``:: + + proxy_http_version 1.1; + + set $via "1.1 $host"; + if ($http_via) { + set $via "$http_via, $via"; + } + + proxy_set_header Via $via; + proxy_set_header Host $http_host; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + + proxy_set_header slipstream-authn-info ""; + proxy_set_header slipstream-ssl-server-name $ssl_server_name; + + proxy_redirect off; + +At the same level as the ``servers`` directory, create a ``ssl`` directory and jump into it. +From there, you will generate key and certificate files :: + + openssl req -x509 -nodes -days 365 -newkey rsa:2048 -keyout nginx.key -out nginx.crt + +At this stage, your Nginx configuration directory should look like:: + + ├── [...] + ├── nginx.conf + ├── servers + │   ├── slipstream-proxy.params + │   └── slipstream.conf + ├── ssl + │   ├── nginx.crt + │   └── nginx.key + + +* Optionally you may want to test your Nginx configuration:: + + sudo nginx -t + +* Finally launch Nginx:: + + sudo nginx + +TCP port 443 which you have configured in ``servers\slipstream.conf`` is the standard TCP port that is used for websites which use SSL, therefore your Slipstream is available at +``https://localhost`` + + You are now ready to :ref:`configure ` your new SlipStream server. From 8c4a7ac912bb9bd2008d5bcaa6a13a80e86db8f3 Mon Sep 17 00:00:00 2001 From: Eric Le Goff Date: Fri, 26 May 2017 11:55:23 +0200 Subject: [PATCH 2/4] Connected to #114 : added some text about how to force the usage of hsqldb not in memory --- docs/developer_guide/running.rst | 59 +++++++++++++++++++++++--------- 1 file changed, 43 insertions(+), 16 deletions(-) diff --git a/docs/developer_guide/running.rst b/docs/developer_guide/running.rst index 85f0382..10a89b8 100644 --- a/docs/developer_guide/running.rst +++ b/docs/developer_guide/running.rst @@ -82,14 +82,25 @@ The service should be started from the ``ssclj/jar`` module of $ cd SlipStreamServer/ssclj/jar -To run the service, export the required environment variables, start Clojure -REPL with boot and in the REPL run the commands listed below:: +To run the service, export the required environment variables:: $ export ES_HOST=localhost $ export ES_PORT=9300 - $ export CONFIG_NAME=ssclj-conf.edn + $ export CONFIG_NAME=config-hsqldb.edn - $ boot dev-env repl +SlipStream authentication requires you define environment variables ``AUTH_PUBLIC_KEY`` and ``AUTH_PRIVATE_KEY`` +pointing to your public and private keys absolute paths. + +Sample files are provided in ```ssclj/jar/test-resources``` :: + + test-resources + ├── auth_privkey.pem + ├── auth_pubkey.pem + ├── ... + +Start Clojure REPL with boot and in the REPL run the commands listed below:: + + $ boot repl boot.user=> (require '[com.sixsq.slipstream.ssclj.app.server :as server :reload true]) nil boot.user=> (def stop-fn (server/start 8201)) @@ -101,19 +112,34 @@ The services will be started on port ``8201``. You can set it as needed, taking into account that it will be required later during the startup of the main SlipStream service. -The directory containing the ``ssclj-conf.edn`` file must be on the classpath. The -``ssclj-conf.edn`` is the path to the file containing the HSQLDB database definition. +The directory containing the ``config-hsqldb.edn`` file must be on the classpath. The +``config-hsqldb.edn`` is the path to the file containing the HSQLDB database definition. Typical content looks like:: - {:db { - :classname "org.hsqldb.jdbc.JDBCDriver" - :subprotocol "hsqldb" - :subname "mem://localhost:9012/devresources" - :make-pool? true}} + {:auth-db { + :classname "org.hsqldb.jdbc.JDBCDriver" + :subprotocol "hsqldb" + :subname "hsql://localhost:9001/slipstream" + :make-pool? true} + + :token-nb-minutes-expiry 120 + + ;; Used by front server when redirecting (URL accessed only locally) + :upstream-server "http://localhost:8080" + + ;; Used by external authentication providers + :auth-server "http://localhost:8201" + :main-server "http://localhost:8080" + + ;; Application must be registered on Github + ;; See https://github.com/settings/applications/new + ;; Homepage URL can be + ;; The Authorization callback URL must be /auth/callback-github + :github-client-id "changeme" + :github-client-secret "changeme"} -The ``ssclj-conf.edn`` is part of the source code and located under -``test-resources/`` subdirectory, which gets appended to the classpath thanks to the ``dev-env`` -option for the ``boot`` command above. +The ``config-hsqldb.edn`` is part of the source code and located under +``resources/`` subdirectory. The service's log file can be found under ``logs/ssclj-.log`` @@ -156,7 +182,7 @@ archive (war file). :: $ cd SlipStreamServer/war - $ mvn jetty:run-war + $ mvn jetty:run-war -Dpersistence.unit=hsqldb-schema If the last command returns an error like ``JettyRunWarMojo : Unsupported major.minor version 51.0`` make sure you @@ -177,11 +203,12 @@ the server pointing to source static location as following: $ export ES_HOST=localhost $ export ES_PORT=9300 $ mvn jetty:run-war \ + -Dpersistence.unit=hsqldb-schema \ -Dstatic.content.location=file:../../SlipStreamUI/clj/src/slipstream/ui/views The server makes use of Elasticsearch as database backend, therefore, you see the need to set the host and port of Elasticsearch. -You can also change the main database backend connection using the +You can also change the main database backend connection updating the ``persistence.unit``. For example: :: From f426318d1d2b467b0e05ae9a2e8376d2643b43b2 Mon Sep 17 00:00:00 2001 From: Khaled Basbous Date: Mon, 29 May 2017 15:24:03 +0200 Subject: [PATCH 3/4] Update running.rst --- docs/developer_guide/running.rst | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/docs/developer_guide/running.rst b/docs/developer_guide/running.rst index 10a89b8..8b89e87 100644 --- a/docs/developer_guide/running.rst +++ b/docs/developer_guide/running.rst @@ -254,7 +254,10 @@ that both ``jar`` and ``conf`` artifacts should be added. Starting the HTTP Server ------------------------ -`Nginx `__ is required to serve SlipsStream pages. +`Nginx `__ is required to serve SlipStream pages and calls done to Api(s). +As there are complementary SlipStream services (SlipStream server, SSCLJ server) which run behind different ports, +and the fact that SlipStream force the usage of secure cookies, all services should be run behind an SSL encryption. +We use following simplified configuration of Nginx to fulfill this need. * Nginx installation @@ -340,6 +343,11 @@ Create a ``servers`` directory realative to your ``nginx.conf`` location and add proxy_pass http://ssclj_servers; include servers/slipstream-proxy.params; } + + location /filter-rank { + proxy_pass http://prs_servers; + include servers/slipstream-proxy.params; + } } From ab4327f79f57d48b98e032adc35eac5a450837d3 Mon Sep 17 00:00:00 2001 From: Konstantin Skaburskas Date: Mon, 19 Jun 2017 17:31:42 +0200 Subject: [PATCH 4/4] updates: running ssclj and nginx. --- docs/developer_guide/running.rst | 124 ++++++++++++++----------------- 1 file changed, 56 insertions(+), 68 deletions(-) diff --git a/docs/developer_guide/running.rst b/docs/developer_guide/running.rst index 8b89e87..0e57ab2 100644 --- a/docs/developer_guide/running.rst +++ b/docs/developer_guide/running.rst @@ -82,25 +82,9 @@ The service should be started from the ``ssclj/jar`` module of $ cd SlipStreamServer/ssclj/jar -To run the service, export the required environment variables:: - - $ export ES_HOST=localhost - $ export ES_PORT=9300 - $ export CONFIG_NAME=config-hsqldb.edn - -SlipStream authentication requires you define environment variables ``AUTH_PUBLIC_KEY`` and ``AUTH_PRIVATE_KEY`` -pointing to your public and private keys absolute paths. - -Sample files are provided in ```ssclj/jar/test-resources``` :: - - test-resources - ├── auth_privkey.pem - ├── auth_pubkey.pem - ├── ... - Start Clojure REPL with boot and in the REPL run the commands listed below:: - $ boot repl + $ boot server-repl boot.user=> (require '[com.sixsq.slipstream.ssclj.app.server :as server :reload true]) nil boot.user=> (def stop-fn (server/start 8201)) @@ -112,36 +96,40 @@ The services will be started on port ``8201``. You can set it as needed, taking into account that it will be required later during the startup of the main SlipStream service. -The directory containing the ``config-hsqldb.edn`` file must be on the classpath. The -``config-hsqldb.edn`` is the path to the file containing the HSQLDB database definition. -Typical content looks like:: +It is assumed that an instance of `Elasticsearch `__ is +running on ``localhost:9300``. If this is not the case, export the following +environment variables defining the coordinates of Elasticsearch:: - {:auth-db { - :classname "org.hsqldb.jdbc.JDBCDriver" - :subprotocol "hsqldb" - :subname "hsql://localhost:9001/slipstream" - :make-pool? true} + $ export ES_HOST= + $ export ES_PORT= - :token-nb-minutes-expiry 120 +The service uses the configuration file defined by ``CONFIG_NAME`` environment +variable. To be found by the service, the file should be on the service's +classpath. ``SlipStreamServer/ssclj/jar/boot.build`` (the project's +configuration file) sets the service configuration file name and extends the +classpath to include the default location containing the file. Typically, the +file is named ``config-hsqldb-mem.edn`` and located in ``test-resources``:: - ;; Used by front server when redirecting (URL accessed only locally) - :upstream-server "http://localhost:8080" + (environ :env {:config-name "config-hsqldb-mem.edn" + ... + (set-env! :source-paths #(set (concat % #{"test" "test-resources"}))) + ... - ;; Used by external authentication providers - :auth-server "http://localhost:8201" - :main-server "http://localhost:8080" +So, both the file name and its location can be modified in ``boot.build``. - ;; Application must be registered on Github - ;; See https://github.com/settings/applications/new - ;; Homepage URL can be - ;; The Authorization callback URL must be /auth/callback-github - :github-client-id "changeme" - :github-client-secret "changeme"} +Apart from other configuration parameters the configuration file contains +HSQLDB configuration definition. Typical content looks like:: + + {:auth-db { + :classname "org.hsqldb.jdbc.JDBCDriver" + :subprotocol "hsqldb" + :subname "mem://localhost:9012/devresources" + :make-pool? true}} The ``config-hsqldb.edn`` is part of the source code and located under ``resources/`` subdirectory. -The service's log file can be found under ``logs/ssclj-.log`` +The service's log file can be found under ``logs/ssclj-.log``. You can add other dependencies to the classpath as needed. This can be done either by editing the list of dependencies in @@ -184,11 +172,11 @@ archive (war file). $ cd SlipStreamServer/war $ mvn jetty:run-war -Dpersistence.unit=hsqldb-schema -If the last command returns an error like -``JettyRunWarMojo : Unsupported major.minor version 51.0`` make sure you -have Java 8 installed. You can find the appropriate download from the -Java web site. You may also want to consult `this -article `__ +If the last command returns an error like ``JettyRunWarMojo : Unsupported +major.minor version 51.0`` make sure you have Java 8 installed. You can find +the appropriate download from the Java web site. You may also want to consult +`this article +`__ for setting up the environment. As you can see, we run SlipStream as a war behind Jetty. Now that the @@ -251,17 +239,21 @@ that both ``jar`` and ``conf`` artifacts should be added. ${project.version} -Starting the HTTP Server ------------------------- +Starting HTTP Server and Reverce Proxy +-------------------------------------- -`Nginx `__ is required to serve SlipStream pages and calls done to Api(s). -As there are complementary SlipStream services (SlipStream server, SSCLJ server) which run behind different ports, -and the fact that SlipStream force the usage of secure cookies, all services should be run behind an SSL encryption. -We use following simplified configuration of Nginx to fulfill this need. +HTTP server and reverse proxy are required to serve SlipStream static content +and calls to API. Below this is done on an example of `Nginx +`__. As there are complementary SlipStream services +(SlipStream server, SSCLJ server) which run on different ports, and the +fact that SlipStream force the usage of secure cookies, all services should be +run behind an SSL encryption. We use following simplified configuration of +Nginx to fulfill this need. * Nginx installation -Linux users should install it from the official `documentation `__ page. +Linux users should install it from the official `documentation +`__ page. Mac OS X users can simply run :: @@ -269,7 +261,9 @@ Mac OS X users can simply run :: * Nginx configuration -By default, the main Nginx configuration file is named ``nginx.conf`` and placed in the directory ``/usr/local/nginx/conf``, ``/etc/nginx``, or ``/usr/local/etc/nginx``. +By default, the main Nginx configuration file is named ``nginx.conf`` and +placed in the directory ``/usr/local/nginx/conf``, ``/etc/nginx``, or +``/usr/local/etc/nginx``. It should contain the following :: @@ -289,10 +283,11 @@ It should contain the following :: } +You need to create configuration for upstream SlipStream services and SSL that will be located +in ``nginx/servers`` and ``nginx/ssl`` respectively :: -Create a ``servers`` directory realative to your ``nginx.conf`` location and add the following two files into it - -- ``slipstream.conf``:: + $ mkdir {servers,ssl} + $ cat > servers/slipstream.conf< servers/slipstream-proxy.params<< EOF proxy_http_version 1.1; @@ -372,9 +361,7 @@ Create a ``servers`` directory realative to your ``nginx.conf`` location and add proxy_redirect off; -At the same level as the ``servers`` directory, create a ``ssl`` directory and jump into it. -From there, you will generate key and certificate files :: - + $ cd ssl openssl req -x509 -nodes -days 365 -newkey rsa:2048 -keyout nginx.key -out nginx.crt At this stage, your Nginx configuration directory should look like:: @@ -397,8 +384,9 @@ At this stage, your Nginx configuration directory should look like:: sudo nginx -TCP port 443 which you have configured in ``servers\slipstream.conf`` is the standard TCP port that is used for websites which use SSL, therefore your Slipstream is available at -``https://localhost`` +TCP port 443 which you have configured in ``servers/slipstream.conf`` is the +standard TCP port that is used for websites which use SSL, therefore your +Slipstream is available at ``https://localhost`` You are now ready to :ref:`configure ` your new SlipStream