diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 0000000..13c76aa --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,77 @@ +--- +name: CI + +on: + push: + pull_request: + +concurrency: + group: ${{ github.workflow }} + +jobs: + build: + name: build + runs-on: ubuntu-latest + container: centos:centos7 + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + - name: Install build requisites + run: | + yum -y install rpmdevtools rpmlint rsync + - name: Build rpm + run: | + chmod 777 rpm/teapot-rpm.sh + rpm/teapot-rpm.sh + - name: Upload rpm + uses: actions/upload-artifact@v3 + with: + name: teapot-rpm + path: | + /github/home/rpmbuild/RPMS/noarch/teapot-v*-1.el7.noarch.rpm + + test: + name: test + needs: build + runs-on: ubuntu-latest + container: + image: dvrbanec/teapot-dependencies:latest + ports: + - 4242:4242 + - 8081:8081 + - 8085:8085 + services: + keycloak: + image: dvrbanec/keycloak-dev:latest + steps: + - uses: actions/download-artifact@v3 + with: + name: teapot-rpm + - name: Install the rpm package and self-signed certs + run: | + rpm -i teapot-v*-1.el7.noarch.rpm --nodeps + /usr/share/teapot/self-signed-cert-gen.sh + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + - name: Test Teapot + run: | + eval "$(oidc-agent)" + python3 keycloak-setup/keycloak-config.py + oidc-gen --flow password -f keycloak-setup/client_config.json --op-username test-user1 --op-password secret1 --prompt none --pw-file keycloak_setup/pw-file test-user1 + oidc-gen --flow password -f keycloak-setup/client_config.json --op-username test-user2 --op-password secret2 --prompt none --pw-file keycloak_setup/pw-file test-user2 + adduser test-user1 + adduser test-user2 + cp robot/storage-areas /etc/teapot/ + chmod 777 robot/User-sub_generator.sh + robot/User-sub_generator.sh + /usr/share/teapot/webdav-automation.sh test-user1 8085 + robot /__w/teapot/teapot/robot/storm-webdav-tests.robot + - name: Upload configs + uses: actions/upload-artifact@v3 + with: + name: test-config + path: | + /var/lib/teapot/user-test-user1 + \ No newline at end of file diff --git a/keycloak-setup/Dockerfile b/keycloak-setup/Dockerfile new file mode 100644 index 0000000..fac4a95 --- /dev/null +++ b/keycloak-setup/Dockerfile @@ -0,0 +1,22 @@ +FROM registry.access.redhat.com/ubi9 AS ubi-micro-build + +RUN mkdir -p /mnt/rootfs + +RUN dnf install --installroot /mnt/rootfs curl --releasever 9 --setopt install_weak_deps=false --nodocs -y && \ + dnf --installroot /mnt/rootfs clean all && \ + rpm --root /mnt/rootfs -e --nodeps setup + +FROM quay.io/keycloak/keycloak:22.0.5 + +COPY --from=ubi-micro-build /mnt/rootfs / + +ENV KEYCLOAK_ADMIN=admin +ENV KEYCLOAK_ADMIN_PASSWORD=testing1 +ENV KC_HOSTNAME=keycloak +ENV KC_HTTP_ENABLED=true +ENV KC_HEALTH_ENABLED=true + +CMD [ "start-dev"] + +HEALTHCHECK --interval=10s --start-period=30s --retries=15 CMD curl --fail http://keycloak:8080 || exit 1 + diff --git a/keycloak-setup/client_config.json b/keycloak-setup/client_config.json new file mode 100644 index 0000000..f050643 --- /dev/null +++ b/keycloak-setup/client_config.json @@ -0,0 +1,9 @@ +{ + "issuer_url": "http://keycloak:8080/realms/test-realm", + "client_id": "test-client", + "client_secret": "test-secret", + "scope": "max", + "redirect_uris": [ + "http://localhost:4242" + ] +} \ No newline at end of file diff --git a/keycloak-setup/keycloak-config.py b/keycloak-setup/keycloak-config.py new file mode 100644 index 0000000..e1f7dcb --- /dev/null +++ b/keycloak-setup/keycloak-config.py @@ -0,0 +1,57 @@ +from keycloak import KeycloakAdmin, KeycloakOpenID, KeycloakOpenIDConnection + +# Keycloak Admin +keycloak_connection = KeycloakOpenIDConnection( + server_url="http://keycloak:8080/", + username="admin", + password="testing1", + realm_name="master", + verify=True, +) + +keycloak_admin = KeycloakAdmin(connection=keycloak_connection) + +# Changing Realm +keycloak_admin.create_realm( + payload={"realm": "test-realm", "enabled": True}, skip_exists=False +) +keycloak_connection.realm_name = "test-realm" + +# Create a client +client = keycloak_admin.create_client( + payload={ + "name": "test-client", + "clientId": "test-client", + "secret": "test-secret", + "redirectUris": ["http://localhost:4242/"], + "enabled": True, + "directAccessGrantsEnabled": True, + } +) + +# Configuring client in the new Realm +keycloak_openid = KeycloakOpenID( + server_url="http://keycloak:8080/", + client_id="test-client", + realm_name="test-realm", + client_secret_key="test-secret", +) + +# Get WellKnown +config_well_known = keycloak_openid.well_known() + +# Adding Users +test_user1 = keycloak_admin.create_user( + { + "username": "test-user1", + "enabled": True, + "credentials": [{"value": "secret1", "type": "password"}], + } +) +test_user2 = keycloak_admin.create_user( + { + "username": "test-user2", + "enabled": True, + "credentials": [{"value": "secret2", "type": "password"}], + } +) diff --git a/keycloak-setup/pw_file b/keycloak-setup/pw_file new file mode 100644 index 0000000..8d1c8b6 --- /dev/null +++ b/keycloak-setup/pw_file @@ -0,0 +1 @@ + diff --git a/robot/Dockerfile b/robot/Dockerfile new file mode 100644 index 0000000..ebd525a --- /dev/null +++ b/robot/Dockerfile @@ -0,0 +1,60 @@ +FROM centos:centos7 + +USER root + +RUN \ +yum -y update && \ +yum -y install epel-release redhat-lsb-core yum-utils && \ +yum -y install initscripts java-11-openjdk && \ +yum -y install perl-IPC-Cmd perl-Test-Simple && \ +yum -y group install "Development Tools" && \ +yum -y install tar jq-1.6 + +WORKDIR /usr/src +ADD https://www.openssl.org/source/openssl-3.0.10.tar.gz . +RUN \ +tar -zxf openssl-3.0.10.tar.gz && \ +rm openssl-3.0.10.tar.gz + +WORKDIR /usr/src/openssl-3.0.10 +RUN \ +./config && \ +make && \ +make install && \ +ln -s /usr/local/lib64/libssl.so.3 /usr/lib64/libssl.so.3 && \ +ln -s /usr/local/lib64/libcrypto.so.3 /usr/lib64/libcrypto.so.3 && \ +ln -s /usr/local/bin/openssl /usr/bin/openssl + +WORKDIR /tmp +ADD https://www.python.org/ftp/python/3.10.12/Python-3.10.12.tgz . +RUN \ +tar -xzf Python-3.10.12.tgz && \ +rm Python-3.10.12.tgz && \ +yum -y install libffi libffi-devel zlib-devel + +WORKDIR /tmp/Python-3.10.12/ +RUN \ +./configure --enable-optimizations && \ +make clean && \ +make install && \ +echo "export PATH=\"\${PATH}:/usr/local/lib/python3.10/site-packages\"" >> ~/.bashrc && \ +pip3 install --upgrade pip && \ +pip3 install pydantic && \ +pip3 install regex && \ +pip3 install httpx && \ +pip3 install liboidcagent && \ +pip3 install logging && \ +pip3 install uvicorn && \ +pip3 install requests && \ +pip3 install flaat && \ +pip3 install fastapi && \ +pip3 install robotframework && \ +pip3 install robotframework-requests && \ +pip3 install python-keycloak + +WORKDIR /tmp +ADD https://repo.data.kit.edu//data-kit-edu-centos7.repo /etc/yum.repos.d/data-kit-edu-centos7.repo +RUN \ +yum -y install oidc-agent && \ +yum -y update && \ +yum clean all diff --git a/robot/User-sub_generator.sh b/robot/User-sub_generator.sh new file mode 100755 index 0000000..8d50842 --- /dev/null +++ b/robot/User-sub_generator.sh @@ -0,0 +1,11 @@ +user1=test-user1 +curl -s -H "Authorization: Bearer $(oidc-token test-user1)" http://keycloak:8080/realms/test-realm/protocol/openid-connect/userinfo | jq . | grep sub | while read column sub; do \ + sub1=`sed -e 's/^"//' -e 's/",$//' <<< $sub` \ + && echo -e "$user1 $sub1" >> /etc/teapot/user-mapping.csv\ + ;done + +user2=test-user2 +curl -s -H "Authorization: Bearer $(oidc-token test-user2)" http://keycloak:8080/realms/test-realm/protocol/openid-connect/userinfo | jq . | grep sub | while read column sub; do \ + sub2=`sed -e 's/^"//' -e 's/",$//' <<< $sub` \ + && echo -e "$user2 $sub2" >> /etc/teapot/user-mapping.csv\ + ;done \ No newline at end of file diff --git a/robot/storage-areas b/robot/storage-areas new file mode 100644 index 0000000..6a6d2ee --- /dev/null +++ b/robot/storage-areas @@ -0,0 +1,2 @@ +default $HOME/interTwin +extra $HOME/interTwin_extra diff --git a/robot/storm-webdav-tests.robot b/robot/storm-webdav-tests.robot index 9fca27a..b629866 100644 --- a/robot/storm-webdav-tests.robot +++ b/robot/storm-webdav-tests.robot @@ -1,6 +1,6 @@ *** Settings *** Library RequestsLibrary -Variables /home/dijana/teapot/robot/variables.py +Variables /__w/teapot/teapot/robot/variables.py *** Test Cases *** diff --git a/robot/teapot-tests.robot b/robot/teapot-tests.robot new file mode 100644 index 0000000..bb09d62 --- /dev/null +++ b/robot/teapot-tests.robot @@ -0,0 +1,46 @@ +*** Settings *** +Library RequestsLibrary +Variables /__w/teapot/teapot/robot/variables.py + + +*** Test Cases *** + +Authentification + [Tags] authentication + ${RESPONSE}= GET ${MAIN_URL} headers=${HEADER1} verify=${false} expected_status=200 + +WRONG TOKEN + ${RESPONSE}= GET ${MAIN_URL} headers=${HEADER2} verify=${false} expected_status=403 + +NO TOKEN + ${RESPONSE}= GET ${MAIN_URL} verify=${false} expected_status=401 + +INVALID TOKEN + ${RESPONSE}= GET ${MAIN_URL} headers=${HEADER4} verify=${false} expected_status=500 + +PUT REQUEST INVALID TOKEN + ${RESPONSE}= PUT ${MAIN_URL}/TestFile1 data=${DATA} headers=${HEADER4} verify=${false} expected_status=500 + +PUT REQUEST WRONG TOKEN + ${RESPONSE}= PUT ${MAIN_URL}/TestFile1 data=${DATA} headers=${HEADER2} verify=${false} expected_status=403 + +PUT REQUEST NO TOKEN + ${RESPONSE}= PUT ${MAIN_URL}/TestFile1 data=${DATA} verify=${false} expected_status=401 + +PUT REQUEST DEFAULT SA + ${RESPONSE}= PUT ${MAIN_URL}/TestFile1 data=${DATA} headers=${HEADER1} verify=${false} expected_status=201 + + +DELETE REQUEST + ${RESPONSE}= DELETE ${MAIN_URL}/TestFile1 headers=${HEADER1} verify=${false} expected_status=204 + +DELETE REQUEST INVALID TOKEN + ${RESPONSE}= DELETE ${MAIN_URL}/TestFile1 headers=${HEADER4} verify=${false} expected_status=500 + +DELETE REQUEST WRONG TOKEN + ${RESPONSE}= DELETE ${MAIN_URL}/TestFile1 headers=${HEADER2} verify=${false} expected_status=403 + +DELETE REQUEST NO TOKEN + ${RESPONSE}= DELETE ${MAIN_URL}/TestFile1 verify=${false} expected_status=401 + + diff --git a/robot/test_file b/robot/test_file new file mode 100644 index 0000000..ec1b959 Binary files /dev/null and b/robot/test_file differ diff --git a/robot/variables.py b/robot/variables.py index 5f1a03d..12f647b 100644 --- a/robot/variables.py +++ b/robot/variables.py @@ -1,15 +1,13 @@ import subprocess -PORT= '8085' -PORT_MAIN='8081' -HOST='131.169.234.115' -STORM_URL="https://"+HOST+":"+PORT -MAIN_URL="https://"+HOST+":"+PORT_MAIN -TOKEN1=subprocess.check_output("oidc-token test-kc1", shell=True) -TOKEN2=subprocess.check_output("oidc-token test-kc2", shell=True) -TOKEN3=subprocess.check_output("oidc-token test-kc3", shell=True) -HEADER1={"Authorization": ("Bearer "+str(TOKEN1, 'ascii')).strip("\n")} -HEADER2={"Authorization": ("Bearer "+str(TOKEN2, 'ascii')).strip("\n")} -HEADER3={"Authorization": ("Bearer "+str(TOKEN3, 'ascii')).strip("\n")} -HEADER4={"Authorization": "Bearer NOT_A_TOKEN"} -DATA='/home/dijana/TestFile1' +PORT = "8085" +PORT_MAIN = "8081" +HOST = str(subprocess.check_output("curl ifconfig.me", shell=True), encoding="utf-8") +STORM_URL = "https://" + HOST + ":" + PORT +MAIN_URL = "https://" + HOST + ":" + PORT_MAIN +TOKEN1 = subprocess.check_output("oidc-token test-user1", shell=True) +TOKEN2 = subprocess.check_output("oidc-token test-user2", shell=True) +HEADER1 = {"Authorization": ("Bearer " + str(TOKEN1, "ascii")).strip("\n")} +HEADER2 = {"Authorization": ("Bearer " + str(TOKEN2, "ascii")).strip("\n")} +HEADER4 = {"Authorization": "Bearer NOT_A_TOKEN"} +DATA = "/__w/teapot/teapot/robot/test_file" diff --git a/rpm/teapot-rpm.sh b/rpm/teapot-rpm.sh index 99f70fa..bba0a31 100644 --- a/rpm/teapot-rpm.sh +++ b/rpm/teapot-rpm.sh @@ -28,5 +28,5 @@ mv storm-webdav-server.tar.gz $HOME/rpmbuild/SOURCES/ #building the RPM package rpmbuild -ba ~/rpmbuild/SPECS/teapot.spec -cp $HOME/rpmbuild/RPMS/noarch/teapot-v*-1.el7.noarch.rpm $HOME/ -rm -r $HOME/rpmbuild +# cp $HOME/rpmbuild/RPMS/noarch/teapot-v*-1.el7.noarch.rpm $HOME/ +# rm -r $HOME/rpmbuild diff --git a/run-teapot.sh b/run-teapot.sh index a316c01..55c8260 100644 --- a/run-teapot.sh +++ b/run-teapot.sh @@ -1,12 +1,12 @@ #!/bin/bash -path_to_pid='/home/dijana/teapot.pid' -if [ -a $path_to_pid ]; then - stormpid=`cat $path_to_pid` - kill $stormpid - rm $path_to_pid -else - python3 /usr/share/teapot/teapot.py &> tmp-storm-proxy.log & - echo $! > $path_to_pid -fi +#path_to_pid='$HOME/teapot.pid' +#if [ -a $path_to_pid ]; then +# stormpid=`cat $path_to_pid` +# kill $stormpid +# rm $path_to_pid +#else +python3 /usr/share/teapot/teapot.py &> tmp-storm-proxy.log & +# echo $! > $path_to_pid +#fi diff --git a/teapot.py b/teapot.py index 19a1350..e480bc0 100644 --- a/teapot.py +++ b/teapot.py @@ -25,23 +25,25 @@ flaat = Flaat() security = HTTPBearer() -flaat.set_access_levels( - [ - AccessLevel("user", HasSubIss()) - ]) +HOST = str(subprocess.check_output("curl ifconfig.me", shell=True), encoding="utf-8") + +flaat.set_access_levels([AccessLevel("user", HasSubIss())]) flaat.set_trusted_OP_list( - [ + [ "https://aai-demo.egi.eu/auth/realms/egi", - "https://keycloak.ci-cd-prep2.desy.de/realms/Testing" + "http://keycloak:8080/realms/test-realm", ] ) # logging is important -LOGLEVEL = os.environ.get('STORM_LOGLEVEL', 'INFO').upper() -logging.basicConfig(filename='/var/lib/teapot/webdav/teapot.log', level=logging.getLevelName(LOGLEVEL)) +LOGLEVEL = os.environ.get("STORM_LOGLEVEL", "INFO").upper() +logging.basicConfig( + filename="/var/lib/teapot/webdav/teapot.log", level=logging.getLevelName(LOGLEVEL) +) logger = logging.getLogger(__name__) + async def _map_fed_to_local(sub): # this func returns the local username for a federated user or None # for this prototype it could just be read from a mapping file on the local file system. # noqa: E501 @@ -52,8 +54,8 @@ async def _map_fed_to_local(sub): # without headers and only the first hit for a federated sub claim is returned. # like this, it is possible to match different subs to a local username but not the other way around. # noqa: E501 - with open('/etc/teapot/user-mapping.csv', 'r') as mapping_file: - mappingreader = csv.reader(mapping_file, delimiter=' ') + with open("/etc/teapot/user-mapping.csv", "r") as mapping_file: + mappingreader = csv.reader(mapping_file, delimiter=" ") for row in mappingreader: logger.info(f"from mapping file: {row}") if row[1] == sub: @@ -61,30 +63,41 @@ async def _map_fed_to_local(sub): return row[0] return None + async def _return_or_create_storm_instance(sub): # get the mapping for the federated user from the sub-claim local_user = await _map_fed_to_local(sub) if not local_user: # local user is unknown, we cannot start or check anything. return None, None - #now check if an instance is running by possibly checking `ps faux | grep local-username # noqa: E501 + # now check if an instance is running by possibly checking `ps faux | grep local-username # noqa: E501 # currently just mocking the return values - if (exists("/var/lib/teapot/user-"+local_user+"/server.pid")): - with open("/var/lib/teapot/user-"+local_user+"/server.pid",'r') as pid_file: - pid=pid_file.read() - with open("/var/lib/teapot/user-"+local_user+"/server.port","r") as port_file: - port=port_file.read().rstrip() - logger.info(f"StoRM-WebDAV instance for {local_user} is running on port {port}.") # noqa: E501 + if exists("/var/lib/teapot/user-" + local_user + "/server.pid"): + with open( + "/var/lib/teapot/user-" + local_user + "/server.pid", "r" + ) as pid_file: + pid = pid_file.read() + with open( + "/var/lib/teapot/user-" + local_user + "/server.port", "r" + ) as port_file: + port = port_file.read().rstrip() + logger.info( + f"StoRM-WebDAV instance for {local_user} is running on port {port}." + ) # noqa: E501 else: - port=8085 - subprocess.run(["sudo", "/usr/share/teapot/webdav-automation.sh", local_user, str(port)]) # noqa: E501 - with open("/var/lib/teapot/user-"+local_user+"/server.pid",'r') as pid_file: - pid=pid_file.read() # noqa: F841 + port = 8085 + subprocess.run( + ["sudo", "/usr/share/teapot/webdav-automation.sh", local_user, str(port)] + ) # noqa: E501 + with open( + "/var/lib/teapot/user-" + local_user + "/server.pid", "r" + ) as pid_file: + pid = pid_file.read() # noqa: F841 running = False while not running: time.sleep(1) try: - resp=httpx.get(f'https://localhost:{port}/') + resp = httpx.get(f"https://{HOST}:{port}/") if resp.status_code == 403 or resp.status_code == 200: running = True except: @@ -93,18 +106,37 @@ async def _return_or_create_storm_instance(sub): logger.info(f"Storm-WebDAV instance for {local_user} started on port {port}.") return None, port -@app.api_route("/{filepath:path}", methods=["HEAD", "GET", "PUT", "POST", "DELETE", "PROPFIND", "MKCOL", "COPY", "MOVE"]) + +@app.api_route( + "/{filepath:path}", + methods=[ + "HEAD", + "GET", + "PUT", + "POST", + "DELETE", + "PROPFIND", + "MKCOL", + "COPY", + "MOVE", + ], +) @flaat.is_authenticated() -async def root(filepath: str, request: Request, response: Response, credentials: HTTPBasicCredentials = Depends(security)): # noqa: E501 +async def root( + filepath: str, + request: Request, + response: Response, + credentials: HTTPBasicCredentials = Depends(security), +): # noqa: E501 # get data from userinfo endpoint user_infos = flaat.get_user_infos_from_request(request) if not user_infos: return 403 logger.info(f"user_info is: {user_infos['sub']}") - sub = user_infos.get('sub', None) + sub = user_infos.get("sub", None) if not sub: - #if there is no sub, user can not be authenticated + # if there is no sub, user can not be authenticated return 403 # user is valid, so check if a storm instance is running for this sub redirect_host, redirect_port = await _return_or_create_storm_instance(sub) @@ -116,7 +148,7 @@ async def root(filepath: str, request: Request, response: Response, credentials: # no port returned, should not happen return 500 if not redirect_host: - redirect_host = "localhost" + redirect_host = HOST logger.info(f"redirect_host: {redirect_host}, redirect_port: {redirect_port}") logger.info(f"request path: {request.url.path}") @@ -124,14 +156,16 @@ async def root(filepath: str, request: Request, response: Response, credentials: storage_prefix = "default_area" redirect_path = f"{storage_prefix}{request.url.path}" redirect_url = f"https://{redirect_host}:{redirect_port}/{redirect_path}" - #redirect_url = httpx.URL(scheme=request.url.scheme, host=redirect_host, port=redirect_port, path=redirect_path) # noqa: E501 + # redirect_url = httpx.URL(scheme=request.url.scheme, host=redirect_host, port=redirect_port, path=redirect_path) # noqa: E501 logger.info(f"redirect_url is formed as {redirect_url}.") async with httpx.AsyncClient(verify=False) as client: - forward_req = client.build_request(request.method, redirect_url, - headers=request.headers.raw, - content=request.stream() - ) + forward_req = client.build_request( + request.method, + redirect_url, + headers=request.headers.raw, + content=request.stream(), + ) forward_resp = await client.send(forward_req, stream=True) return StreamingResponse( forward_resp.aiter_raw(), @@ -143,9 +177,7 @@ async def root(filepath: str, request: Request, response: Response, credentials: # if yes, forward the request to this instance; if no, create an instance and then forward the request. forwarding happens via _reverse_proxy() # noqa: E501 -if __name__ == '__main__': - key="/var/lib/teapot/webdav/localhost.key" - cert="/var/lib/teapot/webdav/localhost.crt" - uvicorn.run(app, host="0.0.0.0", port=8081, - ssl_keyfile=key, ssl_certfile=cert) - +if __name__ == "__main__": + key = "/var/lib/teapot/webdav/localhost.key" + cert = "/var/lib/teapot/webdav/localhost.crt" + uvicorn.run(app, host="0.0.0.0", port=8081, ssl_keyfile=key, ssl_certfile=cert) diff --git a/templates/issuers.yml b/templates/issuers.yml index 00ba520..4c60ecf 100644 --- a/templates/issuers.yml +++ b/templates/issuers.yml @@ -1,5 +1,5 @@ oauth: issuers: - name: production - issuer: https://keycloak.ci-cd-prep2.desy.de/realms/Testing + issuer: http://keycloak:8080/realms/test-realm diff --git a/templates/storage_authorizations.yml b/templates/storage_authorizations.yml index e9c1268..2c5377f 100644 --- a/templates/storage_authorizations.yml +++ b/templates/storage_authorizations.yml @@ -8,7 +8,7 @@ principals: - type: jwt-subject params: - iss: https://keycloak.ci-cd-prep2.desy.de/realms/Testing + iss: http://keycloak:8080/realms/test-realm sub: $sub - sa: $storage_area actions: diff --git a/templates/storage_element.properties b/templates/storage_element.properties index 0200bc9..1bb43d0 100644 --- a/templates/storage_element.properties +++ b/templates/storage_element.properties @@ -2,7 +2,7 @@ #vos=test.vo # Comma-separated list of OAuth/OpenID Connect token issuers trusted in this storage area -orgs=https://aai-demo.egi.eu/auth/realms/egi, https://keycloak.ci-cd-prep2.desy.de/realms/Testing +orgs=https://aai-demo.egi.eu/auth/realms/egi, http://keycloak:8080/realms/test-realm # Enables read access to users authenticated with an X.509 certificate issued by # a trusted CA (users without VOMS credentials). diff --git a/webdav-automation.sh b/webdav-automation.sh index cfea8b9..7a3f9a9 100644 --- a/webdav-automation.sh +++ b/webdav-automation.sh @@ -17,74 +17,77 @@ fi export USER="$1" export port="$2" -export name=teapot echo "StoRM-WebDAV instance for user $USER will be listening on the port $port" #creation of user-specific configuration directories -USER_DIR=/var/lib/$name/user-$USER -if [ ! -d $USER_DIR ]; then - mkdir -p $USER_DIR - chown $USER $USER_DIR +USER_DIR=/var/lib/teapot/user-"$USER" +if [ ! -d "$USER_DIR" ]; then + mkdir -p "$USER_DIR" + chown "$USER" "$USER_DIR" echo "$USER_DIR was created as a directory for user $USER" fi # creation of the log directory for a new user -if [ ! -d $USER_DIR/log ]; then - mkdir -p $USER_DIR/log - chown $USER $USER_DIR/log +if [ ! -d "$USER_DIR"/log ]; then + mkdir -p "$USER_DIR"/log + chown "$USER" "$USER_DIR"/log echo "$USER_DIR/log was created" fi #creation of storage-properties files and folders for a new user -CONFIG_DIR=/etc/$name -if [ ! -f $CONFIG_DIR/storage-areas ]; then +CONFIG_DIR=/etc/teapot +if [ ! -f "$CONFIG_DIR"/storage-areas ]; then echo "Error: $CONFIG_DIR/storage-areas file is missing." echo "It should consist of two variables per storage area: name of the storage area and root path to the storage area's directory separated by a single space." exit 1 fi -TEMPL_DIR=/usr/share/$name -if [ ! -d $USER_DIR/sa.d ]; then - mkdir -p $USER_DIR/sa.d - cat $CONFIG_DIR/storage-areas | while read storage_area path; do \ - echo -e "name=$storage_area\nrootPath=$(su $USER -c "echo $path")\naccessPoints=/${storage_area}_area\n\n" >> $USER_DIR/sa.d/$storage_area.properties \ +TEMPL_DIR=/usr/share/teapot +if [ ! -d "$USER_DIR"/sa.d ]; then + mkdir -p "$USER_DIR"/sa.d + cat "$CONFIG_DIR"/storage-areas | while read storage_area path; do \ + echo -e "name=$storage_area\nrootPath=$(su "$USER" -c "echo $path")\naccessPoints=/${storage_area}_area\n" >> "$USER_DIR"/sa.d/"$storage_area".properties \ && cat $TEMPL_DIR/storage_element.properties >> $USER_DIR/sa.d/$storage_area.properties \ - && chown $USER $USER_DIR/sa.d/$storage_area.properties \ - && chmod g+w $USER_DIR/sa.d/$storage_area.properties \ - && mkdir -p "$(su $USER -c "echo \"$path\"")" \ - && chown $USER "$(su $USER -c "echo \"$path\"")" \ + && chown "$USER" "$USER_DIR"/sa.d/"$storage_area".properties \ + && chmod g+w "$USER_DIR"/sa.d/"$storage_area".properties \ + && mkdir -p "$(su "$USER" -c "echo \"$path\"")" \ + && chown $USER "$(su "$USER" -c "echo \"$path\"")" \ ; done - chown $USER $USER_DIR/sa.d + chown "$USER" "$USER_DIR"/sa.d echo "Storage configuration was created at $USER_DIR/sa.d" fi #creation of the token authorization files -if [ ! -f $CONFIG_DIR/user-mapping.csv ]; then +if [ ! -f "$CONFIG_DIR"/user-mapping.csv ]; then echo "Error: $CONFIG_DIR/user-mapping.csv file is missing." echo "It should consist of two variables per user: username and subject claim separated by a single space." exit 1 fi -if [ ! -d $USER_DIR/config ]; then - mkdir -p $USER_DIR/config - grep "$USER" $CONFIG_DIR/user-mapping.csv | while read user_id sub; do \ - cat $CONFIG_DIR/issuers.yml >> $USER_DIR/config/application.yml \ - && echo -e "storm:\n authz:\n policies:" >> $USER_DIR/config/application.yml \ - && cat $CONFIG_DIR/storage-areas | while read storage_area path; do \ +if [ ! -d "$USER_DIR"/config ]; then + mkdir -p "$USER_DIR"/config + grep "$USER" "$CONFIG_DIR"/user-mapping.csv | while read user_id sub; do \ + cat "$CONFIG_DIR"/issuers.yml >> "$USER_DIR"/config/application.yml \ + && echo -e "storm:\n authz:\n policies:" >> "$USER_DIR"/config/application.yml \ + && cat "$CONFIG_DIR"/storage-areas | while read storage_area path; do \ export sub storage_area \ - && envsubst < $TEMPL_DIR/storage_authorizations.yml >> $USER_DIR/config/application.yml \ + && envsubst < "$TEMPL_DIR"/storage_authorizations.yml >> "$USER_DIR"/config/application.yml \ ; done; done - chown $USER $USER_DIR/config/application.yml - chown $USER $USER_DIR/config + chown "$USER" "$USER_DIR"/config/application.yml + chown "$USER" "$USER_DIR"/config echo "Authorization configuration was created for user $USER" fi #creation of the work directory $USER_DIR/tmp for a new user -if [ ! -d $USER_DIR/tmp ]; then - mkdir -p $USER_DIR/tmp - chown $USER $USER_DIR/tmp +if [ ! -d "$USER_DIR"/tmp ]; then + mkdir -p "$USER_DIR"/tmp + chown "$USER" "$USER_DIR"/tmp echo "$USER_DIR/tmp was created" fi -su -c /usr/share/$name/webdav-server-start.sh $USER +# vomaps +mkdir -p /etc/grid-security +mkdir -p /etc/grid-security/vomsdir + +su -c /usr/share/teapot/webdav-server-start.sh "$USER" diff --git a/webdav-server-start.sh b/webdav-server-start.sh index 070dbc7..6119d46 100644 --- a/webdav-server-start.sh +++ b/webdav-server-start.sh @@ -6,32 +6,34 @@ # This script will exit only once StoRM WebDAV is able to accept # requests. -USER_DIR=/var/lib/$name/user-$USER -STORM_DIR=/var/lib/$name/webdav +USER_DIR=/var/lib/teapot/user-$USER +STORM_DIR=/var/lib/teapot/webdav -echo "Starting StoRM WebDAV as user $USER" +HOST_IP="$(curl ifconfig.me)" +echo HOST_IP is "$HOST_IP" -export STORM_WEBDAV_JVM_OPTS="-Xms2048m -Xmx2048m -Djava.security.egd=file:/dev/./urandom" -export STORM_WEBDAV_SERVER_ADDRESS=127.0.0.1 -export STORM_WEBDAV_HTTPS_PORT=$port -export STORM_WEBDAV_HTTP_PORT=1$port # FIXME remove -export STORM_WEBDAV_CERTIFICATE_PATH=$STORM_DIR/localhost.crt -export STORM_WEBDAV_PRIVATE_KEY_PATH=$STORM_DIR/localhost.key +echo "Starting StoRM WebDAV as user $USER" +export STORM_WEBDAV_JVM_OPTS="-Xms2048M -Xmx2048M -Djava.security.egd=file:/dev/./urandom" +export STORM_WEBDAV_SERVER_ADDRESS="$HOST_IP" +export STORM_WEBDAV_HTTPS_PORT="$port" +export STORM_WEBDAV_HTTP_PORT="1$port" +export STORM_WEBDAV_CERTIFICATE_PATH="$STORM_DIR"/localhost.crt +export STORM_WEBDAV_PRIVATE_KEY_PATH="$STORM_DIR"/localhost.key export STORM_WEBDAV_TRUST_ANCHORS_DIR=/etc/ssl/certs export STORM_WEBDAV_TRUST_ANCHORS_REFRESH_INTERVAL=86400 export STORM_WEBDAV_MAX_CONNECTIONS=300 export STORM_WEBDAV_MAX_QUEUE_SIZE=900 export STORM_WEBDAV_CONNECTOR_MAX_IDLE_TIME=30000 -export STORM_WEBDAV_SA_CONFIG_DIR=$USER_DIR/sa.d +export STORM_WEBDAV_SA_CONFIG_DIR="$USER_DIR"/sa.d export STORM_WEBDAV_JAR=/usr/share/java/storm-webdav/storm-webdav-server.jar -export STORM_WEBDAV_LOG=$USER_DIR/log/server.log -export STORM_WEBDAV_OUT=$USER_DIR/log/server.out -export STORM_WEBDAV_ERR=$USER_DIR/log/server.err +export STORM_WEBDAV_LOG="$USER_DIR"/log/server.log +export STORM_WEBDAV_OUT="$USER_DIR"/log/server.out +export STORM_WEBDAV_ERR="$USER_DIR"/log/server.err -ETC_DIR=/etc/$name -export STORM_WEBDAV_LOG_CONFIGURATION=$ETC_DIR/logback.xml -export STORM_WEBDAV_ACCESS_LOG_CONFIGURATION=$ETC_DIR/logback-access.xml +ETC_DIR=/etc/teapot +export STORM_WEBDAV_LOG_CONFIGURATION="$ETC_DIR"/logback.xml +export STORM_WEBDAV_ACCESS_LOG_CONFIGURATION="$ETC_DIR"/logback-access.xml export STORM_WEBDAV_VO_MAP_FILES_ENABLE=false export STORM_WEBDAV_VO_MAP_FILES_REFRESH_INTERVAL=21600 export STORM_WEBDAV_TPC_MAX_CONNECTIONS=50 @@ -41,21 +43,21 @@ export STORM_WEBDAV_TPC_USE_CONSCRYPT=true #strace -e trace=file -o /tmp/storm-webdav \ /usr/bin/java ${STORM_WEBDAV_JVM_OPTS} \ - -Djava.io.tmpdir=$USER_DIR/tmp \ + -Djava.io.tmpdir="$USER_DIR"/tmp \ -Dlogging.config=${STORM_WEBDAV_LOG_CONFIGURATION} \ -jar ${STORM_WEBDAV_JAR} >${STORM_WEBDAV_OUT} 2>${STORM_WEBDAV_ERR} \ - --spring.config.additional-location=optional:file:/var/lib/$name/user-$USER/config/application.yml& + --spring.config.additional-location=optional:file:/var/lib/teapot/user-"$USER"/config/application.yml& pid=$! -echo "$pid" > $USER_DIR/server.pid -echo "$port" > $USER_DIR/server.port +echo "$pid" > "$USER_DIR"/server.pid +echo "$port" > "$USER_DIR"/server.port -nc -zvw 1 localhost $port &> /dev/null +nc -zvw 1 $STORM_WEBDAV_SERVER_ADDRESS "$port" &> /dev/null status=$? while [ ! $status -eq 0 ]; do echo -n . sleep 0.5s - nc -zvw 1 localhost $port &> /dev/null + nc -zvw 1 $STORM_WEBDAV_SERVER_ADDRESS "$port" &> /dev/null status=$? done diff --git a/webdav-server-stop.sh b/webdav-server-stop.sh index b1e5f1e..76b3791 100644 --- a/webdav-server-stop.sh +++ b/webdav-server-stop.sh @@ -9,11 +9,11 @@ fi USER="$1" -USER_DIR=/var/lib/teapot/user-$USER +USER_DIR=/var/lib/teapot/user-"$USER" -pid=$(cat $USER_DIR/server.pid) +pid=$(cat "$USER_DIR"/server.pid) -kill -15 $pid +kill -15 "$pid" -rm $USER_DIR/server.pid -rm $USER_DIR/server.port +rm "$USER_DIR"/server.pid +rm "$USER_DIR"/server.port