version 0.8.0

.gitignore

@@ -106,23 +106,19 @@ venv.bak/
 # mypy
 .mypy_cache/
 __pycache__/*
+thethe_venv/*
 notes.txt
 tags
 TODO
 sources
 Pipfile
 Pipfile.lock
-node_modules/
-
-# thethe
-external/*
-!external/.gitkeep
-!external/phishtank/.gitkeep
+frontend/public/static/phishtank/*
+frontend/public/static/urlscan/*
+!frontend/public/static/phishtank/.gitkeep
+!frontend/public/static/urlscan/.gitkeep
 utils/db.dump
 utils/db.dump.gz
 db.dump.gz
 mongodb_data/*
-!mongodb_data/.gitkeep
-thethe_server/*
-thethe_frontend/*
-utils/db.dump.gz.old_with_dot_in_keys
+node_modules/

docker-compose.yml

@@ -21,6 +21,7 @@ services:
     command: /usr/local/bin/celery -A tasks.tasks:celery_app worker --concurrency 1 -l info
     volumes:
       - ./external/phishtank:/temp/phishtank
+      - ./external/urlscan:/temp/urlscan
     depends_on:
       - redis
 
@@ -31,6 +32,7 @@ services:
     volumes:
       - ./external/phishtank:/usr/share/nginx/html/static/phishtank
       - ./external/certs:/etc/nginx/certificates
+      - ./external/urlscan:/usr/share/nginx/html/static/urlscan
     ports:
       - 80:80
       - 443:443
@@ -47,6 +49,7 @@ services:
       /usr/local/bin/gunicorn server.main:app -w 4 -t 90 --log-level=info -b 0.0.0.0:8000"
     volumes:
       - ./external/phishtank:/temp/phishtank
+      - ./external/urlscan:/temp/urlscan
     depends_on:
       - mongo
       - celery

docker-compose_dev.yml

@@ -4,7 +4,6 @@ services:
   mongo:
     image: mongo
     container_name: thethe_mongo
-    restart: always
     ports:
       - 27017:27017
     env_file: .env
@@ -14,7 +13,6 @@ services:
   redis:
     image: redis
     container_name: thethe_redis
-    restart: always
     ports:
       - 6379:6379
 
@@ -28,6 +26,7 @@ services:
       - ./thethe_server/server:/usr/src/thethe/server
       - ./thethe_server/tasks:/usr/src/thethe/tasks
       - ./external/phishtank:/temp/phishtank
+      - ./external/urlscan:/temp/urlscan
     depends_on:
       - redis
 
@@ -42,6 +41,7 @@ services:
       - ./thethe_server/server:/usr/src/thethe/server
       - ./thethe_server/tasks:/usr/src/thethe/tasks
       - ./external/phishtank:/temp/phishtank
+      - ./external/urlscan:/temp/urlscan
     ports:
       - 8000:8000
     depends_on:

external/certs/thethe.crt

@@ -0,0 +1,18 @@
+-----BEGIN CERTIFICATE-----
+MIIC9jCCAd4CCQDrFsXimR3SMjANBgkqhkiG9w0BAQUFADA9MQswCQYDVQQGEwJO
+TzEPMA0GA1UECAwGRnJhbmNlMQ4wDAYDVQQHDAVQYXJpczENMAsGA1UECgwETFVM
+VTAeFw0yMDAyMjUyMDA0MDRaFw0yMTAyMjQyMDA0MDRaMD0xCzAJBgNVBAYTAk5P
+MQ8wDQYDVQQIDAZGcmFuY2UxDjAMBgNVBAcMBVBhcmlzMQ0wCwYDVQQKDARMVUxV
+MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAySBADUDCKl3Lucl1q1bt
+lpN0TWBdSGgTjchQj8VUbPB+Ft3NsuMrArv+sZuWoY9ggKkXqsFzR1TliBAz1kZX
+sdLTft5fLpLVfmSQZTL8meIfdP5LQ3R7euXt1XANNM74TKcksKw+XEGivWk0IuYt
+VNb/XHm/aM8rag4F713Hi3RnBcpiQ+eHW5paM8IOSd7rXLca+t85sWUI9yqAAo6N
+FTlFBd71Y1zNt1eEAxkV3SgkfeU+d6ynPGVz1NkTASEtYyo+jj1tJtQ5ofPfPXGL
+8HBwfoGQNoAArYQHympPHYixIEtFRFzPr/hBp9O3lNQmx5ZKBtdi1oAhsx8RWUZT
+KQIDAQABMA0GCSqGSIb3DQEBBQUAA4IBAQAtZRWNsd3POadbDr8ivuCaAed2DWvg
+3KgI1yNxZAmVbDqoSPefV8LF1beE/tCiXMIvu5cLXiseab5s2sctByh+6XMNSL9t
+pujrqUC5KEWgByt8orlwnYPpW0lzvDHt+XLTCRE8r8BSvOmnWmpfhswuIUOaicvP
+izwWchqOUlNYTxK1yh9pXjqWBubafO88YM6ItPT3KiDSxQrZAxekLUMiZLjiL3xm
+SZtS+Hn8s12WPXEyaZIP80RWiOpNRaH2agEjnnvYubdURJMsLGcIbzmQaCTk/mQp
+xTjhR5SHmTt4xj2txVk/TL08iv2UyUAQqFel1PaMHpavTYbcMwv9MZ0u
+-----END CERTIFICATE-----

external/certs/thethe.key

@@ -0,0 +1,28 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQDJIEANQMIqXcu5
+yXWrVu2Wk3RNYF1IaBONyFCPxVRs8H4W3c2y4ysCu/6xm5ahj2CAqReqwXNHVOWI
+EDPWRlex0tN+3l8uktV+ZJBlMvyZ4h90/ktDdHt65e3VcA00zvhMpySwrD5cQaK9
+aTQi5i1U1v9ceb9ozytqDgXvXceLdGcFymJD54dbmlozwg5J3utctxr63zmxZQj3
+KoACjo0VOUUF3vVjXM23V4QDGRXdKCR95T53rKc8ZXPU2RMBIS1jKj6OPW0m1Dmh
+8989cYvwcHB+gZA2gACthAfKak8diLEgS0VEXM+v+EGn07eU1CbHlkoG12LWgCGz
+HxFZRlMpAgMBAAECggEAMWDjmN1fTjt8+v4FD63l4uQM17pi7JXwxwx5CvMtiRl6
+O271Q4gWVTOGkmBamQu0a6IYF2orPUYQnFkuANuASD40gp9hyvNtyvwxn0bnUXh8
+tKKCI+uDLWcp/w7ljyOJPniCv4VqQggzRkF/9VmBo6StSSzy226RgBY1StFBXnEj
+nhf/n11e9hTX4y3ztNXBqX2YXwNGQTcccAi34lCEV60ArPqcLv2dy46GQXAiCkWz
+MX/10JWXJzum/aTJQXFREEOOkjKVb3HrHQoa2i322RCQhVZ01qzSieI3qDQSRZ/1
+gqM290Fs8Gke+s8NVnV81TajNq1aiccI1JAUbLsZaQKBgQD7oBEvoOEIyBlJ8jbS
+1hnxqxYuTajd0SPIXDXqNVdJe65ai8VvyccLEmTxuYt6uak8SPA6bYE/Y+pEYFb9
+yH/QH6XQoCxvfYbPX9qEh+P60IHxTYqyvjWIcjRDza/kmeBVDb3rKxjlfFYyRFwW
+I9iy6JD8ZaJBzI94UA7pGMvqOwKBgQDMn2vIJFypGQc4qxjCCuVcem5U0Sz7vKhD
+MkcZLJO5t16NRgUjT6sBFx3BajIAayEDA0xHppRKt1/hH3xnidw5oYG2GpjvoLiU
+T3HJYMz4RaYcUzAsBJhZIZ64hmPjHuieul1XIcFZpDb9+8tA/+i4qDOYlD9TAvpj
+pEgdWy796wKBgQCsdraw1WzILxYeN9TZx4N6v8/xJqtidbE35/PQNoVASXRkfT5y
+83LopM/pWko+cLiNKFT3Rwz6XMo52WpmgaERDlABYXdzZy8aPdGxWs6on3cg5Av3
+nSVmAxjNjFJNtiYrRcLxR9XFnOYbh/0x85nYbkzJogOHY+TGCuFdZPCG9wKBgQCp
+acRe7FwbVLF55Y6V+Vd0fZkjeBv6QHQ/Tj03RoT4Wi1aYgKW/brjKDRmknnbhtbK
+G8zn9a8JPVGQnfTt9GH1BRbToqnEnBi2EiBvTknltCSXhvh3OUa5ZPB4biv9OvX1
+lO762OOwSBmPpwzdQAN5/yjYxsS3Adw06ysZwwjhCQKBgQDczaHddj1WkHYV595S
+Jo6iGDHuUQIslswvqdQ3vQMyTa0cUv2Vf4/T+4/nd55P+2gFEcxkQALEsC1DOeyf
++1StsxQgfR/yltyYCq9MjjSjNIFzT+z9k8xs5+3rdAdKj6VwM0AxysFem/og7UcR
+0qt068bNs9U2E5L6QNF4ak0KKw==
+-----END PRIVATE KEY-----

install.sh

@@ -54,6 +54,7 @@ function check_dependencies() {
     # docker
     if command -v docker >/dev/null 2>&1; then
         echo -e "${Green}[+] Docker is installed. Good.${Color_Off}"
+        echo -e "${Yellow}[!] Make sure your user is in docker group${Color_Off}"
     else
         echo -e "${Red}[!] Missing docker installation. Bad.${Color_Off}"
         echo "See there how to get docker: https://docs.docker.com/install/"
@@ -78,29 +79,42 @@ function check_dependencies() {
     fi
 }
 
-# If this is a git repo do nothing
-if git rev-parse >/dev/null 2>&1; then
-    echo -e "${Red}[!] Aborting. This is a git repository. Did you mean ${BRed}update.sh?${Color_Off}"
-    echo -e "${BBlue}[!] This script is meant to be ran outside thethe repository.${Color_Off}"
-    exit 1
-fi
+function check_env_file() {
+    if [[ -f ".env" ]]; then
+        echo -e "${Green}[+] .env file exists. Good.${Color_Off}"
+    else
+        echo -e "${Green}[!] .env file not detected. Creating .env file${Color_Off}"
+        touch .env
+        echo -e "${Blue}[?] Enter a username for MongoDB root: ${Color_Off}"
+        read mongo_username
+        printf '%s%s\n' "MONGO_INITDB_ROOT_USERNAME=" $mongo_username >>.env
+        echo -e "${Blue}[?] Enter a password for MongoDB root: ${Color_Off}"
+        read -s mongo_password
+        printf '%s%s\n' "MONGO_INITDB_ROOT_PASSWORD=" $mongo_password >>.env
+        printf '%s\n' "MONGO_INITDB_DATABASE=thethe" >>.env
+        echo -e "${Blue}[?] Setting up thethe secret: ${Color_Off}"
+        printf '%s%s\n' "THETHE_SECRET=" $(openssl rand -hex 32) >>.env
+    fi
+
+}
 
 echo -e "${Green}[+] Installing thethe.${Color_Off}"
 
 check_dependencies
 echo -e "${BGreen}[*] Dependencies checked.${Color_Off}"
 
-echo -e "${Green}[+] Cloning repository thethe from https://github.com/ElevenPaths/thethe...${Color_Off}"
-git clone --recurse-submodules https://github.com/ElevenPaths/thethe >/dev/null 2>&1
-
-echo -e "${Green}[+] Entering repository${Color_Off}"
-cd thethe
+echo -e "${Green}[+] Starting the environment${Color_Off}"
 mkdir -p external/phishtank
+mkdir -p external/urlscan
 
+echo -e "${Green}[+] Checking for certificates${Color_Off}"
 source certs.sh
 
+echo -e "${Green}[+] Checking for .env file${Color_Off}"
+check_env_file
+
 echo -e "${Green}[+] Building docker images. It will take a while.${Color_Off}"
 docker-compose build
 
 echo -e "${BGreen}[*] Installation completed.${Color_Off}"
-echo -e "${Blue}Make sure you have a proper .env file with your desired values. Then, run start.sh (inside thethe directory).${Color_Off}"
+echo -e "${Blue}Run start.sh (inside thethe directory) if you want to launch thethe.${Color_Off}"