From 2194304a15858644a4d1998623ce2f586334ab6f Mon Sep 17 00:00:00 2001 From: Jonas Baumann Date: Fri, 6 Sep 2024 23:09:44 +0200 Subject: [PATCH 1/4] cleanup: remove unused db folder. We are currently using the official elasticsearch image. --- app.code-workspace | 5 ----- db/Dockerfile | 0 2 files changed, 5 deletions(-) delete mode 100644 db/Dockerfile diff --git a/app.code-workspace b/app.code-workspace index 0e42345..d489b60 100644 --- a/app.code-workspace +++ b/app.code-workspace @@ -4,10 +4,6 @@ "name": "api", "path": "api" }, - { - "name": "db", - "path": "db" - }, { "name": "frontend", "path": "frontend" @@ -24,7 +20,6 @@ "settings": { "files.exclude": { "api": true, - "db": true, "frontend": true, "import": true, } diff --git a/db/Dockerfile b/db/Dockerfile deleted file mode 100644 index e69de29..0000000 From 2e82f9f2ecd0190959eea125ecef059193270b4c Mon Sep 17 00:00:00 2001 From: Jonas Baumann Date: Sat, 7 Sep 2024 09:17:54 +0200 Subject: [PATCH 2/4] api: linting and cleanup --- api/app/elastic.py | 9 ++++++--- api/app/importer.py | 18 ++++++++---------- api/app/record.py | 21 +++++++++------------ api/app/server.py | 4 ++-- 4 files changed, 25 insertions(+), 27 deletions(-) diff --git a/api/app/elastic.py b/api/app/elastic.py index 672254f..9e70f99 100644 --- a/api/app/elastic.py +++ b/api/app/elastic.py @@ -58,17 +58,20 @@ def create_index(self) -> None: } }, ) + def import_records(self, records: List[Record]) -> str: self.create_index() success, failed = 0, 0 for ok, action in streaming_bulk( - client=self.connection, index=self.index_name, actions=self.generate_actions(records), + client=self.connection, + index=self.index_name, + actions=self.generate_actions(records), ): if ok: success += 1 else: failed += 1 - return(f"Finished: {success} documents indexed, {failed} failed.") + return f"Finished: {success} documents indexed, {failed} failed." def generate_actions(self, records: List[Record]): """This function is passed into the bulk() @@ -76,4 +79,4 @@ def generate_actions(self, records: List[Record]): """ for record in records: doc = record.record_to_dict() - yield doc \ No newline at end of file + yield doc diff --git a/api/app/importer.py b/api/app/importer.py index cd3c00b..46c041e 100644 --- a/api/app/importer.py +++ b/api/app/importer.py @@ -1,8 +1,7 @@ from csv import DictReader +from typing import List from app.record import Record -from typing import List -import csv class Importer: @@ -10,22 +9,21 @@ def read_csv_to_records(self, csv_reader: DictReader) -> List[Record]: records = [] for row in csv_reader: try: - lat_str, lon_str = row['coordinates'].split(',') - except ValueError as e: + lat_str, lon_str = row["coordinates"].split(",") + except ValueError: # print(str(e)) continue lat = float(lat_str.strip()) lon = float(lon_str.strip()) # Create a Record instance from each row record = Record( - id=row['id'], - title=row['title'], - image_url=row['image_url'], + id=row["id"], + title=row["title"], + image_url=row["image_url"], lat=lat, lon=lon, - iiif_url=row['iiif_url'], - source_system_url=row['source_system_url'] + iiif_url=row["iiif_url"], + source_system_url=row["source_system_url"], ) records.append(record) return records - diff --git a/api/app/record.py b/api/app/record.py index 7d4a2f2..f4ce168 100644 --- a/api/app/record.py +++ b/api/app/record.py @@ -1,13 +1,13 @@ class Record: def __init__( - self, - id: str, - title: str, - image_url: str, - lat: float, - lon: float, - iiif_url: str, - source_system_url: str, + self, + id: str, + title: str, + image_url: str, + lat: float, + lon: float, + iiif_url: str, + source_system_url: str, ): self.id = id self.title = title @@ -23,10 +23,7 @@ def record_to_dict(self) -> dict: "id": self.id, "title": self.title, "image_url": self.image_url, - "coordinates": { - "lat": self.lat, - "lon": self.lon - }, + "coordinates": {"lat": self.lat, "lon": self.lon}, "iiif_url": self.iiif_url, "source_system_url": self.source_system_url, } diff --git a/api/app/server.py b/api/app/server.py index 117c51b..d942233 100644 --- a/api/app/server.py +++ b/api/app/server.py @@ -56,7 +56,7 @@ async def import_data(file: UploadFile = File(...), token: HTTPAuthorizationCred content = await file.read() # Decode the content and use StringIO to emulate a file-like object - csv_content = content.decode('utf-8') + csv_content = content.decode("utf-8") csv_reader = csv.DictReader(StringIO(csv_content)) importer = Importer() @@ -65,4 +65,4 @@ async def import_data(file: UploadFile = File(...), token: HTTPAuthorizationCred return { "status": "ok", "message": message, - } \ No newline at end of file + } From 5dc30c9dd02d4155d673dab9c56d07c28d6f7659 Mon Sep 17 00:00:00 2001 From: Jonas Baumann Date: Sat, 7 Sep 2024 09:18:36 +0200 Subject: [PATCH 3/4] vscode: remove import folder. --- app.code-workspace | 5 ----- 1 file changed, 5 deletions(-) diff --git a/app.code-workspace b/app.code-workspace index d489b60..e478aa2 100644 --- a/app.code-workspace +++ b/app.code-workspace @@ -8,10 +8,6 @@ "name": "frontend", "path": "frontend" }, - { - "name": "import", - "path": "import" - }, { "name": "root", "path": "." @@ -21,7 +17,6 @@ "files.exclude": { "api": true, "frontend": true, - "import": true, } } } From 6d589a75f33fb75d6bddee764d9052879e156179 Mon Sep 17 00:00:00 2001 From: Jonas Baumann Date: Sat, 7 Sep 2024 09:53:20 +0200 Subject: [PATCH 4/4] Setup local nginx for development. The nginx handles the routing to api and frontend, so that we can run both in development mode. --- api/app/server.py | 1 + app.code-workspace | 5 +++++ docker-compose.yml | 40 ++++++++++++++++++++++++++++++---------- frontend/Dockerfile | 22 +++++++++++++++------- frontend/package.json | 3 ++- nginx/nginx.conf | 28 ++++++++++++++++++++++++++++ 6 files changed, 81 insertions(+), 18 deletions(-) create mode 100644 nginx/nginx.conf diff --git a/api/app/server.py b/api/app/server.py index d942233..e22a7a5 100644 --- a/api/app/server.py +++ b/api/app/server.py @@ -23,6 +23,7 @@ version="1.0.0", docs_url="/api/docs", redoc_url="/api/redoc", + openapi_url="/api/openapi.json", ) diff --git a/app.code-workspace b/app.code-workspace index e478aa2..4b8f6a9 100644 --- a/app.code-workspace +++ b/app.code-workspace @@ -8,6 +8,10 @@ "name": "frontend", "path": "frontend" }, + { + "name": "nginx", + "path": "nginx" + }, { "name": "root", "path": "." @@ -17,6 +21,7 @@ "files.exclude": { "api": true, "frontend": true, + "nginx": true, } } } diff --git a/docker-compose.yml b/docker-compose.yml index 5944e9c..f9008a7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,19 +1,42 @@ services: + nginx: + image: nginx:1.27.1-alpine + ports: + - 8000:80 + volumes: + - ./nginx/nginx.conf:/etc/nginx/templates/default.conf.template:ro + environment: + API_HOST: api + FRONTEND_HOST: frontend + depends_on: + api: + condition: service_started + frontend: + condition: service_started + api: build: context: api target: dev ports: - - 8000:8000 + - 8001:8000 volumes: - ./api/app/:/app/app/ + depends_on: + es01: + condition: service_started + es02: + condition: service_started - # frontend: - # build: ./frontend/Dockerfile - - #import: - # build: ./import/Dockerfile - + frontend: + build: + context: frontend + target: dev + ports: + - 8002:3000 + volumes: + - ./frontend/:/app/ + - /app/node_modules es01: image: docker.elastic.co/elasticsearch/elasticsearch:8.15.1 @@ -37,7 +60,6 @@ services: soft: -1 hard: -1 - es02: depends_on: - es01 @@ -59,8 +81,6 @@ services: soft: -1 hard: -1 - - kibana: image: docker.elastic.co/kibana/kibana:8.15.1 volumes: diff --git a/frontend/Dockerfile b/frontend/Dockerfile index b770743..7e38146 100644 --- a/frontend/Dockerfile +++ b/frontend/Dockerfile @@ -1,15 +1,23 @@ -FROM node:22-alpine3.20 - +FROM node:22-alpine3.20 AS base ENV PNPM_HOME="/pnpm" ENV PATH="$PNPM_HOME:$PATH" - RUN corepack enable - -WORKDIR /app/player - +WORKDIR /app COPY pnpm-lock.yaml package.json ./ RUN --mount=type=cache,id=pnpm,target=/pnpm/store pnpm install --frozen-lockfile +FROM base AS dev EXPOSE 3000 - CMD [ "pnpm", "run", "dev", "--host", "0.0.0.0" ] + + +FROM base AS builder +RUN pnpm generate + + +FROM nginx:1.27.1-alpine AS prod +COPY ./docker/frontend.nginx.conf /etc/nginx/templates/default.conf.template +COPY --from=builder /app/dist /usr/share/nginx/html +ENV API_HOST=api +EXPOSE 80 +CMD ["nginx", "-g", "daemon off;"] diff --git a/frontend/package.json b/frontend/package.json index fa78cbf..3c42a20 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -41,5 +41,6 @@ "@vueuse/core": "^11.0.3", "leaflet": "^1.9.4", "lodash-es": "^4.17.21" - } + }, + "packageManager": "pnpm@9.9.0+sha512.60c18acd138bff695d339be6ad13f7e936eea6745660d4cc4a776d5247c540d0edee1a563695c183a66eb917ef88f2b4feb1fc25f32a7adcadc7aaf3438e99c1" } diff --git a/nginx/nginx.conf b/nginx/nginx.conf new file mode 100644 index 0000000..3553fd2 --- /dev/null +++ b/nginx/nginx.conf @@ -0,0 +1,28 @@ +upstream api { + server ${API_HOST}:8000; +} + +upstream frontend { + server ${FRONTEND_HOST}:3000; +} + +server { + listen 80; + listen [::]:80; + server_name localhost; + charset utf-8; + + location ~* ^/(api)($|/) { + proxy_pass http://api; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Host $http_host; + proxy_redirect off; + } + + location / { + proxy_pass http://frontend; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Host $http_host; + proxy_redirect off; + } +}