Skip to content

Commit

Permalink
Merge branch 'main' of github.com:histify/geophotoradar
Browse files Browse the repository at this point in the history
  • Loading branch information
liowalter committed Sep 7, 2024
2 parents 5852190 + 6d589a7 commit fa5df6c
Show file tree
Hide file tree
Showing 10 changed files with 104 additions and 53 deletions.
9 changes: 6 additions & 3 deletions api/app/elastic.py
Original file line number Diff line number Diff line change
Expand Up @@ -58,22 +58,25 @@ def create_index(self) -> None:
}
},
)

def import_records(self, records: List[Record]) -> str:
self.create_index()
success, failed = 0, 0
for ok, action in streaming_bulk(
client=self.connection, index=self.index_name, actions=self.generate_actions(records),
client=self.connection,
index=self.index_name,
actions=self.generate_actions(records),
):
if ok:
success += 1
else:
failed += 1
return(f"Finished: {success} documents indexed, {failed} failed.")
return f"Finished: {success} documents indexed, {failed} failed."

def generate_actions(self, records: List[Record]):
"""This function is passed into the bulk()
helper to create many documents in sequence.
"""
for record in records:
doc = record.record_to_dict()
yield doc
yield doc
18 changes: 8 additions & 10 deletions api/app/importer.py
Original file line number Diff line number Diff line change
@@ -1,31 +1,29 @@
from csv import DictReader
from typing import List

from app.record import Record
from typing import List
import csv


class Importer:
def read_csv_to_records(self, csv_reader: DictReader) -> List[Record]:
records = []
for row in csv_reader:
try:
lat_str, lon_str = row['coordinates'].split(',')
except ValueError as e:
lat_str, lon_str = row["coordinates"].split(",")
except ValueError:
# print(str(e))
continue
lat = float(lat_str.strip())
lon = float(lon_str.strip())
# Create a Record instance from each row
record = Record(
id=row['id'],
title=row['title'],
image_url=row['image_url'],
id=row["id"],
title=row["title"],
image_url=row["image_url"],
lat=lat,
lon=lon,
iiif_url=row['iiif_url'],
source_system_url=row['source_system_url']
iiif_url=row["iiif_url"],
source_system_url=row["source_system_url"],
)
records.append(record)
return records

21 changes: 9 additions & 12 deletions api/app/record.py
Original file line number Diff line number Diff line change
@@ -1,13 +1,13 @@
class Record:
def __init__(
self,
id: str,
title: str,
image_url: str,
lat: float,
lon: float,
iiif_url: str,
source_system_url: str,
self,
id: str,
title: str,
image_url: str,
lat: float,
lon: float,
iiif_url: str,
source_system_url: str,
):
self.id = id
self.title = title
Expand All @@ -23,10 +23,7 @@ def record_to_dict(self) -> dict:
"id": self.id,
"title": self.title,
"image_url": self.image_url,
"coordinates": {
"lat": self.lat,
"lon": self.lon
},
"coordinates": {"lat": self.lat, "lon": self.lon},
"iiif_url": self.iiif_url,
"source_system_url": self.source_system_url,
}
5 changes: 3 additions & 2 deletions api/app/server.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
version="1.0.0",
docs_url="/api/docs",
redoc_url="/api/redoc",
openapi_url="/api/openapi.json",
)


Expand Down Expand Up @@ -56,7 +57,7 @@ async def import_data(file: UploadFile = File(...), token: HTTPAuthorizationCred
content = await file.read()

# Decode the content and use StringIO to emulate a file-like object
csv_content = content.decode('utf-8')
csv_content = content.decode("utf-8")
csv_reader = csv.DictReader(StringIO(csv_content))

importer = Importer()
Expand All @@ -65,4 +66,4 @@ async def import_data(file: UploadFile = File(...), token: HTTPAuthorizationCred
return {
"status": "ok",
"message": message,
}
}
11 changes: 3 additions & 8 deletions app.code-workspace
Original file line number Diff line number Diff line change
Expand Up @@ -4,17 +4,13 @@
"name": "api",
"path": "api"
},
{
"name": "db",
"path": "db"
},
{
"name": "frontend",
"path": "frontend"
},
{
"name": "import",
"path": "import"
"name": "nginx",
"path": "nginx"
},
{
"name": "root",
Expand All @@ -24,9 +20,8 @@
"settings": {
"files.exclude": {
"api": true,
"db": true,
"frontend": true,
"import": true,
"nginx": true,
}
}
}
Empty file removed db/Dockerfile
Empty file.
40 changes: 30 additions & 10 deletions docker-compose.yml
Original file line number Diff line number Diff line change
@@ -1,19 +1,42 @@
services:
nginx:
image: nginx:1.27.1-alpine
ports:
- 8000:80
volumes:
- ./nginx/nginx.conf:/etc/nginx/templates/default.conf.template:ro
environment:
API_HOST: api
FRONTEND_HOST: frontend
depends_on:
api:
condition: service_started
frontend:
condition: service_started

api:
build:
context: api
target: dev
ports:
- 8000:8000
- 8001:8000
volumes:
- ./api/app/:/app/app/
depends_on:
es01:
condition: service_started
es02:
condition: service_started

# frontend:
# build: ./frontend/Dockerfile

#import:
# build: ./import/Dockerfile

frontend:
build:
context: frontend
target: dev
ports:
- 8002:3000
volumes:
- ./frontend/:/app/
- /app/node_modules

es01:
image: docker.elastic.co/elasticsearch/elasticsearch:8.15.1
Expand All @@ -37,7 +60,6 @@ services:
soft: -1
hard: -1


es02:
depends_on:
- es01
Expand All @@ -59,8 +81,6 @@ services:
soft: -1
hard: -1



kibana:
image: docker.elastic.co/kibana/kibana:8.15.1
volumes:
Expand Down
22 changes: 15 additions & 7 deletions frontend/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,15 +1,23 @@
FROM node:22-alpine3.20

FROM node:22-alpine3.20 AS base
ENV PNPM_HOME="/pnpm"
ENV PATH="$PNPM_HOME:$PATH"

RUN corepack enable

WORKDIR /app/player

WORKDIR /app
COPY pnpm-lock.yaml package.json ./
RUN --mount=type=cache,id=pnpm,target=/pnpm/store pnpm install --frozen-lockfile

FROM base AS dev
EXPOSE 3000

CMD [ "pnpm", "run", "dev", "--host", "0.0.0.0" ]


FROM base AS builder
RUN pnpm generate


FROM nginx:1.27.1-alpine AS prod
COPY ./docker/frontend.nginx.conf /etc/nginx/templates/default.conf.template
COPY --from=builder /app/dist /usr/share/nginx/html
ENV API_HOST=api
EXPOSE 80
CMD ["nginx", "-g", "daemon off;"]
3 changes: 2 additions & 1 deletion frontend/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -41,5 +41,6 @@
"@vueuse/core": "^11.0.3",
"leaflet": "^1.9.4",
"lodash-es": "^4.17.21"
}
},
"packageManager": "[email protected]+sha512.60c18acd138bff695d339be6ad13f7e936eea6745660d4cc4a776d5247c540d0edee1a563695c183a66eb917ef88f2b4feb1fc25f32a7adcadc7aaf3438e99c1"
}
28 changes: 28 additions & 0 deletions nginx/nginx.conf
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
upstream api {
server ${API_HOST}:8000;
}

upstream frontend {
server ${FRONTEND_HOST}:3000;
}

server {
listen 80;
listen [::]:80;
server_name localhost;
charset utf-8;

location ~* ^/(api)($|/) {
proxy_pass http://api;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header Host $http_host;
proxy_redirect off;
}

location / {
proxy_pass http://frontend;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header Host $http_host;
proxy_redirect off;
}
}

0 comments on commit fa5df6c

Please sign in to comment.