Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Feat update schemas #803

Merged
merged 13 commits into from
Sep 6, 2023
13 changes: 12 additions & 1 deletion src/backend/app/central/central_crud.py
Original file line number Diff line number Diff line change
Expand Up @@ -330,12 +330,23 @@ def list_submissions(project_id: int, odk_central: project_schemas.ODKCentral =
def get_form_list(db: Session, skip: int, limit: int):
"""Returns the list of id and title of xforms from the database."""
try:
return (
forms = (
db.query(db_models.DbXForm.id, db_models.DbXForm.title)
.offset(skip)
.limit(limit)
.all()
)

result_dict = []
for form in forms:
form_dict = {
'id': form[0], # Assuming the first element is the ID
'title': form[1] # Assuming the second element is the title
}
result_dict.append(form_dict)

return result_dict

except Exception as e:
log.error(e)
raise HTTPException(e) from e
Expand Down
17 changes: 7 additions & 10 deletions src/backend/app/organization/organization_crud.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
import string
from fastapi import HTTPException, File,UploadFile
import re

from sqlalchemy import func
from sqlalchemy.orm import Session
from ..db import db_models

Expand All @@ -44,15 +44,12 @@ def generate_slug(text: str) -> str:


async def get_organisation_by_name(db: Session, name: str):

# Construct the SQL query with the case-insensitive search
query = f"SELECT * FROM organisations WHERE LOWER(name) LIKE LOWER('%{name}%') LIMIT 1"

# Execute the query and retrieve the result
result = db.execute(query)

# Fetch the first row of the result
db_organisation = result.fetchone()
# Use SQLAlchemy's query-building capabilities
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Try to avoid using SQLAlchemy, as we will remove it in the future.
The raw SQL equivalent is probably best 👍

db_organisation = (
db.query(db_models.DbOrganisation)
.filter(func.lower(db_models.DbOrganisation.name).like(func.lower(f'%{name}%')))
.first()
)
return db_organisation


Expand Down
29 changes: 15 additions & 14 deletions src/backend/app/projects/project_crud.py
Original file line number Diff line number Diff line change
Expand Up @@ -44,12 +44,13 @@
from osm_fieldwork.make_data_extract import PostgresClient
from osm_fieldwork.OdkCentral import OdkAppUser
from osm_fieldwork.xlsforms import xlsforms_path
from osm_fieldwork.json2osm import json2osm
from osm_fieldwork import json2osm
from shapely import wkt
from shapely.geometry import MultiPolygon, Polygon, mapping, shape
from sqlalchemy import and_, column, func, inspect, select, table
from sqlalchemy.dialects.postgresql import insert
from sqlalchemy.orm import Session
from sqlalchemy import text
from sqlalchemy.sql import text
from cpuinfo import get_cpu_info
from ..db import database
Expand Down Expand Up @@ -436,14 +437,13 @@ def remove_z_dimension(coord):
)
db.commit()



# Generate project outline from tasks
# query = f'''SELECT ST_AsText(ST_Buffer(ST_Union(outline), 0.5, 'endcap=round')) as oval_envelope
# FROM tasks
# where project_id={project_id};'''
query = text(f"""SELECT ST_AsText(ST_ConvexHull(ST_Collect(outline)))
FROM tasks
WHERE project_id={project_id};""")

query = f"""SELECT ST_AsText(ST_ConvexHull(ST_Collect(outline)))
FROM tasks
WHERE project_id={project_id};"""
log.debug("Generating project outline from tasks")
result = db.execute(query)
data = result.fetchone()
Expand Down Expand Up @@ -700,15 +700,15 @@ def process_polygon(db:Session, project_id:uuid.UUID, boundary_data:str, no_of_b
db.commit()
else:
# Remove the polygons outside of the project AOI using a parameterized query
query = f"""
query = text(f"""
DELETE FROM ways_poly
WHERE NOT ST_Within(ST_Centroid(ways_poly.geom), (SELECT geom FROM project_aoi WHERE project_id = '{project_id}'));
"""
""")
result = db.execute(query)
db.commit()
with open('app/db/split_algorithm.sql', 'r') as sql_file:
query = sql_file.read()
result = db.execute(query, params={'num_buildings': no_of_buildings})
result = db.execute(text(query), params={'num_buildings': no_of_buildings})
result = result.fetchall()
db.query(db_models.DbBuildings).delete()
db.query(db_models.DbOsmLines).delete()
Expand Down Expand Up @@ -1207,7 +1207,7 @@ def generate_task_files(
# Get the features for this task.
# Postgis query to filter task inside this task outline and of this project
# Update those features and set task_id
query = f"""UPDATE features
query = text(f"""UPDATE features
SET task_id={task_id}
WHERE id IN (
SELECT id
Expand All @@ -1216,12 +1216,12 @@ def generate_task_files(
AND ST_IsValid(geometry)
AND ST_IsValid('{task.outline}'::Geometry)
AND ST_Contains('{task.outline}'::Geometry, ST_Centroid(geometry))
)"""
)""")

result = db.execute(query)

# Get the geojson of those features for this task.
query = f"""SELECT jsonb_build_object(
query = text(f"""SELECT jsonb_build_object(
'type', 'FeatureCollection',
'features', jsonb_agg(feature)
)
Expand All @@ -1234,9 +1234,10 @@ def generate_task_files(
) AS feature
FROM features
WHERE project_id={project_id} and task_id={task_id}
) features;"""
) features;""")

result = db.execute(query)

features = result.fetchone()[0]

upload_media = False if features['features'] is None else True
Expand Down
48 changes: 25 additions & 23 deletions src/backend/app/projects/project_schemas.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
# along with FMTM. If not, see <https:#www.gnu.org/licenses/>.
#

from typing import List, Union
from typing import List, Union, Optional

from geojson_pydantic import Feature
from pydantic import BaseModel
Expand Down Expand Up @@ -50,26 +50,33 @@ class BETAProjectUpload(BaseModel):
xform_title: Union[str, None]
odk_central: ODKCentral
hashtags: Union[List[str], None]
organisation_id: int = None
organisation_id: Optional[int] = None
# city: str
# country: str


class Feature(BaseModel):
id: int
project_id: int
task_id: Optional[int] = None
geometry: Optional[Feature] = None


class ProjectSummary(BaseModel):
id: int = -1
priority: ProjectPriority = ProjectPriority.MEDIUM
priority_str: str = priority.name
title: str = None
location_str: str = None
description: str = None
num_contributors: int = None
total_tasks: int = None
tasks_mapped: int = None
tasks_validated: int = None
tasks_bad: int = None
hashtags: List[str] = None
organisation_id: int = None
organisation_logo: str = None
title: Optional[str] = None
location_str: Optional[str] = None
description: Optional[str] = None
total_tasks: Optional[int] = None
tasks_mapped: Optional[int] = None
num_contributors: Optional[int] = None
tasks_validated: Optional[int] = None
tasks_bad: Optional[int] = None
hashtags: Optional[List[str]] = None
organisation_id: Optional[int] = None
organisation_logo: Optional[str] = None


class ProjectBase(BaseModel):
Expand All @@ -79,20 +86,15 @@ class ProjectBase(BaseModel):
project_info: List[ProjectInfo]
status: ProjectStatus
# location_str: str
outline_geojson: Feature = None
project_tasks: List[tasks_schemas.Task] = None
xform_title: str = None
hashtags: List[str] = None
organisation_id: int = None
# outline_geojson: Optional[Feature]
project_tasks: Optional[List[tasks_schemas.Task]]
xform_title: Optional[str] = None
hashtags: Optional[List[str]] = None
organisation_id: Optional[int] = None


class ProjectOut(ProjectBase):
pass



class Feature(BaseModel):
id: int
project_id: int
task_id: int = None
geometry: Feature
16 changes: 12 additions & 4 deletions src/backend/app/tasks/tasks_crud.py
Original file line number Diff line number Diff line change
Expand Up @@ -50,10 +50,18 @@ async def get_task_count_in_project(db: Session, project_id: int):

def get_task_lists(db: Session, project_id: int):
"""Get a list of tasks for a project."""
query = f"""select id from tasks where project_id = {project_id}"""
result = db.execute(query)
tasks = [task[0] for task in result.fetchall()]
return tasks
query = text("""
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do we need the SQL wrapped in text? That locks it in a bit more to SQLAlchemy

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We have used sqlalchemy db session. it was not required in the previous stage. But, it is required since your latest update. It might be because of pydantic??

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

No it's not related to pydantic, but there is no harm having the text() there for now if it works.
We can refactor when we remove SQLAlchemy.

SELECT id
FROM tasks
WHERE project_id = :project_id
""")

# Then execute the query with the desired parameter
result = db.execute(query, {"project_id": project_id})

# Fetch the result
task_ids = [row.id for row in result]
return task_ids


def get_tasks(
Expand Down
18 changes: 9 additions & 9 deletions src/backend/app/tasks/tasks_routes.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
from . import tasks_crud, tasks_schemas
from ..projects import project_crud, project_schemas
from ..central import central_crud
from sqlalchemy.sql import text


router = APIRouter(
Expand Down Expand Up @@ -89,16 +90,15 @@ async def get_point_on_surface(
List[Tuple[int, str]]: A list of tuples containing the task ID and the centroid as a string.
"""

query = f"""
query = text(f"""
SELECT id, ARRAY_AGG(ARRAY[ST_X(ST_PointOnSurface(outline)), ST_Y(ST_PointOnSurface(outline))]) AS point
FROM tasks
WHERE project_id = {project_id}
GROUP BY id;
"""
GROUP BY id; """)

result = db.execute(query)
result = result.fetchall()
return result
result_dict_list = [{"id": row[0], "point": row[1]} for row in result.fetchall()]
return result_dict_list


@router.post("/near_me", response_model=tasks_schemas.TaskOut)
Expand Down Expand Up @@ -178,9 +178,9 @@ async def task_features_count(
)

def process_task(task):
feature_count_query = f"""
select count(*) from features where project_id = {project_id} and task_id = {task}
"""
feature_count_query = text(f"""
select count(*)from features where project_id = {project_id} and task_id = {task}
""")
result = db.execute(feature_count_query)
feature_count = result.fetchone()

Expand All @@ -190,7 +190,7 @@ def process_task(task):
# form_details = central_crud.get_form_full_details(project.odkid, task, odk_credentials)
return {
"task_id": task,
"feature_count": feature_count["count"],
"feature_count": feature_count[0],
# 'submission_count': form_details['submissions'],
"submission_count": len(submission_list)
if isinstance(submission_list, list)
Expand Down
Loading