Merge pull request #796 from hotosm/build/upgrade-pydantic-v2

Upgrade dependencies, Pydantic v2

src/backend/Dockerfile

@@ -64,7 +64,6 @@ RUN set -ex \
         "libspatialindex-dev" \
         "libproj-dev" \
         "libgeos-dev" \
-        "libgdal-dev" \
         "git" \
     && rm -rf /var/lib/apt/lists/*
 COPY --from=extract-deps \
@@ -98,7 +97,6 @@ RUN set -ex \
         "libspatialindex-c6" \
         "libproj25" \
         "libgeos-c1v5" \
-        "libgdal32" \
     && rm -rf /var/lib/apt/lists/*
 COPY container-entrypoint.sh /
 ENTRYPOINT ["/container-entrypoint.sh"]

src/backend/app/config.py

@@ -15,13 +15,13 @@
 #     You should have received a copy of the GNU General Public License
 #     along with FMTM.  If not, see <https:#www.gnu.org/licenses/>.
 #
-
 """Config file for Pydantic and FastAPI, using environment variables."""
 
 from functools import lru_cache
 from typing import Any, Optional, Union
 
-from pydantic import AnyUrl, BaseSettings, PostgresDsn, validator
+from pydantic import AnyUrl, Extra, FieldValidationInfo, PostgresDsn, field_validator
+from pydantic_settings import BaseSettings, SettingsConfigDict
 
 
 class Settings(BaseSettings):
@@ -35,12 +35,15 @@ class Settings(BaseSettings):
     FRONTEND_MAIN_URL: Optional[str]
     FRONTEND_MAP_URL: Optional[str]
 
-    EXTRA_CORS_ORIGINS: Optional[Union[str, list[AnyUrl]]]
+    EXTRA_CORS_ORIGINS: Optional[Union[str, list[AnyUrl]]] = []
 
-    @validator("EXTRA_CORS_ORIGINS", pre=True)
+    @field_validator("EXTRA_CORS_ORIGINS", mode="before")
+    @classmethod
     def assemble_cors_origins(
-        cls, val: Union[str, list[AnyUrl]], values: dict
-    ) -> list[str]:
+        cls,
+        val: Union[str, list[AnyUrl]],
+        info: FieldValidationInfo,
+    ) -> Union[list[str], str]:
         """Build and validate CORS origins list.
 
         By default, the provided frontend URLs are included in the origins list.
@@ -49,9 +52,9 @@ def assemble_cors_origins(
         default_origins = []
 
         # Build default origins from env vars
-        url_scheme = values.get("URL_SCHEME")
-        main_url = values.get("FRONTEND_MAIN_URL")
-        map_url = values.get("FRONTEND_MAP_URL")
+        url_scheme = info.data.get("URL_SCHEME")
+        main_url = info.data.get("URL_SCHEME")
+        map_url = info.data.get("URL_SCHEME")
         if url_scheme and main_url and map_url:
             default_origins = [
                 f"{url_scheme}://{main_url}",
@@ -78,26 +81,23 @@ def assemble_cors_origins(
     FMTM_DB_PASSWORD: Optional[str] = "fmtm"
     FMTM_DB_NAME: Optional[str] = "fmtm"
 
-    FMTM_DB_URL: Optional[PostgresDsn]
+    FMTM_DB_URL: Optional[PostgresDsn] = None
 
-    @validator("FMTM_DB_URL", pre=True)
-    def assemble_db_connection(cls, v: str, values: dict[str, Any]) -> Any:
+    @field_validator("FMTM_DB_URL", mode="after")
+    @classmethod
+    def assemble_db_connection(cls, v: Optional[str], info: FieldValidationInfo) -> Any:
         """Build Postgres connection from environment variables."""
         if isinstance(v, str):
             return v
-        if not (user := values.get("FMTM_DB_USER")):
-            raise ValueError("FMTM_DB_USER is not present in the environment")
-        if not (password := values.get("FMTM_DB_PASSWORD")):
-            raise ValueError("FMTM_DB_PASSWORD is not present in the environment")
-        if not (host := values.get("FMTM_DB_HOST")):
-            raise ValueError("FMTM_DB_HOST is not present in the environment")
-        return PostgresDsn.build(
+        pg_url = PostgresDsn.build(
             scheme="postgresql",
-            user=user,
-            password=password,
-            host=host,
-            path=f"/{values.get('FMTM_DB_NAME') or ''}",
+            username=info.data.get("FMTM_DB_USER"),
+            password=info.data.get("FMTM_DB_PASSWORD"),
+            host=info.data.get("FMTM_DB_HOST"),
+            path=info.data.get("FMTM_DB_NAME", ""),
         )
+        # Convert Url type to string
+        return str(pg_url)
 
     ODK_CENTRAL_URL: Optional[AnyUrl]
     ODK_CENTRAL_USER: Optional[str]
@@ -110,13 +110,11 @@ def assemble_db_connection(cls, v: str, values: dict[str, Any]) -> Any:
     OSM_SCOPE: str = "read_prefs"
     OSM_LOGIN_REDIRECT_URI: AnyUrl = "http://127.0.0.1:8080/osmauth/"
 
-    SENTRY_DSN: Optional[str]
-
-    class Config:
-        """Pydantic settings config."""
+    SENTRY_DSN: Optional[str] = None
 
-        case_sensitive = True
-        env_file = ".env"
+    model_config = SettingsConfigDict(
+        case_sensitive=True, env_file=".env", extra=Extra.allow
+    )
 
 
 @lru_cache

src/backend/app/projects/project_crud.py

@@ -44,6 +44,7 @@
 from osm_fieldwork.make_data_extract import PostgresClient
 from osm_fieldwork.OdkCentral import OdkAppUser
 from osm_fieldwork.xlsforms import xlsforms_path
+from osm_fieldwork.json2osm import json2osm
 from shapely import wkt
 from shapely.geometry import MultiPolygon, Polygon, mapping, shape
 from sqlalchemy import and_, column, func, inspect, select, table
@@ -2286,13 +2287,5 @@ async def get_mbtiles_list(db: Session, project_id: int):
 
 
 async def convert_geojson_to_osm(geojson_file: str):
-    try:
-        osm_filename = "/tmp/task_outline.osm"
-        ogr2osm_command = f"ogr2osm -f -o {osm_filename} {geojson_file}"
-        os.system(ogr2osm_command)
-
-        return osm_filename
-    except Exception as e:
-        return {
-            "error": f"Conversion failed. Error details: {str(e)}"
-        }
+    """Convert a GeoJSON file to OSM format."""
+    return json2osm(geojson_file)

src/backend/app/projects/project_schemas.py

@@ -31,18 +31,12 @@ class ODKCentral(BaseModel):
     odk_central_user: str
     odk_central_password: str
 
-    class Config:
-        orm_mode = True
-
 
 class ProjectInfo(BaseModel):
     name: str
     short_description: str
     description: str
 
-    class Config:
-        orm_mode = True
-
 
 class ProjectUpdate(BaseModel):
     name: Union[str, None]
@@ -77,9 +71,6 @@ class ProjectSummary(BaseModel):
     organisation_id: int = None
     organisation_logo: str = None
 
-    class Config:
-        orm_mode = True
-
 
 class ProjectBase(BaseModel):
     id: int
@@ -94,9 +85,6 @@ class ProjectBase(BaseModel):
     hashtags: List[str] = None
     organisation_id: int = None
 
-    class Config:
-        orm_mode = True
-
 
 class ProjectOut(ProjectBase):
     pass
@@ -108,6 +96,3 @@ class Feature(BaseModel):
     project_id: int
     task_id: int = None
     geometry: Feature
-
-    class Config:
-        orm_mode = True

src/backend/app/submission/submission_crud.py

@@ -174,6 +174,7 @@ def create_zip_file(files, output_file_path):
 
 async def convert_json_to_osm_xml(file_path):
 
+    # TODO refactor to simply use json2osm(file_path)
     jsonin = JsonDump()
     infile = Path(file_path)
 
@@ -218,6 +219,7 @@ async def write_osm_async(features):
 
 async def convert_json_to_osm(file_path):
 
+    # TODO refactor to simply use json2osm(file_path)
     jsonin = JsonDump()
     infile = Path(file_path)
 

src/backend/app/tasks/tasks_schemas.py

@@ -42,9 +42,6 @@ class TaskHistoryBase(BaseModel):
     action_text: str
     action_date: datetime
 
-    class Config:
-        orm_mode = True
-
 
 class TaskHistoryOut(TaskHistoryBase):
     pass
@@ -59,10 +56,6 @@ class TaskBasicInfo(BaseModel):
     locked_by_username: str = None
     task_history: List[TaskHistoryBase]
 
-    class Config:
-        orm_mode = True
-
-
 
 class TaskBase(BaseModel):
     id: int
@@ -77,9 +70,6 @@ class TaskBase(BaseModel):
     locked_by_username: str = None
     task_history: List[TaskHistoryBase]
 
-    class Config:
-        orm_mode = True
-
 
 class Task(TaskBase):
     # geometry_geojson: str

src/backend/app/users/user_schemas.py

@@ -28,17 +28,11 @@ class UserBase(BaseModel):
 class User(UserBase):
     id: int
 
-    class Config:
-        orm_mode = True
-
 
 class UserOut(UserBase):
     id: int
     role: str
 
-    class Config:
-        orm_mode = True
-
 
 class UserRole(BaseModel):
     role: str