manual edits

Author: swelborn

backend/app/app/api/api_v1/endpoints/jobs.py

@@ -27,8 +27,8 @@ async def create_job(job: schemas.JobCreate, db: Session = Depends(get_db)):
 
     job = crud.create_job(db=db, job=job)
 
-    scan = schemas.Scan.from_orm(scan)
-    job = schemas.Job.from_orm(job)
+    scan = schemas.Scan.model_validate(scan)
+    job = schemas.Job.model_validate(job)
 
     await send_submit_job_event_to_kafka(SubmitJobEvent(scan=scan, job=job))
 

backend/app/app/api/api_v1/endpoints/scans.py

@@ -86,7 +86,7 @@ async def create_scan_from_file(
             status_code=status.HTTP_409_CONFLICT, detail="Scan in SHA already exists"
         )
 
-    scan_from_file = schemas.ScanFromFile(sha=sha, **meta.dict())
+    scan_from_file = schemas.ScanFromFile(sha=sha, **meta.model_dump())
 
     scan = crud.create_scan(db=db, scan=scan_from_file)
     ext = Path(file_upload.filename).suffix
@@ -138,7 +138,7 @@ async def create_scan(
     try:
         content_type = request.headers["content-type"]
         if content_type == "application/json":
-            scan = schemas.Scan4DCreate.parse_obj(await request.json())
+            scan = schemas.Scan4DCreate.model_validate(await request.json())
             scan = await create_4d_scan(db, scan)
         elif content_type.startswith("multipart/form-data"):
             form_data = await request.form()
@@ -156,7 +156,7 @@ async def create_scan(
                     detail="Invalid request, scan metadata is required",
                 )
 
-            scan_metadata = schemas.ScanFromFileMetadata.parse_raw(scan_metadata_str)
+            scan_metadata = schemas.ScanFromFileMetadata.model_validate_json(scan_metadata_str)
 
             # See if we have an associated ser file
             file_stem = Path(file.filename).stem.replace("%20", "\\ ")
@@ -175,7 +175,7 @@ async def create_scan(
         )
 
     await send_scan_event_to_kafka(
-        ScanCreatedEvent(**schemas.Scan.from_orm(scan).dict())
+        ScanCreatedEvent(**schemas.Scan.model_validate(scan).model_dump())
     )
 
     return scan
@@ -293,7 +293,7 @@ async def update_scan(
             scan_updated_event.progress = cast(int, scan.progress)
 
         scan_updated_event.locations = [
-            schemas.scan.Location.from_orm(l) for l in scan.locations
+            schemas.scan.Location.model_validate(l) for l in scan.locations
         ]
 
         if scan.notes is not None and scan.notes == payload.notes:
@@ -319,7 +319,7 @@ async def _remove_scan_files(db_scan: Scan, host: Optional[str] = None):
                 status_code=status.HTTP_400_BAD_REQUEST, detail="Invalid request"
             )
 
-        scan = schemas.Scan.from_orm(db_scan)
+        scan = schemas.Scan.model_validate(db_scan)
         await send_remove_scan_files_event_to_kafka(
             RemoveScanFilesEvent(scan=scan, host=host)
         )

backend/app/app/core/config.py

@@ -1,9 +1,9 @@
 import secrets
 from typing import Any, Dict, List, Literal, Optional, Union
 
-from pydantic import AnyHttpUrl, BaseSettings, PostgresDsn, validator
-
 from app.schemas import Machine
+from pydantic import AnyHttpUrl, PostgresDsn, field_validator, ValidationInfo
+from pydantic_settings import BaseSettings, SettingsConfigDict
 
 
 class Settings(BaseSettings):
@@ -18,7 +18,8 @@ class Settings(BaseSettings):
     # "http://localhost:8080", "http://local.dockertoolbox.tiangolo.com"]'
     BACKEND_CORS_ORIGINS: List[Union[AnyHttpUrl, Literal["*"]]] = []
 
-    @validator("BACKEND_CORS_ORIGINS", pre=True)
+    @field_validator("BACKEND_CORS_ORIGINS", mode="before")
+    @classmethod
     def assemble_cors_origins(cls, v: Union[str, List[str]]) -> Union[List[str], str]:
         if isinstance(v, str) and not v.startswith("["):
             return [i.strip() for i in v.split(",")]
@@ -32,19 +33,20 @@ def assemble_cors_origins(cls, v: Union[str, List[str]]) -> Union[List[str], str
     POSTGRES_USER: str
     POSTGRES_PASSWORD: str
     POSTGRES_DB: str
-    SQLALCHEMY_DATABASE_URI: Optional[PostgresDsn] = None
+    SQLALCHEMY_DATABASE_URI: Optional[str] = None
 
-    @validator("SQLALCHEMY_DATABASE_URI", pre=True)
-    def assemble_db_connection(cls, v: Optional[str], values: Dict[str, Any]) -> Any:
+    @field_validator("SQLALCHEMY_DATABASE_URI", mode="before")
+    def assemble_db_connection(cls, v: Optional[str], info: ValidationInfo) -> Any:
         if isinstance(v, str):
             return v
-        return PostgresDsn.build(
+        postgres_build = PostgresDsn.build(
             scheme="postgresql",
-            user=values.get("POSTGRES_USER"),
-            password=values.get("POSTGRES_PASSWORD"),
-            host=values.get("POSTGRES_SERVER"),
-            path=f"/{values.get('POSTGRES_DB') or ''}",
+            username=info.data.get("POSTGRES_USER"),
+            password=info.data.get("POSTGRES_PASSWORD"),
+            host=info.data.get("POSTGRES_SERVER"),
+            path=f"{info.data.get('POSTGRES_DB') or ''}",
         )
+        return postgres_build.unicode_string()
 
     API_KEY_NAME: str
     API_KEY: str
@@ -53,7 +55,7 @@ def assemble_db_connection(cls, v: Optional[str], values: Dict[str, Any]) -> Any
     JWT_ALGORITHM: str
     JWT_ACCESS_TOKEN_EXPIRE_MINUTES: int
     JWT_REFRESH_TOKEN_EXPIRE_MINUTES: int
-    JWT_REFRESH_COOKIE_DOMAIN: str = None
+    JWT_REFRESH_COOKIE_DOMAIN: Optional[str] = None
     JWT_REFRESH_COOKIE_SECURE: bool = False
 
     KAFKA_BOOTSTRAP_SERVERS: List[str]
@@ -67,15 +69,12 @@ def assemble_db_connection(cls, v: Optional[str], values: Dict[str, Any]) -> Any
     # have been reset in in the detector software.
     HAADF_SCAN_AGE_LIMIT: int = 1
 
-    SENTRY_DSN_URL: AnyHttpUrl = None
+    SENTRY_DSN_URL: Optional[AnyHttpUrl] = None
 
     MACHINES: List[Machine]
     NCEMHUB_PATH: str
     NOTEBOOKS: List[str] = ["DPC", "vacuum_scan_prepare", "vacuum_scan_subtract"]
-
-    class Config:
-        case_sensitive = True
-        env_file = ".env"
+    model_config = SettingsConfigDict(case_sensitive=True, env_file=".env", extra="allow")
 
 
 settings = Settings()

backend/app/app/crud/job.py

@@ -32,7 +32,7 @@ def get_jobs(
 
 
 def create_job(db: Session, job: schemas.JobCreate):
-    db_job = models.Job(**job.dict())
+    db_job = models.Job(**job.model_dump())
     db.add(db_job)
     db.commit()
     db.refresh(db_job)

backend/app/app/crud/scan.py

@@ -147,10 +147,10 @@ def create_scan(
         scan.microscope_id = microscope_ids[0]
 
     # Note: We have to pass metadata as metadata_ as metadata is reserved!
-    db_scan = models.Scan(**scan.dict(), image_path=image_path, metadata_=scan.metadata)
+    db_scan = models.Scan(**scan.model_dump(), image_path=image_path, metadata_=scan.metadata)
     db.add(db_scan)
     for l in locations:
-        l = models.Location(**l.dict())
+        l = models.Location(**l.model_dump())
         db_scan.locations.append(l)
         db.add(l)
     db.commit()
@@ -192,7 +192,7 @@ def update_scan(
                 is not None
             )
             if not exists:
-                l = models.Location(**l.dict(), scan_id=id)
+                l = models.Location(**l.model_dump(), scan_id=id)
                 db.add(l)
                 locations_updated = True
 

backend/app/app/schemas/scan.py

@@ -117,12 +117,12 @@ class ScanCreatedEvent(ScanEvent):
     microscope_id: int
     scan_id: Optional[int] = None
     created: datetime
-    event_type = ScanEventType.CREATED
+    event_type: ScanEventType = ScanEventType.CREATED
     image_path: Optional[str] = None
 
 
 class ScanUpdateEvent(ScanEvent):
-    event_type = ScanEventType.UPDATED
+    event_type: ScanEventType = ScanEventType.UPDATED
     jobs: Optional[List[Job]] = None
     image_path: Optional[str] = None
     notes: Optional[str] = None

backend/faust/config.py

@@ -24,7 +24,7 @@ class Settings(BaseSettings):
     JOB_QOS: str
     JOB_QOS_FILTER: str
     JOB_BBCP_EXECUTABLE_PATH: str
-    JOB_MACHINE_OVERRIDES_PATH: Optional[str]
+    JOB_MACHINE_OVERRIDES_PATH: Optional[str] = None
 
     IMAGE_UPLOAD_DIR: str
     HAADF_IMAGE_UPLOAD_DIR_EXPIRATION_HOURS: int
@@ -40,7 +40,7 @@ class Settings(BaseSettings):
     CUSTODIAN_USER: str
     CUSTODIAN_PRIVATE_KEY: str
     CUSTODIAN_VALID_HOSTS: List[str] = []
-    model_config = SettingsConfigDict(case_sensitive=True, env_file=".env")
+    model_config = SettingsConfigDict(case_sensitive=True, env_file=".env", extra="allow")
 
 
 settings = Settings()

backend/faust/scan_worker.py

@@ -164,7 +164,7 @@ async def process_status_file(
     if event.content is None:
         raise Exception("Status file not included!")
 
-    status_file = ScanStatusFile.parse_raw(event.content)
+    status_file = ScanStatusFile.model_validate_json(event.content)
     scan_id = status_file.last_scan_number
     new_scan = scan_id not in scan_id_to_distiller_id
     primary_status_file = (
@@ -337,7 +337,7 @@ async def watch_for_event(file_events):
                 raise Exception("Status file not included!")
 
             try:
-                status_file = ScanStatusFile.parse_raw(event.content)
+                status_file = ScanStatusFile.model_validate_json(event.content)
             except ValidationError:
                 logger.warning(f"Skipping {path}, with validation error.")
                 continue
@@ -373,7 +373,7 @@ async def process_sync_event(session: aiohttp.ClientSession, event: SyncEvent) -
         )
 
         try:
-            status = ScanStatusFile.parse_raw(f.content)
+            status = ScanStatusFile.model_validate_json(f.content)
         except ValidationError:
             logger.warning(f"Skipping {path}, with validation error.")
             continue