Merge pull request #93 from ral-facilities/DSEGOG-267-HDF-ingestion-v…

…alidity-checks

DSEGOG-267 HDF ingestion validity checks

.flake8

@@ -11,6 +11,7 @@ per-file-ignores =
   test/*: S101, S303
   test/experiments/scheduler_mocking/models.py: N815
   operationsgateway_api/src/models.py: B902
+  operationsgateway_api/src/records/waveform.py: E402
 # As recommended on https://github.com/tiangolo/fastapi/discussions/7463
 extend-immutable-calls = Depends, fastapi.Depends, Query, fastapi.Query, Body, fastapi.Body, Cookie, fastapi.Cookie, Path, fastapi.Path
 # As recommended on https://github.com/pydantic/pydantic/issues/568

operationsgateway_api/src/channels/channel_manifest.py

@@ -1,5 +1,6 @@
 from datetime import datetime
 import json
+import logging
 from tempfile import SpooledTemporaryFile
 
 from pydantic import ValidationError
@@ -12,6 +13,9 @@
 from operationsgateway_api.src.mongo.interface import MongoDBInterface
 
 
+log = logging.getLogger()
+
+
 class ChannelManifest:
     def __init__(self, manifest_input: SpooledTemporaryFile) -> None:
         """
@@ -76,6 +80,7 @@ async def get_most_recent_manifest() -> dict:
         """
         Get the most up to date manifest file from MongoDB and return it to the user
         """
+        log.info("Getting most recent channel manifest file")
         manifest_data = await MongoDBInterface.find_one(
             "channels",
             sort=[("_id", pymongo.DESCENDING)],

operationsgateway_api/src/exceptions.py

@@ -3,6 +3,18 @@ class ApiError(Exception):
     status_code = 500
 
 
+class RejectFileError(ApiError):
+    def __init__(self, msg="HDF file rejected", *args, **kwargs):
+        super().__init__(msg, *args, **kwargs)
+        self.status_code = 400
+
+
+class RejectRecordError(ApiError):
+    def __init__(self, msg="HDF file record rejected", *args, **kwargs):
+        super().__init__(msg, *args, **kwargs)
+        self.status_code = 400
+
+
 class DatabaseError(ApiError):
     def __init__(self, msg="Database error", *args, **kwargs):
         super().__init__(msg, *args, **kwargs)

operationsgateway_api/src/models.py

@@ -31,19 +31,19 @@ def validate_from_str(input_value: str) -> ObjectId:
 
 
 class ImageModel(BaseModel):
-    path: str
-    data: np.ndarray
+    path: Optional[Union[str, Any]]
+    data: Optional[Union[np.ndarray, Any]]
     model_config = ConfigDict(arbitrary_types_allowed=True)
 
 
 class WaveformModel(BaseModel):
-    # Path is optional as we need it when ingesting waveforms (so we know where to
-    # store it) but don't want to display it when a user is retrieving a waveform.
-    # Setting `exclude=True` inside `Field()` ensure it's not displayed when returned as
-    # a response
+    # Path is optional as we need it when ingesting waveforms (so we know where to store
+    # it) but don't want to display it when a user is retrieving a waveform. Setting
+    # `exclude=True` inside `Field()` ensures it's not displayed when returned as a
+    # response
     path: Optional[str] = Field(None, exclude=True)
-    x: List[float]
-    y: List[float]
+    x: Optional[Union[List[float], Any]]
+    y: Optional[Union[List[float], Any]]
 
     class Config:
         arbitrary_types_allowed = True
@@ -57,47 +57,49 @@ def encode_values(cls, value):  # noqa: N805
 
 
 class ImageChannelMetadataModel(BaseModel):
-    channel_dtype: str
-    exposure_time_s: Optional[float] = None
-    gain: Optional[float] = None
-    x_pixel_size: Optional[float] = None
-    x_pixel_units: Optional[str] = None
-    y_pixel_size: Optional[float] = None
-    y_pixel_units: Optional[str] = None
+    channel_dtype: Optional[Union[str, Any]]
+    exposure_time_s: Optional[Union[float, Any]] = None
+    gain: Optional[Union[float, Any]] = None
+    x_pixel_size: Optional[Union[float, Any]] = None
+    x_pixel_units: Optional[Union[str, Any]] = None
+    y_pixel_size: Optional[Union[float, Any]] = None
+    y_pixel_units: Optional[Union[str, Any]] = None
 
 
 class ImageChannelModel(BaseModel):
     metadata: ImageChannelMetadataModel
-    image_path: str
-    thumbnail: Optional[bytes] = None
+    image_path: Optional[Union[str, Any]]
+    thumbnail: Optional[Union[bytes, Any]] = None
 
 
 class ScalarChannelMetadataModel(BaseModel):
-    channel_dtype: str
-    units: Optional[str] = None
+    channel_dtype: Optional[Union[str, Any]]
+    units: Optional[Union[str, Any]] = None
 
 
 class ScalarChannelModel(BaseModel):
     metadata: ScalarChannelMetadataModel
-    data: Union[int, float, str]
+    data: Optional[Union[int, float, str]]
 
 
 class WaveformChannelMetadataModel(BaseModel):
-    channel_dtype: str
-    x_units: Optional[str] = None
-    y_units: Optional[str] = None
+    channel_dtype: Optional[Union[str, Any]]
+    x_units: Optional[Union[str, Any]] = None
+    y_units: Optional[Union[str, Any]] = None
 
 
 class WaveformChannelModel(BaseModel):
     metadata: WaveformChannelMetadataModel
-    thumbnail: Optional[bytes] = None
-    waveform_path: str
+    thumbnail: Optional[Union[bytes, Any]] = None
+    waveform_path: Optional[Union[str, Any]]
 
 
 class RecordMetadataModel(BaseModel):
-    epac_ops_data_version: str
+    epac_ops_data_version: Optional[Any] = None
     shotnum: Optional[int] = None
-    timestamp: datetime
+    timestamp: Optional[Any] = None
+    active_area: Optional[Any] = None
+    active_experiment: Optional[Any] = None
 
 
 class RecordModel(BaseModel):

operationsgateway_api/src/records/echo_interface.py

@@ -56,7 +56,7 @@ def __init__(self) -> None:
                 "Bucket cannot be found: %s",
                 Config.config.echo.bucket_name,
             )
-            raise EchoS3Error("Bucket for image storage cannot be found")
+            raise EchoS3Error("Bucket for object storage cannot be found")
 
     def download_file_object(self, object_path: str) -> BytesIO:
         """