overwriting timestamp value after _annotate() is finished, fixed some pydantic warnings

Author: keighrim

clams/app/__init__.py

@@ -77,7 +77,7 @@ def __init__(self):
 
         self.metadata_param_caster = ParameterCaster(self.metadata_param_spec)
         self.annotate_param_caster = ParameterCaster(self.annotate_param_spec)
-        self.logger = logging.getLogger(self.metadata.identifier)
+        self.logger = logging.getLogger(str(self.metadata.identifier))
 
     def appmetadata(self, **kwargs: List[str]) -> str:
         """
@@ -138,6 +138,7 @@ def annotate(self, mmif: Union[str, dict, Mmif], **runtime_params: List[str]) ->
         """
         if not isinstance(mmif, Mmif):
             mmif = Mmif(mmif)
+        existing_view_ids = {view.id for view in mmif.views}
         issued_warnings = []
         for key in runtime_params:
             if key not in self.annotate_param_spec:
@@ -156,7 +157,8 @@ def annotate(self, mmif: Union[str, dict, Mmif], **runtime_params: List[str]) ->
             warnings_view = annotated.new_view()
             self.sign_view(warnings_view, refined)
             warnings_view.metadata.warnings = issued_warnings
-        td = datetime.now() - t
+        run_id = datetime.now()
+        td = run_id - t
         runningTime = refined.get('runningTime', False)
         hwFetch = refined.get('hwFetch', False)
         runtime_recs = {}
@@ -180,7 +182,8 @@ def annotate(self, mmif: Union[str, dict, Mmif], **runtime_params: List[str]) ->
                     name, mem = gpu.split(', ')
                     runtime_recs['cuda'].append(self._cuda_device_name_concat(name, mem))
         for annotated_view in annotated.views:
-            if annotated_view.metadata.app == self.metadata.identifier:
+            if annotated_view.id not in existing_view_ids and annotated_view.metadata.app == str(self.metadata.identifier):
+                annotated_view.metadata.timestamp = run_id
                 profiling_data = {}
                 if runningTime:
                     profiling_data['runningTime'] = str(td)
@@ -265,7 +268,7 @@ def sign_view(self, view: View, runtime_conf: dict) -> None:
         :param view: a view to sign
         :param runtime_conf: runtime configuration of the app as k-v pairs
         """
-        view.metadata.app = self.metadata.identifier
+        view.metadata.app = str(self.metadata.identifier)
         params_map = {p.name: p for p in self.metadata.parameters}
         if self._RAW_PARAMS_KEY in runtime_conf:
             for k, v in runtime_conf.items():
@@ -300,7 +303,7 @@ def set_error_view(self, mmif: Union[str, dict, Mmif], **runtime_conf: List[str]
             mmif = Mmif(mmif)
         error_view: Optional[View] = None
         for view in reversed(mmif.views):
-            if view.metadata.app == self.metadata.identifier:
+            if view.metadata.app == str(self.metadata.identifier):
                 error_view = view
                 break
         if error_view is None:
@@ -358,7 +361,7 @@ def _get_profile_path(self, param_hash: str) -> pathlib.Path:
         :return: Path to the profile file
         """
         # Sanitize app identifier for filesystem use
-        app_id = self.metadata.identifier.replace('/', '-').replace(':', '-')
+        app_id = str(self.metadata.identifier).replace('/', '-').replace(':', '-')
         cache_base = pathlib.Path(os.environ.get('XDG_CACHE_HOME', pathlib.Path.home() / '.cache'))
         cache_dir = cache_base / 'clams' / 'memory_profiles' / app_id
         return cache_dir / f"memory_{param_hash}.json"

clams/appmetadata/__init__.py

@@ -131,11 +131,6 @@ def validate_properties(cls, value):
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
 
-    @pydantic.field_validator('at_type', mode='after')  # because pydantic v2 doesn't auto-convert url to string
-    @classmethod
-    def stringify(cls, val):
-        return str(val)
-    
     @pydantic.field_validator('at_type', mode='before')
     @classmethod
     def at_type_must_be_str(cls, v):
@@ -375,28 +370,26 @@ class AppMetadata(pydantic.BaseModel):
     }
 
     @pydantic.model_validator(mode='after')
-    @classmethod
-    def assign_versions(cls, data):
-        if data.app_version == '':
-            data.app_version = generate_app_version()
-        if data.mmif_version == '':
-            data.mmif_version = get_mmif_specver()
-        return data
+    def assign_versions(self):
+        if self.app_version == '':
+            self.app_version = generate_app_version()
+        if self.mmif_version == '':
+            self.mmif_version = get_mmif_specver()
+        return self
 
     @pydantic.model_validator(mode='after')
-    @classmethod
-    def validate_gpu_memory(cls, data):
+    def validate_gpu_memory(self):
         import warnings
-        if data.est_gpu_mem_typ > 0 and data.est_gpu_mem_min > 0:
-            if data.est_gpu_mem_typ < data.est_gpu_mem_min:
+        if self.est_gpu_mem_typ > 0 and self.est_gpu_mem_min > 0:
+            if self.est_gpu_mem_typ < self.est_gpu_mem_min:
                 warnings.warn(
-                    f"est_gpu_mem_typ ({data.est_gpu_mem_typ} MB) is less than "
-                    f"est_gpu_mem_min ({data.est_gpu_mem_min} MB). "
-                    f"Setting est_gpu_mem_typ to {data.est_gpu_mem_min} MB.",
+                    f"est_gpu_mem_typ ({self.est_gpu_mem_typ} MB) is less than "
+                    f"est_gpu_mem_min ({self.est_gpu_mem_min} MB). "
+                    f"Setting est_gpu_mem_typ to {self.est_gpu_mem_min} MB.",
                     UserWarning
                 )
-                data.est_gpu_mem_typ = data.est_gpu_mem_min
-        return data
+                self.est_gpu_mem_typ = self.est_gpu_mem_min
+        return self
 
     @pydantic.field_validator('identifier', mode='before')
     @classmethod
@@ -405,11 +398,7 @@ def append_version(cls, val):
         suffix = generate_app_version()
         return '/'.join(map(lambda x: x.strip('/'), filter(None, (prefix, val, suffix))))
 
-    @pydantic.field_validator('url', 'identifier', mode='after')  # because pydantic v2 doesn't auto-convert url to string
-    @classmethod
-    def stringify(cls, val):
-        return str(val)
-    
+
     def _check_input_duplicate(self, a_input):
         for elem in self.input:
             if isinstance(elem, list):

tests/test_clamsapp.py

@@ -234,12 +234,12 @@ def test_sign_view(self):
         m = Mmif(self.in_mmif)
         v1 = m.new_view()
         self.app.sign_view(v1, {})
-        self.assertEqual(v1.metadata.app, self.app.metadata.identifier)
+        self.assertEqual(v1.metadata.app, str(self.app.metadata.identifier))
         self.assertEqual(len(v1.metadata.parameters), 0)
         v2 = m.new_view()
         args2 = {'undefined_param1': ['value1']}  # values are lists as our restifier uses `to_dict(flat=False)`
         self.app.sign_view(v2, self.app._refine_params(**args2))
-        self.assertEqual(v2.metadata.app, self.app.metadata.identifier)
+        self.assertEqual(v2.metadata.app, str(self.app.metadata.identifier))
         self.assertEqual(len(v2.metadata.parameters), 1)
         self.assertFalse(clams.ClamsApp._RAW_PARAMS_KEY in v2.metadata.appConfiguration)
         for param in self.app.metadata.parameters:
@@ -271,19 +271,41 @@ def test_annotate(self):
         out_mmif = Mmif(out_mmif)
         self.assertEqual(len(out_mmif.views), 2)
         for v in out_mmif.views:
-            if v.metadata.app == self.app.metadata.identifier:
+            if v.metadata.app == str(self.app.metadata.identifier):
                 self.assertEqual(len(v.metadata.parameters), 0)  # no params were passed when `annotate()` was called
         out_mmif = self.app.annotate(self.in_mmif, pretty=[str(False)])
         out_mmif = Mmif(out_mmif)
         for v in out_mmif.views:
-            if v.metadata.app == self.app.metadata.identifier:
+            if v.metadata.app == str(self.app.metadata.identifier):
                 self.assertEqual(len(v.metadata.parameters), 1)  # 'pretty` parameter was passed 
         out_mmif = Mmif(self.app.annotate(out_mmif))
         self.assertEqual(len(out_mmif.views), 4)
         views = list(out_mmif.views)
         # insertion order is kept
-        self.assertTrue(views[0].metadata.timestamp < views[1].metadata.timestamp)
-    
+        self.assertEqual(views[0].metadata.timestamp, views[1].metadata.timestamp)
+        self.assertEqual(views[2].metadata.timestamp, views[3].metadata.timestamp)
+        self.assertTrue(views[1].metadata.timestamp < views[2].metadata.timestamp)
+
+    def test_run_id(self):
+        # first run
+        out_mmif = Mmif(self.app.annotate(self.in_mmif))
+        app_views = [v for v in out_mmif.views if v.metadata.app == str(self.app.metadata.identifier)]
+        self.assertTrue(len(app_views) > 0)
+        first_timestamp = app_views[0].metadata.timestamp
+        for view in app_views[1:]:
+            self.assertEqual(first_timestamp, view.metadata.timestamp)
+        # second run
+        out_mmif2 = Mmif(self.app.annotate(out_mmif))
+        app_views2 = [v for v in out_mmif2.views if v.metadata.app == str(self.app.metadata.identifier)]
+        self.assertEqual(len(app_views2), len(app_views) * 2)
+        second_timestamp = app_views2[-1].metadata.timestamp
+        self.assertNotEqual(first_timestamp, second_timestamp)
+        for view in app_views2:
+            if view.id in [v.id for v in app_views]:
+                self.assertEqual(first_timestamp, view.metadata.timestamp)
+            else:
+                self.assertEqual(second_timestamp, view.metadata.timestamp)
+
     def test_annotate_returns_invalid_mmif(self):
         m = Mmif(self.in_mmif)
         v = m.new_view()