[uss_qualifier] Record failled queries (#1298)

Author: the-glu

.basedpyright/baseline.json

@@ -2576,78 +2576,6 @@
                     "endColumn": 67,
                     "lineCount": 1
                 }
-            },
-            {
-                "code": "reportAssignmentType",
-                "range": {
-                    "startColumn": 17,
-                    "endColumn": 35,
-                    "lineCount": 1
-                }
-            },
-            {
-                "code": "reportOptionalMemberAccess",
-                "range": {
-                    "startColumn": 23,
-                    "endColumn": 30,
-                    "lineCount": 1
-                }
-            },
-            {
-                "code": "reportOptionalMemberAccess",
-                "range": {
-                    "startColumn": 28,
-                    "endColumn": 49,
-                    "lineCount": 1
-                }
-            },
-            {
-                "code": "reportOptionalMemberAccess",
-                "range": {
-                    "startColumn": 25,
-                    "endColumn": 40,
-                    "lineCount": 1
-                }
-            },
-            {
-                "code": "reportPossiblyUnboundVariable",
-                "range": {
-                    "startColumn": 46,
-                    "endColumn": 48,
-                    "lineCount": 1
-                }
-            },
-            {
-                "code": "reportOperatorIssue",
-                "range": {
-                    "startColumn": 22,
-                    "endColumn": 31,
-                    "lineCount": 1
-                }
-            },
-            {
-                "code": "reportPossiblyUnboundVariable",
-                "range": {
-                    "startColumn": 23,
-                    "endColumn": 25,
-                    "lineCount": 1
-                }
-            },
-            {
-                "code": "reportPossiblyUnboundVariable",
-                "range": {
-                    "startColumn": 28,
-                    "endColumn": 30,
-                    "lineCount": 1
-                }
-            },
-            {
-                "code": "reportPossiblyUnboundVariable",
-                "range": {
-                    "startColumn": 41,
-                    "endColumn": 43,
-                    "lineCount": 1
-                }
             }
         ],
         "./monitoring/monitorlib/fetch/evaluation.py": [

monitoring/mock_uss/versioning/routes.py

@@ -1,11 +1,12 @@
+import copy
 import datetime
 import json
 import os
 import traceback
 import uuid
 from dataclasses import dataclass
 from enum import Enum
-from typing import TypeVar
+from typing import Self, TypeVar
 from urllib.parse import urlparse
 
 import flask
@@ -458,6 +459,9 @@ class Query(ImplicitDict):
     query_type: QueryType | None
     """If specified, the recognized type of this query."""
 
+    _previous_query: Self | None
+    """If specified, the previous, failling query that generated this query as a retry"""
+
     @property
     def timestamp(self) -> datetime.datetime:
         """Safety property to prevent crashes when Query.timestamp is accessed.
@@ -579,10 +583,12 @@ def describe_query(
     initiated_at: datetime.datetime,
     query_type: QueryType | None = None,
     participant_id: str | None = None,
+    previous_query: Query | None = None,
 ) -> Query:
     query = Query(
         request=describe_request(resp.request, initiated_at),
         response=describe_response(resp),
+        _previous_query=previous_query,
     )
     if query_type is not None:
         query.query_type = query_type
@@ -618,10 +624,9 @@ def query_and_describe(
         Query object describing the request and response/result.
     """
     if client is None:
-        utm_session = False
-        client = requests.session()
+        _client = requests.session()
     else:
-        utm_session = True
+        _client = client
     req_kwargs = kwargs.copy()
     if "timeout" not in req_kwargs:
         req_kwargs["timeout"] = (
@@ -655,6 +660,36 @@ def get_location() -> str:
             .strip()
         )
 
+    previous_query = None
+
+    def build_failing_query(t0) -> Query:
+        _req_kwargs = copy.deepcopy(req_kwargs)
+
+        if isinstance(_client, infrastructure.UTMClientSession):
+            _req_kwargs = _client.adjust_request_kwargs(_req_kwargs)
+        del _req_kwargs["timeout"]
+
+        req = requests.Request(verb, url, **_req_kwargs)
+        prepped_req = _client.prepare_request(req)
+
+        t1 = datetime.datetime.now(datetime.UTC)
+
+        query = Query(
+            request=describe_request(prepped_req, t0),
+            response=ResponseDescription(
+                code=None,
+                failure="\n".join(failures),
+                elapsed_s=(t1 - t0).total_seconds(),
+                reported=StringBasedDateTime(t1),
+            ),
+            participant_id=participant_id,
+            _previous_query=previous_query,
+        )
+        if query_type is not None:
+            query.query_type = query_type
+
+        return query
+
     # Note: retry logic could be attached to the `client` Session by `mount`ing an HTTPAdapter with custom
     # `max_retries`, however we do not want to mutate the provided Session.  Instead, retry only on errors we explicitly
     # consider retryable.
@@ -664,13 +699,14 @@ def get_location() -> str:
             if is_netloc_fake:
                 failure_message = f"query_and_describe attempt {attempt + 1} from PID {os.getpid()} to {verb} {url} was not attempted because network location of {url} was identified as fake: {settings.fake_netlocs}\nAt {get_location()}"
                 failures.append(failure_message)
-                break
+                return build_failing_query(t0)
 
             return describe_query(
-                client.request(verb, url, **req_kwargs),
+                _client.request(verb, url, **req_kwargs),
                 t0,
                 query_type=query_type,
                 participant_id=participant_id,
+                previous_query=previous_query,
             )
         except (requests.Timeout, urllib3.exceptions.ReadTimeoutError) as e:
             failure_message = f"query_and_describe attempt {attempt + 1} from PID {os.getpid()} to {verb} {url} failed with timeout {type(e).__name__}: {str(e)}\nAt {get_location()}"
@@ -689,38 +725,28 @@ def get_location() -> str:
             if not expect_failure:
                 logger.warning(failure_message)
             failures.append(failure_message)
+
             if not retryable:
-                break
+                return build_failing_query(t0)
+
         except requests.RequestException as e:
             failure_message = f"query_and_describe attempt {attempt + 1} from PID {os.getpid()} to {verb} {url} failed with non-retryable RequestException {type(e).__name__}: {str(e)}\nAt {get_location()}"
             if not expect_failure:
                 logger.warning(failure_message)
             failures.append(failure_message)
 
-            break
-        finally:
-            t1 = datetime.datetime.now(datetime.UTC)
-
-    # Reconstruct request similar to the one in the query (which is not
-    # accessible at this point)
-    if utm_session:
-        req_kwargs = client.adjust_request_kwargs(req_kwargs)
-    del req_kwargs["timeout"]
-    req = requests.Request(verb, url, **req_kwargs)
-    prepped_req = client.prepare_request(req)
-    result = Query(
-        request=describe_request(prepped_req, t0),
-        response=ResponseDescription(
-            code=None,
-            failure="\n".join(failures),
-            elapsed_s=(t1 - t0).total_seconds(),
-            reported=StringBasedDateTime(t1),
-        ),
-        participant_id=participant_id,
-    )
-    if query_type is not None:
-        result.query_type = query_type
-    return result
+            return build_failing_query(t0)
+
+        previous_query = build_failing_query(
+            t0
+        )  # If we arrive there, query failled, but is retriable
+
+    if not previous_query:
+        raise Exception(
+            "Internal error: arrived after retried without any expected failled query"
+        )
+
+    return previous_query  # Previous query is the last failled one
 
 
 def describe_flask_query(

monitoring/monitorlib/clients/versioning/client_interuss.py

@@ -423,6 +423,11 @@ def record_queries(self, queries: list[fetch.Query]) -> None:
 
     def record_query(self, query: fetch.Query) -> None:
         self._expect_phase({ScenarioPhase.RunningTestStep, ScenarioPhase.CleaningUp})
+
+        # If the query has a previous one, record it first
+        if "_previous_query" in query and query._previous_query:
+            self.record_query(query._previous_query)
+
         if "queries" not in self._step_report:
             self._step_report.queries = []
         for existing_query in self._step_report.queries:

monitoring/monitorlib/fetch/__init__.py

@@ -4,7 +4,7 @@
 import json
 import os
 import sys
-from typing import get_args, get_origin, get_type_hints
+from typing import Self, get_args, get_origin, get_type_hints
 
 import implicitdict
 import implicitdict.jsonschema
@@ -78,7 +78,8 @@ def _make_type_schemas(
                 pending_types.extend(get_args(pending_type))
             else:
                 if (
-                    issubclass(pending_type, ImplicitDict)
+                    pending_type != Self
+                    and issubclass(pending_type, ImplicitDict)
                     and fullname(pending_type) not in already_checked
                 ):
                     _make_type_schemas(