harmonize tag obfuscation handling, tag obfuscation plugin now builtin

Author: soad003

gsrest/__init__.py

@@ -14,6 +14,7 @@
 from gsrest.config import GSRestConfig, LoggingConfig
 from gsrest.dependencies import ConceptsCacheService, ServiceContainer
 from gsrest.plugins import get_subclass
+from gsrest.plugins.builtin.obfuscate_tags.obfuscate_tags import ObfuscateTags
 
 CONFIG_FILE = "./instance/config.yaml"
 
@@ -183,7 +184,26 @@ async def setup_services(app):
     app.app["plugin_contexts"] = {}
     app.app["request_config"] = {}
 
+    obfuscate_private_tags = any(
+        1 for name in config.plugins if name.endswith("obfuscate_tags")
+    )
+
+    if obfuscate_private_tags:
+        app.app.logger.warning(
+            "Tag obfuscation plugin enabled, using built-in version. Skipping load of external plugin."
+        )
+        builtinPlugin = ObfuscateTags
+        name = f"{builtinPlugin.__module__}"
+        app.app["plugins"].append(builtinPlugin)
+        app.app["plugin_contexts"][name] = {}
+        if hasattr(builtinPlugin, "setup"):
+            app.app.cleanup_ctx.append(plugin_setup(builtinPlugin, name))
+
     for name in config.plugins:
+        if name.endswith("obfuscate_tags"):
+            # already loaded builtin plugin
+            continue
+
         subcl = get_subclass(importlib.import_module(name))
         app.app["plugins"].append(subcl)
         app.app["plugin_contexts"][name] = {}

gsrest/dependencies.py

@@ -15,6 +15,10 @@
 from graphsenselib.tagstore.db import TagstoreDbAsync, Taxonomies
 
 from gsrest.config import GSRestConfig
+from gsrest.plugins.builtin.obfuscate_tags.obfuscate_tags import (
+    GROUPS_HEADER_NAME,
+    OBFUSCATION_MARKER_GROUP,
+)
 
 
 class ConceptsCacheService(ConceptProtocol):
@@ -168,3 +172,7 @@ def get_tagstore_access_groups(request):
 def get_username(request) -> Optional[str]:
     """Extract username from request, if available"""
     return request.headers.get("X-Consumer-Username", None)
+
+
+def should_obfuscate_private_tags(request) -> bool:
+    return request.headers.get(GROUPS_HEADER_NAME, "") == OBFUSCATION_MARKER_GROUP

gsrest/plugins/__init__.py

@@ -4,11 +4,13 @@
 
 
 class Plugin(abc.ABC):
-    @abc.abstractclassmethod
+    @classmethod
+    @abc.abstractmethod
     def before_request(cls, context, request: web.Request):
         return request
 
-    @abc.abstractclassmethod
+    @classmethod
+    @abc.abstractmethod
     def before_response(cls, context, request: web.Request, result):
         return
 

gsrest/plugins/builtin/__init__.py

@@ -0,0 +1,102 @@
+from aiohttp import web
+from openapi_server.models.entity import Entity
+from openapi_server.models.address_tags import AddressTags
+from openapi_server.models.neighbor_entities import NeighborEntities
+from multidict import CIMultiDict
+from openapi_server.models.search_result_level1 import SearchResultLevel1
+from openapi_server.models.search_result_level2 import SearchResultLevel2
+from openapi_server.models.search_result_level3 import SearchResultLevel3
+from openapi_server.models.search_result_level4 import SearchResultLevel4
+from openapi_server.models.search_result_level5 import SearchResultLevel5
+from openapi_server.models.search_result_level6 import SearchResultLevel6
+from openapi_server.models.search_result_leaf import SearchResultLeaf
+
+import re
+from gsrest.plugins import Plugin
+
+from graphsenselib.tagstore.algorithms.obfuscate import (
+    obfuscate_entity_actor,
+    obfuscate_tag_if_not_public,
+)
+
+GROUPS_HEADER_NAME = "X-Consumer-Groups"
+NO_OBFUSCATION_MARKER_GROUP = "private"
+OBFUSCATION_MARKER_GROUP = "obfuscate"
+
+
+class ObfuscateTags(Plugin):
+    @classmethod
+    def before_request(cls, context, request: web.Request):
+        groups = [
+            x.strip() for x in request.headers.get(GROUPS_HEADER_NAME, "").split(",")
+        ]
+
+        if NO_OBFUSCATION_MARKER_GROUP in groups:
+            return request
+        if "include_labels=true" in request.query_string.lower():
+            return request
+        if "/search" == request.path:
+            return request
+        if "/bulk" in request.path:
+            return request
+        if re.match(re.compile("/tags"), request.path):
+            return request
+        if re.match(re.compile("/[a-z]{3}/addresses/[^/]+$"), request.path):
+            # to avoid loading actors for address
+            return request
+
+        # setting group for request.
+        headers = dict(request.headers)
+        headers[GROUPS_HEADER_NAME] = OBFUSCATION_MARKER_GROUP
+        headers = CIMultiDict(**headers)
+        return request.clone(headers=headers)
+
+    @classmethod
+    def before_response(cls, context, request: web.Request, result):
+        # request.app.logger.debug(str(request.headers.get(GROUPS_HEADER_NAME, '')))
+        groups = [
+            x.strip() for x in request.headers.get(GROUPS_HEADER_NAME, "").split(",")
+        ]
+
+        if NO_OBFUSCATION_MARKER_GROUP in groups:
+            return
+        if isinstance(result, Entity):
+            cls.obfuscate(result.best_address_tag)
+            obfuscate_entity_actor(result)
+            return
+        if isinstance(result, AddressTags):
+            cls.obfuscate(result.address_tags)
+            return
+        if isinstance(result, NeighborEntities):
+            for neighbor in result.neighbors:
+                cls.obfuscate(neighbor.entity.best_address_tag)
+                obfuscate_entity_actor(neighbor.entity)
+        if (
+            isinstance(result, SearchResultLevel1)
+            or isinstance(result, SearchResultLevel2)
+            or isinstance(result, SearchResultLevel3)
+            or isinstance(result, SearchResultLevel4)
+            or isinstance(result, SearchResultLevel5)
+            or isinstance(result, SearchResultLevel6)
+            or isinstance(result, SearchResultLeaf)
+        ):
+            if result.neighbor:
+                cls.obfuscate(result.neighbor.entity.best_address_tag)
+            if not isinstance(result, SearchResultLeaf) and result.paths:
+                for path in result.paths:
+                    cls.before_response(context, request, path)
+            return
+        if isinstance(result, list):
+            for r in result:
+                cls.before_response(context, request, r)
+            return
+
+    @classmethod
+    def obfuscate(cls, tags):
+        if not tags:
+            return
+        if isinstance(tags, list):
+            for tag in tags:
+                obfuscate_tag_if_not_public(tag)
+        else:
+            obfuscate_tag_if_not_public(tags)

gsrest/plugins/builtin/obfuscate_tags/__init__.py

@@ -1,11 +1,13 @@
 from typing import Optional
 
 from graphsenselib.errors import BadUserInputException
+from graphsenselib.tagstore.algorithms.obfuscate import obfuscate_tag_if_not_public
 
 from gsrest.dependencies import (
     get_request_cache,
     get_service_container,
     get_tagstore_access_groups,
+    should_obfuscate_private_tags,
 )
 from gsrest.service import parse_page_int_optional
 from gsrest.translators import (
@@ -57,6 +59,11 @@ async def get_tag_summary_by_address(
         include_best_cluster_tag,
         include_pubkey_derived_tags=include_pubkey_derived_tags,
         only_propagate_high_confidence_actors=tag_summary_only_propagate_high_confidence_actors,
+        tag_transformer=(
+            None
+            if not should_obfuscate_private_tags(request)
+            else obfuscate_tag_if_not_public
+        ),
     )
 
     return pydantic_to_openapi(pydantic_result)

gsrest/plugins/builtin/obfuscate_tags/obfuscate_tags.py

@@ -94,7 +94,7 @@ exclude = [
 extend-select = ["T201"]
 
 [tool.uv.sources]
-graphsense-lib = { git = "https://github.com/graphsense/graphsense-lib", rev = "v2.8.0-dev11" }
+graphsense-lib = { git = "https://github.com/graphsense/graphsense-lib", rev = "v2.8.0-rc5" }
 
 [dependency-groups]
 dev = [