better search endpoint

soad003 · soad003 · commit a7ee5f8779c0 · 2025-11-05T12:11:32.000+01:00
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -3,6 +3,14 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
+## [25.11.0/1.5.0] - 2025-11-07
+### changed
+- harmonize tagpack obfuscation handling
+- better search features (search with postfixes), scoped search that allows to enable/disable features.
+- stricter actor propagation handling
+- tags are now derived from addresses with the same pubkey (if feature is enabled)
+
+
 ## [25.09.4/1.14.4] - 2025-10-03
 ### fixed
 - improvements thorchain handling on unknown networks (gslib 2.7.7)
diff --git a/Makefile b/Makefile
@@ -2,8 +2,8 @@ all: format lint
 
 -include .env
 
-GS_REST_SERVICE_VERSIONM ?= "25.11.0-rc2"
-GS_REST_SERVICE_VERSION ?= "1.15.0-rc2"
+GS_REST_SERVICE_VERSIONM ?= "25.11.0-rc3"
+GS_REST_SERVICE_VERSION ?= "1.15.0-rc3"
 
 GS_REST_DEV_PORT ?= 9000
 NUM_WORKERS ?= 1
diff --git a/clients/python/README.md b/clients/python/README.md
@@ -3,8 +3,8 @@ GraphSense API provides programmatic access to various ledgers' addresses, entit
 
 This Python package is automatically generated by the [OpenAPI Generator](https://openapi-generator.tech) project:
 
-- API version: 1.15.0-rc2
-- Package version: 1.15.0-rc2
+- API version: 1.15.0-rc3
+- Package version: 1.15.0-rc3
 - Build package: org.openapitools.codegen.languages.PythonClientCodegen
 
 ## Requirements.
diff --git a/clients/python/docs/GeneralApi.md b/clients/python/docs/GeneralApi.md
@@ -115,7 +115,11 @@ with graphsense.ApiClient(configuration) as api_client:
     q = "foo" # str | It can be (the beginning of) an address, a transaction or a label
     currency = "btc" # str | The cryptocurrency (e.g., btc) (optional)
     limit = 10 # int | Maximum number of search results (optional) if omitted the server will use the default value of 10
-    include_sub_tx_identifiers = True # bool | Whether to include sub-transaction identifiers (optional) if omitted the server will use the default value of False
+    include_sub_tx_identifiers = False # bool | Whether to include sub-transaction identifiers (optional) if omitted the server will use the default value of False
+    include_labels = True # bool | Whether to include labels (optional) if omitted the server will use the default value of True
+    include_actors = True # bool | Whether to include actors (optional) if omitted the server will use the default value of True
+    include_txs = True # bool | Whether to include transactions (optional) if omitted the server will use the default value of True
+    include_addresses = True # bool | Whether to include addresses (optional) if omitted the server will use the default value of True
 
     # example passing only required values which don't have defaults set
     try:
@@ -129,7 +133,7 @@ with graphsense.ApiClient(configuration) as api_client:
     # and optional values
     try:
         # Returns matching addresses, transactions and labels
-        api_response = api_instance.search(q, currency=currency, limit=limit, include_sub_tx_identifiers=include_sub_tx_identifiers)
+        api_response = api_instance.search(q, currency=currency, limit=limit, include_sub_tx_identifiers=include_sub_tx_identifiers, include_labels=include_labels, include_actors=include_actors, include_txs=include_txs, include_addresses=include_addresses)
         pprint(api_response)
     except graphsense.ApiException as e:
         print("Exception when calling GeneralApi->search: %s\n" % e)
@@ -144,6 +148,10 @@ Name | Type | Description  | Notes
  **currency** | **str**| The cryptocurrency (e.g., btc) | [optional]
  **limit** | **int**| Maximum number of search results | [optional] if omitted the server will use the default value of 10
  **include_sub_tx_identifiers** | **bool**| Whether to include sub-transaction identifiers | [optional] if omitted the server will use the default value of False
+ **include_labels** | **bool**| Whether to include labels | [optional] if omitted the server will use the default value of True
+ **include_actors** | **bool**| Whether to include actors | [optional] if omitted the server will use the default value of True
+ **include_txs** | **bool**| Whether to include transactions | [optional] if omitted the server will use the default value of True
+ **include_addresses** | **bool**| Whether to include addresses | [optional] if omitted the server will use the default value of True
 **_preload_content** | **bool** | If False, the urllib3.HTTPResponse object will be returned without reading/decoding response data. | [optional] default is True.
 **async_req** | **bool** | Execute request asynchronously | [optional] default is False.
 
diff --git a/clients/python/graphsense/__init__.py b/clients/python/graphsense/__init__.py
@@ -10,7 +10,7 @@
 """
 
 
-__version__ = "1.15.0-rc2"
+__version__ = "1.15.0-rc3"
 
 # import ApiClient
 from graphsense.api_client import ApiClient
diff --git a/clients/python/graphsense/api/general_api.py b/clients/python/graphsense/api/general_api.py
@@ -161,6 +161,10 @@ def __search(
                 currency (str): The cryptocurrency (e.g., btc). [optional]
                 limit (int): Maximum number of search results. [optional] if omitted the server will use the default value of 10
                 include_sub_tx_identifiers (bool): Whether to include sub-transaction identifiers. [optional] if omitted the server will use the default value of False
+                include_labels (bool): Whether to include labels. [optional] if omitted the server will use the default value of True
+                include_actors (bool): Whether to include actors. [optional] if omitted the server will use the default value of True
+                include_txs (bool): Whether to include transactions. [optional] if omitted the server will use the default value of True
+                include_addresses (bool): Whether to include addresses. [optional] if omitted the server will use the default value of True
                 _return_http_data_only (bool): response data without head status
                     code and headers. Default is True.
                 _preload_content (bool): if False, the urllib3.HTTPResponse object
@@ -226,6 +230,10 @@ def __search(
                     'currency',
                     'limit',
                     'include_sub_tx_identifiers',
+                    'include_labels',
+                    'include_actors',
+                    'include_txs',
+                    'include_addresses',
                 ],
                 'required': [
                     'q',
@@ -261,18 +269,34 @@ def __search(
                         (int,),
                     'include_sub_tx_identifiers':
                         (bool,),
+                    'include_labels':
+                        (bool,),
+                    'include_actors':
+                        (bool,),
+                    'include_txs':
+                        (bool,),
+                    'include_addresses':
+                        (bool,),
                 },
                 'attribute_map': {
                     'q': 'q',
                     'currency': 'currency',
                     'limit': 'limit',
                     'include_sub_tx_identifiers': 'include_sub_tx_identifiers',
+                    'include_labels': 'include_labels',
+                    'include_actors': 'include_actors',
+                    'include_txs': 'include_txs',
+                    'include_addresses': 'include_addresses',
                 },
                 'location_map': {
                     'q': 'query',
                     'currency': 'query',
                     'limit': 'query',
                     'include_sub_tx_identifiers': 'query',
+                    'include_labels': 'query',
+                    'include_actors': 'query',
+                    'include_txs': 'query',
+                    'include_addresses': 'query',
                 },
                 'collection_format_map': {
                 }
diff --git a/clients/python/graphsense/api_client.py b/clients/python/graphsense/api_client.py
@@ -76,7 +76,7 @@ def __init__(self, configuration=None, header_name=None, header_value=None,
             self.default_headers[header_name] = header_value
         self.cookie = cookie
         # Set default User-Agent.
-        self.user_agent = 'OpenAPI-Generator/1.15.0-rc2/python'
+        self.user_agent = 'OpenAPI-Generator/1.15.0-rc3/python'
 
     def __enter__(self):
         return self
diff --git a/clients/python/graphsense/configuration.py b/clients/python/graphsense/configuration.py
@@ -405,8 +405,8 @@ def to_debug_report(self):
         return "Python SDK Debug Report:\n"\
                "OS: {env}\n"\
                "Python Version: {pyversion}\n"\
-               "Version of the API: 1.15.0-rc2\n"\
-               "SDK Package Version: 1.15.0-rc2".\
+               "Version of the API: 1.15.0-rc3\n"\
+               "SDK Package Version: 1.15.0-rc3".\
                format(env=sys.platform, pyversion=sys.version)
 
     def get_host_settings(self):
diff --git a/clients/python/pyproject.toml b/clients/python/pyproject.toml
@@ -1,6 +1,6 @@
 [project]
 name = "graphsense-python"
-version = "1.15.0-rc2"
+version = "1.15.0-rc3"
 description = "GraphSense API"
 readme = { file = "README.md", content-type = "text/markdown; charset=UTF-8; variant=GFM" }
 requires-python = ">=3.6"
diff --git a/gsrest/__init__.py b/gsrest/__init__.py
@@ -11,10 +11,10 @@
 from graphsenselib.utils.slack import SlackLogHandler
 
 import gsrest.db
+from gsrest.builtin.plugins.obfuscate_tags.obfuscate_tags import ObfuscateTags
 from gsrest.config import GSRestConfig, LoggingConfig
 from gsrest.dependencies import ConceptsCacheService, ServiceContainer
 from gsrest.plugins import get_subclass
-from gsrest.builtin.plugins.obfuscate_tags.obfuscate_tags import ObfuscateTags
 
 CONFIG_FILE = "./instance/config.yaml"
 
@@ -195,7 +195,9 @@ async def setup_services(app):
         builtinPlugin = ObfuscateTags
         name = f"{builtinPlugin.__module__}"
         app.app["plugins"].append(builtinPlugin)
-        app.app["plugin_contexts"][name] = {}
+        app.app["plugin_contexts"][name] = {
+            "config": app.app["config"].get_plugin_config(name)
+        }
         if hasattr(builtinPlugin, "setup"):
             app.app.cleanup_ctx.append(plugin_setup(builtinPlugin, name))
 
diff --git a/gsrest/builtin/plugins/obfuscate_tags/obfuscate_tags.py b/gsrest/builtin/plugins/obfuscate_tags/obfuscate_tags.py
@@ -1,29 +1,53 @@
+import re
+
 from aiohttp import web
-from openapi_server.models.entity import Entity
+from graphsenselib.tagstore.algorithms.obfuscate import (
+    obfuscate_entity_actor,
+    obfuscate_tag_if_not_public,
+)
+from multidict import CIMultiDict
+
+from gsrest.plugins import Plugin
 from openapi_server.models.address_tags import AddressTags
+from openapi_server.models.entity import Entity
 from openapi_server.models.neighbor_entities import NeighborEntities
-from multidict import CIMultiDict
+from openapi_server.models.search_result_leaf import SearchResultLeaf
 from openapi_server.models.search_result_level1 import SearchResultLevel1
 from openapi_server.models.search_result_level2 import SearchResultLevel2
 from openapi_server.models.search_result_level3 import SearchResultLevel3
 from openapi_server.models.search_result_level4 import SearchResultLevel4
 from openapi_server.models.search_result_level5 import SearchResultLevel5
 from openapi_server.models.search_result_level6 import SearchResultLevel6
-from openapi_server.models.search_result_leaf import SearchResultLeaf
-
-import re
-from gsrest.plugins import Plugin
-
-from graphsenselib.tagstore.algorithms.obfuscate import (
-    obfuscate_entity_actor,
-    obfuscate_tag_if_not_public,
-)
+from functools import partial
 
 GROUPS_HEADER_NAME = "X-Consumer-Groups"
 NO_OBFUSCATION_MARKER_GROUP = "private"
 OBFUSCATION_MARKER_GROUP = "obfuscate"
 
 
+def obfuscate_tagpack_uri_by_rule(rule, tags):
+    if not tags:
+        return
+    if isinstance(tags, list):
+        for tag in tags:
+            obfuscate_tagpack_uri_by_rule(rule, tag)
+    else:
+        # use regex in rule to check if uri needs to be redacted
+        pattern = re.compile(rule)
+        if pattern.match(tags.tagpack_uri):
+            tags.tagpack_uri = ""
+
+
+def obfuscate_private_tags(tags):
+    if not tags:
+        return
+    if isinstance(tags, list):
+        for tag in tags:
+            obfuscate_tag_if_not_public(tag)
+    else:
+        obfuscate_tag_if_not_public(tags)
+
+
 class ObfuscateTags(Plugin):
     @classmethod
     def before_request(cls, context, request: web.Request):
@@ -58,18 +82,38 @@ def before_response(cls, context, request: web.Request, result):
             x.strip() for x in request.headers.get(GROUPS_HEADER_NAME, "").split(",")
         ]
 
+        obfuscate_tagpack_uri_rule = context.get("config", {}).get(
+            "obfuscate_tagpack_uri_rule", None
+        )
+
+        if obfuscate_tagpack_uri_rule is not None:
+            cls.obfuscate_tags_in_objects(
+                context,
+                request,
+                result,
+                partial(obfuscate_tagpack_uri_by_rule, obfuscate_tagpack_uri_rule),
+            )
+
         if NO_OBFUSCATION_MARKER_GROUP in groups:
             return
+
+        else:
+            cls.obfuscate_tags_in_objects(
+                context, request, result, obfuscate_private_tags
+            )
+
+    @classmethod
+    def obfuscate_tags_in_objects(cls, context, request, result, tag_obfuscation_func):
         if isinstance(result, Entity):
-            cls.obfuscate(result.best_address_tag)
+            tag_obfuscation_func(result.best_address_tag)
             obfuscate_entity_actor(result)
             return
         if isinstance(result, AddressTags):
-            cls.obfuscate(result.address_tags)
+            tag_obfuscation_func(result.address_tags)
             return
         if isinstance(result, NeighborEntities):
             for neighbor in result.neighbors:
-                cls.obfuscate(neighbor.entity.best_address_tag)
+                tag_obfuscation_func(neighbor.entity.best_address_tag)
                 obfuscate_entity_actor(neighbor.entity)
         if (
             isinstance(result, SearchResultLevel1)
@@ -81,7 +125,7 @@ def before_response(cls, context, request: web.Request, result):
             or isinstance(result, SearchResultLeaf)
         ):
             if result.neighbor:
-                cls.obfuscate(result.neighbor.entity.best_address_tag)
+                tag_obfuscation_func(result.neighbor.entity.best_address_tag)
             if not isinstance(result, SearchResultLeaf) and result.paths:
                 for path in result.paths:
                     cls.before_response(context, request, path)
@@ -90,13 +134,3 @@ def before_response(cls, context, request: web.Request, result):
             for r in result:
                 cls.before_response(context, request, r)
             return
-
-    @classmethod
-    def obfuscate(cls, tags):
-        if not tags:
-            return
-        if isinstance(tags, list):
-            for tag in tags:
-                obfuscate_tag_if_not_public(tag)
-        else:
-            obfuscate_tag_if_not_public(tags)
diff --git a/gsrest/dependencies.py b/gsrest/dependencies.py
@@ -14,11 +14,11 @@
 from graphsenselib.db.asynchronous.services.txs_service import TxsService
 from graphsenselib.tagstore.db import TagstoreDbAsync, Taxonomies
 
-from gsrest.config import GSRestConfig
 from gsrest.builtin.plugins.obfuscate_tags.obfuscate_tags import (
     GROUPS_HEADER_NAME,
     OBFUSCATION_MARKER_GROUP,
 )
+from gsrest.config import GSRestConfig
 
 
 class ConceptsCacheService(ConceptProtocol):
diff --git a/gsrest/service/general_service.py b/gsrest/service/general_service.py
@@ -5,6 +5,8 @@
     pydantic_stats_to_openapi,
 )
 
+from graphsenselib.db.asynchronous.services.models import SearchRequestConfig
+
 
 async def get_statistics(request):
     """Returns summary statistics on all available currencies"""
@@ -26,16 +28,34 @@ async def search_by_currency(request, currency, q, limit=10):
     return pydantic_search_result_by_currency_to_openapi(pydantic_result)
 
 
-async def search(request, q, currency=None, limit=10, include_sub_tx_identifiers=False):
+async def search(
+    request,
+    q,
+    currency=None,
+    limit=10,
+    include_sub_tx_identifiers=False,
+    include_labels=True,
+    include_actors=True,
+    include_txs=True,
+    include_addresses=True,
+):
     services = get_service_container(request)
     tagstore_groups = get_tagstore_access_groups(request)
 
+    search_config = SearchRequestConfig(
+        include_sub_tx_identifiers=include_sub_tx_identifiers,
+        include_labels=include_labels,
+        include_actors=include_actors,
+        include_txs=include_txs,
+        include_addresses=include_addresses,
+    )
+
     pydantic_result = await services.general_service.search(
         q,
         tagstore_groups,
         currency,
         limit,
-        include_sub_tx_identifiers=include_sub_tx_identifiers,
+        config=search_config,
     )
 
     return pydantic_search_result_to_openapi(pydantic_result)
diff --git a/openapi_server/controllers/general_controller.py b/openapi_server/controllers/general_controller.py
diff --git a/openapi_server/openapi/openapi.yaml b/openapi_server/openapi/openapi.yaml
diff --git a/openapi_spec/graphsense.yaml b/openapi_spec/graphsense.yaml
diff --git a/pyproject.toml b/pyproject.toml
diff --git a/uv.lock b/uv.lock
