diff --git a/packages/react-native-gesture-handler/android/src/main/java/com/swmansion/gesturehandler/react/RNGestureHandlerDetectorView.kt b/packages/react-native-gesture-handler/android/src/main/java/com/swmansion/gesturehandler/react/RNGestureHandlerDetectorView.kt
index 0fa3b16dd8..fb61aaaa69 100644
--- a/packages/react-native-gesture-handler/android/src/main/java/com/swmansion/gesturehandler/react/RNGestureHandlerDetectorView.kt
+++ b/packages/react-native-gesture-handler/android/src/main/java/com/swmansion/gesturehandler/react/RNGestureHandlerDetectorView.kt
@@ -90,7 +90,7 @@ class RNGestureHandlerDetectorView(context: Context) : ReactViewGroup(context) {
     for (tag in newHandlers) {
       handlersToDetach.remove(tag)
       if (!attachedHandlers.contains(tag)) {
-        if (shouldAttachGestureToChildView(tag)) {
+        if (shouldAttachGestureToChildView(tag) && actionType == GestureHandler.ACTION_TYPE_NATIVE_DETECTOR) {
           // It might happen that `attachHandlers` will be called before children are added into view hierarchy. In that case we cannot
           // attach `NativeViewGestureHandlers` here and we have to do it in `addView` method.
           nativeHandlers.add(tag)
@@ -121,12 +121,25 @@ class RNGestureHandlerDetectorView(context: Context) : ReactViewGroup(context) {
   private fun attachVirtualChildren(virtualChildrenToAttach: List<VirtualChildren>) {
     val virtualChildrenToDetach = attachedVirtualHandlers.keys.toMutableSet()
 
+    for (child in virtualChildrenToAttach) {
+      virtualChildrenToDetach.remove(child.viewTag)
+    }
+
+    val registry = RNGestureHandlerModule.registries[moduleId]
+      ?: throw Exception("Tried to access a non-existent registry")
+
+    for (child in virtualChildrenToDetach) {
+      for (tag in attachedVirtualHandlers[child]!!) {
+        registry.detachHandler(tag)
+      }
+      attachedVirtualHandlers.remove(tag)
+    }
+
     for (child in virtualChildrenToAttach) {
       if (!attachedVirtualHandlers.containsKey(child.viewTag)) {
         attachedVirtualHandlers[child.viewTag] = mutableSetOf()
       }
 
-      virtualChildrenToDetach.remove(child.viewTag)
       attachHandlers(
         child.handlerTags,
         child.viewTag,
@@ -134,14 +147,6 @@ class RNGestureHandlerDetectorView(context: Context) : ReactViewGroup(context) {
         attachedVirtualHandlers[child.viewTag]!!,
       )
     }
-
-    val registry = RNGestureHandlerModule.registries[moduleId]
-      ?: throw Exception("Tried to access a non-existent registry")
-
-    for (tag in virtualChildrenToDetach) {
-      registry.detachHandler(tag)
-      attachedVirtualHandlers.remove(tag)
-    }
   }
 
   private fun tryAttachNativeHandlersToChildView(childId: Int) {
diff --git a/packages/react-native-gesture-handler/apple/RNGestureHandlerDetector.mm b/packages/react-native-gesture-handler/apple/RNGestureHandlerDetector.mm
index ce3cd3bb97..ecc4a79322 100644
--- a/packages/react-native-gesture-handler/apple/RNGestureHandlerDetector.mm
+++ b/packages/react-native-gesture-handler/apple/RNGestureHandlerDetector.mm
@@ -171,7 +171,7 @@ - (void)attachHandlers:(const std::vector<int> &)handlerTags
   for (const int tag : handlerTags) {
     [handlersToDetach removeObject:@(tag)];
     if (![attachedHandlers containsObject:@(tag)]) {
-      if ([self shouldAttachGestureToSubview:@(tag)]) {
+      if ([self shouldAttachGestureToSubview:@(tag)] && actionType == RNGestureHandlerActionTypeNativeDetector) {
         // It might happen that `attachHandlers` will be called before children are added into view hierarchy. In that
         // case we cannot attach `NativeViewGestureHandlers` here and we have to do it in `didAddSubview` method.
         [_nativeHandlers addObject:@(tag)];
@@ -236,22 +236,24 @@ - (void)updateVirtualChildren:(const std::vector<RNGestureHandlerDetectorVirtual
   }
 
   for (const auto &child : virtualChildren) {
-    if (_attachedVirtualHandlers.find(child.viewTag) == _attachedVirtualHandlers.end()) {
-      _attachedVirtualHandlers[child.viewTag] = [NSMutableSet set];
-    }
-
     [virtualChildrenToDetach removeObject:@(child.viewTag)];
-
-    [self attachHandlers:child.handlerTags
-              actionType:RNGestureHandlerActionTypeVirtualDetector
-                 viewTag:child.viewTag
-        attachedHandlers:_attachedVirtualHandlers[child.viewTag]];
   }
 
   for (const NSNumber *tag : virtualChildrenToDetach) {
     for (id handlerTag : _attachedVirtualHandlers[tag.intValue]) {
       [handlerManager.registry detachHandlerWithTag:handlerTag];
     }
+    _attachedVirtualHandlers.erase(tag.intValue);
+  }
+
+  for (const auto &child : virtualChildren) {
+    if (_attachedVirtualHandlers.find(child.viewTag) == _attachedVirtualHandlers.end()) {
+      _attachedVirtualHandlers[child.viewTag] = [NSMutableSet set];
+    }
+    [self attachHandlers:child.handlerTags
+              actionType:RNGestureHandlerActionTypeVirtualDetector
+                 viewTag:child.viewTag
+        attachedHandlers:_attachedVirtualHandlers[child.viewTag]];
   }
 }
 
diff --git a/packages/react-native-gesture-handler/src/v3/detectors/HostGestureDetector.web.tsx b/packages/react-native-gesture-handler/src/v3/detectors/HostGestureDetector.web.tsx
index 68d4f41fbe..27542e4ec3 100644
--- a/packages/react-native-gesture-handler/src/v3/detectors/HostGestureDetector.web.tsx
+++ b/packages/react-native-gesture-handler/src/v3/detectors/HostGestureDetector.web.tsx
@@ -53,7 +53,8 @@ const HostGestureDetector = (props: GestureHandlerDetectorProps) => {
       if (
         RNGestureHandlerModule.getGestureHandlerNode(
           tag
-        ).shouldAttachGestureToChildView()
+        ).shouldAttachGestureToChildView() &&
+        actionType === ActionType.NATIVE_DETECTOR
       ) {
         RNGestureHandlerModule.attachGestureHandler(
           tag,
@@ -109,6 +110,14 @@ const HostGestureDetector = (props: GestureHandlerDetectorProps) => {
       attachedVirtualHandlers.current.keys()
     );
 
+    props.virtualChildren?.forEach((child) => {
+      virtualChildrenToDetach.delete(child.viewTag);
+    });
+
+    virtualChildrenToDetach.forEach((tag) => {
+      detachHandlers(EMPTY_HANDLERS, attachedVirtualHandlers.current.get(tag)!);
+    });
+
     props.virtualChildren?.forEach((child) => {
       if (child.viewRef.current == null) {
         // We must check whether viewRef is  not null as otherwise we get an error when intercepting gesture detector
@@ -118,7 +127,6 @@ const HostGestureDetector = (props: GestureHandlerDetectorProps) => {
       if (!attachedVirtualHandlers.current.has(child.viewTag)) {
         attachedVirtualHandlers.current.set(child.viewTag, new Set());
       }
-      virtualChildrenToDetach.delete(child.viewTag);
 
       const currentHandlerTags = new Set(child.handlerTags);
       detachHandlers(
@@ -134,10 +142,6 @@ const HostGestureDetector = (props: GestureHandlerDetectorProps) => {
         ActionType.VIRTUAL_DETECTOR
       );
     });
-
-    virtualChildrenToDetach.forEach((tag) => {
-      detachHandlers(EMPTY_HANDLERS, attachedVirtualHandlers.current.get(tag)!);
-    });
   }, [props.virtualChildren]);
 
   return (
diff --git a/packages/react-native-gesture-handler/src/v3/detectors/VirtualDetector/VirtualDetector.tsx b/packages/react-native-gesture-handler/src/v3/detectors/VirtualDetector/VirtualDetector.tsx
index ba1b9b68a8..7e7e486ecb 100644
--- a/packages/react-native-gesture-handler/src/v3/detectors/VirtualDetector/VirtualDetector.tsx
+++ b/packages/react-native-gesture-handler/src/v3/detectors/VirtualDetector/VirtualDetector.tsx
@@ -23,22 +23,34 @@ export function VirtualDetector<THandlerData, TConfig>(
 
   const viewRef = useRef(null);
   const [viewTag, setViewTag] = useState<number>(-1);
+
   const virtualMethods = useRef(props.gesture.detectorCallbacks);
-  const handleRef = useCallback((node: any) => {
-    viewRef.current = node;
-    if (!node) {
-      return;
-    }
 
-    if (Platform.OS === 'web') {
-      setViewTag(node);
-    } else {
-      const tag = findNodeHandle(node);
+  const handleRef = useCallback(
+    (node: any) => {
+      viewRef.current = node;
+      if (!node) {
+        return;
+      }
+
+      const tag = Platform.OS === 'web' ? node : findNodeHandle(node);
+
       if (tag != null) {
         setViewTag(tag);
       }
-    }
-  }, []);
+
+      return () => {
+        if (tag != null) {
+          const handlerTags = isComposedGesture(props.gesture)
+            ? props.gesture.tags
+            : [props.gesture.tag];
+
+          unregister(tag, handlerTags);
+        }
+      };
+    },
+    [props.children]
+  );
 
   useEffect(() => {
     virtualMethods.current = props.gesture.detectorCallbacks;