From a603a63f823905d76d50f5e004f0411326bf69f5 Mon Sep 17 00:00:00 2001 From: devendra-shardeum Date: Tue, 19 Aug 2025 21:19:34 +0530 Subject: [PATCH 1/7] patches with added logs for fact* algo fixes --- src/state-manager/CachedAppDataManager.ts | 3 +- src/state-manager/TransactionQueue.ts | 14 +- src/utils/fastAggregatedCorrespondingTell.ts | 173 ++++++++----------- 3 files changed, 86 insertions(+), 104 deletions(-) diff --git a/src/state-manager/CachedAppDataManager.ts b/src/state-manager/CachedAppDataManager.ts index fa2ee3c8f..737f716d1 100644 --- a/src/state-manager/CachedAppDataManager.ts +++ b/src/state-manager/CachedAppDataManager.ts @@ -440,7 +440,8 @@ class CachedAppDataManager { globalOffset, targetGroupSize, senderGroupSize, - allNodes.length + allNodes.length, + `factSendCorrespondingCachedAppData`+ txId ) const correspondingNodes: P2PTypes.NodeListTypes.Node[] = [] diff --git a/src/state-manager/TransactionQueue.ts b/src/state-manager/TransactionQueue.ts index eef64e635..8ac73b99b 100644 --- a/src/state-manager/TransactionQueue.ts +++ b/src/state-manager/TransactionQueue.ts @@ -4471,7 +4471,8 @@ class TransactionQueue { queueEntry.correspondingGlobalOffset, targetGroupSize, senderGroupSize, - queueEntry.transactionGroup.length + queueEntry.transactionGroup.length, + 'factTellCorrespondingNodes' + queueEntry.logID + queueEntry.acceptedTx.txId ) // check if we should avoid our index in the corresponding nodes if (Context.config.stateManager.avoidOurIndexInFactTell && correspondingIndices.includes(ourIndexInTxGroup)) { @@ -4500,7 +4501,8 @@ class TransactionQueue { queueEntry.correspondingGlobalOffset, targetGroupSize, senderGroupSize, - queueEntry.transactionGroup.length + queueEntry.transactionGroup.length, + `factTellCorrespondingNodes` + queueEntry.logID + queueEntry.acceptedTx.txId ) if (logFlags.debug) this.mainLogger.debug( @@ -4715,9 +4717,10 @@ class TransactionQueue { false, `tellSender ${queueEntry.logID}` ) + if (isValidFactSender === false && wrappedSenderNodeIndex != null && wrappedSenderNodeIndex >= 0) { // try again with wrapped sender index - isValidFactSender = verifyCorrespondingSender( + let isValidFactSender = verifyCorrespondingSender( receivingNodeIndex, wrappedSenderNodeIndex, queueEntry.correspondingGlobalOffset, @@ -4729,6 +4732,7 @@ class TransactionQueue { false, `tellSenderWrapped ${queueEntry.logID}` ) + console.log(`factValidateCorrespondingTellSender: result: ${isValidFactSender}`) } // it maybe a FACT sender but sender does not cover the account if (senderHasAddress === false) { @@ -4745,7 +4749,7 @@ class TransactionQueue { // it is neither a FACT corresponding node nor an exe neighbour node if (isValidFactSender === false) { this.mainLogger.error( - `factValidateCorrespondingTellSender: logId: ${queueEntry.logID} sender is neither a valid sender nor a neighbour node isValidSender: ${isValidFactSender}` + `factValidateCorrespondingTellSender: logId: ${queueEntry.logID} sender is neither a valid sender nor a neighbour node isValidSender: ${wrappedSenderNodeIndex}` ) nestedCountersInstance.countEvent( 'stateManager', @@ -5086,7 +5090,7 @@ class TransactionQueue { targetGroupSize, senderGroupSize, queueEntry.transactionGroup.length, - queueEntry.logID + `factTellCorrespondingNodesFinalData` + queueEntry.logID + queueEntry.acceptedTx.txId ) for (const key of keysToShare) { diff --git a/src/utils/fastAggregatedCorrespondingTell.ts b/src/utils/fastAggregatedCorrespondingTell.ts index 183f3468c..c0e11f09a 100644 --- a/src/utils/fastAggregatedCorrespondingTell.ts +++ b/src/utils/fastAggregatedCorrespondingTell.ts @@ -12,85 +12,41 @@ export function getCorrespondingNodes( transactionGroupSize: number, note = '' ): number[] { - if (logFlags.verbose) { + // if (logFlags.verbose) { console.log( `getCorrespondingNodes ${note} ${ourIndex} ${startTargetIndex} ${endTargetIndex} ${globalOffset} ${receiverGroupSize} ${sendGroupSize} ${transactionGroupSize}` ) - } - let wrappedIndex: number - let targetNumber: number - let found = false - - let unWrappedEndIndex = -1 - // handle case where receiver group is split (wraps around) - if (startTargetIndex > endTargetIndex) { - unWrappedEndIndex = endTargetIndex - endTargetIndex = endTargetIndex + transactionGroupSize - } - //wrap our index to the send group size - ourIndex = ourIndex % sendGroupSize - - //find our initial staring index into the receiver group (wrappedIndex) - for (let i = startTargetIndex; i < endTargetIndex; i++) { - wrappedIndex = i - if (i >= transactionGroupSize) { - wrappedIndex = i - transactionGroupSize - } - targetNumber = (i + globalOffset) % receiverGroupSize - if (targetNumber === ourIndex) { - found = true - break - } - } - if (!found) { - //return empty array - return [] - } - + // } + const destinationNodes: number[] = [] - //this loop is at most O(k) where k is receiverGroupSize / sendGroupSize - //effectively it is constant time it is required so that a smaller - //group can send to a larger group - while (targetNumber < receiverGroupSize) { - //send all payload to this node - const destinationNode = wrappedIndex - - destinationNodes.push(destinationNode) - //console.log(`sender ${ourIndex} send all payload to node ${destinationNode} targetNumber ${targetNumber} `) - - // //in-place verification check - // let sendingNodeIndex = ourIndex - // let receivingNodeIndex = destinationNode - // //extra step here, remove in production - // verifySender(receivingNodeIndex, sendingNodeIndex) - - //this part is a bit tricky. - //we are incrementing two indexes that control our loop - //wrapped index will have various corrections so that it can - //wrap past the end of a split span, or wrap within the range - //of the receiver group - targetNumber += sendGroupSize - wrappedIndex += sendGroupSize - - //wrap to front of transaction group - if (wrappedIndex >= transactionGroupSize) { - wrappedIndex = wrappedIndex - transactionGroupSize - } - //wrap to front of receiver group - if (wrappedIndex >= endTargetIndex) { - wrappedIndex = wrappedIndex - receiverGroupSize + const normalizedSenderIndex = ourIndex % sendGroupSize + + // Calculate logical position of first receiver + let logicalPosition = 0 + let currentIndex = startTargetIndex + + // Iterate through receiver group + for (let count = 0; count < receiverGroupSize; count++) { + // Calculate which sender this logical position maps to + const mappedSenderIndex = ((logicalPosition + globalOffset) % receiverGroupSize) % sendGroupSize + + if (mappedSenderIndex === normalizedSenderIndex) { + destinationNodes.push(currentIndex) } - //special case to stay in bounds when we have a split index and - //the unWrappedEndIndex is smaller than the start index. - //i.e. startTargetIndex = 45, endTargetIndex = 5 for a 50 node group - if (unWrappedEndIndex != -1 && wrappedIndex >= unWrappedEndIndex) { - const howFarPastUnWrapped = wrappedIndex - unWrappedEndIndex - wrappedIndex = startTargetIndex + howFarPastUnWrapped + + // Move to next receiver + logicalPosition++ + currentIndex++ + + // Handle wrap-around using modular arithmetic + if (currentIndex === transactionGroupSize) { + currentIndex = 0 } } - if (logFlags.verbose) { + + // if (logFlags.verbose) { console.log(`note: ${note} destinationNodes ${destinationNodes}`) - } + // } return destinationNodes } @@ -106,37 +62,58 @@ export function verifyCorrespondingSender( shouldUnwrapSender = false, note = '' ): boolean { - if (logFlags.verbose) { + // if (logFlags.verbose) { console.log( `verifyCorrespondingSender ${note} ${receivingNodeIndex} ${sendingNodeIndex} ${globalOffset} ${receiverGroupSize} ${sendGroupSize} ${receiverStartIndex} ${receiverEndIndex} ${transactionGroupSize}` ) - } - //note, in the gather case, we need to check the address range of the sender node also, to prove - //that it does cover the given account range - let unwrappedReceivingNodeIndex = receivingNodeIndex - - // handle case where receiver group is split (wraps around) - if (receiverStartIndex > unwrappedReceivingNodeIndex) { - unwrappedReceivingNodeIndex = unwrappedReceivingNodeIndex + transactionGroupSize - } - let unwrappedSendingNodeIndex = sendingNodeIndex - if (shouldUnwrapSender) { - unwrappedSendingNodeIndex = sendingNodeIndex + transactionGroupSize - } - - // use unwrappedReceivingNodeIndex to calculate the target index - const targetIndex = ((unwrappedReceivingNodeIndex + globalOffset) % receiverGroupSize) % sendGroupSize - const targetIndex2 = unwrappedSendingNodeIndex % sendGroupSize - if (targetIndex === targetIndex2) { - if (logFlags.verbose) + // } + + // Calculate logical position of receiver in its group + let logicalPosition: number + + if (receiverStartIndex <= receiverEndIndex) { + // Non-wrapped case + if (receivingNodeIndex >= receiverStartIndex && receivingNodeIndex < receiverEndIndex) { + logicalPosition = receivingNodeIndex - receiverStartIndex + } else { + // Receiver not in group console.log( - `note: ${note} verification passed ${targetIndex} === ${targetIndex2} ${unwrappedSendingNodeIndex}->${receivingNodeIndex}` + `note: ${note} receiver not in group ${receivingNodeIndex} ${receiverStartIndex} ${receiverEndIndex}` ) - return true + return false + } } else { - console.log( - `note: ${note} X verification failed ${targetIndex} !== ${targetIndex2} sender: ${unwrappedSendingNodeIndex} receiver: ${receivingNodeIndex}` - ) - return false + // Wrapped case + if (receivingNodeIndex >= receiverStartIndex) { + logicalPosition = receivingNodeIndex - receiverStartIndex + } else if (receivingNodeIndex < receiverEndIndex) { + logicalPosition = (transactionGroupSize - receiverStartIndex) + receivingNodeIndex + } else { + // Receiver not in group + console.log( + `note: ${note} receiver not in group ${receivingNodeIndex} ${receiverStartIndex} ${receiverEndIndex}` + ) + return false + } } -} + + // Calculate expected sender using pure math + const expectedSenderIndex = ((logicalPosition + globalOffset) % receiverGroupSize) % sendGroupSize + const actualSenderIndex = sendingNodeIndex % sendGroupSize + + const result = expectedSenderIndex === actualSenderIndex + + // if (logFlags.verbose) { + if (result) { + console.log( + `note: ${note} verification passed ${expectedSenderIndex} === ${actualSenderIndex} ${sendingNodeIndex}->${receivingNodeIndex}` + ) + } else { + console.log( + `note: ${note} X verification failed ${expectedSenderIndex} !== ${actualSenderIndex} sender: ${sendingNodeIndex} receiver: ${receivingNodeIndex}` + ) + } + // } + + return result +} \ No newline at end of file From 180d0af64e9b7bce1fe4d18745a20e4aa1a6672a Mon Sep 17 00:00:00 2001 From: devendra-shardeum Date: Wed, 20 Aug 2025 03:53:51 +0530 Subject: [PATCH 2/7] added calculation of effective index to correctly compute the index first no need of retry --- src/state-manager/TransactionQueue.ts | 25 +++++-------------------- 1 file changed, 5 insertions(+), 20 deletions(-) diff --git a/src/state-manager/TransactionQueue.ts b/src/state-manager/TransactionQueue.ts index 8ac73b99b..37366f617 100644 --- a/src/state-manager/TransactionQueue.ts +++ b/src/state-manager/TransactionQueue.ts @@ -4693,9 +4693,10 @@ class TransactionQueue { // check if it is a FACT sender const receivingNodeIndex = queueEntry.ourTXGroupIndex // we are the receiver const senderNodeIndex = queueEntry.transactionGroup.findIndex((node) => node.id === senderNodeId) - let wrappedSenderNodeIndex = null + // CHANGE: Determine the correct sender index upfront + let effectiveSenderIndex = senderNodeIndex if (queueEntry.isSenderWrappedTxGroup[senderNodeId] != null) { - wrappedSenderNodeIndex = queueEntry.isSenderWrappedTxGroup[senderNodeId] + effectiveSenderIndex = queueEntry.isSenderWrappedTxGroup[senderNodeId] } const receiverGroupSize = queueEntry.executionNodeIdSorted.length const senderGroupSize = receiverGroupSize @@ -4707,7 +4708,7 @@ class TransactionQueue { let isValidFactSender = verifyCorrespondingSender( receivingNodeIndex, - senderNodeIndex, + effectiveSenderIndex, queueEntry.correspondingGlobalOffset, receiverGroupSize, senderGroupSize, @@ -4718,22 +4719,6 @@ class TransactionQueue { `tellSender ${queueEntry.logID}` ) - if (isValidFactSender === false && wrappedSenderNodeIndex != null && wrappedSenderNodeIndex >= 0) { - // try again with wrapped sender index - let isValidFactSender = verifyCorrespondingSender( - receivingNodeIndex, - wrappedSenderNodeIndex, - queueEntry.correspondingGlobalOffset, - receiverGroupSize, - senderGroupSize, - targetIndices.startIndex, - targetIndices.endIndex, - queueEntry.transactionGroup.length, - false, - `tellSenderWrapped ${queueEntry.logID}` - ) - console.log(`factValidateCorrespondingTellSender: result: ${isValidFactSender}`) - } // it maybe a FACT sender but sender does not cover the account if (senderHasAddress === false) { this.mainLogger.error( @@ -4749,7 +4734,7 @@ class TransactionQueue { // it is neither a FACT corresponding node nor an exe neighbour node if (isValidFactSender === false) { this.mainLogger.error( - `factValidateCorrespondingTellSender: logId: ${queueEntry.logID} sender is neither a valid sender nor a neighbour node isValidSender: ${wrappedSenderNodeIndex}` + `factValidateCorrespondingTellSender: logId: ${queueEntry.logID} sender is neither a valid sender nor a neighbour node isValidSender: ${effectiveSenderIndex}` ) nestedCountersInstance.countEvent( 'stateManager', From 6d536a0cde1fec55908a8e4d672222e1b3168f33 Mon Sep 17 00:00:00 2001 From: Jintu Das Date: Thu, 21 Aug 2025 11:37:17 +0530 Subject: [PATCH 3/7] FACT bug fixes --- src/utils/fastAggregatedCorrespondingTell.ts | 43 +++++++++++--------- 1 file changed, 24 insertions(+), 19 deletions(-) diff --git a/src/utils/fastAggregatedCorrespondingTell.ts b/src/utils/fastAggregatedCorrespondingTell.ts index c0e11f09a..02e545256 100644 --- a/src/utils/fastAggregatedCorrespondingTell.ts +++ b/src/utils/fastAggregatedCorrespondingTell.ts @@ -1,4 +1,5 @@ //get the target nodes for a given corresponding sender +import { Utils } from '@shardeum-foundation/lib-types' import { logFlags } from '../logger' //this only has to be computed once time no matter how many facts are being shared @@ -12,11 +13,11 @@ export function getCorrespondingNodes( transactionGroupSize: number, note = '' ): number[] { - // if (logFlags.verbose) { + if (logFlags.verbose) { console.log( - `getCorrespondingNodes ${note} ${ourIndex} ${startTargetIndex} ${endTargetIndex} ${globalOffset} ${receiverGroupSize} ${sendGroupSize} ${transactionGroupSize}` + `getCorrespondingNodes ${note} ourIndex:${ourIndex} startTarget:${startTargetIndex} endTarget:${endTargetIndex} globalOffset:${globalOffset} receiverGroupSize:${receiverGroupSize} sendGroupSize:${sendGroupSize} transactionGroupSize:${transactionGroupSize}` ) - // } + } const destinationNodes: number[] = [] const normalizedSenderIndex = ourIndex % sendGroupSize @@ -44,9 +45,9 @@ export function getCorrespondingNodes( } } - // if (logFlags.verbose) { - console.log(`note: ${note} destinationNodes ${destinationNodes}`) - // } + if (logFlags.verbose) { + console.log(`getCorrespondingNodes ${note} destinationNodes:${Utils.safeStringify(destinationNodes)}`) + } return destinationNodes } @@ -62,11 +63,11 @@ export function verifyCorrespondingSender( shouldUnwrapSender = false, note = '' ): boolean { - // if (logFlags.verbose) { + if (logFlags.verbose) { console.log( - `verifyCorrespondingSender ${note} ${receivingNodeIndex} ${sendingNodeIndex} ${globalOffset} ${receiverGroupSize} ${sendGroupSize} ${receiverStartIndex} ${receiverEndIndex} ${transactionGroupSize}` + `verifyCorrespondingSender ${note} receivingNode:${receivingNodeIndex} sendingNode:${sendingNodeIndex} globalOffset:${globalOffset} receiverGroupSize:${receiverGroupSize} sendGroupSize:${sendGroupSize} receiverStart:${receiverStartIndex} receiverEnd:${receiverEndIndex} transactionGroupSize:${transactionGroupSize}` ) - // } + } // Calculate logical position of receiver in its group let logicalPosition: number @@ -77,9 +78,11 @@ export function verifyCorrespondingSender( logicalPosition = receivingNodeIndex - receiverStartIndex } else { // Receiver not in group - console.log( - `note: ${note} receiver not in group ${receivingNodeIndex} ${receiverStartIndex} ${receiverEndIndex}` - ) + if (logFlags.verbose) { + console.log( + `verifyCorrespondingSender ${note} receiver not in group receivingNode:${receivingNodeIndex} receiverStart:${receiverStartIndex} receiverEnd:${receiverEndIndex}` + ) + } return false } } else { @@ -90,9 +93,11 @@ export function verifyCorrespondingSender( logicalPosition = (transactionGroupSize - receiverStartIndex) + receivingNodeIndex } else { // Receiver not in group - console.log( - `note: ${note} receiver not in group ${receivingNodeIndex} ${receiverStartIndex} ${receiverEndIndex}` - ) + if (logFlags.verbose) { + console.log( + `verifyCorrespondingSender ${note} receiver not in group (wrapped case) receivingNode:${receivingNodeIndex} receiverStart:${receiverStartIndex} receiverEnd:${receiverEndIndex}` + ) + } return false } } @@ -103,17 +108,17 @@ export function verifyCorrespondingSender( const result = expectedSenderIndex === actualSenderIndex - // if (logFlags.verbose) { + if (logFlags.verbose) { if (result) { console.log( - `note: ${note} verification passed ${expectedSenderIndex} === ${actualSenderIndex} ${sendingNodeIndex}->${receivingNodeIndex}` + `verifyCorrespondingSender ${note} verification PASSED expectedSender:${expectedSenderIndex} === actualSender:${actualSenderIndex} sender:${sendingNodeIndex}->receiver:${receivingNodeIndex}` ) } else { console.log( - `note: ${note} X verification failed ${expectedSenderIndex} !== ${actualSenderIndex} sender: ${sendingNodeIndex} receiver: ${receivingNodeIndex}` + `verifyCorrespondingSender ${note} verification FAILED expectedSender:${expectedSenderIndex} !== actualSender:${actualSenderIndex} sender:${sendingNodeIndex} receiver:${receivingNodeIndex}` ) } - // } + } return result } \ No newline at end of file From 92ee9987d99dfb521eed8547d4f93f88422a52ad Mon Sep 17 00:00:00 2001 From: devendra-shardeum Date: Thu, 21 Aug 2025 21:36:40 +0530 Subject: [PATCH 4/7] disabled the extra logging --- src/utils/fastAggregatedCorrespondingTell.ts | 23 ++++++++++---------- 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/src/utils/fastAggregatedCorrespondingTell.ts b/src/utils/fastAggregatedCorrespondingTell.ts index 02e545256..b18c3e55a 100644 --- a/src/utils/fastAggregatedCorrespondingTell.ts +++ b/src/utils/fastAggregatedCorrespondingTell.ts @@ -78,11 +78,10 @@ export function verifyCorrespondingSender( logicalPosition = receivingNodeIndex - receiverStartIndex } else { // Receiver not in group - if (logFlags.verbose) { - console.log( - `verifyCorrespondingSender ${note} receiver not in group receivingNode:${receivingNodeIndex} receiverStart:${receiverStartIndex} receiverEnd:${receiverEndIndex}` - ) - } + console.log( + `verifyCorrespondingSender ${note} receiver not in group receivingNode:${receivingNodeIndex} receiverStart:${receiverStartIndex} receiverEnd:${receiverEndIndex}` + ) + return false } } else { @@ -93,11 +92,10 @@ export function verifyCorrespondingSender( logicalPosition = (transactionGroupSize - receiverStartIndex) + receivingNodeIndex } else { // Receiver not in group - if (logFlags.verbose) { - console.log( - `verifyCorrespondingSender ${note} receiver not in group (wrapped case) receivingNode:${receivingNodeIndex} receiverStart:${receiverStartIndex} receiverEnd:${receiverEndIndex}` - ) - } + console.log( + `verifyCorrespondingSender ${note} receiver not in group (wrapped case) receivingNode:${receivingNodeIndex} receiverStart:${receiverStartIndex} receiverEnd:${receiverEndIndex}` + ) + return false } } @@ -113,12 +111,13 @@ export function verifyCorrespondingSender( console.log( `verifyCorrespondingSender ${note} verification PASSED expectedSender:${expectedSenderIndex} === actualSender:${actualSenderIndex} sender:${sendingNodeIndex}->receiver:${receivingNodeIndex}` ) - } else { + } + } else { console.log( `verifyCorrespondingSender ${note} verification FAILED expectedSender:${expectedSenderIndex} !== actualSender:${actualSenderIndex} sender:${sendingNodeIndex} receiver:${receivingNodeIndex}` ) } - } + return result } \ No newline at end of file From 1d5549aad91fb18ee5f56684acc5f0feb709863e Mon Sep 17 00:00:00 2001 From: devendra-shardeum Date: Thu, 21 Aug 2025 22:28:56 +0530 Subject: [PATCH 5/7] added tests to check if receivers are covered or not --- .../fastAggregatedCorrespondingTell.test.ts | 531 ++++++++++++++---- 1 file changed, 431 insertions(+), 100 deletions(-) diff --git a/test/unit/src/utils/fastAggregatedCorrespondingTell.test.ts b/test/unit/src/utils/fastAggregatedCorrespondingTell.test.ts index 7376971c8..284f3e964 100644 --- a/test/unit/src/utils/fastAggregatedCorrespondingTell.test.ts +++ b/test/unit/src/utils/fastAggregatedCorrespondingTell.test.ts @@ -2,11 +2,24 @@ import { getCorrespondingNodes, verifyCorrespondingSender } from '../../../../sr const verbose = true -describe('FACT Tests', () => { - //push several test cases into an array - const receiverTestCases = [ - // trivial case +describe('FACT Tests - Comprehensive Sender/Receiver Verification', () => { + // Test configuration type for better clarity + interface TestCase { + name: string + startTargetIndex: number + endTargetIndex: number + transactionGroupSize: number + senderStartRange: number + senderEndRange: number + sendGroupSize: number + globalOffset?: number + expectedCoverage?: number // Expected coverage per receiver + } + + const testCases: TestCase[] = [ + // Basic cases { + name: 'Basic non-wrapped case', startTargetIndex: 13, endTargetIndex: 23, transactionGroupSize: 50, @@ -14,8 +27,8 @@ describe('FACT Tests', () => { senderEndRange: 43, sendGroupSize: 10, }, - // trivial case, half the sender size { + name: 'Half sender size', startTargetIndex: 13, endTargetIndex: 23, transactionGroupSize: 50, @@ -23,8 +36,10 @@ describe('FACT Tests', () => { senderEndRange: 40, sendGroupSize: 5, }, - // wrap around case + + // Wrap-around cases { + name: 'Receiver wrap-around', startTargetIndex: 45, endTargetIndex: 5, transactionGroupSize: 50, @@ -32,17 +47,86 @@ describe('FACT Tests', () => { senderEndRange: 10, sendGroupSize: 10, }, - // smaller receiver group { + name: 'Sender wrap-around - valid consecutive', + startTargetIndex: 10, + endTargetIndex: 20, + transactionGroupSize: 50, + senderStartRange: 45, + senderEndRange: 5, // [45,46,47,48,49,0,1,2,3,4] - 10 consecutive nodes + sendGroupSize: 10, + }, + { + name: 'Both wrap-around', + startTargetIndex: 45, + endTargetIndex: 5, + transactionGroupSize: 50, + senderStartRange: 48, + senderEndRange: 3, // [48,49,0,1,2] - 5 consecutive nodes + sendGroupSize: 5, + }, + + // Edge cases - small groups + { + name: 'Minimal receiver group', startTargetIndex: 13, - endTargetIndex: 18, + endTargetIndex: 14, transactionGroupSize: 50, senderStartRange: 0, senderEndRange: 10, sendGroupSize: 10, }, - // send to whole transaction group (disperse case) { + name: 'Minimal sender group', + startTargetIndex: 10, + endTargetIndex: 20, + transactionGroupSize: 50, + senderStartRange: 5, + senderEndRange: 6, + sendGroupSize: 1, + }, + { + name: 'Both minimal groups', + startTargetIndex: 5, + endTargetIndex: 6, + transactionGroupSize: 20, + senderStartRange: 10, + senderEndRange: 11, + sendGroupSize: 1, + }, + + // Different size ratios + { + name: 'Receiver larger than sender', + startTargetIndex: 10, + endTargetIndex: 30, + transactionGroupSize: 50, + senderStartRange: 35, + senderEndRange: 40, + sendGroupSize: 5, + }, + { + name: 'Sender larger than receiver', + startTargetIndex: 13, + endTargetIndex: 18, + transactionGroupSize: 50, + senderStartRange: 0, + senderEndRange: 20, + sendGroupSize: 20, + }, + { + name: 'Equal size groups', + startTargetIndex: 10, + endTargetIndex: 20, + transactionGroupSize: 50, + senderStartRange: 30, + senderEndRange: 40, + sendGroupSize: 10, + }, + + // Full coverage cases + { + name: 'Full transaction group coverage', startTargetIndex: 0, endTargetIndex: 50, transactionGroupSize: 50, @@ -50,8 +134,8 @@ describe('FACT Tests', () => { senderEndRange: 10, sendGroupSize: 10, }, - // send to whole transaction group (disperse case) { + name: 'Large scale test', startTargetIndex: 3000, endTargetIndex: 3128, transactionGroupSize: 5000, @@ -59,124 +143,371 @@ describe('FACT Tests', () => { senderEndRange: 228, sendGroupSize: 128, }, - // send to whole transaction group (disperse case) + + // Boundary conditions { - startTargetIndex: 15, - endTargetIndex: 25, + name: 'Receiver at start boundary', + startTargetIndex: 0, + endTargetIndex: 10, + transactionGroupSize: 50, + senderStartRange: 20, + senderEndRange: 30, + sendGroupSize: 10, + }, + { + name: 'Receiver at end boundary', + startTargetIndex: 40, + endTargetIndex: 50, transactionGroupSize: 50, senderStartRange: 10, senderEndRange: 20, sendGroupSize: 10, }, + + // With different global offsets + { + name: 'Large global offset', + startTargetIndex: 10, + endTargetIndex: 20, + transactionGroupSize: 50, + senderStartRange: 25, + senderEndRange: 35, + sendGroupSize: 10, + globalOffset: 43776, + }, { - //globalOffset = 43776 + name: 'Prime number sizes', + startTargetIndex: 7, + endTargetIndex: 18, + transactionGroupSize: 53, + senderStartRange: 23, + senderEndRange: 30, + sendGroupSize: 7, + globalOffset: 13, + }, + + // Stress test cases - CORRECTED + { + name: 'Maximum wrap complexity - valid group', startTargetIndex: 2, endTargetIndex: 7, transactionGroupSize: 8, - senderStartRange: 5, - senderEndRange: 2, + senderStartRange: 0, + senderEndRange: 5, // [0,1,2,3,4] - valid group with all normalized indices sendGroupSize: 5, + globalOffset: 43776, + }, + { + name: 'Non-aligned boundaries - consecutive senders', + startTargetIndex: 17, + endTargetIndex: 33, + transactionGroupSize: 47, + senderStartRange: 30, + senderEndRange: 43, // [30,31,...,42] - 13 consecutive nodes + sendGroupSize: 13, + }, + + // Additional wrapped sender test cases + { + name: 'Wrapped senders at boundary - partial indices', + startTargetIndex: 5, + endTargetIndex: 15, + transactionGroupSize: 20, + senderStartRange: 17, + senderEndRange: 5, // [17,18,19,0,1,2,3,4] - normalized [1,2,3,0,1,2,3,4], unique [0,1,2,3,4] + sendGroupSize: 5, // Match the actual unique normalized indices available + }, + { + name: 'Large wrapped sender group - aligned', + startTargetIndex: 100, + endTargetIndex: 200, + transactionGroupSize: 256, + senderStartRange: 192, + senderEndRange: 240, // [192-239] - 48 consecutive nodes, all unique when %48 + sendGroupSize: 48, + }, + + // Real-world scenarios + { + name: 'Typical sharding scenario', + startTargetIndex: 256, + endTargetIndex: 512, + transactionGroupSize: 1024, + senderStartRange: 512, + senderEndRange: 768, + sendGroupSize: 256, + }, + { + name: 'High node count scenario', + startTargetIndex: 1000, + endTargetIndex: 2000, + transactionGroupSize: 10000, + senderStartRange: 3000, + senderEndRange: 4000, + sendGroupSize: 1000, }, ] - receiverTestCases.forEach( - ( - { startTargetIndex, endTargetIndex, transactionGroupSize, senderStartRange, senderEndRange, sendGroupSize }, - index - ) => { - test(`Test case ${index}: startTargetIndex: ${startTargetIndex}, endTargetIndex: ${endTargetIndex} transactionGroupSize:${transactionGroupSize} sendGroupSize:${sendGroupSize}`, () => { - const receiverGroupSize = endTargetIndex - startTargetIndex - const globalOffset = 0 //43776 // Math.round(Math.random() * 1000) - const coverage = new Array(transactionGroupSize).fill(0) - - const senderIndicies: number[] = [] - if (senderStartRange < senderEndRange) { - for (let i = senderStartRange; i < senderEndRange; i++) { - senderIndicies.push(i) + // Helper function to generate sender indices + function generateSenderIndices( + senderStartRange: number, + senderEndRange: number, + transactionGroupSize: number, + sendGroupSize: number + ): number[] { + const indices: number[] = [] + + if (senderStartRange < senderEndRange) { + // Non-wrapped case + for (let i = senderStartRange; i < senderEndRange; i++) { + indices.push(i) + } + } else { + // Wrapped case - we need to ensure we generate a contiguous set of + // normalized indices. The sender group should contain exactly sendGroupSize + // consecutive nodes when considered modulo sendGroupSize. + + // First, check if this wrapped range creates a valid sender group + const expectedSenderCount = (transactionGroupSize - senderStartRange) + senderEndRange + + if (expectedSenderCount !== sendGroupSize) { + console.warn(`Warning: Wrapped sender range has ${expectedSenderCount} senders, but sendGroupSize is ${sendGroupSize}`) + } + + // Generate indices maintaining the correct wrap-around + for (let i = senderStartRange; i < transactionGroupSize; i++) { + indices.push(i) + } + for (let i = 0; i < senderEndRange; i++) { + indices.push(i) + } + } + + // Validate that the sender group is valid + const normalizedIndices = new Set(indices.map(i => i % sendGroupSize)) + if (normalizedIndices.size !== sendGroupSize) { + console.warn(`Warning: Sender group does not contain all normalized indices. Has ${normalizedIndices.size}, needs ${sendGroupSize}`) + console.warn(`Missing indices: ${Array.from({length: sendGroupSize}, (_, i) => i).filter(i => !normalizedIndices.has(i))}`) + } + + return indices + } + + // Helper function to calculate actual receiver group size + function calculateReceiverGroupSize( + startTargetIndex: number, + endTargetIndex: number, + transactionGroupSize: number + ): number { + if (startTargetIndex <= endTargetIndex) { + return endTargetIndex - startTargetIndex + } else { + // Wrapped case + return (transactionGroupSize - startTargetIndex) + endTargetIndex + } + } + + // Main test loop + testCases.forEach((testCase, index) => { + test(`Case ${index + 1}: ${testCase.name}`, () => { + const { + startTargetIndex, + endTargetIndex, + transactionGroupSize, + senderStartRange, + senderEndRange, + sendGroupSize, + globalOffset = 0, + } = testCase + + const receiverGroupSize = calculateReceiverGroupSize( + startTargetIndex, + endTargetIndex, + transactionGroupSize + ) + + // Track coverage for each node + const coverage = new Array(transactionGroupSize).fill(0) + const senderToReceivers: Map = new Map() + + // Generate sender indices + const senderIndices = generateSenderIndices( + senderStartRange, + senderEndRange, + transactionGroupSize, + sendGroupSize + ) + + if (verbose) { + console.log(`\n=== Test Case: ${testCase.name} ===`) + console.log(`Receiver range: [${startTargetIndex}, ${endTargetIndex})`) + console.log(`Sender range: [${senderStartRange}, ${senderEndRange})`) + console.log(`Receiver group size: ${receiverGroupSize}`) + console.log(`Send group size: ${sendGroupSize}`) + console.log(`Global offset: ${globalOffset}`) + console.log(`Transaction group size: ${transactionGroupSize}`) + + // Additional diagnostics for wrapped cases + if (senderStartRange > senderEndRange) { + const actualSenderCount = (transactionGroupSize - senderStartRange) + senderEndRange + console.log(`Wrapped sender count: ${actualSenderCount}`) + const normalizedSet = new Set() + for (let i = senderStartRange; i < transactionGroupSize; i++) { + normalizedSet.add(i % sendGroupSize) } - } else { - const useUnWrapped = true - if (useUnWrapped) { - //indicies can go past the end - for (let i = senderStartRange; i < senderEndRange + transactionGroupSize; i++) { - senderIndicies.push(i) - } - } else { - // wrapped - for (let i = 0; i < senderEndRange; i++) { - senderIndicies.push(i) - } - for (let i = senderStartRange; i < transactionGroupSize; i++) { - senderIndicies.push(i) - } + for (let i = 0; i < senderEndRange; i++) { + normalizedSet.add(i % sendGroupSize) } + console.log(`Unique normalized sender indices: [${Array.from(normalizedSet).sort((a, b) => a - b)}]`) + console.log(`Expected normalized indices: [${Array.from({length: sendGroupSize}, (_, i) => i)}]`) + } + } + + // Test each sender + for (let senderNodeIndex of senderIndices) { + const originalSenderIndex = senderNodeIndex + const destinationNodes = getCorrespondingNodes( + senderNodeIndex, + startTargetIndex, + endTargetIndex, + globalOffset, + receiverGroupSize, + sendGroupSize, + transactionGroupSize, + testCase.name + ) + + // Store sender-receiver mapping + senderToReceivers.set(originalSenderIndex % transactionGroupSize, destinationNodes) - if (verbose) console.log(`wrapped sender ranges ${senderIndicies}`) + if (verbose && destinationNodes.length > 0) { + console.log(`Sender ${senderNodeIndex} -> Receivers ${destinationNodes}`) } - for (let senderNodeIndex of senderIndicies) { - //get a list of destination nodes for this sender - const destinationNodes = getCorrespondingNodes( - senderNodeIndex, - startTargetIndex, - endTargetIndex, + // Update coverage and verify each destination + destinationNodes.forEach((receiverNodeIndex) => { + coverage[receiverNodeIndex]++ + + // Verify the correspondence + const isValid = verifyCorrespondingSender( + receiverNodeIndex, + senderNodeIndex % transactionGroupSize, // Always use actual node index globalOffset, receiverGroupSize, sendGroupSize, - transactionGroupSize + startTargetIndex, + endTargetIndex, + transactionGroupSize, + false, // Don't use shouldUnwrapSender + testCase.name ) - console.log(`sender ${senderNodeIndex} send all payload to nodes ${destinationNodes}`) - //cheap hack to test that verification can refute things - //globalOffset++ - - //this is the list of nodes that we should send to, - //in this test we will increment a coverage array - destinationNodes.forEach((receiverNodeIndex) => { - coverage[receiverNodeIndex]++ - //NOTE, This is where we would send the payload - //for tellCorrespondingNodes we would send all accounts we cover - //for tellCorrespondingNodesFinalData we would look at the receiver storage range and send only the accounts are covered - - //verification check - //extra step here, remove in production - - let shouldUnwrapSender = false - if (senderNodeIndex > transactionGroupSize) { - senderNodeIndex = senderNodeIndex - transactionGroupSize - shouldUnwrapSender = true - } - - expect( - verifyCorrespondingSender( - receiverNodeIndex, - senderNodeIndex, - globalOffset, - receiverGroupSize, - sendGroupSize, - 0, - 0, - transactionGroupSize, - shouldUnwrapSender - ) - ).toBe(true) - }) - } + expect(isValid).toBe(true) + }) + } - // verify coverage - for (let i = startTargetIndex; i < endTargetIndex; i++) { - const wrappedIndex = i >= transactionGroupSize ? i % transactionGroupSize : i - if (verbose) console.log(`Coverage ${i} ${wrappedIndex}: ${coverage[wrappedIndex]}`) - expect(coverage[wrappedIndex]).toBeGreaterThan(0) + // Verify coverage completeness + let totalCoverage = 0 + let minCoverage = Infinity + let maxCoverage = 0 + + // Check receiver coverage + for (let i = 0; i < transactionGroupSize; i++) { + const isInReceiverRange = startTargetIndex <= endTargetIndex + ? i >= startTargetIndex && i < endTargetIndex + : i >= startTargetIndex || i < endTargetIndex + + if (isInReceiverRange) { + expect(coverage[i]).toBeGreaterThan(0) + totalCoverage += coverage[i] + minCoverage = Math.min(minCoverage, coverage[i]) + maxCoverage = Math.max(maxCoverage, coverage[i]) + } else { + expect(coverage[i]).toBe(0) } + } - // check to make sure we did not cover anything outside of the target range - for (let i = 0; i < transactionGroupSize; i++) { - if (i < startTargetIndex || i >= endTargetIndex) { - expect(coverage[i]).toBe(0) + if (verbose) { + console.log(`Coverage stats - Total: ${totalCoverage}, Min: ${minCoverage}, Max: ${maxCoverage}`) + console.log(`Coverage distribution: ${coverage.filter(c => c > 0).join(', ')}`) + } + + // Verify coverage balance (should be relatively even) + if (receiverGroupSize > 0 && minCoverage > 0) { + const coverageRatio = maxCoverage / minCoverage + // Coverage should be reasonably balanced (within 2x) + expect(coverageRatio).toBeLessThanOrEqual(2) + } + + // Verify reverse mapping: each receiver should be able to identify its senders + const receiverToSenders: Map = new Map() + + for (const [sender, receivers] of senderToReceivers.entries()) { + receivers.forEach(receiver => { + if (!receiverToSenders.has(receiver)) { + receiverToSenders.set(receiver, []) } + receiverToSenders.get(receiver)!.push(sender) + }) + } + + // Each receiver should have at least one sender + for (let i = 0; i < transactionGroupSize; i++) { + const isInReceiverRange = startTargetIndex <= endTargetIndex + ? i >= startTargetIndex && i < endTargetIndex + : i >= startTargetIndex || i < endTargetIndex + + if (isInReceiverRange) { + expect(receiverToSenders.has(i)).toBe(true) + expect(receiverToSenders.get(i)!.length).toBeGreaterThan(0) } + } + }) + }) + + // Additional verification test + test('Verify sender-receiver correspondence is bijective', () => { + const testCase = { + startTargetIndex: 10, + endTargetIndex: 20, + transactionGroupSize: 50, + senderStartRange: 25, + senderEndRange: 35, + sendGroupSize: 10, + globalOffset: 0, + } + + const { startTargetIndex, endTargetIndex, transactionGroupSize, senderStartRange, senderEndRange, sendGroupSize, globalOffset } = testCase + const receiverGroupSize = calculateReceiverGroupSize(startTargetIndex, endTargetIndex, transactionGroupSize) + + // Map each receiver to its expected sender + const receiverToExpectedSender: Map = new Map() + + for (let receiverIndex = startTargetIndex; receiverIndex < endTargetIndex; receiverIndex++) { + const logicalPosition = receiverIndex - startTargetIndex + const expectedSenderIndex = ((logicalPosition + globalOffset) % receiverGroupSize) % sendGroupSize + receiverToExpectedSender.set(receiverIndex, expectedSenderIndex) + } + + // Verify each sender sends to the correct receivers + const senderIndices = generateSenderIndices(senderStartRange, senderEndRange, transactionGroupSize, sendGroupSize) + + for (const senderIndex of senderIndices) { + const destinations = getCorrespondingNodes( + senderIndex, + startTargetIndex, + endTargetIndex, + globalOffset, + receiverGroupSize, + sendGroupSize, + transactionGroupSize + ) + + destinations.forEach(receiver => { + const expectedSender = receiverToExpectedSender.get(receiver) + const actualSender = senderIndex % sendGroupSize + expect(actualSender).toBe(expectedSender) }) } - ) -}) + }) +}) \ No newline at end of file From f065722283ab921abd35a68071ab0554161ca885 Mon Sep 17 00:00:00 2001 From: devendra-shardeum Date: Thu, 21 Aug 2025 22:46:50 +0530 Subject: [PATCH 6/7] added counter as well in case of verifyCorrespondingSender fails --- src/utils/fastAggregatedCorrespondingTell.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/utils/fastAggregatedCorrespondingTell.ts b/src/utils/fastAggregatedCorrespondingTell.ts index b18c3e55a..ef55180e0 100644 --- a/src/utils/fastAggregatedCorrespondingTell.ts +++ b/src/utils/fastAggregatedCorrespondingTell.ts @@ -1,7 +1,7 @@ //get the target nodes for a given corresponding sender import { Utils } from '@shardeum-foundation/lib-types' import { logFlags } from '../logger' - +import { nestedCountersInstance } from './nestedCounters' //this only has to be computed once time no matter how many facts are being shared export function getCorrespondingNodes( ourIndex: number, @@ -109,13 +109,14 @@ export function verifyCorrespondingSender( if (logFlags.verbose) { if (result) { console.log( - `verifyCorrespondingSender ${note} verification PASSED expectedSender:${expectedSenderIndex} === actualSender:${actualSenderIndex} sender:${sendingNodeIndex}->receiver:${receivingNodeIndex}` + `verifyCorrespondingSender ${note} verification passed expectedSender:${expectedSenderIndex} === actualSender:${actualSenderIndex} sender:${sendingNodeIndex}->receiver:${receivingNodeIndex}` ) } } else { console.log( - `verifyCorrespondingSender ${note} verification FAILED expectedSender:${expectedSenderIndex} !== actualSender:${actualSenderIndex} sender:${sendingNodeIndex} receiver:${receivingNodeIndex}` + `verifyCorrespondingSender ${note} verification failed expectedSender:${expectedSenderIndex} !== actualSender:${actualSenderIndex} sender:${sendingNodeIndex} receiver:${receivingNodeIndex}` ) + if (nestedCountersInstance) nestedCountersInstance.countEvent('verifyCorrespondingSender', 'failed') } From 8ce2cf5f97b260005d12f48f82cc15a6cbf0ec83 Mon Sep 17 00:00:00 2001 From: devendra-shardeum Date: Thu, 21 Aug 2025 23:03:48 +0530 Subject: [PATCH 7/7] logging format corrected --- src/state-manager/CachedAppDataManager.ts | 2 +- src/state-manager/TransactionQueue.ts | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/state-manager/CachedAppDataManager.ts b/src/state-manager/CachedAppDataManager.ts index 737f716d1..4ebf2468d 100644 --- a/src/state-manager/CachedAppDataManager.ts +++ b/src/state-manager/CachedAppDataManager.ts @@ -441,7 +441,7 @@ class CachedAppDataManager { targetGroupSize, senderGroupSize, allNodes.length, - `factSendCorrespondingCachedAppData`+ txId + `factSendCorrespondingCachedAppData `+ txId ) const correspondingNodes: P2PTypes.NodeListTypes.Node[] = [] diff --git a/src/state-manager/TransactionQueue.ts b/src/state-manager/TransactionQueue.ts index 37366f617..5eeada8df 100644 --- a/src/state-manager/TransactionQueue.ts +++ b/src/state-manager/TransactionQueue.ts @@ -4472,7 +4472,7 @@ class TransactionQueue { targetGroupSize, senderGroupSize, queueEntry.transactionGroup.length, - 'factTellCorrespondingNodes' + queueEntry.logID + queueEntry.acceptedTx.txId + 'factTellCorrespondingNodes ' + queueEntry.logID ) // check if we should avoid our index in the corresponding nodes if (Context.config.stateManager.avoidOurIndexInFactTell && correspondingIndices.includes(ourIndexInTxGroup)) { @@ -4502,7 +4502,7 @@ class TransactionQueue { targetGroupSize, senderGroupSize, queueEntry.transactionGroup.length, - `factTellCorrespondingNodes` + queueEntry.logID + queueEntry.acceptedTx.txId + `factTellCorrespondingNodes ` + queueEntry.logID ) if (logFlags.debug) this.mainLogger.debug( @@ -5075,7 +5075,7 @@ class TransactionQueue { targetGroupSize, senderGroupSize, queueEntry.transactionGroup.length, - `factTellCorrespondingNodesFinalData` + queueEntry.logID + queueEntry.acceptedTx.txId + `factTellCorrespondingNodesFinalData ` + queueEntry.logID ) for (const key of keysToShare) {