diff --git a/.gitignore b/.gitignore index 8a7cf437..fc5e8f11 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,4 @@ test/docker/config/* !test/docker/config/bitcoin.conf docker-commands.md .DS_Store +query.txt diff --git a/src/api/routes/dash.js b/src/api/routes/dash.js index 9d1bd754..6c8346f3 100644 --- a/src/api/routes/dash.js +++ b/src/api/routes/dash.js @@ -1,11 +1,30 @@ const asyncHandler = require('express-async-handler') const router = require('express').Router() const { fromUnixTime, differenceInDays, parseISO, compareAsc, eachDayOfInterval, format } = require('date-fns') - +const debug = require('debug')('liquality:agent:dash') const Market = require('../../models/Market') const Order = require('../../models/Order') const MarketHistory = require('../../models/MarketHistory') +const mergeJSON = (data) => { + const result = {} + + data.forEach((dataSet) => { + for (let [key, value] of Object.entries(dataSet)) { + if (key === '_id' || key === 'wallet:count' || key === 'count') { + result[key] = value + continue + } + if (result[key]) { + result[key] += value + } else { + result[key] = value + } + } + }) + return result +} + const addressHashVariants = (address) => { const addressLowerCase = address.toLowerCase() @@ -244,59 +263,88 @@ router.get( ...req.query, status: ['AGENT_CLAIMED'] }) - const markets = (await Market.find({}, 'from to').lean().exec()).map((market) => `${market.from}-${market.to}`) - - const $group = markets.reduce((acc, market) => { - acc[`market:${market}:sum:fromAmountUsd`] = { - $sum: { $cond: [{ $eq: ['$market', market] }, '$fromAmountUsd', 0] } - } - acc[`market:${market}:sum:toAmountUsd`] = { - $sum: { $cond: [{ $eq: ['$market', market] }, '$toAmountUsd', 0] } - } - acc[`market:${market}:count`] = { - $sum: { $cond: [{ $eq: ['$market', market] }, 1, 0] } - } + let allMarkets = (await Market.find({}, 'from to').lean().exec()).map((market) => `${market.from}-${market.to}`) - return acc - }, {}) + const groupSize = 10 + let result = [] + const marketGroups = allMarkets + .map((e, i) => { + return i % groupSize === 0 ? allMarkets.slice(i, i + groupSize) : null + }) + .filter((e) => { + return e + }) - const result = await Order.aggregate([ - { - $match: query - }, - { - $addFields: { - market: { $concat: ['$from', '-', '$to'] }, - date: { $dateToString: { format: '%Y-%m-%d', date: '$createdAt' } } + for (const markets of marketGroups) { + const $group = markets.reduce((acc, market) => { + acc[`market:${market}:sum:fromAmountUsd`] = { + $sum: { $cond: [{ $eq: ['$market', market] }, '$fromAmountUsd', 0] } } - }, - { - $group: { - _id: '$date', - ...$group, - 'wallet:sum:fromAmountUsd': { - $sum: { - $cond: [{ $regexMatch: { input: '$userAgent', regex: /^Wallet/i } }, '$fromAmountUsd', 0] - } - }, - 'wallet:sum:toAmountUsd': { - $sum: { - $cond: [{ $regexMatch: { input: '$userAgent', regex: /^Wallet/i } }, '$toAmountUsd', 0] - } - }, - 'wallet:count': { - $sum: { - $cond: [{ $regexMatch: { input: '$userAgent', regex: /^Wallet/i } }, 1, 0] - } - }, - 'sum:totalAgentFeeUsd': { $sum: '$totalAgentFeeUsd' }, - 'sum:totalUserFeeUsd': { $sum: '$totalUserFeeUsd' }, - 'sum:fromAmountUsd': { $sum: '$fromAmountUsd' }, - 'sum:toAmountUsd': { $sum: '$toAmountUsd' }, - count: { $sum: 1 } + acc[`market:${market}:sum:toAmountUsd`] = { + $sum: { $cond: [{ $eq: ['$market', market] }, '$toAmountUsd', 0] } + } + acc[`market:${market}:count`] = { + $sum: { $cond: [{ $eq: ['$market', market] }, 1, 0] } } - } - ]).exec() + + return acc + }, {}) + + const marketResult = await Order.aggregate([ + { + $match: query + }, + { + $addFields: { + market: { $concat: ['$from', '-', '$to'] }, + date: { $dateToString: { format: '%Y-%m-%d', date: '$createdAt' } } + } + }, + { + $group: { + _id: '$date', + ...$group, + 'wallet:sum:fromAmountUsd': { + $sum: { + $cond: [{ $or: { input: '$userAgent', regex: /^Wallet/i } }, '$fromAmountUsd', 0] + } + }, + 'wallet:sum:toAmountUsd': { + $sum: { + $cond: [{ $or: { input: '$userAgent', regex: /^Wallet/i } }, '$toAmountUsd', 0] + } + }, + 'wallet:count': { + $sum: { + $cond: [{ $or: { input: '$userAgent', regex: /^Wallet/i } }, 1, 0] + } + }, + 'sum:totalAgentFeeUsd': { $sum: '$totalAgentFeeUsd' }, + 'sum:totalUserFeeUsd': { $sum: '$totalUserFeeUsd' }, + 'sum:fromAmountUsd': { $sum: '$fromAmountUsd' }, + 'sum:toAmountUsd': { $sum: '$toAmountUsd' }, + count: { $sum: 1 } + } + } + ]).exec() + result.push(marketResult) + } + + //flatten and group by date + let flattenResult = result.flat(1).reduce((r, ele) => { + r[ele._id] = r[ele._id] || [] + r[ele._id].push(ele) + return r + }, Object.create(null)) + + debug('market result =>', flattenResult) + //merge all with sum + let finalResults = [] + Object.keys(flattenResult).forEach((key) => { + finalResults.push(mergeJSON(flattenResult[key])) + }) + + result = finalResults const stats = result.map((json) => { json.date = json._id @@ -319,7 +367,7 @@ router.get( return json }) - + debug('result=>', stats) const emptyDataPoint = { 'wallet:sum:fromAmountUsd': 0, 'wallet:sum:toAmountUsd': 0, @@ -349,7 +397,7 @@ router.get( res.json({ count: result.length, result: { - markets, + allMarkets, stats } })