diff --git a/api/api.js b/api/api.js index 3ae153fce89..07f19f6d938 100644 --- a/api/api.js +++ b/api/api.js @@ -1,12 +1,12 @@ const http = require('http'); const https = require('https'); const fs = require('fs'); -const formidable = require('formidable'); const countlyConfig = require('./config', 'dont-enclose'); const plugins = require('../plugins/pluginManager.ts'); const log = require('./utils/log.js')('core:api'); const common = require('./utils/common.js'); const {processRequest} = require('./utils/requestProcessor'); +const {createApiApp} = require('./express/app'); const frontendConfig = require('../frontend/express/config.js'); const {WriteBatcher, ReadBatcher, InsertBatcher} = require('./parts/data/batcher.js'); const QueryRunner = require('./parts/data/QueryRunner.js'); @@ -187,35 +187,36 @@ plugins.connectToAllDatabases().then(function() { plugins.dispatch("/master", {}); // init hook + // Rate limiting as Express middleware (same logic as before) const rateLimitWindow = parseInt(plugins.getConfig("security").api_rate_limit_window, 10) || 0; const rateLimitRequests = parseInt(plugins.getConfig("security").api_rate_limit_requests, 10) || 0; const rateLimiterInstance = new RateLimiterMemory({ points: rateLimitRequests, duration: rateLimitWindow }); const requiresRateLimiting = rateLimitWindow > 0 && rateLimitRequests > 0; const omit = /^\/i(\/bulk)?(\?|$)/; // omit /i endpoint from rate limiting + /** - * Rate Limiting Middleware - * @param {Function} next - The next middleware function - * @returns {Function} - The wrapped middleware function with rate limiting + * Express rate limiting middleware + * @param {object} req - Express request + * @param {object} res - Express response + * @param {Function} next - next middleware */ - const rateLimit = (next) => { - if (!requiresRateLimiting) { - return next; + const rateLimitMiddleware = (req, res, next) => { + if (!requiresRateLimiting || omit.test(req.url)) { + return next(); } - return (req, res) => { - if (omit.test(req.url)) { - return next(req, res); - } - const ip = common.getIpAddress(req); - rateLimiterInstance - .consume(ip) - .then(() => next(req, res)) - .catch(() => { - log.w(`Rate limit exceeded for IP: ${ip}`); - common.returnMessage({ req, res, qstring: {} }, 429, "Too Many Requests"); - }); - }; + const ip = common.getIpAddress(req); + rateLimiterInstance + .consume(ip) + .then(() => next()) + .catch(() => { + log.w(`Rate limit exceeded for IP: ${ip}`); + common.returnMessage({req, res, qstring: {}}, 429, "Too Many Requests"); + }); }; + // Create the Express app with the full middleware stack + const app = createApiApp({countlyConfig, processRequest, plugins, rateLimitMiddleware}); + const serverOptions = { port: common.config.api.port, host: common.config.api.host || '' @@ -230,10 +231,10 @@ plugins.connectToAllDatabases().then(function() { if (common.config.api.ssl.ca) { sslOptions.ca = fs.readFileSync(common.config.api.ssl.ca); } - server = https.createServer(sslOptions, rateLimit(handleRequest)); + server = https.createServer(sslOptions, app); } else { - server = http.createServer(rateLimit(handleRequest)); + server = http.createServer(app); } server.listen(serverOptions.port, serverOptions.host, () => { @@ -249,108 +250,3 @@ plugins.connectToAllDatabases().then(function() { log.e('Database connection failed:', error); process.exit(1); }); - -/** - * Handle incoming HTTP/HTTPS requests - * @param {http.IncomingMessage} req - The request object - * @param {http.ServerResponse} res - The response object - */ -function handleRequest(req, res) { - const params = { - qstring: {}, - res: res, - req: req - }; - - res.setHeader('Connection', 'keep-alive'); - res.setHeader('Keep-Alive', 'timeout=5, max=1000'); - - if (req.method.toLowerCase() === 'post') { - const formidableOptions = { multiples: true }; - if (countlyConfig.api.maxUploadFileSize) { - formidableOptions.maxFileSize = countlyConfig.api.maxUploadFileSize; - } - - const form = new formidable.IncomingForm(formidableOptions); - if (/crash_symbols\/(add_symbol|upload_symbol)/.test(req.url)) { - req.body = []; - req.on('data', (data) => { - req.body.push(data); - }); - } - else { - req.body = ''; - req.on('data', (data) => { - req.body += data; - }); - } - - let multiFormData = false; - // Check if we have 'multipart/form-data' - if (req.headers['content-type']?.startsWith('multipart/form-data')) { - multiFormData = true; - } - - form.parse(req, (err, fields, files) => { - //handle bakcwards compatability with formiddble v1 - for (let i in files) { - if (Array.isArray(files[i])) { - files[i].forEach((file) => { - if (file.filepath) { - file.path = file.filepath; - } - if (file.mimetype) { - file.type = file.mimetype; - } - if (file.originalFilename) { - file.name = file.originalFilename; - } - }); - } - else { - if (files[i].filepath) { - files[i].path = files[i].filepath; - } - if (files[i].mimetype) { - files[i].type = files[i].mimetype; - } - if (files[i].originalFilename) { - files[i].name = files[i].originalFilename; - } - } - } - params.files = files; - if (multiFormData) { - let formDataUrl = []; - for (const i in fields) { - params.qstring[i] = fields[i]; - formDataUrl.push(`${i}=${fields[i]}`); - } - params.formDataUrl = formDataUrl.join('&'); - } - else { - for (const i in fields) { - params.qstring[i] = fields[i]; - } - } - if (!params.apiPath) { - processRequest(params); - } - }); - } - else if (req.method.toLowerCase() === 'options') { - const headers = {}; - headers["Access-Control-Allow-Origin"] = "*"; - headers["Access-Control-Allow-Methods"] = "POST, GET, OPTIONS"; - headers["Access-Control-Allow-Headers"] = "countly-token, Content-Type"; - res.writeHead(200, headers); - res.end(); - } - //attempt process GET request - else if (req.method.toLowerCase() === 'get') { - processRequest(params); - } - else { - common.returnMessage(params, 405, "Method not allowed"); - } -} diff --git a/api/config.sample.js b/api/config.sample.js index a012e06f6ac..0677c28093e 100644 --- a/api/config.sample.js +++ b/api/config.sample.js @@ -419,6 +419,28 @@ var countlyConfig = { */ encryption: {}, + /** + * JWT (JSON Web Token) authentication configuration + * Provides stateless API authentication using access and refresh tokens + * + * IMPORTANT: Set COUNTLY_JWT_SECRET environment variable with a secure secret (minimum 32 characters) + * JWT authentication will not work without this secret configured. + * + * @type {object} + * @property {string} secret - JWT signing secret. MUST be set via COUNTLY_JWT_SECRET env variable (min 32 chars) + * @property {number} [accessTokenExpiry=900] - Access token expiration in seconds (default: 15 minutes) + * @property {number} [refreshTokenExpiry=604800] - Refresh token expiration in seconds (default: 7 days) + * @property {string} [issuer=countly] - Token issuer claim + * @property {string} [algorithm=HS256] - JWT signing algorithm + */ + jwt: { + secret: '', // Set via COUNTLY_JWT_SECRET environment variable + accessTokenExpiry: 900, // 15 minutes + refreshTokenExpiry: 604800, // 7 days + issuer: 'countly', + algorithm: 'HS256' + }, + /** * Specifies where to store files. Value "fs" means file system or basically storing files on hard drive. Another currently supported option is "gridfs" storing files in MongoDB database using GridFS. By default fallback to "fs"; * @type {string} [default=fs] diff --git a/api/express/app.js b/api/express/app.js new file mode 100644 index 00000000000..94c4e6171af --- /dev/null +++ b/api/express/app.js @@ -0,0 +1,82 @@ +/** + * Express app factory for the Countly API server. + * Creates and configures an Express application that replaces the raw + * http.createServer() handler, while preserving all existing behavior + * through a middleware stack that bridges to the legacy request processor. + * @module api/express/app + */ + +const express = require('express'); + +/** + * Create and configure the Express app for the API server + * @param {object} options - Configuration options + * @param {object} options.countlyConfig - Countly API config + * @param {Function} options.processRequest - Legacy processRequest function + * @param {object} options.plugins - Plugin manager instance + * @param {Function} options.rateLimitMiddleware - Express rate limiting middleware + * @returns {express.Application} Configured Express app + */ +function createApiApp({countlyConfig, processRequest, plugins, rateLimitMiddleware}) { + const app = express(); + + app.enable('trust proxy'); + + // Disable express default headers we don't need + app.disable('x-powered-by'); + app.disable('etag'); + + // CORS for OPTIONS requests (replaces handleRequest OPTIONS block) + app.options('*', (req, res) => { + res.setHeader("Access-Control-Allow-Origin", "*"); + res.setHeader("Access-Control-Allow-Methods", "POST, GET, OPTIONS"); + res.setHeader("Access-Control-Allow-Headers", "countly-token, Content-Type, Authorization"); + res.status(200).end(); + }); + + // Body parser - formidable-based, replicates handleRequest POST parsing + app.use(require('./bodyParser').createBodyParser(countlyConfig)); + + // Params middleware - builds req.countlyParams from parsed request + app.use(require('./params')); + + // Rate limiting (same logic as before, now as Express middleware) + app.use(rateLimitMiddleware); + + // Core routes migrated from requestProcessor.js switch/case + app.use(require('../routes/jwt.ts').default); + app.use(require('../routes/ping')); + app.use(require('../routes/token')); + app.use(require('../routes/notes')); + app.use(require('../routes/cms')); + app.use(require('../routes/date_presets')); + app.use(require('../routes/version')); + app.use(require('../routes/sdk')); + app.use(require('../routes/users')); + app.use(require('../routes/apps')); + app.use(require('../routes/event_groups')); + app.use(require('../routes/tasks')); + app.use(require('../routes/render')); + app.use(require('../routes/app_users')); + app.use(require('../routes/events')); + app.use(require('../routes/system')); + app.use(require('../routes/export')); + app.use(require('../routes/analytics')); + // /o with method-based dispatch - must come after specific /o/* routes + app.use(require('../routes/data')); + + // Express router for plugin routes registered via plugins.apiRoute() + const apiRouter = express.Router(); + plugins.mountApiRoutes(apiRouter); + app.use(apiRouter); + + // Store router on app for later access (e.g., adding routes after init) + app.set('apiRouter', apiRouter); + + // Legacy bridge - catch-all that delegates to processRequest + app.use(require('./legacyBridge').createLegacyBridge(processRequest)); + + return app; +} + +module.exports = {createApiApp}; diff --git a/api/express/bodyParser.js b/api/express/bodyParser.js new file mode 100644 index 00000000000..c8e1231b14d --- /dev/null +++ b/api/express/bodyParser.js @@ -0,0 +1,109 @@ +/** + * Body parsing middleware for the Countly API server. + * Replicates the exact formidable-based POST parsing from the original + * handleRequest() function in api.js, preserving all edge cases including + * multipart form-data, crash symbol uploads (raw buffer), and formidable v1 + * backward-compatibility shims. + * @module api/express/bodyParser + */ + +const formidable = require('formidable'); + +/** + * Normalize formidable v3 file objects to have v1-compatible properties + * @param {object} files - formidable files object + */ +function normalizeFiles(files) { + for (let i in files) { + if (Array.isArray(files[i])) { + files[i].forEach((file) => { + if (file.filepath) { + file.path = file.filepath; + } + if (file.mimetype) { + file.type = file.mimetype; + } + if (file.originalFilename) { + file.name = file.originalFilename; + } + }); + } + else { + if (files[i].filepath) { + files[i].path = files[i].filepath; + } + if (files[i].mimetype) { + files[i].type = files[i].mimetype; + } + if (files[i].originalFilename) { + files[i].name = files[i].originalFilename; + } + } + } +} + +/** + * Create the body parser middleware + * @param {object} countlyConfig - Countly API config object + * @returns {Function} Express middleware + */ +function createBodyParser(countlyConfig) { + return function bodyParserMiddleware(req, res, next) { + if (req.method.toLowerCase() !== 'post') { + return next(); + } + + const formidableOptions = {multiples: true}; + if (countlyConfig.api.maxUploadFileSize) { + formidableOptions.maxFileSize = countlyConfig.api.maxUploadFileSize; + } + + const form = new formidable.IncomingForm(formidableOptions); + + // Accumulate raw body - buffer array for crash symbols, string for everything else + if (/crash_symbols\/(add_symbol|upload_symbol)/.test(req.url)) { + req.body = []; + req.on('data', (data) => { + req.body.push(data); + }); + } + else { + req.body = ''; + req.on('data', (data) => { + req.body += data; + }); + } + + let multiFormData = false; + if (req.headers['content-type'] && req.headers['content-type'].startsWith('multipart/form-data')) { + multiFormData = true; + } + + form.parse(req, (err, fields, files) => { + normalizeFiles(files); + + // Store parsed files on req for downstream access + req.countlyFiles = files; + + if (multiFormData) { + req.countlyFields = {}; + let formDataUrl = []; + for (const i in fields) { + req.countlyFields[i] = fields[i]; + formDataUrl.push(`${i}=${fields[i]}`); + } + req.countlyFormDataUrl = formDataUrl.join('&'); + } + else { + req.countlyFields = {}; + for (const i in fields) { + req.countlyFields[i] = fields[i]; + } + } + + next(); + }); + }; +} + +module.exports = {createBodyParser}; diff --git a/api/express/legacyBridge.js b/api/express/legacyBridge.js new file mode 100644 index 00000000000..5b70bf17e10 --- /dev/null +++ b/api/express/legacyBridge.js @@ -0,0 +1,30 @@ +/** + * Legacy bridge middleware for the Countly API server. + * Catch-all middleware that delegates unhandled requests to the existing + * processRequest() function. This ensures all existing routes (both core + * switch/case and plugin dispatch) continue to work unchanged during the + * incremental migration to Express-style routing. + * @module api/express/legacyBridge + */ + +/** + * Create the legacy bridge middleware + * @param {Function} processRequest - The legacy processRequest function from requestProcessor.js + * @returns {Function} Express middleware + */ +function createLegacyBridge(processRequest) { + return function legacyBridgeMiddleware(req, res, next) { + const params = req.countlyParams; + if (!params) { + return next(); + } + + // Mark that params were pre-parsed by Express middleware, + // so processRequest can skip its own URL parsing + params._expressParsed = true; + + processRequest(params); + }; +} + +module.exports = {createLegacyBridge}; diff --git a/api/express/params.js b/api/express/params.js new file mode 100644 index 00000000000..e3b40a10ae9 --- /dev/null +++ b/api/express/params.js @@ -0,0 +1,98 @@ +/** + * Params construction middleware for the Countly API server. + * Builds the Countly `params` object from an Express request and attaches + * it as `req.countlyParams`. This replicates the URL parsing and qstring + * construction logic from processRequest() lines 113-173 of requestProcessor.js. + * @module api/express/params + */ + +const url = require('url'); +const common = require('../utils/common.js'); + +/** + * Express middleware that constructs the Countly params object + */ +function paramsMiddleware(req, res, next) { + // Set keep-alive headers (previously in handleRequest) + res.setHeader('Connection', 'keep-alive'); + res.setHeader('Keep-Alive', 'timeout=5, max=1000'); + + const urlParts = url.parse(req.url, true); + const queryString = urlParts.query; + const paths = urlParts.pathname.split("/"); + + const params = { + qstring: {}, + res: res, + req: req, + href: urlParts.href, + urlParts: urlParts, + paths: paths + }; + + // Fill in request object defaults (same as processRequest) + params.req.method = params.req.method || "custom"; + params.req.headers = params.req.headers || {}; + params.req.socket = params.req.socket || {}; + params.req.connection = params.req.connection || {}; + + // Copy query string params + if (queryString) { + for (let i in queryString) { + params.qstring[i] = queryString[i]; + } + } + + // Copy parsed body fields (from bodyParser middleware) + if (req.countlyFields) { + for (let i in req.countlyFields) { + params.qstring[i] = req.countlyFields[i]; + } + } + + // Copy body as qstring param (for programmatic/raw body access) + if (params.req.body && typeof params.req.body === "object" && !req.countlyFields) { + for (let i in params.req.body) { + params.qstring[i] = params.req.body[i]; + } + } + + // Transfer files and formDataUrl from body parser + if (req.countlyFiles) { + params.files = req.countlyFiles; + } + if (req.countlyFormDataUrl) { + params.formDataUrl = req.countlyFormDataUrl; + } + + // Validate app_id and user_id length early + if (params.qstring.app_id && params.qstring.app_id.length !== 24) { + return common.returnMessage(params, 400, 'Invalid parameter "app_id"'); + } + + if (params.qstring.user_id && params.qstring.user_id.length !== 24) { + return common.returnMessage(params, 400, 'Invalid parameter "user_id"'); + } + + // Remove countly path prefix if configured + if (common.config.path === "/" + paths[1]) { + paths.splice(1, 1); + } + + // Compute apiPath (first 2 path segments) + let apiPath = ''; + for (let i = 1; i < paths.length; i++) { + if (i > 2) { + break; + } + apiPath += "/" + paths[i]; + } + + params.apiPath = apiPath; + params.fullPath = paths.join("/"); + + req.countlyParams = params; + next(); +} + +module.exports = paramsMiddleware; diff --git a/api/parts/mgmt/jwt_tokens.js b/api/parts/mgmt/jwt_tokens.js new file mode 100644 index 00000000000..f01c534c8d7 --- /dev/null +++ b/api/parts/mgmt/jwt_tokens.js @@ -0,0 +1,328 @@ +/** + * Module for JWT token management operations + * @module api/parts/mgmt/jwt_tokens + */ + +const common = require('../../utils/common.js'); +const jwtUtils = require('../../utils/jwt.js'); +const log = require('../../utils/log.js')('core:jwt_tokens'); + +// Import membersUtility for password verification +let membersUtility = null; + +/** + * Lazily load membersUtility to avoid circular dependency issues + * @returns {object} membersUtility module + */ +function getMembersUtility() { + if (!membersUtility) { + membersUtility = require('../../../frontend/express/libs/members.js'); + membersUtility.db = common.db; + } + return membersUtility; +} + +/** + * Login endpoint - authenticates user and returns access/refresh tokens + * @param {object} params - Request params object + * @param {function} done - Callback function + */ +function login(params, done) { + // Check if JWT is properly configured + if (!jwtUtils.isConfigured()) { + common.returnMessage(params, 500, 'JWT authentication is not configured. Set COUNTLY_JWT_SECRET environment variable (minimum 32 characters).'); + if (done) { + done(); + } + return; + } + + const username = params.qstring.username; + const password = params.qstring.password; + + if (!username || !password) { + common.returnMessage(params, 400, 'Missing required parameters: username and password'); + if (done) { + done(); + } + return; + } + + // Use membersUtility.verifyCredentials for password verification + const mu = getMembersUtility(); + mu.verifyCredentials(username, password, function(member) { + if (!member) { + common.returnMessage(params, 401, 'Invalid username or password'); + if (done) { + done(); + } + return; + } + + // Check if user is locked + if (member.locked) { + common.returnMessage(params, 401, 'User account is locked'); + if (done) { + done(); + } + return; + } + + // Generate access token + const accessResult = jwtUtils.signAccessToken(member); + if (!accessResult.success) { + log.e('Failed to sign access token:', accessResult.error); + common.returnMessage(params, 500, 'Failed to generate access token'); + if (done) { + done(); + } + return; + } + + // Generate refresh token + const refreshResult = jwtUtils.signRefreshToken(member._id); + if (!refreshResult.success) { + log.e('Failed to sign refresh token:', refreshResult.error); + common.returnMessage(params, 500, 'Failed to generate refresh token'); + if (done) { + done(); + } + return; + } + + // Return tokens + common.returnOutput(params, { + access_token: accessResult.token, + token_type: 'Bearer', + expires_in: accessResult.expiresIn, + refresh_token: refreshResult.token, + refresh_expires_in: refreshResult.expiresIn + }); + + if (done) { + done(); + } + }); +} + +/** + * Refresh endpoint - exchanges refresh token for new access token + * @param {object} params - Request params object + * @param {function} done - Callback function + */ +function refresh(params, done) { + // Check if JWT is properly configured + if (!jwtUtils.isConfigured()) { + common.returnMessage(params, 500, 'JWT authentication is not configured'); + if (done) { + done(); + } + return; + } + + const refreshToken = params.qstring.refresh_token; + + if (!refreshToken) { + common.returnMessage(params, 400, 'Missing required parameter: refresh_token'); + if (done) { + done(); + } + return; + } + + // Verify the refresh token + const verifyResult = jwtUtils.verifyToken(refreshToken, 'refresh'); + if (!verifyResult.valid) { + let statusCode = 401; + let message = 'Invalid refresh token'; + + if (verifyResult.error === 'TOKEN_EXPIRED') { + message = 'Refresh token has expired. Please login again.'; + } + else if (verifyResult.error === 'INVALID_TOKEN_TYPE') { + message = 'Invalid token type. Expected refresh token.'; + } + + common.returnMessage(params, statusCode, message); + if (done) { + done(); + } + return; + } + + const decoded = verifyResult.decoded; + const memberId = decoded.sub; + const jti = decoded.jti; + + // Check if token is blacklisted + jwtUtils.isTokenBlacklisted(jti, function(err, isBlacklisted) { + if (err) { + common.returnMessage(params, 500, 'Error checking token status'); + if (done) { + done(); + } + return; + } + + if (isBlacklisted) { + common.returnMessage(params, 401, 'Refresh token has been revoked'); + if (done) { + done(); + } + return; + } + + // Fetch the member to ensure they still exist and aren't locked + common.db.collection('members').findOne( + { _id: common.db.ObjectID(memberId) }, + function(findErr, member) { + if (findErr || !member) { + common.returnMessage(params, 401, 'User not found'); + if (done) { + done(); + } + return; + } + + if (member.locked) { + common.returnMessage(params, 401, 'User account is locked'); + if (done) { + done(); + } + return; + } + + // Blacklist the old refresh token (refresh token rotation) + const expiresAt = new Date(decoded.exp * 1000); + jwtUtils.blacklistToken(jti, memberId, expiresAt, function(blacklistErr) { + if (blacklistErr) { + log.e('Failed to blacklist old refresh token:', blacklistErr); + // Continue anyway - token rotation is a security enhancement, not critical + } + + // Generate new access token + const accessResult = jwtUtils.signAccessToken(member); + if (!accessResult.success) { + common.returnMessage(params, 500, 'Failed to generate access token'); + if (done) { + done(); + } + return; + } + + // Generate new refresh token + const refreshResult = jwtUtils.signRefreshToken(memberId); + if (!refreshResult.success) { + common.returnMessage(params, 500, 'Failed to generate refresh token'); + if (done) { + done(); + } + return; + } + + // Return new tokens + common.returnOutput(params, { + access_token: accessResult.token, + token_type: 'Bearer', + expires_in: accessResult.expiresIn, + refresh_token: refreshResult.token, + refresh_expires_in: refreshResult.expiresIn + }); + + if (done) { + done(); + } + }); + } + ); + }); +} + +/** + * Revoke endpoint - invalidates a refresh token (logout) + * @param {object} params - Request params object + * @param {function} done - Callback function + */ +function revoke(params, done) { + // Check if JWT is properly configured + if (!jwtUtils.isConfigured()) { + common.returnMessage(params, 500, 'JWT authentication is not configured'); + if (done) { + done(); + } + return; + } + + const refreshToken = params.qstring.refresh_token; + + if (!refreshToken) { + common.returnMessage(params, 400, 'Missing required parameter: refresh_token'); + if (done) { + done(); + } + return; + } + + // Verify the refresh token (we need to decode it to get the jti) + const verifyResult = jwtUtils.verifyToken(refreshToken, 'refresh'); + + // Even if expired, we should try to blacklist it + let decoded; + if (verifyResult.valid) { + decoded = verifyResult.decoded; + } + else if (verifyResult.error === 'TOKEN_EXPIRED') { + // For expired tokens, try to decode without verification + try { + decoded = require('jsonwebtoken').decode(refreshToken); + if (!decoded || decoded.type !== 'refresh') { + common.returnMessage(params, 400, 'Invalid refresh token format'); + if (done) { + done(); + } + return; + } + } + catch (e) { + common.returnMessage(params, 400, 'Invalid refresh token'); + if (done) { + done(); + } + return; + } + } + else { + common.returnMessage(params, 400, 'Invalid refresh token'); + if (done) { + done(); + } + return; + } + + const jti = decoded.jti; + const memberId = decoded.sub; + const expiresAt = new Date(decoded.exp * 1000); + + // Add to blacklist + jwtUtils.blacklistToken(jti, memberId, expiresAt, function(err) { + if (err) { + log.e('Failed to revoke token:', err); + common.returnMessage(params, 500, 'Failed to revoke token'); + if (done) { + done(); + } + return; + } + + common.returnOutput(params, { success: true, message: 'Token revoked successfully' }); + if (done) { + done(); + } + }); +} + +module.exports = { + login, + refresh, + revoke +}; diff --git a/api/routes/analytics.js b/api/routes/analytics.js new file mode 100644 index 00000000000..7a381e03d13 --- /dev/null +++ b/api/routes/analytics.js @@ -0,0 +1,199 @@ +/** + * Analytics and aggregate data routes (/o/analytics, /o/aggregate). + * Migrated from the legacy switch/case in requestProcessor.js. + * @module api/routes/analytics + */ + +const express = require('express'); +const router = express.Router(); +const common = require('../utils/common.js'); +const { validateUser, validateRead, validateUserForWrite, validateGlobalAdmin } = require('../utils/rights.js'); +const calculatedDataManager = require('../utils/calculatedDataManager.js'); +const plugins = require('../../plugins/pluginManager.ts'); +const log = require('../utils/log.js')('core:api'); + +const validateUserForDataReadAPI = validateRead; +const validateUserForMgmtReadAPI = validateUser; +const validateUserForDataWriteAPI = validateUserForWrite; +const validateUserForGlobalAdmin = validateGlobalAdmin; + +const countlyApi = { + data: { + fetch: require('../parts/data/fetch.js'), + } +}; + +// --- Read endpoints: /o/analytics --- + +router.all('/o/analytics/dashboard', (req, res) => { + const params = req.countlyParams; + if (!params.qstring.app_id) { + common.returnMessage(params, 400, 'Missing parameter "app_id"'); + return false; + } + validateUserForDataReadAPI(params, 'core', countlyApi.data.fetch.fetchDashboard); +}); + +router.all('/o/analytics/countries', (req, res) => { + const params = req.countlyParams; + if (!params.qstring.app_id) { + common.returnMessage(params, 400, 'Missing parameter "app_id"'); + return false; + } + validateUserForDataReadAPI(params, 'core', countlyApi.data.fetch.fetchCountries); +}); + +router.all('/o/analytics/sessions', (req, res) => { + const params = req.countlyParams; + if (!params.qstring.app_id) { + common.returnMessage(params, 400, 'Missing parameter "app_id"'); + return false; + } + //takes also bucket=daily || monthly. extends period to full months if monthly + validateUserForDataReadAPI(params, 'core', countlyApi.data.fetch.fetchSessions); +}); + +router.all('/o/analytics/metric', (req, res) => { + const params = req.countlyParams; + if (!params.qstring.app_id) { + common.returnMessage(params, 400, 'Missing parameter "app_id"'); + return false; + } + validateUserForDataReadAPI(params, 'core', countlyApi.data.fetch.fetchMetric); +}); + +router.all('/o/analytics/tops', (req, res) => { + const params = req.countlyParams; + if (!params.qstring.app_id) { + common.returnMessage(params, 400, 'Missing parameter "app_id"'); + return false; + } + validateUserForDataReadAPI(params, 'core', countlyApi.data.fetch.fetchTops); +}); + +router.all('/o/analytics/loyalty', (req, res) => { + const params = req.countlyParams; + if (!params.qstring.app_id) { + common.returnMessage(params, 400, 'Missing parameter "app_id"'); + return false; + } + validateUserForDataReadAPI(params, 'core', countlyApi.data.fetch.fetchLoyalty); +}); + +router.all('/o/analytics/frequency', (req, res) => { + const params = req.countlyParams; + if (!params.qstring.app_id) { + common.returnMessage(params, 400, 'Missing parameter "app_id"'); + return false; + } + validateUserForDataReadAPI(params, 'core', countlyApi.data.fetch.fetchFrequency); +}); + +router.all('/o/analytics/durations', (req, res) => { + const params = req.countlyParams; + if (!params.qstring.app_id) { + common.returnMessage(params, 400, 'Missing parameter "app_id"'); + return false; + } + validateUserForDataReadAPI(params, 'core', countlyApi.data.fetch.fetchDurations); +}); + +router.all('/o/analytics/events', (req, res) => { + const params = req.countlyParams; + if (!params.qstring.app_id) { + common.returnMessage(params, 400, 'Missing parameter "app_id"'); + return false; + } + //takes also bucket=daily || monthly. extends period to full months if monthly + validateUserForDataReadAPI(params, 'core', countlyApi.data.fetch.fetchEvents); +}); + +// Catch-all for /o/analytics/* - dispatches to plugins or returns error +router.all('/o/analytics/:action', (req, res) => { + const params = req.countlyParams; + const apiPath = '/o/analytics'; + const paths = params.paths; + if (!params.qstring.app_id) { + common.returnMessage(params, 400, 'Missing parameter "app_id"'); + return false; + } + if (!plugins.dispatch(apiPath, { + params: params, + validateUserForDataReadAPI: validateUserForDataReadAPI, + validateUserForMgmtReadAPI: validateUserForMgmtReadAPI, + paths: paths, + validateUserForDataWriteAPI: validateUserForDataWriteAPI, + validateUserForGlobalAdmin: validateUserForGlobalAdmin + })) { + common.returnMessage(params, 400, 'Invalid path, must be one of /dashboard, /countries, /sessions, /metric, /tops, /loyalty, /frequency, /durations, /events'); + } +}); + +// --- Read endpoints: /o/aggregate --- + +router.all('/o/aggregate', (req, res) => { + const params = req.countlyParams; + validateUser(params, () => { + //Long task to run specific drill query. Give back task_id if running, result if done. + if (params.qstring.query) { + + try { + params.qstring.query = JSON.parse(params.qstring.query); + } + catch (ee) { + log.e(ee); + common.returnMessage(params, 400, 'Invalid query parameter'); + return; + } + + if (params.qstring.query.appID) { + if (Array.isArray(params.qstring.query.appID)) { + //make sure member has access to all apps in this list + for (var i = 0; i < params.qstring.query.appID.length; i++) { + if (!params.member.global_admin && params.member.user_of && params.member.user_of.indexOf(params.qstring.query.appID[i]) === -1) { + common.returnMessage(params, 401, 'User does not have access right for this app'); + return; + } + } + } + else { + if (!params.member.global_admin && params.member.user_of && params.member.user_of.indexOf(params.qstring.query.appID) === -1) { + common.returnMessage(params, 401, 'User does not have access right for this app'); + return; + } + } + } + else { + params.qstring.query.appID = params.qstring.app_id; + } + if (params.qstring.period) { + params.qstring.query.period = params.qstring.query.period || params.qstring.period || "30days"; + } + if (params.qstring.periodOffset) { + params.qstring.query.periodOffset = params.qstring.query.periodOffset || params.qstring.periodOffset || 0; + } + + calculatedDataManager.longtask({ + db: common.db, + no_cache: params.qstring.no_cache, + threshold: plugins.getConfig("api").request_threshold, + app_id: params.qstring.query.app_id, + query_data: params.qstring.query, + outputData: function(err, data) { + if (err) { + common.returnMessage(params, 400, err); + } + else { + common.returnMessage(params, 200, data); + } + } + }); + } + else { + common.returnMessage(params, 400, 'Missing parameter "query"'); + } + + }); +}); + +module.exports = router; diff --git a/api/routes/app_users.js b/api/routes/app_users.js new file mode 100644 index 00000000000..f549b0868b8 --- /dev/null +++ b/api/routes/app_users.js @@ -0,0 +1,436 @@ +/** + * App Users routes (/i/app_users, /o/app_users). + * Migrated from the legacy switch/case in requestProcessor.js. + * @module api/routes/app_users + */ + +const express = require('express'); +const router = express.Router(); +const common = require('../utils/common.js'); +const { validateUser, validateRead, validateUserForRead, validateUserForWrite, validateGlobalAdmin } = require('../utils/rights.js'); +const plugins = require('../../plugins/pluginManager.ts'); +const taskmanager = require('../utils/taskmanager.js'); +var countlyFs = require('../utils/countlyFs.js'); + +const validateUserForDataReadAPI = validateRead; +const validateUserForMgmtReadAPI = validateUser; +const validateUserForDataWriteAPI = validateUserForWrite; +const validateUserForGlobalAdmin = validateGlobalAdmin; + +const countlyApi = { + data: { + exports: require('../parts/data/exports.js'), + }, + mgmt: { + appUsers: require('../parts/mgmt/app_users.js'), + } +}; + +// --- Write endpoints: /i/app_users --- + +router.all('/i/app_users/create', (req, res) => { + const params = req.countlyParams; + if (!params.qstring.app_id) { + common.returnMessage(params, 400, 'Missing parameter "app_id"'); + return false; + } + if (!params.qstring.data) { + common.returnMessage(params, 400, 'Missing parameter "data"'); + return false; + } + else if (typeof params.qstring.data === "string") { + try { + params.qstring.data = JSON.parse(params.qstring.data); + } + catch (ex) { + console.log("Could not parse data", params.qstring.data); + common.returnMessage(params, 400, 'Could not parse parameter "data": ' + params.qstring.data); + return false; + } + } + if (!Object.keys(params.qstring.data).length) { + common.returnMessage(params, 400, 'Parameter "data" cannot be empty'); + return false; + } + validateUserForWrite(params, function() { + countlyApi.mgmt.appUsers.create(params.qstring.app_id, params.qstring.data, params, function(err, result) { + if (err) { + common.returnMessage(params, 400, err); + } + else { + common.returnMessage(params, 200, 'User Created: ' + JSON.stringify(result)); + } + }); + }); +}); + +router.all('/i/app_users/update', (req, res) => { + const params = req.countlyParams; + if (!params.qstring.app_id) { + common.returnMessage(params, 400, 'Missing parameter "app_id"'); + return false; + } + if (!params.qstring.update) { + common.returnMessage(params, 400, 'Missing parameter "update"'); + return false; + } + else if (typeof params.qstring.update === "string") { + try { + params.qstring.update = JSON.parse(params.qstring.update); + } + catch (ex) { + console.log("Could not parse update", params.qstring.update); + common.returnMessage(params, 400, 'Could not parse parameter "update": ' + params.qstring.update); + return false; + } + } + if (!Object.keys(params.qstring.update).length) { + common.returnMessage(params, 400, 'Parameter "update" cannot be empty'); + return false; + } + if (!params.qstring.query) { + common.returnMessage(params, 400, 'Missing parameter "query"'); + return false; + } + else if (typeof params.qstring.query === "string") { + try { + params.qstring.query = JSON.parse(params.qstring.query); + } + catch (ex) { + console.log("Could not parse query", params.qstring.query); + common.returnMessage(params, 400, 'Could not parse parameter "query": ' + params.qstring.query); + return false; + } + } + validateUserForWrite(params, function() { + countlyApi.mgmt.appUsers.count(params.qstring.app_id, params.qstring.query, function(err, count) { + if (err || count === 0) { + common.returnMessage(params, 400, 'No users matching criteria'); + return false; + } + if (count > 1 && !params.qstring.force) { + common.returnMessage(params, 400, 'This query would update more than one user'); + return false; + } + countlyApi.mgmt.appUsers.update(params.qstring.app_id, params.qstring.query, params.qstring.update, params, function(err2) { + if (err2) { + common.returnMessage(params, 400, err2); + } + else { + common.returnMessage(params, 200, 'User Updated'); + } + }); + }); + }); +}); + +router.all('/i/app_users/delete', (req, res) => { + const params = req.countlyParams; + if (!params.qstring.app_id) { + common.returnMessage(params, 400, 'Missing parameter "app_id"'); + return false; + } + if (!params.qstring.query) { + common.returnMessage(params, 400, 'Missing parameter "query"'); + return false; + } + else if (typeof params.qstring.query === "string") { + try { + params.qstring.query = JSON.parse(params.qstring.query); + } + catch (ex) { + console.log("Could not parse query", params.qstring.query); + common.returnMessage(params, 400, 'Could not parse parameter "query": ' + params.qstring.query); + return false; + } + } + if (!Object.keys(params.qstring.query).length) { + common.returnMessage(params, 400, 'Parameter "query" cannot be empty, it would delete all users. Use clear app instead'); + return false; + } + validateUserForWrite(params, function() { + countlyApi.mgmt.appUsers.count(params.qstring.app_id, params.qstring.query, function(err, count) { + if (err || count === 0) { + common.returnMessage(params, 400, 'No users matching criteria'); + return false; + } + if (count > 1 && !params.qstring.force) { + common.returnMessage(params, 400, 'This query would delete more than one user'); + return false; + } + countlyApi.mgmt.appUsers.delete(params.qstring.app_id, params.qstring.query, params, function(err2) { + if (err2) { + common.returnMessage(params, 400, err2); + } + else { + common.returnMessage(params, 200, 'User deleted'); + } + }); + }); + }); +}); + +/** + * @api {get} /i/app_users/deleteExport/:id Deletes user export. + * @apiName Delete user export + * @apiGroup App User Management + * @apiDescription Deletes user export. + * + * @apiParam {Number} id Id of export. + * + * @apiQuery {String} app_id Application id + * + * @apiSuccessExample {json} Success-Response: + * HTTP/1.1 200 OK + * { + * "result":"Export deleted" + * } + * @apiErrorExample {json} Error-Response: + * HTTP/1.1 400 Bad Request + * { + * "result": "Missing parameter \"app_id\"" + * } + */ +router.all('/i/app_users/deleteExport/:id', (req, res) => { + const params = req.countlyParams; + const paths = params.paths; + validateUserForWrite(params, function() { + countlyApi.mgmt.appUsers.deleteExport(paths[4], params, function(err) { + if (err) { + common.returnMessage(params, 400, err); + } + else { + common.returnMessage(params, 200, 'Export deleted'); + } + }); + }); +}); + +/** + * @api {get} /i/app_users/export Exports all data collected about app user + * @apiName Export user data + * @apiGroup App User Management + * + * @apiDescription Creates export and stores in database. + * @apiQuery {String} app_id Application id + * @apiQuery {String} query Query to match users to run export on. + * + * @apiSuccessExample {json} Success-Response: + * HTTP/1.1 200 OK + * { + * "result": "appUser_644658291e95e720503d5087_1.json" + * } + * @apiErrorExample {json} Error-Response: + * HTTP/1.1 400 Bad Request + * { + * "result": "Missing parameter \"app_id\"" + * } + */ +router.all('/i/app_users/export', (req, res) => { + const params = req.countlyParams; + if (!params.qstring.app_id) { + common.returnMessage(params, 400, 'Missing parameter "app_id"'); + return false; + } + validateUserForWrite(params, function() { + taskmanager.checkIfRunning({ + db: common.db, + params: params + }, function(task_id) { + if (task_id) { + common.returnOutput(params, {task_id: task_id}); + } + else { + if (!params.qstring.query) { + common.returnMessage(params, 400, 'Missing parameter "query"'); + return false; + } + else if (typeof params.qstring.query === "string") { + try { + params.qstring.query = JSON.parse(params.qstring.query); + } + catch (ex) { + console.log("Could not parse query", params.qstring.query); + common.returnMessage(params, 400, 'Could not parse parameter "query": ' + params.qstring.query); + return false; + } + } + + var my_name = ""; + if (params.qstring.query) { + my_name = JSON.stringify(params.qstring.query); + } + + countlyApi.mgmt.appUsers.export(params.qstring.app_id, params.qstring.query || {}, params, taskmanager.longtask({ + db: common.db, + threshold: plugins.getConfig("api").request_threshold, + force: false, + app_id: params.qstring.app_id, + params: params, + type: "AppUserExport", + report_name: "User export", + meta: JSON.stringify({ + "app_id": params.qstring.app_id, + "query": params.qstring.query || {} + }), + name: my_name, + view: "#/exportedData/AppUserExport/", + processData: function(err, result, callback) { + if (!err) { + callback(null, result); + } + else { + callback(err, ''); + } + }, + outputData: function(err, data) { + if (err) { + common.returnMessage(params, 400, err); + } + else { + common.returnMessage(params, 200, data); + } + } + })); + } + }); + }); +}); + +// Catch-all for /i/app_users/* - dispatches to plugins +router.all('/i/app_users/:action', (req, res) => { + const params = req.countlyParams; + const paths = params.paths; + const apiPath = '/i/app_users'; + if (!plugins.dispatch(apiPath, { + params: params, + validateUserForDataReadAPI: validateUserForDataReadAPI, + validateUserForMgmtReadAPI: validateUserForMgmtReadAPI, + paths: paths, + validateUserForDataWriteAPI: validateUserForDataWriteAPI, + validateUserForGlobalAdmin: validateUserForGlobalAdmin + })) { + common.returnMessage(params, 400, 'Invalid path, must be one of /all or /me'); + } +}); + +// --- Read endpoints: /o/app_users --- + +router.all('/o/app_users/loyalty', (req, res) => { + const params = req.countlyParams; + if (!params.qstring.app_id) { + common.returnMessage(params, 400, 'Missing parameter "app_id"'); + return false; + } + validateUserForRead(params, countlyApi.mgmt.appUsers.loyalty); +}); + +/** + * @api {get} /o/app_users/download/:id Downloads user export. + * @apiName Download user export + * @apiGroup App User Management + * @apiDescription Downloads users export + * + * @apiParam {Number} id Id of export. + * + * @apiQuery {String} app_id Application id + * + * @apiErrorExample {json} Error-Response: + * HTTP/1.1 400 Bad Request + * { + * "result": "Missing parameter \"app_id\"" + * } + */ +router.all('/o/app_users/download/:id', (req, res) => { + const params = req.countlyParams; + const paths = params.paths; + if (paths[4] && paths[4] !== '') { + validateUserForRead(params, function() { + var filename = paths[4].split('.'); + new Promise(function(resolve) { + if (filename[0].startsWith("appUser_")) { + filename[0] = filename[0] + '.tar.gz'; + resolve(); + } + else { //we have task result. Try getting from there + taskmanager.getResult({id: filename[0]}, function(err, result) { + if (result && result.data) { + filename[0] = result.data; + filename[0] = filename[0].replace(/"/g, ''); + } + resolve(); + }); + } + }).then(function() { + var myfile = '../../export/AppUser/' + filename[0]; + countlyFs.gridfs.getSize("appUsers", myfile, {id: filename[0]}, function(error, size) { + if (error) { + common.returnMessage(params, 400, error); + } + else if (parseInt(size) === 0) { + //export does not exist. lets check out export collection. + var eid = filename[0].split("."); + eid = eid[0]; + + var cursor = common.db.collection("exports").find({"_eid": eid}, {"_eid": 0, "_id": 0}); + var options = {"type": "stream", "filename": eid + ".json", params: params}; + params.res.writeHead(200, { + 'Content-Type': 'application/x-gzip', + 'Content-Disposition': 'inline; filename="' + eid + '.json' + }); + options.streamOptions = {}; + if (options.type === "stream" || options.type === "json") { + options.streamOptions.transform = function(doc) { + doc._id = doc.__id; + delete doc.__id; + return JSON.stringify(doc); + }; + } + + options.output = options.output || function(stream) { + countlyApi.data.exports.stream(options.params, stream, options); + }; + options.output(cursor); + } + else { + countlyFs.gridfs.getStream("appUsers", myfile, {id: filename[0]}, function(err, stream) { + if (err) { + common.returnMessage(params, 400, "Export doesn't exist"); + } + else { + params.res.writeHead(200, { + 'Content-Type': 'application/x-gzip', + 'Content-Length': size, + 'Content-Disposition': 'inline; filename="' + filename[0] + }); + stream.pipe(params.res); + } + }); + } + }); + }); + }); + } + else { + common.returnMessage(params, 400, 'Missing filename'); + } +}); + +// Catch-all for /o/app_users/* - dispatches to plugins +router.all('/o/app_users/:action', (req, res) => { + const params = req.countlyParams; + const paths = params.paths; + const apiPath = '/o/app_users'; + if (!plugins.dispatch(apiPath, { + params: params, + validateUserForDataReadAPI: validateUserForDataReadAPI, + validateUserForMgmtReadAPI: validateUserForMgmtReadAPI, + paths: paths, + validateUserForDataWriteAPI: validateUserForDataWriteAPI, + validateUserForGlobalAdmin: validateUserForGlobalAdmin + })) { + common.returnMessage(params, 400, 'Invalid path, must be one of /all or /me'); + } +}); + +module.exports = router; diff --git a/api/routes/apps.js b/api/routes/apps.js new file mode 100644 index 00000000000..d143629b98f --- /dev/null +++ b/api/routes/apps.js @@ -0,0 +1,131 @@ +/** + * App management routes (/i/apps, /o/apps). + * Migrated from the legacy switch/case in requestProcessor.js. + * @module api/routes/apps + */ + +const express = require('express'); +const router = express.Router(); +const common = require('../utils/common.js'); +const { validateAppAdmin, validateUser, validateRead, validateUserForWrite, validateGlobalAdmin } = require('../utils/rights.js'); +const plugins = require('../../plugins/pluginManager.ts'); + +const validateUserForDataReadAPI = validateRead; +const validateUserForMgmtReadAPI = validateUser; +const validateUserForDataWriteAPI = validateUserForWrite; +const validateUserForGlobalAdmin = validateGlobalAdmin; + +const countlyApi = { + mgmt: { + apps: require('../parts/mgmt/apps.js'), + } +}; + +// Helper: parse JSON args for /i/apps endpoints +function parseArgs(params) { + if (params.qstring.args) { + try { + params.qstring.args = JSON.parse(params.qstring.args); + } + catch (SyntaxError) { + console.log('Parse /i/apps JSON failed %s', params.req.url, params.req.body); + } + } +} + +// --- Write endpoints: /i/apps --- + +router.all('/i/apps/create', (req, res) => { + const params = req.countlyParams; + parseArgs(params); + validateUserForGlobalAdmin(params, countlyApi.mgmt.apps.createApp); +}); + +// /i/apps/update/plugins - must come before the generic /i/apps/update route +router.all('/i/apps/update/plugins', (req, res) => { + const params = req.countlyParams; + parseArgs(params); + validateAppAdmin(params, countlyApi.mgmt.apps.updateAppPlugins); +}); + +router.all('/i/apps/update', (req, res) => { + const params = req.countlyParams; + parseArgs(params); + if (params.qstring.app_id) { + validateAppAdmin(params, countlyApi.mgmt.apps.updateApp); + } + else { + validateUserForGlobalAdmin(params, countlyApi.mgmt.apps.updateApp); + } +}); + +router.all('/i/apps/delete', (req, res) => { + const params = req.countlyParams; + parseArgs(params); + validateUserForGlobalAdmin(params, countlyApi.mgmt.apps.deleteApp); +}); + +router.all('/i/apps/reset', (req, res) => { + const params = req.countlyParams; + parseArgs(params); + validateUserForGlobalAdmin(params, countlyApi.mgmt.apps.resetApp); +}); + +// Catch-all for /i/apps/* - dispatches to plugins or returns error +router.all('/i/apps/:action', (req, res) => { + const params = req.countlyParams; + const apiPath = '/i/apps'; + const paths = params.paths; + parseArgs(params); + if (!plugins.dispatch(apiPath, { + params: params, + validateUserForDataReadAPI: validateUserForDataReadAPI, + validateUserForMgmtReadAPI: validateUserForMgmtReadAPI, + paths: paths, + validateUserForDataWriteAPI: validateUserForDataWriteAPI, + validateUserForGlobalAdmin: validateUserForGlobalAdmin + })) { + common.returnMessage(params, 400, 'Invalid path, must be one of /create, /update, /delete or /reset'); + } +}); + +// --- Read endpoints: /o/apps --- + +router.all('/o/apps/all', (req, res) => { + const params = req.countlyParams; + validateUserForGlobalAdmin(params, countlyApi.mgmt.apps.getAllApps); +}); + +router.all('/o/apps/mine', (req, res) => { + const params = req.countlyParams; + validateUser(params, countlyApi.mgmt.apps.getCurrentUserApps); +}); + +router.all('/o/apps/details', (req, res) => { + const params = req.countlyParams; + validateAppAdmin(params, countlyApi.mgmt.apps.getAppsDetails); +}); + +router.all('/o/apps/plugins', (req, res) => { + const params = req.countlyParams; + validateUserForGlobalAdmin(params, countlyApi.mgmt.apps.getAppPlugins); +}); + +// Catch-all for /o/apps/* - dispatches to plugins or returns error +router.all('/o/apps/:action', (req, res) => { + const params = req.countlyParams; + const apiPath = '/o/apps'; + const paths = params.paths; + if (!plugins.dispatch(apiPath, { + params: params, + validateUserForDataReadAPI: validateUserForDataReadAPI, + validateUserForMgmtReadAPI: validateUserForMgmtReadAPI, + paths: paths, + validateUserForDataWriteAPI: validateUserForDataWriteAPI, + validateUserForGlobalAdmin: validateUserForGlobalAdmin + })) { + common.returnMessage(params, 400, 'Invalid path, must be one of /all, /mine, /details or /plugins'); + } +}); + +module.exports = router; diff --git a/api/routes/cms.js b/api/routes/cms.js new file mode 100644 index 00000000000..f3d6ab220bb --- /dev/null +++ b/api/routes/cms.js @@ -0,0 +1,58 @@ +/** + * CMS routes - content management system entries and cache. + * Migrated from the legacy switch/case in requestProcessor.js. + * @module api/routes/cms + */ + +const express = require('express'); +const router = express.Router(); +const common = require('../utils/common.js'); +const plugins = require('../../plugins/pluginManager.ts'); +const { validateUser, validateUserForWrite, validateRead, validateGlobalAdmin } = require('../utils/rights.js'); +const validateUserForMgmtReadAPI = validateUser; +const validateUserForDataReadAPI = validateRead; +const validateUserForDataWriteAPI = validateUserForWrite; +const validateUserForGlobalAdmin = validateGlobalAdmin; +const countlyApi = { + mgmt: { + cms: require('../parts/mgmt/cms.js') + } +}; + +// --- Read endpoints: /o/cms --- + +router.all('/o/cms/entries', (req, res) => { + const params = req.countlyParams; + validateUserForMgmtReadAPI(countlyApi.mgmt.cms.getEntries, params); +}); + +// --- Write endpoints: /i/cms --- + +router.all('/i/cms/save_entries', (req, res) => { + const params = req.countlyParams; + validateUserForWrite(params, countlyApi.mgmt.cms.saveEntries); +}); + +router.all('/i/cms/clear', (req, res) => { + const params = req.countlyParams; + validateUserForWrite(countlyApi.mgmt.cms.clearCache, params); +}); + +// Catch-all for /i/cms/* - dispatches to plugins or returns error +router.all('/i/cms/:action', (req, res) => { + const params = req.countlyParams; + const paths = params.paths; + const apiPath = '/i/cms'; + if (!plugins.dispatch(apiPath, { + params: params, + validateUserForDataReadAPI: validateUserForDataReadAPI, + validateUserForMgmtReadAPI: validateUserForMgmtReadAPI, + paths: paths, + validateUserForDataWriteAPI: validateUserForDataWriteAPI, + validateUserForGlobalAdmin: validateUserForGlobalAdmin + })) { + common.returnMessage(params, 400, 'Invalid path, must be one of /save_entries or /clear'); + } +}); + +module.exports = router; diff --git a/api/routes/data.js b/api/routes/data.js new file mode 100644 index 00000000000..aa053ba025c --- /dev/null +++ b/api/routes/data.js @@ -0,0 +1,224 @@ +/** + * Generic data read routes (/o with method-based dispatch). + * Migrated from the legacy switch/case in requestProcessor.js. + * @module api/routes/data + */ + +const express = require('express'); +const router = express.Router(); +const common = require('../utils/common.js'); +const { validateUser, validateRead, validateUserForWrite, validateGlobalAdmin } = require('../utils/rights.js'); +const plugins = require('../../plugins/pluginManager.ts'); + +const validateUserForDataReadAPI = validateRead; +const validateUserForMgmtReadAPI = validateUser; +const validateUserForDataWriteAPI = validateUserForWrite; +const validateUserForGlobalAdmin = validateGlobalAdmin; + +const countlyApi = { + data: { + fetch: require('../parts/data/fetch.js'), + geoData: require('../parts/data/geoData.ts').default + }, + mgmt: { + users: require('../parts/mgmt/users.js'), + } +}; + +/** + * Generic /o endpoint - method-based dispatch. + * Requires app_id parameter and dispatches based on params.qstring.method. + */ +router.all('/o', (req, res) => { + const params = req.countlyParams; + + if (!params.qstring.app_id) { + common.returnMessage(params, 400, 'Missing parameter "app_id"'); + return false; + } + + switch (params.qstring.method) { + case 'total_users': + validateUserForDataReadAPI(params, 'core', countlyApi.data.fetch.fetchTotalUsersObj, params.qstring.metric || 'users'); + break; + case 'get_period_obj': + validateUserForDataReadAPI(params, 'core', countlyApi.data.fetch.getPeriodObj, 'users'); + break; + case 'locations': + case 'sessions': + case 'users': + validateUserForDataReadAPI(params, 'core', countlyApi.data.fetch.fetchTimeObj, 'users'); + break; + case 'app_versions': + case 'device_details': + validateUserForDataReadAPI(params, 'core', countlyApi.data.fetch.fetchTimeObj, 'device_details'); + break; + case 'devices': + case 'carriers': + validateUserForDataReadAPI(params, 'core', countlyApi.data.fetch.fetchTimeObj, params.qstring.method); + break; + case 'countries': + if (plugins.getConfig("api", params.app && params.app.plugins, true).country_data !== false) { + validateUserForDataReadAPI(params, 'core', countlyApi.data.fetch.fetchTimeObj, params.qstring.method); + } + else { + common.returnOutput(params, {}); + } + break; + case 'cities': + if (plugins.getConfig("api", params.app && params.app.plugins, true).city_data !== false) { + validateUserForDataReadAPI(params, 'core', countlyApi.data.fetch.fetchTimeObj, params.qstring.method); + } + else { + common.returnOutput(params, {}); + } + break; + case 'geodata': { + validateRead(params, 'core', function() { + if (params.qstring.loadFor === "cities") { + countlyApi.data.geoData.loadCityCoordiantes({"query": params.qstring.query}, function(err, data) { + common.returnOutput(params, data); + }); + } + }); + break; + } + case 'get_event_groups': + validateRead(params, 'core', countlyApi.data.fetch.fetchEventGroups); + break; + case 'get_event_group': + validateRead(params, 'core', countlyApi.data.fetch.fetchEventGroupById); + break; + case 'events': + if (params.qstring.events) { + try { + params.qstring.events = JSON.parse(params.qstring.events); + } + catch (SyntaxError) { + console.log('Parse events array failed', params.qstring.events, params.req.url, params.req.body); + } + if (params.qstring.overview) { + validateRead(params, 'core', countlyApi.data.fetch.fetchDataEventsOverview); + } + else { + validateRead(params, 'core', countlyApi.data.fetch.fetchMergedEventData); + } + } + else { + if (params.qstring.event && params.qstring.event.startsWith('[CLY]_group_')) { + validateRead(params, 'core', countlyApi.data.fetch.fetchMergedEventGroups); + } + else { + params.truncateEventValuesList = true; + validateRead(params, 'core', countlyApi.data.fetch.prefetchEventData, params.qstring.method); + } + } + break; + case 'get_events': + validateRead(params, 'core', async function() { + try { + var result = await common.db.collection("events").findOne({ '_id': common.db.ObjectID(params.qstring.app_id) }); + result = result || {}; + result.list = result.list || []; + result.segments = result.segments || {}; + + if (result.list) { + result.list = result.list.filter(function(l) { + return l.indexOf('[CLY]') !== 0; + }); + } + if (result.segments) { + for (let i in result.segments) { + if (i.indexOf('[CLY]') === 0) { + delete result.segments[i]; + } + } + } + const pluginsGetConfig = plugins.getConfig("api", params.app && params.app.plugins, true); + result.limits = { + event_limit: pluginsGetConfig.event_limit, + event_segmentation_limit: pluginsGetConfig.event_segmentation_limit, + event_segmentation_value_limit: pluginsGetConfig.event_segmentation_value_limit, + }; + + var aggregation = []; + aggregation.push({$match: {"app_id": params.qstring.app_id, "type": "e", "biglist": {"$ne": true}}}); + aggregation.push({"$project": {e: 1, _id: 0, "sg": 1}}); + //e does not start with [CLY]_ + aggregation.push({$match: {"e": {"$not": /^(\[CLY\]_)/}}}); + aggregation.push({"$sort": {"e": 1}}); + aggregation.push({"$limit": pluginsGetConfig.event_limit || 500}); + + var drillRes = await common.drillDb.collection("drill_meta").aggregate(aggregation).toArray(); + for (var k = 0; k < drillRes.length; k++) { + if (result.list.indexOf(drillRes[k].e) === -1) { + result.list.push(drillRes[k].e); + } + + if (drillRes[k].sg && Object.keys(drillRes[k].sg).length > 0) { + result.segments[drillRes[k].e] = result.segments[drillRes[k].e] || []; + for (var key in drillRes[k].sg) { + if (result.segments[drillRes[k].e].indexOf(key) === -1) { + result.segments[drillRes[k].e].push(key); + } + } + } + if (result.omitted_segments && result.omitted_segments[drillRes[k].e]) { + for (let kz = 0; kz < result.omitted_segments[drillRes[k].e].length; kz++) { + //remove items that are in omitted list + result.segments[drillRes[k].e].splice(result.segments[drillRes[k].e].indexOf(result.omitted_segments[drillRes[k].e][kz]), 1); + } + } + if (result.whitelisted_segments && result.whitelisted_segments[drillRes[k].e] && Array.isArray(result.whitelisted_segments[drillRes[k].e])) { + //remove all that are not whitelisted + for (let kz = 0; kz < result.segments[drillRes[k].e].length; kz++) { + if (result.whitelisted_segments[drillRes[k].e].indexOf(result.segments[drillRes[k].e][kz]) === -1) { + result.segments[drillRes[k].e].splice(kz, 1); + kz--; + } + } + } + //Sort segments + if (result.segments[drillRes[k].e]) { + result.segments[drillRes[k].e].sort(); + } + } + if (result.list.length === 0) { + delete result.list; + } + if (Object.keys(result.segments).length === 0) { + delete result.segments; + } + common.returnOutput(params, result); + + } + catch (ex) { + console.error("Error fetching events", ex); + common.returnMessage(params, 500, "Error fetching events"); + } + }, 'events'); + break; + case 'top_events': + validateRead(params, 'core', countlyApi.data.fetch.fetchDataTopEvents); + break; + case 'all_apps': + validateUserForGlobalAdmin(params, countlyApi.data.fetch.fetchAllApps); + break; + case 'notes': + validateRead(params, 'core', countlyApi.mgmt.users.fetchNotes); + break; + default: + if (!plugins.dispatch('/o', { + params: params, + validateUserForDataReadAPI: validateUserForDataReadAPI, + validateUserForMgmtReadAPI: validateUserForMgmtReadAPI, + validateUserForDataWriteAPI: validateUserForDataWriteAPI, + validateUserForGlobalAdmin: validateUserForGlobalAdmin + })) { + common.returnMessage(params, 400, 'Invalid method'); + } + break; + } +}); + +module.exports = router; diff --git a/api/routes/date_presets.js b/api/routes/date_presets.js new file mode 100644 index 00000000000..157f5599fb8 --- /dev/null +++ b/api/routes/date_presets.js @@ -0,0 +1,85 @@ +/** + * Date presets routes - custom date range presets. + * Migrated from the legacy switch/case in requestProcessor.js. + * @module api/routes/date_presets + */ + +const express = require('express'); +const router = express.Router(); +const common = require('../utils/common.js'); +const plugins = require('../../plugins/pluginManager.ts'); +const { validateUser, validateUserForWrite, validateRead, validateGlobalAdmin } = require('../utils/rights.js'); +const validateUserForMgmtReadAPI = validateUser; +const validateUserForDataReadAPI = validateRead; +const validateUserForDataWriteAPI = validateUserForWrite; +const validateUserForGlobalAdmin = validateGlobalAdmin; +const countlyApi = { + mgmt: { + datePresets: require('../parts/mgmt/date_presets.js') + } +}; + +// --- Read endpoints: /o/date_presets --- + +router.all('/o/date_presets/getAll', (req, res) => { + const params = req.countlyParams; + validateUserForMgmtReadAPI(countlyApi.mgmt.datePresets.getAll, params); +}); + +router.all('/o/date_presets/getById', (req, res) => { + const params = req.countlyParams; + validateUserForMgmtReadAPI(countlyApi.mgmt.datePresets.getById, params); +}); + +// Catch-all for /o/date_presets/* - dispatches to plugins or returns error +router.all('/o/date_presets/:action', (req, res) => { + const params = req.countlyParams; + const paths = params.paths; + const apiPath = '/o/date_presets'; + if (!plugins.dispatch(apiPath, { + params: params, + validateUserForDataReadAPI: validateUserForDataReadAPI, + validateUserForMgmtReadAPI: validateUserForMgmtReadAPI, + paths: paths, + validateUserForDataWriteAPI: validateUserForDataWriteAPI, + validateUserForGlobalAdmin: validateUserForGlobalAdmin + })) { + common.returnMessage(params, 400, 'Invalid path, must be one of /getAll /getById'); + } +}); + +// --- Write endpoints: /i/date_presets --- + +router.all('/i/date_presets/create', (req, res) => { + const params = req.countlyParams; + validateUserForWrite(params, countlyApi.mgmt.datePresets.create); +}); + +router.all('/i/date_presets/update', (req, res) => { + const params = req.countlyParams; + validateUserForWrite(params, countlyApi.mgmt.datePresets.update); +}); + +router.all('/i/date_presets/delete', (req, res) => { + const params = req.countlyParams; + validateUserForWrite(params, countlyApi.mgmt.datePresets.delete); +}); + +// Catch-all for /i/date_presets/* - dispatches to plugins or returns error +router.all('/i/date_presets/:action', (req, res) => { + const params = req.countlyParams; + const paths = params.paths; + const apiPath = '/i/date_presets'; + if (!plugins.dispatch(apiPath, { + params: params, + validateUserForDataReadAPI: validateUserForDataReadAPI, + validateUserForMgmtReadAPI: validateUserForMgmtReadAPI, + paths: paths, + validateUserForDataWriteAPI: validateUserForDataWriteAPI, + validateUserForGlobalAdmin: validateUserForGlobalAdmin + })) { + common.returnMessage(params, 400, 'Invalid path, must be one of /create /update or /delete'); + } +}); + +module.exports = router; diff --git a/api/routes/event_groups.js b/api/routes/event_groups.js new file mode 100644 index 00000000000..4398f43087a --- /dev/null +++ b/api/routes/event_groups.js @@ -0,0 +1,48 @@ +/** + * Event groups management routes (/i/event_groups). + * Migrated from the legacy switch/case in requestProcessor.js. + * @module api/routes/event_groups + */ + +const express = require('express'); +const router = express.Router(); +const common = require('../utils/common.js'); +const { validateCreate, validateUpdate, validateDelete } = require('../utils/rights.js'); + +const countlyApi = { + mgmt: { + eventGroups: require('../parts/mgmt/event_groups.js'), + } +}; + +// --- Write endpoints: /i/event_groups --- + +router.all('/i/event_groups/create', (req, res) => { + const params = req.countlyParams; + if (!params.qstring.args) { + common.returnMessage(params, 400, 'Error: args not found'); + return false; + } + try { + params.qstring.args = JSON.parse(params.qstring.args); + params.qstring.app_id = params.qstring.args.app_id; + } + catch (SyntaxError) { + console.log('Parse /i/event_groups JSON failed %s', params.req.url, params.req.body); + common.returnMessage(params, 400, 'Error: could not parse args'); + return false; + } + validateCreate(params, 'core', countlyApi.mgmt.eventGroups.create); +}); + +router.all('/i/event_groups/update', (req, res) => { + const params = req.countlyParams; + validateUpdate(params, 'core', countlyApi.mgmt.eventGroups.update); +}); + +router.all('/i/event_groups/delete', (req, res) => { + const params = req.countlyParams; + validateDelete(params, 'core', countlyApi.mgmt.eventGroups.remove); +}); + +module.exports = router; diff --git a/api/routes/events.js b/api/routes/events.js new file mode 100644 index 00000000000..d50aefadc3d --- /dev/null +++ b/api/routes/events.js @@ -0,0 +1,680 @@ +/** + * Events management routes (/i/events). + * Migrated from the legacy switch/case in requestProcessor.js. + * @module api/routes/events + */ + +const express = require('express'); +const router = express.Router(); +const common = require('../utils/common.js'); +const { validateUser, validateRead, validateUserForWrite, validateGlobalAdmin, validateUpdate, validateDelete } = require('../utils/rights.js'); +const plugins = require('../../plugins/pluginManager.ts'); +const log = require('../utils/log.js')('core:api'); + +const validateUserForDataReadAPI = validateRead; +const validateUserForMgmtReadAPI = validateUser; +const validateUserForDataWriteAPI = validateUserForWrite; +const validateUserForGlobalAdmin = validateGlobalAdmin; + +// --- /i/events endpoints --- + +router.all('/i/events/whitelist_segments', (req, res) => { + const params = req.countlyParams; + validateUpdate(params, "events", function() { + common.db.collection('events').findOne({"_id": common.db.ObjectID(params.qstring.app_id)}, function(err, event) { + if (err) { + common.returnMessage(params, 400, err); + return; + } + else if (!event) { + common.returnMessage(params, 400, "Could not find record in event collection"); + return; + } + + //rewrite whitelisted + if (params.qstring.whitelisted_segments && params.qstring.whitelisted_segments !== "") { + try { + params.qstring.whitelisted_segments = JSON.parse(params.qstring.whitelisted_segments); + } + catch (SyntaxError) { + params.qstring.whitelisted_segments = {}; console.log('Parse ' + params.qstring.whitelisted_segments + ' JSON failed', params.req.url, params.req.body); + } + + var update = {}; + var whObj = params.qstring.whitelisted_segments; + for (let k in whObj) { + if (Array.isArray(whObj[k]) && whObj[k].length > 0) { + update.$set = update.$set || {}; + update.$set["whitelisted_segments." + k] = whObj[k]; + } + else { + update.$unset = update.$unset || {}; + update.$unset["whitelisted_segments." + k] = true; + } + } + + common.db.collection('events').update({"_id": common.db.ObjectID(params.qstring.app_id)}, update, function(err2) { + if (err2) { + common.returnMessage(params, 400, err2); + } + else { + var data_arr = {update: {}}; + if (update.$set) { + data_arr.update.$set = update.$set; + } + + if (update.$unset) { + data_arr.update.$unset = update.$unset; + } + data_arr.update = JSON.stringify(data_arr.update); + common.returnMessage(params, 200, 'Success'); + plugins.dispatch("/systemlogs", { + params: params, + action: "segments_whitelisted_for_events", + data: data_arr + }); + } + }); + + } + else { + common.returnMessage(params, 400, "Value for 'whitelisted_segments' missing"); + return; + } + + + }); + }); +}); + +router.all('/i/events/edit_map', (req, res) => { + const params = req.countlyParams; + if (!params.qstring.app_id) { + common.returnMessage(params, 400, 'Missing parameter "app_id"'); + return false; + } + validateUpdate(params, 'events', function() { + common.db.collection('events').findOne({"_id": common.db.ObjectID(params.qstring.app_id)}, async function(err, event) { + if (err) { + common.returnMessage(params, 400, err); + return; + } + else if (!event) { + common.returnMessage(params, 400, "Could not find event"); + return; + } + //Load available events + + const pluginsGetConfig = plugins.getConfig("api", params.app && params.app.plugins, true); + + var list = await common.drillDb.collection("drill_meta").aggregate( + [ + {$match: {"app_id": params.qstring.app_id, "type": "e", "biglist": {"$ne": true}}}, + {$match: {"e": {"$not": /^(\[CLY\]_)/}}}, + {"$group": {"_id": "$e"}}, + {"$sort": {"_id": 1}}, + {"$limit": pluginsGetConfig.event_limit || 500}, + {"$group": {"_id": null, "list": {"$addToSet": "$_id"}}} + ] + , {"allowDiskUse": true}).toArray(); + event.list = list[0].list; + + var update_array = {}; + var update_segments = []; + var pull_us = {}; + if (params.qstring.event_order && params.qstring.event_order !== "") { + try { + update_array.order = JSON.parse(params.qstring.event_order); + } + catch (SyntaxError) { + update_array.order = event.order; console.log('Parse ' + params.qstring.event_order + ' JSON failed', params.req.url, params.req.body); + } + } + else { + update_array.order = event.order || []; + } + + if (params.qstring.event_overview && params.qstring.event_overview !== "") { + try { + update_array.overview = JSON.parse(params.qstring.event_overview); + } + catch (SyntaxError) { + update_array.overview = []; console.log('Parse ' + params.qstring.event_overview + ' JSON failed', params.req.url, params.req.body); + } + if (update_array.overview && Array.isArray(update_array.overview)) { + if (update_array.overview.length > 12) { + common.returnMessage(params, 400, "You can't add more than 12 items in overview"); + return; + } + //sanitize overview + var allowedEventKeys = event.list; + var allowedProperties = ['dur', 'sum', 'count']; + var propertyNames = { + 'dur': 'Dur', + 'sum': 'Sum', + 'count': 'Count' + }; + for (let i = 0; i < update_array.overview.length; i++) { + update_array.overview[i].order = i; + update_array.overview[i].eventKey = update_array.overview[i].eventKey || ""; + update_array.overview[i].eventProperty = update_array.overview[i].eventProperty || ""; + if (allowedEventKeys.indexOf(update_array.overview[i].eventKey) === -1 || allowedProperties.indexOf(update_array.overview[i].eventProperty) === -1) { + update_array.overview.splice(i, 1); + i = i - 1; + } + else { + update_array.overview[i].is_event_group = (typeof update_array.overview[i].is_event_group === 'boolean' && update_array.overview[i].is_event_group) || false; + update_array.overview[i].eventName = update_array.overview[i].eventName || update_array.overview[i].eventKey; + update_array.overview[i].propertyName = propertyNames[update_array.overview[i].eventProperty]; + } + } + //check for duplicates + var overview_map = Object.create(null); + for (let p = 0; p < update_array.overview.length; p++) { + if (!overview_map[update_array.overview[p].eventKey]) { + overview_map[update_array.overview[p].eventKey] = {}; + } + if (!overview_map[update_array.overview[p].eventKey][update_array.overview[p].eventProperty]) { + overview_map[update_array.overview[p].eventKey][update_array.overview[p].eventProperty] = 1; + } + else { + update_array.overview.splice(p, 1); + p = p - 1; + } + } + } + } + else { + update_array.overview = event.overview || []; + } + + update_array.omitted_segments = {}; + + if (event.omitted_segments) { + try { + update_array.omitted_segments = JSON.parse(JSON.stringify(event.omitted_segments)); + } + catch (SyntaxError) { + update_array.omitted_segments = {}; + } + } + + if (params.qstring.omitted_segments && params.qstring.omitted_segments !== "") { + var omitted_segments_empty = false; + try { + params.qstring.omitted_segments = JSON.parse(params.qstring.omitted_segments); + if (JSON.stringify(params.qstring.omitted_segments) === '{}') { + omitted_segments_empty = true; + } + } + catch (SyntaxError) { + params.qstring.omitted_segments = {}; console.log('Parse ' + params.qstring.omitted_segments + ' JSON failed', params.req.url, params.req.body); + } + + for (let k in params.qstring.omitted_segments) { + update_array.omitted_segments[k] = params.qstring.omitted_segments[k]; + update_segments.push({ + "key": k, + "list": params.qstring.omitted_segments[k] + }); + pull_us["segments." + k] = {$in: params.qstring.omitted_segments[k]}; + } + if (omitted_segments_empty) { + var events = JSON.parse(params.qstring.event_map); + for (let k in events) { + if (update_array.omitted_segments[k]) { + delete update_array.omitted_segments[k]; + } + } + } + } + + if (params.qstring.event_map && params.qstring.event_map !== "") { + try { + params.qstring.event_map = JSON.parse(params.qstring.event_map); + } + catch (SyntaxError) { + params.qstring.event_map = {}; console.log('Parse ' + params.qstring.event_map + ' JSON failed', params.req.url, params.req.body); + } + + if (event.map) { + try { + update_array.map = JSON.parse(JSON.stringify(event.map)); + } + catch (SyntaxError) { + update_array.map = {}; + } + } + else { + update_array.map = {}; + } + + + for (let k in params.qstring.event_map) { + if (Object.prototype.hasOwnProperty.call(params.qstring.event_map, k)) { + update_array.map[k] = params.qstring.event_map[k]; + + if (update_array.map[k].is_visible && update_array.map[k].is_visible === true) { + delete update_array.map[k].is_visible; + } + if (update_array.map[k].name && update_array.map[k].name === k) { + delete update_array.map[k].name; + } + + if (update_array.map[k] && typeof update_array.map[k].is_visible !== 'undefined' && update_array.map[k].is_visible === false) { + for (var j = 0; j < update_array.overview.length; j++) { + if (update_array.overview[j].eventKey === k) { + update_array.overview.splice(j, 1); + j = j - 1; + } + } + } + if (Object.keys(update_array.map[k]).length === 0) { + delete update_array.map[k]; + } + } + } + } + var changes = {$set: update_array}; + if (Object.keys(pull_us).length > 0) { + changes = { + $set: update_array, + $pull: pull_us + }; + } + + common.db.collection('events').update({"_id": common.db.ObjectID(params.qstring.app_id)}, changes, function(err2) { + if (err2) { + common.returnMessage(params, 400, err2); + } + else { + var data_arr = {update: update_array}; + data_arr.before = { + order: [], + map: {}, + overview: [], + omitted_segments: {} + }; + if (event.order) { + data_arr.before.order = event.order; + } + if (event.map) { + data_arr.before.map = event.map; + } + if (event.overview) { + data_arr.before.overview = event.overview; + } + if (event.omitted_segments) { + data_arr.before.omitted_segments = event.omitted_segments; + } + + //updated, clear out segments + Promise.all(update_segments.map(function(obj) { + return new Promise(function(resolve) { + var collectionNameWoPrefix = common.crypto.createHash('sha1').update(obj.key + params.qstring.app_id).digest('hex'); + //removes all document for current segment + common.db.collection("events_data").remove({"_id": {"$regex": ("^" + params.qstring.app_id + "_" + collectionNameWoPrefix + "_.*")}, "s": {$in: obj.list}}, {multi: true}, function(err3) { + if (err3) { + console.log(err3); + } + //create query for all segments + var my_query = []; + var unsetUs = {}; + if (obj.list.length > 0) { + for (let p = 0; p < obj.list.length; p++) { + my_query[p] = {}; + my_query[p]["meta_v2.segments." + obj.list[p]] = {$exists: true}; //for select + unsetUs["meta_v2.segments." + obj.list[p]] = ""; //remove from list + unsetUs["meta_v2." + obj.list[p]] = ""; + } + //clears out meta data for segments + common.db.collection("events_data").update({"_id": {"$regex": ("^" + params.qstring.app_id + "_" + collectionNameWoPrefix + "_.*")}, $or: my_query}, {$unset: unsetUs}, {multi: true}, function(err4) { + if (err4) { + console.log(err4); + } + if (plugins.isPluginEnabled('drill')) { + //remove from drill + var eventHash = common.crypto.createHash('sha1').update(obj.key + params.qstring.app_id).digest('hex'); + common.drillDb.collection("drill_meta").findOne({_id: params.qstring.app_id + "_meta_" + eventHash}, function(err5, resEvent) { + if (err5) { + console.log(err5); + } + + var newsg = {}; + var remove_biglists = []; + resEvent = resEvent || {}; + resEvent.sg = resEvent.sg || {}; + for (let p = 0; p < obj.list.length; p++) { + remove_biglists.push(params.qstring.app_id + "_meta_" + eventHash + "_sg." + obj.list[p]); + newsg["sg." + obj.list[p]] = {"type": "s"}; + } + //big list, delete also big list file + if (remove_biglists.length > 0) { + common.drillDb.collection("drill_meta").remove({_id: {$in: remove_biglists}}, function(err6) { + if (err6) { + console.log(err6); + } + common.drillDb.collection("drill_meta").update({_id: params.qstring.app_id + "_meta_" + eventHash}, {$set: newsg}, function(err7) { + if (err7) { + console.log(err7); + } + resolve(); + }); + }); + } + else { + common.drillDb.collection("drill_meta").update({_id: params.qstring.app_id + "_meta_" + eventHash}, {$set: newsg}, function() { + resolve(); + }); + } + }); + } + else { + resolve(); + } + }); + } + else { + resolve(); + } + }); + }); + + })).then(function() { + common.returnMessage(params, 200, 'Success'); + plugins.dispatch("/systemlogs", { + params: params, + action: "events_updated", + data: data_arr + }); + + }) + .catch((error) => { + console.log(error); + common.returnMessage(params, 400, 'Events were updated sucessfully. There was error during clearing segment data. Please look in log for more onformation'); + }); + + } + }); + }); + }); +}); + +/** + * @api {get} /i/events/delete_events Delete event + * @apiName Delete Event + * @apiGroup Events Management + * + * @apiDescription Deletes one or multiple events. + * @apiQuery {String} app_id Application id + * @apiQuery {String} events JSON array of event keys to delete. + * + * @apiSuccessExample {json} Success-Response: + * HTTP/1.1 200 OK + * { + * "result":"Success" + * } + */ +router.all('/i/events/delete_events', (req, res) => { + const params = req.countlyParams; + validateDelete(params, 'events', function() { + var idss = []; + try { + idss = JSON.parse(params.qstring.events); + } + catch (SyntaxError) { + idss = []; + } + + if (!Array.isArray(idss)) { + idss = []; + } + + var app_id = params.qstring.app_id; + var updateThese = {"$unset": {}}; + if (idss.length > 0) { + + common.db.collection('events').findOne({"_id": common.db.ObjectID(params.qstring.app_id)}, function(err, event) { + if (err) { + common.returnMessage(params, 400, err); + } + if (!event) { + common.returnMessage(params, 400, "Could not find event"); + return; + } + let successIds = []; + let failedIds = []; + let promises = []; + for (let i = 0; i < idss.length; i++) { + let collectionNameWoPrefix = common.crypto.createHash('sha1').update(idss[i] + app_id).digest('hex'); + common.db.collection("events" + collectionNameWoPrefix).drop(); + promises.push(new Promise((resolve, reject) => { + plugins.dispatch("/i/event/delete", { + event_key: idss[i], + appId: app_id + }, function(_, otherPluginResults) { + const rejectReasons = otherPluginResults?.reduce((acc, result) => { + if (result?.status === "rejected") { + acc.push((result.reason && result.reason.message) || ''); + } + return acc; + }, []); + + if (rejectReasons?.length) { + failedIds.push(idss[i]); + log.e("Event deletion failed\n%j", rejectReasons.join("\n")); + reject("Event deletion failed. Failed to delete some data related to this Event."); + return; + } + else { + successIds.push(idss[i]); + resolve(); + } + } + ); + })); + } + + Promise.allSettled(promises).then(async() => { + //remove from map, segments, omitted_segments + for (let i = 0; i < successIds.length; i++) { + successIds[i] = successIds[i] + ""; //make sure it is string to do not fail. + if (successIds[i].indexOf('.') !== -1) { + updateThese.$unset["map." + successIds[i].replace(/\./g, '\\u002e')] = 1; + updateThese.$unset["omitted_segments." + successIds[i].replace(/\./g, '\\u002e')] = 1; + } + else { + updateThese.$unset["map." + successIds[i]] = 1; + updateThese.$unset["omitted_segments." + successIds[i]] = 1; + } + successIds[i] = common.decode_html(successIds[i]);//previously escaped, get unescaped id (because segments are using it) + if (successIds[i].indexOf('.') !== -1) { + updateThese.$unset["segments." + successIds[i].replace(/\./g, '\\u002e')] = 1; + } + else { + updateThese.$unset["segments." + successIds[i]] = 1; + } + } + //fix overview + if (event.overview && event.overview.length) { + for (let i = 0; i < successIds.length; i++) { + for (let j = 0; j < event.overview.length; j++) { + if (event.overview[j].eventKey === successIds[i]) { + event.overview.splice(j, 1); + j = j - 1; + } + } + } + if (!updateThese.$set) { + updateThese.$set = {}; + } + updateThese.$set.overview = event.overview; + } + //remove from list + if (typeof event.list !== 'undefined' && Array.isArray(event.list) && event.list.length > 0) { + for (let i = 0; i < successIds.length; i++) { + let index = event.list.indexOf(successIds[i]); + if (index > -1) { + event.list.splice(index, 1); + i = i - 1; + } + } + if (!updateThese.$set) { + updateThese.$set = {}; + } + updateThese.$set.list = event.list; + } + //remove from order + if (typeof event.order !== 'undefined' && Array.isArray(event.order) && event.order.length > 0) { + for (let i = 0; i < successIds.length; i++) { + let index = event.order.indexOf(successIds[i]); + if (index > -1) { + event.order.splice(index, 1); + i = i - 1; + } + } + if (!updateThese.$set) { + updateThese.$set = {}; + } + updateThese.$set.order = event.order; + } + + await common.db.collection('events').update({ "_id": common.db.ObjectID(app_id) }, updateThese); + + plugins.dispatch("/systemlogs", { + params: params, + action: "event_deleted", + data: { + events: successIds, + appID: app_id + } + }); + + common.returnMessage(params, 200, 'Success'); + + }).catch((err2) => { + if (failedIds.length) { + log.e("Event deletion failed for following Event keys:\n%j", failedIds.join("\n")); + } + log.e("Event deletion failed\n%j", err2); + common.returnMessage(params, 500, { errorMessage: "Event deletion failed. Failed to delete some data related to this Event." }); + }); + }); + } + else { + common.returnMessage(params, 400, "Missing events to delete"); + } + }); +}); + +router.all('/i/events/change_visibility', (req, res) => { + const params = req.countlyParams; + validateUpdate(params, 'events', function() { + common.db.collection('events').findOne({"_id": common.db.ObjectID(params.qstring.app_id)}, function(err, event) { + if (err) { + common.returnMessage(params, 400, err); + return; + } + if (!event) { + common.returnMessage(params, 400, "Could not find event"); + return; + } + + var update_array = {}; + var idss = []; + try { + idss = JSON.parse(params.qstring.events); + } + catch (SyntaxError) { + idss = []; + } + if (!Array.isArray(idss)) { + idss = []; + } + + if (event.map) { + try { + update_array.map = JSON.parse(JSON.stringify(event.map)); + } + catch (SyntaxError) { + update_array.map = {}; + console.log('Parse ' + event.map + ' JSON failed', params.req.url, params.req.body); + } + } + else { + update_array.map = {}; + } + + for (let i = 0; i < idss.length; i++) { + + var baseID = idss[i].replace(/\\u002e/g, "."); + if (!update_array.map[idss[i]]) { + update_array.map[idss[i]] = {}; + } + + if (params.qstring.set_visibility === 'hide') { + update_array.map[idss[i]].is_visible = false; + } + else { + update_array.map[idss[i]].is_visible = true; + } + + if (update_array.map[idss[i]].is_visible) { + delete update_array.map[idss[i]].is_visible; + } + + if (Object.keys(update_array.map[idss[i]]).length === 0) { + delete update_array.map[idss[i]]; + } + + if (params.qstring.set_visibility === 'hide' && event && event.overview && Array.isArray(event.overview)) { + for (let j = 0; j < event.overview.length; j++) { + if (event.overview[j].eventKey === baseID) { + event.overview.splice(j, 1); + j = j - 1; + } + } + update_array.overview = event.overview; + } + } + common.db.collection('events').update({"_id": common.db.ObjectID(params.qstring.app_id)}, {'$set': update_array}, function(err2) { + + if (err2) { + common.returnMessage(params, 400, err2); + } + else { + common.returnMessage(params, 200, 'Success'); + var data_arr = {update: update_array}; + data_arr.before = {map: {}}; + if (event.map) { + data_arr.before.map = event.map; + } + plugins.dispatch("/systemlogs", { + params: params, + action: "events_updated", + data: data_arr + }); + } + }); + }); + }); +}); + +// Catch-all for /i/events/* - dispatches to plugins or returns error +router.all('/i/events/:action', (req, res) => { + const params = req.countlyParams; + const paths = params.paths; + const apiPath = '/i/events'; + if (!plugins.dispatch(apiPath, { + params: params, + validateUserForDataReadAPI: validateUserForDataReadAPI, + validateUserForMgmtReadAPI: validateUserForMgmtReadAPI, + paths: paths, + validateUserForDataWriteAPI: validateUserForDataWriteAPI, + validateUserForGlobalAdmin: validateUserForGlobalAdmin + })) { + common.returnMessage(params, 400, 'Invalid path, must be one of /all or /me'); + } +}); + +module.exports = router; diff --git a/api/routes/export.js b/api/routes/export.js new file mode 100644 index 00000000000..c94500a5f3b --- /dev/null +++ b/api/routes/export.js @@ -0,0 +1,369 @@ +/** + * Export routes (/o/export). + * Migrated from the legacy switch/case in requestProcessor.js. + * @module api/routes/export + */ + +const express = require('express'); +const router = express.Router(); +const common = require('../utils/common.js'); +const { validateUser, validateRead, validateUserForWrite, validateGlobalAdmin, dbUserHasAccessToCollection, getBaseAppFilter } = require('../utils/rights.js'); +const plugins = require('../../plugins/pluginManager.ts'); +const taskmanager = require('../utils/taskmanager.js'); +var countlyFs = require('../utils/countlyFs.js'); + +const validateUserForDataReadAPI = validateRead; +const validateUserForMgmtReadAPI = validateUser; +const validateUserForDataWriteAPI = validateUserForWrite; +const validateUserForGlobalAdmin = validateGlobalAdmin; + +const countlyApi = { + data: { + exports: require('../parts/data/exports.js'), + } +}; + +// --- /o/export endpoints --- + +router.all('/o/export/db', (req, res) => { + const params = req.countlyParams; + validateUserForMgmtReadAPI(() => { + if (!params.qstring.collection) { + common.returnMessage(params, 400, 'Missing parameter "collection"'); + return false; + } + if (typeof params.qstring.filter === "string") { + try { + params.qstring.query = JSON.parse(params.qstring.filter, common.reviver); + } + catch (ex) { + common.returnMessage(params, 400, "Failed to parse query. " + ex.message); + return false; + } + } + else if (typeof params.qstring.query === "string") { + try { + params.qstring.query = JSON.parse(params.qstring.query, common.reviver); + } + catch (ex) { + common.returnMessage(params, 400, "Failed to parse query. " + ex.message); + return false; + } + } + if (typeof params.qstring.projection === "string") { + try { + params.qstring.projection = JSON.parse(params.qstring.projection); + } + catch (ex) { + params.qstring.projection = null; + } + } + if (typeof params.qstring.project === "string") { + try { + params.qstring.projection = JSON.parse(params.qstring.project); + } + catch (ex) { + params.qstring.projection = null; + } + } + if (typeof params.qstring.sort === "string") { + try { + params.qstring.sort = JSON.parse(params.qstring.sort); + } + catch (ex) { + params.qstring.sort = null; + } + } + + if (typeof params.qstring.formatFields === "string") { + try { + params.qstring.formatFields = JSON.parse(params.qstring.formatFields); + } + catch (ex) { + params.qstring.formatFields = null; + } + } + + if (typeof params.qstring.get_index === "string") { + try { + params.qstring.get_index = JSON.parse(params.qstring.get_index); + } + catch (ex) { + params.qstring.get_index = null; + } + } + + dbUserHasAccessToCollection(params, params.qstring.collection, (hasAccess) => { + if (hasAccess || (params.qstring.db === "countly_drill" && params.qstring.collection === "drill_events") || (params.qstring.db === "countly" && params.qstring.collection === "events_data")) { + var dbs = { countly: common.db, countly_drill: common.drillDb, countly_out: common.outDb, countly_fs: countlyFs.gridfs.getHandler() }; + var db = ""; + if (params.qstring.db && dbs[params.qstring.db]) { + db = dbs[params.qstring.db]; + } + else { + db = common.db; + } + if (!params.member.global_admin && params.qstring.collection === "drill_events" || params.qstring.collection === "events_data") { + var base_filter = getBaseAppFilter(params.member, params.qstring.db, params.qstring.collection); + if (base_filter && Object.keys(base_filter).length > 0) { + params.qstring.query = params.qstring.query || {}; + for (var key in base_filter) { + if (params.qstring.query[key]) { + params.qstring.query.$and = params.qstring.query.$and || []; + params.qstring.query.$and.push({[key]: base_filter[key]}); + params.qstring.query.$and.push({[key]: params.qstring.query[key]}); + delete params.qstring.query[key]; + } + else { + params.qstring.query[key] = base_filter[key]; + } + } + } + } + countlyApi.data.exports.fromDatabase({ + db: db, + params: params, + collection: params.qstring.collection, + query: params.qstring.query, + projection: params.qstring.projection, + sort: params.qstring.sort, + limit: params.qstring.limit, + skip: params.qstring.skip, + type: params.qstring.type + }); + } + else { + common.returnMessage(params, 401, 'User does not have access right for this collection'); + } + }); + }, params); +}); + +router.all('/o/export/request', (req, res) => { + const params = req.countlyParams; + validateUserForMgmtReadAPI(() => { + if (!params.qstring.path) { + common.returnMessage(params, 400, 'Missing parameter "path"'); + return false; + } + if (typeof params.qstring.data === "string") { + try { + params.qstring.data = JSON.parse(params.qstring.data); + } + catch (ex) { + console.log("Error parsing export request data", params.qstring.data, ex); + params.qstring.data = {}; + } + } + + if (params.qstring.projection) { + try { + params.qstring.projection = JSON.parse(params.qstring.projection); + } + catch (ex) { + params.qstring.projection = {}; + } + } + + if (params.qstring.columnNames) { + try { + params.qstring.columnNames = JSON.parse(params.qstring.columnNames); + } + catch (ex) { + params.qstring.columnNames = {}; + } + } + if (params.qstring.mapper) { + try { + params.qstring.mapper = JSON.parse(params.qstring.mapper); + } + catch (ex) { + params.qstring.mapper = {}; + } + } + countlyApi.data.exports.fromRequest({ + params: params, + path: params.qstring.path, + data: params.qstring.data, + method: params.qstring.method, + prop: params.qstring.prop, + type: params.qstring.type, + filename: params.qstring.filename, + projection: params.qstring.projection, + columnNames: params.qstring.columnNames, + mapper: params.qstring.mapper, + }); + }, params); +}); + +router.all('/o/export/requestQuery', (req, res) => { + const params = req.countlyParams; + validateUserForMgmtReadAPI(() => { + if (!params.qstring.path) { + common.returnMessage(params, 400, 'Missing parameter "path"'); + return false; + } + if (typeof params.qstring.data === "string") { + try { + params.qstring.data = JSON.parse(params.qstring.data); + } + catch (ex) { + console.log("Error parsing export request data", params.qstring.data, ex); + params.qstring.data = {}; + } + } + var my_name = JSON.stringify(params.qstring); + + var ff = taskmanager.longtask({ + db: common.db, + threshold: plugins.getConfig("api").request_threshold, + force: true, + gridfs: true, + binary: true, + app_id: params.qstring.app_id, + params: params, + type: params.qstring.type_name || "tableExport", + report_name: params.qstring.filename + "." + params.qstring.type, + meta: JSON.stringify({ + "app_id": params.qstring.app_id, + "query": params.qstring.query || {} + }), + name: my_name, + view: "#/exportedData/tableExport/", + processData: function(err, result, callback) { + if (!err) { + callback(null, result); + } + else { + callback(err, ''); + } + }, + outputData: function(err, data) { + if (err) { + common.returnMessage(params, 400, err); + } + else { + common.returnMessage(params, 200, data); + } + } + }); + + countlyApi.data.exports.fromRequestQuery({ + db_name: params.qstring.db, + db: (params.qstring.db === "countly_drill") ? common.drillDb : (params.qstring.dbs === "countly_drill") ? common.drillDb : common.db, + params: params, + path: params.qstring.path, + data: params.qstring.data, + method: params.qstring.method, + prop: params.qstring.prop, + type: params.qstring.type, + filename: params.qstring.filename + "." + params.qstring.type, + output: function(data) { + ff(null, data); + } + }); + }, params); +}); + +router.all('/o/export/download/:id', (req, res) => { + const params = req.countlyParams; + const paths = params.paths; + validateRead(params, "core", () => { + if (paths[4] && paths[4] !== '') { + common.db.collection("long_tasks").findOne({_id: paths[4]}, function(err, data) { + if (err) { + common.returnMessage(params, 400, err); + } + else { + var filename = data.report_name; + var type = filename.split("."); + type = type[type.length - 1]; + var myfile = paths[4]; + var headers = {}; + + countlyFs.gridfs.getSize("task_results", myfile, {id: paths[4]}, function(err2, size) { + if (err2) { + common.returnMessage(params, 400, err2); + } + else if (parseInt(size) === 0) { + if (data.type !== "dbviewer") { + common.returnMessage(params, 400, "Export size is 0"); + } + //handling older aggregations that aren't saved in countly_fs + else if (!data.gridfs && data.data) { + type = "json"; + filename = data.name + "." + type; + headers = {}; + headers["Content-Type"] = countlyApi.data.exports.getType(type); + headers["Content-Disposition"] = "attachment;filename=" + encodeURIComponent(filename); + params.res.writeHead(200, headers); + params.res.write(data.data); + params.res.end(); + } + } + else { + countlyFs.gridfs.getStream("task_results", myfile, {id: myfile}, function(err5, stream) { + if (err5) { + common.returnMessage(params, 400, "Export stream does not exist"); + } + else { + headers = {}; + headers["Content-Type"] = countlyApi.data.exports.getType(type); + headers["Content-Disposition"] = "attachment;filename=" + encodeURIComponent(filename); + params.res.writeHead(200, headers); + stream.pipe(params.res); + } + }); + } + }); + } + }); + } + else { + common.returnMessage(params, 400, 'Missing filename'); + } + }); +}); + +router.all('/o/export/data', (req, res) => { + const params = req.countlyParams; + validateUserForMgmtReadAPI(() => { + if (!params.qstring.data) { + common.returnMessage(params, 400, 'Missing parameter "data"'); + return false; + } + if (typeof params.qstring.data === "string" && !params.qstring.raw) { + try { + params.qstring.data = JSON.parse(params.qstring.data); + } + catch (ex) { + common.returnMessage(params, 400, 'Incorrect parameter "data"'); + return false; + } + } + countlyApi.data.exports.fromData(params.qstring.data, { + params: params, + type: params.qstring.type, + filename: params.qstring.filename + }); + }, params); +}); + +// Catch-all for /o/export/* - dispatches to plugins +router.all('/o/export/:action', (req, res) => { + const params = req.countlyParams; + const paths = params.paths; + const apiPath = '/o/export'; + if (!plugins.dispatch(apiPath, { + params: params, + validateUserForDataReadAPI: validateUserForDataReadAPI, + validateUserForMgmtReadAPI: validateUserForMgmtReadAPI, + paths: paths, + validateUserForDataWriteAPI: validateUserForDataWriteAPI, + validateUserForGlobalAdmin: validateUserForGlobalAdmin + })) { + common.returnMessage(params, 400, 'Invalid path'); + } +}); + +module.exports = router; diff --git a/api/routes/jwt.ts b/api/routes/jwt.ts new file mode 100644 index 00000000000..48346dcd02c --- /dev/null +++ b/api/routes/jwt.ts @@ -0,0 +1,36 @@ +/** + * JWT authentication routes. + * Migrated from the legacy switch/case in requestProcessor.js. + * All endpoints are public — auth is handled internally by jwt_tokens module. + * @module api/routes/jwt + */ + +import { createRequire } from 'module'; +import type { Params } from '../../types/requestProcessor'; +import express from "express"; + +// @ts-expect-error TS1470 - import.meta is valid at runtime +const require = createRequire(import.meta.url); +const router = express.Router(); +const jwtTokens = require('../parts/mgmt/jwt_tokens.js') as { + login(params: Params): void; + refresh(params: Params): void; + revoke(params: Params): void; +}; + +// POST /o/jwt/token - Login with username/password, receive JWT tokens +router.post('/o/jwt/token', (req) => { + jwtTokens.login(req.countlyParams); +}); + +// POST /o/jwt/refresh - Exchange refresh token for new token pair +router.post('/o/jwt/refresh', (req) => { + jwtTokens.refresh(req.countlyParams); +}); + +// POST /i/jwt/revoke - Revoke a refresh token (logout) +router.post('/i/jwt/revoke', (req) => { + jwtTokens.revoke(req.countlyParams); +}); + +export default router; diff --git a/api/routes/notes.js b/api/routes/notes.js new file mode 100644 index 00000000000..3e2a2cd8d44 --- /dev/null +++ b/api/routes/notes.js @@ -0,0 +1,53 @@ +/** + * Notes routes - graph annotation notes. + * Migrated from the legacy switch/case in requestProcessor.js. + * @module api/routes/notes + */ + +const express = require('express'); +const router = express.Router(); +const { validateCreate, validateDelete, validateRead } = require('../utils/rights.js'); +const countlyApi = { + mgmt: { + users: require('../parts/mgmt/users.js') + } +}; + +// Helper: parse JSON args for notes endpoints +function parseArgs(params) { + if (params.qstring.args) { + try { + params.qstring.args = JSON.parse(params.qstring.args); + } + catch (SyntaxError) { + console.log('Parse %s JSON failed %s', params.apiPath, params.req.url, params.req.body); + } + } +} + +// --- Write endpoints: /i/notes --- + +router.all('/i/notes/save', (req, res) => { + const params = req.countlyParams; + parseArgs(params); + validateCreate(params, 'core', () => { + countlyApi.mgmt.users.saveNote(params); + }); +}); + +router.all('/i/notes/delete', (req, res) => { + const params = req.countlyParams; + parseArgs(params); + validateDelete(params, 'core', () => { + countlyApi.mgmt.users.deleteNote(params); + }); +}); + +// --- Read endpoints: /o/notes --- + +router.all('/o/notes', (req, res) => { + const params = req.countlyParams; + validateRead(params, 'core', countlyApi.mgmt.users.fetchNotes); +}); + +module.exports = router; diff --git a/api/routes/ping.js b/api/routes/ping.js new file mode 100644 index 00000000000..3aa0bc5ea66 --- /dev/null +++ b/api/routes/ping.js @@ -0,0 +1,24 @@ +/** + * Ping route - health check endpoint. + * First route migrated from the legacy switch/case in requestProcessor.js + * to Express-style routing as a proof of concept. + * @module api/routes/ping + */ + +const express = require('express'); +const router = express.Router(); +const common = require('../utils/common.js'); + +router.all('/o/ping', (req, res) => { + const params = req.countlyParams; + common.db.collection("plugins").findOne({_id: "plugins"}, {_id: 1}, (err) => { + if (err) { + return common.returnMessage(params, 404, 'DB Error'); + } + else { + return common.returnMessage(params, 200, 'Success'); + } + }); +}); + +module.exports = router; diff --git a/api/routes/render.js b/api/routes/render.js new file mode 100644 index 00000000000..82684298ea9 --- /dev/null +++ b/api/routes/render.js @@ -0,0 +1,57 @@ +/** + * Render/screenshot routes (/o/render). + * Migrated from the legacy switch/case in requestProcessor.js. + * @module api/routes/render + */ + +const express = require('express'); +const router = express.Router(); +const common = require('../utils/common.js'); +const { validateUserForRead } = require('../utils/rights.js'); +const authorize = require('../utils/authorizer.js'); +const render = require('../utils/render.js'); +var path = require('path'); + +// --- Read endpoints: /o/render --- + +router.all('/o/render', (req, res) => { + const params = req.countlyParams; + validateUserForRead(params, function() { + var options = {}; + var view = params.qstring.view || ""; + var route = params.qstring.route || ""; + var id = params.qstring.id || ""; + + options.view = view + "#" + route; + options.id = id ? "#" + id : ""; + + var imageName = "screenshot_" + common.crypto.randomBytes(16).toString("hex") + ".png"; + + options.savePath = path.resolve(__dirname, "../../frontend/express/public/images/screenshots/" + imageName); + options.source = "core"; + + authorize.save({ + db: common.db, + multi: false, + owner: params.member._id, + ttl: 300, + purpose: "LoginAuthToken", + callback: function(err2, token) { + if (err2) { + common.returnMessage(params, 400, 'Error creating token: ' + err2); + return false; + } + options.token = token; + render.renderView(options, function(err3) { + if (err3) { + common.returnMessage(params, 400, 'Error creating screenshot. Please check logs for more information.'); + return false; + } + common.returnOutput(params, {path: common.config.path + "/images/screenshots/" + imageName}); + }); + } + }); + }); +}); + +module.exports = router; diff --git a/api/routes/sdk.js b/api/routes/sdk.js new file mode 100644 index 00000000000..93786867495 --- /dev/null +++ b/api/routes/sdk.js @@ -0,0 +1,55 @@ +/** + * SDK routes - SDK data fetch and ingest endpoints. + * Migrated from the legacy switch/case in requestProcessor.js. + * + * NOTE: These routes require validateAppForFetchAPI which is defined + * locally in requestProcessor.js. It must be exported from there for + * these routes to function. + * @module api/routes/sdk + */ + +const express = require('express'); +const router = express.Router(); +const common = require('../utils/common.js'); +const log = require('../utils/log.js')('core:api'); +const { validateAppForFetchAPI } = require('../utils/requestProcessor.js'); + +/** + * Shared SDK request handler for both /o/sdk and /i/sdk. + * Both endpoints have identical logic. + * @param {object} params - Countly params object + * @returns {boolean|void} false on validation failure + */ +function handleSdkRequest(params) { + params.ip_address = params.qstring.ip_address || common.getIpAddress(params.req); + params.user = {}; + + if (!params.qstring.app_key || !params.qstring.device_id) { + common.returnMessage(params, 400, 'Missing parameter "app_key" or "device_id"'); + return false; + } + else { + params.qstring.device_id += ""; + params.app_user_id = common.crypto.createHash('sha1') + .update(params.qstring.app_key + params.qstring.device_id + "") + .digest('hex'); + } + + log.d('processing request %j', params.qstring); + + params.promises = []; + + validateAppForFetchAPI(params, () => { }); +} + +// GET /o/sdk - SDK data fetch +router.all('/o/sdk', (req, res) => { + handleSdkRequest(req.countlyParams); +}); + +// POST/GET /i/sdk - SDK data ingest +router.all('/i/sdk', (req, res) => { + handleSdkRequest(req.countlyParams); +}); + +module.exports = router; diff --git a/api/routes/system.js b/api/routes/system.js new file mode 100644 index 00000000000..767cee4e734 --- /dev/null +++ b/api/routes/system.js @@ -0,0 +1,436 @@ +/** + * System information routes (/o/system). + * Migrated from the legacy switch/case in requestProcessor.js. + * @module api/routes/system + */ + +const express = require('express'); +const router = express.Router(); +const common = require('../utils/common.js'); +const { validateUser } = require('../utils/rights.js'); +const plugins = require('../../plugins/pluginManager.ts'); +const versionInfo = require('../../frontend/express/version.info'); +const log = require('../utils/log.js')('core:api'); + +const validateUserForMgmtReadAPI = validateUser; + +const validateUserForDataReadAPI = require('../utils/rights.js').validateRead; +const validateUserForDataWriteAPI = require('../utils/rights.js').validateUserForWrite; +const validateUserForGlobalAdmin = require('../utils/rights.js').validateGlobalAdmin; + +// Kafka events meta cache (30s TTL) with in-flight dedup +var _kafkaMetaCache = null; +var _kafkaMetaCacheTs = 0; +var _kafkaMetaCachePromise = null; +const KAFKA_META_CACHE_TTL = 30000; + +// --- /o/system endpoints --- + +router.all('/o/system/version', (req, res) => { + const params = req.countlyParams; + validateUserForMgmtReadAPI(() => { + common.returnOutput(params, {"version": versionInfo.version}); + }, params); +}); + +router.all('/o/system/plugins', (req, res) => { + const params = req.countlyParams; + validateUserForMgmtReadAPI(() => { + common.returnOutput(params, plugins.getPlugins()); + }, params); +}); + +router.all('/o/system/observability', (req, res) => { + const params = req.countlyParams; + validateUserForMgmtReadAPI(() => { + plugins.dispatch('/system/observability/collect', {params: params}, function(err, results) { + if (err) { + common.returnMessage(params, 500, 'Error collecting observability data'); + return; + } + const data = (results || []) + .filter(r => r && r.status === 'fulfilled' && r.value) + .map(r => r.value); + common.returnOutput(params, data); + }); + }, params); +}); + +router.all('/o/system/aggregator', (req, res) => { + const params = req.countlyParams; + validateUserForMgmtReadAPI(async() => { + try { + //fetch aggregator status and latest drill cd in parallel + const [pluginsData, drillData] = await Promise.all([ + common.db.collection("plugins").findOne({_id: "_changeStreams"}), + common.drillDb.collection("drill_events").find({}, {projection: {cd: 1}}).sort({cd: -1}).limit(1).toArray() + ]); + + var data = []; + var now = Date.now().valueOf(); + var nowDrill = now; + if (drillData && drillData.length) { + nowDrill = new Date(drillData[0].cd).valueOf(); + } + + if (pluginsData) { + for (var key in pluginsData) { + if (key !== "_id") { + var lastAccepted = new Date(pluginsData[key].cd).valueOf(); + data.push({ + name: key, + last_cd: pluginsData[key].cd, + drill: drillData && drillData[0] && drillData[0].cd, + last_id: pluginsData[key]._id, + diff: (now - lastAccepted) / 1000, + diffDrill: (nowDrill - lastAccepted) / 1000 + }); + } + } + } + common.returnOutput(params, data); + } + catch (err) { + log.e('Error fetching aggregator status:', err); + common.returnMessage(params, 500, 'Error fetching aggregator status'); + } + }, params); +}); + +// /o/system/kafka/events/meta - Get filter options (cached 30s, deduped) +router.all('/o/system/kafka/events/meta', (req, res) => { + const params = req.countlyParams; + validateUserForMgmtReadAPI(async() => { + try { + var now = Date.now(); + if (_kafkaMetaCache && (now - _kafkaMetaCacheTs) < KAFKA_META_CACHE_TTL) { + common.returnOutput(params, _kafkaMetaCache); + return; + } + + // Reuse in-flight fetch to prevent thundering herd on cache expiry + if (!_kafkaMetaCachePromise) { + _kafkaMetaCachePromise = Promise.all([ + common.db.collection('kafka_consumer_events').distinct('type'), + common.db.collection('kafka_consumer_events').distinct('groupId'), + common.db.collection('kafka_consumer_events').distinct('topic'), + common.db.collection('kafka_consumer_events').distinct('clusterId') + ]).then(function([eventTypes, groupIds, topics, clusterIds]) { + _kafkaMetaCache = { + eventTypes: eventTypes.filter(Boolean).sort(), + groupIds: groupIds.filter(Boolean).sort(), + topics: topics.filter(Boolean).sort(), + clusterIds: clusterIds.filter(Boolean).sort() + }; + _kafkaMetaCacheTs = Date.now(); + _kafkaMetaCachePromise = null; + return _kafkaMetaCache; + }).catch(function(err) { + _kafkaMetaCachePromise = null; + throw err; + }); + } + + var result = await _kafkaMetaCachePromise; + common.returnOutput(params, result); + } + catch (err) { + log.e('Error fetching Kafka events meta:', err); + common.returnMessage(params, 500, 'Error fetching Kafka events meta'); + } + }, params); +}); + +// /o/system/kafka/events - Get events list +router.all('/o/system/kafka/events', (req, res) => { + const params = req.countlyParams; + validateUserForMgmtReadAPI(async() => { + try { + // Build query from filters + const query = {}; + + if (params.qstring.eventType && params.qstring.eventType !== 'all') { + query.type = params.qstring.eventType + ""; + } + if (params.qstring.groupId && params.qstring.groupId !== 'all') { + query.groupId = params.qstring.groupId + ""; + } + if (params.qstring.topic && params.qstring.topic !== 'all') { + query.topic = params.qstring.topic + ""; + } + if (params.qstring.clusterId && params.qstring.clusterId !== 'all') { + query.clusterId = params.qstring.clusterId + ""; + } + + // Get accurate counts + const [total, filteredCount] = await Promise.all([ + common.db.collection('kafka_consumer_events').countDocuments({}), + common.db.collection('kafka_consumer_events').countDocuments(query) + ]); + let cursor = common.db.collection('kafka_consumer_events').find(query); + + // Sorting with validated column index + const columns = ['_id', 'ts', 'type', 'groupId', 'topic', 'partition', 'clusterId']; + const sortColIndex = parseInt(params.qstring.iSortCol_0, 10); + if (params.qstring.iSortCol_0 && + params.qstring.sSortDir_0 && + Number.isInteger(sortColIndex) && + sortColIndex >= 0 && + sortColIndex < columns.length) { + const sortObj = {}; + sortObj[columns[sortColIndex]] = params.qstring.sSortDir_0 === 'asc' ? 1 : -1; + cursor = cursor.sort(sortObj); + } + else { + cursor = cursor.sort({ ts: -1 }); + } + + // Pagination with validated parameters + const MAX_DISPLAY_LENGTH = 1000; + let displayStart = parseInt(params.qstring.iDisplayStart, 10); + if (Number.isNaN(displayStart) || displayStart < 0) { + displayStart = 0; + } + if (displayStart > 0) { + cursor = cursor.skip(displayStart); + } + + let displayLength = parseInt(params.qstring.iDisplayLength, 10); + if (Number.isNaN(displayLength) || displayLength < 1 || displayLength > MAX_DISPLAY_LENGTH) { + displayLength = 50; + } + cursor = cursor.limit(displayLength); + + const events = await cursor.toArray(); + + common.returnOutput(params, { + sEcho: params.qstring.sEcho, + iTotalRecords: Math.max(total, 0), + iTotalDisplayRecords: filteredCount, + aaData: events + }); + } + catch (err) { + log.e('Error fetching Kafka consumer events:', err); + common.returnMessage(params, 500, 'Error fetching Kafka consumer events'); + } + }, params); +}); + +// /o/system/kafka - Get Kafka status overview +router.all('/o/system/kafka', (req, res) => { + const params = req.countlyParams; + validateUserForMgmtReadAPI(async() => { + try { + const KAFKA_QUERY_LIMIT = 500; + + // Fetch all Kafka data and summaries in parallel (with projections) + const [consumerState, consumerHealth, lagHistory, connectStatus, stateSummary, healthSummary] = await Promise.all([ + common.db.collection("kafka_consumer_state") + .find({}, { + projection: { + consumerGroup: 1, + topic: 1, + partitions: 1, + lastProcessedAt: 1, + batchCount: 1, + duplicatesSkipped: 1, + lastDuplicateAt: 1, + lastBatchSize: 1, + avgBatchSize: 1 + } + }) + .sort({ lastProcessedAt: -1 }) + .limit(KAFKA_QUERY_LIMIT) + .toArray(), + common.db.collection("kafka_consumer_health") + .find({}, { + projection: { + groupId: 1, + rebalanceCount: 1, + lastRebalanceAt: 1, + lastJoinAt: 1, + lastMemberId: 1, + lastGenerationId: 1, + commitCount: 1, + lastCommitAt: 1, + errorCount: 1, + lastErrorAt: 1, + lastErrorMessage: 1, + recentErrors: 1, + totalLag: 1, + partitionLag: 1, + lagUpdatedAt: 1, + updatedAt: 1 + } + }) + .sort({ updatedAt: -1 }) + .limit(KAFKA_QUERY_LIMIT) + .toArray(), + common.db.collection("kafka_lag_history") + .find({}, {projection: {ts: 1, groups: 1, connectLag: 1}}) + .sort({ ts: -1 }) + .limit(100) + .toArray(), + common.db.collection("kafka_connect_status") + .find({}, { + projection: { + connectorName: 1, + connectorState: 1, + connectorType: 1, + workerId: 1, + tasks: 1, + updatedAt: 1 + } + }) + .sort({ updatedAt: -1 }) + .limit(KAFKA_QUERY_LIMIT) + .toArray(), + // Aggregate consumer state summary via MongoDB pipeline + common.db.collection("kafka_consumer_state") + .aggregate([{ + $group: { + _id: null, + totalBatchesProcessed: { $sum: { $ifNull: ["$batchCount", 0] } }, + totalDuplicatesSkipped: { $sum: { $ifNull: ["$duplicatesSkipped", 0] } }, + avgBatchSizeSum: { $sum: { $ifNull: ["$avgBatchSize", 0] } }, + groupsWithData: { $sum: { $cond: [{ $gt: ["$avgBatchSize", 0] }, 1, 0] } } + } + }]) + .toArray(), + // Aggregate consumer health summary via MongoDB pipeline + common.db.collection("kafka_consumer_health") + .aggregate([{ + $group: { + _id: null, + totalRebalances: { $sum: { $ifNull: ["$rebalanceCount", 0] } }, + totalErrors: { $sum: { $ifNull: ["$errorCount", 0] } }, + totalLag: { $sum: { $ifNull: ["$totalLag", 0] } } + } + }]) + .toArray() + ]); + + // Extract summary from aggregation pipelines + const stSummary = stateSummary[0] || {}; + const totalBatchesProcessed = stSummary.totalBatchesProcessed || 0; + const totalDuplicatesSkipped = stSummary.totalDuplicatesSkipped || 0; + const avgBatchSizeOverall = stSummary.groupsWithData > 0 + ? stSummary.avgBatchSizeSum / stSummary.groupsWithData + : 0; + + const htSummary = healthSummary[0] || {}; + const totalRebalances = htSummary.totalRebalances || 0; + const totalErrors = htSummary.totalErrors || 0; + const totalLagAll = htSummary.totalLag || 0; + + // Transform consumer state rows + const partitionStats = consumerState.map(state => { + const partitions = state.partitions || {}; + const partitionCount = Object.keys(partitions).length; + const activePartitions = Object.values(partitions).filter(p => p.lastProcessedAt).length; + + return { + id: state._id, + consumerGroup: state.consumerGroup, + topic: state.topic, + partitionCount, + activePartitions, + lastProcessedAt: state.lastProcessedAt, + batchCount: state.batchCount || 0, + duplicatesSkipped: state.duplicatesSkipped || 0, + lastDuplicateAt: state.lastDuplicateAt, + lastBatchSize: state.lastBatchSize, + avgBatchSize: state.avgBatchSize ? Math.round(state.avgBatchSize) : null + }; + }); + + // Transform consumer health rows + const consumerStats = consumerHealth.map(health => ({ + id: health._id, + groupId: health.groupId, + rebalanceCount: health.rebalanceCount || 0, + lastRebalanceAt: health.lastRebalanceAt, + lastJoinAt: health.lastJoinAt, + lastMemberId: health.lastMemberId, + lastGenerationId: health.lastGenerationId, + commitCount: health.commitCount || 0, + lastCommitAt: health.lastCommitAt, + errorCount: health.errorCount || 0, + lastErrorAt: health.lastErrorAt, + lastErrorMessage: health.lastErrorMessage, + recentErrors: health.recentErrors || [], + totalLag: health.totalLag || 0, + partitionLag: health.partitionLag || {}, + lagUpdatedAt: health.lagUpdatedAt, + updatedAt: health.updatedAt + })); + + // Process Kafka Connect status + const connectorStats = connectStatus.map(conn => ({ + id: conn._id, + connectorName: conn.connectorName, + connectorState: conn.connectorState, + connectorType: conn.connectorType, + workerId: conn.workerId, + tasks: conn.tasks || [], + tasksRunning: (conn.tasks || []).filter(t => t.state === 'RUNNING').length, + tasksTotal: (conn.tasks || []).length, + updatedAt: conn.updatedAt + })); + + // Get connect consumer group lag for ClickHouse sink + const connectConsumerGroupId = common.config?.kafka?.connectConsumerGroupId; + const connectGroupHealth = connectConsumerGroupId + ? consumerHealth.find(h => h.groupId === connectConsumerGroupId) + : null; + + common.returnOutput(params, { + summary: { + totalBatchesProcessed, + totalDuplicatesSkipped, + avgBatchSizeOverall: Math.round(avgBatchSizeOverall * 100) / 100, + totalRebalances, + totalErrors, + totalLag: totalLagAll, + consumerGroupCount: consumerStats.length, + partitionCount: partitionStats.length + }, + partitions: partitionStats, + consumers: consumerStats, + lagHistory: lagHistory.reverse(), // Oldest first for charts + + // Kafka Connect status + connectStatus: { + enabled: !!common.config?.kafka?.connectApiUrl, + connectors: connectorStats, + sinkLag: connectGroupHealth?.totalLag || 0, + sinkLagUpdatedAt: connectGroupHealth?.lagUpdatedAt + } + }); + } + catch (err) { + log.e('Error fetching Kafka stats:', err); + common.returnMessage(params, 500, 'Error fetching Kafka stats'); + } + }, params); +}); + +// Catch-all for /o/system/* - dispatches to plugins or returns error +router.all('/o/system/:action', (req, res) => { + const params = req.countlyParams; + const paths = params.paths; + const apiPath = '/o/system'; + if (!plugins.dispatch(apiPath, { + params: params, + validateUserForDataReadAPI: validateUserForDataReadAPI, + validateUserForMgmtReadAPI: validateUserForMgmtReadAPI, + paths: paths, + validateUserForDataWriteAPI: validateUserForDataWriteAPI, + validateUserForGlobalAdmin: validateUserForGlobalAdmin + })) { + common.returnMessage(params, 400, 'Invalid path'); + } +}); + +module.exports = router; diff --git a/api/routes/tasks.js b/api/routes/tasks.js new file mode 100644 index 00000000000..5c467ce8427 --- /dev/null +++ b/api/routes/tasks.js @@ -0,0 +1,345 @@ +/** + * Task manager routes (/i/tasks, /o/tasks). + * Migrated from the legacy switch/case in requestProcessor.js. + * @module api/routes/tasks + */ + +const express = require('express'); +const router = express.Router(); +const common = require('../utils/common.js'); +const countlyCommon = require('../lib/countly.common.js'); +const { validateRead, validateUser, validateUserForWrite, validateGlobalAdmin } = require('../utils/rights.js'); +const taskmanager = require('../utils/taskmanager.js'); +const plugins = require('../../plugins/pluginManager.ts'); + +const validateUserForDataReadAPI = validateRead; +const validateUserForMgmtReadAPI = validateUser; +const validateUserForDataWriteAPI = validateUserForWrite; +const validateUserForGlobalAdmin = validateGlobalAdmin; + +// --- Write endpoints: /i/tasks --- + +router.all('/i/tasks/update', (req, res) => { + const params = req.countlyParams; + if (!params.qstring.task_id) { + common.returnMessage(params, 400, 'Missing parameter "task_id"'); + return false; + } + validateUserForWrite(params, () => { + taskmanager.rerunTask({ + db: common.db, + id: params.qstring.task_id + }, (err, res) => { + common.returnMessage(params, 200, res); + }); + }); +}); + +router.all('/i/tasks/delete', (req, res) => { + const params = req.countlyParams; + if (!params.qstring.task_id) { + common.returnMessage(params, 400, 'Missing parameter "task_id"'); + return false; + } + validateUserForWrite(params, () => { + taskmanager.deleteResult({ + db: common.db, + id: params.qstring.task_id + }, (err, task) => { + plugins.dispatch("/systemlogs", {params: params, action: "task_manager_task_deleted", data: task}); + common.returnMessage(params, 200, "Success"); + }); + }); +}); + +router.all('/i/tasks/name', (req, res) => { + const params = req.countlyParams; + if (!params.qstring.task_id) { + common.returnMessage(params, 400, 'Missing parameter "task_id"'); + return false; + } + validateUserForWrite(params, () => { + taskmanager.nameResult({ + db: common.db, + id: params.qstring.task_id, + name: params.qstring.name + }, () => { + common.returnMessage(params, 200, "Success"); + }); + }); +}); + +router.all('/i/tasks/edit', (req, res) => { + const params = req.countlyParams; + if (!params.qstring.task_id) { + common.returnMessage(params, 400, 'Missing parameter "task_id"'); + return false; + } + validateUserForWrite(params, () => { + const data = { + "report_name": params.qstring.report_name, + "report_desc": params.qstring.report_desc, + "global": params.qstring.global + "" === 'true', + "autoRefresh": params.qstring.autoRefresh + "" === 'true', + "period_desc": params.qstring.period_desc + }; + taskmanager.editTask({ + db: common.db, + data: data, + id: params.qstring.task_id + }, (err, d) => { + if (err) { + common.returnMessage(params, 503, "Error"); + } + else { + common.returnMessage(params, 200, "Success"); + } + plugins.dispatch("/systemlogs", {params: params, action: "task_manager_task_updated", data: d}); + }); + }); +}); + +// Catch-all for /i/tasks/* - dispatches to plugins or returns error +router.all('/i/tasks/:action', (req, res) => { + const params = req.countlyParams; + const apiPath = '/i/tasks'; + const paths = params.paths; + if (!params.qstring.task_id) { + common.returnMessage(params, 400, 'Missing parameter "task_id"'); + return false; + } + if (!plugins.dispatch(apiPath, { + params: params, + validateUserForDataReadAPI: validateUserForDataReadAPI, + validateUserForMgmtReadAPI: validateUserForMgmtReadAPI, + paths: paths, + validateUserForDataWriteAPI: validateUserForDataWriteAPI, + validateUserForGlobalAdmin: validateUserForGlobalAdmin + })) { + common.returnMessage(params, 400, 'Invalid path'); + } +}); + +// --- Read endpoints: /o/tasks --- + +router.all('/o/tasks/all', (req, res) => { + const params = req.countlyParams; + validateRead(params, 'core', () => { + if (!params.qstring.query) { + params.qstring.query = {}; + } + if (typeof params.qstring.query === "string") { + try { + params.qstring.query = JSON.parse(params.qstring.query); + } + catch (ex) { + params.qstring.query = {}; + } + } + if (params.qstring.query.$or) { + params.qstring.query.$and = [ + {"$or": Object.assign([], params.qstring.query.$or) }, + {"$or": [{"global": {"$ne": false}}, {"creator": params.member._id + ""}]} + ]; + delete params.qstring.query.$or; + } + else { + params.qstring.query.$or = [{"global": {"$ne": false}}, {"creator": params.member._id + ""}]; + } + params.qstring.query.subtask = {$exists: false}; + params.qstring.query.app_id = params.qstring.app_id; + if (params.qstring.app_ids && params.qstring.app_ids !== "") { + var ll = params.qstring.app_ids.split(","); + if (ll.length > 1) { + params.qstring.query.app_id = {$in: ll}; + } + } + if (params.qstring.period) { + countlyCommon.getPeriodObj(params); + params.qstring.query.ts = countlyCommon.getTimestampRangeQuery(params, false); + } + taskmanager.getResults({ + db: common.db, + query: params.qstring.query + }, (err, res) => { + common.returnOutput(params, res || []); + }); + }); +}); + +router.all('/o/tasks/count', (req, res) => { + const params = req.countlyParams; + validateRead(params, 'core', () => { + if (!params.qstring.query) { + params.qstring.query = {}; + } + if (typeof params.qstring.query === "string") { + try { + params.qstring.query = JSON.parse(params.qstring.query); + } + catch (ex) { + params.qstring.query = {}; + } + } + if (params.qstring.query.$or) { + params.qstring.query.$and = [ + {"$or": Object.assign([], params.qstring.query.$or) }, + {"$or": [{"global": {"$ne": false}}, {"creator": params.member._id + ""}]} + ]; + delete params.qstring.query.$or; + } + else { + params.qstring.query.$or = [{"global": {"$ne": false}}, {"creator": params.member._id + ""}]; + } + if (params.qstring.period) { + countlyCommon.getPeriodObj(params); + params.qstring.query.ts = countlyCommon.getTimestampRangeQuery(params, false); + } + taskmanager.getCounts({ + db: common.db, + query: params.qstring.query + }, (err, res) => { + common.returnOutput(params, res || []); + }); + }); +}); + +router.all('/o/tasks/list', (req, res) => { + const params = req.countlyParams; + validateRead(params, 'core', () => { + if (!params.qstring.query) { + params.qstring.query = {}; + } + if (typeof params.qstring.query === "string") { + try { + params.qstring.query = JSON.parse(params.qstring.query); + } + catch (ex) { + params.qstring.query = {}; + } + } + params.qstring.query.$and = []; + if (params.qstring.query.creator && params.qstring.query.creator === params.member._id) { + params.qstring.query.$and.push({"creator": params.member._id + ""}); + } + else { + params.qstring.query.$and.push({"$or": [{"global": {"$ne": false}}, {"creator": params.member._id + ""}]}); + } + + if (params.qstring.data_source !== "all" && params.qstring.app_id) { + if (params.qstring.data_source === "independent") { + params.qstring.query.$and.push({"app_id": "undefined"}); + } + else { + params.qstring.query.$and.push({"app_id": params.qstring.app_id}); + } + } + + if (params.qstring.query.$or) { + params.qstring.query.$and.push({"$or": Object.assign([], params.qstring.query.$or) }); + delete params.qstring.query.$or; + } + params.qstring.query.subtask = {$exists: false}; + if (params.qstring.period) { + countlyCommon.getPeriodObj(params); + params.qstring.query.ts = countlyCommon.getTimestampRangeQuery(params, false); + } + const skip = params.qstring.iDisplayStart; + const limit = params.qstring.iDisplayLength; + const sEcho = params.qstring.sEcho; + const keyword = params.qstring.sSearch || null; + const sortBy = params.qstring.iSortCol_0 || null; + const sortSeq = params.qstring.sSortDir_0 || null; + taskmanager.getTableQueryResult({ + db: common.db, + query: params.qstring.query, + page: {skip, limit}, + sort: {sortBy, sortSeq}, + keyword: keyword, + }, (err, res) => { + if (!err) { + common.returnOutput(params, {aaData: res.list, iTotalDisplayRecords: res.count, iTotalRecords: res.count, sEcho}); + } + else { + common.returnMessage(params, 500, '"Query failed"'); + } + }); + }); +}); + +router.all('/o/tasks/task', (req, res) => { + const params = req.countlyParams; + validateRead(params, 'core', () => { + if (!params.qstring.task_id) { + common.returnMessage(params, 400, 'Missing parameter "task_id"'); + return false; + } + taskmanager.getResult({ + db: common.db, + id: params.qstring.task_id, + subtask_key: params.qstring.subtask_key + }, (err, res) => { + if (res) { + common.returnOutput(params, res); + } + else { + common.returnMessage(params, 400, 'Task does not exist'); + } + }); + }); +}); + +router.all('/o/tasks/check', (req, res) => { + const params = req.countlyParams; + validateRead(params, 'core', () => { + if (!params.qstring.task_id) { + common.returnMessage(params, 400, 'Missing parameter "task_id"'); + return false; + } + + var tasks = params.qstring.task_id; + + try { + tasks = JSON.parse(tasks); + } + catch (e) { + // ignore + } + + var isMulti = Array.isArray(tasks); + + taskmanager.checkResult({ + db: common.db, + id: tasks + }, (err, res) => { + if (isMulti && res) { + common.returnMessage(params, 200, res); + } + else if (res) { + common.returnMessage(params, 200, res.status); + } + else { + common.returnMessage(params, 400, 'Task does not exist'); + } + }); + }); +}); + +// Catch-all for /o/tasks/* - dispatches to plugins or returns error +router.all('/o/tasks/:action', (req, res) => { + const params = req.countlyParams; + const apiPath = '/o/tasks'; + const paths = params.paths; + if (!plugins.dispatch(apiPath, { + params: params, + validateUserForDataReadAPI: validateUserForDataReadAPI, + validateUserForMgmtReadAPI: validateUserForMgmtReadAPI, + paths: paths, + validateUserForDataWriteAPI: validateUserForDataWriteAPI, + validateUserForGlobalAdmin: validateUserForGlobalAdmin + })) { + common.returnMessage(params, 400, 'Invalid path'); + } +}); + +module.exports = router; diff --git a/api/routes/token.js b/api/routes/token.js new file mode 100644 index 00000000000..8257ad2e1fe --- /dev/null +++ b/api/routes/token.js @@ -0,0 +1,252 @@ +/** + * Token management routes (/i/token, /o/token). + * Migrated from the legacy switch/case in requestProcessor.js. + * @module api/routes/token + */ + +const express = require('express'); +const router = express.Router(); +const common = require('../utils/common.js'); +const { validateUser } = require('../utils/rights.js'); +const authorize = require('../utils/authorizer.js'); + +// --- Write endpoints: /i/token --- + +/** + * @api {get} /i/token/delete + * @apiName deleteToken + * @apiGroup TokenManager + * + * @apiDescription Deletes related token that given id + * @apiQuery {String} tokenid, Token id to be deleted + * + * @apiSuccessExample {json} Success-Response: + * HTTP/1.1 200 OK + * { + * "result": { + * "result": { + * "n": 1, + * "ok": 1 + * }, + * "connection": { + * "_events": {}, + * "_eventsCount": 4, + * "id": 4, + * "address": "127.0.0.1:27017", + * "bson": {}, + * "socketTimeout": 999999999, + * "host": "localhost", + * "port": 27017, + * "monitorCommands": false, + * "closed": false, + * "destroyed": false, + * "lastIsMasterMS": 15 + * }, + * "deletedCount": 1, + * "n": 1, + * "ok": 1 + * } + * } + * + * @apiErrorExample {json} Error-Response: + * HTTP/1.1 400 Bad Request + * { + * "result": "Token id not provided" + * } +*/ +router.all('/i/token/delete', (req, res) => { + const params = req.countlyParams; + validateUser(() => { + if (params.qstring.tokenid) { + common.db.collection("auth_tokens").remove({ + "_id": params.qstring.tokenid, + "owner": params.member._id + "" + }, function(err, result) { + if (err) { + common.returnMessage(params, 404, err.message); + } + else { + common.returnMessage(params, 200, result); + } + }); + } + else { + common.returnMessage(params, 404, "Token id not provided"); + } + }, params); +}); + +/** + * @api {get} /i/token/create + * @apiName createToken + * @apiGroup TokenManager + * + * @apiDescription Creates spesific token + * @apiQuery {String} purpose, Purpose is description of the created token + * @apiQuery {Array} endpointquery, Includes "params" and "endpoint" inside + * {"params":{qString Key: qString Val} + * "endpoint": "_endpointAdress" + * @apiQuery {Boolean} multi, Defines availability multiple times + * @apiQuery {Boolean} apps, App Id of selected application + * @apiQuery {Boolean} ttl, expiration time for token + * + * @apiSuccessExample {json} Success-Response: + * HTTP/1.1 200 OK + * { + * "result": "0e1c012f855e7065e779b57a616792fb5bd03834" + * } + * + * @apiErrorExample {json} Error-Response: + * HTTP/1.1 400 Bad Request + * { + * "result": "Missing parameter \"api_key\" or \"auth_token\"" + * } +*/ +router.all('/i/token/create', (req, res) => { + const params = req.countlyParams; + validateUser(params, () => { + let ttl, multi, endpoint, purpose, apps; + if (params.qstring.ttl) { + ttl = parseInt(params.qstring.ttl); + } + else { + ttl = 1800; + } + multi = true; + if (params.qstring.multi === false || params.qstring.multi === 'false') { + multi = false; + } + apps = params.qstring.apps || ""; + if (params.qstring.apps) { + apps = params.qstring.apps.split(','); + } + + if (params.qstring.endpointquery && params.qstring.endpointquery !== "") { + try { + endpoint = JSON.parse(params.qstring.endpointquery); //structure with also info for qstring params. + } + catch (ex) { + if (params.qstring.endpoint) { + endpoint = params.qstring.endpoint.split(','); + } + else { + endpoint = ""; + } + } + } + else if (params.qstring.endpoint) { + endpoint = params.qstring.endpoint.split(','); + } + + if (params.qstring.purpose) { + purpose = params.qstring.purpose; + } + authorize.save({ + db: common.db, + ttl: ttl, + multi: multi, + owner: params.member._id + "", + app: apps, + endpoint: endpoint, + purpose: purpose, + callback: (err, token) => { + if (err) { + common.returnMessage(params, 404, err); + } + else { + common.returnMessage(params, 200, token); + } + } + }); + }); +}); + +// Default for /i/token +router.all('/i/token', (req, res) => { + const params = req.countlyParams; + common.returnMessage(params, 400, 'Invalid path, must be one of /delete or /create'); +}); + +// --- Read endpoints: /o/token --- + +router.all('/o/token/check', (req, res) => { + const params = req.countlyParams; + if (!params.qstring.token) { + common.returnMessage(params, 400, 'Missing parameter "token"'); + return false; + } + + validateUser(params, function() { + authorize.check_if_expired({ + token: params.qstring.token, + db: common.db, + callback: (err, valid, time_left) => { + if (err) { + common.returnMessage(params, 404, err.message); + } + else { + common.returnMessage(params, 200, { + valid: valid, + time: time_left + }); + } + } + }); + }); +}); + +/** + * @api {get} /o/token/list + * @apiName initialize + * @apiGroup TokenManager + * + * @apiDescription Returns active tokens as an array that uses tokens in order to protect the API key + * @apiQuery {String} app_id, App Id of related application or {String} auth_token + * + * @apiSuccessExample {json} Success-Response: + * HTTP/1.1 200 OK + * { + * "result": [ + * { + * "_id": "884803f9e9eda51f5dbbb45ba91fa7e2b1dbbf4b", + * "ttl": 0, + * "ends": 1650466609, + * "multi": false, + * "owner": "60e42efa5c23ee7ec6259af0", + * "app": "", + * "endpoint": [ + * + * ], + * "purpose": "Test Token", + * "temporary": false + * } + * ] + * } + * + * @apiErrorExample {json} Error-Response: + * HTTP/1.1 400 Bad Request + * { + * "result": "Missing parameter \"api_key\" or \"auth_token\"" + * } +*/ +router.all('/o/token/list', (req, res) => { + const params = req.countlyParams; + validateUser(params, function() { + common.db.collection("auth_tokens").find({"owner": params.member._id + ""}).toArray(function(err, result) { + if (err) { + common.returnMessage(params, 404, err.message); + } + else { + common.returnMessage(params, 200, result); + } + }); + }); +}); + +// Default for /o/token +router.all('/o/token', (req, res) => { + const params = req.countlyParams; + common.returnMessage(params, 400, 'Invalid path, must be one of /list'); +}); + +module.exports = router; diff --git a/api/routes/users.js b/api/routes/users.js new file mode 100644 index 00000000000..d20e24dd90d --- /dev/null +++ b/api/routes/users.js @@ -0,0 +1,261 @@ +/** + * User management routes (/i/users, /o/users). + * Migrated from the legacy switch/case in requestProcessor.js. + * @module api/routes/users + */ + +const express = require('express'); +const router = express.Router(); +const common = require('../utils/common.js'); +const { validateUser, validateRead, validateUserForWrite, validateGlobalAdmin } = require('../utils/rights.js'); +const plugins = require('../../plugins/pluginManager.ts'); + +const validateUserForWriteAPI = validateUser; +const validateUserForDataReadAPI = validateRead; +const validateUserForMgmtReadAPI = validateUser; +const validateUserForDataWriteAPI = validateUserForWrite; +const validateUserForGlobalAdmin = validateGlobalAdmin; + +const countlyApi = { + mgmt: { + users: require('../parts/mgmt/users.js'), + } +}; + +// Helper: parse JSON args for /i/users endpoints +function parseArgs(params) { + if (params.qstring.args) { + try { + params.qstring.args = JSON.parse(params.qstring.args); + } + catch (SyntaxError) { + console.log('Parse /i/users JSON failed. URL: %s, Body: %s', params.req.url, JSON.stringify(params.req.body)); + } + } +} + +// --- Write endpoints: /i/users --- + +/** + * @api {get} /i/users/create Create new user + * @apiName Create User + * @apiGroup User Management + * + * @apiDescription Access database, get collections, indexes and data + * @apiQuery {Object} args User data object + * @apiQuery {String} args.full_name Full name + * @apiQuery {String} args.username Username + * @apiQuery {String} args.password Password + * @apiQuery {String} args.email Email + * @apiQuery {Object} args.permission Permission object + * @apiQuery {Boolean} args.global_admin Global admin flag + * + * @apiSuccessExample {json} Success-Response: + * HTTP/1.1 200 OK + * { + * "full_name":"fn", + * "username":"un", + * "email":"e@ms.cd", + * "permission": { + * "c":{}, + * "r":{}, + * "u":{}, + * "d":{}, + * "_":{ + * "u":[[]], + * "a":[] + * } + * }, + * "global_admin":true, + * "password_changed":0, + * "created_at":1651240780, + * "locked":false, + * "api_key":"1c5e93c6657d76ae8903f14c32cb3796", + * "_id":"626bef4cb00db29a02f8f7a0" + * } + * + * @apiErrorExample {json} Error-Response: + * HTTP/1.1 400 Bad Request + * { + * "result": "Missing parameter \"app_key\" or \"device_id\"" + * } + */ +router.all('/i/users/create', (req, res) => { + const params = req.countlyParams; + parseArgs(params); + validateUserForGlobalAdmin(params, countlyApi.mgmt.users.createUser); +}); + +/** + * @api {get} /i/users/update Update user + * @apiName Update User + * @apiGroup User Management + * + * @apiDescription Access database, get collections, indexes and data + * @apiQuery {Object} args User data object + * @apiQuery {String} args.full_name Full name + * @apiQuery {String} args.username Username + * @apiQuery {String} args.password Password + * @apiQuery {String} args.email Email + * @apiQuery {Object} args.permission Permission object + * @apiQuery {Boolean} args.global_admin Global admin flag + * + * @apiSuccessExample {json} Success-Response: + * HTTP/1.1 200 OK + * { + * "result":"Success" + * } + * + * @apiErrorExample {json} Error-Response: + * HTTP/1.1 400 Bad Request + * { + * "result": "Missing parameter \"app_key\" or \"device_id\"" + * } + */ +router.all('/i/users/update', (req, res) => { + const params = req.countlyParams; + parseArgs(params); + validateUserForGlobalAdmin(params, countlyApi.mgmt.users.updateUser); +}); + +/** + * @api {get} /i/users/delete Delete user + * @apiName Delete User + * @apiGroup User Management + * + * @apiDescription Access database, get collections, indexes and data + * @apiQuery {Object} args User data object + * @apiQuery {String} args.user_ids IDs array for users which will be deleted + * + * @apiSuccessExample {json} Success-Response: + * HTTP/1.1 200 OK + * { + * "result":"Success" + * } + * + * @apiErrorExample {json} Error-Response: + * HTTP/1.1 400 Bad Request + * { + * "result": "Missing parameter \"app_key\" or \"device_id\"" + * } + */ +router.all('/i/users/delete', (req, res) => { + const params = req.countlyParams; + parseArgs(params); + validateUserForGlobalAdmin(params, countlyApi.mgmt.users.deleteUser); +}); + +router.all('/i/users/deleteOwnAccount', (req, res) => { + const params = req.countlyParams; + parseArgs(params); + validateUserForGlobalAdmin(params, countlyApi.mgmt.users.deleteOwnAccount); +}); + +router.all('/i/users/updateHomeSettings', (req, res) => { + const params = req.countlyParams; + parseArgs(params); + validateUserForGlobalAdmin(params, countlyApi.mgmt.users.updateHomeSettings); +}); + +router.all('/i/users/ack', (req, res) => { + const params = req.countlyParams; + parseArgs(params); + validateUserForWriteAPI(countlyApi.mgmt.users.ackNotification, params); +}); + +// Catch-all for /i/users/* - dispatches to plugins or returns error +router.all('/i/users/:action', (req, res) => { + const params = req.countlyParams; + const apiPath = '/i/users'; + const paths = params.paths; + parseArgs(params); + if (!plugins.dispatch(apiPath, { + params: params, + validateUserForDataReadAPI: validateUserForDataReadAPI, + validateUserForMgmtReadAPI: validateUserForMgmtReadAPI, + paths: paths, + validateUserForDataWriteAPI: validateUserForDataWriteAPI, + validateUserForGlobalAdmin: validateUserForGlobalAdmin + })) { + common.returnMessage(params, 400, 'Invalid path, must be one of /create, /update, /deleteOwnAccount or /delete'); + } +}); + +// --- Read endpoints: /o/users --- + +router.all('/o/users/all', (req, res) => { + const params = req.countlyParams; + validateUserForGlobalAdmin(params, countlyApi.mgmt.users.getAllUsers); +}); + +router.all('/o/users/me', (req, res) => { + const params = req.countlyParams; + validateUserForMgmtReadAPI(countlyApi.mgmt.users.getCurrentUser, params); +}); + +router.all('/o/users/id', (req, res) => { + const params = req.countlyParams; + validateUserForGlobalAdmin(params, countlyApi.mgmt.users.getUserById); +}); + +router.all('/o/users/reset_timeban', (req, res) => { + const params = req.countlyParams; + validateUserForGlobalAdmin(params, countlyApi.mgmt.users.resetTimeBan); +}); + +router.all('/o/users/permissions', (req, res) => { + const params = req.countlyParams; + validateRead(params, 'core', function() { + var features = ["core", "events" /* , "global_configurations", "global_applications", "global_users", "global_jobs", "global_upload" */]; + /* + Example structure for featuresPermissionDependency Object + { + [FEATURE name which need other permissions]:{ + [CRUD permission of FEATURE]: { + [DEPENDENT_FEATURE name]:[DEPENDENT_FEATURE required CRUD permissions array] + }, + .... other CRUD permission if necessary + } + }, + { + data_manager: Transformations:{ + c:{ + data_manager:['r','u'] + }, + r:{ + data_manager:['r'] + }, + u:{ + data_manager:['r','u'] + }, + d:{ + data_manager:['r','u'] + }, + } + } + */ + var featuresPermissionDependency = {}; + plugins.dispatch("/permissions/features", { params: params, features: features, featuresPermissionDependency: featuresPermissionDependency }, function() { + common.returnOutput(params, {features, featuresPermissionDependency}); + }); + }); +}); + +// Catch-all for /o/users/* - dispatches to plugins or returns error +router.all('/o/users/:action', (req, res) => { + const params = req.countlyParams; + const apiPath = '/o/users'; + const paths = params.paths; + if (!plugins.dispatch(apiPath, { + params: params, + validateUserForDataReadAPI: validateUserForDataReadAPI, + validateUserForMgmtReadAPI: validateUserForMgmtReadAPI, + paths: paths, + validateUserForDataWriteAPI: validateUserForDataWriteAPI, + validateUserForGlobalAdmin: validateUserForGlobalAdmin + })) { + common.returnMessage(params, 400, 'Invalid path, must be one of /all or /me'); + } +}); + +module.exports = router; diff --git a/api/routes/version.js b/api/routes/version.js new file mode 100644 index 00000000000..c45572b1ce3 --- /dev/null +++ b/api/routes/version.js @@ -0,0 +1,111 @@ +/** + * Version route - returns Countly version information. + * Migrated from the legacy switch/case in requestProcessor.js. + * @module api/routes/version + */ + +const express = require('express'); +const router = express.Router(); +const common = require('../utils/common.js'); +const { validateUser } = require('../utils/rights.js'); +const fs = require('fs'); +const path = require('path'); +const packageJson = require('../../package.json'); + +/** + * Fetches version mark history (filesystem) + * @param {function} callback - callback when response is ready + * @returns {void} void + */ +function loadFsVersionMarks(callback) { + fs.readFile(path.resolve(__dirname, "./../../countly_marked_version.json"), function(err, data) { + if (err) { + callback(err, []); + } + else { + var olderVersions = []; + try { + olderVersions = JSON.parse(data); + } + catch (parseErr) { //unable to parse file + console.log(parseErr); + callback(parseErr, []); + } + if (Array.isArray(olderVersions)) { + //sort versions here. + olderVersions.sort(function(a, b) { + if (typeof a.updated !== "undefined" && typeof b.updated !== "undefined") { + return a.updated - b.updated; + } + else { + return 1; + } + }); + callback(null, olderVersions); + } + } + }); +} + +/** + * Fetches version mark history (database) + * @param {function} callback - callback when response is ready + * @returns {void} void + */ +function loadDbVersionMarks(callback) { + common.db.collection('plugins').find({'_id': 'version'}, {"history": 1}).toArray(function(err, versionDocs) { + if (err) { + console.log(err); + callback(err, []); + return; + } + var history = []; + if (versionDocs[0] && versionDocs[0].history) { + history = versionDocs[0].history; + } + callback(null, history); + }); +} + +// GET /o/countly_version - return version info +router.all('/o/countly_version', (req, res) => { + const params = req.countlyParams; + + validateUser(params, () => { + //load previos version info if exist + loadFsVersionMarks(function(errFs, fsValues) { + loadDbVersionMarks(function(errDb, dbValues) { + //load mongodb version + common.db.command({ buildInfo: 1 }, function(errorV, info) { + var response = {}; + if (errorV) { + response.mongo = errorV; + } + else { + if (info && info.version) { + response.mongo = info.version; + } + } + + if (errFs) { + response.fs = errFs; + } + else { + response.fs = fsValues; + } + if (errDb) { + response.db = errDb; + } + else { + response.db = dbValues; + } + response.pkg = packageJson.version || ""; + var statusCode = (errFs && errDb) ? 400 : 200; + common.returnMessage(params, statusCode, response); + }); + }); + }); + }); +}); + +module.exports = router; diff --git a/api/utils/jwt.js b/api/utils/jwt.js new file mode 100644 index 00000000000..682b32c2617 --- /dev/null +++ b/api/utils/jwt.js @@ -0,0 +1,321 @@ +/** + * Module for JWT authentication utilities + * @module api/utils/jwt + */ + +const jwt = require('jsonwebtoken'); +const crypto = require('crypto'); +const common = require('./common.js'); +const log = require('./log.js')('core:jwt'); + +/** + * Get JWT configuration with defaults + * @returns {object} JWT configuration object + */ +function getConfig() { + const config = common.config.jwt || {}; + return { + secret: process.env.COUNTLY_JWT_SECRET || config.secret || '', + accessTokenExpiry: config.accessTokenExpiry || 900, // 15 minutes + refreshTokenExpiry: config.refreshTokenExpiry || 604800, // 7 days + issuer: config.issuer || 'countly', + algorithm: config.algorithm || 'HS256' + }; +} + +/** + * Check if JWT authentication is properly configured + * @returns {boolean} true if JWT secret is configured + */ +function isConfigured() { + const config = getConfig(); + return config.secret && config.secret.length >= 32; +} + +/** + * Generate a unique token identifier (jti) + * @returns {string} unique identifier + */ +function generateJti() { + return crypto.randomBytes(16).toString('hex'); +} + +/** + * Sign an access token for a member + * @param {object} member - The member object from the database + * @returns {object} Object containing the token and expiration info, or error + */ +function signAccessToken(member) { + const config = getConfig(); + + if (!isConfigured()) { + return { + success: false, + error: 'JWT_NOT_CONFIGURED', + message: 'JWT secret is not configured or is too short (minimum 32 characters)' + }; + } + + const payload = { + sub: member._id.toString(), + type: 'access', + global_admin: member.global_admin || false + }; + + // Include permissions for non-global admins + if (!member.global_admin && member.permission) { + payload.permission = member.permission; + } + + try { + const token = jwt.sign(payload, config.secret, { + algorithm: config.algorithm, + expiresIn: config.accessTokenExpiry, + issuer: config.issuer + }); + + return { + success: true, + token: token, + expiresIn: config.accessTokenExpiry + }; + } + catch (err) { + log.e('Error signing access token:', err); + return { + success: false, + error: 'SIGNING_ERROR', + message: err.message + }; + } +} + +/** + * Sign a refresh token for a member + * @param {string} memberId - The member's _id as a string + * @returns {object} Object containing the token, jti, and expiration info, or error + */ +function signRefreshToken(memberId) { + const config = getConfig(); + + if (!isConfigured()) { + return { + success: false, + error: 'JWT_NOT_CONFIGURED', + message: 'JWT secret is not configured or is too short (minimum 32 characters)' + }; + } + + const jti = generateJti(); + const payload = { + sub: memberId.toString(), + type: 'refresh', + jti: jti + }; + + try { + const token = jwt.sign(payload, config.secret, { + algorithm: config.algorithm, + expiresIn: config.refreshTokenExpiry, + issuer: config.issuer + }); + + return { + success: true, + token: token, + jti: jti, + expiresIn: config.refreshTokenExpiry + }; + } + catch (err) { + log.e('Error signing refresh token:', err); + return { + success: false, + error: 'SIGNING_ERROR', + message: err.message + }; + } +} + +/** + * Verify and decode a JWT token + * @param {string} token - The JWT token to verify + * @param {string} expectedType - Expected token type ('access' or 'refresh') + * @returns {object} Object with valid flag, decoded payload or error info + */ +function verifyToken(token, expectedType) { + const config = getConfig(); + + if (!isConfigured()) { + return { + valid: false, + error: 'JWT_NOT_CONFIGURED', + message: 'JWT secret is not configured' + }; + } + + if (!token) { + return { + valid: false, + error: 'NO_TOKEN', + message: 'No token provided' + }; + } + + try { + const decoded = jwt.verify(token, config.secret, { + algorithms: [config.algorithm], + issuer: config.issuer + }); + + // Validate token type + if (expectedType && decoded.type !== expectedType) { + return { + valid: false, + error: 'INVALID_TOKEN_TYPE', + message: `Expected ${expectedType} token but got ${decoded.type}` + }; + } + + return { + valid: true, + decoded: decoded + }; + } + catch (err) { + if (err.name === 'TokenExpiredError') { + return { + valid: false, + error: 'TOKEN_EXPIRED', + message: 'Token has expired', + expiredAt: err.expiredAt + }; + } + else if (err.name === 'JsonWebTokenError') { + return { + valid: false, + error: 'INVALID_TOKEN', + message: err.message + }; + } + else if (err.name === 'NotBeforeError') { + return { + valid: false, + error: 'TOKEN_NOT_ACTIVE', + message: 'Token is not yet active' + }; + } + else { + log.e('Unexpected error verifying token:', err); + return { + valid: false, + error: 'VERIFICATION_ERROR', + message: err.message + }; + } + } +} + +/** + * Extract Bearer token from request Authorization header + * @param {object} req - The request object + * @returns {string|null} The token if found, null otherwise + */ +function extractBearerToken(req) { + if (!req || !req.headers) { + return null; + } + + const authHeader = req.headers.authorization || req.headers.Authorization; + if (!authHeader) { + return null; + } + + // Check for Bearer token format + const parts = authHeader.split(' '); + if (parts.length !== 2 || parts[0].toLowerCase() !== 'bearer') { + return null; + } + + return parts[1]; +} + +/** + * Check if a refresh token JTI is blacklisted + * @param {string} jti - The token's unique identifier + * @param {function} callback - Callback function(err, isBlacklisted) + */ +function isTokenBlacklisted(jti, callback) { + common.db.collection('jwt_blacklist').findOne({ _id: jti }, function(err, doc) { + if (err) { + log.e('Error checking token blacklist:', err); + callback(err, false); + } + else { + callback(null, !!doc); + } + }); +} + +/** + * Add a refresh token JTI to the blacklist + * @param {string} jti - The token's unique identifier + * @param {string} memberId - The member ID who owned the token + * @param {Date} expiresAt - When the original token would have expired + * @param {function} callback - Callback function(err) + */ +function blacklistToken(jti, memberId, expiresAt, callback) { + const doc = { + _id: jti, + member_id: memberId, + revoked_at: new Date(), + expires_at: expiresAt + }; + + common.db.collection('jwt_blacklist').insertOne(doc, function(err) { + if (err) { + // Ignore duplicate key errors (token already blacklisted) + if (err.code === 11000) { + callback(null); + } + else { + log.e('Error blacklisting token:', err); + callback(err); + } + } + else { + callback(null); + } + }); +} + +/** + * Ensure the jwt_blacklist collection has a TTL index for automatic cleanup + * @param {function} callback - Callback function(err) + */ +function ensureIndexes(callback) { + common.db.collection('jwt_blacklist').createIndex( + { expires_at: 1 }, + { expireAfterSeconds: 0 }, + function(err) { + if (err) { + log.e('Error creating TTL index on jwt_blacklist:', err); + } + if (callback) { + callback(err); + } + } + ); +} + +module.exports = { + getConfig, + isConfigured, + signAccessToken, + signRefreshToken, + verifyToken, + extractBearerToken, + isTokenBlacklisted, + blacklistToken, + ensureIndexes +}; diff --git a/api/utils/requestProcessor.js b/api/utils/requestProcessor.js index 511208d6494..c1722f01b1c 100644 --- a/api/utils/requestProcessor.js +++ b/api/utils/requestProcessor.js @@ -3,6 +3,8 @@ * @module api/utils/requestProcessor */ + + /** * @typedef {import('../../types/requestProcessor').Params} Params * @typedef {import('../../types/common').TimeObject} TimeObject @@ -79,6 +81,7 @@ const reloadConfig = function() { }); }; + /** * Default request processing handler, which requires request context to operate. Check tcp_example.js * @static @@ -115,62 +118,73 @@ const processRequest = (params) => { return common.returnMessage(params, 400, "Please provide request data"); } - const urlParts = url.parse(params.req.url, true), - queryString = urlParts.query, - paths = urlParts.pathname.split("/"); - params.href = urlParts.href; - params.qstring = params.qstring || {}; - params.res = params.res || {}; - params.urlParts = urlParts; - params.paths = paths; - - //request object fillers - params.req.method = params.req.method || "custom"; - params.req.headers = params.req.headers || {}; - params.req.socket = params.req.socket || {}; - params.req.connection = params.req.connection || {}; - - //copying query string data as qstring param - if (queryString) { - for (let i in queryString) { - params.qstring[i] = queryString[i]; + // When called via the Express legacy bridge, URL parsing and qstring + // merging were already done by the params middleware. Skip re-parsing + // to avoid duplicating query/body params. Programmatic callers (e.g. + // star-rating, taskmanager) do not set _expressParsed, so they still + // go through the original parsing path. + if (!params._expressParsed) { + const urlParts = url.parse(params.req.url, true), + queryString = urlParts.query, + paths = urlParts.pathname.split("/"); + params.href = urlParts.href; + params.qstring = params.qstring || {}; + params.res = params.res || {}; + params.urlParts = urlParts; + params.paths = paths; + + //request object fillers + params.req.method = params.req.method || "custom"; + params.req.headers = params.req.headers || {}; + params.req.socket = params.req.socket || {}; + params.req.connection = params.req.connection || {}; + + //copying query string data as qstring param + if (queryString) { + for (let i in queryString) { + params.qstring[i] = queryString[i]; + } } - } - //copying body as qstring param - if (params.req.body && typeof params.req.body === "object") { - for (let i in params.req.body) { - params.qstring[i] = params.req.body[i]; + //copying body as qstring param + if (params.req.body && typeof params.req.body === "object") { + for (let i in params.req.body) { + params.qstring[i] = params.req.body[i]; + } } - } - if (params.qstring.app_id && params.qstring.app_id.length !== 24) { - common.returnMessage(params, 400, 'Invalid parameter "app_id"'); - return false; - } + if (params.qstring.app_id && params.qstring.app_id.length !== 24) { + common.returnMessage(params, 400, 'Invalid parameter "app_id"'); + return false; + } - if (params.qstring.user_id && params.qstring.user_id.length !== 24) { - common.returnMessage(params, 400, 'Invalid parameter "user_id"'); - return false; - } + if (params.qstring.user_id && params.qstring.user_id.length !== 24) { + common.returnMessage(params, 400, 'Invalid parameter "user_id"'); + return false; + } - //remove countly path - if (common.config.path === "/" + paths[1]) { - paths.splice(1, 1); - } + //remove countly path + if (common.config.path === "/" + params.paths[1]) { + params.paths.splice(1, 1); + } + + let apiPath = ''; - let apiPath = ''; + for (let i = 1; i < params.paths.length; i++) { + if (i > 2) { + break; + } - for (let i = 1; i < paths.length; i++) { - if (i > 2) { - break; + apiPath += "/" + params.paths[i]; } - apiPath += "/" + paths[i]; + params.apiPath = apiPath; + params.fullPath = params.paths.join("/"); } - params.apiPath = apiPath; - params.fullPath = paths.join("/"); + const paths = params.paths; + const apiPath = params.apiPath; + const urlParts = params.urlParts; reloadConfig().then(function() { plugins.dispatch("/", { @@ -185,3196 +199,37 @@ const processRequest = (params) => { }); if (!params.cancelRequest) { + console.log("Processing", apiPath); switch (apiPath) { - case '/i/users': { - if (params.qstring.args) { - try { - params.qstring.args = JSON.parse(params.qstring.args); - } - catch (SyntaxError) { - console.log('Parse %s JSON failed. URL: %s, Body: %s', apiPath, params.req.url, JSON.stringify(params.req.body)); - } - } - - switch (paths[3]) { - /** - * @api {get} /i/users/create Create new user - * @apiName Create User - * @apiGroup User Management - * - * @apiDescription Access database, get collections, indexes and data - * @apiQuery {Object} args User data object - * @apiQuery {String} args.full_name Full name - * @apiQuery {String} args.username Username - * @apiQuery {String} args.password Password - * @apiQuery {String} args.email Email - * @apiQuery {Object} args.permission Permission object - * @apiQuery {Boolean} args.global_admin Global admin flag - * - * @apiSuccessExample {json} Success-Response: - * HTTP/1.1 200 OK - * { - * "full_name":"fn", - * "username":"un", - * "email":"e@ms.cd", - * "permission": { - * "c":{}, - * "r":{}, - * "u":{}, - * "d":{}, - * "_":{ - * "u":[[]], - * "a":[] - * } - * }, - * "global_admin":true, - * "password_changed":0, - * "created_at":1651240780, - * "locked":false, - * "api_key":"1c5e93c6657d76ae8903f14c32cb3796", - * "_id":"626bef4cb00db29a02f8f7a0" - * } - * - * @apiErrorExample {json} Error-Response: - * HTTP/1.1 400 Bad Request - * { - * "result": "Missing parameter \"app_key\" or \"device_id\""" - * } - */ - case 'create': - validateUserForGlobalAdmin(params, countlyApi.mgmt.users.createUser); - break; - /** - * @api {get} /i/users/update Update user - * @apiName Update User - * @apiGroup User Management - * - * @apiDescription Access database, get collections, indexes and data - * @apiQuery {Object} args User data object - * @apiQuery {String} args.full_name Full name - * @apiQuery {String} args.username Username - * @apiQuery {String} args.password Password - * @apiQuery {String} args.email Email - * @apiQuery {Object} args.permission Permission object - * @apiQuery {Boolean} args.global_admin Global admin flag - * - * @apiSuccessExample {json} Success-Response: - * HTTP/1.1 200 OK - * { - * "result":"Success" - * } - * - * @apiErrorExample {json} Error-Response: - * HTTP/1.1 400 Bad Request - * { - * "result": "Missing parameter \"app_key\" or \"device_id\""" - * } - */ - case 'update': - validateUserForGlobalAdmin(params, countlyApi.mgmt.users.updateUser); - break; - /** - * @api {get} /i/users/delete Delete user - * @apiName Delete User - * @apiGroup User Management - * - * @apiDescription Access database, get collections, indexes and data - * @apiQuery {Object} args User data object - * @apiQuery {String} args.user_ids IDs array for users which will be deleted - * - * @apiSuccessExample {json} Success-Response: - * HTTP/1.1 200 OK - * { - * "result":"Success" - * } - * - * @apiErrorExample {json} Error-Response: - * HTTP/1.1 400 Bad Request - * { - * "result": "Missing parameter \"app_key\" or \"device_id\""" - * } - */ - case 'delete': - validateUserForGlobalAdmin(params, countlyApi.mgmt.users.deleteUser); - break; - case 'deleteOwnAccount': - validateUserForGlobalAdmin(params, countlyApi.mgmt.users.deleteOwnAccount); - break; - case 'updateHomeSettings': - validateUserForGlobalAdmin(params, countlyApi.mgmt.users.updateHomeSettings); - break; - case 'ack': - validateUserForWriteAPI(countlyApi.mgmt.users.ackNotification, params); - break; - default: - if (!plugins.dispatch(apiPath, { - params: params, - validateUserForDataReadAPI: validateUserForDataReadAPI, - validateUserForMgmtReadAPI: validateUserForMgmtReadAPI, - paths: paths, - validateUserForDataWriteAPI: validateUserForDataWriteAPI, - validateUserForGlobalAdmin: validateUserForGlobalAdmin - })) { - common.returnMessage(params, 400, 'Invalid path, must be one of /create, /update, /deleteOwnAccount or /delete'); - } - break; - } - - break; - } - case '/i/notes': { - if (params.qstring.args) { - try { - params.qstring.args = JSON.parse(params.qstring.args); - } - catch (SyntaxError) { - console.log('Parse %s JSON failed %s', apiPath, params.req.url, params.req.body); - } - } - switch (paths[3]) { - case 'save': - validateCreate(params, 'core', () => { - countlyApi.mgmt.users.saveNote(params); - }); - break; - case 'delete': - validateDelete(params, 'core', () => { - countlyApi.mgmt.users.deleteNote(params); - }); - break; - } - break; - } - case '/o/render': { - validateUserForRead(params, function() { - var options = {}; - var view = params.qstring.view || ""; - var route = params.qstring.route || ""; - var id = params.qstring.id || ""; - - options.view = view + "#" + route; - options.id = id ? "#" + id : ""; - - var imageName = "screenshot_" + common.crypto.randomBytes(16).toString("hex") + ".png"; - - options.savePath = path.resolve(__dirname, "../../frontend/express/public/images/screenshots/" + imageName); - options.source = "core"; - - authorize.save({ - db: common.db, - multi: false, - owner: params.member._id, - ttl: 300, - purpose: "LoginAuthToken", - callback: function(err2, token) { - if (err2) { - common.returnMessage(params, 400, 'Error creating token: ' + err2); - return false; - } - options.token = token; - render.renderView(options, function(err3) { - if (err3) { - common.returnMessage(params, 400, 'Error creating screenshot. Please check logs for more information.'); - return false; - } - common.returnOutput(params, {path: common.config.path + "/images/screenshots/" + imageName}); - }); - } - }); - }); - break; - } - case '/i/app_users': { - switch (paths[3]) { - case 'create': { - if (!params.qstring.app_id) { - common.returnMessage(params, 400, 'Missing parameter "app_id"'); - return false; - } - if (!params.qstring.data) { - common.returnMessage(params, 400, 'Missing parameter "data"'); - return false; - } - else if (typeof params.qstring.data === "string") { - try { - params.qstring.data = JSON.parse(params.qstring.data); - } - catch (ex) { - console.log("Could not parse data", params.qstring.data); - common.returnMessage(params, 400, 'Could not parse parameter "data": ' + params.qstring.data); - return false; - } - } - if (!Object.keys(params.qstring.data).length) { - common.returnMessage(params, 400, 'Parameter "data" cannot be empty'); - return false; - } - validateUserForWrite(params, function() { - countlyApi.mgmt.appUsers.create(params.qstring.app_id, params.qstring.data, params, function(err, res) { - if (err) { - common.returnMessage(params, 400, err); - } - else { - common.returnMessage(params, 200, 'User Created: ' + JSON.stringify(res)); - } - }); - }); - break; - } - case 'update': { - if (!params.qstring.app_id) { - common.returnMessage(params, 400, 'Missing parameter "app_id"'); - return false; - } - if (!params.qstring.update) { - common.returnMessage(params, 400, 'Missing parameter "update"'); - return false; - } - else if (typeof params.qstring.update === "string") { - try { - params.qstring.update = JSON.parse(params.qstring.update); - } - catch (ex) { - console.log("Could not parse update", params.qstring.update); - common.returnMessage(params, 400, 'Could not parse parameter "update": ' + params.qstring.update); - return false; - } - } - if (!Object.keys(params.qstring.update).length) { - common.returnMessage(params, 400, 'Parameter "update" cannot be empty'); - return false; - } - if (!params.qstring.query) { - common.returnMessage(params, 400, 'Missing parameter "query"'); - return false; - } - else if (typeof params.qstring.query === "string") { - try { - params.qstring.query = JSON.parse(params.qstring.query); - } - catch (ex) { - console.log("Could not parse query", params.qstring.query); - common.returnMessage(params, 400, 'Could not parse parameter "query": ' + params.qstring.query); - return false; - } - } - validateUserForWrite(params, function() { - countlyApi.mgmt.appUsers.count(params.qstring.app_id, params.qstring.query, function(err, count) { - if (err || count === 0) { - common.returnMessage(params, 400, 'No users matching criteria'); - return false; - } - if (count > 1 && !params.qstring.force) { - common.returnMessage(params, 400, 'This query would update more than one user'); - return false; - } - countlyApi.mgmt.appUsers.update(params.qstring.app_id, params.qstring.query, params.qstring.update, params, function(err2) { - if (err2) { - common.returnMessage(params, 400, err2); - } - else { - common.returnMessage(params, 200, 'User Updated'); - } - }); - }); - }); - break; - } - case 'delete': { - if (!params.qstring.app_id) { - common.returnMessage(params, 400, 'Missing parameter "app_id"'); - return false; - } - if (!params.qstring.query) { - common.returnMessage(params, 400, 'Missing parameter "query"'); - return false; - } - else if (typeof params.qstring.query === "string") { - try { - params.qstring.query = JSON.parse(params.qstring.query); - } - catch (ex) { - console.log("Could not parse query", params.qstring.query); - common.returnMessage(params, 400, 'Could not parse parameter "query": ' + params.qstring.query); - return false; - } - } - if (!Object.keys(params.qstring.query).length) { - common.returnMessage(params, 400, 'Parameter "query" cannot be empty, it would delete all users. Use clear app instead'); - return false; - } - validateUserForWrite(params, function() { - countlyApi.mgmt.appUsers.count(params.qstring.app_id, params.qstring.query, function(err, count) { - if (err || count === 0) { - common.returnMessage(params, 400, 'No users matching criteria'); - return false; - } - if (count > 1 && !params.qstring.force) { - common.returnMessage(params, 400, 'This query would delete more than one user'); - return false; - } - countlyApi.mgmt.appUsers.delete(params.qstring.app_id, params.qstring.query, params, function(err2) { - if (err2) { - common.returnMessage(params, 400, err2); - } - else { - common.returnMessage(params, 200, 'User deleted'); - } - }); - }); - }); - break; - } - /** - * @api {get} /i/app_users/deleteExport/:id Deletes user export. - * @apiName Delete user export - * @apiGroup App User Management - * @apiDescription Deletes user export. - * - * @apiParam {Number} id Id of export. For single user it would be similar to: appUser_644658291e95e720503d5087_1, but for multiple users - appUser_62e253489315313ffbc2c457_HASH_3e5b86cb367a6b8c0689ffd80652d2bbcb0a3edf - * - * @apiQuery {String} app_id Application id - * - * @apiSuccessExample {json} Success-Response: - * HTTP/1.1 200 OK - * { - * "result":"Export deleted" - * } - * @apiErrorExample {json} Error-Response: - * HTTP/1.1 400 Bad Request - * { - * "result": "Missing parameter \"app_id\"" - * } - */ - case 'deleteExport': { - validateUserForWrite(params, function() { - countlyApi.mgmt.appUsers.deleteExport(paths[4], params, function(err) { - if (err) { - common.returnMessage(params, 400, err); - } - else { - common.returnMessage(params, 200, 'Export deleted'); - } - }); - }); - break; - } - /** - * @api {get} /i/app_users/export Exports all data collected about app user - * @apiName Export user data - * @apiGroup App User Management - * - * @apiDescription Creates export and stores in database. export is downloadable on demand. - * @apiQuery {String} app_id Application id - * @apiQuery {String} query Query to match users to run export on. Query should be runnable on mongodb database. For example: {"uid":"1"} will find user, for whuch uid === "1" If is possible to export also multiple users in same export. - * - * @apiSuccessExample {json} Success-Response: - * HTTP/1.1 200 OK - * { - * "result": "appUser_644658291e95e720503d5087_1.json" - * } - * @apiErrorExample {json} Error-Response: - * HTTP/1.1 400 Bad Request - * { - * "result": "Missing parameter \"app_id\"" - * } - */ - case 'export': { - if (!params.qstring.app_id) { - common.returnMessage(params, 400, 'Missing parameter "app_id"'); - return false; - } - validateUserForWrite(params, function() { - taskmanager.checkIfRunning({ - db: common.db, - params: params //allow generate request from params, as it is what identifies task in drill - }, function(task_id) { - //check if task already running - if (task_id) { - common.returnOutput(params, {task_id: task_id}); - } - else { - if (!params.qstring.query) { - common.returnMessage(params, 400, 'Missing parameter "query"'); - return false; - } - else if (typeof params.qstring.query === "string") { - try { - params.qstring.query = JSON.parse(params.qstring.query); - } - catch (ex) { - console.log("Could not parse query", params.qstring.query); - common.returnMessage(params, 400, 'Could not parse parameter "query": ' + params.qstring.query); - return false; - } - } - - var my_name = ""; - if (params.qstring.query) { - my_name = JSON.stringify(params.qstring.query); - } - - countlyApi.mgmt.appUsers.export(params.qstring.app_id, params.qstring.query || {}, params, taskmanager.longtask({ - db: common.db, - threshold: plugins.getConfig("api").request_threshold, - force: false, - app_id: params.qstring.app_id, - params: params, - type: "AppUserExport", - report_name: "User export", - meta: JSON.stringify({ - "app_id": params.qstring.app_id, - "query": params.qstring.query || {} - }), - name: my_name, - view: "#/exportedData/AppUserExport/", - processData: function(err, res, callback) { - if (!err) { - callback(null, res); - } - else { - callback(err, ''); - } - }, - outputData: function(err, data) { - if (err) { - common.returnMessage(params, 400, err); - } - else { - common.returnMessage(params, 200, data); - } - } - })); - } - }); - }); - break; - } - default: - if (!plugins.dispatch(apiPath, { - params: params, - validateUserForDataReadAPI: validateUserForDataReadAPI, - validateUserForMgmtReadAPI: validateUserForMgmtReadAPI, - paths: paths, - validateUserForDataWriteAPI: validateUserForDataWriteAPI, - validateUserForGlobalAdmin: validateUserForGlobalAdmin - })) { - common.returnMessage(params, 400, 'Invalid path, must be one of /all or /me'); - } - break; - } - break; - } - case '/i/apps': { - if (params.qstring.args) { - try { - params.qstring.args = JSON.parse(params.qstring.args); - } - catch (SyntaxError) { - console.log('Parse %s JSON failed %s', apiPath, params.req.url, params.req.body); - } - } - - switch (paths[3]) { - case 'create': - validateUserForGlobalAdmin(params, countlyApi.mgmt.apps.createApp); - break; - case 'update': - if (paths[4] === 'plugins') { - validateAppAdmin(params, countlyApi.mgmt.apps.updateAppPlugins); - } - else { - if (params.qstring.app_id) { - validateAppAdmin(params, countlyApi.mgmt.apps.updateApp); - } - else { - validateUserForGlobalAdmin(params, countlyApi.mgmt.apps.updateApp); - } - } - break; - case 'delete': - validateUserForGlobalAdmin(params, countlyApi.mgmt.apps.deleteApp); - break; - case 'reset': - validateUserForGlobalAdmin(params, countlyApi.mgmt.apps.resetApp); - break; - default: - if (!plugins.dispatch(apiPath, { - params: params, - validateUserForDataReadAPI: validateUserForDataReadAPI, - validateUserForMgmtReadAPI: validateUserForMgmtReadAPI, - paths: paths, - validateUserForDataWriteAPI: validateUserForDataWriteAPI, - validateUserForGlobalAdmin: validateUserForGlobalAdmin - })) { - common.returnMessage(params, 400, 'Invalid path, must be one of /create, /update, /delete or /reset'); - } - break; - } - - break; - } - case '/i/event_groups': - switch (paths[3]) { - case 'create': - if (!params.qstring.args) { - common.returnMessage(params, 400, 'Error: args not found'); - return false; - } - try { - params.qstring.args = JSON.parse(params.qstring.args); - params.qstring.app_id = params.qstring.args.app_id; - } - catch (SyntaxError) { - console.log('Parse %s JSON failed %s', apiPath, params.req.url, params.req.body); - common.returnMessage(params, 400, 'Error: could not parse args'); - return false; - } - validateCreate(params, 'core', countlyApi.mgmt.eventGroups.create); - break; - case 'update': - validateUpdate(params, 'core', countlyApi.mgmt.eventGroups.update); - break; - case 'delete': - validateDelete(params, 'core', countlyApi.mgmt.eventGroups.remove); - break; - default: - break; - } - break; - case '/i/tasks': { - if (!params.qstring.task_id) { - common.returnMessage(params, 400, 'Missing parameter "task_id"'); - return false; - } - - switch (paths[3]) { - case 'update': - validateUserForWrite(params, () => { - taskmanager.rerunTask({ - db: common.db, - id: params.qstring.task_id - }, (err, res) => { - common.returnMessage(params, 200, res); - }); - }); - break; - case 'delete': - validateUserForWrite(params, () => { - taskmanager.deleteResult({ - db: common.db, - id: params.qstring.task_id - }, (err, task) => { - plugins.dispatch("/systemlogs", {params: params, action: "task_manager_task_deleted", data: task}); - common.returnMessage(params, 200, "Success"); - }); - }); - break; - case 'name': - validateUserForWrite(params, () => { - taskmanager.nameResult({ - db: common.db, - id: params.qstring.task_id, - name: params.qstring.name - }, () => { - common.returnMessage(params, 200, "Success"); - }); - }); - break; - case 'edit': - validateUserForWrite(params, () => { - const data = { - "report_name": params.qstring.report_name, - "report_desc": params.qstring.report_desc, - "global": params.qstring.global + "" === 'true', - "autoRefresh": params.qstring.autoRefresh + "" === 'true', - "period_desc": params.qstring.period_desc - }; - taskmanager.editTask({ - db: common.db, - data: data, - id: params.qstring.task_id - }, (err, d) => { - if (err) { - common.returnMessage(params, 503, "Error"); - } - else { - common.returnMessage(params, 200, "Success"); - } - plugins.dispatch("/systemlogs", {params: params, action: "task_manager_task_updated", data: d}); - }); - }); - break; - default: - if (!plugins.dispatch(apiPath, { - params: params, - validateUserForDataReadAPI: validateUserForDataReadAPI, - validateUserForMgmtReadAPI: validateUserForMgmtReadAPI, - paths: paths, - validateUserForDataWriteAPI: validateUserForDataWriteAPI, - validateUserForGlobalAdmin: validateUserForGlobalAdmin - })) { - common.returnMessage(params, 400, 'Invalid path'); - } - break; - } - - break; - } - case '/i/events': { - switch (paths[3]) { - case 'whitelist_segments': - { - validateUpdate(params, "events", function() { - common.db.collection('events').findOne({"_id": common.db.ObjectID(params.qstring.app_id)}, function(err, event) { - if (err) { - common.returnMessage(params, 400, err); - return; - } - else if (!event) { - common.returnMessage(params, 400, "Could not find record in event collection"); - return; - } - - //rewrite whitelisted - if (params.qstring.whitelisted_segments && params.qstring.whitelisted_segments !== "") { - try { - params.qstring.whitelisted_segments = JSON.parse(params.qstring.whitelisted_segments); - } - catch (SyntaxError) { - params.qstring.whitelisted_segments = {}; console.log('Parse ' + params.qstring.whitelisted_segments + ' JSON failed', params.req.url, params.req.body); - } - - var update = {}; - var whObj = params.qstring.whitelisted_segments; - for (let k in whObj) { - if (Array.isArray(whObj[k]) && whObj[k].length > 0) { - update.$set = update.$set || {}; - update.$set["whitelisted_segments." + k] = whObj[k]; - } - else { - update.$unset = update.$unset || {}; - update.$unset["whitelisted_segments." + k] = true; - } - } - - common.db.collection('events').update({"_id": common.db.ObjectID(params.qstring.app_id)}, update, function(err2) { - if (err2) { - common.returnMessage(params, 400, err2); - } - else { - var data_arr = {update: {}}; - if (update.$set) { - data_arr.update.$set = update.$set; - } - - if (update.$unset) { - data_arr.update.$unset = update.$unset; - } - data_arr.update = JSON.stringify(data_arr.update); - common.returnMessage(params, 200, 'Success'); - plugins.dispatch("/systemlogs", { - params: params, - action: "segments_whitelisted_for_events", - data: data_arr - }); - } - }); - - } - else { - common.returnMessage(params, 400, "Value for 'whitelisted_segments' missing"); - return; - } - - - }); - }); - break; - } - case 'edit_map': - { - if (!params.qstring.app_id) { - common.returnMessage(params, 400, 'Missing parameter "app_id"'); - return false; - } - validateUpdate(params, 'events', function() { - common.db.collection('events').findOne({"_id": common.db.ObjectID(params.qstring.app_id)}, async function(err, event) { - if (err) { - common.returnMessage(params, 400, err); - return; - } - else if (!event) { - common.returnMessage(params, 400, "Could not find event"); - return; - } - //Load available events - - const pluginsGetConfig = plugins.getConfig("api", params.app && params.app.plugins, true); - - var list = await common.drillDb.collection("drill_meta").aggregate( - [ - {$match: {"app_id": params.qstring.app_id, "type": "e", "biglist": {"$ne": true}}}, - {$match: {"e": {"$not": /^(\[CLY\]_)/}}}, - {"$group": {"_id": "$e"}}, - {"$sort": {"_id": 1}}, - {"$limit": pluginsGetConfig.event_limit || 500}, - {"$group": {"_id": null, "list": {"$addToSet": "$_id"}}} - ] - , {"allowDiskUse": true}).toArray(); - event.list = list[0].list; - - var update_array = {}; - var update_segments = []; - var pull_us = {}; - if (params.qstring.event_order && params.qstring.event_order !== "") { - try { - update_array.order = JSON.parse(params.qstring.event_order); - } - catch (SyntaxError) { - update_array.order = event.order; console.log('Parse ' + params.qstring.event_order + ' JSON failed', params.req.url, params.req.body); - } - } - else { - update_array.order = event.order || []; - } - - if (params.qstring.event_overview && params.qstring.event_overview !== "") { - try { - update_array.overview = JSON.parse(params.qstring.event_overview); - } - catch (SyntaxError) { - update_array.overview = []; console.log('Parse ' + params.qstring.event_overview + ' JSON failed', params.req.url, params.req.body); - } - if (update_array.overview && Array.isArray(update_array.overview)) { - if (update_array.overview.length > 12) { - common.returnMessage(params, 400, "You can't add more than 12 items in overview"); - return; - } - //sanitize overview - var allowedEventKeys = event.list; - var allowedProperties = ['dur', 'sum', 'count']; - var propertyNames = { - 'dur': 'Dur', - 'sum': 'Sum', - 'count': 'Count' - }; - for (let i = 0; i < update_array.overview.length; i++) { - update_array.overview[i].order = i; - update_array.overview[i].eventKey = update_array.overview[i].eventKey || ""; - update_array.overview[i].eventProperty = update_array.overview[i].eventProperty || ""; - if (allowedEventKeys.indexOf(update_array.overview[i].eventKey) === -1 || allowedProperties.indexOf(update_array.overview[i].eventProperty) === -1) { - update_array.overview.splice(i, 1); - i = i - 1; - } - else { - update_array.overview[i].is_event_group = (typeof update_array.overview[i].is_event_group === 'boolean' && update_array.overview[i].is_event_group) || false; - update_array.overview[i].eventName = update_array.overview[i].eventName || update_array.overview[i].eventKey; - update_array.overview[i].propertyName = propertyNames[update_array.overview[i].eventProperty]; - } - } - //check for duplicates - var overview_map = Object.create(null); - for (let p = 0; p < update_array.overview.length; p++) { - if (!overview_map[update_array.overview[p].eventKey]) { - overview_map[update_array.overview[p].eventKey] = {}; - } - if (!overview_map[update_array.overview[p].eventKey][update_array.overview[p].eventProperty]) { - overview_map[update_array.overview[p].eventKey][update_array.overview[p].eventProperty] = 1; - } - else { - update_array.overview.splice(p, 1); - p = p - 1; - } - } - } - } - else { - update_array.overview = event.overview || []; - } - - update_array.omitted_segments = {}; - - if (event.omitted_segments) { - try { - update_array.omitted_segments = JSON.parse(JSON.stringify(event.omitted_segments)); - } - catch (SyntaxError) { - update_array.omitted_segments = {}; - } - } - - if (params.qstring.omitted_segments && params.qstring.omitted_segments !== "") { - var omitted_segments_empty = false; - try { - params.qstring.omitted_segments = JSON.parse(params.qstring.omitted_segments); - if (JSON.stringify(params.qstring.omitted_segments) === '{}') { - omitted_segments_empty = true; - } - } - catch (SyntaxError) { - params.qstring.omitted_segments = {}; console.log('Parse ' + params.qstring.omitted_segments + ' JSON failed', params.req.url, params.req.body); - } - - for (let k in params.qstring.omitted_segments) { - update_array.omitted_segments[k] = params.qstring.omitted_segments[k]; - update_segments.push({ - "key": k, - "list": params.qstring.omitted_segments[k] - }); - pull_us["segments." + k] = {$in: params.qstring.omitted_segments[k]}; - } - if (omitted_segments_empty) { - var events = JSON.parse(params.qstring.event_map); - for (let k in events) { - if (update_array.omitted_segments[k]) { - delete update_array.omitted_segments[k]; - } - } - } - } - - if (params.qstring.event_map && params.qstring.event_map !== "") { - try { - params.qstring.event_map = JSON.parse(params.qstring.event_map); - } - catch (SyntaxError) { - params.qstring.event_map = {}; console.log('Parse ' + params.qstring.event_map + ' JSON failed', params.req.url, params.req.body); - } - - if (event.map) { - try { - update_array.map = JSON.parse(JSON.stringify(event.map)); - } - catch (SyntaxError) { - update_array.map = {}; - } - } - else { - update_array.map = {}; - } - - - for (let k in params.qstring.event_map) { - if (Object.prototype.hasOwnProperty.call(params.qstring.event_map, k)) { - update_array.map[k] = params.qstring.event_map[k]; - - if (update_array.map[k].is_visible && update_array.map[k].is_visible === true) { - delete update_array.map[k].is_visible; - } - if (update_array.map[k].name && update_array.map[k].name === k) { - delete update_array.map[k].name; - } - - if (update_array.map[k] && typeof update_array.map[k].is_visible !== 'undefined' && update_array.map[k].is_visible === false) { - for (var j = 0; j < update_array.overview.length; j++) { - if (update_array.overview[j].eventKey === k) { - update_array.overview.splice(j, 1); - j = j - 1; - } - } - } - if (Object.keys(update_array.map[k]).length === 0) { - delete update_array.map[k]; - } - } - } - } - var changes = {$set: update_array}; - if (Object.keys(pull_us).length > 0) { - changes = { - $set: update_array, - $pull: pull_us - }; - } - - common.db.collection('events').update({"_id": common.db.ObjectID(params.qstring.app_id)}, changes, function(err2) { - if (err2) { - common.returnMessage(params, 400, err2); - } - else { - var data_arr = {update: update_array}; - data_arr.before = { - order: [], - map: {}, - overview: [], - omitted_segments: {} - }; - if (event.order) { - data_arr.before.order = event.order; - } - if (event.map) { - data_arr.before.map = event.map; - } - if (event.overview) { - data_arr.before.overview = event.overview; - } - if (event.omitted_segments) { - data_arr.before.omitted_segments = event.omitted_segments; - } - - //updated, clear out segments - Promise.all(update_segments.map(function(obj) { - return new Promise(function(resolve) { - var collectionNameWoPrefix = common.crypto.createHash('sha1').update(obj.key + params.qstring.app_id).digest('hex'); - //removes all document for current segment - common.db.collection("events_data").remove({"_id": {"$regex": ("^" + params.qstring.app_id + "_" + collectionNameWoPrefix + "_.*")}, "s": {$in: obj.list}}, {multi: true}, function(err3) { - if (err3) { - console.log(err3); - } - //create query for all segments - var my_query = []; - var unsetUs = {}; - if (obj.list.length > 0) { - for (let p = 0; p < obj.list.length; p++) { - my_query[p] = {}; - my_query[p]["meta_v2.segments." + obj.list[p]] = {$exists: true}; //for select - unsetUs["meta_v2.segments." + obj.list[p]] = ""; //remove from list - unsetUs["meta_v2." + obj.list[p]] = ""; - } - //clears out meta data for segments - common.db.collection("events_data").update({"_id": {"$regex": ("^" + params.qstring.app_id + "_" + collectionNameWoPrefix + "_.*")}, $or: my_query}, {$unset: unsetUs}, {multi: true}, function(err4) { - if (err4) { - console.log(err4); - } - if (plugins.isPluginEnabled('drill')) { - //remove from drill - var eventHash = common.crypto.createHash('sha1').update(obj.key + params.qstring.app_id).digest('hex'); - common.drillDb.collection("drill_meta").findOne({_id: params.qstring.app_id + "_meta_" + eventHash}, function(err5, resEvent) { - if (err5) { - console.log(err5); - } - - var newsg = {}; - var remove_biglists = []; - resEvent = resEvent || {}; - resEvent.sg = resEvent.sg || {}; - for (let p = 0; p < obj.list.length; p++) { - remove_biglists.push(params.qstring.app_id + "_meta_" + eventHash + "_sg." + obj.list[p]); - newsg["sg." + obj.list[p]] = {"type": "s"}; - } - //big list, delete also big list file - if (remove_biglists.length > 0) { - common.drillDb.collection("drill_meta").remove({_id: {$in: remove_biglists}}, function(err6) { - if (err6) { - console.log(err6); - } - common.drillDb.collection("drill_meta").update({_id: params.qstring.app_id + "_meta_" + eventHash}, {$set: newsg}, function(err7) { - if (err7) { - console.log(err7); - } - resolve(); - }); - }); - } - else { - common.drillDb.collection("drill_meta").update({_id: params.qstring.app_id + "_meta_" + eventHash}, {$set: newsg}, function() { - resolve(); - }); - } - }); - } - else { - resolve(); - } - }); - } - else { - resolve(); - } - }); - }); - - })).then(function() { - common.returnMessage(params, 200, 'Success'); - plugins.dispatch("/systemlogs", { - params: params, - action: "events_updated", - data: data_arr - }); - - }) - .catch((error) => { - console.log(error); - common.returnMessage(params, 400, 'Events were updated sucessfully. There was error during clearing segment data. Please look in log for more onformation'); - }); - - } - }); - }); - }); - break; - } - /** - * @api {get} /i/events/delete_events Delete event - * @apiName Delete Event - * @apiGroup Events Management - * - * @apiDescription Deletes one or multiple events. Params can be send as POST and also as GET. - * @apiQuery {String} app_id Application id - * @apiQuery {String} events JSON array of event keys to delete. For example: ["event1", "event2"]. Value must be passed as string. (Array must be stringified before passing to API) - * - * @apiSuccessExample {json} Success-Response: - * HTTP/1.1 200 OK - * { - * "result":"Success" - * } - * - * @apiErrorExample {json} Error-Response: - * HTTP/1.1 400 Bad Request - * { - * "result":"Missing parameter \"api_key\" or \"auth_token\"" - * } - * - * @apiErrorExample {json} Error-Response: - * HTTP/1.1 400 Bad Request - * { - * "result":"Could not find event" - * } - */ - case 'delete_events': - { - validateDelete(params, 'events', function() { - var idss = []; - try { - idss = JSON.parse(params.qstring.events); - } - catch (SyntaxError) { - idss = []; - } - - if (!Array.isArray(idss)) { - idss = []; - } - - var app_id = params.qstring.app_id; - var updateThese = {"$unset": {}}; - if (idss.length > 0) { - - common.db.collection('events').findOne({"_id": common.db.ObjectID(params.qstring.app_id)}, function(err, event) { - if (err) { - common.returnMessage(params, 400, err); - } - if (!event) { - common.returnMessage(params, 400, "Could not find event"); - return; - } - let successIds = []; - let failedIds = []; - let promises = []; - for (let i = 0; i < idss.length; i++) { - let collectionNameWoPrefix = common.crypto.createHash('sha1').update(idss[i] + app_id).digest('hex'); - common.db.collection("events" + collectionNameWoPrefix).drop(); - promises.push(new Promise((resolve, reject) => { - plugins.dispatch("/i/event/delete", { - event_key: idss[i], - appId: app_id - }, function(_, otherPluginResults) { - const rejectReasons = otherPluginResults?.reduce((acc, result) => { - if (result?.status === "rejected") { - acc.push((result.reason && result.reason.message) || ''); - } - return acc; - }, []); - - if (rejectReasons?.length) { - failedIds.push(idss[i]); - log.e("Event deletion failed\n%j", rejectReasons.join("\n")); - reject("Event deletion failed. Failed to delete some data related to this Event."); - return; - } - else { - successIds.push(idss[i]); - resolve(); - } - } - ); - })); - } - - Promise.allSettled(promises).then(async() => { - //remove from map, segments, omitted_segments - for (let i = 0; i < successIds.length; i++) { - successIds[i] = successIds[i] + ""; //make sure it is string to do not fail. - if (successIds[i].indexOf('.') !== -1) { - updateThese.$unset["map." + successIds[i].replace(/\./g, '\\u002e')] = 1; - updateThese.$unset["omitted_segments." + successIds[i].replace(/\./g, '\\u002e')] = 1; - } - else { - updateThese.$unset["map." + successIds[i]] = 1; - updateThese.$unset["omitted_segments." + successIds[i]] = 1; - } - successIds[i] = common.decode_html(successIds[i]);//previously escaped, get unescaped id (because segments are using it) - if (successIds[i].indexOf('.') !== -1) { - updateThese.$unset["segments." + successIds[i].replace(/\./g, '\\u002e')] = 1; - } - else { - updateThese.$unset["segments." + successIds[i]] = 1; - } - } - //fix overview - if (event.overview && event.overview.length) { - for (let i = 0; i < successIds.length; i++) { - for (let j = 0; j < event.overview.length; j++) { - if (event.overview[j].eventKey === successIds[i]) { - event.overview.splice(j, 1); - j = j - 1; - } - } - } - if (!updateThese.$set) { - updateThese.$set = {}; - } - updateThese.$set.overview = event.overview; - } - //remove from list - if (typeof event.list !== 'undefined' && Array.isArray(event.list) && event.list.length > 0) { - for (let i = 0; i < successIds.length; i++) { - let index = event.list.indexOf(successIds[i]); - if (index > -1) { - event.list.splice(index, 1); - i = i - 1; - } - } - if (!updateThese.$set) { - updateThese.$set = {}; - } - updateThese.$set.list = event.list; - } - //remove from order - if (typeof event.order !== 'undefined' && Array.isArray(event.order) && event.order.length > 0) { - for (let i = 0; i < successIds.length; i++) { - let index = event.order.indexOf(successIds[i]); - if (index > -1) { - event.order.splice(index, 1); - i = i - 1; - } - } - if (!updateThese.$set) { - updateThese.$set = {}; - } - updateThese.$set.order = event.order; - } - - await common.db.collection('events').update({ "_id": common.db.ObjectID(app_id) }, updateThese); - - plugins.dispatch("/systemlogs", { - params: params, - action: "event_deleted", - data: { - events: successIds, - appID: app_id - } - }); - - common.returnMessage(params, 200, 'Success'); - - }).catch((err2) => { - if (failedIds.length) { - log.e("Event deletion failed for following Event keys:\n%j", failedIds.join("\n")); - } - log.e("Event deletion failed\n%j", err2); - common.returnMessage(params, 500, { errorMessage: "Event deletion failed. Failed to delete some data related to this Event." }); - }); - }); - } - else { - common.returnMessage(params, 400, "Missing events to delete"); - } - }); - break; - } - case 'change_visibility': - { - validateUpdate(params, 'events', function() { - common.db.collection('events').findOne({"_id": common.db.ObjectID(params.qstring.app_id)}, function(err, event) { - if (err) { - common.returnMessage(params, 400, err); - return; - } - if (!event) { - common.returnMessage(params, 400, "Could not find event"); - return; - } - - var update_array = {}; - var idss = []; - try { - idss = JSON.parse(params.qstring.events); - } - catch (SyntaxError) { - idss = []; - } - if (!Array.isArray(idss)) { - idss = []; - } - - if (event.map) { - try { - update_array.map = JSON.parse(JSON.stringify(event.map)); - } - catch (SyntaxError) { - update_array.map = {}; - console.log('Parse ' + event.map + ' JSON failed', params.req.url, params.req.body); - } - } - else { - update_array.map = {}; - } - - for (let i = 0; i < idss.length; i++) { - - var baseID = idss[i].replace(/\\u002e/g, "."); - if (!update_array.map[idss[i]]) { - update_array.map[idss[i]] = {}; - } - - if (params.qstring.set_visibility === 'hide') { - update_array.map[idss[i]].is_visible = false; - } - else { - update_array.map[idss[i]].is_visible = true; - } - - if (update_array.map[idss[i]].is_visible) { - delete update_array.map[idss[i]].is_visible; - } - - if (Object.keys(update_array.map[idss[i]]).length === 0) { - delete update_array.map[idss[i]]; - } - - if (params.qstring.set_visibility === 'hide' && event && event.overview && Array.isArray(event.overview)) { - for (let j = 0; j < event.overview.length; j++) { - if (event.overview[j].eventKey === baseID) { - event.overview.splice(j, 1); - j = j - 1; - } - } - update_array.overview = event.overview; - } - } - common.db.collection('events').update({"_id": common.db.ObjectID(params.qstring.app_id)}, {'$set': update_array}, function(err2) { - - if (err2) { - common.returnMessage(params, 400, err2); - } - else { - common.returnMessage(params, 200, 'Success'); - var data_arr = {update: update_array}; - data_arr.before = {map: {}}; - if (event.map) { - data_arr.before.map = event.map; - } - plugins.dispatch("/systemlogs", { - params: params, - action: "events_updated", - data: data_arr - }); - } - }); - }); - }); - break; - } - default: - if (!plugins.dispatch(apiPath, { - params: params, - validateUserForDataReadAPI: validateUserForDataReadAPI, - validateUserForMgmtReadAPI: validateUserForMgmtReadAPI, - paths: paths, - validateUserForDataWriteAPI: validateUserForDataWriteAPI, - validateUserForGlobalAdmin: validateUserForGlobalAdmin - })) { - common.returnMessage(params, 400, 'Invalid path, must be one of /all or /me'); - } - break; - } - break; - } - case '/o/users': { - switch (paths[3]) { - case 'all': - validateUserForGlobalAdmin(params, countlyApi.mgmt.users.getAllUsers); - break; - case 'me': - validateUserForMgmtReadAPI(countlyApi.mgmt.users.getCurrentUser, params); - break; - case 'id': - validateUserForGlobalAdmin(params, countlyApi.mgmt.users.getUserById); - break; - case 'reset_timeban': - validateUserForGlobalAdmin(params, countlyApi.mgmt.users.resetTimeBan); - break; - case 'permissions': - validateRead(params, 'core', function() { - var features = ["core", "events" /* , "global_configurations", "global_applications", "global_users", "global_jobs", "global_upload" */]; - /* - Example structure for featuresPermissionDependency Object - { - [FEATURE name which need other permissions]:{ - [CRUD permission of FEATURE]: { - [DEPENDENT_FEATURE name]:[DEPENDENT_FEATURE required CRUD permissions array] - }, - .... other CRUD permission if necessary - } - }, - { - data_manager: Transformations:{ - c:{ - data_manager:['r','u'] - }, - r:{ - data_manager:['r'] - }, - u:{ - data_manager:['r','u'] - }, - d:{ - data_manager:['r','u'] - }, - } - } - */ - var featuresPermissionDependency = {}; - plugins.dispatch("/permissions/features", { params: params, features: features, featuresPermissionDependency: featuresPermissionDependency }, function() { - common.returnOutput(params, {features, featuresPermissionDependency}); - }); - }); - break; - default: - if (!plugins.dispatch(apiPath, { - params: params, - validateUserForDataReadAPI: validateUserForDataReadAPI, - validateUserForMgmtReadAPI: validateUserForMgmtReadAPI, - paths: paths, - validateUserForDataWriteAPI: validateUserForDataWriteAPI, - validateUserForGlobalAdmin: validateUserForGlobalAdmin - })) { - common.returnMessage(params, 400, 'Invalid path, must be one of /all or /me'); - } - break; - } - - break; - } - case '/o/app_users': { - switch (paths[3]) { - case 'loyalty': { - if (!params.qstring.app_id) { - common.returnMessage(params, 400, 'Missing parameter "app_id"'); - return false; - } - validateUserForRead(params, countlyApi.mgmt.appUsers.loyalty); - break; - } - /** - * @api {get} /o/app_users/download/:id Downloads user export. - * @apiName Download user export - * @apiGroup App User Management - * @apiDescription Downloads users export - * - * @apiParam {Number} id Id of export. For single user it would be similar to: appUser_644658291e95e720503d5087_1, but for multiple users - appUser_62e253489315313ffbc2c457_HASH_3e5b86cb367a6b8c0689ffd80652d2bbcb0a3edf - * - * @apiQuery {String} app_id Application id - * - * @apiErrorExample {json} Error-Response: - * HTTP/1.1 400 Bad Request - * { - * "result": "Missing parameter \"app_id\"" - * } - */ - case 'download': { - if (paths[4] && paths[4] !== '') { - validateUserForRead(params, function() { - var filename = paths[4].split('.'); - new Promise(function(resolve) { - if (filename[0].startsWith("appUser_")) { - filename[0] = filename[0] + '.tar.gz'; - resolve(); - } - else { //we have task result. Try getting from there - taskmanager.getResult({id: filename[0]}, function(err, res) { - if (res && res.data) { - filename[0] = res.data; - filename[0] = filename[0].replace(/\"/g, ''); - } - resolve(); - }); - } - }).then(function() { - var myfile = '../../export/AppUser/' + filename[0]; - countlyFs.gridfs.getSize("appUsers", myfile, {id: filename[0]}, function(error, size) { - if (error) { - common.returnMessage(params, 400, error); - } - else if (parseInt(size) === 0) { - //export does not exist. lets check out export collection. - var eid = filename[0].split("."); - eid = eid[0]; - - var cursor = common.db.collection("exports").find({"_eid": eid}, {"_eid": 0, "_id": 0}); - var options = {"type": "stream", "filename": eid + ".json", params: params}; - params.res.writeHead(200, { - 'Content-Type': 'application/x-gzip', - 'Content-Disposition': 'inline; filename="' + eid + '.json' - }); - options.streamOptions = {}; - if (options.type === "stream" || options.type === "json") { - options.streamOptions.transform = function(doc) { - doc._id = doc.__id; - delete doc.__id; - return JSON.stringify(doc); - }; - } - - options.output = options.output || function(stream) { - countlyApi.data.exports.stream(options.params, stream, options); - }; - options.output(cursor); - - - } - else { - countlyFs.gridfs.getStream("appUsers", myfile, {id: filename[0]}, function(err, stream) { - if (err) { - common.returnMessage(params, 400, "Export doesn't exist"); - } - else { - params.res.writeHead(200, { - 'Content-Type': 'application/x-gzip', - 'Content-Length': size, - 'Content-Disposition': 'inline; filename="' + filename[0] - }); - stream.pipe(params.res); - } - }); - } - }); - }); - }); - } - else { - common.returnMessage(params, 400, 'Missing filename'); - } - break; - } - default: - if (!plugins.dispatch(apiPath, { - params: params, - validateUserForDataReadAPI: validateUserForDataReadAPI, - validateUserForMgmtReadAPI: validateUserForMgmtReadAPI, - paths: paths, - validateUserForDataWriteAPI: validateUserForDataWriteAPI, - validateUserForGlobalAdmin: validateUserForGlobalAdmin - })) { - common.returnMessage(params, 400, 'Invalid path, must be one of /all or /me'); - } - break; - } - break; - } - case '/o/apps': { - switch (paths[3]) { - case 'all': - validateUserForGlobalAdmin(params, countlyApi.mgmt.apps.getAllApps); - break; - case 'mine': - validateUser(params, countlyApi.mgmt.apps.getCurrentUserApps); - break; - case 'details': - validateAppAdmin(params, countlyApi.mgmt.apps.getAppsDetails); - break; - case 'plugins': - validateUserForGlobalAdmin(params, countlyApi.mgmt.apps.getAppPlugins); - break; - default: - if (!plugins.dispatch(apiPath, { - params: params, - validateUserForDataReadAPI: validateUserForDataReadAPI, - validateUserForMgmtReadAPI: validateUserForMgmtReadAPI, - paths: paths, - validateUserForDataWriteAPI: validateUserForDataWriteAPI, - validateUserForGlobalAdmin: validateUserForGlobalAdmin - })) { - common.returnMessage(params, 400, 'Invalid path, must be one of /all, /mine, /details or /plugins'); - } - break; - } - - break; - } - case '/o/tasks': { - switch (paths[3]) { - case 'all': - validateRead(params, 'core', () => { - if (!params.qstring.query) { - params.qstring.query = {}; - } - if (typeof params.qstring.query === "string") { - try { - params.qstring.query = JSON.parse(params.qstring.query); - } - catch (ex) { - params.qstring.query = {}; - } - } - if (params.qstring.query.$or) { - params.qstring.query.$and = [ - {"$or": Object.assign([], params.qstring.query.$or) }, - {"$or": [{"global": {"$ne": false}}, {"creator": params.member._id + ""}]} - ]; - delete params.qstring.query.$or; - } - else { - params.qstring.query.$or = [{"global": {"$ne": false}}, {"creator": params.member._id + ""}]; - } - params.qstring.query.subtask = {$exists: false}; - params.qstring.query.app_id = params.qstring.app_id; - if (params.qstring.app_ids && params.qstring.app_ids !== "") { - var ll = params.qstring.app_ids.split(","); - if (ll.length > 1) { - params.qstring.query.app_id = {$in: ll}; - } - } - if (params.qstring.period) { - countlyCommon.getPeriodObj(params); - params.qstring.query.ts = countlyCommon.getTimestampRangeQuery(params, false); - } - taskmanager.getResults({ - db: common.db, - query: params.qstring.query - }, (err, res) => { - common.returnOutput(params, res || []); - }); - }); - break; - case 'count': - validateRead(params, 'core', () => { - if (!params.qstring.query) { - params.qstring.query = {}; - } - if (typeof params.qstring.query === "string") { - try { - params.qstring.query = JSON.parse(params.qstring.query); - } - catch (ex) { - params.qstring.query = {}; - } - } - if (params.qstring.query.$or) { - params.qstring.query.$and = [ - {"$or": Object.assign([], params.qstring.query.$or) }, - {"$or": [{"global": {"$ne": false}}, {"creator": params.member._id + ""}]} - ]; - delete params.qstring.query.$or; - } - else { - params.qstring.query.$or = [{"global": {"$ne": false}}, {"creator": params.member._id + ""}]; - } - if (params.qstring.period) { - countlyCommon.getPeriodObj(params); - params.qstring.query.ts = countlyCommon.getTimestampRangeQuery(params, false); - } - taskmanager.getCounts({ - db: common.db, - query: params.qstring.query - }, (err, res) => { - common.returnOutput(params, res || []); - }); - }); - break; - case 'list': - validateRead(params, 'core', () => { - if (!params.qstring.query) { - params.qstring.query = {}; - } - if (typeof params.qstring.query === "string") { - try { - params.qstring.query = JSON.parse(params.qstring.query); - } - catch (ex) { - params.qstring.query = {}; - } - } - params.qstring.query.$and = []; - if (params.qstring.query.creator && params.qstring.query.creator === params.member._id) { - params.qstring.query.$and.push({"creator": params.member._id + ""}); - } - else { - params.qstring.query.$and.push({"$or": [{"global": {"$ne": false}}, {"creator": params.member._id + ""}]}); - } - - if (params.qstring.data_source !== "all" && params.qstring.app_id) { - if (params.qstring.data_source === "independent") { - params.qstring.query.$and.push({"app_id": "undefined"}); - } - else { - params.qstring.query.$and.push({"app_id": params.qstring.app_id}); - } - } - - if (params.qstring.query.$or) { - params.qstring.query.$and.push({"$or": Object.assign([], params.qstring.query.$or) }); - delete params.qstring.query.$or; - } - params.qstring.query.subtask = {$exists: false}; - if (params.qstring.period) { - countlyCommon.getPeriodObj(params); - params.qstring.query.ts = countlyCommon.getTimestampRangeQuery(params, false); - } - const skip = params.qstring.iDisplayStart; - const limit = params.qstring.iDisplayLength; - const sEcho = params.qstring.sEcho; - const keyword = params.qstring.sSearch || null; - const sortBy = params.qstring.iSortCol_0 || null; - const sortSeq = params.qstring.sSortDir_0 || null; - taskmanager.getTableQueryResult({ - db: common.db, - query: params.qstring.query, - page: {skip, limit}, - sort: {sortBy, sortSeq}, - keyword: keyword, - }, (err, res) => { - if (!err) { - common.returnOutput(params, {aaData: res.list, iTotalDisplayRecords: res.count, iTotalRecords: res.count, sEcho}); - } - else { - common.returnMessage(params, 500, '"Query failed"'); - } - }); - }); - break; - case 'task': - validateRead(params, 'core', () => { - if (!params.qstring.task_id) { - common.returnMessage(params, 400, 'Missing parameter "task_id"'); - return false; - } - taskmanager.getResult({ - db: common.db, - id: params.qstring.task_id, - subtask_key: params.qstring.subtask_key - }, (err, res) => { - if (res) { - common.returnOutput(params, res); - } - else { - common.returnMessage(params, 400, 'Task does not exist'); - } - }); - }); - break; - case 'check': - validateRead(params, 'core', () => { - if (!params.qstring.task_id) { - common.returnMessage(params, 400, 'Missing parameter "task_id"'); - return false; - } - - var tasks = params.qstring.task_id; - - try { - tasks = JSON.parse(tasks); - } - catch { - // ignore - } - - var isMulti = Array.isArray(tasks); - - taskmanager.checkResult({ - db: common.db, - id: tasks - }, (err, res) => { - if (isMulti && res) { - common.returnMessage(params, 200, res); - } - else if (res) { - common.returnMessage(params, 200, res.status); - } - else { - common.returnMessage(params, 400, 'Task does not exist'); - } - }); - }); - break; - default: - if (!plugins.dispatch(apiPath, { - params: params, - validateUserForDataReadAPI: validateUserForDataReadAPI, - validateUserForMgmtReadAPI: validateUserForMgmtReadAPI, - paths: paths, - validateUserForDataWriteAPI: validateUserForDataWriteAPI, - validateUserForGlobalAdmin: validateUserForGlobalAdmin - })) { - common.returnMessage(params, 400, 'Invalid path'); - } - break; - } - - break; - } - case '/o/system': { - switch (paths[3]) { - case 'version': - validateUserForMgmtReadAPI(() => { - common.returnOutput(params, {"version": versionInfo.version}); - }, params); - break; - case 'plugins': - validateUserForMgmtReadAPI(() => { - common.returnOutput(params, plugins.getPlugins()); - }, params); - break; - case 'observability': - validateUserForMgmtReadAPI(() => { - plugins.dispatch('/system/observability/collect', {params: params}, function(err, results) { - if (err) { - common.returnMessage(params, 500, 'Error collecting observability data'); - return; - } - const data = (results || []) - .filter(r => r && r.status === 'fulfilled' && r.value) - .map(r => r.value); - common.returnOutput(params, data); - }); - }, params); - break; - case 'aggregator': - validateUserForMgmtReadAPI(async() => { - try { - //fetch aggregator status and latest drill cd in parallel - const [pluginsData, drillData] = await Promise.all([ - common.db.collection("plugins").findOne({_id: "_changeStreams"}), - common.drillDb.collection("drill_events").find({}, {projection: {cd: 1}}).sort({cd: -1}).limit(1).toArray() - ]); - - var data = []; - var now = Date.now().valueOf(); - var nowDrill = now; - if (drillData && drillData.length) { - nowDrill = new Date(drillData[0].cd).valueOf(); - } - - if (pluginsData) { - for (var key in pluginsData) { - if (key !== "_id") { - var lastAccepted = new Date(pluginsData[key].cd).valueOf(); - data.push({ - name: key, - last_cd: pluginsData[key].cd, - drill: drillData && drillData[0] && drillData[0].cd, - last_id: pluginsData[key]._id, - diff: (now - lastAccepted) / 1000, - diffDrill: (nowDrill - lastAccepted) / 1000 - }); - } - } - } - common.returnOutput(params, data); - } - catch (err) { - log.e('Error fetching aggregator status:', err); - common.returnMessage(params, 500, 'Error fetching aggregator status'); - } - }, params); - break; - case 'kafka': - // Handle sub-routes: /o/system/kafka/events and /o/system/kafka/events/meta - if (paths[4] === 'events') { - if (paths[5] === 'meta') { - // /o/system/kafka/events/meta - Get filter options (cached 30s, deduped) - validateUserForMgmtReadAPI(async() => { - try { - var now = Date.now(); - if (_kafkaMetaCache && (now - _kafkaMetaCacheTs) < KAFKA_META_CACHE_TTL) { - common.returnOutput(params, _kafkaMetaCache); - return; - } - - // Reuse in-flight fetch to prevent thundering herd on cache expiry - if (!_kafkaMetaCachePromise) { - _kafkaMetaCachePromise = Promise.all([ - common.db.collection('kafka_consumer_events').distinct('type'), - common.db.collection('kafka_consumer_events').distinct('groupId'), - common.db.collection('kafka_consumer_events').distinct('topic'), - common.db.collection('kafka_consumer_events').distinct('clusterId') - ]).then(function([eventTypes, groupIds, topics, clusterIds]) { - _kafkaMetaCache = { - eventTypes: eventTypes.filter(Boolean).sort(), - groupIds: groupIds.filter(Boolean).sort(), - topics: topics.filter(Boolean).sort(), - clusterIds: clusterIds.filter(Boolean).sort() - }; - _kafkaMetaCacheTs = Date.now(); - _kafkaMetaCachePromise = null; - return _kafkaMetaCache; - }).catch(function(err) { - _kafkaMetaCachePromise = null; - throw err; - }); - } - - var result = await _kafkaMetaCachePromise; - common.returnOutput(params, result); - } - catch (err) { - log.e('Error fetching Kafka events meta:', err); - common.returnMessage(params, 500, 'Error fetching Kafka events meta'); - } - }, params); - } - else { - // /o/system/kafka/events - Get events list - validateUserForMgmtReadAPI(async() => { - try { - // Build query from filters - const query = {}; - - if (params.qstring.eventType && params.qstring.eventType !== 'all') { - query.type = params.qstring.eventType + ""; - } - if (params.qstring.groupId && params.qstring.groupId !== 'all') { - query.groupId = params.qstring.groupId + ""; - } - if (params.qstring.topic && params.qstring.topic !== 'all') { - query.topic = params.qstring.topic + ""; - } - if (params.qstring.clusterId && params.qstring.clusterId !== 'all') { - query.clusterId = params.qstring.clusterId + ""; - } - - // Get accurate counts - const [total, filteredCount] = await Promise.all([ - common.db.collection('kafka_consumer_events').countDocuments({}), - common.db.collection('kafka_consumer_events').countDocuments(query) - ]); - let cursor = common.db.collection('kafka_consumer_events').find(query); - - // Sorting with validated column index - const columns = ['_id', 'ts', 'type', 'groupId', 'topic', 'partition', 'clusterId']; - const sortColIndex = parseInt(params.qstring.iSortCol_0, 10); - if (params.qstring.iSortCol_0 && - params.qstring.sSortDir_0 && - Number.isInteger(sortColIndex) && - sortColIndex >= 0 && - sortColIndex < columns.length) { - const sortObj = {}; - sortObj[columns[sortColIndex]] = params.qstring.sSortDir_0 === 'asc' ? 1 : -1; - cursor = cursor.sort(sortObj); - } - else { - cursor = cursor.sort({ ts: -1 }); - } - - // Pagination with validated parameters - const MAX_DISPLAY_LENGTH = 1000; - let displayStart = parseInt(params.qstring.iDisplayStart, 10); - if (Number.isNaN(displayStart) || displayStart < 0) { - displayStart = 0; - } - if (displayStart > 0) { - cursor = cursor.skip(displayStart); - } - - let displayLength = parseInt(params.qstring.iDisplayLength, 10); - if (Number.isNaN(displayLength) || displayLength < 1 || displayLength > MAX_DISPLAY_LENGTH) { - displayLength = 50; - } - cursor = cursor.limit(displayLength); - - const events = await cursor.toArray(); - - common.returnOutput(params, { - sEcho: params.qstring.sEcho, - iTotalRecords: Math.max(total, 0), - iTotalDisplayRecords: filteredCount, - aaData: events - }); - } - catch (err) { - log.e('Error fetching Kafka consumer events:', err); - common.returnMessage(params, 500, 'Error fetching Kafka consumer events'); - } - }, params); - } - break; - } - - // Default: /o/system/kafka - Get Kafka status overview - validateUserForMgmtReadAPI(async() => { - try { - const KAFKA_QUERY_LIMIT = 500; - - // Fetch all Kafka data and summaries in parallel (with projections) - const [consumerState, consumerHealth, lagHistory, connectStatus, stateSummary, healthSummary] = await Promise.all([ - common.db.collection("kafka_consumer_state") - .find({}, { - projection: { - consumerGroup: 1, - topic: 1, - partitions: 1, - lastProcessedAt: 1, - batchCount: 1, - duplicatesSkipped: 1, - lastDuplicateAt: 1, - lastBatchSize: 1, - avgBatchSize: 1 - } - }) - .sort({ lastProcessedAt: -1 }) - .limit(KAFKA_QUERY_LIMIT) - .toArray(), - common.db.collection("kafka_consumer_health") - .find({}, { - projection: { - groupId: 1, - rebalanceCount: 1, - lastRebalanceAt: 1, - lastJoinAt: 1, - lastMemberId: 1, - lastGenerationId: 1, - commitCount: 1, - lastCommitAt: 1, - errorCount: 1, - lastErrorAt: 1, - lastErrorMessage: 1, - recentErrors: 1, - totalLag: 1, - partitionLag: 1, - lagUpdatedAt: 1, - updatedAt: 1 - } - }) - .sort({ updatedAt: -1 }) - .limit(KAFKA_QUERY_LIMIT) - .toArray(), - common.db.collection("kafka_lag_history") - .find({}, {projection: {ts: 1, groups: 1, connectLag: 1}}) - .sort({ ts: -1 }) - .limit(100) - .toArray(), - common.db.collection("kafka_connect_status") - .find({}, { - projection: { - connectorName: 1, - connectorState: 1, - connectorType: 1, - workerId: 1, - tasks: 1, - updatedAt: 1 - } - }) - .sort({ updatedAt: -1 }) - .limit(KAFKA_QUERY_LIMIT) - .toArray(), - // Aggregate consumer state summary via MongoDB pipeline - common.db.collection("kafka_consumer_state") - .aggregate([{ - $group: { - _id: null, - totalBatchesProcessed: { $sum: { $ifNull: ["$batchCount", 0] } }, - totalDuplicatesSkipped: { $sum: { $ifNull: ["$duplicatesSkipped", 0] } }, - avgBatchSizeSum: { $sum: { $ifNull: ["$avgBatchSize", 0] } }, - groupsWithData: { $sum: { $cond: [{ $gt: ["$avgBatchSize", 0] }, 1, 0] } } - } - }]) - .toArray(), - // Aggregate consumer health summary via MongoDB pipeline - common.db.collection("kafka_consumer_health") - .aggregate([{ - $group: { - _id: null, - totalRebalances: { $sum: { $ifNull: ["$rebalanceCount", 0] } }, - totalErrors: { $sum: { $ifNull: ["$errorCount", 0] } }, - totalLag: { $sum: { $ifNull: ["$totalLag", 0] } } - } - }]) - .toArray() - ]); - - // Extract summary from aggregation pipelines - const stSummary = stateSummary[0] || {}; - const totalBatchesProcessed = stSummary.totalBatchesProcessed || 0; - const totalDuplicatesSkipped = stSummary.totalDuplicatesSkipped || 0; - const avgBatchSizeOverall = stSummary.groupsWithData > 0 - ? stSummary.avgBatchSizeSum / stSummary.groupsWithData - : 0; - - const htSummary = healthSummary[0] || {}; - const totalRebalances = htSummary.totalRebalances || 0; - const totalErrors = htSummary.totalErrors || 0; - const totalLagAll = htSummary.totalLag || 0; - - // Transform consumer state rows - const partitionStats = consumerState.map(state => { - const partitions = state.partitions || {}; - const partitionCount = Object.keys(partitions).length; - const activePartitions = Object.values(partitions).filter(p => p.lastProcessedAt).length; - - return { - id: state._id, - consumerGroup: state.consumerGroup, - topic: state.topic, - partitionCount, - activePartitions, - lastProcessedAt: state.lastProcessedAt, - batchCount: state.batchCount || 0, - duplicatesSkipped: state.duplicatesSkipped || 0, - lastDuplicateAt: state.lastDuplicateAt, - lastBatchSize: state.lastBatchSize, - avgBatchSize: state.avgBatchSize ? Math.round(state.avgBatchSize) : null - }; - }); - - // Transform consumer health rows - const consumerStats = consumerHealth.map(health => ({ - id: health._id, - groupId: health.groupId, - rebalanceCount: health.rebalanceCount || 0, - lastRebalanceAt: health.lastRebalanceAt, - lastJoinAt: health.lastJoinAt, - lastMemberId: health.lastMemberId, - lastGenerationId: health.lastGenerationId, - commitCount: health.commitCount || 0, - lastCommitAt: health.lastCommitAt, - errorCount: health.errorCount || 0, - lastErrorAt: health.lastErrorAt, - lastErrorMessage: health.lastErrorMessage, - recentErrors: health.recentErrors || [], - totalLag: health.totalLag || 0, - partitionLag: health.partitionLag || {}, - lagUpdatedAt: health.lagUpdatedAt, - updatedAt: health.updatedAt - })); - - // Process Kafka Connect status - const connectorStats = connectStatus.map(conn => ({ - id: conn._id, - connectorName: conn.connectorName, - connectorState: conn.connectorState, - connectorType: conn.connectorType, - workerId: conn.workerId, - tasks: conn.tasks || [], - tasksRunning: (conn.tasks || []).filter(t => t.state === 'RUNNING').length, - tasksTotal: (conn.tasks || []).length, - updatedAt: conn.updatedAt - })); - - // Get connect consumer group lag for ClickHouse sink - const connectConsumerGroupId = common.config?.kafka?.connectConsumerGroupId; - const connectGroupHealth = connectConsumerGroupId - ? consumerHealth.find(h => h.groupId === connectConsumerGroupId) - : null; - - common.returnOutput(params, { - summary: { - totalBatchesProcessed, - totalDuplicatesSkipped, - avgBatchSizeOverall: Math.round(avgBatchSizeOverall * 100) / 100, - totalRebalances, - totalErrors, - totalLag: totalLagAll, - consumerGroupCount: consumerStats.length, - partitionCount: partitionStats.length - }, - partitions: partitionStats, - consumers: consumerStats, - lagHistory: lagHistory.reverse(), // Oldest first for charts - - // Kafka Connect status - connectStatus: { - enabled: !!common.config?.kafka?.connectApiUrl, - connectors: connectorStats, - sinkLag: connectGroupHealth?.totalLag || 0, - sinkLagUpdatedAt: connectGroupHealth?.lagUpdatedAt - } - }); - } - catch (err) { - log.e('Error fetching Kafka stats:', err); - common.returnMessage(params, 500, 'Error fetching Kafka stats'); - } - }, params); - break; - default: - if (!plugins.dispatch(apiPath, { - params: params, - validateUserForDataReadAPI: validateUserForDataReadAPI, - validateUserForMgmtReadAPI: validateUserForMgmtReadAPI, - paths: paths, - validateUserForDataWriteAPI: validateUserForDataWriteAPI, - validateUserForGlobalAdmin: validateUserForGlobalAdmin - })) { - common.returnMessage(params, 400, 'Invalid path'); - } - break; - } - - break; - } - case '/o/export': { - switch (paths[3]) { - case 'db': - validateUserForMgmtReadAPI(() => { - if (!params.qstring.collection) { - common.returnMessage(params, 400, 'Missing parameter "collection"'); - return false; - } - if (typeof params.qstring.filter === "string") { - try { - params.qstring.query = JSON.parse(params.qstring.filter, common.reviver); - } - catch (ex) { - common.returnMessage(params, 400, "Failed to parse query. " + ex.message); - return false; - } - } - else if (typeof params.qstring.query === "string") { - try { - params.qstring.query = JSON.parse(params.qstring.query, common.reviver); - } - catch (ex) { - common.returnMessage(params, 400, "Failed to parse query. " + ex.message); - return false; - } - } - if (typeof params.qstring.projection === "string") { - try { - params.qstring.projection = JSON.parse(params.qstring.projection); - } - catch (ex) { - params.qstring.projection = null; - } - } - if (typeof params.qstring.project === "string") { - try { - params.qstring.projection = JSON.parse(params.qstring.project); - } - catch (ex) { - params.qstring.projection = null; - } - } - if (typeof params.qstring.sort === "string") { - try { - params.qstring.sort = JSON.parse(params.qstring.sort); - } - catch (ex) { - params.qstring.sort = null; - } - } - - if (typeof params.qstring.formatFields === "string") { - try { - params.qstring.formatFields = JSON.parse(params.qstring.formatFields); - } - catch (ex) { - params.qstring.formatFields = null; - } - } - - if (typeof params.qstring.get_index === "string") { - try { - params.qstring.get_index = JSON.parse(params.qstring.get_index); - } - catch (ex) { - params.qstring.get_index = null; - } - } - - dbUserHasAccessToCollection(params, params.qstring.collection, (hasAccess) => { - if (hasAccess || (params.qstring.db === "countly_drill" && params.qstring.collection === "drill_events") || (params.qstring.db === "countly" && params.qstring.collection === "events_data")) { - var dbs = { countly: common.db, countly_drill: common.drillDb, countly_out: common.outDb, countly_fs: countlyFs.gridfs.getHandler() }; - var db = ""; - if (params.qstring.db && dbs[params.qstring.db]) { - db = dbs[params.qstring.db]; - } - else { - db = common.db; - } - if (!params.member.global_admin && params.qstring.collection === "drill_events" || params.qstring.collection === "events_data") { - var base_filter = getBaseAppFilter(params.member, params.qstring.db, params.qstring.collection); - if (base_filter && Object.keys(base_filter).length > 0) { - params.qstring.query = params.qstring.query || {}; - for (var key in base_filter) { - if (params.qstring.query[key]) { - params.qstring.query.$and = params.qstring.query.$and || []; - params.qstring.query.$and.push({[key]: base_filter[key]}); - params.qstring.query.$and.push({[key]: params.qstring.query[key]}); - delete params.qstring.query[key]; - } - else { - params.qstring.query[key] = base_filter[key]; - } - } - } - } - countlyApi.data.exports.fromDatabase({ - db: db, - params: params, - collection: params.qstring.collection, - query: params.qstring.query, - projection: params.qstring.projection, - sort: params.qstring.sort, - limit: params.qstring.limit, - skip: params.qstring.skip, - type: params.qstring.type - }); - } - else { - common.returnMessage(params, 401, 'User does not have access right for this collection'); - } - }); - }, params); - break; - case 'request': - validateUserForMgmtReadAPI(() => { - if (!params.qstring.path) { - common.returnMessage(params, 400, 'Missing parameter "path"'); - return false; - } - if (typeof params.qstring.data === "string") { - try { - params.qstring.data = JSON.parse(params.qstring.data); - } - catch (ex) { - console.log("Error parsing export request data", params.qstring.data, ex); - params.qstring.data = {}; - } - } - - if (params.qstring.projection) { - try { - params.qstring.projection = JSON.parse(params.qstring.projection); - } - catch (ex) { - params.qstring.projection = {}; - } - } - - if (params.qstring.columnNames) { - try { - params.qstring.columnNames = JSON.parse(params.qstring.columnNames); - } - catch (ex) { - params.qstring.columnNames = {}; - } - } - if (params.qstring.mapper) { - try { - params.qstring.mapper = JSON.parse(params.qstring.mapper); - } - catch (ex) { - params.qstring.mapper = {}; - } - } - countlyApi.data.exports.fromRequest({ - params: params, - path: params.qstring.path, - data: params.qstring.data, - method: params.qstring.method, - prop: params.qstring.prop, - type: params.qstring.type, - filename: params.qstring.filename, - projection: params.qstring.projection, - columnNames: params.qstring.columnNames, - mapper: params.qstring.mapper, - }); - }, params); - break; - case 'requestQuery': - validateUserForMgmtReadAPI(() => { - if (!params.qstring.path) { - common.returnMessage(params, 400, 'Missing parameter "path"'); - return false; - } - if (typeof params.qstring.data === "string") { - try { - params.qstring.data = JSON.parse(params.qstring.data); - } - catch (ex) { - console.log("Error parsing export request data", params.qstring.data, ex); - params.qstring.data = {}; - } - } - var my_name = JSON.stringify(params.qstring); - - var ff = taskmanager.longtask({ - db: common.db, - threshold: plugins.getConfig("api").request_threshold, - force: true, - gridfs: true, - binary: true, - app_id: params.qstring.app_id, - params: params, - type: params.qstring.type_name || "tableExport", - report_name: params.qstring.filename + "." + params.qstring.type, - meta: JSON.stringify({ - "app_id": params.qstring.app_id, - "query": params.qstring.query || {} - }), - name: my_name, - view: "#/exportedData/tableExport/", - processData: function(err, res, callback) { - if (!err) { - callback(null, res); - } - else { - callback(err, ''); - } - }, - outputData: function(err, data) { - if (err) { - common.returnMessage(params, 400, err); - } - else { - common.returnMessage(params, 200, data); - } - } - }); - - countlyApi.data.exports.fromRequestQuery({ - db_name: params.qstring.db, - db: (params.qstring.db === "countly_drill") ? common.drillDb : (params.qstring.dbs === "countly_drill") ? common.drillDb : common.db, - params: params, - path: params.qstring.path, - data: params.qstring.data, - method: params.qstring.method, - prop: params.qstring.prop, - type: params.qstring.type, - filename: params.qstring.filename + "." + params.qstring.type, - output: function(data) { - ff(null, data); - } - }); - }, params); - break; - case 'download': { - validateRead(params, "core", () => { - if (paths[4] && paths[4] !== '') { - common.db.collection("long_tasks").findOne({_id: paths[4]}, function(err, data) { - if (err) { - common.returnMessage(params, 400, err); - } - else { - var filename = data.report_name; - var type = filename.split("."); - type = type[type.length - 1]; - var myfile = paths[4]; - var headers = {}; - - countlyFs.gridfs.getSize("task_results", myfile, {id: paths[4]}, function(err2, size) { - if (err2) { - common.returnMessage(params, 400, err2); - } - else if (parseInt(size) === 0) { - if (data.type !== "dbviewer") { - common.returnMessage(params, 400, "Export size is 0"); - } - //handling older aggregations that aren't saved in countly_fs - else if (!data.gridfs && data.data) { - type = "json"; - filename = data.name + "." + type; - headers = {}; - headers["Content-Type"] = countlyApi.data.exports.getType(type); - headers["Content-Disposition"] = "attachment;filename=" + encodeURIComponent(filename); - params.res.writeHead(200, headers); - params.res.write(data.data); - params.res.end(); - } - } - else { - countlyFs.gridfs.getStream("task_results", myfile, {id: myfile}, function(err5, stream) { - if (err5) { - common.returnMessage(params, 400, "Export stream does not exist"); - } - else { - headers = {}; - headers["Content-Type"] = countlyApi.data.exports.getType(type); - headers["Content-Disposition"] = "attachment;filename=" + encodeURIComponent(filename); - params.res.writeHead(200, headers); - stream.pipe(params.res); - } - }); - } - }); - } - }); - } - else { - common.returnMessage(params, 400, 'Missing filename'); - } - }); - break; - } - case 'data': - validateUserForMgmtReadAPI(() => { - if (!params.qstring.data) { - common.returnMessage(params, 400, 'Missing parameter "data"'); - return false; - } - if (typeof params.qstring.data === "string" && !params.qstring.raw) { - try { - params.qstring.data = JSON.parse(params.qstring.data); - } - catch (ex) { - common.returnMessage(params, 400, 'Incorrect parameter "data"'); - return false; - } - } - countlyApi.data.exports.fromData(params.qstring.data, { - params: params, - type: params.qstring.type, - filename: params.qstring.filename - }); - }, params); - break; - default: - if (!plugins.dispatch(apiPath, { - params: params, - validateUserForDataReadAPI: validateUserForDataReadAPI, - validateUserForMgmtReadAPI: validateUserForMgmtReadAPI, - paths: paths, - validateUserForDataWriteAPI: validateUserForDataWriteAPI, - validateUserForGlobalAdmin: validateUserForGlobalAdmin - })) { - common.returnMessage(params, 400, 'Invalid path'); - } - break; - } - - break; - } - case '/o/ping': { - common.db.collection("plugins").findOne({_id: "plugins"}, {_id: 1}, (err) => { - if (err) { - return common.returnMessage(params, 404, 'DB Error'); - } - else { - return common.returnMessage(params, 200, 'Success'); - } - }); - break; - } - case '/i/token': { - switch (paths[3]) { - /** - * @api {get} /i/token/delete - * @apiName deleteToken - * @apiGroup TokenManager - * - * @apiDescription Deletes related token that given id - * @apiQuery {String} tokenid, Token id to be deleted - * - * @apiSuccessExample {json} Success-Response: - * HTTP/1.1 200 OK - * { - * "result": { - * "result": { - * "n": 1, - * "ok": 1 - * }, - * "connection": { - * "_events": {}, - * "_eventsCount": 4, - * "id": 4, - * "address": "127.0.0.1:27017", - * "bson": {}, - * "socketTimeout": 999999999, - * "host": "localhost", - * "port": 27017, - * "monitorCommands": false, - * "closed": false, - * "destroyed": false, - * "lastIsMasterMS": 15 - * }, - * "deletedCount": 1, - * "n": 1, - * "ok": 1 - * } - * } - * - * @apiErrorExample {json} Error-Response: - * HTTP/1.1 400 Bad Request - * { - * "result": "Token id not provided" - * } - */ - case 'delete': - validateUser(() => { - if (params.qstring.tokenid) { - common.db.collection("auth_tokens").remove({ - "_id": params.qstring.tokenid, - "owner": params.member._id + "" - }, function(err, res) { - if (err) { - common.returnMessage(params, 404, err.message); - } - else { - common.returnMessage(params, 200, res); - } - }); - } - else { - common.returnMessage(params, 404, "Token id not provided"); - } - }, params); - break; - /** - * @api {get} /i/token/create - * @apiName createToken - * @apiGroup TokenManager - * - * @apiDescription Creates spesific token - * @apiQuery {String} purpose, Purpose is description of the created token - * @apiQuery {Array} endpointquery, Includes "params" and "endpoint" inside - * {"params":{qString Key: qString Val} - * "endpoint": "_endpointAdress" - * @apiQuery {Boolean} multi, Defines availability multiple times - * @apiQuery {Boolean} apps, App Id of selected application - * @apiQuery {Boolean} ttl, expiration time for token - * - * @apiSuccessExample {json} Success-Response: - * HTTP/1.1 200 OK - * { - * "result": "0e1c012f855e7065e779b57a616792fb5bd03834" - * } - * - * @apiErrorExample {json} Error-Response: - * HTTP/1.1 400 Bad Request - * { - * "result": "Missing parameter "api_key" or "auth_token"" - * } - */ - case 'create': - validateUser(params, () => { - let ttl, multi, endpoint, purpose, apps; - if (params.qstring.ttl) { - ttl = parseInt(params.qstring.ttl); - } - else { - ttl = 1800; - } - multi = true; - if (params.qstring.multi === false || params.qstring.multi === 'false') { - multi = false; - } - apps = params.qstring.apps || ""; - if (params.qstring.apps) { - apps = params.qstring.apps.split(','); - } - - if (params.qstring.endpointquery && params.qstring.endpointquery !== "") { - try { - endpoint = JSON.parse(params.qstring.endpointquery); //structure with also info for qstring params. - } - catch (ex) { - if (params.qstring.endpoint) { - endpoint = params.qstring.endpoint.split(','); - } - else { - endpoint = ""; - } - } - } - else if (params.qstring.endpoint) { - endpoint = params.qstring.endpoint.split(','); - } - - if (params.qstring.purpose) { - purpose = params.qstring.purpose; - } - authorize.save({ - db: common.db, - ttl: ttl, - multi: multi, - owner: params.member._id + "", - app: apps, - endpoint: endpoint, - purpose: purpose, - callback: (err, token) => { - if (err) { - common.returnMessage(params, 404, err); - } - else { - common.returnMessage(params, 200, token); - } - } - }); - }); - break; - default: - common.returnMessage(params, 400, 'Invalid path, must be one of /delete or /create'); - } - break; - } - case '/o/token': { //returns all my tokens - switch (paths[3]) { - case 'check': - if (!params.qstring.token) { - common.returnMessage(params, 400, 'Missing parameter "token"'); - return false; - } - - validateUser(params, function() { - authorize.check_if_expired({ - token: params.qstring.token, - db: common.db, - callback: (err, valid, time_left)=>{ - if (err) { - common.returnMessage(params, 404, err.message); - } - else { - common.returnMessage(params, 200, { - valid: valid, - time: time_left - }); - } - } - }); - }); - break; - /** - * @api {get} /o/token/list - * @apiName initialize - * @apiGroup TokenManager - * - * @apiDescription Returns active tokens as an array that uses tokens in order to protect the API key - * @apiQuery {String} app_id, App Id of related application or {String} auth_token - * - * @apiSuccessExample {json} Success-Response: - * HTTP/1.1 200 OK - * { - * "result": [ - * { - * "_id": "884803f9e9eda51f5dbbb45ba91fa7e2b1dbbf4b", - * "ttl": 0, - * "ends": 1650466609, - * "multi": false, - * "owner": "60e42efa5c23ee7ec6259af0", - * "app": "", - * "endpoint": [ - * - * ], - * "purpose": "Test Token", - * "temporary": false - * }, - * { - * "_id": "08976f4a2037d39a9e8a7ada8afe1707769b7878", - * "ttl": 1, - * "ends": 1650632001, - * "multi": true, - * "owner": "60e42efa5c23ee7ec6259af0", - * "app": "", - * "endpoint": "", - * "purpose": "LoggedInAuth", - * "temporary": false - * } - * ] - * } - * - * @apiErrorExample {json} Error-Response: - * HTTP/1.1 400 Bad Request - * { - * "result": "Missing parameter "api_key" or "auth_token"" - * } - */ - case 'list': - validateUser(params, function() { - common.db.collection("auth_tokens").find({"owner": params.member._id + ""}).toArray(function(err, res) { - if (err) { - common.returnMessage(params, 404, err.message); - } - else { - common.returnMessage(params, 200, res); - } - }); - }); - break; - default: - common.returnMessage(params, 400, 'Invalid path, must be one of /list'); - } - break; - } - case '/o': { - if (!params.qstring.app_id) { - common.returnMessage(params, 400, 'Missing parameter "app_id"'); - return false; - } - - switch (params.qstring.method) { - case 'total_users': - validateUserForDataReadAPI(params, 'core', countlyApi.data.fetch.fetchTotalUsersObj, params.qstring.metric || 'users'); - break; - case 'get_period_obj': - validateUserForDataReadAPI(params, 'core', countlyApi.data.fetch.getPeriodObj, 'users'); - break; - case 'locations': - case 'sessions': - case 'users': - validateUserForDataReadAPI(params, 'core', countlyApi.data.fetch.fetchTimeObj, 'users'); - break; - case 'app_versions': - case 'device_details': - validateUserForDataReadAPI(params, 'core', countlyApi.data.fetch.fetchTimeObj, 'device_details'); - break; - case 'devices': - case 'carriers': - validateUserForDataReadAPI(params, 'core', countlyApi.data.fetch.fetchTimeObj, params.qstring.method); - break; - case 'countries': - if (plugins.getConfig("api", params.app && params.app.plugins, true).country_data !== false) { - validateUserForDataReadAPI(params, 'core', countlyApi.data.fetch.fetchTimeObj, params.qstring.method); - } - else { - common.returnOutput(params, {}); - } - break; - case 'cities': - if (plugins.getConfig("api", params.app && params.app.plugins, true).city_data !== false) { - validateUserForDataReadAPI(params, 'core', countlyApi.data.fetch.fetchTimeObj, params.qstring.method); - } - else { - common.returnOutput(params, {}); - } - break; - case 'geodata': { - validateRead(params, 'core', function() { - if (params.qstring.loadFor === "cities") { - countlyApi.data.geoData.loadCityCoordiantes({"query": params.qstring.query}, function(err, data) { - common.returnOutput(params, data); - }); - } - }); - break; - } - case 'get_event_groups': - validateRead(params, 'core', countlyApi.data.fetch.fetchEventGroups); - break; - case 'get_event_group': - validateRead(params, 'core', countlyApi.data.fetch.fetchEventGroupById); - break; - case 'events': - if (params.qstring.events) { - try { - params.qstring.events = JSON.parse(params.qstring.events); - } - catch (SyntaxError) { - console.log('Parse events array failed', params.qstring.events, params.req.url, params.req.body); - } - if (params.qstring.overview) { - validateRead(params, 'core', countlyApi.data.fetch.fetchDataEventsOverview); - } - else { - validateRead(params, 'core', countlyApi.data.fetch.fetchMergedEventData); - } - } - else { - if (params.qstring.event && params.qstring.event.startsWith('[CLY]_group_')) { - validateRead(params, 'core', countlyApi.data.fetch.fetchMergedEventGroups); - } - else { - params.truncateEventValuesList = true; - validateRead(params, 'core', countlyApi.data.fetch.prefetchEventData, params.qstring.method); - } - } - break; - case 'get_events': - //validateRead(params, 'core', countlyApi.data.fetch.fetchCollection, 'events'); - validateRead(params, 'core', async function() { - try { - var result = await common.db.collection("events").findOne({ '_id': common.db.ObjectID(params.qstring.app_id) }); - result = result || {}; - result.list = result.list || []; - result.segments = result.segments || {}; - - if (result.list) { - result.list = result.list.filter(function(l) { - return l.indexOf('[CLY]') !== 0; - }); - } - if (result.segments) { - for (let i in result.segments) { - if (i.indexOf('[CLY]') === 0) { - delete result.segments[i]; - } - } - } - // result.list = []; - commented out to do not clear list. Keeping existing events. - //result.segments = {}; - const pluginsGetConfig = plugins.getConfig("api", params.app && params.app.plugins, true); - result.limits = { - event_limit: pluginsGetConfig.event_limit, - event_segmentation_limit: pluginsGetConfig.event_segmentation_limit, - event_segmentation_value_limit: pluginsGetConfig.event_segmentation_value_limit, - }; - - var aggregation = []; - aggregation.push({$match: {"app_id": params.qstring.app_id, "type": "e", "biglist": {"$ne": true}}}); - aggregation.push({"$project": {e: 1, _id: 0, "sg": 1}}); - //e does not start with [CLY]_ - aggregation.push({$match: {"e": {"$not": /^(\[CLY\]_)/}}}); - aggregation.push({"$sort": {"e": 1}}); - aggregation.push({"$limit": pluginsGetConfig.event_limit || 500}); - - var res = await common.drillDb.collection("drill_meta").aggregate(aggregation).toArray(); - for (var k = 0; k < res.length; k++) { - if (result.list.indexOf(res[k].e) === -1) { - result.list.push(res[k].e); - } - - if (res[k].sg && Object.keys(res[k].sg).length > 0) { - result.segments[res[k].e] = result.segments[res[k].e] || []; - for (var key in res[k].sg) { - if (result.segments[res[k].e].indexOf(key) === -1) { - result.segments[res[k].e].push(key); - } - } - } - if (result.omitted_segments && result.omitted_segments[res[k].e]) { - for (let kz = 0; kz < result.omitted_segments[res[k].e].length; kz++) { - //remove items that are in omitted list - result.segments[res[k].e].splice(result.segments[res[k].e].indexOf(result.omitted_segments[res[k].e][kz]), 1); - } - } - if (result.whitelisted_segments && result.whitelisted_segments[res[k].e] && Array.isArray(result.whitelisted_segments[res[k].e])) { - //remove all that are not whitelisted - for (let kz = 0; kz < result.segments[res[k].e].length; kz++) { - if (result.whitelisted_segments[res[k].e].indexOf(result.segments[res[k].e][kz]) === -1) { - result.segments[res[k].e].splice(kz, 1); - kz--; - } - } - } - //Sort segments - if (result.segments[res[k].e]) { - result.segments[res[k].e].sort(); - } - } - if (result.list.length === 0) { - delete result.list; - } - if (Object.keys(result.segments).length === 0) { - delete result.segments; - } - common.returnOutput(params, result); - - } - catch (ex) { - console.error("Error fetching events", ex); - common.returnMessage(params, 500, "Error fetching events"); - } - }, 'events'); - break; - case 'top_events': - validateRead(params, 'core', countlyApi.data.fetch.fetchDataTopEvents); - break; - case 'all_apps': - validateUserForGlobalAdmin(params, countlyApi.data.fetch.fetchAllApps); - break; - case 'notes': - validateRead(params, 'core', countlyApi.mgmt.users.fetchNotes); - break; - default: - if (!plugins.dispatch(apiPath, { - params: params, - validateUserForDataReadAPI: validateUserForDataReadAPI, - validateUserForMgmtReadAPI: validateUserForMgmtReadAPI, - validateUserForDataWriteAPI: validateUserForDataWriteAPI, - validateUserForGlobalAdmin: validateUserForGlobalAdmin - })) { - common.returnMessage(params, 400, 'Invalid method'); - } - break; - } - - break; - } - case '/o/analytics': { - if (!params.qstring.app_id) { - common.returnMessage(params, 400, 'Missing parameter "app_id"'); - return false; - } - - switch (paths[3]) { - case 'dashboard': - validateUserForDataReadAPI(params, 'core', countlyApi.data.fetch.fetchDashboard); - break; - case 'countries': - validateUserForDataReadAPI(params, 'core', countlyApi.data.fetch.fetchCountries); - break; - case 'sessions': - //takes also bucket=daily || monthly. extends period to full months if monthly - validateUserForDataReadAPI(params, 'core', countlyApi.data.fetch.fetchSessions); - break; - case 'metric': - validateUserForDataReadAPI(params, 'core', countlyApi.data.fetch.fetchMetric); - break; - case 'tops': - validateUserForDataReadAPI(params, 'core', countlyApi.data.fetch.fetchTops); - break; - case 'loyalty': - validateUserForDataReadAPI(params, 'core', countlyApi.data.fetch.fetchLoyalty); - break; - case 'frequency': - validateUserForDataReadAPI(params, 'core', countlyApi.data.fetch.fetchFrequency); - break; - case 'durations': - validateUserForDataReadAPI(params, 'core', countlyApi.data.fetch.fetchDurations); - break; - case 'events': - //takes also bucket=daily || monthly. extends period to full months if monthly - validateUserForDataReadAPI(params, 'core', countlyApi.data.fetch.fetchEvents); - break; - default: - if (!plugins.dispatch(apiPath, { - params: params, - validateUserForDataReadAPI: validateUserForDataReadAPI, - validateUserForMgmtReadAPI: validateUserForMgmtReadAPI, - paths: paths, - validateUserForDataWriteAPI: validateUserForDataWriteAPI, - validateUserForGlobalAdmin: validateUserForGlobalAdmin - })) { - common.returnMessage(params, 400, 'Invalid path, must be one of /dashboard, /countries, /sessions, /metric, /tops, /loyalty, /frequency, /durations, /events'); - } - break; - } - - break; - } - case '/o/aggregate': { - validateUser(params, () => { - //Long task to run specific drill query. Give back task_id if running, result if done. - if (params.qstring.query) { - - try { - params.qstring.query = JSON.parse(params.qstring.query); - } - catch (ee) { - log.e(ee); - common.returnMessage(params, 400, 'Invalid query parameter'); - return; - } - - if (params.qstring.query.appID) { - if (Array.isArray(params.qstring.query.appID)) { - //make sure member has access to all apps in this list - for (var i = 0; i < params.qstring.query.appID.length; i++) { - if (!params.member.global_admin && params.member.user_of && params.member.user_of.indexOf(params.qstring.query.appID[i]) === -1) { - common.returnMessage(params, 401, 'User does not have access right for this app'); - return; - } - } - } - else { - if (!params.member.global_admin && params.member.user_of && params.member.user_of.indexOf(params.qstring.query.appID) === -1) { - common.returnMessage(params, 401, 'User does not have access right for this app'); - return; - } - } - } - else { - params.qstring.query.appID = params.qstring.app_id; - } - if (params.qstring.period) { - params.qstring.query.period = params.qstring.query.period || params.qstring.period || "30days"; - } - if (params.qstring.periodOffset) { - params.qstring.query.periodOffset = params.qstring.query.periodOffset || params.qstring.periodOffset || 0; - } - - calculatedDataManager.longtask({ - db: common.db, - no_cache: params.qstring.no_cache, - threshold: plugins.getConfig("api").request_threshold, - app_id: params.qstring.query.app_id, - query_data: params.qstring.query, - outputData: function(err, data) { - if (err) { - common.returnMessage(params, 400, err); - } - else { - common.returnMessage(params, 200, data); - } - } - }); - } - else { - common.returnMessage(params, 400, 'Missing parameter "query"'); - } - - }); - break; - } - case '/o/countly_version': { - validateUser(params, () => { - //load previos version info if exist - loadFsVersionMarks(function(errFs, fsValues) { - loadDbVersionMarks(function(errDb, dbValues) { - //load mongodb version - common.db.command({ buildInfo: 1 }, function(errorV, info) { - var response = {}; - if (errorV) { - response.mongo = errorV; - } - else { - if (info && info.version) { - response.mongo = info.version; - } - } - - if (errFs) { - response.fs = errFs; - } - else { - response.fs = fsValues; - } - if (errDb) { - response.db = errDb; - } - else { - response.db = dbValues; - } - response.pkg = packageJson.version || ""; - var statusCode = (errFs && errDb) ? 400 : 200; - common.returnMessage(params, statusCode, response); - }); - }); - }); - }); - break; - } - case '/o/sdk': { - params.ip_address = params.qstring.ip_address || common.getIpAddress(params.req); - params.user = {}; - - if (!params.qstring.app_key || !params.qstring.device_id) { - common.returnMessage(params, 400, 'Missing parameter "app_key" or "device_id"'); - return false; - } - else { - params.qstring.device_id += ""; - params.app_user_id = common.crypto.createHash('sha1') - .update(params.qstring.app_key + params.qstring.device_id + "") - .digest('hex'); - } - - log.d('processing request %j', params.qstring); - - params.promises = []; - - validateAppForFetchAPI(params, () => { }); - - break; - } - case '/i/sdk': { - params.ip_address = params.qstring.ip_address || common.getIpAddress(params.req); - params.user = {}; - - if (!params.qstring.app_key || !params.qstring.device_id) { - common.returnMessage(params, 400, 'Missing parameter "app_key" or "device_id"'); - return false; - } - else { - params.qstring.device_id += ""; - params.app_user_id = common.crypto.createHash('sha1') - .update(params.qstring.app_key + params.qstring.device_id + "") - .digest('hex'); - } - - log.d('processing request %j', params.qstring); - - params.promises = []; - - validateAppForFetchAPI(params, () => { }); - - break; - } - case '/o/notes': { - validateUserForDataReadAPI(params, 'core', countlyApi.mgmt.users.fetchNotes); - break; - } - case '/o/cms': { - switch (paths[3]) { - case 'entries': - validateUserForMgmtReadAPI(countlyApi.mgmt.cms.getEntries, params); - break; - } - break; - } - case '/i/cms': { - switch (paths[3]) { - case 'save_entries': - validateUserForWrite(params, countlyApi.mgmt.cms.saveEntries); - break; - case 'clear': - validateUserForWrite(countlyApi.mgmt.cms.clearCache, params); - break; - default: - if (!plugins.dispatch(apiPath, { - params: params, - validateUserForDataReadAPI: validateUserForDataReadAPI, - validateUserForMgmtReadAPI: validateUserForMgmtReadAPI, - paths: paths, - validateUserForDataWriteAPI: validateUserForDataWriteAPI, - validateUserForGlobalAdmin: validateUserForGlobalAdmin - })) { - common.returnMessage(params, 400, 'Invalid path, must be one of /save_entries or /clear'); - } - break; - } - break; - } - case '/o/date_presets': { - switch (paths[3]) { - case 'getAll': - validateUserForMgmtReadAPI(countlyApi.mgmt.datePresets.getAll, params); - break; - case 'getById': - validateUserForMgmtReadAPI(countlyApi.mgmt.datePresets.getById, params); - break; - default: - if (!plugins.dispatch(apiPath, { - params: params, - validateUserForDataReadAPI: validateUserForDataReadAPI, - validateUserForMgmtReadAPI: validateUserForMgmtReadAPI, - paths: paths, - validateUserForDataWriteAPI: validateUserForDataWriteAPI, - validateUserForGlobalAdmin: validateUserForGlobalAdmin - })) { - common.returnMessage(params, 400, 'Invalid path, must be one of /getAll /getById'); - } - break; - } - break; - } - case '/i/date_presets': { - switch (paths[3]) { - case 'create': - validateUserForWrite(params, countlyApi.mgmt.datePresets.create); - break; - case 'update': - validateUserForWrite(params, countlyApi.mgmt.datePresets.update); - break; - case 'delete': - validateUserForWrite(params, countlyApi.mgmt.datePresets.delete); - break; - default: - if (!plugins.dispatch(apiPath, { - params: params, - validateUserForDataReadAPI: validateUserForDataReadAPI, - validateUserForMgmtReadAPI: validateUserForMgmtReadAPI, - paths: paths, - validateUserForDataWriteAPI: validateUserForDataWriteAPI, - validateUserForGlobalAdmin: validateUserForGlobalAdmin - })) { - common.returnMessage(params, 400, 'Invalid path, must be one of /create /update or /delete'); - } - break; - } - break; - } + // '/i/users' has been migrated to core/api/routes/users.js (Express route) + // '/i/notes' has been migrated to core/api/routes/notes.js (Express route) + // '/o/render' has been migrated to core/api/routes/render.js (Express route) + // '/i/app_users' has been migrated to core/api/routes/app_users.js (Express route) + // '/i/apps' has been migrated to core/api/routes/apps.js (Express route) + // '/i/event_groups' has been migrated to core/api/routes/event_groups.js (Express route) + // '/i/tasks' has been migrated to core/api/routes/tasks.js (Express route) + // '/i/events' has been migrated to core/api/routes/events.js (Express route) + // '/o/users' has been migrated to core/api/routes/users.js (Express route) + // '/o/app_users' has been migrated to core/api/routes/app_users.js (Express route) + // '/o/apps' has been migrated to core/api/routes/apps.js (Express route) + // '/o/tasks' has been migrated to core/api/routes/tasks.js (Express route) + // '/o/system' has been migrated to core/api/routes/system.js (Express route) + // '/o/export' has been migrated to core/api/routes/export.js (Express route) + // '/o/ping' has been migrated to core/api/routes/ping.js (Express route) + // '/o/jwt' and '/i/jwt' have been migrated to core/api/routes/jwt.js (Express routes) + // '/i/token' has been migrated to core/api/routes/token.js (Express route) + // '/o/token' has been migrated to core/api/routes/token.js (Express route) + // '/o' has been migrated to core/api/routes/data.js (Express route) + // '/o/analytics' has been migrated to core/api/routes/analytics.js (Express route) + // '/o/aggregate' has been migrated to core/api/routes/analytics.js (Express route) + // '/o/countly_version' has been migrated to core/api/routes/version.js (Express route) + // '/o/sdk' has been migrated to core/api/routes/sdk.js (Express route) + // '/i/sdk' has been migrated to core/api/routes/sdk.js (Express route) + // '/o/notes' has been migrated to core/api/routes/notes.js (Express route) + // '/o/cms' has been migrated to core/api/routes/cms.js (Express route) + // '/i/cms' has been migrated to core/api/routes/cms.js (Express route) + // '/o/date_presets' has been migrated to core/api/routes/date_presets.js (Express route) + // '/i/date_presets' has been migrated to core/api/routes/date_presets.js (Express route) default: if (!plugins.dispatch(apiPath, { params: params, @@ -4074,4 +929,4 @@ function loadDbVersionMarks(callback) { } /** @lends module:api/utils/requestProcessor */ -module.exports = {processRequest: processRequest, processUserFunction: processUser}; +module.exports = {processRequest: processRequest, processUserFunction: processUser, validateAppForFetchAPI: validateAppForFetchAPI}; \ No newline at end of file diff --git a/api/utils/rights.js b/api/utils/rights.js index b250c3da5e5..981b5103e4b 100644 --- a/api/utils/rights.js +++ b/api/utils/rights.js @@ -12,7 +12,8 @@ var common = require("./common.js"), plugins = require('../../plugins/pluginManager.ts'), Promise = require("bluebird"), crypto = require('crypto'), - log = require('./log.js')('core:rights'); + log = require('./log.js')('core:rights'), + jwtUtils = require('./jwt.js'); /** @type {Authorizer} */ var authorize = require('./authorizer.js'); //for token validations @@ -31,6 +32,24 @@ var cachedSchema = {}; */ function validate_token_if_exists(params) { return new Promise(function(resolve) { + // Check for JWT Bearer token first + var bearerToken = jwtUtils.extractBearerToken(params.req); + if (bearerToken) { + var jwtResult = jwtUtils.verifyToken(bearerToken, 'access'); + if (jwtResult.valid) { + // Store JWT claims in params for potential use + params.jwt_claims = jwtResult.decoded; + resolve(jwtResult.decoded.sub); + return; + } + else if (jwtResult.error === 'TOKEN_EXPIRED') { + resolve('token-expired'); + return; + } + // Fall through to existing auth if JWT invalid (allows other auth methods) + } + + // Existing auth_token logic var token = params.qstring.auth_token || params.req.headers["countly-token"] || ""; if (token && token !== "") { authorize.verify_return({ @@ -70,7 +89,7 @@ exports.validateUserForRead = function(params, callback, callbackParam) { validate_token_if_exists(params).then(function(result) { var query = ""; // then result is owner id - if (result !== 'token-not-given' && result !== 'token-invalid') { + if (result !== 'token-not-given' && result !== 'token-invalid' && result !== 'token-expired') { query = {'_id': common.db.ObjectID(result)}; } else { @@ -79,6 +98,10 @@ exports.validateUserForRead = function(params, callback, callbackParam) { common.returnMessage(params, 400, 'Token not valid'); return false; } + else if (result === 'token-expired') { + common.returnMessage(params, 401, 'Token has expired'); + return false; + } else { common.returnMessage(params, 400, 'Missing parameter "api_key" or "auth_token"'); return false; @@ -166,7 +189,7 @@ exports.validateUserForWrite = function(params, callback, callbackParam) { validate_token_if_exists(params).then(function(result) { var query = ""; // then result is owner id - if (result !== 'token-not-given' && result !== 'token-invalid') { + if (result !== 'token-not-given' && result !== 'token-invalid' && result !== 'token-expired') { query = {'_id': common.db.ObjectID(result)}; } else { @@ -175,6 +198,10 @@ exports.validateUserForWrite = function(params, callback, callbackParam) { common.returnMessage(params, 400, 'Token not valid'); return false; } + else if (result === 'token-expired') { + common.returnMessage(params, 401, 'Token has expired'); + return false; + } else { common.returnMessage(params, 400, 'Missing parameter "api_key" or "auth_token"'); return false; @@ -255,7 +282,7 @@ exports.validateGlobalAdmin = function(params, callback, callbackParam) { validate_token_if_exists(params).then(function(result) { var query = ""; // then result is owner id - if (result !== 'token-not-given' && result !== 'token-invalid') { + if (result !== 'token-not-given' && result !== 'token-invalid' && result !== 'token-expired') { query = {'_id': common.db.ObjectID(result)}; } else { @@ -264,6 +291,10 @@ exports.validateGlobalAdmin = function(params, callback, callbackParam) { common.returnMessage(params, 400, 'Token not valid'); return false; } + else if (result === 'token-expired') { + common.returnMessage(params, 401, 'Token has expired'); + return false; + } else { common.returnMessage(params, 400, 'Missing parameter "api_key" or "auth_token"'); return false; @@ -326,7 +357,7 @@ exports.validateAppAdmin = function(params, callback, callbackParam) { validate_token_if_exists(params).then(function(result) { var query = ""; // then result is owner id - if (result !== 'token-not-given' && result !== 'token-invalid') { + if (result !== 'token-not-given' && result !== 'token-invalid' && result !== 'token-expired') { query = {'_id': common.db.ObjectID(result)}; } else { @@ -335,6 +366,10 @@ exports.validateAppAdmin = function(params, callback, callbackParam) { common.returnMessage(params, 400, 'Token not valid'); return false; } + else if (result === 'token-expired') { + common.returnMessage(params, 401, 'Token has expired'); + return false; + } else { common.returnMessage(params, 400, 'Missing parameter "api_key" or "auth_token"'); return false; @@ -411,7 +446,7 @@ exports.validateUser = function(params, callback, callbackParam) { validate_token_if_exists(params).then(function(result) { var query = ""; // then result is owner id - if (result !== 'token-not-given' && result !== 'token-invalid') { + if (result !== 'token-not-given' && result !== 'token-invalid' && result !== 'token-expired') { query = {'_id': common.db.ObjectID(result)}; } else { @@ -420,6 +455,10 @@ exports.validateUser = function(params, callback, callbackParam) { common.returnMessage(params, 400, 'Token not valid'); return false; } + else if (result === 'token-expired') { + common.returnMessage(params, 401, 'Token has expired'); + return false; + } else { common.returnMessage(params, 400, 'Missing parameter "api_key" or "auth_token"'); return false; @@ -809,7 +848,7 @@ exports.validateRead = function(params, feature, callback, callbackParam) { validate_token_if_exists(params).then(function(result) { var query = ""; // then result is owner id - if (result !== 'token-not-given' && result !== 'token-invalid') { + if (result !== 'token-not-given' && result !== 'token-invalid' && result !== 'token-expired') { query = {'_id': common.db.ObjectID(result)}; } else { @@ -818,6 +857,10 @@ exports.validateRead = function(params, feature, callback, callbackParam) { common.returnMessage(params, 400, 'Token not valid'); return false; } + else if (result === 'token-expired') { + common.returnMessage(params, 401, 'Token has expired'); + return false; + } else { common.returnMessage(params, 400, 'Missing parameter "api_key" or "auth_token"'); return false; @@ -963,7 +1006,7 @@ function validateWrite(params, feature, accessType, callback, callbackParam) { var query = ""; //var appIdExceptions = ['global_users', 'global_applications', 'global_jobs', 'global_plugins', 'global_configurations', 'global_upload']; // then result is owner id - if (result !== 'token-not-given' && result !== 'token-invalid') { + if (result !== 'token-not-given' && result !== 'token-invalid' && result !== 'token-expired') { query = {'_id': common.db.ObjectID(result)}; } else { @@ -972,6 +1015,10 @@ function validateWrite(params, feature, accessType, callback, callbackParam) { common.returnMessage(params, 400, 'Token not valid'); return false; } + else if (result === 'token-expired') { + common.returnMessage(params, 401, 'Token has expired'); + return false; + } else { common.returnMessage(params, 400, 'Missing parameter "api_key" or "auth_token"'); return false; diff --git a/package.json b/package.json index aa70a7e95d0..93e01ec1252 100644 --- a/package.json +++ b/package.json @@ -96,6 +96,7 @@ "@opentelemetry/semantic-conventions": "^1.36.0", "@pulsecron/pulse": "^1.6.8", "@pyroscope/nodejs": "^0.4.5", + "@types/express": "^5.0.6", "all-the-cities": "3.1.0", "argon2": "0.41.1", "async": "3.2.6", diff --git a/plugins/pluginManager.ts b/plugins/pluginManager.ts index c964cf56641..ba8bf2d5f7d 100644 --- a/plugins/pluginManager.ts +++ b/plugins/pluginManager.ts @@ -50,6 +50,14 @@ interface EventsRegistry { [eventName: string]: EventHandler[]; } +/** API route definition for Express-style routing */ +interface ApiRouteDefinition { + method: string; + path: string; + pluginName: string; + handler: (req: any, res: any, next: any) => void; +} + /** Plugin API methods */ interface PluginApi { [methodName: string]: Function; @@ -337,6 +345,9 @@ class PluginManager { /** Core plugin list */ coreList: string[] = ['api', 'core']; + /** Express-style API route definitions registered by plugins */ + apiRoutes: ApiRouteDefinition[] = []; + /** Dependency graph */ dependencyMap: any = {}; @@ -1136,6 +1147,68 @@ class PluginManager { } } + /** + * Register an Express-style API HTTP route for a plugin. + * Use this for HTTP endpoint registration on the API server (port 3001). + * For lifecycle hooks and internal events, continue using plugins.register(). + * @param method - HTTP method: 'GET', 'POST', 'DELETE', 'PUT', or 'ALL' + * @param path - Express route path, e.g. '/o/myfeature' or '/o/myfeature/:id' + * @param handler - Express route handler (req, res, next) => void. Access params via req.countlyParams. + * @param featureName - optional plugin name override + */ + apiRoute(method: string, path: string, handler: (req: any, res: any, next: any) => void, featureName?: string): void { + if (!featureName) { + featureName = this.getFeatureName(); + featureName = featureName || 'core'; + } + this.apiRoutes.push({ + method: method.toUpperCase(), + path, + pluginName: featureName, + handler + }); + } + + /** + * Mount all registered API routes onto an Express Router. + * Called during server startup in api.js after all plugins have loaded. + * Each route is wrapped with plugin enable/disable checking, matching + * the same pattern used by loadAppPlugins() for the frontend server. + * @param router - Express Router instance + */ + mountApiRoutes(router: any): void { + for (const route of this.apiRoutes) { + const pluginName = route.pluginName; + const handler = route.handler; + const fullPluginsMap = this.fullPluginsMap; + + const wrappedHandler = (req: any, res: any, next: any) => { + // Skip if this is a non-core plugin that has been disabled + if (fullPluginsMap[pluginName] && pluginConfig[pluginName] === false) { + return next(); + } + handler(req, res, next); + }; + + const m = route.method; + if (m === 'GET') { + router.get(route.path, wrappedHandler); + } + else if (m === 'POST') { + router.post(route.path, wrappedHandler); + } + else if (m === 'DELETE') { + router.delete(route.path, wrappedHandler); + } + else if (m === 'PUT') { + router.put(route.path, wrappedHandler); + } + else { + router.all(route.path, wrappedHandler); + } + } + } + /** * Dispatch specific event on api side * @param event - event to dispatch diff --git a/test/unit/api.utils.jwt.unit.js b/test/unit/api.utils.jwt.unit.js new file mode 100644 index 00000000000..8fa916cf04e --- /dev/null +++ b/test/unit/api.utils.jwt.unit.js @@ -0,0 +1,295 @@ +var should = require("should"); +var jwt = require("jsonwebtoken"); + +// Mock the common module before requiring jwt module +var mockConfig = { + jwt: { + secret: 'test-secret-key-with-at-least-32-characters-for-testing', + accessTokenExpiry: 900, + refreshTokenExpiry: 604800, + issuer: 'countly-test', + algorithm: 'HS256' + } +}; + +var mockDb = { + collection: function() { + return { + findOne: function(query, callback) { + callback(null, null); + }, + insertOne: function(doc, callback) { + callback(null, { insertedId: doc._id }); + }, + createIndex: function(keys, options, callback) { + if (callback) { + callback(null); + } + } + }; + }, + ObjectID: function(id) { + return id; + } +}; + +// Mock common module +require.cache[require.resolve("../../api/utils/common.js")] = { + id: require.resolve("../../api/utils/common.js"), + filename: require.resolve("../../api/utils/common.js"), + loaded: true, + exports: { + config: mockConfig, + db: mockDb + } +}; + +// Mock log module +require.cache[require.resolve("../../api/utils/log.js")] = { + id: require.resolve("../../api/utils/log.js"), + filename: require.resolve("../../api/utils/log.js"), + loaded: true, + exports: function() { + return { + d: function() {}, + i: function() {}, + w: function() {}, + e: function() {} + }; + } +}; + +// Now require the jwt module +var jwtUtils = require("../../api/utils/jwt.js"); + +describe("JWT Utility Functions", function() { + describe("getConfig", function() { + it("should return configuration with defaults", function() { + var config = jwtUtils.getConfig(); + config.should.have.property('secret'); + config.should.have.property('accessTokenExpiry', 900); + config.should.have.property('refreshTokenExpiry', 604800); + config.should.have.property('issuer', 'countly-test'); + config.should.have.property('algorithm', 'HS256'); + }); + }); + + describe("isConfigured", function() { + it("should return true when secret is properly configured", function() { + jwtUtils.isConfigured().should.be.true(); + }); + }); + + describe("signAccessToken", function() { + var mockMember = { + _id: '507f1f77bcf86cd799439011', + global_admin: true, + permission: null + }; + + it("should sign an access token successfully", function() { + var result = jwtUtils.signAccessToken(mockMember); + result.should.have.property('success', true); + result.should.have.property('token'); + result.should.have.property('expiresIn', 900); + + // Verify the token is valid + var decoded = jwt.verify(result.token, mockConfig.jwt.secret); + decoded.should.have.property('sub', '507f1f77bcf86cd799439011'); + decoded.should.have.property('type', 'access'); + decoded.should.have.property('global_admin', true); + }); + + it("should include permissions for non-global admins", function() { + var nonAdminMember = { + _id: '507f1f77bcf86cd799439012', + global_admin: false, + permission: { + c: { 'app1': { all: true } }, + r: { 'app1': { all: true } }, + u: { 'app1': { all: true } }, + d: { 'app1': { all: true } }, + _: { a: ['app1'], u: [[]] } + } + }; + + var result = jwtUtils.signAccessToken(nonAdminMember); + result.should.have.property('success', true); + + var decoded = jwt.verify(result.token, mockConfig.jwt.secret); + decoded.should.have.property('permission'); + decoded.permission.should.have.property('c'); + }); + }); + + describe("signRefreshToken", function() { + it("should sign a refresh token with JTI", function() { + var result = jwtUtils.signRefreshToken('507f1f77bcf86cd799439011'); + result.should.have.property('success', true); + result.should.have.property('token'); + result.should.have.property('jti'); + result.should.have.property('expiresIn', 604800); + + // Verify the token + var decoded = jwt.verify(result.token, mockConfig.jwt.secret); + decoded.should.have.property('sub', '507f1f77bcf86cd799439011'); + decoded.should.have.property('type', 'refresh'); + decoded.should.have.property('jti', result.jti); + }); + + it("should generate unique JTIs for each token", function() { + var result1 = jwtUtils.signRefreshToken('507f1f77bcf86cd799439011'); + var result2 = jwtUtils.signRefreshToken('507f1f77bcf86cd799439011'); + + result1.jti.should.not.equal(result2.jti); + }); + }); + + describe("verifyToken", function() { + it("should verify a valid access token", function() { + var mockMember = { + _id: '507f1f77bcf86cd799439011', + global_admin: true + }; + var signResult = jwtUtils.signAccessToken(mockMember); + var verifyResult = jwtUtils.verifyToken(signResult.token, 'access'); + + verifyResult.should.have.property('valid', true); + verifyResult.should.have.property('decoded'); + verifyResult.decoded.should.have.property('sub', '507f1f77bcf86cd799439011'); + verifyResult.decoded.should.have.property('type', 'access'); + }); + + it("should verify a valid refresh token", function() { + var signResult = jwtUtils.signRefreshToken('507f1f77bcf86cd799439011'); + var verifyResult = jwtUtils.verifyToken(signResult.token, 'refresh'); + + verifyResult.should.have.property('valid', true); + verifyResult.decoded.should.have.property('type', 'refresh'); + verifyResult.decoded.should.have.property('jti'); + }); + + it("should reject token with wrong type", function() { + var signResult = jwtUtils.signRefreshToken('507f1f77bcf86cd799439011'); + var verifyResult = jwtUtils.verifyToken(signResult.token, 'access'); + + verifyResult.should.have.property('valid', false); + verifyResult.should.have.property('error', 'INVALID_TOKEN_TYPE'); + }); + + it("should reject expired tokens", function() { + // Create a token that's already expired + var expiredToken = jwt.sign( + { sub: '507f1f77bcf86cd799439011', type: 'access' }, + mockConfig.jwt.secret, + { expiresIn: -10, issuer: mockConfig.jwt.issuer } + ); + + var verifyResult = jwtUtils.verifyToken(expiredToken, 'access'); + verifyResult.should.have.property('valid', false); + verifyResult.should.have.property('error', 'TOKEN_EXPIRED'); + }); + + it("should reject invalid tokens", function() { + var verifyResult = jwtUtils.verifyToken('invalid.token.here', 'access'); + verifyResult.should.have.property('valid', false); + verifyResult.should.have.property('error', 'INVALID_TOKEN'); + }); + + it("should reject tokens with wrong secret", function() { + var token = jwt.sign( + { sub: '507f1f77bcf86cd799439011', type: 'access' }, + 'wrong-secret', + { expiresIn: 900, issuer: mockConfig.jwt.issuer } + ); + + var verifyResult = jwtUtils.verifyToken(token, 'access'); + verifyResult.should.have.property('valid', false); + verifyResult.should.have.property('error', 'INVALID_TOKEN'); + }); + + it("should return error for missing token", function() { + var verifyResult = jwtUtils.verifyToken(null, 'access'); + verifyResult.should.have.property('valid', false); + verifyResult.should.have.property('error', 'NO_TOKEN'); + }); + }); + + describe("extractBearerToken", function() { + it("should extract Bearer token from Authorization header", function() { + var req = { + headers: { + authorization: 'Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9' + } + }; + var token = jwtUtils.extractBearerToken(req); + token.should.equal('eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9'); + }); + + it("should extract Bearer token with capitalized header", function() { + var req = { + headers: { + Authorization: 'Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9' + } + }; + var token = jwtUtils.extractBearerToken(req); + token.should.equal('eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9'); + }); + + it("should return null for missing header", function() { + var req = { headers: {} }; + should(jwtUtils.extractBearerToken(req)).be.null(); + }); + + it("should return null for non-Bearer auth", function() { + var req = { + headers: { + authorization: 'Basic dXNlcm5hbWU6cGFzc3dvcmQ=' + } + }; + should(jwtUtils.extractBearerToken(req)).be.null(); + }); + + it("should return null for malformed Bearer header", function() { + var req = { + headers: { + authorization: 'Bearer' + } + }; + should(jwtUtils.extractBearerToken(req)).be.null(); + }); + + it("should return null for missing request", function() { + should(jwtUtils.extractBearerToken(null)).be.null(); + }); + + it("should handle case-insensitive Bearer", function() { + var req = { + headers: { + authorization: 'BEARER token123' + } + }; + jwtUtils.extractBearerToken(req).should.equal('token123'); + }); + }); + + describe("isTokenBlacklisted", function() { + it("should check blacklist status via callback", function(done) { + jwtUtils.isTokenBlacklisted('test-jti', function(err, isBlacklisted) { + should(err).be.null(); + isBlacklisted.should.be.false(); + done(); + }); + }); + }); + + describe("blacklistToken", function() { + it("should add token to blacklist via callback", function(done) { + var expiresAt = new Date(Date.now() + 86400000); + jwtUtils.blacklistToken('test-jti', 'member-id', expiresAt, function(err) { + should(err).be.null(); + done(); + }); + }); + }); +}); diff --git a/tsconfig.json b/tsconfig.json index 074faf3d4e8..5fe8a149160 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -81,11 +81,11 @@ "api/jobs/appExpire.ts", "api/jobs/clearAutoTasks.ts", "api/jobs/clearTokens.ts", - "api/jobs/task.ts", - "api/jobs/ttlCleanup.ts", - "api/jobs/userMerge.ts", - "api/jobs/mutationManagerJob.ts" - ], + "api/jobs/task.ts", + "api/jobs/ttlCleanup.ts", + "api/jobs/userMerge.ts", + "api/jobs/mutationManagerJob.ts" + ], "exclude": [ "node_modules/*", "ui-tests/*", diff --git a/types/requestProcessor.d.ts b/types/requestProcessor.d.ts index 3308f622d6a..a5143ad1e94 100644 --- a/types/requestProcessor.d.ts +++ b/types/requestProcessor.d.ts @@ -1,326 +1,336 @@ -import { ServerResponse, IncomingMessage } from "http"; -import { ObjectId } from "mongodb"; -import { TimeObject } from "./common"; -/** Application user interface representing a document from the app_users collection */ -export interface AppUser { - /** MongoDB document ID */ - _id?: string; - /** Application user ID */ - uid: string; - /** Device ID */ - did: string; - /** User's country */ - country: string; - /** User's city */ - city: string; - /** User's timezone offset (in minutes) */ - tz: number; - /** Custom properties for the application user */ - custom?: Record; - /** Last session timestamp of the app user */ - ls?: object; - /** Last session ID */ - lsid?: string; - /** Flag indicating if the user has an ongoing session */ - has_ongoing_session?: boolean; - /** Timestamp of the user's last request */ - last_req?: number; - /** Last GET request URL */ - last_req_get?: string; - /** Last POST request body */ - last_req_post?: string; - /** First access timestamp */ - fac?: number; - /** First seen timestamp */ - fs?: number; - /** Last access timestamp */ - lac?: number; - /** SDK information */ - sdk?: { - /** SDK name */ - name?: string; - /** SDK version */ - version?: string; - }; - /** Request count */ - req_count?: number; - /** Type or token identifier */ - t?: string | number; - /** Flag indicating if the user has information */ - hasInfo?: boolean; - /** Information about merged users */ - merges?: any; -} - -/** Main request processing object containing all information shared through all the parts of the same request */ -export interface Params { - /** full URL href */ - href: string; - /** The HTTP response object */ - res: ServerResponse; - /** The HTTP request object */ - req: IncomingMessage; - /** API output handler. Which should handle API response */ - APICallback: (error: boolean, response: any, headers?: any, returnCode?: number, params?: Params) => void; - /** all the passed fields either through query string in GET requests or body and query string for POST requests */ - qstring: Record; - /** two top level url path, for example /i/analytics, first two segments from the fullPath */ - apiPath: string; - /** full url path, for example /i/analytics/dashboards */ - fullPath: string; - /** object with uploaded files, available in POST requests which upload files */ - files: { - app_image?: { - /** The temporary path of the uploaded app image file */ - path: string; - /** The original name of the uploaded app image file */ - name: string; - /** The MIME type of the uploaded app image file */ - type: string; - /** The size (in bytes) of the uploaded app image file */ - size: number; - }; - }; - /** Used for skipping SDK requests, if contains true, then request should be ignored and not processed. Can be set at any time by any plugin, but API only checks for it in beggining after / and /sdk events, so that is when plugins should set it if needed. Should contain reason for request cancelation */ - cancelRequest?: string; - /** [blockResponses=false] Flag to block responses from being sent */ - blockResponses?: boolean; - /** [forceProcessingRequestTimeout=false] Flag to force processing request timeout */ - forceProcessingRequestTimeout?: boolean; - /** True if this SDK request is processed from the bulk method */ - bulk?: boolean; - /** Array of the promises by different events. When all promises are fulfilled, request counts as processed */ - promises: Array>; - /** IP address of the device submitted request, exists in all SDK requests */ - ip_address: string; - /** Data with some user info, like country geolocation, etc from the request, exists in all SDK requests */ - user: { - /** User's country */ - country: string; - /** User's city */ - city: string; - /** User's timezone offset (in minutes) */ - tz?: number; - }; - /** Document from the app_users collection for current user, exists in all SDK requests after validation */ - app_user: AppUser; - /** ID of app_users document for the user, exists in all SDK requests after validation */ - app_user_id: string; - /** Document for the app sending request, exists in all SDK requests after validation and after validateUserForDataReadAPI validation */ - app: { - /** ID of the app document */ - _id: string; - /** Name of the app */ - name: string; - /** Country of the app */ - country: string; - /** Category of the app */ - category: string; - /** Timezone of the app */ - timezone: string; - /** Type of the app */ - type: string; - /** Flag indicating if the app is locked */ - locked: boolean; - /** Plugin-specific configuration for the app */ - plugins: Record; - }; - /** ObjectID of the app document, available after validation */ - app_id: ObjectId; - /** Selected app country, available after validation */ - app_cc: string; - /** Selected app timezone, available after validation */ - appTimezone: string; - /** All data about dashboard user sending the request, exists on all requests containing api_key, after validation through validation methods */ - member: { - /** ID of the dashboard user */ - _id: string; - /** Flag indicating if the user has global admin rights */ - global_admin: boolean; - /** The authentication token for the user */ - auth_token: string; - /** Flag indicating if the user is locked */ - locked: boolean; - /** Array of app IDs the user is an admin of (legacy) */ - admin_of?: Array; - /** Array of app IDs the user has user access to (legacy) */ - user_of?: Array; - /** Username of the dashboard user */ - username: string; - /** Email address of the dashboard user */ - email: string; - /** Full name of the dashboard user */ - full_name: string; - /** API key of the dashboard user */ - api_key: string; - /** Object containing user's access permissions */ - permission: { - _: { - /** Array of arrays containing app IDs the user has user access to */ - u: Array>; - /** Array of app IDs the user has admin access to */ - a: Array; - }; - /** Object containing create permissions for specific apps */ - c?: Record }>; - /** Object containing read permissions for specific apps */ - r?: Record }>; - /** Object containing update permissions for specific apps */ - u?: Record }>; - /** Object containing delete permissions for specific apps */ - d?: Record }>; - }; - /** Object containing event collections with replaced app names */ - eventList: Record; - /** Object containing view collections with replaced app names */ - viewList: Record; - }; - /** Time object for the request */ - time: TimeObject; - /** Hash of the request data */ - request_hash: string; - /** ID of the user's previous session */ - previous_session?: string; - /** Start timestamp of the user's previous session */ - previous_session_start?: number; - /** Unique ID for this request */ - request_id: string; - /** ID of the user making the request */ - user_id?: string; - /** URL encoded form data */ - formDataUrl?: string; - /** Flag indicating if this is a retry of a failed request */ - retry_request?: boolean; - /** Flag indicating if this request is from the populator */ - populator?: boolean; - /** Parsed URL parts */ - urlParts: { - /** Parsed query string as key-value pairs */ - query: Record; - /** The URL path */ - pathname: string; - /** The full URL */ - href: string; - }; - /** The URL path split into segments */ - paths: Array; - /** Callback function to handle the API response */ - output?: (response: any) => void; - /** If false, return immediately and do not wait for plugin chain execution to complete */ - waitForResponse?: boolean; - /** Name of the app */ - app_name?: string; - /** [truncateEventValuesList=false] Flag indicating whether to truncate event values list */ - truncateEventValuesList?: boolean; - /** Total session duration */ - session_duration?: number; - /** [is_os_processed=false] Flag indicating if OS and OS version have been processed */ - is_os_processed?: boolean; - /** Processed metrics data */ - processed_metrics?: Record; - /** Object with date IDs for different time periods */ - dbDateIds?: Record; - /** [dataTransformed=false] Flag indicating if the data is already transformed */ - dataTransformed?: boolean; - response?: { - /** HTTP response code */ - code: number; - /** Response body */ - body: string; - }; - /** Default value for metrics */ - defaultValue?: number; - - /** Allow any additional properties for legacy compatibility */ - [key: string]: any; -} - -/** Bulk request structure */ -export interface BulkRequest { - [key: string]: any; -} - -/** API validation functions */ -export interface ValidationFunctions { - validateAppForWriteAPI: (params: Params, callback: Function, callbackParam?: any) => Promise; - validateUserForDataReadAPI: (params: Params, feature: string | string[], callback: Function, callbackParam?: any) => Promise; - validateUserForDataWriteAPI: (params: Params, callback: Function, callbackParam?: any) => Promise; - validateUserForGlobalAdmin: (params: Params, callback: Function, callbackParam?: any) => Promise; -} - -/** Version mark entry */ -export interface VersionMark { - version: string; - timestamp: number; - [key: string]: any; -} - -/** Countly API modules structure */ -export interface CountlyAPI { - data: { - usage: any; - fetch: any; - events: any; - exports: any; - geoData: any; - }; - mgmt: { - users: any; - apps: any; - appUsers: any; - eventGroups: any; - cms: any; - datePresets: any; - }; -} - -/** - * Default request processing handler, which requires request context to operate - * @param params - Request context parameters with minimum required properties - * @returns void - * @example - * // Creating request context - * var params = { - * // providing data in request object - * 'req': {"url": "/i", "body": {"device_id": "test", "app_key": "APP_KEY", "begin_session": 1, "metrics": {}}}, - * // adding custom processing for API responses - * 'APICallback': function(err, data, headers, returnCode, params) { - * // handling api response, like sending to client or verifying - * if (err) { - * // there was problem processing request - * console.log(data, returnCode); - * } - * else { - * // request was processed, let's handle response data - * handle(data); - * } - * } - * }; - * - * // processing request - * processRequest(params); - */ -export declare function processRequest(params: Params): void; - -/** - * Process user data and app_user document with retry logic - * @param params - Request context parameters - * @param initiator - The initiating component or identifier - * @param done - Callback function when processing is complete - * @param try_times - Number of retry attempts (optional, defaults to appropriate value) - * @returns Promise that resolves when user processing is complete - */ -export declare function processUserFunction( - params: Params, - initiator: any, - done: (error?: any) => void, - try_times?: number -): Promise; - -/** Request processor module exports */ -declare const requestProcessor: { - processRequest: typeof processRequest; - processUserFunction: typeof processUserFunction; -}; - -export default requestProcessor; +import { ServerResponse, IncomingMessage } from "http"; +import { ObjectId } from "mongodb"; +import { TimeObject } from "./common"; + +// this is required because of the name conflict with Params here and the one +// inside express-serve-static-core. +type CountlyParams = Params; +declare module 'express-serve-static-core' { + export interface Request { + countlyParams: CountlyParams; + } +} + +/** Application user interface representing a document from the app_users collection */ +export interface AppUser { + /** MongoDB document ID */ + _id?: string; + /** Application user ID */ + uid: string; + /** Device ID */ + did: string; + /** User's country */ + country: string; + /** User's city */ + city: string; + /** User's timezone offset (in minutes) */ + tz: number; + /** Custom properties for the application user */ + custom?: Record; + /** Last session timestamp of the app user */ + ls?: object; + /** Last session ID */ + lsid?: string; + /** Flag indicating if the user has an ongoing session */ + has_ongoing_session?: boolean; + /** Timestamp of the user's last request */ + last_req?: number; + /** Last GET request URL */ + last_req_get?: string; + /** Last POST request body */ + last_req_post?: string; + /** First access timestamp */ + fac?: number; + /** First seen timestamp */ + fs?: number; + /** Last access timestamp */ + lac?: number; + /** SDK information */ + sdk?: { + /** SDK name */ + name?: string; + /** SDK version */ + version?: string; + }; + /** Request count */ + req_count?: number; + /** Type or token identifier */ + t?: string | number; + /** Flag indicating if the user has information */ + hasInfo?: boolean; + /** Information about merged users */ + merges?: any; +} + +/** Main request processing object containing all information shared through all the parts of the same request */ +export interface Params { + /** full URL href */ + href: string; + /** The HTTP response object */ + res: ServerResponse; + /** The HTTP request object */ + req: IncomingMessage; + /** API output handler. Which should handle API response */ + APICallback: (error: boolean, response: any, headers?: any, returnCode?: number, params?: Params) => void; + /** all the passed fields either through query string in GET requests or body and query string for POST requests */ + qstring: Record; + /** two top level url path, for example /i/analytics, first two segments from the fullPath */ + apiPath: string; + /** full url path, for example /i/analytics/dashboards */ + fullPath: string; + /** object with uploaded files, available in POST requests which upload files */ + files: { + app_image?: { + /** The temporary path of the uploaded app image file */ + path: string; + /** The original name of the uploaded app image file */ + name: string; + /** The MIME type of the uploaded app image file */ + type: string; + /** The size (in bytes) of the uploaded app image file */ + size: number; + }; + }; + /** Used for skipping SDK requests, if contains true, then request should be ignored and not processed. Can be set at any time by any plugin, but API only checks for it in beggining after / and /sdk events, so that is when plugins should set it if needed. Should contain reason for request cancelation */ + cancelRequest?: string; + /** [blockResponses=false] Flag to block responses from being sent */ + blockResponses?: boolean; + /** [forceProcessingRequestTimeout=false] Flag to force processing request timeout */ + forceProcessingRequestTimeout?: boolean; + /** True if this SDK request is processed from the bulk method */ + bulk?: boolean; + /** Array of the promises by different events. When all promises are fulfilled, request counts as processed */ + promises: Array>; + /** IP address of the device submitted request, exists in all SDK requests */ + ip_address: string; + /** Data with some user info, like country geolocation, etc from the request, exists in all SDK requests */ + user: { + /** User's country */ + country: string; + /** User's city */ + city: string; + /** User's timezone offset (in minutes) */ + tz?: number; + }; + /** Document from the app_users collection for current user, exists in all SDK requests after validation */ + app_user: AppUser; + /** ID of app_users document for the user, exists in all SDK requests after validation */ + app_user_id: string; + /** Document for the app sending request, exists in all SDK requests after validation and after validateUserForDataReadAPI validation */ + app: { + /** ID of the app document */ + _id: string; + /** Name of the app */ + name: string; + /** Country of the app */ + country: string; + /** Category of the app */ + category: string; + /** Timezone of the app */ + timezone: string; + /** Type of the app */ + type: string; + /** Flag indicating if the app is locked */ + locked: boolean; + /** Plugin-specific configuration for the app */ + plugins: Record; + }; + /** ObjectID of the app document, available after validation */ + app_id: ObjectId; + /** Selected app country, available after validation */ + app_cc: string; + /** Selected app timezone, available after validation */ + appTimezone: string; + /** All data about dashboard user sending the request, exists on all requests containing api_key, after validation through validation methods */ + member: { + /** ID of the dashboard user */ + _id: string; + /** Flag indicating if the user has global admin rights */ + global_admin: boolean; + /** The authentication token for the user */ + auth_token: string; + /** Flag indicating if the user is locked */ + locked: boolean; + /** Array of app IDs the user is an admin of (legacy) */ + admin_of?: Array; + /** Array of app IDs the user has user access to (legacy) */ + user_of?: Array; + /** Username of the dashboard user */ + username: string; + /** Email address of the dashboard user */ + email: string; + /** Full name of the dashboard user */ + full_name: string; + /** API key of the dashboard user */ + api_key: string; + /** Object containing user's access permissions */ + permission: { + _: { + /** Array of arrays containing app IDs the user has user access to */ + u: Array>; + /** Array of app IDs the user has admin access to */ + a: Array; + }; + /** Object containing create permissions for specific apps */ + c?: Record }>; + /** Object containing read permissions for specific apps */ + r?: Record }>; + /** Object containing update permissions for specific apps */ + u?: Record }>; + /** Object containing delete permissions for specific apps */ + d?: Record }>; + }; + /** Object containing event collections with replaced app names */ + eventList: Record; + /** Object containing view collections with replaced app names */ + viewList: Record; + }; + /** Time object for the request */ + time: TimeObject; + /** Hash of the request data */ + request_hash: string; + /** ID of the user's previous session */ + previous_session?: string; + /** Start timestamp of the user's previous session */ + previous_session_start?: number; + /** Unique ID for this request */ + request_id: string; + /** ID of the user making the request */ + user_id?: string; + /** URL encoded form data */ + formDataUrl?: string; + /** Flag indicating if this is a retry of a failed request */ + retry_request?: boolean; + /** Flag indicating if this request is from the populator */ + populator?: boolean; + /** Parsed URL parts */ + urlParts: { + /** Parsed query string as key-value pairs */ + query: Record; + /** The URL path */ + pathname: string; + /** The full URL */ + href: string; + }; + /** The URL path split into segments */ + paths: Array; + /** Callback function to handle the API response */ + output?: (response: any) => void; + /** If false, return immediately and do not wait for plugin chain execution to complete */ + waitForResponse?: boolean; + /** Name of the app */ + app_name?: string; + /** [truncateEventValuesList=false] Flag indicating whether to truncate event values list */ + truncateEventValuesList?: boolean; + /** Total session duration */ + session_duration?: number; + /** [is_os_processed=false] Flag indicating if OS and OS version have been processed */ + is_os_processed?: boolean; + /** Processed metrics data */ + processed_metrics?: Record; + /** Object with date IDs for different time periods */ + dbDateIds?: Record; + /** [dataTransformed=false] Flag indicating if the data is already transformed */ + dataTransformed?: boolean; + response?: { + /** HTTP response code */ + code: number; + /** Response body */ + body: string; + }; + /** Default value for metrics */ + defaultValue?: number; + + /** Allow any additional properties for legacy compatibility */ + [key: string]: any; +} + +/** Bulk request structure */ +export interface BulkRequest { + [key: string]: any; +} + +/** API validation functions */ +export interface ValidationFunctions { + validateAppForWriteAPI: (params: Params, callback: Function, callbackParam?: any) => Promise; + validateUserForDataReadAPI: (params: Params, feature: string | string[], callback: Function, callbackParam?: any) => Promise; + validateUserForDataWriteAPI: (params: Params, callback: Function, callbackParam?: any) => Promise; + validateUserForGlobalAdmin: (params: Params, callback: Function, callbackParam?: any) => Promise; +} + +/** Version mark entry */ +export interface VersionMark { + version: string; + timestamp: number; + [key: string]: any; +} + +/** Countly API modules structure */ +export interface CountlyAPI { + data: { + usage: any; + fetch: any; + events: any; + exports: any; + geoData: any; + }; + mgmt: { + users: any; + apps: any; + appUsers: any; + eventGroups: any; + cms: any; + datePresets: any; + }; +} + +/** + * Default request processing handler, which requires request context to operate + * @param params - Request context parameters with minimum required properties + * @returns void + * @example + * // Creating request context + * var params = { + * // providing data in request object + * 'req': {"url": "/i", "body": {"device_id": "test", "app_key": "APP_KEY", "begin_session": 1, "metrics": {}}}, + * // adding custom processing for API responses + * 'APICallback': function(err, data, headers, returnCode, params) { + * // handling api response, like sending to client or verifying + * if (err) { + * // there was problem processing request + * console.log(data, returnCode); + * } + * else { + * // request was processed, let's handle response data + * handle(data); + * } + * } + * }; + * + * // processing request + * processRequest(params); + */ +export declare function processRequest(params: Params): void; + +/** + * Process user data and app_user document with retry logic + * @param params - Request context parameters + * @param initiator - The initiating component or identifier + * @param done - Callback function when processing is complete + * @param try_times - Number of retry attempts (optional, defaults to appropriate value) + * @returns Promise that resolves when user processing is complete + */ +export declare function processUserFunction( + params: Params, + initiator: any, + done: (error?: any) => void, + try_times?: number +): Promise; + +/** Request processor module exports */ +declare const requestProcessor: { + processRequest: typeof processRequest; + processUserFunction: typeof processUserFunction; +}; + +export default requestProcessor;