Skip to content

Commit 37ff8dc

Browse files
authored
Merge pull request #8 from Contrast-Security-OSS/NODE-3593-real-app
Node 3593 real app & Node 3594 basic reporter
2 parents cd3417b + 9fa1c31 commit 37ff8dc

File tree

14 files changed

+8850
-1669
lines changed

14 files changed

+8850
-1669
lines changed

.gitignore

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,5 @@
11
node_modules
22
.env
3+
.contrast
4+
.swc
5+
__pycache__/

README.md

Lines changed: 22 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,8 +15,28 @@ For more information on how to this works with other frontends/backends, head ov
1515
# Getting started
1616

1717
1. install npm
18-
1. Run `npm install` in the project folder
19-
1. Run `npm run dev` for dev mode and `npm run start` for regualr mode
18+
1. make sure mongo is running
19+
1. define needed env vars
20+
- `DATABASE_URI` - the uri to the mongo database: mongodb://127.0.0.1:27017/somedbname (assuming mongo is running on localhost:27017)
21+
- `ACCESS_TOKEN_SECRET` - the secret used for the JWT
22+
1. execute `node api/index.js`
23+
- e.g., `ACCESS_TOKEN_SECRET=xyzzy-plover-boom DATABASE_URI=mongodb://127.0.0.1:27017/test node api/index.js`
24+
25+
## Contrast-specific
26+
27+
1. A contrast_security.yaml config file should be present and configured appropriately.
28+
1. The contrast agent should be installed as a dependency.
29+
1. For developmental testing, linking to the local node-mono repo is useful.
30+
1. To enable perf use a command line like: `CSI_PERF_INTERVAL=10000 CSI_PERF=1 ACCESS_TOKEN_SECRET=xyzzy-plover-boom DATABASE_URI=mongodb://127.0.0.1:27017/somedb node --import @contrast/agent api/index.js`
31+
1. loads the agent with perf enabled, using a 10 second interval for writing the log.
32+
1. set up `locust` per instructions in the `script-locust/README.md`
33+
1. run the request-generating script, `script-locust/locustfile.py` using `locust -f script-locust/locustfile.py --headless -i 1`.
34+
1. `--headless` just means don't use the web UI, i.e., pure command line
35+
1. `-f` specifies the file (more TBD, exercising different aspects of the code)
36+
1. `-i 1` specifies 1 iteration.
37+
1. the agent writes `agent-perf.jsonl`
38+
1. `agent-perf.jsonl` can be analyzed using tools in `script-analysis/`.
39+
1. `summarize.mjs` will summarize the data. it's primitive, but provides basic data.
2040

2141
# How it works
2242

models/Article.js

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -71,7 +71,7 @@ articleSchema.methods.toArticleResponse = async function (user) {
7171
tagList: this.tagList,
7272
favorited: user ? user.isFavourite(this._id) : false,
7373
favoritesCount: this.favouritesCount,
74-
author: authorObj.toProfileJSON(user)
74+
author: authorObj?.toProfileJSON(user) || 'anon',
7575
}
7676
}
7777

package-lock.json

Lines changed: 5454 additions & 1663 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

package.json

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -12,16 +12,23 @@
1212
"author": "",
1313
"license": "ISC",
1414
"dependencies": {
15+
"@aws-sdk/credential-providers": "3.654",
16+
"@contrast/agent": "^5.21.0",
1517
"bcrypt": "^5.1.0",
18+
"braces": "^3.0.3",
1619
"cookie-parser": "^1.4.6",
1720
"cors": "^2.8.5",
1821
"dotenv": "^16.0.3",
1922
"express": "^4.21.0",
2023
"express-async-handler": "^1.2.0",
21-
"jsonwebtoken": "^8.5.1",
22-
"mongoose": "^6.8.1",
24+
"fast-xml-parser": "4.4",
25+
"jsonwebtoken": "^9.0.2",
26+
"mongodb": "4.17",
27+
"mongoose": "6.13",
2328
"mongoose-unique-validator": "^3.1.0",
24-
"slugify": "^1.6.5"
29+
"newman": "^6.2.1",
30+
"slugify": "^1.6.5",
31+
"tar": "6.2"
2532
},
2633
"devDependencies": {
2734
"nodemon": "^2.0.20"

script-analysis/summarize.mjs

Lines changed: 166 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,166 @@
1+
2+
import fs from 'fs';
3+
4+
//
5+
// the only command line option is to supply the filename.
6+
//
7+
// this is a work in progress.
8+
//
9+
// definitions:
10+
// - wrapper - the code that the agent wraps a patched function with
11+
// - original - the code that the agent is wrapping, e.g., String.prototype.concat
12+
// - delta - the total difference, for all invocations, between the time taken
13+
// to execute the wrapper and the time taken to execute the original
14+
// - deltaPer - delta divided by number of invocations
15+
// - ratio - the ratio of the wrapper time to the original time.
16+
//
17+
// set env var SORT to `delta`, `ratio`, or `deltaPer` to sort the output
18+
// set env var VERBOSE to 1 to see skipped items.
19+
//
20+
const prefixHandlers = {
21+
'contrast-ui-reporter': reporter,
22+
'patcher': patcher,
23+
};
24+
25+
const kRunOrig = ':native:runOriginalFunction';
26+
const kRunOrigLen = kRunOrig.length;
27+
const kWrapper = ':wrapper';
28+
const kWrapperLen = kWrapper.length;
29+
const kPost = ':post';
30+
const kPostLen = kPost.length;
31+
const kPut = ':put';
32+
const kPutLen = kPut.length;
33+
const kGet = ':get';
34+
const kGetLen = kGet.length;
35+
36+
const verbose = process.env.VERBOSE === '1';
37+
38+
const filename = process.argv[2] || 'agent-perf.jsonl';
39+
40+
let json = fs.readFileSync(filename, 'utf-8');
41+
42+
json = json.split('\n').slice(0, -1).map(JSON.parse);
43+
44+
let lastRequests = 0;
45+
46+
for (let i = 0; i < json.length; i++) {
47+
let { timestamp, requests, prefix, measurements } = json[i];
48+
49+
if (!(prefix in prefixHandlers)) {
50+
verbose && console.log(`skipping ${timestamp} unknown prefix ${prefix}`);
51+
continue;
52+
}
53+
const prefixLen = prefix.length + 1;
54+
55+
if (requests === lastRequests) {
56+
verbose && console.log(`skipping ${timestamp} no new requests`);
57+
continue;
58+
}
59+
lastRequests = requests;
60+
61+
62+
const summarized = [];
63+
const unified = {};
64+
for (const measurement of measurements) {
65+
const { tag, n, totalMicros, mean } = measurement;
66+
if (tag.endsWith(kRunOrig)) {
67+
const unifiedTag = tag.slice(prefixLen, -kRunOrigLen);
68+
if (unifiedTag in unified) {
69+
throw new Error(`wrapper came first1 ${timestamp} ${unifiedTag}`);
70+
// merge, but this should never happen because the native
71+
// function should always complete before the wrapper.
72+
} else {
73+
unified[unifiedTag] = { tag: unifiedTag, n, nativeMicros: totalMicros, nativeMean: mean };
74+
}
75+
} else if (tag.endsWith(kWrapper)) {
76+
const unifiedTag = tag.slice(prefixLen, -kWrapperLen);
77+
if (unifiedTag in unified) {
78+
unified[unifiedTag].wrapperMicros = totalMicros;
79+
unified[unifiedTag].wrapperMean = mean;
80+
81+
const ratio = totalMicros / unified[unifiedTag].nativeMicros;
82+
const delta = totalMicros - unified[unifiedTag].nativeMicros;
83+
84+
unified[unifiedTag].ratio = ratio;
85+
unified[unifiedTag].delta = delta;
86+
87+
summarized.push(unified[unifiedTag]);
88+
} else {
89+
throw new Error(`wrapper came first2 ${timestamp} ${unifiedTag}`);
90+
// this should never happen either.
91+
unified[unifiedTag] = { n, wrapperMicros: totalMicros, wrapperMean: mean };
92+
}
93+
} else if (tag.endsWith(kPost)) {
94+
const unifiedTag = tag.slice(prefixLen, -kPostLen);
95+
96+
unified[unifiedTag] = { n, wrapperMicros: totalMicros, wrapperMean: mean };
97+
unified[unifiedTag].tag = unifiedTag + ' post';
98+
99+
summarized.push(unified[unifiedTag]);
100+
} else if (tag.endsWith(kPut)) {
101+
const unifiedTag = tag.slice(prefixLen, -kPutLen);
102+
103+
unified[unifiedTag] = { n, wrapperMicros: totalMicros, wrapperMean: mean };
104+
unified[unifiedTag].tag = unifiedTag + ' put';
105+
106+
summarized.push(unified[unifiedTag]);
107+
} else if (tag.endsWith(kGet)) {
108+
const unifiedTag = tag.slice(prefixLen, -kGetLen);
109+
110+
unified[unifiedTag] = { n, wrapperMicros: totalMicros, wrapperMean: mean };
111+
unified[unifiedTag].tag = unifiedTag + ' get';
112+
113+
summarized.push(unified[unifiedTag]);
114+
}
115+
116+
}
117+
118+
if (summarized.length) {
119+
if (process.env.SORT === 'delta') {
120+
summarized.sort(sortDelta);
121+
} else if (process.env.SORT === 'ratio') {
122+
summarized.sort(sortRatio);
123+
} else if (process.env.SORT === 'deltaPer') {
124+
summarized.sort(sortDeltaPer);
125+
}
126+
127+
let deltaPerReq = 0;
128+
129+
console.log(`\n${timestamp}`);
130+
for (const { tag, n, nativeMicros, nativeMean, wrapperMicros, wrapperMean, ratio, delta } of summarized) {
131+
const raw = `(raw w ${wrapperMean}, o ${nativeMean})`;
132+
if (ratio) {
133+
console.log(`${tag} ${n} ratio ${f2(ratio)} delta ${f2(delta)} deltaPer ${f2(delta / n)} ${raw}`);
134+
} else {
135+
const raw = `(raw w ${wrapperMean}, total ${wrapperMicros})`;
136+
console.log(`${tag} ${n} ${raw}`);
137+
}
138+
// total number of invocations / requests => invocations/request
139+
// invocations/request * deltaPer => deltaPerReq
140+
const deltaPer = delta / n;
141+
deltaPerReq += (n / requests) * deltaPer;
142+
// console.log(`${tag} ${n} ${f2(nativeMicros)} ${f2(nativeMean)} ${f2(wrapperMicros)} ${f2(wrapperMean)} ${f2(ratio)} ${f2(delta)}`);
143+
}
144+
145+
if (deltaPerReq) console.log(`deltaPerReq ${f2(deltaPerReq)}`);
146+
147+
break;
148+
}
149+
150+
}
151+
152+
function f2(x) {
153+
return x.toFixed(2);
154+
}
155+
156+
function sortRatio(a, b) {
157+
return b.ratio - a.ratio;
158+
}
159+
160+
function sortDelta(a, b) {
161+
return b.delta - a.delta;
162+
}
163+
164+
function sortDeltaPer(a, b) {
165+
return b.delta / b.n - a.delta / a.n;
166+
}

script-locust/README.md

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
# locust
2+
3+
`locust` is used to generate a specific, reproducible load against the server. It does not use random timing or random data.
4+
5+
## Setting up locust
6+
7+
`locust` is a python program, so install a recent version of python. Development was done with version 3.10.12.
8+
9+
1. Install `locust` using `pip3 install locust`
10+
1. Install `locust-plugins` using `pip3 install locust-plugins` (used for `-i`)
11+
12+
## Writing locustfiles
13+
14+
https://docs.locust.io/en/stable/what-is-locust.html

script-locust/data/__init__.py

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
import data.articles as articles
2+
3+
class UserData:
4+
_user_seq = 0
5+
6+
def __init__(self):
7+
self._article_seq = 0
8+
9+
UserData._user_seq += 1
10+
self._user_seq = f"{UserData._user_seq:02d}"
11+
12+
self.username = self.make_username()
13+
self.email = self.make_email()
14+
self.password = "password-might-be-long-enough"
15+
16+
def get_user(self):
17+
return {
18+
"email": self.email,
19+
"password": "password-might-be-long-enough",
20+
"username": self.username,
21+
}
22+
23+
def make_email(self):
24+
return f"{self.make_username()}@abysmal.com"
25+
26+
def make_username(self, name = "john.q.customer"):
27+
return f"{name}-{self._user_seq}"
28+
29+
def get_articles(self, n = 1, paragraphs = 1):
30+
unique_articles = articles.get_articles(n, paragraphs)
31+
# make the title unique by combining user and user-specific sequence
32+
for article in unique_articles:
33+
self._article_seq += 1
34+
title = article["title"]
35+
article["title"] = f"{title} by {self.username} ({self._article_seq})"
36+
37+
return unique_articles

0 commit comments

Comments
 (0)