Skip to content

Commit b0d1a1b

Browse files
mstangemstange
committed
Add merge-android-profiles script
1 parent ddd385e commit b0d1a1b

File tree

3 files changed

+310
-0
lines changed

3 files changed

+310
-0
lines changed

‎package.json‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,8 @@
1919
"build-l10n-prod:quiet": "yarn build:clean && yarn build-photon && cross-env NODE_ENV=production L10N=1 webpack",
2020
"build-l10n-prod": "yarn build-l10n-prod:quiet --progress",
2121
"build-photon": "webpack --config res/photon/webpack.config.js",
22+
"build-merge-android-profiles": "yarn build-merge-android-profiles:quiet --progress",
23+
"build-merge-android-profiles:quiet": "yarn build:clean && cross-env NODE_ENV=production webpack --config src/merge-android-profiles/webpack.config.js",
2224
"build-symbolicator-cli": "yarn build-symbolicator-cli:quiet --progress",
2325
"build-symbolicator-cli:quiet": "yarn build:clean && cross-env NODE_ENV=production webpack --config src/symbolicator-cli/webpack.config.js",
2426
"lint": "node bin/output-fixing-commands.js run-p lint-js lint-css prettier-run",

‎src/merge-android-profiles/index.ts‎

Lines changed: 266 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,266 @@
1+
/**
2+
* Merge two existing profiles, taking the samples from the first profile and
3+
* the markers from the second profile.
4+
*
5+
* This was useful during early 2025 when the Mozilla Performance team was
6+
* doing a lot of Android startup profiling:
7+
*
8+
* - The "samples" profile would be collected using simpleperf and converted
9+
* with samply import.
10+
* - The "markers" profile would be collected using the Gecko profiler.
11+
*
12+
* To use this script, it first needs to be built:
13+
* yarn build-merge-android-profiles
14+
*
15+
* Then it can be run from the `dist` directory:
16+
* node dist/merge-android-profiles.js --samples-hash warg8azfac0z5b5sy92h4a69bfrj2fqsjc6ty58 --markers-hash mb6220c2rx3mmhegv82d84tsvgn6a5p8r7g4je8 --output-file ~/Downloads/merged-profile.json
17+
*
18+
* For example:
19+
* yarn build-merge-android-profiles && node dist/merge-android-profiles.js --samples-hash warg8azfac0z5b5sy92h4a69bfrj2fqsjc6ty58 --markers-hash mb6220c2rx3mmhegv82d84tsvgn6a5p8r7g4je8 --output-file ~/Downloads/merged-profile.json
20+
*
21+
*/
22+
23+
const fs = require('fs');
24+
25+
import {
26+
unserializeProfileOfArbitraryFormat,
27+
adjustMarkerTimestamps,
28+
} from '../profile-logic/process-profile';
29+
import { getProfileUrlForHash } from '../actions/receive-profile';
30+
import { computeStringIndexMarkerFieldsByDataType } from '../profile-logic/marker-schema';
31+
import { ensureExists } from '../utils/flow';
32+
import { StringTable } from '../utils/string-table';
33+
34+
import type { Profile, RawThread, Tid } from '../types/profile';
35+
import { compress } from 'firefox-profiler/utils/gz';
36+
37+
type ProfileSource =
38+
| {
39+
type: 'HASH';
40+
hash: string;
41+
}
42+
| {
43+
type: 'FILE';
44+
file: string;
45+
};
46+
47+
interface CliOptions {
48+
samplesProf: ProfileSource;
49+
markersProf: ProfileSource;
50+
filterByProcessPrefix: string | undefined;
51+
outputFile: string;
52+
}
53+
54+
async function fetchProfileWithHash(hash: string): Promise<Profile> {
55+
const response = await fetch(getProfileUrlForHash(hash));
56+
const serializedProfile = await response.json();
57+
return unserializeProfileOfArbitraryFormat(serializedProfile);
58+
}
59+
60+
async function loadProfileFromFile(path: string): Promise<Profile> {
61+
const uint8Array = fs.readFileSync(path, null);
62+
return unserializeProfileOfArbitraryFormat(uint8Array.buffer);
63+
}
64+
65+
async function loadProfile(source: ProfileSource): Promise<Profile> {
66+
switch (source.type) {
67+
case 'HASH':
68+
return fetchProfileWithHash(source.hash);
69+
case 'FILE':
70+
return loadProfileFromFile(source.file);
71+
default:
72+
return source;
73+
}
74+
}
75+
76+
export async function run(options: CliOptions) {
77+
const profileWithSamples: Profile = await loadProfile(options.samplesProf);
78+
const profileWithMarkers: Profile = await loadProfile(options.markersProf);
79+
80+
// const referenceSampleTime = 169912951.547432; // filteredThread.samples.time[0] after zooming in on samples in mozilla::dom::indexedDB::BackgroundTransactionChild::RecvComplete
81+
// const referenceMarkerTime = 664.370158 ; // selectedMarker.start after selecting the marker for the "complete" DOMEvent
82+
83+
// console.log(profileWithSamples.meta);
84+
// console.log(profileWithMarkers.meta);
85+
86+
let timeDelta =
87+
profileWithMarkers.meta.startTime - profileWithSamples.meta.startTime;
88+
if (
89+
profileWithSamples.meta.startTimeAsClockMonotonicNanosecondsSinceBoot !==
90+
undefined &&
91+
profileWithMarkers.meta.startTimeAsClockMonotonicNanosecondsSinceBoot !==
92+
undefined
93+
) {
94+
timeDelta =
95+
(profileWithMarkers.meta.startTimeAsClockMonotonicNanosecondsSinceBoot -
96+
profileWithSamples.meta.startTimeAsClockMonotonicNanosecondsSinceBoot) /
97+
1000000;
98+
}
99+
100+
// console.log({ timeDelta });
101+
102+
const profile = profileWithSamples;
103+
profile.meta.markerSchema = profileWithMarkers.meta.markerSchema;
104+
profile.pages = profileWithMarkers.pages;
105+
106+
const markerProfileCategoryToCategory = new Map();
107+
const markerProfileCategories = ensureExists(
108+
profileWithMarkers.meta.categories
109+
);
110+
const profileCategories = ensureExists(profile.meta.categories);
111+
for (
112+
let markerCategoryIndex = 0;
113+
markerCategoryIndex < markerProfileCategories.length;
114+
markerCategoryIndex++
115+
) {
116+
const category = markerProfileCategories[markerCategoryIndex];
117+
let categoryIndex = profileCategories.findIndex(
118+
(c) => c.name === category.name
119+
);
120+
if (categoryIndex === -1) {
121+
categoryIndex = profileCategories.length;
122+
profileCategories[categoryIndex] = {
123+
name: category.name,
124+
color: category.color,
125+
subcategories: ['Other'],
126+
};
127+
}
128+
markerProfileCategoryToCategory.set(markerCategoryIndex, categoryIndex);
129+
}
130+
131+
const markerThreadsByTid = new Map<Tid, RawThread>(
132+
profileWithMarkers.threads.map((thread) => ['' + thread.tid, thread])
133+
);
134+
// console.log([...markerThreadsByTid.keys()]);
135+
136+
// console.log(profile.threads.map((thread) => thread.tid));
137+
138+
const stringIndexMarkerFieldsByDataType =
139+
computeStringIndexMarkerFieldsByDataType(profile.meta.markerSchema);
140+
141+
const sampleThreadTidsWithoutCorrespondingMarkerThreads = new Set();
142+
143+
const stringTable = StringTable.withBackingArray(profile.shared.stringArray);
144+
const markerStringArray = profileWithMarkers.shared.stringArray;
145+
const keptThreads = [];
146+
for (const thread of profile.threads) {
147+
if (options.filterByProcessPrefix !== undefined) {
148+
if (!thread.processName!.startsWith(options.filterByProcessPrefix)) {
149+
continue;
150+
}
151+
}
152+
keptThreads.push(thread);
153+
const tid = thread.tid;
154+
const markerThread = markerThreadsByTid.get(tid);
155+
if (markerThread === undefined) {
156+
sampleThreadTidsWithoutCorrespondingMarkerThreads.add(tid);
157+
continue;
158+
}
159+
markerThreadsByTid.delete(tid);
160+
161+
thread.markers = adjustMarkerTimestamps(markerThread.markers, timeDelta);
162+
for (let i = 0; i < thread.markers.length; i++) {
163+
thread.markers.category[i] = ensureExists(
164+
markerProfileCategoryToCategory.get(thread.markers.category[i])
165+
);
166+
thread.markers.name[i] = stringTable.indexForString(
167+
markerStringArray[thread.markers.name[i]]
168+
);
169+
const data = thread.markers.data[i];
170+
if (data !== null && data.type) {
171+
const markerType = data.type;
172+
const stringIndexMarkerFields =
173+
stringIndexMarkerFieldsByDataType.get(markerType);
174+
if (stringIndexMarkerFields !== undefined) {
175+
for (const fieldKey of stringIndexMarkerFields) {
176+
const stringIndex = (data as any)[fieldKey];
177+
if (typeof stringIndex === 'number') {
178+
const newStringIndex = stringTable.indexForString(
179+
markerStringArray[stringIndex]
180+
);
181+
(data as any)[fieldKey] = newStringIndex;
182+
}
183+
}
184+
}
185+
}
186+
}
187+
}
188+
189+
profile.threads = keptThreads;
190+
191+
// console.log(
192+
// `Have ${markerThreadsByTid.size} marker threads left over which weren't slurped up by sample threads:`,
193+
// [...markerThreadsByTid.keys()]
194+
// );
195+
// if (markerThreadsByTid.size !== 0) {
196+
// console.log(
197+
// `Have ${sampleThreadTidsWithoutCorrespondingMarkerThreads.size} sample threads which didn't find corresponding marker threads:`,
198+
// [...sampleThreadTidsWithoutCorrespondingMarkerThreads]
199+
// );
200+
// }
201+
202+
if (options.outputFile.endsWith('.gz')) {
203+
fs.writeFileSync(
204+
options.outputFile,
205+
await compress(JSON.stringify(profile))
206+
);
207+
} else {
208+
fs.writeFileSync(options.outputFile, JSON.stringify(profile));
209+
}
210+
}
211+
212+
export function makeOptionsFromArgv(processArgv: string[]): CliOptions {
213+
const argv = require('minimist')(processArgv.slice(2));
214+
215+
const hasSamplesHash =
216+
'samples-hash' in argv && typeof argv['samples-hash'] === 'string';
217+
const hasSamplesFile =
218+
'samples-file' in argv && typeof argv['samples-file'] === 'string';
219+
const hasMarkersHash =
220+
'markers-hash' in argv && typeof argv['markers-hash'] === 'string';
221+
const hasMarkersFile =
222+
'markers-file' in argv && typeof argv['markers-file'] === 'string';
223+
224+
if (!hasSamplesHash && !hasSamplesFile) {
225+
throw new Error('Either --samples-file or --samples-hash must be supplied');
226+
}
227+
if (hasSamplesHash && hasSamplesFile) {
228+
throw new Error(
229+
'Only one of --samples-file or --samples-hash can be supplied'
230+
);
231+
}
232+
if (!hasMarkersHash && !hasMarkersFile) {
233+
throw new Error('Either --markers-file or --markers-hash must be supplied');
234+
}
235+
if (hasMarkersHash && hasMarkersFile) {
236+
throw new Error(
237+
'Only one of --markers-file or --markers-hash can be supplied'
238+
);
239+
}
240+
241+
const samplesProf: ProfileSource = hasSamplesHash
242+
? { type: 'HASH', hash: argv['samples-hash'] }
243+
: { type: 'FILE', file: argv['samples-file'] };
244+
const markersProf: ProfileSource = hasMarkersHash
245+
? { type: 'HASH', hash: argv['markers-hash'] }
246+
: { type: 'FILE', file: argv['markers-file'] };
247+
248+
return {
249+
samplesProf,
250+
markersProf,
251+
filterByProcessPrefix: argv['filter-by-process-prefix'],
252+
outputFile: argv['output-file'],
253+
};
254+
}
255+
256+
if (!module.parent) {
257+
try {
258+
const options = makeOptionsFromArgv(process.argv);
259+
run(options).catch((err) => {
260+
throw err;
261+
});
262+
} catch (e) {
263+
console.error(e);
264+
process.exit(1);
265+
}
266+
}

‎src/merge-android-profiles/webpack.config.js‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
const path = require('path');
2+
const projectRoot = path.join(__dirname, '../..');
3+
const includes = [path.join(projectRoot, 'src')];
4+
5+
module.exports = {
6+
name: 'merge-android-profiles',
7+
target: 'node',
8+
mode: process.env.NODE_ENV,
9+
resolve: {
10+
extensions: ['.js', '.jsx', '.ts', '.tsx'],
11+
alias: {
12+
'firefox-profiler-res': path.resolve(projectRoot, 'res'),
13+
},
14+
},
15+
output: {
16+
path: path.resolve(projectRoot, 'dist'),
17+
filename: 'merge-android-profiles.js',
18+
},
19+
entry: './src/merge-android-profiles/index.ts',
20+
module: {
21+
rules: [
22+
{
23+
test: /\.(ts|tsx)$/,
24+
use: ['babel-loader'],
25+
include: includes,
26+
},
27+
{
28+
test: /\.js$/,
29+
include: path.resolve(projectRoot, 'res'),
30+
type: 'asset/resource',
31+
},
32+
{
33+
test: /\.svg$/,
34+
type: 'asset/resource',
35+
},
36+
],
37+
},
38+
experiments: {
39+
// Make WebAssembly work just like in webpack v4
40+
syncWebAssembly: true,
41+
},
42+
};

0 commit comments

Comments
 (0)