Support reading profiles from JsonSlabs files

jest.config.js

@@ -18,7 +18,7 @@ const browserEnvConfig = {
   // Transform ESM modules to CommonJS for Jest
   // These packages ship as pure ESM and need to be transformed by Babel
   transformIgnorePatterns: [
-    '/node_modules/(?!(query-string|decode-uri-component|iongraph-web|split-on-first|filter-obj|fetch-mock|devtools-reps)/)',
+    '/node_modules/(?!(query-string|decode-uri-component|iongraph-web|split-on-first|filter-obj|fetch-mock|devtools-reps|json-slabs)/)',
   ],
 
   // Mock static assets (images, CSS, etc.)

package.json

@@ -91,6 +91,7 @@
     "gecko-profiler-demangle": "^0.4.0",
     "idb": "^8.0.3",
     "iongraph-web": "0.2.1",
+    "json-slabs": "^0.3.0",
     "jszip": "^3.10.1",
     "long": "^5.3.2",
     "memoize-immutable": "^3.0.0",

src/actions/publish.ts

@@ -60,7 +60,7 @@ import type {
   ProfileIndexTranslationMaps,
 } from 'firefox-profiler/types';
 import { compress } from 'firefox-profiler/utils/gz';
-import { serializeProfile } from 'firefox-profiler/profile-logic/process-profile';
+import { serializeProfileToJsonString } from 'firefox-profiler/profile-logic/process-profile';
 
 export function updateSharingOption(
   slug: keyof CheckedSharingOptions,
@@ -322,7 +322,9 @@ export function encodeSanitizedProfile(
     const encodingPromise: Promise<ProfileEncodingResult> = (async function () {
       try {
         dispatch(sanitizedProfileEncodingStarted(sanitizedProfile));
-        const gzipData = await compress(serializeProfile(sanitizedProfile));
+        const gzipData = await compress(
+          serializeProfileToJsonString(sanitizedProfile)
+        );
         const blob = new Blob([gzipData], { type: 'application/octet-binary' });
         dispatch(sanitizedProfileEncodingCompleted(sanitizedProfile, blob));
         return { type: 'SUCCESS', profileData: blob };

src/node-tools/profiler-edit.ts

@@ -4,7 +4,11 @@
 import fs from 'fs';
 import minimist from 'minimist';
 
-import { unserializeProfileOfArbitraryFormat } from 'firefox-profiler/profile-logic/process-profile';
+import {
+  serializeProfileToJsonSlabsFile,
+  serializeProfileToJsonString,
+  unserializeProfileOfArbitraryFormat,
+} from 'firefox-profiler/profile-logic/process-profile';
 import { computeCompactedProfile } from 'firefox-profiler/profile-logic/profile-compacting';
 import { GOOGLE_STORAGE_BUCKET } from 'firefox-profiler/app-logic/constants';
 import { compress } from 'firefox-profiler/utils/gz';
@@ -128,6 +132,24 @@ async function loadProfile(source: ProfileSource): Promise<Profile> {
   }
 }
 
+async function encodeProfileWithFilename(
+  profile: Profile,
+  filename: string
+): Promise<Uint8Array> {
+  if (filename.endsWith('.jslb') || filename.endsWith('.jslb.gz')) {
+    const bytes = serializeProfileToJsonSlabsFile(profile);
+    if (filename.endsWith('.jslb.gz')) {
+      return compress(bytes);
+    }
+    return bytes;
+  }
+  const s = serializeProfileToJsonString(profile);
+  if (filename.endsWith('.gz')) {
+    return compress(s);
+  }
+  return new TextEncoder().encode(s);
+}
+
 export async function run(options: CliOptions) {
   const profile = await loadProfile(options.input);
 
@@ -185,15 +207,13 @@ export async function run(options: CliOptions) {
 
   const { profile: compactedProfile } = computeCompactedProfile(profile);
 
-  console.log(`Saving profile to ${options.output}`);
-  if (options.output.endsWith('.gz')) {
-    fs.writeFileSync(
-      options.output,
-      await compress(JSON.stringify(compactedProfile))
-    );
-  } else {
-    fs.writeFileSync(options.output, JSON.stringify(compactedProfile));
-  }
+  const outputFilename = options.output;
+  console.log(`Saving profile to ${outputFilename}`);
+  const bytes = await encodeProfileWithFilename(
+    compactedProfile,
+    outputFilename
+  );
+  fs.writeFileSync(outputFilename, bytes);
   console.log('Finished.');
 }
 

src/profile-logic/process-profile.ts

@@ -1,6 +1,12 @@
 /* This Source Code Form is subject to the terms of the Mozilla Public
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+import {
+  isJsonSlabsFile,
+  decode as decodeJsonSlabs,
+  encode as encodeJsonSlabs,
+} from 'json-slabs';
+
 import { attemptToConvertChromeProfile } from './import/chrome';
 import { attemptToConvertDhat } from './import/dhat';
 import { GlobalDataCollector } from './global-data-collector';
@@ -1922,10 +1928,42 @@ export function processGeckoProfile(geckoProfile: GeckoProfile): Profile {
 /**
  * Take a processed profile and convert it to a string.
  */
-export function serializeProfile(profile: Profile): string {
+export function serializeProfileToJsonString(profile: Profile): string {
   return JSON.stringify(profile);
 }
 
+/**
+ * Take a profile and convert it to a Uint8Array in the JsonSlabs format.
+ *
+ * This is more efficient than JSON if the profile contains large typed arrays.
+ */
+export function serializeProfileToJsonSlabsFile(
+  profile: Profile
+): Uint8Array<ArrayBuffer> {
+  // Encode the profile object with the binary JsonSlabs container format.
+  return encodeJsonSlabs(profile, [
+    // "Split-out" slabs:
+    //
+    // This second argument to the encode function is an array of objects which
+    // should be pulled out into their own dedicated slabs. This is totally
+    // optional and doesn't change what the decoded object will look like.
+    // We use it to "split out" some large tables as long as we haven't converted
+    // them to use typed arrays. This already gives us a benefit: It means that
+    // decoding will use several JSON.parse calls rather than just one single
+    // JSON.parse call, and each individual JSON.parse will act on a smaller
+    // string, which means it's less likely to hit any string size limits.
+    //
+    // As we convert more and more tables / columns to typed arrays, the "skeleton
+    // JSON" for these tables will become much smaller and we won't need to split
+    // out those tables anymore.
+    profile.threads,
+    profile.shared.stackTable,
+    profile.shared.frameTable,
+    profile.shared.funcTable,
+    profile.shared.stringArray,
+  ]);
+}
+
 // If applicable, this function will try to "fix" a processed profile that was
 // copied from the console on an old version of the UI, where such a profile
 // would have a `stringTable` property rather than a `stringArray` property on
@@ -2062,7 +2100,9 @@ export async function unserializeProfileOfArbitraryFormat(
         profileBytes = await decompress(profileBytes);
       }
 
-      if (isArtTraceFormat(profileBytes)) {
+      if (isJsonSlabsFile(profileBytes)) {
+        arbitraryFormat = decodeJsonSlabs(profileBytes);
+      } else if (isArtTraceFormat(profileBytes)) {
         arbitraryFormat = convertArtTraceProfile(profileBytes);
       } else if (verifyMagic(SIMPLEPERF_MAGIC, profileBytes)) {
         const { convertSimpleperfTraceProfile } =

src/test/components/DragAndDrop.test.tsx

@@ -12,7 +12,7 @@ import {
   DragAndDropOverlay,
 } from '../../components/app/DragAndDrop';
 import { getProfileFromTextSamples } from '../fixtures/profiles/processed-profile';
-import { serializeProfile } from '../../profile-logic/process-profile';
+import { serializeProfileToJsonString } from '../../profile-logic/process-profile';
 import { getView } from 'firefox-profiler/selectors';
 import { updateBrowserConnectionStatus } from 'firefox-profiler/actions/app';
 import { mockWebChannel } from '../fixtures/mocks/web-channel';
@@ -100,9 +100,13 @@ describe('app/DragAndDrop', () => {
     const [dragAndDrop, overlay] = container.children;
 
     const { profile } = getProfileFromTextSamples('A');
-    const file = new File([serializeProfile(profile)], 'profile.json', {
-      type: 'application/json',
-    });
+    const file = new File(
+      [serializeProfileToJsonString(profile)],
+      'profile.json',
+      {
+        type: 'application/json',
+      }
+    );
     const files = [file];
 
     fireEvent.dragEnter(dragAndDrop);

src/test/fixtures/profiles/zip-file.ts

@@ -2,7 +2,7 @@
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 import { getProfileFromTextSamples } from '../../fixtures/profiles/processed-profile';
-import { serializeProfile } from '../../../profile-logic/process-profile';
+import { serializeProfileToJsonString } from '../../../profile-logic/process-profile';
 import { receiveZipFile } from '../../../actions/receive-profile';
 import { setDataSource } from '../../../actions/profile-view';
 import type {
@@ -17,7 +17,7 @@ import JSZip from 'jszip';
  */
 export function getZippedProfiles(files: string[] = []): JSZip {
   const { profile } = getProfileFromTextSamples('A');
-  const profileText = serializeProfile(profile);
+  const profileText = serializeProfileToJsonString(profile);
 
   const zip = new JSZip();
   files.forEach((fileName) => {

src/test/store/receive-profile.test.ts

@@ -36,7 +36,7 @@ import { createGeckoProfile } from '../fixtures/profiles/gecko-profile';
 import { blankStore, storeWithProfile } from '../fixtures/stores';
 import {
   processGeckoProfile,
-  serializeProfile,
+  serializeProfileToJsonString,
 } from '../../profile-logic/process-profile';
 import {
   getProfileFromTextSamples,
@@ -1044,7 +1044,7 @@ describe('actions/receive-profile', function () {
       const expectedUrl = 'https://profiles.club/shared.json';
       window.fetchMock.get(
         expectedUrl,
-        compress(serializeProfile(_getSimpleProfile()))
+        compress(serializeProfileToJsonString(_getSimpleProfile()))
       );
       const store = blankStore();
       await store.dispatch(retrieveProfileOrZipFromUrl(expectedUrl));
@@ -1174,7 +1174,7 @@ describe('actions/receive-profile', function () {
 
       const { getState, view } = await setupTestWithFile({
         type: 'application/json',
-        payload: serializeProfile(profile),
+        payload: serializeProfileToJsonString(profile),
       });
       expect(view.phase).toBe('DATA_LOADED');
       expect(ProfileViewSelectors.getProfile(getState()).meta.product).toEqual(
@@ -1194,7 +1194,7 @@ describe('actions/receive-profile', function () {
 
       const { getState, view } = await setupTestWithFile({
         type: 'application/json',
-        payload: JSON.stringify(profile), // Note: No serializeProfile call!
+        payload: JSON.stringify(profile), // Note: No serializeProfileToJsonString call!
       });
 
       expect(view.phase).toBe('DATA_LOADED');
@@ -1243,7 +1243,7 @@ describe('actions/receive-profile', function () {
 
       const { getState, view } = await setupTestWithFile({
         type: '',
-        payload: serializeProfile(profile),
+        payload: serializeProfileToJsonString(profile),
       });
       expect(view.phase).toBe('DATA_LOADED');
       expect(ProfileViewSelectors.getProfile(getState()).meta.product).toEqual(
@@ -1257,7 +1257,9 @@ describe('actions/receive-profile', function () {
 
       const { getState, view } = await setupTestWithFile({
         type: '',
-        payload: extractArrayBuffer(await compress(serializeProfile(profile))),
+        payload: extractArrayBuffer(
+          await compress(serializeProfileToJsonString(profile))
+        ),
       });
       expect(view.phase).toBe('DATA_LOADED');
       expect(ProfileViewSelectors.getProfile(getState()).meta.product).toEqual(
@@ -1289,7 +1291,9 @@ describe('actions/receive-profile', function () {
 
       const { getState, view } = await setupTestWithFile({
         type: 'application/gzip',
-        payload: extractArrayBuffer(await compress(serializeProfile(profile))),
+        payload: extractArrayBuffer(
+          await compress(serializeProfileToJsonString(profile))
+        ),
       });
       expect(view.phase).toBe('DATA_LOADED');
       expect(ProfileViewSelectors.getProfile(getState()).meta.product).toEqual(
@@ -1303,7 +1307,9 @@ describe('actions/receive-profile', function () {
 
       const { getState, view } = await setupTestWithFile({
         type: 'application/json',
-        payload: extractArrayBuffer(await compress(serializeProfile(profile))),
+        payload: extractArrayBuffer(
+          await compress(serializeProfileToJsonString(profile))
+        ),
       });
       expect(view.phase).toBe('DATA_LOADED');
       expect(ProfileViewSelectors.getProfile(getState()).meta.product).toEqual(
@@ -1345,7 +1351,7 @@ describe('actions/receive-profile', function () {
     it('can load a zipped profile', async function () {
       const { getState, view } = await setupZipTestWithProfile(
         'profile.json',
-        serializeProfile(_getSimpleProfile())
+        serializeProfileToJsonString(_getSimpleProfile())
       );
       expect(view.phase).toBe('DATA_LOADED');
       const zipInStore = ZippedProfilesSelectors.getZipFile(getState());
@@ -1358,7 +1364,7 @@ describe('actions/receive-profile', function () {
     it('will load and view a simple profile with no errors', async function () {
       const { getState, dispatch } = await setupZipTestWithProfile(
         'profile.json',
-        serializeProfile(_getSimpleProfile())
+        serializeProfileToJsonString(_getSimpleProfile())
       );
 
       expect(ZippedProfilesSelectors.getZipFileState(getState()).phase).toEqual(
@@ -1376,7 +1382,7 @@ describe('actions/receive-profile', function () {
     it('will be an error to view a profile with no threads', async function () {
       const { getState, dispatch } = await setupZipTestWithProfile(
         'profile.json',
-        serializeProfile(getEmptyProfile())
+        serializeProfileToJsonString(getEmptyProfile())
       );
 
       expect(ZippedProfilesSelectors.getZipFileState(getState()).phase).toEqual(

src/test/store/zipped-profiles.test.ts

@@ -16,7 +16,7 @@ import * as ZippedProfilesActions from '../../actions/zipped-profiles';
 import * as ReceiveProfileActions from '../../actions/receive-profile';
 import * as ProfileViewActions from '../../actions/profile-view';
 import { getProfileFromTextSamples } from '../fixtures/profiles/processed-profile';
-import { serializeProfile } from '../../profile-logic/process-profile';
+import { serializeProfileToJsonString } from '../../profile-logic/process-profile';
 import { compress } from '../../utils/gz';
 import type { PreviewSelection } from 'firefox-profiler/types';
 
@@ -112,7 +112,7 @@ describe('reducer zipFileState', function () {
     // returns a typed array whose `instanceof Uint8Array` check fails against
     // the main realm's global, so JSZip wouldn't recognize it directly.
     const gzippedProfile = new Uint8Array(
-      await compress(serializeProfile(profile))
+      await compress(serializeProfileToJsonString(profile))
     );
 
     const zip = new JSZip();

src/test/unit/process-profile.test.ts

@@ -1,10 +1,13 @@
 /* This Source Code Form is subject to the terms of the Mozilla Public
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+import { isJsonSlabsFile } from 'json-slabs';
+
 import {
   extractFuncsAndResourcesFromFrameLocations,
   processGeckoProfile,
-  serializeProfile,
+  serializeProfileToJsonSlabsFile,
+  serializeProfileToJsonString,
   unserializeProfileOfArbitraryFormat,
 } from '../../profile-logic/process-profile';
 import { GlobalDataCollector } from 'firefox-profiler/profile-logic/global-data-collector';
@@ -382,25 +385,47 @@ describe('gecko profilerOverhead processing', function () {
 describe('serializeProfile', function () {
   it('should produce a parsable profile string', async function () {
     const profile = processGeckoProfile(createGeckoProfile());
-    const serialized = serializeProfile(profile);
+    const serialized = serializeProfileToJsonString(profile);
     expect(JSON.parse.bind(null, serialized)).not.toThrow();
   });
 
   it('should produce the same profile in a roundtrip', async function () {
     const profile = processGeckoProfile(createGeckoProfile());
-    const serialized = serializeProfile(profile);
+    const serialized = serializeProfileToJsonString(profile);
     const roundtrip = await unserializeProfileOfArbitraryFormat(serialized);
     // FIXME: Uncomment this line after resolving `undefined` serialization issue
     // See: https://github.com/firefox-devtools/profiler/issues/1599
     // expect(profile).toEqual(roundtrip);
 
-    const secondSerialized = serializeProfile(roundtrip);
+    const secondSerialized = serializeProfileToJsonString(roundtrip);
     const secondRountrip =
       await unserializeProfileOfArbitraryFormat(secondSerialized);
     expect(roundtrip).toEqual(secondRountrip);
   });
 });
 
+describe('serializeProfileToJsonSlabsFile', function () {
+  it('should produce bytes recognized as a JsonSlabs file', function () {
+    const profile = processGeckoProfile(createGeckoProfile());
+    const bytes = serializeProfileToJsonSlabsFile(profile);
+    expect(isJsonSlabsFile(bytes)).toBe(true);
+  });
+
+  it('should produce the same profile in a roundtrip', async function () {
+    const profile = processGeckoProfile(createGeckoProfile());
+    const bytes = serializeProfileToJsonSlabsFile(profile);
+    const roundtrip = await unserializeProfileOfArbitraryFormat(bytes);
+
+    // Two roundtrips should be stable, mirroring the JSON serializer test
+    // above (see issue #1599 for why we can't compare against the original
+    // profile directly).
+    const secondBytes = serializeProfileToJsonSlabsFile(roundtrip);
+    const secondRoundtrip =
+      await unserializeProfileOfArbitraryFormat(secondBytes);
+    expect(roundtrip).toEqual(secondRoundtrip);
+  });
+});
+
 describe('js allocation processing', function () {
   function getAllocationMarkerHelper(geckoThread: GeckoThread) {
     let time = 0;