create better debuggable source maps

Summary:
Introduces a new mechanism to build source maps that allows us to use real mapping segments instead of just mapping line-by-line.

This mechanism is only used when building development bundles to improve the debugging experience in Chrome.

The new mechanism takes advantage of a new feature in babel-generator that exposes raw mapping objects. These raw mapping objects are converted to arrays with 2, 4, or 5 for the most compact representation possible.
We no longer generate a source map for the bundle that maps each line to itself in conjunction with configuring babel generator to retain lines.

Instead, we create a source map with a large mappings object produced from the mappings of each individual file in conjunction with a “carry over” – the number of preceding lines in the bundle.

The implementation makes a couple of assumptions that hold true for babel transform results, e.g. mappings being in the order of the generated code, and that a block of mappings always belongs to the same source file. In addition, the implementation avoids allocation of objects and strings at all costs. All calculations are purely numeric, and base64 vlq produces numeric ascii character codes. These are written to a preallocated buffer objects, which is turned to a string only at the end of the building process. This implementation is ~5x faster than using the source-map library.

In addition to providing development source maps that work better, we can now also produce individual high-quality source maps for production builds and combine them to an “index source map”. This approach is unfeasable for development source maps, because index source map consistently crash Chrome.

Better production source maps are useful to get precise information about source location and symbol names when symbolicating stack traces from crashes in production.

Reviewed By: jeanlauliac

Differential Revision: D4382290

fbshipit-source-id: 365a176fa142729d0a4cef43edeb81084361e54d

Author: davidaurelio

flow/babel.js.flow

@@ -87,7 +87,7 @@ type __TransformOptions = {
 
 type _TransformOptions =
   __TransformOptions & {env?: {[key: string]: __TransformOptions}};
-declare class _Ast {};
+declare class _Ast {}
 type TransformResult = {
   ast: _Ast,
   code: ?string,
@@ -119,9 +119,17 @@ declare module 'babel-core' {
   ): TransformResult;
 }
 
+type RawMapping = {
+  generated: {column: number, line: number},
+  name?: string,
+  original?: {column: number, line: number},
+  source?: string,
+};
+
 declare module 'babel-generator' {
+  declare type RawMapping = RawMapping;
   declare function exports(
     ast: _Ast,
     options?: GeneratorOptions,
-  ): TransformResult;
+  ): TransformResult & {rawMappings: ?Array<RawMapping>};
 }

jest/preprocessor.js

@@ -12,14 +12,17 @@ const babel = require('babel-core');
 const babelRegisterOnly = require('../packager/babelRegisterOnly');
 const createCacheKeyFunction = require('fbjs-scripts/jest/createCacheKeyFunction');
 const path = require('path');
-const transformer = require('../packager/transformer.js');
 
 const nodeFiles = RegExp([
   '/local-cli/',
   '/packager/(?!react-packager/src/Resolver/polyfills/)',
 ].join('|'));
 const nodeOptions = babelRegisterOnly.config([nodeFiles]);
 
+babelRegisterOnly([]);
+// has to be required after setting up babelRegisterOnly
+const transformer = require('../packager/transformer.js');
+
 module.exports = {
   process(src, file) {
     // Don't transform node_modules, except react-tools which includes the

local-cli/bundle/output/bundle.js

@@ -26,7 +26,10 @@ function buildBundle(packagerClient: Server, requestOptions: RequestOptions) {
 }
 
 function createCodeWithMap(bundle: Bundle, dev: boolean, sourceMapSourcesRoot?: string): * {
-  const sourceMap = relativizeSourceMap(bundle.getSourceMap({dev}), sourceMapSourcesRoot);
+  const map = bundle.getSourceMap({dev});
+  const sourceMap = relativizeSourceMap(
+    typeof map === 'string' ? JSON.parse(map) : map,
+    sourceMapSourcesRoot);
   return {
     code: bundle.getSource({dev}),
     map: JSON.stringify(sourceMap),

local-cli/bundle/output/unbundle/util.js

@@ -10,6 +10,8 @@
  */
 'use strict';
 
+const invariant = require('fbjs/lib/invariant');
+
 import type {ModuleGroups, ModuleTransportLike, SourceMap} from '../../types.flow';
 
 const newline = /\r\n?|\n|\u2028|\u2029/g;
@@ -99,6 +101,10 @@ function combineSourceMaps({
       column = wrapperEnd(code);
     }
 
+    invariant(
+      !Array.isArray(map),
+      'Random Access Bundle source maps cannot be built from raw mappings',
+    );
     sections.push(Section(line, column, map || lineToLineSourceMap(code, name)));
     if (hasOffset) {
       offsets[id] = line;

local-cli/bundle/types.flow.js

@@ -26,7 +26,7 @@ export type ModuleGroups = {|
 export type ModuleTransportLike = {
   code: string,
   id: number,
-  map?: ?MixedSourceMap,
+  map?: $PropertyType<ModuleTransport, 'map'>,
   +name?: string,
 };
 

packager/package.json

@@ -1,5 +1,5 @@
 {
-  "version": "0.4.0",
+  "version": "0.5.0",
   "name": "react-native-packager",
   "description": "Build native apps with React!",
   "repository": {

packager/react-packager/src/Bundler/Bundle.js

@@ -15,8 +15,11 @@ const BundleBase = require('./BundleBase');
 const ModuleTransport = require('../lib/ModuleTransport');
 
 const _ = require('lodash');
-const base64VLQ = require('./base64-vlq');
 const crypto = require('crypto');
+const debug = require('debug')('RNP:Bundle');
+const invariant = require('fbjs/lib/invariant');
+
+const {fromRawMappings} = require('./source-map');
 
 import type {SourceMap, CombinedSourceMap, MixedSourceMap} from '../lib/SourceMap';
 import type {GetSourceOptions, FinalizeOptions} from './BundleBase';
@@ -27,6 +30,8 @@ export type Unbundle = {
   groups: Map<number, Set<number>>,
 };
 
+type SourceMapFormat = 'undetermined' | 'indexed' | 'flattened';
+
 const SOURCEMAPPING_URL = '\n\/\/# sourceMappingURL=';
 
 class Bundle extends BundleBase {
@@ -37,8 +42,8 @@ class Bundle extends BundleBase {
   _numRequireCalls: number;
   _ramBundle: Unbundle | null;
   _ramGroups: Array<string> | void;
-  _shouldCombineSourceMaps: boolean;
-  _sourceMap: boolean;
+  _sourceMap: string | null;
+  _sourceMapFormat: SourceMapFormat;
   _sourceMapUrl: string | void;
 
   constructor({sourceMapUrl, dev, minify, ramGroups}: {
@@ -48,9 +53,9 @@ class Bundle extends BundleBase {
     ramGroups?: Array<string>,
   } = {}) {
     super();
-    this._sourceMap = false;
+    this._sourceMap = null;
+    this._sourceMapFormat = 'undetermined';
     this._sourceMapUrl = sourceMapUrl;
-    this._shouldCombineSourceMaps = false;
     this._numRequireCalls = 0;
     this._dev = dev;
     this._minify = minify;
@@ -86,8 +91,22 @@ class Bundle extends BundleBase {
     }).then(({code, map}) => {
       // If we get a map from the transformer we'll switch to a mode
       // were we're combining the source maps as opposed to
-      if (!this._shouldCombineSourceMaps && map != null) {
-        this._shouldCombineSourceMaps = true;
+      if (map) {
+        const usesRawMappings = isRawMappings(map);
+
+        if (this._sourceMapFormat === 'undetermined') {
+          this._sourceMapFormat = usesRawMappings ? 'flattened' : 'indexed';
+        } else if (usesRawMappings && this._sourceMapFormat === 'indexed') {
+          throw new Error(
+            `Got at least one module with a full source map, but ${
+            moduleTransport.sourcePath} has raw mappings`
+          );
+        } else if (!usesRawMappings && this._sourceMapFormat === 'flattened') {
+          throw new Error(
+            `Got at least one module with raw mappings, but ${
+            moduleTransport.sourcePath} has a full source map`
+          );
+        }
       }
 
       this.replaceModuleAt(
@@ -103,7 +122,7 @@ class Bundle extends BundleBase {
       options.runBeforeMainModule.forEach(this._addRequireCall, this);
       /* $FlowFixMe: this is unsound, as nothing enforces the module ID to have
        * been set beforehand. */
-      this._addRequireCall(super.getMainModuleId());
+      this._addRequireCall(this.getMainModuleId());
     }
 
     super.finalize(options);
@@ -126,16 +145,16 @@ class Bundle extends BundleBase {
 
   _getInlineSourceMap(dev) {
     if (this._inlineSourceMap == null) {
-      const sourceMap = this.getSourceMap({excludeSource: true, dev});
+      const sourceMap = this.getSourceMapString({excludeSource: true, dev});
       /*eslint-env node*/
-      const encoded = new Buffer(JSON.stringify(sourceMap)).toString('base64');
+      const encoded = new Buffer(sourceMap).toString('base64');
       this._inlineSourceMap = 'data:application/json;base64,' + encoded;
     }
     return this._inlineSourceMap;
   }
 
   getSource(options: GetSourceOptions) {
-    super.assertFinalized();
+    this.assertFinalized();
 
     options = options || {};
 
@@ -175,6 +194,12 @@ class Bundle extends BundleBase {
     return this._ramBundle;
   }
 
+  invalidateSource() {
+    debug('invalidating bundle');
+    super.invalidateSource();
+    this._sourceMap = null;
+  }
+
   /**
    * Combine each of the sourcemaps multiple modules have into a single big
    * one. This works well thanks to a neat trick defined on the sourcemap spec
@@ -190,23 +215,22 @@ class Bundle extends BundleBase {
 
     let line = 0;
     this.getModules().forEach(module => {
-      let map = module.map;
+      let map = module.map == null || module.virtual
+        ? generateSourceMapForVirtualModule(module)
+        : module.map;
 
-      if (module.virtual) {
-        map = generateSourceMapForVirtualModule(module);
-      }
+      invariant(
+        !Array.isArray(map),
+        `Unexpected raw mappings for ${module.sourcePath}`,
+      );
 
-      if (options.excludeSource) {
-        /* $FlowFixMe: assume the map is not empty if we got here. */
-        if (map.sourcesContent && map.sourcesContent.length) {
-          map = Object.assign({}, map, {sourcesContent: []});
-        }
+      if (options.excludeSource && 'sourcesContent' in map) {
+        map = {...map, sourcesContent: []};
       }
 
       result.sections.push({
         offset: { line: line, column: 0 },
-        /* $FlowFixMe: assume the map is not empty if we got here. */
-        map: map,
+        map: (map: MixedSourceMap),
       });
       line += module.code.split('\n').length;
     });
@@ -215,23 +239,30 @@ class Bundle extends BundleBase {
   }
 
   getSourceMap(options: {excludeSource?: boolean}): MixedSourceMap {
-    super.assertFinalized();
+    this.assertFinalized();
+
+    return this._sourceMapFormat === 'indexed'
+      ? this._getCombinedSourceMaps(options)
+      : fromRawMappings(this.getModules()).toMap();
+  }
 
-    if (this._shouldCombineSourceMaps) {
-      return this._getCombinedSourceMaps(options);
+  getSourceMapString(options: {excludeSource?: boolean}): string {
+    if (this._sourceMapFormat === 'indexed') {
+      return JSON.stringify(this.getSourceMap(options));
     }
 
-    const mappings = this._getMappings();
-    const modules = this.getModules();
-    const map = {
-      file: this._getSourceMapFile(),
-      sources: modules.map(module => module.sourcePath),
-      version: 3,
-      names: [],
-      mappings: mappings,
-      sourcesContent: options.excludeSource
-        ? [] : modules.map(module => module.sourceCode),
-    };
+    // The following code is an optimization specific to the development server:
+    // 1. generator.toSource() is faster than JSON.stringify(generator.toMap()).
+    // 2. caching the source map unless there are changes saves time in
+    //    development settings.
+    let map = this._sourceMap;
+    if (map == null) {
+      debug('Start building flat source map');
+      map = this._sourceMap = fromRawMappings(this.getModules()).toString();
+      debug('End building flat source map');
+    } else {
+      debug('Returning cached source map');
+    }
     return map;
   }
 
@@ -248,53 +279,6 @@ class Bundle extends BundleBase {
       : 'bundle.js';
   }
 
-  _getMappings() {
-    const modules = super.getModules();
-
-    // The first line mapping in our package is basically the base64vlq code for
-    // zeros (A).
-    const firstLine = 'AAAA';
-
-    // Most other lines in our mappings are all zeros (for module, column etc)
-    // except for the lineno mappinp: curLineno - prevLineno = 1; Which is C.
-    const line = 'AACA';
-
-    const moduleLines = Object.create(null);
-    let mappings = '';
-    for (let i = 0; i < modules.length; i++) {
-      const module = modules[i];
-      const code = module.code;
-      let lastCharNewLine  = false;
-      moduleLines[module.sourcePath] = 0;
-      for (let t = 0; t < code.length; t++) {
-        if (t === 0 && i === 0) {
-          mappings += firstLine;
-        } else if (t === 0) {
-          mappings += 'AC';
-
-          // This is the only place were we actually don't know the mapping ahead
-          // of time. When it's a new module (and not the first) the lineno
-          // mapping is 0 (current) - number of lines in prev module.
-          mappings += base64VLQ.encode(
-            0 - moduleLines[modules[i - 1].sourcePath]
-          );
-          mappings += 'A';
-        } else if (lastCharNewLine) {
-          moduleLines[module.sourcePath]++;
-          mappings += line;
-        }
-        lastCharNewLine = code[t] === '\n';
-        if (lastCharNewLine) {
-          mappings += ';';
-        }
-      }
-      if (i !== modules.length - 1) {
-        mappings += ';';
-      }
-    }
-    return mappings;
-  }
-
   getJSModulePaths() {
     return this.getModules()
       // Filter out non-js files. Like images etc.
@@ -305,7 +289,7 @@ class Bundle extends BundleBase {
   getDebugInfo() {
     return [
       /* $FlowFixMe: this is unsound as the module ID could be unset. */
-      '<div><h3>Main Module:</h3> ' + super.getMainModuleId() + '</div>',
+      '<div><h3>Main Module:</h3> ' + this.getMainModuleId() + '</div>',
       '<style>',
       'pre.collapsed {',
       '  height: 10px;',
@@ -328,30 +312,6 @@ class Bundle extends BundleBase {
   setRamGroups(ramGroups: Array<string>) {
     this._ramGroups = ramGroups;
   }
-
-  toJSON() {
-    this.assertFinalized('Cannot serialize bundle unless finalized');
-
-    return {
-      ...super.toJSON(),
-      sourceMapUrl: this._sourceMapUrl,
-      numRequireCalls: this._numRequireCalls,
-      shouldCombineSourceMaps: this._shouldCombineSourceMaps,
-    };
-  }
-
-  static fromJSON(json) {
-    const bundle = new Bundle({sourceMapUrl: json.sourceMapUrl});
-
-    bundle._sourceMapUrl = json.sourceMapUrl;
-    bundle._numRequireCalls = json.numRequireCalls;
-    bundle._shouldCombineSourceMaps = json.shouldCombineSourceMaps;
-
-    BundleBase.fromJSON(bundle, json);
-
-    /* $FlowFixMe: this modifies BundleBase#fromJSON() signature. */
-    return bundle;
-  }
 }
 
 function generateSourceMapForVirtualModule(module): SourceMap {
@@ -472,4 +432,6 @@ function createGroups(ramGroups: Array<string>, lazyModules) {
   return result;
 }
 
+const isRawMappings = Array.isArray;
+
 module.exports = Bundle;