Add support for module groups to iOS Random Access Bundle format

Summary:
Adds the possibility to specify an array of files (group roots) that are used to bundle modules outside of the “startup section” into bigger groups by colocating their code.
A require call for any grouped module will load all modules in that group.
Files contained by multiple groups are deoptimized (i.e. bundled as individual script)

Reviewed By: martinbigio

Differential Revision: D3841780

fbshipit-source-id: 8d37782792efd66b5f557c7567489f68c9b229d8

Author: davidaurelio

local-cli/bundle/output/unbundle/as-indexed-file.js

@@ -32,19 +32,20 @@ function saveAsIndexedFile(bundle, options, log) {
   } = options;
 
   log('start');
-  const {startupModules, lazyModules} = bundle.getUnbundle();
+  const {startupModules, lazyModules, groups} = bundle.getUnbundle();
   log('finish');
 
+  const moduleGroups = ModuleGroups(groups, lazyModules);
   const startupCode = joinModules(startupModules);
 
   log('Writing unbundle output to:', bundleOutput);
   const writeUnbundle = writeBuffers(
     fs.createWriteStream(bundleOutput),
-    buildTableAndContents(startupCode, lazyModules, encoding)
+    buildTableAndContents(startupCode, lazyModules, moduleGroups, encoding)
   ).then(() => log('Done writing unbundle output'));
 
   const sourceMap =
-    buildSourceMapWithMetaData({startupModules, lazyModules});
+    buildSourceMapWithMetaData({startupModules, lazyModules, moduleGroups});
 
   return Promise.all([
     writeUnbundle,
@@ -84,7 +85,7 @@ function entryOffset(n) {
   return (2 + n * 2) * SIZEOF_UINT32;
 }
 
-function buildModuleTable(startupCode, buffers) {
+function buildModuleTable(startupCode, buffers, moduleGroups) {
   // table format:
   // - num_entries:      uint_32  number of entries
   // - startup_code_len: uint_32  length of the startup section
@@ -94,7 +95,8 @@ function buildModuleTable(startupCode, buffers) {
   //  - module_offset:   uint_32  offset into the modules blob
   //  - module_length:   uint_32  length of the module code in bytes
 
-  const maxId = buffers.reduce((max, {id}) => Math.max(max, id), 0);
+  const moduleIds = Array.from(moduleGroups.modulesById.keys());
+  const maxId = moduleIds.reduce((max, id) => Math.max(max, id));
   const numEntries = maxId + 1;
   const table = new Buffer(entryOffset(numEntries)).fill(0);
 
@@ -107,32 +109,59 @@ function buildModuleTable(startupCode, buffers) {
   // entries
   let codeOffset = startupCode.length;
   buffers.forEach(({id, buffer}) => {
-    const offset = entryOffset(id);
-    // module_offset
-    table.writeUInt32LE(codeOffset, offset);
-    // module_length
-    table.writeUInt32LE(buffer.length, offset + SIZEOF_UINT32);
+    const idsInGroup = moduleGroups.groups.has(id)
+      ? [id].concat(Array.from(moduleGroups.groups.get(id)))
+      : [id];
+
+    idsInGroup.forEach(moduleId => {
+      const offset = entryOffset(moduleId);
+      // module_offset
+      table.writeUInt32LE(codeOffset, offset);
+      // module_length
+      table.writeUInt32LE(buffer.length, offset + SIZEOF_UINT32);
+    });
     codeOffset += buffer.length;
   });
 
   return table;
 }
 
-function buildModuleBuffers(modules, encoding) {
-  return modules.map(
-    module => moduleToBuffer(module.id, module.code, encoding));
+function groupCode(rootCode, moduleGroup, modulesById) {
+  if (!moduleGroup || !moduleGroup.size) {
+    return rootCode;
+  }
+  const code = [rootCode];
+  for (const id of moduleGroup) {
+    code.push(modulesById.get(id).code);
+  }
+
+  return code.join('\n');
+}
+
+function buildModuleBuffers(modules, moduleGroups, encoding) {
+  return modules
+    .filter(m => !moduleGroups.modulesInGroups.has(m.id))
+    .map(({id, code}) => moduleToBuffer(
+      id,
+      groupCode(
+        code,
+        moduleGroups.groups.get(id),
+        moduleGroups.modulesById,
+      ),
+      encoding
+    ));
 }
 
-function buildTableAndContents(startupCode, modules, encoding) {
+function buildTableAndContents(startupCode, modules, moduleGroups, encoding) {
   // file contents layout:
   // - magic number      char[4]  0xE5 0xD1 0x0B 0xFB (0xFB0BD1E5 uint32 LE)
   // - offset table      table    see `buildModuleTables`
   // - code blob         char[]   null-terminated code strings, starting with
   //                              the startup code
 
   const startupCodeBuffer = nullTerminatedBuffer(startupCode, encoding);
-  const moduleBuffers = buildModuleBuffers(modules, encoding);
-  const table = buildModuleTable(startupCodeBuffer, moduleBuffers);
+  const moduleBuffers = buildModuleBuffers(modules, moduleGroups, encoding);
+  const table = buildModuleTable(startupCodeBuffer, moduleBuffers, moduleGroups);
 
   return [
     fileHeader,
@@ -141,4 +170,18 @@ function buildTableAndContents(startupCode, modules, encoding) {
   ].concat(moduleBuffers.map(({buffer}) => buffer));
 }
 
+function ModuleGroups(groups, modules) {
+  return {
+    groups,
+    modulesById: new Map(modules.map(m => [m.id, m])),
+    modulesInGroups: new Set(concat(groups.values())),
+  };
+}
+
+function * concat(iterators) {
+  for (const it of iterators) {
+    yield * it;
+  }
+}
+
 module.exports = saveAsIndexedFile;

local-cli/bundle/output/unbundle/build-unbundle-sourcemap-with-metadata.js

@@ -10,13 +10,14 @@
 
 const {combineSourceMaps, joinModules} = require('./util');
 
-module.exports = ({startupModules, lazyModules}) => {
+module.exports = ({startupModules, lazyModules, moduleGroups}) => {
   const startupModule = {
     code: joinModules(startupModules),
     map: combineSourceMaps({modules: startupModules}),
   };
   return combineSourceMaps({
     modules: [startupModule].concat(lazyModules),
+    moduleGroups,
     withCustomOffsets: true,
   });
 };

local-cli/bundle/output/unbundle/util.js

@@ -31,7 +31,7 @@ const wrapperEnd = wrappedCode => wrappedCode.indexOf('{') + 1;
 
 const Section = (line, column, map) => ({map, offset: {line, column}});
 
-function combineSourceMaps({modules, withCustomOffsets}) {
+function combineSourceMaps({modules, withCustomOffsets, moduleGroups}) {
   let offsets;
   const sections = [];
   const sourceMap = {
@@ -45,13 +45,41 @@ function combineSourceMaps({modules, withCustomOffsets}) {
 
   let line = 0;
   modules.forEach(({code, id, map, name}) => {
-    const hasOffset = withCustomOffsets && id != null;
-    const column = hasOffset ? wrapperEnd(code) : 0;
+    let column = 0;
+    let hasOffset = false;
+    let group;
+    let groupLines = 0;
+
+    if (withCustomOffsets) {
+      if (moduleGroups && moduleGroups.modulesInGroups.has(id)) {
+        // this is a module appended to another module
+        return;
+      }
+
+      if (moduleGroups && moduleGroups.groups.has(id)) {
+        group = moduleGroups.groups.get(id);
+        const otherModules = Array.from(group).map(
+          moduleId => moduleGroups.modulesById.get(moduleId));
+        otherModules.forEach(m => {
+          groupLines += countLines(m.code);
+        });
+        map = combineSourceMaps({
+          modules: [{code, id, map, name}].concat(otherModules),
+        });
+      }
+
+      hasOffset = id != null;
+      column = wrapperEnd(code);
+    }
+
     sections.push(Section(line, column, map || lineToLineSourceMap(code, name)));
     if (hasOffset) {
       offsets[id] = line;
+      for (const moduleId of group || []) {
+        offsets[moduleId] = line;
+      }
     }
-    line += countLines(code);
+    line += countLines(code) + groupLines;
   });
 
   return sourceMap;

packager/react-packager/src/Bundler/Bundle.js

@@ -17,7 +17,7 @@ const crypto = require('crypto');
 const SOURCEMAPPING_URL = '\n\/\/# sourceMappingURL=';
 
 class Bundle extends BundleBase {
-  constructor({sourceMapUrl, dev, minify} = {}) {
+  constructor({sourceMapUrl, dev, minify, ramGroups} = {}) {
     super();
     this._sourceMap = false;
     this._sourceMapUrl = sourceMapUrl;
@@ -26,6 +26,7 @@ class Bundle extends BundleBase {
     this._dev = dev;
     this._minify = minify;
 
+    this._ramGroups = ramGroups;
     this._ramBundle = null; // cached RAM Bundle
   }
 
@@ -111,9 +112,17 @@ class Bundle extends BundleBase {
       // separate modules we need to preload from the ones we don't
       const [startupModules, lazyModules] = partition(modules, shouldPreload);
 
+      const ramGroups = this._ramGroups;
+      let groups;
       this._ramBundle = {
         startupModules,
         lazyModules,
+        get groups() {
+          if (!groups) {
+            groups = createGroups(ramGroups || [], lazyModules);
+          }
+          return groups;
+        }
       };
     }
 
@@ -265,6 +274,10 @@ class Bundle extends BundleBase {
     ].join('\n');
   }
 
+  setRamGroups(ramGroups) {
+    this._ramGroups = ramGroups;
+  }
+
   toJSON() {
     this.assertFinalized('Cannot serialize bundle unless finalized');
 
@@ -318,4 +331,89 @@ function partition(array, predicate) {
   return [included, excluded];
 }
 
+function * filter(iterator, predicate) {
+  for (const value of iterator) {
+    if (predicate(value)) {
+      yield value;
+    }
+  }
+}
+
+function * subtree(moduleTransport, moduleTransportsByPath, seen = new Set()) {
+  seen.add(moduleTransport.id);
+  for (const [, {path}] of moduleTransport.meta.dependencyPairs) {
+    const dependency = moduleTransportsByPath.get(path);
+    if (dependency && !seen.has(dependency.id)) {
+      yield dependency.id;
+      yield * subtree(dependency, moduleTransportsByPath, seen);
+    }
+  }
+}
+
+class ArrayMap extends Map {
+  get(key) {
+    let array = super.get(key);
+    if (!array) {
+      array = [];
+      this.set(key, array);
+    }
+    return array;
+  }
+}
+
+function createGroups(ramGroups, lazyModules) {
+  // build two maps that allow to lookup module data
+  // by path or (numeric) module id;
+  const byPath = new Map();
+  const byId = new Map();
+  lazyModules.forEach(m => {
+    byPath.set(m.sourcePath, m);
+    byId.set(m.id, m.sourcePath);
+  });
+
+  // build a map of group root IDs to an array of module IDs in the group
+  const result = new Map(
+    ramGroups
+      .map(modulePath => {
+        const root = byPath.get(modulePath);
+        if (!root) {
+          throw Error(`Group root ${modulePath} is not part of the bundle`);
+        }
+        return [
+          root.id,
+          // `subtree` yields the IDs of all transitive dependencies of a module
+          new Set(subtree(byPath.get(root.sourcePath), byPath)),
+        ];
+      })
+  );
+
+  if (ramGroups.length > 1) {
+    // build a map of all grouped module IDs to an array of group root IDs
+    const all = new ArrayMap();
+    for (const [parent, children] of result) {
+      for (const module of children) {
+        all.get(module).push(parent);
+      }
+    }
+
+    // find all module IDs that are part of more than one group
+    const doubles = filter(all, ([, parents]) => parents.length > 1);
+    for (const [moduleId, parents] of doubles) {
+      // remove them from their groups
+      parents.forEach(p => result.get(p).delete(moduleId));
+
+      // print a warning for each removed module
+      const parentNames = parents.map(byId.get, byId);
+      const lastName = parentNames.pop();
+      console.warn(
+        `Module ${byId.get(moduleId)} belongs to groups ${
+          parentNames.join(', ')}, and ${lastName
+          }. Removing it from all groups.`
+      );
+    }
+  }
+
+  return result;
+}
+
 module.exports = Bundle;