feat: add option to merge ASARs (#34)

* feat: fuse ASARs

* Rename, improve

* Rename option

* Drop universal from MACHO_MAGIC

package.json

@@ -10,6 +10,10 @@
     "apple silicon",
     "universal"
   ],
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/electron/universal.git"
+  },
   "engines": {
     "node": ">=8.6"
   },
@@ -29,6 +33,7 @@
     "@continuous-auth/semantic-release-npm": "^2.0.0",
     "@types/debug": "^4.1.5",
     "@types/fs-extra": "^9.0.4",
+    "@types/minimatch": "^3.0.5",
     "@types/node": "^14.14.7",
     "@types/plist": "^3.0.2",
     "husky": "^4.3.0",
@@ -43,6 +48,7 @@
     "debug": "^4.3.1",
     "dir-compare": "^2.4.0",
     "fs-extra": "^9.0.1",
+    "minimatch": "^3.0.4",
     "plist": "^3.0.4"
   },
   "husky": {

src/asar-utils.ts

@@ -1,14 +1,38 @@
 import * as asar from 'asar';
+import { execFileSync } from 'child_process';
 import * as crypto from 'crypto';
 import * as fs from 'fs-extra';
 import * as path from 'path';
+import * as minimatch from 'minimatch';
+import * as os from 'os';
 import { d } from './debug';
 
+const LIPO = 'lipo';
+
 export enum AsarMode {
   NO_ASAR,
   HAS_ASAR,
 }
 
+export type MergeASARsOptions = {
+  x64AsarPath: string;
+  arm64AsarPath: string;
+  outputAsarPath: string;
+
+  singleArchFiles?: string;
+};
+
+// See: https://github.com/apple-opensource-mirror/llvmCore/blob/0c60489d96c87140db9a6a14c6e82b15f5e5d252/include/llvm/Object/MachOFormat.h#L108-L112
+const MACHO_MAGIC = new Set([
+  // 32-bit Mach-O
+  0xfeedface,
+  0xcefaedfe,
+
+  // 64-bit Mach-O
+  0xfeedfacf,
+  0xcffaedfe,
+]);
+
 export const detectAsarMode = async (appPath: string) => {
   d('checking asar mode of', appPath);
   const asarPath = path.resolve(appPath, 'Contents', 'Resources', 'app.asar');
@@ -31,3 +55,152 @@ export const generateAsarIntegrity = (asarPath: string) => {
       .digest('hex'),
   };
 };
+
+function toRelativePath(file: string): string {
+  return file.replace(/^\//, '');
+}
+
+function isDirectory(a: string, file: string): boolean {
+  return Boolean('files' in asar.statFile(a, file));
+}
+
+function checkSingleArch(archive: string, file: string, allowList?: string): void {
+  if (allowList === undefined || !minimatch(file, allowList, { matchBase: true })) {
+    throw new Error(
+      `Detected unique file "${file}" in "${archive}" not covered by ` +
+        `allowList rule: "${allowList}"`,
+    );
+  }
+}
+
+export const mergeASARs = async ({
+  x64AsarPath,
+  arm64AsarPath,
+  outputAsarPath,
+  singleArchFiles,
+}: MergeASARsOptions): Promise<void> => {
+  d(`merging ${x64AsarPath} and ${arm64AsarPath}`);
+
+  const x64Files = new Set(asar.listPackage(x64AsarPath).map(toRelativePath));
+  const arm64Files = new Set(asar.listPackage(arm64AsarPath).map(toRelativePath));
+
+  //
+  // Build set of unpacked directories and files
+  //
+
+  const unpackedFiles = new Set<string>();
+
+  function buildUnpacked(a: string, fileList: Set<string>): void {
+    for (const file of fileList) {
+      const stat = asar.statFile(a, file);
+
+      if (!('unpacked' in stat) || !stat.unpacked) {
+        continue;
+      }
+
+      if ('files' in stat) {
+        continue;
+      }
+      unpackedFiles.add(file);
+    }
+  }
+
+  buildUnpacked(x64AsarPath, x64Files);
+  buildUnpacked(arm64AsarPath, arm64Files);
+
+  //
+  // Build list of files/directories unique to each asar
+  //
+
+  for (const file of x64Files) {
+    if (!arm64Files.has(file)) {
+      checkSingleArch(x64AsarPath, file, singleArchFiles);
+    }
+  }
+  const arm64Unique = [];
+  for (const file of arm64Files) {
+    if (!x64Files.has(file)) {
+      checkSingleArch(arm64AsarPath, file, singleArchFiles);
+      arm64Unique.push(file);
+    }
+  }
+
+  //
+  // Find common bindings with different content
+  //
+
+  const commonBindings = [];
+  for (const file of x64Files) {
+    if (!arm64Files.has(file)) {
+      continue;
+    }
+
+    // Skip directories
+    if (isDirectory(x64AsarPath, file)) {
+      continue;
+    }
+
+    const x64Content = asar.extractFile(x64AsarPath, file);
+    const arm64Content = asar.extractFile(arm64AsarPath, file);
+
+    if (x64Content.compare(arm64Content) === 0) {
+      continue;
+    }
+
+    if (!MACHO_MAGIC.has(x64Content.readUInt32LE(0))) {
+      throw new Error(`Can't reconcile two non-macho files ${file}`);
+    }
+
+    commonBindings.push(file);
+  }
+
+  //
+  // Extract both
+  //
+
+  const x64Dir = await fs.mkdtemp(path.join(os.tmpdir(), 'x64-'));
+  const arm64Dir = await fs.mkdtemp(path.join(os.tmpdir(), 'arm64-'));
+
+  try {
+    d(`extracting ${x64AsarPath} to ${x64Dir}`);
+    asar.extractAll(x64AsarPath, x64Dir);
+
+    d(`extracting ${arm64AsarPath} to ${arm64Dir}`);
+    asar.extractAll(arm64AsarPath, arm64Dir);
+
+    for (const file of arm64Unique) {
+      const source = path.resolve(arm64Dir, file);
+      const destination = path.resolve(x64Dir, file);
+
+      if (isDirectory(arm64AsarPath, file)) {
+        d(`creating unique directory: ${file}`);
+        await fs.mkdirp(destination);
+        continue;
+      }
+
+      d(`xopying unique file: ${file}`);
+      await fs.mkdirp(path.dirname(destination));
+      await fs.copy(source, destination);
+    }
+
+    for (const binding of commonBindings) {
+      const source = await fs.realpath(path.resolve(arm64Dir, binding));
+      const destination = await fs.realpath(path.resolve(x64Dir, binding));
+
+      d(`merging binding: ${binding}`);
+      execFileSync(LIPO, [source, destination, '-create', '-output', destination]);
+    }
+
+    d(`creating archive at ${outputAsarPath}`);
+
+    const resolvedUnpack = Array.from(unpackedFiles).map((file) => path.join(x64Dir, file));
+
+    await asar.createPackageWithOptions(x64Dir, outputAsarPath, {
+      unpack: `{${resolvedUnpack.join(',')}}`,
+    });
+
+    d('done merging');
+  } finally {
+    await Promise.all([fs.remove(x64Dir), fs.remove(arm64Dir)]);
+  }
+};

src/index.ts

@@ -8,7 +8,7 @@ import * as plist from 'plist';
 import * as dircompare from 'dir-compare';
 
 import { AppFile, AppFileType, getAllAppFiles } from './file-utils';
-import { AsarMode, detectAsarMode, generateAsarIntegrity } from './asar-utils';
+import { AsarMode, detectAsarMode, generateAsarIntegrity, mergeASARs } from './asar-utils';
 import { sha } from './sha';
 import { d } from './debug';
 
@@ -31,6 +31,14 @@ type MakeUniversalOpts = {
    * Forcefully overwrite any existing files that are in the way of generating the universal application
    */
   force: boolean;
+  /**
+   * Merge x64 and arm64 ASARs into one.
+   */
+  mergeASARs?: boolean;
+  /**
+   * Minimatch pattern of paths that are allowed to be present in one of the ASAR files, but not in the other.
+   */
+  singleArchFiles?: string;
 };
 
 const dupedFiles = (files: AppFile[]) =>
@@ -186,7 +194,18 @@ export const makeUniversalApp = async (opts: MakeUniversalOpts): Promise<void> =
      * look at codifying that assumption as actual logic.
      */
     // FIXME: Codify the assumption that app.asar.unpacked only contains native modules
-    if (x64AsarMode === AsarMode.HAS_ASAR) {
+    if (x64AsarMode === AsarMode.HAS_ASAR && opts.mergeASARs) {
+      d('merging x64 and arm64 asars');
+      const output = path.resolve(tmpApp, 'Contents', 'Resources', 'app.asar');
+      await mergeASARs({
+        x64AsarPath: path.resolve(tmpApp, 'Contents', 'Resources', 'app.asar'),
+        arm64AsarPath: path.resolve(opts.arm64AppPath, 'Contents', 'Resources', 'app.asar'),
+        outputAsarPath: output,
+        singleArchFiles: opts.singleArchFiles,
+      });
+
+      generatedIntegrity['Resources/app.asar'] = generateAsarIntegrity(output);
+    } else if (x64AsarMode === AsarMode.HAS_ASAR) {
       d('checking if the x64 and arm64 asars are identical');
       const x64AsarSha = await sha(path.resolve(tmpApp, 'Contents', 'Resources', 'app.asar'));
       const arm64AsarSha = await sha(

yarn.lock

@@ -294,6 +294,11 @@
   resolved "https://registry.yarnpkg.com/@types/minimatch/-/minimatch-3.0.3.tgz#3dca0e3f33b200fc7d1139c0cd96c1268cadfd9d"
   integrity sha512-tHq6qdbT9U1IRSGf14CL0pUlULksvY9OZ+5eEgl1N7t+OA3tGvNpxJCzuKQlsNgCVwbAs670L1vcVQi8j9HjnA==
 
+"@types/minimatch@^3.0.5":
+  version "3.0.5"
+  resolved "https://registry.yarnpkg.com/@types/minimatch/-/minimatch-3.0.5.tgz#1001cc5e6a3704b83c236027e77f2f58ea010f40"
+  integrity sha512-Klz949h02Gz2uZCMGwDUSDS1YBlTdDDgbWHi+81l29tQALUtvz4rAYi5uoVhE5Lagoq6DeqAUlbrHvW/mXDgdQ==
+
 "@types/minimist@^1.2.0":
   version "1.2.1"
   resolved "https://registry.yarnpkg.com/@types/minimist/-/minimist-1.2.1.tgz#283f669ff76d7b8260df8ab7a4262cc83d988256"