• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1/*
2 * Copyright (c) 2021 Huawei Device Co., Ltd.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 *     http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16import * as fs from 'fs';
17import * as path from 'path';
18import cluster from 'cluster';
19import process from 'process';
20import Compiler from 'webpack/lib/Compiler';
21import { logger } from './compile_info';
22import {
23  toUnixPath,
24  toHashData,
25  validateFilePathLength
26} from './utils';
27import {
28  TEMPORARY,
29  NODE_MODULES,
30  FAIL
31} from './pre_define';
32
33const genAbcScript = 'gen_abc.js';
34let output: string;
35let isWin: boolean = false;
36let isMac: boolean = false;
37let isDebug: boolean = false;
38let arkDir: string;
39let nodeJs: string;
40
41interface File {
42  path: string,
43  size: number,
44  cacheOutputPath: string
45}
46const intermediateJsBundle: Array<File> = [];
47let fileterIntermediateJsBundle: Array<File> = [];
48let hashJsonObject = {};
49let buildPathInfo = '';
50
51const red: string = '\u001b[31m';
52const reset: string = '\u001b[39m';
53const hashFile = 'gen_hash.json';
54const ARK = '/ark/';
55
56export class GenAbcPlugin {
57  constructor(output_, arkDir_, nodeJs_, isDebug_) {
58    output = output_;
59    arkDir = arkDir_;
60    nodeJs = nodeJs_;
61    isDebug = isDebug_;
62  }
63  apply(compiler: Compiler) {
64    if (fs.existsSync(path.resolve(arkDir, 'build-win'))) {
65      isWin = true;
66    } else {
67      if (fs.existsSync(path.resolve(arkDir, 'build-mac'))) {
68        isMac = true;
69      } else {
70        if (!fs.existsSync(path.resolve(arkDir, 'build'))) {
71          logger.debug(red, 'ArkTS:ERROR find build fail', reset);
72          process.exitCode = FAIL;
73          return;
74        }
75      }
76    }
77
78    if (!checkNodeModules()) {
79      process.exitCode = FAIL;
80      return;
81    }
82
83    compiler.hooks.emit.tap('GenAbcPlugin', (compilation) => {
84      buildPathInfo = output;
85      Object.keys(compilation.assets).forEach(key => {
86        // choose *.js
87        if (output && path.extname(key) === '.js') {
88          const newContent: string = compilation.assets[key].source();
89          const keyPath: string = key.replace(/\.js$/, ".temp.js");
90          writeFileSync(newContent, output, keyPath, key);
91        }
92      });
93    });
94
95    compiler.hooks.afterEmit.tap('GenAbcPluginMultiThread', () => {
96      if (intermediateJsBundle.length === 0) {
97        return;
98      }
99      buildPathInfo = output;
100      invokeWorkersToGenAbc();
101    });
102  }
103}
104
105function writeFileSync(inputString: string, buildPath: string, keyPath: string, jsBundleFile: string): void {
106  let output = path.resolve(buildPath, keyPath);
107  validateFilePathLength(output);
108  let parent: string = path.join(output, '..');
109  if (!(fs.existsSync(parent) && fs.statSync(parent).isDirectory())) {
110    mkDir(parent);
111  }
112  let cacheOutputPath: string = "";
113  if (process.env.cachePath) {
114    let buildDirArr: string[] = buildPathInfo.split(path.sep);
115    let abilityDir: string = buildDirArr[buildDirArr.length - 1];
116    cacheOutputPath = path.join(process.env.cachePath, TEMPORARY, abilityDir, keyPath);
117  } else {
118    cacheOutputPath = output;
119  }
120  validateFilePathLength(cacheOutputPath);
121  parent = path.join(cacheOutputPath, '..');
122  if (!(fs.existsSync(parent) && fs.statSync(parent).isDirectory())) {
123    mkDir(parent);
124  }
125  fs.writeFileSync(cacheOutputPath, inputString);
126  if (fs.existsSync(cacheOutputPath)) {
127    const fileSize = fs.statSync(cacheOutputPath).size;
128    output = toUnixPath(output);
129    cacheOutputPath = toUnixPath(cacheOutputPath);
130    intermediateJsBundle.push({path: output, size: fileSize, cacheOutputPath: cacheOutputPath});
131  } else {
132    logger.debug(red, `ArkTS:ERROR Failed to convert file ${jsBundleFile} to bin. ${output} is lost`, reset);
133    process.exitCode = FAIL;
134  }
135}
136
137function mkDir(path_: string): void {
138  const parent: string = path.join(path_, '..');
139  if (!(fs.existsSync(parent) && !fs.statSync(parent).isFile())) {
140    mkDir(parent);
141  }
142  fs.mkdirSync(path_);
143}
144
145function getSmallestSizeGroup(groupSize: Map<number, number>) {
146  const groupSizeArray = Array.from(groupSize);
147  groupSizeArray.sort(function(g1, g2) {
148    return g1[1] - g2[1]; // sort by size
149  });
150  return groupSizeArray[0][0];
151}
152
153function splitJsBundlesBySize(bundleArray: Array<File>, groupNumber: number) {
154  const result = [];
155  if (bundleArray.length < groupNumber) {
156    result.push(bundleArray);
157    return result;
158  }
159
160  bundleArray.sort(function(f1: File, f2: File) {
161    return f2.size - f1.size;
162  });
163  const groupFileSize = new Map();
164  for (let i = 0; i < groupNumber; ++i) {
165    result.push([]);
166    groupFileSize.set(i, 0);
167  }
168
169  let index = 0;
170  while (index < bundleArray.length) {
171    const smallestGroup = getSmallestSizeGroup(groupFileSize);
172    result[smallestGroup].push(bundleArray[index]);
173    const sizeUpdate = groupFileSize.get(smallestGroup) + bundleArray[index].size;
174    groupFileSize.set(smallestGroup, sizeUpdate);
175    index++;
176  }
177  return result;
178}
179
180function invokeWorkersToGenAbc() {
181  let param: string = '';
182  if (isDebug) {
183    param += ' --debug';
184  }
185
186  let js2abc: string = path.join(arkDir, 'build', 'src', 'index.js');
187  if (isWin) {
188    js2abc = path.join(arkDir, 'build-win', 'src', 'index.js');
189  } else if (isMac) {
190    js2abc = path.join(arkDir, 'build-mac', 'src', 'index.js');
191  }
192  validateFilePathLength(js2abc);
193
194  filterIntermediateJsBundleByHashJson(buildPathInfo, intermediateJsBundle);
195  const maxWorkerNumber = 3;
196  const splitedBundles = splitJsBundlesBySize(fileterIntermediateJsBundle, maxWorkerNumber);
197  const workerNumber = maxWorkerNumber < splitedBundles.length ? maxWorkerNumber : splitedBundles.length;
198  const cmdPrefix: string = `${nodeJs} --expose-gc "${js2abc}" ${param} `;
199
200  const clusterNewApiVersion = 16;
201  const currentNodeVersion = parseInt(process.version.split('.')[0]);
202  const useNewApi = currentNodeVersion >= clusterNewApiVersion;
203
204  if (useNewApi && cluster.isPrimary || !useNewApi && cluster.isMaster) {
205    if (useNewApi) {
206      cluster.setupPrimary({
207        exec: path.resolve(__dirname, genAbcScript)
208      });
209    } else {
210      cluster.setupMaster({
211        exec: path.resolve(__dirname, genAbcScript)
212      });
213    }
214
215    for (let i = 0; i < workerNumber; ++i) {
216      const workerData = {
217        'inputs': JSON.stringify(splitedBundles[i]),
218        'cmd': cmdPrefix
219      };
220      cluster.fork(workerData);
221    }
222
223    cluster.on('exit', (worker, code, signal) => {
224      if (code === FAIL || process.exitCode === FAIL) {
225        process.exitCode = FAIL;
226        return;
227      }
228      logger.debug(`worker ${worker.process.pid} finished`);
229    });
230
231    process.on('exit', (code) => {
232      writeHashJson();
233      copyFileCachePathToBuildPath()
234    });
235  }
236}
237
238function filterIntermediateJsBundleByHashJson(buildPath: string, inputPaths: File[]) {
239  for (let i = 0; i < inputPaths.length; ++i) {
240    fileterIntermediateJsBundle.push(inputPaths[i]);
241  }
242  const hashFilePath = genHashJsonPath(buildPath);
243  if (hashFilePath.length === 0) {
244    return;
245  }
246  const updateJsonObject = {};
247  let jsonObject = {};
248  let jsonFile = '';
249  if (fs.existsSync(hashFilePath)) {
250    jsonFile = fs.readFileSync(hashFilePath).toString();
251    jsonObject = JSON.parse(jsonFile);
252    fileterIntermediateJsBundle = [];
253    for (let i = 0; i < inputPaths.length; ++i) {
254      const cacheOutputPath: string = inputPaths[i].cacheOutputPath;
255      const cacheAbcFilePath: string = cacheOutputPath.replace(/\.temp\.js$/, '.abc');
256      if (!fs.existsSync(cacheOutputPath)) {
257        logger.error(red, `ArkTS:ERROR ${cacheOutputPath} is lost`, reset);
258        process.exitCode = FAIL;
259        continue;
260      }
261      if (fs.existsSync(cacheAbcFilePath)) {
262        const hashInputContentData = toHashData(cacheOutputPath);
263        const hashAbcContentData = toHashData(cacheAbcFilePath);
264        if (jsonObject[cacheOutputPath] === hashInputContentData && jsonObject[cacheAbcFilePath] === hashAbcContentData) {
265          updateJsonObject[cacheOutputPath] = hashInputContentData;
266          updateJsonObject[cacheAbcFilePath] = hashAbcContentData;
267        } else {
268          fileterIntermediateJsBundle.push(inputPaths[i]);
269        }
270      } else {
271        fileterIntermediateJsBundle.push(inputPaths[i]);
272      }
273    }
274  }
275
276  hashJsonObject = updateJsonObject;
277}
278
279function writeHashJson() {
280  for (let i = 0; i < fileterIntermediateJsBundle.length; ++i) {
281    const cacheOutputPath: string = fileterIntermediateJsBundle[i].cacheOutputPath;
282    const cacheAbcFilePath: string = cacheOutputPath.replace(/\.temp\.js$/, '.abc');
283    if (!fs.existsSync(cacheOutputPath) || !fs.existsSync(cacheAbcFilePath)) {
284      logger.debug(red, `ArkTS:ERROR ${cacheOutputPath} is lost`, reset);
285      process.exitCode = FAIL;
286      continue;
287    }
288    const hashInputContentData: any = toHashData(cacheOutputPath);
289    const hashAbcContentData: any = toHashData(cacheAbcFilePath);
290    hashJsonObject[cacheOutputPath] = hashInputContentData;
291    hashJsonObject[cacheAbcFilePath] = hashAbcContentData;
292  }
293  const hashFilePath = genHashJsonPath(buildPathInfo);
294  if (hashFilePath.length === 0) {
295    return;
296  }
297  fs.writeFileSync(hashFilePath, JSON.stringify(hashJsonObject));
298}
299
300function genHashJsonPath(buildPath: string) {
301  buildPath = toUnixPath(buildPath);
302  if (process.env.cachePath) {
303    if (!fs.existsSync(process.env.cachePath) || !fs.statSync(process.env.cachePath).isDirectory()) {
304      logger.debug(red, `ArkTS:ERROR hash path does not exist`, reset);
305      return '';
306    }
307    let buildDirArr: string[] = buildPathInfo.split(path.sep);
308    let abilityDir: string = buildDirArr[buildDirArr.length - 1];
309    let hashJsonPath: string = path.join(process.env.cachePath, TEMPORARY, abilityDir, hashFile);
310    validateFilePathLength(hashJsonPath);
311    let parent = path.join(hashJsonPath, '..');
312    if (!(fs.existsSync(parent) && fs.statSync(parent).isDirectory())) {
313      mkDir(parent);
314    }
315    return hashJsonPath;
316  } else if (buildPath.indexOf(ARK) >= 0) {
317    const dataTmps = buildPath.split(ARK);
318    const hashPath = path.join(dataTmps[0], ARK);
319    if (!fs.existsSync(hashPath) || !fs.statSync(hashPath).isDirectory()) {
320      logger.debug(red, `ArkTS:ERROR hash path does not exist`, reset);
321      return '';
322    }
323    let hashJsonPath: string = path.join(hashPath, hashFile);
324    validateFilePathLength(hashJsonPath);
325    return hashJsonPath;
326  } else {
327    logger.debug(red, `ArkTS:ERROR not cache exist`, reset);
328    return '';
329  }
330}
331
332function checkNodeModules() {
333  let arkEntryPath: string = path.join(arkDir, 'build');
334  if (isWin) {
335    arkEntryPath = path.join(arkDir, 'build-win');
336  } else if (isMac) {
337    arkEntryPath = path.join(arkDir, 'build-mac');
338  }
339  let nodeModulesPath: string = path.join(arkEntryPath, NODE_MODULES);
340  validateFilePathLength(nodeModulesPath);
341  if (!(fs.existsSync(nodeModulesPath) && fs.statSync(nodeModulesPath).isDirectory())) {
342    logger.error(red, `ERROR: node_modules for ark compiler not found.
343      Please make sure switch to non-root user before runing "npm install" for safity requirements and try re-run "npm install" under ${arkEntryPath}`, reset);
344    return false;
345  }
346
347  return true;
348}
349
350function copyFileCachePathToBuildPath() {
351  for (let i = 0; i < intermediateJsBundle.length; ++i) {
352    const abcFile: string = intermediateJsBundle[i].path.replace(/\.temp\.js$/, ".abc");
353    const cacheOutputPath: string = intermediateJsBundle[i].cacheOutputPath;
354    const cacheAbcFilePath: string = intermediateJsBundle[i].cacheOutputPath.replace(/\.temp\.js$/, ".abc");
355    if (!fs.existsSync(cacheAbcFilePath)) {
356      logger.debug(red, `ArkTS:ERROR ${cacheAbcFilePath} is lost`, reset);
357      process.exitCode = FAIL;
358      break;
359    }
360    let parent: string = path.join(abcFile, '..');
361    if (!(fs.existsSync(parent) && fs.statSync(parent).isDirectory())) {
362      mkDir(parent);
363    }
364    if (process.env.cachePath !== undefined) {
365      fs.copyFileSync(cacheAbcFilePath, abcFile);
366    }
367    if (process.env.cachePath === undefined && fs.existsSync(cacheOutputPath)) {
368      fs.unlinkSync(cacheOutputPath);
369    }
370  }
371}
372