• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1// Arborist.rebuild({path = this.path}) will do all the binlinks and
2// bundle building needed.  Called by reify, and by `npm rebuild`.
3
4const localeCompare = require('@isaacs/string-locale-compare')('en')
5const { depth: dfwalk } = require('treeverse')
6const promiseAllRejectLate = require('promise-all-reject-late')
7const rpj = require('read-package-json-fast')
8const binLinks = require('bin-links')
9const runScript = require('@npmcli/run-script')
10const { callLimit: promiseCallLimit } = require('promise-call-limit')
11const { resolve } = require('path')
12const {
13  isNodeGypPackage,
14  defaultGypInstallScript,
15} = require('@npmcli/node-gyp')
16const log = require('proc-log')
17
18const boolEnv = b => b ? '1' : ''
19const sortNodes = (a, b) =>
20  (a.depth - b.depth) || localeCompare(a.path, b.path)
21
22const _checkBins = Symbol.for('checkBins')
23
24// defined by reify mixin
25const _handleOptionalFailure = Symbol.for('handleOptionalFailure')
26const _trashList = Symbol.for('trashList')
27
28module.exports = cls => class Builder extends cls {
29  #doHandleOptionalFailure
30  #oldMeta = null
31  #queues
32
33  constructor (options) {
34    super(options)
35
36    this.scriptsRun = new Set()
37    this.#resetQueues()
38  }
39
40  async rebuild ({ nodes, handleOptionalFailure = false } = {}) {
41    // nothing to do if we're not building anything!
42    if (this.options.ignoreScripts && !this.options.binLinks) {
43      return
44    }
45
46    // when building for the first time, as part of reify, we ignore
47    // failures in optional nodes, and just delete them.  however, when
48    // running JUST a rebuild, we treat optional failures as real fails
49    this.#doHandleOptionalFailure = handleOptionalFailure
50
51    if (!nodes) {
52      nodes = await this.#loadDefaultNodes()
53    }
54
55    // separates links nodes so that it can run
56    // prepare scripts and link bins in the expected order
57    process.emit('time', 'build')
58
59    const {
60      depNodes,
61      linkNodes,
62    } = this.#retrieveNodesByType(nodes)
63
64    // build regular deps
65    await this.#build(depNodes, {})
66
67    // build link deps
68    if (linkNodes.size) {
69      this.#resetQueues()
70      await this.#build(linkNodes, { type: 'links' })
71    }
72
73    process.emit('timeEnd', 'build')
74  }
75
76  // if we don't have a set of nodes, then just rebuild
77  // the actual tree on disk.
78  async #loadDefaultNodes () {
79    let nodes
80    const tree = await this.loadActual()
81    let filterSet
82    if (!this.options.workspacesEnabled) {
83      filterSet = this.excludeWorkspacesDependencySet(tree)
84      nodes = tree.inventory.filter(node =>
85        filterSet.has(node) || node.isProjectRoot
86      )
87    } else if (this.options.workspaces.length) {
88      filterSet = this.workspaceDependencySet(
89        tree,
90        this.options.workspaces,
91        this.options.includeWorkspaceRoot
92      )
93      nodes = tree.inventory.filter(node => filterSet.has(node))
94    } else {
95      nodes = tree.inventory.values()
96    }
97    return nodes
98  }
99
100  #retrieveNodesByType (nodes) {
101    const depNodes = new Set()
102    const linkNodes = new Set()
103    const storeNodes = new Set()
104
105    for (const node of nodes) {
106      if (node.isStoreLink) {
107        storeNodes.add(node)
108      } else if (node.isLink) {
109        linkNodes.add(node)
110      } else {
111        depNodes.add(node)
112      }
113    }
114    // Make sure that store linked nodes are processed last.
115    // We can't process store links separately or else lifecycle scripts on
116    // standard nodes might not have bin links yet.
117    for (const node of storeNodes) {
118      depNodes.add(node)
119    }
120
121    // deduplicates link nodes and their targets, avoids
122    // calling lifecycle scripts twice when running `npm rebuild`
123    // ref: https://github.com/npm/cli/issues/2905
124    //
125    // we avoid doing so if global=true since `bin-links` relies
126    // on having the target nodes available in global mode.
127    if (!this.options.global) {
128      for (const node of linkNodes) {
129        depNodes.delete(node.target)
130      }
131    }
132
133    return {
134      depNodes,
135      linkNodes,
136    }
137  }
138
139  #resetQueues () {
140    this.#queues = {
141      preinstall: [],
142      install: [],
143      postinstall: [],
144      prepare: [],
145      bin: [],
146    }
147  }
148
149  async #build (nodes, { type = 'deps' }) {
150    process.emit('time', `build:${type}`)
151
152    await this.#buildQueues(nodes)
153
154    if (!this.options.ignoreScripts) {
155      await this.#runScripts('preinstall')
156    }
157
158    // links should run prepare scripts and only link bins after that
159    if (type === 'links') {
160      await this.#runScripts('prepare')
161    }
162    if (this.options.binLinks) {
163      await this.#linkAllBins()
164    }
165
166    if (!this.options.ignoreScripts) {
167      await this.#runScripts('install')
168      await this.#runScripts('postinstall')
169    }
170
171    process.emit('timeEnd', `build:${type}`)
172  }
173
174  async #buildQueues (nodes) {
175    process.emit('time', 'build:queue')
176    const set = new Set()
177
178    const promises = []
179    for (const node of nodes) {
180      promises.push(this.#addToBuildSet(node, set))
181
182      // if it has bundle deps, add those too, if rebuildBundle
183      if (this.options.rebuildBundle !== false) {
184        const bd = node.package.bundleDependencies
185        if (bd && bd.length) {
186          dfwalk({
187            tree: node,
188            leave: node => promises.push(this.#addToBuildSet(node, set)),
189            getChildren: node => [...node.children.values()],
190            filter: node => node.inBundle,
191          })
192        }
193      }
194    }
195    await promiseAllRejectLate(promises)
196
197    // now sort into the queues for the 4 things we have to do
198    // run in the same predictable order that buildIdealTree uses
199    // there's no particular reason for doing it in this order rather
200    // than another, but sorting *somehow* makes it consistent.
201    const queue = [...set].sort(sortNodes)
202
203    for (const node of queue) {
204      const { package: { bin, scripts = {} } } = node.target
205      const { preinstall, install, postinstall, prepare } = scripts
206      const tests = { bin, preinstall, install, postinstall, prepare }
207      for (const [key, has] of Object.entries(tests)) {
208        if (has) {
209          this.#queues[key].push(node)
210        }
211      }
212    }
213    process.emit('timeEnd', 'build:queue')
214  }
215
216  async [_checkBins] (node) {
217    // if the node is a global top, and we're not in force mode, then
218    // any existing bins need to either be missing, or a symlink into
219    // the node path.  Otherwise a package can have a preinstall script
220    // that unlinks something, to allow them to silently overwrite system
221    // binaries, which is unsafe and insecure.
222    if (!node.globalTop || this.options.force) {
223      return
224    }
225    const { path, package: pkg } = node
226    await binLinks.checkBins({ pkg, path, top: true, global: true })
227  }
228
229  async #addToBuildSet (node, set, refreshed = false) {
230    if (set.has(node)) {
231      return
232    }
233
234    if (this.#oldMeta === null) {
235      const { root: { meta } } = node
236      this.#oldMeta = meta && meta.loadedFromDisk &&
237        !(meta.originalLockfileVersion >= 2)
238    }
239
240    const { package: pkg, hasInstallScript } = node.target
241    const { gypfile, bin, scripts = {} } = pkg
242
243    const { preinstall, install, postinstall, prepare } = scripts
244    const anyScript = preinstall || install || postinstall || prepare
245    if (!refreshed && !anyScript && (hasInstallScript || this.#oldMeta)) {
246      // we either have an old metadata (and thus might have scripts)
247      // or we have an indication that there's install scripts (but
248      // don't yet know what they are) so we have to load the package.json
249      // from disk to see what the deal is.  Failure here just means
250      // no scripts to add, probably borked package.json.
251      // add to the set then remove while we're reading the pj, so we
252      // don't accidentally hit it multiple times.
253      set.add(node)
254      const pkg = await rpj(node.path + '/package.json').catch(() => ({}))
255      set.delete(node)
256
257      const { scripts = {} } = pkg
258      node.package.scripts = scripts
259      return this.#addToBuildSet(node, set, true)
260    }
261
262    // Rebuild node-gyp dependencies lacking an install or preinstall script
263    // note that 'scripts' might be missing entirely, and the package may
264    // set gypfile:false to avoid this automatic detection.
265    const isGyp = gypfile !== false &&
266      !install &&
267      !preinstall &&
268      await isNodeGypPackage(node.path)
269
270    if (bin || preinstall || install || postinstall || prepare || isGyp) {
271      if (bin) {
272        await this[_checkBins](node)
273      }
274      if (isGyp) {
275        scripts.install = defaultGypInstallScript
276        node.package.scripts = scripts
277      }
278      set.add(node)
279    }
280  }
281
282  async #runScripts (event) {
283    const queue = this.#queues[event]
284
285    if (!queue.length) {
286      return
287    }
288
289    process.emit('time', `build:run:${event}`)
290    const stdio = this.options.foregroundScripts ? 'inherit' : 'pipe'
291    const limit = this.options.foregroundScripts ? 1 : undefined
292    await promiseCallLimit(queue.map(node => async () => {
293      const {
294        path,
295        integrity,
296        resolved,
297        optional,
298        peer,
299        dev,
300        devOptional,
301        package: pkg,
302        location,
303        isStoreLink,
304      } = node.target
305
306      // skip any that we know we'll be deleting
307      // or storeLinks
308      if (this[_trashList].has(path) || isStoreLink) {
309        return
310      }
311
312      const timer = `build:run:${event}:${location}`
313      process.emit('time', timer)
314      log.info('run', pkg._id, event, location, pkg.scripts[event])
315      const env = {
316        npm_package_resolved: resolved,
317        npm_package_integrity: integrity,
318        npm_package_json: resolve(path, 'package.json'),
319        npm_package_optional: boolEnv(optional),
320        npm_package_dev: boolEnv(dev),
321        npm_package_peer: boolEnv(peer),
322        npm_package_dev_optional:
323          boolEnv(devOptional && !dev && !optional),
324      }
325      const runOpts = {
326        event,
327        path,
328        pkg,
329        stdio,
330        env,
331        scriptShell: this.options.scriptShell,
332      }
333      const p = runScript(runOpts).catch(er => {
334        const { code, signal } = er
335        log.info('run', pkg._id, event, { code, signal })
336        throw er
337      }).then(({ args, code, signal, stdout, stderr }) => {
338        this.scriptsRun.add({
339          pkg,
340          path,
341          event,
342          // I do not know why this needs to be on THIS line but refactoring
343          // this function would be quite a process
344          // eslint-disable-next-line promise/always-return
345          cmd: args && args[args.length - 1],
346          env,
347          code,
348          signal,
349          stdout,
350          stderr,
351        })
352        log.info('run', pkg._id, event, { code, signal })
353      })
354
355      await (this.#doHandleOptionalFailure
356        ? this[_handleOptionalFailure](node, p)
357        : p)
358
359      process.emit('timeEnd', timer)
360    }), { limit })
361    process.emit('timeEnd', `build:run:${event}`)
362  }
363
364  async #linkAllBins () {
365    const queue = this.#queues.bin
366    if (!queue.length) {
367      return
368    }
369
370    process.emit('time', 'build:link')
371    const promises = []
372    // sort the queue by node path, so that the module-local collision
373    // detector in bin-links will always resolve the same way.
374    for (const node of queue.sort(sortNodes)) {
375      // TODO these run before they're awaited
376      promises.push(this.#createBinLinks(node))
377    }
378
379    await promiseAllRejectLate(promises)
380    process.emit('timeEnd', 'build:link')
381  }
382
383  async #createBinLinks (node) {
384    if (this[_trashList].has(node.path)) {
385      return
386    }
387
388    process.emit('time', `build:link:${node.location}`)
389
390    const p = binLinks({
391      pkg: node.package,
392      path: node.path,
393      top: !!(node.isTop || node.globalTop),
394      force: this.options.force,
395      global: !!node.globalTop,
396    })
397
398    await (this.#doHandleOptionalFailure
399      ? this[_handleOptionalFailure](node, p)
400      : p)
401
402    process.emit('timeEnd', `build:link:${node.location}`)
403  }
404}
405