• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1'use strict'
2
3const BB = require('bluebird')
4
5const binLink = require('bin-links')
6const buildLogicalTree = require('npm-logical-tree')
7const extract = require('./lib/extract.js')
8const figgyPudding = require('figgy-pudding')
9const fs = require('graceful-fs')
10const getPrefix = require('find-npm-prefix')
11const lifecycle = require('npm-lifecycle')
12const lockVerify = require('lock-verify')
13const mkdirp = BB.promisify(require('mkdirp'))
14const npa = require('npm-package-arg')
15const path = require('path')
16const readPkgJson = BB.promisify(require('read-package-json'))
17const rimraf = BB.promisify(require('rimraf'))
18
19const readFileAsync = BB.promisify(fs.readFile)
20const statAsync = BB.promisify(fs.stat)
21const symlinkAsync = BB.promisify(fs.symlink)
22const writeFileAsync = BB.promisify(fs.writeFile)
23
24const LifecycleOpts = figgyPudding({
25  config: {},
26  'script-shell': {},
27  scriptShell: 'script-shell',
28  'ignore-scripts': {},
29  ignoreScripts: 'ignore-scripts',
30  'ignore-prepublish': {},
31  ignorePrepublish: 'ignore-prepublish',
32  'scripts-prepend-node-path': {},
33  scriptsPrependNodePath: 'scripts-prepend-node-path',
34  'unsafe-perm': {},
35  unsafePerm: 'unsafe-perm',
36  prefix: {},
37  dir: 'prefix',
38  failOk: { default: false }
39}, { other () { return true } })
40
41class Installer {
42  constructor (opts) {
43    this.opts = opts
44
45    // Stats
46    this.startTime = Date.now()
47    this.runTime = 0
48    this.timings = { scripts: 0 }
49    this.pkgCount = 0
50
51    // Misc
52    this.log = this.opts.log || require('./lib/silentlog.js')
53    this.pkg = null
54    this.tree = null
55    this.failedDeps = new Set()
56  }
57
58  timedStage (name) {
59    const start = Date.now()
60    return BB.resolve(this[name].apply(this, [].slice.call(arguments, 1)))
61      .tap(() => {
62        this.timings[name] = Date.now() - start
63        this.log.info(name, `Done in ${this.timings[name] / 1000}s`)
64      })
65  }
66
67  run () {
68    return this.timedStage('prepare')
69      .then(() => this.timedStage('extractTree', this.tree))
70      .then(() => this.timedStage('updateJson', this.tree))
71      .then(pkgJsons => this.timedStage('buildTree', this.tree, pkgJsons))
72      .then(() => this.timedStage('garbageCollect', this.tree))
73      .then(() => this.timedStage('runScript', 'prepublish', this.pkg, this.prefix))
74      .then(() => this.timedStage('runScript', 'prepare', this.pkg, this.prefix))
75      .then(() => this.timedStage('teardown'))
76      .then(() => {
77        this.runTime = Date.now() - this.startTime
78        this.log.info(
79          'run-scripts',
80          `total script time: ${this.timings.scripts / 1000}s`
81        )
82        this.log.info(
83          'run-time',
84          `total run time: ${this.runTime / 1000}s`
85        )
86      })
87      .catch(err => {
88        this.timedStage('teardown')
89        if (err.message.match(/aggregate error/)) {
90          throw err[0]
91        } else {
92          throw err
93        }
94      })
95      .then(() => this)
96  }
97
98  prepare () {
99    this.log.info('prepare', 'initializing installer')
100    this.log.level = this.opts.loglevel
101    this.log.verbose('prepare', 'starting workers')
102    extract.startWorkers()
103
104    return (
105      this.opts.prefix && this.opts.global
106        ? BB.resolve(this.opts.prefix)
107        // There's some Special™ logic around the `--prefix` config when it
108        // comes from a config file or env vs when it comes from the CLI
109        : process.argv.some(arg => arg.match(/^\s*--prefix\s*/i))
110          ? BB.resolve(this.opts.prefix)
111          : getPrefix(process.cwd())
112    )
113      .then(prefix => {
114        this.prefix = prefix
115        this.log.verbose('prepare', 'installation prefix: ' + prefix)
116        return BB.join(
117          readJson(prefix, 'package.json'),
118          readJson(prefix, 'package-lock.json', true),
119          readJson(prefix, 'npm-shrinkwrap.json', true),
120          (pkg, lock, shrink) => {
121            if (shrink) {
122              this.log.verbose('prepare', 'using npm-shrinkwrap.json')
123            } else if (lock) {
124              this.log.verbose('prepare', 'using package-lock.json')
125            }
126            pkg._shrinkwrap = shrink || lock
127            this.pkg = pkg
128          }
129        )
130      })
131      .then(() => statAsync(
132        path.join(this.prefix, 'node_modules')
133      ).catch(err => { if (err.code !== 'ENOENT') { throw err } }))
134      .then(stat => {
135        stat && this.log.warn(
136          'prepare', 'removing existing node_modules/ before installation'
137        )
138        return BB.join(
139          this.checkLock(),
140          stat && rimraf(path.join(this.prefix, 'node_modules/*'))
141        )
142      }).then(() => {
143      // This needs to happen -after- we've done checkLock()
144        this.tree = buildLogicalTree(this.pkg, this.pkg._shrinkwrap)
145        this.log.silly('tree', this.tree)
146        this.expectedTotal = 0
147        this.tree.forEach((dep, next) => {
148          this.expectedTotal++
149          next()
150        })
151      })
152  }
153
154  teardown () {
155    this.log.verbose('teardown', 'shutting down workers.')
156    return extract.stopWorkers()
157  }
158
159  checkLock () {
160    this.log.verbose('checkLock', 'verifying package-lock data')
161    const pkg = this.pkg
162    const prefix = this.prefix
163    if (!pkg._shrinkwrap || !pkg._shrinkwrap.lockfileVersion) {
164      return BB.reject(
165        new Error(`cipm can only install packages with an existing package-lock.json or npm-shrinkwrap.json with lockfileVersion >= 1. Run an install with npm@5 or later to generate it, then try again.`)
166      )
167    }
168    return lockVerify(prefix).then(result => {
169      if (result.status) {
170        result.warnings.forEach(w => this.log.warn('lockfile', w))
171      } else {
172        throw new Error(
173          'cipm can only install packages when your package.json and package-lock.json or ' +
174          'npm-shrinkwrap.json are in sync. Please update your lock file with `npm install` ' +
175          'before continuing.\n\n' +
176          result.warnings.map(w => 'Warning: ' + w).join('\n') + '\n' +
177          result.errors.join('\n') + '\n'
178        )
179      }
180    }).catch(err => {
181      throw err
182    })
183  }
184
185  extractTree (tree) {
186    this.log.verbose('extractTree', 'extracting dependencies to node_modules/')
187    const cg = this.log.newItem('extractTree', this.expectedTotal)
188    return tree.forEachAsync((dep, next) => {
189      if (!this.checkDepEnv(dep)) { return }
190      const depPath = dep.path(this.prefix)
191      const spec = npa.resolve(dep.name, dep.version, this.prefix)
192      if (dep.isRoot) {
193        return next()
194      } else if (spec.type === 'directory') {
195        const relative = path.relative(path.dirname(depPath), spec.fetchSpec)
196        this.log.silly('extractTree', `${dep.name}@${spec.fetchSpec} -> ${depPath} (symlink)`)
197        return mkdirp(path.dirname(depPath))
198          .then(() => symlinkAsync(relative, depPath, 'junction'))
199          .catch(
200            () => rimraf(depPath)
201              .then(() => symlinkAsync(relative, depPath, 'junction'))
202          ).then(() => next())
203          .then(() => {
204            this.pkgCount++
205            cg.completeWork(1)
206          })
207      } else {
208        this.log.silly('extractTree', `${dep.name}@${dep.version} -> ${depPath}`)
209        return (
210          dep.bundled
211            ? statAsync(path.join(depPath, 'package.json')).catch(err => {
212              if (err.code !== 'ENOENT') { throw err }
213            })
214            : BB.resolve(false)
215        )
216          .then(wasBundled => {
217          // Don't extract if a bundled dep is actually present
218            if (wasBundled) {
219              cg.completeWork(1)
220              return next()
221            } else {
222              return BB.resolve(extract.child(
223                dep.name, dep, depPath, this.opts
224              ))
225                .then(() => cg.completeWork(1))
226                .then(() => { this.pkgCount++ })
227                .then(next)
228            }
229          })
230      }
231    }, {concurrency: 50, Promise: BB})
232      .then(() => cg.finish())
233  }
234
235  checkDepEnv (dep) {
236    const includeDev = (
237      // Covers --dev and --development (from npm config itself)
238      this.opts.dev ||
239      (
240        !/^prod(uction)?$/.test(this.opts.only) &&
241        !this.opts.production
242      ) ||
243      /^dev(elopment)?$/.test(this.opts.only) ||
244      /^dev(elopment)?$/.test(this.opts.also)
245    )
246    const includeProd = !/^dev(elopment)?$/.test(this.opts.only)
247    const includeOptional = includeProd && this.opts.optional
248    return (dep.dev && includeDev) ||
249      (dep.optional && includeOptional) ||
250      (!dep.dev && !dep.optional && includeProd)
251  }
252
253  updateJson (tree) {
254    this.log.verbose('updateJson', 'updating json deps to include _from')
255    const pkgJsons = new Map()
256    return tree.forEachAsync((dep, next) => {
257      if (!this.checkDepEnv(dep)) { return }
258      const spec = npa.resolve(dep.name, dep.version)
259      const depPath = dep.path(this.prefix)
260      return next()
261        .then(() => readJson(depPath, 'package.json'))
262        .then(pkg => (spec.registry || spec.type === 'directory')
263          ? pkg
264          : this.updateFromField(dep, pkg).then(() => pkg)
265        )
266        .then(pkg => (pkg.scripts && pkg.scripts.install)
267          ? pkg
268          : this.updateInstallScript(dep, pkg).then(() => pkg)
269        )
270        .tap(pkg => { pkgJsons.set(dep, pkg) })
271    }, {concurrency: 100, Promise: BB})
272      .then(() => pkgJsons)
273  }
274
275  buildTree (tree, pkgJsons) {
276    this.log.verbose('buildTree', 'finalizing tree and running scripts')
277    return tree.forEachAsync((dep, next) => {
278      if (!this.checkDepEnv(dep)) { return }
279      const spec = npa.resolve(dep.name, dep.version)
280      const depPath = dep.path(this.prefix)
281      const pkg = pkgJsons.get(dep)
282      this.log.silly('buildTree', `linking ${spec}`)
283      return this.runScript('preinstall', pkg, depPath)
284        .then(next) // build children between preinstall and binLink
285      // Don't link root bins
286        .then(() => {
287          if (
288            dep.isRoot ||
289          !(pkg.bin || pkg.man || (pkg.directories && pkg.directories.bin))
290          ) {
291          // We skip the relatively expensive readPkgJson if there's no way
292          // we'll actually be linking any bins or mans
293            return
294          }
295          return readPkgJson(path.join(depPath, 'package.json'))
296            .then(pkg => binLink(pkg, depPath, false, {
297              force: this.opts.force,
298              ignoreScripts: this.opts['ignore-scripts'],
299              log: Object.assign({}, this.log, { info: () => {} }),
300              name: pkg.name,
301              pkgId: pkg.name + '@' + pkg.version,
302              prefix: this.prefix,
303              prefixes: [this.prefix],
304              umask: this.opts.umask
305            }), e => {
306              this.log.verbose('buildTree', `error linking ${spec}: ${e.message} ${e.stack}`)
307            })
308        })
309        .then(() => this.runScript('install', pkg, depPath))
310        .then(() => this.runScript('postinstall', pkg, depPath))
311        .then(() => this)
312        .catch(e => {
313          if (dep.optional) {
314            this.failedDeps.add(dep)
315          } else {
316            throw e
317          }
318        })
319    }, {concurrency: 1, Promise: BB})
320  }
321
322  updateFromField (dep, pkg) {
323    const depPath = dep.path(this.prefix)
324    const depPkgPath = path.join(depPath, 'package.json')
325    const parent = dep.requiredBy.values().next().value
326    return readJson(parent.path(this.prefix), 'package.json')
327      .then(ppkg =>
328        (ppkg.dependencies && ppkg.dependencies[dep.name]) ||
329      (ppkg.devDependencies && ppkg.devDependencies[dep.name]) ||
330      (ppkg.optionalDependencies && ppkg.optionalDependencies[dep.name])
331      )
332      .then(from => npa.resolve(dep.name, from))
333      .then(from => { pkg._from = from.toString() })
334      .then(() => writeFileAsync(depPkgPath, JSON.stringify(pkg, null, 2)))
335      .then(() => pkg)
336  }
337
338  updateInstallScript (dep, pkg) {
339    const depPath = dep.path(this.prefix)
340    return statAsync(path.join(depPath, 'binding.gyp'))
341      .catch(err => { if (err.code !== 'ENOENT') { throw err } })
342      .then(stat => {
343        if (stat) {
344          if (!pkg.scripts) {
345            pkg.scripts = {}
346          }
347          pkg.scripts.install = 'node-gyp rebuild'
348        }
349      })
350      .then(() => pkg)
351  }
352
353  // A cute little mark-and-sweep collector!
354  garbageCollect (tree) {
355    if (!this.failedDeps.size) { return }
356    return sweep(
357      tree,
358      this.prefix,
359      mark(tree, this.failedDeps)
360    )
361      .then(purged => {
362        this.purgedDeps = purged
363        this.pkgCount -= purged.size
364      })
365  }
366
367  runScript (stage, pkg, pkgPath) {
368    const start = Date.now()
369    if (!this.opts['ignore-scripts']) {
370      // TODO(mikesherov): remove pkg._id when npm-lifecycle no longer relies on it
371      pkg._id = pkg.name + '@' + pkg.version
372      return BB.resolve(lifecycle(
373        pkg, stage, pkgPath, LifecycleOpts(this.opts).concat({
374          // TODO: can be removed once npm-lifecycle is updated to modern
375          //       config practices.
376          config: Object.assign({}, this.opts, {
377            log: null,
378            dirPacker: null
379          }),
380          dir: this.prefix
381        }))
382      ).tap(() => { this.timings.scripts += Date.now() - start })
383    }
384    return BB.resolve()
385  }
386}
387module.exports = Installer
388
389function mark (tree, failed) {
390  const liveDeps = new Set()
391  tree.forEach((dep, next) => {
392    if (!failed.has(dep)) {
393      liveDeps.add(dep)
394      next()
395    }
396  })
397  return liveDeps
398}
399
400function sweep (tree, prefix, liveDeps) {
401  const purged = new Set()
402  return tree.forEachAsync((dep, next) => {
403    return next().then(() => {
404      if (
405        !dep.isRoot && // never purge root! ��
406        !liveDeps.has(dep) &&
407        !purged.has(dep)
408      ) {
409        purged.add(dep)
410        return rimraf(dep.path(prefix))
411      }
412    })
413  }, {concurrency: 50, Promise: BB}).then(() => purged)
414}
415
416function stripBOM (str) {
417  return str.replace(/^\uFEFF/, '')
418}
419
420module.exports._readJson = readJson
421function readJson (jsonPath, name, ignoreMissing) {
422  return readFileAsync(path.join(jsonPath, name), 'utf8')
423    .then(str => JSON.parse(stripBOM(str)))
424    .catch({code: 'ENOENT'}, err => {
425      if (!ignoreMissing) {
426        throw err
427      }
428    })
429}
430