npm-packlist/lib/index.js

'use strict'

const { Walker: IgnoreWalker } = require('ignore-walk')
const { lstatSync: lstat, readFileSync: readFile } = require('fs')
const { basename, dirname, extname, join, relative, resolve, sep } = require('path')

// symbols used to represent synthetic rule sets
const defaultRules = Symbol('npm-packlist.rules.default')
const strictRules = Symbol('npm-packlist.rules.strict')

// There may be others, but :?|<> are handled by node-tar
const nameIsBadForWindows = file => /\*/.test(file)

// these are the default rules that are applied to everything except for non-link bundled deps
const defaults = [
  '.npmignore',
  '.gitignore',
  '**/.git',
  '**/.svn',
  '**/.hg',
  '**/CVS',
  '**/.git/**',
  '**/.svn/**',
  '**/.hg/**',
  '**/CVS/**',
  '/.lock-wscript',
  '/.wafpickle-*',
  '/build/config.gypi',
  'npm-debug.log',
  '**/.npmrc',
  '.*.swp',
  '.DS_Store',
  '**/.DS_Store/**',
  '._*',
  '**/._*/**',
  '*.orig',
  '/archived-packages/**',
]

const strictDefaults = [
  // these are forcibly excluded
  '/.git',
]

const normalizePath = (path) => path.split('\\').join('/')

const readOutOfTreeIgnoreFiles = (root, rel, result = []) => {
  for (const file of ['.npmignore', '.gitignore']) {
    try {
      const ignoreContent = readFile(join(root, file), { encoding: 'utf8' })
      result.push(ignoreContent)
      // break the loop immediately after reading, this allows us to prioritize
      // the .npmignore and discard the .gitignore if one is present
      break
    } catch (err) {
      // we ignore ENOENT errors completely because we don't care if the file doesn't exist
      // but we throw everything else because failing to read a file that does exist is
      // something that the user likely wants to know about
      // istanbul ignore next -- we do not need to test a thrown error
      if (err.code !== 'ENOENT') {
        throw err
      }
    }
  }

  if (!rel) {
    return result
  }

  const firstRel = rel.split(sep, 1)[0]
  const newRoot = join(root, firstRel)
  const newRel = relative(newRoot, join(root, rel))

  return readOutOfTreeIgnoreFiles(newRoot, newRel, result)
}

class PackWalker extends IgnoreWalker {
  constructor (tree, opts) {
    const options = {
      ...opts,
      includeEmpty: false,
      follow: false,
      // we path.resolve() here because ignore-walk doesn't do it and we want full paths
      path: resolve(opts?.path || tree.path).replace(/\\/g, '/'),
      ignoreFiles: opts?.ignoreFiles || [
        defaultRules,
        'package.json',
        '.npmignore',
        '.gitignore',
        strictRules,
      ],
    }

    super(options)
    this.isPackage = options.isPackage
    this.seen = options.seen || new Set()
    this.tree = tree
    this.requiredFiles = options.requiredFiles || []

    const additionalDefaults = []
    if (options.prefix && options.workspaces) {
      const path = normalizePath(options.path)
      const prefix = normalizePath(options.prefix)
      const workspaces = options.workspaces.map((ws) => normalizePath(ws))

      // istanbul ignore else - this does nothing unless we need it to
      if (path !== prefix && workspaces.includes(path)) {
        // if path and prefix are not the same directory, and workspaces has path in it
        // then we know path is a workspace directory. in order to not drop ignore rules
        // from directories between the workspaces root (prefix) and the workspace itself
        // (path) we need to find and read those now
        const relpath = relative(options.prefix, dirname(options.path))
        additionalDefaults.push(...readOutOfTreeIgnoreFiles(options.prefix, relpath))
      } else if (path === prefix) {
        // on the other hand, if the path and prefix are the same, then we ignore workspaces
        // so that we don't pack a workspace as part of the root project. append them as
        // normalized relative paths from the root
        additionalDefaults.push(...workspaces.map((w) => normalizePath(relative(options.path, w))))
      }
    }

    // go ahead and inject the default rules now
    this.injectRules(defaultRules, [...defaults, ...additionalDefaults])

    if (!this.isPackage) {
      // if this instance is not a package, then place some strict default rules, and append
      // known required files for this directory
      this.injectRules(strictRules, [
        ...strictDefaults,
        ...this.requiredFiles.map((file) => `!${file}`),
      ])
    }
  }

  // overridden method: we intercept the reading of the package.json file here so that we can
  // process it into both the package.json file rules as well as the strictRules synthetic rule set
  addIgnoreFile (file, callback) {
    // if we're adding anything other than package.json, then let ignore-walk handle it
    if (file !== 'package.json' || !this.isPackage) {
      return super.addIgnoreFile(file, callback)
    }

    return this.processPackage(callback)
  }

  // overridden method: if we're done, but we're a package, then we also need to evaluate bundles
  // before we actually emit our done event
  emit (ev, data) {
    if (ev !== 'done' || !this.isPackage) {
      return super.emit(ev, data)
    }

    // we intentionally delay the done event while keeping the function sync here
    // eslint-disable-next-line promise/catch-or-return, promise/always-return
    this.gatherBundles().then(() => {
      super.emit('done', this.result)
    })
    return true
  }

  // overridden method: before actually filtering, we make sure that we've removed the rules for
  // files that should no longer take effect due to our order of precedence
  filterEntries () {
    if (this.ignoreRules['package.json']) {
      // package.json means no .npmignore or .gitignore
      this.ignoreRules['.npmignore'] = null
      this.ignoreRules['.gitignore'] = null
    } else if (this.ignoreRules['.npmignore']) {
      // .npmignore means no .gitignore
      this.ignoreRules['.gitignore'] = null
    }

    return super.filterEntries()
  }

  // overridden method: we never want to include anything that isn't a file or directory
  onstat (opts, callback) {
    if (!opts.st.isFile() && !opts.st.isDirectory()) {
      return callback()
    }

    return super.onstat(opts, callback)
  }

  // overridden method: we want to refuse to pack files that are invalid, node-tar protects us from
  // a lot of them but not all
  stat (opts, callback) {
    if (nameIsBadForWindows(opts.entry)) {
      return callback()
    }

    return super.stat(opts, callback)
  }

  // overridden method: this is called to create options for a child walker when we step
  // in to a normal child directory (this will never be a bundle). the default method here
  // copies the root's `ignoreFiles` value, but we don't want to respect package.json for
  // subdirectories, so we override it with a list that intentionally omits package.json
  walkerOpt (entry, opts) {
    let ignoreFiles = null

    // however, if we have a tree, and we have workspaces, and the directory we're about
    // to step into is a workspace, then we _do_ want to respect its package.json
    if (this.tree.workspaces) {
      const workspaceDirs = [...this.tree.workspaces.values()]
        .map((dir) => dir.replace(/\\/g, '/'))

      const entryPath = join(this.path, entry).replace(/\\/g, '/')
      if (workspaceDirs.includes(entryPath)) {
        ignoreFiles = [
          defaultRules,
          'package.json',
          '.npmignore',
          '.gitignore',
          strictRules,
        ]
      }
    } else {
      ignoreFiles = [
        defaultRules,
        '.npmignore',
        '.gitignore',
        strictRules,
      ]
    }

    return {
      ...super.walkerOpt(entry, opts),
      ignoreFiles,
      // we map over our own requiredFiles and pass ones that are within this entry
      requiredFiles: this.requiredFiles
        .map((file) => {
          if (relative(file, entry) === '..') {
            return relative(entry, file).replace(/\\/g, '/')
          }
          return false
        })
        .filter(Boolean),
    }
  }

  // overridden method: we want child walkers to be instances of this class, not ignore-walk
  walker (entry, opts, callback) {
    new PackWalker(this.tree, this.walkerOpt(entry, opts)).on('done', callback).start()
  }

  // overridden method: we use a custom sort method to help compressibility
  sort (a, b) {
    // optimize for compressibility
    // extname, then basename, then locale alphabetically
    // https://twitter.com/isntitvacant/status/1131094910923231232
    const exta = extname(a).toLowerCase()
    const extb = extname(b).toLowerCase()
    const basea = basename(a).toLowerCase()
    const baseb = basename(b).toLowerCase()

    return exta.localeCompare(extb, 'en') ||
      basea.localeCompare(baseb, 'en') ||
      a.localeCompare(b, 'en')
  }

  // convenience method: this joins the given rules with newlines, appends a trailing newline,
  // and calls the internal onReadIgnoreFile method
  injectRules (filename, rules, callback = () => {}) {
    this.onReadIgnoreFile(filename, `${rules.join('\n')}\n`, callback)
  }

  // custom method: this is called by addIgnoreFile when we find a package.json, it uses the
  // arborist tree to pull both default rules and strict rules for the package
  processPackage (callback) {
    const {
      bin,
      browser,
      files,
      main,
    } = this.tree.package

    // rules in these arrays are inverted since they are patterns we want to _not_ ignore
    const ignores = []
    const strict = [
      ...strictDefaults,
      '!/package.json',
      '!/readme{,.*[^~$]}',
      '!/copying{,.*[^~$]}',
      '!/license{,.*[^~$]}',
      '!/licence{,.*[^~$]}',
      '/.git',
      '/node_modules',
      '.npmrc',
      '/package-lock.json',
      '/yarn.lock',
      '/pnpm-lock.yaml',
    ]

    // if we have a files array in our package, we need to pull rules from it
    if (files) {
      for (let file of files) {
        // invert the rule because these are things we want to include
        if (file.startsWith('./')) {
          file = file.slice(1)
        }
        if (file.endsWith('/*')) {
          file += '*'
        }
        const inverse = `!${file}`
        try {
          // if an entry in the files array is a specific file, then we need to include it as a
          // strict requirement for this package. if it's a directory or a pattern, it's a default
          // pattern instead. this is ugly, but we have to stat to find out if it's a file
          const stat = lstat(join(this.path, file.replace(/^!+/, '')).replace(/\\/g, '/'))
          // if we have a file and we know that, it's strictly required
          if (stat.isFile()) {
            strict.unshift(inverse)
            this.requiredFiles.push(file.startsWith('/') ? file.slice(1) : file)
          } else if (stat.isDirectory()) {
            // otherwise, it's a default ignore, and since we got here we know it's not a pattern
            // so we include the directory contents
            ignores.push(inverse)
            ignores.push(`${inverse}/**`)
          }
          // if the thing exists, but is neither a file or a directory, we don't want it at all
        } catch (err) {
          // if lstat throws, then we assume we're looking at a pattern and treat it as a default
          ignores.push(inverse)
        }
      }

      // we prepend a '*' to exclude everything, followed by our inverted file rules
      // which now mean to include those
      this.injectRules('package.json', ['*', ...ignores])
    }

    // browser is required
    if (browser) {
      strict.push(`!/${browser}`)
    }

    // main is required
    if (main) {
      strict.push(`!/${main}`)
    }

    // each bin is required
    if (bin) {
      for (const key in bin) {
        strict.push(`!/${bin[key]}`)
      }
    }

    // and now we add all of the strict rules to our synthetic file
    this.injectRules(strictRules, strict, callback)
  }

  // custom method: after we've finished gathering the files for the root package, we call this
  // before emitting the 'done' event in order to gather all of the files for bundled deps
  async gatherBundles () {
    if (this.seen.has(this.tree)) {
      return
    }

    // add this node to our seen tracker
    this.seen.add(this.tree)

    // if we're the project root, then we look at our bundleDependencies, otherwise we got here
    // because we're a bundled dependency of the root, which means we need to include all prod
    // and optional dependencies in the bundle
    let toBundle
    if (this.tree.isProjectRoot) {
      const { bundleDependencies } = this.tree.package
      toBundle = bundleDependencies || []
    } else {
      const { dependencies, optionalDependencies } = this.tree.package
      toBundle = Object.keys(dependencies || {}).concat(Object.keys(optionalDependencies || {}))
    }

    for (const dep of toBundle) {
      const edge = this.tree.edgesOut.get(dep)
      // no edgeOut = missing node, so skip it. we can't pack it if it's not here
      // we also refuse to pack peer dependencies and dev dependencies
      if (!edge || edge.peer || edge.dev) {
        continue
      }

      // get a reference to the node we're bundling
      const node = this.tree.edgesOut.get(dep).to
      // if there's no node, this is most likely an optional dependency that hasn't been
      // installed. just skip it.
      if (!node) {
        continue
      }
      // we use node.path for the path because we want the location the node was linked to,
      // not where it actually lives on disk
      const path = node.path
      // but link nodes don't have edgesOut, so we need to pass in the target of the node
      // in order to make sure we correctly traverse its dependencies
      const tree = node.target

      // and start building options to be passed to the walker for this package
      const walkerOpts = {
        path,
        isPackage: true,
        ignoreFiles: [],
        seen: this.seen, // pass through seen so we can prevent infinite circular loops
      }

      // if our node is a link, we apply defaultRules. we don't do this for regular bundled
      // deps because their .npmignore and .gitignore files are excluded by default and may
      // override defaults
      if (node.isLink) {
        walkerOpts.ignoreFiles.push(defaultRules)
      }

      // _all_ nodes will follow package.json rules from their package root
      walkerOpts.ignoreFiles.push('package.json')

      // only link nodes will obey .npmignore or .gitignore
      if (node.isLink) {
        walkerOpts.ignoreFiles.push('.npmignore')
        walkerOpts.ignoreFiles.push('.gitignore')
      }

      // _all_ nodes follow strict rules
      walkerOpts.ignoreFiles.push(strictRules)

      // create a walker for this dependency and gather its results
      const walker = new PackWalker(tree, walkerOpts)
      const bundled = await new Promise((pResolve, pReject) => {
        walker.on('error', pReject)
        walker.on('done', pResolve)
        walker.start()
      })

      // now we make sure we have our paths correct from the root, and accumulate everything into
      // our own result set to deduplicate
      const relativeFrom = relative(this.root, walker.path)
      for (const file of bundled) {
        this.result.add(join(relativeFrom, file).replace(/\\/g, '/'))
      }
    }
  }
}

const walk = (tree, options, callback) => {
  if (typeof options === 'function') {
    callback = options
    options = {}
  }
  const p = new Promise((pResolve, pReject) => {
    new PackWalker(tree, { ...options, isPackage: true })
      .on('done', pResolve).on('error', pReject).start()
  })
  return callback ? p.then(res => callback(null, res), callback) : p
}

module.exports = walk
walk.Walker = PackWalker