• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1'use strict'
2
3let Cache
4const url = require('url')
5const CachePolicy = require('http-cache-semantics')
6const fetch = require('node-fetch-npm')
7const pkg = require('./package.json')
8const retry = require('promise-retry')
9let ssri
10const Stream = require('stream')
11const getAgent = require('./agent')
12const setWarning = require('./warning')
13
14const isURL = /^https?:/
15const USER_AGENT = `${pkg.name}/${pkg.version} (+https://npm.im/${pkg.name})`
16
17const RETRY_ERRORS = [
18  'ECONNRESET', // remote socket closed on us
19  'ECONNREFUSED', // remote host refused to open connection
20  'EADDRINUSE', // failed to bind to a local port (proxy?)
21  'ETIMEDOUT' // someone in the transaction is WAY TOO SLOW
22  // Known codes we do NOT retry on:
23  // ENOTFOUND (getaddrinfo failure. Either bad hostname, or offline)
24]
25
26const RETRY_TYPES = [
27  'request-timeout'
28]
29
30// https://fetch.spec.whatwg.org/#http-network-or-cache-fetch
31module.exports = cachingFetch
32cachingFetch.defaults = function (_uri, _opts) {
33  const fetch = this
34  if (typeof _uri === 'object') {
35    _opts = _uri
36    _uri = null
37  }
38
39  function defaultedFetch (uri, opts) {
40    const finalOpts = Object.assign({}, _opts || {}, opts || {})
41    return fetch(uri || _uri, finalOpts)
42  }
43
44  defaultedFetch.defaults = fetch.defaults
45  defaultedFetch.delete = fetch.delete
46  return defaultedFetch
47}
48
49cachingFetch.delete = cacheDelete
50function cacheDelete (uri, opts) {
51  opts = configureOptions(opts)
52  if (opts.cacheManager) {
53    const req = new fetch.Request(uri, {
54      method: opts.method,
55      headers: opts.headers
56    })
57    return opts.cacheManager.delete(req, opts)
58  }
59}
60
61function initializeCache (opts) {
62  if (typeof opts.cacheManager === 'string') {
63    if (!Cache) {
64      // Default cacache-based cache
65      Cache = require('./cache')
66    }
67
68    opts.cacheManager = new Cache(opts.cacheManager, opts)
69  }
70
71  opts.cache = opts.cache || 'default'
72
73  if (opts.cache === 'default' && isHeaderConditional(opts.headers)) {
74    // If header list contains `If-Modified-Since`, `If-None-Match`,
75    // `If-Unmodified-Since`, `If-Match`, or `If-Range`, fetch will set cache
76    // mode to "no-store" if it is "default".
77    opts.cache = 'no-store'
78  }
79}
80
81function configureOptions (_opts) {
82  const opts = Object.assign({}, _opts || {})
83  opts.method = (opts.method || 'GET').toUpperCase()
84
85  if (opts.retry && typeof opts.retry === 'number') {
86    opts.retry = { retries: opts.retry }
87  }
88
89  if (opts.retry === false) {
90    opts.retry = { retries: 0 }
91  }
92
93  if (opts.cacheManager) {
94    initializeCache(opts)
95  }
96
97  return opts
98}
99
100function initializeSsri () {
101  if (!ssri) {
102    ssri = require('ssri')
103  }
104}
105
106function cachingFetch (uri, _opts) {
107  const opts = configureOptions(_opts)
108
109  if (opts.integrity) {
110    initializeSsri()
111    // if verifying integrity, node-fetch must not decompress
112    opts.compress = false
113  }
114
115  const isCachable = (opts.method === 'GET' || opts.method === 'HEAD') &&
116    opts.cacheManager &&
117    opts.cache !== 'no-store' &&
118    opts.cache !== 'reload'
119
120  if (isCachable) {
121    const req = new fetch.Request(uri, {
122      method: opts.method,
123      headers: opts.headers
124    })
125
126    return opts.cacheManager.match(req, opts).then(res => {
127      if (res) {
128        const warningCode = (res.headers.get('Warning') || '').match(/^\d+/)
129        if (warningCode && +warningCode >= 100 && +warningCode < 200) {
130          // https://tools.ietf.org/html/rfc7234#section-4.3.4
131          //
132          // If a stored response is selected for update, the cache MUST:
133          //
134          // * delete any Warning header fields in the stored response with
135          //   warn-code 1xx (see Section 5.5);
136          //
137          // * retain any Warning header fields in the stored response with
138          //   warn-code 2xx;
139          //
140          res.headers.delete('Warning')
141        }
142
143        if (opts.cache === 'default' && !isStale(req, res)) {
144          return res
145        }
146
147        if (opts.cache === 'default' || opts.cache === 'no-cache') {
148          return conditionalFetch(req, res, opts)
149        }
150
151        if (opts.cache === 'force-cache' || opts.cache === 'only-if-cached') {
152          //   112 Disconnected operation
153          // SHOULD be included if the cache is intentionally disconnected from
154          // the rest of the network for a period of time.
155          // (https://tools.ietf.org/html/rfc2616#section-14.46)
156          setWarning(res, 112, 'Disconnected operation')
157          return res
158        }
159      }
160
161      if (!res && opts.cache === 'only-if-cached') {
162        const errorMsg = `request to ${
163          uri
164        } failed: cache mode is 'only-if-cached' but no cached response available.`
165
166        const err = new Error(errorMsg)
167        err.code = 'ENOTCACHED'
168        throw err
169      }
170
171      // Missing cache entry, or mode is default (if stale), reload, no-store
172      return remoteFetch(req.url, opts)
173    })
174  }
175
176  return remoteFetch(uri, opts)
177}
178
179function iterableToObject (iter) {
180  const obj = {}
181  for (let k of iter.keys()) {
182    obj[k] = iter.get(k)
183  }
184  return obj
185}
186
187function makePolicy (req, res) {
188  const _req = {
189    url: req.url,
190    method: req.method,
191    headers: iterableToObject(req.headers)
192  }
193  const _res = {
194    status: res.status,
195    headers: iterableToObject(res.headers)
196  }
197
198  return new CachePolicy(_req, _res, { shared: false })
199}
200
201// https://tools.ietf.org/html/rfc7234#section-4.2
202function isStale (req, res) {
203  if (!res) {
204    return null
205  }
206
207  const _req = {
208    url: req.url,
209    method: req.method,
210    headers: iterableToObject(req.headers)
211  }
212
213  const policy = makePolicy(req, res)
214
215  const responseTime = res.headers.get('x-local-cache-time') ||
216    res.headers.get('date') ||
217    0
218
219  policy._responseTime = new Date(responseTime)
220
221  const bool = !policy.satisfiesWithoutRevalidation(_req)
222  return bool
223}
224
225function mustRevalidate (res) {
226  return (res.headers.get('cache-control') || '').match(/must-revalidate/i)
227}
228
229function conditionalFetch (req, cachedRes, opts) {
230  const _req = {
231    url: req.url,
232    method: req.method,
233    headers: Object.assign({}, opts.headers || {})
234  }
235
236  const policy = makePolicy(req, cachedRes)
237  opts.headers = policy.revalidationHeaders(_req)
238
239  return remoteFetch(req.url, opts)
240    .then(condRes => {
241      const revalidatedPolicy = policy.revalidatedPolicy(_req, {
242        status: condRes.status,
243        headers: iterableToObject(condRes.headers)
244      })
245
246      if (condRes.status >= 500 && !mustRevalidate(cachedRes)) {
247        //   111 Revalidation failed
248        // MUST be included if a cache returns a stale response because an
249        // attempt to revalidate the response failed, due to an inability to
250        // reach the server.
251        // (https://tools.ietf.org/html/rfc2616#section-14.46)
252        setWarning(cachedRes, 111, 'Revalidation failed')
253        return cachedRes
254      }
255
256      if (condRes.status === 304) { // 304 Not Modified
257        condRes.body = cachedRes.body
258        return opts.cacheManager.put(req, condRes, opts)
259          .then(newRes => {
260            newRes.headers = new fetch.Headers(revalidatedPolicy.policy.responseHeaders())
261            return newRes
262          })
263      }
264
265      return condRes
266    })
267    .then(res => res)
268    .catch(err => {
269      if (mustRevalidate(cachedRes)) {
270        throw err
271      } else {
272        //   111 Revalidation failed
273        // MUST be included if a cache returns a stale response because an
274        // attempt to revalidate the response failed, due to an inability to
275        // reach the server.
276        // (https://tools.ietf.org/html/rfc2616#section-14.46)
277        setWarning(cachedRes, 111, 'Revalidation failed')
278        //   199 Miscellaneous warning
279        // The warning text MAY include arbitrary information to be presented to
280        // a human user, or logged. A system receiving this warning MUST NOT take
281        // any automated action, besides presenting the warning to the user.
282        // (https://tools.ietf.org/html/rfc2616#section-14.46)
283        setWarning(
284          cachedRes,
285          199,
286          `Miscellaneous Warning ${err.code}: ${err.message}`
287        )
288
289        return cachedRes
290      }
291    })
292}
293
294function remoteFetchHandleIntegrity (res, integrity) {
295  const oldBod = res.body
296  const newBod = ssri.integrityStream({
297    integrity
298  })
299  oldBod.pipe(newBod)
300  res.body = newBod
301  oldBod.once('error', err => {
302    newBod.emit('error', err)
303  })
304  newBod.once('error', err => {
305    oldBod.emit('error', err)
306  })
307}
308
309function remoteFetch (uri, opts) {
310  const agent = getAgent(uri, opts)
311  const headers = Object.assign({
312    'connection': agent ? 'keep-alive' : 'close',
313    'user-agent': USER_AGENT
314  }, opts.headers || {})
315
316  const reqOpts = {
317    agent,
318    body: opts.body,
319    compress: opts.compress,
320    follow: opts.follow,
321    headers: new fetch.Headers(headers),
322    method: opts.method,
323    redirect: 'manual',
324    size: opts.size,
325    counter: opts.counter,
326    timeout: opts.timeout
327  }
328
329  return retry(
330    (retryHandler, attemptNum) => {
331      const req = new fetch.Request(uri, reqOpts)
332      return fetch(req)
333        .then(res => {
334          res.headers.set('x-fetch-attempts', attemptNum)
335
336          if (opts.integrity) {
337            remoteFetchHandleIntegrity(res, opts.integrity)
338          }
339
340          const isStream = req.body instanceof Stream
341
342          if (opts.cacheManager) {
343            const isMethodGetHead = req.method === 'GET' ||
344              req.method === 'HEAD'
345
346            const isCachable = opts.cache !== 'no-store' &&
347              isMethodGetHead &&
348              makePolicy(req, res).storable() &&
349              res.status === 200 // No other statuses should be stored!
350
351            if (isCachable) {
352              return opts.cacheManager.put(req, res, opts)
353            }
354
355            if (!isMethodGetHead) {
356              return opts.cacheManager.delete(req).then(() => {
357                if (res.status >= 500 && req.method !== 'POST' && !isStream) {
358                  if (typeof opts.onRetry === 'function') {
359                    opts.onRetry(res)
360                  }
361
362                  return retryHandler(res)
363                }
364
365                return res
366              })
367            }
368          }
369
370          const isRetriable = req.method !== 'POST' &&
371            !isStream && (
372              res.status === 408 || // Request Timeout
373              res.status === 420 || // Enhance Your Calm (usually Twitter rate-limit)
374              res.status === 429 || // Too Many Requests ("standard" rate-limiting)
375              res.status >= 500 // Assume server errors are momentary hiccups
376            )
377
378          if (isRetriable) {
379            if (typeof opts.onRetry === 'function') {
380              opts.onRetry(res)
381            }
382
383            return retryHandler(res)
384          }
385
386          if (!fetch.isRedirect(res.status) || opts.redirect === 'manual') {
387            return res
388          }
389
390          // handle redirects - matches behavior of npm-fetch: https://github.com/bitinn/node-fetch
391          if (opts.redirect === 'error') {
392            const err = new Error(`redirect mode is set to error: ${uri}`)
393            err.code = 'ENOREDIRECT'
394            throw err
395          }
396
397          if (!res.headers.get('location')) {
398            const err = new Error(`redirect location header missing at: ${uri}`)
399            err.code = 'EINVALIDREDIRECT'
400            throw err
401          }
402
403          if (req.counter >= req.follow) {
404            const err = new Error(`maximum redirect reached at: ${uri}`)
405            err.code = 'EMAXREDIRECT'
406            throw err
407          }
408
409          const resolvedUrl = url.resolve(req.url, res.headers.get('location'))
410          let redirectURL = url.parse(resolvedUrl)
411
412          if (isURL.test(res.headers.get('location'))) {
413            redirectURL = url.parse(res.headers.get('location'))
414          }
415
416          // Remove authorization if changing hostnames (but not if just
417          // changing ports or protocols).  This matches the behavior of request:
418          // https://github.com/request/request/blob/b12a6245/lib/redirect.js#L134-L138
419          if (url.parse(req.url).hostname !== redirectURL.hostname) {
420            req.headers.delete('authorization')
421          }
422
423          // for POST request with 301/302 response, or any request with 303 response,
424          // use GET when following redirect
425          if (res.status === 303 ||
426            ((res.status === 301 || res.status === 302) && req.method === 'POST')) {
427            opts.method = 'GET'
428            opts.body = null
429            req.headers.delete('content-length')
430          }
431
432          opts.headers = {}
433          req.headers.forEach((value, name) => {
434            opts.headers[name] = value
435          })
436
437          opts.counter = ++req.counter
438          return cachingFetch(resolvedUrl, opts)
439        })
440        .catch(err => {
441          const code = err.code === 'EPROMISERETRY' ? err.retried.code : err.code
442
443          const isRetryError = RETRY_ERRORS.indexOf(code) === -1 &&
444            RETRY_TYPES.indexOf(err.type) === -1
445
446          if (req.method === 'POST' || isRetryError) {
447            throw err
448          }
449
450          if (typeof opts.onRetry === 'function') {
451            opts.onRetry(err)
452          }
453
454          return retryHandler(err)
455        })
456    },
457    opts.retry
458  ).catch(err => {
459    if (err.status >= 400) {
460      return err
461    }
462
463    throw err
464  })
465}
466
467function isHeaderConditional (headers) {
468  if (!headers || typeof headers !== 'object') {
469    return false
470  }
471
472  const modifiers = [
473    'if-modified-since',
474    'if-none-match',
475    'if-unmodified-since',
476    'if-match',
477    'if-range'
478  ]
479
480  return Object.keys(headers)
481    .some(h => modifiers.indexOf(h.toLowerCase()) !== -1)
482}
483