1'use strict' 2 3let Cache 4const url = require('url') 5const CachePolicy = require('http-cache-semantics') 6const fetch = require('node-fetch-npm') 7const pkg = require('./package.json') 8const retry = require('promise-retry') 9let ssri 10const Stream = require('stream') 11const getAgent = require('./agent') 12const setWarning = require('./warning') 13 14const isURL = /^https?:/ 15const USER_AGENT = `${pkg.name}/${pkg.version} (+https://npm.im/${pkg.name})` 16 17const RETRY_ERRORS = [ 18 'ECONNRESET', // remote socket closed on us 19 'ECONNREFUSED', // remote host refused to open connection 20 'EADDRINUSE', // failed to bind to a local port (proxy?) 21 'ETIMEDOUT' // someone in the transaction is WAY TOO SLOW 22 // Known codes we do NOT retry on: 23 // ENOTFOUND (getaddrinfo failure. Either bad hostname, or offline) 24] 25 26const RETRY_TYPES = [ 27 'request-timeout' 28] 29 30// https://fetch.spec.whatwg.org/#http-network-or-cache-fetch 31module.exports = cachingFetch 32cachingFetch.defaults = function (_uri, _opts) { 33 const fetch = this 34 if (typeof _uri === 'object') { 35 _opts = _uri 36 _uri = null 37 } 38 39 function defaultedFetch (uri, opts) { 40 const finalOpts = Object.assign({}, _opts || {}, opts || {}) 41 return fetch(uri || _uri, finalOpts) 42 } 43 44 defaultedFetch.defaults = fetch.defaults 45 defaultedFetch.delete = fetch.delete 46 return defaultedFetch 47} 48 49cachingFetch.delete = cacheDelete 50function cacheDelete (uri, opts) { 51 opts = configureOptions(opts) 52 if (opts.cacheManager) { 53 const req = new fetch.Request(uri, { 54 method: opts.method, 55 headers: opts.headers 56 }) 57 return opts.cacheManager.delete(req, opts) 58 } 59} 60 61function initializeCache (opts) { 62 if (typeof opts.cacheManager === 'string') { 63 if (!Cache) { 64 // Default cacache-based cache 65 Cache = require('./cache') 66 } 67 68 opts.cacheManager = new Cache(opts.cacheManager, opts) 69 } 70 71 opts.cache = opts.cache || 'default' 72 73 if (opts.cache === 'default' && isHeaderConditional(opts.headers)) { 74 // If header list contains `If-Modified-Since`, `If-None-Match`, 75 // `If-Unmodified-Since`, `If-Match`, or `If-Range`, fetch will set cache 76 // mode to "no-store" if it is "default". 77 opts.cache = 'no-store' 78 } 79} 80 81function configureOptions (_opts) { 82 const opts = Object.assign({}, _opts || {}) 83 opts.method = (opts.method || 'GET').toUpperCase() 84 85 if (opts.retry && typeof opts.retry === 'number') { 86 opts.retry = { retries: opts.retry } 87 } 88 89 if (opts.retry === false) { 90 opts.retry = { retries: 0 } 91 } 92 93 if (opts.cacheManager) { 94 initializeCache(opts) 95 } 96 97 return opts 98} 99 100function initializeSsri () { 101 if (!ssri) { 102 ssri = require('ssri') 103 } 104} 105 106function cachingFetch (uri, _opts) { 107 const opts = configureOptions(_opts) 108 109 if (opts.integrity) { 110 initializeSsri() 111 // if verifying integrity, node-fetch must not decompress 112 opts.compress = false 113 } 114 115 const isCachable = (opts.method === 'GET' || opts.method === 'HEAD') && 116 opts.cacheManager && 117 opts.cache !== 'no-store' && 118 opts.cache !== 'reload' 119 120 if (isCachable) { 121 const req = new fetch.Request(uri, { 122 method: opts.method, 123 headers: opts.headers 124 }) 125 126 return opts.cacheManager.match(req, opts).then(res => { 127 if (res) { 128 const warningCode = (res.headers.get('Warning') || '').match(/^\d+/) 129 if (warningCode && +warningCode >= 100 && +warningCode < 200) { 130 // https://tools.ietf.org/html/rfc7234#section-4.3.4 131 // 132 // If a stored response is selected for update, the cache MUST: 133 // 134 // * delete any Warning header fields in the stored response with 135 // warn-code 1xx (see Section 5.5); 136 // 137 // * retain any Warning header fields in the stored response with 138 // warn-code 2xx; 139 // 140 res.headers.delete('Warning') 141 } 142 143 if (opts.cache === 'default' && !isStale(req, res)) { 144 return res 145 } 146 147 if (opts.cache === 'default' || opts.cache === 'no-cache') { 148 return conditionalFetch(req, res, opts) 149 } 150 151 if (opts.cache === 'force-cache' || opts.cache === 'only-if-cached') { 152 // 112 Disconnected operation 153 // SHOULD be included if the cache is intentionally disconnected from 154 // the rest of the network for a period of time. 155 // (https://tools.ietf.org/html/rfc2616#section-14.46) 156 setWarning(res, 112, 'Disconnected operation') 157 return res 158 } 159 } 160 161 if (!res && opts.cache === 'only-if-cached') { 162 const errorMsg = `request to ${ 163 uri 164 } failed: cache mode is 'only-if-cached' but no cached response available.` 165 166 const err = new Error(errorMsg) 167 err.code = 'ENOTCACHED' 168 throw err 169 } 170 171 // Missing cache entry, or mode is default (if stale), reload, no-store 172 return remoteFetch(req.url, opts) 173 }) 174 } 175 176 return remoteFetch(uri, opts) 177} 178 179function iterableToObject (iter) { 180 const obj = {} 181 for (let k of iter.keys()) { 182 obj[k] = iter.get(k) 183 } 184 return obj 185} 186 187function makePolicy (req, res) { 188 const _req = { 189 url: req.url, 190 method: req.method, 191 headers: iterableToObject(req.headers) 192 } 193 const _res = { 194 status: res.status, 195 headers: iterableToObject(res.headers) 196 } 197 198 return new CachePolicy(_req, _res, { shared: false }) 199} 200 201// https://tools.ietf.org/html/rfc7234#section-4.2 202function isStale (req, res) { 203 if (!res) { 204 return null 205 } 206 207 const _req = { 208 url: req.url, 209 method: req.method, 210 headers: iterableToObject(req.headers) 211 } 212 213 const policy = makePolicy(req, res) 214 215 const responseTime = res.headers.get('x-local-cache-time') || 216 res.headers.get('date') || 217 0 218 219 policy._responseTime = new Date(responseTime) 220 221 const bool = !policy.satisfiesWithoutRevalidation(_req) 222 return bool 223} 224 225function mustRevalidate (res) { 226 return (res.headers.get('cache-control') || '').match(/must-revalidate/i) 227} 228 229function conditionalFetch (req, cachedRes, opts) { 230 const _req = { 231 url: req.url, 232 method: req.method, 233 headers: Object.assign({}, opts.headers || {}) 234 } 235 236 const policy = makePolicy(req, cachedRes) 237 opts.headers = policy.revalidationHeaders(_req) 238 239 return remoteFetch(req.url, opts) 240 .then(condRes => { 241 const revalidatedPolicy = policy.revalidatedPolicy(_req, { 242 status: condRes.status, 243 headers: iterableToObject(condRes.headers) 244 }) 245 246 if (condRes.status >= 500 && !mustRevalidate(cachedRes)) { 247 // 111 Revalidation failed 248 // MUST be included if a cache returns a stale response because an 249 // attempt to revalidate the response failed, due to an inability to 250 // reach the server. 251 // (https://tools.ietf.org/html/rfc2616#section-14.46) 252 setWarning(cachedRes, 111, 'Revalidation failed') 253 return cachedRes 254 } 255 256 if (condRes.status === 304) { // 304 Not Modified 257 condRes.body = cachedRes.body 258 return opts.cacheManager.put(req, condRes, opts) 259 .then(newRes => { 260 newRes.headers = new fetch.Headers(revalidatedPolicy.policy.responseHeaders()) 261 return newRes 262 }) 263 } 264 265 return condRes 266 }) 267 .then(res => res) 268 .catch(err => { 269 if (mustRevalidate(cachedRes)) { 270 throw err 271 } else { 272 // 111 Revalidation failed 273 // MUST be included if a cache returns a stale response because an 274 // attempt to revalidate the response failed, due to an inability to 275 // reach the server. 276 // (https://tools.ietf.org/html/rfc2616#section-14.46) 277 setWarning(cachedRes, 111, 'Revalidation failed') 278 // 199 Miscellaneous warning 279 // The warning text MAY include arbitrary information to be presented to 280 // a human user, or logged. A system receiving this warning MUST NOT take 281 // any automated action, besides presenting the warning to the user. 282 // (https://tools.ietf.org/html/rfc2616#section-14.46) 283 setWarning( 284 cachedRes, 285 199, 286 `Miscellaneous Warning ${err.code}: ${err.message}` 287 ) 288 289 return cachedRes 290 } 291 }) 292} 293 294function remoteFetchHandleIntegrity (res, integrity) { 295 const oldBod = res.body 296 const newBod = ssri.integrityStream({ 297 integrity 298 }) 299 oldBod.pipe(newBod) 300 res.body = newBod 301 oldBod.once('error', err => { 302 newBod.emit('error', err) 303 }) 304 newBod.once('error', err => { 305 oldBod.emit('error', err) 306 }) 307} 308 309function remoteFetch (uri, opts) { 310 const agent = getAgent(uri, opts) 311 const headers = Object.assign({ 312 'connection': agent ? 'keep-alive' : 'close', 313 'user-agent': USER_AGENT 314 }, opts.headers || {}) 315 316 const reqOpts = { 317 agent, 318 body: opts.body, 319 compress: opts.compress, 320 follow: opts.follow, 321 headers: new fetch.Headers(headers), 322 method: opts.method, 323 redirect: 'manual', 324 size: opts.size, 325 counter: opts.counter, 326 timeout: opts.timeout 327 } 328 329 return retry( 330 (retryHandler, attemptNum) => { 331 const req = new fetch.Request(uri, reqOpts) 332 return fetch(req) 333 .then(res => { 334 res.headers.set('x-fetch-attempts', attemptNum) 335 336 if (opts.integrity) { 337 remoteFetchHandleIntegrity(res, opts.integrity) 338 } 339 340 const isStream = req.body instanceof Stream 341 342 if (opts.cacheManager) { 343 const isMethodGetHead = req.method === 'GET' || 344 req.method === 'HEAD' 345 346 const isCachable = opts.cache !== 'no-store' && 347 isMethodGetHead && 348 makePolicy(req, res).storable() && 349 res.status === 200 // No other statuses should be stored! 350 351 if (isCachable) { 352 return opts.cacheManager.put(req, res, opts) 353 } 354 355 if (!isMethodGetHead) { 356 return opts.cacheManager.delete(req).then(() => { 357 if (res.status >= 500 && req.method !== 'POST' && !isStream) { 358 if (typeof opts.onRetry === 'function') { 359 opts.onRetry(res) 360 } 361 362 return retryHandler(res) 363 } 364 365 return res 366 }) 367 } 368 } 369 370 const isRetriable = req.method !== 'POST' && 371 !isStream && ( 372 res.status === 408 || // Request Timeout 373 res.status === 420 || // Enhance Your Calm (usually Twitter rate-limit) 374 res.status === 429 || // Too Many Requests ("standard" rate-limiting) 375 res.status >= 500 // Assume server errors are momentary hiccups 376 ) 377 378 if (isRetriable) { 379 if (typeof opts.onRetry === 'function') { 380 opts.onRetry(res) 381 } 382 383 return retryHandler(res) 384 } 385 386 if (!fetch.isRedirect(res.status) || opts.redirect === 'manual') { 387 return res 388 } 389 390 // handle redirects - matches behavior of npm-fetch: https://github.com/bitinn/node-fetch 391 if (opts.redirect === 'error') { 392 const err = new Error(`redirect mode is set to error: ${uri}`) 393 err.code = 'ENOREDIRECT' 394 throw err 395 } 396 397 if (!res.headers.get('location')) { 398 const err = new Error(`redirect location header missing at: ${uri}`) 399 err.code = 'EINVALIDREDIRECT' 400 throw err 401 } 402 403 if (req.counter >= req.follow) { 404 const err = new Error(`maximum redirect reached at: ${uri}`) 405 err.code = 'EMAXREDIRECT' 406 throw err 407 } 408 409 const resolvedUrl = url.resolve(req.url, res.headers.get('location')) 410 let redirectURL = url.parse(resolvedUrl) 411 412 if (isURL.test(res.headers.get('location'))) { 413 redirectURL = url.parse(res.headers.get('location')) 414 } 415 416 // Remove authorization if changing hostnames (but not if just 417 // changing ports or protocols). This matches the behavior of request: 418 // https://github.com/request/request/blob/b12a6245/lib/redirect.js#L134-L138 419 if (url.parse(req.url).hostname !== redirectURL.hostname) { 420 req.headers.delete('authorization') 421 } 422 423 // for POST request with 301/302 response, or any request with 303 response, 424 // use GET when following redirect 425 if (res.status === 303 || 426 ((res.status === 301 || res.status === 302) && req.method === 'POST')) { 427 opts.method = 'GET' 428 opts.body = null 429 req.headers.delete('content-length') 430 } 431 432 opts.headers = {} 433 req.headers.forEach((value, name) => { 434 opts.headers[name] = value 435 }) 436 437 opts.counter = ++req.counter 438 return cachingFetch(resolvedUrl, opts) 439 }) 440 .catch(err => { 441 const code = err.code === 'EPROMISERETRY' ? err.retried.code : err.code 442 443 const isRetryError = RETRY_ERRORS.indexOf(code) === -1 && 444 RETRY_TYPES.indexOf(err.type) === -1 445 446 if (req.method === 'POST' || isRetryError) { 447 throw err 448 } 449 450 if (typeof opts.onRetry === 'function') { 451 opts.onRetry(err) 452 } 453 454 return retryHandler(err) 455 }) 456 }, 457 opts.retry 458 ).catch(err => { 459 if (err.status >= 400) { 460 return err 461 } 462 463 throw err 464 }) 465} 466 467function isHeaderConditional (headers) { 468 if (!headers || typeof headers !== 'object') { 469 return false 470 } 471 472 const modifiers = [ 473 'if-modified-since', 474 'if-none-match', 475 'if-unmodified-since', 476 'if-match', 477 'if-range' 478 ] 479 480 return Object.keys(headers) 481 .some(h => modifiers.indexOf(h.toLowerCase()) !== -1) 482} 483