Weird behaviour of the fetch when running npm install - node.js

Question

My setup is very simple and small:
C:\temp\npm [master]> tree /f
Folder PATH listing for volume OSDisk
Volume serial number is F6C4-7BEF
C:.
│ .gitignore
│ 1.js
│ package.json
│
└───.vscode
launch.json
C:\temp\npm [master]> cat .\package.json
{
"name": "node-modules",
"version": "1.0.0",
"description": "",
"main": "index.js",
"dependencies": {
"emitter": "http://github.com/component/emitter/archive/1.0.1.tar.gz",
"global": "https://github.com/component/global/archive/v2.0.1.tar.gz"
},
"author": "",
"license": "ISC"
}
C:\temp\npm [master]> npm config list
; cli configs
metrics-registry = "https://registry.npmjs.org/"
scope = ""
user-agent = "npm/6.14.12 node/v14.16.1 win32 x64"
; userconfig C:\Users\p11f70f\.npmrc
https-proxy = "http://127.0.0.1:8888/"
proxy = "http://127.0.0.1:8888/"
strict-ssl = false
; builtin config undefined
prefix = "C:\\Users\\p11f70f\\AppData\\Roaming\\npm"
; node bin location = C:\Program Files\nodejs\node.exe
; cwd = C:\temp\npm
; HOME = C:\Users\p11f70f
; "npm config ls -l" to show all defaults.
C:\temp\npm [master]>
Notes:
The proxy addresses correspond to Fiddler.
Notice that the emitter dependency url uses http whereas the global uses https.
When I run npm install it starts and then hangs very quickly. And I know why, because Fiddler tells me:
The request is:
GET http://github.com:80/component/emitter/archive/1.0.1.tar.gz HTTP/1.1
connection: keep-alive
user-agent: npm/6.14.12 node/v14.16.1 win32 x64
npm-in-ci: false
npm-scope:
npm-session: 74727385b32ebcbf
referer: install
pacote-req-type: tarball
pacote-pkg-id: registry:undefined#http://github.com/component/emitter/archive/1.0.1.tar.gz
accept: */*
accept-encoding: gzip,deflate
Host: github.com:80
And the response is:
HTTP/1.1 301 Moved Permanently
Content-Length: 0
Location: https://github.com:80/component/emitter/archive/1.0.1.tar.gz
Now this is BS, pardon my French, because the returned Location value of https://github.com:80/component/emitter/archive/1.0.1.tar.gz is invalid. But I suppose the server is not very smart - it just redirects to https without changing anything else, including the port, which remains 80 - good for http, wrong for https. This explains the hanging - the fetch API used by npm seems to retry at progressively longer delays which creates an illusion of hanging.
Debugging npm brings me to the following code inside C:\Program Files\nodejs\node_modules\npm\node_modules\npm-registry-fetch\index.js:
return opts.Promise.resolve(body).then(body => fetch(uri, {
agent: opts.agent,
algorithms: opts.algorithms,
body,
cache: getCacheMode(opts),
cacheManager: opts.cache,
ca: opts.ca,
cert: opts.cert,
headers,
integrity: opts.integrity,
key: opts.key,
localAddress: opts['local-address'],
maxSockets: opts.maxsockets,
memoize: opts.memoize,
method: opts.method || 'GET',
noProxy: opts['no-proxy'] || opts.noproxy,
Promise: opts.Promise,
proxy: opts['https-proxy'] || opts.proxy,
referer: opts.refer,
retry: opts.retry != null ? opts.retry : {
retries: opts['fetch-retries'],
factor: opts['fetch-retry-factor'],
minTimeout: opts['fetch-retry-mintimeout'],
maxTimeout: opts['fetch-retry-maxtimeout']
},
strictSSL: !!opts['strict-ssl'],
timeout: opts.timeout
}).then(res => checkResponse(
opts.method || 'GET', res, registry, startTime, opts
)))
And when I stop at the right moment, this boils down to the following values:
uri
'http://github.com/component/emitter/archive/1.0.1.tar.gz'
agent:undefined
algorithms:['sha1']
body:undefined
ca:null
cache:'default'
cacheManager:'C:\\Users\\p11f70f\\AppData\\Roaming\\npm-cache\\_cacache'
cert:null
headers:{
npm-in-ci:false
npm-scope:''
npm-session:'413f9b25525c452a'
pacote-pkg-id:'registry:undefined#http://github.com/component/emitter/archive/1.0.1.tar.gz'
pacote-req-type:'tarball'
referer:'install'
user-agent:'npm/6.14.12 node/v14.16.1 win32 x64'
}
integrity:undefined
key:null
localAddress:undefined
maxSockets:50
method:'GET'
noProxy:null
proxy:'http://127.0.0.1:8888/'
referer:'install'
retry:{retries: 2, factor: 10, minTimeout: 10000, maxTimeout: 60000}
strictSSL:false
timeout:0
I have omitted two values and I truly do not know their significance - opt.Promise and memoize. It is possible that they are crucial, I do not know.
Anyway, when I step over this statement, the aforementioned session appears in Fiddler with the bogus url of http://github.com:80/component/emitter/archive/1.0.1.tar.gz and I do not understand - how come? The debugger clearly shows that the uri parameter passed to fetch does not specify the port number.
I thought maybe it is some kind of a non string type, but typeof uri returns 'string'.
I have even written a tiny reproduction to execute just this request using the same arguments, except for the opt.Promise and memoize:
const fetch = require('make-fetch-happen')
fetch('http://github.com/component/emitter/archive/1.0.1.tar.gz', {
algorithms: ['sha1'],
cache: 'default',
cacheManager: 'C:\\Users\\p11f70f\\AppData\\Roaming\\npm-cache\\_cacache',
headers:{
"npm-in-ci":false,
"npm-scope":"",
"npm-session":"00b5bb97075e3c35",
"user-agent":"npm/6.14.12 node/v14.16.1 win32 x64",
"referer":"install",
"pacote-req-type":"tarball",
"pacote-pkg-id":"registry:undefined#http://github.com/component/emitter/archive/1.0.1.tar.gz"
},
maxSockets: 50,
method: 'GET',
proxy: 'http://127.0.0.1:8888',
referer: 'install',
retry: {
retries: 2,
factor: 10,
minTimeout: 10000,
maxTimeout: 60000
},
strictSSL: false,
timeout: 0
}).then(res => console.log(res))
But it shows up correctly in Fiddler - no port is added and hence the redirection works fine.
When there is no Fiddler (and hence no proxy) everything works correctly too, but I am very much curious to know why it does not work with Fiddler.
What is going on here?