Skip to content

Remove node-fetch dependency and use built-in https/http modules instead #963

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Shooteger wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Remove node-fetch dependency and use built-in https/http modules instead #963

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
143 changes: 90 additions & 53 deletions lib/install.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,8 @@ const versioning = require('./util/versioning.js');
const napi = require('./util/napi.js');
const s3_setup = require('./util/s3_setup.js');
const url = require('url');
// for fetching binaries
const fetch = require('node-fetch');
const https = require('https');
const http = require('http');
const tar = require('tar');

let npgVersion = 'unknown';
Expand Down Expand Up @@ -84,6 +84,51 @@ function place_binary_authenticated(opts, targetDir, callback) {
}
}

/**
* Downloads a URL following redirects, using Node's built-in http/https modules.
* Calls back with (err, IncomingMessage) — the response is a Readable stream
* that can be .pipe()'d directly.
*
* @param {string} uri
* @param {{ headers?: object, agent?: object, ca?: string|Buffer }} opts
* @param {number} maxRedirects
* @param {(err: Error|null, res?: import('http').IncomingMessage) => void} callback
*/
function fetch_with_redirects(uri, opts, maxRedirects, callback) {
let parsed;
try {
parsed = new URL(uri);
} catch (e) {
return callback(new Error(`Invalid URL: ${uri}`));
}

const mod = parsed.protocol === 'https:' ? https : http;
const reqOpts = {
hostname: parsed.hostname,
port: parsed.port || (parsed.protocol === 'https:' ? 443 : 80),
path: parsed.pathname + parsed.search,
headers: opts.headers || {},
agent: opts.agent || undefined,
ca: opts.ca || undefined
};

const req = mod.get(reqOpts, (res) => {
if (res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) {
res.resume(); // release socket
if (maxRedirects === 0) {
return callback(new Error('Too many redirects'));
}
const next = new URL(res.headers.location, uri).href;
return fetch_with_redirects(next, opts, maxRedirects - 1, callback);
}
callback(null, res);
});

req.on('error', callback);
}

exports.fetch_with_redirects = fetch_with_redirects;

function place_binary(uri, targetDir, opts, callback) {
log.log('GET', uri);

Expand All @@ -92,22 +137,16 @@ function place_binary(uri, targetDir, opts, callback) {
'node ' + process.version;

const sanitized = uri.replace('+', '%2B');
const requestOpts = {
uri: sanitized,
headers: {
'User-Agent': 'node-pre-gyp (v' + npgVersion + ', ' + envVersionInfo + ')'
},
follow_max: 10
};

let ca;
if (opts.cafile) {
try {
requestOpts.ca = fs.readFileSync(opts.cafile);
ca = fs.readFileSync(opts.cafile);
} catch (e) {
return callback(e);
}
} else if (opts.ca) {
requestOpts.ca = opts.ca;
ca = opts.ca;
}

const proxyUrl = opts.proxy ||
Expand All @@ -121,51 +160,49 @@ function place_binary(uri, targetDir, opts, callback) {
log.log('download', `proxy agent configured using: "${proxyUrl}"`);
}

fetch(sanitized, { agent })
.then((res) => {
if (!res.ok) {
// If we get 403 Forbidden, the binary might be private - try authenticated download
if (res.status === 403) {
log.info('install', 'Received 403 Forbidden - attempting authenticated download');
// Call place_binary_authenticated and return a special marker
// to prevent the promise chain from calling callback again
place_binary_authenticated(opts, targetDir, callback);
return { authenticated: true };
}
throw new Error(`response status ${res.status} ${res.statusText} on ${sanitized}`);
fetch_with_redirects(sanitized, {
headers: { 'User-Agent': 'node-pre-gyp (v' + npgVersion + ', ' + envVersionInfo + ')' },
agent,
ca
}, 10, (err, res) => {
if (err) {
log.error(`install ${err.message}`);
return callback(err);
}

const ok = res.statusCode >= 200 && res.statusCode < 300;
if (!ok) {
res.resume(); // release socket
if (res.statusCode === 403) {
log.info('install', 'Received 403 Forbidden - attempting authenticated download');
return place_binary_authenticated(opts, targetDir, callback);
}
const dataStream = res.body;

return new Promise((resolve, reject) => {
let extractions = 0;
const countExtractions = (entry) => {
extractions += 1;
log.info('install', `unpacking ${entry.path}`);
};

dataStream.pipe(extract(targetDir, countExtractions))
.on('error', (e) => {
reject(e);
});
dataStream.on('end', () => {
resolve(`extracted file count: ${extractions}`);
});
dataStream.on('error', (e) => {
reject(e);
});
return callback(new Error(`response status ${res.statusCode} ${res.statusMessage} on ${sanitized}`));
}

new Promise((resolve, reject) => {
let extractions = 0;
const countExtractions = (entry) => {
extractions += 1;
log.info('install', `unpacking ${entry.path}`);
};

res.pipe(extract(targetDir, countExtractions))
.on('error', reject);
res.on('end', () => {
resolve(`extracted file count: ${extractions}`);
});
res.on('error', reject);
})
.then((text) => {
if (text && text.authenticated) {
return; // Don't call callback - place_binary_authenticated will handle it
}
log.info(text);
callback();
})
.catch((e) => {
log.error(`install ${e.message}`);
callback(e);
});
.then((text) => {
log.info(text);
callback();
})
.catch((e) => {
log.error(`install ${e.message}`);
callback(e);
});
});
}

function extract(to, onentry) {
Expand Down
Loading
Loading