http: rewrite request module.

This commit is contained in:
Christopher Jeffrey 2016-12-11 08:18:11 -08:00
parent 48fc7549ce
commit fc6798d3f0
No known key found for this signature in database
GPG Key ID: 8962AB9DE6666BBD

View File

@ -1,6 +1,5 @@
/*!
/*
* request.js - http request for bcoin
* Copyright (c) 2014-2015, Fedor Indutny (MIT License)
* Copyright (c) 2014-2016, Christopher Jeffrey (MIT License).
* https://github.com/bcoin-org/bcoin
*/
@ -9,17 +8,470 @@
/* jshint -W069 */
/**
* @module request
*/
var Stream = require('stream').Stream;
var assert = require('assert');
var url, qs, http, https, StringDecoder;
/*
* Constants
*/
// Spoof by default
var USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1)'
+ ' AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.1 Safari/537.36';
/**
* Request Options
* @constructor
* @param {Object} options
*/
function RequestOptions(options) {
if (!(this instanceof RequestOptions))
return new RequestOptions(options);
this.uri = 'http://localhost:80/';
this.host = 'localhost';
this.path = '/';
this.port = 80;
this.ssl = false;
this.method = 'GET';
this.strictSSL = true;
this.agent = USER_AGENT;
this.type = null;
this.expect = null;
this.query = null;
this.body = null;
this.auth = null;
this.limit = 5 << 20;
this.maxRedirects = 5;
this.timeout = 5000;
this.buffer = false;
// Hack
ensureRequires();
if (options)
this.fromOptions(options);
}
RequestOptions.prototype.setURI = function setURI(uri) {
var parts;
assert(typeof uri === 'string');
if (!/:\/\//.test(uri))
uri = (this.ssl ? 'https://' : 'http://') + uri;
uri = url.parse(uri);
assert(uri.protocol === 'http:' || uri.protocol === 'https:');
this.uri = uri;
this.ssl = uri.protocol === 'https:';
if (uri.search)
this.query = qs.parse(uri.search);
this.host = uri.hostname;
this.path = uri.pathname;
this.port = uri.port || (this.ssl ? 443 : 80);
if (uri.auth) {
parts = uri.auth.split(':');
this.auth = {
username: parts[0] || '',
password: parts[1] || ''
};
}
};
RequestOptions.prototype.fromOptions = function fromOptions(options) {
if (typeof options === 'string')
options = { uri: options };
if (options.ssl != null) {
assert(typeof options.ssl === 'boolean');
this.ssl = options.ssl;
}
if (options.uri != null)
this.setURI(options.uri);
if (options.url != null)
this.setURI(options.url);
if (options.method != null) {
assert(typeof options.method === 'string');
this.method = options.method.toUpperCase();
}
if (options.agent != null) {
assert(typeof options.agent === 'string');
this.agent = options.agent;
}
if (options.strictSSL != null) {
assert(typeof options.strictSSL === 'boolean');
this.strictSSL = options.strictSSL;
}
if (options.auth != null) {
assert(typeof options.auth === 'object');
assert(typeof options.auth.username === 'string');
assert(typeof options.auth.password === 'string');
this.auth = options.auth;
}
if (options.query != null) {
if (typeof options.query === 'string') {
this.query = qs.stringify(options.query);
} else {
assert(typeof options.query === 'object');
this.query = options.query;
}
}
if (options.json != null) {
assert(typeof options.json === 'object');
this.body = new Buffer(JSON.stringify(options.json), 'utf8');
this.type = 'json';
}
if (options.form != null) {
assert(typeof options.form === 'object');
this.body = new Buffer(qs.stringify(options.form), 'utf8');
this.type = 'form';
}
if (options.type != null) {
assert(typeof options.type === 'string');
assert(getType(options.type));
this.type = options.type;
}
if (options.expect != null) {
assert(typeof options.expect === 'string');
assert(getType(options.expect));
this.expect = options.expect;
}
if (options.body != null) {
if (typeof options.body === 'string') {
this.body = new Buffer(options.body, 'utf8');
} else {
assert(Buffer.isBuffer(options.body));
this.body = options.body;
}
}
if (options.timeout != null) {
assert(typeof options.timeout === 'number');
this.timeout = options.timeout;
}
if (options.limit != null) {
assert(typeof options.limit === 'number');
this.limit = options.limit;
}
if (options.maxRedirects != null) {
assert(typeof options.maxRedirects === 'number');
this.maxRedirects = options.maxRedirects;
}
if (options.buffer != null) {
assert(typeof options.buffer === 'boolean');
this.buffer = options.buffer;
}
};
RequestOptions.prototype.expected = function expected(type) {
if (!this.expect)
return true;
return this.expect === type;
};
RequestOptions.prototype.getBackend = function getBackend() {
ensureRequires(this.ssl);
return this.ssl ? https : http;
};
RequestOptions.prototype.toHTTP = function toHTTP() {
var query = '';
var headers = {};
var auth;
if (this.query)
query = '?' + qs.stringify(this.query);
headers['User-Agent'] = this.agent;
if (this.body) {
headers['Content-Type'] = getType(this.type);
headers['Content-Length'] = this.body.length + '';
}
if (this.auth) {
auth = this.auth.username + ':' + this.auth.password;
headers['Authorization'] =
'Basic ' + new Buffer(auth, 'utf8').toString('base64');
}
return {
method: this.method,
host: this.host,
port: this.port,
path: this.path + query,
headers: headers,
rejectUnauthorized: this.strictSSL
};
};
/**
* Request
* @constructor
* @param {Object} options
*/
function Request(options) {
if (!(this instanceof Request))
return new Request(options);
Stream.call(this);
this.options = new RequestOptions(options);
this.request = null;
this.response = null;
this.statusCode = 0;
this.headers = null;
this.type = 'binary';
this.redirects = 0;
this.timeout = null;
this.finished = false;
this.onResponse = this._onResponse.bind(this);
this.onData = this._onData.bind(this);
this.onEnd = this._onEnd.bind(this);
this.total = 0;
this.decoder = null;
this.body = null;
}
Request.prototype.__proto__ = Stream.prototype;
Request.prototype.startTimeout = function startTimeout() {
var self = this;
if (!this.options.timeout)
return;
this.timeout = setTimeout(function() {
self.finish(new Error('Request timed out.'));
}, this.options.timeout);
};
Request.prototype.stopTimeout = function stopTimeout() {
if (this.timeout != null) {
clearTimeout(this.timeout);
this.timeout = null;
}
};
Request.prototype.cleanup = function cleanup() {
this.stopTimeout();
if (this.request) {
this.request.removeListener('response', this.onResponse);
this.request.removeListener('error', this.onEnd);
}
if (this.response) {
this.response.removeListener('data', this.onData);
this.response.removeListener('error', this.onEnd);
this.response.removeListener('end', this.onEnd);
if (this.response.socket)
this.response.socket.removeListener('end', this.onEnd);
}
};
Request.prototype.close = function close() {
this.cleanup();
if (this.request) {
try {
this.request.abort();
} catch (e) {
;
}
this.request = null;
}
if (this.response) {
try {
this.response.destroy();
} catch (e) {
;
}
if (this.response.socket) {
try {
this.response.socket.destroy();
} catch (e) {
;
}
}
this.response = null;
}
};
Request.prototype.destroy = function destroy() {
this.close();
};
Request.prototype.start = function start() {
var backend = this.options.getBackend();
var options = this.options.toHTTP();
this.startTimeout();
this.request = backend.request(options);
this.response = null;
if (this.options.body)
this.request.write(this.options.body);
this.request.on('response', this.onResponse);
this.request.on('error', this.onEnd);
};
Request.prototype.write = function write(data) {
return this.request.write(data);
};
Request.prototype.end = function end() {
return this.request.end();
};
Request.prototype.finish = function finish(err) {
if (this.finished)
return;
this.finished = true;
if (err) {
this.destroy();
this.emit('error', err);
return;
}
this.cleanup();
if (this.options.buffer && this.body) {
switch (this.type) {
case 'binary':
this.body = Buffer.concat(this.body);
break;
case 'json':
try {
this.body = JSON.parse(this.body);
} catch (e) {
this.emit('error', e);
return;
}
break;
case 'form':
try {
this.body = qs.parse(this.body);
} catch (e) {
this.emit('error', e);
return;
}
break;
}
}
this.emit('end');
this.emit('close');
};
Request.prototype._onResponse = function _onResponse(response) {
var location = response.headers['location'];
var type = response.headers['content-type'];
if (location) {
if (++this.redirects > this.options.maxRedirects) {
this.finish(new Error('Too many redirects.'));
return;
}
this.close();
this.options.setURI(location);
this.start();
this.end();
return;
}
type = parseType(type);
if (!this.options.expected(type)) {
this.finish(new Error('Wrong content-type for response.'));
return;
}
this.response = response;
this.statusCode = response.statusCode;
this.headers = response.headers;
this.type = type;
this.emit('headers', response.headers);
this.emit('type', this.type);
this.emit('response', response);
if (this.options.buffer) {
if (this.type !== 'binary') {
this.decoder = new StringDecoder('utf8');
this.body = '';
} else {
this.body = [];
}
}
this.response.on('data', this.onData);
this.response.on('error', this.onEnd);
this.response.on('end', this.onEnd);
// An agent socket's `end` sometimes
// won't be emitted on the response.
if (this.response.socket)
this.response.socket.on('end', this.onEnd);
};
Request.prototype._onData = function _onData(data) {
this.total += data.length;
if (this.options.limit) {
if (this.total > this.options.limit) {
this.finish(new Error('Response exceeded limit.'));
return;
}
}
if (this.options.buffer) {
if (this.decoder) {
this.body += this.decoder.write(data);
return;
}
this.body.push(data);
}
this.emit('data', data);
};
Request.prototype._onEnd = function _onEnd(err) {
this.finish(err);
};
/**
* Make an HTTP request.
* @param {Object} options
@ -42,338 +494,99 @@ var USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1)'
* @param {Function?} callback - Will return a stream if not present.
*/
function request(options, callback, stream) {
var qs = require('querystring');
var url = require('url');
var uri = options.uri;
var query = options.query;
var body = options.body;
var json = options.json;
var form = options.form;
var type = options.type;
var http, req, opt;
function request(options, callback) {
var stream;
if (callback)
return request._buffer(options, callback);
if (json && typeof json === 'object') {
body = json;
json = true;
if (!callback) {
stream = new Request(options);
stream.start();
return stream;
}
if (form && typeof form === 'object') {
body = qs.stringify(form);
form = true;
}
if (typeof options === 'string')
options = { uri: options };
if (typeof uri !== 'object') {
if (!/:\/\//.test(uri))
uri = 'http://' + uri;
uri = url.parse(uri);
}
options.buffer = true;
if (uri.protocol === 'https:')
http = require('https');
else
http = require('http');
stream = new Request(options);
if (uri.search)
query = qs.parse(uri.search);
if (query && typeof query !== 'string')
query = qs.stringify(query);
if (query)
query = '?' + query;
else
query = '';
opt = {
host: uri.hostname,
port: uri.port || (uri.protocol === 'https:' ? 443 : 80),
path: uri.pathname + query,
headers: {
'User-Agent': options.agent || USER_AGENT
},
rejectUnauthorized: options.strictSSL !== false
};
if (body) {
if (!type) {
if (form)
type = 'form';
else if (json || (typeof body === 'object' && !Buffer.isBuffer(body)))
type = 'json';
else if (typeof body === 'string')
type = 'text';
else
type = 'binary';
}
if (type === 'json' && typeof body === 'object')
body = JSON.stringify(body);
else if (type === 'form' && typeof body === 'object')
body = qs.stringify(body);
if (type === 'form')
type = 'application/x-www-form-urlencoded; charset=utf-8';
else if (type === 'json')
type = 'application/json; charset=utf-8';
else if (type === 'text')
type = 'text/plain; charset=utf-8';
else if (type === 'binary')
type = 'application/octet-stream';
if (typeof body === 'string')
body = new Buffer(body, 'utf8');
assert(Buffer.isBuffer(body));
opt.headers['Content-Type'] = type;
opt.headers['Content-Length'] = body.length + '';
opt.method = options.method || 'POST';
} else {
opt.method = options.method || 'GET';
}
if (options.auth)
uri.auth = options.auth.username + ':' + options.auth.password;
if (uri.auth) {
opt.headers['Authorization'] =
'Basic ' + new Buffer(uri.auth, 'utf8').toString('base64');
}
opt.method = opt.method.toUpperCase();
req = http.request(opt);
if (!stream)
stream = new ReqStream(options);
stream.req = req;
req.on('response', function(res) {
var called = false;
var type = res.headers['content-type'];
if (res.headers['location']) {
if (++stream._redirects > stream.maxRedirects)
return done(new Error('Too many redirects.'));
options.uri = res.headers['location'];
return request(options, null, stream);
}
if (/\/json/i.test(type))
type = 'json';
else if (/form-urlencoded/i.test(type))
type = 'form';
else if (/text\/plain/i.test(type))
type = 'text';
else if (/\/x?html/i.test(type))
type = 'html';
else
type = 'binary';
stream.res = res;
stream.headers = res.headers;
stream.type = type;
if (options.expect && type !== options.expect)
return done(new Error('Wrong content-type for response.'));
stream.emit('headers', res.headers);
stream.emit('type', type);
stream.emit('response', res);
function done(err) {
if (called)
return;
called = true;
if (res.socket)
res.socket.removeListener('end', done);
stream.finish();
if (err) {
stream.destroy();
stream.emit('error', err);
return;
}
stream.emit('end');
stream.emit('close');
}
res.on('data', function(data) {
stream.emit('data', data);
});
res.on('error', done);
res.on('end', done);
// An agent socket's `end` sometimes
// won't be emitted on the response.
if (res.socket)
res.socket.on('end', done);
stream.on('error', function(err) {
callback(err);
});
req.on('error', function(err) {
stream.destroy();
stream.emit('error', err);
stream.on('end', function() {
callback(null, stream, stream.body);
});
if (body)
req.write(body);
req.end();
stream.start();
stream.end();
return stream;
}
request._buffer = function(options, callback) {
var qs = require('querystring');
var StringDecoder = require('string_decoder').StringDecoder;
var stream = request(options);
var total = 0;
var called = false;
var decoder, body;
function done(err) {
if (called)
return;
called = true;
if (err)
return callback(err);
if (stream.type === 'binary') {
body = Buffer.concat(body);
} else if (stream.type === 'json') {
try {
body = JSON.parse(body);
} catch (e) {
return callback(e);
}
} else if (stream.type === 'form') {
try {
body = qs.parse(body);
} catch (e) {
return callback(e);
}
}
callback(null, stream.res, body, stream.type);
}
stream.on('type', function(type) {
if (type !== 'binary') {
decoder = new StringDecoder('utf8');
body = '';
} else {
body = [];
}
});
stream.on('data', function(data) {
total += data.length;
if (options.limit && total > options.limit) {
stream.destroy();
return done();
}
if (decoder)
body += decoder.write(data);
else
body.push(data);
});
stream.on('error', done);
stream.on('end', done);
return stream;
};
request.promise = function promise(options) {
return new Promise(function(resolve, reject) {
request(options, function(err, res, body) {
if (err)
return reject(err);
res.body = body;
request(options, function(err, res) {
if (err) {
reject(err);
return;
}
resolve(res);
});
});
};
/*
* ReqStream
* Helpers
*/
function ReqStream(options) {
if (!(this instanceof ReqStream))
return new ReqStream(options);
function parseType(type) {
if (/\/json/i.test(type))
return 'json';
Stream.call(this);
if (/form-urlencoded/i.test(type))
return 'form';
this.req = null;
this.res = null;
this.headers = null;
this.type = null;
this._redirects = 0;
this.maxRedirects = options.maxRedirects || 5;
this.timeout = options.timeout;
this._timeout = null;
if (/text\/plain/i.test(type))
return 'text';
this._init();
if (/\/x?html/i.test(type))
return 'html';
return 'binary';
}
ReqStream.prototype._init = function _init() {
var self = this;
if (this.timeout) {
this._timeout = setTimeout(function() {
self.emit('error', new Error('Request timed out.'));
self.destroy();
}, this.timeout);
function getType(type) {
switch (type) {
case 'form':
return 'application/x-www-form-urlencoded; charset=utf-8';
case 'json':
return 'application/json; charset=utf-8';
case 'text':
return 'text/plain; charset=utf-8';
case 'binary':
return 'application/octet-stream';
default:
throw new Error('Unknown type: ' + type);
}
};
}
ReqStream.prototype.__proto__ = Stream.prototype;
function ensureRequires(ssl) {
if (!url)
url = require('url');
ReqStream.prototype.destroy = function destroy() {
try {
this.req.abort();
} catch (e) {
;
}
if (!qs)
qs = require('querystring');
try {
this.res.destroy();
} catch (e) {
;
}
if (!http)
http = require('http');
try {
this.res.socket.destroy();
} catch (e) {
;
}
if (ssl && !https)
https = require('https');
this.finish();
};
ReqStream.prototype.finish = function finish() {
if (this._timeout != null) {
clearTimeout(this._timeout);
this._timeout = null;
}
};
if (!StringDecoder)
StringDecoder = require('string_decoder').StringDecoder;
}
/*
* Expose