Node.js nginx/sails.js:文件上载不完整
我们正在使用sails.js开发一个应用程序。 在此应用程序中,我们有一个上载控制器:Node.js nginx/sails.js:文件上载不完整,node.js,nginx,sails.js,skipper,Node.js,Nginx,Sails.js,Skipper,我们正在使用sails.js开发一个应用程序。 在此应用程序中,我们有一个上载控制器: 如文档中所述,此控制器在发动机罩下使用skipper 现在的问题是,当我们上传大文件时,它们的存储不完整,上传的大小从来都不一样,对于一个15MB的文件,大小从7mb到14mb不等 架构如下: haproxy->nginx->node.js/sails 如果我们用一个简单的apache+proxypass配置替换nginx反向代理,那么上传工作将完美无缺 如果我们用一个简单的python上传控制器(例如,在
如文档中所述,此控制器在发动机罩下使用skipper 现在的问题是,当我们上传大文件时,它们的存储不完整,上传的大小从来都不一样,对于一个15MB的文件,大小从7mb到14mb不等 架构如下:
haproxy->nginx->node.js/sails 如果我们用一个简单的apache+proxypass配置替换nginx反向代理,那么上传工作将完美无缺 如果我们用一个简单的python上传控制器(例如,在flask中)替换node.js应用程序,那么上传也会显示正确的长度和数据 当然,nginx已经正确地配置了缓冲区大小、client_body_timeout和client_max_body_size,正如我所说的,flask刚刚正确地接收了上传 nginx应用程序的上传结果是200响应,因此文件似乎是上传的,但实际上,在磁盘上,文件是不完整的 在nginx调试日志中,我们可以看到:
2014/12/03 01:57:23 [debug] 39583#0: *1 http proxy header:
"POST /admin/edit_object/6 HTTP/1.1^M
Host: xxxxxx.makina-corpus.net^M
X-Real-IP: xxxx^M
X-Forwarded-For: xxxxx^M
X-NginX-Proxy: true^M
X-Forwarded-Proto: http^M
Connection: upgrade^M
Content-Length: 15361775^M
Origin: http://bai.makina-corpus.net^M
User-Agent: Mozilla/5.0 (Unknown; Linux x86_64) AppleWebKit/534.34 (KHTML, like Gecko) CasperJS/1.1.0-beta3+PhantomJS/1.9.8 Safari/534.34^M
Content-Type: multipart/form-data; boundary=----WebKitFormBoundaryRt4v4f7RkrlzUEX2^M
Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8^M
Referer: http://xxxxxxxxxx.makina-corpus.net/admin/edit_object/6^M
Cookie: sails.sid=s%3Akv_Gxxxxxxxx2F5iaDWA^M
Accept-Encoding: gzip^M
Accept-Language: en,*^M
Authorization: Basic xxxx=^M
^M
"
2014/12/03 01:57:23 [debug] 39583#0: *1 http cleanup add: 00000000011CC520
2014/12/03 01:57:23 [debug] 39583#0: *1 init keepalive peer
2014/12/03 01:57:23 [debug] 39583#0: *1 get keepalive peer
2014/12/03 01:57:23 [debug] 39583#0: *1 get rr peer, try: 1
2014/12/03 01:57:23 [debug] 39583#0: *1 get keepalive peer: using connection 0000000001156018
2014/12/03 01:57:23 [debug] 39583#0: *1 http upstream connect: -4
2014/12/03 01:57:23 [debug] 39583#0: *1 http upstream send request
2014/12/03 01:57:23 [debug] 39583#0: *1 chain writer buf fl:0 s:806
2014/12/03 01:57:23 [debug] 39583#0: *1 chain writer buf fl:1 s:15361775
2014/12/03 01:57:23 [debug] 39583#0: *1 chain writer in: 00000000011CC5C0
2014/12/03 01:57:23 [debug] 39583#0: *1 tcp_nopush
2014/12/03 01:57:23 [debug] 39583#0: *1 writev: 806
2014/12/03 01:57:23 [debug] 39583#0: *1 sendfile: @0 15361775
2014/12/03 01:57:23 [debug] 39583#0: *1 sendfile: 2776864, @0 2776864:15361775
2014/12/03 01:57:23 [debug] 39583#0: *1 chain writer out: 00000000011CC5D0
2014/12/03 01:57:23 [debug] 39583#0: *1 event timer add: 35: 60000:1417568303245
2014/12/03 01:57:23 [debug] 39583#0: *1 http run request: "/admin/edit_object/6?"
2014/12/03 01:57:23 [debug] 39583#0: *1 http request empty handler
2014/12/03 01:57:23 [debug] 39583#0: *1 http upstream request: "/admin/edit_object/6?"
2014/12/03 01:57:23 [debug] 39583#0: *1 http upstream send request handler
2014/12/03 01:57:23 [debug] 39583#0: *1 http upstream send request
2014/12/03 01:57:23 [debug] 39583#0: *1 chain writer in: 00000000011CC5D0
2014/12/03 01:57:23 [debug] 39583#0: *1 sendfile: @2776864 12584911
2014/12/03 01:57:23 [debug] 39583#0: *1 sendfile: 2488810, @2776864 2488810:12584911
2014/12/03 01:57:23 [debug] 39583#0: *1 chain writer out: 00000000011CC5D0
2014/12/03 01:57:23 [debug] 39583#0: *1 event timer del: 35: 1417568303245
2014/12/03 01:57:23 [debug] 39583#0: *1 event timer add: 35: 60000:1417568303254
2014/12/03 01:57:23 [debug] 39583#0: *1 http upstream request: "/admin/edit_object/6?"
2014/12/03 01:57:23 [debug] 39583#0: *1 http upstream process header
2014/12/03 01:57:23 [debug] 39583#0: *1 malloc: 00000000011CD000:262144
2014/12/03 01:57:23 [debug] 39583#0: *1 recv: fd:35 369 of 262144
2014/12/03 01:57:23 [debug] 39583#0: *1 http proxy status 200 "200 OK"
2014/12/03 01:57:23 [debug] 39583#0: *1 http proxy header: "X-Powered-By: Sails <sailsjs.org>"
2014/12/03 01:57:23 [debug] 39583#0: *1 http proxy header: "Access-Control-Allow-Origin: "
2014/12/03 01:57:23 [debug] 39583#0: *1 http proxy header: "Access-Control-Allow-Credentials: "
2014/12/03 01:57:23 [debug] 39583#0: *1 http proxy header: "Access-Control-Allow-Methods: "
2014/12/03 01:57:23 [debug] 39583#0: *1 http proxy header: "Access-Control-Allow-Headers: "
2014/12/03 01:57:23 [debug] 39583#0: *1 http proxy header: "Content-Type: application/json; charset=utf-8"
2014/12/03 01:57:23 [debug] 39583#0: *1 http proxy header: "Content-Length: 33"
2014/12/03 01:57:23 [debug] 39583#0: *1 http proxy header: "Vary: Accept-Encoding"
2014/12/03 01:57:23 [debug] 39583#0: *1 http proxy header: "Date: Wed, 03 Dec 2014 00:57:23 GMT"
2014/12/03 01:57:23 [debug] 39583#0: *1 http proxy header: "Connection: keep-alive"
2014/12/03 01:57:23 [debug] 39583#0: *1 http proxy header done
2014/12/03 01:57:23 [debug] 39583#0: *1 uploadprogress error-tracker error: 0
2014/12/03 01:57:23 [debug] 39583#0: *1 xslt filter header
2014/12/03 01:57:23 [debug] 39583#0: *1 HTTP/1.1 200 OK^M
Server: nginx^M
Date: Wed, 03 Dec 2014 00:57:23 GMT^M
有趣的是,上传后磁盘上的文件最后包含来自另一个无关请求的头:
07bb890: 3130 3130 3130 3130 3130 3130 3130 3130 1010101010101010
07bb8a0: 3130 3130 3130 3130 3130 3130 3130 3130 1010101010101010
07bb8b0: 3130 3130 3130 3130 3130 3130 3130 3130 1010101010101010
07bb8c0: 3130 3130 3130 3130 3130 3130 4745 5420 101010101010GET
07bb8d0: 2f20 4854 5450 2f31 2e31 0d0a 486f 7374 / HTTP/1.1..Host
07bb8e0: xxxx xxxx xxxx xxxx xxxx xxxx 2d63 6f72 : xxx.makina-cor
07bb8f0: 7075 732e 6e65 740d 0a58 2d52 6561 6c2d pus.net..X-Real-
07bb900: 4950
07bb910: 3134 0d0a 582d 466f 7277 6172 6465 642d 14..X-Forwarded-
07bb920: 466f For: xxxxxxxxxxx
07bb930: 2e31
07bb940: 2e31 340d 0a58 2d4e 6769 6e58 2d50 726f .14..X-NginX-Pro
07bb950: 7879 3a20 7472 7565 0d0a 582d 466f 7277 xy: true..X-Forw
07bb960: 6172 6465 642d 5072 6f74 6f3a 2068 7474 arded-Proto: htt
07bb970: 700d 0a43 6f6e 6e65 6374 696f 6e3a 2075 p..Connection: u
07bb980: 7067 7261 6465 0d0a 5573 6572 2d41 6765 pgrade..User-Age
07bb990: 6e74 3a20 4d6f 7a69 6c6c 612f 352e 3020 nt: Mozilla/5.0
07bb9a0: 2855 6e6b 6e6f 776e 3b20 4c69 6e75 7820 (Unknown; Linux
07bb9b0: 7838 365f 3634 2920 4170 706c 6557 6562 x86_64) AppleWeb
07bb9c0: 4b69 742f 3533 342e 3334 2028 4b48 544d Kit/534.34 (KHTM
07bb9d0: 4c2c 206c 696b 6520 4765 636b 6f29 2043 L, like Gecko) C
07bb9e0: 6173 7065 724a 532f 312e 312e 302d 6265 asperJS/1.1.0-be
07bb9f0: 7461 332b 5068 616e 746f 6d4a 532f 312e ta3+PhantomJS/1.
07bba00: 392e 3820 5361 6661 7269 2f35 3334 2e33 9.8 Safari/534.3
07bba10: 340d 0a41 6363 6570 743a 2074 6578 742f 4..Accept: text/
07bba20: 6874 6d6c 2c61 7070 6c69 6361 7469 6f6e html,application
07bba30: 2f78 6874 6d6c 2b78 6d6c 2c61 7070 6c69 /xhtml+xml,appli
07bba40: 6361 7469 6f6e 2f78 6d6c 3b71 3d30 2e39 cation/xml;q=0.9
07bba50: 2c2a 2f2a 3b71 3d30 2e38 0d0a 4163 6365 ,*/*;q=0.8..Acce
07bba60: 7074 2d45 6e63 6f64 696e 673a 2067 7a69 pt-Encoding: gzi
07bba70: 700d 0a41 6363 6570 742d 4c61 6e67 7561 p..Accept-Langua
07bba80: 6765 3a20 656e 2c2a 0d0a 4175 7468 6f72 ge: en,*..Author
07bba90: 697a 6174 696f 6e3a 2042 6173 6963 2063 ization: Basic c
07bbaa0: 6d39 xxxx xxxx xxxx xxxx 3d0d 0a xxxx=..
(END)
在其他请求中,我们没有其他请求头,只有一个不完整的文件。
在这里,缺少的位来自原始文件的结尾,开始总是正确的
请注意,与apache的主要区别在于,nginx向sails应用程序发送数据的速度非常快。相反,apache正在流式传输请求。
这是因为nginx确实请求缓冲
如果有人有一个想法,从哪里继续在船长挖出上传问题
如果我用这个示例替换save方法,我会看到来自nginx的位被正确写入,我在发布的数据中有完整且正确的文件,因此错误显然在skipper请求消耗中的某个地方
var body = "";
req.on('data', function (chunk) {
body += chunk;
});
req.on('end', function () {
console.log('POSTed: ' + body.length);
console.log('POSTed: ' + body.slice(-400));
res.writeHead(200);
res.end('<html/>');
});
var body=”“;
请求on('data',函数(块){
body+=块;
});
请求开启('end',函数(){
console.log('POSTed:'+body.length);
console.log('POSTed:'+body.slice(-400));
书面记录(200);
res.end(“”);
});
我们面临着类似的问题。我不知道我们的解决方案是否适用于您,但现在开始
对于非常大的文件,csrf
从请求包中删除。因此,我们需要在request header
中发送csrf
,而不是request body
。为此,我们稍微更改了XMLHttpRequest
/*
Putting csrf in Header as some large
files need this mechanism to upload
*/
(function() {
var send = XMLHttpRequest.prototype.send,
token = csrfToken; //csrfToken is global
XMLHttpRequest.prototype.send = function(data) {
this.setRequestHeader('X-CSRF-Token', token);
return send.apply(this, arguments);
};
}());
从现在起,每个请求都将在标题中包含csrf
。这为我们解决了问题。希望这对您也有帮助。从nginx调试日志来看,问题似乎是由于从后端提前返回响应造成的-请注意,在上一次sendfile()调用中,nginx只能发送12584911字节中的2488810字节:
...
2014/12/03 01:57:23 [debug] 39583#0: *1 chain writer in: 00000000011CC5D0
2014/12/03 01:57:23 [debug] 39583#0: *1 sendfile: @2776864 12584911
2014/12/03 01:57:23 [debug] 39583#0: *1 sendfile: 2488810, @2776864 2488810:12584911
2014/12/03 01:57:23 [debug] 39583#0: *1 chain writer out: 00000000011CC5D0
2014/12/03 01:57:23 [debug] 39583#0: *1 event timer del: 35: 1417568303245
2014/12/03 01:57:23 [debug] 39583#0: *1 event timer add: 35: 60000:1417568303254
2014/12/03 01:57:23 [debug] 39583#0: *1 http upstream request: "/admin/edit_object/6?"
2014/12/03 01:57:23 [debug] 39583#0: *1 http upstream process header
2014/12/03 01:57:23 [debug] 39583#0: *1 malloc: 00000000011CD000:262144
2014/12/03 01:57:23 [debug] 39583#0: *1 recv: fd:35 369 of 262144
2014/12/03 01:57:23 [debug] 39583#0: *1 http proxy status 200 "200 OK"
2014/12/03 01:57:23 [debug] 39583#0: *1 http proxy header: "X-Powered-By: Sails <sailsjs.org>"
...
。。。
2014/12/03 01:57:23[调试]39583#0:*1个链写入器输入:000000000 11CC5D0
2014/12/03 01:57:23[调试]39583#0:*1发送文件:@2776864 12584911
2014/12/03 01:57:23[调试]39583#0:*1发送文件:2488810,@2776864 2488810:12584911
2014/12/03 01:57:23[调试]39583#0:*1个链写入器输出:000000000 11CC5D0
2014/12/03 01:57:23[调试]39583#0:*1事件计时器del:35:1417568303245
2014/12/03 01:57:23[调试]39583#0:*1事件计时器添加:35:60000:1417568303254
2014/12/03 01:57:23[调试]39583#0:*1 http上游请求:“/admin/edit#u object/6?”
2014/12/03 01:57:23[调试]39583#0:*1 http上游进程头
2014/12/03 01:57:23[调试]39583#0:*1 malloc:000000000 11CD000:262144
2014/12/03 01:57:23[调试]39583#0:*1记录:fd:35 369/262144
2014/12/03 01:57:23[调试]39583#0:*1 http代理状态200“200正常”
2014/12/03 01:57:23[调试]39583#0:*1 http代理标头:“X-Powered-By:Sails”
...
后端返回了一个200 OK的答案。此时,nginx认为没有理由发送请求正文的其余部分并停止发送它-这就是导致上传文件不完整的原因。此外,您已经配置了keepalive上游连接,并且正在点击-这就是为什么您会看到不相关请求的头
教后端代码只在请求被完全读取后发送响应,就像在测试代码中一样,应该可以解决这个问题。因此,我找到的解决方案是破解一个使用强大功能的bodyparser
没有问题:)
为了记录在案,在中间件中切换bodyparser有点像黑客:
config/http.js
module.exports.http = {
middleware: {
bodyParser: false,
cbodyParser: require('../bodyParser')(
{urls: [/\/admin\/edit_object/]}),
order: [
'startRequestTimer',
'cookieParser',
'session',
'cbodyParser',
'handleBodyParserError',
'compress',
'methodOverride',
'poweredBy',
'$custom',
'router',
'www',
'favicon',
'404',
'500'
],
}
};
bodyparser.js:
/**
* Module dependencies
// Configure body parser components
*/
var _ = require('lodash');
var util = require('util');
var formidable = require('formidable');
function mime(req) {
var str = req.headers['content-type'] || '';
return str.split(';')[0];
}
function parseMultipart(req, res, next) {
req.form = new formidable.IncomingForm();
req.form.uploadDir = sails.config.data.__uploadData;
req.form.maxFieldsSize = sails.config.maxsize;
req.form.multiple = true;
// res.setTimeout(0);
req.form.parse(req, function(err, fields, files) {
if (err)
return next(err);
else {
req.files = files;
req.fields = fields;
req.body = extend(fields, files);
next();
}
});
}
function extend(target) {
var key, obj;
for (var i = 1, l = arguments.length; i < l; i++) {
if ((obj = arguments[i])) {
for (key in obj)
target[key] = obj[key];
}
}
return target;
}
function disable_parser(opts, req, res) {
var matched = false;
try {
var method = null;
try {method = req.method.toLowerCase();}
catch(err){ /* */}
if(method) {
_(opts.urls).forEach(function(turl) {
if (method === 'post' && req.url.match(turl)) {
// console.log("matched"+ req.url);
if(!matched) matched = true;
};});
}
} catch(err) { debug(err);/* pass */ }
return matched;
}
module.exports = function toParseHTTPBody(options) {
options = options || {};
var bodyparser = require('skipper')(options);
// NAME of anynonymous func IS IMPORTANT (same as the middleware in config) !!!
return function cbodyParser(req, res, next) {
var err_hdler = function(err) {};
if (disable_parser(options, req, res) && mime(req) == 'multipart/form-data') {
return parseMultipart(req, res, next);
} else {
return bodyparser(req, res, next);
}
};
};
/**
*模块依赖关系
//配置主体解析器组件
*/
var=要求('lodash');
var util=require('util');
var Required=要求(“Required”);
函数mime(req){
var str=req.headers['content-type']| |';
返回str.split(“;”)[0];
}
函数parseMultipart(req、res、next){
req.form=new.IncomingForm();
req.form.uploadDir=sails.config.data.\uu uploadData;
req.form.maxFieldsSize=sails.config.maxsize;
req.form.multiple=真;
//res.setTimeout(0);
解析(请求,函数(错误,字段,文件){
如果(错误)
返回下一个(错误);
否则{
req.files=文件;
请求字段=字段;
req.body=扩展(字段、文件);
next();
}
});
}
功能扩展(目标){
var-key,obj;
for(var i=1,l=arguments.length;i/**
* Module dependencies
// Configure body parser components
*/
var _ = require('lodash');
var util = require('util');
var formidable = require('formidable');
function mime(req) {
var str = req.headers['content-type'] || '';
return str.split(';')[0];
}
function parseMultipart(req, res, next) {
req.form = new formidable.IncomingForm();
req.form.uploadDir = sails.config.data.__uploadData;
req.form.maxFieldsSize = sails.config.maxsize;
req.form.multiple = true;
// res.setTimeout(0);
req.form.parse(req, function(err, fields, files) {
if (err)
return next(err);
else {
req.files = files;
req.fields = fields;
req.body = extend(fields, files);
next();
}
});
}
function extend(target) {
var key, obj;
for (var i = 1, l = arguments.length; i < l; i++) {
if ((obj = arguments[i])) {
for (key in obj)
target[key] = obj[key];
}
}
return target;
}
function disable_parser(opts, req, res) {
var matched = false;
try {
var method = null;
try {method = req.method.toLowerCase();}
catch(err){ /* */}
if(method) {
_(opts.urls).forEach(function(turl) {
if (method === 'post' && req.url.match(turl)) {
// console.log("matched"+ req.url);
if(!matched) matched = true;
};});
}
} catch(err) { debug(err);/* pass */ }
return matched;
}
module.exports = function toParseHTTPBody(options) {
options = options || {};
var bodyparser = require('skipper')(options);
// NAME of anynonymous func IS IMPORTANT (same as the middleware in config) !!!
return function cbodyParser(req, res, next) {
var err_hdler = function(err) {};
if (disable_parser(options, req, res) && mime(req) == 'multipart/form-data') {
return parseMultipart(req, res, next);
} else {
return bodyparser(req, res, next);
}
};
};