Node.js 使用NodeJS将文件上载到AmazonS3
我在尝试将文件上载到S3存储桶时遇到问题。除了我的文件参数似乎不合适之外,一切都正常。我正在使用AmazonS3SDK从nodejs上传到S3 以下是我的路线设置:Node.js 使用NodeJS将文件上载到AmazonS3,node.js,amazon-s3,multipartform-data,Node.js,Amazon S3,Multipartform Data,我在尝试将文件上载到S3存储桶时遇到问题。除了我的文件参数似乎不合适之外,一切都正常。我正在使用AmazonS3SDK从nodejs上传到S3 以下是我的路线设置: var multiparty = require('connect-multiparty'), multipartyMiddleware = multiparty(); app.route('/api/items/upload').post(multipartyMiddleware, items.upload); 这是it
var multiparty = require('connect-multiparty'),
multipartyMiddleware = multiparty();
app.route('/api/items/upload').post(multipartyMiddleware, items.upload);
这是items.upload()函数:
exports.upload = function(req, res) {
var file = req.files.file;
var s3bucket = new AWS.S3({params: {Bucket: 'mybucketname'}});
s3bucket.createBucket(function() {
var params = {
Key: file.name,
Body: file
};
s3bucket.upload(params, function(err, data) {
console.log("PRINT FILE:", file);
if (err) {
console.log('ERROR MSG: ', err);
} else {
console.log('Successfully uploaded data');
}
});
});
};
将Body
param设置为类似“hello”
的字符串可以正常工作。根据,Body
param必须获取(缓冲区、类型化数组、Blob、字符串、ReadableStream)对象数据。但是,上载文件对象失败,并显示以下错误消息:
[Error: Unsupported body payload object]
这是文件对象:
{ fieldName: 'file',
originalFilename: 'second_fnp.png',
path: '/var/folders/ps/l8lvygws0w93trqz7yj1t5sr0000gn/T/26374-7ttwvc.png',
headers:
{ 'content-disposition': 'form-data; name="file"; filename="second_fnp.png"',
'content-type': 'image/png' },
ws:
{ _writableState:
{ highWaterMark: 16384,
objectMode: false,
needDrain: true,
ending: true,
ended: true,
finished: true,
decodeStrings: true,
defaultEncoding: 'utf8',
length: 0,
writing: false,
sync: false,
bufferProcessing: false,
onwrite: [Function],
writecb: null,
writelen: 0,
buffer: [],
errorEmitted: false },
writable: true,
domain: null,
_events: { error: [Object], close: [Object] },
_maxListeners: 10,
path: '/var/folders/ps/l8lvygws0w93trqz7yj1t5sr0000gn/T/26374-7ttwvc.png',
fd: null,
flags: 'w',
mode: 438,
start: undefined,
pos: undefined,
bytesWritten: 261937,
closed: true },
size: 261937,
name: 'second_fnp.png',
type: 'image/png' }
任何帮助都将不胜感激 看来这里有些地方出了问题。根据您的帖子,您似乎正在尝试使用
connectmultiparty
中间件支持文件上传。该中间件所做的是获取上传的文件,将其写入本地文件系统,然后将req.files
设置为上传的文件
路由的配置看起来不错,问题似乎出在items.upload()
函数上。特别是本部分:
var params = {
Key: file.name,
Body: file
};
正如我在回答的开头所提到的,connect multiparty将文件写入本地文件系统,因此您需要打开文件并读取它,然后上载它,然后在本地文件系统上删除它
也就是说,您可以将方法更新为以下内容:
var fs = require('fs');
exports.upload = function (req, res) {
var file = req.files.file;
fs.readFile(file.path, function (err, data) {
if (err) throw err; // Something went wrong!
var s3bucket = new AWS.S3({params: {Bucket: 'mybucketname'}});
s3bucket.createBucket(function () {
var params = {
Key: file.originalFilename, //file.name doesn't exist as a property
Body: data
};
s3bucket.upload(params, function (err, data) {
// Whether there is an error or not, delete the temp file
fs.unlink(file.path, function (err) {
if (err) {
console.error(err);
}
console.log('Temp File Delete');
});
console.log("PRINT FILE:", file);
if (err) {
console.log('ERROR MSG: ', err);
res.status(500).send(err);
} else {
console.log('Successfully uploaded data');
res.status(200).end();
}
});
});
});
};
这样做的目的是从本地文件系统读取上传的文件,然后将其上传到S3,然后删除临时文件并发送响应
这种方法存在一些问题。首先,它的效率不如它可能的那样高,因为对于大型文件,您将在写入之前加载整个文件。其次,这个过程不支持大文件的多部分上传(我认为在你必须进行多部分上传之前,截止时间是5MB)
相反,我建议您使用我一直在研究的一个模块,该模块提供了与Node.JS中本机类似的接口,但抽象掉了一些细节,如多部分上传和S3API(以及添加了一些其他功能,如递归方法)
如果要拉入库,代码将如下所示:
var fs = require('fs'),
S3FS = require('s3fs'),
s3fsImpl = new S3FS('mybucketname', {
accessKeyId: XXXXXXXXXXX,
secretAccessKey: XXXXXXXXXXXXXXXXX
});
// Create our bucket if it doesn't exist
s3fsImpl.create();
exports.upload = function (req, res) {
var file = req.files.file;
var stream = fs.createReadStream(file.path);
return s3fsImpl.writeFile(file.originalFilename, stream).then(function () {
fs.unlink(file.path, function (err) {
if (err) {
console.error(err);
}
});
res.status(200).end();
});
};
这将为提供的bucket和AWS凭据实例化模块,然后创建bucket(如果不存在)。然后,当上传文件的请求通过时,我们将打开一个文件流,并使用它将文件写入S3的指定路径。这将在后台处理多部分上传文件(如果需要的话),并且有通过流完成的好处,因此您不必等待读取整个文件后再开始上传
如果愿意,可以将代码更改为从回调。或者使用事件侦听器的方法来确定结束/错误
如果您正在寻找一些其他方法,请查看s3fs的文档,如果您正在寻找一些其他方法或存在问题,请随时提出问题。我发现以下是一个有效的解决方案:
npm install aws-sdk
一旦安装了aws sdk,请使用以下代码在需要时替换值
var AWS = require('aws-sdk');
var fs = require('fs');
var s3 = new AWS.S3();
// Bucket names must be unique across all S3 users
var myBucket = 'njera';
var myKey = 'jpeg';
//for text file
//fs.readFile('demo.txt', function (err, data) {
//for Video file
//fs.readFile('demo.avi', function (err, data) {
//for image file
fs.readFile('demo.jpg', function (err, data) {
if (err) { throw err; }
params = {Bucket: myBucket, Key: myKey, Body: data };
s3.putObject(params, function(err, data) {
if (err) {
console.log(err)
} else {
console.log("Successfully uploaded data to myBucket/myKey");
}
});
});
我在这里找到了关于这个主题的完整教程,以防您在寻找参考资料:
或使用承诺:
const AWS = require('aws-sdk');
AWS.config.update({
accessKeyId: 'accessKeyId',
secretAccessKey: 'secretAccessKey',
region: 'region'
});
let params = {
Bucket: "yourBucketName",
Key: 'someUniqueKey',
Body: 'someFile'
};
try {
let uploadPromise = await new AWS.S3().putObject(params).promise();
console.log("Successfully uploaded data to bucket");
} catch (e) {
console.log("Error uploading data: ", e);
}
将文件上载到AWS s3并发送url作为访问文件的响应 Multer是一个node.js中间件,用于处理多部分/表单数据,主要用于上传文件。它写在巴士司机的上面,以达到最大的效率。检查此npm模块 发送请求时,请确保标题“have Content Type”为“multipart/form data”。 我们将在响应中发送文件位置,这将给出url,但是如果您想访问该url,请将存储桶公开,否则您将无法访问它 upload.router.js app.js
感谢David,他的解决方案帮助我提出了将多部分文件从Heroku托管站点上传到S3 bucket的解决方案。我使用强大的工具来处理传入的表单,并使用fs来获取文件内容。希望它能帮助你 api.service.ts
public upload(files): Observable<any> {
const formData: FormData = new FormData();
files.forEach(file => {
// create a new multipart-form for every file
formData.append('file', file, file.name);
});
return this.http.post(uploadUrl, formData).pipe(
map(this.extractData),
catchError(this.handleError));
}
}
upload.js
const IncomingForm = require('formidable').IncomingForm;
const fs = require('fs');
const AWS = require('aws-sdk');
module.exports = function upload(req, res) {
var form = new IncomingForm();
const bucket = new AWS.S3(
{
signatureVersion: 'v4',
accessKeyId: process.env.AWS_ACCESS_KEY_ID,
secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY,
region: 'us-east-1'
}
);
form.on('file', (field, file) => {
const fileContent = fs.readFileSync(file.path);
const s3Params = {
Bucket: process.env.AWS_S3_BUCKET,
Key: 'folder/' + file.name,
Expires: 60,
Body: fileContent,
ACL: 'public-read'
};
bucket.upload(s3Params, function(err, data) {
if (err) {
throw err;
}
console.log('File uploaded to: ' + data.Location);
fs.unlink(file.path, function (err) {
if (err) {
console.error(err);
}
console.log('Temp File Delete');
});
});
});
// The second callback is called when the form is completely parsed.
// In this case, we want to send back a success status code.
form.on('end', () => {
res.status(200).json('upload ok');
});
form.parse(req);
}
upload-image.component.ts
import { Component, OnInit, ViewChild, Output, EventEmitter, Input } from '@angular/core';
import { ApiService } from '../api.service';
import { MatSnackBar } from '@angular/material/snack-bar';
@Component({
selector: 'app-upload-image',
templateUrl: './upload-image.component.html',
styleUrls: ['./upload-image.component.css']
})
export class UploadImageComponent implements OnInit {
public files: Set<File> = new Set();
@ViewChild('file', { static: false }) file;
public uploadedFiles: Array<string> = new Array<string>();
public uploadedFileNames: Array<string> = new Array<string>();
@Output() filesOutput = new EventEmitter<Array<string>>();
@Input() CurrentImage: string;
@Input() IsPublic: boolean;
@Output() valueUpdate = new EventEmitter();
strUploadedFiles:string = '';
filesUploaded: boolean = false;
constructor(private api: ApiService, public snackBar: MatSnackBar,) { }
ngOnInit() {
}
updateValue(val) {
this.valueUpdate.emit(val);
}
reset()
{
this.files = new Set();
this.uploadedFiles = new Array<string>();
this.uploadedFileNames = new Array<string>();
this.filesUploaded = false;
}
upload() {
this.api.upload(this.files).subscribe(res => {
this.filesOutput.emit(this.uploadedFiles);
if (res == 'upload ok')
{
this.reset();
}
}, err => {
console.log(err);
});
}
onFilesAdded() {
var txt = '';
const files: { [key: string]: File } = this.file.nativeElement.files;
for (let key in files) {
if (!isNaN(parseInt(key))) {
var currentFile = files[key];
var sFileExtension = currentFile.name.split('.')[currentFile.name.split('.').length - 1].toLowerCase();
var iFileSize = currentFile.size;
if (!(sFileExtension === "jpg"
|| sFileExtension === "png")
|| iFileSize > 671329) {
txt = "File type : " + sFileExtension + "\n\n";
txt += "Size: " + iFileSize + "\n\n";
txt += "Please make sure your file is in jpg or png format and less than 655 KB.\n\n";
alert(txt);
return false;
}
this.files.add(files[key]);
this.uploadedFiles.push('https://gourmet-philatelist-assets.s3.amazonaws.com/folder/' + files[key].name);
this.uploadedFileNames.push(files[key].name);
if (this.IsPublic && this.uploadedFileNames.length == 1)
{
this.filesUploaded = true;
this.updateValue(files[key].name);
break;
}
else if (!this.IsPublic && this.uploadedFileNames.length == 3)
{
this.strUploadedFiles += files[key].name;
this.updateValue(this.strUploadedFiles);
this.filesUploaded = true;
break;
}
else
{
this.strUploadedFiles += files[key].name + ",";
this.updateValue(this.strUploadedFiles);
}
}
}
}
addFiles() {
this.file.nativeElement.click();
}
openSnackBar(message: string, action: string) {
this.snackBar.open(message, action, {
duration: 2000,
verticalPosition: 'top'
});
}
}
从'@angular/core'导入{Component,OnInit,ViewChild,Output,EventEmitter,Input};
从“../api.service”导入{ApiService};
从“@angular/material/snakp bar”导入{matsnakbar};
@组成部分({
选择器:“应用程序上载图像”,
templateUrl:“./upload image.component.html”,
样式URL:['./上传image.component.css']
})
导出类UploadImageComponent实现OnInit{
公共文件:Set=new Set();
@ViewChild('file',{static:false})文件;
公共上传文件:Array=new Array();
公共上载文件名:Array=new Array();
@Output()fileoutput=neweventemitter();
@Input()CurrentImage:字符串;
@Input()IsPublic:布尔值;
@Output()valueUpdate=neweventemitter();
strUploadedFiles:string='';
文件上传:boolean=false;
构造函数(私有api:ApiService,公共snackBar:matsnakbar,){}
ngOnInit(){
}
更新值(val){
this.valueUpdate.emit(val);
}
重置()
{
this.files=new Set();
this.uploadedFiles=新数组();
this.uploadedFileNames=新数组();
this.filessupload=false;
}
上载(){
this.api.upload(this.files).subscribe(res=>{
this.filesOutput.emit(this.uploadedFiles);
如果(res=='upload ok')
{
这是reset();
}
},err=>{
控制台日志(err);
});
}
onFilesAdded(){
var-txt='';
常量文件:{[key:string]:File}=this.File.nativeElement.files;
用于(让输入文件){
如果(!isNaN(parseInt(key))){
var currentFile=files[key];
变量文件
public upload(files): Observable<any> {
const formData: FormData = new FormData();
files.forEach(file => {
// create a new multipart-form for every file
formData.append('file', file, file.name);
});
return this.http.post(uploadUrl, formData).pipe(
map(this.extractData),
catchError(this.handleError));
}
}
app.post('/api/upload', upload);
app.use('/api/upload', router);
const IncomingForm = require('formidable').IncomingForm;
const fs = require('fs');
const AWS = require('aws-sdk');
module.exports = function upload(req, res) {
var form = new IncomingForm();
const bucket = new AWS.S3(
{
signatureVersion: 'v4',
accessKeyId: process.env.AWS_ACCESS_KEY_ID,
secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY,
region: 'us-east-1'
}
);
form.on('file', (field, file) => {
const fileContent = fs.readFileSync(file.path);
const s3Params = {
Bucket: process.env.AWS_S3_BUCKET,
Key: 'folder/' + file.name,
Expires: 60,
Body: fileContent,
ACL: 'public-read'
};
bucket.upload(s3Params, function(err, data) {
if (err) {
throw err;
}
console.log('File uploaded to: ' + data.Location);
fs.unlink(file.path, function (err) {
if (err) {
console.error(err);
}
console.log('Temp File Delete');
});
});
});
// The second callback is called when the form is completely parsed.
// In this case, we want to send back a success status code.
form.on('end', () => {
res.status(200).json('upload ok');
});
form.parse(req);
}
import { Component, OnInit, ViewChild, Output, EventEmitter, Input } from '@angular/core';
import { ApiService } from '../api.service';
import { MatSnackBar } from '@angular/material/snack-bar';
@Component({
selector: 'app-upload-image',
templateUrl: './upload-image.component.html',
styleUrls: ['./upload-image.component.css']
})
export class UploadImageComponent implements OnInit {
public files: Set<File> = new Set();
@ViewChild('file', { static: false }) file;
public uploadedFiles: Array<string> = new Array<string>();
public uploadedFileNames: Array<string> = new Array<string>();
@Output() filesOutput = new EventEmitter<Array<string>>();
@Input() CurrentImage: string;
@Input() IsPublic: boolean;
@Output() valueUpdate = new EventEmitter();
strUploadedFiles:string = '';
filesUploaded: boolean = false;
constructor(private api: ApiService, public snackBar: MatSnackBar,) { }
ngOnInit() {
}
updateValue(val) {
this.valueUpdate.emit(val);
}
reset()
{
this.files = new Set();
this.uploadedFiles = new Array<string>();
this.uploadedFileNames = new Array<string>();
this.filesUploaded = false;
}
upload() {
this.api.upload(this.files).subscribe(res => {
this.filesOutput.emit(this.uploadedFiles);
if (res == 'upload ok')
{
this.reset();
}
}, err => {
console.log(err);
});
}
onFilesAdded() {
var txt = '';
const files: { [key: string]: File } = this.file.nativeElement.files;
for (let key in files) {
if (!isNaN(parseInt(key))) {
var currentFile = files[key];
var sFileExtension = currentFile.name.split('.')[currentFile.name.split('.').length - 1].toLowerCase();
var iFileSize = currentFile.size;
if (!(sFileExtension === "jpg"
|| sFileExtension === "png")
|| iFileSize > 671329) {
txt = "File type : " + sFileExtension + "\n\n";
txt += "Size: " + iFileSize + "\n\n";
txt += "Please make sure your file is in jpg or png format and less than 655 KB.\n\n";
alert(txt);
return false;
}
this.files.add(files[key]);
this.uploadedFiles.push('https://gourmet-philatelist-assets.s3.amazonaws.com/folder/' + files[key].name);
this.uploadedFileNames.push(files[key].name);
if (this.IsPublic && this.uploadedFileNames.length == 1)
{
this.filesUploaded = true;
this.updateValue(files[key].name);
break;
}
else if (!this.IsPublic && this.uploadedFileNames.length == 3)
{
this.strUploadedFiles += files[key].name;
this.updateValue(this.strUploadedFiles);
this.filesUploaded = true;
break;
}
else
{
this.strUploadedFiles += files[key].name + ",";
this.updateValue(this.strUploadedFiles);
}
}
}
}
addFiles() {
this.file.nativeElement.click();
}
openSnackBar(message: string, action: string) {
this.snackBar.open(message, action, {
duration: 2000,
verticalPosition: 'top'
});
}
}
<input type="file" #file style="display: none" (change)="onFilesAdded()" multiple />
<button mat-raised-button color="primary"
[disabled]="filesUploaded" (click)="$event.preventDefault(); addFiles()">
Add Files
</button>
<button class="btn btn-success" [disabled]="uploadedFileNames.length == 0" (click)="$event.preventDefault(); upload()">
Upload
</button>
const fs = require('fs');
const AWS = require('aws-sdk');
const s3 = new AWS.S3({
accessKeyId: XXXXXXXXX,
secretAccessKey: XXXXXXXXX
});
const absoluteFilePath = "C:\\Project\\test.xlsx";
const uploadFile = () => {
fs.readFile(absoluteFilePath, (err, data) => {
if (err) throw err;
const params = {
Bucket: 'testBucket', // pass your bucket name
Key: 'folderName/key.xlsx', // file will be saved in <folderName> folder
Body: data
};
s3.upload(params, function (s3Err, data) {
if (s3Err) throw s3Err
console.log(`File uploaded successfully at ${data.Location}`);
debugger;
});
});
};
uploadFile();