Fatal编程技术网
  • javascript/
  • Javascript 节点和错误：EMFILE，打开的文件太多

Javascript 节点和错误：EMFILE，打开的文件太多

javascript node.js macos

Javascript 节点和错误：EMFILE，打开的文件太多,javascript,node.js,macos,Javascript,Node.js,Macos,几天来，我一直在寻找解决错误的有效方法 错误：EMFILE，打开的文件太多 似乎很多人都有同样的问题。通常的答案是增加文件描述符的数量。所以，我试过这个： sysctl -w kern.maxfiles=20480 默认值为10240。在我看来这有点奇怪，因为我在目录中处理的文件数不到10240。更奇怪的是，在我增加了文件描述符的数量之后，仍然会收到相同的错误 第二个问题： openFiles = child_process.execSync(`lsof -p ${process.pid}`)

几天来，我一直在寻找解决错误的有效方法

错误：EMFILE，打开的文件太多

似乎很多人都有同样的问题。通常的答案是增加文件描述符的数量。所以，我试过这个：

sysctl -w kern.maxfiles=20480
默认值为10240。在我看来这有点奇怪，因为我在目录中处理的文件数不到10240。更奇怪的是，在我增加了文件描述符的数量之后，仍然会收到相同的错误

第二个问题：

openFiles = child_process.execSync(`lsof -p ${process.pid}`);
经过多次搜索，我找到了解决“打开的文件太多”问题的方法：

var requestBatches={}；
函数batchingReadFile（文件名，回调）{
//首先检查是否已经有批次
if（requestBatches.hasOwnProperty（文件名））{
RequestBatchs[filename]。推送（回调）；
返回；
}
//否则，启动一个新的，并提出真正的要求
var batch=requestBatches[filename]=[callback]；
readFile（文件名，onRealRead）；
//完成后冲洗该批次
函数onRealRead（）{
删除请求批次[文件名]；
对于（变量i=0，l=batch.length；i
不幸的是，我仍然收到同样的错误。
此代码有什么问题？

您读取的文件太多。节点异步读取文件，它将一次读取所有文件。因此，您可能正在读取10240限制

看看这是否有效：

var fs = require('fs')
var events = require('events')
var util = require('util')
var path = require('path')

var FsPool = module.exports = function(dir) {
    events.EventEmitter.call(this)
    this.dir = dir;
    this.files = [];
    this.active = [];
    this.threads = 1;
    this.on('run', this.runQuta.bind(this))
};
// So will act like an event emitter
util.inherits(FsPool, events.EventEmitter);

FsPool.prototype.runQuta = function() {
    if(this.files.length === 0 && this.active.length === 0) {
        return this.emit('done');
    }
    if(this.active.length < this.threads) {
        var name = this.files.shift()

        this.active.push(name)
        var fileName = path.join(this.dir, name);
        var self = this;
        fs.stat(fileName, function(err, stats) {
            if(err)
                throw err;
            if(stats.isFile()) {
                fs.readFile(fileName, function(err, data) {
                    if(err)
                        throw err;
                    self.active.splice(self.active.indexOf(name), 1)
                    self.emit('file', name, data);
                    self.emit('run');

                });
            } else {
                self.active.splice(self.active.indexOf(name), 1)
                self.emit('dir', name);
                self.emit('run');
            }
        });
    }
    return this
};
FsPool.prototype.init = function() {
    var dir = this.dir;
    var self = this;
    fs.readdir(dir, function(err, files) {
        if(err)
            throw err;
        self.files = files
        self.emit('run');
    })
    return this
};
var fsPool = new FsPool(__dirname)

fsPool.on('file', function(fileName, fileData) {
    console.log('file name: ' + fileName)
    console.log('file data: ', fileData.toString('utf8'))

})
fsPool.on('dir', function(dirName) {
    console.log('dir name: ' + dirName)

})
fsPool.on('done', function() {
    console.log('done')
});
fsPool.init()

var fs=require（'fs'））
var events=require（'事件'）
var util=require（'util'）
var path=require（'路径'）
var FsPool=module.exports=function（dir）{
events.EventEmitter.call（此）
this.dir=dir；
this.files=[]；
this.active=[]；
此参数为1；
this.on（'run'，this.runQuta.bind（this））
};
//因此将像事件发射器一样工作
继承（FsPool、events.EventEmitter）；
FsPool.prototype.runQuta=函数（）{
if（this.files.length==0&&this.active.length==0）{
返回此.emit（'done'）；
}
if（this.active.length
使用风笛，你只需要换衣服

FS.readFile(filename, onRealRead);
=>

风笛帮助您限制并行。更多详细信息：

我自己刚刚完成了一段代码来解决这个问题，所有其他解决方案都显得过于繁重，需要您更改程序结构

此解决方案只会暂停任何fs.readFile或fs.writeFile调用，以便在任何给定时间都不会有超过一个设置数的调用

// Queuing reads and writes, so your nodejs script doesn't overwhelm system limits catastrophically
global.maxFilesInFlight = 100; // Set this value to some number safeish for your system
var origRead = fs.readFile;
var origWrite = fs.writeFile;

var activeCount = 0;
var pending = [];

var wrapCallback = function(cb){
    return function(){
        activeCount--;
        cb.apply(this,Array.prototype.slice.call(arguments));
        if (activeCount < global.maxFilesInFlight && pending.length){
            console.log("Processing Pending read/write");
            pending.shift()();
        }
    };
};
fs.readFile = function(){
    var args = Array.prototype.slice.call(arguments);
    if (activeCount < global.maxFilesInFlight){
        if (args[1] instanceof Function){
            args[1] = wrapCallback(args[1]);
        } else if (args[2] instanceof Function) {
            args[2] = wrapCallback(args[2]);
        }
        activeCount++;
        origRead.apply(fs,args);
    } else {
        console.log("Delaying read:",args[0]);
        pending.push(function(){
            fs.readFile.apply(fs,args);
        });
    }
};

fs.writeFile = function(){
    var args = Array.prototype.slice.call(arguments);
    if (activeCount < global.maxFilesInFlight){
        if (args[1] instanceof Function){
            args[1] = wrapCallback(args[1]);
        } else if (args[2] instanceof Function) {
            args[2] = wrapCallback(args[2]);
        }
        activeCount++;
        origWrite.apply(fs,args);
    } else {
        console.log("Delaying write:",args[0]);
        pending.push(function(){
            fs.writeFile.apply(fs,args);
        });
    }
};

//排队读写，这样您的nodejs脚本就不会灾难性地超过系统限制
global.maxFilesInFlight=100；//将此值设置为系统的某个数字
var origRead=fs.readFile；
var origWrite=fs.writeFile；
var-activeCount=0；
var未决=[]；
var wrapCallback=函数（cb）{
返回函数（）{
活动计数--；
apply（这个，Array.prototype.slice.call（参数））；
if（activeCount
我今天遇到了这个问题，但没有找到好的解决方案，于是我创建了一个模块来解决这个问题 
// Queuing reads and writes, so your nodejs script doesn't overwhelm system limits catastrophically
global.maxFilesInFlight = 100; // Set this value to some number safeish for your system
var origRead = fs.readFile;
var origWrite = fs.writeFile;

var activeCount = 0;
var pending = [];

var wrapCallback = function(cb){
    return function(){
        activeCount--;
        cb.apply(this,Array.prototype.slice.call(arguments));
        if (activeCount < global.maxFilesInFlight && pending.length){
            console.log("Processing Pending read/write");
            pending.shift()();
        }
    };
};
fs.readFile = function(){
    var args = Array.prototype.slice.call(arguments);
    if (activeCount < global.maxFilesInFlight){
        if (args[1] instanceof Function){
            args[1] = wrapCallback(args[1]);
        } else if (args[2] instanceof Function) {
            args[2] = wrapCallback(args[2]);
        }
        activeCount++;
        origRead.apply(fs,args);
    } else {
        console.log("Delaying read:",args[0]);
        pending.push(function(){
            fs.readFile.apply(fs,args);
        });
    }
};

fs.writeFile = function(){
    var args = Array.prototype.slice.call(arguments);
    if (activeCount < global.maxFilesInFlight){
        if (args[1] instanceof Function){
            args[1] = wrapCallback(args[1]);
        } else if (args[2] instanceof Function) {
            args[2] = wrapCallback(args[2]);
        }
        activeCount++;
        origWrite.apply(fs,args);
    } else {
        console.log("Delaying write:",args[0]);
        pending.push(function(){
            fs.writeFile.apply(fs,args);
        });
    }
};

var Filequeue = require('filequeue');
var fq = new Filequeue(200); // max number of files to open at once

fq.readdir('/Users/xaver/Downloads/xaver/xxx/xxx/', function(err, files) {
    if(err) {
        throw err;
    }
    files.forEach(function(file) {
        fq.readFile('/Users/xaver/Downloads/xaver/xxx/xxx/' + file, function(err, data) {
            // do something here
        }
    });
});

lsof -i -n -P | grep nodejs

COMMAND     PID    USER   FD   TYPE    DEVICE SIZE/OFF NODE NAME
...
nodejs    12211    root 1012u  IPv4 151317015      0t0  TCP 10.101.42.209:40371->54.236.3.170:80 (ESTABLISHED)
nodejs    12211    root 1013u  IPv4 151279902      0t0  TCP 10.101.42.209:43656->54.236.3.172:80 (ESTABLISHED)
nodejs    12211    root 1014u  IPv4 151317016      0t0  TCP 10.101.42.209:34450->54.236.3.168:80 (ESTABLISHED)
nodejs    12211    root 1015u  IPv4 151289728      0t0  TCP 10.101.42.209:52691->54.236.3.173:80 (ESTABLISHED)
nodejs    12211    root 1016u  IPv4 151305607      0t0  TCP 10.101.42.209:47707->54.236.3.172:80 (ESTABLISHED)
nodejs    12211    root 1017u  IPv4 151289730      0t0  TCP 10.101.42.209:45423->54.236.3.171:80 (ESTABLISHED)
nodejs    12211    root 1018u  IPv4 151289731      0t0  TCP 10.101.42.209:36090->54.236.3.170:80 (ESTABLISHED)
nodejs    12211    root 1019u  IPv4 151314874      0t0  TCP 10.101.42.209:49176->54.236.3.172:80 (ESTABLISHED)
nodejs    12211    root 1020u  IPv4 151289768      0t0  TCP 10.101.42.209:45427->54.236.3.171:80 (ESTABLISHED)
nodejs    12211    root 1021u  IPv4 151289769      0t0  TCP 10.101.42.209:36094->54.236.3.170:80 (ESTABLISHED)
nodejs    12211    root 1022u  IPv4 151279903      0t0  TCP 10.101.42.209:43836->54.236.3.171:80 (ESTABLISHED)
nodejs    12211    root 1023u  IPv4 151281403      0t0  TCP 10.101.42.209:43930->54.236.3.172:80 (ESTABLISHED)
....

lsof -i -n -P | grep "8465" | wc -l

# lsof -i -n -P | grep "nodejs.*8465" | wc -l
28
# lsof -i -n -P | grep "nodejs.*8465" | wc -l
31
# lsof -i -n -P | grep "nodejs.*8465" | wc -l
34

ulimit -a

open files                      (-n) 1024

  session required pam_limits.so

  root soft  nofile 40000
  root hard  nofile 100000

var Promise = require('bluebird');
var cwait = require('cwait');

// Allow max. 10 concurrent file reads.
var queue = new cwait.TaskQueue(Promise, 10);
var read = queue.wrap(Promise.promisify(batchingReadFile));

Promise.map(files, function(filename) {
    console.log(filename);
    return(read(filename));
})

const fs = require('fs')
const async = require("async")

var q = async.queue(function(task, callback) {
    console.log(task.filename);
    fs.readFile(task.filename,"utf-8",function (err, data_read) {
            callback(err,task.filename,data_read);
        }
    );
}, 4);

var files = [1,2,3,4,5,6,7,8,9,10]

for (var file in files) {
    q.push({filename:file+".txt"}, function (err,filename,res) {
        console.log(filename + " read");
    });
}

> node ./queue.js
0.txt
    1.txt
2.txt
0.txt read
3.txt
3.txt read
4.txt
2.txt read
5.txt
4.txt read
6.txt
5.txt read
7.txt
    1.txt read (biggest file than other)
8.txt
6.txt read
9.txt
7.txt read
8.txt read
9.txt read

brew update
brew install watchman

openFiles = child_process.execSync(`lsof -p ${process.pid}`);

nodemon app.js --ignore node_modules/ --ignore data/

user@ubuntu:~$ ulimit -n 65535

user@ubuntu:~$ ulimit -n
1024

user@ubuntu:~$ sudo nano /etc/sysctl.conf

fs.file-max = 65535

user@ubuntu:~$ sudo sysctl -p

user@ubuntu:~$ sudo vim /etc/security/limits.conf

root soft     nproc          65535    
root hard     nproc          65535   
root soft     nofile         65535   
root hard     nofile         65535

user@ubuntu:~$ sudo vim /etc/pam.d/common-session

session required pam_limits.so

user@ubuntu:~$ ulimit -n
65535

DefaultLimitNOFILE=65535

fs.readdir(<directory path></directory>, async (err, filenames) => {
    if (err) {
        console.log(err);
    }

    try {
        for (let filename of filenames) {
            const fileContent = await new Promise((resolve, reject) => {
                fs.readFile(<dirctory path + filename>, 'utf-8', (err, content) => {
                    if (err) {
                        reject(err);
                    }
                    resolve(content);
                });
            });
            ... // do things with fileContent
        }
    } catch (err) {
        console.log(err);
    }
});

// import graceful-fs or normal fs
const fs = require("graceful-fs"); // or use: const fs = require("fs") 

// Create output file and set it up to receive streamed data
// Flag is to say "append" so that data can be recursively added to the same file 
let fakeCSV = fs.createWriteStream("./output/document.csv", {
  flags: "a",
});

// create custom streamer that can be invoked when needed
const customStreamer = (dataToWrite) => {
  fakeCSV.write(dataToWrite + "\n");
};

const dataToWrite = "batman" + ";" + "superman"
customStreamer(dataToWrite);