Fatal编程技术网
  • node.js/
  • Node.js ExpressJS中用于多个子进程生成的错误处理中间件

Node.js ExpressJS中用于多个子进程生成的错误处理中间件

node.js express

Node.js ExpressJS中用于多个子进程生成的错误处理中间件,node.js,express,winston,Node.js,Express,Winston,我已经编写了一个不错的小错误报告中间件，它位于所有GET和POST处理之后（在app.use（app.router）；）。见下文 这对于简单的快速获取和发布到PostGIS数据库等非常有用 但是我有一个POST请求，它被设计用来创建一堆目录、许多文件，然后生成1->8个子进程任务 childProcess.execFile(job.config.FMEPath, ["PARAMETER_FILE", job.fmeParamFile], { cwd: job.root }, 所有这些设置都不会

我已经编写了一个不错的小错误报告中间件，它位于所有GET和POST处理之后（在app.use（app.router）；）。见下文

这对于简单的快速获取和发布到PostGIS数据库等非常有用

但是我有一个POST请求，它被设计用来创建一堆目录、许多文件，然后生成1->8个子进程任务

childProcess.execFile(job.config.FMEPath, ["PARAMETER_FILE", job.fmeParamFile], { cwd: job.root },
所有这些设置都不会花费太多时间（不到一秒钟，而且都是异步的（我在一个点上使用异步库来安排5个步骤的顺序（见下文）

我的问题是错误处理。现在，在创建所有文件和执行所有步骤之前，我会立即返回一个响应。这意味着next（err）没有按预期工作。报告错误的好范例是什么？我正在使用WINSTON记录错误[logger.log（）]，但我应该只在服务器上记录错误，还是也应该向原始请求报告错误。这是当前的post请求（请记住，我必须将其余的、req和next对象保留一段时间才能调用next（错误）

jobBatchRoot（）（然后我将进行大量处理，我没有包括所有代码

exports.bugs = function (err, req, res, next) {
    global.app.settings.stats.errors += 1;
    if (err.code == undefined) {
        err.code = 500;
        err.message = "Server Error";
    }
    res.status(err.code);
    logger.log('error', '%s url: %s status: %d \n', req.method, req.url, err.code, { query: req.query, body: req.body, message: err.message, stack: err.stack });
    var desc = req.method + " " + req.url;
    var body = util.format("%j", req.body);
    var query = util.format("%j", req.query);
    var stack = err.stack.split('\n');
    res.format({
        text: function () {
            res.send(util.format("%j", { title: err.message, code: err.code, desc: desc, query: query, message: err.message, stack: err.stack, body: body}));
        },

        html: function () {
            query = tools.pretty(req.query);
            res.render('error', { title: err.message, code: err.code, desc: desc, query: query, message: err.message, stack: stack, body: body });
        }, 

        json: function () {
            res.send({ title: err.message, code: err.code, desc: desc, query: query, message: err.message, stack: err.stack, body: body });
        }
    });

};
也许我应该重新考虑这一点（也许是面向对象的），无论如何，我想我会在这里发布完整的模块，我想要的只是一些关于结构和最佳实践的提示

var util = require('util');
var query = require("pg-query");
var timespan = require('timespan');
var _ = require('lodash');
var path = require('path');
var fs = require('fs');
var query = require("pg-query");
var async = require("async");
var childProcess = require("child_process");
var tools = require("../tools/tools");
var nitfMultiplier = 99;
var manifestVersionID = 5;

exports.setup = function (app) {
};

exports.estimate = function (req, res, next) {
    var config = global.app.settings.config;
    var fileCount = req.body.FrameList.length * nitfMultiplier;
    var ts = timespan.fromSeconds(fileCount / config.TileRate);
    var estimate = ts.hours + ":" + tools.pad(ts.minutes, 2) + ":" + tools.pad(ts.seconds, 2);
    res.set({ 'Content-Type': 'application/json; charset=utf-8' });
    res.send({ ticket: "Estimate", maxTiles: fileCount, timeEstimate: estimate, tileRate: config.TileRate, wwwURL: config.WWWUrl });
};

exports.build = function (req, res, next) {
    var config = global.app.settings.config;
    var jobBatch = groupJobs(req.body.FrameList);
    var ticket = tools.newGuid("", true);
    var fileCount = req.body.FrameList.length * nitfMultiplier;
    var ts = timespan.fromSeconds(fileCount / config.TileRate);
    var estimate = ts.hours + ":" + tools.pad(ts.minutes, 2) + ":" + tools.pad(ts.seconds, 2);
    res.set({ 'Content-Type': 'application/json; charset=utf-8' });
    res.send({ ticket: ticket, maxTiles: fileCount, timeEstimate: estimate, tileRate: config.TileRate, wwwURL: config.WWWUrl });
    jobBatchRoot(req, res, jobBatch, config, ticket, next);
};

function groupJobs(list) {
    var jobBatch = {};
    _.forEach(list, function (obj) {
        if (jobBatch[obj.type] == undefined) {
            jobBatch[obj.type] = [];
        }
        jobBatch[obj.type].push(obj);
    });
    return jobBatch;
};

function jobBatchRoot(req, res, jobBatch, config, ticket, next) {
    var batchRoot = path.join(config.JobsPath, ticket);
    fs.mkdir(batchRoot, function (err) {
        if (err) return next(err);
        var mapInfoFile = path.join(batchRoot, "MapInfo.json");
        var mapInfo = {
            Date: (new Date()).toISOString(),
            Version: manifestVersionID,
            Zoom: req.body.Zoom,
            CenterLat: req.body.CenterLat,
            CenterLon: req.body.CenterLon
        };
        fs.writeFile(mapInfoFile, tools.pretty(mapInfo), function (err) {
            if (err) return next(err);
            spawnJobs(req, res, batchRoot, mapInfo, config, ticket, jobBatch, next);
        });
    });
};

function spawnJobs(req, res, root, mapInfo, config, ticket, jobBatch, next) {
    _.forEach(jobBatch, function (files, key) {
        var job = {
            req: req,
            res: res,
            type: key,
            files: files,
            batchRoot: root,
            mapInfo: mapInfo,
            config: config,
            ticket: ticket,
            missingFiles: [],
            run: true,
            next: next
        };
        setup(job);
    });
};

function setup(job) {
    job.root = path.join(job.batchRoot, job.type);
    job.fmeParamFile = path.join(job.root, "fmeParameters.txt");
    job.fmeWorkSpace = path.join(job.config.LibrarianPath, "TileBuilder.fmw");
    job.fmeLogFile = path.join(job.root, "jobLog.log");
    job.errorLog = path.join(job.root, "errorLog.log");
    job.jobFile = path.join(job.root, "jobFile.json");
    job.manifestFile = path.join(job.root, "manifest.json");
    async.series({
        one: function (callback) {
            maxZoom(job, callback);
        },
        two: function (callback) {
            fs.mkdir(job.root, function (err) {
                if (err) return job.next(err);
                callback(null, "Job Root Created");
            });
        },
        three: function (callback) {
            makeParamFile(job, callback);
        },
        four: function (callback) {
            delete job.req;
            delete job.res;
            fs.writeFile(job.jobFile, tools.pretty(job), function (err) {
                if (err) return job.next(err);
                callback(null, "Wrote Job File");
            });
        },
        five: function (callback) {
            runJob(job, callback);
        },
        six: function (callback) {
            tileList(job, callback);
        },
        seven: function (callback) {
            finish(job, callback);
        },
    },
    function (err, results) {
        if (err) return job.next(err);
        console.log(tools.pretty(results));
    });
}

function maxZoom(job, callback) {
    var qString = util.format('SELECT type, "maxZoom" FROM portal.m_type WHERE type=\'%s\'', job.type);
    query(qString, function (err, rows, result) {
        if (err) {
            var err = new Error(queryName);
            err.message = err.message + " - " + qString;
            err.code = 400;
            return job.next(err);
        }
        job.maxZoom = rows[0].maxZoom - 1; // kludge for 2x1 root layer in leaflet
        return callback(null, "Got MaxZoom");
    });
}

function makeParamFile(job, callback) {
    var text = util.format("%s\n", job.fmeWorkSpace);
    text += util.format("--OutputDir %s\n", job.root);
    text += util.format("--LogFile %s\n", job.fmeLogFile);
    var source = "";
    _.forEach(job.files, function (file) {
        var path = ('development' == process.env.NODE_ENV) ? file.path.replace(job.config.SourceRootRaw, job.config.SourceRoot) : file.path;
        if (fs.existsSync(path)) {
            source += wrap(path, '\\"') + " ";
        }
        else {
            job.missingFiles.push(path);
        }
    });
    source = wrap(wrap(source, '\\"'), '"');
    text += "--Sources " + source;
    if (job.missingFiles.length == job.files.length) job.run = false;
    fs.writeFile(job.fmeParamFile, text, function (err) {
        if (err) return job.next(err);
        return callback(null, "Wrote Paramaters File");
    })
};

function wrap(content, edge) {
    return edge+content+edge;
    }

function runJob(job, callback) {
    if (!job.run) return callback(null, "Skipped JOB, no files");
    childProcess.execFile(job.config.FMEPath, ["PARAMETER_FILE", job.fmeParamFile], { cwd: job.root },
        function (err, stdout, stderr) {
            if (err) return job.next(err);
            job.stdout = stdout;
            job.stderr = stderr;
            var bar = "\n--------------------------------------------------------------------------------------------------------\n";
            var results = util.format("%s STDOUT: \n %s%s STDERR: \n %s", bar, job.stdout, bar, job.stderr);
            fs.appendFile(job.fmeLogFile, results, function (err) {
                return callback(err, "FME JOB " + job.type + " run completed");
            });
        });
}

function tileList(job, callback) {
    var tiles = [];
    var byteCount = 0;
    fs.readdir(job.root, function (err, files) {
        if (err) {
            logger.log('error', 'tileList directory read: %s \n', job.root, { message: err.message, stack: err.stack });
            return job.next(err);
        }
        async.each(files, function (file, done) {
            var fileName = file.split(".");
            fs.lstat(job.root + "\\" + file, function (err, stats) {
                if (!err && stats.isFile() && (fileName[1] == "png")) {
                    tiles.push({ id: fileName[0], size: stats.size });
                    byteCount += stats.size;
                };
                done(null);
            });
        }, function (err) {
        job.tileList = tiles;
        job.byteCount = byteCount;
        return callback(null, "got tile list");}
        );
    });
}

function finish(job, callback) {
    var manifest = {
        Date: (new Date()).toISOString(),
        Version: manifestVersionID,
        MaxZoom: job.maxZoom,
        Class: "OVERLAY",
        FileCount: job.tileList.length,
        Size: job.byteCount / (1024 * 1024), // Mbytes
        files: job.tileList
    };
    fs.writeFile(job.manifestFile, tools.pretty(manifest), function (err) {
        if (err) {
            logger.log('error', 'manifest write: %s \n', job.manifestFile, { message: err.message, stack: err.stack });
            return job.next(err);
        }
        return callback(null, "JOB " + job.type + " completed");
    });
}
我去重新考虑了这个问题，创建了一个模块，module.exports=function（..）{…}

然后添加了大量的状态和方法来创建一个类。该类包含作业定义。因此，现在我创建顶级目录，返回响应，并生成子作业。它们都在快速响应后异步运行。但它们不应该出现错误，如果出现错误，则使用WINSTON将它们记录到服务器中，并返回作业当所有构建完成时，将信息发送给用户

var util = require('util');
var query = require("pg-query");
var timespan = require('timespan');
var _ = require('lodash');
var path = require('path');
var fs = require('fs');
var query = require("pg-query");
var async = require("async");
var childProcess = require("child_process");
var tools = require("../tools/tools");
var nitfMultiplier = 99;
var manifestVersionID = 5;

exports.setup = function (app) {
};

exports.estimate = function (req, res, next) {
    var config = global.app.settings.config;
    var fileCount = req.body.FrameList.length * nitfMultiplier;
    var ts = timespan.fromSeconds(fileCount / config.TileRate);
    var estimate = ts.hours + ":" + tools.pad(ts.minutes, 2) + ":" + tools.pad(ts.seconds, 2);
    res.set({ 'Content-Type': 'application/json; charset=utf-8' });
    res.send({ ticket: "Estimate", maxTiles: fileCount, timeEstimate: estimate, tileRate: config.TileRate, wwwURL: config.WWWUrl });
};

exports.build = function (req, res, next) {
    var config = global.app.settings.config;
    var jobBatch = groupJobs(req.body.FrameList);
    var ticket = tools.newGuid("", true);
    var fileCount = req.body.FrameList.length * nitfMultiplier;
    var ts = timespan.fromSeconds(fileCount / config.TileRate);
    var estimate = ts.hours + ":" + tools.pad(ts.minutes, 2) + ":" + tools.pad(ts.seconds, 2);
    res.set({ 'Content-Type': 'application/json; charset=utf-8' });
    res.send({ ticket: ticket, maxTiles: fileCount, timeEstimate: estimate, tileRate: config.TileRate, wwwURL: config.WWWUrl });
    jobBatchRoot(req, res, jobBatch, config, ticket, next);
};

function groupJobs(list) {
    var jobBatch = {};
    _.forEach(list, function (obj) {
        if (jobBatch[obj.type] == undefined) {
            jobBatch[obj.type] = [];
        }
        jobBatch[obj.type].push(obj);
    });
    return jobBatch;
};

function jobBatchRoot(req, res, jobBatch, config, ticket, next) {
    var batchRoot = path.join(config.JobsPath, ticket);
    fs.mkdir(batchRoot, function (err) {
        if (err) return next(err);
        var mapInfoFile = path.join(batchRoot, "MapInfo.json");
        var mapInfo = {
            Date: (new Date()).toISOString(),
            Version: manifestVersionID,
            Zoom: req.body.Zoom,
            CenterLat: req.body.CenterLat,
            CenterLon: req.body.CenterLon
        };
        fs.writeFile(mapInfoFile, tools.pretty(mapInfo), function (err) {
            if (err) return next(err);
            spawnJobs(req, res, batchRoot, mapInfo, config, ticket, jobBatch, next);
        });
    });
};

function spawnJobs(req, res, root, mapInfo, config, ticket, jobBatch, next) {
    _.forEach(jobBatch, function (files, key) {
        var job = {
            req: req,
            res: res,
            type: key,
            files: files,
            batchRoot: root,
            mapInfo: mapInfo,
            config: config,
            ticket: ticket,
            missingFiles: [],
            run: true,
            next: next
        };
        setup(job);
    });
};

function setup(job) {
    job.root = path.join(job.batchRoot, job.type);
    job.fmeParamFile = path.join(job.root, "fmeParameters.txt");
    job.fmeWorkSpace = path.join(job.config.LibrarianPath, "TileBuilder.fmw");
    job.fmeLogFile = path.join(job.root, "jobLog.log");
    job.errorLog = path.join(job.root, "errorLog.log");
    job.jobFile = path.join(job.root, "jobFile.json");
    job.manifestFile = path.join(job.root, "manifest.json");
    async.series({
        one: function (callback) {
            maxZoom(job, callback);
        },
        two: function (callback) {
            fs.mkdir(job.root, function (err) {
                if (err) return job.next(err);
                callback(null, "Job Root Created");
            });
        },
        three: function (callback) {
            makeParamFile(job, callback);
        },
        four: function (callback) {
            delete job.req;
            delete job.res;
            fs.writeFile(job.jobFile, tools.pretty(job), function (err) {
                if (err) return job.next(err);
                callback(null, "Wrote Job File");
            });
        },
        five: function (callback) {
            runJob(job, callback);
        },
        six: function (callback) {
            tileList(job, callback);
        },
        seven: function (callback) {
            finish(job, callback);
        },
    },
    function (err, results) {
        if (err) return job.next(err);
        console.log(tools.pretty(results));
    });
}

function maxZoom(job, callback) {
    var qString = util.format('SELECT type, "maxZoom" FROM portal.m_type WHERE type=\'%s\'', job.type);
    query(qString, function (err, rows, result) {
        if (err) {
            var err = new Error(queryName);
            err.message = err.message + " - " + qString;
            err.code = 400;
            return job.next(err);
        }
        job.maxZoom = rows[0].maxZoom - 1; // kludge for 2x1 root layer in leaflet
        return callback(null, "Got MaxZoom");
    });
}

function makeParamFile(job, callback) {
    var text = util.format("%s\n", job.fmeWorkSpace);
    text += util.format("--OutputDir %s\n", job.root);
    text += util.format("--LogFile %s\n", job.fmeLogFile);
    var source = "";
    _.forEach(job.files, function (file) {
        var path = ('development' == process.env.NODE_ENV) ? file.path.replace(job.config.SourceRootRaw, job.config.SourceRoot) : file.path;
        if (fs.existsSync(path)) {
            source += wrap(path, '\\"') + " ";
        }
        else {
            job.missingFiles.push(path);
        }
    });
    source = wrap(wrap(source, '\\"'), '"');
    text += "--Sources " + source;
    if (job.missingFiles.length == job.files.length) job.run = false;
    fs.writeFile(job.fmeParamFile, text, function (err) {
        if (err) return job.next(err);
        return callback(null, "Wrote Paramaters File");
    })
};

function wrap(content, edge) {
    return edge+content+edge;
    }

function runJob(job, callback) {
    if (!job.run) return callback(null, "Skipped JOB, no files");
    childProcess.execFile(job.config.FMEPath, ["PARAMETER_FILE", job.fmeParamFile], { cwd: job.root },
        function (err, stdout, stderr) {
            if (err) return job.next(err);
            job.stdout = stdout;
            job.stderr = stderr;
            var bar = "\n--------------------------------------------------------------------------------------------------------\n";
            var results = util.format("%s STDOUT: \n %s%s STDERR: \n %s", bar, job.stdout, bar, job.stderr);
            fs.appendFile(job.fmeLogFile, results, function (err) {
                return callback(err, "FME JOB " + job.type + " run completed");
            });
        });
}

function tileList(job, callback) {
    var tiles = [];
    var byteCount = 0;
    fs.readdir(job.root, function (err, files) {
        if (err) {
            logger.log('error', 'tileList directory read: %s \n', job.root, { message: err.message, stack: err.stack });
            return job.next(err);
        }
        async.each(files, function (file, done) {
            var fileName = file.split(".");
            fs.lstat(job.root + "\\" + file, function (err, stats) {
                if (!err && stats.isFile() && (fileName[1] == "png")) {
                    tiles.push({ id: fileName[0], size: stats.size });
                    byteCount += stats.size;
                };
                done(null);
            });
        }, function (err) {
        job.tileList = tiles;
        job.byteCount = byteCount;
        return callback(null, "got tile list");}
        );
    });
}

function finish(job, callback) {
    var manifest = {
        Date: (new Date()).toISOString(),
        Version: manifestVersionID,
        MaxZoom: job.maxZoom,
        Class: "OVERLAY",
        FileCount: job.tileList.length,
        Size: job.byteCount / (1024 * 1024), // Mbytes
        files: job.tileList
    };
    fs.writeFile(job.manifestFile, tools.pretty(manifest), function (err) {
        if (err) {
            logger.log('error', 'manifest write: %s \n', job.manifestFile, { message: err.message, stack: err.stack });
            return job.next(err);
        }
        return callback(null, "JOB " + job.type + " completed");
    });
}