Javascript CSV到CSV格式的JSON

通过REST API端点，我获得了具有以下结构的相当大的CSV文件（CSV文件中的JSON）：

对于另一个工具，我需要一个具有平面结构（没有嵌套JSON）的CSV。所以，最后，我希望有一个CSV，看起来像这样

A,B,C,E,F,G,H
1,2,3,1,2,3,
1,2,3,1,,,2

（尽管列标题看起来是结构化的，但这对于我的用例并不重要）

由于CSV文件相当大，我正在寻找一种性能相对较好的方法。我将用JavaScript（Node.JS）编写这篇文章（因为这是脚本所有其他部分使用的语言）。然而，现在我只是在寻找一种理论上的方法/伪代码来实现性能问题

据我所知，我可能不得不在CSV文件上循环两次。第一次我只需要获取所有JSON密钥。第二次，我可以创建一个新的CSV文件并设置所有值。但是，我会正确地找出我必须在哪一列中写入值吗

或者在一个循环中将CSV文件“转换”为一组对象，然后使用类似CSV解析器（）的方法将其转换回CSV，这会更有效吗

var express = require('express');
var app = express();
var bodyParser = require('body-parser');
var mysql=require('mysql');
var fs= require('fs');
var csv = require('fast-csv');
var formidable = require('formidable');
var urlencodedParser = bodyParser.urlencoded({ extended: false })
var con=mysql.createConnection({
host:'localhost',
user:'dheeraj',
password:'123',
database:'dheeraj'
});
app.use('/assets',express.static('assets'));
app.get('/d', function (req, res) {
   res.sendFile( __dirname + "/" + "/d.html" );
})

app.post('/file_upload', urlencodedParser, function (req, res) {

  //{
  var form = new formidable.IncomingForm();
  form.parse(req, function (err, fields, files) {
    res.write('File uploaded');
    //console.log(files.filetoupload);

    fs.createReadStream(files.filetoupload.name)
      .pipe(csv())
      .on('data',function(data){
        var d1=data[0];
          var d2=data[1];
            var d3=data[2];
              var d4=data[3];
                var d5=data[4];
        con.query('insert into demo values(\''+d1+'\',\''+d2+'\',\''+d3+'\',\''+d4+'\',\''+d5+'\')',function(err,result)
            {
              console.log('inserted');
            })
        console.log(data);
      })
      .on('end',function(data){
      console.log('read finished');
      });

    res.end();

})
})

var server = app.listen(8081, function () {
var host = server.address().address
var port = server.address().port

console.log("Example app listening at http://%s:%s", host, port)

})

如果文件

filter.jq

包含

[
  split("\n")                                                  # split string into lines
| (.[0]    | split(",")) as $headers                           # split header
| (.[1:][] | split(","))                                       # split data rows
| select(length>0)                                             # get rid of empty lines
| $headers[:-1] as $h1                                         # fixed headers
| .[:($h1|length)] as $p1                                      # fixed part
| .[($h1|length):] as $p2                                      # variable part
| (
     [   [ $h1, $p1 ]                                          # \  
       | transpose[]                                           #  \ assemble fixed object
       | {key:.[0], value:.[1]|tonumber}                       #  / from fixed keys and values
     ] | from_entries                                          # /
  ) + (
     $p2 | join(",") | fromjson                                # assemble variable object
  )
]

| (map(keys) | add | unique) as $all                           # compute final headers
| [$all] + (                                                   # add headers to
       map(. as $b | reduce $all[] as $a ([];. + [$b[$a]]))    # objects with all keys
     | map(map(if . == null then "" else tostring end))        # convert values to strings
  )
| .[]                                                          # scan final array
| @csv                                                         # convert to csv

您的数据在一个名为

data

的文件中，然后

jq -M -R -s -r -f filter.jq data

将产生

"A","B","C","E","F","G","H"
"1","2","3","1","2","3",""
"1","2","3","1","","","2"

---

这是我的代码，用于将csv文件上载到页面并检索可插入数据库的数据。我希望它能帮助你。谢谢，但这并不能解决我的问题，因为我的CSV数据中也有JSON数据。

"A","B","C","E","F","G","H"
"1","2","3","1","2","3",""
"1","2","3","1","","","2"