Google cloud platform Google Cloud中的流媒体插入/更新-BigQuery

我正在尝试将Salesforce数据流传输到Google Cloud Bigquery。设法实现插入流，例如：每当在SF中创建新Lead时，都会将其插入到Biguerry表中。检查，是否有一种方法可以用于数据的上传。我知道有一个流式缓冲区，它不允许对插入的数据执行任何DML操作，因为这些数据将在流式缓冲区中保留很短的时间

真的很感谢你给我的提示

编辑-2019年6月6日

使用云下功能插入记录

    /**
     * Responds to any HTTP request.
     *
     * @param {!express:Request} req HTTP request context.
     * @param {!express:Response} res HTTP response context.
     */
    exports.helloWorld = (req, res) => {
    let message = req.query.mes || req.body.mes || 'Hello World!';
    res.status(200).send(req.body);
    var d =JSON.stringify(req.body);
      console.log(d);
    var e = d.replace(/:""/g, '');
  var f = e.replace(/\\/g, '');
  var g = f.replace(/"{n /g, '');
  var h = g.replace(/n}"/g, '');
  var i = h.replace(/n /g, '');
  console.log(i);
  const {BigQuery} = require('@google-cloud/bigquery');
  const bigquery = new BigQuery();
  var instance = "DEMO";
  var table = "HTTP";
  bigquery
    .dataset(instance)
    .table(table)
    .insert(JSON.parse(i),
    {'ignoreUnknownValues':true, 'raw':false})
    .then ((data) => {
      console.log('Inserted 1 rows');
      console.log(data);
    })
    };

---

用于更新的代码

exports.helloWorld = (req, res) => {
  let message = req.query.mes || req.body.mes || 'Hello World!';
  res.status(200).send(req.body);
  var d =JSON.stringify(req.body);
  console.log(d);
  var e = d.replace(/:""/g, '');
  var f = e.replace(/\\/g, '');
  var g = f.replace(/"{n /g, '');
  var h = g.replace(/n}"/g, '');
  var i = h.replace(/n /g, '');
  console.log(i);
  var j = JSON.parse(i);
  var k = JSON.stringify(j.Id);
  var id = k.replace(/"/g, '');
  console.log(k);
  console.log(id);
  const {BigQuery} = require('@google-cloud/bigquery');
  const bigquery = new BigQuery();
  var instance = "DEMO";
  var table = "LEADS_STG";
  bigquery
    .dataset(instance)
    .table(table)
    .insert(JSON.parse(i),
    {'ignoreUnknownValues':true, 'raw':false})
    .then ((data) => {
      console.log('Inserted 1 rows');
      console.log(data);
    })
  /*const bigqueryClient = new BigQuery();*/
  var delayInMilliseconds = 1000;
  setTimeout(function() {
  bigquery.query({
  query: [
    'MERGE DEMO.LEADS_D T USING  (SELECT ID,NAME,LEADSOURCE,COMPANY FROM DEMO.LEADS_STG where ID= ? order by LASTMODIFIEDDATE DESC LIMIT 1) S ON  T.ID = S.ID  WHEN MATCHED  THEN  UPDATE   SET  NAME = S.NAME,  LEADSOURCE = S.LEADSOURCE,  COMPANY = S.COMPANY  WHEN NOT MATCHED  THEN INSERT  ( ID,    NAME,    LEADSOURCE,    COMPANY) VALUES( ID, NAME,LEADSOURCE,COMPANY)'  
  ].join(' '),
  params: [
    id
  ]
}, function(err, rows) {});
    }, delayInMilliseconds);
};

---

用于更新的代码

exports.helloWorld = (req, res) => {
  let message = req.query.mes || req.body.mes || 'Hello World!';
  res.status(200).send(req.body);
  var d =JSON.stringify(req.body);
  console.log(d);
  var e = d.replace(/:""/g, '');
  var f = e.replace(/\\/g, '');
  var g = f.replace(/"{n /g, '');
  var h = g.replace(/n}"/g, '');
  var i = h.replace(/n /g, '');
  console.log(i);
  var j = JSON.parse(i);
  var k = JSON.stringify(j.Id);
  var id = k.replace(/"/g, '');
  console.log(k);
  console.log(id);
  const {BigQuery} = require('@google-cloud/bigquery');
  const bigquery = new BigQuery();
  var instance = "DEMO";
  var table = "LEADS_STG";
  bigquery
    .dataset(instance)
    .table(table)
    .insert(JSON.parse(i),
    {'ignoreUnknownValues':true, 'raw':false})
    .then ((data) => {
      console.log('Inserted 1 rows');
      console.log(data);
    })
  /*const bigqueryClient = new BigQuery();*/
  var delayInMilliseconds = 1000;
  setTimeout(function() {
  bigquery.query({
  query: [
    'MERGE DEMO.LEADS_D T USING  (SELECT ID,NAME,LEADSOURCE,COMPANY FROM DEMO.LEADS_STG where ID= ? order by LASTMODIFIEDDATE DESC LIMIT 1) S ON  T.ID = S.ID  WHEN MATCHED  THEN  UPDATE   SET  NAME = S.NAME,  LEADSOURCE = S.LEADSOURCE,  COMPANY = S.COMPANY  WHEN NOT MATCHED  THEN INSERT  ( ID,    NAME,    LEADSOURCE,    COMPANY) VALUES( ID, NAME,LEADSOURCE,COMPANY)'  
  ].join(' '),
  params: [
    id
  ]
}, function(err, rows) {});
    }, delayInMilliseconds);
};