Google bigquery 为什么bigquery去客户端库给出;错误:bigquery:递归类型*bigquery.FieldSchema的架构推断

Google bigquery 为什么bigquery去客户端库给出;错误:bigquery:递归类型*bigquery.FieldSchema的架构推断,google-bigquery,Google Bigquery,以下代码使用用于BigQuery的go客户端库返回一个错误 type GceQuotaRow struct { CustomerProjectNumber int64 ExecutionProjectNumber int64 Region string Metric string Limit float64 } ds := b.bq.Dataset(task.Options.

以下代码使用用于BigQuery的go客户端库返回一个错误

type GceQuotaRow struct {
  CustomerProjectNumber  int64
  ExecutionProjectNumber int64
  Region                 string
  Metric                 string
  Limit                  float64
}

ds := b.bq.Dataset(task.Options.Dataset)
table := ds.Table(task.Options.Table)

u := table.Uploader()

rows := []bigquery.StructSaver{}
// rows := []*GceQuotaRow{}

schema, err := bigquery.InferSchema(GceQuotaRow{})
if err != nil {
    log.Errorf("Inferring BigQuery scheme failed; %v", util.Pformat(err))
    return backfill.TaskPermanentError
}

// Construct an insertID to dedupe insertions on a best effort basis.

for _, region := range res.GetRegionalQuotas() {
    for _, q := range region.GetQuotas() {

        r := &GceQuotaRow{
            CustomerProjectNumber:  task.CustomerProjectNumber,
            ExecutionProjectNumber: res.GetExecutionProjectNumber(),
            Region:                 region.GetRegion(),
            Metric:                 q.GetMetric(),
            Limit:                  q.GetLimit(),
        }
        h := sha256.New()
        h.Write([]byte(util.Pformat(r)))

        // Use the sha256 of the row as the InsertId to avoid duplicates.
        insertId := fmt.Sprintf("%x", h.Sum(nil))

        rows = append(rows, bigquery.StructSaver{
            Schema:   schema,
            InsertID: insertId,
            Struct:   r,
        })
        // rows = append(rows, r)
    }
}

err = u.Put(ctx, rows)
if err != nil {
    putErr, isErr := err.(bigquery.PutMultiError)
    if isErr {
        for _, e := range putErr {
            log.Errorf("There was a problem writing Row for customer project: %v, Error: %v", task.CustomerProjectNumber, util.Pformat(e))
        }
    } else {
        log.Errorf("There was a problem writing GCE Quota Rows for customer project: %v, Error: %v", task.CustomerProjectNumber, err)
    }
}
调用Put返回错误:

Error: bigquery: schema inference for recursive type *bigquery.FieldSchema
我不明白为什么Put会返回一个关于推断模式的错误,因为推断模式调用成功,所以应该将模式传递给structsaver


我想使用StructSaver,以便提供一个插入id来删除重复行

我只是偶然发现了这一点,并把它弄明白了。发生这种情况的原因是,具体的
bigquery.StructSaver
没有实现
bigquery.ValueSaver
接口,因为其方法在指针类型
*bigquery.StructSaver
上操作。这会导致golang客户端库在您的结构(在本例中为
bigquery.StructSaver
)上退回到其通用模式推理逻辑,然后失败,因为
bigquery.StructSaver
嵌套了其他结构,因此它本身不是保存在bigquery中的有效对象。您可以按如下方式修复代码:

for _, region := range res.GetRegionalQuotas() {
    for _, q := range region.GetQuotas() {

        r := &GceQuotaRow{
            CustomerProjectNumber:  task.CustomerProjectNumber,
            ExecutionProjectNumber: res.GetExecutionProjectNumber(),
            Region:                 region.GetRegion(),
            Metric:                 q.GetMetric(),
            Limit:                  q.GetLimit(),
        }
        h := sha256.New()
        h.Write([]byte(util.Pformat(r)))

        // Use the sha256 of the row as the InsertId to avoid duplicates.
        insertId := fmt.Sprintf("%x", h.Sum(nil))

        rows = append(rows, &bigquery.StructSaver{
            Schema:   schema,
            InsertID: insertId,
            Struct:   r,
        })
        // rows = append(rows, r)
    }
}

err = u.Put(ctx, rows)
if err != nil {
    putErr, isErr := err.(bigquery.PutMultiError)
    if isErr {
        for _, e := range putErr {
            log.Errorf("There was a problem writing Row for customer project: %v, Error: %v", task.CustomerProjectNumber, util.Pformat(e))
        }
    } else {
        log.Errorf("There was a problem writing GCE Quota Rows for customer project: %v, Error: %v", task.CustomerProjectNumber, err)
    }
}

我希望这能有所帮助(如果不是你,也许是两年后的事实,那么其他任何人偶然发现了这一点)

我只是偶然发现了这一点,并把它弄明白了。发生这种情况的原因是,具体的
bigquery.StructSaver
没有实现
bigquery.ValueSaver
接口,因为其方法在指针类型
*bigquery.StructSaver
上操作。这会导致golang客户端库在您的结构(在本例中为
bigquery.StructSaver
)上退回到其通用模式推理逻辑,然后失败,因为
bigquery.StructSaver
嵌套了其他结构,因此它本身不是保存在bigquery中的有效对象。您可以按如下方式修复代码:

for _, region := range res.GetRegionalQuotas() {
    for _, q := range region.GetQuotas() {

        r := &GceQuotaRow{
            CustomerProjectNumber:  task.CustomerProjectNumber,
            ExecutionProjectNumber: res.GetExecutionProjectNumber(),
            Region:                 region.GetRegion(),
            Metric:                 q.GetMetric(),
            Limit:                  q.GetLimit(),
        }
        h := sha256.New()
        h.Write([]byte(util.Pformat(r)))

        // Use the sha256 of the row as the InsertId to avoid duplicates.
        insertId := fmt.Sprintf("%x", h.Sum(nil))

        rows = append(rows, &bigquery.StructSaver{
            Schema:   schema,
            InsertID: insertId,
            Struct:   r,
        })
        // rows = append(rows, r)
    }
}

err = u.Put(ctx, rows)
if err != nil {
    putErr, isErr := err.(bigquery.PutMultiError)
    if isErr {
        for _, e := range putErr {
            log.Errorf("There was a problem writing Row for customer project: %v, Error: %v", task.CustomerProjectNumber, util.Pformat(e))
        }
    } else {
        log.Errorf("There was a problem writing GCE Quota Rows for customer project: %v, Error: %v", task.CustomerProjectNumber, err)
    }
}

我希望这能有所帮助(如果不是你,也许是两年后的事实,那么其他任何人偶然发现了这一点)

我也面临类似的问题,但我的结构中嵌入了字段,这似乎引发了错误。有关于这个问题的更新吗?你解决了吗?也面临类似的问题,但我的结构中嵌入了字段,这似乎引发了错误。有关于这个问题的更新吗?你解决了吗?