Google bigquery 为什么bigquery去客户端库给出;错误:bigquery:递归类型*bigquery.FieldSchema的架构推断
以下代码使用用于BigQuery的go客户端库返回一个错误Google bigquery 为什么bigquery去客户端库给出;错误:bigquery:递归类型*bigquery.FieldSchema的架构推断,google-bigquery,Google Bigquery,以下代码使用用于BigQuery的go客户端库返回一个错误 type GceQuotaRow struct { CustomerProjectNumber int64 ExecutionProjectNumber int64 Region string Metric string Limit float64 } ds := b.bq.Dataset(task.Options.
type GceQuotaRow struct {
CustomerProjectNumber int64
ExecutionProjectNumber int64
Region string
Metric string
Limit float64
}
ds := b.bq.Dataset(task.Options.Dataset)
table := ds.Table(task.Options.Table)
u := table.Uploader()
rows := []bigquery.StructSaver{}
// rows := []*GceQuotaRow{}
schema, err := bigquery.InferSchema(GceQuotaRow{})
if err != nil {
log.Errorf("Inferring BigQuery scheme failed; %v", util.Pformat(err))
return backfill.TaskPermanentError
}
// Construct an insertID to dedupe insertions on a best effort basis.
for _, region := range res.GetRegionalQuotas() {
for _, q := range region.GetQuotas() {
r := &GceQuotaRow{
CustomerProjectNumber: task.CustomerProjectNumber,
ExecutionProjectNumber: res.GetExecutionProjectNumber(),
Region: region.GetRegion(),
Metric: q.GetMetric(),
Limit: q.GetLimit(),
}
h := sha256.New()
h.Write([]byte(util.Pformat(r)))
// Use the sha256 of the row as the InsertId to avoid duplicates.
insertId := fmt.Sprintf("%x", h.Sum(nil))
rows = append(rows, bigquery.StructSaver{
Schema: schema,
InsertID: insertId,
Struct: r,
})
// rows = append(rows, r)
}
}
err = u.Put(ctx, rows)
if err != nil {
putErr, isErr := err.(bigquery.PutMultiError)
if isErr {
for _, e := range putErr {
log.Errorf("There was a problem writing Row for customer project: %v, Error: %v", task.CustomerProjectNumber, util.Pformat(e))
}
} else {
log.Errorf("There was a problem writing GCE Quota Rows for customer project: %v, Error: %v", task.CustomerProjectNumber, err)
}
}
调用Put返回错误:
Error: bigquery: schema inference for recursive type *bigquery.FieldSchema
我不明白为什么Put会返回一个关于推断模式的错误,因为推断模式调用成功,所以应该将模式传递给structsaver
我想使用StructSaver,以便提供一个插入id来删除重复行 我只是偶然发现了这一点,并把它弄明白了。发生这种情况的原因是,具体的
bigquery.StructSaver
没有实现bigquery.ValueSaver
接口,因为其方法在指针类型*bigquery.StructSaver
上操作。这会导致golang客户端库在您的结构(在本例中为bigquery.StructSaver
)上退回到其通用模式推理逻辑,然后失败,因为bigquery.StructSaver
嵌套了其他结构,因此它本身不是保存在bigquery中的有效对象。您可以按如下方式修复代码:
for _, region := range res.GetRegionalQuotas() {
for _, q := range region.GetQuotas() {
r := &GceQuotaRow{
CustomerProjectNumber: task.CustomerProjectNumber,
ExecutionProjectNumber: res.GetExecutionProjectNumber(),
Region: region.GetRegion(),
Metric: q.GetMetric(),
Limit: q.GetLimit(),
}
h := sha256.New()
h.Write([]byte(util.Pformat(r)))
// Use the sha256 of the row as the InsertId to avoid duplicates.
insertId := fmt.Sprintf("%x", h.Sum(nil))
rows = append(rows, &bigquery.StructSaver{
Schema: schema,
InsertID: insertId,
Struct: r,
})
// rows = append(rows, r)
}
}
err = u.Put(ctx, rows)
if err != nil {
putErr, isErr := err.(bigquery.PutMultiError)
if isErr {
for _, e := range putErr {
log.Errorf("There was a problem writing Row for customer project: %v, Error: %v", task.CustomerProjectNumber, util.Pformat(e))
}
} else {
log.Errorf("There was a problem writing GCE Quota Rows for customer project: %v, Error: %v", task.CustomerProjectNumber, err)
}
}
我希望这能有所帮助(如果不是你,也许是两年后的事实,那么其他任何人偶然发现了这一点) 我只是偶然发现了这一点,并把它弄明白了。发生这种情况的原因是,具体的
bigquery.StructSaver
没有实现bigquery.ValueSaver
接口,因为其方法在指针类型*bigquery.StructSaver
上操作。这会导致golang客户端库在您的结构(在本例中为bigquery.StructSaver
)上退回到其通用模式推理逻辑,然后失败,因为bigquery.StructSaver
嵌套了其他结构,因此它本身不是保存在bigquery中的有效对象。您可以按如下方式修复代码:
for _, region := range res.GetRegionalQuotas() {
for _, q := range region.GetQuotas() {
r := &GceQuotaRow{
CustomerProjectNumber: task.CustomerProjectNumber,
ExecutionProjectNumber: res.GetExecutionProjectNumber(),
Region: region.GetRegion(),
Metric: q.GetMetric(),
Limit: q.GetLimit(),
}
h := sha256.New()
h.Write([]byte(util.Pformat(r)))
// Use the sha256 of the row as the InsertId to avoid duplicates.
insertId := fmt.Sprintf("%x", h.Sum(nil))
rows = append(rows, &bigquery.StructSaver{
Schema: schema,
InsertID: insertId,
Struct: r,
})
// rows = append(rows, r)
}
}
err = u.Put(ctx, rows)
if err != nil {
putErr, isErr := err.(bigquery.PutMultiError)
if isErr {
for _, e := range putErr {
log.Errorf("There was a problem writing Row for customer project: %v, Error: %v", task.CustomerProjectNumber, util.Pformat(e))
}
} else {
log.Errorf("There was a problem writing GCE Quota Rows for customer project: %v, Error: %v", task.CustomerProjectNumber, err)
}
}
我希望这能有所帮助(如果不是你,也许是两年后的事实,那么其他任何人偶然发现了这一点) 我也面临类似的问题,但我的结构中嵌入了字段,这似乎引发了错误。有关于这个问题的更新吗?你解决了吗?也面临类似的问题,但我的结构中嵌入了字段,这似乎引发了错误。有关于这个问题的更新吗?你解决了吗?