BigQuery-使用_avro_logical_类型和#xB4；我不能在python脚本中工作
这个疑问类似于这个问题：但是我已经更新了我使用谷歌的库，但没有成功。所以我想了解我的情况。我将发送avro文件的接收脚本部分。注意：BigQuery的多个avro文件摄取管道中会出现此问题 编辑：上面提到的更改模式类型的解决方案不起作用，它给出了另一个错误 BQ中的模式： BQ中的数据： Avro模式：BigQuery-使用_avro_logical_类型和#xB4；我不能在python脚本中工作,python,google-bigquery,avro,Python,Google Bigquery,Avro,这个疑问类似于这个问题：但是我已经更新了我使用谷歌的库，但没有成功。所以我想了解我的情况。我将发送avro文件的接收脚本部分。注意：BigQuery的多个avro文件摄取管道中会出现此问题 编辑：上面提到的更改模式类型的解决方案不起作用，它给出了另一个错误 BQ中的模式： BQ中的数据： Avro模式： "fields": [ { "name": "id", "type": { "type": "string", "avro.j
"fields": [
{
"name": "id",
"type": {
"type": "string",
"avro.java.string": "String"
},
"doc": "the payment id"
},
{
"name": "merchant",
"type": {
"type": "string",
"avro.java.string": "String"
},
"doc": "the merchant who owns the payment"
},
{
"name": "date",
"type": {
"type": "long",
"logicalType": "timestamp-millis"
},
"doc": "the date where the transaction happend"
},
{
"name": "amount",
"type": {
"type": "record",
"name": "amount",
"fields": [
{
"name": "amount",
"type": [
"null",
{
"type": "bytes",
"logicalType": "decimal",
"precision": 5,
"scale": 5
}
],
"doc": "the original currency amount",
"default": null
},
{
"name": "foreignAmount",
"type": [
"null",
{
"type": "bytes",
"logicalType": "decimal",
"precision": 5,
"scale": 5
}
],
"doc": "the foreign amount for the payment",
"default": null
},
{
"name": "code",
"type": {
"type": "string",
"avro.java.string": "String"
},
"doc": "the destination currency code"
}
],
"default": null
}
},
{
"name": "exchange_rate",
"type": {
"type": "record",
"name": "code",
"fields": [
{
"name": "currency_code",
"type": [
"null",
{
"type": "string",
"avro.java.string": "String"
}
],
"doc": "the exchange rate currency code",
"default": null
},
{
"name": "rate",
"type": [
"null",
{
"type": "bytes",
"logicalType": "decimal",
"precision": 5,
"scale": 5
}
],
"default": null
},
{
"name": "online_rate",
"type": [
"null",
{
"type": "bytes",
"logicalType": "decimal",
"precision": 5,
"scale": 5
}
],
"default": null
}
]
},
"doc": "The transaction exchange rate"
},
{
"name": "consumer",
"type": {
"type": "record",
"name": "Consumer",
"fields": [
{
"name": "name",
"type": [
"null",
{
"type": "string",
"avro.java.string": "String"
}
],
"doc": "the consumer's name",
"default": null
},
{
"name": "email",
"type": [
"null",
{
"type": "string",
"avro.java.string": "String"
}
],
"doc": "the consumer's email address",
"default": null
},
{
"name": "external_id",
"type": [
"null",
{
"type": "string",
"avro.java.string": "String"
}
],
"doc": "the consumer's external id when needed",
"default": null
},
{
"name": "national_id",
"type": {
"type": "string",
"avro.java.string": "String"
},
"doc": "the national id"
},
{
"name": "phone",
"type": {
"type": "string",
"avro.java.string": "String"
},
"doc": "the consumer's phone number",
"default": ""
}
]
}
},
{
"name": "soft_descriptor",
"type": [
"null",
{
"type": "string",
"avro.java.string": "String"
}
],
"doc": "the description as it will be shown at the customer's invoice",
"default": null
},
{
"name": "merchant_contract",
"type": {
"type": "enum",
"name": "merchant_contract_type",
"symbols": [
"PAY",
"BANK"
]
},
"default": "PAY"
},
{
"name": "type",
"type": {
"type": "enum",
"name": "payment_type",
"symbols": [
"INITIAL",
"CREDIT_CARD",
"DEBIT_CARD",
"ONLINE_DEBIT",
"BANK_SLIP",
"DIGITAL_WALLET",
"ELECTRONIC_BANK_TRANSFER"
]
},
"default": "INITIAL"
},
{
"name": "card",
"type": {
"type": "record",
"name": "card",
"fields": [
{
"name": "type",
"type": [
"null",
{
"type": "enum",
"name": "card_type",
"symbols": [
"CARD",
"TOKEN"
]
}
],
"default": null
},
{
"name": "mask_number",
"type": [
"null",
{
"type": "string",
"avro.java.string": "String"
}
],
"default": null
},
{
"name": "card_holder",
"type": [
"null",
{
"type": "string",
"avro.java.string": "String"
}
],
"default": null
},
{
"name": "brand",
"type": [
"null",
{
"type": "string",
"avro.java.string": "String"
}
],
"default": null
}
]
}
},
{
"name": "confirm",
"type": "boolean",
"doc": "indicates whether is self confirmed",
"default": false
},
{
"name": "installments",
"type": "int",
"doc": "Number of installments for the payment",
"default": 1
},
{
"name": "due_date",
"type": [
"null",
{
"type": "int",
"logicalType": "date"
}
],
"default": null
},
{
"name": "correlation_id",
"type": [
"null",
{
"type": "string",
"avro.java.string": "String"
}
],
"doc": "the external customer correlationid",
"default": null
},
{
"name": "billing",
"type": {
"type": "record",
"name": "Billing",
"fields": [
{
"name": "national_id",
"type": [
"null",
{
"type": "string",
"avro.java.string": "String"
}
],
"doc": "billing info",
"default": null
},
{
"name": "name",
"type": [
"null",
{
"type": "string",
"avro.java.string": "String"
}
],
"doc": "the consumer address name",
"default": null
}
]
}
},
{
"name": "status",
"type": {
"type": "enum",
"name": "payment_status",
"symbols": [
"INITIAL",
"CONSUMER",
"AUTHORIZED",
"WAITING_CONFIRMATION",
"CANCELED",
"WAITING_CLEARING",
"CLEARED",
"TRANSFERENCE",
"DECLINED_BY_ISSUER",
"DECLINED_BY_BUSINESS_RULES",
"CONFIRMED",
"WAITING_CANCELING",
"WAITING_CONSUMER",
"TRANSFER_REQUESTED"
],
"default": "INITIAL"
}
},
{
"name": "metadata",
"type": {
"type": "map",
"values": {
"type": "string",
"avro.java.string": "String"
},
"avro.java.string": "String"
}
},
{
"name": "events",
"type": {
"type": "array",
"items": {
"type": "record",
"name": "event",
"fields": [
{
"name": "id",
"type": {
"type": "string",
"avro.java.string": "String"
},
"default": "0"
},
{
"name": "type",
"type": {
"type": "enum",
"name": "event_type",
"symbols": [
"AUTHORIZATION",
"AUTHENTICATION",
"CONFIRMATION",
"CANCELATION",
"CHECKOUT_CREATION",
"SETTLEMENT",
"TRANSFER_VALIDATION",
"TRANSFER_SCHEDULE",
"TRANSFERRED"
]
}
},
{
"name": "gateway",
"type": [
"null",
{
"type": "string",
"avro.java.string": "String"
}
],
"default": null
},
{
"name": "breadcrumb_id",
"type": [
"null",
{
"type": "string",
"avro.java.string": "String"
}
],
"default": null
},
{
"name": "request_time",
"type": {
"type": "long",
"logicalType": "timestamp-millis"
},
"doc": "the moment where this request was received by the platform"
},
{
"name": "response_time",
"type": {
"type": "long",
"logicalType": "timestamp-millis"
},
"doc": "the moment where this request was returned by the platform"
},
{
"name": "status",
"type": {
"type": "enum",
"name": "event_status",
"symbols": [
"SUCCESS",
"DENIED",
"ERROR",
"TIMEOUT",
"PENDING"
]
}
},
{
"name": "actor",
"type": {
"type": "enum",
"name": "actor",
"symbols": [
"AQ",
"GTW",
"CONCIL"
],
"default": "GTW"
},
"default": "GTW"
},
{
"name": "amount",
"type": [
"null",
{
"type": "bytes",
"logicalType": "decimal",
"precision": 5,
"scale": 5
}
],
"doc": "the original currency amount",
"default": null
},
{
"name": "foreign_amount",
"type": [
"null",
{
"type": "bytes",
"logicalType": "decimal",
"precision": 5,
"scale": 5
}
],
"doc": "the foreign amount for the payment",
"default": null
},
{
"name": "error",
"type": {
"type": "record",
"name": "Error",
"fields": [
{
"name": "code",
"type": [
"null",
{
"type": "string",
"avro.java.string": "String"
}
],
"default": null
},
{
"name": "message",
"type": [
"null",
{
"type": "string",
"avro.java.string": "String"
}
],
"default": null
}
]
}
},
{
"name": "message",
"type": [
"null",
{
"type": "string",
"avro.java.string": "String"
}
],
"default": null
},
{
"name": "fee_amount",
"type": [
"null",
{
"type": "bytes",
"logicalType": "decimal",
"precision": 5,
"scale": 5
}
],
"doc": "the fee amount",
"default": null
},
{
"name": "net_amount",
"type": [
"null",
{
"type": "bytes",
"logicalType": "decimal",
"precision": 5,
"scale": 5
}
],
"doc": "the net amount",
"default": null
},
{
"name": "metadata",
"type": {
"type": "map",
"values": {
"type": "string",
"avro.java.string": "String"
},
"avro.java.string": "String"
}
},
{
"name": "internal_metadata",
"type": {
"type": "map",
"values": {
"type": "string",
"avro.java.string": "String"
},
"avro.java.string": "String"
}
}
]
}
}
},
{
"name": "bank_account",
"type": [
"null",
{
"type": "record",
"name": "bank_account",
"fields": [
{
"name": "name",
"type": {
"type": "string",
"avro.java.string": "String"
},
"doc": "the bank name",
"default": ""
},
{
"name": "code",
"type": {
"type": "string",
"avro.java.string": "String"
},
"doc": "the bank code",
"default": ""
},
{
"name": "agency",
"type": {
"type": "string",
"avro.java.string": "String"
},
"doc": "the bank agency",
"default": ""
},
{
"name": "account",
"type": {
"type": "string",
"avro.java.string": "String"
},
"doc": "the bank account",
"default": ""
},
{
"name": "document_number",
"type": {
"type": "string",
"avro.java.string": "String"
},
"doc": "the bank document number (CNPJ)",
"default": ""
}
]
}
],
"doc": "The bank account values",
"default": null
}
]
"type": ["null", {"type": "long", "logicalType": "timestamp-millis"}]
我尝试了，但收到了以下消息：无法访问分支索引1574092545874以与2个分支联合编写器的模式：[“null”，“type”：“long”，“logicalType”：“timestamp milliss”}]读取器的模式：[“null”，“type”：“long”，“logicalType”：“timestamp millis”}]你能提供你的模式吗？我知道你也有日期类型，所以我之前的建议不起作用。请参阅本节和第1节。加载作业完成后，能否发布BigQuery表的结果架构？2.您正在尝试加载到现有表中（然后使用哪种写入模式，WRITE_APPEND还是WRITE_TRUNCATE），还是正在加载到新表中。3.哪一列提供了您不希望的类型？@NhanNguyen您好，我刚刚提供了所需的信息。如果该表不存在，其想法是生成一个表，但它都位于WRITE\u APPEND之上。应更改的列：日期（时间戳）、截止日期（日期）、响应时间（时间戳）、请求时间（时间戳）。@NhanNguyen您对如何操作有何建议？
"type": ["null", {"type": "long", "logicalType": "timestamp-millis"}]