Google bigquery Bigquery存储API多处理故障
长期读者，第一次海报。我使用的是BigQuery存储API Python客户端库，在使用Python多处理分离阅读器时遇到了一些问题 文件中包含一个注释，说明： 因为此客户端使用grpcio库，所以共享实例是安全的 跨越线程。在多处理场景中，最佳实践是 在调用os.fork（）之后创建客户端实例 multiprocessing.Pool或multiprocessing.Process 我想我这样做是对的…但我不能这样做 这是我目前的代码。目标是在多个并行流中读取BQ表，然后将数据行写入单个CSV文件。一旦创建了所有CSV文件，我将执行一个简单的cat命令来组合它们 作为旁注，这段代码实际上适用于小的BigQuery表，但在尝试下载大的BQ表时，SEGFULT会失败Google bigquery Bigquery存储API多处理故障,google-bigquery,multiprocessing,Google Bigquery,Multiprocessing,长期读者，第一次海报。我使用的是BigQuery存储API Python客户端库，在使用Python多处理分离阅读器时遇到了一些问题 文件中包含一个注释，说明： 因为此客户端使用grpcio库，所以共享实例是安全的 跨越线程。在多处理场景中，最佳实践是 在调用os.fork（）之后创建客户端实例 multiprocessing.Pool或multiprocessing.Process 我想我这样做是对的…但我不能这样做 这是我目前的代码。目标是在多个并行流中读取BQ表，然后将数据行写入单个CSV
import faulthandler
faulthandler.enable()
from google.cloud.bigquery_storage import BigQueryReadClient
from google.cloud.bigquery_storage import types
import multiprocessing as mp
import psutil
import os
import sys
import csv
from datetime import datetime
def extract_table(i):
client_in = BigQueryReadClient()
reader_in = client_in.read_rows(session.streams[i].name, timeout=10000)
rows = reader_in.rows(session)
csv_file = "/home/user/sas/" + table_name + "_" + str(i) + ".csv"
print(f"Starting at time {datetime.now()} for file {csv_file}")
try:
with open(csv_file, 'w') as csvfile:
writer = csv.DictWriter(csvfile, fieldnames=csv_columns)
if i == 0:
writer.writeheader()
else:
pass
for data in rows:
# print(data)
writer.writerow(data)
except IOError:
print("I/O error")
print(f"Finished at time {datetime.now()} for file {csv_file}")
return
if __name__ == '__main__':
# Get input args
project_id = sys.argv[1]
db_name = sys.argv[2]
table_name = sys.argv[3]
n = len(sys.argv[4])
a = sys.argv[4][1:n - 1]
csv_columns = a.replace("'", '').split(', ')
output_type = sys.argv[5] # csv or sas
bucket_root = sys.argv[6]
# The read session is created in this project. This project can be
# different from that which contains the table.
client = BigQueryReadClient()
table = "projects/{}/datasets/{}/tables/{}".format(
project_id, db_name, table_name
)
requested_session = types.ReadSession()
requested_session.table = table
# This API can also deliver data serialized in Apache Arrow format.
# This example leverages Apache Avro.
requested_session.data_format = types.DataFormat.AVRO
# We limit the output columns to a subset of those allowed in the table
requested_session.read_options.selected_fields = csv_columns
ncpus = psutil.cpu_count(logical=False)
if ncpus <= 2:
ncpus_buffer = 2
else:
ncpus_buffer = ncpus - 2
print(f"You have {ncpus} cores according to psutil. Using {ncpus_buffer} cores")
parent = "projects/{}".format(project_id)
session = client.create_read_session(
parent=parent,
read_session=requested_session,
max_stream_count=ncpus_buffer,
)
print(f"There are {len(session.streams)} streams")
num_streams = int(len(session.streams))
with mp.Pool(processes=ncpus_buffer) as p:
result = p.map(extract_table, list(range(0, num_streams)), chunksize=1)
os.fork()
Process 1 Process 2 Process 3
Session1 Session2 Session3
*Stream1 - 10 rows Stream1 - 8 rows Stream1 - 9 rows
Stream2 - 10 rows *Stream2 - 12 rows Stream2 - 11 rows
Stream3 - 10 rows Stream3 - 10 rows *Stream3 - 10 rows
# create read session here
# Then call the target worker function with one thread per worker
for i in range(0, num_streams):
t = threading.Thread(target=extract_table, args=(i,))
t.start()
# create read session here
# Then call the target worker function with one thread per worker
for i in range(0, num_streams):
t = threading.Thread(target=extract_table, args=(i,))
t.start()
with mp.Pool(processes=ncpus_buffer) as p:
result = p.map(extract_table, list(range(0, num_streams)), chunksize=1)
with concurrent.futures.ThreadPoolExecutor(max_workers=num_streams) as p:
result = p.map(extract_table, list(range(0, num_streams)), chunksize=1)