Apache beam 运行Apache Beam Python SplittableDoFn时出错
尝试Apache beam 运行Apache Beam Python SplittableDoFn时出错,apache-beam,apache-beam-internals,Apache Beam,Apache Beam Internals,尝试pubsub io>splittable dofn时遇到错误 RuntimeError: Transform node AppliedPTransform(ParDo(TestDoFn)/ProcessKeyedElements/GroupByKey/GroupByKey, _GroupByKeyOnly) was not replaced as expected. 有没有人能帮我检查一下代码,看看有没有什么地方我做得不对 代码: 我也犯了同样的错误。删除流媒体选项为我解决了问题。我也
pubsub io>splittable dofn时遇到错误
RuntimeError: Transform node
AppliedPTransform(ParDo(TestDoFn)/ProcessKeyedElements/GroupByKey/GroupByKey,
_GroupByKeyOnly) was not replaced as expected.
有没有人能帮我检查一下代码,看看有没有什么地方我做得不对
代码:
我也犯了同样的错误。删除流媒体选项为我解决了问题。我也有同样的错误。删除流媒体选项为我解决了这个问题。您正在通过流媒体从pub/sub接收数据。然后,在应用这种转换之前,您必须按窗口创建批:(ParDo(TestDoFn)/ProcessKeyedElements/GroupByKey/GroupByKey,_GroupByKeyOnly)
带有窗口示例的发布/订阅:
试着这样做:
class GroupWindowsInToBatchs(beam.ptTransform):
“”“对发布/订阅消息进行分组的复合转换
"""
定义初始值(自身、窗口大小):
#把分钟转换成秒。
self.window\u size=int(window\u size*60)
def扩展(自我,pcoll):
返回(
pcoll
#根据每个发布/订阅消息的窗口信息为其分配窗口信息
#发布时间戳。
|“固定间隔窗口”
>>梁.窗入(窗.固定窗(自窗尺寸))
)
def运行(argv=None,save_main_session=True):
parser=argparse.ArgumentParser()
parser.add_参数('--topic',type=str,help='Pub/subtopicto read from')
args,pipeline_args=parser.parse_known_args(argv)
选项=管道选项(管道参数)
选项。按(标准选项)查看。流=真
窗口大小=1.0
梁管道(选项=选项)为p:
得分=(p
|beam.io.ReadFromPubSub(topic=args.topic)
|“WindowInto”>>GroupWindowsIntoBatchs(窗口大小)
|beam.ParDo(TestDoFn())
)
您正在通过蒸汽从pub/sub接收数据。然后,在应用这种转换之前,您必须按窗口创建批:(ParDo(TestDoFn)/ProcessKeyedElements/GroupByKey/GroupByKey,_GroupByKeyOnly)
带有窗口示例的发布/订阅:
试着这样做:
class GroupWindowsInToBatchs(beam.ptTransform):
“”“对发布/订阅消息进行分组的复合转换
"""
定义初始值(自身、窗口大小):
#把分钟转换成秒。
self.window\u size=int(window\u size*60)
def扩展(自我,pcoll):
返回(
pcoll
#根据每个发布/订阅消息的窗口信息为其分配窗口信息
#发布时间戳。
|“固定间隔窗口”
>>梁.窗入(窗.固定窗(自窗尺寸))
)
def运行(argv=None,save_main_session=True):
parser=argparse.ArgumentParser()
parser.add_参数('--topic',type=str,help='Pub/subtopicto read from')
args,pipeline_args=parser.parse_known_args(argv)
选项=管道选项(管道参数)
选项。按(标准选项)查看。流=真
窗口大小=1.0
梁管道(选项=选项)为p:
得分=(p
|beam.io.ReadFromPubSub(topic=args.topic)
|“WindowInto”>>GroupWindowsIntoBatchs(窗口大小)
|beam.ParDo(TestDoFn())
)
请提供更多解释请提供更多解释
"""
python examples/test_restriction_unbounded.py --project mk2 --topic projects/mk2/topics/testing
"""
# pytype: skip-file
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import argparse
import csv
import logging
import sys
import time
from datetime import datetime
import apache_beam as beam
from apache_beam.options.pipeline_options import GoogleCloudOptions
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.options.pipeline_options import StandardOptions
from apache_beam.io.restriction_trackers import OffsetRestrictionTracker, OffsetRange
from apache_beam.transforms.core import RestrictionProvider
class TestProvider(RestrictionProvider):
def initial_restriction(self, element):
return OffsetRange(0, 1)
def create_tracker(self, restriction):
return OffsetRestrictionTracker(restriction)
def restriction_size(self, element, restriction):
return restriction.size()
class TestDoFn(beam.DoFn):
def process(
self,
element,
restriction_tracker=beam.DoFn.RestrictionParam(
TestProvider())):
import pdb; pdb.set_trace()
cur = restriction_tracker.current_restriction().start
while restriction_tracker.try_claim(cur):
return element
def run(argv=None, save_main_session=True):
parser = argparse.ArgumentParser()
parser.add_argument('--topic', type=str, help='Pub/Sub topic to read from')
args, pipeline_args = parser.parse_known_args(argv)
options = PipelineOptions(pipeline_args)
options.view_as(StandardOptions).streaming = True
with beam.Pipeline(options=options) as p:
# data = ['abc', 'defghijklmno', 'pqrstuv', 'wxyz']
# actual = (p | beam.Create(data) | beam.ParDo(ExpandingStringsDoFn()))
scores = p | beam.io.ReadFromPubSub(topic=args.topic) | beam.ParDo(TestDoFn())
if __name__ == '__main__':
logging.getLogger().setLevel(logging.INFO)
run()