Python 气流单元测试中未呈现模板
我有一个Python模块，其结构如下：Python 气流单元测试中未呈现模板,python,unit-testing,airflow,Python,Unit Testing,Airflow,我有一个Python模块，其结构如下： my_module/ ... tests/ __init__.py my_test.py ... from __future__ import print_function, unicode_literals import os import unittest from datetime import timedelta, date from airflow import configuration from airf
my_module/
...
tests/
__init__.py
my_test.py
...
from __future__ import print_function, unicode_literals
import os
import unittest
from datetime import timedelta, date
from airflow import configuration
from airflow.models import TaskInstance as TI, DAG, DagRun
from airflow.operators.python_operator import PythonOperator
from airflow.settings import Session
from airflow.utils import timezone
from airflow.utils.state import State
DEFAULT_DATE = timezone.datetime(2016, 1, 1)
END_DATE = timezone.datetime(2016, 1, 2)
INTERVAL = timedelta(hours=12)
FROZEN_NOW = timezone.datetime(2016, 1, 2, 12, 1, 1)
TI_CONTEXT_ENV_VARS = ['AIRFLOW_CTX_DAG_ID',
'AIRFLOW_CTX_TASK_ID',
'AIRFLOW_CTX_EXECUTION_DATE',
'AIRFLOW_CTX_DAG_RUN_ID']
class Call:
def __init__(self, *args, **kwargs):
self.args = args
self.kwargs = kwargs
def build_recording_function(calls_collection):
"""
We can not use a Mock instance as a PythonOperator callable function or some tests fail with a
TypeError: Object of type Mock is not JSON serializable
Then using this custom function recording custom Call objects for further testing
(replacing Mock.assert_called_with assertion method)
"""
def recording_function(*args, **kwargs):
calls_collection.append(Call(*args, **kwargs))
return recording_function
class PythonOperatorTest(unittest.TestCase):
@classmethod
def setUpClass(cls):
super(PythonOperatorTest, cls).setUpClass()
session = Session()
session.query(DagRun).delete()
session.query(TI).delete()
session.commit()
session.close()
def setUp(self):
super(PythonOperatorTest, self).setUp()
configuration.load_test_config()
self.dag = DAG(
'test_dag',
default_args={
'owner': 'airflow',
'start_date': DEFAULT_DATE},
schedule_interval=INTERVAL)
self.addCleanup(self.dag.clear)
self.clear_run()
self.addCleanup(self.clear_run)
def tearDown(self):
super(PythonOperatorTest, self).tearDown()
session = Session()
session.query(DagRun).delete()
session.query(TI).delete()
print(len(session.query(DagRun).all()))
session.commit()
session.close()
for var in TI_CONTEXT_ENV_VARS:
if var in os.environ:
del os.environ[var]
def do_run(self):
self.run = True
def clear_run(self):
self.run = False
def is_run(self):
return self.run
def _assertCallsEqual(self, first, second):
self.assertIsInstance(first, Call)
self.assertIsInstance(second, Call)
self.assertTupleEqual(first.args, second.args)
self.assertDictEqual(first.kwargs, second.kwargs)
def test_python_callable_arguments_are_templatized(self):
"""Test PythonOperator op_args are templatized"""
recorded_calls = []
task = PythonOperator(
task_id='python_operator',
# a Mock instance cannot be used as a callable function or test fails with a
# TypeError: Object of type Mock is not JSON serializable
python_callable=(build_recording_function(recorded_calls)),
op_args=[
4,
date(2019, 1, 1),
"dag {{dag.dag_id}} ran on {{ds}}."
],
dag=self.dag)
self.dag.create_dagrun(
run_id='manual__' + DEFAULT_DATE.isoformat(),
execution_date=DEFAULT_DATE,
start_date=DEFAULT_DATE,
state=State.RUNNING
)
task.run(start_date=DEFAULT_DATE, end_date=DEFAULT_DATE)
self.assertEqual(1, len(recorded_calls))
self._assertCallsEqual(
recorded_calls[0],
Call(4,
date(2019, 1, 1),
"dag {} ran on {}.".format(self.dag.dag_id, DEFAULT_DATE.date().isoformat()))
)
my_test.pymy_module/
...
tests/
__init__.py
my_test.py
...
from __future__ import print_function, unicode_literals
import os
import unittest
from datetime import timedelta, date
from airflow import configuration
from airflow.models import TaskInstance as TI, DAG, DagRun
from airflow.operators.python_operator import PythonOperator
from airflow.settings import Session
from airflow.utils import timezone
from airflow.utils.state import State
DEFAULT_DATE = timezone.datetime(2016, 1, 1)
END_DATE = timezone.datetime(2016, 1, 2)
INTERVAL = timedelta(hours=12)
FROZEN_NOW = timezone.datetime(2016, 1, 2, 12, 1, 1)
TI_CONTEXT_ENV_VARS = ['AIRFLOW_CTX_DAG_ID',
'AIRFLOW_CTX_TASK_ID',
'AIRFLOW_CTX_EXECUTION_DATE',
'AIRFLOW_CTX_DAG_RUN_ID']
class Call:
def __init__(self, *args, **kwargs):
self.args = args
self.kwargs = kwargs
def build_recording_function(calls_collection):
"""
We can not use a Mock instance as a PythonOperator callable function or some tests fail with a
TypeError: Object of type Mock is not JSON serializable
Then using this custom function recording custom Call objects for further testing
(replacing Mock.assert_called_with assertion method)
"""
def recording_function(*args, **kwargs):
calls_collection.append(Call(*args, **kwargs))
return recording_function
class PythonOperatorTest(unittest.TestCase):
@classmethod
def setUpClass(cls):
super(PythonOperatorTest, cls).setUpClass()
session = Session()
session.query(DagRun).delete()
session.query(TI).delete()
session.commit()
session.close()
def setUp(self):
super(PythonOperatorTest, self).setUp()
configuration.load_test_config()
self.dag = DAG(
'test_dag',
default_args={
'owner': 'airflow',
'start_date': DEFAULT_DATE},
schedule_interval=INTERVAL)
self.addCleanup(self.dag.clear)
self.clear_run()
self.addCleanup(self.clear_run)
def tearDown(self):
super(PythonOperatorTest, self).tearDown()
session = Session()
session.query(DagRun).delete()
session.query(TI).delete()
print(len(session.query(DagRun).all()))
session.commit()
session.close()
for var in TI_CONTEXT_ENV_VARS:
if var in os.environ:
del os.environ[var]
def do_run(self):
self.run = True
def clear_run(self):
self.run = False
def is_run(self):
return self.run
def _assertCallsEqual(self, first, second):
self.assertIsInstance(first, Call)
self.assertIsInstance(second, Call)
self.assertTupleEqual(first.args, second.args)
self.assertDictEqual(first.kwargs, second.kwargs)
def test_python_callable_arguments_are_templatized(self):
"""Test PythonOperator op_args are templatized"""
recorded_calls = []
task = PythonOperator(
task_id='python_operator',
# a Mock instance cannot be used as a callable function or test fails with a
# TypeError: Object of type Mock is not JSON serializable
python_callable=(build_recording_function(recorded_calls)),
op_args=[
4,
date(2019, 1, 1),
"dag {{dag.dag_id}} ran on {{ds}}."
],
dag=self.dag)
self.dag.create_dagrun(
run_id='manual__' + DEFAULT_DATE.isoformat(),
execution_date=DEFAULT_DATE,
start_date=DEFAULT_DATE,
state=State.RUNNING
)
task.run(start_date=DEFAULT_DATE, end_date=DEFAULT_DATE)
self.assertEqual(1, len(recorded_calls))
self._assertCallsEqual(
recorded_calls[0],
Call(4,
date(2019, 1, 1),
"dag {} ran on {}.".format(self.dag.dag_id, DEFAULT_DATE.date().isoformat()))
)
nosetests test/my_test.py======================================================================
FAIL: Test PythonOperator op_args are templatized
----------------------------------------------------------------------
Traceback (most recent call last):
File "/home/user/my_module/tests/my_test.py", line 120, in test_python_callable_arguments_are_templatized
"dag {} ran on {}.".format(self.dag.dag_id, DEFAULT_DATE.date().isoformat()))
File "/home/user/my_module/tests/my_test.py", line 88, in _assertCallsEqual
self.assertTupleEqual(first.args, second.args)
AssertionError: Tuples differ: (4, datetime.date(2019, 1, 1), 'dag {{dag.dag_id}} ran on {{ds}}.') != (4, datetime.date(2019, 1, 1), 'dag test_dag ran on 2016-01-01.')
First differing element 2:
'dag {{dag.dag_id}} ran on {{ds}}.'
'dag test_dag ran on 2016-01-01.'
- (4, datetime.date(2019, 1, 1), 'dag {{dag.dag_id}} ran on {{ds}}.')
? ^^ --------- ^^^^^^
+ (4, datetime.date(2019, 1, 1), 'dag test_dag ran on 2016-01-01.')
? ^^^^^ ^^^^^^^^^^
-------------------- >> begin captured logging << --------------------
airflow.utils.log.logging_mixin.LoggingMixin: INFO: Reading the config from /home/user/airflow/airflow.cfg
airflow.settings: INFO: Configured default timezone <Timezone [UTC]>
airflow.logging_config: DEBUG: Unable to load custom logging, using default config instead
--------------------- >> end captured logging << ---------------------
======================================================================
失败：测试PythonOperator op_参数已模板化
----------------------------------------------------------------------
回溯（最近一次呼叫最后一次）：
test\u python\u可调用参数\u中的文件“/home/user/my\u module/tests/my\u test.py”第120行被模板化
“dag{}在{}上运行。”.format（self.dag.dag_id，默认_DATE.DATE（）.isoformat（））
文件“/home/user/my_module/tests/my_test.py”，第88行，在_assertCallsEqual中
self.assertTupleEqual（first.args，second.args）
断言错误：元组不同：（4，datetime.date（2019，1，1），'dag{{dag.dag_id}在{ds}上运行。'）！=（4，日期时间。日期（2019年1月1日），“dag测试于2016年1月1日进行。”
第一个不同的要素2：
'dag{{dag.dag_id}}在{{ds}上运行。'
“dag测试于2016年1月1日进行。”
-（4，datetime.date（2019，1，1），“dag{{dag.dag_id}在{{ds}上运行。”）
? ^^ --------- ^^^^^^
+（4，日期时间。日期（2019年1月1日），“dag测试于2016年1月1日进行。”
? ^^^^^ ^^^^^^^^^^
-------------------->>开始捕获日志>结束捕获日志您最有可能使用1.10
或更早版本来运行测试。在该版本中，PythonOperator
中的op_args
未模板化。但是在master
中，您很可能正在使用来自的测试，op_args
被模板化，并相应地进行测试。如果您真的想使用气流测试作为示例，您应该从与您安装的版本相对应的任何分支获取它们。这是因为，PythonOperator的'op_args'
，'op_kwargs'
字段不是气流1.10.2中的模板化字段。您的问题中的链接是气流存储库的主分支
'op_args'
，'op_kwargs'
是在气流1.10.2发布后添加的
将包含这些字段的提交到模板\u字段
（这仍在主版本中，未包含在任何发布版本中）：
对于气流1.10.2：检查此文件-
另外，不要使用v1-10-stable
分支，因为它包含即将发布的1.10.3的代码。您应该改用1.10.2标记：
蟒蛇操作员（1.10.2）：
蟒蛇运营商（硕士-开发分支）：
谢谢你的回答。我编辑了我的帖子。我使用的是ApacheAirflow 1.10.2，测试文件的代码来自v1-10-stable分支。我认为问题出在某个地方……在3月27日，该系统被合并为v1-10-stable
。我相信它不是1.10.2
的一部分。您可以查看本地PythonOperator
的代码，并查看是否有：template\u字段=（'templates\u dict'，'op\u args'，'op\u kwargs'）