Airflow 自定义气流回调插件赢得'；行不通

我们的代码库多次粘贴了许多相同的松弛回调函数，因此我希望将它们移动到一个单独的插件，并从那里调用这些函数。然而，我有问题的代码，甚至能够达到插件。我在

try/except

块中出错，这意味着我在

DAG

中做错了什么，但是我运气不太好

DAG的

code

from airflow import DAG
from airflow.operators.slack_airflow_plugins import SlackSuccessAlert
from airflow.operators.python_operator import PythonOperator
from airflow.operators.dummy_operator import DummyOperator
from datetime import datetime
import urllib


def slack_callback(context):
    operator = SlackSuccessAlert(
        dag_id = context.get("task_instance").dag_id,
        task_id = context.get("task_instance").task_id,
        execution_date = context.get("execution_date").isoformat()
        )

    return operator.execute(context=context)

default_args = {
    "depends_on_past": False,
    'start_date': datetime(2021, 5, 1)
}


dag = DAG(
    dag_id='slack_plugin_test',
    default_args=default_args,
    catchup=False,
    max_active_runs=1
)

success_task_test = DummyOperator(
    task_id='my_task',
    provide_context=True,
    on_success_callback=slack_callback,
    dag=dag)

success_task_test


from airflow.utils.decorators import apply_defaults
from airflow.operators.slack_operator import SlackAPIPostOperator
from airflow.models import BaseOperator
from airflow.hooks.base_hook import BaseHook
import urllib
import os
from airflow.plugins_manager import AirflowPlugin


# Slack Alerts
# HOSTNAME variable set in docker-compose
base_url = "https://"+os.getenv('HOSTNAME')
slack_channel = 'airflow-alerts-test'
slack_conn_id = 'slack'
slack_token = str(BaseHook.get_connection("slack").password)


class SlackSuccessAlert(BaseOperator):

    @apply_defaults
    def __init__(
            self,
            dag_id,
            task_id,
            execution_date,
            emoji = ':green_check:',
            message = 'has succeeded',
            *args,
            **kwargs):
        super().__init__(*args, **kwargs)
        self.dag_id = dag_id
        self.task_id = task_id
        self.execution_date = execution_date
        self.emoji = emoji
        self.message = message

    def execute(self, context):
        #dag_id = self.dag_id
        #task_id = self.task_id
        #execution_date = self.execution_date
        encoded_execution_date = urllib.parse.quote_plus(self.execution_date)
        dag_url = (f'{base_url}/graph?dag_id={self.dag_id}'
                   f'&execution_date={encoded_execution_date}')
        slack_message = (f'{self.emoji} *{self.dag_id}* <{dag_url}|*{self.message}*>.')
        slack_alert = SlackAPIPostOperator(
            task_id='slack_alert',
            channel=slack_channel,
            token=slack_token,
            text=slack_message
        )
        try:
            return slack_alert.execute(context=context)
        except:
            print("something going wrong with the callback...")



class SlackAlertsTest(AirflowPlugin):
    """
    Custom Slack Plugins for Airflow
    """
    name = "slack_airflow_plugins"
    operators = [SlackSuccessAlert]


插件代码

from airflow import DAG
from airflow.operators.slack_airflow_plugins import SlackSuccessAlert
from airflow.operators.python_operator import PythonOperator
from airflow.operators.dummy_operator import DummyOperator
from datetime import datetime
import urllib


def slack_callback(context):
    operator = SlackSuccessAlert(
        dag_id = context.get("task_instance").dag_id,
        task_id = context.get("task_instance").task_id,
        execution_date = context.get("execution_date").isoformat()
        )

    return operator.execute(context=context)

default_args = {
    "depends_on_past": False,
    'start_date': datetime(2021, 5, 1)
}


dag = DAG(
    dag_id='slack_plugin_test',
    default_args=default_args,
    catchup=False,
    max_active_runs=1
)

success_task_test = DummyOperator(
    task_id='my_task',
    provide_context=True,
    on_success_callback=slack_callback,
    dag=dag)

success_task_test


from airflow.utils.decorators import apply_defaults
from airflow.operators.slack_operator import SlackAPIPostOperator
from airflow.models import BaseOperator
from airflow.hooks.base_hook import BaseHook
import urllib
import os
from airflow.plugins_manager import AirflowPlugin


# Slack Alerts
# HOSTNAME variable set in docker-compose
base_url = "https://"+os.getenv('HOSTNAME')
slack_channel = 'airflow-alerts-test'
slack_conn_id = 'slack'
slack_token = str(BaseHook.get_connection("slack").password)


class SlackSuccessAlert(BaseOperator):

    @apply_defaults
    def __init__(
            self,
            dag_id,
            task_id,
            execution_date,
            emoji = ':green_check:',
            message = 'has succeeded',
            *args,
            **kwargs):
        super().__init__(*args, **kwargs)
        self.dag_id = dag_id
        self.task_id = task_id
        self.execution_date = execution_date
        self.emoji = emoji
        self.message = message

    def execute(self, context):
        #dag_id = self.dag_id
        #task_id = self.task_id
        #execution_date = self.execution_date
        encoded_execution_date = urllib.parse.quote_plus(self.execution_date)
        dag_url = (f'{base_url}/graph?dag_id={self.dag_id}'
                   f'&execution_date={encoded_execution_date}')
        slack_message = (f'{self.emoji} *{self.dag_id}* <{dag_url}|*{self.message}*>.')
        slack_alert = SlackAPIPostOperator(
            task_id='slack_alert',
            channel=slack_channel,
            token=slack_token,
            text=slack_message
        )
        try:
            return slack_alert.execute(context=context)
        except:
            print("something going wrong with the callback...")



class SlackAlertsTest(AirflowPlugin):
    """
    Custom Slack Plugins for Airflow
    """
    name = "slack_airflow_plugins"
    operators = [SlackSuccessAlert]


在反复敲打我的脑袋之后，我终于想出了一个解决办法。与使用类创建插件不同，我只是将松弛回调移到一个单独的python文件中，该文件只包含函数，并将其作为模块导入并调用函数。
在本例中，正在导入的“插件”将被称为
basicplugintest

下面是一个示例DAG，它允许我测试“触发器”、“成功”和“失败”回调。您会注意到“trigger”和“success”都使用了
on\u success\u callback
气流功能，并且使用
partial
库，您可以在插件中自定义/创建case语句，将代码缩短几行

最困难的部分是弄清楚如何使用
context
，并能够使用
partial
模块定制某些回调

DAG

from airflow import DAG
import basicplugintest #this is the 'plugin' I created
from airflow.operators.dummy_operator import DummyOperator
from airflow.operators.bash_operator import BashOperator
from datetime import datetime, timedelta
import urllib
from functools import partial

# callback_type can be specified and used with if/elif statements
# probably best to only use them for trigger/success callbacks since
# they are both considered 'on_success' callbacks
# callback_type must be ordered after context 
def trigger_success_callback(context, callback_type):
    basicplugintest.success_callback(context, callback_type)

def retry_callback(context):
    basicplugintest.retry_callback(context)

def failure_callback(context):
    basicplugintest.failure_callback(context)


default_args = {
    "depends_on_past": False,
    'start_date': datetime(2021, 5, 1)
}


dag = DAG(
    dag_id='slack_plugin_test_functions',
    default_args=default_args,
    catchup=False,
    max_active_runs=1
)

start_task = DummyOperator(
    task_id='start_task',
    provide_context=True,
    # specify the callback_type to correspond to case statement in the separate plugin
    on_success_callback=partial(trigger_success_callback, callback_type='started'),
    dag=dag)

success_task = DummyOperator(
    task_id='success_task',
    provide_context=True,
    on_success_callback=partial(trigger_success_callback, callback_type='success'),
    dag=dag)

fail_task = BashOperator(
    task_id='fail_task',
    provide_context=True,
    bash_command='exit 123',
    retries=1,
    retry_delay=timedelta(seconds=10),
    on_retry_callback=retry_callback,
    on_failure_callback=failure_callback,
    dag=dag)

start_task >> success_task >> fail_task

下面是插件：

# added in the dags folder instead of plugins
# functions are referenced by the dags themselves
from airflow.models import DAG
from airflow.hooks.base_hook import BaseHook
from airflow.operators.slack_operator import SlackAPIPostOperator
import urllib
from airflow.plugins_manager import AirflowPlugin
import os
import traceback

# Slack Alerts
# HOSTNAME variable set in docker-compose
base_url = os.getenv('AIRFLOW__WEBSERVER__BASE_URL')
slack_channel = 'airflow-alerts-test'
slack_token = str(BaseHook.get_connection("slack").password)


def success_callback(context, callback_type):
    if callback_type == 'success':
        emoji=':green_check:'
        message='has succeeded'
        dag_id=context.get("task_instance").dag_id
        task_id=context.get("task_instance").task_id
        execution_date = context.get("execution_date").isoformat()
        encoded_execution_date = urllib.parse.quote_plus(execution_date)
        dag_url = (f'{base_url}/graph?dag_id={dag_id}'
                   f'&execution_date={encoded_execution_date}')
        run_id = context.get("task_instance").xcom_pull(key='RUN_ID', task_ids='xcom_run_id')
        slack_message = (f'{emoji} *{dag_id}* {message} for dp_run_id <{dag_url}|*{run_id}*>.')
        slack_alert = SlackAPIPostOperator(
            task_id='slack_alert',
            channel=slack_channel,
            token=slack_token,
            text=slack_message
        )
        return slack_alert.execute(context=context)
    
    elif callback_type == 'started':
        emoji=':airflow:'
        message='has been triggered'
        dag_id=context.get("task_instance").dag_id
        task_id=context.get("task_instance").task_id
        execution_date = context.get("execution_date").isoformat()
        encoded_execution_date = urllib.parse.quote_plus(execution_date)
        dag_url = (f'{base_url}/graph?dag_id={dag_id}'
                   f'&execution_date={encoded_execution_date}')
        run_id = context.get("task_instance").xcom_pull(key='RUN_ID', task_ids='xcom_run_id')
        slack_message = (f'{emoji} *{dag_id}* {message} for dp_run_id <{dag_url}|*{run_id}*>.')
        slack_alert = SlackAPIPostOperator(
            task_id='slack_alert',
            channel=slack_channel,
            token=slack_token,
            text=slack_message
        )
        return slack_alert.execute(context=context)

def retry_callback(context):
    emoji=':alert:'
    message='has missed SLA'
    dag_id=context.get("task_instance").dag_id
    task_id=context.get("task_instance").task_id
    try_number=context.get("task_instance").try_number - 1 #bug in airflow sets try_number 1 higher than it should be
    max_tries=context.get("task_instance").max_tries + 1 #max_tries is equal to number of retries set, which does not include the first attempt
    execution_date = context.get("execution_date").isoformat()
    encoded_execution_date = urllib.parse.quote_plus(execution_date)
    dag_url = (f'{base_url}/graph?dag_id={dag_id}'
               f'&execution_date={encoded_execution_date}')
    run_id = context.get("task_instance").xcom_pull(key='RUN_ID', task_ids='xcom_run_id')

    exception = context.get("exception")
    formatted_exception = ''.join(
        traceback.format_exception(etype=type(exception),
        value=exception, tb=exception.__traceback__)
    ).strip()

    slack_message = (f'{emoji} *{dag_id}* {message} for dp_run_id <{dag_url}|*{run_id}*>! Try number {try_number} out of {max_tries}. <!channel|channel>'
                f'\n*Task:* {task_id}'
                f'\n*Error:* {formatted_exception}')
    slack_alert = SlackAPIPostOperator(
        task_id='slack_alert',
        channel=slack_channel,
        token=slack_token,
        text=slack_message
    )

    return slack_alert.execute(context=context)


def failure_callback(context):
    emoji = ':alert:'
    message = 'has failed'
    dag_id = context.get("task_instance").dag_id
    task_id = context.get("task_instance").task_id
    execution_date = context.get("execution_date").isoformat()
    encoded_execution_date = urllib.parse.quote_plus(execution_date)
    dag_url = (f'{base_url}/graph?dag_id={dag_id}'
               f'&execution_date={encoded_execution_date}')
    run_id = context.get("task_instance").xcom_pull(key='RUN_ID', task_ids='xcom_run_id')

    exception = context.get("exception")
    formatted_exception = ''.join(
        traceback.format_exception(etype=type(exception),
        value=exception, tb=exception.__traceback__)
    ).strip()

    slack_message = (f'{emoji} *{dag_id}* {message} for dp_run_id <{dag_url}|*{run_id}*>! <!channel|channel>'
                f'\n*Task:* {task_id}'
                f'\n*Error:* {formatted_exception}')
    slack_alert = SlackAPIPostOperator(
        task_id='slack_alert',
        channel=slack_channel,
        token=slack_token,
        text=slack_message
    )
    return slack_alert.execute(context=context)

#添加到dags文件夹中，而不是插件
#函数由DAG本身引用
从airflow.models导入DAG
从afflow.hooks.base\u hook导入BaseHook
从afflow.operators.slack\u operator导入slackapipost operator
导入URL库
从airflow.plugins\u管理器导入AirflowPlugin
导入操作系统
导入回溯
#松弛警报
#docker compose中设置的主机名变量
base\u url=os.getenv（'AIRFLOW\uu WEBSERVER\uuuu base\u url'））
松弛通道=‘气流警报测试’
slack\u token=str（BaseHook.get\u连接（“slack”）.password）
def success_回调（上下文、回调类型）：
如果回调类型==“成功”：
表情符号=':green_check:'
消息=“已成功”
dag\u id=context.get（“任务\u实例”）.dag\u id
task\u id=context.get（“task\u实例”）.task\u id
执行日期=context.get（“执行日期”）.isoformat（）
encoded_execution_date=urllib.parse.quote_plus（执行日期）
dag_url=（f'{base_url}/graph？dag_id={dag_id}'
f'&execution_date={encoded_execution_date}'）
run\u id=context.get（“task\u instance”）.xcom\u pull（key='run\u id'，task\u id='xcom\u run\u id'）
slack_message=（dp_run_id的f'{emoji}*{dag_id}*{message}）
松弛警报=松弛操作员(
任务\u id='slack\u alert'，
信道=松弛信道，
令牌=松弛令牌，
text=slack\u消息
)
返回slack\u alert.execute（上下文=上下文）
elif回调类型==“已启动”：
表情符号='：气流：'
消息=“已被触发”
dag\u id=context.get（“任务\u实例”）.dag\u id
task\u id=context.get（“task\u实例”）.task\u id
执行日期=context.get（“执行日期”）.isoformat（）
encoded_execution_date=urllib.parse.quote_plus（执行日期）
dag_url=（f'{base_url}/graph？dag_id={dag_id}'
f'&execution_date={encoded_execution_date}'）
run\u id=context.get（“task\u instance”）.xcom\u pull（key='run\u id'，task\u id='xcom\u run\u id'）
slack_message=（dp_run_id的f'{emoji}*{dag_id}*{message}）
松弛警报=松弛操作员(
任务\u id='slack\u alert'，
信道=松弛信道，
令牌=松弛令牌，
text=slack\u消息
)
返回slack\u alert.execute（上下文=上下文）
def retry_回调（上下文）：
表情符号='：警报：'
消息=“已错过SLA”
dag\u id=context.get（“任务\u实例”）.dag\u id
task\u id=context.get（“task\u实例”）.task\u id
try_number=context.get（“task_instance”）。try_number-1#气流设置中的bug try_number 1高于其应有值
max_trys=context.get（“任务_实例”）。max_trys+1#max_trys等于设置的重试次数，不包括第一次尝试
执行日期=context.get（“执行日期”）.isoformat（）
encoded_execution_date=urllib.parse.quote_plus（执行日期）
dag_url=（f'{base_url}/graph？dag_id={dag_id}'
f'&execution_date={encoded_execution_date}'）
run\u id=context.get（“task\u instance”）.xcom\u pull（key='run\u id'，task\u id='xcom\u run\u id'）
exception=context.get（“异常”）
格式化的\u异常=“”。加入(
traceback.format_异常（etype=type（异常），
值=异常，tb=异常
).strip（）
slack_message=（dp_run_id的f'{emoji}*{dag_id}*{message}！在{max_tries}中尝试第{Try_number}个
f'\n*任务：{Task_id}'
f'\n*错误：{格式化的_异常}）
松弛警报=松弛操作员(
任务\u id='slack\u alert'，
信道=松弛信道，
令牌=松弛令牌，
text=slack\u消息
)
返回slack\u alert.execute（上下文=上下文）
def故障_回调（上下文）：
表情符号='：警报：'
消息='已失败'
dag\u id=context.get（“任务\u实例”）.dag\u id
task\u id=context.get（“task\u实例”）.task\u id
执行日期=context.get（“执行日期”）.isoformat（）
encoded_execution_date=urllib.parse.quote_plus（执行日期）
dag_url=（f'{base_url}/graph？dag_id={dag_id}'
f'&execution_date={encoded_execution_date}'）
run\u id=context.get（“task\u instance”）.xcom\u pull（key='run\u id'，task\u id='xcom\u run\u id'）
exception=context.get（“异常”）
格式化的\u异常=“”。加入(
traceback.format_异常（etype=type（异常），
值=异常，tb=异常
).strip（）
slack_message=（f'{emoji}*{dag_id}*{message}用于dp_run_id！'