在使用火山引擎机器学习平台(MLPlatform, MLP)进行 AI 开发过程中,常因以下原因导致任务异常:
用户无法实时感知任务失败、异常或排队等情况,事件无法及时处理。现支持通过集成 veFaaS(函数服务)与 SNS(消息通知服务),实现将 MLP 工作负载状态变化实时推送,解决告警滞后问题。
除了 MLP 产品外,实现事件通知还需要下面组件/产品:
创建事件处理函数,便于后续 SNS 中配置。
requirements.txt 文件,内容为:volcengine-python-sdk volcengine
index.py 文件内容。参考文档获取飞书机器人 webhook 地址,格式如下,替换到代码中。
https://open.feishu.cn/open-apis/bot/v2/hook/xxxxxxxxxxxxxxxxx
import json import requests from typing import Dict import volcenginesdkiam import volcenginesdkcore from volcengine.ApiInfo import ApiInfo from volcengine.Credentials import Credentials from volcengine.base.Service import Service from volcengine.ServiceInfo import ServiceInfo # --------------------------- 用户需要自行修改的部分 --------------------------- AK = "您的AccessKey" # 火山引擎API密钥 SK = "您的SecretKey" # 火山引擎SecretKey Region = "cn-beijing" # 区域 RobotWebhook = "飞书机器人Webhook地址" # 飞书机器人配置中获取的Webhook # --------------------------- 用户需要自行修改的部分 --------------------------- configuration = volcenginesdkcore.Configuration() configuration.ak = AK configuration.sk = SK configuration.region = Region configuration.client_side_validation = True # set default configuration volcenginesdkcore.Configuration.set_default(configuration) # use global default configuration api_instance = volcenginesdkiam.IAMApi() type_color = { "Start": "blue", "Run": "blue", "Success": "green", "Fail": "red", "Stop": "yellow", "SuccessHolding": "green", "FailedHolding": "red", } event_name_map = { "CreateDevInstanceEvent": "创建开发机", "StartDevInstanceEvent": "启动开发机", "StopDevInstanceEvent": "停止开发机", "ForceShutdownEvent": "强制关机", "TriggerIdleDevInstanceEvent": "触发闲时关机", "DeleteDevInstanceEvent": "删除开发机", "RebootDevInstanceEvent": "重启开发机", "DevInstanceMachineFailureEvent": "节点故障", "DevInstanceMachineSuspiciousEvent": "节点疑似故障", "DevInstanceMachineRedeployEvent": "节点待下线", "DevInstanceUvmSpinLoopEvent": "节点 UVM 自旋循环", "CreateDevInstanceWorkerEvent": "创建开发机Worker", "FinishDevInstanceWorkerEvent": "完成开发机Worker", "DeleteDevInstanceWorkerEvent": "删除开发机Worker", } def get_card( sid, name, resource_group_id, resource_queue_id, creator_trn, event_time, message_event, event_category, ): return { "config": {"update_multi": True}, "i18n_elements": { "zh_cn": [ { "tag": "column_set", "horizontal_spacing": "8px", "horizontal_align": "left", "columns": [ { "tag": "column", "width": "weighted", "elements": [ { "tag": "markdown", "content": f"**开发机**: {name}(ID: {sid})", "text_align": "left", "text_size": "normal", "icon": { "tag": "standard_icon", "token": "card_outlined", "color": "grey", }, }, { "tag": "markdown", "content": f"**开发机所属资源组ID:**{resource_group_id}", "text_align": "left", "text_size": "normal", "icon": { "tag": "standard_icon", "token": "hot_outlined", "color": "grey", }, }, { "tag": "markdown", "content": f"**开发机所属队列ID:**{resource_queue_id}", "text_align": "left", "text_size": "normal", "icon": { "tag": "standard_icon", "token": "hot_outlined", "color": "grey", }, }, { "tag": "markdown", "content": f"**创建人TRN:**{creator_trn}", "text_align": "left", "text_size": "normal", "icon": { "tag": "standard_icon", "token": "member_outlined", "color": "grey", }, }, { "tag": "markdown", "content": f"**事件时间:** {event_time}", "text_align": "left", "text_size": "normal", "icon": { "tag": "standard_icon", "token": "time_outlined", "color": "grey", }, }, ], "vertical_align": "top", "vertical_spacing": "8px", "weight": 2, } ], }, {"tag": "hr"}, { "tag": "action", "actions": [ { "tag": "button", "text": {"tag": "plain_text", "content": "查看详情"}, "url": f"https://console.volcengine.com/ml-platform/region:ml-platform+cn-beijing/devinstance/detail?InstanceId={sid}", "type": "default", "width": "default", "size": "medium", "icon": { "tag": "standard_icon", "token": "details_outlined", }, } ], }, {"tag": "hr"}, ] }, "i18n_header": { "zh_cn": { "title": { "tag": "plain_text", "content": f"【火山引擎-机器学习平台】【{Region}】开发机 {sid} {event_name_map[message_event]}", }, "subtitle": {"tag": "plain_text", "content": ""}, "template": f"{event_category}", } }, } def handler(event, context): print(f"received new request, event content: {event}") records = event.get("data", {}).get("Records", []) for record in records: print(f"get record: {record}") message = json.loads(record.get("Sns", "{}").get("Message", "{}")) print(f"get message: {message}") if message.get("EventResourceType") == "DevInstance": event_resource_id = message.get("EventResourceID") message_event = message.get("EventType", "") event_time = message.get("EventTime") devinstance_detail = get_devinstance_detail(event_resource_id) creator_trn = devinstance_detail.creator_trn card = { "msg_type": "interactive", "card": get_card( devinstance_detail.id, devinstance_detail.name, devinstance_detail.resource_group_id, devinstance_detail.resource_queue_id, creator_trn, event_time, message_event, type_color.get(message_event, "gray"), ), } print(f"get card: {card}") res = requests.post(RobotWebhook, json=card) print(res) result = {"statusCode": 200, "headers": {"Content-Type": "application/json"}} return result # ==================== MLP Client ==================== SERVICE_VERSION_V2 = "2024-07-01" SERVICE_NAME = "ml_platform_stg" service_info = ServiceInfo( "open.volcengineapi.com", {"Accept": "application/json"}, Credentials(AK, SK, SERVICE_NAME, Region), 60 * 5, 60 * 5, "https", ) api_info_v2 = { "GetDevInstance": ApiInfo( "POST", "/", {"Action": "GetDevInstance", "Version": SERVICE_VERSION_V2}, {}, {}, ), "GetResourceQueue": ApiInfo( "POST", "/", {"Action": "GetResourceQueue", "Version": SERVICE_VERSION_V2}, {}, {}, ), } class MLPClient(Service): def __init__(self) -> None: super(MLPClient, self).__init__( MLPClient.get_service_info(), MLPClient.get_api_info() ) @staticmethod def get_service_info() -> ServiceInfo: return service_info @staticmethod def get_api_info() -> Dict[str, ApiInfo]: return api_info_v2 def get_devinstance(self, instance_id: str) -> dict: res_str = self.json("GetDevInstance", [], json.dumps({"Id": instance_id})) res = json.loads(res_str) if not res.get("Result"): raise Exception(f"GetDevInstance failed, res: {res}") return res.get("Result") def get_resource_queue(self, resource_queue_id: str) -> dict: res_str = self.json( "GetResourceQueue", [], json.dumps({"Id": resource_queue_id}) ) res = json.loads(res_str) if not res.get("Result"): raise Exception(f"GetResourceQueue failed, res: {res}") return res.get("Result") class DevInstanceDetail: id: str name: str resource_group_id: str resource_queue_id: str creator_trn: str def get_devinstance_detail(eventResourceId: str) -> DevInstanceDetail: return get_devinstance_detail_by_devinstance_id(eventResourceId) def get_devinstance_detail_by_devinstance_id(devinstance_id: str) -> DevInstanceDetail: client = MLPClient() res = client.get_devinstance(devinstance_id) detail = DevInstanceDetail() detail.id = devinstance_id detail.name = res.get("Name") detail.resource_queue_id = res.get("ResourceQueueId") detail.resource_group_id = client.get_resource_queue(detail.resource_queue_id).get( "ResourceGroupId" ) detail.creator_trn = res.get("CreatorTrn") return detail
目前效果如图:
参考文档获取飞书机器人 webhook 地址,格式如下,替换到代码中。
https://open.feishu.cn/open-apis/bot/v2/hook/xxxxxxxxxxxxxxxxx
import json import requests import hashlib import base64 import hmac import time import re import volcenginesdkiam import volcenginesdkcore from volcenginesdkiam import GetUserResponse from volcenginesdkcore.rest import ApiException # --------------------------- 用户需要自行修改的部分 --------------------------- AK = "您的AccessKey" # 火山引擎API密钥 SK = "您的SecretKey" # 火山引擎SecretKey Region = "cn-beijing" # 区域 RobotWebhook = "飞书机器人Webhook地址" # 飞书机器人配置中获取的Webhook # --------------------------- 用户需要自行修改的部分 --------------------------- configuration = volcenginesdkcore.Configuration() configuration.ak = AK configuration.sk = SK configuration.region = Region configuration.client_side_validation = True volcenginesdkcore.Configuration.set_default(configuration) api_instance = volcenginesdkiam.IAMApi() def is_email(email): """验证字符串是否为邮箱格式""" pattern = r'^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+.[a-zA-Z]{2,}$' return bool(re.match(pattern, email)) def get_user_email(user_id: str): """根据用户ID获取邮箱地址""" if user_id is None or user_id == 0: return "主账号" try: resp = api_instance.get_user(volcenginesdkiam.GetUserRequest(id=user_id)) email = _get_user_email(resp) return email if email else user_id except ApiException as e: print(f"获取用户邮箱失败: {e}") return user_id def _get_user_email(resp: GetUserResponse): """从IAM响应中提取用户邮箱""" if is_email(resp.user.user_name): return resp.user.user_name if is_email(resp.user.user_email): return resp.user.user_email return None # 事件类型对应的飞书卡片颜色 type_color = { "Start": "blue", "Run": "blue", "Success": "green", "Fail": "red", "Stop": "yellow", "SuccessHolding": "green", "FailedHolding": "red", } def get_card( custom_task_id, custom_task_name, resource_group_id, resource_queue_id, creator, operator, event_time, message_event, event_categray ): """生成飞书交互卡片内容""" return { "config": {"update_multi": True}, "i18n_elements": { "zh_cn": [ { "tag": "column_set", "horizontal_spacing": "8px", "horizontal_align": "left", "columns": [ { "tag": "column", "width": "weighted", "elements": [ { "tag": "markdown", "content": f"**任务**: {custom_task_name}(ID: {custom_task_id})", "text_align": "left", "text_size": "normal", "icon": {"tag": "standard_icon", "token": "card_outlined", "color": "grey"} }, {"tag": "markdown", "content": f"**资源组ID:**{resource_group_id}", "icon": {"token": "hot_outlined", "color": "grey"}}, {"tag": "markdown", "content": f"**队列ID:**{resource_queue_id}", "icon": {"token": "hot_outlined", "color": "grey"}}, {"tag": "markdown", "content": f"**创建人:**{creator}", "icon": {"token": "member_outlined", "color": "grey"}}, {"tag": "markdown", "content": f"**操作人:**{operator}", "icon": {"token": "member_outlined", "color": "grey"}}, {"tag": "markdown", "content": f"**事件时间:** {event_time}", "icon": {"token": "time_outlined", "color": "grey"}} ], "vertical_align": "top", "vertical_spacing": "8px", "weight": 2 } ] }, {"tag": "hr"}, { "tag": "action", "actions": [ { "tag": "button", "text": {"tag": "plain_text", "content": "查看详情"}, "url": f"https://console.volcengine.com/ml-platform/region:ml-platform+cn-beijing/customTask/detail?Id={custom_task_id}", "type": "default", "icon": {"tag": "standard_icon", "token": "details_outlined"} } ] }, {"tag": "hr"} ] }, "i18n_header": { "zh_cn": { "title": {"tag": "plain_text", "content": f"【火山引擎-机器学习平台】【{Region}】自定义任务 {custom_task_id} {message_event}"}, "subtitle": {"tag": "plain_text", "content": ""}, "template": f"{event_categray}" } } } def handler(event, context): """函数入口处理MLP事件""" print(f"收到新请求,事件内容: {event}") records = event.get('data', {}).get('Records', []) for record in records: print(f"处理记录: {record}") message = json.loads(record.get('Sns', '{}').get('Message', '{}')) print(f"解析消息: {message}") if message.get('EventResourceType') == 'CustomTask': custom_task_id = message.get('EventResourceID') message_event = message.get('EventType', '')[10:] # 提取事件类型 event_time = message.get('EventTime') # 提取任务详情 custom_task = message.get('Detail', {}).get('New') or message.get('Detail', {}).get('Old') custom_task_name = custom_task.get('Name', '未知任务') resource_group_id = custom_task.get('ResourceGroupId', '未设置') resource_queue_id = custom_task.get('ResourceQueueId', '未设置') # 处理创建者信息 creator_email = get_user_email(custom_task.get('CreatorUserId')) creator = f'<at email="{creator_email}"></at>' if is_email(creator_email) else creator_email # 处理操作人信息 if message.get('EventTigerByUser'): operator_email = get_user_email(message.get('UserID')) operator = f'<at email="{operator_email}"></at>' if is_email(operator_email) else operator_email else: operator = "平台" # 生成飞书卡片 card = { "msg_type": "interactive", "card": get_card( custom_task_id, custom_task_name, resource_group_id, resource_queue_id, creator, operator, event_time, message_event, type_color.get(message_event, "grey") ) } print(f"生成卡片: {card}") # 发送消息到飞书 res = requests.post(RobotWebhook, json=card) print(f"发送结果: {res.status_code}, {res.text}") return { 'statusCode': 200, 'headers': {'Content-Type': 'application/json'} }
目前效果如图:
tip
如果只关心自己创建的训练任务,以上代码可以稍作修改,例如取消红框里的两行注释,只处理自己的 ID 相关的训练任务,
参考文档获取飞书机器人 webhook 地址,格式如下,替换到代码中。
https://open.feishu.cn/open-apis/bot/v2/hook/xxxxxxxxxxxxxxxxx
import json import requests import hashlib import base64 import hmac import time import re from typing import Dict import volcenginesdkiam import volcenginesdkcore from volcenginesdkiam import GetUserResponse from volcenginesdkcore.rest import ApiException from volcengine.ApiInfo import ApiInfo from volcengine.Credentials import Credentials from volcengine.base.Service import Service from volcengine.ServiceInfo import ServiceInfo # --------------------------- 用户需要自行修改的部分 --------------------------- AK = "您的AccessKey" # 火山引擎API密钥 SK = "您的SecretKey" # 火山引擎SecretKey Region = "cn-beijing" # 区域 RobotWebhook = "飞书机器人Webhook地址" # 飞书机器人配置中获取的Webhook # --------------------------- 用户需要自行修改的部分 --------------------------- configuration = volcenginesdkcore.Configuration() configuration.ak = AK configuration.sk = SK configuration.region = Region configuration.client_side_validation = True # set default configuration volcenginesdkcore.Configuration.set_default(configuration) type_color = { "Start" : "blue", "Run" : "blue", "Success" : "green", "Fail" : "red", "Stop" : "yellow", "SuccessHolding" : "green", "FailedHolding" : "red", } def get_card( id, service_id, service_name, resource_group_id, resource_queue_id, event_time, message_event, event_categray ): return { "config": { "update_multi": True }, "i18n_elements": { "zh_cn": [ { "tag": "column_set", "horizontal_spacing": "8px", "horizontal_align": "left", "columns": [ { "tag": "column", "width": "weighted", "elements": [ { "tag": "markdown", "content": f"**服务**: {service_name}(ID: {service_id})", "text_align": "left", "text_size": "normal", "icon": { "tag": "standard_icon", "token": "card_outlined", "color": "grey" } }, { "tag": "markdown", "content": f"**部署所属资源组ID:**{resource_group_id}", "text_align": "left", "text_size": "normal", "icon": { "tag": "standard_icon", "token": "hot_outlined", "color": "grey" } }, { "tag": "markdown", "content": f"**部署所属队列ID:**{resource_queue_id}", "text_align": "left", "text_size": "normal", "icon": { "tag": "standard_icon", "token": "hot_outlined", "color": "grey" } }, { "tag": "markdown", "content": f"**事件时间:** {event_time}", "text_align": "left", "text_size": "normal", "icon": { "tag": "standard_icon", "token": "time_outlined", "color": "grey" } } ], "vertical_align": "top", "vertical_spacing": "8px", "weight": 2 } ] }, { "tag": "hr" }, { "tag": "action", "actions": [ { "tag": "button", "text": { "tag": "plain_text", "content": "查看详情" }, "url": f"https://console.volcengine.com/ml-platform/region:ml-platform+cn-beijing/serviceNew/detail?id={service_id}", "type": "default", "width": "default", "size": "medium", "icon": { "tag": "standard_icon", "token": "details_outlined" } } ] }, { "tag": "hr" } ] }, "i18n_header": { "zh_cn": { "title": { "tag": "plain_text", "content": f"【火山引擎-机器学习平台】【{Region}】在线服务 {id} {message_event}" }, "subtitle": { "tag": "plain_text", "content": "" }, "template": f"{event_categray}" } } } def handler(event, context): # Log incoming request. # # NOTE: the log here is only for debug purpose. It's recommended to delete those debug/info # logs and only print log where error occurs after your business logic is verified and ready to # go production, nor the log amount may be too huge. print(f"received new request, event content: {event}") records = event.get('data', {}).get('Records', []) for record in records: print(f"get record: {record}") message = json.loads(record.get('Sns', '{}').get('Message', '{}')) print(f"get message: {message}") if message.get('EventResourceType') == 'ServiceEvent': event_resource_id = message.get('EventResourceID') message_event = message.get('EventType', '') event_time = message.get('EventTime') inferenceDetail = get_inference_detail(message_event, event_resource_id) card = { "msg_type": "interactive", "card": get_card( inferenceDetail.id, inferenceDetail.service_id, inferenceDetail.service_name, inferenceDetail.resource_group_id, inferenceDetail.resource_queue_id, event_time, message_event, type_color.get(message_event, "gray") ) } print(f"get card: {card}") res = requests.post(RobotWebhook, json=card) print(res) result = { 'statusCode': 200, 'headers': { 'Content-Type': 'application/json' } } return result SERVICE_VERSION_V1 = '2021-10-01' SERVICE_VERSION_V2 = '2024-07-01' service_info = ServiceInfo( 'open.volcengineapi.com', {'Accept': 'application/json'}, Credentials(AK, SK, 'ml_platform_stg', Region), 60 * 5, 60 * 5, "https" ) api_info_v1 = { 'GetDeployment': ApiInfo('POST', '/',{ "Action": "GetDeployment", "Version": SERVICE_VERSION_V1}, {}, {}), 'GetService': ApiInfo('POST', '/',{ "Action": "GetService", "Version": SERVICE_VERSION_V1}, {}, {}), } api_info_v2 = { 'GetDeployment': ApiInfo('POST', '/',{ "Action": "GetDeployment", "Version": SERVICE_VERSION_V2}, {}, {}), 'GetService': ApiInfo('POST', '/',{ "Action": "GetService", "Version": SERVICE_VERSION_V2}, {}, {}), 'GetResourceQueue': ApiInfo('POST', '/',{ "Action": "GetResourceQueue", "Version": SERVICE_VERSION_V2}, {}, {}), } class MLPClientV1(Service): def __init__(self) -> None: super(MLPClientV1, self).__init__(MLPClientV1.get_service_info(), MLPClientV1.get_api_info()) @staticmethod def get_service_info() -> ServiceInfo: return service_info @staticmethod def get_api_info() -> Dict[str, ApiInfo]: return api_info_v1 def get_deployment(self, service_id: str, deployment_id: str) -> dict: res_str = self.json('GetDeployment', [],json.dumps({ "DeploymentId": deployment_id, "ServiceId": service_id })) res = json.loads(res_str) if not res.get('Result'): raise Exception(f'GetDeployment failed, res: {res}') return res.get('Result') def get_service(self, service_id: str) -> dict: res_str = self.json('GetService', [],json.dumps({ "ServiceID": service_id })) res = json.loads(res_str) if not res.get('Result'): raise Exception(f'GetService failed, res: {res}') return res.get('Result') class MLPClientV2(Service): def __init__(self) -> None: super(MLPClientV2, self).__init__(MLPClientV2.get_service_info(), MLPClientV2.get_api_info()) @staticmethod def get_service_info() -> ServiceInfo: return service_info @staticmethod def get_api_info() -> Dict[str, ApiInfo]: return api_info_v2 def get_deployment(self, deployment_id: str) -> dict: res_str = self.json('GetDeployment', [],json.dumps({ "Id": deployment_id })) res = json.loads(res_str) if not res.get('Result'): raise Exception(f'GetDeployment failed, res: {res}') return res.get('Result') def get_service(self, service_id: str) -> dict: res_str = self.json('GetService',[], json.dumps({ "Id": service_id })) res = json.loads(res_str) if not res.get('Result'): raise Exception(f'GetService failed, res: {res}') return res.get('Result') def get_resource_queue(self, resource_queue_id: str) -> dict: res_str = self.json('GetResourceQueue', [], json.dumps({ "Id": resource_queue_id })) res = json.loads(res_str) if not res.get('Result'): raise Exception(f'GetResourceQueue failed, res: {res}') return res.get('Result') class InferenceDetail(): id: str name: str resource_group_id: str resource_queue_id: str service_id: str service_name: str def get_inference_detail(eventType: str, eventResourceId: str) -> InferenceDetail: if eventType.startswith('Deployment'): return get_inference_detail_by_deployment_id(eventResourceId) elif eventType.startswith('Service'): return get_inference_detail_by_service_id(eventResourceId) elif eventType.startswith('Instance'): return get_inference_detail_by_instance_id(eventResourceId) else: return None def get_inference_detail_by_service_id(service_id: str) -> InferenceDetail: detail = InferenceDetail() detail.id = service_id if UseMultiRole: client = MLPClientV2() res = client.get_service(service_id) detail.name = res.get('Name') detail.service_id = service_id detail.service_name = res.get('Name') else: client = MLPClientV1() res = client.get_service(service_id) detail.name = res.get('ServiceName') detail.service_id = service_id detail.service_name = res.get('Name') return detail def get_inference_detail_by_deployment_id(deployment_id: str) -> InferenceDetail: detail = InferenceDetail() detail.id = deployment_id detail.resource_type = 'Deployment' if UseMultiRole: clientv2 = MLPClientV2() res = clientv2.get_deployment(deployment_id) detail.name = res.get('Name') detail.resource_queue_id = res.get('ResourceQueueId') resource_group_id = clientv2.get_resource_queue(detail.resource_queue_id).get('ResourceGroupId') detail.resource_group_id = resource_group_id service_id = deployment_id[:22] service = clientv2.get_service(service_id) detail.service_id = service_id detail.service_name = service.get('Name') else: clientv1 = MLPClientV1() clientv2 = MLPClientV2() service_id = deployment_id[:22] res = clientv1.get_deployment(service_id, deployment_id) detail.name = res.get('Name') detail.resource_queue_id = res.get('ResourceQueueId') resource_group_id = clientv2.get_resource_queue(detail.resource_queue_id).get('ResourceGroupId') detail.resource_group_id = resource_group_id detail.service_id = service_id service_info = clientv1.get_service(service_id) detail.service_name = service_info.get('ServiceName') return detail def get_inference_detail_by_instance_id(instance_id: str) -> InferenceDetail: detail = InferenceDetail() detail.id = instance_id detail.resource_type = 'Instance' if UseMultiRole: clientv2 = MLPClientV2() deployment_id = instance_id[:31] res = clientv2.get_deployment(deployment_id) detail.resource_queue_id = res.get('ResourceQueueId') resource_group_id = clientv2.get_resource_queue(detail.resource_queue_id).get('ResourceGroupId') detail.resource_group_id = resource_group_id service_id = deployment_id[:22] service = clientv2.get_service(service_id) detail.service_id = service_id detail.service_name = service.get('Name') else: clientv1 = MLPClientV1() clientv2 = MLPClientV2() service_id = instance_id[:22] deployment_id = instance_id[:28] res = clientv1.get_deployment(service_id, deployment_id) detail.resource_queue_id = res.get('ResourceQueueId') resource_group_id = clientv2.get_resource_queue(detail.resource_queue_id).get('ResourceGroupId') detail.resource_group_id = resource_group_id detail.service_id = service_id service_info = clientv1.get_service(service_id) detail.service_name = service_info.get('ServiceName') return detail
参考文档获取飞书机器人 webhook 地址,格式如下,替换到代码中。
https://open.feishu.cn/open-apis/bot/v2/hook/xxxxxxxxxxxxxxxxx
import os import re import json import requests import volcenginesdkiam import volcenginesdkcore from volcenginesdkiam import GetUserResponse from volcenginesdkcore.rest import ApiException # --------------------------- 用户需要自行修改的部分 --------------------------- AK = "您的AccessKey" # 火山引擎API密钥 SK = "您的SecretKey" # 火山引擎SecretKey Region = "cn-beijing" # 区域 RobotWebhook = "飞书机器人Webhook地址" # 飞书机器人配置中获取的Webhook # --------------------------- 用户需要自行修改的部分 --------------------------- # --- Dictionaries for Card Formatting (Unchanged) --- type_color = { "CreatePipelineEvent": "green", "UpdatePipelineEvent": "green", "DeletePipelineEvent": "yellow", "PipelineInstanceInitializedEvent": "blue", "PipelineInstanceRunningEvent": "green", "PipelineInstanceSuccessEvent": "green", "PipelineInstanceFailedEvent": "red", "PipelineInstanceKillingEvent": "green", "PipelineInstanceKilledEvent": "yellow", } status_hint= { "CreatePipelineEvent": "创建成功", "UpdatePipelineEvent": "更新成功", "DeletePipelineEvent": "删除成功", "PipelineInstanceInitializedEvent": "初始化中", "PipelineInstanceRunningEvent": "运行中", "PipelineInstanceSuccessEvent": "运行成功", "PipelineInstanceFailedEvent": "运行失败", "PipelineInstanceKillingEvent": "停止中", "PipelineInstanceKilledEvent": "已停止" } # --- Helper Functions for User Info and Card Generation (Unchanged) --- def is_email(email): # Defines the regular expression for an email pattern = r'^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+.[a-zA-Z]{2,}$' # Match using the regular expression if re.match(pattern, str(email)): return True else: return False def _get_user_email(resp: GetUserResponse): if is_email(resp.user.user_name): return resp.user.user_name if is_email(resp.user.user_email): return resp.user.user_email return None def get_pipeline_card( region, pipeline_id, pipeline_name, creator, operator, event_time, event_type, event_service, event_template ): return { "config": { "update_multi": True }, "i18n_elements": { "zh_cn": [ { "tag": "column_set", "horizontal_spacing": "8px", "horizontal_align": "left", "columns": [ { "tag": "column", "width": "weighted", "elements": [ { "tag": "markdown", "content": f"**工作流**: {pipeline_name}(ID: {pipeline_id})", "text_align": "left", "text_size": "normal", "icon": { "tag": "standard_icon", "token": "card_outlined", "color": "grey" } }, { "tag": "markdown", "content": f"**创建人:**{creator}", "text_align": "left", "text_size": "normal", "icon": { "tag": "standard_icon", "token": "member_outlined", "color": "grey" } }, { "tag": "markdown", "content": f"**操作人:**{operator}", "text_align": "left", "text_size": "normal", "icon": { "tag": "standard_icon", "token": "member_outlined", "color": "grey" } }, { "tag": "markdown", "content": f"**事件时间:** {event_time}", "text_align": "left", "text_size": "normal", "icon": { "tag": "standard_icon", "token": "time_outlined", "color": "grey" } } ], "vertical_align": "top", "vertical_spacing": "8px", "weight": 2 } ] }, { "tag": "hr" }, { "tag": "action", "actions": [ { "tag": "button", "text": { "tag": "plain_text", "content": "查看详情" }, "url": f"https://console.volcengine.com/{event_service.replace('_', '-')}/region:{event_service.replace('_', '-')}+{region}/pipeline/detail/{pipeline_id}", "type": "default", "width": "default", "size": "medium", "icon": { "tag": "standard_icon", "token": "details_outlined" } } ] }, { "tag": "hr" } ] }, "i18n_header": { "zh_cn": { "title": { "tag": "plain_text", "content": f"【火山引擎-机器学习平台】【{region}】 工作流 {pipeline_id} {status_hint.get(event_type)}" }, "subtitle": { "tag": "plain_text", "content": "" }, "template": f"{event_template}" } } } def get_pipeline_instance_card( region, pipeline_instance_id, pipeline_instance_name, pipeline_id, parent_pipeline_instance_id, creator, operator, event_time, event_type, event_service, event_template ): return { "config": { "update_multi": True }, "i18n_elements": { "zh_cn": [ { "tag": "column_set", "horizontal_spacing": "8px", "horizontal_align": "left", "columns": [ { "tag": "column", "width": "weighted", "elements": [ { "tag": "markdown", "content": f"**工作流**: {pipeline_id}", "text_align": "left", "text_size": "normal", "icon": { "tag": "standard_icon", "token": "card_outlined", "color": "grey" } }, { "tag": "markdown", "content": f"**工作流实例**: {pipeline_instance_name}(ID: {pipeline_instance_id})", "text_align": "left", "text_size": "normal", "icon": { "tag": "standard_icon", "token": "card_outlined", "color": "grey" } }, { "tag": "markdown", "content": f"**是否重试**: {'是' if parent_pipeline_instance_id is not None else '否'}", "text_align": "left", "text_size": "normal", "icon": { "tag": "standard_icon", "token": "card_outlined", "color": "grey" } }, { "tag": "markdown", "content": f"**创建人:**{creator}", "text_align": "left", "text_size": "normal", "icon": { "tag": "standard_icon", "token": "member_outlined", "color": "grey" } }, { "tag": "markdown", "content": f"**操作人:**{operator}", "text_align": "left", "text_size": "normal", "icon": { "tag": "standard_icon", "token": "member_outlined", "color": "grey" } }, { "tag": "markdown", "content": f"**事件时间:** {event_time}", "text_align": "left", "text_size": "normal", "icon": { "tag": "standard_icon", "token": "time_outlined", "color": "grey" } } ], "vertical_align": "top", "vertical_spacing": "8px", "weight": 2 } ] }, { "tag": "hr" }, { "tag": "action", "actions": [ { "tag": "button", "text": { "tag": "plain_text", "content": "查看详情" }, "url": f"https://console.volcengine.com/{event_service.replace('_', '-')}/region:{event_service.replace('_', '-')}+{region}/pipeline/detail/{pipeline_id}?activeBar=operationRecord&selectedId={pipeline_instance_id}", "type": "default", "width": "default", "size": "medium", "icon": { "tag": "standard_icon", "token": "details_outlined" } } ] } ] }, "i18n_header": { "zh_cn": { "title": { "tag": "plain_text", "content": f"【火山引擎-机器学习平台】【{region}】 工作流实例 {pipeline_instance_id} {status_hint.get(event_type)}" }, "subtitle": { "tag": "plain_text", "content": "" }, "template": f"{event_template}" } } } # --- Simplified Class for Lark (Feishu) Notifications Only --- class PipelineEvent: def __init__(self, region, webhook): self.region = region self.webhook = webhook # Configure and initialize the IAM client to get user details configuration = volcenginesdkcore.Configuration() configuration.ak = AK configuration.sk = SK configuration.region = region configuration.client_side_validation = True volcenginesdkcore.Configuration.set_default(configuration) self.iam_client = volcenginesdkiam.IAMApi() def get_user_email(self, user_id: str): if user_id is None: return user_id # In Volcengine, UserID 0 represents the root account if user_id == '0' or user_id == 0: return "主账号" try: resp = self.iam_client.get_user( volcenginesdkiam.GetUserRequest( # The SDK might expect the ID as a string user_name=str(user_id), )) email = _get_user_email(resp) return email if email else str(user_id) except ApiException as e: print(f"Exception when calling IAMApi->get_user for UserID '{user_id}': {e} ") return str(user_id) def send_webhook(self, card): try: print(f"Sending card to Lark: {json.dumps(card, indent=2)}") result = requests.post(self.webhook, json=card) result.raise_for_status() # Raise an exception for bad status codes (4xx or 5xx) print(f"Lark API response: {result.status_code} {result.text}") except Exception as e: print(f"Failed to send webhook: {str(e)}") def handle(self, msg: dict): resource_id = msg.get('EventResourceID') # pipeline id or pipeline instance id event_type = msg.get('EventType') event_time = msg.get('EventTime') event_service = msg.get('EventService') resource_type = msg.get('EventResourceType') operator = "平台" operator_email = None if msg.get('EventTigerByUser'): operator_email = self.get_user_email(msg.get('UserID')) # Format for Lark @mention if it's a valid email operator = f'<at email="{operator_email}"></at>' if is_email(operator_email) else operator_email extra_info = json.loads(msg.get('Detail', {}).get('Message', '{}')) creator_email = self.get_user_email(extra_info.get('OwnerId')) # Format for Lark @mention if it's a valid email creator = f'<at email="{creator_email}"></at>' if is_email(creator_email) else creator_email card = {} if resource_type == 'Pipeline': pipeline_name = extra_info.get('PipelineName') card = { "msg_type": "interactive", "card": get_pipeline_card( self.region, resource_id, pipeline_name, creator, operator, event_time, event_type, event_service, type_color.get(event_type, "grey") ) } elif resource_type == 'PipelineInstance': pipeline_instance_name = extra_info.get('PipelineInstanceName') pipeline_id = extra_info.get('PipelineId') parent_pipeline_instance_id = extra_info.get("ParentPipelineInstanceId") card = { "msg_type": "interactive", "card": get_pipeline_instance_card( self.region, resource_id, pipeline_instance_name, pipeline_id, parent_pipeline_instance_id, creator, operator, event_time, event_type, event_service, type_color.get(event_type, "grey") ) } else: print(f"Unsupported resource type: {resource_type}. Skipping.") return # Only send the webhook now self.send_webhook(card) # --- Instantiation and Handler Function --- # Initialize the event handler with only the region and webhook URL pipeline_event = PipelineEvent( region=Region, webhook=RobotWebhook ) def handler(event, context): print(f"received new request, event content: {event}") records = event.get('data', {}).get('Records', []) for record in records: print(f"get record: {record}") try: message = json.loads(record.get('Sns', {}).get('Message', '{}')) if not message: # Skip if message is empty print("Skipping empty message.") continue print(f"get message: {message}") pipeline_event.handle(message) except json.JSONDecodeError as e: print(f"Error decoding message JSON: {e}") except Exception as e: print(f"An unexpected error occurred while processing a record: {e}") result = { 'statusCode': 200, 'headers': { 'Content-Type': 'application/json' }, 'body': json.dumps({'message': 'Processing complete'}) } return result
通过消息通知服务(SNS),订阅和消费 MLP 产生的结构化消息(JSON 格式)
当前 SNS 服务需要申请白名单权限,申请方式:
函数名称:函数ID)dev 关键词Customtask 关键词负载类型 | 通知事件 |
|---|---|
开发机 | 创建开发机 |
创建远程开发机 | |
启动开发机 | |
关闭开发机 | |
低利用率即将关机 | |
闲时即将关机 | |
强制关机 | |
重启开发机 | |
删除开发机 | |
删除远程开发机 | |
远程开发机运行完成 | |
机器故障 | |
自定义任务 | 创建自定义任务 |
自定义任务等待中 | |
自定义任务排队中 | |
自定义任务部署中 | |
自定义任务运行中 | |
自定义任务失败 | |
自定义任务完成 | |
自定义任务失败并进入保留阶段 | |
自定义任务完成并进入保留阶段 | |
自定义任务停止 | |
自定义任务取消 | |
自定义任务重试 | |
自定义任务重试结束 | |
自定义任务优先级调整 | |
自定义任务机器故障 | |
自定义任务低利用率停止 | |
在线服务 | 在线服务异常 |
在线服务停止 | |
在线服务部署异常 | |
在线服务部署停止 | |
在线服务部署部署中 | |
在线服务部署运行中 | |
在线服务部署扩缩容已完成 | |
在线服务实例异常 | |
工作流 | 运行失败 |
工作流终止 | |
工作流运行完成 | |
工作流运行中 | |
创建成功 | |
工作流配置更新 |