8000 feat(notifications): add per-task incident policies by cuisongliu · Pull Request #2 · cuisongliu/macmini-task-monitor · GitHub
[go: up one dir, main page]

Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
8000
Diff view
19 changes: 19 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,25 @@ AI 脚本库与普通任务分开管理。脚本配置包含 AI 提供商、模
- 日历任务显示计划时间;间隔任务根据最近启动时间估计,不能保证休眠、唤醒后的实际时刻。
- SQLite 当前保留全部记录,不自动删除历史。原业务日志仍由原脚本负责维护。

## 飞书通知

runner 支持在任务结束后按状态发送飞书和邮件通知。打开页面的“通知设置”保存渠道即可;配置写入监控自己的 SQLite,读取接口不会返回凭据。每个渠道可单独启用并选择触发状态,未设置触发状态时不发送;通知失败不会改变业务退出码,也不会重试。配置示例:

```text
TASK_MONITOR_FEISHU_EVENTS=failed
[{"id":"feishu","type":"feishu_webhook","enabled":true,"events":["failed"],"webhook_url":"https://open.feishu.cn/open-apis/bot/v2/hook/...","secret":"可选签名密钥"},{"id":"mail","type":"email","enabled":true,"events":["failed"],"smtp_host":"smtp.example.com","smtp_port":465,"security":"ssl","from":"monitor@example.com","to":["ops@example.com"],"username":"monitor","password":"SMTP 密码"}]
```

`events` 支持 `success`、`failed`、`interrupted`、`skipped`,多个状态直接写入数组。自定义机器人可配置签名密钥;传输层只接受飞书官方 HTTPS Webhook 域名。

每个渠道卡片上的测试按钮会立即向该渠道发送一条“Mac mini 任务监控测试通知”,使用当前表单内容但不会保存配置;测试失败会显示接口错误类别或 SMTP/网络错误类型。

飞书应用机器人渠道使用 `type: "feishu_app"`,并填写 `app_id`、`app_secret`、`receive_id`;接收方类型可设为 `open_id`(默认)、`user_id`、`union_id`、`email` 或 `chat_id`。应用需具备获取 tenant_access_token 与发送 IM 消息所需权限,并已能向目标接收方发消息。邮件支持 SMTP SSL、STARTTLS 和明文连接。

通知正文仅包含任务 ID、最终状态和执行摘要,不包含日志。为兼容旧任务,仍支持 `TASK_MONITOR_FEISHU_*` 环境变量;数据库有通知配置时优先使用数据库配置。凭据只保存在本机,诊断输出会脱敏;不要把真实凭据写入代码或提交到仓库。可用本仓库测试验证通知封装,但不要用真实任务验证发送。

任务是否通知在“本机任务”编辑表单中单独设置,默认关闭。推荐使用“故障合并”策略:首次失败或中断立即通知,连续故障按提醒间隔合并,恢复时只通知一次;默认冷却 30 分钟、持续故障提醒 4 小时、单任务每小时最多 3 条。监控全局每小时最多发送 20 条,超出的通知会记录为抑制,避免异常时通知风暴。通知状态和发送记录持久化在本机 SQLite,服务重启不会清空限流状态。

## 维护

局部验证(使用临时 SQLite,不执行真实业务或发送通知):
Expand Down
91 changes: 91 additions & 0 deletions feishu_notifications.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,91 @@
"""飞书通知传输。

通知是任务的旁路能力:配置缺失或发送失败都不应改变业务任务结果。
支持自定义机器人 Webhook 和飞书应用机器人 IM API;调用方负责决定何时发送。
"""

import base64
import hashlib
import hmac
import json
import time
from urllib.error import HTTPError, URLError
from urllib.parse import urlencode, urlparse
from urllib.request import Request, urlopen


class FeishuError(RuntimeError):
"""可安全显示给本机日志的通知错误。"""


def _json_request(url, payload, headers=None, timeout=10, opener=urlopen):
body = json.dumps(payload, ensure_ascii=False).encode('utf-8')
request = Request(url, data=body, headers={
'Content-Type': 'application/json; charset=utf-8',
'User-Agent': 'macmini-task-monitor/1',
**(headers or {}),
}, method='POST')
try:
with opener(request, timeout=timeout) as response:
raw = response.read(1024 * 1024)
except (HTTPError, URLError, TimeoutError, OSError) as error:
raise FeishuError(f'飞书请求失败:{type(error).__name__}') from error
try:
result = json.loads(raw.decode('utf-8'))
except (ValueError, UnicodeDecodeError) as error:
raise FeishuError('飞书响应不是有效 JSON') from error
if not isinstance(result, dict) or result.get('code', 0) not in (0, None):
raise FeishuError('飞书接口返回失败')
return result


def _check_webhook_url(url):
parsed = urlparse(str(url))
if parsed.scheme != 'https' or parsed.hostname not in ('open.feishu.cn', 'open.larksuite.com'):
raise FeishuError('飞书 Webhook 地址无效')
if '/open-apis/bot/' not in parsed.path:
raise FeishuError('飞书 Webhook 地址无效')


def send_webhook(text, webhook_url, secret=None, *, timeout=10, opener=urlopen):
"""通过自定义机器人发送文本;secret 存在时按飞书规则签名。"""
_check_webhook_url(webhook_url)
payload = {'msg_type': 'text', 'content': {'text': str(text)[:30000]}}
if secret:
timestamp = str(int(time.time()))
digest = hmac.new(f'{timestamp}\n{secret}'.encode('utf-8'), digestmod=hashlib.sha256).digest()
payload.update({'timestamp': timestamp, 'sign': base64.b64encode(digest).decode('ascii')})
return _json_request(webhook_url, payload, timeout=timeout, opener=opener)


def _tenant_token(app_id, app_secret, *, timeout=10, opener=urlopen):
result = _json_request('https://open.feishu.cn/open-apis/auth/v3/tenant_access_token/internal',
{'app_id': app_id, 'app_secret': app_secret}, timeout=timeout, opener=opener)
token = result.get('tenant_access_token')
if not token:
raise FeishuError('飞书应用未返回租户令牌')
return token


def send_app(text, app_id, app_secret, receive_id, receive_id_type='open_id', *, timeout=10, opener=urlopen):
"""通过飞书应用机器人给一个接收方发送文本。"""
if not all((app_id, app_secret, receive_id)):
raise FeishuError('飞书应用配置不完整')
if receive_id_type not in ('open_id', 'user_id', 'union_id', 'email', 'chat_id'):
raise FeishuError('飞书接收方类型无效')
token = _tenant_token(app_id, app_secret, timeout=timeout, opener=opener)
query = urlencode({'receive_id_type': receive_id_type})
return _json_request('https://open.feishu.cn/open-apis/im/v1/messages?' + query,
{'receive_id': receive_id, 'msg_type': 'text', 'content': json.dumps({'text': str(text)[:30000]}, ensure_ascii=False)},
headers={'Authorization': 'Bearer ' + token}, timeout=timeout, opener=opener)


def send(text, *, webhook_url='', webhook_secret='', app_id='', app_secret='', receive_id='', receive_id_type='open_id', timeout=10, opener=urlopen):
"""根据配置选择唯一发送通道;没有配置时返回 False。"""
if webhook_url:
send_webhook(text, webhook_url, webhook_secret, timeout=timeout, opener=opener)
return True
if app_id or app_secret or receive_id:
send_app(text, app_id, app_secret, receive_id, receive_id_type, timeout=timeout, opener=opener)
return True
return False
2 changes: 2 additions & 0 deletions launchd_config.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@

import collector
import store
import notifications
from config import AGENTS, DATA, ROOT, redact

LOCK = threading.RLock()
Expand Down Expand Up @@ -319,6 +320,7 @@ def metadata(body, source, original=None):
value['ai_script_id'] = script_id
else:
value.pop('ai_script_id', None)
value['notification'] = notifications.validate_task_rule(body.get('notification', (original or {}).get('notification')))
return value


Expand Down
237 changes: 237 additions & 0 deletions notifications.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,237 @@
"""通用通知配置与发送层。

配置保存在监控自己的 SQLite 中,凭据只在发送时读取;HTTP 展示始终脱敏。
"""

import json
import os
import smtplib
import ssl
import time
from email.message import EmailMessage

import feishu_notifications

EVENTS = {'success', 'failed', 'interrupted', 'skipped'}
TYPES = {'feishu_webhook', 'feishu_app', 'email'}
SECRET_FIELDS = {'webhook_url', 'secret', 'app_secret', 'password'}
DEFAULT_TASK_RULE = {'enabled': False, 'channels': [], 'events': ['failed', 'interrupted'], 'mode': 'incident',
'cooldown_seconds': 1800, 'reminder_seconds': 14400, 'notify_recovery': True, 'max_p 8000 er_hour': 3}
GLOBAL_MAX_PER_HOUR = 20


class NotificationError(ValueError):
pass


def validate_task_rule(value=None):
rule = {**DEFAULT_TASK_RULE, **(value or {})}
if not isinstance(rule.get('enabled'), bool) or not isinstance(rule.get('notify_recovery'), bool):
raise NotificationError('任务通知开关必须为布尔值')
if not isinstance(rule.get('channels'), list) or not all(isinstance(item, str) and item for item in rule['channels']):
raise NotificationError('任务通知渠道无效')
if not isinstance(rule.get('events'), list) or not rule['events'] or not set(rule['events']).issubset(EVENTS):
raise NotificationError('任务通知事件无效')
if rule.get('mode') not in ('incident', 'every_run'):
raise NotificationError('任务通知模式无效')
for field, maximum in (('cooldown_seconds', 604800), ('reminder_seconds', 2592000), ('max_per_hour', 20)):
try:
rule[field] = int(rule[field])
except (TypeError, ValueError):
raise NotificationError(f'{field} 必须是整数')
if rule[field] < (1 if field != 'max_per_hour' else 1) or rule[field] > maximum:
raise NotificationError(f'{field} 超出允许范围')
return rule


def validate(config, existing=None):
if not isinstance(config, dict) or not isinstance(config.get('channels'), list):
raise NotificationError('通知配置必须包含 channels 数组')
old = {item.get('id'): item for item in (existing or {}).get('channels', []) if isinstance(item, dict)}
result = []
for item in config['channels']:
if not isinstance(item, dict):
raise NotificationError('通知渠道必须是对象')
channel = dict(item)
channel_id = str(channel.get('id', '')).strip()
kind = channel.get('type')
if not channel_id or len(channel_id) > 80 or kind not in TYPES:
raise NotificationError('通知渠道 ID 或类型无效')
if any(row.get('id') == channel_id for row in result):
raise NotificationError('通知渠道 ID 不能重复')
events = channel.get('events', [])
if not isinstance(events, list) or not events or not set(events).issubset(EVENTS):
raise NotificationError('通知事件必须是 success、failed、interrupted 或 skipped')
channel['id'] = channel_id
channel['enabled'] = bool(channel.get('enabled', True))
channel['events'] = list(dict.fromkeys(events))
previous = old.get(channel_id, {})
for key in SECRET_FIELDS:
if not channel.get(key) and previous.get(key):
channel[key] = previous[key]
if kind == 'feishu_webhook':
if not channel.get('webhook_url'):
raise NotificationError(f'渠道 {channel_id} 缺少 Webhook 地址')
feishu_notifications._check_webhook_url(channel['webhook_url'])
elif kind == 'feishu_app':
for key in ('app_id', 'app_secret', 'receive_id'):
if not channel.get(key):
raise NotificationError(f'渠道 {channel_id} 缺少 {key}')
if channel.get('receive_id_type', 'open_id') not in ('open_id', 'user_id', 'union_id', 'email', 'chat_id'):
raise NotificationError('飞书接收方类型无效')
channel['receive_id_type'] = channel.get('receive_id_type', 'open_id')
else:
required = ('smtp_host', 'from', 'to')
if any(not channel.get(key) for key in required) or not isinstance(channel['to'], list) or not channel['to']:
raise NotificationError(f'渠道 {channel_id} 的邮件配置不完整')
try:
channel['smtp_port'] = int(channel.get('smtp_port', 465))
except (TypeError, ValueError):
raise NotificationError('SMTP 端口无效')
if channel['smtp_port'] < 1 or channel['smtp_port'] > 65535:
raise NotificationError('SMTP 端口无效')
if channel.get('security', 'ssl') not in ('ssl', 'starttls', 'plain'):
raise NotificationError('SMTP 安全方式无效')
channel['security'] = channel.get('security', 'ssl')
result.append(channel)
return {'channels': result}


def public(config):
"""返回可供页面编辑的脱敏配置;空敏感字段表示保留原值。"""
output = {'channels': []}
for item in (config or {}).get('channels', []):
row = {key: value for key, value in item.items() if key not in SECRET_FIELDS}
for key in SECRET_FIELDS:
if item.get(key):
row[key] = ''
row[key + '_configured'] = True
output['channels'].append(row)
return output


def send_channel(channel, text, timeout=10):
kind = channel['type']
if kind == 'feishu_webhook':
return feishu_notifications.send_webhook(text, channel['webhook_url'], channel.get('secret', ''), timeout=timeout)
if kind == 'feishu_app':
return feishu_notifications.send_app(text, channel['app_id'], channel['app_secret'], channel['receive_id'], channel.get('receive_id_type', 'open_id'), timeout=timeout)
message = EmailMessage()
message['Subject'] = channel.get('subject', 'Mac mini 任务通知')
message['From'] = channel['from']
message['To'] = ', '.join(channel['to'])
message.set_content(str(text)[:30000])
security = channel.get('security', 'ssl')
if security == 'ssl':
client = smtplib.SMTP_SSL(channel['smtp_host'], channel.get('smtp_port', 465), timeout=timeout)
else:
client = smtplib.SMTP(channel['smtp_host'], channel.get('smtp_port', 587), timeout=timeout)
try:
if security == 'starttls':
client.starttls(context=ssl.create_default_context())
if channel.get('username'):
client.login(channel['username'], channel.get('password', ''))
client.send_message(message)
finally:
try:
client.quit()
except OSError:
client.close()
return True


def send_for_event(config, event, text):
sent = 0
errors = []
for channel in (config or {}).get('channels', []):
if channel.get('enabled', True) and event in channel.get('events', []):
try:
send_channel(channel, text)
sent += 1
except Exception as error:
errors.append(f"{channel.get('id', 'unknown')}: {type(error).__name__}")
return sent, errors


def notify_task(task_id, rule_value, channel_config, event, summary, now=None):
"""Apply incident merging, reminders, recovery, and durable per-task/global limits."""
import store

now = time.time() if now is None else now
rule = validate_task_rule(rule_value)
if not rule['enabled']:
return 0, []
selected = set(rule['channels'])
channels = [item for item in (channel_config or {}).get('channels', [])
if item.get('id') in selected and item.get('enabled', True)]
state = store.get_notification_state(task_id)
active = bool(state.get('active'))
should_send = False
message_event = event
if event in ('failed', 'interrupted') and event in rule['events']:
state['failure_count'] = int(state.get('failure_count', 0)) + 1
state.setdefault('incident_started_at', now)
if not active:
state['active'] = True
should_send = True
elif rule['mode'] == 'every_run' and now - state.get('last_sent_at', 0) >= rule['cooldown_seconds']:
should_send = True
elif rule['mode'] == 'incident' and now - state.get('last_sent_at', 0) >= rule['reminder_seconds']:
should_send = True
message_event = 'reminder'
else:
state['suppressed_count'] = int(state.get('suppressed_count', 0)) + 1
elif event == 'success' and active:
should_send = bool(rule['notify_recovery'])
message_event = 'recovered'
state.update({'active': False, 'recovered_at': now, 'failure_count': 0})
elif event in rule['events'] and rule['mode'] == 'every_run':
should_send = True
if not should_send:
store.save_notification_state(task_id, state)
return 0, []

message = f'任务通知\n任务:{task_id}\n事件:{message_event}\n摘要:{summary}'
if message_event == 'reminder':
message += f"\n已连续故障 {state.get('failure_count', 1)} 次"
if message_event == 'recovered' and state.get('suppressed_count'):
message += f"\n故障期间合并了 {state['suppressed_count']} 次通知"
sent, errors = 0, []
for channel in channels:
if message_event not in ('recovered', 'reminder') and event not in channel.get('events', []):
continue
# Recovery/reminder are lifecycle messages for an incident; honor channel's failure subscription.
if message_event in ('recovered', 'reminder') and not ({'failed', 'interrupted'} & set(channel.get('events', []))):
continue
if not store.reserve_notification_delivery(task_id, channel['id'], message_event, now,
rule['max_per_hour'], GLOBAL_MAX_PER_HOUR):
state['suppressed_count'] = int(state.get('suppressed_count', 0)) + 1
continue
try:
send_channel(channel, message)
sent += 1
except Exception as error:
errors.append(f"{channel.get('id', 'unknown')}: {type(error).__name__}")
if sent:
state['last_sent_at'] = now
else:
state['suppressed_count'] = int(state.get('suppressed_count', 0)) + 1
store.save_notification_state(task_id, state)
return sent, errors


def environment_config():
"""兼容旧版显式环境变量配置;数据库有配置时由调用方优先使用数据库。"""
events = [item.strip() for item in os.environ.get('TASK_MONITOR_FEISHU_EVENTS', '').split(',') if item.strip()]
webhook = os.environ.get('TASK_MONITOR_FEISHU_WEBHOOK', '')
if webhook:
return {'channels': [{'id': 'environment-feishu', 'type': 'feishu_webhook', 'enabled': True,
'events': events, 'webhook_url': webhook,
'secret': os.environ.get('TASK_MONITOR_FEISHU_SECRET', '')}]}
if os.environ.get('TASK_MONITOR_FEISHU_APP_ID'):
return {'channels': [{'id': 'environment-feishu-app', 'type': 'feishu_app', 'enabled': True,
'events': events, 'app_id': os.environ.get('TASK_MONITOR_FEISHU_APP_ID'),
'app_secret': os.environ.get('TASK_MONITOR_FEISHU_APP_SECRET', ''),
'receive_id': os.environ.get('TASK_MONITOR_FEISHU_RECEIVE_ID', ''),
'receive_id_type': os.environ.get('TASK_MONITOR_FEISHU_RECEIVE_ID_TYPE', 'open_id')}]}
return {'channels': []}
Loading
0