automation_service.py 53 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320
  1. from __future__ import annotations
  2. import base64
  3. import io
  4. import json
  5. import mimetypes
  6. import re
  7. import sqlite3
  8. import time
  9. import uuid
  10. import zipfile
  11. from pathlib import Path
  12. from typing import Any
  13. import psutil
  14. from fastapi import HTTPException
  15. from . import ai_service, settings_service, windows_automation
  16. from .automation import get_node_definitions, get_node_executor
  17. from .automation.context import WorkflowContext, WorkflowPaused
  18. from .database import DATA_DIR, get_db
  19. from .scanner import now_iso
  20. from .schemas import (
  21. AutomationKeyboardActionRequest,
  22. AutomationMouseActionRequest,
  23. AutomationElementLocateRequest,
  24. AutomationScreenshotCaptureRequest,
  25. AutomationStartProgramRequest,
  26. AutomationTextInputRequest,
  27. AutomationVisionAnalyzeRequest,
  28. AutomationWorkflowRunRequest,
  29. AutomationWorkflowSaveRequest,
  30. AutomationWorkflowPlanRequest,
  31. AutomationWorkflowPlanContinueRequest,
  32. )
  33. AUTOMATION_DIR = DATA_DIR / "automation"
  34. SCREEN_DIR = AUTOMATION_DIR / "screens"
  35. ERROR_DIR = AUTOMATION_DIR / "errors"
  36. RUNTIME_DIR = AUTOMATION_DIR / "runtime"
  37. OPENED_PROCESS_IDS: set[int] = set()
  38. SCREEN_ANALYZE_PROMPT = """请作为 AI 视觉自动化助手分析这张 Windows 屏幕截图,并严格只输出 JSON 对象。
  39. 输出字段:
  40. - interface_name:界面名称,简洁中文。
  41. - description:界面描述,说明当前主要窗口或桌面内容。
  42. - is_windows_desktop:boolean,截图是否处于 Windows 桌面。
  43. - is_browser_webpage:boolean,截图是否为浏览器中的网页。
  44. - elements:可操作元素数组。
  45. 元素字段:
  46. - name:元素名称。
  47. - approximate_location:元素在界面中的大致位置文字描述,例如“窗口右上角”“左侧导航栏中部”“底部任务栏靠左”。不要输出具体坐标或百分比。
  48. 判断规则:
  49. 1. 如果截图位于 Windows 桌面,请识别桌面图标、开始菜单入口、任务栏应用、托盘区域等可操作元素。
  50. 2. 如果不是 Windows 桌面,也就是存在打开的前台窗口或全屏界面,只识别该前台窗口内的可操作元素,不要识别被遮挡的桌面元素。
  51. 3. 不要输出 Markdown,不要解释,只输出 JSON。
  52. """
  53. ELEMENT_LOCATE_PROMPT = """请作为 AI 视觉定位助手,在这张 Windows 屏幕截图中查找一个具体的可操作元素。
  54. 目标元素名称:
  55. {name}
  56. 目标元素大致位置描述:
  57. {approximate_location}
  58. 所在界面描述:
  59. {screen_description}
  60. 请严格只输出 JSON 对象,字段为:
  61. - has_element:boolean,图片中是否能找到该目标元素。
  62. - x_percent:元素中心点 X 相对整张截图宽度的百分比,范围 0-100,可以保留 2 位小数。找不到时为 null。
  63. - y_percent:元素中心点 Y 相对整张截图高度的百分比,范围 0-100,可以保留 2 位小数。找不到时为 null。
  64. - reason:简短中文原因。
  65. 只定位这个目标元素,不要列出其他元素。不要输出 Markdown,不要解释,只输出 JSON。
  66. """
  67. SCREEN_COMPARE_PROMPT = """请作为 AI 视觉自动化校验器判断两张截图是否处于同一个目标界面。
  68. 图片1是当前实际屏幕截图。图片2是数据库中保存的目标界面截图。
  69. 目标界面描述如下:
  70. {description}
  71. 请严格只输出 JSON 对象,字段为:
  72. - is_match:boolean,图片1是否仍然处于目标界面。
  73. - similarity:0 到 1 的数值,表示相似度。
  74. - reason:简短中文原因。
  75. 判断时可以允许小的光标位置、时间、列表内容滚动或轻微刷新差异,但如果前台窗口、网页、弹窗、主要页面或应用已经不同,应返回 false。
  76. """
  77. def ensure_dirs() -> None:
  78. """确保自动化截图、错误截图和运行时目录存在。"""
  79. for path in [screen_dir(), error_dir(), runtime_dir()]:
  80. path.mkdir(parents=True, exist_ok=True)
  81. def screen_dir() -> Path:
  82. """根据系统设置获取已识别界面截图目录。"""
  83. return settings_service.resolve_data_path("automation_screen_path", "automation/screens")
  84. def error_dir() -> Path:
  85. """根据系统设置获取错误截图目录。"""
  86. return settings_service.resolve_data_path("automation_error_path", "automation/errors")
  87. def runtime_dir() -> Path:
  88. """根据系统设置获取临时截图目录。"""
  89. return settings_service.resolve_data_path("automation_runtime_path", "automation/runtime")
  90. def image_to_base64(path: str | Path) -> dict[str, str]:
  91. """读取图片文件并转为 AI 服务可接收的 base64 结构。"""
  92. file_path = stored_path(path)
  93. mime_type = mimetypes.guess_type(file_path.name)[0] or "image/png"
  94. return {
  95. "base64": base64.b64encode(file_path.read_bytes()).decode("ascii"),
  96. "mime_type": mime_type,
  97. }
  98. def json_from_ai(content: str) -> dict[str, Any]:
  99. """从 AI 输出中提取 JSON 对象,兼容模型误加代码块的情况。"""
  100. parsed = json.loads(ai_service.extract_json_text(content))
  101. if not isinstance(parsed, dict):
  102. raise ValueError("AI output must be a JSON object")
  103. return parsed
  104. def take_screenshot_file(folder: Path, prefix: str) -> dict[str, Any]:
  105. """截取当前屏幕并保存为 PNG 文件,同时返回 base64 和分辨率信息。"""
  106. ensure_dirs()
  107. filename = f"{prefix}_{int(time.time() * 1000)}.png"
  108. path = folder / filename
  109. result = windows_automation.take_screenshot(str(path), include_base64=True)
  110. result["path"] = str(path)
  111. result["db_path"] = data_relative_path(path)
  112. return result
  113. def data_relative_path(path: str | Path) -> str:
  114. """把 data 目录下的文件路径转换为数据库保存用的相对路径。"""
  115. file_path = Path(path).resolve()
  116. try:
  117. return file_path.relative_to(DATA_DIR.resolve()).as_posix()
  118. except ValueError:
  119. return str(file_path)
  120. def stored_path(path: str | Path) -> Path:
  121. """把数据库中的相对路径还原成真实文件路径,同时兼容旧的绝对路径。"""
  122. file_path = Path(path)
  123. if file_path.is_absolute():
  124. return file_path
  125. return (DATA_DIR / file_path).resolve()
  126. def resolve_ai_params(
  127. provider_id: int | None,
  128. model_id: int | None,
  129. temperature: float | None,
  130. ) -> tuple[int, int, float]:
  131. """合并请求参数和系统默认 AI 参数。"""
  132. defaults = settings_service.default_ai_params()
  133. resolved_provider = provider_id or defaults.get("provider_id")
  134. resolved_model = model_id or defaults.get("model_id")
  135. resolved_temperature = temperature if temperature is not None else defaults.get("temperature", 0.1)
  136. if not resolved_provider or not resolved_model:
  137. raise HTTPException(status_code=400, detail="AI provider and model are required. Configure system defaults or pass them explicitly.")
  138. return int(resolved_provider), int(resolved_model), float(resolved_temperature)
  139. def capture_screenshot(payload: AutomationScreenshotCaptureRequest) -> dict[str, Any]:
  140. """截取当前屏幕并返回给前端显示,不进行 AI 分析。"""
  141. if payload.save:
  142. screenshot = take_screenshot_file(runtime_dir(), "manual_screenshot")
  143. else:
  144. screenshot = windows_automation.take_screenshot(None, include_base64=True)
  145. screenshot["path"] = None
  146. screenshot["db_path"] = None
  147. return {
  148. "width": screenshot["width"],
  149. "height": screenshot["height"],
  150. "image_base64": screenshot["image_base64"],
  151. "mime_type": screenshot["mime_type"],
  152. "path": screenshot.get("db_path"),
  153. }
  154. def analyze_screen(payload: AutomationVisionAnalyzeRequest) -> dict[str, Any]:
  155. """截图当前屏幕,调用 AI 识别界面和可操作元素,并保存识别结果。"""
  156. provider_id, model_id, temperature = resolve_ai_params(payload.provider_id, payload.model_id, payload.temperature)
  157. screenshot = take_screenshot_file(screen_dir(), "screen")
  158. image = image_to_base64(screenshot["path"])
  159. ai_result = ai_service.chat_with_images(
  160. provider_id,
  161. model_id,
  162. SCREEN_ANALYZE_PROMPT,
  163. [image],
  164. temperature,
  165. )
  166. try:
  167. parsed = json_from_ai(ai_result["content"])
  168. except (json.JSONDecodeError, ValueError) as exc:
  169. raise HTTPException(status_code=502, detail=f"AI vision output is not valid JSON: {exc}") from exc
  170. width = int(screenshot["width"])
  171. height = int(screenshot["height"])
  172. elements = normalize_elements(parsed.get("elements"), width, height)
  173. now = now_iso()
  174. with get_db() as conn:
  175. cursor = conn.execute(
  176. """
  177. INSERT INTO automation_screens (
  178. interface_name, description, image_path, width, height,
  179. is_windows_desktop, is_browser_webpage, raw_ai_json, created_at, updated_at
  180. )
  181. VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
  182. """,
  183. (
  184. str(parsed.get("interface_name") or "未命名界面")[:160],
  185. parsed.get("description"),
  186. screenshot["db_path"],
  187. width,
  188. height,
  189. 1 if bool(parsed.get("is_windows_desktop")) else 0,
  190. 1 if bool(parsed.get("is_browser_webpage")) else 0,
  191. json.dumps(parsed, ensure_ascii=False),
  192. now,
  193. now,
  194. ),
  195. )
  196. screen_id = cursor.lastrowid
  197. for index, element in enumerate(elements, start=1):
  198. conn.execute(
  199. """
  200. INSERT INTO automation_screen_elements (
  201. screen_id, element_index, name, x_percent, y_percent, x, y,
  202. approximate_location, is_located, raw_json, created_at
  203. )
  204. VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
  205. """,
  206. (
  207. screen_id,
  208. index,
  209. element["name"],
  210. element["x_percent"],
  211. element["y_percent"],
  212. element["x"],
  213. element["y"],
  214. element["approximate_location"],
  215. 1 if element["is_located"] else 0,
  216. json.dumps(element.get("raw") or element, ensure_ascii=False),
  217. now,
  218. ),
  219. )
  220. detail = get_screen(screen_id)
  221. detail["image_base64"] = screenshot["image_base64"]
  222. detail["mime_type"] = screenshot["mime_type"]
  223. detail["ai_raw_content"] = ai_result["content"]
  224. return detail
  225. def normalize_elements(raw_elements: Any, width: int, height: int) -> list[dict[str, Any]]:
  226. """规范化 AI 返回的可操作元素清单;初始分析阶段不要求坐标。"""
  227. if not isinstance(raw_elements, list):
  228. return []
  229. result = []
  230. for item in raw_elements:
  231. if not isinstance(item, dict):
  232. continue
  233. name = str(item.get("name") or f"元素 {len(result) + 1}")[:160]
  234. approximate_location = str(item.get("approximate_location") or item.get("location") or "未定位")[:300]
  235. x_percent = normalize_percent(item.get("x_percent")) if item.get("x_percent") is not None else 0.0
  236. y_percent = normalize_percent(item.get("y_percent")) if item.get("y_percent") is not None else 0.0
  237. is_located = item.get("x_percent") is not None and item.get("y_percent") is not None
  238. x = round(width * x_percent / 100)
  239. y = round(height * y_percent / 100)
  240. result.append(
  241. {
  242. "name": name,
  243. "x_percent": x_percent,
  244. "y_percent": y_percent,
  245. "x": max(0, min(width - 1, x)),
  246. "y": max(0, min(height - 1, y)),
  247. "approximate_location": approximate_location,
  248. "is_located": is_located,
  249. "raw": item,
  250. }
  251. )
  252. return result
  253. def locate_element(screen_id: int, element_id: int, payload: AutomationElementLocateRequest) -> dict[str, Any]:
  254. """针对单个可操作元素调用 AI 精确定位,并更新该元素的像素坐标。"""
  255. provider_id, model_id, temperature = resolve_ai_params(payload.provider_id, payload.model_id, payload.temperature)
  256. screen = get_screen(screen_id)
  257. element = next((item for item in screen.get("elements", []) if item["id"] == element_id), None)
  258. if not element:
  259. raise HTTPException(status_code=404, detail="Automation screen element not found")
  260. prompt = (
  261. ELEMENT_LOCATE_PROMPT
  262. .replace("{name}", element.get("name") or "")
  263. .replace("{approximate_location}", element.get("approximate_location") or "")
  264. .replace("{screen_description}", screen.get("description") or screen.get("interface_name") or "")
  265. )
  266. ai_result = ai_service.chat_with_images(
  267. provider_id,
  268. model_id,
  269. prompt,
  270. [image_to_base64(screen["image_path"])],
  271. temperature,
  272. )
  273. try:
  274. parsed = json_from_ai(ai_result["content"])
  275. except (json.JSONDecodeError, ValueError) as exc:
  276. raise HTTPException(status_code=502, detail=f"AI locate output is not valid JSON: {exc}") from exc
  277. if not bool(parsed.get("has_element")) or parsed.get("x_percent") is None or parsed.get("y_percent") is None:
  278. return {"located": False, "element": element, "ai_result": parsed, "ai_raw_content": ai_result["content"]}
  279. x_percent = normalize_percent(parsed.get("x_percent"))
  280. y_percent = normalize_percent(parsed.get("y_percent"))
  281. x = max(0, min(int(screen["width"]) - 1, round(int(screen["width"]) * x_percent / 100)))
  282. y = max(0, min(int(screen["height"]) - 1, round(int(screen["height"]) * y_percent / 100)))
  283. raw = {**parsed, "previous": element.get("raw_json")}
  284. with get_db() as conn:
  285. conn.execute(
  286. """
  287. UPDATE automation_screen_elements
  288. SET x_percent = ?, y_percent = ?, x = ?, y = ?, is_located = 1, raw_json = ?
  289. WHERE id = ? AND screen_id = ?
  290. """,
  291. (x_percent, y_percent, x, y, json.dumps(raw, ensure_ascii=False), element_id, screen_id),
  292. )
  293. updated = get_screen(screen_id, include_image=True)
  294. updated_element = next(item for item in updated["elements"] if item["id"] == element_id)
  295. return {
  296. "located": True,
  297. "element": updated_element,
  298. "screen": updated,
  299. "ai_result": parsed,
  300. "ai_raw_content": ai_result["content"],
  301. }
  302. def normalize_percent(value: Any) -> float:
  303. """规范化百分比数值,兼容模型偶尔输出 0-1 小数的情况。"""
  304. try:
  305. number = float(value)
  306. except (TypeError, ValueError):
  307. return 0.0
  308. if 0 <= number <= 1:
  309. number *= 100
  310. return max(0.0, min(100.0, round(number, 2)))
  311. def list_screens(page: int, page_size: int) -> dict[str, Any]:
  312. """分页查询已识别界面列表。"""
  313. offset = (page - 1) * page_size
  314. with get_db() as conn:
  315. total = conn.execute("SELECT COUNT(*) AS total FROM automation_screens").fetchone()["total"]
  316. rows = conn.execute(
  317. """
  318. SELECT s.*, COUNT(e.id) AS element_count
  319. FROM automation_screens s
  320. LEFT JOIN automation_screen_elements e ON e.screen_id = s.id
  321. GROUP BY s.id
  322. ORDER BY s.created_at DESC
  323. LIMIT ? OFFSET ?
  324. """,
  325. (page_size, offset),
  326. ).fetchall()
  327. return {"items": [public_screen(row) for row in rows], "total": total, "page": page, "page_size": page_size}
  328. def get_screen(screen_id: int, include_image: bool = False) -> dict[str, Any]:
  329. """读取单个已识别界面的详情和可操作元素。"""
  330. with get_db() as conn:
  331. screen = conn.execute("SELECT * FROM automation_screens WHERE id = ?", (screen_id,)).fetchone()
  332. if not screen:
  333. raise HTTPException(status_code=404, detail="Automation screen not found")
  334. elements = conn.execute(
  335. "SELECT * FROM automation_screen_elements WHERE screen_id = ? ORDER BY element_index ASC",
  336. (screen_id,),
  337. ).fetchall()
  338. item = public_screen(screen)
  339. item["elements"] = [public_element(row) for row in elements]
  340. if include_image and stored_path(item["image_path"]).exists():
  341. image = image_to_base64(item["image_path"])
  342. item["image_base64"] = image["base64"]
  343. item["mime_type"] = image["mime_type"]
  344. return item
  345. def delete_screen(screen_id: int) -> dict[str, Any]:
  346. """删除已识别界面记录,图片文件保留用于审计。"""
  347. with get_db() as conn:
  348. cursor = conn.execute("DELETE FROM automation_screens WHERE id = ?", (screen_id,))
  349. if cursor.rowcount == 0:
  350. raise HTTPException(status_code=404, detail="Automation screen not found")
  351. return {"deleted": cursor.rowcount}
  352. def public_screen(row: dict[str, Any]) -> dict[str, Any]:
  353. """把数据库中的界面行转换为接口返回格式。"""
  354. item = dict(row)
  355. item["is_windows_desktop"] = bool(item.get("is_windows_desktop"))
  356. item["is_browser_webpage"] = bool(item.get("is_browser_webpage"))
  357. return item
  358. def public_element(row: dict[str, Any]) -> dict[str, Any]:
  359. """把数据库中的元素行转换为接口返回格式。"""
  360. item = dict(row)
  361. item["is_located"] = bool(item.get("is_located"))
  362. return item
  363. def process_snapshot() -> dict[int, dict[str, Any]]:
  364. """获取当前进程快照,只用于自动化动作前后对比,不写入进程扫描表。"""
  365. snapshot: dict[int, dict[str, Any]] = {}
  366. for proc in psutil.process_iter(["pid", "name", "exe"]):
  367. try:
  368. snapshot[int(proc.info["pid"])] = {
  369. "pid": int(proc.info["pid"]),
  370. "name": proc.info.get("name"),
  371. "exe": proc.info.get("exe"),
  372. }
  373. except (psutil.Error, OSError, TypeError, ValueError):
  374. continue
  375. return snapshot
  376. def diff_new_processes(before: dict[int, dict[str, Any]], after: dict[int, dict[str, Any]]) -> list[dict[str, Any]]:
  377. """比较动作前后的进程快照,找出本次自动化动作新增的进程。"""
  378. new_items = [after[pid] for pid in sorted(set(after) - set(before))]
  379. OPENED_PROCESS_IDS.update(item["pid"] for item in new_items)
  380. return new_items
  381. def validate_screen_before_action(
  382. screen_id: int | None,
  383. provider_id: int | None,
  384. model_id: int | None,
  385. temperature: float,
  386. action_type: str,
  387. workflow_id: int | None = None,
  388. node_id: int | None = None,
  389. ) -> dict[str, Any] | None:
  390. """如果动作绑定了界面 ID,则先用 AI 判断当前屏幕是否仍处于目标界面。"""
  391. if screen_id is None:
  392. return None
  393. provider_id, model_id, temperature = resolve_ai_params(provider_id, model_id, temperature)
  394. target = get_screen(screen_id)
  395. current = take_screenshot_file(error_dir(), "compare_current")
  396. prompt = SCREEN_COMPARE_PROMPT.replace("{description}", target.get("description") or target.get("interface_name") or "")
  397. ai_result = ai_service.chat_with_images(
  398. provider_id,
  399. model_id,
  400. prompt,
  401. [image_to_base64(current["path"]), image_to_base64(target["image_path"])],
  402. temperature,
  403. )
  404. try:
  405. parsed = json_from_ai(ai_result["content"])
  406. except (json.JSONDecodeError, ValueError) as exc:
  407. raise HTTPException(status_code=502, detail=f"AI compare output is not valid JSON: {exc}") from exc
  408. is_match = bool(parsed.get("is_match"))
  409. similarity = safe_float(parsed.get("similarity"))
  410. if not is_match:
  411. error = record_error(
  412. action_type=action_type,
  413. message=str(parsed.get("reason") or "界面对比失败,当前屏幕不是目标界面"),
  414. screen_id=screen_id,
  415. workflow_id=workflow_id,
  416. node_id=node_id,
  417. similarity=similarity,
  418. expected_image_path=target["image_path"],
  419. actual_image_path=current["db_path"],
  420. compare_result=parsed,
  421. )
  422. raise HTTPException(status_code=409, detail={"message": error["message"], "error": error})
  423. return parsed
  424. def safe_float(value: Any) -> float | None:
  425. """安全转换浮点数。"""
  426. try:
  427. return float(value)
  428. except (TypeError, ValueError):
  429. return None
  430. def record_error(
  431. action_type: str,
  432. message: str,
  433. screen_id: int | None = None,
  434. workflow_id: int | None = None,
  435. node_id: int | None = None,
  436. similarity: float | None = None,
  437. expected_image_path: str | None = None,
  438. actual_image_path: str | None = None,
  439. compare_result: dict[str, Any] | None = None,
  440. ) -> dict[str, Any]:
  441. """保存自动化错误记录,便于在错误记录菜单中回看。"""
  442. now = now_iso()
  443. with get_db() as conn:
  444. cursor = conn.execute(
  445. """
  446. INSERT INTO automation_errors (
  447. workflow_id, node_id, screen_id, action_type, message, similarity,
  448. expected_image_path, actual_image_path, compare_result_json, created_at
  449. )
  450. VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
  451. """,
  452. (
  453. workflow_id,
  454. node_id,
  455. screen_id,
  456. action_type,
  457. message,
  458. similarity,
  459. expected_image_path,
  460. actual_image_path,
  461. json.dumps(compare_result or {}, ensure_ascii=False),
  462. now,
  463. ),
  464. )
  465. row = conn.execute("SELECT * FROM automation_errors WHERE id = ?", (cursor.lastrowid,)).fetchone()
  466. return public_error(row)
  467. def execute_mouse_action(payload: AutomationMouseActionRequest) -> dict[str, Any]:
  468. """执行鼠标点击类动作,并记录动作前后新增进程。"""
  469. before = process_snapshot()
  470. compare = validate_screen_before_action(
  471. payload.screen_id,
  472. payload.provider_id,
  473. payload.model_id,
  474. payload.temperature,
  475. f"mouse_{payload.mouse_action}",
  476. payload.workflow_id,
  477. payload.node_id,
  478. )
  479. action_map = {"click": "click", "double_click": "double_click", "right_click": "right_click"}
  480. result = windows_automation.mouse_action(action_map[payload.mouse_action], x=payload.x, y=payload.y)
  481. time.sleep(0.5)
  482. new_processes = diff_new_processes(before, process_snapshot())
  483. return {"result": result, "compare": compare, "new_processes": new_processes}
  484. def execute_keyboard_action(payload: AutomationKeyboardActionRequest) -> dict[str, Any]:
  485. """执行键盘组合键动作,并记录动作前后新增进程。"""
  486. before = process_snapshot()
  487. compare = validate_screen_before_action(
  488. payload.screen_id,
  489. payload.provider_id,
  490. payload.model_id,
  491. payload.temperature,
  492. "keyboard",
  493. payload.workflow_id,
  494. payload.node_id,
  495. )
  496. result = windows_automation.keyboard_action("hotkey" if len(payload.keys) > 1 else "press", key=payload.keys[0], keys=payload.keys)
  497. time.sleep(0.5)
  498. new_processes = diff_new_processes(before, process_snapshot())
  499. return {"result": result, "compare": compare, "new_processes": new_processes}
  500. def execute_text_input(payload: AutomationTextInputRequest) -> dict[str, Any]:
  501. """通过剪贴板粘贴文本,避免直接模拟按键时中文输入不稳定。"""
  502. before = process_snapshot()
  503. compare = validate_screen_before_action(
  504. payload.screen_id,
  505. payload.provider_id,
  506. payload.model_id,
  507. payload.temperature,
  508. "text_input",
  509. payload.workflow_id,
  510. payload.node_id,
  511. )
  512. try:
  513. import pyperclip
  514. except ImportError as exc:
  515. raise HTTPException(status_code=500, detail="pyperclip is not installed") from exc
  516. pyperclip.copy(payload.text)
  517. result = windows_automation.keyboard_action("hotkey", keys=["ctrl", "v"])
  518. time.sleep(0.5)
  519. new_processes = diff_new_processes(before, process_snapshot())
  520. return {"result": result, "compare": compare, "new_processes": new_processes}
  521. def execute_start_program(payload: AutomationStartProgramRequest) -> dict[str, Any]:
  522. """启动程序,并把动作后新增的进程记录为本次自动化打开的程序。"""
  523. before = process_snapshot()
  524. compare = validate_screen_before_action(
  525. payload.screen_id,
  526. payload.provider_id,
  527. payload.model_id,
  528. payload.temperature,
  529. "start_program",
  530. payload.workflow_id,
  531. payload.node_id,
  532. )
  533. result = windows_automation.start_program(payload.command, payload.cwd, payload.shell)
  534. time.sleep(1)
  535. new_processes = diff_new_processes(before, process_snapshot())
  536. if result.get("pid"):
  537. OPENED_PROCESS_IDS.add(int(result["pid"]))
  538. return {"result": result, "compare": compare, "new_processes": new_processes}
  539. def close_opened_programs(pids: list[int] | None = None) -> dict[str, Any]:
  540. """关闭本次自动化过程中记录的新进程。"""
  541. targets = sorted(set(pids or list(OPENED_PROCESS_IDS)))
  542. closed = []
  543. for pid in targets:
  544. try:
  545. closed.append(windows_automation.stop_program(pid=pid))
  546. OPENED_PROCESS_IDS.discard(pid)
  547. except HTTPException as exc:
  548. closed.append({"pid": pid, "error": exc.detail})
  549. return {"action": "close_opened_programs", "items": closed}
  550. def save_workflow(payload: AutomationWorkflowSaveRequest) -> dict[str, Any]:
  551. """保存 workflow/v1 工作流图。"""
  552. now = now_iso()
  553. workflow_json = normalize_workflow_payload(payload)
  554. workflow_key = normalize_workflow_key(payload.workflow_key)
  555. try:
  556. with get_db() as conn:
  557. cursor = conn.execute(
  558. """
  559. INSERT INTO automation_workflows (workflow_key, name, description, raw_json, created_at, updated_at)
  560. VALUES (?, ?, ?, ?, ?, ?)
  561. """,
  562. (workflow_key, payload.name.strip(), payload.description, json.dumps(workflow_json, ensure_ascii=False), now, now),
  563. )
  564. workflow_id = cursor.lastrowid
  565. conn.execute("DELETE FROM automation_workflow_nodes WHERE workflow_id = ?", (workflow_id,))
  566. except sqlite3.IntegrityError as exc:
  567. raise HTTPException(status_code=409, detail="Workflow key already exists") from exc
  568. return get_workflow(workflow_id)
  569. def update_workflow(workflow_id: int, payload: AutomationWorkflowSaveRequest) -> dict[str, Any]:
  570. """更新 workflow/v1 工作流图。"""
  571. now = now_iso()
  572. workflow_json = normalize_workflow_payload(payload)
  573. workflow_key = normalize_workflow_key(payload.workflow_key)
  574. try:
  575. with get_db() as conn:
  576. existing = conn.execute("SELECT id FROM automation_workflows WHERE id = ?", (workflow_id,)).fetchone()
  577. if not existing:
  578. raise HTTPException(status_code=404, detail="Automation workflow not found")
  579. conn.execute(
  580. """
  581. UPDATE automation_workflows
  582. SET workflow_key = ?, name = ?, description = ?, raw_json = ?, updated_at = ?
  583. WHERE id = ?
  584. """,
  585. (workflow_key, payload.name.strip(), payload.description, json.dumps(workflow_json, ensure_ascii=False), now, workflow_id),
  586. )
  587. conn.execute("DELETE FROM automation_workflow_nodes WHERE workflow_id = ?", (workflow_id,))
  588. except sqlite3.IntegrityError as exc:
  589. raise HTTPException(status_code=409, detail="Workflow key already exists") from exc
  590. return get_workflow(workflow_id)
  591. def import_workflow(payload: AutomationWorkflowSaveRequest, conflict_strategy: str) -> dict[str, Any]:
  592. """导入 workflow/v1;replace 模式按 workflow_key 覆盖目标设备中的同名工作流。"""
  593. workflow_key = normalize_workflow_key(payload.workflow_key)
  594. if conflict_strategy == "replace" and workflow_key:
  595. with get_db() as conn:
  596. existing = conn.execute(
  597. "SELECT id FROM automation_workflows WHERE workflow_key = ?",
  598. (workflow_key,),
  599. ).fetchone()
  600. if existing:
  601. return update_workflow(int(existing["id"]), payload)
  602. return save_workflow(payload)
  603. def export_workflows_zip() -> bytes:
  604. """把全部 workflow 打成 ZIP,便于在多台设备间批量迁移。"""
  605. with get_db() as conn:
  606. rows = conn.execute(
  607. """
  608. SELECT *
  609. FROM automation_workflows
  610. ORDER BY workflow_key ASC, name ASC, id ASC
  611. """
  612. ).fetchall()
  613. buffer = io.BytesIO()
  614. exported_items: list[dict[str, Any]] = []
  615. with zipfile.ZipFile(buffer, "w", compression=zipfile.ZIP_DEFLATED) as archive:
  616. for row in rows:
  617. workflow = workflow_export_payload(workflow_to_public(row))
  618. key = normalize_workflow_key(str(workflow.get("workflow_key") or "")) or f"workflow-{row['id']}"
  619. filename = f"workflows/{safe_zip_name(key)}.workflow.json"
  620. archive.writestr(filename, json.dumps(workflow, ensure_ascii=False, indent=2))
  621. exported_items.append({"workflow_key": workflow.get("workflow_key"), "name": workflow.get("name"), "path": filename})
  622. archive.writestr(
  623. "manifest.json",
  624. json.dumps(
  625. {
  626. "schema_version": "workflow-zip/v1",
  627. "exported_at": now_iso(),
  628. "count": len(exported_items),
  629. "items": exported_items,
  630. },
  631. ensure_ascii=False,
  632. indent=2,
  633. ),
  634. )
  635. return buffer.getvalue()
  636. def import_workflows_zip(content: bytes) -> dict[str, Any]:
  637. """从 ZIP 批量导入 workflow;遇到重复 workflow_key 时跳过,不覆盖本机已有配置。"""
  638. if not content:
  639. raise HTTPException(status_code=400, detail="zip content is required")
  640. created: list[dict[str, Any]] = []
  641. skipped: list[dict[str, Any]] = []
  642. failed: list[dict[str, Any]] = []
  643. try:
  644. archive = zipfile.ZipFile(io.BytesIO(content))
  645. except zipfile.BadZipFile as exc:
  646. raise HTTPException(status_code=400, detail="Invalid workflow zip file") from exc
  647. with archive:
  648. names = [name for name in archive.namelist() if name.lower().endswith(".json") and Path(name).name != "manifest.json"]
  649. for name in names:
  650. try:
  651. with archive.open(name) as file:
  652. raw = json.loads(file.read().decode("utf-8"))
  653. payload = AutomationWorkflowSaveRequest.model_validate(raw)
  654. workflow_key = normalize_workflow_key(payload.workflow_key)
  655. if workflow_key and workflow_key_exists(workflow_key):
  656. skipped.append({"path": name, "workflow_key": workflow_key, "reason": "workflow_key already exists"})
  657. continue
  658. saved = save_workflow(payload)
  659. created.append({"path": name, "id": saved["id"], "workflow_key": saved.get("workflow_key"), "name": saved.get("name")})
  660. except Exception as exc:
  661. failed.append({"path": name, "error": str(exc)})
  662. return {
  663. "created_count": len(created),
  664. "skipped_count": len(skipped),
  665. "failed_count": len(failed),
  666. "created": created,
  667. "skipped": skipped,
  668. "failed": failed,
  669. }
  670. def workflow_key_exists(workflow_key: str) -> bool:
  671. """检查目标设备中是否已存在同名 workflow key。"""
  672. with get_db() as conn:
  673. row = conn.execute("SELECT id FROM automation_workflows WHERE workflow_key = ?", (workflow_key,)).fetchone()
  674. return bool(row)
  675. def safe_zip_name(value: str) -> str:
  676. """生成安全的 ZIP 文件名片段,避免不同系统路径规则不一致。"""
  677. name = re.sub(r"[^A-Za-z0-9_.-]+", "-", value).strip(".-")
  678. return name or "workflow"
  679. def workflow_export_payload(workflow: dict[str, Any]) -> dict[str, Any]:
  680. """提取可迁移的 workflow/v1 字段,不包含本机数据库 ID 和时间戳。"""
  681. return {
  682. key: workflow.get(key)
  683. for key in [
  684. "schema_version",
  685. "workflow_key",
  686. "name",
  687. "description",
  688. "variables",
  689. "settings",
  690. "nodes",
  691. "edges",
  692. ]
  693. }
  694. def normalize_workflow_payload(payload: AutomationWorkflowSaveRequest) -> dict[str, Any]:
  695. """把请求模型转换为持久化的 workflow/v1 JSON。"""
  696. # 排除 Pydantic 为可选引用字段补出的 null,保证导入后再导出仍保持简洁稳定。
  697. workflow_json = payload.model_dump(exclude_none=True)
  698. workflow_json["schema_version"] = "workflow/v1"
  699. workflow_json["workflow_key"] = normalize_workflow_key(payload.workflow_key)
  700. workflow_json["name"] = payload.name.strip()
  701. workflow_json.setdefault("variables", {})
  702. workflow_json.setdefault("settings", {})
  703. workflow_json.setdefault("edges", [])
  704. return workflow_json
  705. def normalize_workflow_key(value: str | None) -> str | None:
  706. key = (value or "").strip()
  707. if not key:
  708. return None
  709. if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_-]*", key):
  710. raise HTTPException(status_code=400, detail="Workflow key can only contain letters, numbers, underscores, and hyphens")
  711. return key
  712. def list_workflows(page: int, page_size: int) -> dict[str, Any]:
  713. """分页查询自动化工作流列表。"""
  714. offset = (page - 1) * page_size
  715. with get_db() as conn:
  716. total = conn.execute("SELECT COUNT(*) AS total FROM automation_workflows").fetchone()["total"]
  717. rows = conn.execute(
  718. """
  719. SELECT *
  720. FROM automation_workflows
  721. ORDER BY updated_at DESC
  722. LIMIT ? OFFSET ?
  723. """,
  724. (page_size, offset),
  725. ).fetchall()
  726. return {"items": [workflow_summary(row) for row in rows], "total": total, "page": page, "page_size": page_size}
  727. def web_search_workflow_template() -> dict[str, Any]:
  728. """返回可供外部程序调用的 AI 多轮网页研究 workflow。"""
  729. return {
  730. "schema_version": "workflow/v1",
  731. "workflow_key": "ai-web-research",
  732. "name": "AI 多轮网页搜索研究",
  733. "description": "AI 制定搜索计划,使用视觉浏览器多轮研究,并按调用方提供的 JSON Schema 返回结果。",
  734. "variables": {
  735. "objective": {
  736. "type": "string",
  737. "default": "",
  738. "description": "要搜索和研究的目标",
  739. },
  740. "output_schema": {
  741. "type": "object",
  742. "default": {
  743. "type": "object",
  744. "required": ["summary", "facts"],
  745. "properties": {
  746. "summary": {"type": "string"},
  747. "facts": {"type": "array", "items": {"type": "string"}},
  748. },
  749. "additionalProperties": False,
  750. },
  751. "description": "最终 data 字段必须满足的 JSON Schema",
  752. },
  753. "constraints": {
  754. "type": "object",
  755. "default": {"language": "zh-CN", "min_sources": 1},
  756. "description": "来源、语言、时间范围等研究约束",
  757. },
  758. "max_attempts": {
  759. "type": "number",
  760. "default": 3,
  761. "description": "AI 评估未达标时最多执行的搜索轮数",
  762. },
  763. },
  764. "settings": {
  765. "max_steps": 10,
  766. "default_timeout_ms": 1800000,
  767. "on_unhandled_error": "pause_for_user",
  768. "return": {"node_id": "research"},
  769. },
  770. "nodes": [
  771. {
  772. "id": "start",
  773. "type": "flow.start",
  774. "title": "开始",
  775. "position": {"x": 80, "y": 180},
  776. "params": {},
  777. "inputs": {},
  778. },
  779. {
  780. "id": "research",
  781. "type": "research.ai_web_research",
  782. "title": "AI 规划并循环研究",
  783. "position": {"x": 360, "y": 180},
  784. "params": {
  785. "search_engine": "bing",
  786. "browser": "edge",
  787. "max_search_pages": 2,
  788. "result_count": 2,
  789. "detail_max_pages": 2,
  790. },
  791. "inputs": {
  792. "objective": {"source": "variable", "name": "objective"},
  793. "output_schema": {"source": "variable", "name": "output_schema"},
  794. "constraints": {"source": "variable", "name": "constraints"},
  795. "max_attempts": {"source": "variable", "name": "max_attempts"},
  796. },
  797. },
  798. {
  799. "id": "end",
  800. "type": "flow.end",
  801. "title": "结束",
  802. "position": {"x": 680, "y": 180},
  803. "params": {},
  804. "inputs": {},
  805. },
  806. ],
  807. "edges": [
  808. {
  809. "id": "start_to_research",
  810. "kind": "control",
  811. "source": "start",
  812. "source_port": "next",
  813. "target": "research",
  814. "target_port": "run",
  815. },
  816. {
  817. "id": "research_to_end",
  818. "kind": "control",
  819. "source": "research",
  820. "source_port": "success",
  821. "target": "end",
  822. "target_port": "run",
  823. },
  824. {
  825. "id": "partial_to_end",
  826. "kind": "control",
  827. "source": "research",
  828. "source_port": "partial",
  829. "target": "end",
  830. "target_port": "run",
  831. },
  832. ],
  833. }
  834. def create_web_search_workflow() -> dict[str, Any]:
  835. """创建完善版研究 workflow;已存在时直接返回,避免重复覆盖用户调整。"""
  836. template = AutomationWorkflowSaveRequest.model_validate(web_search_workflow_template())
  837. try:
  838. return get_workflow_by_key(str(template.workflow_key))
  839. except HTTPException as exc:
  840. if exc.status_code != 404:
  841. raise
  842. return save_workflow(template)
  843. def get_workflow(workflow_id: int) -> dict[str, Any]:
  844. """读取 workflow/v1 工作流详情。"""
  845. with get_db() as conn:
  846. workflow = conn.execute("SELECT * FROM automation_workflows WHERE id = ?", (workflow_id,)).fetchone()
  847. if not workflow:
  848. raise HTTPException(status_code=404, detail="Automation workflow not found")
  849. item = workflow_to_public(workflow)
  850. return item
  851. def get_workflow_by_key(workflow_key: str) -> dict[str, Any]:
  852. """按稳定 key 读取 workflow/v1 工作流详情。"""
  853. key = normalize_workflow_key(workflow_key)
  854. if not key:
  855. raise HTTPException(status_code=400, detail="Workflow key is required")
  856. with get_db() as conn:
  857. workflow = conn.execute("SELECT * FROM automation_workflows WHERE workflow_key = ?", (key,)).fetchone()
  858. if not workflow:
  859. raise HTTPException(status_code=404, detail="Automation workflow not found")
  860. return workflow_to_public(workflow)
  861. def export_workflow_by_key(workflow_key: str) -> dict[str, Any]:
  862. """导出不含数据库 ID 和时间字段的可迁移 workflow/v1 JSON。"""
  863. workflow = get_workflow_by_key(workflow_key)
  864. return workflow_export_payload(workflow)
  865. def delete_workflow(workflow_id: int) -> dict[str, Any]:
  866. """删除工作流及其节点。"""
  867. with get_db() as conn:
  868. cursor = conn.execute("DELETE FROM automation_workflows WHERE id = ?", (workflow_id,))
  869. if cursor.rowcount == 0:
  870. raise HTTPException(status_code=404, detail="Automation workflow not found")
  871. return {"deleted": cursor.rowcount}
  872. def execute_workflow(workflow: dict[str, Any], payload: AutomationWorkflowRunRequest) -> dict[str, Any]:
  873. """同步执行 workflow 快照,仅供后台任务 worker 调用。"""
  874. workflow_id = workflow.get("id")
  875. defaults = settings_service.default_ai_params()
  876. provider_id = payload.provider_id or defaults.get("provider_id")
  877. model_id = payload.model_id or defaults.get("model_id")
  878. temperature = payload.temperature if payload.temperature is not None else defaults.get("temperature", 0.1)
  879. context = WorkflowContext(
  880. workflow_id=workflow_id,
  881. provider_id=provider_id,
  882. model_id=model_id,
  883. temperature=float(temperature),
  884. variables=workflow_variables(workflow, payload.variables),
  885. )
  886. nodes = workflow.get("nodes") or []
  887. edges = workflow.get("edges") or []
  888. node_map = {node["id"]: node for node in nodes}
  889. start_id = first_workflow_node_id(nodes, edges)
  890. if not start_id:
  891. return {"workflow_id": workflow_id, "status": "SUCCESS", "results": []}
  892. results: list[dict[str, Any]] = []
  893. current_id: str | None = start_id
  894. visited_steps = 0
  895. max_steps = int(workflow.get("settings", {}).get("max_steps") or 100)
  896. while current_id and visited_steps < max_steps:
  897. visited_steps += 1
  898. node = node_map.get(current_id)
  899. if not node:
  900. return {"workflow_id": workflow_id, "status": "FAILED", "detail": f"Missing node: {current_id}", "results": results}
  901. try:
  902. resolved_inputs = resolve_node_inputs(node, edges, context)
  903. outputs = execute_workflow_node(node, resolved_inputs, context)
  904. context.outputs[node["id"]] = outputs
  905. results.append({"node_id": node["id"], "node": node, "status": "SUCCESS", "inputs": resolved_inputs, "outputs": outputs})
  906. if node.get("type") == "flow.end":
  907. return {"workflow_id": workflow_id, "status": "SUCCESS", "results": results, "outputs": context.outputs}
  908. next_port = str(outputs.get("next_port") or "success")
  909. current_id = next_control_node_id(node["id"], next_port, edges) or next_control_node_id(node["id"], "next", edges)
  910. except HTTPException as exc:
  911. failure = {
  912. "node_id": node.get("id"),
  913. "node": node,
  914. "status": "FAILED",
  915. "detail": exc.detail,
  916. "artifacts": capture_failure_artifacts(context),
  917. }
  918. results.append(failure)
  919. return {"workflow_id": workflow_id, "status": "FAILED", "failed": failure, "results": results}
  920. except WorkflowPaused as exc:
  921. paused = {"node_id": node.get("id"), "node": node, "status": "PAUSED", "detail": exc.payload}
  922. results.append(paused)
  923. return {"workflow_id": workflow_id, "status": "PAUSED", "paused": paused, "results": results}
  924. except Exception as exc:
  925. failure = {
  926. "node_id": node.get("id"),
  927. "node": node,
  928. "status": "FAILED",
  929. "detail": str(exc),
  930. "artifacts": capture_failure_artifacts(context),
  931. }
  932. results.append(failure)
  933. return {"workflow_id": workflow_id, "status": "FAILED", "failed": failure, "results": results}
  934. if visited_steps >= max_steps:
  935. return {"workflow_id": workflow_id, "status": "FAILED", "detail": f"Workflow exceeded max_steps={max_steps}", "results": results}
  936. return {"workflow_id": workflow_id, "status": "SUCCESS", "results": results, "outputs": context.outputs}
  937. def run_workflow(workflow_id: int, payload: AutomationWorkflowRunRequest) -> dict[str, Any]:
  938. """兼容内部调用;HTTP 接口不再按数据库 ID 同步执行。"""
  939. return execute_workflow(get_workflow(workflow_id), payload)
  940. def workflow_return_data(workflow: dict[str, Any], result: dict[str, Any]) -> Any:
  941. """按 workflow.settings.return 提取给外部调用方的最终数据。"""
  942. return_config = (workflow.get("settings") or {}).get("return") or {}
  943. node_id = str(return_config.get("node_id") or "").strip()
  944. output_name = str(return_config.get("output") or "").strip()
  945. if not node_id:
  946. return result
  947. node_outputs = (result.get("outputs") or {}).get(node_id)
  948. if not output_name:
  949. return node_outputs
  950. if isinstance(node_outputs, dict):
  951. return node_outputs.get(output_name)
  952. return None
  953. def execute_workflow_node(
  954. node: dict[str, Any],
  955. inputs: dict[str, Any],
  956. context: WorkflowContext,
  957. ) -> dict[str, Any]:
  958. """通过节点注册表执行 workflow/v1 节点。"""
  959. try:
  960. executor = get_node_executor(str(node.get("type") or ""))
  961. except KeyError as exc:
  962. raise HTTPException(status_code=400, detail=str(exc)) from exc
  963. return executor(node, inputs, context)
  964. def capture_failure_artifacts(context: WorkflowContext) -> dict[str, Any]:
  965. """工作流失败时尽量保存一张当前屏幕截图,供前端询问用户。"""
  966. artifacts: dict[str, Any] = {}
  967. try:
  968. screenshot = take_screenshot_file(error_dir(), "workflow_failure")
  969. except Exception as exc:
  970. artifacts["screenshot_error"] = str(exc)
  971. return artifacts
  972. artifacts["screenshot_path"] = screenshot.get("db_path") or screenshot.get("path")
  973. artifacts["width"] = screenshot.get("width")
  974. artifacts["height"] = screenshot.get("height")
  975. context.runtime["current_screenshot_path"] = artifacts["screenshot_path"]
  976. return artifacts
  977. def workflow_to_public(row: dict[str, Any]) -> dict[str, Any]:
  978. item = workflow_summary(row)
  979. workflow_json = parse_workflow_json(row.get("raw_json"))
  980. item.update(workflow_json)
  981. item["id"] = row["id"]
  982. item["workflow_key"] = row.get("workflow_key") or workflow_json.get("workflow_key")
  983. item["created_at"] = row["created_at"]
  984. item["updated_at"] = row["updated_at"]
  985. item["node_count"] = len(item.get("nodes") or [])
  986. item["edge_count"] = len(item.get("edges") or [])
  987. return item
  988. def workflow_summary(row: dict[str, Any]) -> dict[str, Any]:
  989. workflow_json = parse_workflow_json(row.get("raw_json"))
  990. return {
  991. "id": row["id"],
  992. "workflow_key": row.get("workflow_key") or workflow_json.get("workflow_key"),
  993. "name": row["name"],
  994. "description": row.get("description"),
  995. "schema_version": workflow_json.get("schema_version") or "workflow/v1",
  996. "node_count": len(workflow_json.get("nodes") or []),
  997. "edge_count": len(workflow_json.get("edges") or []),
  998. "created_at": row.get("created_at"),
  999. "updated_at": row.get("updated_at"),
  1000. }
  1001. def parse_workflow_json(raw_json: str | None) -> dict[str, Any]:
  1002. if not raw_json:
  1003. return empty_workflow_json()
  1004. try:
  1005. parsed = json.loads(raw_json)
  1006. except json.JSONDecodeError:
  1007. return empty_workflow_json()
  1008. if not isinstance(parsed, dict):
  1009. return empty_workflow_json()
  1010. parsed.setdefault("schema_version", "workflow/v1")
  1011. parsed.setdefault("variables", {})
  1012. parsed.setdefault("settings", {})
  1013. parsed.setdefault("nodes", [])
  1014. parsed.setdefault("edges", [])
  1015. return parsed
  1016. def empty_workflow_json() -> dict[str, Any]:
  1017. return {"schema_version": "workflow/v1", "variables": {}, "settings": {}, "nodes": [], "edges": []}
  1018. def workflow_variables(workflow: dict[str, Any], overrides: dict[str, Any]) -> dict[str, Any]:
  1019. variables: dict[str, Any] = {}
  1020. for name, definition in (workflow.get("variables") or {}).items():
  1021. if isinstance(definition, dict):
  1022. variables[name] = definition.get("default")
  1023. else:
  1024. variables[name] = definition
  1025. variables.update(overrides or {})
  1026. return variables
  1027. def first_workflow_node_id(nodes: list[dict[str, Any]], edges: list[dict[str, Any]]) -> str | None:
  1028. if not nodes:
  1029. return None
  1030. for node in nodes:
  1031. if node.get("type") == "flow.start":
  1032. return node.get("id")
  1033. targeted = {edge.get("target") for edge in edges if edge.get("kind") == "control"}
  1034. for node in nodes:
  1035. if node.get("id") not in targeted:
  1036. return node.get("id")
  1037. return nodes[0].get("id")
  1038. def next_control_node_id(source_id: str, source_port: str, edges: list[dict[str, Any]]) -> str | None:
  1039. fallback = None
  1040. for edge in edges:
  1041. if edge.get("kind") != "control" or edge.get("source") != source_id:
  1042. continue
  1043. if edge.get("source_port") == source_port:
  1044. return edge.get("target")
  1045. if edge.get("source_port") in (None, "", "success", "next") and fallback is None:
  1046. fallback = edge.get("target")
  1047. return fallback
  1048. def resolve_node_inputs(node: dict[str, Any], edges: list[dict[str, Any]], context: WorkflowContext) -> dict[str, Any]:
  1049. resolved: dict[str, Any] = {}
  1050. for key, value in (node.get("inputs") or {}).items():
  1051. resolved[key] = resolve_value_ref(value, context)
  1052. for edge in edges:
  1053. if edge.get("kind") != "data" or edge.get("target") != node.get("id"):
  1054. continue
  1055. source_outputs = context.outputs.get(edge.get("source") or "", {})
  1056. resolved[edge.get("target_port") or "value"] = source_outputs.get(edge.get("source_port") or "value")
  1057. return resolved
  1058. def resolve_value_ref(value: Any, context: WorkflowContext) -> Any:
  1059. if not isinstance(value, dict) or "source" not in value:
  1060. return value
  1061. source = value.get("source")
  1062. if source == "literal":
  1063. return value.get("value")
  1064. if source == "variable":
  1065. return context.variables.get(value.get("name") or "")
  1066. if source == "node_output":
  1067. return context.outputs.get(value.get("node_id") or "", {}).get(value.get("output") or "")
  1068. if source == "runtime":
  1069. return context.runtime.get(value.get("name") or "")
  1070. return None
  1071. def list_workflow_node_definitions() -> dict[str, Any]:
  1072. """返回前端可用于生成节点库和属性表单的节点定义。"""
  1073. return {"schema_version": "workflow/v1", "items": get_node_definitions()}
  1074. def plan_workflow(payload: AutomationWorkflowPlanRequest) -> dict[str, Any]:
  1075. """让 AI 根据用户需求和节点定义生成 workflow/v1 草稿。"""
  1076. provider_id, model_id, temperature = resolve_ai_params(payload.provider_id, payload.model_id, payload.temperature)
  1077. prompt = build_workflow_plan_prompt(payload.requirement)
  1078. ai_result = ai_service.chat(provider_id, model_id, prompt, temperature)
  1079. try:
  1080. parsed = json_from_ai(ai_result["content"])
  1081. except (json.JSONDecodeError, ValueError) as exc:
  1082. raise HTTPException(status_code=502, detail=f"AI workflow plan output is not valid JSON: {exc}") from exc
  1083. session_id = str(uuid.uuid4())
  1084. return {"session_id": session_id, "plan": parsed, "ai_raw_content": ai_result["content"]}
  1085. def continue_workflow_plan(payload: AutomationWorkflowPlanContinueRequest) -> dict[str, Any]:
  1086. """继续一次 AI 工作流规划对话,返回新的草稿建议。"""
  1087. provider_id, model_id, temperature = resolve_ai_params(payload.provider_id, payload.model_id, payload.temperature)
  1088. prompt = build_workflow_plan_prompt(payload.user_message, session_id=payload.session_id)
  1089. ai_result = ai_service.chat(provider_id, model_id, prompt, temperature)
  1090. try:
  1091. parsed = json_from_ai(ai_result["content"])
  1092. except (json.JSONDecodeError, ValueError) as exc:
  1093. raise HTTPException(status_code=502, detail=f"AI workflow plan output is not valid JSON: {exc}") from exc
  1094. return {"session_id": payload.session_id, "plan": parsed, "ai_raw_content": ai_result["content"]}
  1095. def build_workflow_plan_prompt(requirement: str, session_id: str | None = None) -> str:
  1096. node_defs = json.dumps(get_node_definitions(), ensure_ascii=False, indent=2)
  1097. return f"""请作为 Windows 自动化工作流规划器,根据用户需求生成 workflow/v1 JSON 草稿。
  1098. 要求:
  1099. 1. 只能使用节点定义列表中的 type。
  1100. 2. 输出严格 JSON 对象,不要 Markdown。
  1101. 3. JSON 字段必须包含 summary、questions、workflow。
  1102. 4. workflow 必须包含 schema_version、name、description、variables、settings、nodes、edges。
  1103. 5. 不确定的坐标或界面状态,优先添加 human.ask_user 节点或 screen.screenshot 节点。
  1104. 6. 控制流连线 kind 使用 control,数据连线 kind 使用 data。
  1105. 会话 ID:{session_id or "new"}
  1106. 用户需求:
  1107. {requirement}
  1108. 可用节点定义:
  1109. {node_defs}
  1110. """
  1111. def list_errors(page: int, page_size: int) -> dict[str, Any]:
  1112. """分页查询自动化错误记录。"""
  1113. offset = (page - 1) * page_size
  1114. with get_db() as conn:
  1115. total = conn.execute("SELECT COUNT(*) AS total FROM automation_errors").fetchone()["total"]
  1116. rows = conn.execute(
  1117. """
  1118. SELECT e.*, s.interface_name
  1119. FROM automation_errors e
  1120. LEFT JOIN automation_screens s ON s.id = e.screen_id
  1121. ORDER BY e.created_at DESC
  1122. LIMIT ? OFFSET ?
  1123. """,
  1124. (page_size, offset),
  1125. ).fetchall()
  1126. return {"items": [public_error(row) for row in rows], "total": total, "page": page, "page_size": page_size}
  1127. def get_error(error_id: int, include_images: bool = False) -> dict[str, Any]:
  1128. """读取单条自动化错误详情,可附带目标截图和实际截图。"""
  1129. with get_db() as conn:
  1130. row = conn.execute("SELECT * FROM automation_errors WHERE id = ?", (error_id,)).fetchone()
  1131. if not row:
  1132. raise HTTPException(status_code=404, detail="Automation error not found")
  1133. item = public_error(row)
  1134. if include_images:
  1135. for key in ["expected_image_path", "actual_image_path"]:
  1136. path = item.get(key)
  1137. if path and stored_path(path).exists():
  1138. image = image_to_base64(path)
  1139. item[key.replace("_path", "_base64")] = image["base64"]
  1140. item[key.replace("_path", "_mime_type")] = image["mime_type"]
  1141. return item
  1142. def public_error(row: dict[str, Any]) -> dict[str, Any]:
  1143. """把错误记录行转换为接口返回格式。"""
  1144. item = dict(row)
  1145. try:
  1146. item["compare_result"] = json.loads(item.pop("compare_result_json") or "{}")
  1147. except json.JSONDecodeError:
  1148. item["compare_result"] = {}
  1149. return item