Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion config/template.json
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,9 @@
"combat_screenshot_interval": 1.0,
"task_hoarding_duration": 0,
"when_task_queue_empty": "goto_main",
"schedule_rule": "Filter"
"schedule_rule": "Filter",
"queue_mode": false,
"queue_idle_threshold": 10
}
},
"restart": {
Expand Down
310 changes: 310 additions & 0 deletions module/config/instance_guard.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,310 @@
# This Python file uses the following encoding: utf-8
# @author handman123
# github https://github.com/runhey
"""跨进程实例看守器。

通过共享 JSON 状态文件协调多个进程的实例执行顺序,确保同一时间
只有一个实例操作模拟器。持有执行权(token)的实例可执行任务,
其他实例排队等待。
"""
import json
import threading
from pathlib import Path
from datetime import datetime

from filelock import FileLock
from module.logger import logger


class _QueueState:
"""共享状态文件的持久化层。

负责 JSON 文件的原子读写和持有者心跳过期判断。
文件锁(FileLock)保证跨进程写入安全。

文件格式:
{
"current": "oas1", # 当前持有执行权的实例名
"timestamp": "2026-07-08 ...", # 持有者心跳时间戳
"queue": ["oas2", "oas3"] # 等待队列
}
"""

STATE_FILE = Path.cwd() / "log" / ".queue_state.json"
STALE_TIMEOUT = 300 # 心跳超时 5 分钟,超过视为持有者崩溃

def __init__(self, config_name: str):
"""Args:
config_name: 配置文件名称,如 'oas1'。
"""
self.config_name = config_name
self._lock = FileLock(str(self.STATE_FILE) + ".lock", timeout=5)

def read(self) -> dict:
"""读取状态文件。

调用方需持有 _lock。文件不存在或损坏时返回空状态。

Returns:
dict: 包含 current、timestamp、queue 的状态字典。
"""
if not self.STATE_FILE.exists():
return {"current": None, "timestamp": None, "queue": []}
try:
with open(self.STATE_FILE, "r", encoding="utf-8") as f:
return json.load(f)
except (json.JSONDecodeError, IOError):
logger.warning("[InstanceGuard] State file corrupted, resetting")
return {"current": None, "timestamp": None, "queue": []}

def write(self, state: dict) -> None:
"""写入状态文件。调用方需持有 _lock。"""
self.STATE_FILE.parent.mkdir(parents=True, exist_ok=True)
with open(self.STATE_FILE, "w", encoding="utf-8") as f:
json.dump(state, f, indent=2, ensure_ascii=False)

def is_stale(self, state: dict) -> bool:
"""判断持有者心跳是否过期。

如果 timestamp 距现在超过 STALE_TIMEOUT 秒,认为持有者崩溃。

Args:
state: 从 read() 获取的状态字典。

Returns:
bool: True 表示心跳过期,可安全接管。
"""
ts_str = state.get("timestamp")
if not ts_str:
return False
try:
ts = datetime.fromisoformat(ts_str)
elapsed = (datetime.now() - ts).total_seconds()
return elapsed > self.STALE_TIMEOUT
except (ValueError, TypeError):
return True

def __enter__(self):
self._lock.acquire()
return self.read()

def __exit__(self, *args):
self._lock.release()


class _Heartbeat:
"""心跳保活线程。

持有 token 期间定期更新 timestamp,防止被误判崩溃。
同时监控 token 是否被其他实例抢占(例如被管理员从状态文件强制接管),
检测到丢失时设置 token_lost 标志通知主线程。
"""

INTERVAL = 180 # 心跳间隔 3 分钟,小于 STALE_TIMEOUT(5min) 留出缓冲

def __init__(self, config_name: str, state: _QueueState):
self._config_name = config_name
self._state = state
self._thread: threading.Thread = None
self._stop_event: threading.Event = None
self._token_lost = threading.Event()

@property
def token_lost(self) -> bool:
"""token 是否被其他实例抢占。"""
return self._token_lost.is_set()

def start(self) -> None:
"""启动心跳 daemon 线程。已运行则跳过。"""
if self._thread and self._thread.is_alive():
return
self._token_lost.clear()
self._stop_event = threading.Event()
self._thread = threading.Thread(target=self._loop, daemon=True)
self._thread.start()
logger.info(
f"[InstanceGuard] Heartbeat started for '{self._config_name}'")

def stop(self) -> None:
"""停止心跳线程。"""
if self._stop_event:
self._stop_event.set()
self._thread = None
self._stop_event = None

def _loop(self) -> None:
"""心跳循环。每 INTERVAL 秒刷新 timestamp。"""
while not self._stop_event.wait(timeout=self.INTERVAL):
try:
with self._state._lock:
state = self._state.read()
if state["current"] == self._config_name:
state["timestamp"] = datetime.now().isoformat()
self._state.write(state)
elif self._config_name not in state.get("queue", []):
# 不在 current 也不在 queue,说明被强制移除了
logger.warning(
f"[InstanceGuard] '{self._config_name}' lost token, "
f"current holder: {state['current']}")
self._token_lost.set()
return
except Exception as e:
logger.error(f"[InstanceGuard] Heartbeat error: {e}")


class InstanceGuard:
"""跨进程实例看守器。

通过共享状态文件协调多个实例的执行顺序,确保同一时间只有一个实例
操作模拟器。集成心跳保活和崩溃检测。

用法:
guard = InstanceGuard(config_name)
if guard.try_acquire():
# 执行业务逻辑
guard.release()
"""

def __init__(self, config_name: str):
"""Args:
config_name: 配置文件名称。
"""
self.config_name = config_name
self._state = _QueueState(config_name)
self._heartbeat = _Heartbeat(config_name, self._state)

@property
def token_lost(self) -> bool:
"""token 是否被其他实例抢占。"""
return self._heartbeat.token_lost

# ==================== 执行权操作 ====================

def try_acquire(self) -> bool:
"""尝试获取执行权。

四种情况:
1. 自己已是持有者 — 刷新 heartbeat,返回 True
2. 持有者心跳过期 — 强制接管,返回 True
3. 无人持有 — 自己成为持有者,启动心跳,返回 True
4. 他人持有 — 加入等待队列,返回 False

Returns:
True: 已获得执行权。
False: 执行权被他人持有,已加入队列。
"""
with self._state._lock:
state = self._state.read()

# 自己已是持有者,刷新心跳
if state["current"] == self.config_name:
state["timestamp"] = datetime.now().isoformat()
self._state.write(state)
self._heartbeat.start()
logger.info(f"[InstanceGuard] '{self.config_name}' already holds token")
return True

# 持有者崩溃,强制接管
if state["current"] and self._state.is_stale(state):
logger.warning(
f"[InstanceGuard] Holder '{state['current']}' "
f"heartbeat stale, taking over")
state["current"] = None
state["timestamp"] = None

# 无人持有,自己成为持有者
if state["current"] is None:
state["current"] = self.config_name
state["timestamp"] = datetime.now().isoformat()
if self.config_name in state["queue"]:
state["queue"].remove(self.config_name)
self._state.write(state)
logger.info(
f"[InstanceGuard] '{self.config_name}' acquired token")
self._heartbeat.start()
return True

# 他人持有,加入等待队列
if self.config_name not in state["queue"]:
state["queue"].append(self.config_name)
self._state.write(state)
logger.info(
f"[InstanceGuard] '{self.config_name}' waiting, "
f"holder: '{state['current']}', "
f"queue: {state['queue']}")
return False

def release(self) -> None:
"""释放执行权。

清空 current 和 timestamp,停止心跳。
等待队列中的实例自行获取。
如果自己不是持有者,静默返回。
"""
with self._state._lock:
state = self._state.read()
if state["current"] != self.config_name:
return

self._heartbeat.stop()
state["current"] = None
state["timestamp"] = None
self._state.write(state)
logger.info(f"[InstanceGuard] '{self.config_name}' released token")

def should_release(self, pending_task: list, waiting_task: list,
idle_threshold_minutes: int = 10) -> bool:
"""判断是否应该在进入空闲前释放执行权。

条件:
1. pending_task 为空(当前无待执行任务)
2. waiting_task 为空,或最早的任务距现在超过 idle_threshold_minutes 分钟

Args:
pending_task: 当前到期的任务列表。
waiting_task: 未来任务列表,按 next_run 升序。
idle_threshold_minutes: 空闲阈值(分钟)。

Returns:
True: 空闲时间足够长,应释放执行权。
False: 即将有任务,应继续持有。
"""
if pending_task:
return False
if not waiting_task:
return True

next_task_time = waiting_task[0].next_run
idle_seconds = (next_task_time - datetime.now()).total_seconds()
return idle_seconds > idle_threshold_minutes * 60

def remove_from_queue(self) -> None:
"""从排队系统中完全移除自己。

进程停止时调用,防止残留状态导致死锁。
如果自己是持有者,将执行权移交给下一个等待者。
"""
with self._state._lock:
state = self._state.read()

if self.config_name in state["queue"]:
state["queue"].remove(self.config_name)
logger.info(
f"[InstanceGuard] '{self.config_name}' removed from queue")

if state["current"] == self.config_name:
self._heartbeat.stop()
if state["queue"]:
next_holder = state["queue"].pop(0)
state["current"] = next_holder
state["timestamp"] = datetime.now().isoformat()
logger.info(
f"[InstanceGuard] '{self.config_name}' removed, "
f"token → '{next_holder}'")
else:
state["current"] = None
state["timestamp"] = None
self._state.write(state)
return

self._state.write(state)
10 changes: 9 additions & 1 deletion module/device/method/adb.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
from module.device.method.minitouch import smooth_path
from module.device.method.utils import (RETRY_TRIES, retry_sleep, remove_prefix, handle_adb_error,
ImageTruncated, PackageNotInstalled)
from module.exception import RequestHumanTakeover, ScriptError
from module.exception import RequestHumanTakeover, ScriptError, EmulatorNotRunningError
from module.logger import logger


Expand All @@ -33,6 +33,14 @@ def retry_wrapper(self, *args, **kwargs):
# Can't handle
except RequestHumanTakeover:
break
# Emulator not running, try to restart it
except EmulatorNotRunningError:
logger.error('Emulator not running, attempting to restart')
if hasattr(self, 'emulator_start'):
self.emulator_start()

def init():
self.adb_reconnect()
# When adb server was killed
except ConnectionResetError as e:
logger.error(e)
Expand Down
1 change: 1 addition & 0 deletions module/server/config_manager.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ def all_script_files() -> list[str]:
# 如果没有脚本文件 则创建一个
ConfigManager.copy(file='oas1', template='template')
result.append('oas1')
result.sort()
return result

@staticmethod
Expand Down
Loading