-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathmodels.py
More file actions
184 lines (132 loc) · 4.49 KB
/
models.py
File metadata and controls
184 lines (132 loc) · 4.49 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
"""
Pydantic models for Sentience SDK - matches spec/snapshot.schema.json
"""
from typing import Literal
from pydantic import BaseModel, Field
class BBox(BaseModel):
"""Bounding box coordinates"""
x: float
y: float
width: float
height: float
class Viewport(BaseModel):
"""Viewport dimensions"""
width: float
height: float
class VisualCues(BaseModel):
"""Visual analysis cues"""
is_primary: bool
background_color_name: str | None = None
is_clickable: bool
class Element(BaseModel):
"""Element from snapshot"""
id: int
role: str
text: str | None = None
importance: int
bbox: BBox
visual_cues: VisualCues
in_viewport: bool = True
is_occluded: bool = False
z_index: int = 0
class Snapshot(BaseModel):
"""Snapshot response from extension"""
status: Literal["success", "error"]
timestamp: str | None = None
url: str
viewport: Viewport | None = None
elements: list[Element]
screenshot: str | None = None
screenshot_format: Literal["png", "jpeg"] | None = None
error: str | None = None
requires_license: bool | None = None
def save(self, filepath: str) -> None:
"""Save snapshot as JSON file"""
import json
with open(filepath, "w") as f:
json.dump(self.model_dump(), f, indent=2)
class ActionResult(BaseModel):
"""Result of an action (click, type, press)"""
success: bool
duration_ms: int
outcome: Literal["navigated", "dom_updated", "no_change", "error"] | None = None
url_changed: bool | None = None
snapshot_after: Snapshot | None = None
error: dict | None = None
class WaitResult(BaseModel):
"""Result of wait_for operation"""
found: bool
element: Element | None = None
duration_ms: int
timeout: bool
# ========== Agent Layer Models ==========
class ScreenshotConfig(BaseModel):
"""Screenshot format configuration"""
format: Literal["png", "jpeg"] = "png"
quality: int | None = Field(None, ge=1, le=100) # Only for JPEG (1-100)
class SnapshotFilter(BaseModel):
"""Filter options for snapshot elements"""
min_area: int | None = Field(None, ge=0)
allowed_roles: list[str] | None = None
min_z_index: int | None = None
class SnapshotOptions(BaseModel):
"""
Configuration for snapshot calls.
Matches TypeScript SnapshotOptions interface from sdk-ts/src/snapshot.ts
"""
screenshot: bool | ScreenshotConfig = False # Union type: boolean or config
limit: int = Field(50, ge=1, le=500)
filter: SnapshotFilter | None = None
use_api: bool | None = None # Force API vs extension
save_trace: bool = False # Save raw_elements to JSON for benchmarking/training
trace_path: str | None = None # Path to save trace (default: "trace_{timestamp}.json")
class Config:
arbitrary_types_allowed = True
class AgentActionResult(BaseModel):
"""Result of a single agent action (from agent.act())"""
success: bool
action: Literal["click", "type", "press", "finish", "error"]
goal: str
duration_ms: int
attempt: int
# Optional fields based on action type
element_id: int | None = None
text: str | None = None
key: str | None = None
outcome: Literal["navigated", "dom_updated", "no_change", "error"] | None = None
url_changed: bool | None = None
error: str | None = None
message: str | None = None # For FINISH action
def __getitem__(self, key):
"""
Support dict-style access for backward compatibility.
This allows existing code using result["success"] to continue working.
"""
import warnings
warnings.warn(
f"Dict-style access result['{key}'] is deprecated. Use result.{key} instead.",
DeprecationWarning,
stacklevel=2,
)
return getattr(self, key)
class ActionTokenUsage(BaseModel):
"""Token usage for a single action"""
goal: str
prompt_tokens: int
completion_tokens: int
total_tokens: int
model: str
class TokenStats(BaseModel):
"""Token usage statistics for an agent session"""
total_prompt_tokens: int
total_completion_tokens: int
total_tokens: int
by_action: list[ActionTokenUsage]
class ActionHistory(BaseModel):
"""Single history entry from agent execution"""
goal: str
action: str # The raw action string from LLM
result: dict # Will be AgentActionResult but stored as dict for flexibility
success: bool
attempt: int
duration_ms: int