Add runtime abstraction and agent loop
Co-authored-by: codegirl007 <codegirl-007@users.noreply.github.com>
This commit is contained in:
Cursor Agent
co-authored by
codegirl007
codegirl007
parent
54dc5658c6
commit
94146fcae2
@@ -0,0 +1,39 @@
|
||||
package agent
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"agentbox/internal/environment"
|
||||
)
|
||||
|
||||
type Observation struct {
|
||||
Screenshot []byte
|
||||
Timestamp time.Time
|
||||
Logs []environment.LogEntry
|
||||
PreviousActions []environment.InputAction
|
||||
}
|
||||
|
||||
type Step struct {
|
||||
Number int
|
||||
Timestamp time.Time
|
||||
ScreenshotPath string
|
||||
Message string
|
||||
Action *environment.InputAction
|
||||
}
|
||||
|
||||
type Decision struct {
|
||||
Reason string
|
||||
Action *environment.InputAction
|
||||
Done bool
|
||||
}
|
||||
|
||||
type Agent interface {
|
||||
Name() string
|
||||
NextAction(
|
||||
ctx context.Context,
|
||||
task string,
|
||||
history []Step,
|
||||
observation Observation,
|
||||
) (Decision, error)
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
package agent
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"agentbox/internal/environment"
|
||||
)
|
||||
|
||||
type Deterministic struct {
|
||||
next int
|
||||
}
|
||||
|
||||
func (a *Deterministic) Name() string {
|
||||
return "deterministic-right"
|
||||
}
|
||||
|
||||
func (a *Deterministic) NextAction(
|
||||
_ context.Context,
|
||||
_ string,
|
||||
_ []Step,
|
||||
_ Observation,
|
||||
) (Decision, error) {
|
||||
var decision Decision
|
||||
switch a.next {
|
||||
case 0:
|
||||
action := environment.InputAction{Type: environment.KeyDown, Key: "RIGHT"}
|
||||
decision = Decision{Reason: "Press RIGHT to start moving.", Action: &action}
|
||||
case 1:
|
||||
action := environment.InputAction{Type: environment.Wait, Duration: time.Second}
|
||||
decision = Decision{Reason: "Keep RIGHT held for one second.", Action: &action}
|
||||
case 2:
|
||||
action := environment.InputAction{Type: environment.KeyUp, Key: "RIGHT"}
|
||||
decision = Decision{Reason: "Release RIGHT after the movement.", Action: &action}
|
||||
default:
|
||||
decision = Decision{Reason: "The movement sequence is complete.", Done: true}
|
||||
}
|
||||
a.next++
|
||||
return decision, nil
|
||||
}
|
||||
|
||||
var _ Agent = (*Deterministic)(nil)
|
||||
@@ -0,0 +1,33 @@
|
||||
package agent
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"agentbox/internal/environment"
|
||||
)
|
||||
|
||||
func TestDeterministicSequence(t *testing.T) {
|
||||
controller := &Deterministic{}
|
||||
want := []struct {
|
||||
action environment.InputType
|
||||
done bool
|
||||
}{
|
||||
{environment.KeyDown, false},
|
||||
{environment.Wait, false},
|
||||
{environment.KeyUp, false},
|
||||
{"", true},
|
||||
}
|
||||
for index, expected := range want {
|
||||
decision, err := controller.NextAction(context.Background(), "", nil, Observation{})
|
||||
if err != nil {
|
||||
t.Fatalf("step %d: %v", index, err)
|
||||
}
|
||||
if decision.Done != expected.done {
|
||||
t.Fatalf("step %d done = %v, want %v", index, decision.Done, expected.done)
|
||||
}
|
||||
if expected.action != "" && (decision.Action == nil || decision.Action.Type != expected.action) {
|
||||
t.Fatalf("step %d action = %#v, want %s", index, decision.Action, expected.action)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,244 @@
|
||||
package agent
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"agentbox/internal/environment"
|
||||
)
|
||||
|
||||
type OpenAIConfig struct {
|
||||
APIKey string
|
||||
Model string
|
||||
BaseURL string
|
||||
Client *http.Client
|
||||
}
|
||||
|
||||
type OpenAI struct {
|
||||
config OpenAIConfig
|
||||
}
|
||||
|
||||
func NewOpenAI(config OpenAIConfig) (*OpenAI, error) {
|
||||
if config.APIKey == "" {
|
||||
return nil, errors.New("OPENAI_API_KEY is required for --agent openai")
|
||||
}
|
||||
if config.Model == "" {
|
||||
config.Model = "gpt-5"
|
||||
}
|
||||
if config.BaseURL == "" {
|
||||
config.BaseURL = "https://api.openai.com/v1"
|
||||
}
|
||||
if config.Client == nil {
|
||||
config.Client = &http.Client{Timeout: 90 * time.Second}
|
||||
}
|
||||
return &OpenAI{config: config}, nil
|
||||
}
|
||||
|
||||
func (a *OpenAI) Name() string {
|
||||
return "openai:" + a.config.Model
|
||||
}
|
||||
|
||||
func (a *OpenAI) NextAction(
|
||||
ctx context.Context,
|
||||
task string,
|
||||
history []Step,
|
||||
observation Observation,
|
||||
) (Decision, error) {
|
||||
prompt, err := modelPrompt(task, history, observation)
|
||||
if err != nil {
|
||||
return Decision{}, err
|
||||
}
|
||||
requestBody := map[string]any{
|
||||
"model": a.config.Model,
|
||||
"input": []any{
|
||||
map[string]any{
|
||||
"role": "user",
|
||||
"content": []any{
|
||||
map[string]any{"type": "input_text", "text": prompt},
|
||||
map[string]any{
|
||||
"type": "input_image",
|
||||
"image_url": "data:image/png;base64," +
|
||||
base64.StdEncoding.EncodeToString(observation.Screenshot),
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
"text": map[string]any{
|
||||
"format": map[string]any{
|
||||
"type": "json_schema",
|
||||
"name": "agentbox_action",
|
||||
"strict": true,
|
||||
"schema": decisionSchema(),
|
||||
},
|
||||
},
|
||||
}
|
||||
body, err := json.Marshal(requestBody)
|
||||
if err != nil {
|
||||
return Decision{}, err
|
||||
}
|
||||
request, err := http.NewRequestWithContext(ctx, http.MethodPost,
|
||||
strings.TrimRight(a.config.BaseURL, "/")+"/responses", bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return Decision{}, err
|
||||
}
|
||||
request.Header.Set("Authorization", "Bearer "+a.config.APIKey)
|
||||
request.Header.Set("Content-Type", "application/json")
|
||||
|
||||
response, err := a.config.Client.Do(request)
|
||||
if err != nil {
|
||||
return Decision{}, err
|
||||
}
|
||||
defer response.Body.Close()
|
||||
responseBody, err := io.ReadAll(io.LimitReader(response.Body, 4<<20))
|
||||
if err != nil {
|
||||
return Decision{}, err
|
||||
}
|
||||
if response.StatusCode < 200 || response.StatusCode >= 300 {
|
||||
return Decision{}, fmt.Errorf("OpenAI response %s: %s",
|
||||
response.Status, strings.TrimSpace(string(responseBody)))
|
||||
}
|
||||
text, err := responseText(responseBody)
|
||||
if err != nil {
|
||||
return Decision{}, err
|
||||
}
|
||||
return parseModelDecision(text)
|
||||
}
|
||||
|
||||
func modelPrompt(task string, history []Step, observation Observation) (string, error) {
|
||||
contextData := struct {
|
||||
Task string `json:"task"`
|
||||
History []Step `json:"history"`
|
||||
RecentLogs []environment.LogEntry `json:"recent_logs"`
|
||||
PreviousActions []environment.InputAction `json:"previous_actions"`
|
||||
}{
|
||||
Task: task,
|
||||
History: history,
|
||||
RecentLogs: observation.Logs,
|
||||
PreviousActions: observation.PreviousActions,
|
||||
}
|
||||
data, err := json.Marshal(contextData)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return "You control an interactive Linux application from screenshots. " +
|
||||
"Choose exactly one safe input action, or mark done when the task is complete. " +
|
||||
"Use X11 key names such as RIGHT, Return, or Escape. Keep waits under 5000 ms.\n" +
|
||||
string(data), nil
|
||||
}
|
||||
|
||||
func decisionSchema() map[string]any {
|
||||
actionProperties := map[string]any{
|
||||
"type": map[string]any{
|
||||
"type": "string",
|
||||
"enum": []string{
|
||||
string(environment.KeyDown), string(environment.KeyUp),
|
||||
string(environment.MouseMove), string(environment.MouseDown),
|
||||
string(environment.MouseUp), string(environment.Wait),
|
||||
},
|
||||
},
|
||||
"key": map[string]any{"type": "string"},
|
||||
"x": map[string]any{"type": "integer"},
|
||||
"y": map[string]any{"type": "integer"},
|
||||
"button": map[string]any{"type": "integer"},
|
||||
"duration_ms": map[string]any{"type": "integer"},
|
||||
}
|
||||
return map[string]any{
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"properties": map[string]any{
|
||||
"reason": map[string]any{"type": "string"},
|
||||
"done": map[string]any{"type": "boolean"},
|
||||
"action": map[string]any{
|
||||
"anyOf": []any{
|
||||
map[string]any{
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"properties": actionProperties,
|
||||
"required": []string{
|
||||
"type", "key", "x", "y", "button", "duration_ms",
|
||||
},
|
||||
},
|
||||
map[string]any{"type": "null"},
|
||||
},
|
||||
},
|
||||
},
|
||||
"required": []string{"reason", "done", "action"},
|
||||
}
|
||||
}
|
||||
|
||||
func responseText(data []byte) (string, error) {
|
||||
var response struct {
|
||||
Output []struct {
|
||||
Content []struct {
|
||||
Type string `json:"type"`
|
||||
Text string `json:"text"`
|
||||
} `json:"content"`
|
||||
} `json:"output"`
|
||||
}
|
||||
if err := json.Unmarshal(data, &response); err != nil {
|
||||
return "", fmt.Errorf("decode OpenAI response: %w", err)
|
||||
}
|
||||
for _, output := range response.Output {
|
||||
for _, content := range output.Content {
|
||||
if content.Type == "output_text" && content.Text != "" {
|
||||
return content.Text, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
return "", errors.New("OpenAI response contained no output_text")
|
||||
}
|
||||
|
||||
func parseModelDecision(text string) (Decision, error) {
|
||||
var result struct {
|
||||
Reason string `json:"reason"`
|
||||
Done bool `json:"done"`
|
||||
Action *struct {
|
||||
Type environment.InputType `json:"type"`
|
||||
Key string `json:"key"`
|
||||
X int `json:"x"`
|
||||
Y int `json:"y"`
|
||||
Button int `json:"button"`
|
||||
DurationMS int `json:"duration_ms"`
|
||||
} `json:"action"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(text), &result); err != nil {
|
||||
return Decision{}, fmt.Errorf("decode model decision: %w", err)
|
||||
}
|
||||
if result.Reason == "" {
|
||||
return Decision{}, errors.New("model decision requires reason")
|
||||
}
|
||||
if result.Done {
|
||||
return Decision{Reason: result.Reason, Done: true}, nil
|
||||
}
|
||||
if result.Action == nil {
|
||||
return Decision{}, errors.New("model decision requires action when not done")
|
||||
}
|
||||
if result.Action.DurationMS < 0 || result.Action.DurationMS > 5000 {
|
||||
return Decision{}, errors.New("model wait must be between 0 and 5000 ms")
|
||||
}
|
||||
action := environment.InputAction{
|
||||
Type: result.Action.Type,
|
||||
Key: result.Action.Key,
|
||||
X: result.Action.X,
|
||||
Y: result.Action.Y,
|
||||
Button: result.Action.Button,
|
||||
Duration: time.Duration(result.Action.DurationMS) * time.Millisecond,
|
||||
}
|
||||
switch action.Type {
|
||||
case environment.KeyDown, environment.KeyUp, environment.MouseMove,
|
||||
environment.MouseDown, environment.MouseUp, environment.Wait:
|
||||
default:
|
||||
return Decision{}, fmt.Errorf("model returned unsupported action %q", action.Type)
|
||||
}
|
||||
return Decision{Reason: result.Reason, Action: &action}, nil
|
||||
}
|
||||
|
||||
var _ Agent = (*OpenAI)(nil)
|
||||
@@ -0,0 +1,59 @@
|
||||
package agent
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"agentbox/internal/environment"
|
||||
)
|
||||
|
||||
func TestOpenAISendsScreenshotAndParsesDecision(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) {
|
||||
if request.URL.Path != "/responses" {
|
||||
t.Errorf("path = %q, want /responses", request.URL.Path)
|
||||
}
|
||||
if request.Header.Get("Authorization") != "Bearer test-key" {
|
||||
t.Errorf("missing bearer token")
|
||||
}
|
||||
body, err := io.ReadAll(request.Body)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(string(body), "data:image/png;base64,AQID") {
|
||||
t.Errorf("request does not contain screenshot data URL: %s", body)
|
||||
}
|
||||
response.Header().Set("Content-Type", "application/json")
|
||||
_, _ = io.WriteString(response, `{
|
||||
"output":[{"content":[{"type":"output_text","text":"{\"reason\":\"move right\",\"done\":false,\"action\":{\"type\":\"key_down\",\"key\":\"RIGHT\",\"x\":0,\"y\":0,\"button\":0,\"duration_ms\":0}}"}]}]
|
||||
}`)
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
controller, err := NewOpenAI(OpenAIConfig{
|
||||
APIKey: "test-key", Model: "test-model", BaseURL: server.URL, Client: server.Client(),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
decision, err := controller.NextAction(context.Background(), "move", nil, Observation{
|
||||
Screenshot: []byte{1, 2, 3},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if decision.Action == nil || decision.Action.Type != environment.KeyDown ||
|
||||
decision.Action.Key != "RIGHT" {
|
||||
t.Fatalf("decision = %#v", decision)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecisionSchemaIsJSONSerializable(t *testing.T) {
|
||||
if _, err := json.Marshal(decisionSchema()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user