From c4f8dbeaaf3abf86c93272844299987a36752425 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Mon, 17 Aug 2026 16:49:37 +0000 Subject: [PATCH] Verify deterministic movement from observations Co-authored-by: codegirl007 --- agentbox/internal/agent/deterministic.go | 52 +++++++++++++++++-- agentbox/internal/agent/deterministic_test.go | 44 +++++++++++++++- agentbox/internal/agent/openai.go | 12 ++--- .../internal/environment/dockerx11/docker.go | 37 +++++++++++-- .../environment/dockerx11/docker_test.go | 18 +++++++ agentbox/internal/environment/environment.go | 12 ++--- agentbox/internal/runtime/runtime.go | 2 +- 7 files changed, 154 insertions(+), 23 deletions(-) create mode 100644 agentbox/internal/environment/dockerx11/docker_test.go diff --git a/agentbox/internal/agent/deterministic.go b/agentbox/internal/agent/deterministic.go index fcd2557..3efab10 100644 --- a/agentbox/internal/agent/deterministic.go +++ b/agentbox/internal/agent/deterministic.go @@ -1,14 +1,18 @@ package agent import ( + "bytes" "context" - "time" + "fmt" + "image/png" "agentbox/internal/environment" ) type Deterministic struct { - next int + next int + initialX float64 + hasInitial bool } func (a *Deterministic) Name() string { @@ -19,19 +23,35 @@ func (a *Deterministic) NextAction( _ context.Context, _ string, _ []Step, - _ Observation, + observation Observation, ) (Decision, error) { var decision Decision switch a.next { case 0: + x, err := greenCentroidX(observation.Screenshot) + if err != nil { + return Decision{}, fmt.Errorf("inspect initial screenshot: %w", err) + } + a.initialX = x + a.hasInitial = true action := environment.InputAction{Type: environment.KeyDown, Key: "RIGHT"} decision = Decision{Reason: "Press RIGHT to start moving.", Action: &action} case 1: - action := environment.InputAction{Type: environment.Wait, Duration: time.Second} + action := environment.InputAction{Type: environment.Wait, DurationMS: 1000} decision = Decision{Reason: "Keep RIGHT held for one second.", Action: &action} case 2: + x, err := greenCentroidX(observation.Screenshot) + if err != nil { + return Decision{}, fmt.Errorf("inspect moved screenshot: %w", err) + } + if !a.hasInitial || x-a.initialX < 100 { + return Decision{}, fmt.Errorf("visual verification failed: square moved %.1f pixels right, want at least 100", x-a.initialX) + } action := environment.InputAction{Type: environment.KeyUp, Key: "RIGHT"} - decision = Decision{Reason: "Release RIGHT after the movement.", Action: &action} + decision = Decision{ + Reason: fmt.Sprintf("The square moved %.1f pixels right; release RIGHT.", x-a.initialX), + Action: &action, + } default: decision = Decision{Reason: "The movement sequence is complete.", Done: true} } @@ -39,4 +59,26 @@ func (a *Deterministic) NextAction( return decision, nil } +func greenCentroidX(screenshot []byte) (float64, error) { + image, err := png.Decode(bytes.NewReader(screenshot)) + if err != nil { + return 0, err + } + var sumX, count uint64 + bounds := image.Bounds() + for y := bounds.Min.Y; y < bounds.Max.Y; y++ { + for x := bounds.Min.X; x < bounds.Max.X; x++ { + red, green, blue, _ := image.At(x, y).RGBA() + if green > 0xc000 && red < 0x4000 && blue < 0x8000 { + sumX += uint64(x) + count++ + } + } + } + if count < 1000 { + return 0, fmt.Errorf("found only %d green square pixels", count) + } + return float64(sumX) / float64(count), nil +} + var _ Agent = (*Deterministic)(nil) diff --git a/agentbox/internal/agent/deterministic_test.go b/agentbox/internal/agent/deterministic_test.go index f9ca40f..fb20e20 100644 --- a/agentbox/internal/agent/deterministic_test.go +++ b/agentbox/internal/agent/deterministic_test.go @@ -1,7 +1,11 @@ package agent import ( + "bytes" "context" + "image" + "image/color" + "image/png" "testing" "agentbox/internal/environment" @@ -19,7 +23,13 @@ func TestDeterministicSequence(t *testing.T) { {"", true}, } for index, expected := range want { - decision, err := controller.NextAction(context.Background(), "", nil, Observation{}) + x := 20 + if index >= 2 { + x = 140 + } + decision, err := controller.NextAction(context.Background(), "", nil, Observation{ + Screenshot: screenshotWithSquare(t, x), + }) if err != nil { t.Fatalf("step %d: %v", index, err) } @@ -31,3 +41,35 @@ func TestDeterministicSequence(t *testing.T) { } } } + +func TestDeterministicRejectsMissingMovement(t *testing.T) { + controller := &Deterministic{} + for step := 0; step < 2; step++ { + if _, err := controller.NextAction(context.Background(), "", nil, Observation{ + Screenshot: screenshotWithSquare(t, 20), + }); err != nil { + t.Fatal(err) + } + } + if _, err := controller.NextAction(context.Background(), "", nil, Observation{ + Screenshot: screenshotWithSquare(t, 20), + }); err == nil { + t.Fatal("movement verification error = nil") + } +} + +func screenshotWithSquare(t *testing.T, startX int) []byte { + t.Helper() + img := image.NewRGBA(image.Rect(0, 0, 240, 100)) + green := color.RGBA{G: 255, B: 102, A: 255} + for y := 20; y < 60; y++ { + for x := startX; x < startX+40; x++ { + img.Set(x, y, green) + } + } + var output bytes.Buffer + if err := png.Encode(&output, img); err != nil { + t.Fatal(err) + } + return output.Bytes() +} diff --git a/agentbox/internal/agent/openai.go b/agentbox/internal/agent/openai.go index ae91ad6..a115771 100644 --- a/agentbox/internal/agent/openai.go +++ b/agentbox/internal/agent/openai.go @@ -225,12 +225,12 @@ func parseModelDecision(text string) (Decision, error) { return Decision{}, errors.New("model wait must be between 0 and 5000 ms") } action := environment.InputAction{ - Type: result.Action.Type, - Key: result.Action.Key, - X: result.Action.X, - Y: result.Action.Y, - Button: result.Action.Button, - Duration: time.Duration(result.Action.DurationMS) * time.Millisecond, + Type: result.Action.Type, + Key: result.Action.Key, + X: result.Action.X, + Y: result.Action.Y, + Button: result.Action.Button, + DurationMS: result.Action.DurationMS, } switch action.Type { case environment.KeyDown, environment.KeyUp, environment.MouseMove, diff --git a/agentbox/internal/environment/dockerx11/docker.go b/agentbox/internal/environment/dockerx11/docker.go index 3272c48..a248466 100644 --- a/agentbox/internal/environment/dockerx11/docker.go +++ b/agentbox/internal/environment/dockerx11/docker.go @@ -157,12 +157,12 @@ func (e *Environment) SendInput(ctx context.Context, action environment.InputAct if action.Key == "" { return errors.New("key_down requires key") } - args = []string{"keydown", action.Key} + args = []string{"keydown", x11Key(action.Key)} case environment.KeyUp: if action.Key == "" { return errors.New("key_up requires key") } - args = []string{"keyup", action.Key} + args = []string{"keyup", x11Key(action.Key)} case environment.MouseMove: args = []string{"mousemove", strconv.Itoa(action.X), strconv.Itoa(action.Y)} case environment.MouseDown: @@ -170,13 +170,13 @@ func (e *Environment) SendInput(ctx context.Context, action environment.InputAct case environment.MouseUp: args = []string{"mouseup", strconv.Itoa(action.Button)} case environment.Wait: - if action.Duration < 0 { + if action.DurationMS < 0 { return errors.New("wait duration cannot be negative") } select { case <-ctx.Done(): return ctx.Err() - case <-time.After(action.Duration): + case <-time.After(time.Duration(action.DurationMS) * time.Millisecond): return nil } default: @@ -263,4 +263,33 @@ func shellQuote(value string) string { return "'" + strings.ReplaceAll(value, "'", "'\"'\"'") + "'" } +func x11Key(key string) string { + switch strings.ToUpper(key) { + case "LEFT": + return "Left" + case "RIGHT": + return "Right" + case "UP": + return "Up" + case "DOWN": + return "Down" + case "ENTER", "RETURN": + return "Return" + case "ESC", "ESCAPE": + return "Escape" + case "SPACE": + return "space" + case "TAB": + return "Tab" + case "BACKSPACE": + return "BackSpace" + case "DELETE": + return "Delete" + } + if len(key) == 1 { + return strings.ToLower(key) + } + return key +} + var _ environment.Environment = (*Environment)(nil) diff --git a/agentbox/internal/environment/dockerx11/docker_test.go b/agentbox/internal/environment/dockerx11/docker_test.go new file mode 100644 index 0000000..4239a40 --- /dev/null +++ b/agentbox/internal/environment/dockerx11/docker_test.go @@ -0,0 +1,18 @@ +package dockerx11 + +import "testing" + +func TestX11KeyTranslatesLogicalNames(t *testing.T) { + tests := map[string]string{ + "RIGHT": "Right", + "LEFT": "Left", + "ENTER": "Return", + "ESCAPE": "Escape", + "A": "a", + } + for input, want := range tests { + if got := x11Key(input); got != want { + t.Errorf("x11Key(%q) = %q, want %q", input, got, want) + } + } +} diff --git a/agentbox/internal/environment/environment.go b/agentbox/internal/environment/environment.go index d7051aa..77c50ab 100644 --- a/agentbox/internal/environment/environment.go +++ b/agentbox/internal/environment/environment.go @@ -24,12 +24,12 @@ const ( ) type InputAction struct { - Type InputType `json:"type"` - Key string `json:"key,omitempty"` - X int `json:"x,omitempty"` - Y int `json:"y,omitempty"` - Button int `json:"button,omitempty"` - Duration time.Duration `json:"duration,omitempty"` + Type InputType `json:"type"` + Key string `json:"key,omitempty"` + X int `json:"x,omitempty"` + Y int `json:"y,omitempty"` + Button int `json:"button,omitempty"` + DurationMS int `json:"duration_ms,omitempty"` } type LogEntry struct { diff --git a/agentbox/internal/runtime/runtime.go b/agentbox/internal/runtime/runtime.go index f28ac04..3708449 100644 --- a/agentbox/internal/runtime/runtime.go +++ b/agentbox/internal/runtime/runtime.go @@ -142,7 +142,7 @@ func actionDetail(action environment.InputAction) string { case environment.MouseDown, environment.MouseUp: return fmt.Sprintf(" %d", action.Button) case environment.Wait: - return " " + action.Duration.String() + return fmt.Sprintf(" %dms", action.DurationMS) default: return "" }