Verify deterministic movement from observations

Co-authored-by: codegirl007 <codegirl-007@users.noreply.github.com>

agentbox/internal/agent/deterministic.go

@@ -1,14 +1,18 @@
 package agent
 
 import (
+	"bytes"
 	"context"
-	"time"
+	"fmt"
+	"image/png"
 
 	"agentbox/internal/environment"
 )
 
 type Deterministic struct {
-	next int
+	next       int
+	initialX   float64
+	hasInitial bool
 }
 
 func (a *Deterministic) Name() string {
@@ -19,19 +23,35 @@ func (a *Deterministic) NextAction(
 	_ context.Context,
 	_ string,
 	_ []Step,
-	_ Observation,
+	observation Observation,
 ) (Decision, error) {
 	var decision Decision
 	switch a.next {
 	case 0:
+		x, err := greenCentroidX(observation.Screenshot)
+		if err != nil {
+			return Decision{}, fmt.Errorf("inspect initial screenshot: %w", err)
+		}
+		a.initialX = x
+		a.hasInitial = true
 		action := environment.InputAction{Type: environment.KeyDown, Key: "RIGHT"}
 		decision = Decision{Reason: "Press RIGHT to start moving.", Action: &action}
 	case 1:
-		action := environment.InputAction{Type: environment.Wait, Duration: time.Second}
+		action := environment.InputAction{Type: environment.Wait, DurationMS: 1000}
 		decision = Decision{Reason: "Keep RIGHT held for one second.", Action: &action}
 	case 2:
+		x, err := greenCentroidX(observation.Screenshot)
+		if err != nil {
+			return Decision{}, fmt.Errorf("inspect moved screenshot: %w", err)
+		}
+		if !a.hasInitial || x-a.initialX < 100 {
+			return Decision{}, fmt.Errorf("visual verification failed: square moved %.1f pixels right, want at least 100", x-a.initialX)
+		}
 		action := environment.InputAction{Type: environment.KeyUp, Key: "RIGHT"}
-		decision = Decision{Reason: "Release RIGHT after the movement.", Action: &action}
+		decision = Decision{
+			Reason: fmt.Sprintf("The square moved %.1f pixels right; release RIGHT.", x-a.initialX),
+			Action: &action,
+		}
 	default:
 		decision = Decision{Reason: "The movement sequence is complete.", Done: true}
 	}
@@ -39,4 +59,26 @@ func (a *Deterministic) NextAction(
 	return decision, nil
 }
 
+func greenCentroidX(screenshot []byte) (float64, error) {
+	image, err := png.Decode(bytes.NewReader(screenshot))
+	if err != nil {
+		return 0, err
+	}
+	var sumX, count uint64
+	bounds := image.Bounds()
+	for y := bounds.Min.Y; y < bounds.Max.Y; y++ {
+		for x := bounds.Min.X; x < bounds.Max.X; x++ {
+			red, green, blue, _ := image.At(x, y).RGBA()
+			if green > 0xc000 && red < 0x4000 && blue < 0x8000 {
+				sumX += uint64(x)
+				count++
+			}
+		}
+	}
+	if count < 1000 {
+		return 0, fmt.Errorf("found only %d green square pixels", count)
+	}
+	return float64(sumX) / float64(count), nil
+}
+
 var _ Agent = (*Deterministic)(nil)

agentbox/internal/agent/deterministic_test.go

@@ -1,7 +1,11 @@
 package agent
 
 import (
+	"bytes"
 	"context"
+	"image"
+	"image/color"
+	"image/png"
 	"testing"
 
 	"agentbox/internal/environment"
@@ -19,7 +23,13 @@ func TestDeterministicSequence(t *testing.T) {
 		{"", true},
 	}
 	for index, expected := range want {
-		decision, err := controller.NextAction(context.Background(), "", nil, Observation{})
+		x := 20
+		if index >= 2 {
+			x = 140
+		}
+		decision, err := controller.NextAction(context.Background(), "", nil, Observation{
+			Screenshot: screenshotWithSquare(t, x),
+		})
 		if err != nil {
 			t.Fatalf("step %d: %v", index, err)
 		}
@@ -31,3 +41,35 @@ func TestDeterministicSequence(t *testing.T) {
 		}
 	}
 }
+
+func TestDeterministicRejectsMissingMovement(t *testing.T) {
+	controller := &Deterministic{}
+	for step := 0; step < 2; step++ {
+		if _, err := controller.NextAction(context.Background(), "", nil, Observation{
+			Screenshot: screenshotWithSquare(t, 20),
+		}); err != nil {
+			t.Fatal(err)
+		}
+	}
+	if _, err := controller.NextAction(context.Background(), "", nil, Observation{
+		Screenshot: screenshotWithSquare(t, 20),
+	}); err == nil {
+		t.Fatal("movement verification error = nil")
+	}
+}
+
+func screenshotWithSquare(t *testing.T, startX int) []byte {
+	t.Helper()
+	img := image.NewRGBA(image.Rect(0, 0, 240, 100))
+	green := color.RGBA{G: 255, B: 102, A: 255}
+	for y := 20; y < 60; y++ {
+		for x := startX; x < startX+40; x++ {
+			img.Set(x, y, green)
+		}
+	}
+	var output bytes.Buffer
+	if err := png.Encode(&output, img); err != nil {
+		t.Fatal(err)
+	}
+	return output.Bytes()
+}

agentbox/internal/agent/openai.go

@@ -225,12 +225,12 @@ func parseModelDecision(text string) (Decision, error) {
 		return Decision{}, errors.New("model wait must be between 0 and 5000 ms")
 	}
 	action := environment.InputAction{
-		Type:     result.Action.Type,
-		Key:      result.Action.Key,
-		X:        result.Action.X,
-		Y:        result.Action.Y,
-		Button:   result.Action.Button,
-		Duration: time.Duration(result.Action.DurationMS) * time.Millisecond,
+		Type:       result.Action.Type,
+		Key:        result.Action.Key,
+		X:          result.Action.X,
+		Y:          result.Action.Y,
+		Button:     result.Action.Button,
+		DurationMS: result.Action.DurationMS,
 	}
 	switch action.Type {
 	case environment.KeyDown, environment.KeyUp, environment.MouseMove,

agentbox/internal/environment/dockerx11/docker.go

@@ -157,12 +157,12 @@ func (e *Environment) SendInput(ctx context.Context, action environment.InputAct
 		if action.Key == "" {
 			return errors.New("key_down requires key")
 		}
-		args = []string{"keydown", action.Key}
+		args = []string{"keydown", x11Key(action.Key)}
 	case environment.KeyUp:
 		if action.Key == "" {
 			return errors.New("key_up requires key")
 		}
-		args = []string{"keyup", action.Key}
+		args = []string{"keyup", x11Key(action.Key)}
 	case environment.MouseMove:
 		args = []string{"mousemove", strconv.Itoa(action.X), strconv.Itoa(action.Y)}
 	case environment.MouseDown:
@@ -170,13 +170,13 @@ func (e *Environment) SendInput(ctx context.Context, action environment.InputAct
 	case environment.MouseUp:
 		args = []string{"mouseup", strconv.Itoa(action.Button)}
 	case environment.Wait:
-		if action.Duration < 0 {
+		if action.DurationMS < 0 {
 			return errors.New("wait duration cannot be negative")
 		}
 		select {
 		case <-ctx.Done():
 			return ctx.Err()
-		case <-time.After(action.Duration):
+		case <-time.After(time.Duration(action.DurationMS) * time.Millisecond):
 			return nil
 		}
 	default:
@@ -263,4 +263,33 @@ func shellQuote(value string) string {
 	return "'" + strings.ReplaceAll(value, "'", "'\"'\"'") + "'"
 }
 
+func x11Key(key string) string {
+	switch strings.ToUpper(key) {
+	case "LEFT":
+		return "Left"
+	case "RIGHT":
+		return "Right"
+	case "UP":
+		return "Up"
+	case "DOWN":
+		return "Down"
+	case "ENTER", "RETURN":
+		return "Return"
+	case "ESC", "ESCAPE":
+		return "Escape"
+	case "SPACE":
+		return "space"
+	case "TAB":
+		return "Tab"
+	case "BACKSPACE":
+		return "BackSpace"
+	case "DELETE":
+		return "Delete"
+	}
+	if len(key) == 1 {
+		return strings.ToLower(key)
+	}
+	return key
+}
+
 var _ environment.Environment = (*Environment)(nil)

agentbox/internal/environment/dockerx11/docker_test.go

@@ -0,0 +1,18 @@
+package dockerx11
+
+import "testing"
+
+func TestX11KeyTranslatesLogicalNames(t *testing.T) {
+	tests := map[string]string{
+		"RIGHT":  "Right",
+		"LEFT":   "Left",
+		"ENTER":  "Return",
+		"ESCAPE": "Escape",
+		"A":      "a",
+	}
+	for input, want := range tests {
+		if got := x11Key(input); got != want {
+			t.Errorf("x11Key(%q) = %q, want %q", input, got, want)
+		}
+	}
+}

agentbox/internal/environment/environment.go

@@ -24,12 +24,12 @@ const (
 )
 
 type InputAction struct {
-	Type     InputType     `json:"type"`
-	Key      string        `json:"key,omitempty"`
-	X        int           `json:"x,omitempty"`
-	Y        int           `json:"y,omitempty"`
-	Button   int           `json:"button,omitempty"`
-	Duration time.Duration `json:"duration,omitempty"`
+	Type       InputType `json:"type"`
+	Key        string    `json:"key,omitempty"`
+	X          int       `json:"x,omitempty"`
+	Y          int       `json:"y,omitempty"`
+	Button     int       `json:"button,omitempty"`
+	DurationMS int       `json:"duration_ms,omitempty"`
 }
 
 type LogEntry struct {

agentbox/internal/runtime/runtime.go

@@ -142,7 +142,7 @@ func actionDetail(action environment.InputAction) string {
 	case environment.MouseDown, environment.MouseUp:
 		return fmt.Sprintf(" %d", action.Button)
 	case environment.Wait:
-		return " " + action.Duration.String()
+		return fmt.Sprintf(" %dms", action.DurationMS)
 	default:
 		return ""
 	}