Verify deterministic movement from observations

Co-authored-by: codegirl007 <codegirl-007@users.noreply.github.com>

agentbox/internal/agent/deterministic.go

@@ -1,14 +1,18 @@
 package agent
 
 import (
+	"bytes"
 	"context"
-	"time"
+	"fmt"
+	"image/png"
 
 	"agentbox/internal/environment"
 )
 
 type Deterministic struct {
-	next int
+	next       int
+	initialX   float64
+	hasInitial bool
 }
 
 func (a *Deterministic) Name() string {
@@ -19,19 +23,35 @@ func (a *Deterministic) NextAction(
 	_ context.Context,
 	_ string,
 	_ []Step,
-	_ Observation,
+	observation Observation,
 ) (Decision, error) {
 	var decision Decision
 	switch a.next {
 	case 0:
+		x, err := greenCentroidX(observation.Screenshot)
+		if err != nil {
+			return Decision{}, fmt.Errorf("inspect initial screenshot: %w", err)
+		}
+		a.initialX = x
+		a.hasInitial = true
 		action := environment.InputAction{Type: environment.KeyDown, Key: "RIGHT"}
 		decision = Decision{Reason: "Press RIGHT to start moving.", Action: &action}
 	case 1:
-		action := environment.InputAction{Type: environment.Wait, Duration: time.Second}
+		action := environment.InputAction{Type: environment.Wait, DurationMS: 1000}
 		decision = Decision{Reason: "Keep RIGHT held for one second.", Action: &action}
 	case 2:
+		x, err := greenCentroidX(observation.Screenshot)
+		if err != nil {
+			return Decision{}, fmt.Errorf("inspect moved screenshot: %w", err)
+		}
+		if !a.hasInitial || x-a.initialX < 100 {
+			return Decision{}, fmt.Errorf("visual verification failed: square moved %.1f pixels right, want at least 100", x-a.initialX)
+		}
 		action := environment.InputAction{Type: environment.KeyUp, Key: "RIGHT"}
-		decision = Decision{Reason: "Release RIGHT after the movement.", Action: &action}
+		decision = Decision{
+			Reason: fmt.Sprintf("The square moved %.1f pixels right; release RIGHT.", x-a.initialX),
+			Action: &action,
+		}
 	default:
 		decision = Decision{Reason: "The movement sequence is complete.", Done: true}
 	}
@@ -39,4 +59,26 @@ func (a *Deterministic) NextAction(
 	return decision, nil
 }
 
+func greenCentroidX(screenshot []byte) (float64, error) {
+	image, err := png.Decode(bytes.NewReader(screenshot))
+	if err != nil {
+		return 0, err
+	}
+	var sumX, count uint64
+	bounds := image.Bounds()
+	for y := bounds.Min.Y; y < bounds.Max.Y; y++ {
+		for x := bounds.Min.X; x < bounds.Max.X; x++ {
+			red, green, blue, _ := image.At(x, y).RGBA()
+			if green > 0xc000 && red < 0x4000 && blue < 0x8000 {
+				sumX += uint64(x)
+				count++
+			}
+		}
+	}
+	if count < 1000 {
+		return 0, fmt.Errorf("found only %d green square pixels", count)
+	}
+	return float64(sumX) / float64(count), nil
+}
+
 var _ Agent = (*Deterministic)(nil)

agentbox/internal/agent/deterministic_test.go

@@ -1,7 +1,11 @@
 package agent
 
 import (
+	"bytes"
 	"context"
+	"image"
+	"image/color"
+	"image/png"
 	"testing"
 
 	"agentbox/internal/environment"
@@ -19,7 +23,13 @@ func TestDeterministicSequence(t *testing.T) {
 		{"", true},
 	}
 	for index, expected := range want {
-		decision, err := controller.NextAction(context.Background(), "", nil, Observation{})
+		x := 20
+		if index >= 2 {
+			x = 140
+		}
+		decision, err := controller.NextAction(context.Background(), "", nil, Observation{
+			Screenshot: screenshotWithSquare(t, x),
+		})
 		if err != nil {
 			t.Fatalf("step %d: %v", index, err)
 		}
@@ -31,3 +41,35 @@ func TestDeterministicSequence(t *testing.T) {
 		}
 	}
 }
+
+func TestDeterministicRejectsMissingMovement(t *testing.T) {
+	controller := &Deterministic{}
+	for step := 0; step < 2; step++ {
+		if _, err := controller.NextAction(context.Background(), "", nil, Observation{
+			Screenshot: screenshotWithSquare(t, 20),
+		}); err != nil {
+			t.Fatal(err)
+		}
+	}
+	if _, err := controller.NextAction(context.Background(), "", nil, Observation{
+		Screenshot: screenshotWithSquare(t, 20),
+	}); err == nil {
+		t.Fatal("movement verification error = nil")
+	}
+}
+
+func screenshotWithSquare(t *testing.T, startX int) []byte {
+	t.Helper()
+	img := image.NewRGBA(image.Rect(0, 0, 240, 100))
+	green := color.RGBA{G: 255, B: 102, A: 255}
+	for y := 20; y < 60; y++ {
+		for x := startX; x < startX+40; x++ {
+			img.Set(x, y, green)
+		}
+	}
+	var output bytes.Buffer
+	if err := png.Encode(&output, img); err != nil {
+		t.Fatal(err)
+	}
+	return output.Bytes()
+}

agentbox/internal/agent/openai.go

@@ -225,12 +225,12 @@ func parseModelDecision(text string) (Decision, error) {
 		return Decision{}, errors.New("model wait must be between 0 and 5000 ms")
 	}
 	action := environment.InputAction{
-		Type:     result.Action.Type,
-		Key:      result.Action.Key,
-		X:        result.Action.X,
-		Y:        result.Action.Y,
-		Button:   result.Action.Button,
-		Duration: time.Duration(result.Action.DurationMS) * time.Millisecond,
+		Type:       result.Action.Type,
+		Key:        result.Action.Key,
+		X:          result.Action.X,
+		Y:          result.Action.Y,
+		Button:     result.Action.Button,
+		DurationMS: result.Action.DurationMS,
 	}
 	switch action.Type {
 	case environment.KeyDown, environment.KeyUp, environment.MouseMove,