Organize Agentbox runtime code

Co-authored-by: codegirl007 <codegirl-007@users.noreply.github.com>

agentbox/internal/agent/agent.go

@@ -14,6 +14,7 @@ type Observation struct {
 	PreviousActions []environment.InputAction
 }
 
+// Step is the compact history passed back to an agent on its next decision.
 type Step struct {
 	Number         int
 	Timestamp      time.Time
@@ -22,12 +23,14 @@ type Step struct {
 	Action         *environment.InputAction
 }
 
+// Decision contains either one action or Done. Returning neither is invalid.
 type Decision struct {
 	Reason string
 	Action *environment.InputAction
 	Done   bool
 }
 
+// Agent chooses one backend-neutral action from the latest observation.
 type Agent interface {
 	Name() string
 	NextAction(

agentbox/internal/agent/deterministic.go

@@ -10,75 +10,82 @@ import (
 )
 
 type Deterministic struct {
-	next       int
-	initialX   float64
-	hasInitial bool
+	nextStep           int
+	initialSquareX     float64
+	hasInitialPosition bool
 }
 
-func (a *Deterministic) Name() string {
+func (deterministicAgent *Deterministic) Name() string {
 	return "deterministic-right"
 }
 
-func (a *Deterministic) NextAction(
+func (deterministicAgent *Deterministic) NextAction(
 	_ context.Context,
 	_ string,
 	_ []Step,
 	observation Observation,
 ) (Decision, error) {
 	var decision Decision
-	switch a.next {
+	switch deterministicAgent.nextStep {
 	case 0:
-		x, err := greenCentroidX(observation.Screenshot)
+		initialSquareX, err := greenCentroidX(observation.Screenshot)
 		if err != nil {
 			return Decision{}, fmt.Errorf("inspect initial screenshot: %w", err)
 		}
-		a.initialX = x
-		a.hasInitial = true
+		deterministicAgent.initialSquareX = initialSquareX
+		deterministicAgent.hasInitialPosition = true
 		action := environment.InputAction{Type: environment.KeyDown, Key: "RIGHT"}
 		decision = Decision{Reason: "Press RIGHT to start moving.", Action: &action}
 	case 1:
 		action := environment.InputAction{Type: environment.Wait, DurationMS: 1000}
 		decision = Decision{Reason: "Keep RIGHT held for one second.", Action: &action}
 	case 2:
-		x, err := greenCentroidX(observation.Screenshot)
+		currentSquareX, err := greenCentroidX(observation.Screenshot)
 		if err != nil {
 			return Decision{}, fmt.Errorf("inspect moved screenshot: %w", err)
 		}
-		if !a.hasInitial || x-a.initialX < 100 {
-			return Decision{}, fmt.Errorf("visual verification failed: square moved %.1f pixels right, want at least 100", x-a.initialX)
+		distanceMoved := currentSquareX - deterministicAgent.initialSquareX
+		if !deterministicAgent.hasInitialPosition || distanceMoved < 100 {
+			return Decision{}, fmt.Errorf(
+				"visual verification failed: square moved %.1f pixels right, want at least 100",
+				distanceMoved,
+			)
 		}
 		action := environment.InputAction{Type: environment.KeyUp, Key: "RIGHT"}
 		decision = Decision{
-			Reason: fmt.Sprintf("The square moved %.1f pixels right; release RIGHT.", x-a.initialX),
+			Reason: fmt.Sprintf(
+				"The square moved %.1f pixels right; release RIGHT.",
+				distanceMoved,
+			),
 			Action: &action,
 		}
 	default:
 		decision = Decision{Reason: "The movement sequence is complete.", Done: true}
 	}
-	a.next++
+	deterministicAgent.nextStep++
 	return decision, nil
 }
 
 func greenCentroidX(screenshot []byte) (float64, error) {
-	image, err := png.Decode(bytes.NewReader(screenshot))
+	renderedImage, err := png.Decode(bytes.NewReader(screenshot))
 	if err != nil {
 		return 0, err
 	}
-	var sumX, count uint64
-	bounds := image.Bounds()
+	var xCoordinateSum, greenPixelCount uint64
+	bounds := renderedImage.Bounds()
 	for y := bounds.Min.Y; y < bounds.Max.Y; y++ {
 		for x := bounds.Min.X; x < bounds.Max.X; x++ {
-			red, green, blue, _ := image.At(x, y).RGBA()
+			red, green, blue, _ := renderedImage.At(x, y).RGBA()
 			if green > 0xc000 && red < 0x4000 && blue < 0x8000 {
-				sumX += uint64(x)
-				count++
+				xCoordinateSum += uint64(x)
+				greenPixelCount++
 			}
 		}
 	}
-	if count < 1000 {
-		return 0, fmt.Errorf("found only %d green square pixels", count)
+	if greenPixelCount < 1000 {
+		return 0, fmt.Errorf("found only %d green square pixels", greenPixelCount)
 	}
-	return float64(sumX) / float64(count), nil
+	return float64(xCoordinateSum) / float64(greenPixelCount), nil
 }
 
 var _ Agent = (*Deterministic)(nil)

agentbox/internal/agent/openai.go

@@ -3,16 +3,12 @@ package agent
 import (
 	"bytes"
 	"context"
-	"encoding/base64"
-	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
 	"net/http"
 	"strings"
 	"time"
-
-	"agentbox/internal/environment"
 )
 
 type OpenAIConfig struct {
@@ -22,6 +18,8 @@ type OpenAIConfig struct {
 	Client  *http.Client
 }
 
+// OpenAI adapts the Responses API to Agentbox's provider-neutral Agent
+// contract. HTTP and wire-format details do not leak into the runtime.
 type OpenAI struct {
 	config OpenAIConfig
 }
@@ -42,203 +40,59 @@ func NewOpenAI(config OpenAIConfig) (*OpenAI, error) {
 	return &OpenAI{config: config}, nil
 }
 
-func (a *OpenAI) Name() string {
-	return "openai:" + a.config.Model
+func (openAI *OpenAI) Name() string {
+	return "openai:" + openAI.config.Model
 }
 
-func (a *OpenAI) NextAction(
+func (openAI *OpenAI) NextAction(
 	ctx context.Context,
 	task string,
 	history []Step,
 	observation Observation,
 ) (Decision, error) {
-	prompt, err := modelPrompt(task, history, observation)
+	requestBody, err := buildResponsesRequest(
+		openAI.config.Model,
+		task,
+		history,
+		observation,
+	)
 	if err != nil {
 		return Decision{}, err
 	}
-	requestBody := map[string]any{
-		"model": a.config.Model,
-		"input": []any{
-			map[string]any{
-				"role": "user",
-				"content": []any{
-					map[string]any{"type": "input_text", "text": prompt},
-					map[string]any{
-						"type": "input_image",
-						"image_url": "data:image/png;base64," +
-							base64.StdEncoding.EncodeToString(observation.Screenshot),
-					},
-				},
-			},
-		},
-		"text": map[string]any{
-			"format": map[string]any{
-				"type":   "json_schema",
-				"name":   "agentbox_action",
-				"strict": true,
-				"schema": decisionSchema(),
-			},
-		},
-	}
-	body, err := json.Marshal(requestBody)
+	request, err := http.NewRequestWithContext(
+		ctx,
+		http.MethodPost,
+		strings.TrimRight(openAI.config.BaseURL, "/")+"/responses",
+		bytes.NewReader(requestBody),
+	)
 	if err != nil {
 		return Decision{}, err
 	}
-	request, err := http.NewRequestWithContext(ctx, http.MethodPost,
-		strings.TrimRight(a.config.BaseURL, "/")+"/responses", bytes.NewReader(body))
-	if err != nil {
-		return Decision{}, err
-	}
-	request.Header.Set("Authorization", "Bearer "+a.config.APIKey)
+	request.Header.Set("Authorization", "Bearer "+openAI.config.APIKey)
 	request.Header.Set("Content-Type", "application/json")
 
-	response, err := a.config.Client.Do(request)
+	httpResponse, err := openAI.config.Client.Do(request)
 	if err != nil {
 		return Decision{}, err
 	}
-	defer response.Body.Close()
-	responseBody, err := io.ReadAll(io.LimitReader(response.Body, 4<<20))
+	defer httpResponse.Body.Close()
+
+	responseBody, err := io.ReadAll(io.LimitReader(httpResponse.Body, 4<<20))
 	if err != nil {
 		return Decision{}, err
 	}
-	if response.StatusCode < 200 || response.StatusCode >= 300 {
-		return Decision{}, fmt.Errorf("OpenAI response %s: %s",
-			response.Status, strings.TrimSpace(string(responseBody)))
+	if httpResponse.StatusCode < 200 || httpResponse.StatusCode >= 300 {
+		return Decision{}, fmt.Errorf(
+			"OpenAI response %s: %s",
+			httpResponse.Status,
+			strings.TrimSpace(string(responseBody)),
+		)
 	}
-	text, err := responseText(responseBody)
+	outputText, err := extractResponseText(responseBody)
 	if err != nil {
 		return Decision{}, err
 	}
-	return parseModelDecision(text)
-}
-
-func modelPrompt(task string, history []Step, observation Observation) (string, error) {
-	contextData := struct {
-		Task            string                    `json:"task"`
-		History         []Step                    `json:"history"`
-		RecentLogs      []environment.LogEntry    `json:"recent_logs"`
-		PreviousActions []environment.InputAction `json:"previous_actions"`
-	}{
-		Task:            task,
-		History:         history,
-		RecentLogs:      observation.Logs,
-		PreviousActions: observation.PreviousActions,
-	}
-	data, err := json.Marshal(contextData)
-	if err != nil {
-		return "", err
-	}
-	return "You control an interactive Linux application from screenshots. " +
-		"Choose exactly one safe input action, or mark done when the task is complete. " +
-		"Use X11 key names such as RIGHT, Return, or Escape. Keep waits under 5000 ms.\n" +
-		string(data), nil
-}
-
-func decisionSchema() map[string]any {
-	actionProperties := map[string]any{
-		"type": map[string]any{
-			"type": "string",
-			"enum": []string{
-				string(environment.KeyDown), string(environment.KeyUp),
-				string(environment.MouseMove), string(environment.MouseDown),
-				string(environment.MouseUp), string(environment.Wait),
-			},
-		},
-		"key":         map[string]any{"type": "string"},
-		"x":           map[string]any{"type": "integer"},
-		"y":           map[string]any{"type": "integer"},
-		"button":      map[string]any{"type": "integer"},
-		"duration_ms": map[string]any{"type": "integer"},
-	}
-	return map[string]any{
-		"type":                 "object",
-		"additionalProperties": false,
-		"properties": map[string]any{
-			"reason": map[string]any{"type": "string"},
-			"done":   map[string]any{"type": "boolean"},
-			"action": map[string]any{
-				"anyOf": []any{
-					map[string]any{
-						"type":                 "object",
-						"additionalProperties": false,
-						"properties":           actionProperties,
-						"required": []string{
-							"type", "key", "x", "y", "button", "duration_ms",
-						},
-					},
-					map[string]any{"type": "null"},
-				},
-			},
-		},
-		"required": []string{"reason", "done", "action"},
-	}
-}
-
-func responseText(data []byte) (string, error) {
-	var response struct {
-		Output []struct {
-			Content []struct {
-				Type string `json:"type"`
-				Text string `json:"text"`
-			} `json:"content"`
-		} `json:"output"`
-	}
-	if err := json.Unmarshal(data, &response); err != nil {
-		return "", fmt.Errorf("decode OpenAI response: %w", err)
-	}
-	for _, output := range response.Output {
-		for _, content := range output.Content {
-			if content.Type == "output_text" && content.Text != "" {
-				return content.Text, nil
-			}
-		}
-	}
-	return "", errors.New("OpenAI response contained no output_text")
-}
-
-func parseModelDecision(text string) (Decision, error) {
-	var result struct {
-		Reason string `json:"reason"`
-		Done   bool   `json:"done"`
-		Action *struct {
-			Type       environment.InputType `json:"type"`
-			Key        string                `json:"key"`
-			X          int                   `json:"x"`
-			Y          int                   `json:"y"`
-			Button     int                   `json:"button"`
-			DurationMS int                   `json:"duration_ms"`
-		} `json:"action"`
-	}
-	if err := json.Unmarshal([]byte(text), &result); err != nil {
-		return Decision{}, fmt.Errorf("decode model decision: %w", err)
-	}
-	if result.Reason == "" {
-		return Decision{}, errors.New("model decision requires reason")
-	}
-	if result.Done {
-		return Decision{Reason: result.Reason, Done: true}, nil
-	}
-	if result.Action == nil {
-		return Decision{}, errors.New("model decision requires action when not done")
-	}
-	if result.Action.DurationMS < 0 || result.Action.DurationMS > 5000 {
-		return Decision{}, errors.New("model wait must be between 0 and 5000 ms")
-	}
-	action := environment.InputAction{
-		Type:       result.Action.Type,
-		Key:        result.Action.Key,
-		X:          result.Action.X,
-		Y:          result.Action.Y,
-		Button:     result.Action.Button,
-		DurationMS: result.Action.DurationMS,
-	}
-	switch action.Type {
-	case environment.KeyDown, environment.KeyUp, environment.MouseMove,
-		environment.MouseDown, environment.MouseUp, environment.Wait:
-	default:
-		return Decision{}, fmt.Errorf("model returned unsupported action %q", action.Type)
-	}
-	return Decision{Reason: result.Reason, Action: &action}, nil
+	return parseModelDecision(outputText)
 }
 
 var _ Agent = (*OpenAI)(nil)

agentbox/internal/agent/openai_protocol.go

@@ -0,0 +1,204 @@
+package agent
+
+import (
+	"encoding/base64"
+	"encoding/json"
+	"errors"
+	"fmt"
+
+	"agentbox/internal/environment"
+)
+
+type responsesRequest struct {
+	Model string                `json:"model"`
+	Input []responsesInput      `json:"input"`
+	Text  responsesTextSettings `json:"text"`
+}
+
+type responsesInput struct {
+	Role    string             `json:"role"`
+	Content []responsesContent `json:"content"`
+}
+
+type responsesContent struct {
+	Type     string `json:"type"`
+	Text     string `json:"text,omitempty"`
+	ImageURL string `json:"image_url,omitempty"`
+}
+
+type responsesTextSettings struct {
+	Format responsesFormat `json:"format"`
+}
+
+type responsesFormat struct {
+	Type   string         `json:"type"`
+	Name   string         `json:"name"`
+	Strict bool           `json:"strict"`
+	Schema map[string]any `json:"schema"`
+}
+
+func buildResponsesRequest(
+	model string,
+	task string,
+	history []Step,
+	observation Observation,
+) ([]byte, error) {
+	prompt, err := modelPrompt(task, history, observation)
+	if err != nil {
+		return nil, err
+	}
+	request := responsesRequest{
+		Model: model,
+		Input: []responsesInput{{
+			Role: "user",
+			Content: []responsesContent{
+				{Type: "input_text", Text: prompt},
+				{
+					Type: "input_image",
+					ImageURL: "data:image/png;base64," +
+						base64.StdEncoding.EncodeToString(observation.Screenshot),
+				},
+			},
+		}},
+		Text: responsesTextSettings{Format: responsesFormat{
+			Type:   "json_schema",
+			Name:   "agentbox_action",
+			Strict: true,
+			Schema: decisionSchema(),
+		}},
+	}
+	return json.Marshal(request)
+}
+
+func modelPrompt(task string, history []Step, observation Observation) (string, error) {
+	contextData := struct {
+		Task            string                    `json:"task"`
+		History         []Step                    `json:"history"`
+		RecentLogs      []environment.LogEntry    `json:"recent_logs"`
+		PreviousActions []environment.InputAction `json:"previous_actions"`
+	}{
+		Task:            task,
+		History:         history,
+		RecentLogs:      observation.Logs,
+		PreviousActions: observation.PreviousActions,
+	}
+	data, err := json.Marshal(contextData)
+	if err != nil {
+		return "", err
+	}
+	return "You control an interactive Linux application from screenshots. " +
+		"Choose exactly one safe input action, or mark done when the task is complete. " +
+		"Use logical key names such as RIGHT, ENTER, or ESCAPE. Keep waits under 5000 ms.\n" +
+		string(data), nil
+}
+
+func decisionSchema() map[string]any {
+	actionProperties := map[string]any{
+		"type": map[string]any{
+			"type": "string",
+			"enum": []string{
+				string(environment.KeyDown), string(environment.KeyUp),
+				string(environment.MouseMove), string(environment.MouseDown),
+				string(environment.MouseUp), string(environment.Wait),
+			},
+		},
+		"key":         map[string]any{"type": "string"},
+		"x":           map[string]any{"type": "integer"},
+		"y":           map[string]any{"type": "integer"},
+		"button":      map[string]any{"type": "integer"},
+		"duration_ms": map[string]any{"type": "integer"},
+	}
+	return map[string]any{
+		"type":                 "object",
+		"additionalProperties": false,
+		"properties": map[string]any{
+			"reason": map[string]any{"type": "string"},
+			"done":   map[string]any{"type": "boolean"},
+			"action": map[string]any{
+				"anyOf": []any{
+					map[string]any{
+						"type":                 "object",
+						"additionalProperties": false,
+						"properties":           actionProperties,
+						"required": []string{
+							"type", "key", "x", "y", "button", "duration_ms",
+						},
+					},
+					map[string]any{"type": "null"},
+				},
+			},
+		},
+		"required": []string{"reason", "done", "action"},
+	}
+}
+
+func extractResponseText(data []byte) (string, error) {
+	var response struct {
+		Output []struct {
+			Content []struct {
+				Type string `json:"type"`
+				Text string `json:"text"`
+			} `json:"content"`
+		} `json:"output"`
+	}
+	if err := json.Unmarshal(data, &response); err != nil {
+		return "", fmt.Errorf("decode OpenAI response: %w", err)
+	}
+	for _, output := range response.Output {
+		for _, content := range output.Content {
+			if content.Type == "output_text" && content.Text != "" {
+				return content.Text, nil
+			}
+		}
+	}
+	return "", errors.New("OpenAI response contained no output_text")
+}
+
+type modelDecision struct {
+	Reason string       `json:"reason"`
+	Done   bool         `json:"done"`
+	Action *modelAction `json:"action"`
+}
+
+type modelAction struct {
+	Type       environment.InputType `json:"type"`
+	Key        string                `json:"key"`
+	X          int                   `json:"x"`
+	Y          int                   `json:"y"`
+	Button     int                   `json:"button"`
+	DurationMS int                   `json:"duration_ms"`
+}
+
+func parseModelDecision(text string) (Decision, error) {
+	var modelOutput modelDecision
+	if err := json.Unmarshal([]byte(text), &modelOutput); err != nil {
+		return Decision{}, fmt.Errorf("decode model decision: %w", err)
+	}
+	if modelOutput.Reason == "" {
+		return Decision{}, errors.New("model decision requires reason")
+	}
+	if modelOutput.Done {
+		return Decision{Reason: modelOutput.Reason, Done: true}, nil
+	}
+	if modelOutput.Action == nil {
+		return Decision{}, errors.New("model decision requires action when not done")
+	}
+	if modelOutput.Action.DurationMS < 0 || modelOutput.Action.DurationMS > 5000 {
+		return Decision{}, errors.New("model wait must be between 0 and 5000 ms")
+	}
+	action := environment.InputAction{
+		Type:       modelOutput.Action.Type,
+		Key:        modelOutput.Action.Key,
+		X:          modelOutput.Action.X,
+		Y:          modelOutput.Action.Y,
+		Button:     modelOutput.Action.Button,
+		DurationMS: modelOutput.Action.DurationMS,
+	}
+	switch action.Type {
+	case environment.KeyDown, environment.KeyUp, environment.MouseMove,
+		environment.MouseDown, environment.MouseUp, environment.Wait:
+	default:
+		return Decision{}, fmt.Errorf("model returned unsupported action %q", action.Type)
+	}
+	return Decision{Reason: modelOutput.Reason, Action: &action}, nil
+}