Organize Agentbox runtime code

Co-authored-by: codegirl007 <codegirl-007@users.noreply.github.com>

agentbox/cmd/agentbox/history_commands.go

@@ -0,0 +1,64 @@
+package main
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"path/filepath"
+
+	"agentbox/internal/trace"
+)
+
+func listRuns() error {
+	projectRoot, err := findProjectRoot()
+	if err != nil {
+		return err
+	}
+	runs, err := trace.List(projectRoot)
+	if err != nil {
+		return err
+	}
+	if len(runs) == 0 {
+		fmt.Println("No recorded runs.")
+		return nil
+	}
+	for _, run := range runs {
+		fmt.Printf("%s  %-8s  %-20s  %s\n",
+			run.ID, run.Status, run.Agent, run.Task)
+	}
+	return nil
+}
+
+func inspect(args []string) error {
+	if len(args) != 1 {
+		return errors.New("inspect requires one run ID")
+	}
+	projectRoot, err := findProjectRoot()
+	if err != nil {
+		return err
+	}
+	runID := args[0]
+	run, err := trace.Read(projectRoot, runID)
+	if err != nil {
+		return err
+	}
+	steps, err := trace.ReadSteps(projectRoot, runID)
+	if err != nil {
+		return err
+	}
+	output := struct {
+		Run       trace.Run          `json:"run"`
+		Steps     []trace.StepRecord `json:"steps"`
+		Directory string             `json:"directory"`
+	}{
+		Run:       run,
+		Steps:     steps,
+		Directory: filepath.Join(projectRoot, ".agentbox", "runs", run.ID),
+	}
+	data, err := json.MarshalIndent(output, "", "  ")
+	if err != nil {
+		return err
+	}
+	fmt.Println(string(data))
+	return nil
+}

agentbox/cmd/agentbox/main.go

@@ -1,24 +1,13 @@
 package main
 
 import (
-	"bytes"
 	"context"
-	"encoding/json"
-	"errors"
 	"fmt"
 	"os"
 	"os/signal"
-	"path/filepath"
-	"strconv"
-	"strings"
 	"syscall"
 
-	"agentbox/internal/agent"
-	"agentbox/internal/appspec"
-	"agentbox/internal/environment/dockerx11"
 	"agentbox/internal/phase1"
-	agentRuntime "agentbox/internal/runtime"
-	"agentbox/internal/trace"
 )
 
 func main() {
@@ -50,188 +39,6 @@ func main() {
 	}
 }
 
-type runOptions struct {
-	path     string
-	task     string
-	agent    string
-	model    string
-	maxSteps int
-}
-
-func run(ctx context.Context, args []string) error {
-	options, err := parseRunOptions(args)
-	if err != nil {
-		return err
-	}
-	root, err := findRoot()
-	if err != nil {
-		return err
-	}
-	command, err := appspec.Resolve(options.path)
-	if err != nil {
-		return err
-	}
-	controller, err := selectAgent(options)
-	if err != nil {
-		return err
-	}
-	store, err := trace.New(root, options.task, options.path, controller.Name())
-	if err != nil {
-		return err
-	}
-	env := dockerx11.New(dockerx11.Config{
-		ProjectRoot: root,
-		RunID:       store.ID(),
-		Output:      os.Stdout,
-	})
-	err = agentRuntime.Run(ctx, agentRuntime.Config{
-		Task:     options.task,
-		Command:  command,
-		Agent:    controller,
-		Env:      env,
-		Trace:    store,
-		MaxSteps: options.maxSteps,
-		Output:   os.Stdout,
-	})
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "Run artifacts: %s\n", store.Directory())
-		return err
-	}
-	fmt.Printf("Replay written to:\n%s\n", store.Directory())
-	return nil
-}
-
-func parseRunOptions(args []string) (runOptions, error) {
-	options := runOptions{agent: "deterministic", model: "gpt-5", maxSteps: 20}
-	for index := 0; index < len(args); index++ {
-		arg := args[index]
-		var name, value string
-		if strings.HasPrefix(arg, "--") {
-			name, value, _ = strings.Cut(strings.TrimPrefix(arg, "--"), "=")
-			if value == "" {
-				index++
-				if index >= len(args) {
-					return options, fmt.Errorf("--%s requires a value", name)
-				}
-				value = args[index]
-			}
-			switch name {
-			case "task":
-				options.task = value
-			case "agent":
-				options.agent = value
-			case "model":
-				options.model = value
-			case "max-steps":
-				number, err := strconv.Atoi(value)
-				if err != nil || number < 1 {
-					return options, errors.New("--max-steps must be a positive integer")
-				}
-				options.maxSteps = number
-			default:
-				return options, fmt.Errorf("unknown flag --%s", name)
-			}
-			continue
-		}
-		if options.path != "" {
-			return options, errors.New("run accepts exactly one application path")
-		}
-		options.path = arg
-	}
-	if options.path == "" {
-		return options, errors.New("run requires an application path")
-	}
-	if options.task == "" {
-		return options, errors.New("run requires --task")
-	}
-	return options, nil
-}
-
-func selectAgent(options runOptions) (agent.Agent, error) {
-	switch options.agent {
-	case "deterministic":
-		return &agent.Deterministic{}, nil
-	case "openai":
-		return agent.NewOpenAI(agent.OpenAIConfig{
-			APIKey: os.Getenv("OPENAI_API_KEY"),
-			Model:  options.model,
-		})
-	default:
-		return nil, fmt.Errorf("unknown agent %q (want deterministic or openai)", options.agent)
-	}
-}
-
-func listRuns() error {
-	root, err := findRoot()
-	if err != nil {
-		return err
-	}
-	runs, err := trace.List(root)
-	if err != nil {
-		return err
-	}
-	if len(runs) == 0 {
-		fmt.Println("No recorded runs.")
-		return nil
-	}
-	for _, run := range runs {
-		fmt.Printf("%s  %-8s  %-20s  %s\n",
-			run.ID, run.Status, run.Agent, run.Task)
-	}
-	return nil
-}
-
-func inspect(args []string) error {
-	if len(args) != 1 {
-		return errors.New("inspect requires one run ID")
-	}
-	root, err := findRoot()
-	if err != nil {
-		return err
-	}
-	run, err := trace.Read(root, args[0])
-	if err != nil {
-		return err
-	}
-	steps, err := trace.ReadSteps(root, args[0])
-	if err != nil {
-		return err
-	}
-	output := struct {
-		Run       trace.Run          `json:"run"`
-		Steps     []trace.StepRecord `json:"steps"`
-		Directory string             `json:"directory"`
-	}{
-		Run:       run,
-		Steps:     steps,
-		Directory: filepath.Join(root, ".agentbox", "runs", run.ID),
-	}
-	data, err := json.MarshalIndent(output, "", "  ")
-	if err != nil {
-		return err
-	}
-	fmt.Println(string(data))
-	return nil
-}
-
-func findRoot() (string, error) {
-	current, err := os.Getwd()
-	if err != nil {
-		return "", err
-	}
-	for {
-		data, readErr := os.ReadFile(filepath.Join(current, "go.mod"))
-		if readErr == nil && bytes.Contains(data, []byte("module agentbox")) {
-			return current, nil
-		}
-		parent := filepath.Dir(current)
-		if parent == current {
-			return "", errors.New("run agentbox from inside its module directory")
-		}
-		current = parent
-	}
-}
-
 func usage() {
 	fmt.Fprintln(os.Stderr, `usage:
   agentbox run <path> --task "<task>" [--agent deterministic|openai] [--model <model>]

agentbox/cmd/agentbox/main_test.go

@@ -9,7 +9,9 @@ func TestParseRunOptionsAllowsFlagsAfterPath(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	if options.path != "./game" || options.task != "move right" || options.maxSteps != 7 {
+	if options.applicationPath != "./game" ||
+		options.task != "move right" ||
+		options.maxSteps != 7 {
 		t.Fatalf("options = %#v", options)
 	}
 }

agentbox/cmd/agentbox/project.go

@@ -0,0 +1,28 @@
+package main
+
+import (
+	"bytes"
+	"errors"
+	"os"
+	"path/filepath"
+)
+
+// findProjectRoot lets commands work from any directory inside the Agentbox
+// module. The module declaration is a stronger marker than a directory name.
+func findProjectRoot() (string, error) {
+	currentDirectory, err := os.Getwd()
+	if err != nil {
+		return "", err
+	}
+	for {
+		goModule, readErr := os.ReadFile(filepath.Join(currentDirectory, "go.mod"))
+		if readErr == nil && bytes.Contains(goModule, []byte("module agentbox")) {
+			return currentDirectory, nil
+		}
+		parentDirectory := filepath.Dir(currentDirectory)
+		if parentDirectory == currentDirectory {
+			return "", errors.New("run agentbox from inside its module directory")
+		}
+		currentDirectory = parentDirectory
+	}
+}

agentbox/cmd/agentbox/run_command.go

@@ -0,0 +1,141 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"os"
+	"strconv"
+	"strings"
+
+	"agentbox/internal/agent"
+	"agentbox/internal/appspec"
+	"agentbox/internal/environment/dockerx11"
+	agentRuntime "agentbox/internal/runtime"
+	"agentbox/internal/trace"
+)
+
+type runOptions struct {
+	applicationPath string
+	task            string
+	agentName       string
+	modelName       string
+	maxSteps        int
+}
+
+func run(ctx context.Context, args []string) error {
+	options, err := parseRunOptions(args)
+	if err != nil {
+		return err
+	}
+	projectRoot, err := findProjectRoot()
+	if err != nil {
+		return err
+	}
+	applicationCommand, err := appspec.Resolve(options.applicationPath)
+	if err != nil {
+		return err
+	}
+	controller, err := newAgent(options)
+	if err != nil {
+		return err
+	}
+	traceStore, err := trace.New(
+		projectRoot,
+		options.task,
+		options.applicationPath,
+		controller.Name(),
+	)
+	if err != nil {
+		return err
+	}
+	desktopEnvironment := dockerx11.New(dockerx11.Config{
+		ProjectRoot: projectRoot,
+		RunID:       traceStore.ID(),
+		Output:      os.Stdout,
+	})
+	err = agentRuntime.Run(ctx, agentRuntime.Config{
+		Task:        options.task,
+		Command:     applicationCommand,
+		Controller:  controller,
+		Environment: desktopEnvironment,
+		TraceStore:  traceStore,
+		MaxSteps:    options.maxSteps,
+		Output:      os.Stdout,
+	})
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Run artifacts: %s\n", traceStore.Directory())
+		return err
+	}
+	fmt.Printf("Replay written to:\n%s\n", traceStore.Directory())
+	return nil
+}
+
+// parseRunOptions accepts flags before or after the application path so the CLI
+// matches the natural "agentbox run ./app --task ..." form shown in the docs.
+func parseRunOptions(args []string) (runOptions, error) {
+	options := runOptions{
+		agentName: "deterministic",
+		modelName: "gpt-5",
+		maxSteps:  20,
+	}
+	for index := 0; index < len(args); index++ {
+		argument := args[index]
+		if !strings.HasPrefix(argument, "--") {
+			if options.applicationPath != "" {
+				return options, errors.New("run accepts exactly one application path")
+			}
+			options.applicationPath = argument
+			continue
+		}
+
+		flagName, flagValue, _ := strings.Cut(strings.TrimPrefix(argument, "--"), "=")
+		if flagValue == "" {
+			index++
+			if index >= len(args) {
+				return options, fmt.Errorf("--%s requires a value", flagName)
+			}
+			flagValue = args[index]
+		}
+		switch flagName {
+		case "task":
+			options.task = flagValue
+		case "agent":
+			options.agentName = flagValue
+		case "model":
+			options.modelName = flagValue
+		case "max-steps":
+			maxSteps, err := strconv.Atoi(flagValue)
+			if err != nil || maxSteps < 1 {
+				return options, errors.New("--max-steps must be a positive integer")
+			}
+			options.maxSteps = maxSteps
+		default:
+			return options, fmt.Errorf("unknown flag --%s", flagName)
+		}
+	}
+	if options.applicationPath == "" {
+		return options, errors.New("run requires an application path")
+	}
+	if options.task == "" {
+		return options, errors.New("run requires --task")
+	}
+	return options, nil
+}
+
+func newAgent(options runOptions) (agent.Agent, error) {
+	switch options.agentName {
+	case "deterministic":
+		return &agent.Deterministic{}, nil
+	case "openai":
+		return agent.NewOpenAI(agent.OpenAIConfig{
+			APIKey: os.Getenv("OPENAI_API_KEY"),
+			Model:  options.modelName,
+		})
+	default:
+		return nil, fmt.Errorf(
+			"unknown agent %q (want deterministic or openai)",
+			options.agentName,
+		)
+	}
+}

agentbox/internal/agent/agent.go

@@ -14,6 +14,7 @@ type Observation struct {
 	PreviousActions []environment.InputAction
 }
 
+// Step is the compact history passed back to an agent on its next decision.
 type Step struct {
 	Number         int
 	Timestamp      time.Time
@@ -22,12 +23,14 @@ type Step struct {
 	Action         *environment.InputAction
 }
 
+// Decision contains either one action or Done. Returning neither is invalid.
 type Decision struct {
 	Reason string
 	Action *environment.InputAction
 	Done   bool
 }
 
+// Agent chooses one backend-neutral action from the latest observation.
 type Agent interface {
 	Name() string
 	NextAction(

agentbox/internal/agent/deterministic.go

@@ -10,75 +10,82 @@ import (
 )
 
 type Deterministic struct {
-	next       int
-	initialX   float64
-	hasInitial bool
+	nextStep           int
+	initialSquareX     float64
+	hasInitialPosition bool
 }
 
-func (a *Deterministic) Name() string {
+func (deterministicAgent *Deterministic) Name() string {
 	return "deterministic-right"
 }
 
-func (a *Deterministic) NextAction(
+func (deterministicAgent *Deterministic) NextAction(
 	_ context.Context,
 	_ string,
 	_ []Step,
 	observation Observation,
 ) (Decision, error) {
 	var decision Decision
-	switch a.next {
+	switch deterministicAgent.nextStep {
 	case 0:
-		x, err := greenCentroidX(observation.Screenshot)
+		initialSquareX, err := greenCentroidX(observation.Screenshot)
 		if err != nil {
 			return Decision{}, fmt.Errorf("inspect initial screenshot: %w", err)
 		}
-		a.initialX = x
-		a.hasInitial = true
+		deterministicAgent.initialSquareX = initialSquareX
+		deterministicAgent.hasInitialPosition = true
 		action := environment.InputAction{Type: environment.KeyDown, Key: "RIGHT"}
 		decision = Decision{Reason: "Press RIGHT to start moving.", Action: &action}
 	case 1:
 		action := environment.InputAction{Type: environment.Wait, DurationMS: 1000}
 		decision = Decision{Reason: "Keep RIGHT held for one second.", Action: &action}
 	case 2:
-		x, err := greenCentroidX(observation.Screenshot)
+		currentSquareX, err := greenCentroidX(observation.Screenshot)
 		if err != nil {
 			return Decision{}, fmt.Errorf("inspect moved screenshot: %w", err)
 		}
-		if !a.hasInitial || x-a.initialX < 100 {
-			return Decision{}, fmt.Errorf("visual verification failed: square moved %.1f pixels right, want at least 100", x-a.initialX)
+		distanceMoved := currentSquareX - deterministicAgent.initialSquareX
+		if !deterministicAgent.hasInitialPosition || distanceMoved < 100 {
+			return Decision{}, fmt.Errorf(
+				"visual verification failed: square moved %.1f pixels right, want at least 100",
+				distanceMoved,
+			)
 		}
 		action := environment.InputAction{Type: environment.KeyUp, Key: "RIGHT"}
 		decision = Decision{
-			Reason: fmt.Sprintf("The square moved %.1f pixels right; release RIGHT.", x-a.initialX),
+			Reason: fmt.Sprintf(
+				"The square moved %.1f pixels right; release RIGHT.",
+				distanceMoved,
+			),
 			Action: &action,
 		}
 	default:
 		decision = Decision{Reason: "The movement sequence is complete.", Done: true}
 	}
-	a.next++
+	deterministicAgent.nextStep++
 	return decision, nil
 }
 
 func greenCentroidX(screenshot []byte) (float64, error) {
-	image, err := png.Decode(bytes.NewReader(screenshot))
+	renderedImage, err := png.Decode(bytes.NewReader(screenshot))
 	if err != nil {
 		return 0, err
 	}
-	var sumX, count uint64
-	bounds := image.Bounds()
+	var xCoordinateSum, greenPixelCount uint64
+	bounds := renderedImage.Bounds()
 	for y := bounds.Min.Y; y < bounds.Max.Y; y++ {
 		for x := bounds.Min.X; x < bounds.Max.X; x++ {
-			red, green, blue, _ := image.At(x, y).RGBA()
+			red, green, blue, _ := renderedImage.At(x, y).RGBA()
 			if green > 0xc000 && red < 0x4000 && blue < 0x8000 {
-				sumX += uint64(x)
-				count++
+				xCoordinateSum += uint64(x)
+				greenPixelCount++
 			}
 		}
 	}
-	if count < 1000 {
-		return 0, fmt.Errorf("found only %d green square pixels", count)
+	if greenPixelCount < 1000 {
+		return 0, fmt.Errorf("found only %d green square pixels", greenPixelCount)
 	}
-	return float64(sumX) / float64(count), nil
+	return float64(xCoordinateSum) / float64(greenPixelCount), nil
 }
 
 var _ Agent = (*Deterministic)(nil)

agentbox/internal/agent/openai.go

@@ -3,16 +3,12 @@ package agent
 import (
 	"bytes"
 	"context"
-	"encoding/base64"
-	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
 	"net/http"
 	"strings"
 	"time"
-
-	"agentbox/internal/environment"
 )
 
 type OpenAIConfig struct {
@@ -22,6 +18,8 @@ type OpenAIConfig struct {
 	Client  *http.Client
 }
 
+// OpenAI adapts the Responses API to Agentbox's provider-neutral Agent
+// contract. HTTP and wire-format details do not leak into the runtime.
 type OpenAI struct {
 	config OpenAIConfig
 }
@@ -42,203 +40,59 @@ func NewOpenAI(config OpenAIConfig) (*OpenAI, error) {
 	return &OpenAI{config: config}, nil
 }
 
-func (a *OpenAI) Name() string {
-	return "openai:" + a.config.Model
+func (openAI *OpenAI) Name() string {
+	return "openai:" + openAI.config.Model
 }
 
-func (a *OpenAI) NextAction(
+func (openAI *OpenAI) NextAction(
 	ctx context.Context,
 	task string,
 	history []Step,
 	observation Observation,
 ) (Decision, error) {
-	prompt, err := modelPrompt(task, history, observation)
+	requestBody, err := buildResponsesRequest(
+		openAI.config.Model,
+		task,
+		history,
+		observation,
+	)
 	if err != nil {
 		return Decision{}, err
 	}
-	requestBody := map[string]any{
-		"model": a.config.Model,
-		"input": []any{
-			map[string]any{
-				"role": "user",
-				"content": []any{
-					map[string]any{"type": "input_text", "text": prompt},
-					map[string]any{
-						"type": "input_image",
-						"image_url": "data:image/png;base64," +
-							base64.StdEncoding.EncodeToString(observation.Screenshot),
-					},
-				},
-			},
-		},
-		"text": map[string]any{
-			"format": map[string]any{
-				"type":   "json_schema",
-				"name":   "agentbox_action",
-				"strict": true,
-				"schema": decisionSchema(),
-			},
-		},
-	}
-	body, err := json.Marshal(requestBody)
+	request, err := http.NewRequestWithContext(
+		ctx,
+		http.MethodPost,
+		strings.TrimRight(openAI.config.BaseURL, "/")+"/responses",
+		bytes.NewReader(requestBody),
+	)
 	if err != nil {
 		return Decision{}, err
 	}
-	request, err := http.NewRequestWithContext(ctx, http.MethodPost,
-		strings.TrimRight(a.config.BaseURL, "/")+"/responses", bytes.NewReader(body))
-	if err != nil {
-		return Decision{}, err
-	}
-	request.Header.Set("Authorization", "Bearer "+a.config.APIKey)
+	request.Header.Set("Authorization", "Bearer "+openAI.config.APIKey)
 	request.Header.Set("Content-Type", "application/json")
 
-	response, err := a.config.Client.Do(request)
+	httpResponse, err := openAI.config.Client.Do(request)
 	if err != nil {
 		return Decision{}, err
 	}
-	defer response.Body.Close()
-	responseBody, err := io.ReadAll(io.LimitReader(response.Body, 4<<20))
+	defer httpResponse.Body.Close()
+
+	responseBody, err := io.ReadAll(io.LimitReader(httpResponse.Body, 4<<20))
 	if err != nil {
 		return Decision{}, err
 	}
-	if response.StatusCode < 200 || response.StatusCode >= 300 {
-		return Decision{}, fmt.Errorf("OpenAI response %s: %s",
-			response.Status, strings.TrimSpace(string(responseBody)))
+	if httpResponse.StatusCode < 200 || httpResponse.StatusCode >= 300 {
+		return Decision{}, fmt.Errorf(
+			"OpenAI response %s: %s",
+			httpResponse.Status,
+			strings.TrimSpace(string(responseBody)),
+		)
 	}
-	text, err := responseText(responseBody)
+	outputText, err := extractResponseText(responseBody)
 	if err != nil {
 		return Decision{}, err
 	}
-	return parseModelDecision(text)
-}
-
-func modelPrompt(task string, history []Step, observation Observation) (string, error) {
-	contextData := struct {
-		Task            string                    `json:"task"`
-		History         []Step                    `json:"history"`
-		RecentLogs      []environment.LogEntry    `json:"recent_logs"`
-		PreviousActions []environment.InputAction `json:"previous_actions"`
-	}{
-		Task:            task,
-		History:         history,
-		RecentLogs:      observation.Logs,
-		PreviousActions: observation.PreviousActions,
-	}
-	data, err := json.Marshal(contextData)
-	if err != nil {
-		return "", err
-	}
-	return "You control an interactive Linux application from screenshots. " +
-		"Choose exactly one safe input action, or mark done when the task is complete. " +
-		"Use X11 key names such as RIGHT, Return, or Escape. Keep waits under 5000 ms.\n" +
-		string(data), nil
-}
-
-func decisionSchema() map[string]any {
-	actionProperties := map[string]any{
-		"type": map[string]any{
-			"type": "string",
-			"enum": []string{
-				string(environment.KeyDown), string(environment.KeyUp),
-				string(environment.MouseMove), string(environment.MouseDown),
-				string(environment.MouseUp), string(environment.Wait),
-			},
-		},
-		"key":         map[string]any{"type": "string"},
-		"x":           map[string]any{"type": "integer"},
-		"y":           map[string]any{"type": "integer"},
-		"button":      map[string]any{"type": "integer"},
-		"duration_ms": map[string]any{"type": "integer"},
-	}
-	return map[string]any{
-		"type":                 "object",
-		"additionalProperties": false,
-		"properties": map[string]any{
-			"reason": map[string]any{"type": "string"},
-			"done":   map[string]any{"type": "boolean"},
-			"action": map[string]any{
-				"anyOf": []any{
-					map[string]any{
-						"type":                 "object",
-						"additionalProperties": false,
-						"properties":           actionProperties,
-						"required": []string{
-							"type", "key", "x", "y", "button", "duration_ms",
-						},
-					},
-					map[string]any{"type": "null"},
-				},
-			},
-		},
-		"required": []string{"reason", "done", "action"},
-	}
-}
-
-func responseText(data []byte) (string, error) {
-	var response struct {
-		Output []struct {
-			Content []struct {
-				Type string `json:"type"`
-				Text string `json:"text"`
-			} `json:"content"`
-		} `json:"output"`
-	}
-	if err := json.Unmarshal(data, &response); err != nil {
-		return "", fmt.Errorf("decode OpenAI response: %w", err)
-	}
-	for _, output := range response.Output {
-		for _, content := range output.Content {
-			if content.Type == "output_text" && content.Text != "" {
-				return content.Text, nil
-			}
-		}
-	}
-	return "", errors.New("OpenAI response contained no output_text")
-}
-
-func parseModelDecision(text string) (Decision, error) {
-	var result struct {
-		Reason string `json:"reason"`
-		Done   bool   `json:"done"`
-		Action *struct {
-			Type       environment.InputType `json:"type"`
-			Key        string                `json:"key"`
-			X          int                   `json:"x"`
-			Y          int                   `json:"y"`
-			Button     int                   `json:"button"`
-			DurationMS int                   `json:"duration_ms"`
-		} `json:"action"`
-	}
-	if err := json.Unmarshal([]byte(text), &result); err != nil {
-		return Decision{}, fmt.Errorf("decode model decision: %w", err)
-	}
-	if result.Reason == "" {
-		return Decision{}, errors.New("model decision requires reason")
-	}
-	if result.Done {
-		return Decision{Reason: result.Reason, Done: true}, nil
-	}
-	if result.Action == nil {
-		return Decision{}, errors.New("model decision requires action when not done")
-	}
-	if result.Action.DurationMS < 0 || result.Action.DurationMS > 5000 {
-		return Decision{}, errors.New("model wait must be between 0 and 5000 ms")
-	}
-	action := environment.InputAction{
-		Type:       result.Action.Type,
-		Key:        result.Action.Key,
-		X:          result.Action.X,
-		Y:          result.Action.Y,
-		Button:     result.Action.Button,
-		DurationMS: result.Action.DurationMS,
-	}
-	switch action.Type {
-	case environment.KeyDown, environment.KeyUp, environment.MouseMove,
-		environment.MouseDown, environment.MouseUp, environment.Wait:
-	default:
-		return Decision{}, fmt.Errorf("model returned unsupported action %q", action.Type)
-	}
-	return Decision{Reason: result.Reason, Action: &action}, nil
+	return parseModelDecision(outputText)
 }
 
 var _ Agent = (*OpenAI)(nil)

agentbox/internal/agent/openai_protocol.go

@@ -0,0 +1,204 @@
+package agent
+
+import (
+	"encoding/base64"
+	"encoding/json"
+	"errors"
+	"fmt"
+
+	"agentbox/internal/environment"
+)
+
+type responsesRequest struct {
+	Model string                `json:"model"`
+	Input []responsesInput      `json:"input"`
+	Text  responsesTextSettings `json:"text"`
+}
+
+type responsesInput struct {
+	Role    string             `json:"role"`
+	Content []responsesContent `json:"content"`
+}
+
+type responsesContent struct {
+	Type     string `json:"type"`
+	Text     string `json:"text,omitempty"`
+	ImageURL string `json:"image_url,omitempty"`
+}
+
+type responsesTextSettings struct {
+	Format responsesFormat `json:"format"`
+}
+
+type responsesFormat struct {
+	Type   string         `json:"type"`
+	Name   string         `json:"name"`
+	Strict bool           `json:"strict"`
+	Schema map[string]any `json:"schema"`
+}
+
+func buildResponsesRequest(
+	model string,
+	task string,
+	history []Step,
+	observation Observation,
+) ([]byte, error) {
+	prompt, err := modelPrompt(task, history, observation)
+	if err != nil {
+		return nil, err
+	}
+	request := responsesRequest{
+		Model: model,
+		Input: []responsesInput{{
+			Role: "user",
+			Content: []responsesContent{
+				{Type: "input_text", Text: prompt},
+				{
+					Type: "input_image",
+					ImageURL: "data:image/png;base64," +
+						base64.StdEncoding.EncodeToString(observation.Screenshot),
+				},
+			},
+		}},
+		Text: responsesTextSettings{Format: responsesFormat{
+			Type:   "json_schema",
+			Name:   "agentbox_action",
+			Strict: true,
+			Schema: decisionSchema(),
+		}},
+	}
+	return json.Marshal(request)
+}
+
+func modelPrompt(task string, history []Step, observation Observation) (string, error) {
+	contextData := struct {
+		Task            string                    `json:"task"`
+		History         []Step                    `json:"history"`
+		RecentLogs      []environment.LogEntry    `json:"recent_logs"`
+		PreviousActions []environment.InputAction `json:"previous_actions"`
+	}{
+		Task:            task,
+		History:         history,
+		RecentLogs:      observation.Logs,
+		PreviousActions: observation.PreviousActions,
+	}
+	data, err := json.Marshal(contextData)
+	if err != nil {
+		return "", err
+	}
+	return "You control an interactive Linux application from screenshots. " +
+		"Choose exactly one safe input action, or mark done when the task is complete. " +
+		"Use logical key names such as RIGHT, ENTER, or ESCAPE. Keep waits under 5000 ms.\n" +
+		string(data), nil
+}
+
+func decisionSchema() map[string]any {
+	actionProperties := map[string]any{
+		"type": map[string]any{
+			"type": "string",
+			"enum": []string{
+				string(environment.KeyDown), string(environment.KeyUp),
+				string(environment.MouseMove), string(environment.MouseDown),
+				string(environment.MouseUp), string(environment.Wait),
+			},
+		},
+		"key":         map[string]any{"type": "string"},
+		"x":           map[string]any{"type": "integer"},
+		"y":           map[string]any{"type": "integer"},
+		"button":      map[string]any{"type": "integer"},
+		"duration_ms": map[string]any{"type": "integer"},
+	}
+	return map[string]any{
+		"type":                 "object",
+		"additionalProperties": false,
+		"properties": map[string]any{
+			"reason": map[string]any{"type": "string"},
+			"done":   map[string]any{"type": "boolean"},
+			"action": map[string]any{
+				"anyOf": []any{
+					map[string]any{
+						"type":                 "object",
+						"additionalProperties": false,
+						"properties":           actionProperties,
+						"required": []string{
+							"type", "key", "x", "y", "button", "duration_ms",
+						},
+					},
+					map[string]any{"type": "null"},
+				},
+			},
+		},
+		"required": []string{"reason", "done", "action"},
+	}
+}
+
+func extractResponseText(data []byte) (string, error) {
+	var response struct {
+		Output []struct {
+			Content []struct {
+				Type string `json:"type"`
+				Text string `json:"text"`
+			} `json:"content"`
+		} `json:"output"`
+	}
+	if err := json.Unmarshal(data, &response); err != nil {
+		return "", fmt.Errorf("decode OpenAI response: %w", err)
+	}
+	for _, output := range response.Output {
+		for _, content := range output.Content {
+			if content.Type == "output_text" && content.Text != "" {
+				return content.Text, nil
+			}
+		}
+	}
+	return "", errors.New("OpenAI response contained no output_text")
+}
+
+type modelDecision struct {
+	Reason string       `json:"reason"`
+	Done   bool         `json:"done"`
+	Action *modelAction `json:"action"`
+}
+
+type modelAction struct {
+	Type       environment.InputType `json:"type"`
+	Key        string                `json:"key"`
+	X          int                   `json:"x"`
+	Y          int                   `json:"y"`
+	Button     int                   `json:"button"`
+	DurationMS int                   `json:"duration_ms"`
+}
+
+func parseModelDecision(text string) (Decision, error) {
+	var modelOutput modelDecision
+	if err := json.Unmarshal([]byte(text), &modelOutput); err != nil {
+		return Decision{}, fmt.Errorf("decode model decision: %w", err)
+	}
+	if modelOutput.Reason == "" {
+		return Decision{}, errors.New("model decision requires reason")
+	}
+	if modelOutput.Done {
+		return Decision{Reason: modelOutput.Reason, Done: true}, nil
+	}
+	if modelOutput.Action == nil {
+		return Decision{}, errors.New("model decision requires action when not done")
+	}
+	if modelOutput.Action.DurationMS < 0 || modelOutput.Action.DurationMS > 5000 {
+		return Decision{}, errors.New("model wait must be between 0 and 5000 ms")
+	}
+	action := environment.InputAction{
+		Type:       modelOutput.Action.Type,
+		Key:        modelOutput.Action.Key,
+		X:          modelOutput.Action.X,
+		Y:          modelOutput.Action.Y,
+		Button:     modelOutput.Action.Button,
+		DurationMS: modelOutput.Action.DurationMS,
+	}
+	switch action.Type {
+	case environment.KeyDown, environment.KeyUp, environment.MouseMove,
+		environment.MouseDown, environment.MouseUp, environment.Wait:
+	default:
+		return Decision{}, fmt.Errorf("model returned unsupported action %q", action.Type)
+	}
+	return Decision{Reason: modelOutput.Reason, Action: &action}, nil
+}

agentbox/internal/appspec/appspec.go

@@ -17,6 +17,8 @@ type manifest struct {
 	WindowTitle string            `json:"window_title"`
 }
 
+// Resolve turns either an executable path or a directory containing
+// agentbox.json into the backend-neutral launch command.
 func Resolve(path string) (environment.Command, error) {
 	absolute, err := filepath.Abs(path)
 	if err != nil {

agentbox/internal/environment/dockerx11/application.go

@@ -0,0 +1,175 @@
+package dockerx11
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"os/exec"
+	"strings"
+	"time"
+
+	"agentbox/internal/environment"
+)
+
+const stagedApplicationPath = "/tmp/application"
+
+func (environmentBackend *Environment) Launch(
+	ctx context.Context,
+	application environment.Command,
+) error {
+	fmt.Fprintln(environmentBackend.config.Output, "Uploading build...")
+	executable, err := os.Open(application.Path)
+	if err != nil {
+		return fmt.Errorf("open application: %w", err)
+	}
+	defer executable.Close()
+
+	// Stream instead of bind-mounting the developer's directory. The container
+	// sees only the requested executable, and the copy disappears at teardown.
+	stageCommand := exec.CommandContext(
+		ctx,
+		"docker",
+		"exec",
+		"-i",
+		environmentBackend.containerName,
+		"sh",
+		"-c",
+		"cat > "+stagedApplicationPath+" && chmod 0500 "+stagedApplicationPath,
+	)
+	stageCommand.Stdin = executable
+	if output, err := stageCommand.CombinedOutput(); err != nil {
+		return fmt.Errorf(
+			"stage application: %w: %s",
+			err,
+			strings.TrimSpace(string(output)),
+		)
+	}
+
+	fmt.Fprintln(environmentBackend.config.Output, "Launching application...")
+	dockerArguments := []string{"exec", "-d"}
+	for variableName, variableValue := range application.Env {
+		dockerArguments = append(
+			dockerArguments,
+			"-e",
+			variableName+"="+variableValue,
+		)
+	}
+	dockerArguments = append(
+		dockerArguments,
+		environmentBackend.containerName,
+		"sh",
+		"-c",
+	)
+	shellCommand := []string{"exec", stagedApplicationPath}
+	for _, applicationArgument := range application.Args {
+		shellCommand = append(shellCommand, shellQuote(applicationArgument))
+	}
+	shellCommand = append(
+		shellCommand,
+		">/tmp/stdout.log",
+		"2>/tmp/stderr.log",
+	)
+	dockerArguments = append(dockerArguments, strings.Join(shellCommand, " "))
+	if _, err := environmentBackend.runDocker(ctx, dockerArguments...); err != nil {
+		return fmt.Errorf("launch application: %w", err)
+	}
+
+	return environmentBackend.focusApplicationWindow(ctx, application.WindowTitle)
+}
+
+func (environmentBackend *Environment) focusApplicationWindow(
+	ctx context.Context,
+	windowTitle string,
+) error {
+	if windowTitle == "" {
+		// Applications without a title cannot be searched reliably. Give the
+		// process a brief startup window before the first screenshot.
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case <-time.After(500 * time.Millisecond):
+			return nil
+		}
+	}
+
+	var windowID string
+	if err := waitFor(ctx, 10*time.Second, func() bool {
+		output, searchErr := environmentBackend.runDocker(
+			ctx,
+			"exec",
+			environmentBackend.containerName,
+			"xdotool",
+			"search",
+			"--name",
+			windowTitle,
+		)
+		if searchErr != nil {
+			return false
+		}
+		windowID = strings.TrimSpace(strings.Split(string(output), "\n")[0])
+		return windowID != ""
+	}); err != nil {
+		return fmt.Errorf("wait for application window %q: %w", windowTitle, err)
+	}
+	if _, err := environmentBackend.runDocker(
+		ctx,
+		"exec",
+		environmentBackend.containerName,
+		"xdotool",
+		"windowactivate",
+		"--sync",
+		windowID,
+	); err != nil {
+		return fmt.Errorf("focus application window: %w", err)
+	}
+	return nil
+}
+
+func (environmentBackend *Environment) Screenshot(ctx context.Context) ([]byte, error) {
+	const screenshotPath = "/tmp/screenshot.png"
+	if _, err := environmentBackend.runDocker(
+		ctx,
+		"exec",
+		environmentBackend.containerName,
+		"scrot",
+		"-o",
+		screenshotPath,
+	); err != nil {
+		return nil, fmt.Errorf("capture screenshot: %w", err)
+	}
+	screenshot, err := environmentBackend.runDocker(
+		ctx,
+		"exec",
+		environmentBackend.containerName,
+		"cat",
+		screenshotPath,
+	)
+	if err != nil {
+		return nil, fmt.Errorf("extract screenshot: %w", err)
+	}
+	return screenshot, nil
+}
+
+func (environmentBackend *Environment) Logs(
+	ctx context.Context,
+) ([]environment.LogEntry, error) {
+	var logEntries []environment.LogEntry
+	for _, streamName := range []string{"stdout", "stderr"} {
+		logContents, err := environmentBackend.runDocker(
+			ctx,
+			"exec",
+			environmentBackend.containerName,
+			"cat",
+			"/tmp/"+streamName+".log",
+		)
+		if err != nil || len(logContents) == 0 {
+			continue
+		}
+		logEntries = append(logEntries, environment.LogEntry{
+			Stream:  streamName,
+			Message: string(logContents),
+			Time:    time.Now().UTC(),
+		})
+	}
+	return logEntries, nil
+}

agentbox/internal/environment/dockerx11/commands.go

@@ -0,0 +1,61 @@
+package dockerx11
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"os/exec"
+	"strings"
+	"time"
+)
+
+func (environmentBackend *Environment) runDocker(
+	ctx context.Context,
+	arguments ...string,
+) ([]byte, error) {
+	command := exec.CommandContext(ctx, "docker", arguments...)
+	output, err := command.CombinedOutput()
+	if err != nil {
+		return output, fmt.Errorf(
+			"docker %s: %w: %s",
+			arguments[0],
+			err,
+			strings.TrimSpace(string(output)),
+		)
+	}
+	return output, nil
+}
+
+func (environmentBackend *Environment) streamDockerOutput(
+	ctx context.Context,
+	arguments ...string,
+) error {
+	command := exec.CommandContext(ctx, "docker", arguments...)
+	command.Stdout = environmentBackend.config.Output
+	command.Stderr = environmentBackend.config.Output
+	return command.Run()
+}
+
+func waitFor(ctx context.Context, timeout time.Duration, ready func() bool) error {
+	timeoutTimer := time.NewTimer(timeout)
+	defer timeoutTimer.Stop()
+	retryTicker := time.NewTicker(100 * time.Millisecond)
+	defer retryTicker.Stop()
+
+	for {
+		if ready() {
+			return nil
+		}
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case <-timeoutTimer.C:
+			return errors.New("timed out")
+		case <-retryTicker.C:
+		}
+	}
+}
+
+func shellQuote(value string) string {
+	return "'" + strings.ReplaceAll(value, "'", "'\"'\"'") + "'"
+}

agentbox/internal/environment/dockerx11/docker.go

@@ -1,35 +1,30 @@
 package dockerx11
 
 import (
-	"context"
-	"errors"
-	"fmt"
 	"io"
-	"os"
-	"os/exec"
-	"path/filepath"
-	"strconv"
-	"strings"
 	"sync"
-	"time"
 
 	"agentbox/internal/environment"
 )
 
-const imageName = "agentbox-runtime:local"
+const runtimeImageName = "agentbox-runtime:local"
 
+// Config contains host-side values needed to create one isolated desktop.
 type Config struct {
 	ProjectRoot string
 	RunID       string
 	Output      io.Writer
 }
 
+// Environment implements the backend-neutral environment contract with one
+// Docker container, one X11 display, and one application process.
 type Environment struct {
 	config        Config
 	containerName string
-	created       bool
-	stopped       bool
-	mu            sync.Mutex
+
+	stopMutex        sync.Mutex
+	containerCreated bool
+	containerStopped bool
 }
 
 func New(config Config) *Environment {
@@ -39,257 +34,4 @@ func New(config Config) *Environment {
 	}
 }
 
-func (e *Environment) Start(ctx context.Context) error {
-	if e.config.Output == nil {
-		e.config.Output = io.Discard
-	}
-	if _, err := e.docker(ctx, "version", "--format", "{{.Server.Version}}"); err != nil {
-		return fmt.Errorf("Docker is required and the daemon must be accessible: %w", err)
-	}
-	fmt.Fprintln(e.config.Output, "Creating environment...")
-	if err := e.dockerStream(ctx, "build", "-q", "-t", imageName, "-f",
-		filepath.Join(e.config.ProjectRoot, "environment", "Dockerfile"), e.config.ProjectRoot); err != nil {
-		return fmt.Errorf("build environment image: %w", err)
-	}
-	_, err := e.docker(ctx,
-		"create",
-		"--name", e.containerName,
-		"--init",
-		"--network=none",
-		"--read-only",
-		"--tmpfs=/tmp:rw,exec,nosuid,nodev,size=128m",
-		"--cap-drop=ALL",
-		"--security-opt=no-new-privileges",
-		"--pids-limit=128",
-		"--memory=512m",
-		"--cpus=1",
-		imageName,
-	)
-	if err != nil {
-		return fmt.Errorf("create environment: %w", err)
-	}
-	e.created = true
-	if _, err := e.docker(ctx, "start", e.containerName); err != nil {
-		return fmt.Errorf("start environment: %w", err)
-	}
-	if err := e.waitFor(ctx, 10*time.Second, func() bool {
-		_, readyErr := e.docker(ctx, "exec", e.containerName, "test", "-f", "/tmp/agentbox-ready")
-		return readyErr == nil
-	}); err != nil {
-		return fmt.Errorf("wait for graphical environment: %w", err)
-	}
-	return nil
-}
-
-func (e *Environment) Launch(ctx context.Context, command environment.Command) error {
-	fmt.Fprintln(e.config.Output, "Uploading build...")
-	file, err := os.Open(command.Path)
-	if err != nil {
-		return fmt.Errorf("open application: %w", err)
-	}
-	defer file.Close()
-
-	stage := exec.CommandContext(ctx, "docker", "exec", "-i", e.containerName, "sh", "-c",
-		"cat > /tmp/application && chmod 0500 /tmp/application")
-	stage.Stdin = file
-	if output, err := stage.CombinedOutput(); err != nil {
-		return fmt.Errorf("stage application: %w: %s", err, strings.TrimSpace(string(output)))
-	}
-
-	fmt.Fprintln(e.config.Output, "Launching application...")
-	args := []string{"exec", "-d"}
-	for key, value := range command.Env {
-		args = append(args, "-e", key+"="+value)
-	}
-	args = append(args, e.containerName, "sh", "-c")
-	parts := []string{"exec", "/tmp/application"}
-	for _, arg := range command.Args {
-		parts = append(parts, shellQuote(arg))
-	}
-	parts = append(parts, ">/tmp/stdout.log", "2>/tmp/stderr.log")
-	args = append(args, strings.Join(parts, " "))
-	if _, err := e.docker(ctx, args...); err != nil {
-		return fmt.Errorf("launch application: %w", err)
-	}
-
-	if command.WindowTitle != "" {
-		var windowID string
-		if err := e.waitFor(ctx, 10*time.Second, func() bool {
-			output, searchErr := e.docker(ctx, "exec", e.containerName, "xdotool",
-				"search", "--name", command.WindowTitle)
-			if searchErr != nil {
-				return false
-			}
-			windowID = strings.TrimSpace(strings.Split(string(output), "\n")[0])
-			return windowID != ""
-		}); err != nil {
-			return fmt.Errorf("wait for application window %q: %w", command.WindowTitle, err)
-		}
-		if _, err := e.docker(ctx, "exec", e.containerName, "xdotool",
-			"windowactivate", "--sync", windowID); err != nil {
-			return fmt.Errorf("focus application window: %w", err)
-		}
-	} else {
-		select {
-		case <-ctx.Done():
-			return ctx.Err()
-		case <-time.After(500 * time.Millisecond):
-		}
-	}
-	return nil
-}
-
-func (e *Environment) Screenshot(ctx context.Context) ([]byte, error) {
-	if _, err := e.docker(ctx, "exec", e.containerName, "scrot", "-o", "/tmp/screenshot.png"); err != nil {
-		return nil, fmt.Errorf("capture screenshot: %w", err)
-	}
-	data, err := e.docker(ctx, "exec", e.containerName, "cat", "/tmp/screenshot.png")
-	if err != nil {
-		return nil, fmt.Errorf("extract screenshot: %w", err)
-	}
-	return data, nil
-}
-
-func (e *Environment) SendInput(ctx context.Context, action environment.InputAction) error {
-	var args []string
-	switch action.Type {
-	case environment.KeyDown:
-		if action.Key == "" {
-			return errors.New("key_down requires key")
-		}
-		args = []string{"keydown", x11Key(action.Key)}
-	case environment.KeyUp:
-		if action.Key == "" {
-			return errors.New("key_up requires key")
-		}
-		args = []string{"keyup", x11Key(action.Key)}
-	case environment.MouseMove:
-		args = []string{"mousemove", strconv.Itoa(action.X), strconv.Itoa(action.Y)}
-	case environment.MouseDown:
-		args = []string{"mousedown", strconv.Itoa(action.Button)}
-	case environment.MouseUp:
-		args = []string{"mouseup", strconv.Itoa(action.Button)}
-	case environment.Wait:
-		if action.DurationMS < 0 {
-			return errors.New("wait duration cannot be negative")
-		}
-		select {
-		case <-ctx.Done():
-			return ctx.Err()
-		case <-time.After(time.Duration(action.DurationMS) * time.Millisecond):
-			return nil
-		}
-	default:
-		return fmt.Errorf("unsupported input action %q", action.Type)
-	}
-	_, err := e.docker(ctx, append([]string{"exec", e.containerName, "xdotool"}, args...)...)
-	if err != nil {
-		return fmt.Errorf("send %s: %w", action.Type, err)
-	}
-	return nil
-}
-
-func (e *Environment) Logs(ctx context.Context) ([]environment.LogEntry, error) {
-	var entries []environment.LogEntry
-	for _, stream := range []string{"stdout", "stderr"} {
-		data, err := e.docker(ctx, "exec", e.containerName, "cat", "/tmp/"+stream+".log")
-		if err != nil {
-			continue
-		}
-		if len(data) > 0 {
-			entries = append(entries, environment.LogEntry{
-				Stream: stream, Message: string(data), Time: time.Now().UTC(),
-			})
-		}
-	}
-	return entries, nil
-}
-
-func (e *Environment) Stop(ctx context.Context) error {
-	e.mu.Lock()
-	defer e.mu.Unlock()
-	if !e.created || e.stopped {
-		return nil
-	}
-	e.stopped = true
-	fmt.Fprintln(e.config.Output, "Shutting environment down...")
-	_, stopErr := e.docker(ctx, "stop", "--time=3", e.containerName)
-	_, removeErr := e.docker(ctx, "rm", "-f", e.containerName)
-	if stopErr != nil {
-		return fmt.Errorf("stop environment: %w", stopErr)
-	}
-	if removeErr != nil {
-		return fmt.Errorf("remove environment: %w", removeErr)
-	}
-	return nil
-}
-
-func (e *Environment) waitFor(ctx context.Context, timeout time.Duration, check func() bool) error {
-	timer := time.NewTimer(timeout)
-	defer timer.Stop()
-	ticker := time.NewTicker(100 * time.Millisecond)
-	defer ticker.Stop()
-	for {
-		if check() {
-			return nil
-		}
-		select {
-		case <-ctx.Done():
-			return ctx.Err()
-		case <-timer.C:
-			return errors.New("timed out")
-		case <-ticker.C:
-		}
-	}
-}
-
-func (e *Environment) docker(ctx context.Context, args ...string) ([]byte, error) {
-	command := exec.CommandContext(ctx, "docker", args...)
-	output, err := command.CombinedOutput()
-	if err != nil {
-		return output, fmt.Errorf("docker %s: %w: %s", args[0], err, strings.TrimSpace(string(output)))
-	}
-	return output, nil
-}
-
-func (e *Environment) dockerStream(ctx context.Context, args ...string) error {
-	command := exec.CommandContext(ctx, "docker", args...)
-	command.Stdout = e.config.Output
-	command.Stderr = e.config.Output
-	return command.Run()
-}
-
-func shellQuote(value string) string {
-	return "'" + strings.ReplaceAll(value, "'", "'\"'\"'") + "'"
-}
-
-func x11Key(key string) string {
-	switch strings.ToUpper(key) {
-	case "LEFT":
-		return "Left"
-	case "RIGHT":
-		return "Right"
-	case "UP":
-		return "Up"
-	case "DOWN":
-		return "Down"
-	case "ENTER", "RETURN":
-		return "Return"
-	case "ESC", "ESCAPE":
-		return "Escape"
-	case "SPACE":
-		return "space"
-	case "TAB":
-		return "Tab"
-	case "BACKSPACE":
-		return "BackSpace"
-	case "DELETE":
-		return "Delete"
-	}
-	if len(key) == 1 {
-		return strings.ToLower(key)
-	}
-	return key
-}
-
 var _ environment.Environment = (*Environment)(nil)

agentbox/internal/environment/dockerx11/docker_test.go

@@ -11,8 +11,8 @@ func TestX11KeyTranslatesLogicalNames(t *testing.T) {
 		"A":      "a",
 	}
 	for input, want := range tests {
-		if got := x11Key(input); got != want {
-			t.Errorf("x11Key(%q) = %q, want %q", input, got, want)
+		if got := toX11KeyName(input); got != want {
+			t.Errorf("toX11KeyName(%q) = %q, want %q", input, got, want)
 		}
 	}
 }

agentbox/internal/environment/dockerx11/input.go

@@ -0,0 +1,95 @@
+package dockerx11
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"strconv"
+	"strings"
+	"time"
+
+	"agentbox/internal/environment"
+)
+
+func (environmentBackend *Environment) SendInput(
+	ctx context.Context,
+	action environment.InputAction,
+) error {
+	var xdotoolArguments []string
+	switch action.Type {
+	case environment.KeyDown:
+		if action.Key == "" {
+			return errors.New("key_down requires key")
+		}
+		xdotoolArguments = []string{"keydown", toX11KeyName(action.Key)}
+	case environment.KeyUp:
+		if action.Key == "" {
+			return errors.New("key_up requires key")
+		}
+		xdotoolArguments = []string{"keyup", toX11KeyName(action.Key)}
+	case environment.MouseMove:
+		xdotoolArguments = []string{
+			"mousemove",
+			strconv.Itoa(action.X),
+			strconv.Itoa(action.Y),
+		}
+	case environment.MouseDown:
+		xdotoolArguments = []string{"mousedown", strconv.Itoa(action.Button)}
+	case environment.MouseUp:
+		xdotoolArguments = []string{"mouseup", strconv.Itoa(action.Button)}
+	case environment.Wait:
+		if action.DurationMS < 0 {
+			return errors.New("wait duration cannot be negative")
+		}
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case <-time.After(time.Duration(action.DurationMS) * time.Millisecond):
+			return nil
+		}
+	default:
+		return fmt.Errorf("unsupported input action %q", action.Type)
+	}
+
+	dockerArguments := []string{
+		"exec",
+		environmentBackend.containerName,
+		"xdotool",
+	}
+	dockerArguments = append(dockerArguments, xdotoolArguments...)
+	if _, err := environmentBackend.runDocker(ctx, dockerArguments...); err != nil {
+		return fmt.Errorf("send %s: %w", action.Type, err)
+	}
+	return nil
+}
+
+// toX11KeyName keeps X11 spellings out of the public action API. Agents can use
+// logical names such as RIGHT even though xdotool expects Right.
+func toX11KeyName(logicalName string) string {
+	switch strings.ToUpper(logicalName) {
+	case "LEFT":
+		return "Left"
+	case "RIGHT":
+		return "Right"
+	case "UP":
+		return "Up"
+	case "DOWN":
+		return "Down"
+	case "ENTER", "RETURN":
+		return "Return"
+	case "ESC", "ESCAPE":
+		return "Escape"
+	case "SPACE":
+		return "space"
+	case "TAB":
+		return "Tab"
+	case "BACKSPACE":
+		return "BackSpace"
+	case "DELETE":
+		return "Delete"
+	}
+	if len(logicalName) == 1 {
+		return strings.ToLower(logicalName)
+	}
+	return logicalName
+}

agentbox/internal/environment/dockerx11/lifecycle.go

@@ -0,0 +1,119 @@
+package dockerx11
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"path/filepath"
+	"time"
+)
+
+func (environmentBackend *Environment) Start(ctx context.Context) error {
+	if environmentBackend.config.Output == nil {
+		environmentBackend.config.Output = io.Discard
+	}
+	if _, err := environmentBackend.runDocker(
+		ctx,
+		"version",
+		"--format",
+		"{{.Server.Version}}",
+	); err != nil {
+		return fmt.Errorf("Docker is required and the daemon must be accessible: %w", err)
+	}
+
+	fmt.Fprintln(environmentBackend.config.Output, "Creating environment...")
+	dockerfilePath := filepath.Join(
+		environmentBackend.config.ProjectRoot,
+		"environment",
+		"Dockerfile",
+	)
+	if err := environmentBackend.streamDockerOutput(
+		ctx,
+		"build",
+		"-q",
+		"-t",
+		runtimeImageName,
+		"-f",
+		dockerfilePath,
+		environmentBackend.config.ProjectRoot,
+	); err != nil {
+		return fmt.Errorf("build environment image: %w", err)
+	}
+
+	// These restrictions reduce accidental damage. They are defense in depth,
+	// not a safe boundary for hostile customer code; see docs/architecture.md.
+	_, err := environmentBackend.runDocker(
+		ctx,
+		"create",
+		"--name", environmentBackend.containerName,
+		"--init",
+		"--network=none",
+		"--read-only",
+		"--tmpfs=/tmp:rw,exec,nosuid,nodev,size=128m",
+		"--cap-drop=ALL",
+		"--security-opt=no-new-privileges",
+		"--pids-limit=128",
+		"--memory=512m",
+		"--cpus=1",
+		runtimeImageName,
+	)
+	if err != nil {
+		return fmt.Errorf("create environment: %w", err)
+	}
+	environmentBackend.containerCreated = true
+
+	if _, err := environmentBackend.runDocker(
+		ctx,
+		"start",
+		environmentBackend.containerName,
+	); err != nil {
+		return fmt.Errorf("start environment: %w", err)
+	}
+	if err := waitFor(ctx, 10*time.Second, func() bool {
+		_, readyErr := environmentBackend.runDocker(
+			ctx,
+			"exec",
+			environmentBackend.containerName,
+			"test",
+			"-f",
+			"/tmp/agentbox-ready",
+		)
+		return readyErr == nil
+	}); err != nil {
+		return fmt.Errorf("wait for graphical environment: %w", err)
+	}
+	return nil
+}
+
+// Stop is idempotent because both normal completion and deferred cleanup may
+// try to tear down the same environment.
+func (environmentBackend *Environment) Stop(ctx context.Context) error {
+	environmentBackend.stopMutex.Lock()
+	defer environmentBackend.stopMutex.Unlock()
+
+	if !environmentBackend.containerCreated || environmentBackend.containerStopped {
+		return nil
+	}
+	environmentBackend.containerStopped = true
+	fmt.Fprintln(environmentBackend.config.Output, "Shutting environment down...")
+
+	_, stopErr := environmentBackend.runDocker(
+		ctx,
+		"stop",
+		"--time=3",
+		environmentBackend.containerName,
+	)
+	_, removeErr := environmentBackend.runDocker(
+		ctx,
+		"rm",
+		"-f",
+		environmentBackend.containerName,
+	)
+	if stopErr != nil {
+		return fmt.Errorf("stop environment: %w", stopErr)
+	}
+	if removeErr != nil {
+		return fmt.Errorf("remove environment: %w", removeErr)
+	}
+	return nil
+}

agentbox/internal/environment/environment.go

@@ -5,6 +5,8 @@ import (
 	"time"
 )
 
+// Command describes a host executable and how it should start in an
+// environment. Environment implementations decide how to stage the file.
 type Command struct {
 	Path        string
 	Args        []string
@@ -12,6 +14,7 @@ type Command struct {
 	WindowTitle string
 }
 
+// InputType identifies one backend-neutral keyboard, mouse, or timing action.
 type InputType string
 
 const (
@@ -23,6 +26,7 @@ const (
 	Wait      InputType = "wait"
 )
 
+// InputAction contains only fields relevant to Type. DurationMS is used by Wait.
 type InputAction struct {
 	Type       InputType `json:"type"`
 	Key        string    `json:"key,omitempty"`
@@ -32,12 +36,15 @@ type InputAction struct {
 	DurationMS int       `json:"duration_ms,omitempty"`
 }
 
+// LogEntry is a snapshot of one application output stream.
 type LogEntry struct {
 	Stream  string    `json:"stream"`
 	Message string    `json:"message"`
 	Time    time.Time `json:"time"`
 }
 
+// Environment is the programmable-computer boundary used by the runtime.
+// Implementations may use containers, virtual machines, or remote hosts.
 type Environment interface {
 	Start(context.Context) error
 	Launch(context.Context, Command) error

agentbox/internal/runtime/runtime.go

@@ -12,14 +12,16 @@ import (
 	"agentbox/internal/trace"
 )
 
+// Config wires replaceable environment, agent, and trace implementations into
+// one run. The runtime itself has no Docker, X11, or model-provider knowledge.
 type Config struct {
-	Task     string
-	Command  environment.Command
-	Agent    agent.Agent
-	Env      environment.Environment
-	Trace    *trace.Store
-	MaxSteps int
-	Output   io.Writer
+	Task        string
+	Command     environment.Command
+	Controller  agent.Agent
+	Environment environment.Environment
+	TraceStore  *trace.Store
+	MaxSteps    int
+	Output      io.Writer
 }
 
 func Run(ctx context.Context, config Config) (runErr error) {
@@ -29,88 +31,85 @@ func Run(ctx context.Context, config Config) (runErr error) {
 	if config.Output == nil {
 		config.Output = io.Discard
 	}
-	if config.Agent == nil || config.Env == nil || config.Trace == nil {
+	if config.Controller == nil || config.Environment == nil || config.TraceStore == nil {
 		return errors.New("runtime requires agent, environment, and trace")
 	}
 
-	started := false
 	defer func() {
-		cleanupCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-		defer cancel()
-		if started {
-			if logs, err := config.Env.Logs(cleanupCtx); err == nil {
-				if logErr := config.Trace.WriteLogs(logs); runErr == nil && logErr != nil {
-					runErr = logErr
-				}
-			}
-			if stopErr := config.Env.Stop(cleanupCtx); runErr == nil && stopErr != nil {
-				runErr = stopErr
-			}
-		}
-		if finishErr := config.Trace.Finish(runErr); runErr == nil && finishErr != nil {
-			runErr = finishErr
-		}
+		runErr = finalizeRun(config, runErr)
 	}()
 
-	if err := config.Env.Start(ctx); err != nil {
+	if err := config.Environment.Start(ctx); err != nil {
 		return err
 	}
-	started = true
-	if err := config.Env.Launch(ctx, config.Command); err != nil {
+	if err := config.Environment.Launch(ctx, config.Command); err != nil {
 		return err
 	}
 	fmt.Fprintln(config.Output, "Agent attached.")
 
-	var history []agent.Step
-	var actions []environment.InputAction
-	startedAt := time.Now()
-	for number := 1; number <= config.MaxSteps; number++ {
-		screenshot, err := config.Env.Screenshot(ctx)
+	var stepHistory []agent.Step
+	var actionHistory []environment.InputAction
+	runStartedAt := time.Now()
+	for stepNumber := 1; stepNumber <= config.MaxSteps; stepNumber++ {
+		screenshot, err := config.Environment.Screenshot(ctx)
 		if err != nil {
 			return err
 		}
-		screenshotPath, err := config.Trace.SaveScreenshot(number, screenshot)
+		screenshotPath, err := config.TraceStore.SaveScreenshot(stepNumber, screenshot)
 		if err != nil {
 			return err
 		}
-		logs, err := config.Env.Logs(ctx)
+		applicationLogs, err := config.Environment.Logs(ctx)
 		if err != nil {
 			return err
 		}
-		now := time.Now().UTC()
-		fmt.Fprintf(config.Output, "[%s] screenshot captured\n", elapsed(startedAt))
-		decision, err := config.Agent.NextAction(ctx, config.Task, history, agent.Observation{
+		observedAt := time.Now().UTC()
+		previousActions := append([]environment.InputAction(nil), actionHistory...)
+		observation := agent.Observation{
 			Screenshot:      screenshot,
-			Timestamp:       now,
-			Logs:            logs,
-			PreviousActions: append([]environment.InputAction(nil), actions...),
-		})
+			Timestamp:       observedAt,
+			Logs:            applicationLogs,
+			PreviousActions: previousActions,
+		}
+		fmt.Fprintf(config.Output, "[%s] screenshot captured\n", elapsed(runStartedAt))
+
+		decision, err := config.Controller.NextAction(
+			ctx,
+			config.Task,
+			stepHistory,
+			observation,
+		)
 		if err != nil {
 			return fmt.Errorf("agent next action: %w", err)
 		}
-		fmt.Fprintf(config.Output, "[%s] agent: %s\n", elapsed(startedAt), decision.Reason)
-		record := trace.StepRecord{
-			Step:      number,
-			Timestamp: now,
+		fmt.Fprintf(
+			config.Output,
+			"[%s] agent: %s\n",
+			elapsed(runStartedAt),
+			decision.Reason,
+		)
+		if err := config.TraceStore.Record(trace.StepRecord{
+			Step:      stepNumber,
+			Timestamp: observedAt,
 			Observation: trace.ObservationRecord{
 				Screenshot:      screenshotPath,
-				Logs:            logs,
-				PreviousActions: append([]environment.InputAction(nil), actions...),
+				Logs:            applicationLogs,
+				PreviousActions: previousActions,
 			},
 			Agent:  trace.AgentRecord{Message: decision.Reason},
 			Action: decision.Action,
 			Done:   decision.Done,
-		}
-		if err := config.Trace.Record(record); err != nil {
+		}); err != nil {
 			return err
 		}
-		history = append(history, agent.Step{
-			Number:         number,
-			Timestamp:      now,
+		stepHistory = append(stepHistory, agent.Step{
+			Number:         stepNumber,
+			Timestamp:      observedAt,
 			ScreenshotPath: screenshotPath,
 			Message:        decision.Reason,
 			Action:         decision.Action,
 		})
+
 		if decision.Done {
 			fmt.Fprintln(config.Output, "Task complete.")
 			return nil
@@ -118,19 +117,53 @@ func Run(ctx context.Context, config Config) (runErr error) {
 		if decision.Action == nil {
 			return errors.New("agent returned neither action nor completion")
 		}
-		fmt.Fprintf(config.Output, "[%s] action: %s%s\n",
-			elapsed(startedAt), decision.Action.Type, actionDetail(*decision.Action))
-		if err := config.Env.SendInput(ctx, *decision.Action); err != nil {
+		fmt.Fprintf(
+			config.Output,
+			"[%s] action: %s%s\n",
+			elapsed(runStartedAt),
+			decision.Action.Type,
+			actionDetail(*decision.Action),
+		)
+		if err := config.Environment.SendInput(ctx, *decision.Action); err != nil {
 			return err
 		}
-		actions = append(actions, *decision.Action)
+		actionHistory = append(actionHistory, *decision.Action)
 	}
 	return fmt.Errorf("agent exceeded maximum of %d steps", config.MaxSteps)
 }
 
+// finalizeRun uses a fresh timeout because the caller's context may already be
+// canceled. Preserving logs and removing the environment must still be tried.
+func finalizeRun(
+	config Config,
+	runErr error,
+) error {
+	cleanupCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	if logs, err := config.Environment.Logs(cleanupCtx); err == nil {
+		if logErr := config.TraceStore.WriteLogs(logs); runErr == nil && logErr != nil {
+			runErr = logErr
+		}
+	}
+	// Stop is idempotent, so always call it. Start may have created a container
+	// before returning an error while waiting for its graphical services.
+	if stopErr := config.Environment.Stop(cleanupCtx); runErr == nil && stopErr != nil {
+		runErr = stopErr
+	}
+	if finishErr := config.TraceStore.Finish(runErr); runErr == nil && finishErr != nil {
+		runErr = finishErr
+	}
+	return runErr
+}
+
 func elapsed(start time.Time) string {
 	duration := time.Since(start).Round(time.Second)
-	return fmt.Sprintf("%02d:%02d", int(duration.Minutes()), int(duration.Seconds())%60)
+	return fmt.Sprintf(
+		"%02d:%02d",
+		int(duration.Minutes()),
+		int(duration.Seconds())%60,
+	)
 }
 
 func actionDetail(action environment.InputAction) string {

agentbox/internal/runtime/runtime_test.go

@@ -0,0 +1,63 @@
+package runtime
+
+import (
+	"context"
+	"errors"
+	"io"
+	"testing"
+
+	"agentbox/internal/agent"
+	"agentbox/internal/environment"
+	"agentbox/internal/trace"
+)
+
+type failingStartEnvironment struct {
+	stopCalled bool
+}
+
+func (fake *failingStartEnvironment) Start(context.Context) error {
+	return errors.New("startup failed")
+}
+
+func (fake *failingStartEnvironment) Launch(context.Context, environment.Command) error {
+	return nil
+}
+
+func (fake *failingStartEnvironment) Screenshot(context.Context) ([]byte, error) {
+	return nil, nil
+}
+
+func (fake *failingStartEnvironment) SendInput(context.Context, environment.InputAction) error {
+	return nil
+}
+
+func (fake *failingStartEnvironment) Logs(context.Context) ([]environment.LogEntry, error) {
+	return nil, nil
+}
+
+func (fake *failingStartEnvironment) Stop(context.Context) error {
+	fake.stopCalled = true
+	return nil
+}
+
+func TestRunStopsEnvironmentWhenStartFails(t *testing.T) {
+	traceStore, err := trace.New(t.TempDir(), "task", "application", "test-agent")
+	if err != nil {
+		t.Fatal(err)
+	}
+	fakeEnvironment := &failingStartEnvironment{}
+
+	err = Run(context.Background(), Config{
+		Task:        "task",
+		Controller:  &agent.Deterministic{},
+		Environment: fakeEnvironment,
+		TraceStore:  traceStore,
+		Output:      io.Discard,
+	})
+	if err == nil {
+		t.Fatal("Run() error = nil, want startup error")
+	}
+	if !fakeEnvironment.stopCalled {
+		t.Fatal("Run() did not stop environment after startup failure")
+	}
+}

agentbox/internal/trace/reader.go

@@ -0,0 +1,94 @@
+package trace
+
+import (
+	"bufio"
+	"encoding/json"
+	"errors"
+	"os"
+	"path/filepath"
+	"sort"
+)
+
+func List(projectRoot string) ([]Run, error) {
+	runsDirectory := filepath.Join(projectRoot, ".agentbox", "runs")
+	directoryEntries, err := os.ReadDir(runsDirectory)
+	if errors.Is(err, os.ErrNotExist) {
+		return nil, nil
+	}
+	if err != nil {
+		return nil, err
+	}
+
+	var runs []Run
+	for _, entry := range directoryEntries {
+		if !entry.IsDir() {
+			continue
+		}
+		run, err := Read(projectRoot, entry.Name())
+		if err == nil && run.SchemaVersion == SchemaVersion {
+			runs = append(runs, run)
+		}
+	}
+	sort.Slice(runs, func(left, right int) bool {
+		return runs[left].StartedAt.After(runs[right].StartedAt)
+	})
+	return runs, nil
+}
+
+func Read(projectRoot, runID string) (Run, error) {
+	if err := validateRunID(runID); err != nil {
+		return Run{}, err
+	}
+	runPath := filepath.Join(
+		projectRoot,
+		".agentbox",
+		"runs",
+		runID,
+		"run.json",
+	)
+	data, err := os.ReadFile(runPath)
+	if err != nil {
+		return Run{}, err
+	}
+	var run Run
+	if err := json.Unmarshal(data, &run); err != nil {
+		return Run{}, err
+	}
+	return run, nil
+}
+
+func ReadSteps(projectRoot, runID string) ([]StepRecord, error) {
+	if err := validateRunID(runID); err != nil {
+		return nil, err
+	}
+	stepsPath := filepath.Join(
+		projectRoot,
+		".agentbox",
+		"runs",
+		runID,
+		"steps.jsonl",
+	)
+	stepsFile, err := os.Open(stepsPath)
+	if err != nil {
+		return nil, err
+	}
+	defer stepsFile.Close()
+
+	var steps []StepRecord
+	scanner := bufio.NewScanner(stepsFile)
+	for scanner.Scan() {
+		var step StepRecord
+		if err := json.Unmarshal(scanner.Bytes(), &step); err != nil {
+			return nil, err
+		}
+		steps = append(steps, step)
+	}
+	return steps, scanner.Err()
+}
+
+func validateRunID(runID string) error {
+	if filepath.Base(runID) != runID {
+		return errors.New("invalid run ID")
+	}
+	return nil
+}

agentbox/internal/trace/store.go

@@ -1,85 +1,52 @@
 package trace
 
 import (
-	"bufio"
 	"crypto/rand"
 	"encoding/hex"
 	"encoding/json"
-	"errors"
 	"fmt"
 	"os"
 	"path/filepath"
-	"sort"
 	"time"
 
 	"agentbox/internal/environment"
 )
 
-const SchemaVersion = "1"
-
-type Run struct {
-	SchemaVersion string    `json:"schema_version"`
-	ID            string    `json:"id"`
-	Task          string    `json:"task"`
-	Application   string    `json:"application"`
-	Agent         string    `json:"agent"`
-	Status        string    `json:"status"`
-	StartedAt     time.Time `json:"started_at"`
-	FinishedAt    time.Time `json:"finished_at,omitempty"`
-	StepCount     int       `json:"step_count"`
-	Error         string    `json:"error,omitempty"`
-}
-
-type ObservationRecord struct {
-	Screenshot      string                    `json:"screenshot"`
-	Logs            []environment.LogEntry    `json:"logs,omitempty"`
-	PreviousActions []environment.InputAction `json:"previous_actions,omitempty"`
-}
-
-type AgentRecord struct {
-	Message string `json:"message"`
-}
-
-type StepRecord struct {
-	Step        int                      `json:"step"`
-	Timestamp   time.Time                `json:"timestamp"`
-	Observation ObservationRecord        `json:"observation"`
-	Agent       AgentRecord              `json:"agent"`
-	Action      *environment.InputAction `json:"action,omitempty"`
-	Done        bool                     `json:"done,omitempty"`
-}
-
+// Store writes one run's metadata and append-only event streams.
 type Store struct {
-	root    string
-	runDir  string
-	run     Run
-	steps   *os.File
-	actions *os.File
+	runDirectory string
+	run          Run
+	stepsFile    *os.File
+	actionsFile  *os.File
 }
 
-func New(root, task, application, agentName string) (*Store, error) {
-	id, err := newID()
+func New(projectRoot, task, application, agentName string) (*Store, error) {
+	runID, err := newRunID()
 	if err != nil {
 		return nil, err
 	}
-	runDir := filepath.Join(root, ".agentbox", "runs", id)
-	if err := os.MkdirAll(filepath.Join(runDir, "screenshots"), 0o755); err != nil {
+	runDirectory := filepath.Join(projectRoot, ".agentbox", "runs", runID)
+	screenshotDirectory := filepath.Join(runDirectory, "screenshots")
+	if err := os.MkdirAll(screenshotDirectory, 0o755); err != nil {
 		return nil, fmt.Errorf("create run directory: %w", err)
 	}
-	steps, err := os.Create(filepath.Join(runDir, "steps.jsonl"))
+	stepsFile, err := os.Create(filepath.Join(runDirectory, "steps.jsonl"))
 	if err != nil {
 		return nil, fmt.Errorf("create step trace: %w", err)
 	}
-	actions, err := os.Create(filepath.Join(runDir, "actions.jsonl"))
+	actionsFile, err := os.Create(filepath.Join(runDirectory, "actions.jsonl"))
 	if err != nil {
-		_ = steps.Close()
+		_ = stepsFile.Close()
 		return nil, fmt.Errorf("create action trace: %w", err)
 	}
+
 	store := &Store{
-		root: root, runDir: runDir, steps: steps, actions: actions,
+		runDirectory: runDirectory,
+		stepsFile:    stepsFile,
+		actionsFile:  actionsFile,
 		run: Run{
 			SchemaVersion: SchemaVersion,
-			ID:            id,
+			ID:            runID,
 			Task:          task,
 			Application:   application,
 			Agent:         agentName,
@@ -87,143 +54,103 @@ func New(root, task, application, agentName string) (*Store, error) {
 			StartedAt:     time.Now().UTC(),
 		},
 	}
-	if err := store.writeRun(); err != nil {
-		_ = steps.Close()
-		_ = actions.Close()
+	if err := store.writeRunSummary(); err != nil {
+		_ = stepsFile.Close()
+		_ = actionsFile.Close()
 		return nil, err
 	}
 	return store, nil
 }
 
-func (s *Store) ID() string {
-	return s.run.ID
+func (store *Store) ID() string {
+	return store.run.ID
 }
 
-func (s *Store) Directory() string {
-	return s.runDir
+func (store *Store) Directory() string {
+	return store.runDirectory
 }
 
-func (s *Store) SaveScreenshot(step int, data []byte) (string, error) {
-	relative := filepath.Join("screenshots", fmt.Sprintf("%04d.png", step))
-	if err := os.WriteFile(filepath.Join(s.runDir, relative), data, 0o644); err != nil {
+func (store *Store) SaveScreenshot(stepNumber int, screenshot []byte) (string, error) {
+	relativePath := filepath.Join(
+		"screenshots",
+		fmt.Sprintf("%04d.png", stepNumber),
+	)
+	absolutePath := filepath.Join(store.runDirectory, relativePath)
+	if err := os.WriteFile(absolutePath, screenshot, 0o644); err != nil {
 		return "", fmt.Errorf("write screenshot: %w", err)
 	}
-	return filepath.ToSlash(relative), nil
+	// Trace paths always use slash separators so traces are portable.
+	return filepath.ToSlash(relativePath), nil
 }
 
-func (s *Store) Record(record StepRecord) error {
-	if err := appendJSON(s.steps, record); err != nil {
+func (store *Store) Record(step StepRecord) error {
+	if err := appendJSONLine(store.stepsFile, step); err != nil {
 		return fmt.Errorf("record step: %w", err)
 	}
-	if record.Action != nil {
-		action := struct {
-			Step      int                     `json:"step"`
-			Timestamp time.Time               `json:"timestamp"`
-			Action    environment.InputAction `json:"action"`
-		}{record.Step, record.Timestamp, *record.Action}
-		if err := appendJSON(s.actions, action); err != nil {
+	if step.Action != nil {
+		action := actionRecord{
+			Step:      step.Step,
+			Timestamp: step.Timestamp,
+			Action:    *step.Action,
+		}
+		if err := appendJSONLine(store.actionsFile, action); err != nil {
 			return fmt.Errorf("record action: %w", err)
 		}
 	}
-	s.run.StepCount = record.Step
-	return s.writeRun()
+	store.run.StepCount = step.Step
+	return store.writeRunSummary()
 }
 
-func (s *Store) WriteLogs(logs []environment.LogEntry) error {
-	var stdout, stderr string
-	for _, entry := range logs {
-		if entry.Stream == "stderr" {
-			stderr = entry.Message
-		} else if entry.Stream == "stdout" {
-			stdout = entry.Message
+func (store *Store) WriteLogs(logEntries []environment.LogEntry) error {
+	var standardOutput, standardError string
+	for _, entry := range logEntries {
+		switch entry.Stream {
+		case "stdout":
+			standardOutput = entry.Message
+		case "stderr":
+			standardError = entry.Message
 		}
 	}
-	if err := os.WriteFile(filepath.Join(s.runDir, "stdout.log"), []byte(stdout), 0o644); err != nil {
+	if err := os.WriteFile(
+		filepath.Join(store.runDirectory, "stdout.log"),
+		[]byte(standardOutput),
+		0o644,
+	); err != nil {
 		return err
 	}
-	return os.WriteFile(filepath.Join(s.runDir, "stderr.log"), []byte(stderr), 0o644)
+	return os.WriteFile(
+		filepath.Join(store.runDirectory, "stderr.log"),
+		[]byte(standardError),
+		0o644,
+	)
 }
 
-func (s *Store) Finish(runErr error) error {
-	if s.steps != nil {
-		_ = s.steps.Close()
-		s.steps = nil
-	}
-	if s.actions != nil {
-		_ = s.actions.Close()
-		s.actions = nil
-	}
-	s.run.FinishedAt = time.Now().UTC()
+func (store *Store) Finish(runErr error) error {
+	store.closeEventFiles()
+	store.run.FinishedAt = time.Now().UTC()
 	if runErr != nil {
-		s.run.Status = "failed"
-		s.run.Error = runErr.Error()
+		store.run.Status = "failed"
+		store.run.Error = runErr.Error()
 	} else {
-		s.run.Status = "complete"
+		store.run.Status = "complete"
 	}
-	return s.writeRun()
+	return store.writeRunSummary()
 }
 
-func List(root string) ([]Run, error) {
-	directories, err := os.ReadDir(filepath.Join(root, ".agentbox", "runs"))
-	if errors.Is(err, os.ErrNotExist) {
-		return nil, nil
+func (store *Store) closeEventFiles() {
+	if store.stepsFile != nil {
+		_ = store.stepsFile.Close()
+		store.stepsFile = nil
 	}
-	if err != nil {
-		return nil, err
+	if store.actionsFile != nil {
+		_ = store.actionsFile.Close()
+		store.actionsFile = nil
 	}
-	var runs []Run
-	for _, directory := range directories {
-		if !directory.IsDir() {
-			continue
-		}
-		run, err := Read(root, directory.Name())
-		if err == nil && run.SchemaVersion == SchemaVersion {
-			runs = append(runs, run)
-		}
-	}
-	sort.Slice(runs, func(i, j int) bool {
-		return runs[i].StartedAt.After(runs[j].StartedAt)
-	})
-	return runs, nil
 }
 
-func Read(root, id string) (Run, error) {
-	if filepath.Base(id) != id {
-		return Run{}, errors.New("invalid run ID")
-	}
-	data, err := os.ReadFile(filepath.Join(root, ".agentbox", "runs", id, "run.json"))
-	if err != nil {
-		return Run{}, err
-	}
-	var run Run
-	if err := json.Unmarshal(data, &run); err != nil {
-		return Run{}, err
-	}
-	return run, nil
-}
-
-func ReadSteps(root, id string) ([]StepRecord, error) {
-	if filepath.Base(id) != id {
-		return nil, errors.New("invalid run ID")
-	}
-	file, err := os.Open(filepath.Join(root, ".agentbox", "runs", id, "steps.jsonl"))
-	if err != nil {
-		return nil, err
-	}
-	defer file.Close()
-	var steps []StepRecord
-	scanner := bufio.NewScanner(file)
-	for scanner.Scan() {
-		var step StepRecord
-		if err := json.Unmarshal(scanner.Bytes(), &step); err != nil {
-			return nil, err
-		}
-		steps = append(steps, step)
-	}
-	return steps, scanner.Err()
-}
-
-func appendJSON(file *os.File, value any) error {
+// Sync each JSONL record so an interrupted run retains its latest complete
+// decision. Performance is secondary to debuggability in this local spike.
+func appendJSONLine(file *os.File, value any) error {
 	data, err := json.Marshal(value)
 	if err != nil {
 		return err
@@ -234,18 +161,20 @@ func appendJSON(file *os.File, value any) error {
 	return file.Sync()
 }
 
-func (s *Store) writeRun() error {
-	data, err := json.MarshalIndent(s.run, "", "  ")
+func (store *Store) writeRunSummary() error {
+	data, err := json.MarshalIndent(store.run, "", "  ")
 	if err != nil {
 		return err
 	}
-	return os.WriteFile(filepath.Join(s.runDir, "run.json"), append(data, '\n'), 0o644)
+	runPath := filepath.Join(store.runDirectory, "run.json")
+	return os.WriteFile(runPath, append(data, '\n'), 0o644)
 }
 
-func newID() (string, error) {
-	random := make([]byte, 3)
-	if _, err := rand.Read(random); err != nil {
+func newRunID() (string, error) {
+	randomSuffix := make([]byte, 3)
+	if _, err := rand.Read(randomSuffix); err != nil {
 		return "", err
 	}
-	return time.Now().UTC().Format("20060102T150405") + "-" + hex.EncodeToString(random), nil
+	timestamp := time.Now().UTC().Format("20060102T150405")
+	return timestamp + "-" + hex.EncodeToString(randomSuffix), nil
 }

agentbox/internal/trace/types.go

@@ -0,0 +1,50 @@
+package trace
+
+import (
+	"time"
+
+	"agentbox/internal/environment"
+)
+
+// SchemaVersion changes only when existing trace readers need new logic.
+const SchemaVersion = "1"
+
+// Run is the summary stored in run.json.
+type Run struct {
+	SchemaVersion string    `json:"schema_version"`
+	ID            string    `json:"id"`
+	Task          string    `json:"task"`
+	Application   string    `json:"application"`
+	Agent         string    `json:"agent"`
+	Status        string    `json:"status"`
+	StartedAt     time.Time `json:"started_at"`
+	FinishedAt    time.Time `json:"finished_at,omitempty"`
+	StepCount     int       `json:"step_count"`
+	Error         string    `json:"error,omitempty"`
+}
+
+type ObservationRecord struct {
+	Screenshot      string                    `json:"screenshot"`
+	Logs            []environment.LogEntry    `json:"logs,omitempty"`
+	PreviousActions []environment.InputAction `json:"previous_actions,omitempty"`
+}
+
+type AgentRecord struct {
+	Message string `json:"message"`
+}
+
+// StepRecord is one append-only line in steps.jsonl.
+type StepRecord struct {
+	Step        int                      `json:"step"`
+	Timestamp   time.Time                `json:"timestamp"`
+	Observation ObservationRecord        `json:"observation"`
+	Agent       AgentRecord              `json:"agent"`
+	Action      *environment.InputAction `json:"action,omitempty"`
+	Done        bool                     `json:"done,omitempty"`
+}
+
+type actionRecord struct {
+	Step      int                     `json:"step"`
+	Timestamp time.Time               `json:"timestamp"`
+	Action    environment.InputAction `json:"action"`
+}