diff --git a/agentbox/cmd/agentbox/history_commands.go b/agentbox/cmd/agentbox/history_commands.go new file mode 100644 index 0000000..78ec5e8 --- /dev/null +++ b/agentbox/cmd/agentbox/history_commands.go @@ -0,0 +1,64 @@ +package main + +import ( + "encoding/json" + "errors" + "fmt" + "path/filepath" + + "agentbox/internal/trace" +) + +func listRuns() error { + projectRoot, err := findProjectRoot() + if err != nil { + return err + } + runs, err := trace.List(projectRoot) + if err != nil { + return err + } + if len(runs) == 0 { + fmt.Println("No recorded runs.") + return nil + } + for _, run := range runs { + fmt.Printf("%s %-8s %-20s %s\n", + run.ID, run.Status, run.Agent, run.Task) + } + return nil +} + +func inspect(args []string) error { + if len(args) != 1 { + return errors.New("inspect requires one run ID") + } + projectRoot, err := findProjectRoot() + if err != nil { + return err + } + runID := args[0] + run, err := trace.Read(projectRoot, runID) + if err != nil { + return err + } + steps, err := trace.ReadSteps(projectRoot, runID) + if err != nil { + return err + } + output := struct { + Run trace.Run `json:"run"` + Steps []trace.StepRecord `json:"steps"` + Directory string `json:"directory"` + }{ + Run: run, + Steps: steps, + Directory: filepath.Join(projectRoot, ".agentbox", "runs", run.ID), + } + data, err := json.MarshalIndent(output, "", " ") + if err != nil { + return err + } + fmt.Println(string(data)) + return nil +} diff --git a/agentbox/cmd/agentbox/main.go b/agentbox/cmd/agentbox/main.go index e8fdb33..32b0a17 100644 --- a/agentbox/cmd/agentbox/main.go +++ b/agentbox/cmd/agentbox/main.go @@ -1,24 +1,13 @@ package main import ( - "bytes" "context" - "encoding/json" - "errors" "fmt" "os" "os/signal" - "path/filepath" - "strconv" - "strings" "syscall" - "agentbox/internal/agent" - "agentbox/internal/appspec" - "agentbox/internal/environment/dockerx11" "agentbox/internal/phase1" - agentRuntime "agentbox/internal/runtime" - "agentbox/internal/trace" ) func main() { @@ -50,188 +39,6 @@ func main() { } } -type runOptions struct { - path string - task string - agent string - model string - maxSteps int -} - -func run(ctx context.Context, args []string) error { - options, err := parseRunOptions(args) - if err != nil { - return err - } - root, err := findRoot() - if err != nil { - return err - } - command, err := appspec.Resolve(options.path) - if err != nil { - return err - } - controller, err := selectAgent(options) - if err != nil { - return err - } - store, err := trace.New(root, options.task, options.path, controller.Name()) - if err != nil { - return err - } - env := dockerx11.New(dockerx11.Config{ - ProjectRoot: root, - RunID: store.ID(), - Output: os.Stdout, - }) - err = agentRuntime.Run(ctx, agentRuntime.Config{ - Task: options.task, - Command: command, - Agent: controller, - Env: env, - Trace: store, - MaxSteps: options.maxSteps, - Output: os.Stdout, - }) - if err != nil { - fmt.Fprintf(os.Stderr, "Run artifacts: %s\n", store.Directory()) - return err - } - fmt.Printf("Replay written to:\n%s\n", store.Directory()) - return nil -} - -func parseRunOptions(args []string) (runOptions, error) { - options := runOptions{agent: "deterministic", model: "gpt-5", maxSteps: 20} - for index := 0; index < len(args); index++ { - arg := args[index] - var name, value string - if strings.HasPrefix(arg, "--") { - name, value, _ = strings.Cut(strings.TrimPrefix(arg, "--"), "=") - if value == "" { - index++ - if index >= len(args) { - return options, fmt.Errorf("--%s requires a value", name) - } - value = args[index] - } - switch name { - case "task": - options.task = value - case "agent": - options.agent = value - case "model": - options.model = value - case "max-steps": - number, err := strconv.Atoi(value) - if err != nil || number < 1 { - return options, errors.New("--max-steps must be a positive integer") - } - options.maxSteps = number - default: - return options, fmt.Errorf("unknown flag --%s", name) - } - continue - } - if options.path != "" { - return options, errors.New("run accepts exactly one application path") - } - options.path = arg - } - if options.path == "" { - return options, errors.New("run requires an application path") - } - if options.task == "" { - return options, errors.New("run requires --task") - } - return options, nil -} - -func selectAgent(options runOptions) (agent.Agent, error) { - switch options.agent { - case "deterministic": - return &agent.Deterministic{}, nil - case "openai": - return agent.NewOpenAI(agent.OpenAIConfig{ - APIKey: os.Getenv("OPENAI_API_KEY"), - Model: options.model, - }) - default: - return nil, fmt.Errorf("unknown agent %q (want deterministic or openai)", options.agent) - } -} - -func listRuns() error { - root, err := findRoot() - if err != nil { - return err - } - runs, err := trace.List(root) - if err != nil { - return err - } - if len(runs) == 0 { - fmt.Println("No recorded runs.") - return nil - } - for _, run := range runs { - fmt.Printf("%s %-8s %-20s %s\n", - run.ID, run.Status, run.Agent, run.Task) - } - return nil -} - -func inspect(args []string) error { - if len(args) != 1 { - return errors.New("inspect requires one run ID") - } - root, err := findRoot() - if err != nil { - return err - } - run, err := trace.Read(root, args[0]) - if err != nil { - return err - } - steps, err := trace.ReadSteps(root, args[0]) - if err != nil { - return err - } - output := struct { - Run trace.Run `json:"run"` - Steps []trace.StepRecord `json:"steps"` - Directory string `json:"directory"` - }{ - Run: run, - Steps: steps, - Directory: filepath.Join(root, ".agentbox", "runs", run.ID), - } - data, err := json.MarshalIndent(output, "", " ") - if err != nil { - return err - } - fmt.Println(string(data)) - return nil -} - -func findRoot() (string, error) { - current, err := os.Getwd() - if err != nil { - return "", err - } - for { - data, readErr := os.ReadFile(filepath.Join(current, "go.mod")) - if readErr == nil && bytes.Contains(data, []byte("module agentbox")) { - return current, nil - } - parent := filepath.Dir(current) - if parent == current { - return "", errors.New("run agentbox from inside its module directory") - } - current = parent - } -} - func usage() { fmt.Fprintln(os.Stderr, `usage: agentbox run --task "" [--agent deterministic|openai] [--model ] diff --git a/agentbox/cmd/agentbox/main_test.go b/agentbox/cmd/agentbox/main_test.go index 2476668..0d01ba3 100644 --- a/agentbox/cmd/agentbox/main_test.go +++ b/agentbox/cmd/agentbox/main_test.go @@ -9,7 +9,9 @@ func TestParseRunOptionsAllowsFlagsAfterPath(t *testing.T) { if err != nil { t.Fatal(err) } - if options.path != "./game" || options.task != "move right" || options.maxSteps != 7 { + if options.applicationPath != "./game" || + options.task != "move right" || + options.maxSteps != 7 { t.Fatalf("options = %#v", options) } } diff --git a/agentbox/cmd/agentbox/project.go b/agentbox/cmd/agentbox/project.go new file mode 100644 index 0000000..d43a8c7 --- /dev/null +++ b/agentbox/cmd/agentbox/project.go @@ -0,0 +1,28 @@ +package main + +import ( + "bytes" + "errors" + "os" + "path/filepath" +) + +// findProjectRoot lets commands work from any directory inside the Agentbox +// module. The module declaration is a stronger marker than a directory name. +func findProjectRoot() (string, error) { + currentDirectory, err := os.Getwd() + if err != nil { + return "", err + } + for { + goModule, readErr := os.ReadFile(filepath.Join(currentDirectory, "go.mod")) + if readErr == nil && bytes.Contains(goModule, []byte("module agentbox")) { + return currentDirectory, nil + } + parentDirectory := filepath.Dir(currentDirectory) + if parentDirectory == currentDirectory { + return "", errors.New("run agentbox from inside its module directory") + } + currentDirectory = parentDirectory + } +} diff --git a/agentbox/cmd/agentbox/run_command.go b/agentbox/cmd/agentbox/run_command.go new file mode 100644 index 0000000..2c7d8b0 --- /dev/null +++ b/agentbox/cmd/agentbox/run_command.go @@ -0,0 +1,141 @@ +package main + +import ( + "context" + "errors" + "fmt" + "os" + "strconv" + "strings" + + "agentbox/internal/agent" + "agentbox/internal/appspec" + "agentbox/internal/environment/dockerx11" + agentRuntime "agentbox/internal/runtime" + "agentbox/internal/trace" +) + +type runOptions struct { + applicationPath string + task string + agentName string + modelName string + maxSteps int +} + +func run(ctx context.Context, args []string) error { + options, err := parseRunOptions(args) + if err != nil { + return err + } + projectRoot, err := findProjectRoot() + if err != nil { + return err + } + applicationCommand, err := appspec.Resolve(options.applicationPath) + if err != nil { + return err + } + controller, err := newAgent(options) + if err != nil { + return err + } + traceStore, err := trace.New( + projectRoot, + options.task, + options.applicationPath, + controller.Name(), + ) + if err != nil { + return err + } + desktopEnvironment := dockerx11.New(dockerx11.Config{ + ProjectRoot: projectRoot, + RunID: traceStore.ID(), + Output: os.Stdout, + }) + err = agentRuntime.Run(ctx, agentRuntime.Config{ + Task: options.task, + Command: applicationCommand, + Controller: controller, + Environment: desktopEnvironment, + TraceStore: traceStore, + MaxSteps: options.maxSteps, + Output: os.Stdout, + }) + if err != nil { + fmt.Fprintf(os.Stderr, "Run artifacts: %s\n", traceStore.Directory()) + return err + } + fmt.Printf("Replay written to:\n%s\n", traceStore.Directory()) + return nil +} + +// parseRunOptions accepts flags before or after the application path so the CLI +// matches the natural "agentbox run ./app --task ..." form shown in the docs. +func parseRunOptions(args []string) (runOptions, error) { + options := runOptions{ + agentName: "deterministic", + modelName: "gpt-5", + maxSteps: 20, + } + for index := 0; index < len(args); index++ { + argument := args[index] + if !strings.HasPrefix(argument, "--") { + if options.applicationPath != "" { + return options, errors.New("run accepts exactly one application path") + } + options.applicationPath = argument + continue + } + + flagName, flagValue, _ := strings.Cut(strings.TrimPrefix(argument, "--"), "=") + if flagValue == "" { + index++ + if index >= len(args) { + return options, fmt.Errorf("--%s requires a value", flagName) + } + flagValue = args[index] + } + switch flagName { + case "task": + options.task = flagValue + case "agent": + options.agentName = flagValue + case "model": + options.modelName = flagValue + case "max-steps": + maxSteps, err := strconv.Atoi(flagValue) + if err != nil || maxSteps < 1 { + return options, errors.New("--max-steps must be a positive integer") + } + options.maxSteps = maxSteps + default: + return options, fmt.Errorf("unknown flag --%s", flagName) + } + } + if options.applicationPath == "" { + return options, errors.New("run requires an application path") + } + if options.task == "" { + return options, errors.New("run requires --task") + } + return options, nil +} + +func newAgent(options runOptions) (agent.Agent, error) { + switch options.agentName { + case "deterministic": + return &agent.Deterministic{}, nil + case "openai": + return agent.NewOpenAI(agent.OpenAIConfig{ + APIKey: os.Getenv("OPENAI_API_KEY"), + Model: options.modelName, + }) + default: + return nil, fmt.Errorf( + "unknown agent %q (want deterministic or openai)", + options.agentName, + ) + } +} diff --git a/agentbox/internal/agent/agent.go b/agentbox/internal/agent/agent.go index eb9e52e..d599272 100644 --- a/agentbox/internal/agent/agent.go +++ b/agentbox/internal/agent/agent.go @@ -14,6 +14,7 @@ type Observation struct { PreviousActions []environment.InputAction } +// Step is the compact history passed back to an agent on its next decision. type Step struct { Number int Timestamp time.Time @@ -22,12 +23,14 @@ type Step struct { Action *environment.InputAction } +// Decision contains either one action or Done. Returning neither is invalid. type Decision struct { Reason string Action *environment.InputAction Done bool } +// Agent chooses one backend-neutral action from the latest observation. type Agent interface { Name() string NextAction( diff --git a/agentbox/internal/agent/deterministic.go b/agentbox/internal/agent/deterministic.go index 3efab10..2c2aa59 100644 --- a/agentbox/internal/agent/deterministic.go +++ b/agentbox/internal/agent/deterministic.go @@ -10,75 +10,82 @@ import ( ) type Deterministic struct { - next int - initialX float64 - hasInitial bool + nextStep int + initialSquareX float64 + hasInitialPosition bool } -func (a *Deterministic) Name() string { +func (deterministicAgent *Deterministic) Name() string { return "deterministic-right" } -func (a *Deterministic) NextAction( +func (deterministicAgent *Deterministic) NextAction( _ context.Context, _ string, _ []Step, observation Observation, ) (Decision, error) { var decision Decision - switch a.next { + switch deterministicAgent.nextStep { case 0: - x, err := greenCentroidX(observation.Screenshot) + initialSquareX, err := greenCentroidX(observation.Screenshot) if err != nil { return Decision{}, fmt.Errorf("inspect initial screenshot: %w", err) } - a.initialX = x - a.hasInitial = true + deterministicAgent.initialSquareX = initialSquareX + deterministicAgent.hasInitialPosition = true action := environment.InputAction{Type: environment.KeyDown, Key: "RIGHT"} decision = Decision{Reason: "Press RIGHT to start moving.", Action: &action} case 1: action := environment.InputAction{Type: environment.Wait, DurationMS: 1000} decision = Decision{Reason: "Keep RIGHT held for one second.", Action: &action} case 2: - x, err := greenCentroidX(observation.Screenshot) + currentSquareX, err := greenCentroidX(observation.Screenshot) if err != nil { return Decision{}, fmt.Errorf("inspect moved screenshot: %w", err) } - if !a.hasInitial || x-a.initialX < 100 { - return Decision{}, fmt.Errorf("visual verification failed: square moved %.1f pixels right, want at least 100", x-a.initialX) + distanceMoved := currentSquareX - deterministicAgent.initialSquareX + if !deterministicAgent.hasInitialPosition || distanceMoved < 100 { + return Decision{}, fmt.Errorf( + "visual verification failed: square moved %.1f pixels right, want at least 100", + distanceMoved, + ) } action := environment.InputAction{Type: environment.KeyUp, Key: "RIGHT"} decision = Decision{ - Reason: fmt.Sprintf("The square moved %.1f pixels right; release RIGHT.", x-a.initialX), + Reason: fmt.Sprintf( + "The square moved %.1f pixels right; release RIGHT.", + distanceMoved, + ), Action: &action, } default: decision = Decision{Reason: "The movement sequence is complete.", Done: true} } - a.next++ + deterministicAgent.nextStep++ return decision, nil } func greenCentroidX(screenshot []byte) (float64, error) { - image, err := png.Decode(bytes.NewReader(screenshot)) + renderedImage, err := png.Decode(bytes.NewReader(screenshot)) if err != nil { return 0, err } - var sumX, count uint64 - bounds := image.Bounds() + var xCoordinateSum, greenPixelCount uint64 + bounds := renderedImage.Bounds() for y := bounds.Min.Y; y < bounds.Max.Y; y++ { for x := bounds.Min.X; x < bounds.Max.X; x++ { - red, green, blue, _ := image.At(x, y).RGBA() + red, green, blue, _ := renderedImage.At(x, y).RGBA() if green > 0xc000 && red < 0x4000 && blue < 0x8000 { - sumX += uint64(x) - count++ + xCoordinateSum += uint64(x) + greenPixelCount++ } } } - if count < 1000 { - return 0, fmt.Errorf("found only %d green square pixels", count) + if greenPixelCount < 1000 { + return 0, fmt.Errorf("found only %d green square pixels", greenPixelCount) } - return float64(sumX) / float64(count), nil + return float64(xCoordinateSum) / float64(greenPixelCount), nil } var _ Agent = (*Deterministic)(nil) diff --git a/agentbox/internal/agent/openai.go b/agentbox/internal/agent/openai.go index a115771..3220cbc 100644 --- a/agentbox/internal/agent/openai.go +++ b/agentbox/internal/agent/openai.go @@ -3,16 +3,12 @@ package agent import ( "bytes" "context" - "encoding/base64" - "encoding/json" "errors" "fmt" "io" "net/http" "strings" "time" - - "agentbox/internal/environment" ) type OpenAIConfig struct { @@ -22,6 +18,8 @@ type OpenAIConfig struct { Client *http.Client } +// OpenAI adapts the Responses API to Agentbox's provider-neutral Agent +// contract. HTTP and wire-format details do not leak into the runtime. type OpenAI struct { config OpenAIConfig } @@ -42,203 +40,59 @@ func NewOpenAI(config OpenAIConfig) (*OpenAI, error) { return &OpenAI{config: config}, nil } -func (a *OpenAI) Name() string { - return "openai:" + a.config.Model +func (openAI *OpenAI) Name() string { + return "openai:" + openAI.config.Model } -func (a *OpenAI) NextAction( +func (openAI *OpenAI) NextAction( ctx context.Context, task string, history []Step, observation Observation, ) (Decision, error) { - prompt, err := modelPrompt(task, history, observation) + requestBody, err := buildResponsesRequest( + openAI.config.Model, + task, + history, + observation, + ) if err != nil { return Decision{}, err } - requestBody := map[string]any{ - "model": a.config.Model, - "input": []any{ - map[string]any{ - "role": "user", - "content": []any{ - map[string]any{"type": "input_text", "text": prompt}, - map[string]any{ - "type": "input_image", - "image_url": "data:image/png;base64," + - base64.StdEncoding.EncodeToString(observation.Screenshot), - }, - }, - }, - }, - "text": map[string]any{ - "format": map[string]any{ - "type": "json_schema", - "name": "agentbox_action", - "strict": true, - "schema": decisionSchema(), - }, - }, - } - body, err := json.Marshal(requestBody) + request, err := http.NewRequestWithContext( + ctx, + http.MethodPost, + strings.TrimRight(openAI.config.BaseURL, "/")+"/responses", + bytes.NewReader(requestBody), + ) if err != nil { return Decision{}, err } - request, err := http.NewRequestWithContext(ctx, http.MethodPost, - strings.TrimRight(a.config.BaseURL, "/")+"/responses", bytes.NewReader(body)) - if err != nil { - return Decision{}, err - } - request.Header.Set("Authorization", "Bearer "+a.config.APIKey) + request.Header.Set("Authorization", "Bearer "+openAI.config.APIKey) request.Header.Set("Content-Type", "application/json") - response, err := a.config.Client.Do(request) + httpResponse, err := openAI.config.Client.Do(request) if err != nil { return Decision{}, err } - defer response.Body.Close() - responseBody, err := io.ReadAll(io.LimitReader(response.Body, 4<<20)) + defer httpResponse.Body.Close() + + responseBody, err := io.ReadAll(io.LimitReader(httpResponse.Body, 4<<20)) if err != nil { return Decision{}, err } - if response.StatusCode < 200 || response.StatusCode >= 300 { - return Decision{}, fmt.Errorf("OpenAI response %s: %s", - response.Status, strings.TrimSpace(string(responseBody))) + if httpResponse.StatusCode < 200 || httpResponse.StatusCode >= 300 { + return Decision{}, fmt.Errorf( + "OpenAI response %s: %s", + httpResponse.Status, + strings.TrimSpace(string(responseBody)), + ) } - text, err := responseText(responseBody) + outputText, err := extractResponseText(responseBody) if err != nil { return Decision{}, err } - return parseModelDecision(text) -} - -func modelPrompt(task string, history []Step, observation Observation) (string, error) { - contextData := struct { - Task string `json:"task"` - History []Step `json:"history"` - RecentLogs []environment.LogEntry `json:"recent_logs"` - PreviousActions []environment.InputAction `json:"previous_actions"` - }{ - Task: task, - History: history, - RecentLogs: observation.Logs, - PreviousActions: observation.PreviousActions, - } - data, err := json.Marshal(contextData) - if err != nil { - return "", err - } - return "You control an interactive Linux application from screenshots. " + - "Choose exactly one safe input action, or mark done when the task is complete. " + - "Use X11 key names such as RIGHT, Return, or Escape. Keep waits under 5000 ms.\n" + - string(data), nil -} - -func decisionSchema() map[string]any { - actionProperties := map[string]any{ - "type": map[string]any{ - "type": "string", - "enum": []string{ - string(environment.KeyDown), string(environment.KeyUp), - string(environment.MouseMove), string(environment.MouseDown), - string(environment.MouseUp), string(environment.Wait), - }, - }, - "key": map[string]any{"type": "string"}, - "x": map[string]any{"type": "integer"}, - "y": map[string]any{"type": "integer"}, - "button": map[string]any{"type": "integer"}, - "duration_ms": map[string]any{"type": "integer"}, - } - return map[string]any{ - "type": "object", - "additionalProperties": false, - "properties": map[string]any{ - "reason": map[string]any{"type": "string"}, - "done": map[string]any{"type": "boolean"}, - "action": map[string]any{ - "anyOf": []any{ - map[string]any{ - "type": "object", - "additionalProperties": false, - "properties": actionProperties, - "required": []string{ - "type", "key", "x", "y", "button", "duration_ms", - }, - }, - map[string]any{"type": "null"}, - }, - }, - }, - "required": []string{"reason", "done", "action"}, - } -} - -func responseText(data []byte) (string, error) { - var response struct { - Output []struct { - Content []struct { - Type string `json:"type"` - Text string `json:"text"` - } `json:"content"` - } `json:"output"` - } - if err := json.Unmarshal(data, &response); err != nil { - return "", fmt.Errorf("decode OpenAI response: %w", err) - } - for _, output := range response.Output { - for _, content := range output.Content { - if content.Type == "output_text" && content.Text != "" { - return content.Text, nil - } - } - } - return "", errors.New("OpenAI response contained no output_text") -} - -func parseModelDecision(text string) (Decision, error) { - var result struct { - Reason string `json:"reason"` - Done bool `json:"done"` - Action *struct { - Type environment.InputType `json:"type"` - Key string `json:"key"` - X int `json:"x"` - Y int `json:"y"` - Button int `json:"button"` - DurationMS int `json:"duration_ms"` - } `json:"action"` - } - if err := json.Unmarshal([]byte(text), &result); err != nil { - return Decision{}, fmt.Errorf("decode model decision: %w", err) - } - if result.Reason == "" { - return Decision{}, errors.New("model decision requires reason") - } - if result.Done { - return Decision{Reason: result.Reason, Done: true}, nil - } - if result.Action == nil { - return Decision{}, errors.New("model decision requires action when not done") - } - if result.Action.DurationMS < 0 || result.Action.DurationMS > 5000 { - return Decision{}, errors.New("model wait must be between 0 and 5000 ms") - } - action := environment.InputAction{ - Type: result.Action.Type, - Key: result.Action.Key, - X: result.Action.X, - Y: result.Action.Y, - Button: result.Action.Button, - DurationMS: result.Action.DurationMS, - } - switch action.Type { - case environment.KeyDown, environment.KeyUp, environment.MouseMove, - environment.MouseDown, environment.MouseUp, environment.Wait: - default: - return Decision{}, fmt.Errorf("model returned unsupported action %q", action.Type) - } - return Decision{Reason: result.Reason, Action: &action}, nil + return parseModelDecision(outputText) } var _ Agent = (*OpenAI)(nil) diff --git a/agentbox/internal/agent/openai_protocol.go b/agentbox/internal/agent/openai_protocol.go new file mode 100644 index 0000000..1ada81c --- /dev/null +++ b/agentbox/internal/agent/openai_protocol.go @@ -0,0 +1,204 @@ +package agent + +import ( + "encoding/base64" + "encoding/json" + "errors" + "fmt" + + "agentbox/internal/environment" +) + +type responsesRequest struct { + Model string `json:"model"` + Input []responsesInput `json:"input"` + Text responsesTextSettings `json:"text"` +} + +type responsesInput struct { + Role string `json:"role"` + Content []responsesContent `json:"content"` +} + +type responsesContent struct { + Type string `json:"type"` + Text string `json:"text,omitempty"` + ImageURL string `json:"image_url,omitempty"` +} + +type responsesTextSettings struct { + Format responsesFormat `json:"format"` +} + +type responsesFormat struct { + Type string `json:"type"` + Name string `json:"name"` + Strict bool `json:"strict"` + Schema map[string]any `json:"schema"` +} + +func buildResponsesRequest( + model string, + task string, + history []Step, + observation Observation, +) ([]byte, error) { + prompt, err := modelPrompt(task, history, observation) + if err != nil { + return nil, err + } + request := responsesRequest{ + Model: model, + Input: []responsesInput{{ + Role: "user", + Content: []responsesContent{ + {Type: "input_text", Text: prompt}, + { + Type: "input_image", + ImageURL: "data:image/png;base64," + + base64.StdEncoding.EncodeToString(observation.Screenshot), + }, + }, + }}, + Text: responsesTextSettings{Format: responsesFormat{ + Type: "json_schema", + Name: "agentbox_action", + Strict: true, + Schema: decisionSchema(), + }}, + } + return json.Marshal(request) +} + +func modelPrompt(task string, history []Step, observation Observation) (string, error) { + contextData := struct { + Task string `json:"task"` + History []Step `json:"history"` + RecentLogs []environment.LogEntry `json:"recent_logs"` + PreviousActions []environment.InputAction `json:"previous_actions"` + }{ + Task: task, + History: history, + RecentLogs: observation.Logs, + PreviousActions: observation.PreviousActions, + } + data, err := json.Marshal(contextData) + if err != nil { + return "", err + } + return "You control an interactive Linux application from screenshots. " + + "Choose exactly one safe input action, or mark done when the task is complete. " + + "Use logical key names such as RIGHT, ENTER, or ESCAPE. Keep waits under 5000 ms.\n" + + string(data), nil +} + +func decisionSchema() map[string]any { + actionProperties := map[string]any{ + "type": map[string]any{ + "type": "string", + "enum": []string{ + string(environment.KeyDown), string(environment.KeyUp), + string(environment.MouseMove), string(environment.MouseDown), + string(environment.MouseUp), string(environment.Wait), + }, + }, + "key": map[string]any{"type": "string"}, + "x": map[string]any{"type": "integer"}, + "y": map[string]any{"type": "integer"}, + "button": map[string]any{"type": "integer"}, + "duration_ms": map[string]any{"type": "integer"}, + } + return map[string]any{ + "type": "object", + "additionalProperties": false, + "properties": map[string]any{ + "reason": map[string]any{"type": "string"}, + "done": map[string]any{"type": "boolean"}, + "action": map[string]any{ + "anyOf": []any{ + map[string]any{ + "type": "object", + "additionalProperties": false, + "properties": actionProperties, + "required": []string{ + "type", "key", "x", "y", "button", "duration_ms", + }, + }, + map[string]any{"type": "null"}, + }, + }, + }, + "required": []string{"reason", "done", "action"}, + } +} + +func extractResponseText(data []byte) (string, error) { + var response struct { + Output []struct { + Content []struct { + Type string `json:"type"` + Text string `json:"text"` + } `json:"content"` + } `json:"output"` + } + if err := json.Unmarshal(data, &response); err != nil { + return "", fmt.Errorf("decode OpenAI response: %w", err) + } + for _, output := range response.Output { + for _, content := range output.Content { + if content.Type == "output_text" && content.Text != "" { + return content.Text, nil + } + } + } + return "", errors.New("OpenAI response contained no output_text") +} + +type modelDecision struct { + Reason string `json:"reason"` + Done bool `json:"done"` + Action *modelAction `json:"action"` +} + +type modelAction struct { + Type environment.InputType `json:"type"` + Key string `json:"key"` + X int `json:"x"` + Y int `json:"y"` + Button int `json:"button"` + DurationMS int `json:"duration_ms"` +} + +func parseModelDecision(text string) (Decision, error) { + var modelOutput modelDecision + if err := json.Unmarshal([]byte(text), &modelOutput); err != nil { + return Decision{}, fmt.Errorf("decode model decision: %w", err) + } + if modelOutput.Reason == "" { + return Decision{}, errors.New("model decision requires reason") + } + if modelOutput.Done { + return Decision{Reason: modelOutput.Reason, Done: true}, nil + } + if modelOutput.Action == nil { + return Decision{}, errors.New("model decision requires action when not done") + } + if modelOutput.Action.DurationMS < 0 || modelOutput.Action.DurationMS > 5000 { + return Decision{}, errors.New("model wait must be between 0 and 5000 ms") + } + action := environment.InputAction{ + Type: modelOutput.Action.Type, + Key: modelOutput.Action.Key, + X: modelOutput.Action.X, + Y: modelOutput.Action.Y, + Button: modelOutput.Action.Button, + DurationMS: modelOutput.Action.DurationMS, + } + switch action.Type { + case environment.KeyDown, environment.KeyUp, environment.MouseMove, + environment.MouseDown, environment.MouseUp, environment.Wait: + default: + return Decision{}, fmt.Errorf("model returned unsupported action %q", action.Type) + } + return Decision{Reason: modelOutput.Reason, Action: &action}, nil +} diff --git a/agentbox/internal/appspec/appspec.go b/agentbox/internal/appspec/appspec.go index a469048..b4f0f79 100644 --- a/agentbox/internal/appspec/appspec.go +++ b/agentbox/internal/appspec/appspec.go @@ -17,6 +17,8 @@ type manifest struct { WindowTitle string `json:"window_title"` } +// Resolve turns either an executable path or a directory containing +// agentbox.json into the backend-neutral launch command. func Resolve(path string) (environment.Command, error) { absolute, err := filepath.Abs(path) if err != nil { diff --git a/agentbox/internal/environment/dockerx11/application.go b/agentbox/internal/environment/dockerx11/application.go new file mode 100644 index 0000000..357be43 --- /dev/null +++ b/agentbox/internal/environment/dockerx11/application.go @@ -0,0 +1,175 @@ +package dockerx11 + +import ( + "context" + "fmt" + "os" + "os/exec" + "strings" + "time" + + "agentbox/internal/environment" +) + +const stagedApplicationPath = "/tmp/application" + +func (environmentBackend *Environment) Launch( + ctx context.Context, + application environment.Command, +) error { + fmt.Fprintln(environmentBackend.config.Output, "Uploading build...") + executable, err := os.Open(application.Path) + if err != nil { + return fmt.Errorf("open application: %w", err) + } + defer executable.Close() + + // Stream instead of bind-mounting the developer's directory. The container + // sees only the requested executable, and the copy disappears at teardown. + stageCommand := exec.CommandContext( + ctx, + "docker", + "exec", + "-i", + environmentBackend.containerName, + "sh", + "-c", + "cat > "+stagedApplicationPath+" && chmod 0500 "+stagedApplicationPath, + ) + stageCommand.Stdin = executable + if output, err := stageCommand.CombinedOutput(); err != nil { + return fmt.Errorf( + "stage application: %w: %s", + err, + strings.TrimSpace(string(output)), + ) + } + + fmt.Fprintln(environmentBackend.config.Output, "Launching application...") + dockerArguments := []string{"exec", "-d"} + for variableName, variableValue := range application.Env { + dockerArguments = append( + dockerArguments, + "-e", + variableName+"="+variableValue, + ) + } + dockerArguments = append( + dockerArguments, + environmentBackend.containerName, + "sh", + "-c", + ) + shellCommand := []string{"exec", stagedApplicationPath} + for _, applicationArgument := range application.Args { + shellCommand = append(shellCommand, shellQuote(applicationArgument)) + } + shellCommand = append( + shellCommand, + ">/tmp/stdout.log", + "2>/tmp/stderr.log", + ) + dockerArguments = append(dockerArguments, strings.Join(shellCommand, " ")) + if _, err := environmentBackend.runDocker(ctx, dockerArguments...); err != nil { + return fmt.Errorf("launch application: %w", err) + } + + return environmentBackend.focusApplicationWindow(ctx, application.WindowTitle) +} + +func (environmentBackend *Environment) focusApplicationWindow( + ctx context.Context, + windowTitle string, +) error { + if windowTitle == "" { + // Applications without a title cannot be searched reliably. Give the + // process a brief startup window before the first screenshot. + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(500 * time.Millisecond): + return nil + } + } + + var windowID string + if err := waitFor(ctx, 10*time.Second, func() bool { + output, searchErr := environmentBackend.runDocker( + ctx, + "exec", + environmentBackend.containerName, + "xdotool", + "search", + "--name", + windowTitle, + ) + if searchErr != nil { + return false + } + windowID = strings.TrimSpace(strings.Split(string(output), "\n")[0]) + return windowID != "" + }); err != nil { + return fmt.Errorf("wait for application window %q: %w", windowTitle, err) + } + if _, err := environmentBackend.runDocker( + ctx, + "exec", + environmentBackend.containerName, + "xdotool", + "windowactivate", + "--sync", + windowID, + ); err != nil { + return fmt.Errorf("focus application window: %w", err) + } + return nil +} + +func (environmentBackend *Environment) Screenshot(ctx context.Context) ([]byte, error) { + const screenshotPath = "/tmp/screenshot.png" + if _, err := environmentBackend.runDocker( + ctx, + "exec", + environmentBackend.containerName, + "scrot", + "-o", + screenshotPath, + ); err != nil { + return nil, fmt.Errorf("capture screenshot: %w", err) + } + screenshot, err := environmentBackend.runDocker( + ctx, + "exec", + environmentBackend.containerName, + "cat", + screenshotPath, + ) + if err != nil { + return nil, fmt.Errorf("extract screenshot: %w", err) + } + return screenshot, nil +} + +func (environmentBackend *Environment) Logs( + ctx context.Context, +) ([]environment.LogEntry, error) { + var logEntries []environment.LogEntry + for _, streamName := range []string{"stdout", "stderr"} { + logContents, err := environmentBackend.runDocker( + ctx, + "exec", + environmentBackend.containerName, + "cat", + "/tmp/"+streamName+".log", + ) + if err != nil || len(logContents) == 0 { + continue + } + logEntries = append(logEntries, environment.LogEntry{ + Stream: streamName, + Message: string(logContents), + Time: time.Now().UTC(), + }) + } + return logEntries, nil +} diff --git a/agentbox/internal/environment/dockerx11/commands.go b/agentbox/internal/environment/dockerx11/commands.go new file mode 100644 index 0000000..a5ec0ac --- /dev/null +++ b/agentbox/internal/environment/dockerx11/commands.go @@ -0,0 +1,61 @@ +package dockerx11 + +import ( + "context" + "errors" + "fmt" + "os/exec" + "strings" + "time" +) + +func (environmentBackend *Environment) runDocker( + ctx context.Context, + arguments ...string, +) ([]byte, error) { + command := exec.CommandContext(ctx, "docker", arguments...) + output, err := command.CombinedOutput() + if err != nil { + return output, fmt.Errorf( + "docker %s: %w: %s", + arguments[0], + err, + strings.TrimSpace(string(output)), + ) + } + return output, nil +} + +func (environmentBackend *Environment) streamDockerOutput( + ctx context.Context, + arguments ...string, +) error { + command := exec.CommandContext(ctx, "docker", arguments...) + command.Stdout = environmentBackend.config.Output + command.Stderr = environmentBackend.config.Output + return command.Run() +} + +func waitFor(ctx context.Context, timeout time.Duration, ready func() bool) error { + timeoutTimer := time.NewTimer(timeout) + defer timeoutTimer.Stop() + retryTicker := time.NewTicker(100 * time.Millisecond) + defer retryTicker.Stop() + + for { + if ready() { + return nil + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-timeoutTimer.C: + return errors.New("timed out") + case <-retryTicker.C: + } + } +} + +func shellQuote(value string) string { + return "'" + strings.ReplaceAll(value, "'", "'\"'\"'") + "'" +} diff --git a/agentbox/internal/environment/dockerx11/docker.go b/agentbox/internal/environment/dockerx11/docker.go index a248466..5ce5b94 100644 --- a/agentbox/internal/environment/dockerx11/docker.go +++ b/agentbox/internal/environment/dockerx11/docker.go @@ -1,35 +1,30 @@ package dockerx11 import ( - "context" - "errors" - "fmt" "io" - "os" - "os/exec" - "path/filepath" - "strconv" - "strings" "sync" - "time" "agentbox/internal/environment" ) -const imageName = "agentbox-runtime:local" +const runtimeImageName = "agentbox-runtime:local" +// Config contains host-side values needed to create one isolated desktop. type Config struct { ProjectRoot string RunID string Output io.Writer } +// Environment implements the backend-neutral environment contract with one +// Docker container, one X11 display, and one application process. type Environment struct { config Config containerName string - created bool - stopped bool - mu sync.Mutex + + stopMutex sync.Mutex + containerCreated bool + containerStopped bool } func New(config Config) *Environment { @@ -39,257 +34,4 @@ func New(config Config) *Environment { } } -func (e *Environment) Start(ctx context.Context) error { - if e.config.Output == nil { - e.config.Output = io.Discard - } - if _, err := e.docker(ctx, "version", "--format", "{{.Server.Version}}"); err != nil { - return fmt.Errorf("Docker is required and the daemon must be accessible: %w", err) - } - fmt.Fprintln(e.config.Output, "Creating environment...") - if err := e.dockerStream(ctx, "build", "-q", "-t", imageName, "-f", - filepath.Join(e.config.ProjectRoot, "environment", "Dockerfile"), e.config.ProjectRoot); err != nil { - return fmt.Errorf("build environment image: %w", err) - } - _, err := e.docker(ctx, - "create", - "--name", e.containerName, - "--init", - "--network=none", - "--read-only", - "--tmpfs=/tmp:rw,exec,nosuid,nodev,size=128m", - "--cap-drop=ALL", - "--security-opt=no-new-privileges", - "--pids-limit=128", - "--memory=512m", - "--cpus=1", - imageName, - ) - if err != nil { - return fmt.Errorf("create environment: %w", err) - } - e.created = true - if _, err := e.docker(ctx, "start", e.containerName); err != nil { - return fmt.Errorf("start environment: %w", err) - } - if err := e.waitFor(ctx, 10*time.Second, func() bool { - _, readyErr := e.docker(ctx, "exec", e.containerName, "test", "-f", "/tmp/agentbox-ready") - return readyErr == nil - }); err != nil { - return fmt.Errorf("wait for graphical environment: %w", err) - } - return nil -} - -func (e *Environment) Launch(ctx context.Context, command environment.Command) error { - fmt.Fprintln(e.config.Output, "Uploading build...") - file, err := os.Open(command.Path) - if err != nil { - return fmt.Errorf("open application: %w", err) - } - defer file.Close() - - stage := exec.CommandContext(ctx, "docker", "exec", "-i", e.containerName, "sh", "-c", - "cat > /tmp/application && chmod 0500 /tmp/application") - stage.Stdin = file - if output, err := stage.CombinedOutput(); err != nil { - return fmt.Errorf("stage application: %w: %s", err, strings.TrimSpace(string(output))) - } - - fmt.Fprintln(e.config.Output, "Launching application...") - args := []string{"exec", "-d"} - for key, value := range command.Env { - args = append(args, "-e", key+"="+value) - } - args = append(args, e.containerName, "sh", "-c") - parts := []string{"exec", "/tmp/application"} - for _, arg := range command.Args { - parts = append(parts, shellQuote(arg)) - } - parts = append(parts, ">/tmp/stdout.log", "2>/tmp/stderr.log") - args = append(args, strings.Join(parts, " ")) - if _, err := e.docker(ctx, args...); err != nil { - return fmt.Errorf("launch application: %w", err) - } - - if command.WindowTitle != "" { - var windowID string - if err := e.waitFor(ctx, 10*time.Second, func() bool { - output, searchErr := e.docker(ctx, "exec", e.containerName, "xdotool", - "search", "--name", command.WindowTitle) - if searchErr != nil { - return false - } - windowID = strings.TrimSpace(strings.Split(string(output), "\n")[0]) - return windowID != "" - }); err != nil { - return fmt.Errorf("wait for application window %q: %w", command.WindowTitle, err) - } - if _, err := e.docker(ctx, "exec", e.containerName, "xdotool", - "windowactivate", "--sync", windowID); err != nil { - return fmt.Errorf("focus application window: %w", err) - } - } else { - select { - case <-ctx.Done(): - return ctx.Err() - case <-time.After(500 * time.Millisecond): - } - } - return nil -} - -func (e *Environment) Screenshot(ctx context.Context) ([]byte, error) { - if _, err := e.docker(ctx, "exec", e.containerName, "scrot", "-o", "/tmp/screenshot.png"); err != nil { - return nil, fmt.Errorf("capture screenshot: %w", err) - } - data, err := e.docker(ctx, "exec", e.containerName, "cat", "/tmp/screenshot.png") - if err != nil { - return nil, fmt.Errorf("extract screenshot: %w", err) - } - return data, nil -} - -func (e *Environment) SendInput(ctx context.Context, action environment.InputAction) error { - var args []string - switch action.Type { - case environment.KeyDown: - if action.Key == "" { - return errors.New("key_down requires key") - } - args = []string{"keydown", x11Key(action.Key)} - case environment.KeyUp: - if action.Key == "" { - return errors.New("key_up requires key") - } - args = []string{"keyup", x11Key(action.Key)} - case environment.MouseMove: - args = []string{"mousemove", strconv.Itoa(action.X), strconv.Itoa(action.Y)} - case environment.MouseDown: - args = []string{"mousedown", strconv.Itoa(action.Button)} - case environment.MouseUp: - args = []string{"mouseup", strconv.Itoa(action.Button)} - case environment.Wait: - if action.DurationMS < 0 { - return errors.New("wait duration cannot be negative") - } - select { - case <-ctx.Done(): - return ctx.Err() - case <-time.After(time.Duration(action.DurationMS) * time.Millisecond): - return nil - } - default: - return fmt.Errorf("unsupported input action %q", action.Type) - } - _, err := e.docker(ctx, append([]string{"exec", e.containerName, "xdotool"}, args...)...) - if err != nil { - return fmt.Errorf("send %s: %w", action.Type, err) - } - return nil -} - -func (e *Environment) Logs(ctx context.Context) ([]environment.LogEntry, error) { - var entries []environment.LogEntry - for _, stream := range []string{"stdout", "stderr"} { - data, err := e.docker(ctx, "exec", e.containerName, "cat", "/tmp/"+stream+".log") - if err != nil { - continue - } - if len(data) > 0 { - entries = append(entries, environment.LogEntry{ - Stream: stream, Message: string(data), Time: time.Now().UTC(), - }) - } - } - return entries, nil -} - -func (e *Environment) Stop(ctx context.Context) error { - e.mu.Lock() - defer e.mu.Unlock() - if !e.created || e.stopped { - return nil - } - e.stopped = true - fmt.Fprintln(e.config.Output, "Shutting environment down...") - _, stopErr := e.docker(ctx, "stop", "--time=3", e.containerName) - _, removeErr := e.docker(ctx, "rm", "-f", e.containerName) - if stopErr != nil { - return fmt.Errorf("stop environment: %w", stopErr) - } - if removeErr != nil { - return fmt.Errorf("remove environment: %w", removeErr) - } - return nil -} - -func (e *Environment) waitFor(ctx context.Context, timeout time.Duration, check func() bool) error { - timer := time.NewTimer(timeout) - defer timer.Stop() - ticker := time.NewTicker(100 * time.Millisecond) - defer ticker.Stop() - for { - if check() { - return nil - } - select { - case <-ctx.Done(): - return ctx.Err() - case <-timer.C: - return errors.New("timed out") - case <-ticker.C: - } - } -} - -func (e *Environment) docker(ctx context.Context, args ...string) ([]byte, error) { - command := exec.CommandContext(ctx, "docker", args...) - output, err := command.CombinedOutput() - if err != nil { - return output, fmt.Errorf("docker %s: %w: %s", args[0], err, strings.TrimSpace(string(output))) - } - return output, nil -} - -func (e *Environment) dockerStream(ctx context.Context, args ...string) error { - command := exec.CommandContext(ctx, "docker", args...) - command.Stdout = e.config.Output - command.Stderr = e.config.Output - return command.Run() -} - -func shellQuote(value string) string { - return "'" + strings.ReplaceAll(value, "'", "'\"'\"'") + "'" -} - -func x11Key(key string) string { - switch strings.ToUpper(key) { - case "LEFT": - return "Left" - case "RIGHT": - return "Right" - case "UP": - return "Up" - case "DOWN": - return "Down" - case "ENTER", "RETURN": - return "Return" - case "ESC", "ESCAPE": - return "Escape" - case "SPACE": - return "space" - case "TAB": - return "Tab" - case "BACKSPACE": - return "BackSpace" - case "DELETE": - return "Delete" - } - if len(key) == 1 { - return strings.ToLower(key) - } - return key -} - var _ environment.Environment = (*Environment)(nil) diff --git a/agentbox/internal/environment/dockerx11/docker_test.go b/agentbox/internal/environment/dockerx11/docker_test.go index 4239a40..c7f614b 100644 --- a/agentbox/internal/environment/dockerx11/docker_test.go +++ b/agentbox/internal/environment/dockerx11/docker_test.go @@ -11,8 +11,8 @@ func TestX11KeyTranslatesLogicalNames(t *testing.T) { "A": "a", } for input, want := range tests { - if got := x11Key(input); got != want { - t.Errorf("x11Key(%q) = %q, want %q", input, got, want) + if got := toX11KeyName(input); got != want { + t.Errorf("toX11KeyName(%q) = %q, want %q", input, got, want) } } } diff --git a/agentbox/internal/environment/dockerx11/input.go b/agentbox/internal/environment/dockerx11/input.go new file mode 100644 index 0000000..ee48089 --- /dev/null +++ b/agentbox/internal/environment/dockerx11/input.go @@ -0,0 +1,95 @@ +package dockerx11 + +import ( + "context" + "errors" + "fmt" + "strconv" + "strings" + "time" + + "agentbox/internal/environment" +) + +func (environmentBackend *Environment) SendInput( + ctx context.Context, + action environment.InputAction, +) error { + var xdotoolArguments []string + switch action.Type { + case environment.KeyDown: + if action.Key == "" { + return errors.New("key_down requires key") + } + xdotoolArguments = []string{"keydown", toX11KeyName(action.Key)} + case environment.KeyUp: + if action.Key == "" { + return errors.New("key_up requires key") + } + xdotoolArguments = []string{"keyup", toX11KeyName(action.Key)} + case environment.MouseMove: + xdotoolArguments = []string{ + "mousemove", + strconv.Itoa(action.X), + strconv.Itoa(action.Y), + } + case environment.MouseDown: + xdotoolArguments = []string{"mousedown", strconv.Itoa(action.Button)} + case environment.MouseUp: + xdotoolArguments = []string{"mouseup", strconv.Itoa(action.Button)} + case environment.Wait: + if action.DurationMS < 0 { + return errors.New("wait duration cannot be negative") + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(time.Duration(action.DurationMS) * time.Millisecond): + return nil + } + default: + return fmt.Errorf("unsupported input action %q", action.Type) + } + + dockerArguments := []string{ + "exec", + environmentBackend.containerName, + "xdotool", + } + dockerArguments = append(dockerArguments, xdotoolArguments...) + if _, err := environmentBackend.runDocker(ctx, dockerArguments...); err != nil { + return fmt.Errorf("send %s: %w", action.Type, err) + } + return nil +} + +// toX11KeyName keeps X11 spellings out of the public action API. Agents can use +// logical names such as RIGHT even though xdotool expects Right. +func toX11KeyName(logicalName string) string { + switch strings.ToUpper(logicalName) { + case "LEFT": + return "Left" + case "RIGHT": + return "Right" + case "UP": + return "Up" + case "DOWN": + return "Down" + case "ENTER", "RETURN": + return "Return" + case "ESC", "ESCAPE": + return "Escape" + case "SPACE": + return "space" + case "TAB": + return "Tab" + case "BACKSPACE": + return "BackSpace" + case "DELETE": + return "Delete" + } + if len(logicalName) == 1 { + return strings.ToLower(logicalName) + } + return logicalName +} diff --git a/agentbox/internal/environment/dockerx11/lifecycle.go b/agentbox/internal/environment/dockerx11/lifecycle.go new file mode 100644 index 0000000..a378d7d --- /dev/null +++ b/agentbox/internal/environment/dockerx11/lifecycle.go @@ -0,0 +1,119 @@ +package dockerx11 + +import ( + "context" + "fmt" + "io" + "path/filepath" + "time" +) + +func (environmentBackend *Environment) Start(ctx context.Context) error { + if environmentBackend.config.Output == nil { + environmentBackend.config.Output = io.Discard + } + if _, err := environmentBackend.runDocker( + ctx, + "version", + "--format", + "{{.Server.Version}}", + ); err != nil { + return fmt.Errorf("Docker is required and the daemon must be accessible: %w", err) + } + + fmt.Fprintln(environmentBackend.config.Output, "Creating environment...") + dockerfilePath := filepath.Join( + environmentBackend.config.ProjectRoot, + "environment", + "Dockerfile", + ) + if err := environmentBackend.streamDockerOutput( + ctx, + "build", + "-q", + "-t", + runtimeImageName, + "-f", + dockerfilePath, + environmentBackend.config.ProjectRoot, + ); err != nil { + return fmt.Errorf("build environment image: %w", err) + } + + // These restrictions reduce accidental damage. They are defense in depth, + // not a safe boundary for hostile customer code; see docs/architecture.md. + _, err := environmentBackend.runDocker( + ctx, + "create", + "--name", environmentBackend.containerName, + "--init", + "--network=none", + "--read-only", + "--tmpfs=/tmp:rw,exec,nosuid,nodev,size=128m", + "--cap-drop=ALL", + "--security-opt=no-new-privileges", + "--pids-limit=128", + "--memory=512m", + "--cpus=1", + runtimeImageName, + ) + if err != nil { + return fmt.Errorf("create environment: %w", err) + } + environmentBackend.containerCreated = true + + if _, err := environmentBackend.runDocker( + ctx, + "start", + environmentBackend.containerName, + ); err != nil { + return fmt.Errorf("start environment: %w", err) + } + if err := waitFor(ctx, 10*time.Second, func() bool { + _, readyErr := environmentBackend.runDocker( + ctx, + "exec", + environmentBackend.containerName, + "test", + "-f", + "/tmp/agentbox-ready", + ) + return readyErr == nil + }); err != nil { + return fmt.Errorf("wait for graphical environment: %w", err) + } + return nil +} + +// Stop is idempotent because both normal completion and deferred cleanup may +// try to tear down the same environment. +func (environmentBackend *Environment) Stop(ctx context.Context) error { + environmentBackend.stopMutex.Lock() + defer environmentBackend.stopMutex.Unlock() + + if !environmentBackend.containerCreated || environmentBackend.containerStopped { + return nil + } + environmentBackend.containerStopped = true + fmt.Fprintln(environmentBackend.config.Output, "Shutting environment down...") + + _, stopErr := environmentBackend.runDocker( + ctx, + "stop", + "--time=3", + environmentBackend.containerName, + ) + _, removeErr := environmentBackend.runDocker( + ctx, + "rm", + "-f", + environmentBackend.containerName, + ) + if stopErr != nil { + return fmt.Errorf("stop environment: %w", stopErr) + } + if removeErr != nil { + return fmt.Errorf("remove environment: %w", removeErr) + } + return nil +} diff --git a/agentbox/internal/environment/environment.go b/agentbox/internal/environment/environment.go index 77c50ab..c7667e3 100644 --- a/agentbox/internal/environment/environment.go +++ b/agentbox/internal/environment/environment.go @@ -5,6 +5,8 @@ import ( "time" ) +// Command describes a host executable and how it should start in an +// environment. Environment implementations decide how to stage the file. type Command struct { Path string Args []string @@ -12,6 +14,7 @@ type Command struct { WindowTitle string } +// InputType identifies one backend-neutral keyboard, mouse, or timing action. type InputType string const ( @@ -23,6 +26,7 @@ const ( Wait InputType = "wait" ) +// InputAction contains only fields relevant to Type. DurationMS is used by Wait. type InputAction struct { Type InputType `json:"type"` Key string `json:"key,omitempty"` @@ -32,12 +36,15 @@ type InputAction struct { DurationMS int `json:"duration_ms,omitempty"` } +// LogEntry is a snapshot of one application output stream. type LogEntry struct { Stream string `json:"stream"` Message string `json:"message"` Time time.Time `json:"time"` } +// Environment is the programmable-computer boundary used by the runtime. +// Implementations may use containers, virtual machines, or remote hosts. type Environment interface { Start(context.Context) error Launch(context.Context, Command) error diff --git a/agentbox/internal/runtime/runtime.go b/agentbox/internal/runtime/runtime.go index 3708449..dd561f8 100644 --- a/agentbox/internal/runtime/runtime.go +++ b/agentbox/internal/runtime/runtime.go @@ -12,14 +12,16 @@ import ( "agentbox/internal/trace" ) +// Config wires replaceable environment, agent, and trace implementations into +// one run. The runtime itself has no Docker, X11, or model-provider knowledge. type Config struct { - Task string - Command environment.Command - Agent agent.Agent - Env environment.Environment - Trace *trace.Store - MaxSteps int - Output io.Writer + Task string + Command environment.Command + Controller agent.Agent + Environment environment.Environment + TraceStore *trace.Store + MaxSteps int + Output io.Writer } func Run(ctx context.Context, config Config) (runErr error) { @@ -29,88 +31,85 @@ func Run(ctx context.Context, config Config) (runErr error) { if config.Output == nil { config.Output = io.Discard } - if config.Agent == nil || config.Env == nil || config.Trace == nil { + if config.Controller == nil || config.Environment == nil || config.TraceStore == nil { return errors.New("runtime requires agent, environment, and trace") } - started := false defer func() { - cleanupCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - if started { - if logs, err := config.Env.Logs(cleanupCtx); err == nil { - if logErr := config.Trace.WriteLogs(logs); runErr == nil && logErr != nil { - runErr = logErr - } - } - if stopErr := config.Env.Stop(cleanupCtx); runErr == nil && stopErr != nil { - runErr = stopErr - } - } - if finishErr := config.Trace.Finish(runErr); runErr == nil && finishErr != nil { - runErr = finishErr - } + runErr = finalizeRun(config, runErr) }() - if err := config.Env.Start(ctx); err != nil { + if err := config.Environment.Start(ctx); err != nil { return err } - started = true - if err := config.Env.Launch(ctx, config.Command); err != nil { + if err := config.Environment.Launch(ctx, config.Command); err != nil { return err } fmt.Fprintln(config.Output, "Agent attached.") - var history []agent.Step - var actions []environment.InputAction - startedAt := time.Now() - for number := 1; number <= config.MaxSteps; number++ { - screenshot, err := config.Env.Screenshot(ctx) + var stepHistory []agent.Step + var actionHistory []environment.InputAction + runStartedAt := time.Now() + for stepNumber := 1; stepNumber <= config.MaxSteps; stepNumber++ { + screenshot, err := config.Environment.Screenshot(ctx) if err != nil { return err } - screenshotPath, err := config.Trace.SaveScreenshot(number, screenshot) + screenshotPath, err := config.TraceStore.SaveScreenshot(stepNumber, screenshot) if err != nil { return err } - logs, err := config.Env.Logs(ctx) + applicationLogs, err := config.Environment.Logs(ctx) if err != nil { return err } - now := time.Now().UTC() - fmt.Fprintf(config.Output, "[%s] screenshot captured\n", elapsed(startedAt)) - decision, err := config.Agent.NextAction(ctx, config.Task, history, agent.Observation{ + observedAt := time.Now().UTC() + previousActions := append([]environment.InputAction(nil), actionHistory...) + observation := agent.Observation{ Screenshot: screenshot, - Timestamp: now, - Logs: logs, - PreviousActions: append([]environment.InputAction(nil), actions...), - }) + Timestamp: observedAt, + Logs: applicationLogs, + PreviousActions: previousActions, + } + fmt.Fprintf(config.Output, "[%s] screenshot captured\n", elapsed(runStartedAt)) + + decision, err := config.Controller.NextAction( + ctx, + config.Task, + stepHistory, + observation, + ) if err != nil { return fmt.Errorf("agent next action: %w", err) } - fmt.Fprintf(config.Output, "[%s] agent: %s\n", elapsed(startedAt), decision.Reason) - record := trace.StepRecord{ - Step: number, - Timestamp: now, + fmt.Fprintf( + config.Output, + "[%s] agent: %s\n", + elapsed(runStartedAt), + decision.Reason, + ) + if err := config.TraceStore.Record(trace.StepRecord{ + Step: stepNumber, + Timestamp: observedAt, Observation: trace.ObservationRecord{ Screenshot: screenshotPath, - Logs: logs, - PreviousActions: append([]environment.InputAction(nil), actions...), + Logs: applicationLogs, + PreviousActions: previousActions, }, Agent: trace.AgentRecord{Message: decision.Reason}, Action: decision.Action, Done: decision.Done, - } - if err := config.Trace.Record(record); err != nil { + }); err != nil { return err } - history = append(history, agent.Step{ - Number: number, - Timestamp: now, + stepHistory = append(stepHistory, agent.Step{ + Number: stepNumber, + Timestamp: observedAt, ScreenshotPath: screenshotPath, Message: decision.Reason, Action: decision.Action, }) + if decision.Done { fmt.Fprintln(config.Output, "Task complete.") return nil @@ -118,19 +117,53 @@ func Run(ctx context.Context, config Config) (runErr error) { if decision.Action == nil { return errors.New("agent returned neither action nor completion") } - fmt.Fprintf(config.Output, "[%s] action: %s%s\n", - elapsed(startedAt), decision.Action.Type, actionDetail(*decision.Action)) - if err := config.Env.SendInput(ctx, *decision.Action); err != nil { + fmt.Fprintf( + config.Output, + "[%s] action: %s%s\n", + elapsed(runStartedAt), + decision.Action.Type, + actionDetail(*decision.Action), + ) + if err := config.Environment.SendInput(ctx, *decision.Action); err != nil { return err } - actions = append(actions, *decision.Action) + actionHistory = append(actionHistory, *decision.Action) } return fmt.Errorf("agent exceeded maximum of %d steps", config.MaxSteps) } +// finalizeRun uses a fresh timeout because the caller's context may already be +// canceled. Preserving logs and removing the environment must still be tried. +func finalizeRun( + config Config, + runErr error, +) error { + cleanupCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + if logs, err := config.Environment.Logs(cleanupCtx); err == nil { + if logErr := config.TraceStore.WriteLogs(logs); runErr == nil && logErr != nil { + runErr = logErr + } + } + // Stop is idempotent, so always call it. Start may have created a container + // before returning an error while waiting for its graphical services. + if stopErr := config.Environment.Stop(cleanupCtx); runErr == nil && stopErr != nil { + runErr = stopErr + } + if finishErr := config.TraceStore.Finish(runErr); runErr == nil && finishErr != nil { + runErr = finishErr + } + return runErr +} + func elapsed(start time.Time) string { duration := time.Since(start).Round(time.Second) - return fmt.Sprintf("%02d:%02d", int(duration.Minutes()), int(duration.Seconds())%60) + return fmt.Sprintf( + "%02d:%02d", + int(duration.Minutes()), + int(duration.Seconds())%60, + ) } func actionDetail(action environment.InputAction) string { diff --git a/agentbox/internal/runtime/runtime_test.go b/agentbox/internal/runtime/runtime_test.go new file mode 100644 index 0000000..f8963d7 --- /dev/null +++ b/agentbox/internal/runtime/runtime_test.go @@ -0,0 +1,63 @@ +package runtime + +import ( + "context" + "errors" + "io" + "testing" + + "agentbox/internal/agent" + "agentbox/internal/environment" + "agentbox/internal/trace" +) + +type failingStartEnvironment struct { + stopCalled bool +} + +func (fake *failingStartEnvironment) Start(context.Context) error { + return errors.New("startup failed") +} + +func (fake *failingStartEnvironment) Launch(context.Context, environment.Command) error { + return nil +} + +func (fake *failingStartEnvironment) Screenshot(context.Context) ([]byte, error) { + return nil, nil +} + +func (fake *failingStartEnvironment) SendInput(context.Context, environment.InputAction) error { + return nil +} + +func (fake *failingStartEnvironment) Logs(context.Context) ([]environment.LogEntry, error) { + return nil, nil +} + +func (fake *failingStartEnvironment) Stop(context.Context) error { + fake.stopCalled = true + return nil +} + +func TestRunStopsEnvironmentWhenStartFails(t *testing.T) { + traceStore, err := trace.New(t.TempDir(), "task", "application", "test-agent") + if err != nil { + t.Fatal(err) + } + fakeEnvironment := &failingStartEnvironment{} + + err = Run(context.Background(), Config{ + Task: "task", + Controller: &agent.Deterministic{}, + Environment: fakeEnvironment, + TraceStore: traceStore, + Output: io.Discard, + }) + if err == nil { + t.Fatal("Run() error = nil, want startup error") + } + if !fakeEnvironment.stopCalled { + t.Fatal("Run() did not stop environment after startup failure") + } +} diff --git a/agentbox/internal/trace/reader.go b/agentbox/internal/trace/reader.go new file mode 100644 index 0000000..2ceb3ca --- /dev/null +++ b/agentbox/internal/trace/reader.go @@ -0,0 +1,94 @@ +package trace + +import ( + "bufio" + "encoding/json" + "errors" + "os" + "path/filepath" + "sort" +) + +func List(projectRoot string) ([]Run, error) { + runsDirectory := filepath.Join(projectRoot, ".agentbox", "runs") + directoryEntries, err := os.ReadDir(runsDirectory) + if errors.Is(err, os.ErrNotExist) { + return nil, nil + } + if err != nil { + return nil, err + } + + var runs []Run + for _, entry := range directoryEntries { + if !entry.IsDir() { + continue + } + run, err := Read(projectRoot, entry.Name()) + if err == nil && run.SchemaVersion == SchemaVersion { + runs = append(runs, run) + } + } + sort.Slice(runs, func(left, right int) bool { + return runs[left].StartedAt.After(runs[right].StartedAt) + }) + return runs, nil +} + +func Read(projectRoot, runID string) (Run, error) { + if err := validateRunID(runID); err != nil { + return Run{}, err + } + runPath := filepath.Join( + projectRoot, + ".agentbox", + "runs", + runID, + "run.json", + ) + data, err := os.ReadFile(runPath) + if err != nil { + return Run{}, err + } + var run Run + if err := json.Unmarshal(data, &run); err != nil { + return Run{}, err + } + return run, nil +} + +func ReadSteps(projectRoot, runID string) ([]StepRecord, error) { + if err := validateRunID(runID); err != nil { + return nil, err + } + stepsPath := filepath.Join( + projectRoot, + ".agentbox", + "runs", + runID, + "steps.jsonl", + ) + stepsFile, err := os.Open(stepsPath) + if err != nil { + return nil, err + } + defer stepsFile.Close() + + var steps []StepRecord + scanner := bufio.NewScanner(stepsFile) + for scanner.Scan() { + var step StepRecord + if err := json.Unmarshal(scanner.Bytes(), &step); err != nil { + return nil, err + } + steps = append(steps, step) + } + return steps, scanner.Err() +} + +func validateRunID(runID string) error { + if filepath.Base(runID) != runID { + return errors.New("invalid run ID") + } + return nil +} diff --git a/agentbox/internal/trace/store.go b/agentbox/internal/trace/store.go index 51dcbd4..6efcc74 100644 --- a/agentbox/internal/trace/store.go +++ b/agentbox/internal/trace/store.go @@ -1,85 +1,52 @@ package trace import ( - "bufio" "crypto/rand" "encoding/hex" "encoding/json" - "errors" "fmt" "os" "path/filepath" - "sort" "time" "agentbox/internal/environment" ) -const SchemaVersion = "1" - -type Run struct { - SchemaVersion string `json:"schema_version"` - ID string `json:"id"` - Task string `json:"task"` - Application string `json:"application"` - Agent string `json:"agent"` - Status string `json:"status"` - StartedAt time.Time `json:"started_at"` - FinishedAt time.Time `json:"finished_at,omitempty"` - StepCount int `json:"step_count"` - Error string `json:"error,omitempty"` -} - -type ObservationRecord struct { - Screenshot string `json:"screenshot"` - Logs []environment.LogEntry `json:"logs,omitempty"` - PreviousActions []environment.InputAction `json:"previous_actions,omitempty"` -} - -type AgentRecord struct { - Message string `json:"message"` -} - -type StepRecord struct { - Step int `json:"step"` - Timestamp time.Time `json:"timestamp"` - Observation ObservationRecord `json:"observation"` - Agent AgentRecord `json:"agent"` - Action *environment.InputAction `json:"action,omitempty"` - Done bool `json:"done,omitempty"` -} - +// Store writes one run's metadata and append-only event streams. type Store struct { - root string - runDir string - run Run - steps *os.File - actions *os.File + runDirectory string + run Run + stepsFile *os.File + actionsFile *os.File } -func New(root, task, application, agentName string) (*Store, error) { - id, err := newID() +func New(projectRoot, task, application, agentName string) (*Store, error) { + runID, err := newRunID() if err != nil { return nil, err } - runDir := filepath.Join(root, ".agentbox", "runs", id) - if err := os.MkdirAll(filepath.Join(runDir, "screenshots"), 0o755); err != nil { + runDirectory := filepath.Join(projectRoot, ".agentbox", "runs", runID) + screenshotDirectory := filepath.Join(runDirectory, "screenshots") + if err := os.MkdirAll(screenshotDirectory, 0o755); err != nil { return nil, fmt.Errorf("create run directory: %w", err) } - steps, err := os.Create(filepath.Join(runDir, "steps.jsonl")) + stepsFile, err := os.Create(filepath.Join(runDirectory, "steps.jsonl")) if err != nil { return nil, fmt.Errorf("create step trace: %w", err) } - actions, err := os.Create(filepath.Join(runDir, "actions.jsonl")) + actionsFile, err := os.Create(filepath.Join(runDirectory, "actions.jsonl")) if err != nil { - _ = steps.Close() + _ = stepsFile.Close() return nil, fmt.Errorf("create action trace: %w", err) } + store := &Store{ - root: root, runDir: runDir, steps: steps, actions: actions, + runDirectory: runDirectory, + stepsFile: stepsFile, + actionsFile: actionsFile, run: Run{ SchemaVersion: SchemaVersion, - ID: id, + ID: runID, Task: task, Application: application, Agent: agentName, @@ -87,143 +54,103 @@ func New(root, task, application, agentName string) (*Store, error) { StartedAt: time.Now().UTC(), }, } - if err := store.writeRun(); err != nil { - _ = steps.Close() - _ = actions.Close() + if err := store.writeRunSummary(); err != nil { + _ = stepsFile.Close() + _ = actionsFile.Close() return nil, err } return store, nil } -func (s *Store) ID() string { - return s.run.ID +func (store *Store) ID() string { + return store.run.ID } -func (s *Store) Directory() string { - return s.runDir +func (store *Store) Directory() string { + return store.runDirectory } -func (s *Store) SaveScreenshot(step int, data []byte) (string, error) { - relative := filepath.Join("screenshots", fmt.Sprintf("%04d.png", step)) - if err := os.WriteFile(filepath.Join(s.runDir, relative), data, 0o644); err != nil { +func (store *Store) SaveScreenshot(stepNumber int, screenshot []byte) (string, error) { + relativePath := filepath.Join( + "screenshots", + fmt.Sprintf("%04d.png", stepNumber), + ) + absolutePath := filepath.Join(store.runDirectory, relativePath) + if err := os.WriteFile(absolutePath, screenshot, 0o644); err != nil { return "", fmt.Errorf("write screenshot: %w", err) } - return filepath.ToSlash(relative), nil + // Trace paths always use slash separators so traces are portable. + return filepath.ToSlash(relativePath), nil } -func (s *Store) Record(record StepRecord) error { - if err := appendJSON(s.steps, record); err != nil { +func (store *Store) Record(step StepRecord) error { + if err := appendJSONLine(store.stepsFile, step); err != nil { return fmt.Errorf("record step: %w", err) } - if record.Action != nil { - action := struct { - Step int `json:"step"` - Timestamp time.Time `json:"timestamp"` - Action environment.InputAction `json:"action"` - }{record.Step, record.Timestamp, *record.Action} - if err := appendJSON(s.actions, action); err != nil { + if step.Action != nil { + action := actionRecord{ + Step: step.Step, + Timestamp: step.Timestamp, + Action: *step.Action, + } + if err := appendJSONLine(store.actionsFile, action); err != nil { return fmt.Errorf("record action: %w", err) } } - s.run.StepCount = record.Step - return s.writeRun() + store.run.StepCount = step.Step + return store.writeRunSummary() } -func (s *Store) WriteLogs(logs []environment.LogEntry) error { - var stdout, stderr string - for _, entry := range logs { - if entry.Stream == "stderr" { - stderr = entry.Message - } else if entry.Stream == "stdout" { - stdout = entry.Message +func (store *Store) WriteLogs(logEntries []environment.LogEntry) error { + var standardOutput, standardError string + for _, entry := range logEntries { + switch entry.Stream { + case "stdout": + standardOutput = entry.Message + case "stderr": + standardError = entry.Message } } - if err := os.WriteFile(filepath.Join(s.runDir, "stdout.log"), []byte(stdout), 0o644); err != nil { + if err := os.WriteFile( + filepath.Join(store.runDirectory, "stdout.log"), + []byte(standardOutput), + 0o644, + ); err != nil { return err } - return os.WriteFile(filepath.Join(s.runDir, "stderr.log"), []byte(stderr), 0o644) + return os.WriteFile( + filepath.Join(store.runDirectory, "stderr.log"), + []byte(standardError), + 0o644, + ) } -func (s *Store) Finish(runErr error) error { - if s.steps != nil { - _ = s.steps.Close() - s.steps = nil - } - if s.actions != nil { - _ = s.actions.Close() - s.actions = nil - } - s.run.FinishedAt = time.Now().UTC() +func (store *Store) Finish(runErr error) error { + store.closeEventFiles() + store.run.FinishedAt = time.Now().UTC() if runErr != nil { - s.run.Status = "failed" - s.run.Error = runErr.Error() + store.run.Status = "failed" + store.run.Error = runErr.Error() } else { - s.run.Status = "complete" + store.run.Status = "complete" } - return s.writeRun() + return store.writeRunSummary() } -func List(root string) ([]Run, error) { - directories, err := os.ReadDir(filepath.Join(root, ".agentbox", "runs")) - if errors.Is(err, os.ErrNotExist) { - return nil, nil +func (store *Store) closeEventFiles() { + if store.stepsFile != nil { + _ = store.stepsFile.Close() + store.stepsFile = nil } - if err != nil { - return nil, err + if store.actionsFile != nil { + _ = store.actionsFile.Close() + store.actionsFile = nil } - var runs []Run - for _, directory := range directories { - if !directory.IsDir() { - continue - } - run, err := Read(root, directory.Name()) - if err == nil && run.SchemaVersion == SchemaVersion { - runs = append(runs, run) - } - } - sort.Slice(runs, func(i, j int) bool { - return runs[i].StartedAt.After(runs[j].StartedAt) - }) - return runs, nil } -func Read(root, id string) (Run, error) { - if filepath.Base(id) != id { - return Run{}, errors.New("invalid run ID") - } - data, err := os.ReadFile(filepath.Join(root, ".agentbox", "runs", id, "run.json")) - if err != nil { - return Run{}, err - } - var run Run - if err := json.Unmarshal(data, &run); err != nil { - return Run{}, err - } - return run, nil -} - -func ReadSteps(root, id string) ([]StepRecord, error) { - if filepath.Base(id) != id { - return nil, errors.New("invalid run ID") - } - file, err := os.Open(filepath.Join(root, ".agentbox", "runs", id, "steps.jsonl")) - if err != nil { - return nil, err - } - defer file.Close() - var steps []StepRecord - scanner := bufio.NewScanner(file) - for scanner.Scan() { - var step StepRecord - if err := json.Unmarshal(scanner.Bytes(), &step); err != nil { - return nil, err - } - steps = append(steps, step) - } - return steps, scanner.Err() -} - -func appendJSON(file *os.File, value any) error { +// Sync each JSONL record so an interrupted run retains its latest complete +// decision. Performance is secondary to debuggability in this local spike. +func appendJSONLine(file *os.File, value any) error { data, err := json.Marshal(value) if err != nil { return err @@ -234,18 +161,20 @@ func appendJSON(file *os.File, value any) error { return file.Sync() } -func (s *Store) writeRun() error { - data, err := json.MarshalIndent(s.run, "", " ") +func (store *Store) writeRunSummary() error { + data, err := json.MarshalIndent(store.run, "", " ") if err != nil { return err } - return os.WriteFile(filepath.Join(s.runDir, "run.json"), append(data, '\n'), 0o644) + runPath := filepath.Join(store.runDirectory, "run.json") + return os.WriteFile(runPath, append(data, '\n'), 0o644) } -func newID() (string, error) { - random := make([]byte, 3) - if _, err := rand.Read(random); err != nil { +func newRunID() (string, error) { + randomSuffix := make([]byte, 3) + if _, err := rand.Read(randomSuffix); err != nil { return "", err } - return time.Now().UTC().Format("20060102T150405") + "-" + hex.EncodeToString(random), nil + timestamp := time.Now().UTC().Format("20060102T150405") + return timestamp + "-" + hex.EncodeToString(randomSuffix), nil } diff --git a/agentbox/internal/trace/types.go b/agentbox/internal/trace/types.go new file mode 100644 index 0000000..7f8e203 --- /dev/null +++ b/agentbox/internal/trace/types.go @@ -0,0 +1,50 @@ +package trace + +import ( + "time" + + "agentbox/internal/environment" +) + +// SchemaVersion changes only when existing trace readers need new logic. +const SchemaVersion = "1" + +// Run is the summary stored in run.json. +type Run struct { + SchemaVersion string `json:"schema_version"` + ID string `json:"id"` + Task string `json:"task"` + Application string `json:"application"` + Agent string `json:"agent"` + Status string `json:"status"` + StartedAt time.Time `json:"started_at"` + FinishedAt time.Time `json:"finished_at,omitempty"` + StepCount int `json:"step_count"` + Error string `json:"error,omitempty"` +} + +type ObservationRecord struct { + Screenshot string `json:"screenshot"` + Logs []environment.LogEntry `json:"logs,omitempty"` + PreviousActions []environment.InputAction `json:"previous_actions,omitempty"` +} + +type AgentRecord struct { + Message string `json:"message"` +} + +// StepRecord is one append-only line in steps.jsonl. +type StepRecord struct { + Step int `json:"step"` + Timestamp time.Time `json:"timestamp"` + Observation ObservationRecord `json:"observation"` + Agent AgentRecord `json:"agent"` + Action *environment.InputAction `json:"action,omitempty"` + Done bool `json:"done,omitempty"` +} + +type actionRecord struct { + Step int `json:"step"` + Timestamp time.Time `json:"timestamp"` + Action environment.InputAction `json:"action"` +}