From 94146fcae295c75dd0086de5fafff25a9d457670 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Mon, 17 Aug 2026 16:47:22 +0000 Subject: [PATCH] Add runtime abstraction and agent loop Co-authored-by: codegirl007 --- agentbox/.gitignore | 3 +- agentbox/Makefile | 11 +- agentbox/cmd/agentbox/main.go | 241 ++++++++++++++++ agentbox/cmd/agentbox/main_test.go | 15 + agentbox/examples/mover/agentbox.json | 4 + agentbox/internal/agent/agent.go | 39 +++ agentbox/internal/agent/deterministic.go | 42 +++ agentbox/internal/agent/deterministic_test.go | 33 +++ agentbox/internal/agent/openai.go | 244 ++++++++++++++++ agentbox/internal/agent/openai_test.go | 59 ++++ agentbox/internal/appspec/appspec.go | 64 +++++ agentbox/internal/appspec/appspec_test.go | 28 ++ .../internal/environment/dockerx11/docker.go | 266 ++++++++++++++++++ agentbox/internal/environment/environment.go | 48 ++++ agentbox/internal/runtime/runtime.go | 149 ++++++++++ agentbox/internal/trace/store.go | 251 +++++++++++++++++ agentbox/scripts/build_demo.sh | 17 ++ 17 files changed, 1512 insertions(+), 2 deletions(-) create mode 100644 agentbox/cmd/agentbox/main.go create mode 100644 agentbox/cmd/agentbox/main_test.go create mode 100644 agentbox/examples/mover/agentbox.json create mode 100644 agentbox/internal/agent/agent.go create mode 100644 agentbox/internal/agent/deterministic.go create mode 100644 agentbox/internal/agent/deterministic_test.go create mode 100644 agentbox/internal/agent/openai.go create mode 100644 agentbox/internal/agent/openai_test.go create mode 100644 agentbox/internal/appspec/appspec.go create mode 100644 agentbox/internal/appspec/appspec_test.go create mode 100644 agentbox/internal/environment/dockerx11/docker.go create mode 100644 agentbox/internal/environment/environment.go create mode 100644 agentbox/internal/runtime/runtime.go create mode 100644 agentbox/internal/trace/store.go create mode 100755 agentbox/scripts/build_demo.sh diff --git a/agentbox/.gitignore b/agentbox/.gitignore index bf2562a..ea591ad 100644 --- a/agentbox/.gitignore +++ b/agentbox/.gitignore @@ -1,2 +1,3 @@ .agentbox/ -agentbox +/agentbox +examples/mover/mover diff --git a/agentbox/Makefile b/agentbox/Makefile index 579a31f..d2cef21 100644 --- a/agentbox/Makefile +++ b/agentbox/Makefile @@ -1,4 +1,13 @@ -.PHONY: phase1 test +.PHONY: build demo demo-binary phase1 test + +build: + go build -o agentbox ./cmd/agentbox + +demo-binary: + ./scripts/build_demo.sh + +demo: build demo-binary + ./agentbox run ./examples/mover --task "Launch the application, move the character to the right, and describe what happened." phase1: go run ./cmd/agentbox phase1 diff --git a/agentbox/cmd/agentbox/main.go b/agentbox/cmd/agentbox/main.go new file mode 100644 index 0000000..e8fdb33 --- /dev/null +++ b/agentbox/cmd/agentbox/main.go @@ -0,0 +1,241 @@ +package main + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "os" + "os/signal" + "path/filepath" + "strconv" + "strings" + "syscall" + + "agentbox/internal/agent" + "agentbox/internal/appspec" + "agentbox/internal/environment/dockerx11" + "agentbox/internal/phase1" + agentRuntime "agentbox/internal/runtime" + "agentbox/internal/trace" +) + +func main() { + ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer cancel() + + if len(os.Args) < 2 { + usage() + os.Exit(2) + } + + var err error + switch os.Args[1] { + case "phase1": + err = phase1.Run(ctx) + case "run": + err = run(ctx, os.Args[2:]) + case "runs": + err = listRuns() + case "inspect": + err = inspect(os.Args[2:]) + default: + usage() + os.Exit(2) + } + if err != nil { + fmt.Fprintf(os.Stderr, "agentbox: %v\n", err) + os.Exit(1) + } +} + +type runOptions struct { + path string + task string + agent string + model string + maxSteps int +} + +func run(ctx context.Context, args []string) error { + options, err := parseRunOptions(args) + if err != nil { + return err + } + root, err := findRoot() + if err != nil { + return err + } + command, err := appspec.Resolve(options.path) + if err != nil { + return err + } + controller, err := selectAgent(options) + if err != nil { + return err + } + store, err := trace.New(root, options.task, options.path, controller.Name()) + if err != nil { + return err + } + env := dockerx11.New(dockerx11.Config{ + ProjectRoot: root, + RunID: store.ID(), + Output: os.Stdout, + }) + err = agentRuntime.Run(ctx, agentRuntime.Config{ + Task: options.task, + Command: command, + Agent: controller, + Env: env, + Trace: store, + MaxSteps: options.maxSteps, + Output: os.Stdout, + }) + if err != nil { + fmt.Fprintf(os.Stderr, "Run artifacts: %s\n", store.Directory()) + return err + } + fmt.Printf("Replay written to:\n%s\n", store.Directory()) + return nil +} + +func parseRunOptions(args []string) (runOptions, error) { + options := runOptions{agent: "deterministic", model: "gpt-5", maxSteps: 20} + for index := 0; index < len(args); index++ { + arg := args[index] + var name, value string + if strings.HasPrefix(arg, "--") { + name, value, _ = strings.Cut(strings.TrimPrefix(arg, "--"), "=") + if value == "" { + index++ + if index >= len(args) { + return options, fmt.Errorf("--%s requires a value", name) + } + value = args[index] + } + switch name { + case "task": + options.task = value + case "agent": + options.agent = value + case "model": + options.model = value + case "max-steps": + number, err := strconv.Atoi(value) + if err != nil || number < 1 { + return options, errors.New("--max-steps must be a positive integer") + } + options.maxSteps = number + default: + return options, fmt.Errorf("unknown flag --%s", name) + } + continue + } + if options.path != "" { + return options, errors.New("run accepts exactly one application path") + } + options.path = arg + } + if options.path == "" { + return options, errors.New("run requires an application path") + } + if options.task == "" { + return options, errors.New("run requires --task") + } + return options, nil +} + +func selectAgent(options runOptions) (agent.Agent, error) { + switch options.agent { + case "deterministic": + return &agent.Deterministic{}, nil + case "openai": + return agent.NewOpenAI(agent.OpenAIConfig{ + APIKey: os.Getenv("OPENAI_API_KEY"), + Model: options.model, + }) + default: + return nil, fmt.Errorf("unknown agent %q (want deterministic or openai)", options.agent) + } +} + +func listRuns() error { + root, err := findRoot() + if err != nil { + return err + } + runs, err := trace.List(root) + if err != nil { + return err + } + if len(runs) == 0 { + fmt.Println("No recorded runs.") + return nil + } + for _, run := range runs { + fmt.Printf("%s %-8s %-20s %s\n", + run.ID, run.Status, run.Agent, run.Task) + } + return nil +} + +func inspect(args []string) error { + if len(args) != 1 { + return errors.New("inspect requires one run ID") + } + root, err := findRoot() + if err != nil { + return err + } + run, err := trace.Read(root, args[0]) + if err != nil { + return err + } + steps, err := trace.ReadSteps(root, args[0]) + if err != nil { + return err + } + output := struct { + Run trace.Run `json:"run"` + Steps []trace.StepRecord `json:"steps"` + Directory string `json:"directory"` + }{ + Run: run, + Steps: steps, + Directory: filepath.Join(root, ".agentbox", "runs", run.ID), + } + data, err := json.MarshalIndent(output, "", " ") + if err != nil { + return err + } + fmt.Println(string(data)) + return nil +} + +func findRoot() (string, error) { + current, err := os.Getwd() + if err != nil { + return "", err + } + for { + data, readErr := os.ReadFile(filepath.Join(current, "go.mod")) + if readErr == nil && bytes.Contains(data, []byte("module agentbox")) { + return current, nil + } + parent := filepath.Dir(current) + if parent == current { + return "", errors.New("run agentbox from inside its module directory") + } + current = parent + } +} + +func usage() { + fmt.Fprintln(os.Stderr, `usage: + agentbox run --task "" [--agent deterministic|openai] [--model ] + agentbox runs + agentbox inspect + agentbox phase1`) +} diff --git a/agentbox/cmd/agentbox/main_test.go b/agentbox/cmd/agentbox/main_test.go new file mode 100644 index 0000000..2476668 --- /dev/null +++ b/agentbox/cmd/agentbox/main_test.go @@ -0,0 +1,15 @@ +package main + +import "testing" + +func TestParseRunOptionsAllowsFlagsAfterPath(t *testing.T) { + options, err := parseRunOptions([]string{ + "./game", "--task", "move right", "--max-steps=7", + }) + if err != nil { + t.Fatal(err) + } + if options.path != "./game" || options.task != "move right" || options.maxSteps != 7 { + t.Fatalf("options = %#v", options) + } +} diff --git a/agentbox/examples/mover/agentbox.json b/agentbox/examples/mover/agentbox.json new file mode 100644 index 0000000..0562b46 --- /dev/null +++ b/agentbox/examples/mover/agentbox.json @@ -0,0 +1,4 @@ +{ + "command": "mover", + "window_title": "Agentbox Mover" +} diff --git a/agentbox/internal/agent/agent.go b/agentbox/internal/agent/agent.go new file mode 100644 index 0000000..eb9e52e --- /dev/null +++ b/agentbox/internal/agent/agent.go @@ -0,0 +1,39 @@ +package agent + +import ( + "context" + "time" + + "agentbox/internal/environment" +) + +type Observation struct { + Screenshot []byte + Timestamp time.Time + Logs []environment.LogEntry + PreviousActions []environment.InputAction +} + +type Step struct { + Number int + Timestamp time.Time + ScreenshotPath string + Message string + Action *environment.InputAction +} + +type Decision struct { + Reason string + Action *environment.InputAction + Done bool +} + +type Agent interface { + Name() string + NextAction( + ctx context.Context, + task string, + history []Step, + observation Observation, + ) (Decision, error) +} diff --git a/agentbox/internal/agent/deterministic.go b/agentbox/internal/agent/deterministic.go new file mode 100644 index 0000000..fcd2557 --- /dev/null +++ b/agentbox/internal/agent/deterministic.go @@ -0,0 +1,42 @@ +package agent + +import ( + "context" + "time" + + "agentbox/internal/environment" +) + +type Deterministic struct { + next int +} + +func (a *Deterministic) Name() string { + return "deterministic-right" +} + +func (a *Deterministic) NextAction( + _ context.Context, + _ string, + _ []Step, + _ Observation, +) (Decision, error) { + var decision Decision + switch a.next { + case 0: + action := environment.InputAction{Type: environment.KeyDown, Key: "RIGHT"} + decision = Decision{Reason: "Press RIGHT to start moving.", Action: &action} + case 1: + action := environment.InputAction{Type: environment.Wait, Duration: time.Second} + decision = Decision{Reason: "Keep RIGHT held for one second.", Action: &action} + case 2: + action := environment.InputAction{Type: environment.KeyUp, Key: "RIGHT"} + decision = Decision{Reason: "Release RIGHT after the movement.", Action: &action} + default: + decision = Decision{Reason: "The movement sequence is complete.", Done: true} + } + a.next++ + return decision, nil +} + +var _ Agent = (*Deterministic)(nil) diff --git a/agentbox/internal/agent/deterministic_test.go b/agentbox/internal/agent/deterministic_test.go new file mode 100644 index 0000000..f9ca40f --- /dev/null +++ b/agentbox/internal/agent/deterministic_test.go @@ -0,0 +1,33 @@ +package agent + +import ( + "context" + "testing" + + "agentbox/internal/environment" +) + +func TestDeterministicSequence(t *testing.T) { + controller := &Deterministic{} + want := []struct { + action environment.InputType + done bool + }{ + {environment.KeyDown, false}, + {environment.Wait, false}, + {environment.KeyUp, false}, + {"", true}, + } + for index, expected := range want { + decision, err := controller.NextAction(context.Background(), "", nil, Observation{}) + if err != nil { + t.Fatalf("step %d: %v", index, err) + } + if decision.Done != expected.done { + t.Fatalf("step %d done = %v, want %v", index, decision.Done, expected.done) + } + if expected.action != "" && (decision.Action == nil || decision.Action.Type != expected.action) { + t.Fatalf("step %d action = %#v, want %s", index, decision.Action, expected.action) + } + } +} diff --git a/agentbox/internal/agent/openai.go b/agentbox/internal/agent/openai.go new file mode 100644 index 0000000..ae91ad6 --- /dev/null +++ b/agentbox/internal/agent/openai.go @@ -0,0 +1,244 @@ +package agent + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "strings" + "time" + + "agentbox/internal/environment" +) + +type OpenAIConfig struct { + APIKey string + Model string + BaseURL string + Client *http.Client +} + +type OpenAI struct { + config OpenAIConfig +} + +func NewOpenAI(config OpenAIConfig) (*OpenAI, error) { + if config.APIKey == "" { + return nil, errors.New("OPENAI_API_KEY is required for --agent openai") + } + if config.Model == "" { + config.Model = "gpt-5" + } + if config.BaseURL == "" { + config.BaseURL = "https://api.openai.com/v1" + } + if config.Client == nil { + config.Client = &http.Client{Timeout: 90 * time.Second} + } + return &OpenAI{config: config}, nil +} + +func (a *OpenAI) Name() string { + return "openai:" + a.config.Model +} + +func (a *OpenAI) NextAction( + ctx context.Context, + task string, + history []Step, + observation Observation, +) (Decision, error) { + prompt, err := modelPrompt(task, history, observation) + if err != nil { + return Decision{}, err + } + requestBody := map[string]any{ + "model": a.config.Model, + "input": []any{ + map[string]any{ + "role": "user", + "content": []any{ + map[string]any{"type": "input_text", "text": prompt}, + map[string]any{ + "type": "input_image", + "image_url": "data:image/png;base64," + + base64.StdEncoding.EncodeToString(observation.Screenshot), + }, + }, + }, + }, + "text": map[string]any{ + "format": map[string]any{ + "type": "json_schema", + "name": "agentbox_action", + "strict": true, + "schema": decisionSchema(), + }, + }, + } + body, err := json.Marshal(requestBody) + if err != nil { + return Decision{}, err + } + request, err := http.NewRequestWithContext(ctx, http.MethodPost, + strings.TrimRight(a.config.BaseURL, "/")+"/responses", bytes.NewReader(body)) + if err != nil { + return Decision{}, err + } + request.Header.Set("Authorization", "Bearer "+a.config.APIKey) + request.Header.Set("Content-Type", "application/json") + + response, err := a.config.Client.Do(request) + if err != nil { + return Decision{}, err + } + defer response.Body.Close() + responseBody, err := io.ReadAll(io.LimitReader(response.Body, 4<<20)) + if err != nil { + return Decision{}, err + } + if response.StatusCode < 200 || response.StatusCode >= 300 { + return Decision{}, fmt.Errorf("OpenAI response %s: %s", + response.Status, strings.TrimSpace(string(responseBody))) + } + text, err := responseText(responseBody) + if err != nil { + return Decision{}, err + } + return parseModelDecision(text) +} + +func modelPrompt(task string, history []Step, observation Observation) (string, error) { + contextData := struct { + Task string `json:"task"` + History []Step `json:"history"` + RecentLogs []environment.LogEntry `json:"recent_logs"` + PreviousActions []environment.InputAction `json:"previous_actions"` + }{ + Task: task, + History: history, + RecentLogs: observation.Logs, + PreviousActions: observation.PreviousActions, + } + data, err := json.Marshal(contextData) + if err != nil { + return "", err + } + return "You control an interactive Linux application from screenshots. " + + "Choose exactly one safe input action, or mark done when the task is complete. " + + "Use X11 key names such as RIGHT, Return, or Escape. Keep waits under 5000 ms.\n" + + string(data), nil +} + +func decisionSchema() map[string]any { + actionProperties := map[string]any{ + "type": map[string]any{ + "type": "string", + "enum": []string{ + string(environment.KeyDown), string(environment.KeyUp), + string(environment.MouseMove), string(environment.MouseDown), + string(environment.MouseUp), string(environment.Wait), + }, + }, + "key": map[string]any{"type": "string"}, + "x": map[string]any{"type": "integer"}, + "y": map[string]any{"type": "integer"}, + "button": map[string]any{"type": "integer"}, + "duration_ms": map[string]any{"type": "integer"}, + } + return map[string]any{ + "type": "object", + "additionalProperties": false, + "properties": map[string]any{ + "reason": map[string]any{"type": "string"}, + "done": map[string]any{"type": "boolean"}, + "action": map[string]any{ + "anyOf": []any{ + map[string]any{ + "type": "object", + "additionalProperties": false, + "properties": actionProperties, + "required": []string{ + "type", "key", "x", "y", "button", "duration_ms", + }, + }, + map[string]any{"type": "null"}, + }, + }, + }, + "required": []string{"reason", "done", "action"}, + } +} + +func responseText(data []byte) (string, error) { + var response struct { + Output []struct { + Content []struct { + Type string `json:"type"` + Text string `json:"text"` + } `json:"content"` + } `json:"output"` + } + if err := json.Unmarshal(data, &response); err != nil { + return "", fmt.Errorf("decode OpenAI response: %w", err) + } + for _, output := range response.Output { + for _, content := range output.Content { + if content.Type == "output_text" && content.Text != "" { + return content.Text, nil + } + } + } + return "", errors.New("OpenAI response contained no output_text") +} + +func parseModelDecision(text string) (Decision, error) { + var result struct { + Reason string `json:"reason"` + Done bool `json:"done"` + Action *struct { + Type environment.InputType `json:"type"` + Key string `json:"key"` + X int `json:"x"` + Y int `json:"y"` + Button int `json:"button"` + DurationMS int `json:"duration_ms"` + } `json:"action"` + } + if err := json.Unmarshal([]byte(text), &result); err != nil { + return Decision{}, fmt.Errorf("decode model decision: %w", err) + } + if result.Reason == "" { + return Decision{}, errors.New("model decision requires reason") + } + if result.Done { + return Decision{Reason: result.Reason, Done: true}, nil + } + if result.Action == nil { + return Decision{}, errors.New("model decision requires action when not done") + } + if result.Action.DurationMS < 0 || result.Action.DurationMS > 5000 { + return Decision{}, errors.New("model wait must be between 0 and 5000 ms") + } + action := environment.InputAction{ + Type: result.Action.Type, + Key: result.Action.Key, + X: result.Action.X, + Y: result.Action.Y, + Button: result.Action.Button, + Duration: time.Duration(result.Action.DurationMS) * time.Millisecond, + } + switch action.Type { + case environment.KeyDown, environment.KeyUp, environment.MouseMove, + environment.MouseDown, environment.MouseUp, environment.Wait: + default: + return Decision{}, fmt.Errorf("model returned unsupported action %q", action.Type) + } + return Decision{Reason: result.Reason, Action: &action}, nil +} + +var _ Agent = (*OpenAI)(nil) diff --git a/agentbox/internal/agent/openai_test.go b/agentbox/internal/agent/openai_test.go new file mode 100644 index 0000000..8b0078f --- /dev/null +++ b/agentbox/internal/agent/openai_test.go @@ -0,0 +1,59 @@ +package agent + +import ( + "context" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "agentbox/internal/environment" +) + +func TestOpenAISendsScreenshotAndParsesDecision(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) { + if request.URL.Path != "/responses" { + t.Errorf("path = %q, want /responses", request.URL.Path) + } + if request.Header.Get("Authorization") != "Bearer test-key" { + t.Errorf("missing bearer token") + } + body, err := io.ReadAll(request.Body) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(body), "data:image/png;base64,AQID") { + t.Errorf("request does not contain screenshot data URL: %s", body) + } + response.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(response, `{ + "output":[{"content":[{"type":"output_text","text":"{\"reason\":\"move right\",\"done\":false,\"action\":{\"type\":\"key_down\",\"key\":\"RIGHT\",\"x\":0,\"y\":0,\"button\":0,\"duration_ms\":0}}"}]}] + }`) + })) + defer server.Close() + + controller, err := NewOpenAI(OpenAIConfig{ + APIKey: "test-key", Model: "test-model", BaseURL: server.URL, Client: server.Client(), + }) + if err != nil { + t.Fatal(err) + } + decision, err := controller.NextAction(context.Background(), "move", nil, Observation{ + Screenshot: []byte{1, 2, 3}, + }) + if err != nil { + t.Fatal(err) + } + if decision.Action == nil || decision.Action.Type != environment.KeyDown || + decision.Action.Key != "RIGHT" { + t.Fatalf("decision = %#v", decision) + } +} + +func TestDecisionSchemaIsJSONSerializable(t *testing.T) { + if _, err := json.Marshal(decisionSchema()); err != nil { + t.Fatal(err) + } +} diff --git a/agentbox/internal/appspec/appspec.go b/agentbox/internal/appspec/appspec.go new file mode 100644 index 0000000..a469048 --- /dev/null +++ b/agentbox/internal/appspec/appspec.go @@ -0,0 +1,64 @@ +package appspec + +import ( + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + + "agentbox/internal/environment" +) + +type manifest struct { + Command string `json:"command"` + Args []string `json:"args"` + Env map[string]string `json:"env"` + WindowTitle string `json:"window_title"` +} + +func Resolve(path string) (environment.Command, error) { + absolute, err := filepath.Abs(path) + if err != nil { + return environment.Command{}, fmt.Errorf("resolve application path: %w", err) + } + info, err := os.Stat(absolute) + if err != nil { + return environment.Command{}, fmt.Errorf("inspect application path: %w", err) + } + if !info.IsDir() { + if !info.Mode().IsRegular() { + return environment.Command{}, errors.New("application must be a regular executable file") + } + return environment.Command{Path: absolute}, nil + } + + data, err := os.ReadFile(filepath.Join(absolute, "agentbox.json")) + if err != nil { + return environment.Command{}, fmt.Errorf("read directory manifest agentbox.json: %w", err) + } + var config manifest + if err := json.Unmarshal(data, &config); err != nil { + return environment.Command{}, fmt.Errorf("parse agentbox.json: %w", err) + } + if config.Command == "" { + return environment.Command{}, errors.New("agentbox.json requires command") + } + commandPath := config.Command + if !filepath.IsAbs(commandPath) { + commandPath = filepath.Join(absolute, commandPath) + } + commandInfo, err := os.Stat(commandPath) + if err != nil { + return environment.Command{}, fmt.Errorf("inspect manifest command: %w", err) + } + if !commandInfo.Mode().IsRegular() { + return environment.Command{}, errors.New("manifest command must be a regular executable file") + } + return environment.Command{ + Path: commandPath, + Args: config.Args, + Env: config.Env, + WindowTitle: config.WindowTitle, + }, nil +} diff --git a/agentbox/internal/appspec/appspec_test.go b/agentbox/internal/appspec/appspec_test.go new file mode 100644 index 0000000..19beb2a --- /dev/null +++ b/agentbox/internal/appspec/appspec_test.go @@ -0,0 +1,28 @@ +package appspec + +import ( + "os" + "path/filepath" + "testing" +) + +func TestResolveDirectoryManifest(t *testing.T) { + directory := t.TempDir() + executable := filepath.Join(directory, "game") + if err := os.WriteFile(executable, []byte("binary"), 0o755); err != nil { + t.Fatal(err) + } + manifest := `{"command":"game","args":["--demo"],"window_title":"Demo"}` + if err := os.WriteFile(filepath.Join(directory, "agentbox.json"), []byte(manifest), 0o644); err != nil { + t.Fatal(err) + } + + command, err := Resolve(directory) + if err != nil { + t.Fatal(err) + } + if command.Path != executable || command.WindowTitle != "Demo" || + len(command.Args) != 1 || command.Args[0] != "--demo" { + t.Fatalf("command = %#v", command) + } +} diff --git a/agentbox/internal/environment/dockerx11/docker.go b/agentbox/internal/environment/dockerx11/docker.go new file mode 100644 index 0000000..86fd713 --- /dev/null +++ b/agentbox/internal/environment/dockerx11/docker.go @@ -0,0 +1,266 @@ +package dockerx11 + +import ( + "context" + "errors" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "sync" + "time" + + "agentbox/internal/environment" +) + +const imageName = "agentbox-runtime:local" + +type Config struct { + ProjectRoot string + RunID string + Output io.Writer +} + +type Environment struct { + config Config + containerName string + created bool + stopped bool + mu sync.Mutex +} + +func New(config Config) *Environment { + return &Environment{ + config: config, + containerName: "agentbox-run-" + config.RunID, + } +} + +func (e *Environment) Start(ctx context.Context) error { + if e.config.Output == nil { + e.config.Output = io.Discard + } + if _, err := e.docker(ctx, "version", "--format", "{{.Server.Version}}"); err != nil { + return fmt.Errorf("Docker is required and the daemon must be accessible: %w", err) + } + fmt.Fprintln(e.config.Output, "Creating environment...") + if err := e.dockerStream(ctx, "build", "-q", "-t", imageName, "-f", + filepath.Join(e.config.ProjectRoot, "environment", "Dockerfile"), e.config.ProjectRoot); err != nil { + return fmt.Errorf("build environment image: %w", err) + } + _, err := e.docker(ctx, + "create", + "--name", e.containerName, + "--init", + "--network=none", + "--read-only", + "--tmpfs=/tmp:rw,nosuid,nodev,size=128m", + "--cap-drop=ALL", + "--security-opt=no-new-privileges", + "--pids-limit=128", + "--memory=512m", + "--cpus=1", + imageName, + ) + if err != nil { + return fmt.Errorf("create environment: %w", err) + } + e.created = true + if _, err := e.docker(ctx, "start", e.containerName); err != nil { + return fmt.Errorf("start environment: %w", err) + } + if err := e.waitFor(ctx, 10*time.Second, func() bool { + _, readyErr := e.docker(ctx, "exec", e.containerName, "test", "-f", "/tmp/agentbox-ready") + return readyErr == nil + }); err != nil { + return fmt.Errorf("wait for graphical environment: %w", err) + } + return nil +} + +func (e *Environment) Launch(ctx context.Context, command environment.Command) error { + fmt.Fprintln(e.config.Output, "Uploading build...") + file, err := os.Open(command.Path) + if err != nil { + return fmt.Errorf("open application: %w", err) + } + defer file.Close() + + stage := exec.CommandContext(ctx, "docker", "exec", "-i", e.containerName, "sh", "-c", + "cat > /tmp/application && chmod 0500 /tmp/application") + stage.Stdin = file + if output, err := stage.CombinedOutput(); err != nil { + return fmt.Errorf("stage application: %w: %s", err, strings.TrimSpace(string(output))) + } + + fmt.Fprintln(e.config.Output, "Launching application...") + args := []string{"exec", "-d"} + for key, value := range command.Env { + args = append(args, "-e", key+"="+value) + } + args = append(args, e.containerName, "sh", "-c") + parts := []string{"exec", "/tmp/application"} + for _, arg := range command.Args { + parts = append(parts, shellQuote(arg)) + } + parts = append(parts, ">/tmp/stdout.log", "2>/tmp/stderr.log") + args = append(args, strings.Join(parts, " ")) + if _, err := e.docker(ctx, args...); err != nil { + return fmt.Errorf("launch application: %w", err) + } + + if command.WindowTitle != "" { + var windowID string + if err := e.waitFor(ctx, 10*time.Second, func() bool { + output, searchErr := e.docker(ctx, "exec", e.containerName, "xdotool", + "search", "--name", command.WindowTitle) + if searchErr != nil { + return false + } + windowID = strings.TrimSpace(strings.Split(string(output), "\n")[0]) + return windowID != "" + }); err != nil { + return fmt.Errorf("wait for application window %q: %w", command.WindowTitle, err) + } + if _, err := e.docker(ctx, "exec", e.containerName, "xdotool", + "windowactivate", "--sync", windowID); err != nil { + return fmt.Errorf("focus application window: %w", err) + } + } else { + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(500 * time.Millisecond): + } + } + return nil +} + +func (e *Environment) Screenshot(ctx context.Context) ([]byte, error) { + if _, err := e.docker(ctx, "exec", e.containerName, "scrot", "-o", "/tmp/screenshot.png"); err != nil { + return nil, fmt.Errorf("capture screenshot: %w", err) + } + data, err := e.docker(ctx, "exec", e.containerName, "cat", "/tmp/screenshot.png") + if err != nil { + return nil, fmt.Errorf("extract screenshot: %w", err) + } + return data, nil +} + +func (e *Environment) SendInput(ctx context.Context, action environment.InputAction) error { + var args []string + switch action.Type { + case environment.KeyDown: + if action.Key == "" { + return errors.New("key_down requires key") + } + args = []string{"keydown", action.Key} + case environment.KeyUp: + if action.Key == "" { + return errors.New("key_up requires key") + } + args = []string{"keyup", action.Key} + case environment.MouseMove: + args = []string{"mousemove", strconv.Itoa(action.X), strconv.Itoa(action.Y)} + case environment.MouseDown: + args = []string{"mousedown", strconv.Itoa(action.Button)} + case environment.MouseUp: + args = []string{"mouseup", strconv.Itoa(action.Button)} + case environment.Wait: + if action.Duration < 0 { + return errors.New("wait duration cannot be negative") + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(action.Duration): + return nil + } + default: + return fmt.Errorf("unsupported input action %q", action.Type) + } + _, err := e.docker(ctx, append([]string{"exec", e.containerName, "xdotool"}, args...)...) + if err != nil { + return fmt.Errorf("send %s: %w", action.Type, err) + } + return nil +} + +func (e *Environment) Logs(ctx context.Context) ([]environment.LogEntry, error) { + var entries []environment.LogEntry + for _, stream := range []string{"stdout", "stderr"} { + data, err := e.docker(ctx, "exec", e.containerName, "cat", "/tmp/"+stream+".log") + if err != nil { + continue + } + if len(data) > 0 { + entries = append(entries, environment.LogEntry{ + Stream: stream, Message: string(data), Time: time.Now().UTC(), + }) + } + } + return entries, nil +} + +func (e *Environment) Stop(ctx context.Context) error { + e.mu.Lock() + defer e.mu.Unlock() + if !e.created || e.stopped { + return nil + } + e.stopped = true + fmt.Fprintln(e.config.Output, "Shutting environment down...") + _, stopErr := e.docker(ctx, "stop", "--time=3", e.containerName) + _, removeErr := e.docker(ctx, "rm", "-f", e.containerName) + if stopErr != nil { + return fmt.Errorf("stop environment: %w", stopErr) + } + if removeErr != nil { + return fmt.Errorf("remove environment: %w", removeErr) + } + return nil +} + +func (e *Environment) waitFor(ctx context.Context, timeout time.Duration, check func() bool) error { + timer := time.NewTimer(timeout) + defer timer.Stop() + ticker := time.NewTicker(100 * time.Millisecond) + defer ticker.Stop() + for { + if check() { + return nil + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-timer.C: + return errors.New("timed out") + case <-ticker.C: + } + } +} + +func (e *Environment) docker(ctx context.Context, args ...string) ([]byte, error) { + command := exec.CommandContext(ctx, "docker", args...) + output, err := command.CombinedOutput() + if err != nil { + return output, fmt.Errorf("docker %s: %w: %s", args[0], err, strings.TrimSpace(string(output))) + } + return output, nil +} + +func (e *Environment) dockerStream(ctx context.Context, args ...string) error { + command := exec.CommandContext(ctx, "docker", args...) + command.Stdout = e.config.Output + command.Stderr = e.config.Output + return command.Run() +} + +func shellQuote(value string) string { + return "'" + strings.ReplaceAll(value, "'", "'\"'\"'") + "'" +} + +var _ environment.Environment = (*Environment)(nil) diff --git a/agentbox/internal/environment/environment.go b/agentbox/internal/environment/environment.go new file mode 100644 index 0000000..d7051aa --- /dev/null +++ b/agentbox/internal/environment/environment.go @@ -0,0 +1,48 @@ +package environment + +import ( + "context" + "time" +) + +type Command struct { + Path string + Args []string + Env map[string]string + WindowTitle string +} + +type InputType string + +const ( + KeyDown InputType = "key_down" + KeyUp InputType = "key_up" + MouseMove InputType = "mouse_move" + MouseDown InputType = "mouse_down" + MouseUp InputType = "mouse_up" + Wait InputType = "wait" +) + +type InputAction struct { + Type InputType `json:"type"` + Key string `json:"key,omitempty"` + X int `json:"x,omitempty"` + Y int `json:"y,omitempty"` + Button int `json:"button,omitempty"` + Duration time.Duration `json:"duration,omitempty"` +} + +type LogEntry struct { + Stream string `json:"stream"` + Message string `json:"message"` + Time time.Time `json:"time"` +} + +type Environment interface { + Start(context.Context) error + Launch(context.Context, Command) error + Screenshot(context.Context) ([]byte, error) + SendInput(context.Context, InputAction) error + Logs(context.Context) ([]LogEntry, error) + Stop(context.Context) error +} diff --git a/agentbox/internal/runtime/runtime.go b/agentbox/internal/runtime/runtime.go new file mode 100644 index 0000000..f28ac04 --- /dev/null +++ b/agentbox/internal/runtime/runtime.go @@ -0,0 +1,149 @@ +package runtime + +import ( + "context" + "errors" + "fmt" + "io" + "time" + + "agentbox/internal/agent" + "agentbox/internal/environment" + "agentbox/internal/trace" +) + +type Config struct { + Task string + Command environment.Command + Agent agent.Agent + Env environment.Environment + Trace *trace.Store + MaxSteps int + Output io.Writer +} + +func Run(ctx context.Context, config Config) (runErr error) { + if config.MaxSteps <= 0 { + config.MaxSteps = 20 + } + if config.Output == nil { + config.Output = io.Discard + } + if config.Agent == nil || config.Env == nil || config.Trace == nil { + return errors.New("runtime requires agent, environment, and trace") + } + + started := false + defer func() { + cleanupCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + if started { + if logs, err := config.Env.Logs(cleanupCtx); err == nil { + if logErr := config.Trace.WriteLogs(logs); runErr == nil && logErr != nil { + runErr = logErr + } + } + if stopErr := config.Env.Stop(cleanupCtx); runErr == nil && stopErr != nil { + runErr = stopErr + } + } + if finishErr := config.Trace.Finish(runErr); runErr == nil && finishErr != nil { + runErr = finishErr + } + }() + + if err := config.Env.Start(ctx); err != nil { + return err + } + started = true + if err := config.Env.Launch(ctx, config.Command); err != nil { + return err + } + fmt.Fprintln(config.Output, "Agent attached.") + + var history []agent.Step + var actions []environment.InputAction + startedAt := time.Now() + for number := 1; number <= config.MaxSteps; number++ { + screenshot, err := config.Env.Screenshot(ctx) + if err != nil { + return err + } + screenshotPath, err := config.Trace.SaveScreenshot(number, screenshot) + if err != nil { + return err + } + logs, err := config.Env.Logs(ctx) + if err != nil { + return err + } + now := time.Now().UTC() + fmt.Fprintf(config.Output, "[%s] screenshot captured\n", elapsed(startedAt)) + decision, err := config.Agent.NextAction(ctx, config.Task, history, agent.Observation{ + Screenshot: screenshot, + Timestamp: now, + Logs: logs, + PreviousActions: append([]environment.InputAction(nil), actions...), + }) + if err != nil { + return fmt.Errorf("agent next action: %w", err) + } + fmt.Fprintf(config.Output, "[%s] agent: %s\n", elapsed(startedAt), decision.Reason) + record := trace.StepRecord{ + Step: number, + Timestamp: now, + Observation: trace.ObservationRecord{ + Screenshot: screenshotPath, + Logs: logs, + PreviousActions: append([]environment.InputAction(nil), actions...), + }, + Agent: trace.AgentRecord{Message: decision.Reason}, + Action: decision.Action, + Done: decision.Done, + } + if err := config.Trace.Record(record); err != nil { + return err + } + history = append(history, agent.Step{ + Number: number, + Timestamp: now, + ScreenshotPath: screenshotPath, + Message: decision.Reason, + Action: decision.Action, + }) + if decision.Done { + fmt.Fprintln(config.Output, "Task complete.") + return nil + } + if decision.Action == nil { + return errors.New("agent returned neither action nor completion") + } + fmt.Fprintf(config.Output, "[%s] action: %s%s\n", + elapsed(startedAt), decision.Action.Type, actionDetail(*decision.Action)) + if err := config.Env.SendInput(ctx, *decision.Action); err != nil { + return err + } + actions = append(actions, *decision.Action) + } + return fmt.Errorf("agent exceeded maximum of %d steps", config.MaxSteps) +} + +func elapsed(start time.Time) string { + duration := time.Since(start).Round(time.Second) + return fmt.Sprintf("%02d:%02d", int(duration.Minutes()), int(duration.Seconds())%60) +} + +func actionDetail(action environment.InputAction) string { + switch action.Type { + case environment.KeyDown, environment.KeyUp: + return " " + action.Key + case environment.MouseMove: + return fmt.Sprintf(" %d,%d", action.X, action.Y) + case environment.MouseDown, environment.MouseUp: + return fmt.Sprintf(" %d", action.Button) + case environment.Wait: + return " " + action.Duration.String() + default: + return "" + } +} diff --git a/agentbox/internal/trace/store.go b/agentbox/internal/trace/store.go new file mode 100644 index 0000000..51dcbd4 --- /dev/null +++ b/agentbox/internal/trace/store.go @@ -0,0 +1,251 @@ +package trace + +import ( + "bufio" + "crypto/rand" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "sort" + "time" + + "agentbox/internal/environment" +) + +const SchemaVersion = "1" + +type Run struct { + SchemaVersion string `json:"schema_version"` + ID string `json:"id"` + Task string `json:"task"` + Application string `json:"application"` + Agent string `json:"agent"` + Status string `json:"status"` + StartedAt time.Time `json:"started_at"` + FinishedAt time.Time `json:"finished_at,omitempty"` + StepCount int `json:"step_count"` + Error string `json:"error,omitempty"` +} + +type ObservationRecord struct { + Screenshot string `json:"screenshot"` + Logs []environment.LogEntry `json:"logs,omitempty"` + PreviousActions []environment.InputAction `json:"previous_actions,omitempty"` +} + +type AgentRecord struct { + Message string `json:"message"` +} + +type StepRecord struct { + Step int `json:"step"` + Timestamp time.Time `json:"timestamp"` + Observation ObservationRecord `json:"observation"` + Agent AgentRecord `json:"agent"` + Action *environment.InputAction `json:"action,omitempty"` + Done bool `json:"done,omitempty"` +} + +type Store struct { + root string + runDir string + run Run + steps *os.File + actions *os.File +} + +func New(root, task, application, agentName string) (*Store, error) { + id, err := newID() + if err != nil { + return nil, err + } + runDir := filepath.Join(root, ".agentbox", "runs", id) + if err := os.MkdirAll(filepath.Join(runDir, "screenshots"), 0o755); err != nil { + return nil, fmt.Errorf("create run directory: %w", err) + } + steps, err := os.Create(filepath.Join(runDir, "steps.jsonl")) + if err != nil { + return nil, fmt.Errorf("create step trace: %w", err) + } + actions, err := os.Create(filepath.Join(runDir, "actions.jsonl")) + if err != nil { + _ = steps.Close() + return nil, fmt.Errorf("create action trace: %w", err) + } + store := &Store{ + root: root, runDir: runDir, steps: steps, actions: actions, + run: Run{ + SchemaVersion: SchemaVersion, + ID: id, + Task: task, + Application: application, + Agent: agentName, + Status: "running", + StartedAt: time.Now().UTC(), + }, + } + if err := store.writeRun(); err != nil { + _ = steps.Close() + _ = actions.Close() + return nil, err + } + return store, nil +} + +func (s *Store) ID() string { + return s.run.ID +} + +func (s *Store) Directory() string { + return s.runDir +} + +func (s *Store) SaveScreenshot(step int, data []byte) (string, error) { + relative := filepath.Join("screenshots", fmt.Sprintf("%04d.png", step)) + if err := os.WriteFile(filepath.Join(s.runDir, relative), data, 0o644); err != nil { + return "", fmt.Errorf("write screenshot: %w", err) + } + return filepath.ToSlash(relative), nil +} + +func (s *Store) Record(record StepRecord) error { + if err := appendJSON(s.steps, record); err != nil { + return fmt.Errorf("record step: %w", err) + } + if record.Action != nil { + action := struct { + Step int `json:"step"` + Timestamp time.Time `json:"timestamp"` + Action environment.InputAction `json:"action"` + }{record.Step, record.Timestamp, *record.Action} + if err := appendJSON(s.actions, action); err != nil { + return fmt.Errorf("record action: %w", err) + } + } + s.run.StepCount = record.Step + return s.writeRun() +} + +func (s *Store) WriteLogs(logs []environment.LogEntry) error { + var stdout, stderr string + for _, entry := range logs { + if entry.Stream == "stderr" { + stderr = entry.Message + } else if entry.Stream == "stdout" { + stdout = entry.Message + } + } + if err := os.WriteFile(filepath.Join(s.runDir, "stdout.log"), []byte(stdout), 0o644); err != nil { + return err + } + return os.WriteFile(filepath.Join(s.runDir, "stderr.log"), []byte(stderr), 0o644) +} + +func (s *Store) Finish(runErr error) error { + if s.steps != nil { + _ = s.steps.Close() + s.steps = nil + } + if s.actions != nil { + _ = s.actions.Close() + s.actions = nil + } + s.run.FinishedAt = time.Now().UTC() + if runErr != nil { + s.run.Status = "failed" + s.run.Error = runErr.Error() + } else { + s.run.Status = "complete" + } + return s.writeRun() +} + +func List(root string) ([]Run, error) { + directories, err := os.ReadDir(filepath.Join(root, ".agentbox", "runs")) + if errors.Is(err, os.ErrNotExist) { + return nil, nil + } + if err != nil { + return nil, err + } + var runs []Run + for _, directory := range directories { + if !directory.IsDir() { + continue + } + run, err := Read(root, directory.Name()) + if err == nil && run.SchemaVersion == SchemaVersion { + runs = append(runs, run) + } + } + sort.Slice(runs, func(i, j int) bool { + return runs[i].StartedAt.After(runs[j].StartedAt) + }) + return runs, nil +} + +func Read(root, id string) (Run, error) { + if filepath.Base(id) != id { + return Run{}, errors.New("invalid run ID") + } + data, err := os.ReadFile(filepath.Join(root, ".agentbox", "runs", id, "run.json")) + if err != nil { + return Run{}, err + } + var run Run + if err := json.Unmarshal(data, &run); err != nil { + return Run{}, err + } + return run, nil +} + +func ReadSteps(root, id string) ([]StepRecord, error) { + if filepath.Base(id) != id { + return nil, errors.New("invalid run ID") + } + file, err := os.Open(filepath.Join(root, ".agentbox", "runs", id, "steps.jsonl")) + if err != nil { + return nil, err + } + defer file.Close() + var steps []StepRecord + scanner := bufio.NewScanner(file) + for scanner.Scan() { + var step StepRecord + if err := json.Unmarshal(scanner.Bytes(), &step); err != nil { + return nil, err + } + steps = append(steps, step) + } + return steps, scanner.Err() +} + +func appendJSON(file *os.File, value any) error { + data, err := json.Marshal(value) + if err != nil { + return err + } + if _, err := file.Write(append(data, '\n')); err != nil { + return err + } + return file.Sync() +} + +func (s *Store) writeRun() error { + data, err := json.MarshalIndent(s.run, "", " ") + if err != nil { + return err + } + return os.WriteFile(filepath.Join(s.runDir, "run.json"), append(data, '\n'), 0o644) +} + +func newID() (string, error) { + random := make([]byte, 3) + if _, err := rand.Read(random); err != nil { + return "", err + } + return time.Now().UTC().Format("20060102T150405") + "-" + hex.EncodeToString(random), nil +} diff --git a/agentbox/scripts/build_demo.sh b/agentbox/scripts/build_demo.sh new file mode 100755 index 0000000..4e1dd0e --- /dev/null +++ b/agentbox/scripts/build_demo.sh @@ -0,0 +1,17 @@ +#!/bin/sh +set -eu + +root=$(CDPATH= cd -- "$(dirname "$0")/.." && pwd) +image=agentbox-runtime:local +output="$root/examples/mover/mover" +temporary="$output.tmp" + +cleanup() { + rm -f "$temporary" +} +trap cleanup EXIT INT TERM + +docker build -q -t "$image" -f "$root/environment/Dockerfile" "$root" >/dev/null +docker run --rm --network=none --entrypoint cat "$image" /opt/agentbox/mover >"$temporary" +chmod 0755 "$temporary" +mv "$temporary" "$output"