diff --git a/agentbox/cmd/agentbox/run_command.go b/agentbox/cmd/agentbox/run_command.go index 2c7d8b0..d0631be 100644 --- a/agentbox/cmd/agentbox/run_command.go +++ b/agentbox/cmd/agentbox/run_command.go @@ -67,7 +67,7 @@ func run(ctx context.Context, args []string) error { fmt.Fprintf(os.Stderr, "Run artifacts: %s\n", traceStore.Directory()) return err } - fmt.Printf("Replay written to:\n%s\n", traceStore.Directory()) + fmt.Printf("Run artifacts written to:\n%s\n", traceStore.Directory()) return nil } diff --git a/agentbox/internal/agent/deterministic.go b/agentbox/internal/agent/deterministic.go index 2c2aa59..9ccafc3 100644 --- a/agentbox/internal/agent/deterministic.go +++ b/agentbox/internal/agent/deterministic.go @@ -76,6 +76,8 @@ func greenCentroidX(screenshot []byte) (float64, error) { for y := bounds.Min.Y; y < bounds.Max.Y; y++ { for x := bounds.Min.X; x < bounds.Max.X; x++ { red, green, blue, _ := renderedImage.At(x, y).RGBA() + // Match the mover's #00ff66 square. Requiring 1,000 matching pixels + // prevents a small green UI detail from passing verification. if green > 0xc000 && red < 0x4000 && blue < 0x8000 { xCoordinateSum += uint64(x) greenPixelCount++ diff --git a/agentbox/internal/agent/openai_protocol.go b/agentbox/internal/agent/openai_protocol.go index 1ada81c..5e137d4 100644 --- a/agentbox/internal/agent/openai_protocol.go +++ b/agentbox/internal/agent/openai_protocol.go @@ -87,12 +87,14 @@ func modelPrompt(task string, history []Step, observation Observation) (string, return "", err } return "You control an interactive Linux application from screenshots. " + - "Choose exactly one safe input action, or mark done when the task is complete. " + + "Choose exactly one allowed input action, or mark done when the task is complete. " + "Use logical key names such as RIGHT, ENTER, or ESCAPE. Keep waits under 5000 ms.\n" + string(data), nil } func decisionSchema() map[string]any { + // Strict structured output requires every action property to be present, + // even when a particular action ignores most of them. actionProperties := map[string]any{ "type": map[string]any{ "type": "string", diff --git a/agentbox/internal/appspec/appspec.go b/agentbox/internal/appspec/appspec.go index b4f0f79..33bfaec 100644 --- a/agentbox/internal/appspec/appspec.go +++ b/agentbox/internal/appspec/appspec.go @@ -11,10 +11,10 @@ import ( ) type manifest struct { - Command string `json:"command"` - Args []string `json:"args"` - Env map[string]string `json:"env"` - WindowTitle string `json:"window_title"` + Command string `json:"command"` + Args []string `json:"args"` + EnvironmentVariables map[string]string `json:"env"` + WindowTitle string `json:"window_title"` } // Resolve turns either an executable path or a directory containing @@ -58,9 +58,9 @@ func Resolve(path string) (environment.Command, error) { return environment.Command{}, errors.New("manifest command must be a regular executable file") } return environment.Command{ - Path: commandPath, - Args: config.Args, - Env: config.Env, - WindowTitle: config.WindowTitle, + Path: commandPath, + Args: config.Args, + EnvironmentVariables: config.EnvironmentVariables, + WindowTitle: config.WindowTitle, }, nil } diff --git a/agentbox/internal/environment/dockerx11/application.go b/agentbox/internal/environment/dockerx11/application.go index 357be43..1e79561 100644 --- a/agentbox/internal/environment/dockerx11/application.go +++ b/agentbox/internal/environment/dockerx11/application.go @@ -17,7 +17,7 @@ func (environmentBackend *Environment) Launch( ctx context.Context, application environment.Command, ) error { - fmt.Fprintln(environmentBackend.config.Output, "Uploading build...") + fmt.Fprintln(environmentBackend.config.Output, "Staging application...") executable, err := os.Open(application.Path) if err != nil { return fmt.Errorf("open application: %w", err) @@ -47,7 +47,7 @@ func (environmentBackend *Environment) Launch( fmt.Fprintln(environmentBackend.config.Output, "Launching application...") dockerArguments := []string{"exec", "-d"} - for variableName, variableValue := range application.Env { + for variableName, variableValue := range application.EnvironmentVariables { dockerArguments = append( dockerArguments, "-e", diff --git a/agentbox/internal/environment/dockerx11/lifecycle.go b/agentbox/internal/environment/dockerx11/lifecycle.go index a378d7d..fe8fd0a 100644 --- a/agentbox/internal/environment/dockerx11/lifecycle.go +++ b/agentbox/internal/environment/dockerx11/lifecycle.go @@ -69,6 +69,8 @@ func (environmentBackend *Environment) Start(ctx context.Context) error { ); err != nil { return fmt.Errorf("start environment: %w", err) } + // entrypoint.sh creates this file only after both Xvfb and Openbox accept + // requests. It is the readiness handshake between host and container. if err := waitFor(ctx, 10*time.Second, func() bool { _, readyErr := environmentBackend.runDocker( ctx, diff --git a/agentbox/internal/environment/environment.go b/agentbox/internal/environment/environment.go index c7667e3..66d9b4c 100644 --- a/agentbox/internal/environment/environment.go +++ b/agentbox/internal/environment/environment.go @@ -8,10 +8,10 @@ import ( // Command describes a host executable and how it should start in an // environment. Environment implementations decide how to stage the file. type Command struct { - Path string - Args []string - Env map[string]string - WindowTitle string + Path string + Args []string + EnvironmentVariables map[string]string + WindowTitle string } // InputType identifies one backend-neutral keyboard, mouse, or timing action. @@ -36,7 +36,8 @@ type InputAction struct { DurationMS int `json:"duration_ms,omitempty"` } -// LogEntry is a snapshot of one application output stream. +// LogEntry is a cumulative snapshot of one application output stream. Time is +// when Agentbox captured the snapshot, not when the application emitted it. type LogEntry struct { Stream string `json:"stream"` Message string `json:"message"` diff --git a/agentbox/internal/phase1/run.go b/agentbox/internal/phase1/run.go index b072d5d..39cdd7d 100644 --- a/agentbox/internal/phase1/run.go +++ b/agentbox/internal/phase1/run.go @@ -117,7 +117,7 @@ func Run(ctx context.Context) (runErr error) { return fmt.Errorf("build environment image: %w", err) } - fmt.Println("Creating isolated graphical environment...") + fmt.Println("Creating restricted graphical environment...") _, err = r.docker(ctx, "create", "--name", r.containerName, @@ -327,6 +327,8 @@ func squareCentroid(path string) (point, error) { for y := bounds.Min.Y; y < bounds.Max.Y; y++ { for x := bounds.Min.X; x < bounds.Max.X; x++ { red, green, blue, _ := img.At(x, y).RGBA() + // Match the demo square's #00ff66 color with enough tolerance for + // image conversion while rejecting the black background. if green > 0xc000 && red < 0x4000 && blue < 0x8000 { sumX += uint64(x) sumY += uint64(y) diff --git a/agentbox/internal/runtime/runtime.go b/agentbox/internal/runtime/runtime.go index dd561f8..c82b632 100644 --- a/agentbox/internal/runtime/runtime.go +++ b/agentbox/internal/runtime/runtime.go @@ -45,7 +45,7 @@ func Run(ctx context.Context, config Config) (runErr error) { if err := config.Environment.Launch(ctx, config.Command); err != nil { return err } - fmt.Fprintln(config.Output, "Agent attached.") + fmt.Fprintln(config.Output, "Agent loop started.") var stepHistory []agent.Step var actionHistory []environment.InputAction diff --git a/agentbox/internal/trace/doc.go b/agentbox/internal/trace/doc.go new file mode 100644 index 0000000..97f90ec --- /dev/null +++ b/agentbox/internal/trace/doc.go @@ -0,0 +1,4 @@ +// Package trace stores durable run records. steps.jsonl contains complete +// observations and decisions, actions.jsonl is an action-only view, and PNG +// screenshots remain separate files referenced by relative paths. +package trace diff --git a/agentbox/internal/trace/reader.go b/agentbox/internal/trace/reader.go index 2ceb3ca..833b9fe 100644 --- a/agentbox/internal/trace/reader.go +++ b/agentbox/internal/trace/reader.go @@ -25,6 +25,8 @@ func List(projectRoot string) ([]Run, error) { continue } run, err := Read(projectRoot, entry.Name()) + // Old or malformed trace directories are intentionally hidden rather + // than making the entire run listing fail. if err == nil && run.SchemaVersion == SchemaVersion { runs = append(runs, run) } @@ -87,6 +89,7 @@ func ReadSteps(projectRoot, runID string) ([]StepRecord, error) { } func validateRunID(runID string) error { + // Run IDs become path components, so reject separators and traversal. if filepath.Base(runID) != runID { return errors.New("invalid run ID") }