SWE-bench Single

minisweagent.run.extra.swebench_single

Run on a single SWE-Bench instance.

app module-attribute

app = Typer(add_completion=False)

DEFAULT_OUTPUT module-attribute

DEFAULT_OUTPUT = (
    global_config_dir / "last_swebench_single_run.traj.json"
)

main

main(
    subset: str = Option(
        "lite",
        "--subset",
        help="SWEBench subset to use or path to a dataset",
        rich_help_panel="Data selection",
    ),
    split: str = Option(
        "dev",
        "--split",
        help="Dataset split",
        rich_help_panel="Data selection",
    ),
    instance_spec: str = Option(
        0,
        "-i",
        "--instance",
        help="SWE-Bench instance ID or index",
        rich_help_panel="Data selection",
    ),
    model_name: str | None = Option(
        None,
        "-m",
        "--model",
        help="Model to use",
        rich_help_panel="Basic",
    ),
    model_class: str | None = Option(
        None,
        "-c",
        "--model-class",
        help="Model class to use (e.g., 'anthropic' or 'minisweagent.models.anthropic.AnthropicModel')",
        rich_help_panel="Advanced",
    ),
    config_path: Path = Option(
        builtin_config_dir / "extra" / "swebench.yaml",
        "-c",
        "--config",
        help="Path to a config file",
        rich_help_panel="Basic",
    ),
    environment_class: str | None = Option(
        None,
        "--environment-class",
        rich_help_panel="Advanced",
    ),
    exit_immediately: bool = Option(
        False,
        "--exit-immediately",
        help="Exit immediately when the agent wants to finish instead of prompting.",
        rich_help_panel="Basic",
    ),
    output: Path = Option(
        DEFAULT_OUTPUT,
        "-o",
        "--output",
        help="Output trajectory file",
        rich_help_panel="Basic",
    ),
) -> None

Run on a single SWE-Bench instance.

Source code in src/minisweagent/run/extra/swebench_single.py

@app.command()
def main(
    subset: str = typer.Option("lite", "--subset", help="SWEBench subset to use or path to a dataset", rich_help_panel="Data selection"),
    split: str = typer.Option("dev", "--split", help="Dataset split", rich_help_panel="Data selection"),
    instance_spec: str = typer.Option(0, "-i", "--instance", help="SWE-Bench instance ID or index", rich_help_panel="Data selection"),
    model_name: str | None = typer.Option(None, "-m", "--model", help="Model to use", rich_help_panel="Basic"),
    model_class: str | None = typer.Option(None, "-c", "--model-class", help="Model class to use (e.g., 'anthropic' or 'minisweagent.models.anthropic.AnthropicModel')", rich_help_panel="Advanced"),
    config_path: Path = typer.Option( builtin_config_dir / "extra" / "swebench.yaml", "-c", "--config", help="Path to a config file", rich_help_panel="Basic"),
    environment_class: str | None = typer.Option(None, "--environment-class", rich_help_panel="Advanced"),
    exit_immediately: bool = typer.Option( False, "--exit-immediately", help="Exit immediately when the agent wants to finish instead of prompting.", rich_help_panel="Basic"),
    output: Path = typer.Option(DEFAULT_OUTPUT, "-o", "--output", help="Output trajectory file", rich_help_panel="Basic"),
) -> None:
    # fmt: on
    """Run on a single SWE-Bench instance."""
    dataset_path = DATASET_MAPPING.get(subset, subset)
    logger.info(f"Loading dataset from {dataset_path}, split {split}...")
    instances = {
        inst["instance_id"]: inst  # type: ignore
        for inst in load_dataset(dataset_path, split=split)
    }
    if instance_spec.isnumeric():
        instance_spec = sorted(instances.keys())[int(instance_spec)]
    instance: dict = instances[instance_spec]  # type: ignore

    config = yaml.safe_load(get_config_path(config_path).read_text())
    if environment_class is not None:
        config.setdefault("environment", {})["environment_class"] = environment_class
    if model_class is not None:
        config.setdefault("model", {})["model_class"] = model_class
    if exit_immediately:
        config.setdefault("agent", {})["confirm_exit"] = False
    env = get_sb_environment(config, instance)
    agent = InteractiveAgent(
        get_model(model_name, config.get("model", {})),
        env,
        **({"mode": "yolo"} | config.get("agent", {})),
    )

    exit_status, result, extra_info = None, None, None
    try:
        exit_status, result = agent.run(instance["problem_statement"])  # type: ignore[arg-type]
    except Exception as e:
        logger.error(f"Error processing instance {instance_spec}: {e}", exc_info=True)
        exit_status, result = type(e).__name__, str(e)
        extra_info = {"traceback": traceback.format_exc()}
    finally:
        save_traj(agent, output, exit_status=exit_status, result=result, extra_info=extra_info)  # type: ignore[arg-type]

bug_report Something broken/unclear?

Open an issue on GitHub!

help Open-ended discussions

Join our Slack!

Our projects

SWE-agent SWE-ReX SWE-smith SWE-bench sb-cli