ResearchForge Documentation

pip install researchforge          # includes built-in AI providers (Anthropic, Gemini, OpenAI)
pip install "researchforge[serve]" # add only if you want the live web monitor

# After pip install, register skills/rules:
researchforge all install --user

# Claude Code only:
researchforge claude install

# Cursor only:
researchforge cursor install

git clone https://github.com/forger-labs-hq/researchforge
cd researchforge
pip install -e ".[serve,dev]"

docker run --rm -v "$PWD":/workspace -w /workspace \
  ghcr.io/forger-labs-hq/researchforge:latest \
  researchforge research search "your query"

# Recommended — start in the IDE
/researchforge-start
# or
@researchforge-start

# 1. Install
pip install "researchforge[serve]"

# 2. Open Claude Code or Cursor
#    Type one of these:
/researchforge-start
# or
@researchforge-start

# 3. Approve the contract, run the baseline, and let the agent do the rest

pip install "researchforge[serve]"
researchforge all install --user   # → ~/.claude/skills/ and ~/.cursor/rules/
researchforge all status

"""
Your benchmark script. Lives in a protected path.
ResearchForge runs this to measure the baseline, then runs it again
inside each experiment worktree (with the AI's patch applied to src/).
"""
import json
import pathlib
from my_model import load_and_eval  # ← AI can patch this

# Run your evaluation
accuracy, p95_ms, cost = load_and_eval(dataset="benchmark-v2")

# Write results in the standard ResearchForge format
pathlib.Path("artifacts").mkdir(exist_ok=True)
pathlib.Path("artifacts/results.json").write_text(json.dumps({
    "schema_version": 1,
    "primary_metric": {"name": "accuracy", "value": accuracy},
    "secondary_metrics": {
        "p95_latency_ms": p95_ms,
        "average_cost_usd": cost,
    },
    "sample_count": 1200,
    "seed": 42,
    "metadata": {"dataset_version": "benchmark-v2"},
}), encoding="utf-8")

print("evaluation complete")

{
  "schema_version": 1,
  "primary_metric": {"name": "accuracy", "value": 0.891},
  "secondary_metrics": {
    "p95_latency_ms": 143.2,
    "average_cost_usd": 0.0031,
    "f1_macro": 0.877
  },
  "sample_count": 1200,
  "seed": 42,
  "metadata": {"dataset_version": "benchmark-v2", "model_params": 7340032}
}

objective:
  description: >
    Improve accuracy on the classification benchmark while keeping
    p95 latency under 200ms and cost under $0.005 per query.
  primary_metric:
    name: accuracy
    direction: maximize
  hard_constraints:
    - name: p95_latency_ms
      operator: <=
      value: 200
    - name: average_cost_usd
      operator: <=
      value: 0.005

execution:
  screening_command: python benchmarks/evaluate.py --subset screening
  full_command:      python benchmarks/evaluate.py --subset full
  result_file: artifacts/results.json

import sys
import json, pathlib

subset = "screening" if "--subset" in sys.argv and "screening" in sys.argv else "full"
# screening = fast 10% sample; full = complete eval
dataset_size = 120 if subset == "screening" else 1200

accuracy = run_eval(n_samples=dataset_size)

pathlib.Path("artifacts").mkdir(exist_ok=True)
pathlib.Path("artifacts/results.json").write_text(json.dumps({
    "schema_version": 1,
    "primary_metric": {"name": "accuracy", "value": accuracy},
    "sample_count": dataset_size,
    "seed": 42,
}), encoding="utf-8")

researchforge research search [--query/-q TEXT]... [flags]

# List stored papers
researchforge papers list

# Show details for a specific paper
researchforge papers show paper-003

researchforge baseline run [--check] [--json]

# Check current baseline
researchforge baseline show
researchforge baseline status    # alias for show

# Generate hypotheses using a direct AI API key (no IDE needed)
researchforge hypotheses generate [--provider anthropic|google|openai] [--model TEXT] [--no-import] [--json]
# Equivalent alias:
researchforge research synthesize [--provider ...] [--model ...] [--no-import] [--json]

# Import a hypotheses.yaml written by Claude/Cursor (validates schema)
researchforge hypotheses import .researchforge/synthesis/hypotheses.yaml

# List all hypotheses and their status
researchforge hypotheses list

# Show details for one hypothesis
researchforge hypotheses show hyp-002

# Generate a plan template for a hypothesis (Claude/Cursor fills in the patches)
researchforge experiment plan <hyp-id>

# Import a plan.yaml (written by Claude/Cursor or by hand)
researchforge experiment import plan.yaml

# Start a run (import + approval + execute)
researchforge experiment start plan.yaml
researchforge run plan.yaml          # alias

# List experiments and status
researchforge experiment list

# Resume or discard an interrupted run
researchforge experiment resume run-001
researchforge experiment abandon run-001

# Validate the contract (not the plan)
researchforge contract validate

# Start an overnight run
researchforge serve --background        # start live monitor first
researchforge autorun --target 0.85 --max-hours 8 --yes

# Short run to see how the loop behaves before committing overnight
researchforge autorun --max-rounds 2 --observe

# Continue an interrupted loop
researchforge autorun --resume

researchforge run <plan.yaml> [--yes] [--monitor/--no-monitor] [--json]

researchforge validate <run-id> [--experiment/-e TEXT] [--yes] [--json]

# Create a clean local branch from the frozen baseline
researchforge ship branch [experiment_id] [--branch TEXT] [--yes] [--json]

# Build the engineering report
researchforge report build

# OPT-IN: push branch + open a DRAFT PR (requires gh CLI + 3-gate approval)
researchforge ship pr [experiment_id] [--yes] [--json]

# Start the machine-wide hub dashboard (http://127.0.0.1:9000)
researchforge hub --background

# Start the per-project live monitor
researchforge serve --background

# Install both Claude Code skills and Cursor rules
researchforge all install [--user] [--global]

# --user: installs to ~/.claude/skills/ and ~/.cursor/rules/
# --global: installs to system-wide config (requires admin)

# Verify installation
researchforge all status

# researchforge.yaml — complete reference (every key the contract accepts)
version: 1                      # required, literal 1

project:
  name: my-project
  mode: improve_repository      # improve_repository | explore_research_idea

objective:
  description: "Improve detection mAP@0.5 without exceeding the inference budget"
  primary_metric:
    name: map50
    direction: maximize         # maximize | minimize
    target_value: 0.85          # optional — autorun stops when reached
  hard_constraints:             # optional, repeatable
    - name: inference_ms
      operator: "<="            # <= | >= | < | > | ==
      value: 200
  secondary_metrics:            # optional — recorded, never enforced
    - inference_ms

repository:
  baseline_ref: main            # the ref the baseline commit is resolved from

execution:
  mode: auto                    # auto (default) | docker | venv
                                # auto prefers Docker when available
  trusted_repository: false
  setup_command: "pip install -r requirements.txt"
  screening_command: "python benchmarks/evaluate.py --quick"   # optional
  test_command: null            # optional — must pass before benchmark runs
  full_command: "python benchmarks/evaluate.py"                # required
  result_file: artifacts/results.json
  timeout_minutes: 20
  cpu_limit: 2
  memory_mb: 4096
  max_experiments: 8
  stall: 3                      # stop after N consecutive non-improvements

permissions:
  editable_paths:               # the only paths a patch may touch
    - src/
  protected_paths:              # patch touching these → rejected at import, never runs
    - benchmarks/
    - tests/

network:
  mode: none                    # none (default) | enabled
                                # use enabled for repos that download weights

secrets:
  forward_environment_variables: []   # nothing forwarded unless named here

validation:
  repeat_finalists: 3           # repeats to earn "validated"
  require_existing_tests: true

shipping:
  allow_branch_creation: true
  allow_draft_pr: false         # gate 1 of 3 for ship pr

hypothesis_id: hyp-001          # required — must be a stored hypothesis
approach_summary: "Tune confidence and NMS thresholds"

experiments:
  # A) patch variant — a unified diff applied to the worktree
  - key: conf-low               # ^[a-z0-9][a-z0-9-]{0,40}$
    title: "Lower the confidence threshold"
    change_summary: "CONF 0.001 → 0.0001 in src/config.py"
    patch_file: patches/conf-low.patch
    expected_effect: improvement   # optional
    notes: "Grounded in paper-004"  # optional

  # B) env-only variant — no patch file needed
  - key: iou-065
    title: "Raise the NMS IoU threshold"
    change_summary: "IOU 0.6 → 0.65"
    env_overrides:              # injected into the experiment subprocess
      IOU: "0.65"

  # C) build on a measured ancestor (builds on exp-NNN already in the DB)
  - key: conf-plus-size
    title: "Confidence tuning at larger input size"
    change_summary: "Adds IMGSZ 800 on top of the conf-low winner"
    parent: conf-low            # key in this plan, or exp-NNN already measured
    patch_file: patches/conf-plus-size.patch

  # D) merge two independent winners
  - key: merge-001
    title: "Combine both winners"
    change_summary: "Composes conf-low and iou-065"
    parents: [conf-low, iou-065]
    # patch_file / env_overrides may be omitted for a pure merge

# AI providers (built in — no extra install, auto-detected)
ANTHROPIC_API_KEY=sk-ant-...          # → claude-opus-4-5 (default)
GEMINI_API_KEY=...                    # → gemini-2.0-flash (default)
GOOGLE_API_KEY=...                    # alias for GEMINI_API_KEY
OPENAI_API_KEY=sk-...                 # → gpt-4o (default)
RESEARCHFORGE_LLM=claude-opus-4-5    # override the model for any provider
RESEARCHFORGE_LLM=http://localhost:11434/api   # or point at Ollama (air-gapped)

# Local behaviour
RESEARCHFORGE_HOME=~/.researchforge   # where machine-wide state (hub registry) lives
RESEARCHFORGE_NO_HUB=1               # opt out of the hub auto-starting

import os
from sklearn.ensemble import GradientBoostingRegressor, RandomForestRegressor
from sklearn.preprocessing import StandardScaler
from sklearn.metrics import mean_squared_error
import numpy as np

# ResearchForge injects these via worktree env vars
model_type = os.environ.get("MODEL_TYPE", "gbm")
n_estimators = int(os.environ.get("N_ESTIMATORS", "100"))
normalize = os.environ.get("NORMALIZE", "false").lower() == "true"

X_train, X_val, y_train, y_val = load_data()

if normalize:
    scaler = StandardScaler()
    X_train = scaler.fit_transform(X_train)
    X_val = scaler.transform(X_val)

if model_type == "rf":
    model = RandomForestRegressor(n_estimators=n_estimators, random_state=42)
else:
    model = GradientBoostingRegressor(n_estimators=n_estimators, random_state=42)

model.fit(X_train, y_train)
preds = model.predict(X_val)
rmse = np.sqrt(mean_squared_error(y_val, preds))

# Write results.json — NOT print(RF_METRIC)
import json, pathlib
pathlib.Path("artifacts").mkdir(exist_ok=True)
pathlib.Path("artifacts/results.json").write_text(json.dumps({
    "schema_version": 1,
    "primary_metric": {"name": "rmse", "value": float(rmse)},
    "sample_count": len(y_val),
    "seed": 42,
}), encoding="utf-8")

hypothesis_id: hyp-001
approach_summary: "Test model type and normalisation"
experiments:
  - key: gbm-200
    title: "GBM 200 estimators"
    change_summary: "MODEL_TYPE=gbm N_ESTIMATORS=200"
    env_overrides: { MODEL_TYPE: gbm, N_ESTIMATORS: "200" }
  - key: rf-200
    title: "Random forest 200 estimators"
    change_summary: "MODEL_TYPE=rf N_ESTIMATORS=200"
    env_overrides: { MODEL_TYPE: rf, N_ESTIMATORS: "200" }
  - key: gbm-norm
    title: "GBM with normalisation"
    change_summary: "Adds NORMALIZE=true on top of gbm-200 winner"
    parent: gbm-200
    env_overrides: { NORMALIZE: "true" }

import os
import torch
import pytorch_lightning as pl

lr = float(os.environ.get("LR", "1e-3"))
hidden = int(os.environ.get("HIDDEN_SIZE", "256"))
dropout = float(os.environ.get("DROPOUT", "0.1"))
use_batchnorm = os.environ.get("BATCHNORM", "false") == "true"

class MyModel(pl.LightningModule):
    def __init__(self):
        super().__init__()
        self.net = build_net(hidden, dropout, use_batchnorm)
        self.lr = lr

    def training_step(self, batch, idx):
        loss = self.net(batch)
        return loss

    def validation_step(self, batch, idx):
        val_loss = self.net(batch)
        # Emit to ResearchForge
        self.log("rf_val_loss", val_loss)
        return val_loss

trainer.fit(model, train_dl, val_dl)

# Write results.json from best checkpoint metrics
import json, pathlib
best_val = trainer.callback_metrics.get("val_loss", float("inf"))
pathlib.Path("artifacts").mkdir(exist_ok=True)
pathlib.Path("artifacts/results.json").write_text(json.dumps({
    "schema_version": 1,
    "primary_metric": {"name": "val_loss", "value": float(best_val)},
    "sample_count": len(val_dl.dataset),
    "seed": 42,
}), encoding="utf-8")

import os
from transformers import AutoModelForSequenceClassification, TrainingArguments, Trainer
from datasets import load_dataset
import numpy as np

model_name = os.environ.get("MODEL_NAME", "distilbert-base-uncased")
lr = float(os.environ.get("LR", "2e-5"))
epochs = int(os.environ.get("EPOCHS", "3"))
warmup = float(os.environ.get("WARMUP_RATIO", "0.1"))

model = AutoModelForSequenceClassification.from_pretrained(model_name)

args = TrainingArguments(
    output_dir="./out",
    learning_rate=lr,
    num_train_epochs=epochs,
    warmup_ratio=warmup,
    evaluation_strategy="epoch",
    save_strategy="no",
    load_best_model_at_end=False,
    report_to="none",  # disable wandb/mlflow — RF handles tracking
)

trainer = Trainer(model=model, args=args, ...)
trainer.train()
results = trainer.evaluate()

# Write results.json
import json, pathlib
pathlib.Path("artifacts").mkdir(exist_ok=True)
pathlib.Path("artifacts/results.json").write_text(json.dumps({
    "schema_version": 1,
    "primary_metric": {"name": "f1", "value": results["eval_f1"]},
    "secondary_metrics": {"eval_loss": results["eval_loss"]},
    "sample_count": len(eval_dataset),
    "seed": 42,
}), encoding="utf-8")

# Activate your env first, then run baseline:
source .venv/bin/activate
researchforge baseline run

# Worktrees are created at:
# .researchforge/worktrees/exp-001/   ← git worktree at baseline commit
# venv is cloned inside the worktree

# See all file locations:
researchforge paths

execution:
  mode: docker     # or auto (default — Docker when Dockerfile + daemon exist)

# Generate a Dockerfile from the repo scan (no API key needed)
researchforge generate dockerfile

# AI-tailored version
researchforge generate dockerfile --provider anthropic

# GPU base image
researchforge generate dockerfile --cuda

name: ResearchForge experiments

on:
  workflow_dispatch:
  push:
    branches: [research/**]

jobs:
  run-experiments:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with: { python-version: "3.12" }

      - name: Install ResearchForge
        run: pip install "researchforge[serve]"

      - name: Install project deps
        run: pip install -r requirements.txt

      - name: Freeze baseline
        run: researchforge baseline run

      - name: Run experiments
        run: researchforge experiment start .researchforge/experiments/plan.yaml --yes

# Start the local monitoring server for this project
researchforge serve --background

# Start the machine-wide hub dashboard
researchforge hub --background

# Inspect project state and monitor status
researchforge status
researchforge paths

execution:
  stall: 3          # stop after 3 consecutive non-improvements

# Override for one run
researchforge experiment run <plan-id> --stall 3

# In autorun
researchforge autorun --stall 2 --global-stall 3

Orchestrator
│
├─ experiment exp-001  (.researchforge/worktrees/exp-001/)
│    env: MODEL_TYPE=gru, LR=1e-3
│    runs: python benchmarks/evaluate.py --subset full
│    writes: .researchforge/artifacts/exp-001/results.json
│    cleaned up after: ✓
│
├─ experiment exp-002  (.researchforge/worktrees/exp-002/)  ← runs next
│    ...
│
└─ experiment exp-003  (.researchforge/worktrees/exp-003/)  ← runs after
     ...

permissions:
  editable_paths:        # AI may only touch these
    - src/
    - config/
  protected_paths:       # patch touching any of these → rejected at import
    - benchmarks/        # eval script
    - tests/             # test suite

# View the trail (oldest first)
researchforge audit log
researchforge audit log --last 20
researchforge audit log --kind contract_approved   # filter by event kind

# Export as JSON (includes gate findings)
researchforge audit export audit.json

# Pull and start the hub
docker pull ghcr.io/forger-labs-hq/researchforge-hub:latest

docker run -d \
  --name rf-hub \
  -p 8080:8080 \
  -v /data/rf-hub:/data \
  -e RF_SECRET_KEY=$(openssl rand -hex 32) \
  -e RF_ADMIN_EMAIL=admin@yourcompany.com \
  ghcr.io/forger-labs-hq/researchforge-hub:latest

# Dashboard available at http://your-server:8080

RESEARCHFORGE_HUB_URL=https://hub.yourcompany.com
RESEARCHFORGE_API_KEY=rf_live_xxxxxxxxxxxx

RF_HUB_URL=https://hub.yourcompany.com \
RF_API_KEY=rf_live_xxxx \
researchforge run --workload search-ranking-v3

name: ResearchForge experiments

on:
  workflow_dispatch:
  push:
    branches: [research/**]

jobs:
  run-experiments:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with: { python-version: "3.12" }
      - run: pip install researchforge
      - run: pip install -r requirements.txt
      - run: researchforge contract approve --yes
      - run: researchforge baseline run
      - run: researchforge run .researchforge/experiments/plan.yaml --yes
      - run: researchforge results show run-001 --json

# On a connected machine: export pip wheels
pip download researchforge -d ./rf-wheels/
tar -czf rf-wheels.tar.gz rf-wheels/

# On a connected machine: export your paper set
researchforge papers export papers.json

# Transfer both to the air-gapped machine, then:
tar -xzf rf-wheels.tar.gz
pip install --no-index --find-links=./rf-wheels researchforge
researchforge papers import papers.json

# Use Ollama for AI calls locally:
export RESEARCHFORGE_LLM=http://localhost:11434/api

# Share project state via git
git add .researchforge/
git commit -m "share research state"
git push

# Teammate pulls and continues:
git pull
researchforge status          # exact next step
researchforge results show run-001

.researchforge/
├── researchforge.db          ← all state: papers, hypotheses, contract,
│                               baseline, plans, experiments, runs, decisions
├── config.json               ← project id + settings
├── synthesis/                ← landscape.yaml + hypotheses.yaml the AI writes
├── experiments/
│   ├── context.json          ← planning context exported for the AI
│   ├── plan.yaml             ← the plan being imported / last imported
│   └── patches/              ← one unified diff per variant
├── worktrees/                ← isolated checkouts at the baseline commit
├── artifacts/                ← per-execution stdout, stderr, diff, results.json
├── reports/                  ← engineering + research reports
├── research-log.md           ← living context autorun feeds back to the AI
└── autorun.json              ← loop state for autorun --resume

# Upgrade to latest
pip install --upgrade researchforge

# Re-register IDE skills/rules after upgrading
researchforge all install --user

# Confirm the installed version
pip show researchforge

# Verify git/Python/Docker are still usable
researchforge doctor